在Python中使用CasperJS获取JS渲染生成的HTML内容的教程

文章摘要：其实这里casperjs与python没有直接关系,主要依赖casperjs调用phantomjs webkit获取html文件内容。长期以来，爬虫抓取 客户端javascript渲染生成的html页面 都极为 困难, Java里面有 HtmlUnit, 而Python里，我们可以使用独立的跨平台的CasperJS。

    创建site.js(接口文件，输入:url，输出:html file)  

  //USAGE: E:\toolkit\n1k0-casperjs-e3a77d0\bin>python casperjs site.js --url=http://spys.ru/free-proxy-list/IE/ --outputfile='temp.html'

var fs = require('fs');
   var casper = require('casper').create({
    pageSettings: {
    loadImages: false,    
    loadPlugins: false,    
    userAgent: 'Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/34.0.1847.137 Safari/537.36 LBBROWSER'
   },
   logLevel: "debug",//日志等级
   verbose: true  // 记录日志到控制台
    });
   var url = casper.cli.raw.get('url');
   var outputfile = casper.cli.raw.get('outputfile');
   //请求页面
   casper.start(url, function () {
   fs.write(outputfile, this.getHTML(), 'w');
   });

casper.run();

    python 代码, checkout_proxy.py      

import json
   import sys
   #import requests
   #import requests.utils, pickle
   from bs4 import BeautifulSoup
   import os.path,os
   import threading
   #from multiprocessing import Process, Manager
   from datetime import datetime
   import traceback
   import logging
   import re,random
   import subprocess
   import shutil
   import platform

output_file = os.path.join(os.path.dirname(os.path.realpath(__file__)),'proxy.txt')
   global_log = 'http_proxy' + datetime.now().strftime('％Y-％m-％d') + '.log'
   if not os.path.exists(os.path.join(os.path.dirname(os.path.realpath(__file__)),'logs')):
     os.mkdir(os.path.join(os.path.dirname(os.path.realpath(__file__)),'logs'))
   global_log = os.path.join(os.path.dirname(os.path.realpath(__file__)),'logs',global_log)

logging.basicConfig(level=logging.DEBUG,format='[％(asctime)s] [％(levelname)s] [％(module)s] [％(funcName)s] [％(lineno)d] ％(message)s',filename=global_log,filemode='a')
   log = logging.getLogger(__name__)  
   #manager = Manager()
   #PROXY_LIST = manager.list()
   mutex = threading.Lock()
   PROXY_LIST = []

def isWindows():
     if "Windows" in str(platform.uname()):
     return True
     else:
     return False

def getTagsByAttrs(tagName,pageContent,attrName,attrRegValue):
     soup = BeautifulSoup(pageContent)                                                
     return soup.find_all(tagName, { attrName : re.compile(attrRegValue) })

def getTagsByAttrsExt(tagName,filename,attrName,attrRegValue):
     if os.path.isfile(filename):
     f = open(filename,'r')    
     soup = BeautifulSoup(f)
     f.close()
     return soup.find_all(tagName, { attrName : re.compile(attrRegValue) })
     else:
     return None

class Site1Thread(threading.Thread):
     def __init__(self,outputFilePath):
       threading.Thread.__init__(self)
     self.outputFilePath = outputFilePath
     self.fileName = str(random.randint(100,1000)) + ".html"
     self.setName('Site1Thread')

def run(self):
     site1_file = os.path.join(os.path.dirname(os.path.realpath(__file__)),'site.js')
     site2_file = os.path.join(self.outputFilePath,'site.js')
     if not os.path.isfile(site2_file) and os.path.isfile(site1_file):
       shutil.copy(site1_file,site2_file)
     #proc = subprocess.Popen(["bash","-c", "cd ％s && ./casperjs site.js --url=http://spys.ru/free-proxy-list/IE/ --outputfile=％s" ％ (self.outputFilePath,self.fileName) ],stdout=subprocess.PIPE)
     if isWindows():
       proc = subprocess.Popen(["cmd","/c", "％s/casperjs site.js --url=http://spys.ru/free-proxy-list/IE/ --outputfile=％s" ％ (self.outputFilePath,self.fileName) ],stdout=subprocess.PIPE)
     else:
       proc = subprocess.Popen(["bash","-c", "cd ％s && ./casperjs site.js --url=http://spys.ru/free-proxy-list/IE/ --outputfile=％s" ％ (self.outputFilePath,self.fileName) ],stdout=subprocess.PIPE)
     out=proc.communicate()[0]
     htmlFileName = ''
     #因为输出路径在windows不确定，所以这里加了所有可能的路径判断
     if os.path.isfile(self.fileName):
       htmlFileName = self.fileName
     elif os.path.isfile(os.path.join(self.outputFilePath,self.fileName)):
       htmlFileName = os.path.join(self.outputFilePath,self.fileName)
     elif os.path.isfile(os.path.join(os.path.dirname(os.path.realpath(__file__)),self.fileName)):
       htmlFileName = os.path.join(os.path.dirname(os.path.realpath(__file__)),self.fileName)  
     if (not os.path.isfile(htmlFileName)):
       print 'Failed to get html content from http://spys.ru/free-proxy-list/IE/'
       print out
       sys.exit(3)  
     mutex.acquire()
     PROXYList= getTagsByAttrsExt('font',htmlFileName,'class','spy14$')
     for proxy in PROXYList:
       tdContent = proxy.renderContents()
       lineElems = re.split('[<>]',tdContent)
       if re.compile(r'\d+').search(lineElems[-1]) and re.compile('(\d+\.\d+\.\d+)').search(lineElems[0]):
       print lineElems[0],lineElems[-1]
       PROXY_LIST.append("％s:％s" ％ (lineElems[0],lineElems[-1]))
     mutex.release()
     try:
       if os.path.isfile(htmlFileName):
       os.remove(htmlFileName)
     except:
       pass

if __name__ == '__main__':
     try:
     if(len(sys.argv)) < 2:
       print "Usage:％s [casperjs path]" ％ (sys.argv[0])
       sys.exit(1)  
     if not os.path.exists(sys.argv[1]):
       print "casperjs path: ％s does not exist!" ％ (sys.argv[1])
       sys.exit(2)  
     if os.path.isfile(output_file):
       f = open(output_file)
       lines = f.readlines()
       f.close
       for line in lines:
       PROXY_LIST.append(line.strip())
     thread1 = Site1Thread(sys.argv[1])
     thread1.start()
     thread1.join()

f = open(output_file,'w')
     for proxy in set(PROXY_LIST):
       f.write(proxy+"\n")
     f.close()
     print "Done!"
     except SystemExit:
     pass
     except:
       errMsg = traceback.format_exc()
       print errMsg
       log.error(errMsg)