phantomjs 设置代理
phantomjs 可通过以下设置代理ip
#coding=utf-8
import os
import re
import time
import requests
from scrapy.selector import HtmlXPathSelector
from scrapy.http import HtmlResponse
from selenium import webdriver
from selenium.webdriver.common.proxy import ProxyType
import sys
reload(sys)
sys.setdefaultencoding("utf-8")
import warnings
warnings.filterwarnings("ignore")
if __name__ == '__main__':
PATH_PHANTOMJS=r'D:\phantomjs\bin\phantomjs.exe'
browser=webdriver.PhantomJS(PATH_PHANTOMJS)
#真实ip
browser.get('http://icanhazip.com/') #获取ip网址
response = HtmlResponse(url='',body=str(browser.page_source))
hxs = HtmlXPathSelector(response)
print 'your ip is:',''.join(hxs.select('//text()').extract()).strip()
#代理ip
proxy=webdriver.Proxy()
proxy.proxy_type=ProxyType.MANUAL
proxy.http_proxy='220.248.229.45:3128'
#将代理设置添加到webdriver.DesiredCapabilities.PHANTOMJS中
proxy.add_to_capabilities(webdriver.DesiredCapabilities.PHANTOMJS)
browser.start_session(webdriver.DesiredCapabilities.PHANTOMJS)
browser.get('http://icanhazip.com/') #获取ip网址
response = HtmlResponse(url='',body=str(browser.page_source))
hxs = HtmlXPathSelector(response)
print 'your proxy ip is:',''.join(hxs.select('//text()').extract()).strip()
经测试,实际有效,截图如下:
后期可以考虑将此方法运用至下载中,减少爬虫被封的几率。