使用Selenium 结合PhantonJS 百度关键字搜索批量获取URL
- Python2.7 如果想在Python3 环境使用，请修改print
- PhantonJS 将PhantonJS.exe 加入环境变量:PhantonJS下载地址
- Selenium包安装
pip install selenium
#CMD 查看安装信息
pip show selenium
- 如果有问题，重启电脑，或者手动指定下PhantoJS.exe路径
browser = webdriver.PhantomJS(executable_path="phantomjs.exe")
详细代码和注释
# -*- coding: utf-8 -*-
"""
@author:随时静听
@file: baiduSpider2.py
@time: 2018/09/21
"""
from selenium import webdriver
import time
#抓取多少页
pages=10
#搜索的关键字
keyword=u"inurl:?.asp学校"
#百度搜索URL地址
url="https://www.baidu.com/"
#获取页面的有效地址
def getUrls(driver):
urls=[]
#获取当前窗口
current_window=driver.current_window_handle
#百度跳转地址
for a in driver.find_elements_by_xpath("//div[@id='content_left']//div/h3/a"):
a.click()#点击进行跳转
#获取打开的所有窗口，一会要将打开的窗口关闭，防止耗用电脑性能
all_windows=driver.window_handles
#所有窗口包含打开的搜索页面，不能将手搜页面关闭，所以将搜索页面剔除
all_windows.remove(current_window)
#对打开的页面进行关闭操作，这里一定要先关闭，再切换窗口，当前的driver是最后一个打开页面的窗口
#因此如果先关闭的话容易导致出错，自行switchs 切换窗口已经销毁
for window in all_windows:
driver.switch_to_window(window)
urls.append(driver.current_url)#统计获取到的URL
print "[-] "+driver.current_url #打印下当前的URL
print "[-] The number of pages parsed to URL is :"+str(len(all_windows))#打印页面获取了多少URL
#复原
#执行 其实这里可以和上面的循环一起处理
for window in all_windows[:]:
driver.switch_to_window(window)
driver.close()
#重新切换到搜索页面的窗口
driver.switch_to_window(current_window)
#返回抓取的URL和切换后的web对象
return urls,driver
#执行搜索动作，并将改变状态后的web对象返回
def doSearch(url,keyword,driver):
driver.get(url)
driver.find_element_by_id('kw').send_keys(keyword)
driver.find_element_by_id('su').submit()
time.sleep(3)
return driver
#执行点击下一页操作
def toNextpage(driver):
pages = driver.find_elements_by_xpath("//div[@id='page']//a")
if pages:
pages[-1].click()
return driver
#运行
def run():
pages=11
urls=[]
# driver = webdriver.Chrome()
try:
driver = webdriver.PhantomJS()
driver.implicitly_wait(6)
driver=doSearch(url,keyword,driver)
for i in range(pages+1):
url_lst,driver=getUrls(driver)
urls.extend(url_lst)
driver=toNextpage(driver)
time.sleep(2)
except Exception as e:
print "[!] There seems to be a mistake! The error message is as follows: "
print "--"*6+"Error Message Info"+"--"*6
print e
print "--" * 6 + "Error Message Info END" + "--" * 6
finally:
#关闭浏览器
driver.quit()
run()
if __name__ == '__main__':
pass
基本使用
- 由于比较懒，没有加入参数解析，也没将结果写入文件，有需要自己写一下，修改Keyword 直接运行即可
- 修改pages值指定搜索的页面数量
- 获取数度挺慢，不知道是不是我的电脑问题，使用Chrome挺快的
更新加入保存至文件
# -*- coding: utf-8 -*-
"""
@author:随时静听
@file: baiduSpider2.py
@time: 2018/09/21
@email:wang_di@topsec.com.cn
"""
from selenium import webdriver
from selenium.webdriver.common.desired_capabilities import DesiredCapabilities
import time
#抓取多少页
pages=10
#搜索的关键字
keyword=u"inurl:?.asp学校"
#百度搜索URL地址
url="https://www.baidu.com/"
#获取页面的有效地址
def getUrls(driver):
urls=[]
#获取当前窗口
current_window=driver.current_window_handle
#百度跳转地址
for a in driver.find_elements_by_xpath("//div[@id='content_left']//div/h3/a"):
a.click()#点击进行跳转
#获取打开的所有窗口，一会要将打开的窗口关闭，防止耗用电脑性能
all_windows=driver.window_handles
#所有窗口包含打开的搜索页面，不能将手搜页面关闭，所以将搜索页面剔除
all_windows.remove(current_window)
#对打开的页面进行关闭操作，这里一定要先关闭，再切换窗口，当前的driver是最后一个打开页面的窗口
#因此如果先关闭的话容易导致出错，自行switchs 切换窗口已经销毁
for window in all_windows:
driver.switch_to_window(window)
urls.append(driver.current_url)#统计获取到的URL
print "\t[-] "+driver.current_url #打印下当前的URL
print "[*] The number of pages parsed to URL is :"+str(len(all_windows))#打印页面获取了多少URL
#复原
#执行 其实这里可以和上面的循环一起处理
for window in all_windows[:]:
driver.switch_to_window(window)
driver.close()
#重新切换到搜索页面的窗口
driver.switch_to_window(current_window)
#返回抓取的URL和切换后的web对象
return urls,driver
#执行搜索动作，并将改变状态后的web对象返回
def doSearch(url,keyword,driver):
driver.get(url)
driver.find_element_by_id('kw').send_keys(keyword)
driver.find_element_by_id('su').submit()
time.sleep(3)
return driver
#执行点击下一页操作
def toNextpage(driver):
pages = driver.find_elements_by_xpath("//div[@id='page']//a")
if pages:
pages[-1].click()
return driver
# 获取传递的参数:
def getargs():
import sys
usage='''
Usage: BaiduSpider keywords [nums] [filename]
keywords: your Search keywords
nums: default value is 6,seach max pages
filename: result saved file name，default value is result.txt
'''
if len(sys.argv)<2:
print usage
exit()
else:
try:
keywords= sys.argv[1]
if len(sys.argv)==2:
# keywords= sys.argv[1] if sys.argv[1] else ""
nums= 6
filename="result.txt"
return (keywords,nums,filename)
if len(sys.argv)==3:
nums= int(sys.argv[2])
filename = "result.txt"
return (keywords, nums, filename)
if len(sys.argv)==4:
nums = int(sys.argv[2])
filename=sys.argv[3]
return (keywords, nums, filename)
except:
print "[!] Parameter error !"
print usage
exit()
def saveData(urls,filename):
if urls:
with open(filename,'a+') as f:
for url in urls:
f.write(url+'\n')
#运行
def run():
# pages=11
#
# urls=[]
# driver = webdriver.Chrome()
keyword,pages,filename=getargs()
print "[*] Keywords:"+keyword
print "[*] nums:"+str(pages)
print "[*] save file name:"+filename
print "\n"
print "--"*30
urls=[]
try:
# options=webdriver.P
# options.add_argument('--ignore-certificate-errors')
dcap = dict(DesiredCapabilities.PHANTOMJS)
dcap[
"phantomjs.page.settings.userAgent"] = "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/49.0.2623.221 Safari/537.36 SE 2.X MetaSr 1.0"
# 不加载图片
dcap["phantomjs.page.settings.loadImages"] = False
driver = webdriver.PhantomJS(desired_capabilities=dcap)
driver.set_page_load_timeout(20)
driver.implicitly_wait(6)
driver=doSearch(url,keyword,driver)
for i in range(pages+1):
print "[*] Page data loading: "+str(i+1)
url_lst,driver=getUrls(driver)
urls.extend(url_lst)
driver=toNextpage(driver)
time.sleep(5)
saveData(url_lst,filename)
print "[*] The urls total num is:" + str(len(urls))
print "[*] The spider urls save as file:" + filename
except Exception as e:
print "[!] There seems to be a mistake! The error message is as follows: "
print "--"*10+"Error Message Info"+"--"*10
print e
print "--" * 10 + "Error Message Info END" + "--" * 10
finally:
#关闭浏览器
driver.quit()
if __name__ == '__main__':
run()
pass
