selenium+BeautifulSoup实现强大的爬虫功能

阅读目录

• 1 下载并安装必要的插件
• 2 核心代码
• 3 源码

sublime下运行

1 下载并安装必要的插件

BeautifulSoup

selenium

phantomjs

采用方式可以下载后安装，本文采用pip

pip install BeautifulSoup

pip install selenium

pip install phantomjs

2 核心代码

phantomjs解析

def driver_open():
    dcap = dict(DesiredCapabilities.PHANTOMJS)
    dcap["phantomjs.page.settings.userAgent"] = (r"Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3100.0 Safari/537.36")
    driver = webdriver.PhantomJS(executable_path=r'C:\Users\Administrator\AppData\Roaming\Sublime Text 3\Packages\Anaconda\phantomjs.exe', desired_capabilities=dcap)
    return driver

BeautifulSoup

def get_content(driver,url):
    driver.get(url)
    time.sleep(30)
    content = driver.page_source.encode('utf-8')
    driver.close()
    soup = BeautifulSoup(content, 'lxml')
    return soup

3 源码

#!/usr/bin/env python
# -*- coding:utf-8 -*-
from selenium import webdriver
import time
from bs4 import BeautifulSoup
from selenium.webdriver.common.desired_capabilities import DesiredCapabilities

def driver_open():
    dcap = dict(DesiredCapabilities.PHANTOMJS)
    dcap["phantomjs.page.settings.userAgent"] = (r"Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3100.0 Safari/537.36")
    driver = webdriver.PhantomJS(executable_path=r'C:\Users\Administrator\AppData\Roaming\Sublime Text 3\Packages\Anaconda\phantomjs.exe', desired_capabilities=dcap)
    return driver
def get_content(driver,url):
    driver.get(url)
    time.sleep(30)
    content = driver.page_source.encode('utf-8')
    driver.close()
    soup = BeautifulSoup(content, 'lxml')
    return soup

def get_basic_info(soup):
    basic_info = soup.select('.baseInfo_model2017')

    zt = soup.select('.td-regStatus-value > p ')[0].text.replace("\n","").replace(" ","")
    basics = soup.select('.basic-td > .c8 > .ng-binding ')
    zzjgdm = basics[3].text
    tyshxydm = basics[7].text
    print (u'公司名称：'+company)
    print (u'公司状态：'+zt)
    # print basics
    print (u'组织机构代码：'+zzjgdm)
    print (u'统一社会信用代码：'+tyshxydm)

if __name__=='__main__':
    url = "http://www.tianyancha.com/company/2310290454"
    driver = driver_open()
    soup = get_content(driver, url)
    print(soup.body.text)
    print('----获取基础信息----')
    get_basic_info(soup)