Python爬虫-爬取豆瓣信息(selenium+xpath)

======================

====================================

利用selenium模拟豆瓣登录：

'''
利用selenium模拟登录豆瓣
需要输入验证码
思路：
1. 保存页面成快照
2. 等待用户手动输入验证码
3. 继续自动执行提交等动作

'''

from selenium import webdriver
from selenium.webdriver.common.keys import  Keys
import time


url = 'https://accounts.douban.com/login?alias=&redir=https%3A%2F%2Fwww.douban.com%2F&source=index_nav&error=1001'
driver = webdriver.Chrome()
driver.get(url)

time.sleep(4)

# 生成快照，用来查看验证码
driver.save_screenshot('douban_index.png')

captcha = input("plz input youre code:")

# 利用账户信息和验证码登录
driver.find_element_by_id("email").send_keys("1366798119@qq.com")
driver.find_element_by_id("password").send_keys("haha123456")
driver.find_element_by_id("captcha_field").send_keys(captcha)


driver.find_element_by_xpath("//input[@class='btn-submit']").click()

time.sleep(5)

driver.save_screenshot("logined.png")

with open("douban_home.html", 'w', encoding='utf-8') as file:
    file.write(driver.page_source)

driver.quit()

============================================

利用selenium+xpath爬取豆瓣书籍信息

'''
https://book.douban.com/subject_search?search_text=python&cat=1001&start=%s0
使用selenium爬去页面
保存内容后用xpath进行分析
'''

from selenium import webdriver
import time
from lxml import etree


def get_web(url):
    driver = webdriver.Chrome()
    driver.get(url)

    print('waitting for .......')
    time.sleep(20)
    print('waitting done .......')

    driver.save_screenshot('douban_reader.png')

#豆瓣中没有页面源码，需要先把页面源码download下来，driver.page_source
    fn = 'douban_reader.html'
    with open(fn, 'w', encoding='utf-8') as f:
        f.write(driver.page_source)

    content_parse(fn)
    driver.quit()

def content_parse(fn):
    html = ''

    with open(fn, 'r', encoding='utf-8') as f:
        html = f.read()


    # 生成xml树,用xpath解析
    tree = etree.HTML(html)

    #查找book
    books = tree.xpath('//div[@class="item-root"]')

    for book in books:
        book_name = book.xpath(".//div[@class='title']/a")
        print(book_name[0].text)


if __name__ == '__main__':
    url = 'https://book.douban.com/subject_search?search_text=python&cat=1001&start=%s0'
    get_web(url)