selenium+谷歌无头浏览器爬取网易新闻国内板块

网页分析

 首先来看下要爬取的网站的页面

查看网页源代码：你会发现它是由js动态加载显示的

所以采用selenium+谷歌无头浏览器来爬取它

1 加载网站，并拖动到底，发现其还有个加载更多

 2 模拟点击它，然后再次拖动到底，，就可以加载完整个页面

示例代码

from selenium import webdriver
from selenium.webdriver.chrome.options import Options
from selenium.webdriver.common.by import By
from time import sleep
from lxml import etree
import os
import requests

# 使用谷歌无头浏览器来加载动态js
def main():
    # 创建一个无头浏览器对象
    chrome_options = Options()
    # 设置它为无框模式
    chrome_options.add_argument('--headless')
    # 如果在windows上运行需要加代码
    chrome_options.add_argument('--disable-gpu')
    browser = webdriver.Chrome(chrome_options=chrome_options)
    # 设置一个10秒的隐式等待
    browser.implicitly_wait(10)
    browser.get(url)
    sleep(1)
    # 翻到页底
    browser.execute_script('window.scrollTo(0,document.body.scrollHeight)')
    # 点击加载更多
    browser.find_element(By.CSS_SELECTOR, '.load_more_btn').click()
    sleep(1)
    # 再次翻页到底
    browser.execute_script('window.scrollTo(0,document.body.scrollHeight)')
    # 拿到页面源代码
    source = browser.page_source
    browser.quit()
    with open('xinwen.html', 'w', encoding='utf-8') as f:
        f.write(source)
        parse_page(source)

# 对新闻列表页面进行解析
def parse_page(html):
    # 创建etree对象
    tree = etree.HTML(html)
    new_lst = tree.xpath('//div[@class="ndi_main"]/div')
    for one_new in new_lst:
        title = one_new.xpath('.//div[@class="news_title"]/h3/a/text()')[0]
        link = one_new.xpath('.//div[@class="news_title"]/h3/a/@href')[0]
        write_in(title, link)

# 将其写入到文件
def write_in(title, link):
    print('开始写入篇新闻{}'.format(title))
    response = requests.get(url=link, headers=headers)
    tree = etree.HTML(response.text)
    content_lst = tree.xpath('//div[@class="post_text"]//p')
    title = title.replace('?', '')
    with open('new/' + title + '.txt', 'a+', encoding='utf-8') as f:
        for one_content in content_lst:
            if one_content.text:
                    con = one_content.text.strip()
                    f.write(con + '\n')


if __name__ == '__main__':
    url = 'https://news.163.com/domestic/'
    headers = {"User-Agent": 'Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0'}
    if not os.path.exists('new'):
        os.mkdir('new')
    main()

 得到结果：

随意打开一个txt:

Scrapy版

wangyi.py

# -*- coding: utf-8 -*-
import scrapy
from selenium import webdriver
from selenium.webdriver.chrome.options import Options
from happy1.items import Happy1Item

class WangyiSpider(scrapy.Spider):
    name = 'wangyi'
    # allowed_domains = ['https://news.163.com/domestic/']
    start_urls = ['http://news.163.com/domestic/']

    def __init__(self):
        # 创建一个无头浏览器对象
        chrome_options = Options()
        # 设置它为无框模式
        chrome_options.add_argument('--headless')
        # 如果在windows上运行需要加代码
        chrome_options.add_argument('--disable-gpu')
        # 示例话一个浏览器对象(实例化一次)
        self.bro = webdriver.Chrome(chrome_options=chrome_options)

    def parse(self, response):
        new_lst = response.xpath('//div[@class="ndi_main"]/div')
        for one_new in new_lst:
            item = Happy1Item()
            title = one_new.xpath('.//div[@class="news_title"]/h3/a/text()')[0].extract()
            link = one_new.xpath('.//div[@class="news_title"]/h3/a/@href')[0].extract()
            item['title'] = title
            yield scrapy.Request(url=link,callback=self.parse_detail, meta={'item':item})

    def parse_detail(self, response):
        item = response.meta['item']
        content_list = response.xpath('//div[@class="post_text"]//p/text()').extract()
        item['content'] = content_list
        yield item

    # 在爬虫结束后，关闭浏览器
    def close(self, spider):
        print('爬虫结束')
        self.bro.quit()

pipelines.py

class Happy1Pipeline(object):
    def __init__(self):
        self.fp = None

    def open_spider(self, spider):
        print('开始爬虫')

    def process_item(self, item, spider):
        title = item['title'].replace('?', '')
        self.fp = open('news/' + title + '.txt', 'a+', encoding='utf-8')
        for one in item['content']:
            self.fp.write(one.strip() + '\n')
        self.fp.close()
        return item

items.py

import scrapy


class Happy1Item(scrapy.Item):
    # define the fields for your item here like:
    # name = scrapy.Field()
    title = scrapy.Field()
    content = scrapy.Field()

middlewares.py

    def process_response(self, request, response, spider):
        if request.url in ['http://news.163.com/domestic/']:
            spider.bro.get(url=request.url)
            time.sleep(1)
            spider.bro.execute_script('window.scrollTo(0,document.body.scrollHeight)')
            spider.bro.find_element(By.CSS_SELECTOR, '.load_more_btn').click()
            time.sleep(1)
            spider.bro.execute_script('window.scrollTo(0,document.body.scrollHeight)')
            page_text = spider.bro.page_source
            return HtmlResponse(url=spider.bro.current_url, body=page_text, encoding='utf-8', request=request)
        else:
            return response

settings.py

DOWNLOADER_MIDDLEWARES = {
   'happy1.middlewares.Happy1DownloaderMiddleware': 543,
}

ITEM_PIPELINES = {
   'happy1.pipelines.Happy1Pipeline': 300,
}

 得到结果

总结：

1 其实主要的工作还是模拟浏览器来进行操作。

2 处理动态的js其实还有其他办法。

3 爬虫的方法有好多种，主要还是选择适合自己的。

4 自己的代码写的太烂了。