Python爬虫实践~BeautifulSoup+urllib+Flask实现静态网页的爬取

爬取的网站类型：

论坛类网站类型

涉及主要的第三方模块：

BeautifulSoup：解析、遍历页面

urllib：处理URL请求

Flask：简易的WEB框架

介绍：

本次主要使用urllib获取网页数据，然后通过BeautifulSoup进行页面解析，返回json结构的数据。

功能点：

urllib根据URL通过GET方式获取网页内容；
通过JSON文件配置
解析页面结构，返回JSON结构的数据
提供REST服务进行调用

特点：

1、提供数据描述服务，总页面，每页条数，总条数；

2、增量请求数据，只获取当前时间和上次请求时间之间的数据；

3、控制请求时间间隔，防治IP被封杀

4、分页数据请求

5、修改请求关键字，并记录上次的请求关键字

主要代码结构：

- 公共请求端封装

#!/usr/bin/env python 
# -*- coding:utf-8 -*-
from urllib import request
from urllib.parse import quote
import string
import requests

# 静态页面基类
class StaticBase():
    # 获取网页内容
    def __getHTMLText(self,url,code="utf-8"):
        try:
            r = requests.get(url)
            r.raise_for_status()
            r.encoding = code
            return r.text
        except:
            return ""
    # get方式请求数据
    def getUrl(self,url,code='utf-8'):
        url = quote(url,safe=string.printable)
        req = request.Request(url)
        req.add_header('User-Agent','Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/45.0.2454.101 Safari/537.36')
        with request.urlopen(req) as f:
            print('Status:',f.status,f.reason)
            return f.read().decode(code)

#s = StaticBase()
#print(s.getUrl('http://www.baidu.com/','utf-8'))

- 配置文件

{
   "host": "http://shangyu.108sq.cn", 
   "base_url": "http://shangyu.108sq.cn/shuo/search?sertype=4", 
   "key_words": "污染", 
   "page_size": 30, 
   "search_key": "", 
   "last_request_time": 1562142204.149511, 
   "request_gap": 60
  }

- 解析服务

#!/usr/bin/env python 
# -*- coding:utf-8 -*-
from bs4 import BeautifulSoup
from datetime import datetime
import json
from common.staticBase import StaticBase

class Shangyh108(StaticBase):

    __config_file = "config.json"   #配置文件
    __text = ""                     #解析的网页内容
    __config_dict = {}              #配置集合
    __url = ""                      #请求的URL
    __keywords = ""                 #关键字
    __last_request_time = 0         #上次请求时间
    # 构造函数
    def __init__(self):
        self.__config_dict = self.__getConfig()
        # 查询关键字
        if len(self.__config_dict['search_key']) >0 :
            self.__keywords = self.__config_dict['search_key']
        else:
            self.__keywords = self.__config_dict['key_words']
        self.__url = self.__getUrl()
    # 获取网页内容
    def getText(self):
        print(self.__url)
        self.__text = StaticBase.getUrl(self,self.__url)

    # 获取第一页内容
    def getFirstPageText(self,url=''):
        if self.checkRquestTime():
            if len(url)==0 :
                url = self.__getUrl()
            self.__text = StaticBase.getUrl(self,url)
            return self.parseHTML()
        else:
            print("操作频繁请稍后重新")
    # 获取下一页
    def getNextPage(self,url):
        url = self.__config_dict['host']+url
        print(url)
        self.__text = StaticBase.getUrl(self,url)
        return self.parseHTML()

    # 为防止请求对服务器造成太大压力，控制请求的时间间隔，最少为5分钟
    def checkRquestTime(self):
        request_gap = self.__config_dict['request_gap']
        last_request_time = self.__config_dict['last_request_time']
        dt_now = datetime.now().timestamp()
        self.__last_request_time = last_request_time  # 记录上次请求的时间，为了获取阶段性的数据
        if last_request_time == 0:  #第一次请求，直接通过
            last_request_time = dt_now
        elif last_request_time+request_gap > dt_now:
            print("请求过度频繁，请稍后重试")
            return False
        else:
            last_request_time = dt_now

        self.__setConfig('last_request_time',last_request_time)
        return True

    # 获取网页描述信息
    def getDesc(self):
        self.getText()
        soup = BeautifulSoup(self.__text,'html.parser')
        obj_count = soup.select('.count')[0]
        count_str = str(obj_count.string).replace("(共","").replace("条)","")
        count = int(count_str)
        pageSize = int(self.__config_dict['page_size'])
        host = self.__config_dict['host']
        if count % pageSize == 0 :
            pages = count//pageSize
        else:
            pages = count // pageSize + 1
        desc = {}
        desc['host'] = host
        desc['count'] = count
        desc['page_size'] = pageSize
        desc['total_page'] = pages
        # 增加分页的URL
        if pages > 0 :
            pageUrls = soup.select(".TCPage__middle > a")
            page_url = []
            for i in range(len(pageUrls)-1) :
                tag = pageUrls[i+1]
                page_url.append(tag['href'])
            desc['page_url'] = page_url
        return json.dumps(desc,ensure_ascii=False)

    # 解析网页内容
    def parseHTML(self):
        soup = BeautifulSoup(self.__text, 'html.parser')
        list_li = soup.select('.TCSayList .TCSayList_li')
        data_list = []
        for i in range(len(list_li)):
            item = {}
            temp = list_li[i]
            publish_time = temp.select('.TCSayList_li_time')[0]
            int_dt = int(publish_time['data-time'])

            if self.__last_request_time == 0 or self.__last_request_time < int_dt :
                # 发布时间
                item['publish_time_long'] = publish_time['data-time']
                item['publish_time_str'] = datetime.fromtimestamp(int(publish_time['data-time'])).strftime(
                    '%Y-%m-%d %H:%M:%S')

                # 数据标签
                item['data-tag'] = temp['data-tag']
                # 用户
                author = temp.select('.TCSayList_li_author')[0]
                item['author_name'] = author.string
                item['author_url'] = author['href']

                # 标题
                if len(temp.select('.TCSayList__title a')) >0:
                    title = temp.select('.TCSayList__title a')[0]
                    item['title'] = title.string
                    item['link_url'] = title['href']
                # 内容
                item['content'] = temp.select('.TCSayList_li_content')[0]['data-short']

                data_list.append(item)
        return data_list
    # 获取请求配置信息
    def __getConfig(self):
        with open(self.__config_file, "r",encoding="utf-8") as load_f:
            load_dict = json.load(load_f)
        return load_dict
    # 设置配置项
    def __setConfig(self,key,value):
        self.__config_dict[key] = value
        print(self.__config_dict)
        with open(self.__config_file,'w',encoding="utf-8") as f:
            f.write(json.dumps(self.__config_dict,ensure_ascii=False))

    def getKeywords(self):
        self.__keywords = input("请输入查询的关键字，多个关键字用“+”连接，默认关键字：环保+污染+投诉，使用默认关键字可直接按Enter:")
        if len(self.__keywords) == 0:
            self.__keywords = self.__config_dict['key_words']
        else:
            self.__setConfig("search_key",self.__keywords)

    # 获取请求的URL
    def __getUrl(self):
        base_url = self.__config_dict['base_url']
        # 组装查询参数
        url = base_url + "&key=" + self.__keywords
        return url

- REST服务端（目前服务没有暴露很多，但相应服务实现都已经实现）

#!/usr/bin/env python 
# -*- coding:utf-8 -*-
from flask import Flask,jsonify,abort,request
from changshuo108.shangyu.Shangyh108 import Shangyh108

app = Flask(__name__)
# 支持中文
app.config['JSON_AS_ASCII'] = False
shangyue = Shangyh108()
@app.route('/shangyu108/api/desc',methods=['GET'])
def get_desc():
    return shangyue.getDesc()

@app.route('/shangyu108/api/first_page',methods=['GET'])
def get_firstPage():
    return jsonify({'data':shangyue.getFirstPageText('')})

@app.route('/shangyu108/api/page',methods=['POST'])
def get_article():
   if not request.json or not 'url' in request.json:
      abort(400)
   print(request.json['url'])
   return jsonify({'data':shangyue.getNextPage(request.json['url'])})

if __name__ == '__main__':
    app.run(debug=True)