静态网站爬取文本信息实例（requests,re,pool)

import requests
import re
import json
from requests.exceptions import RequestException
from multiprocessing import Pool

def get_one_page(url):
    headers = {
        "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.99 Safari/537.36"}
    try:
        r=requests.get(url,headers=headers)
        r.encoding='utf-8'
        if r.status_code==200:
            return r.text
        return None
    except RequestException:
        return None

def parse_one_page(html):
    #pattern=re.compile('<div.*?790.*?<a.*?>(.*?)</a>.*?<div.*?>"(.*?)"</div>',re.S)
    pattern=re.compile('<h3>.*?">(.*?)</a>.*?">(.*?)</div>',re.S)
    items=re.findall(pattern,html)
    for item in items:
        yield {
            'title':item[0],
            'content':item[1].replace(u'\u3000',u'')
        }

def write(content):
    with open('1.txt','a',encoding='utf-8') as f:
        f.write(json.dumps(content,ensure_ascii=False)+'\n')
def main(p):
    url="http://www.neihan8.com/article/index_"+str(p)+".html"
    html=get_one_page(url)
    for item in parse_one_page(html):
        print(item)
        write(item)

if __name__=='__main__':
    #for p in range(2,100):
        #main(p)
    pool=Pool()
    pool.map(main,[p for p in range(2,50)])