#代码来源 雨敲窗博客
# -*- coding: utf-8 -*-
import re
import requests
import html
import time
def crawl_joke_list(page=1):
url = "http://www.qiushibaike.com/8hr/page/" + str(page)
res = requests.get(url)
#抽取每个段子的div的正则
pattern = re.compile("<div class=\"article block untagged mb15 typs_long.*?<div class=\"content\">.*?</div>",re.S) #re.S匹配多行
#把<br/>替换成换行
body = html.unescape(res.text).replace("<br/>","\n")
body = re.sub(r'<span>',"",body)
body = re.sub(r'</span>', "", body)
body = re.sub(r'<span class="contentForAll">查看全文</span>',"", body)
m = pattern.findall(body)
#抽取用户名的正则
user_pattern = re.compile("<div class=\"author clearfix\">.*?<h2>(.*?)</h2>",re.S)
#抽取段子的正则
content_pattern = re.compile("<div class=\"content\">(.*?)</div>",re.S)
for joke in m:
user = user_pattern.findall(joke)
output = []
if len(user) > 0:
output.append(user[0])
content = content_pattern.findall(joke)
if len(content) > 0:
output.append(content[0].replace("\n",""))
print("\t".join(output))
time.sleep(1)
if __name__ == '__main__':
for i in range(1,10):
crawl_joke_list(i)
