<爬虫实战>糗事百科
1.糗事百科段子.py
# 目标:爬取糗事百科段子信息(文字) # 信息包括:作者头像,作者名字,作者等级,段子内容,好笑数目,评论数目 # 解析用学过的几种方法都实验一下①正则表达式.②BeautifulSoup③xpath import requests import re # 正则表达式 import json from bs4 import BeautifulSoup # BS from lxml import etree # xpath def get_one_page(url): response = requests.get(url) if response.status_code == 200: return response.text return None def zhengze_parse(html): pattern = re.compile( '<img src="//(.*?)".*?alt="(.*?)".*?<a.*?<div class=".*?">(.*?)</div>' + '.*?<div class=.*?<span>(.*?)</span>.*?<span class=".*?".*?<i.*?>(.*?)<' + '.*?<i.*?>(.*?)<', re.S) items = re.findall(pattern, html) for item in items: content = item[3].replace('<br/>', '').strip() content = content.replace('\x01', '') if item[5] == '京公网安备11010502031601号': break yield { 'image': "http://" + item[0], 'name': item[1], 'grade': item[2], 'content': content, 'fun_Num': item[4], 'com_Num': item[5] } def soup_parse(html): soup = BeautifulSoup(html, 'lxml') for data in soup.find_all('div', class_='article'): image = "http:" + data.img['src'] name = data.img['alt'] # 匿名用户没有等级 if name=="匿名用户": grade = "匿名用户" else: grade = data.find('div', class_='articleGender').text content = data.find('div', class_='content').span.text.strip() fun_Num = data.find('i', class_='number').text com_Num = data.find('a', class_='qiushi_comments').i.text yield { 'image': image, 'name': name, 'grade': grade, 'content': content, 'fun_Num': fun_Num, 'com_Num': com_Num, } def xpath_parse(html): html = etree.HTML(html) for data in html.xpath('//div[@class="col1"]/div'): image = "http:"+ str(data.xpath('.//img/@src')[0]) name = data.xpath('.//img/@alt')[0] if name == '匿名用户': grade = '匿名用户' else: grade = data.xpath('./div[1]/div/text()')[0] content = data.xpath('./a/div/span/text()')[0:] content = str(content).strip().replace('\\n','') fun_Num = data.xpath('./div[2]/span[1]/i/text()')[0] com_Num = data.xpath('.//div[2]/span[2]/a/i/text()')[0] # print(image, name, grade, content, fun_Num, com_Num) yield { 'image': image, 'name': name, 'grade': grade, 'content': content, 'fun_Num': fun_Num, 'com_Num': com_Num, } def write_to_file(content, flag): with open('糗百段子(' + str(flag) + ').txt', 'a', encoding='utf-8')as f: f.write(json.dumps(content, ensure_ascii=False) + '\n') def search(Num): url = 'https://www.qiushibaike.com/text/page/' + str(Num) + '/' html = get_one_page(url) # 正则匹配不到匿名用户的等级,不会匹配匿名用户的段子,所以少一些数据 # 稍微加个判断逻辑就行了,懒得弄了 for item in zhengze_parse(html): write_to_file(item, '正则表达式') for item in soup_parse(html): write_to_file(item, 'BS4') for item in xpath_parse(html): write_to_file(item, 'xpath') page = str(Num) print("正在爬取第" + page + '页') def main(): # 提供页码 for Num in range(1, 14): search(Num) print("爬取完成") if __name__ == '__main__': # 入口 main()
2.打包
pyinstaller -F 糗事百科段子.py
3.运行效果
网页上匿名用户段子的显示情况