AcWing做题数爬取
请务必保证是在账号登录的状态下食用
Python
准备步骤
F12
开启网站控制台- 在
网络
界面下刷新, 找到名称为www.acwing.com
的选项
- 点击, 并在
请求标头
下寻找Cookie
值, 复制到代码中去
- 在代码中修改网页总数
代码
import re
import os
# 检查库文件,没有就自动安装
os.system('pip3 install -i https://pypi.tuna.tsinghua.edu.cn/simple requests==2.24.0')
import requests
from lxml import etree
class Spider(object):
def __init__(self):
self.base_url = 'https://www.acwing.com/problem/{}/'
self.headers = {
# 修改网站`Cookie`
'Cookie': '',
'Referer': 'https://www.acwing.com/about/',
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36'
' (KHTML, like Gecko) Chrome/93.0.4577.63 Safari/537.36'
}
self.count = 0
def get_html(self, url):
html = requests.get(
url=url,
headers=self.headers
).text
return html
def xpath_func(self, html):
name_bds = '//tbody/tr[./td/span[@title="已通过这道题目"]]/td/a/text()'
base_obj = etree.HTML(html)
name_lists = base_obj.xpath(name_bds)
L = []
for i in name_lists:
L.append(i.strip())
return L
def re_func(self, html, re_bds):
pattern = re.compile(re_bds, re.S)
re_list = pattern.findall(html)
return re_list
def parse_html(self, url):
html = self.get_html(url)
L = self.xpath_func(html)
return L
def run(self):
warning = input('您马上就要爬取acwing了,看一下你的做题数,您的劳动成果将会在下面展示出来,确定要看吗?(Y/N)')
if warning == 'Y':
print('爬虫系统已经启动...正在努力抓取,请稍等....')
print('+---------------------------------+')
print('| name |')
print('+---------------------------------+')
# 修改网站总数
for i in range(1, 85):
url = self.base_url.format(i)
L = self.parse_html(url)
for _ in L:
self.count += 1
print('| ' + _)
print('+---------------------------------+')
print('经过您的不懈努力,您一共做了' + str(self.count) + '道题,继续努力!!')
else:
print('已经退出,你这个弱者')
if __name__ == '__main__':
spider = Spider()
spider.run()