【fenbi】python爬虫用于做题复习
#!/usr/bin/env python
# coding: utf-8
###此函数用于解析题目和每道题的答案
def jiexi(liebiao):
new = []
timu_last = []
for each in liebiao:
new.append(re.sub(r'flag=\\"tex\\" ','',each))
for each in new:
timu_last.append(re.sub(r'\\','',each))
return timu_last
###此函数用于解析选项
def xuanxiang(liebiao):
xuanxiang_v2 = []
xuanxiang_v3 = []
for each in liebiao:
a = re.sub('<p>','',each)
a = re.sub('</p>','',a)
xuanxiang_v2.append(a)
for each in xuanxiang_v2:
each = each+'</p>'
xuanxiang_v3.append(each)
return xuanxiang_v3
import requests
import re
#import pdfkit
import os
import codecs
print "请输入练习的网址："
url = '填入粉笔网你做的15道题的网址'#raw_input(unicode('网址','utf-8').encode('gbk'))
print "url:"+url
###获取本节练习id
id_ = re.findall(r'https://www.fenbi.com/spa/tiku.*?/xingce/xingce/(.*?)/',url,re.S)[0]
mid_url = 'https://tiku.fenbi.com/api/xingce/exercises/'+str(id_)+'?app=web&kav=12&version=3.0.0.0'
print "mid_url:"+mid_url
headers = {
'Cookie': '你自己的cookie; persistent=你的信息; sess=你的信息; userid=你的id'
#####完整的headers
}
response = requests.get(url=mid_url,headers=headers)
response.encoding = 'utf-8'
page_text_1 = response.text
###获取题目组参数
id_list = re.findall('\"questionIds\"\:\[(.*?)\]\,',page_text_1,re.S)
###获取已答题目的编码（排除没做的题）
your_answerid = re.findall(r'"(.{1,2})":{"questionId',page_text_1,re.S)
###获取自己的答案
your_answer = re.findall(r'"answer":{"choice":"(.*?)",',page_text_1,re.S)
###此练习名称
name = re.findall(r'"name":"(.*?)",',page_text_1,re.S)[0]
###真正存储数据的包
timu_url = 'https://tiku.fenbi.com/api/xingce/questions'
params = {
'ids': id_list
}
response = requests.get(url=timu_url,headers=headers,params=params)
response.encoding = 'utf-8'
page_text_2 = response.text #【问题】文本数据
###获取正确答案
true_answer = re.findall('"correctAnswer":{"choice":"(.*?)"',page_text_2,re.S)
###真正存储数据的包
solution_url = 'https://tiku.fenbi.com/api/xingce/solutions'
response = requests.get(url=solution_url,headers=headers,params=params)
response.encoding = 'utf-8'
page_text_3 = response.text#【解析】文本数据
###获取解析
solution_list = re.findall('"solution":"(.*?)","source"',page_text_3,re.S) #正则匹配以solution开始，source结束
solution_last = jiexi(solution_list)
cailiao = []
timu = []
###获取单选题题目和复合题的题目
for each in response.json():
timu.append(each['content'])
try:
cailiao.append(each['material']['content'])
except:
cailiao.append('none')
###获取选项信息
A_option = re.findall('\"options\"\:\[\"(.*?)\"\,\".*?\"\,\".*?\"\,\".*?\"\]',page_text_2,re.S)
B_option = re.findall('\"options\"\:\[\".*?\"\,\"(.*?)\"\,\".*?\"\,\".*?\"\]',page_text_2,re.S)
C_option = re.findall('\"options\"\:\[\".*?\"\,\".*?\"\,\"(.*?)\"\,\".*?\"\]',page_text_2,re.S)
D_option = re.findall('\"options\"\:\[\".*?\"\,\".*?\"\,\".*?\"\,\"(.*?)\"\]',page_text_2,re.S)
"""
A_option = xuanxiang(A_option)
B_option = xuanxiang(B_option)
C_option = xuanxiang(C_option)
D_option = xuanxiang(D_option)
A_option = jiexi(A_option)
B_option = jiexi(B_option)
C_option = jiexi(C_option)
D_option = jiexi(D_option)
"""
###构造HTML代码
count = 0
count_1 = 0
all_content = "<!DOCTYPE html>\n<meta charset='utf-8'>\n<html>"
all_content_0 = "<!DOCTYPE html>\n<meta charset='utf-8'>\n<html>"
print 'true_answer ',true_answer
print 'your_answer ',your_answer
print 'your_answerid ',your_answerid
for each in true_answer:
if your_answerid[count_1]==str(count):
if each != your_answer[count_1]:
###处理复合题
if cailiao[count_1] != 'none' and cailiao[count_1] not in all_content:
all_content += cailiao[count_1]
all_content += str(count_1+1)
all_content += '、'
all_content += timu[count_1][3:]
all_content += 'A、'
all_content += A_option[count_1]
all_content += 'B、'
all_content += B_option[count_1]
all_content += 'C、'
all_content += C_option[count_1]
all_content += 'D、'
all_content += D_option[count_1]
all_content += '<br>'
if count_1 < len(your_answerid)-1:
count_1 += 1
if count < len(true_answer)-1:
count += 1
count = 0
count_1 = 0
all_content += '<br>'
for each in true_answer:
if your_answerid[count_1]==str(count):
each_res = your_answer[count_1]
if count_1 < len(your_answerid)-1:
count_1 += 1
else:
each_res = "NULL_ANSWER"
if each != each_res:
#如果答案中含有img属性，则添加缺失部分正常显示图片
if A_option[count].find('img') != -1:
A_option_list = list(A_option[count])
nPos_1=A_option[count].index('"//fb.')-1
nPos_2=A_option[count].index('formulas?')+8 #插入此字符串尾部
nPos_3=A_option[count].index('\" /></p>')-1
A_option_list.pop(nPos_1)
A_option_list.insert(nPos_2,'fontSize=18&')
A_option_list.pop(nPos_3)
A_option[count] = ''.join(A_option_list)
if B_option[count].find('img') != -1:
B_option_list = list(B_option[count])
nPos_1=B_option[count].index('"//fb.')-1
nPos_2=B_option[count].index('formulas?')+8 #插入此字符串尾部
nPos_3=B_option[count].index('\" /></p>')-1
B_option_list.pop(nPos_1)
B_option_list.insert(nPos_2,'fontSize=18&')
B_option_list.pop(nPos_3)
B_option[count] = ''.join(B_option_list)
if C_option[count].find('img') != -1:
C_option_list = list(C_option[count])
nPos_1=C_option[count].index('"//fb.')-1
nPos_2=C_option[count].index('formulas?')+8 #插入此字符串尾部
nPos_3=C_option[count].index('\" /></p>')-1
C_option_list.pop(nPos_1)
C_option_list.insert(nPos_2,'fontSize=18&')
C_option_list.pop(nPos_3)
C_option[count] = ''.join(C_option_list)
if D_option[count].find('img') != -1:
D_option_list = list(D_option[count])
nPos_1=D_option[count].index('"//fb.')-1
nPos_2=D_option[count].index('formulas?')+8 #插入此字符串尾部
nPos_3=D_option[count].index('\" /></p>')-1
D_option_list.pop(nPos_1)
D_option_list.insert(nPos_2,'fontSize=18&')
D_option_list.pop(nPos_3)
D_option[count] = ''.join(D_option_list)
all_content = all_content + u'第'+str(count+1)+u'题' + timu[count] + A_option[count] + '***' + B_option[count] + '***' + C_option[count] + '***' + D_option[count] + '<br>'
all_content_0 = all_content_0 + u'第'+str(count+1)+u'题' + timu[count] + A_option[count] + '***' + B_option[count] + '***' + C_option[count] + '***' + D_option[count] + '<br>'
temp = u'第'+str(count+1)+u'题的正确答案为'
all_content += temp
if true_answer[count]=='0':
all_content += 'A'
elif true_answer[count]=='1':
all_content += 'B'
elif true_answer[count]=='2':
all_content += 'C'
elif true_answer[count]=='3':
all_content += 'D'
all_content += solution_last[count]
all_content += '<br>'
if count < len(true_answer)-1:
count += 1
all_content += '</html>'
all_content_0 += '</html>'
path_name_0 = 'G:\\lqj\\' + name + u'(题目).html'
path_name_1 = 'G:\\lqj\\' + name + u'(解析).html'
###保存为HTML文件
with codecs.open(path_name_0,'w',encoding='utf-8') as fp:
fp.write(all_content_0)
with codecs.open(path_name_1,'w',encoding='utf-8') as fp:
fp.write(all_content)
###删除HTML文件
#os.remove(path_name)
print "完成任务"
代码来源
Python爬虫自动化获取华图和粉笔网站的错题(推荐)
参照这位博主的代码，进行部分修正，适合自己使用，望各位惠存
study just for life！
