scrapy模拟知乎登录(无验证码机制)
---恢复内容开始---
spiders 文件夹下新建zhihu.py文件（从dos窗口中进入虚拟环境，再进入工程目录之后输入命令 scrapy genspider zhihu www.zhihu.com）
#zhihu.py
import scrapy
import re
import json
from Item import ZhihuQuestionItem,ZhihuAnswerItem
import datatime
from scrapy.loader import ItemLoader
try:
import urlparse as parse
except:
from urllib import parse
class ZhuhuSpider(scrapy.Spider):
name='zhihu'
allow_domains=["www.zhihu.com"]
start_urls=["http://www.zhihu.com/"]
headers={
"HOST":"www.zhihu.com",
"Referer":"https://www.zhihu.com",
"User-Agent":"Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/59.0.3071.86 Safari/537.36"
}
start_answer_url="https://www.zhihu.com/api/v4/questions/{0}/answers? sort_by=default&include=data%5B%2A%5D.is_normal%2Cis_sticky%2Ccollapsed_by%2Csuggest_edit%2Ccomment_count%2Ccollapsed_counts%2Creviewing_comments_count%2Ccan_comment%2Ccontent%2Ceditable_content%2Cvoteup_count%2Creshipment_settings%2Ccomment_permission%2Cmark_infos%2Ccreated_time%2Cupdated_time%2Crelationship.is_author%2Cvoting%2Cis_thanked%2Cis_nothelp%2Cupvoted_followees%3Bdata%5B%2A%5D.author.is_blocking%2Cis_blocked%2Cis_followed%2Cvoteup_count%2Cmessage_thread_token%2Cbadge%5B%3F%28type%3Dbest_answerer%29%5D.topics&limit={1}&offset={2}" #question 第一页answer请求的url
def parse(self,response):
"""
提取出html页面中的所有url,并跟踪这些url进行进一步爬取
如果提取出的url中格式为/requestion/xxx,就下载之后直接进入解析函数
"""
all_urls=response.css("a::attr(href)").extract()
all_urls=[parse.urljoin(response.url,url) for url in all_urls]
all_urls=fliter(lambda x:True if x.startswith("https") else False,all_urls)
for url in all_urls:
match_obj=re.match("(.*zhihu.com/requestion/(\d))(/|$).*",url)
if match_obj:
#如果提取到requestion相关页面，交由parse_question进行解析
request_url=match_obj.group(1)
yield scrapy.Request(request_url,headers=self.headers,callback=self.parse_question)
else:
#如果未提取到相关页面，则直接进一步跟踪
yield scrapy.Request(url,headers=self.headers,callback=self.parse)
def parse_question(self,response):
#处理question页面，从页面中提取出具体的question item
match_obj=re.match("(.*zhuhu.com/question/(\d))(/|$).*",response.url)
if match_obj:
question_id=int(match_obj.group(2))
item_loader=ItemLoader(item=zhuhuQuestionItem(),response=response)
if "QuestionHeader-title" in response.text: #处理新版本
item_loader.add_css("title","h1.QuestionHeader-title::text")
item_loader.add_css("content",".QuestionHeader-detail")
item_loader.add_value("url",response.url)
item_loader.add_value("zhuhu_id",question_id)
item_loader.add_css("answer_num",".List-headerText span::text")
item_loader.add_css("comment_num",".QuestionHeader-actions button::text")
item_loader.add_css("watch_user_num",".NumberBoard-value::text")
item_loader.add_css("topic",".QuestionHeader-topics .Popover div::text")
else: #处理旧版本页面item的提取
item_loader.add_xpath("title","//*[@id='zh-question-title']/h2/a/text()|//*[@id='zh-question-title']/h2/span/text()")
item_loader.add_css("content","#zh-question-detail")
item_loader.add_value("url",response.url)
item_loader.add_value("zhuhu_id",question_id)
item_loader.add_css("answer_num", "#zh-question-answer-num::text")
item_loader.add_css("comment_num","#zh-question-meta-wrap a[name='addcomment']::text")
item_loader.add_xpath("watch_user_num","//*[@id='zh-question-side-header-wrap']/text()|//*[@class='zh-question-followers-sidebar']/div/a/strong/text()")
item_loader.add_css("topic",".zm-tag-editor-labels a::text")
question_item=item_loader.load_item()
yield scrapy.Request(self.start_answer_url,format(question_id,20,0),headers=self.headers,callback=self.parse_answer)
yield question_item
def parse_answer(self,response):
#处理question中的answer
ans_json=json.load(response.text)
is_end=ans_json["paging"]["is_end"]
next_url=ans_json["paging"]["next"]
#提取answer的具体字段
for answer in ans_json["data"]:
answer_item=ZhihuAnswerItem()
answer_item["zhihu_id"]=answer["id"]
answer_item["url"]=answer["url"]
answer_item["question_id"]=answer["question"]["id"]
answer_item["author_id"] = answer["author"]["id"] if "id" in answer["author"] else None
answer_item["content"] = answer["content"] if "content" in answer else None
answer_item["praise_num"]=answer["voteup_count"]
answer_item["comment_num"]=answer["comment_count"]
answer_item["creat_time"]=answer["created_time"]
answer_item["update_time"]=answer["update_time"]
answer_item["crawl_time"]=datatime.datatime.now()
yield answer_item
if not is_end:
yield scrapy.Request(next_url,headers=self.headers,callback=self.answer.parse_answer)
#重写start_Request方法
def start_requests(self):
return [scrapy.Request("https://www.zhihu.com/#signin",headers=self.headers,callback=self.login)] #使用scrapy.Request一定要使用回调函数，否则会默认回调parse(self,response)
def login(self,response):
response_text=response.text
match_obj=re,match(' .*name="_xsrf" value="(.*?)" ',response_text,re.DOTALL) #注意使用单双引号
xsrf=""
if match_obj:
xsrf=(match_obj.group(1))
if xsrf:
post_url="https://www.zhihu.com/login/phone_num"
post_data={
"_xsrf" : xsrf,
"phone_num" : "18282902586",
"password" : "admin123"
}
return [scrapy.FormRequest(
url=post_url,
formdata=post_data,
headers=self.headers,
callback=self.check_login
)]
def check_login(self,response):
text_json=json.loads(response.text)
if "msg" in text_json and text_json["msg"]=="登陆成功":
for url in self.start_urls:
yield scrapy.Request(url,dont.fliter=True,headers=self.headers)
---恢复内容结束---
