# 爬虫流程
# UA伪装
# 指定url
# 发起请求,输入搜索内容
#获取数据
#持久化存储
各个模块作用
requests 发送请求、获取数据、处理数据自动转码等
response=requests.get(url)
response.text
|
*****text中存的是Beautifulsoup根据猜测的编码方式将content内容编码成字符串******
|
response.content
|
content中间存的是字节码(二进制数据)
|
response.status_code
|
http响应码
|
headers
|
打印请求头
|
cookies
|
打印cookies
|
1 import requests 2 3 # UA伪装 4 headers={ 5 "User-Agent": "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.198 Safari/537.36" 6 ,"Connection": "keep-alive" 7 8 } 9 10 11 if __name__ == '__main__': 12 # 指定url 13 url="https://www.sogou.com/web?" 14 # 发起请求,输入搜索内容 15 sou=input("输入搜搜内容") 16 params={ 17 "query":sou 18 } 19 print(url) 20 request=requests.get(url=url,params=params,headers=headers) 21 #获取数据 22 text=request.text 23 # print(text) 24 #持久化存储 25 filename=sou+".html" 26 with open(filename,"w+",encoding="utf-8") as fp: 27 fp.write(text) 28 print("完成")