Python中的urllib模块(爬虫)
#普通爬虫
import urllib.request
repon=urllib.request.urlopen("http://www.baidu.com") #获取访问url返回的数据
data=repon.read().decode("utf-8") #将返回的数据保存到data中并指定编码格式为utf-8
#decode()指定编码格式
data1=repon.geturl() #url
data2=repon.getcode() #状态吗
data3=repon.info() #当前对象的属性
url="http://112.124.20.112/tester/admin/admin/do.c?username=wfw123&password=Wang%40123&userPwTimeStamp=6471310007861248&mvcode=1234"
newurl=urllib.request.unquote(url) #对url进行编码/解码
解码 unquote() 编码:quote()
#反爬虫(模拟浏览器-url+请求头+请求参数+请求体)
import urllib.request
import random
url="http://www.baidu.com"
'''
#模拟请求头
headers={
"User-Agent":
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.106 Safari/537.36",
}
req=urllib.request.Request(url,headers=headers)
'''
#模拟多个浏览器的请求头
AgentList=[
"Mozilla/5.0 (Windows NT 10.0; …) Gecko/20100101 Firefox/63.0",
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.106 Safari/537.36"
]
a=random.choice(AgentList)
req=urllib.request.Request(url)
req.add_header("User-Agent",a)
response=urllib.request.urlopen(req)
data=response.read().decode("utf-8")
#print(data)
#把获取的内容写入到文件内
path=r"C:\Users\admin\Desktop\1.txt"
#需要指定utf-8,否则出现编码格式错误
with open(path,"a+",encoding='utf-8') as f:
f.writelines(data)