爬虫学习--Requests库详解 Day2

什么是Requests

Requests是用python语言编写，基于urllib，采用Apache2 licensed开源协议的HTTP库，它比urllib更加方便，可以节约我们大量的工作，完全满足HTTP测试需求。
一句话总结：它是Python实现的简单易用的HTTP库

安装Requests  pip install requests

验证没有报错，表示已经成功的安装了

实例引入

import requests
response = requests.get('https://www.baidu.com')
print(type(response))
print(response.status_code) # 状态码
print(type(response.text))
print(response.text) # 响应的内容
print(response.cookies) # 获取cookie

各种请求方式

import requests
print(requests.post('http://httpbin.org/post')) #
print(requests.put('http://httpbin.org/put'))
print(requests.delete('http://httpbin.org/delete'))
print(requests.head('http://httpbin.org/get'))
print(requests.options('http://httpbin.org/get'))

Requests库请求具体怎么用的
基本GET请求-----------------------------------------------------------------------------------------------------------
基本写法

import requests

response = requests.get('http://httpbin.org/get')
print(response.text) # 请求头啊，请求的IP地址，请求的链接

带参数的GET请求

import requests

response = requests.get('http://httpbin.org/get?name=germey&age=22')
print(response.text)

也可以用字典的形式传参 params

import requests

data = {
    'name':'xiaohu',
    'age':'21'
}
response = requests.get('http://httpbin.org/get',params=data)
print(response.text)

解析json

import requests

response = requests.get('http://httpbin.org/get')
print(type(response.text))
print(response.json())
print(type(response.json())) # 它是一个字典的类型

区别用json.loads与直接.json有什么不一样,结果其实是一样的

import requests
import json

response = requests.get('http://httpbin.org/get')
print(type(response.text))
print(response.json())
print(json.loads(response.text)) #区别用json.loads与直接.json有什么不一样,结果其实是一样的
print(type(response.json())) # 它是一个字典的类型

获取二进制数据

import requests

response = requests.get('https://github.com/favicon.ico')
print(type(response.text),type(response.content))  # .content是二进制内容
print(response.text)
print(response.content)

获取到图片把它保存

 import requests
3 response = requests.get('https://github.com/favicon.ico')
with open('favicon','wb') as f:
    f.write(response.content)
    f.close()

添加headers 如果不加headers 报错500
import requests

headers = {
    'User-Agent':'Mozilla/5.0(Macintosh;intel Mac OS X 10_11_4)AppleWebKit/537.36(KHTML,like Gecko)Chrome/52.0.2743.116 Safari/537.36'
}
response = requests.get("https://www.zhihu.com/explore",headers=headers)
print(response.text)

基本POST请求

import requests

data = {
    'name':'xiaohu',
    'age':'21',
    'job':'IT'
}
response = requests.post('http://httpbin.org/post',data=data)
print(response.text)

import requests

data = {
    'name':'xiaohu',
    'age':'21',
    'job':'IT'
}
headers = {
    'User-Agent':'Mozilla/5.0(Macintosh;intel Mac OS X 10_11_4)AppleWebKit/537.36(KHTML,like Gecko)Chrome/52.0.2743.116 Safari/537.36'
}
response = requests.post('http://httpbin.org/post',data=data,headers=headers)
print(response.json())

响应
response属性

import requests

response = requests.get('http://www.jianshu.com')
print(type(response.status_code),response.status_code) # 状态码
print(type(response.headers),response.headers) # 请求头
print(type(response.cookies),response.cookies) #
print(type(response.url),response.url)
print(type(response.history),response.history) # 访问的历史记录

高级操作
文件上传

import requests

files = {
    'file':open('favicon','rb')
}
response = requests.post('http://httpbin.org/post',files=files)
print(response.text)

获取cookie

import requests

response = requests.get('https://www.baidu.com')
print(response.cookies)
print(type(response.cookies))
for key,value in response.cookies.items():
    print(key+'='+value)

会话维持

import requests
requests.get('http://httpbin.org/cookies/set/number/1165872335') # 为网站设置一个cookies
response = requests.get('http://httpbin.org/cookies') # 再用get访问这个cookies
print(response.text) # 为空，因为这里进行了两次get请求，相当于两个浏览器分别设置cookies和访问cookies,相对独立的

改进

import requests
S = requests.Session() # 声明对象
S.get('http://httpbin.org/cookies/set/number/1165872335') # 实现在同一个浏览器进行设置rookies和访问rookies
response = S.get('http://httpbin.org/cookies') # 再用get访问这个cookies
print(response.text) # 此时不为空

证书验证

import requests
from requests.packages import urllib3
urllib3.disable_warnings() # 消除警告信息

response = requests.get('https://www.12306.cn',verify=False) # verify=False 不需要验证进入，但是有警告
print(response.status_code)

import requests

response = requests.get('https://www.12306.cn',cert=('/path/server.crt','/path/key')) # 指定的证书
print(response.status_code)

代理设置

import requests
import socks5

proxies = {
    "http":"socks5://127.0.0.1:8080",
    "https":"socks5://127.0.0.1:8080",
}

response = requests.get('https://www.taobao.com',proxies=proxies)
print(response.status_code)

超时设置

import requests
from requests.exceptions import ReadTimeout

try:
    response = requests.get('https://www.httpbin.org/get',timeout = 0.2)
    print(response.status_code)
except ReadTimeout:
    print("timeout")

认证设置

import requests
from requests.auth import HTTPBasicAuth

r = requests.get('http://127.27.34.24:9001',auth=HTTPBasicAuth('user','123'))
print(r.status_code)

# 第二种方式
import requests

r = requests.get('http://127.27.34.24:9001',auth=('user','123'))
print(r.status_code)

异常处理

import requests
from requests.exceptions import ReadTimeout,HTTPError,RequestException,ConnectionError
try:
    response = requests.get('http://httpbin.org/get',timeout = 0.2)
    print(response.status_code)
except ReadTimeout:
    print('Timeout')
except HTTPError:
    print('Http error')
except ConnectionError:
    print('Connection Error')
except RequestException:
    print('error')