requests库基本使用

请求

基本GET请求

基本写法

import requests 

response = requests.get(url='http://httpbin.org/get')

print(response.text)

{
  "args": {}, 
  "headers": {
    "Accept": "*/*", 
    "Accept-Encoding": "gzip, deflate", 
    "Connection": "close", 
    "Host": "httpbin.org", 
    "User-Agent": "python-requests/2.6.0 CPython/2.7.12 Linux/2.6.32-504.30.3.el6.x86_64"
  }, 
  "origin": "123.207.159.116", 
  "url": "http://httpbin.org/get"
}

带参数GET请求

response = requests.get(url='http://httpbin.org/get?name=yang&age=22')
print(response.text)

{
  "args": {
    "age": "22", 
    "name": "yang"
  }, 
  "headers": {
    "Accept": "*/*", 
    "Accept-Encoding": "gzip, deflate", 
    "Connection": "close", 
    "Host": "httpbin.org", 
    "User-Agent": "python-requests/2.6.0 CPython/2.7.12 Linux/2.6.32-504.30.3.el6.x86_64"
  }, 
  "origin": "123.207.159.116", 
  "url": "http://httpbin.org/get?name=yang&age=22"
}

data = {
    'name': 'yang',
    'age': '23',
}

response = requests.get(url='http://httpbin.org/get', params=data)
print(response.text)

{
  "args": {
    "age": "23", 
    "name": "yang"
  }, 
  "headers": {
    "Accept": "*/*", 
    "Accept-Encoding": "gzip, deflate", 
    "Connection": "close", 
    "Host": "httpbin.org", 
    "User-Agent": "python-requests/2.6.0 CPython/2.7.12 Linux/2.6.32-504.30.3.el6.x86_64"
  }, 
  "origin": "123.207.159.116", 
  "url": "http://httpbin.org/get?age=23&name=yang"
}

解析json

response = requests.get(url='http://httpbin.org/get')
response.json()

# 调用json 方法和 json.loads(response.text) 结果相同

{u'args': {},
 u'headers': {u'Accept': u'*/*',
  u'Accept-Encoding': u'gzip, deflate',
  u'Connection': u'close',
  u'Host': u'httpbin.org',
  u'User-Agent': u'python-requests/2.6.0 CPython/2.7.12 Linux/2.6.32-504.30.3.el6.x86_64'},
 u'origin': u'123.207.159.116',
 u'url': u'http://httpbin.org/get'}

获取二进制数据, 可用来下载图片，视频等

response = requests.get(url='https://github.com/favicon.ico')
# print(type(response.text), type(response.content))

# with open('/tmp/pic.ico', 'wb') as f:
#     f.write(response.content)
    
# print(response.text)
# print(response.content)

添加headers

# headers 请根据实际情况添加
headers = {
    'User-Agent': "Mozilla/5.0 (iPhone; CPU iPhone OS 9_3_4 like Mac OS X) AppleWebKit/601.1.46 (KHTML, like Gecko) Mobile/13G35 QQ/6.5.3.410 V1_IPH_SQ_6.5.3_1_APP_A Pixel/750 Core/UIWebView NetType/2G Mem/117",
}

response = requests.get('https://www.zhihu.com/explore', headers=headers)

# print(response.text)

基本POST请求

data = {'name': 'yang', 'age': 23}

response = requests.post('http://httpbin.org/post', data=data, headers=headers)
print(response.text)

{
  "args": {}, 
  "data": "", 
  "files": {}, 
  "form": {
    "age": "23", 
    "name": "yang"
  }, 
  "headers": {
    "Accept": "*/*", 
    "Accept-Encoding": "gzip, deflate", 
    "Connection": "close", 
    "Content-Length": "16", 
    "Content-Type": "application/x-www-form-urlencoded", 
    "Host": "httpbin.org", 
    "User-Agent": "Mozilla/5.0 (iPhone; CPU iPhone OS 9_3_4 like Mac OS X) AppleWebKit/601.1.46 (KHTML, like Gecko) Mobile/13G35 QQ/6.5.3.410 V1_IPH_SQ_6.5.3_1_APP_A Pixel/750 Core/UIWebView NetType/2G Mem/117"
  }, 
  "json": null, 
  "origin": "123.207.159.116", 
  "url": "http://httpbin.org/post"
}

响应

常用respnse 属性

# 此处的response只是本例中用的变量名称，可以依个人习惯或实际场景用其他变量名

response.status_code   # 返回的状态码 
response.headers       # headers 
response.cookies       # 获取 cookie
response.url           # 请求的url 
response.history       # 访问历史记录 

[]

高级操作

文件上传

file = {'file': open('/tmp/pic.ico', 'rb')}

response = requests.post('http://httpbin.org/post', files=file)

会话维持

s = requests.Session() 
s.get('http://httpbin.org/cookies/set/number/2132')
response = s.get('http://httpbin.org/cookies')
# print（response.status_code, response.cookies）

证书验证

'''
# from requests.packages import urllib3
# urllib3.disable.warnings()     # 取消警告信息
response = requests.get('https://12306.cn', verify=False)   # verify=False 不进行证书验证

# 指定证书 
response = requests.get('https://12306.cn', cert=('/path/server.crt', '/path/key'))

'''
                        

"\n# from requests.packages import urllib3\n# urllib3.disable.warnings()     # \xe5\x8f\x96\xe6\xb6\x88\xe8\xad\xa6\xe5\x91\x8a\xe4\xbf\xa1\xe6\x81\xaf\nresponse = requests.get('https://12306.cn', verify=False)   # verify=False \xe4\xb8\x8d\xe8\xbf\x9b\xe8\xa1\x8c\xe8\xaf\x81\xe4\xb9\xa6\xe9\xaa\x8c\xe8\xaf\x81\n\n# \xe6\x8c\x87\xe5\xae\x9a\xe8\xaf\x81\xe4\xb9\xa6 \nresponse = requests.get('https://12306.cn', cert=('/path/server.crt', '/path/key'))\n\n"

代理设置

proxies = {
    'http': 'http://127.0.0.1:9743',
    'https': 'https://127.0.0.1:9743',
    'http': 'http://user:password@127.0.0.1:9743/',
}

response = requests.get('https://wwww.taobao.com', proxies=proxies)

socks代理

# 安装 
pip install requests[socks]

proxies = {
    'http': 'socks5://127.0.0.1:9743',
    'https': 'socks4://127.0.0.1:9743',
}

response = requests.get('https://wwww.taobao.com', proxies=proxies)

超时设置

response = requests.get('http://www.gogole.com', timeout=10)

认证设置

from requests.auth import HTTPBasicAuth 

r = requests.get('http://123.207.159.116:9001', auth=HTTPBasicAuth('userName', 'password'))
# 或 
r = requests.get('http://123.207.159.116:9001', auth=('userName', 'password'))

异常处理

from requests.exception import ReadTimeout, HTTPError, RequestException, ConnectionError

try:
    r = requests.get('http://www.baidu.com', timeout=10)
    
except ReadTimeout:
    print('ReadTimeout')
    
except HTTPError:
    print('HTTPError')
    
except RequestException:
    print('RequestException')    
    
except ConnectionError:
    print('ConnectionError')