参考网址

https://blog.csdn.net/qq_35709559/article/details/84859927

爬虫示例: 将百度的图片下载到本地


'''
1.找到图片的url
2.发送请求获取响应
3.保存图片 (流媒体必须以二进制方式写入)
'''

import requests

def uploadBaidu():
    url = "https://ss0.bdstatic.com/5aV1bjqh_Q23odCf/static/superman/img/logo/logo_redBlue_32fe2c69.png"
    headers = {
        "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.99 Safari/537.36"
    }
    res = requests.get(url=url,headers=headers)
    # 打开图片
    with open("baidu_log.png", "wb") as f:
        f.write(res.content)
    print("OK")

if __name__ == '__main__':
    uploadBaidu()

发送带参数的请求

  • 以下示例的响应结果,被百度拦截了

import requests

url = "https://www.baidu.com/s"
# 确定查询字符串参数字典
params = {
    'wd':"python"
}

# 设置header信息
headers = {
    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.99 Safari/537.36"
}

# 带上请求头和请求参数
resp = requests.get(url, params=params, headers=headers)

print(resp.content.decode())


import requests

# 确定查询字符串参数字典
url = "https://www.baidu.com/s?wd=python"

# 设置header信息
headers = {
    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.99 Safari/537.36"
}

# 带上请求头和请求参数
resp = requests.get(url, headers=headers)

print(resp.content.decode())
  • 模拟登录
# 百度翻译移动端
# -*-coding:utf-8-*-
import requests
import json
class BaiduTran(object):
    """百度翻译接口"""
    def __init__(self, f="zh", to="en"):
        self.url = "https://fanyi.baidu.com/basetrans"
        self.f = f
        self.to = to
        self.headers = {
            "User-Agent": "Mozilla / 5.0(Linux;Android 5.0;SM - G900P Build / LRX21T) AppleWebKit / 537.36(KHTML, like Gecko) Chrome / 69.0.3497.81 Mobile Safari / 537.36"
        }
        
    def get_resp(self, query):
        """
        获取翻译响应
        :param query:
        :return:
        """
        data = {
            "query":query,
            "from":self.f,
            "to":self.to
        }
        self.resp = requests.post(url=self.url,headers=self.headers, data=data)
        return self.resp

    def get_result(self):
        result = json.loads(self.resp.content.decode())["trans"][0]["dst"]
        return result


if __name__ == '__main__':
    tran = BaiduTran(f="zh", to="en")
    resp = tran.get_resp("今天天气真不错")
    # print(resp.content.decode())
    print(tran.get_result())
  • 使用IP代理
# -*-coding:utf-8-*-
import requests
import json
query = input("请输入要翻译的中文:")
url = "https://fanyi.baidu.com/basetrans"
headers = {
            "User-Agent": "Mozilla / 5.0(Linux;Android 5.0;SM - G900P Build / LRX21T) AppleWebKit / 537.36(KHTML, like Gecko) Chrome / 69.0.3497.81 Mobile Safari / 537.36"
        }
data = {
    'query': query,
    'from': 'zh',
    'to': 'en'
}
# 循环获取代理ip
while True:
    try:
        proxy_str = requests.get("代理ip池的url", timeout=3).text # 获取代理ip
        print("代理ip:%s" % proxy_str)
        proxies = {"https":proxy_str}
        resp = requests.post(url=url,data=data,headers=headers,proxies=proxies)
        break
    except:
        print("更换代理ip...")

print(resp.content.decode())
# 处理响应
print(json.loads(resp.content.decode())["trans"][0]["dst"])
  • 使用requests处理cookie有三种方法
- cookie字符串放在headers中

    headers = {
        "User-Agent":"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_4) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/65.0.3325.181 Safari/537.36",
        "Cookie":" Pycharm-26c2d973=dbb9b300-2483-478f-9f5a-16ca4580177e; Hm_lvt_98b9d8c2fd6608d564bf2ac2ae642948=1512607763; Pycharm-26c2d974=f645329f-338e-486c-82c2-29e2a0205c74; _xsrf=2|d1a3d8ea|c5b07851cbce048bd5453846445de19d|1522379036"}

    requests.get(url,headers=headers)


- 把cookie字典放传给请求方法的cookies参数接收

    requests.get(url,headers=headers,cookies=cookie_dict)


- 使用requests提供的session模块

    session = requests.session()
    response = session.get(url,headers)

    session实例在请求了一个网站后，对方服务器设置在本地的cookie会保存在session中
    下一次再使用session请求对方服务器的时候，会带上前一次的cookie
  • 使用 cookies
import requests

url = "http://www.baidu.com"
#发送请求，获取resposne
response = requests.get(url)
print(type(response.cookies)) # <class 'requests.cookies.RequestsCookieJar'>

#使用方法从cookiejar中提取数据
cookies = requests.utils.dict_from_cookiejar(response.cookies)
print(cookies) # {'BDORZ': '27315'}
  • 证书错误的处理
import requests

url = "https://www.12306.cn/mormhweb/"

# requests.packages.urllib3.disable_warnings() # 不显示安全提示 

response = requests.get(url,verify=False)
  • 超时响应
response = requests.get(url,timeout=3) # 发送请求之后,3秒内返回响应，否则会报错