requests的get方法
1 在百度里面查询关键字的方法,并获取带百度当前页面
import requests keywords = input('请输入>>>').strip() response = requests.get('https://www.baidu.com/s?', params={ 'wd': keywords, 'pn':20 }, headers = { 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.131 Safari/537.36' } ) if response.status_code == 200: with open('b.html', 'wt', encoding='utf-8') as f: f.write(response.text)
2 get请求给知乎
import requests response = requests.get('https://www.zhihu.com', headers={ 'Referer': 'https://www.zhihu.com/', 'user-agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.131 Safari/537.36' } ) with open('c.html', 'wt', encoding='utf-8')as f: f.write(response.text) print(response.status_code) print(response.text)
3 get请求给githup
import requests response = requests.get(url='https://github.com/', headers={ 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.131 Safari/537.36', 'Cookie': '_octo=GH1.1.1333562301.1559296277; _ga=GA1.2.392559115.1559296287; has_recent_activity=1; _gat=1; tz=Asia%2FShanghai; _device_id=0dcf09aab9c4d288aaa33f26fecd1309; user_session=Yp-WRUHkznMCmRXO6-WsL8QRfVCau3k7gQ56zIZHMHfVTRCB; __Host-user_session_same_site=Yp-WRUHkznMCmRXO6-WsL8QRfVCau3k7gQ56zIZHMHfVTRCB; logged_in=yes; dotcom_user=andygouyong; _gh_sess=TTFoakY4c0ZtcHVMc2wrdjJiMmtSejhvN0VsVnhqU01PdW9yL01CMFNHYjZOaUNGUTFmNjlQK0o5NXFmVU40L1AzeUxCV2x0VHBka2VkR3ZBRUtxVnU2YUJPTUM0T3RWM0E5OVJtSklJTmswMXl6WS9lY3lrMGYvd1FoU0NnNVNla0lrZE13TzlIekhoRDA5a1JHcXBIeDNBUXlLZnoxVkd5elNNRmdCUHVZbGttREtyd2JDUWcxS1ZaZFpJZ3pnWUx1Z2p3MEppTGZOZkVMWEMrQ01HRGJxcU5kMWJPa3V5d001OHVsNElaWUowYitYYlFxeDgxNXd4YVdlZEJ5bFViVFdtTCtGQTFHYWZWTjFiSzhodVBPNXdQLzMxSkx3ZkJCeFpUdWJQdzR2dkRhcFhTeTUvZkROczZpWC9GMlVaZjgzTmxhWG5wakh1WnpDOFZpdzZ3PT0tLVFZRmowSjkva3RGY3dqaU15b0VHTkE9PQ%3D%3D--4508766204caae7d9c3ecc0c6e7c0fc8ae887a7f' } ) print(response.status_code) print(response.text) with open('d.html','wt',encoding='utf-8')as f: f.write(response.text)
requests的post方法(模拟登陆githup)
#!/user/bin/env python3 # -*- coding: utf-8 -*- import re, requests import time # 先获取登陆页面,拿到authenticity_token # 然后请求的url为'https://www.githuo.com/login # 请求方式为git r1 = requests.get('https://github.com/login', headers={ 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.131 Safari/537.36' } ) authenticity_token = re.findall('name="authenticity_token" value="(.*?)"', r1.text, re.S)[0] r1_cookies = r1.cookies.get_dict() print(authenticity_token) print(r1_cookies) # 提交数据表单,完成登陆 # 请求方法POST # https://github.com/session # 请求头 # Referer: https://github.com/login # User-Agent: Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.131 Safari/537.36 # cookies=r1_cookies # 请求体 # form date # commit: Sign in # utf8: ✓ # authenticity_token: qGeaCNP3aTAb5B13GiLwYrrO9uth09TU9Wm0CnXBg3cNQowPJJDHHMj0BXjziy1M6uuQVpEScoa9SzubrXDNMg== # login: 你的githup登录名 # password: 你的githup密码 r2 = requests.post( # 请求的url 'https://github.com/session', # 请求的cookies # 请求头 headers={ 'Referer': 'https://github.com', 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.131 Safari/537.36' }, cookies=r1_cookies, # 请求体,勇哥这里得特别别注意 data={ 'commit': 'Sign in', 'utf8': '✓', # 这里里面的authenticity_token就是前面get请求中的,在login页面中拿到 'authenticity_token': authenticity_token, 'login': '你的githup登录名', 'password': '你以为我会吧密码贴出来吗,还是太年轻啊' }, allow_redirects=True ) with open('e.html', 'wt', encoding='utf-8')as f: f.write(r2.text) print(r2.status_code) print('Repositories' in r2.text)
3 爬取梨视频
废话不多说,看勇哥写的代码
import requests import re import os from threading import Thread ppth = os.path.dirname(__file__) def get_index_page(url): # 向目标网站发起请求 response = requests.get(url) # 如果相应的状态吗是200,说明请求成功 if response.status_code == 200: return response.text def parse_index_page(htmll): url = re.findall('class="vervideo-bd".*?href="(.*?)"', htmll, re.S) return url def get_detail_page(url): movie_text = requests.get(url).text return movie_text def parse_detail_page(text): movie_mp4 = re.findall('srcUrl="(.*?)"', text, re.S) title = re.findall('<h1 class="video-tt">(.*?)</h1>', text, re.S) # print(title) if movie_mp4: # print(movie_mp4[0]) return {'title': title[0], 'movie': movie_mp4[0]} def download(movie_mp4): print(movie_mp4) title=movie_mp4['title'] movie_url=movie_mp4['movie'] response=requests.get(movie_url) if response.status_code==200: title=title.replace('"', ' ').replace("'"," ").replace("?"," ").strip() print(title) filename=ppth+'/Download/'+title+'.mp4' with open(filename,'wb') as f: f.write(response.content) # def main(): # # 基础的url # base_url = 'https://www.pearvideo.com/category_{page}' # for i in range(5): # # 获取五条网站数据 # url = base_url.format(page=i) # # 获取网站的html代码 # htmll = get_index_page(url) # # 解析出视频网址 # video_num = parse_index_page(htmll) # for j in video_num: # # 获取到每条视频的url # url_end = base_url[0:26] + j # # print(url_end) # # 解析视频的url数据,拿到.mp4结尾的数据 # movie_text = get_detail_page(url_end) # # 这是一个字典{'title':none,'movie':none} # movie_mp4 = parse_detail_page(movie_text) # # print(movie_mp4) # if movie_mp4: # download(movie_mp4) def main(base_url,i): # 获取五条网站数据 url = base_url.format(page=i) # 获取网站的html代码 htmll = get_index_page(url) # 解析出视频网址 video_num = parse_index_page(htmll) for j in video_num: # 获取到每条视频的url url_end = base_url[0:26] + j # print(url_end) # 解析视频的url数据,拿到.mp4结尾的数据 movie_text = get_detail_page(url_end) # 这是一个字典{'title':none,'movie':none} movie_mp4 = parse_detail_page(movie_text) # print(movie_mp4) if movie_mp4: download(movie_mp4) if __name__ == '__main__': # 基础的url base_url = 'https://www.pearvideo.com/category_{page}' for i in range(5): t=Thread(target=main,args=(base_url,i,)) t.start()
4 响应response
1 response的属性
import requests respone=requests.get('http://www.jianshu.com') # respone属性 print(respone.text) print(respone.content) print(respone.status_code) print(respone.headers) print(respone.cookies) print(respone.cookies.get_dict()) print(respone.cookies.items()) print(respone.url) print(respone.history) print(respone.encoding) #关闭:response.close() from contextlib import closing with closing(requests.get('xxx',stream=True)) as response: for line in response.iter_content(): pass
2 编码问题
#编码问题 import requests,re response=requests.get( 'https://www.autohome.com.cn/shanghai/', headers={ 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.131 Safari/537.36' } ) #汽车之家网站返回的页面内容为gb2312编码的,而requests的默认编码为ISO-8859-1,如果不设置成gbk则中文乱码 response.encoding='gbk' print(response.text) with open('f.html','wt',encoding='gbk')as f: f.write(response.text)
3 获取二进制数据
import requests response=requests.get('https://images.cnblogs.com/cnblogs_com/ouyang99-/1395591/o_1521768608804.jpg') with open('a.jpg','wb')as f: #写二进制数据的时候使用content f.write(tesponse.content)
4 当数据过大时,就容易发生内存撑爆的现象,这时
import requests response=requests.get('https://images.cnblogs.com/cnblogs_com/ouyang99-/1395591/o_1521768608804.jpg') with open('a.jpg','wb')as f: #写二进制数据的时候使用content for line in response.iter_content(): f.write(line)
#这样来一段一段的写入文件,就可以很好的避免上述的问题
5 解析json
#解析json import requests response=requests.get('http://httpbin.org/get') import json res1=json.loads(response.text) #太麻烦 res2=response.json() #直接获取json数据 print(res1 == res2) #True
5response进阶用法
1、SSL Cert Verification
![](https://images.cnblogs.com/OutliningIndicators/ContractedBlock.gif)
#证书验证(大部分网站都是https) import requests respone=requests.get('https://www.12306.cn') #如果是ssl请求,首先检查证书是否合法,不合法则报错,程序终端 #改进1:去掉报错,但是会报警告 import requests respone=requests.get('https://www.12306.cn',verify=False) #不验证证书,报警告,返回200 print(respone.status_code) #改进2:去掉报错,并且去掉警报信息 import requests from requests.packages import urllib3 urllib3.disable_warnings() #关闭警告 respone=requests.get('https://www.12306.cn',verify=False) print(respone.status_code) #改进3:加上证书 #很多网站都是https,但是不用证书也可以访问,大多数情况都是可以携带也可以不携带证书 #知乎\百度等都是可带可不带 #有硬性要求的,则必须带,比如对于定向的用户,拿到证书后才有权限访问某个特定网站 import requests respone=requests.get('https://www.12306.cn', cert=('/path/server.crt', '/path/key')) print(respone.status_code)
2、使用代理
![](https://images.cnblogs.com/OutliningIndicators/ContractedBlock.gif)
#官网链接: http://docs.python-requests.org/en/master/user/advanced/#proxies #代理设置:先发送请求给代理,然后由代理帮忙发送(封ip是常见的事情) import requests proxies={ 'http':'http://egon:123@localhost:9743',#带用户名密码的代理,@符号前是用户名与密码 'http':'http://localhost:9743', 'https':'https://localhost:9743', } respone=requests.get('https://www.12306.cn', proxies=proxies) print(respone.status_code) #支持socks代理,安装:pip install requests[socks] import requests proxies = { 'http': 'socks5://user:pass@host:port', 'https': 'socks5://user:pass@host:port' } respone=requests.get('https://www.12306.cn', proxies=proxies) print(respone.status_code)
3、超时设置
![](https://images.cnblogs.com/OutliningIndicators/ContractedBlock.gif)
#超时设置 #两种超时:float or tuple #timeout=0.1 #代表接收数据的超时时间 #timeout=(0.1,0.2)#0.1代表链接超时 0.2代表接收数据的超时时间 import requests respone=requests.get('https://www.baidu.com', timeout=0.0001)
4、 认证设置
![](https://images.cnblogs.com/OutliningIndicators/ContractedBlock.gif)
#官网链接:http://docs.python-requests.org/en/master/user/authentication/ #认证设置:登陆网站是,弹出一个框,要求你输入用户名密码(与alter很类似),此时是无法获取html的 # 但本质原理是拼接成请求头发送 # r.headers['Authorization'] = _basic_auth_str(self.username, self.password) # 一般的网站都不用默认的加密方式,都是自己写 # 那么我们就需要按照网站的加密方式,自己写一个类似于_basic_auth_str的方法 # 得到加密字符串后添加到请求头 # r.headers['Authorization'] =func('.....') #看一看默认的加密方式吧,通常网站都不会用默认的加密设置 import requests from requests.auth import HTTPBasicAuth r=requests.get('xxx',auth=HTTPBasicAuth('user','password')) print(r.status_code) #HTTPBasicAuth可以简写为如下格式 import requests r=requests.get('xxx',auth=('user','password')) print(r.status_code)
5、异常处理
![](https://images.cnblogs.com/OutliningIndicators/ContractedBlock.gif)
#异常处理 import requests from requests.exceptions import * #可以查看requests.exceptions获取异常类型 try: r=requests.get('http://www.baidu.com',timeout=0.00001) except ReadTimeout: print('===:') # except ConnectionError: #网络不通 # print('-----') # except Timeout: # print('aaaaa') except RequestException: print('Error')
6、上传文件
![](https://images.cnblogs.com/OutliningIndicators/ContractedBlock.gif)
import requests files={'file':open('a.jpg','rb')} respone=requests.post('http://httpbin.org/post',files=files) print(respone.status_code)