- 需求：爬取搜狗指定词条搜索后的页面数据
import requests
import os
#指定搜索关键字
word = input('enter a word you want to search:')
#自定义请求头信息
headers={
'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3497.100 Safari/537.36',
}
#指定url
url = 'https://www.sogou.com/web'
#封装get请求参数
prams = {
'query':word,
'ie':'utf-8'
}
#发起请求
response = requests.get(url=url,params=param)
#获取响应数据，字符串
page_text = response.text
#获取响应数据，二进制，byte
page_text = response.content
-
#返回状态码
page_text = response.status_code
-
#返回响应头信息
page_text = response.headers
-
#获取请求的url
page_text = response.url
-
with open('./sougou.html','w',encoding='utf-8') as fp: fp.write(page_text)
- 需求：登录豆瓣电影，爬取登录成功后的页面数据
import requests
import os
url = 'https://accounts.douban.com/login'
#封装请求参数
data = {
"source": "movie",
"redir": "https://movie.douban.com/",
"form_email": "15027900535",
"form_password": "bobo@15027900535",
"login": "登录",
}
#自定义请求头信息
headers={
'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3497.100 Safari/537.36',
}
response = requests.post(url=url,data=data)
page_text = response.text
with open('./douban111.html','w',encoding='utf-8') as fp:
fp.write(page_text)
- 需求：爬取豆瓣电影分类排行榜 https://movie.douban.com/中的电影详情数据
#!/usr/bin/env python
# -*- coding:utf-8 -*-
import requests
import urllib.request
if __name__ == "__main__":
#指定ajax-get请求的url（通过抓包进行获取）
url = 'https://movie.douban.com/j/chart/top_list?'
#定制请求头信息，相关的头信息必须封装在字典结构中
headers = {
#定制请求头中的User-Agent参数，当然也可以定制请求头中其他的参数
'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/66.0.3359.181 Safari/537.36',
}
#定制get请求携带的参数(从抓包工具中获取)
param = {
'type':'5',
'interval_id':'100:90',
'action':'',
'start':'0',
'limit':'20'
}
#发起get请求，获取响应对象
response = requests.get(url=url,headers=headers,params=param)
#获取响应内容：响应内容为json串
print(response.text)
- 需求：爬取肯德基餐厅查询http://www.kfc.com.cn/kfccda/index.aspx中指定地点的餐厅数据
#!/usr/bin/env python
# -*- coding:utf-8 -*-
import requests
import urllib.request
if __name__ == "__main__":
#指定ajax-post请求的url（通过抓包进行获取）
url = 'http://www.kfc.com.cn/kfccda/ashx/GetStoreList.ashx?op=keyword'
#定制请求头信息，相关的头信息必须封装在字典结构中
headers = {
#定制请求头中的User-Agent参数，当然也可以定制请求头中其他的参数
'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/66.0.3359.181 Safari/537.36',
}
#定制post请求携带的参数(从抓包工具中获取)
data = {
'cname':'',
'pid':'',
'keyword':'北京',
'pageIndex': '1',
'pageSize': '10'
}
#发起post请求，获取响应对象
response = requests.get(url=url,headers=headers,data=data)
#获取响应内容：响应内容为json串
print(response.text)
- 需求：爬取搜狗知乎指定词条指定页码下的页面数据
import requests
import os
#指定搜索关键字
word = input('enter a word you want to search:')
#指定起始页码
start_page = int(input('enter start page num:'))
end_page = int(input('enter end page num:'))
#自定义请求头信息
headers={
'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3497.100 Safari/537.36',
}
#指定url
url = 'https://zhihu.sogou.com/zhihu'
#创建文件夹
if not os.path.exists('./sougou'):
os.mkdir('./sougou')
for page in range(start_page,end_page+1):
#封装get请求参数
params = {
'query':word,
'ie':'utf-8',
'page':str(page)
}
#发起post请求，获取响应对象
response = requests.get(url=url,params=params)
#获取页面数据
page_text = response.text
fileName = word+'_'+str(page)+'.html'
filePath = './sougou/'+fileName
with open(filePath,'w',encoding='utf-8') as fp:
fp.write(page_text)
print('爬取'+str(page)+'页结束')
