51Job多页信息爬取
01 导入模块
import requests
import chardet
from bs4 import BeautifulSoup
import csv
from openpyxl import Workbook
02 定义函数getOnePageInfo()
def getOnePageInfo(url):
# 访问链接
res=requests.get(url,
headers={'User-Agent':'Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_8; en-us) AppleWebKit/534.50 (KHTML, like Gecko) Version/5.1 Safari/534.50'}
)
# 转为beautifulsoup对象
soup=BeautifulSoup(res.text,'html.parser')
# 那么我们只能按照实际得到的对象来找信息
allstring=soup.find_all('script')[-4].string
# allstring=soup.find_all('script')[-4].text
# 1：使用 = 分割1次， 的第二个值就是所有数据
data=allstring.split('=',1)[-1]
# 2 :
index=allstring.find('{')
data2=allstring[index:]
# 1使用eval()将字符串转换为相关数据
dict_data=eval(data)
bigdata = []
for each in dict_data['engine_search_result']:
oneInfo=[]
# 职位名 job_name
oneInfo.append(each.get('job_name'))
# 公司名 commpany_name
oneInfo.append(each.get('company_name'))
# 薪资 providesalary_text
oneInfo.append(each.get('providesalary_text'))
# 工作地点 workarea_text
oneInfo.append(each.get('workarea_text'))
# 发布日期 updatedate
oneInfo.append(each.get('updatedate'))
# 公司类型 companytype_text
oneInfo.append(each.get('companytype_text'))
# 额外信息 attribute_text
oneInfo.append(str(each.get('attribute_text')))
# 所属行业 companyind_text
oneInfo.append(each.get('companyind_text'))
#将最后一条信息放入bigdata
bigdata.append(oneInfo)
return bigdata
03 定义存储函数MySave()
# 存储二维列表专用类
class MySave():
def __init__(self):
pass
def saveToCsv(self,data,fileName:str,mode='w'):
with open(fileName,mode=mode,encoding='utf-8',newline='')as f:
csvfile=csv.writer(f)
#写入data
for each in data:
csvfile.writerow(each)
print(fileName,'存储完成')
def saveToExcel(self,data,fileName):
#实例化工作簿对象
wb=Workbook()
#准备工作表
sheet=wb.active
#写入数据
for each in data:
sheet.append(each)
wb.save(fileName)
print(fileName,'存储完成')
04 爬取多页数据
save=MySave()
import random
import time
#抓多页
for i in range(1,11):
time.sleep(random.randint(1,3))
#拼接链接
url=f'https://search.51job.com/list/020000%252c080200,000000,0000,00,9,99,%25E6%2595%25B0%25E6%258D%25AE%25E5%2588%2586%25E6%259E%2590%25E5%25B8%2588,2,{i}.html?lang=c&postchannel=0000&workyear=99&cotype=99°reefrom=99&jobterm=99&companysize=99&ord_field=0&dibiaoid=0&line=&welfare='
print(f'正在抓取第{i}页')
#运行函数访问url，返回数据
data=getOnePageInfo(url)
#存储到csv
save.saveToCsv(data,'10页数据.csv','a')
完整代码
import requests
import chardet
from bs4 import BeautifulSoup
import csv
from openpyxl import Workbook
def getOnePageInfo(url):
# 访问链接
res=requests.get(url,
headers={'User-Agent':'Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_8; en-us) AppleWebKit/534.50 (KHTML, like Gecko) Version/5.1 Safari/534.50'}
)
# 转为beautifulsoup对象
soup=BeautifulSoup(res.text,'html.parser')
# 那么我们只能按照实际得到的对象来找信息
allstring=soup.find_all('script')[-4].string
# allstring=soup.find_all('script')[-4].text
# 1：使用 = 分割1次， 的第二个值就是所有数据
data=allstring.split('=',1)[-1]
# 2 :
index=allstring.find('{')
data2=allstring[index:]
# 1使用eval()将字符串转换为相关数据
dict_data=eval(data)
bigdata = []
for each in dict_data['engine_search_result']:
oneInfo=[]
# 职位名 job_name
oneInfo.append(each.get('job_name'))
# 公司名 commpany_name
oneInfo.append(each.get('company_name'))
# 薪资 providesalary_text
oneInfo.append(each.get('providesalary_text'))
# 工作地点 workarea_text
oneInfo.append(each.get('workarea_text'))
# 发布日期 updatedate
oneInfo.append(each.get('updatedate'))
# 公司类型 companytype_text
oneInfo.append(each.get('companytype_text'))
# 额外信息 attribute_text
oneInfo.append(str(each.get('attribute_text')))
# 所属行业 companyind_text
oneInfo.append(each.get('companyind_text'))
#将最后一条信息放入bigdata
bigdata.append(oneInfo)
return bigdata
# 存储二维列表专用类
class MySave():
def __init__(self):
pass
def saveToCsv(self,data,fileName:str,mode='w'):
with open(fileName,mode=mode,encoding='utf-8',newline='')as f:
csvfile=csv.writer(f)
#写入data
for each in data:
csvfile.writerow(each)
print(fileName,'存储完成')
def saveToExcel(self,data,fileName):
#实例化工作簿对象
wb=Workbook()
#准备工作表
sheet=wb.active
#写入数据
for each in data:
sheet.append(each)
wb.save(fileName)
print(fileName,'存储完成')
save=MySave()
import random
import time
#抓多页
for i in range(1,11):
time.sleep(random.randint(1,3))
#拼接链接
url=f'https://search.51job.com/list/020000%252c080200,000000,0000,00,9,99,%25E6%2595%25B0%25E6%258D%25AE%25E5%2588%2586%25E6%259E%2590%25E5%25B8%2588,2,{i}.html?lang=c&postchannel=0000&workyear=99&cotype=99°reefrom=99&jobterm=99&companysize=99&ord_field=0&dibiaoid=0&line=&welfare='
print(f'正在抓取第{i}页')
#运行函数访问url，返回数据
data=getOnePageInfo(url)
#存储到csv
save.saveToCsv(data,'10页数据.csv','a')
