将爬取到的数据存入数据框并导出

import requests
from lxml import etree
from pandas import DataFrame

url='https://search.51job.com/list/120800,000000,0000,32,9,99,%25E4%25BA%25A7%25E5%2593%2581%25E7%25BB%258F%25E7%2590%2586,2,1.html'
res=requests.get(url)
res.encoding='gbk'
print(res)
#用etree生成xpath解析对象
root=etree.HTML(res.text)
print(root)
#利用xpath提取信息
position=root.xpath('//p[@class="t1 "]/span/a/@title')
extract=root.xpath('//p[@class="t1 "]/span/a/text()')
extract=[extract[i].strip() for i in range(len(extract))]
company=root.xpath('//span[@class="t2"]/a/@title')
place=root.xpath('//div[@class="el"]/span[@class="t3"]/text()')  #同一标签下的多属性时并列div[@class="el"][@id="22"]
salary=root.xpath('//div[@class="el"]/span[@class="t4"]/text()')
jobinfo=DataFrame([position,company,place,salary]).T
jobinfo.columns=['职位','公司','地点','薪资']
jobinfo.to_csv('51jbob.csv',encoding='gbk')

 

#利用正则匹配     正则表达式中的模式修饰符及应用

#I忽略大小写    S 让 . 匹配换行符   M多行匹配

import re
import requests
from pandas import DataFrame
import pandas as pd

jobinfoAll=DataFrame()

for i in range(1,6):
url='https://search.51job.com/list/120800%252C010000,000000,0000,33,9,99,%25E9%2594%2580%25E5%2594%25AE%25E7%25BB%258F%25E7%2590%2586,2,str(i).html'
res=requests.get(url)
res.encoding='gbk'

# 职位
pat='<a target="_blank" title="(.*)" href=".*" onmousedown="">'
position=re.findall(pat,res.text)
# 公司
company_pat='<span class="t2"><a target="_blank" title="(.*)" href=".*">.*</a></span>'
company=re.findall(company_pat,res.text)
# 地点
place_pat='<div class="el">.*?<span class="t3">(.*?)</span>'
place=re.findall(place_pat,res.text,re.S)
# 薪资
salary_pat='<div class="el">.*?<span class="t4">(.*?)</span>'
salary=re.findall(salary_pat,res.text,re.S)

jobinfo=DataFrame([position,company,place,salary]).T
jobinfo.columns=['职位','公司','地点','薪资']
jobinfoAll=pd.concat([jobinfoAll,jobinfo]) #把两个合成一个
# print(jobinfo)
jobinfoAll.to_csv('51jbob1.csv',encoding='gbk')
# len(jobinfoAll)