方式一 selenium
from selenium import webdriver
import time
from selenium.webdriver.common.keys import Keys
bro=webdriver.Chrome()
def get_resume(bro):
li_list=bro.find_elements_by_class_name('con_list_item')
for li in li_list:
title = li.find_element_by_css_selector('.position_link h3').text
work_time = li.find_element_by_css_selector('.li_b_l').text
company_name = li.find_element_by_css_selector('.company_name').text
add = li.find_element_by_css_selector('.add').text
welfare = li.find_element_by_css_selector('.li_b_r').text
print('''
标题：%s
工作时间：%s
公司名称：%s
公司地址：%s
公司福利：%s
'''%(title, work_time, company_name, add, welfare))
#查找下一页按钮
next = bro.find_element_by_css_selector('.pager_next ')
next.click()
# 点击翻页之后要休息两秒，不能让浏览器操作太频繁，否则会报错
time.sleep(2)
# 继续抓取下一页
get_resume(bro)
time.sleep(5)
try:
bro.get('https://www.lagou.com/')
#隐士等待
bro.implicitly_wait(10)
addr_click = bro.find_element_by_partial_link_text('上海站').click()
input_search=bro.find_element_by_id('search_input')
input_search.send_keys("python")
#模拟键盘操作(模拟键盘敲回车)
input_search.send_keys(Keys.ENTER)
time.sleep(1)
no = bro.find_element_by_css_selector('.body-btn').click()
get_resume(bro)
except Exception as e:
print(e)
finally:
bro.close()
方式二普通方法
import requests
import json
headers = {
'Accept-Language': "zh-CN,zh;q=0.9",
'Host': 'www.lagou.com',
'User-Agent': "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3493.3 Safari/537.36",
'Referer': "https://www.lagou.com/jobs/list_python?city=%E4%B8%8A%E6%B5%B7&cl=false&fromSearch=true&labelWords=&suginput=",
'Cookie': "_ga=GA1.2.1497342262.1582512706; user_trace_token=20200224105145-cc6cf3ff-ce98-45e4-987a-b86e8c600c0a; LGUID=20200224105145-b49fd29f-44e3-4e76-99b1-81d214d196c6; _gid=GA1.2.1855310676.1586229731; JSESSIONID=ABAAAECAAFDAAEH3E452981BE6E4D71C27858CD16B95E47; WEBTJ-ID=20200407223352-171550fd3077e1-099a2b39121da1-396f7f07-1764000-171550fd309d5c; X_MIDDLE_TOKEN=4541125f78b04f020d1ab29fee30c15f; sensorsdata2015jssdkcross=%7B%22distinct_id%22%3A%221709905d08a239-0d00b368bd6473-39697407-1764000-1709905d08b9bd%22%2C%22%24device_id%22%3A%221709905d08a239-0d00b368bd6473-39697407-1764000-1709905d08b9bd%22%2C%22props%22%3A%7B%22%24os%22%3A%22MacOS%22%2C%22%24browser%22%3A%22Chrome%22%2C%22%24browser_version%22%3A%2280.0.3987.149%22%2C%22%24latest_traffic_source_type%22%3A%22%E7%9B%B4%E6%8E%A5%E6%B5%81%E9%87%8F%22%2C%22%24latest_referrer%22%3A%22%22%2C%22%24latest_referrer_host%22%3A%22%22%2C%22%24latest_search_keyword%22%3A%22%E6%9C%AA%E5%8F%96%E5%88%B0%E5%80%BC_%E7%9B%B4%E6%8E%A5%E6%89%93%E5%BC%80%22%7D%7D; index_location_city=%E4%B8%8A%E6%B5%B7; PRE_UTM=; PRE_LAND=https%3A%2F%2Fwww.lagou.com%2F; LGSID=20200408152358-9c69660c-a345-498a-8034-10252ffe3e1b; PRE_HOST=www.baidu.com; PRE_SITE=https%3A%2F%2Fwww.baidu.com%2Flink%3Furl%3DtVZflBXEaVsOXF%5FjFHCfFYOeqdRD4HXjo5Hn4EMTkmG%26ck%3D9551.3.94.252.155.241.150.180%26shh%3Dwww.baidu.com%26sht%3Dbaiduhome%5Fpg%26wd%3D%26eqid%3Df1483e950008cf70000000045e8d7c0b; TG-TRACK-CODE=index_search; _gat=1; Hm_lvt_4233e74dff0ae5bd0a3d81c6ccf756e6=1586229731,1586270033,1586330638,1586332101; SEARCH_ID=42fc072280a0477c92659467a1ad8b00; X_HTTP_TOKEN=dde9d3565777ba88401233685139c3c6eb8d6e4588; Hm_lpvt_4233e74dff0ae5bd0a3d81c6ccf756e6=1586332105; LGRID=20200408154824-69c24f2c-ed0d-40aa-acbc-1df8cfac23d1",
'Accept': "application/json, text/javascript, */*; q=0.01",
'X-Anit-Forge-Code': "0",
'X-Anit-Forge-Token': None,
'X-Requested-With': 'XMLHttpRequest'
}
form_data = {
'first': 'false',
'pn': 1,
'kd': 'python'
}
ret=requests.post('https://www.lagou.com/jobs/positionAjax.json?city=%E4%B8%8A%E6%B5%B7&needAddtionalResult=false',
headers=headers,data=form_data).text
ret = json.loads(ret)
res = ret['content']['positionResult']['result']
for i in res:
title = i['positionName']
company = i['companyFullName']
company_class = i['industryField']
salary = i['salary']
financeStage = i['financeStage']
welfare = i['companyLabelList']
job_type = i['firstType']
addr = i['district']
wyear = i['workYear']
education = i['education']
print("""
职位：%s
公司名称：%s
公司类型：%s
薪资：%s
融资阶段：%s
公司福利：%s
工作分类：%s
公司地址：%s
工作时间：%s
学历：%s
""" % (title, company, company_class, salary, financeStage, welfare, job_type, addr, wyear, education))
posted on
