python爬虫爬取腾讯招聘信息 （静态爬虫）
环境：
windows7，python3.4
代码：（亲测可正常执行）
1 import requests 2 from bs4 import BeautifulSoup 3 from math import ceil 4 5 header = { 6 'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.99 Safari/537.36'} 7 8 9 # 获取岗位页数 10 def getJobPage(url): 11 ret = requests.get(url, headers=header) 12 ret.encoding = "utf-8" # 解决乱码问题 13 html = ret.text 14 soup = BeautifulSoup(html, 'html.parser') 15 # 获取岗位总数，< span class ="lightblue total" > 512 < / span > 16 totalJob = soup.select('span[class="lightblue total"]')[0].text 17 jobPage = ceil(int(totalJob) / 10) 18 return jobPage 19 20 21 def getJobOrder(url): 22 ret = requests.get(url, headers=header) 23 ret.encoding = "utf-8" # 解决乱码问题 24 html = ret.text 25 soup = BeautifulSoup(html, 'html.parser') 26 # 工作职责 27 jobRequests = soup.select('ul[class="squareli"]')[0].text 28 # 工作要求 29 jobOrder = soup.select('ul[class="squareli"]')[1].text 30 return jobRequests, jobOrder 31 32 33 # 获取岗位信息 34 def getJobInfo(url): 35 myfile = open("tencent_job.txt", "a", encoding='gb18030', errors='ignore') # 解决乱码问题 36 ret = requests.get(url, headers=header) 37 ret.encoding = "utf-8" # 解决乱码问题 38 html = ret.text 39 soup = BeautifulSoup(html, 'html.parser') 40 jobList = soup.find_all('tr', class_=['even', 'odd']) 41 for job in jobList: 42 # url 43 jobUrl = "https://hr.tencent.com/" + job.select('td:nth-of-type(1) > a')[0]['href'] 44 # 职位名称 45 jobName = job.select('td:nth-of-type(1) > a')[0].text 46 # 人数 47 jobPeople = job.select('td:nth-of-type(3)')[0].text 48 # 地点 49 jobAddre = job.select('td:nth-of-type(4)')[0].text 50 # 发布时间 51 jobTime = job.select('td:nth-of-type(5)')[0].text 52 # 工作职责 53 jobRequests = getJobOrder(jobUrl)[0] 54 # 工作要求 55 jobOrder = getJobOrder(jobUrl)[1] 56 57 #print(jobName, jobUrl, jobAddre, jobPeople, jobTime, jobRequests, jobOrder) 58 59 tt = jobName + " " + jobUrl + " " + jobAddre + " " + jobPeople + " " + jobTime + " " + jobRequests + " " + jobOrder 60 myfile.write(tt + "\n") 61 62 63 if __name__ == '__main__': 64 mainurl = 'https://hr.tencent.com/position.php?keywords=python' 65 jobPage = getJobPage(mainurl) 66 print(jobPage) 67 for page in range(jobPage): 68 pageUrl = 'https://hr.tencent.com/position.php?keywords=python&start=' + str(page * 10) + '#a' 69 print("第" + str(page + 1) + "页") 70 getJobInfo(pageUrl)
