爬取招聘职位一

 爬取百度招聘上关于IT职业的有关信息进行分析，只是对常见职业进行分析，未解决异步加载，只能显示第一页，而且最多显示100条信息，爬取二升级若对全部职业和城市，则需对数据库进行写入

    #-*- coding:UTF-8 -*-  
    #@author:若鸟  
    #functions: 爬取百度招聘上关于IT职业的有关信息进行分析，只是对常见职业进行分析，未解决异步加载，只能显示第一页，而且最多显示100条信息，爬取二升级若对全部职业和城市，则需对数据库进行写入  
    import requests  
    import re  
    import xlwt  
    from bs4 import BeautifulSoup  
    from Get_jobList import GetJobList  
    def getHtml(url):  
        try:  
            r = requests.get(url)  
            r.raise_for_status()  
            r.encoding = r.apparent_encoding  
            return r.text  
        except Exception as e:  
            print(url,"爬取失败，失败信息：",e)  
    def getJoblist():  
        citys = ['肥城','泰安','北京','上海','深圳','杭州','济南']  
        jobs = GetJobList()[:50]                         #获取部分职位名称，爬取自拉勾网  
        url_job = []  
        for city in citys:  
            for job in jobs:  
                url_job.append("http://zhaopin.baidu.com/quanzhi?query={}&city={}&rn=100".format(job,city))  #网站实现的js翻页，经过每页最多显示100条  
          
        return url_job  
    def getJobData(urls):  
        count = 0  
        job_data = []           #<p class="area line-clamp1">济南  |  济南宅客信息科技有限公司</p>  
        for url in urls:  
              
            html = getHtml(url)  
            job = re.findall(r'<p class="title-h3 line-clamp1">(.+)</p>',html)  
            addr = re.findall(r'<p class="area line-clamp1">(.+)  \|  .+</p>',html)  
            company = re.findall(r'<p class="area line-clamp1">.+ \|  (.+)</p>',html)  
            salary = re.findall(r'<p class="salary">(.+)</p>',html)  
            source = re.findall(r'<p>.+ \| 来自(.+)</p>',html)  
            num = len(job)  
            for i in range(num):  
                job_data.append([job[i],addr[i],company[i],salary[i],source[i]])  
      
              
            count +=1  
            print(count)  
          
        return job_data  
    def writeXls(data_list):  
        #print(data_list)  
        num = len(data_list)  
        f = xlwt.Workbook()  
        sheet1 = f.add_sheet('sheet1',cell_overwrite_ok = True)  
        for i in range(num):  
            num_ = len(data_list[i])  
            for x in range(num_):  
                sheet1.write(i,x,data_list[i][x])  
        f.save("JobData.xls")  
    if __name__ == '__main__':  
        job_data = getJobData(getJoblist())  
        writeXls(job_data)