实习僧招聘网站信息采集

 

# _*_ coding:utf-8 _*_

import requests
from lxml import etree
import json


def htmlpage(start_url):
    """
    根据输入的职位请求相关页面，调用单个页面解析函数，同时调用写入函数
    :param start_url: 
    :return: 
    """
    headers={'User-Agent':"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.1 (KHTML, like Gecko) Chrome/22.0.1207.1 Safari/537.1"}
    r=requests.get(start_url)
    html=r.text
    content = etree.HTML(html)
    links=content.xpath('//div[@class="names cutom_font"]/a/@href')
    for link in links:
        full_url="https://www.shixiseng.com"+link
        pageparse(full_url)

def pageparse(full_url):
    """
    单个页面解析函数，爬取职位名称和相应公司名称。
    :param full_url: 
    :return: 
    """
    headers = {
        'User-Agent': "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.1 (KHTML, like Gecko) Chrome/22.0.1207.1 Safari/537.1"}
    r = requests.get(full_url)
    html = r.text
    content = etree.HTML(html)
    contents={}
    jobname=content.xpath('//div[@class="new_job_name"]/@title')
    #jobdetail=content.xpath('//div[@class="job_detail"]//div/text()')
    #jobdetails=jobdetail.split(',')
    comname=content.xpath('//div[@class="job_com_name cutom_font"]/text()')
    content={
        "jobname":jobname,
        #"jobdetail":jobdetail,
        "comname":comname
    }
    writecontent(content)

def writecontent(content):
    """
    把爬取的职位信息写入本地
    :param content: 
    :return: 
    """
    with open("shixi.json","a") as f:
        f.write(json.dumps(content,ensure_ascii = False)+"\n")

def main(base_url,begain,end):
    """
    调度函数
    :param base_url: 
    :param begain: 
    :param end: 
    :return: 
    """
    for page in range(begain,end):
        start_url=base_url+"&p="+str(page)
        htmlpage(start_url)

if __name__=="__main__":
    key=input("job:")  #用户输入职位名称
    begain=int(input("start:")) #爬取的初始页码
    end=int(input("end:")) #爬取的结束页码
    url="https://www.shixiseng.com/interns?k=" #搜索页码
    base_url=url+key #启始页码
    main(base_url,begain,end) #调用主函数

 结果

{"jobname": ["爬虫实习"], "comname": ["宜信"]}
{"jobname": ["数据挖掘研究实习生(爬虫方向)"], "comname": ["网易游戏"]}
{"jobname": ["金融 Java-爬虫方向实习生（广州）"], "comname": ["唯品会"]}
{"jobname": ["爬虫工程师"], "comname": ["比地科技"]}
{"jobname": ["爬虫工程师"], "comname": ["踪履"]}
{"jobname": ["Java爬虫/数据采集工程师实习"], "comname": ["搜狐"]}
{"jobname": ["爬虫实习生"], "comname": ["地平线机器人"]}
{"jobname": ["Java开发实习生-爬虫开发"], "comname": ["京东金融"]}
{"jobname": ["爬虫工程师"], "comname": ["指食针"]}
{"jobname": ["爬虫实习生"], "comname": ["同花顺"]}
{"jobname": ["爬虫工程师"], "comname": ["TransferEasy"]}
{"jobname": ["数据采集（爬虫）工程师"], "comname": ["乐职网"]}
{"jobname": ["爬虫工程师"], "comname": ["探迹"]}
{"jobname": ["爬虫开发实习生"], "comname": ["妙计旅行"]}
{"jobname": ["网络爬虫实习生"], "comname": ["海天瑞声"]}
{"jobname": ["爬虫实习生"], "comname": ["阿博茨"]}
{"jobname": ["爬虫工程师实习生"], "comname": ["阿博茨"]}
{"jobname": ["助理爬虫工程师"], "comname": ["有数金服"]}
{"jobname": ["数据采集/爬虫工程师/软件工程师"], "comname": ["上海中估联"]}
{"jobname": ["网页爬虫"], "comname": ["赛迪技术"]}
{"jobname": ["爬虫实习生"], "comname": ["阿博茨"]}
{"jobname": ["JavaEE爬虫数据实习生"], "comname": ["行圆汽车"]}
{"jobname": ["Python爬虫数据实习生"], "comname": ["行圆汽车"]}
{"jobname": ["Python爬虫实习生"], "comname": ["商智通"]}
{"jobname": ["搜狐爬虫开发实习生（python）"], "comname": ["搜狐媒体"]}
{"jobname": ["爬虫开发实习生"], "comname": ["北京阿博茨"]}
{"jobname": ["爬虫开发实习生"], "comname": ["勤智数码"]}
{"jobname": ["爬虫系统工程师（实习）"], "comname": ["爱奇艺"]}