boss直聘

"""
author:张鑫
date:2021/11/3 11:30
https://m.zhipin.com/wapi/zpgeek/mobile/search/joblist.json?query=python%E7%88%AC%E8%99%AB%E5%B7%A5%E7%A8%8B%E5%B8%88&page=6&city=101010100&query=python%E7%88%AC%E8%99%AB%E5%B7%A5%E7%A8%8B%E5%B8%88
"""
import json

import pymongo
# 连接数据库
from lxml import etree

client = pymongo.MongoClient('localhost', 27017)
db = client['zhaopin']
boss_list = db['boss_list']


# 抓取数据
def response(flow):
    if 'm.zhipin.com/wapi/zpgeek/mobile/search/joblist.json' in flow.request.url:
        html = json.loads(flow.response.text)['zpData']['html']
        tree = etree.HTML(html)
        # 公司名称
        # name = tree.xpath('//span[@class="title-text"]//text()')
        name = tree.xpath('//span[@class="company"]//text()')
        # 工资
        salary = tree.xpath('//span[@class="salary"]//text()')
        # 公司所在地
        workplace = tree.xpath('//span[@class="workplace"]//text()')
        # 工作年限要求
        year = tree.xpath('//div[@class="labels"]//span[1]//text()')
        # 学历要求
        education = tree.xpath('//div[@class="labels"]//span[2]//text()')
        # 技术要求1
        style = tree.xpath('//div[@class="labels"]//span//text()')
        # 技术要求2
        # jobstyle = tree.xpath('//div[@class="labels"]//span[4]//text()')
        # # 技术要求3
        # jobname = tree.xpath('//div[@class="labels"]//span[5]//text()')
        for i in range(len(name)):
            boss = {}
            boss['公司名称'] = name[i]
            boss['工资'] = salary[i]
            boss['公司所在地'] = workplace[i]
            boss['工作年限要求'] = year[i]
            boss['学历要求'] = education[i]
            boss['技术要求'] = style[i]

            print('********************************************')
            boss_list.insert(boss)
            print(boss)
            print('********************************************')