使用selenium爬取51Job职位信息 入库mongoDB

selenium_51job_com.py

#!/usr/bin/env python3
# coding=utf-8
# Version:python3.6.1
# File:51job_com.py
# Author:LGSP_Harold
import pymongo
from selenium import webdriver
from selenium.webdriver.firefox.options import Options
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from lxml import etree
import time


class HandleWebdriver:
    def __init__(self):
        # 设置无头模式
        options = Options()
        options.add_argument('--headless')

        self.browser = webdriver.Firefox(firefox_options=options)
        # self.browser.maximize_window()

    def handle_job(self):
        # 打开目的地址
        self.browser.get(
            'https://search.51job.com/list/000000,000000,0000,00,9,99,+,2,1.html?lang=c&postchannel=0000&workyear=99&cotype=99&degreefrom=99&jobterm=99&companysize=99&ord_field=0&dibiaoid=0&line=&welfare=')

        # 通过WebDriverWait进行显式等待，等待搜索框
        if WebDriverWait(self.browser, 5, 0.5).until(EC.presence_of_element_located((By.ID, 'keywordInput'))):
            # 外部获取输入岗位信息
            input_keyword = input('请输入要查找的岗位：')
            # 将要查找的信息发送到搜索框
            self.browser.find_element_by_id('keywordInput').send_keys(input_keyword)
            # 点击搜索
            self.browser.find_element_by_id('search_btn').click()

        if WebDriverWait(self.browser, 5, 0.5).until(EC.presence_of_element_located((By.CLASS_NAME, 'j_joblist'))):
            # 查看网页源代码
            # print(self.browser.page_source)
            while True:
                time.sleep(2)
                self.handle_parse(self.browser.page_source)
                try:
                    if self.browser.find_element_by_xpath('//li[@class="next"]/a'):
                        self.browser.find_element_by_xpath('//li[@class="next"]/a').click()
                except:
                    break

            self.browser.quit()

    def handle_parse(self, page_source):
        html_obj = etree.HTML(page_source)
        items = html_obj.xpath('//div[@class="j_joblist"]/div[@class="e"]')
        data_list = []
        for item in items:
            data = {}
            data['job_name'] = item.xpath('.//a/p[@class="t"]/span[@class="jname at"]/text()')[0]
            data['time'] = item.xpath('.//a/p[@class="t"]/span[@class="time"]/text()')[0]
            try:
                data['money'] = item.xpath('.//a/p[@class="info"]/span[@class="sal"]/text()')[0]
            except:
                data['money'] = '面议'
            data['address'] = item.xpath('.//a/p[@class="info"]/span[@class="d at"]/text()')[0]
            try:
                tags = item.xpath('.//a/p[@class="tags"]/span/i/text()')
                text = ''
                for tag in tags:
                    text += tag + ' | '
                data['tags'] = text
            except:
                data['tags'] = '暂无'
            data_list.append(data)
        # print(data_list)
        self.handle_mongodb(data_list)

    def handle_mongodb(self, data_list):
        client = pymongo.MongoClient('mongodb://admin:admin@127.0.0.1:27017')
        db = client['db_51job_com']
        collections = db['collections_51job']
        collections.insert_many(data_list)


selenium = HandleWebdriver()
selenium.handle_job()