python3 scrapy 爬取腾讯招聘

安装scrapy不再赘述，

在控制台中输入scrapy startproject tencent 创建爬虫项目名字为 tencent

接着cd tencent

用pycharm打开tencent项目

构建item文件

# -*- coding: utf-8 -*-

# Define here the models for your scraped items
#
# See documentation in:
# http://doc.scrapy.org/en/latest/topics/items.html

import scrapy


class TencentItem(scrapy.Item):
    # define the fields for your item here like:
    # name = scrapy.Field()
    #职位名
    positionname = scrapy.Field()
    #详细链接
    positionLink = scrapy.Field()
    #职位类别
    positionType = scrapy.Field()
    #招聘人数
    peopleNum = scrapy.Field()
    #工作地点
    workLocation = scrapy.Field()
    #发布时间
    publishTime = scrapy.Field()

　　接着在spiders文件夹中新建tencentPostition.py文件代码如下注释写的很清楚

# -*- coding: utf-8 -*-
import scrapy
from tencent.items import TencentItem

class TencentpostitionSpider(scrapy.Spider):
    #爬虫名
    name = 'tencent'
    #爬虫域
    allowed_domains = ['tencent.com']
    #设置URL
    url = 'http://hr.tencent.com/position.php?&start='
    #设置页码
    offset = 0
    #默认url
    start_urls = [url+str(offset)]

    def parse(self, response):
        #xpath匹配规则
        for each in response.xpath("//tr[@class='even'] | //tr[@class='odd']"):
            item = TencentItem()
            # 职位名
            item["positionname"] = each.xpath("./td[1]/a/text()").extract()[0]
            # 详细链接
            item["positionLink"] = each.xpath("./td[1]/a/@href").extract()[0]
            # 职位类别
            try:
                item["positionType"] = each.xpath("./td[2]/text()").extract()[0]
            except:
                item["positionType"] = '空'
            # 招聘人数
            item["peopleNum"] = each.xpath("./td[3]/text()").extract()[0]
            # 工作地点
            item["workLocation"] = each.xpath("./td[4]/text()").extract()[0]
            # 发布时间
            item["publishTime"] = each.xpath("./td[5]/text()").extract()[0]
            #把数据交给管道文件
            yield item
        #设置新URL页码
        if(self.offset<2620):
            self.offset += 10
        #把请求交给控制器
        yield scrapy.Request(self.url+str(self.offset),callback=self.parse)

　　接着配置管道文件pipelines.py代码如下

# -*- coding: utf-8 -*-

# Define your item pipelines here
#
# Don't forget to add your pipeline to the ITEM_PIPELINES setting
# See: http://doc.scrapy.org/en/latest/topics/item-pipeline.html

import json
class TencentPipeline(object):
    def __init__(self):
        #在初始化方法中打开文件
        self.fileName = open("tencent.json","wb")

    def process_item(self, item, spider):
        #把数据转换为字典再转换成json
        text = json.dumps(dict(item),ensure_ascii=False)+"\n"
        #写到文件中编码设置为utf-8
        self.fileName.write(text.encode("utf-8"))
        #返回item
        return item

    def close_spider(self,spider):
        #关闭时关闭文件
        self.fileName.close()

　　接下来需要配置settings.py文件

不遵循ROBOTS规则

ROBOTSTXT_OBEY = False

　　

#下载延迟
DOWNLOAD_DELAY = 3

　　

#设置请求头
DEFAULT_REQUEST_HEADERS = {
    'User-Agent':'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.84 Safari/537.36',
    'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
}

　

#交给哪个管道文件处理 文件夹.管道文件名.类名
ITEM_PIPELINES = {
    'tencent.pipelines.TencentPipeline': 300,
}

　接下来再控制台中输入　

scrapy crawl tencent

即可爬取

源码地址

https://github.com/ingxx/scrapy_to_tencent