爬取腾讯招聘岗位 scrapy框架
1,腾讯岗位招聘
# -*- coding: utf-8 -*-
import scrapy
from lxml import etree
from ..items import TencentTtem
class TxSpider(scrapy.Spider):
name = 'tx'
allowed_domains = ['hr.tencent.com']
start_urls = []
url = 'https://hr.tencent.com/position.php?lid=&tid=&keywords=python&start=%d'
for page in range(55):
url1 = url % (page * 10)
start_urls.append(url1)
def parse(self, response):
content=response.body.decode('utf-8')
# with open('11111.html','w',encoding='utf-8')as fp:
# fp.write(content)
tree=etree.HTML(content)
tr_list=tree.xpath('//table[@class="tablelist"]/tr')
# print(tr_list)
tr_list.pop()
tr_list.pop(0)
for tr in tr_list:
item=TencentTtem()
##职位名称
name=tr.xpath('./td[1]/a/text()')
item['name']=name[0]
type=tr.xpath('./td[2]/text()')
item['type'] = type[0]
num=tr.xpath('./td[3]/text()')
item['num'] = num[0]
address=tr.xpath('./td[4]/text()')
item['address']=address[0]
time=tr.xpath('./td[5]/text()')
item['time']=time[0]
#详情页 url
deile=tr.xpath('./td[1]/a/@href')
item['deile']=deile[0]
##拼接域名
delai_url='http://hr.tencent.com/'+deile[0]
print(delai_url)
yield scrapy.Request(delai_url,callback=self.deile_parse,meta={'data':item},dont_filter=False)
def deile_parse(self, response):
content=response.body.decode('utf-8')
tree=etree.HTML(content)
item=response.meta['data']
##获取所有的内容
duty_list1=tree.xpath('//table[@class="tablelist textl"]/tr[3]//text()')
tr=''
for tr3 in duty_list1:
tr=tr + tr3.strip()
item['duty'] = tr
duty_list2=tree.xpath('//table[@class="tablelist textl"]/tr[4]//text()')
tr_4=''
for tr4 in duty_list2:
tr_4=tr_4+tr4.strip()
item['duty1']=tr_4
yield item
2.pipelines 管道部分
class TencentPipeline(object):
def __init__(self):
#建立连接
self.client = pymongo.MongoClient('localhost')
#创建库
self.db = self.client['Tencent']
self.table = self.db['Tencentjob']
def process_item(self, item, spider):
self.table.insert(dict(item))
return item
3，items 部分
class TencentTtem(scrapy.Item):
name = scrapy.Field()
type = scrapy.Field()
num = scrapy.Field()
address = scrapy.Field()
time = scrapy.Field()
deile = scrapy.Field()
duty= scrapy.Field()
duty1=scrapy.Field()
4.settings部分
# -*- coding: utf-8 -*-
# Scrapy settings for tengxu project
#
# For simplicity, this file contains only settings considered important or
# commonly used. You can find more settings consulting the documentation:
#
# https://doc.scrapy.org/en/latest/topics/settings.html
# https://doc.scrapy.org/en/latest/topics/downloader-middleware.html
# https://doc.scrapy.org/en/latest/topics/spider-middleware.html
BOT_NAME = 'tengxu'
SPIDER_MODULES = ['tengxu.spiders']
NEWSPIDER_MODULE = 'tengxu.spiders'
# Crawl responsibly by identifying yourself (and your website) on the user-agent
USER_AGENT = 'tengxu (+http://www.yourdomain.com)'
# Obey robots.txt rules
ROBOTSTXT_OBEY = False
# Configure maximum concurrent requests performed by Scrapy (default: 16)
#CONCURRENT_REQUESTS = 32
# Configure a delay for requests for the same website (default: 0)
# See https://doc.scrapy.org/en/latest/topics/settings.html#download-delay
# See also autothrottle settings and docs
#DOWNLOAD_DELAY = 3
# The download delay setting will honor only one of:
#CONCURRENT_REQUESTS_PER_DOMAIN = 16
#CONCURRENT_REQUESTS_PER_IP = 16
# Disable cookies (enabled by default)
#COOKIES_ENABLED = False
# Disable Telnet Console (enabled by default)
#TELNETCONSOLE_ENABLED = False
# Override the default request headers:
#DEFAULT_REQUEST_HEADERS = {
# 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
# 'Accept-Language': 'en',
#}
# Enable or disable spider middlewares
# See https://doc.scrapy.org/en/latest/topics/spider-middleware.html
#SPIDER_MIDDLEWARES = {
# 'tengxu.middlewares.TengxuSpiderMiddleware': 543,
#}
# Enable or disable downloader middlewares
# See https://doc.scrapy.org/en/latest/topics/downloader-middleware.html
# DOWNLOADER_MIDDLEWARES = {
# 'tengxu.mymiddleware.TengxuSpiderMiddleware': 543,
# 'tengxu.mymiddleware.ProxyMiddleware': 1,
# }
# Enable or disable extensions
# See https://doc.scrapy.org/en/latest/topics/extensions.html
#EXTENSIONS = {
# 'scrapy.extensions.telnet.TelnetConsole': None,
#}
# Configure item pipelines
# See https://doc.scrapy.org/en/latest/topics/item-pipeline.html
ITEM_PIPELINES = {
'tengxu.pipelines.TencentPipeline': 300, ##
# 'tengxu.pipelines.MeijuPipeline': 300, ##美剧
}
# Enable and configure the AutoThrottle extension (disabled by default)
# See https://doc.scrapy.org/en/latest/topics/autothrottle.html
#AUTOTHROTTLE_ENABLED = True
# The initial download delay
#AUTOTHROTTLE_START_DELAY = 5
# The maximum download delay to be set in case of high latencies
#AUTOTHROTTLE_MAX_DELAY = 60
# The average number of requests Scrapy should be sending in parallel to
# each remote server
#AUTOTHROTTLE_TARGET_CONCURRENCY = 1.0
# Enable showing throttling stats for every response received:
#AUTOTHROTTLE_DEBUG = False
# Enable and configure HTTP caching (disabled by default)
# See https://doc.scrapy.org/en/latest/topics/downloader-middleware.html#httpcache-middleware-settings
#HTTPCACHE_ENABLED = True
#HTTPCACHE_EXPIRATION_SECS = 0
#HTTPCACHE_DIR = 'httpcache'
#HTTPCACHE_IGNORE_HTTP_CODES = []
#HTTPCACHE_STORAGE = 'scrapy.extensions.httpcache.FilesystemCacheStorage'
##设置ip代理ip的设置
# PROXIES=[
#
# ]
LOG_FILE='tx.log' ##日志文件
LOG_ENABLED=True ##启用log
LOG_ENCODING='UTF-8'##编码
LOG_LEVEL='DEBUG'##日志登记
