1 #!/usr/bin/env python
2 # -*- encoding: utf-8 -*-
3 # Created on 2017-06-11 10:10:53
4 # Project: london
5
6 from pyspider.libs.base_handler import *
7 import pymongo
8
9
10 class Handler(BaseHandler):
11 crawl_config = {
12 }
13 client = pymongo.MongoClient('localhost')
14 db = client['trip']
15
16 @every(minutes=24 * 60)
17 def on_start(self):
18 self.crawl('https://www.tripadvisor.cn/Attractions-g186338-Activities-c47-London_England.html', callback=self.index_page)
19
20 @config(age=10 * 24 * 60 * 60)
21 def index_page(self, response):
22 for each in response.doc('.listing_title > a').items():
23 self.crawl(each.attr.href, callback=self.detail_page)
24 next_page = response.doc('.pagination .nav.next').attr.href
25 self.crawl(next_page,callback = self.index_page)
26
27 @config(priority=2)
28 def detail_page(self, response):
29 return {
30 "name":response.doc('h1').text(),
31 "url": response.url,
32 'comment':response.doc('.heading_ratings .taLnk').text(),
33 'address':response.doc('.addressReset > span.format_address').text(),
34 'phone':response.doc('.phoneNumber').text(),
35 'duration':response.doc('#MAP_AND_LISTING > div.main_section.listingbar > div > div.above_fold_listing_details > div > div:nth-child(5) > div > div:nth-child(1)').text(),
36 'instruction':response.doc('#MAP_AND_LISTING > div.main_section.listingbar > div > div.above_fold_listing_details > div > div:nth-child(6) > div > b').text()
37 }
38 def on_result(self,result):
39 if result:
40 self.save_to_mongo(result)
41
42 def save_to_mongo(self,result):
43 if self.db['london'].insert(result):
44 print('saved to mongo',result)
45
