scrapy爬取极客学院全部课程
1 # -*- coding: utf-8 -*- 2 # scrapy爬取极客学院全部课程 3 import scrapy 4 from pyquery import PyQuery as pq 5 from jike.items import JikeItem 6 7 class JikespiderSpider(scrapy.Spider): 8 name = "jikespider" 9 allowed_domains = ["www.jikexueyuan.com"] 10 base_url = 'http://www.jikexueyuan.com/course/?pageNum=' 11 12 def start_requests(self): 13 for page_num in range(1,96): 14 url = self.base_url + str(page_num) 15 yield scrapy.Request(url, callback=self.parse_index) 16 17 def parse_index(self, response): 18 doc = pq(response.text) 19 lis = doc('.lesson-list .cf li').items() 20 # pyquery心得, 以为pyquery有点问题而导致无法遍历数据结构, 21 # 研究发现是'http:' + item('.lessonimg-box a').attr('href') 22 # 的问题, href是相对路径没有得到一个有效的请求链接 23 for item in lis: 24 detail_url = 'http:' + item('.lessonimg-box a').attr('href') 25 yield scrapy.Request(url=detail_url,callback=self.parse_detail) 26 27 def parse_detail(self, response): 28 item = JikeItem() 29 doc = pq(response.text) 30 item['title'] = doc('.lesson-teacher .bc-box h2').text() 31 item['time'] = doc('.lesson-teacher .bc-box .timebox').text() 32 item['content'] = doc('.lesson-teacher .infor-content').text() 33 34 yield item