基础
1 安装scrapy
2 pip install wheel
3 下载Twisted https://www.lfd.uci.edu/~gohlke/pythonlibs/
4 pip install Twisted-xxxxxx
5 pip install scrapy
创建工程
scrapy startproject xxxpro
scrapy genspider spidername www.xxx.com
运行
scrapy crawl qiubai
extract()
1 def parse(self, response): 2 title_list=response.xpath("/html/body/main/div/div/div//posts") 3 for title in title_list: 4 title_name=title.xpath(".//div[2]/h2/a/text()")[0].extract() 5 title_eyes=title.xpath(".//item[@class='meta-view']/text()")[0].extract() 6 7 # print(title_name,title_eyes) 8 9 item=TestproItem()#前面需要引入文件item 10 item['title_name']=title_name 11 item['title_eyes'] = title_eyes 12 yield item
pipelines.py 管道文件 一个管道类存一种方式
TXT管道
1 class TestproPipeline: 2 fp=None 3 def open_spider(self,spider): 4 print("开始爬虫") 5 self.fp =open('./qiu.txt','w',encoding='utf-8') 6 7 def process_item(self, item, spider): 8 title_name = item['title_name'] 9 title_eyes = item['title_eyes'] 10 # print(title_name,title_eyes) 11 self.fp.write(title_name+":"+title_eyes+"\n") 12 return item #两种管道必须都加返回，否则第二个管道无法获取item 13 def close_spider(self,spider): 14 print("爬虫结束") 15 self.fp.close()
mysql 管道
1 class mysqlPipeline: 2 conn=None 3 cursor=None 4 def open_spider(self, spider): 5 print("开始爬虫") 6 self.conn=pymysql.Connect(host="127.0.0.1",user="root",password="root",database="xijing",port=3306) 7 8 def process_item(self, item, spider): 9 title_name = item['title_name'] 10 title_eyes = item['title_eyes'] 11 # print(title_name,title_eyes) 12 self.cursor=self.conn.cursor() 13 try: 14 self.cursor.execute('insert into water values ("%s","%s")'%(title_name,title_eyes)) 15 self.conn.commit() 16 except Exception as e: 17 print(e) 18 self.conn.rollback() 19 return item #两种管道必须都加返回，否则第二个管道无法获取item 20 21 def close_spider(self, spider): 22 self.cursor.close() 23 self.conn.close() 24 print("爬虫结束")
