爬虫例子入门玩玩
目标：爬取百度百科关于Python的100个网页
工具环境：Python 3.5 , Sublime Text 3
爬虫调度程序：spider_main.py
# coding:utf8 # from baike_spider import url_manager, html_downloader, html_parser, \ # html_outputer import url_manager, html_downloader, html_parser, \ html_outputer import socket class SpiderMain(object): #构造方法 def __init__(self): #url管理器 self.urls = url_manager.UrlManager() #下载器 self.downloader = html_downloader.HtmlDownloader() #解析顺 self.parser = html_parser.HtmlParser() #输出器 self.outputer = html_outputer.HtmlOutputer() self.log_datas = [] #log列表 #爬虫调度程序 def craw(self, root_url): count = 1 self.urls.add_new_url(root_url) while self.urls.has_new_url(): try: new_url = self.urls.get_new_url() print ('craw %d : %s'%(count,new_url)) html_cont = self.downloader.download(new_url) #这里有下载不动或超时，程序一直卡在这里的问题 new_urls, new_data = self.parser.parse(new_url, html_cont) self.urls.add_new_urls(new_urls) self.outputer.collect_data(new_data) # logs self.collect_log('craw %d : %s'%(count,new_url)) if count == 100: break count = count +1 except: print('craw failed!') self.outputer.output_html() # logs self.write_log() #收集日志 def collect_log(self, data): if data is None: return self.log_datas.append(data) #写日志 def write_log(self): fout = open('output.txt', 'w', encoding='utf-8') # 文件的输出对象 W 写模式 , 同时指定文件的编码格式 #fout.write(str(self.log_datas)) for data in self.log_datas: #fout.write(str(data)) fout.write(data+'\r\n') fout.close() if __name__=="__main__": socket.setdefaulttimeout(5) #设置全局的超时时间为5s root_url="http://baike.baidu.com/item/Python" #root_url=r"http://baike.baidu.com/item/%E9%BB%91%E6%B2%B3%E5%AD%A6%E9%99%A2" obj_spider = SpiderMain() print ('Spider starting....') obj_spider.craw(root_url)
url管理器：url_manager.py
# coding:utf8 #url管理器 class UrlManager(object): def __init__(self): self.new_urls = set() self.old_urls = set() print ('init url manager') #向管理器中添加一个新的url def add_new_url(self, url): if url is None: return if url not in self.new_urls and url not in self.old_urls: self.new_urls.add(url) #向管理器中添加多个新的url def add_new_urls(self, urls): if urls is None or len(urls) == 0: return for url in urls: self.add_new_url(url) #调用上面单个添加url方法add_new_url #在管理器中查找一个url def has_new_url(self): return len(self.new_urls) !=0 #在管理器中获取一个url，获取后，删除它在新的url中，并保存在旧的url中 def get_new_url(self): new_url = self.new_urls.pop() #查找并删除 self.old_urls.add(new_url) return new_url
下载器：html_downloader.py
# coding:utf8 # import urllib import urllib.request #下载器 class HtmlDownloader(object): def __init__(self): print ('init html downloader') def download(self, url): if url is None: return None # 下面这种方式，容易卡死，或下载超时，解决方法在爬虫的主入口中，增加全局超时设置：socket.setdefaulttimeout(10) #设置全局的超时时间为10s # 这种方式比下面的方式，要节省时间 response = urllib.request.urlopen(url) # 下面另外一种防止卡死，或下载超时方法： # response = urllib.request.urlopen(url,timeout=3) if response.getcode() != 200: return None return response.read()
解析器：html_parser.py
# coding:utf8 from bs4 import BeautifulSoup import re # import urlparse import urllib.parse #解析器 class HtmlParser(object): def __init__(self): print ('init html parser') def _get_new_urls(self, page_url, soup): # /view/123.htm new_urls = set() #links = soup.find_all('a', href=re.compile(r"/view/\d+\.htm")) links = soup.find_all('a', href=re.compile(r"/item/")) for link in links: new_url = link['href'] #new_full_url = urlparse.urljoin(page_url, new_url) new_full_url =urllib.parse.urljoin(page_url,new_url) new_urls.add(new_full_url) return new_urls def _get_new_data(self, page_url, soup): # 定义一个字典 res_data = {} # url res_data['url'] = page_url #<dd class="lemmaWgt-lemmaTitle-title"> <h1>Python</h1> title_node = soup.find('dd', class_="lemmaWgt-lemmaTitle-title").find("h1") res_data['title'] = title_node.get_text() #<div class="lemma-summary" label-module="lemmaSummary"> summary_node = soup.find('div', class_="lemma-summary") res_data['summary'] = summary_node.get_text() return res_data def parse(self, page_url, html_cont): if page_url is None or html_cont is None: return soup = BeautifulSoup(html_cont, 'html.parser', from_encoding='utf-8') new_urls = self._get_new_urls(page_url, soup) new_data = self._get_new_data(page_url, soup) return new_urls, new_data
输出器：html_outputer.py
# coding:utf8 #输出器 class HtmlOutputer(object): # 构造函数 def __init__(self): self.datas = [] #列表 print ('init html outputer') def collect_data(self, data): if data is None: return self.datas.append(data) def output_html(self): #fout = open('output.html','w') # 文件的输出对象 W 写模式 fout = open('output.html', 'w', encoding='utf-8') # 文件的输出对象 W 写模式 , 同时指定文件的编码格式 fout.write("<html>") fout.write("<body>") fout.write("<table>") count = 1 for data in self.datas: fout.write("<tr>") #fout.write("<td>%s</td>" % data['url'].encode('utf-8'))# python默认输出编码ascii #fout.write("<td>%s</td>" % data['title'].encode('utf-8')) #fout.write("<td>%s</td>" % data['summary'].encode('utf-8')) fout.write("<td>%s</td>" % count) fout.write("<td>%s</td>" % data['url'])# python3默认输出编码Unicode fout.write("<td>%s</td>" % data['title']) fout.write("<td>%s</td>" % data['summary']) fout.write("</tr>") count = count + 1 fout.write("</table>") fout.write("</body>") fout.write("</html>") fout.close()
最后效果：
总结：
对Python爬虫的基本了解掌握，还需学习更多知识，探索未知！
