简单爬虫
一.简单爬虫
目标——写一个爬虫来爬取常用搜索引擎（比如百度、搜狗等）的首页。
手段——使用 python 的 requests 、BeautifulSoup4 与 lxml 库。
实现方式——
首先，我们要获取到网页，这个不难，可以使用 requests.get() 解决，如下：
# coding = utf-8 import requests url = "http://www.baidu.com" #设置获取网页的时间限制，超时就报错，防止网络延时而太久没反馈 r = requests.get(url, timeout = 1) #如果返回状态码为 200，则说明网页连接成功 print(r.status_code)
在 windows 10 的 cmd 下，运行结果如下
# encoding = utf-8 import requests from lxml import html from bs4 import BeautifulSoup # 教训不要给文件取名与库名相同，否则会给程序扫描文件出错（找库时出错） url = [ "http://www.baidu.com", "http://www.google.com", "http://www.sogou.com", "http://www.bing.com", "http://www.so.com" ] name = ["baidu", "google", "sogou", "bing", "360"] ''' for i in range(5): r = requests.get( url[i], timeout = 10) r.encoding = 'utf-8' tree = html.fromstring(r.text) urls = [] if r.status_code == 200: #create html doc and save it with open("D:\\{}.html".format(name[i]), 'w', encoding='utf-8') as f: f.write(r.text) print("This is {} times: Successful!".format(i)) for i in tree.xpath("//@href"): urls.append(i) for i in range(len(urls)): print(urls[i]) else: print("This is {} times: False!".format(i)) for i in range(20): r = requests.get(url[0], timeout = 1) print(r.status_code) r.encoding = 'utf-8' print("type of text: ", type(r.text)) print("type of content: ", type(r.content)) soup = BeautifulSoup(r) print(soup.get_text()) #print(r.text)
处理一个 HTML 文件
a.打印 body 标签的内容
b.获取 body 标签的内容
c.获取 id 为 first 的标签对象
d.获取并打印 HTML 页面的中文字符
from lxml import html
from bs4 import BeautifulSoup
html_doc = """ <!DOCTYPE html> <html> <head> <meta charset="utf-8"> <title>菜鸟教程(runoob.com)</title> </head> <body> <h1>我的第一个标题</h1> <p>我的第一个段落。</p> </body> <table border="1"> <tr> <td>row 1, cell 1</td> <td>row 1, cell 2</td> </tr> <tr> <td>row 2, cell 1</td> <td>row 2, cell 2</td> </tr> </table> </html> """ #使用 BeautifulSoup 解析网页，并得到一个 BeautifulSoup 的对象 soup = BeautifulSoup(html_doc) #输出网页源码 print(soup.prettify()) #输出网页中的文本信息 #print(soup.get_text()) text = soup.get_text() print("---------------") #print() #输出网页中文本信息的长度（行数） print(len(soup.contents))
牛刀小试
爬取中国大学排名内容，http://www.zuihaodaxue.com/zuihaodaxuepaiming2018.html
爬取2015年的信息。
# coding = utf-8 import requests from bs4 import BeautifulSoup import pandas as pd client = MongoClient() db = client['UnivRanking'] collection = db['UnivRanking'] def save_to_mongo(result): try: if collection.insert(result): print('Save to Mongo') except: print("错误") allUniv = [] def getHTMLText(url): try: r = requests.get(url, timeout = 30) r.raise_for_status() r.encoding = 'utf-8' return r.text except: return "" def fillUnivList(soup): data = soup.find_all('tr') for tr in data: ltd = tr.find_all('td') if len(ltd) == 0: continue singleUniv = [] for td in ltd: singleUniv.append(td.string) allUniv.append(singleUniv) def printUnivList(num): print("{1:^4}{2:{0}^10}{3:{0}^5}{4:{0}^8}{5:{0}^8}{6:{0}^8}{7:{0}^8}".format(chr(12288), "排名", "学校名称", "省市", "总分", "人才培养得分", "科学研究得分", "社会服务得分")) for i in range(num): u = allUniv[i] print("{1:{0}^4}{2:{0}^10}{3:{0}^5}{4:{0}^8}{5:{0}^10}{6:{0}^10}{7:{0}^10}".format(chr(12288), u[0], u[1], u[2], u[3], u[4], u[5], u[6])) name = ["排名", "学校名称", "省市", "总分", "人才培养得分", "科学研究得分", "社会服务得分"] test=pd.DataFrame(columns=name, data= allUniv) print(test) test.to_csv('testcsv.csv',encoding='utf-8') def main(num): url = "http://www.zuihaodaxue.cn/zuihaodaxuepaiming2015_0.html" html = getHTMLText(url) soup = BeautifulSoup(html, "html.parser") fillUnivList(soup) printUnivList(num) save_to_mongo(allUniv) if __name__ == "__main__": main(100)
