定向网页爬虫
1 import requests 2 from bs4 import BeautifulSoup 3 import bs4 4 5 #爬取定向网页的内容 6 def getHTMLText(url): 7 try: 8 r = requests.get(url, timeout = 30) 9 r.raise_for_status() 10 r.encoding = r.apparent_encoding 11 return r.text 12 except: 13 print('error') 14 15 16 def fillUnivList(ulist, html): 17 soup = BeautifulSoup(html, "html.parser") 18 for tr in soup.find('tbody').children: #遍历tbody下的子标签 19 if isinstance(tr, bs4.element.Tag): #是标签类型,非字符串 20 tds = tr('td') #简写,输出列表 21 ulist.append([tds[0].string, tds[1].string, tds[2].string]) 22 23 24 def printUnivList(ulist, num): 25 tplt = "{0:^10}\t{1:{3}^10}\t{2:^10}" #采用宽度不够,中文空格填充,保证中文对齐 26 print(tplt.format("排名","学校名称","省份", chr(12288))) 27 for i in range(num): 28 u=ulist[i] 29 print(tplt.format(u[0],u[1],u[2],chr(12288))) 30 31 def main(): 32 uinfo = [] 33 url = 'http://www.zuihaodaxue.com/zuihaodaxuepaiming2016.html' 34 html = getHTMLText(url) 35 fillUnivList(uinfo, html) 36 printUnivList(uinfo, 20) 37 38 main()