第一个爬虫测试

一、爬虫目标

目标网页：http://www.gaosan.com/gaokao/239012.html

二、写爬虫代码

打开Anaconda3中的spyder

代码如下：

import requests
from bs4 import BeautifulSoup
allUniv=[]
def getHtmlText(url):
    try:
        r=requests.get(url,timeout=30)
        r.raise_for_status()
        r.encoding=r.apparent_encoding
        return r.text
    except:
        return ""
def fillUnivList(soup):
    data=soup.find_all('tr')
    for tr in data:
        ltd=tr.find_all('td')
        if len(ltd)==0:
            continue
        singleUniv=[]
        for td in ltd:
            singleUniv.append(td.string)
        allUniv.append(singleUniv)
def printUnivList(num):
    print("{:^10}{:^10}{:^5}{:^8}".format("排名","学校名称","综合得分","省份"))
    for i in range(num):
        u=allUniv[i]
        print("{:^10}{:^10}{:^5}{:^8}".format(u[0],u[1],u[2],u[3]))
def main(num):
    url="http://www.gaosan.com/gaokao/239012.html"
    html=getHtmlText(url)
    soup=BeautifulSoup(html,"html.parser")
    fillUnivList(soup)
    printUnivList(num)
main(251)

效果如下：