爬虫作业

import requests

url = 'https://cn.bing.com/'

#请求头
headers = {'user-agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.25 Safari/537.36 Core/1.70.3741.400 QQBrowser/10.5.3863.400'}

def Access(url,headers = headers):
    try:
        r = requests.get(url, timeout = 30, headers = headers)
        r.raise_for_status()
        status = r.status_code
        print('响应状态码为:{}'.format(status))
        return r
    except:
        print('请求访问失败')
for i in range(1,21):
    print('第{}次'.format(i))
    r = Access(url)

text = r.text
content = r.content
print('text内容:\n{}'.format(text))
print('text属性返回的网页内容长度:{}'.format(len(text)))
print('contents属性返回的网页内容长度:{}'.format(len(content)))

 

这是一个简单的html页面,请保持为字符串,完成后面的计算要求。(良好)

 

<!DOCTYPE html>
<html>
<head>
<meta charset="utf-8">
<title>菜鸟教程(runoob.com)</title>
</head>
<body>
    <h1>我的第一个标题</h1>
    <p id=first>我的第一个段落。</p >
</body><table border="1">
    <tr>
        <td>row 1, cell 1</td>
        <td>row 1, cell 2</td>
    </tr>
    <tr>
        <td>row 2, cell 1</td>        
        <td>row 2, cell 2</td>        
    </tr>
</table>
</html>

 

from bs4 import BeautifulSoup
import re
soup=BeautifulSoup('''<!DOCTYPE html>
<html>
<head>
<meta charset="utf-8">
<title>菜鸟教程(runoob.com)</title>
</head>
<body>
<h1>好好学习</h1>
<p id="first">天天向上</p>
</body>
<table border="1">
<tr>
<td>row 1, cell 1</td>
<td>row 1, cell 2</td>
</tr>
<tr>
<td>row 2, cell 1</td>
<td>row 2, cell 2</td>
</tr>
</table>
</html>''')
print("head标签:\n",soup.head,"\n学号后两位:39")
print("body标签:\n",soup.body)
print("id为first的标签对象:\n",soup.find_all(id="first"))
st=soup.text
pp = re.findall(u'[\u1100-\uFFFDh]+?',st)
print("html页面中的中文字符")
print(pp)

 

,爬取大学排名(学号尾号9,0,爬取年费2019,)

import requests
import csv
import os
import codecs
from bs4 import BeautifulSoup
allUniv = []
def getHTMLText(url):
    try:
        r = requests.get(url, timeout=30)
        r.raise_for_status()
        r.encoding = 'utf-8'
        return r.text
    except:
        return ""
def fillUnivList(soup):
    data = soup.find_all('tr')
    for tr in data:
        ltd = tr.find_all('td')
        if len(ltd)==0:
            continue
        singleUniv = []
        for td in ltd:
            singleUniv.append(td.string)
        allUniv.append(singleUniv)
def printUnivList(num):
    print("{:^4}{:^10}{:^5}{:^8}{:^10}".format("排名","学校名称","省市","总分","培养规模"))
    for i in range(num):
        u=allUniv[i]
        print("{:^4}{:^10}{:^5}{:^8}{:^10}".format(u[0],u[1],u[2],u[3],u[6]))

'''def write_csv_file(path, head, data):
    try:
        with open(path, 'w', newline='') as csv_file:
            writer = csv.writer(csv_file, dialect='excel')
 
            if head is not None:
                writer.writerow(head)
 
            for row in data:
                writer.writerow(row)
 
            print("Write a CSV file to path %s Successful." % path)
    except Exception as e:
        print("Write an CSV file to path: %s, Case: %s" % (path, e))'''
def writercsv(save_road,num,title):
    if os.path.isfile(save_road):
        with open(save_road,'a',newline='')as f:
            csv_write=csv.writer(f,dialect='excel')
            for i in range(num):
                u=allUniv[i]
                csv_write.writerow(u)
    else:
         with open(save_road,'w',newline='')as f:
            csv_write=csv.writer(f,dialect='excel')
            csv_write.writerow(title)
            for i in range(num):
                u=allUniv[i]
                csv_write.writerow(u)
 
title=["排名","学校名称","省市","总分","生源质量","培养结果","科研规模","科研质量","顶尖成果","顶尖人才","科技服务","产学研究合作","成果转化"]
save_road="F:\\python\csvData.csv"
def main():
    url = 'http://www.zuihaodaxue.cn/zuihaodaxuepaiming2019.html'
    html = getHTMLText(url)
    soup = BeautifulSoup(html, "html.parser")
    fillUnivList(soup)
    printUnivList(10)
    writercsv('F:\\python\csvData.csv',10,title)
main()

 

posted @ 2020-12-13 22:58  唯闻卿卿  阅读(45)  评论(0编辑  收藏  举报