爬取大学论文

"""

姓名 ：赵康乐

职业 ：学生

日期 ：2022-2-7

任务 ：爬取大学论文范文20篇并保存在当前目录

"""

import requests
from bs4 import BeautifulSoup

url = "https://www.wenmi.com/8561/"
headers = {
    "User-Agent":
        "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:96.0) Gecko/20100101 Firefox/96.0"
}
response = requests.get(url,headers).text
# print(response)
soup = BeautifulSoup(response,"lxml")
total = soup.find("div",class_="mt10 conlist-total")
# print(total)
lis = total.find_all("div",class_="conlist-num")
# print(lis)
for li in lis:
    title = li.h5.text
    new_url = "https://www.wenmi.com" + li.h5.a["href"]
    # print(title,new_url)
    new_resp = requests.get(new_url,headers=headers).text
    new_soup = BeautifulSoup(new_resp,"lxml")
    content = new_soup.find("div",class_="art-text").text
    with open("./"+title+".txt",mode="a",encoding="utf-8") as fp:
        fp.write(title)
        fp.write(content)
    print(title+"下载完成")