import requests
from bs4 import BeautifulSoup
import re
class Get_links(object):
def __init__(self):
self.url = 'http://www.jianfeiketang.com'
self.headers = {
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3724.8 Safari/537.36",
}
def parse(self):
response = requests.get(url=self.url, headers=self.headers)
html = response.content.decode()
links = []
soup = BeautifulSoup(html, "html.parser")
url_list = soup.find_all('a')
for link in url_list:
links.append(link.get('href'))
return links
def save(self, links):
for link in links:
with open('../result/jf.txt', 'a', encoding='utf-8') as f:
f.write(link + '\n')
print('保存成功...')
def guolv(self):
with open('../result/jf.txt', 'r', encoding='utf-8') as f:
lines = f.readlines()
for line in lines: # 循环处理每行数据
urls = re.findall(r"http[s]?://(?:[a-zA-Z]|[0-9]|[$-_@.&+]|[!*\(\),]|(?:%[0-9a-fA-F][0-9a-fA-F]))+",
line)
for url in urls:
with open('./jfresult.txt', 'a', encoding='utf-8') as f:
f.write(url + '\n')
print('清洗完毕。。。')
if __name__ == '__main__':
get_links = Get_links()
links = get_links.parse()
get_links.save(links)
get_links.guolv()
