爬取博客并保存到数据库
import requests
import pymysql
from bs4 import BeautifulSoup
# 输入博客网址
url = "https://www.cnblogs.com/java-six/default.html?page="
header={
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/103.0.0.0 Safari/537.36'
}
# 连接数据库
conn = pymysql.connect(host='192.168.43.162', user='root', password='heziqaz1234', database='ssmbuild', charset='utf8', port=3306)
# 创建游标对象
cursor = conn.cursor()
# 按条件将数据插入数据库
sql = 'INSERT INTO blogs(date, title, link) select %s,%s,%s from DUAL where not exists (select * from blogs where link = %s)'
pageindex=0
while(1):
pageindex = pageindex+1
nowurl = url+str(pageindex)
response = requests.get(nowurl,headers=header)
print(nowurl)
print("This is "+str(pageindex)+"page")
if(response.status_code!=200):
print("The status_code is "+response.status_code)
break
soup = BeautifulSoup(response.text,'html.parser')
# 获得该页的博客数量
resSize = len(soup.find_all('div','post'))
arr = soup.find_all('div','post')
# print(arr[1].span.string)
for i in range(resSize):
# print("++++++"+str(arr[i].div.a.span.string)+"+++++")
# .strip() 能消除字符串中的空格等特殊字符
date = str(arr[i].find_all('div','postFoot')[0].text.strip())
# get date of this blog
date = date[9:19]
title = str(arr[i].div.a.span.string).strip()
print(title+"++++++++++++++")
link = str(arr[i].div.a['href'])
data = [date, title,link,link]
response = requests.get(link,headers=header)
soup = BeautifulSoup(response.text,'html.parser')
#prettify() get html file
html = soup.find('div','post').prettify()
# 执行sql语句插入数据库
cursor.execute(sql, data)
conn.commit()
# print(soup.find_all('a','postTitle2')).
# 关闭游标
cursor.close()
# 关闭连接
conn.close()
爬取博客保存到word或pdf中
import requests
from bs4 import BeautifulSoup
import pypandoc
import pdfkit
home = input("请输入博客网址:")
# 输入博客网址
url = home+"/default.html?page="
header={
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/103.0.0.0 Safari/537.36'
}
pageindex=0
# 博客下下标
blogsnum = 0
while(1):
pageindex = pageindex+1
nowurl = url+str(pageindex)
response = requests.get(nowurl,headers=header)
print(nowurl)
print("This is "+str(pageindex)+"page")
soup = BeautifulSoup(response.text,'html.parser')
# 获得该页的博客数量
resSize = len(soup.find_all('div','post'))
arr = soup.find_all('div','post')
if(resSize==0):
break
# print(arr[1].span.string)
for i in range(resSize):
# print("++++++"+str(arr[i].div.a.span.string)+"+++++")
# .strip() 能消除字符串中的空格等特殊字符
date = str(arr[i].find_all('div','postFoot')[0].text.strip())
# get date of this blog
date = date[9:19]
title = str(arr[i].div.a.span.string).strip()
print(title+"++++++++++++++")
link = str(arr[i].div.a['href'])
data = [date, title,link,link]
response = requests.get(link,headers=header)
soup = BeautifulSoup(response.text,'html.parser')
#prettify() get html file
html = soup.find('div','post').prettify()
# print(str(html)+"--------")
# 延时5秒
# time.sleep( 5 )
# convert_file('原文件','目标格式','目标文件') Save to word
# 生成word版文件
docoutput = pypandoc.convert_text(html, 'docx','html', outputfile="blogs-"+str(blogsnum)+".docx")
# 生成pdf版文件,指定格式为utf-8,否则会中文乱码
pdfkit_options = {'encoding': 'UTF-8'}
pdfoutput = pdfkit.from_string(html,"blogs-"+str(blogsnum)+".pdf",options=pdfkit_options)
blogsnum = blogsnum+1
# print(soup.find_all('a','postTitle2')).
大杂烩
import requests
import pymysql
from bs4 import BeautifulSoup
import pypandoc
import pdfkit
import time
# 输入博客网址
url = "https://www.cnblogs.com/java-six/default.html?page="
header={
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/103.0.0.0 Safari/537.36'
}
# 连接数据库
conn = pymysql.connect(host='192.168.43.162', user='root', password='heziqaz1234', database='ssmbuild', charset='utf8', port=3306)
# 创建游标对象
cursor = conn.cursor()
# 按条件将数据插入数据库
sql = 'INSERT INTO blogs(date, title, link) select %s,%s,%s from DUAL where not exists (select * from blogs where link = %s)'
pageindex=0
# 博客下下标
blogsnum = 0
while(1):
pageindex = pageindex+1
nowurl = url+str(pageindex)
response = requests.get(nowurl,headers=header)
print(nowurl)
print("This is "+str(pageindex)+"page")
if(response.status_code!=200):
print("The status_code is "+response.status_code)
break
soup = BeautifulSoup(response.text,'html.parser')
# 获得该页的博客数量
resSize = len(soup.find_all('div','post'))
arr = soup.find_all('div','post')
# print(arr[1].span.string)
for i in range(resSize):
# print("++++++"+str(arr[i].div.a.span.string)+"+++++")
# .strip() 能消除字符串中的空格等特殊字符
date = str(arr[i].find_all('div','postFoot')[0].text.strip())
# get date of this blog
date = date[9:19]
title = str(arr[i].div.a.span.string).strip()
print(title+"++++++++++++++")
link = str(arr[i].div.a['href'])
data = [date, title,link,link]
response = requests.get(link,headers=header)
soup = BeautifulSoup(response.text,'html.parser')
#prettify() get html file
html = soup.find('div','post').prettify()
# print(str(html)+"--------")
# 延时5秒
# time.sleep( 5 )
# convert_file('原文件','目标格式','目标文件') Save to word
# 生成word版文件
docoutput = pypandoc.convert_text(html, 'docx','html', outputfile="blogs-"+str(blogsnum)+".docx")
# 生成pdf版文件,指定格式为utf-8,否则会中文乱码
pdfkit_options = {'encoding': 'UTF-8'}
pdfoutput = pdfkit.from_string(html,"blogs-"+str(blogsnum)+".pdf",options=pdfkit_options)
blogsnum = blogsnum+1
# 执行sql语句插入数据库
cursor.execute(sql, data)
conn.commit()
# print(soup.find_all('a','postTitle2')).
# 关闭游标
cursor.close()
# 关闭连接
conn.close()