Python 爬虫 CSDN 网页下载
import re
import urllib.request
import urllib.error
url="http://blog.csdn.net"
header=("User-Agent",'User-Agent:Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/55.0.2883.87 Safari/537.36')
opn=urllib.request.build_opener()
opn.addheaders=[header]
data=opn.open(url).read().decode()
pat='<li class=""><a href="(.*?)">'
menu_data=re.compile(pat).findall(data)
file_num=0
for all_link in menu_data:
data1=opn.open('http://blog.csdn.net/'+all_link ).read().decode()
pat1='<a href="(http://blog.csdn.net/.*[0-9].*?)" target=.*'
sub_menu=re.compile(pat1).findall(data1)
try:
for link in sub_menu:
file_num+=1
urllib.request.urlretrieve(link,"D:\\data\\"+str(file_num)+".html")
except urllib.error.URLError as err:
if hasattr(err,"code"):
print(err.code)
if hasattr(err,"reason"):
print(err.reason)