import re
import requests
import random
import time
import os
class GetPhoto(object):
def __init__(self, url1, class_):
self.agent = "user_agents.txt"
self.head_url_list = {}
self.pic_item_url = []
self.pic_url_list = []
self.page_url_list = [] #
self.url = url1
self.class_ = class_
with open(self.agent, 'r', encoding='utf-8') as f:
agents = f.readlines()
self.agents = list(map(lambda x: {"user-agent": x.strip()}, agents))
def chartset(self, rsp):
"""
解决中文乱码问题
:param rsp:
:return:
"""
_chart = requests.utils.get_encoding_from_headers(rsp.headers)
if _chart == 'ISO-8859-1':
rsp.encoding = requests.utils.get_encodings_from_content(rsp.text)[0]
# def get_first_url(self):
# rsp = requests.get(self.url, headers=random.choice(self.agents))
# self.chartset(rsp)
# content = rsp.text
# pattern = r'<div class="nav both">.*?<!-- top end -->'
# com = re.compile(pattern, re.S)
# content_info = com.findall(content)
# pattern2 = 'href="(http://(?:[a-zA-Z0-9]+\.){1,2}[a-zA-Z]{2,6}.*?)".*?<span>(.+?)</span>'
# com2 = re.compile(pattern2, re.S)
# self.head_url_list.update(map(lambda x:x[::-1],com2.findall(content_info[0])))
def get_page_urllist(self):
print('开始获取首页下方链接列表url！')
rsp = requests.get(self.url, headers=random.choice(self.agents))
self.chartset(rsp)
content = rsp.text
pattern = r'<div class="page both">.*?</div>'
com = re.compile(pattern, re.S)
content_info = com.findall(content)
pattern2 = r"href='(.*?)'"
com2 = re.compile(pattern2, re.S)
self.page_url_list.append(self.url)
self.page_url_list.extend(list(map(lambda x: url.rsplit("/", 1)[0]+"/"+x, com2.findall(content_info[0]))))
print("首页下方链接列表url获取完成！")
print(self.page_url_list)
def get_sub_url(self):
self.get_page_urllist()
print("开始获取整页图片项目链接：")
for i in self.page_url_list[:10]:
try:
rsp = requests.get(i, headers=random.choice(self.agents))
except Exception as e:
with open('error.log', 'a', encoding='utf-8') as f:
new_time = time.strftime('%Y-%m-%d %H:%M:%S')
f.write('{} {} {}+'.format(new_time, i, e.args[0]))
print('访问失败，暂停10s!')
time.sleep(10)
return
self.chartset(rsp)
content = rsp.text
pattern = r'<div class="imgList2">.*?</div>'
com = re.compile(pattern, re.S)
content_info = com.findall(content)
# print(content_info)
pattern2 = r'href="(?P<url>http://(?:[\w]+\.){1,2}[a-zA-Z]{2,6}.*?)".*?target="_blank" title="(.*?)">'
com2 = re.compile(pattern2, re.S)
sub_url_list = com2.findall(content_info[0])
self.pic_item_url.append(sub_url_list)
print(sub_url_list)
time.sleep(random.randint(1, 3)) #延时1-3秒
print("获取所有页面图片项目完成！")
def get_pic_url(self, url2):
rsp = requests.get(url2, headers=random.choice(self.agents))
self.chartset(rsp)
content = rsp.text
pattern = r'<div class="page".*?</div>'
com = re.compile(pattern, re.S)
content_info = com.findall(content)
pattern2 = r"<a href='([^#]+?)'>"
com2 = re.compile(pattern2, re.S)
pic_url_list = list(map(lambda x: url2.rsplit("/",1)[0]+"/"+x, com2.findall(content_info[0])))
return pic_url_list
def get_pic(self, pic_page_url, dirname, count):
try:
rsp = requests.get(pic_page_url, headers=random.choice(self.agents))
except Exception as e:
with open('error.log', 'a', encoding='utf-8') as f:
new_time = time.strftime('%Y-%m-%d %H:%M:%S')
f.write('{} {} {}\n'.format(new_time, pic_page_url, e.args[0]))
print('访问失败，暂停30s!')
time.sleep(30)
print('下载继续')
return
self.chartset(rsp)
content = rsp.text
# pattern = r"<p align=\"center\" id=\"contents\">.*?img\ssrc='(.*?)'"
pattern = '<p align="center">.*?src="(.*?)"'
# print(content)
com = re.compile(pattern, re.S)
pic_url = com.findall(content)
image_name = dirname + str(count) + '.jpg'
if pic_url:
try:
pic_rsp = requests.get(pic_url[0], headers=random.choice(self.agents))
except Exception as e:
with open('error.log', 'a', encoding='utf-8') as f:
new_time = time.strftime('%Y-%m-%d %H:%M:%S')
f.write('{} {} {}\n'.format(new_time, pic_url[0], e.args[0]))
print('访问失败，暂停30s!')
time.sleep(30)
print('下载继续')
return
img_dirpath = os.path.join(self.class_, dirname)
if not os.path.exists(img_dirpath):
os.makedirs(img_dirpath)
img_path = os.path.join(img_dirpath, image_name)
# print(img_dirpath)
# print(image_name)
# print(img_path)
# return
with open(img_path, 'wb') as f:
f.write(pic_rsp.content)
print('{}下载完成!'.format(image_name))
else:
print('{}下载失败!'.format(image_name))
time.sleep(random.randint(1, 3))
def download(self):
self.get_sub_url()
for i in self.pic_item_url[:3]:
for z in i:
pic_item, dir_name = z
lst = self.get_pic_url(pic_item)
print("开始下载{}系列!".format(dir_name))
for index, j in enumerate(lst, 1):
self.get_pic(j, dir_name, index)
print("{}系列下载完成!".format(dir_name))
if __name__ == '__main__':
url = 'http://www.5442.com/youxi/'
pic = GetPhoto(url, '游戏壁纸')
pic.download()
