【pyhon】nvshens按目录图片批量下载爬虫1.00（多线程版）

# nvshens按目录图片批量下载爬虫1.00（多线程版）
from bs4 import BeautifulSoup
import requests
import datetime
import urllib.request
import os
import threading

user_agent='Mozilla/4.0 (compatible;MEIE 5.5;windows NT)'
headers={'User-Agent':user_agent}


# 下载图片到本地
def downloadPics(pictures):
    while(len(pictures)>0):
        pic=pictures.pop()

        name=pic.split('/')[-1]
        folder=pic.split('/')[-2]

        # 判断目录是否存在，不存在则创建之
        if os.path.exists('./'+folder)==False:
            os.makedirs('./'+folder)

        try:
            rsp=urllib.request.urlopen(pic)
            img=rsp.read()
            with open('./'+folder+"/"+name,'wb') as f:
                f.write(img)
            print('图片'+pic+'下载完成')
        except Exception as e:
            print('图片'+pic+'下载异常,塞回重试')
            pictures.append(pic);


#下载线程类
class dldThread(threading.Thread):
    def __init__(self,name,url):
        threading.Thread.__init__(self,name=name)
        self.name=name
        self.url=url
        self.pictures=[]
    
    def run(self):
        while(self.url!="none"):
            print("线程"+self.name+"开始爬取页面"+self.url);

            try:
                rsp=requests.get(self.url,headers=headers)
                self.url="none"#用完之后置空，看下一页能否取到值
                soup= BeautifulSoup(rsp.text,'html.parser',from_encoding='utf-8')                

                for divs in soup.find_all(class_="gallery_wrapper"):
                    # 把找到的图片放到数组里去
                    for img in divs.find_all('img'):
                        print(img.get("src"))
                        self.pictures.append(img.get("src"))

                    #找下一页
                    for link in divs.find_all('a',class_='a1'):
                        if link.string=='下一页' and link.get("href").find('.html')!=-1:
                            self.url='https://www.nvshens.com'+link.get("href")


                if self.url!="none":
                    print("线程"+self.name+"前往下一页")
                    continue
                else:
                    print("线程"+self.name+'爬取结束，开始下载...')
                    downloadPics(self.pictures)
                    print("线程"+self.name+'下载图片结束.')
            except Exception as e:
                print("线程"+self.name+"发生异常。重新爬行")# 不管怎么出现的异常，就让它一直爬到底
                continue


# 循环下载图片
def main():
    for i in range(10000,20000):#范围自己调整
        url='https://www.nvshens.com/g/'+str(i)+'/'

        th=dldThread(name=str(i),url=url)
        th.start()

# Kickoff Start
main()