新版爬虫

# -*- coding:UTF-8 -*-
import requests
from bs4 import BeautifulSoup
import io
import time
import sys
import os

# 主链接
base_url = "https://www.tupianzj.com/"


def get(url):
    headers = {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:61.0)Gecko/20100101 Firefox/61.0"}
    requests.adapters.DEFAULT_RETRIES = 5
    s = requests.session()
    s.keep_alive = False
    res = s.get(url, headers=headers)
    return res


def get_html(url):
    res = get(url)
    res.encoding = "gb2312"    # 需要根据网页编码调整
    html = res.text
    return html


def get_pic(url, filename):
    html = get_html(url)
    soup = BeautifulSoup(html, 'html.parser')
    div = soup.find("div", id="bigpic")
    img = div.find("img")
    img_src = img.get("src")
    pic = get(img_src)
    with io.open(filename, 'wb') as f:
        f.write(pic.content)
    div_1 = soup.find("div", class_="pages")
    li = div_1.find_all("li")
    pages = int(li[0].text[1:][:-3])
    return pages


def main():
    html = get_html(base_url+"meinv/xinggan/")
    soup = BeautifulSoup(html, 'html.parser')
    div = soup.find("div", class_="list_con_box")
    ul = div.find("ul", class_="list_con_box_ul")
    li = ul.find_all("li")
    for i in li:
        a = i.find("a")
        href = a.get("href")
        title = a.get("title")
        path = u"C:/Users/Administrator/Desktop/MM/{name}".format(name=title)
        os.makedirs(path)
        filename = path + "/1.jpg"
        pages = get_pic(base_url+href, filename)
        for j in range(2, pages+1):
            url = base_url + href[:-5] + "_" + str(j) + ".html"

            filename = path + "/" + str(j) + ".jpg"
            get_pic(url, filename)
        print(title+"下载好了!")
    print("Complet!")


if __name__ == '__main__':
    main()