concurrent 多线程

# -*- coding: utf-8 -*-
from bs4 import BeautifulSoup
import concurrent.futures
import requests



hd = {
    "cept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9",
    "Accept-Encoding": "gzip, deflate",
    "Accept-Language": "zh-CN,zh;q=0.9",
    "Cache-Control": "max-age=0",
    "Connection": "keep-alive",
    "Host": "www.xxxx.com",
    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.116 Safari/537.36"
    }

# 输出到文件
def write(path, text):
    f1 = open(path, 'ab')
    f1.write(bytes(text, encoding="utf-8"))
    f1.close()

# 爬取动作
def start(url):
    demo = BeautifulSoup(requests.get(url, headers = hd).text, "html.parser")
    for a in demo.find_all('div', class_='textlist-body'):
        print(a)
        # write("out.txt", url)
        write("out.txt", '{}, {}\n'.format(url, a))


def Country_url():
    url = "https://www.xxxx.com/"
    demo = BeautifulSoup(requests.get(url+"airports", headers = hd).text, "html.parser")

    for i in demo.find_all('div', class_='textlist-body'):
        url_li = [url+x.string.replace(" ", "-") for x in i if x != " "]  # 生成URL列表
        with concurrent.futures.ThreadPoolExecutor() as executor:
            results = executor.map(start, url_li)  # 利用map对列表传递
            for result in results:
                print(result)



if __name__ == "__main__":
    Country_url()