多线程池——爬取新发地300页菜价

#线程池一次性开辟一些线程，用户给线程提交任务，线程任务的调用交给线程池来完成
#
# from concurrent.futures import ThreadPoolExecutor,ProcessPoolExecutor  #线程池和进程池
#
# def fn(name):
#     for i in range(1000):
#         print(name,i)
#
# if __name__ == '__main__':
#     #创建线程池
#     with ThreadPoolExecutor(50) as t:
#         for i in range(100):
#             t.submit(fn,name=f"线程{i}")
#      #等待线程名完毕，才继续执行
#     print("over")

#思路
#1，如何提取单个页面的数据
#2.上线程池，多个页面同时抓取
import csv
import requests
from concurrent.futures import ThreadPoolExecutor
f=open("菜价.csv",mode="w",encoding="utf-8",newline='')
csvwriter=csv.writer(f)
def down_one_page(url):
    resp=requests.get(url)
    for i in range(20):
        list=[]
        prodName=resp.json()["list"][i]['prodName']
        avgPrice=resp.json()["list"][i]['avgPrice']
        place=resp.json()["list"][i]['place']
        list.append(prodName)
        list.append(avgPrice)
        list.append(place)
        print(list)
        csvwriter.writerow(list)




if __name__ == '__main__':
    #线程池 down_one_page("http://www.xinfadi.com.cn/getPriceData.html?current=1")
    with ThreadPoolExecutor(50) as t:
        for i in range(200): #爬取200页数据
            t.submit(down_one_page,f"http://www.xinfadi.com.cn/getPriceData.html?current={i}")
    print("下载完毕")

多线程爬取新发地300页菜价，速度得到很大的提升