python线程池 批量抓取蔬菜价格 并写入csv文件

# http://www.xinfadi.com.cn/index.html
# 接口 http://www.xinfadi.com.cn/getPriceData.html
# 想提取单个页面的数据
# 上线程池，多页面同时抓取


import requests
from concurrent.futures import ThreadPoolExecutor
import csv
import time




def get_one_page(current):
    data = {"limit":20,"current":current}
    r = requests.post("http://www.xinfadi.com.cn/getPriceData.html",data=data)
    list = r.json()["list"]
    write_csv(list)





def write_csv(list):
    # 写数据
    with open("蔬菜价格.csv","a",encoding="utf-8",newline="") as f:
        w = csv.writer(f)
        for v in list:
            w.writerow((v["prodName"],v["lowPrice"],v["avgPrice"],v["highPrice"],v["place"],v["unitInfo"],v["pubDate"]))
            print(v["prodName"],v["lowPrice"],v["avgPrice"],v["highPrice"],v["place"],v["unitInfo"],v["pubDate"])
    time.sleep(1)





if __name__ == "__main__":
    # 写表头
    with open("蔬菜价格.csv","a",encoding="utf-8",newline="") as f:
        w = csv.writer(f)
        w.writerow(("品名","最低价","平均价","最高价","产地","单位","发布时间"))




    # 创建线程池 
    with ThreadPoolExecutor(20) as t:
        for i in range(100): # 投递100个任务
            t.submit(get_one_page,current=i)




    print("OK")

 

 结果：

 表格打开：