DAY-04

爬取豌豆荚多页app数据

import requests
import re
from bs4 import BeautifulSoup

# 1.发送请求
def get_page(url):
    response = requests.get(url)
    return response

# 2.解析数据
def parse_index(code):
    app_info = code.find_all(name='li', class_='card')
    return app_info

# 3.保存数据
def save_data(app):
    app_name = app.h2.a.text
    detail_url = app.h2.a.attrs['href']
    download_num = app.find(class_='install-count').text
    app_size = app.find(name='span',attrs={"title":re.compile('\d+MB')}).text
    data = f"""
    app名称：{app_name}
    详情页url：{detail_url}
    下载人数：{download_num}
    app大小：{app_size}
    \n
    """
    print(data)
    with open('豌豆荚.txt','a',encoding='utf-8') as f:
        f.write(data)
        f.flush() # 写一次刷新

if __name__ == '__main__':
    for line in range(1,31):
        url = f'https://www.wandoujia.com/wdjweb/api/category/more?catId=6001&subCatId=0&page={line}&ctoken=pWcDYXidvX0QXPcolC5uu3T2'
        index_response = get_page(url)
        # print(url)
        data = index_response.json()
        text = data.get('data').get('content')
        app_list = parse_index(text)
        for app in app_list:
            save_data(app)