python 爬取懂车帝详情页“全部车型模块信息”

 

 

需要爬取的内容：汽车具体型号，指导价，经销商报价，车主参考价

 

import requests
import pymysql


# 车型库
def Dongchedi(offset):
    url = 'https://www.dongchedi.com/motor/brand/m/v6/select/series/?city_name=%E6%AD%A6%E6%B1%89'
    headers = {'user-agent': 'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/89.0.4389.114 Safari/537.36'}
    data = {
        'offset': '{}'.format(offset),
        'limit': 30,
        'is_refresh': 1,
        'city_name': '武汉'
    }

    response = requests.post(url, headers=headers, data=data).json()
    all_cak = response['data']['series']
    for caks in all_cak:
        caks_id = caks['concern_id']
        caks_url = 'https://www.dongchedi.com/auto/series/' + '{}'.format(caks_id)
        cak_name = caks['outter_name']
        print(cak_name, caks_url)
        return caks_id


# 详情页
def Detail():
    for offset in range(0, 2000):
        caks_id = Dongchedi(offset)
        datail_url = 'https://www.dongchedi.com/motor/car_page/m/v1/series_all_json/?series_id=' + str(caks_id) + '&city_name=武汉&show_city_price=1&m_station_dealer_price_v=1'
        response = requests.get(datail_url).json()
        series_all = response['data']
        online_name = series_all['online']
        for data_name in online_name:
            name_info = data_name['info']  # 全部车型模块所有数据
            try:
                if name_info['brand_name']:  # 判断是否为总型号
                    series_name = name_info['name']  # 所有详细车型名称
                    car_name = name_info['series_name']
                    name = str(car_name) + '-' + str(series_name)
                    # 指导价, 经销商报价 -价格字典
                    price_info = name_info['price_info']
                    dealer_price = str(price_info['official_price'])  # 指导价
                    official_price1 = name_info['dealer_price']  # 经销商报价
                    official_price = official_price1.replace('万', '')  # 去除 ‘万’
                    # 车主参考价 -价格字典
                    owner_price_summary = name_info['owner_price_summary']
                    naked_price_avg = owner_price_summary['naked_price_avg']  # 车主参考价
                    # 保存数据库
                    data_info = (name, dealer_price, official_price, naked_price_avg)
                    SaveMysql(data_info)
            except Exception as error:
                # print(error)
                pass


# 保存到数据库
def SaveMysql(data_info):
    conn = pymysql.connect(host='127.0.0.1', port=3306, user='root', password='root', db='spiders', charset="utf8")  # 连接数据库 192.168.20.124
    cursor = conn.cursor()  # 创建游标
    insert_sql = 'insert into dongchedi(car_name,guide_price,dealer_price,owner_price) values(%s, %s, %s, %s)'  # 创建写入sql
    cursor.execute(insert_sql, data_info)  # 写入操作
    print('save to mysql')
    conn.commit()  # 提交
    cursor.close()  # 关闭游标
    conn.close()  # 关闭数据库


if __name__ == '__main__':
    try:
        Detail()
    except Exception:
        pass

 

 

爬取信息：

 

 

遇到的问题：

• 数据方面还没有处理，比如三个价格有一个或者多个不存在时，现在的情况是直接报错（当价格不存在时，该价格字段是没有的）

　　   因为try过了所以价格不存在时那条数据就直接过率掉了。

　　   我们需要的是当价格不存在时，用None填充（后续有时间再解决一下）。

 

• incorrect string value：'xx\\xx..' for column 'car_name' at row 1

 

报错的大概意思就是存入的字符串不正确。但是debug得到的数据是没有问题的，后来重新创建了数据库和表，问题就解决了。

错误原因：创建表的时候没有指定 ‘ 字符集’ 和 ‘排序规则’。