爬虫_scrapy__西瓜视频并保存数据库

#https://www.ixigua.com/search/%E5%B9%BF%E5%9C%BA%E8%88%9E/?logTag=da15511cf98419ebe3b8&tab_name=search
import requests
import pymysql
url = 'https://www.ixigua.com/search/广场舞/?'

headers = {
    'cookie': '__ac_signature=_02B4Z6wo00f01TYwvFQAAIDBtjJGFd2KXiE2ELjAAC-.41; ttcid=bcb6f1be446c4cd582bb5e23d160443d30; MONITOR_WEB_ID=aa2d700f-6664-4ab9-8a72-fa92af0e86d2; s_v_web_id=verify_l3lbglna_w7Mnw0NE_fqKL_4EkF_8q0i_zGaLOrhiCfb0; ixigua-a-s=1; support_webp=true; support_avif=true; tt_scid=BspiwRlA2llAGDMktZImoq9Kr3B6ZTmhR0riM5pG0NEV8emPK.MYbls6zXrwYiqI3354; ttwid=1%7CEXTC5VvDMV8v4FXyn9V_k2BQi6I_NEtP38oaZVfrlN4%7C1654050277%7Ca257ef8c42c66b05a1311ae5408f46c7e53985764f651e94ae95aefe24d50bed; msToken=pj7OSkhJrdi4bl3jAuTSCI4bxpHTMhp-yjmlB7nNIO5KoLT2cr6XYJl09mJXdgSieqzl9_DQhZD-8qum9MwOZFfPLeN7pziEAWhlTUcmWbtzW5YN4G4VOFG4cO28tx8=',
    'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/101.0.4951.67 Safari/537.36'
}
data={
    'logTag': 'da15511cf98419ebe3b8',
    'tab_name': 'search'
}
# url 请求资源路径
# params 参数
# kwargs 字典
response = requests.get(url=url,data=data,headers=headers)
#设置响应的编码格式
response.encoding='utf-8'
content = response.text
# print(content)

#解析网页源码来获取我们想要的数据
from lxml import etree
#解析服务器响应的文件
tree = etree.HTML(content)

#获取想要的数据 xpath的返回值值一个列表类型的数据
src_list =tree.xpath('//div[@class="searchPageV2__complex"]//div[@class="HorizontalFeedCard__coverContainer"]//div[@class="tt-img-wrapper"]/img/@src')
alt_list = tree.xpath('//div[@class="searchPageV2__complex"]//div[@class="HorizontalFeedCard__coverContainer"]//div[@class="tt-img-wrapper"]/img/@alt')
user_list = tree.xpath('//div[@class="searchPageV2__complex"]//div[@class="HorizontalFeedCard__contentWrapper"]//a[@class="user__name"]/text()')
play_list = tree.xpath('//div[@class="searchPageV2__complex"]//div[@class="HorizontalFeedCard__contentWrapper"]//span[@class="HorizontalFeedCard-accessories-bottomInfo__statistics"]')

# print(len(play_list),len(user_list),len(src_list),len(alt_list))

for i in range(len(src_list)):
    href = 'https://'+src_list[i]
    alt = alt_list[i]
    user = user_list[i]
    play = play_list[i].text
    # user_list[i], alt_list[i], src_list[i]
    # print(play_list[i].text,play_list[i][0].text)
    conn = pymysql.connect(
        host='127.0.0.1',
        port=3306,
        user='sa',
        password='HRBJYKJ-Battery',
        db='spider01',
        charset='utf8'
    )
    cursor = conn.cursor()
    # 插入数据库
    sql = 'insert into ixigua(src,alt,user,play) values("{}","{}","{}","{}")'.format(href,alt,user,play)
    # 执行sql语句
    cursor.execute(sql)
    # 提交
    conn.commit()
    cursor.close()
    conn.close()