Python爬取imdb电影数据并存储到mysql数据库

数据获取方式：微信搜索关注【靠谱杨阅读人生】回复【电影】。
整理不易，资源付费，谢谢支持。

Python爬虫代码：

  1 import re
  2 import time
  3 import traceback
  4 
  5 from bs4 import BeautifulSoup
  6 from lxml import etree
  7 import pymysql
  8 import requests
  9 #连接数据库  获取游标
 10 def get_conn():
 11     """
 12     :return: 连接，游标
 13     """
 14     # 创建连接
 15     conn = pymysql.connect(host="82.157.112.34",
 16                     user="root",
 17                     password="root",
 18                     db="MovieRankings",
 19                     charset="utf8")
 20     # 创建游标
 21     cursor = conn.cursor()  # 执行完毕返回的结果集默认以元组显示
 22     if ((conn != None) & (cursor != None)):
 23         print("数据库连接成功！游标创建成功！")
 24     else:
 25         print("数据库连接失败！")
 26     return conn, cursor
 27 #关闭数据库连接和游标
 28 def close_conn(conn, cursor):
 29     if cursor:
 30         cursor.close()
 31     if conn:
 32         conn.close()
 33     return 1
 34 def get_imdb():
 35 
 36     # url='https://www.imdb.cn/feature-film/1-0-0-0/?page=1'
 37     headers={
 38         'user-agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/90.0.4430.93 Safari/537.36'
 39     }
 40     dataRes=[]      #最终结果集
 41     temp_list=[]        #暂时结果集
 42 
 43     # print(all_li)
 44 
 45     for i in range(200,202):
 46             url_='https://www.imdb.cn/feature-film/1-0-0-0/?page='+str(i)
 47             response = requests.get(url=url_, headers=headers)
 48             # print(response)
 49             response.encoding = 'utf-8'
 50             page_text = response.text
 51             # print(page_text)
 52             etree_ = etree.HTML(page_text)
 53             all_li = etree_.xpath('//div[@class="hot_box"]/ul/li')      #获取所有li
 54             #判断all_li是否为空
 55             if(len(all_li)==0):
 56                 print("爬取结束，all_list为空！")
 57                 if(len(dataRes)!=0):
 58                     return dataRes;
 59                 else:
 60                     return ;
 61             print(url_)
 62             for li in all_li:
 63                 name=li.xpath('./a[1]/img/@alt')
 64                 if(len(name)==0):
 65                     name.append("电影名错误")
 66                 # print(name)
 67                 #存姓名
 68                 temp_list.append(name[0])
 69 
 70                 score=li.xpath('./span[@class="img_score"]/@title')
 71                 if(len(score)==0):
 72                     score.append("imdb暂无评分")
 73                 # print(score)
 74                 #存分数
 75                 temp_list.append(score[0])
 76                 # print(temp_list)
 77                 #存到dataRes 把temp_list置为空
 78                 dataRes.append(temp_list)
 79                 temp_list=[]
 80             print(dataRes)
 81     return dataRes
 82 def insert_imdb():
 83     """
 84         插入imdb数据
 85         :return:
 86         """
 87     cursor = None
 88     conn = None
 89     try:
 90         list_=[]
 91         list = get_imdb()
 92         if(type(list)!=type(list_)):
 93             return ;
 94         print(f"{time.asctime()}开始插入imdb数据")
 95         conn, cursor = get_conn()
 96         sql = "insert into movieimdb (id,name,score) values(%s,%s,%s)"
 97         for item in list:
 98             try:
 99                 print(item)
100                 cursor.execute(sql, [0, item[0], item[1]])
101             except pymysql.err.IntegrityError:
102                 print("重复！跳过！")
103             conn.commit()  # 提交事务 update delete insert操作
104             print(f"{time.asctime()}插入imdb数据完毕")
105     finally:
106         close_conn(conn, cursor)
107     return;
108 # def get_dblen():
109 #     conn,cursor=
110 #     num_=
111 
112 if __name__ == '__main__':
113     # get_imdb()
114     insert_imdb()