网易云评论爬虫

import requests
import json
import my_fake_useragent
import threading

import pymysql

#连接本地数据库
conn = pymysql.connect(host='',user='root',password='yangzaigang',database='wangyiyun',charset='utf8',port=3306)
#创建游标
cur = conn.cursor()


def getHTMLText(url):
    try:
        headers = {"user-agent": my_fake_useragent.UserAgent().random()}
        r = requests.get(url, headers=headers)
        r.raise_for_status()
        # r.encoding = r.apparent_encoding              #这里自动识别编码方式会乱码，注释掉就行了
        return r.text
    except:
        #print("getHTMLText失败！")
        #return ""
        pass

def fillList(music_id, url, commentlist):

    new_url = url + "{0}".format(music_id)
    #print("new_url======>", new_url)

    html = getHTMLText(new_url)
    #print("html======>", html)

    try:
        json_dict = json.loads(html)  # 利用json方法把json类型转成dict
        #print(json_dict)

        # likedCount 点赞数量
        likedCount = json_dict['hotComments'][0]['likedCount']
        # print(type(likedCount))
        #print("点赞数======>",likedCount)
        # 如果点赞数大于30万
        if likedCount > 300000:
            comments = json_dict['hotComments']
            # print("comments======>",comments)
            # print("type======>",type(comments))
            # print("type======>",len(comments))

            # commentlist.append([item['user']['nickname'], item['content']])
            m = "点赞数===>" + str(likedCount) + " 歌曲ID===>" + str(music_id) + " 用户===>" + comments[0]['user']['nickname'] + " 评论===>" + comments[0][
                'content']
            print(m)
            name = comments[0]['user']['nickname']
            content = comments[0]['content']
            #创建写入的语句
            #sql = 'insert into doubanfilm6(m_xuhao,m_title, m_pingfen,m_comment, m_link) values ("%s","%s","%s","%s","%s")' % (xuhao,title, pingfen, comment, link)

            sql = 'insert into wangyiyun(likedCount,music_id,name,content) values(%s,%s,%s,%s)'
            args = (likedCount,music_id,name,content)
            #执行写入操作
            cur.execute(sql,args)
            #进行提交。不提交的话中间写入了也查找不到数据
            conn.commit()
            save(m, "./网抑云.txt")
    except:
        pass

def save(m, path):
    with open(path, 'a', encoding='utf-8') as f:
        f.write(m + "\n")
        f.close()

def main():
    mutex = threading.Lock()
    mutex.acquire()
    i = 59867
    mutex.release()
    #新歌的ID   1882041535  18亿
    while i <= 2000000000:
        music_id = str(i)
        url = "http://music.163.com/api/v1/resource/comments/R_SO_4_"
        commentlist = []
        fillList(music_id, url, commentlist)
        i = i + 1
        #print("url_id======>",music_id)

    # 关闭游标和数据库连接
    cur.close()
    conn.close()

def thread():
    for i in range(6):
        t = threading.Thread(target=main())
        t.start()
    main_thread = threading.currentThread()
    for t in threading.enumerate():
        if t is main_thread:
            continue
        t.join()

thread()