新闻分类-中文分词+词云展示(1)
1、导入数据创建存储关键词表(此处使用MySQL)
2、使用jieba进行分词统计并存储到表中
# -*- coding: utf-8 -*- import pandas as pd import pymysql import jieba
#此处只是其中一个表的分词 def getdata(): dbconn=pymysql.connect(host="127.0.0.1", database="test1125", user="root", password="", port=3306, charset='utf8') #sql语句 sqlcmd="select content from sheet_car limit 80" #利用pandas 模块导入mysql数据 titles=pd.read_sql(sqlcmd,dbconn) keywords ="" print(titles.values) for i in range(len(titles)): str =(",").join(titles.values[i]) word_list = jieba.cut(str) keywords = list(word_list) count = 0 for count in range(len(keywords)): if checkword(keywords[count]): flag = checkre(pymysql.connect(host="127.0.0.1", database="test1125", user="root", password="", port=3306, charset='utf8'), keywords[count]) if flag: save_keywords(pymysql.connect(host="127.0.0.1", database="test1125", user="root", password="", port=3306, charset='utf8'), keywords[count]) print(keywords[count]) else: updatenum(pymysql.connect(host="127.0.0.1", database="test1125", user="root", password="", port=3306, charset='utf8'), keywords[count]) else: print("未知词语") def checkword(word): invalid_words = [',', '.', ',', '。', ':', '“', '”', '"', '?', '?', '《', '》', '(', '{', ')', '}', '!', '%', '℃', '¥', '#'] if word.lower() in invalid_words: return False else: return True def save_keywords(db, keyword): # 使用cursor()方法获取操作游标 cursor = db.cursor() # SQL 插入语句 sql = "INSERT INTO key_sheet_car(keywords,num) VALUES ('%s',1)" % (keyword) try: # 执行sql语句 cursor.execute(sql) # 执行sql语句 print("true") db.commit() except: print("数据插入失败") # 发生错误时回滚 db.rollback() # 关闭数据库连接 db.close() def updatenum(db,keyword): # 使用cursor()方法获取操作游标 cursor = db.cursor() # SQL 插入语句 sql = "update key_sheet_car set num=num+1 where keywords = '%s' " % keyword try: # 执行sql语句 cursor.execute(sql) # 执行sql语句 db.commit() except: print("数据更新失败") # 发生错误时回滚 db.rollback() # 关闭数据库连接 db.close() def checkre(db, keyword): # 使用cursor()方法获取操作游标 cursor = db.cursor() ket = [] # SQL 插入语句 ket = [] sql = "select keywords from key_sheet_car where keywords = '%s'" % keyword try: # 执行sql语句 cursor.execute(sql) ket = list(cursor.fetchall()) db.commit() except: print("查询数据失败") # 发生错误时回滚 db.rollback() # 关闭数据库连接 db.close() if ket: return False else: return True if __name__ == '__main__': getdata()
3、项目结构