Python常用功能函数系列总结（五）

本节目录

常用函数一：向量距离和相似度计算

常用函数二：pagerank

常用函数三：TF-IDF

常用函数四：关键词提取

常用函数一：向量距离和相似度计算

• KL距离、JS距离、余弦距离

# -*- coding: utf-8 -*-

"""
@Datetime: 2019/3/30
@Author: Zhang Yafei
"""
import numpy as np
import pandas as pd
import scipy.stats
from scipy.stats import norm
from sklearn.metrics.pairwise import cosine_distances

p = np.asarray([0.65, 0.25, 0.07, 0.03])
q = np.array([0.6, 0.25, 0.1, 0.05])
q2 = np.array([0.1, 0.2, 0.3, 0.4])


def KL_divergence(p, q):
    """
    有时也称为相对熵，KL距离。对于两个概率分布P、Q，二者越相似，KL散度越小。
    KL散度满足非负性
    KL散度是不对称的，交换P、Q的位置将得到不同结果。
    :param p:
    :param q:
    :return:
    """
    return scipy.stats.entropy(p, q)


def JS_divergence(p, q):
    """
    JS散度基于KL散度，同样是二者越相似，JS散度越小。
        JS散度的取值范围在0-1之间，完全相同时为0
        JS散度是对称的
    :param p:
    :param q:
    :return:
    """
    M = (p + q) / 2
    return 0.5 * scipy.stats.entropy(p, M) + 0.5 * scipy.stats.entropy(q, M)


# 1000个均值170，标准差10的正态分布身高样本
h_real = norm.rvs(loc=170, scale=10, size=1000)
h_predict1 = norm.rvs(loc=168, scale=9, size=1000)
h_predict2 = norm.rvs(loc=160, scale=20, size=1000)


def JS_div(arr1, arr2, num_bins):
    max0 = max(np.max(arr1), np.max(arr2))
    min0 = min(np.min(arr1), np.min(arr2))
    bins = np.linspace(min0 - 1e-4, max0 - 1e-4, num=num_bins)
    PDF1 = pd.cut(arr1, bins).value_counts() / len(arr1)
    PDF2 = pd.cut(arr2, bins).value_counts() / len(arr2)
    return JS_divergence(PDF1.values, PDF2.values)


def cos_distance(p, q):
    """
    余弦距离
    @param p:
    @param q:
    @return:
    """
    res = cosine_distances(np.array([p, q]))
    return res[0][1]


if __name__ == '__main__':
    print(KL_divergence(p, q))  # 0.011735745199107783
    print(KL_divergence(q, p))  # 0.013183150978050884
    print(KL_divergence(p, q2))  # 1.092879181568733
    print(JS_divergence(p, q))  # 0.003093977084273652
    print(JS_divergence(p, q2))  # 0.24719159952098618
    print(JS_divergence(p, p))  # 0.0
    print(JS_div(h_real, h_predict1, num_bins=20))  # 0.011333885999505239
    print(JS_div(h_real, h_predict2, num_bins=20))  # 0.14933522936149402
    print(cos_distance(p, q))

• 余弦相似度

# -*- coding: utf-8 -*-

"""
@Datetime: 2019/3/28
@Author: Zhang Yafei
"""
# 余弦计算相似度度量 http://blog.csdn.net/u012160689/article/details/15341303
import functools
import math
import re
import time
import numpy as np
from sklearn.metrics.pairwise import cosine_similarity

text1 = "This game is one of the very best. games ive  played. the  ;pictures? " \
        "cant descripe the real graphics in the game."
text2 = "this game have/ is3 one of the very best. games ive  played. the  ;pictures? " \
        "cant descriPe now the real graphics in the game."
text3 = "So in the picture i saw a nice size detailed metal puzzle. Eager to try since I enjoy 3d wood puzzles, i ordered it. Well to my disappointment I got in the mail a small square about 4 inches around. And to add more disappointment when I built it it was smaller than the palm of my hand. For the price it should of been much much larger. Don't be fooled. It's only worth $5.00.Update 4/15/2013I have bought and completed 13 of these MODELS from A.C. Moore for $5.99 a piece, so i stand by my comment that thiss one is overpriced. It was still fun to build just like all the others from the maker of this brand.Just be warned, They are small."
text4 = "I love it when an author can bring you into their made up world and make you feel like a friend, confidant, or family. Having a special child of my own I could relate to the teacher and her madcap class. I've also spent time in similar classrooms and enjoyed the uniqueness of each and every child. Her story drew me into their world and had me laughing so hard my family thought I had lost my mind, so I shared the passage so they could laugh with me. Read this book if you enjoy a book with strong women, you won't regret it."


def timeit(func):
    @functools.wraps(func)
    def wrap(*args, **kwargs):
        start = time.time()
        res = func(*args, **kwargs)
        print('运行时间为: {0:.4f}' .format(time.time() - start))
        return res

    return wrap


def preprocess(text):
    """
    文本预处理，可根据具体情况书写逻辑
    :param text:
    :return:
    """
    return text.split()


@timeit
def compute_cosine(words1, words2):
    """
    计算两段文本的余弦相似度
    :param text_a:
    :param text_b:
    :return:
    """
    # 1. 统计词频
    words1_dict = {}
    words2_dict = {}

    for word in words1:
        word = re.sub('[^a-zA-Z]', '', word).lower()
        if word != '' and word in words1_dict:
            words1_dict[word] += 1
        elif word != '':
            words1_dict[word] = 1
        else:
            continue
    for word in words2:
        word = re.sub('[^a-zA-Z]', '', word).lower()
        if word != '' and word in words2_dict:
            words2_dict[word] += 1
        elif word != '':
            words2_dict[word] = 1
        else:
            continue

    # 2. 按照频率排序
    dic1 = sorted(words1_dict.items(), key=lambda x: x[1], reverse=True)
    dic2 = sorted(words2_dict.items(), key=lambda x: x[1], reverse=True)

    # 3. 得到词向量
    words_key = []
    list(map(lambda x: words_key.append(x[0]), dic1))
    list(map(lambda x: words_key.append(x[0]), filter(lambda x: x[0] not in words_key, dic2)))

    vect1 = []
    vect2 = []
    for word in words_key:
        if word in words1_dict:
            vect1.append(words1_dict[word])
        else:
            vect1.append(0)
        if word in words2_dict:
            vect2.append(words2_dict[word])
        else:
            vect2.append(0)

    # 4. 计算余弦相似度
    print(cosin_sim(vect1, vect2))
    print(cosin_sim2(vect1, vect2))
    result = cosine_sim3(vect1, vect2)

    return result


def cosin_sim(vector1, vector2):
    """
     K(X, Y) = <X, Y> / (||X||*||Y||)
    :param vector1:
    :param vector2:
    :return:
    """
    dot_product = normA = normB = 0.0
    for a, b in zip(vector1, vector2):
        dot_product += a * b
        normA += a ** 2
        normB += b ** 2
    if normA == 0.0 or normB == 0.0:
        return 0.0
    else:
        return dot_product / ((normA * normB) ** 0.5)


def cosin_sim2(vec1, vec2):
    user_tag_matric = np.matrix(np.array([vec1, vec2]))
    user_similarity = cosine_similarity(user_tag_matric)
    return user_similarity[0][1]


def cosine_sim3(vec1, vec2):
    """
    @param vec1:
    @param vec2:
    @return:
    """
    sum = sq1 = sq2 = 0
    for i in range(len(vec1)):
        sum += vec1[i] * vec2[i]
        sq1 += pow(vec1[i], 2)
        sq2 += pow(vec2[i], 2)
    try:
        result = float(sum) / (math.sqrt(sq1) * math.sqrt(sq2))
    except ZeroDivisionError:
        result = 0.0
    return result


if __name__ == '__main__':
    text1 = preprocess(text1)
    text2 = preprocess(text2)
    print(compute_cosine(text1, text2))

• 余弦相似度（矩阵计算）

# -*- coding: utf-8 -*-

"""
Datetime: 2020/07/19
author: Zhang Yafei
description: 余弦相似度
"""
import pandas as pd
from sklearn.metrics.pairwise import cosine_similarity


def matrix_cos_sim_clac(file, columns=None, drop_columns=None, to_file='result.xlsx'):
    """
    计算矩阵相似度  数据源格式：文件 多列 每列代表一个向量
    :param file 读取文件路径    支持excel格式
    :param columns 选取的列
    :param drop_columns 排除的列
    :param to_file 保存的文件路径
    """
    data = pd.read_excel(file)
    if columns:
        data = data[columns]
    elif drop_columns:
        data.drop(columns=drop_columns, inplace=True)
    columns = data.columns
    data_matrix = data.values.T

    cos_matrix = cosine_similarity(X=data_matrix)
    cos_sim_df = pd.DataFrame(data=cos_matrix, columns=columns, index=columns)
    cos_sim_df.to_excel(to_file)
    print('计算成功')


if __name__ == "__main__":
    matrix_cos_sim_clac(file='data/entity_2020_01_04.xlsx', drop_columns=['Id', 'Label'], to_file='cis_sim_data.xlsx')

常用函数二：pagerank

# encoding=utf-8

"""
@Datetime: 2019/3/27
@Author: Zhang Yafei
@Description: pagerank值计算
"""

import traceback
import warnings

import numpy as np
from sklearn.metrics.pairwise import cosine_similarity

warnings.filterwarnings(action='ignore', category=UserWarning, module='gensim')

from gensim.models.doc2vec import Doc2Vec


class PageRank(object):
    """ 计算pagerank """

    def __init__(self, X):
        self.X = X  # 转移矩阵
        self.pr = None
        self.move_matrix = np.zeros((X.shape), dtype=np.float)  # 转移矩阵初始化
        self.pagerank = np.zeros((X.shape[0], 1), dtype=np.float)  # pr值的矩阵初始化

    def graph_move(self):
        """
        构造转移矩阵
        0≤ w[i][j] ≤1
        np.sum(w[j]) = 1
        :param a:
        :return:
        """
        X = np.where(abs(self.X) >= 0.4, self.X, 0)
        X_t = np.transpose(X)  # b为a的转置矩阵
        bcol = [X_t[j].sum() for j in range(X.shape[1])]            #归一化
        # bcol = [np.where(X_t[j]>0, 1, 0).sum() for j in range(X.shape[1])]  # 存在相似度的度数
        for i in range(X.shape[0]):
            for j in range(X.shape[1]):
                self.move_matrix[i][j] = X[i][j] / bcol[j]  # 完成转移矩阵初始化

    def first_pr(self):
        """
        pr值初始化
        :param c:
        :return:
        """
        for i in range(self.X.shape[0]):
            self.pagerank[i] = float(1) / self.X.shape[0]

    def compute_pagerank(self, p, iter_num=100):
        """
        迭代计算pagerank值
        :param p:
        :param move_matrix: 转移矩阵
        :param pr: 初始pr值
        :param iter:
        :return:
        """
        for i in range(iter_num):
            self.pagerank = np.dot(self.move_matrix, self.pagerank)
            print('iteration {0}'.format(i))
        # ===========================================================================
        # i=1
        # while ((v == dot(m, v)).all() == False):  # 判断pr矩阵是否收敛,(v == p*dot(m,v) + (1-p)*v).all()判断前后的pr矩阵是否相等，若相等则停止循环
        #     # print v
        #     v = dot(m, v)
        #     #print((v == p*dot(m,v) + (1-p)*v).all())
        #     print(i)
        #     i+=1
        # ===========================================================================

    def train(self, iter_num=100):
        """ 训练模型 """
        # 1. 计算转义矩阵
        self.graph_move()
        # print(self.move_matrix.shape)  # (1585, 1585)

        # 2. pr值初始化
        self.first_pr()
        # print(self.pr.shape)  # (1585, 1)

        # 3. 迭代更新pr值
        p = 0.85  # 引入浏览当前网页的概率为p,假设p=0.8
        self.compute_pagerank(p=p, iter_num=iter_num)
        print(self.pagerank)

    @staticmethod
    def save(fname, X, delimiter=',', fmt='%.18e'):
        """ 将计算得到的矩阵保存到文件中 """
        try:
            np.savetxt(fname=fname, X=X, delimiter=delimiter, fmt=fmt)
            print('保存到 {} 成功'.format(fname))
        except Exception:
            traceback.print_exc()


def compute_cosine_similarity(model):
    """ 计算doc2vec文档词向量的余弦相似度 """

    # 1. 计算文档词矩阵
    size_document = len(model.docvecs.vectors_docs)
    doc_words_matrix = np.zeros((size_document, 10), dtype=np.float)
    for i, vector in enumerate(model.docvecs.vectors_docs):
        doc_words_matrix[i] = vector

    # 2. 计算文档词矩阵的余弦相似度矩阵
    cosine_a_matrix = cosine_similarity(doc_words_matrix)
    # print(cosine_a_matrix.shape)  # (1585, 1585)

    return cosine_a_matrix


if __name__ == '__main__':
    # 1. 加载doc2vec文档词向量
    doc2vec_model = Doc2Vec.load("results_data/all_model_titles")
    # print(len(model.docvecs.vectors_docs))    # 1585

    # 2. 计算文档词向量的余弦相似度矩阵
    cosine_a_matrix = compute_cosine_similarity(model=doc2vec_model)
    # np.savetxt('results_data/cosine_a_matrix.csv', cosine_a_matrix, delimiter=',', fmt='%.2f')
    # print(cosine_a_matrix)
    # 3. 计算文档的pagerank值
    model = PageRank(X=cosine_a_matrix)
    model.train()
    model.save(fname='results_data/move_matrix.csv', X=model.move_matrix, fmt='%.8f')
    model.save(fname='results_data/pagerank_titles.csv', X=model.pagerank, fmt='%.8f')
    # import math

    # with open('results_data/pagerank_titles.csv') as f:
    #     for line in f:
    #         print(line.strip())
    #         print(math.exp(-float(line.strip())))

常用函数三：TF-IDF

# -*- coding: utf-8 -*-

"""
@Datetime: 2019/3/28
@Author: Zhang Yafei
"""


# #################################  nltk计算tfidf ####################################333

def nltk_tfidf():
    import nltk
    from nltk.text import TextCollection
    sents = ['this is sentence one', 'this is sentence two', 'this is sentence three']
    sents = [nltk.word_tokenize(sent) for sent in sents]
    corpus = TextCollection(sents)
    # 直接就能算出tfidf
    # (term: ⼀句话中的某个term, text: 这句话)
    print(corpus.idf('four'))
    print(corpus.tf('four', nltk.word_tokenize('this is a sentence four')))
    print(corpus.tf_idf('four', nltk.word_tokenize('this is a sentence four')))


# #################################  sklearn计算tfidf ####################################333
def sklearn_count_tfidf():
    from sklearn.feature_extraction.text import CountVectorizer, TfidfVectorizer

    corpus = ["我 来到 北京 清华大学",  # 第一类文本切词后的结果，词之间以空格隔开
              "他 来到 了 网易 杭研 大厦",  # 第二类文本的切词结果
              "小明 硕士 毕业 与 中国 科学院",  # 第三类文本的切词结果
              "我 爱 北京 天安门"]  # 第四类文本的切词结果
    vectorizer = CountVectorizer()  # 该类会将文本中的词语转换为词频矩阵，矩阵元素a[i][j] 表示j词在i类文本下的词频
    count_corpus = vectorizer.fit_transform(corpus)

    print(count_corpus)

    vectorizer2 = TfidfVectorizer()  # 该类会统计每个词语的tf-idf权值
    tfidf_corpus = vectorizer2.fit_transform(corpus)

    print(tfidf_corpus)

    word = vectorizer2.get_feature_names()  # 获取词袋模型中的所有词语

    weight = tfidf_corpus.toarray()  # 将tf-idf矩阵抽取出来，元素a[i][j]表示j词在i类文本中的tf-idf权重

    for i in range(len(weight)):  # 打印每类文本的tf-idf词语权重，第一个for遍历所有文本，第二个for便利某一类文本下的词语权重
        print("-------这里输出第", i, "类文本的词语tf-idf权重------")
        for j in range(len(word)):
            print(word[j], weight[i][j])
    print(weight)


# #################################  gensim计算tfidf ####################################333
def gensim_tdidf():
    import warnings

    warnings.filterwarnings(action='ignore', category=UserWarning, module='gensim')

    from gensim import corpora, models, similarities
    corpus = ["我 来到 北京 清华大学",  # 第一类文本切词后的结果，词之间以空格隔开
              "他 来到 了 网易 杭研 大厦",  # 第二类文本的切词结果
              "小明 硕士 毕业 与 中国 科学院",  # 第三类文本的切词结果
              "我 爱 北京 天安门"]  # 第四类文本的切词结果
    texts = [[word for word in text.split()] for text in corpus]

    dictionary = corpora.Dictionary(texts)
    print(dictionary)
    # 9、对语料库进一步处理，得到新语料库
    corpus = [dictionary.doc2bow(text) for text in texts]
    print(corpus)
    # 10、将新语料库通过tf-idf model 进行处理，得到tfidf
    tfidf = models.TfidfModel(corpus)
    for vec in tfidf[corpus]:
        print(vec)
    # 8、将要对比的文档通过doc2bow转化为稀疏向量
    data3 = "基本思路 我 爱 北京 天安门"
    new_xs = dictionary.doc2bow(data3.split())
    print(new_xs)

    featurenum = len(dictionary.token2id.keys())

    # #12、稀疏矩阵相似度，从而建立索引
    index = similarities.SparseMatrixSimilarity(tfidf[corpus], num_features=featurenum)
    # #13、得到最终相似结果
    sim = index[tfidf[new_xs]]
    print(sim)

常用函数四：关键词提取

# -*- coding: utf-8 -*-

"""
Datetime: 2020/07/26
Author: Zhang Yafei
Description: 文本关键词抽取
"""
from jieba.analyse import extract_tags, textrank, set_stop_words, set_idf_path
from pandas import read_csv, read_excel, read_table, DataFrame, Series
from sklearn.feature_extraction.text import TfidfVectorizer
import numpy as np


class KeywordsExtract(object):
    def __init__(self, file, text_col, new_col, stopwords_file_path=None, idf_path=None):
        self.data = self.read_file(file)
        self.text_col = text_col
        self.new_col = new_col
        if stopwords_file_path:
            set_stop_words(stop_words_path=stopwords_file_path)
        if idf_path:
            set_idf_path(idf_path=idf_path)

    @staticmethod
    def read_file(file):
        if file.endswith('csv'):
            return read_csv(file)
        elif file.endswith('xls') or file.endswith('xlsx'):
            return read_excel(file)
        elif file.endswith('tsv'):
            return read_table(file)

    @staticmethod
    def func_tfidf(row):
        keywords = extract_tags(row, topK=20, allowPOS=('n', 'nr', 'ns'))
        return '; '.join(keywords)

    @staticmethod
    def func_textrank(row):
        keywords = textrank(row, topK=20, allowPOS=('n', 'nr', 'ns'))
        return '; '.join(keywords)

    def extract_keywords_with_tfidf(self, path):
        """ jieba tfidf关键词抽取 """
        self.data[self.text_col].apply(self.func_tfidf)
        self.to_file(self.data, path)

    def extract_keyword_with_textrank(self, path):
        """ jieba textrank关键词抽取 """
        self.data[self.new_col] = self.data[self.text_col].apply(self.func_textrank)
        self.to_file(self.data, path)

    def tfidf_keywords_extract(self, path):
        """ sklearn tfidf关键词抽取 """
        documents = self.data[self.text_col]
        vectorizer = TfidfVectorizer()
        # vectorizer = TfidfVectorizer(token_pattern=r'(?u)\b\w+\b')  # 匹配单个字符
        corpus = vectorizer.fit_transform(documents).toarray()
        vocabs = vectorizer.get_feature_names()
        words_list = []
        for index, doc in enumerate(corpus):
            # words = '; '.join(np.array(vocabs)[np.argsort(-doc)[:15]])
            words = '; '.join(np.array(vocabs)[np.where(doc > 0.2)])
            words_list.append(words)
        self.data[self.new_col] = Series(words_list)
        self.to_file(self.data, path)

    @staticmethod
    def to_file(df: DataFrame, path):
        if path.endswith('csv'):
            return df.to_csv(path, index=False, encoding='utf_8_sig')
        elif path.endswith('xls') or file.endswith('xlsx'):
            return df.to_excel(path, index=False)
        elif path.endswith('tsv'):
            return df.to_csv(path, sep='\t', index=False)


if __name__ == '__main__':
    key_extract = KeywordsExtract(file='data.xlsx', text_col='摘要', new_col='TFID关键词')
    # key_extract.extract_keywords_with_tfidf()
    # key_extract.extract_keyword_with_textrank()
    key_extract.tfidf_keywords_extract()