文本向量系列-如何基于词频矩阵和TF-IDF权重构建词向量

documents = ["Human machine interface for lab abc computer applications",
             "A survey of user opinion of computer system response time",
             "The EPS user interface management system",
             "System and human system engineering testing of EPS",
             "Relation of user perceived response time to error measurement",
             "The generation of random binary unordered trees",
             "The intersection graph of paths in trees",
             "Graph minors IV Widths of trees and well quasi ordering",
             "Graph minors A survey"]

from collections import Counter
from itertools import chain
import numpy as np

def word_matrix(documents):
    '''计算词频矩阵'''
    # 所有字母转换位小写
    docs = [d.lower() for d in documents]
    # 分词
    docs = [d.split() for d in docs]
    # 获取所有词
    words = list(set(chain(*docs)))
    # 词到ID的映射, 使得每个词有一个ID
    dictionary = dict(zip(words, range(len(words))))
    # 创建一个空的矩阵, 行数等于词数, 列数等于文档数
    matrix = np.zeros((len(words), len(docs)))
    # 逐个文档统计词频
    for col, d in enumerate(docs):
        # 统计词频
        count = Counter(d)
        for word in count:
            # 用word的id表示word在矩阵中的行数
            id = dictionary[word]
            # 把词频赋值给矩阵
            matrix[id, col] = count[word]
    return matrix, dictionary

matrix, dictionary = word_matrix(documents)

print(matrix[:10,:10])
dictionary

print(dictionary)

def tf(matrix):
    # 计算每个文档的总次数
    sm = np.sum(matrix, axis=0)
    print(sm)
    # 每个词的词频除以每个文档的词频
    return matrix / sm

tf(matrix)[:5, :5]

def idf(matrix):
    '''计算IDF'''
    # 文档总数
    D = matrix.shape[1]
    # 包含每个词的文档数
    j = np.sum(matrix>0, axis=1)
    print(j)
    return D / j

idf(matrix)

def tf_idf(matrx):
    return tf(matrix) * idf(matrix).reshape(matrix.shape[0], 1)

weights = tf_idf(matrix)
weights[:5, :5]

%matplotlib inline
import matplotlib.pyplot as plt
from scipy.spatial.distance import cosine


def word_relations(weights, ):
    relations = np.zeros((len(weights), len(weights)))
    for i in range(len(weights)):
        vec1 = weights[i]
        for j in range(i, len(weights)):
            vec2 = weights[j]
            relations[i, j] = cosine(vec1, vec2)
    reverse = dict(zip(dictionary.values(), dictionary.keys()))
    plt.matshow(relations)
    
word_relations(weights)
    

def doc_relations(weights):
    n = weights.shape[1]
    relations = np.zeros((n, n))
    for i in range(n):
        vec1 = weights[:, i]
        for j in range(i, n):
            vec2 = weights[:, j]
            relations[i, j] = cosine(vec1, vec2)
    reverse = dict(zip(dictionary.values(), dictionary.keys()))
    plt.matshow(relations)
    print(relations)
    
doc_relations(weights)