文本向量系列-基于潜在语义分析的词向量

documents = ["Human machine interface for lab abc computer applications",
             "A survey of user opinion of computer system response time",
             "The EPS user interface management system",
             "System and human system engineering testing of EPS",
             "Relation of user perceived response time to error measurement",
             "The generation of random binary unordered trees",
             "The intersection graph of paths in trees",
             "Graph minors IV Widths of trees and well quasi ordering",
             "Graph minors A survey"]

from collections import Counter
from itertools import chain
import numpy as np

def word_matrix(documents):
    '''计算词频矩阵'''
    # 所有字母转换位小写
    docs = [d.lower() for d in documents]
    # 分词
    docs = [d.split() for d in docs]
    # 获取所有词
    words = list(set(chain(*docs)))
    # 词到ID的映射, 使得每个词有一个ID
    dictionary = dict(zip(words, range(len(words))))
    # 创建一个空的矩阵, 行数等于词数, 列数等于文档数
    matrix = np.zeros((len(words), len(docs)))
    # 逐个文档统计词频
    for col, d in enumerate(docs):
        # 统计词频
        count = Counter(d)
        for word in count:
            # 用word的id表示word在矩阵中的行数
            id = dictionary[word]
            # 把词频赋值给矩阵
            matrix[id, col] = count[word]
    return matrix, dictionary

matrix, dictionary = word_matrix(documents)

matrix[:10, :10]

print(dictionary)

from scipy import linalg
# 使用scipy模块进行svd分解, 得到三个矩阵
U, sigma, VT = linalg.svd(matrix)

print('U.shape:', U.shape)
print('sigma.shape:', sigma.shape)
print('VT.shape:', VT.shape)

# 降维后的维数
n = 3

# 取前n个向量
U2=U[:, :n]
VT2=VT[:n]

# 把向量转换位对角矩阵
sigma_matrix=linalg.diagsvd(sigma, U2.shape[0], VT2.shape[1])
# 截取相应的部分
sigma_matrix=sigma_matrix[:U2.shape[1],:VT2.shape[0]]

# SVD的逆运算, 得到新的语义空间, 它和词频矩阵有相同的形状
X_new=np.dot(U2, sigma_matrix)
X_new=np.dot(X_new, VT2)
print('估计得到的潜在语义空间')
print('X_new.shape:', X_new.shape)

%matplotlib inline
import matplotlib.pyplot as plt
from scipy.spatial.distance import cosine


def word_relations(weights, ):
    relations = np.zeros((len(weights), len(weights)))
    for i in range(len(weights)):
        vec1 = weights[i]
        for j in range(i, len(weights)):
            vec2 = weights[j]
            relations[i, j] = cosine(vec1, vec2)
    reverse = dict(zip(dictionary.values(), dictionary.keys()))
    plt.matshow(relations)
    
word_relations(X_new)

def doc_relations(weights):
    n = weights.shape[1]
    relations = np.zeros((n, n))
    for i in range(n):
        vec1 = weights[:, i]
        for j in range(i, n):
            vec2 = weights[:, j]
            relations[j, i] = cosine(vec1, vec2)
    reverse = dict(zip(dictionary.values(), dictionary.keys()))
    plt.matshow(relations)
    
doc_relations(X_new)