大数据揭示心理学规律(利用gensim训练词向量)

# author: mlln.cn
# email: xxxspy@126.com
# qq: 675495787
from gensim.corpora.textcorpus import TextDirectoryCorpus
import jieba
from pyltp import Segmentor
from pyltp import SentenceSplitter
from gensim.corpora.dictionary import Dictionary
class MyTextDirCorpus(TextDirectoryCorpus):
    
    def __init__(self, input, **kwargs):

        kwargs['tokenizer'] = self.tokenizer
        super().__init__(input, **kwargs)
    
    def tokenizer(self, text):
        if not hasattr(self, '_segmentor'):
            model_path = r'D:\mysites\text-characters\tcharacters\ltp\ltp_data\cws.model'
            segmentor = Segmentor()  # 初始化实例
            segmentor.load(model_path)
            self._segmentor = segmentor
        segmentor = self._segmentor
        sents = SentenceSplitter.split(text)
        words = [list(segmentor.segment(s)) for s in sents]
        return words
    
    def __del__(self):
        '''释放资源'''
        if hasattr(self, '_segmentor'):
            self._segmentor.release()
        try:
            super().__del__()
        except AttributeError:
            pass
    
    # 为了强制使用'utf8'编码, 我们复写了这个方法
    def getstream(self):
        """Yield documents from the underlying plain text collection (of one or more files).
        Each item yielded from this method will be considered a document by subsequent
        preprocessing methods.
        If `lines_are_documents` was set to True, items will be lines from files. Otherwise
        there will be one item per file, containing the entire contents of the file.
        """
        num_texts = 0
        for path in self.iter_filepaths():
            with open(path, 'rt', encoding='utf8') as f:
                if self.lines_are_documents:
                    for line in f:
                        yield line.strip()
                        num_texts += 1
                else:
                    content = f.read().strip()
                    yield content
                    num_texts += 1

        self.length = num_texts
        
    def get_texts_from_tokens(self):
        for fpath in self.iter_filepaths():
            fpath = Path(fpath)
            token_path = fpath.parent / (fpath.name + '.cached_tokens')
            yield pickle.loads(token_path.read_bytes())
            
    def save_tokens(self):
        '''保存tokens到硬盘, 只需要运行一次'''
        for fpath in self.iter_filepaths():
            fpath = Path(fpath)
            token_path = fpath.parent / (fpath.name + '.cached_tokens')
            txt = fpath.read_text(encoding='utf8').strip()
            tokens = self.tokenizer(txt)
            token_path.write_bytes(pickle.dumps(list(tokens)))
    
    def save_dictionary(self, dpath):
        '''把字典保存到硬盘'''
        self.dictionary.save_as_text(fname=dpath)
        
    def sentence_iter(self):
        texts = self.get_texts_from_tokens()
        for text in texts:
            for sent in text:
                yield sent

dpath = r'D:\mysites\cnki-search\OUTPUTS'
dic_path = 'dictionary.dict'
dic = Dictionary.load_from_text(dic_path)
corpus = MyTextDirCorpus(dpath, 
                min_depth=1, 
                dictionary=dic,
                 pattern='.*\.abstract$',  
                 lines_are_documents=False, 
                 token_filters=[])
len(dic)

# 保存tokens
from pathlib import Path
import pickle
corpus.save_tokens()

from gensim.models.word2vec import Word2Vec   


model= Word2Vec()  
model.build_vocab(corpus.sentence_iter())  
model.train(corpus.sentence_iter(), total_examples=model.corpus_count, epochs=model.epochs)  

model_path = r"D:\mysites\cnki-search\OUTPUTS\vector.model"
model.save(model_path)

model['基于']

model.most_similar(['我'])

v =  model['完美主义'] - model['自尊']
model.most_similar(positive=[v], topn=10)

model.most_similar(positive=['完美主义',  '消极'], negative=[], topn=20)

model.most_similar(positive=['完美主义', '幸福'], negative=[], topn=20)

model.most_similar(positive=['完美主义'], negative=['卓越'], topn=20)

model.most_similar(positive=['完美主义', '成功'], negative=[], topn=20)

model.most_similar(positive=['完美主义', '失败'], negative=[], topn=20)