[NLP][Python] 使用 FastText 訓練詞向量

# coding: utf-8
"""
Extracted the content in the wiki_database
"""
from gensim.corpora import WikiCorpus


# Load data
wiki_corpus = WikiCorpus('../data/zhwiki-latest-pages-articles-multistream.xml.bz2', dictionary={})


# Save data
with open('wiki_text.txt', 'w', encoding='utf-8') as f:
    print('Start to preprocess.')
    for times, text in enumerate(wiki_corpus.get_texts()):
        f.write(' '.join(text)+'\n')

        if (times+1) % 10000 == 0:
            print(times+1)

# coding: utf-8
"""
Tokenize
"""
import jieba
from opencc import OpenCC


# Initial
cc = OpenCC('s2t')


# Tokenize
with open('wiki_text_seg.txt', 'w', encoding='utf-8') as new_f:
    with open('wiki_text.txt', 'r', encoding='utf-8') as f:
        for data in f:
            data = cc.convert(data)
            data = jieba.cut(data)
            data = [word for word in data if word != ' ']
            data = ' '.join(data)

            new_f.write(data)

# coding: utf-8
from gensim.models import word2vec, fasttext


# Settings
seed = 666
sg = 0
window_size = 10
vector_size = 100
min_count = 1
workers = 8
epochs = 5
batch_words = 10000


# Train
train_data = word2vec.LineSentence('wiki_text_seg.txt')
model = fasttext.FastText(
    train_data,
    min_count=min_count,
    size=vector_size,
    workers=workers,
    iter=epochs,
    window=window_size,
    sg=sg,
    seed=seed,
    batch_words=batch_words,
)


model.save('fasttext.model')

# coding: utf-8
"""
Test the w2v model
"""
from gensim.models import word2vec


# Load the model
model = word2vec.Word2Vec.load('models/fasttext.model')


# Test
print(model['生物'].shape)

for item in model.most_similar('生物'):
    print(item)

model.save('fasttext.model')