[Py3.5] DOC2BOW w/ Gensim

/*******************************************************************************************************************
-- Title : [Py3.5] DOC2BOW w/ Gensim
-- Reference : googling
-- Key word : nlp gensim stopwords stopword stop word word token word tokenizing word tokenization
                  word count vector word frequency word dictionary word feature doc2bow token2id 자연어처리
                  불용어 단어 토큰 워드 토큰 단어 빈도수 사전 단어 피처 
*******************************************************************************************************************/

■ Scripts

from gensim import corpora
from collections import defaultdict
from pprint import pprint  # pretty-printer
 
documents = ["Human machine interface for lab abc computer applications",
             "A survey of user opinion of computer system response time",
             "The EPS user interface management system",
             "System and human system engineering testing of EPS",
             "Relation of user perceived response time to error measurement",
             "The generation of random binary unordered trees",
             "The intersection graph of paths in trees",
             "Graph minors IV Widths of trees and well quasi ordering",
             "Graph minors A survey"]
 
print("\n", documents)
print("... corpus", "." * 100, "\n")
 
 
# =======================================
# -- Remove common words and word_tokenize
# =======================================
 
# --
# -- Add stopwords
# --
stopwords = set('for a of the and to in'.split())
 
print(stopwords)
print(",,, stopwords", "," * 100, "\n")
 
# --
# -- Word Tokenizing
# --
word_token_by_sent = [[word for word in sent_token.lower().split() if word not in stopwords]
                      for sent_token in documents]
 
pprint(word_token_by_sent)
print(",,, word_token_by_sent", "," * 100, "\n")
 
 
# =======================================
# -- Remove words that appear only once
# =======================================
word_frequency = defaultdict(int)
 
# --
# -- Word Counting / word frequency
# --
for word_token in word_token_by_sent:
    for wtoken in word_token:
        word_frequency[wtoken] += 1
 
print(type(word_frequency))
print(word_frequency)
print("::: word frequency", ":" * 100, "\n")
 
# --
# -- Remove words that appear only once
# --
word_token_by_sent2 = [[token for token in text if word_frequency[token] > 1]
                       for text in word_token_by_sent]
 
pprint(word_token_by_sent2)
print("::: word_token_by_sent2", ":" * 100, "\n")
 
 
# =======================================
# -- dictionary / word feature / word token
# =======================================
word_dictionary = corpora.Dictionary(word_token_by_sent2)
word_dictionary.save('d:/word_dictionary.dict')  # store the dictionary, for future reference
 
print("count:", len(word_dictionary), "   ", word_dictionary, )
 
for idx, val in enumerate(word_dictionary):
    print(idx, word_dictionary[idx])
print(";;; enumerate(word_dictionary)", ";" * 100, "\n")
 
print(word_dictionary.token2id)
print(";;; word_dictionary.token2id", ";" * 100, "\n")
 
 
# =======================================
# -- Get doc2bow
# =======================================
new_doc = "Human computer interaction, human. human computer graph science."
new_vec = word_dictionary.doc2bow(new_doc.lower().split())
 
print(new_vec)  # the word "interaction" does not appear in the dictionary and is ignored
print("!!! doc2bow", "!" * 100, "\n")