en/AdvancedNlpCourse2020/TopicModelling: models.py

import logging, gensim
logging.basicConfig(format='%(asctime)s : %(levelname)s : %(message)s', level=logging.INFO)

# load id->word mapping (the dictionary), one of the results of step 2 above
id2word = gensim.corpora.Dictionary.load_from_text('wiki_cs/wiki_cs_wordids.txt')

# load corpus iterator
mm = gensim.corpora.MmCorpus('wiki_cs/wiki_cs_tfidf.mm')

# extract 10 LSA topics; use the default one-pass algorithm
lsa = gensim.models.lsimodel.LsiModel(corpus=mm, id2word=id2word, num_topics=10)
lsa2 = gensim.models.lsimodel.LsiModel(corpus=mm, id2word=id2word, num_topics=5)
lsa3 = gensim.models.lsimodel.LsiModel(corpus=mm, id2word=id2word, num_topics=10, power_iters=5)

# extract 10 LDA topics, using 1 pass and updating once every 1 chunk (5,000 documents)
lda = gensim.models.ldamodel.LdaModel(corpus=mm, id2word=id2word, num_topics=10, update_every=1, chunksize=5000, passes=1)

print("LSA 10 topics")
print(lsa.show_topics())
print("LSA 5 topics")
print(lsa2.show_topics())
print("LSA 10 topics, 5 iters")
print(lsa3.show_topics())
print("LDA topics")
print(lda.show_topics())