topic_modeling.py

import math
import os
import pickle
import re
from collections import Counter

import numpy as np
import plotly
import plotly.graph_objs as go
from gensim.corpora import Dictionary
from gensim.models import TfidfModel
from gensim.models import ldamodel
from matplotlib import pyplot as plt
from nltk import ngrams
from nltk.stem import WordNetLemmatizer
from nltk.tokenize import word_tokenize
from plotly import tools

from sentiment_afinn import afinn


lemmatizer = WordNetLemmatizer()

# generating a list of stopwords from file stopwords.txt.
stop_words = []
with open("stopwords.txt") as f:
    stop_words = f.readlines()
stop_words = [elem.strip() for elem in stop_words if not elem == '\n']


# This function is used to preprocess the reviews. Input to this function is
# a list of strings, which are reviews. In this function after tokenizing a review
# into words,We remove the stopwords and apply lemmatizing on the words to replace
# the word with its basic lemma.
def preprocess_data(reviews):
    preprocessed_reviews = []
    for text in reviews:
        words = word_tokenize(text)
        words = [word.lower() for word in words if word.isalpha()]
        words = [word for word in words if word not in stop_words]
        lemmatized_words = [lemmatizer.lemmatize(
            word) for word in words]
        preprocessed_reviews.append(lemmatized_words)
    with open("preprocessed_reviews.pkl", "wb") as f:
        pickle.dump(preprocessed_reviews, f)
    return preprocessed_reviews


# THis function is used to extract the topics from the preprocessed reviews.
# In this function we are using gensim tool for applying lda on a corpus.
# we are using unigram model with term frequency or tfidf as a featureset.
def topic_extraction(reviews, title):
    nooftopics = 10
    # Joining bigrams for a review to capture the phrases like 'tasty pizza'
    for id, review in enumerate(reviews):
        reviews[id] = review + ["_".join(w) for w in ngrams(review, 2)]
    # Create a dictionary of words from overall reviews.
    dicionary = Dictionary(reviews)
    # change reviews into Bag of words model/ unigram model, here feature is term frequency.
    corpus = [dicionary.doc2bow(review) for review in reviews]
    # to transform from term frequency to tfidf matrix.
    tfidf = TfidfModel(corpus)
    tfidf_corpus = tfidf[corpus]
    # Making a lda model.
    lda_tf = ldamodel.LdaModel(corpus, id2word=dicionary,
                               alpha='auto', num_topics=nooftopics, passes=5)
    lda_tfidf = ldamodel.LdaModel(tfidf_corpus, id2word=dicionary, alpha='auto',
                                  num_topics=nooftopics, passes=5)
    # with open("lda.pkl", "wb") as f:
    #     pickle.dump(lda, f)
    topic_list = lda_tf.print_topics(num_topics=10, num_words=10)
    topic_list_tfidf = lda_tfidf.print_topics(num_topics=10, num_words=10)

    with open("topic_list.pkl", "wb") as f:
        pickle.dump(topic_list, f)
    draw_graph_for_topics(topic_list, title, nooftopics)
    return True


# This is a helper function which is used to generate number of rows,columns for subplots.
def generate_row_column(nooftopics):
    rows = int(nooftopics ** 0.5)
    if rows ** 2 == nooftopics:
        cols = rows
    else:
        cols = rows + 1
    return rows, cols


# This is a helper function which is used to generate co-ordinates for subplots.
def generate_coordinates(cnt, row, col):
    xc = math.ceil(cnt / col)
    if cnt % col == 0:
        yc = col
    else:
        yc = cnt % col
    return xc, yc


# This function is used to generate graph for the topics distribution
# of lda. Input to this function is topic_list which is generated by print_topics()
# function of lda and parameter title is the title of generated graph.
def draw_graph_for_topics(topic_list, title, nooftopics):
    f_words = open('words.txt', 'w')
    # with open("topic_list.pkl", "rb") as f:
    #     topic_list = pickle.load(f)
    topics = []
    for topic, details in topic_list:
        details = re.sub('"', '', details)
        details = details.split(' + ')
        details = [detail.split('*') for detail in details]
        topics.append(details)
    row, col = generate_row_column(nooftopics=nooftopics)
    cnt = 1
    fig = tools.make_subplots(rows=row, cols=col)
    for topic in topics:
        xs = []
        ys = []
        for x, y in topic:
            ys.append(y)
            xs.append(math.fabs(float(x)))
            f_words.write(str(y) + "\n")
        xs.reverse()
        ys.reverse()
        trace = go.Bar(
            x=xs,
            y=ys,
            orientation='h',
        )
        xc, yc = generate_coordinates(cnt, row, col)
        fig.append_trace(trace, xc, yc)
        cnt += 1
    f_words.close()
    fig['layout'].update(height=1000, width=1000,
                         title=title, showlegend=False)
    plotly.offline.plot(fig)


# This function is used to predict the sentiment accuracy using AFINN sentiment
# analysis on the preprocessed reviews. In this function we also binarize the ratings
# to calculate the actual sentiment of reviews.



# This is a helper function to plot the pie-chart of ratings distribution in database.
def ratings_distribution_graph(ratings):
    ratings = dict(Counter(ratings))
    ratings = dict(sorted(ratings.items()))
    print(ratings, type(ratings))
    ratings = [b for a, b in ratings.items()]
    labels = ['1', '2', '3', '4', '5']
    plt.pie(ratings, labels=labels, autopct='%1.0f%%', radius=0.8)
    plt.title('Restaurant ratings distribution')
    plt.legend(['1', '2', '3', '4', '5'], title='Ratings', fontsize='xx-small')
    plt.show()
    return


# This helper function is used to plot a graph between sentiment score of afinn and
# ratings. This shows that which positive words appear more in higher ratings and same for
# negative words in lower ratings.
def afinnvsrating_graph(reviews, ratings):
    x = [[], [], [], [], [], []]
    for _, r in enumerate(ratings):
        x[r].append(reviews[_])
    data = []
    for rating, t in enumerate(x[1:]):
        words = []
        for review in t:
            for word in set(review):
                if word in afinn:
                    words.append(word)
        print(len(t))
        cutoff = 1000
        words = dict(Counter(words))
        keys = [a for a, b in words.items()]
        xs = []
        for x in keys:
            if words[x] > cutoff:
                xs.append(x)
        keys = xs
        xs = list(map(lambda x: afinn.get(x), xs))
        ys = np.linspace(rating, rating + 1, num=len(xs))
        cnt = 0
        trace = go.Scatter(x=xs, y=ys, mode='markers+text', text=keys, textposition='right')
        data.append(trace)
    layout = go.Layout(title='Figures showing distribution of rating and Afinn sentiment over words',
                       xaxis=dict(
                           title='Afinn-senitment score',
                           titlefont=dict(
                               family='Courier New, monospace',
                               size=15,
                               color='#7f7f7f'
                           )
                       ),
                       yaxis=dict(
                           title='Rating',
                           titlefont=dict(
                               family='Courier New, monospace',
                               size=15,
                               color='#7f7f7f'
                           )
                       ), showlegend=False, autosize=False, width=1200, height=800)
    fig = go.Figure(data=data, layout=layout)
    plotly.offline.plot(fig)


# This function is used to generate a file named docs.pkl which contain the noofreviews
# which you want to extract from database. Once it is generated you don't need to again iterate
# the database. load this file and work. This file contain reviews, ratings and their corresponding
# businessids.



# this function is used to generate a frequency distribution graph of words in the reviews.
# It shows which words occur more frequently in reviews.
def most_common_words_graph(reviews):
    counter = Counter()
    for review in reviews:
        counter.update(review)
    x = counter.most_common(20)
    print(x, type(x))
    xs = []
    ys = []
    for a, b in x:
        xs.append(a)
        ys.append(b)
    # print(xs,ys)
    xs.reverse()
    ys.reverse()
    fig, ax = plt.subplots()
    width = 0.25  # the width of the bars
    ind = np.arange(len(ys))  # the x locations for the groups
    ax.barh(ind, ys, width)
    ax.set_yticks(ind + width / 2)
    ax.set_yticklabels(xs, minor=False)
    for i, v in enumerate(ys):
        ax.text(v + 3, i, str(v), color='blue', )
    plt.title('Frequency distribution of most common words')
    plt.xlabel('Frequency')
    plt.ylabel('Words')
    # plt.show()
    plt.savefig(os.path.join('freq_words.png'), dpi=300, format='png', bbox_inches='tight')


if __name__ == '__main__':
    with open("docs_preprocessed.pkl", "rb") as f:
        obj = pickle.load(f)
    reviews = obj[0][:500]
    ratings = obj[1][:500]
    topic_extraction(reviews, "Overall topic analysis")
    # pos_reviews=[]
    # neg_reviews=[]
    # for x in range(len(reviews)):
    #     if ratings[x]>3:
    #         pos_reviews.append(reviews[x])
    #     elif ratings[x]<3:
    #         neg_reviews.append(reviews[x])
    # topic_extraction(pos_reviews,"Positive topic analysis")
    # topic_extraction(neg_reviews,"Negative topic analysis")