# Compute Sentiment Measures¶

In [1]:

    import pandas as pd
    import numpy as np
    import os, re, multiprocessing
    import dask.dataframe as dd

In [2]:

    import nltk
    nltk.download('punkt')
    nltk.download('punkt_tab')
    from nltk.tokenize import sent_tokenize
    from nltk.tokenize import word_tokenize

    [nltk_data] Downloading package punkt to
    [nltk_data]     C:\Users\rache\AppData\Roaming\nltk_data...
    [nltk_data]   Package punkt is already up-to-date!
    [nltk_data] Downloading package punkt_tab to
    [nltk_data]     C:\Users\rache\AppData\Roaming\nltk_data...
    [nltk_data]   Package punkt_tab is already up-to-date!

In [3]:

    %matplotlib inline
    %load_ext autoreload
    %autoreload 2

In [4]:

    pd.set_option('display.memory_usage', 'deep')
    pd.set_option('display.precision', 2)
    pd.set_option('display.width', 200)
    pd.set_option('display.max_rows', 300)
    pd.set_option('display.max_columns', None)
    pd.set_option('display.max_colwidth', 80)
    pd.options.display.float_format = '{:,.4f}'.format

In [5]:

    from filepath_03 import * # change to reflect input and output file path

In [6]:

    complong = pd.read_pickle(filepath_complong)

In [7]:

    LM_dictionary = pd.read_csv(filepath_LMdict)

In [8]:

    Fin_Neg = LM_dictionary[LM_dictionary.Negative > 0]['Word'].str.lower()
    Fin_Pos = LM_dictionary[LM_dictionary.Positive > 0]['Word'].str.lower()

In [9]:

    def neg_count(text):
        wordlist = word_tokenize(text)
        count = 0
        for word in wordlist:
            if word in Fin_Neg.tolist():
                count+=1
        return count

    def pos_count(text):
        wordlist = word_tokenize(text)
        count = 0
        for word in wordlist:
            if word in Fin_Pos.tolist():
                count+=1
        return count

In [10]:

    %%time
    ddf = dd.from_pandas(complong, npartitions=(multiprocessing.cpu_count()-1)*1)
    meta_df = pd.Series(dtype=int)
    complong['pos_ct'] = ddf.apply(lambda x: pos_count(x['text']), axis=1, meta=meta_df).compute(scheduler="multiprocessing")
    complong['neg_ct'] = ddf.apply(lambda x: neg_count(x['text']), axis=1, meta=meta_df).compute(scheduler="multiprocessing")

    CPU times: total: 1min 57s
    Wall time: 6h 34min 42s

In [11]:

    complong['pos_pct'] = complong['pos_ct'] / complong['nwords']
    complong['neg_pct'] = complong['neg_ct'] / complong['nwords']
    complong['sentiment'] = (complong['pos_ct'] - complong['neg_ct']) / complong['nwords']

In [12]:

    cols = ['transcriptid','section','seq',
            'pos_ct', 'neg_ct', 'nwords',
            'pos_pct', 'neg_pct', 'sentiment']
    complong2 = complong[cols].copy()

In [13]:

    complong2.to_pickle(outputpath_compsent)

In [ ]: