Compute Sentiment Measures¶

In [1]:
import pandas as pd
import numpy as np
import os, re, multiprocessing
import dask.dataframe as dd
In [2]:
import nltk
nltk.download('punkt')
nltk.download('punkt_tab')
from nltk.tokenize import sent_tokenize
from nltk.tokenize import word_tokenize

[nltk_data] Downloading package punkt to
[nltk_data]     C:\Users\rache\AppData\Roaming\nltk_data...
[nltk_data]   Package punkt is already up-to-date!
[nltk_data] Downloading package punkt_tab to
[nltk_data]     C:\Users\rache\AppData\Roaming\nltk_data...
[nltk_data]   Package punkt_tab is already up-to-date!
In [3]:
%matplotlib inline 
%load_ext autoreload
%autoreload 2
In [4]:
pd.set_option('display.memory_usage', 'deep')
pd.set_option('display.precision', 2) 
pd.set_option('display.width', 200)
pd.set_option('display.max_rows', 300)
pd.set_option('display.max_columns', None)
pd.set_option('display.max_colwidth', 80)
pd.options.display.float_format = '{:,.4f}'.format
In [5]:
from filepath_03 import * # change to reflect input and output file path
In [6]:
complong = pd.read_pickle(filepath_complong)
In [7]:
LM_dictionary = pd.read_csv(filepath_LMdict)
In [8]:
Fin_Neg = LM_dictionary[LM_dictionary.Negative > 0]['Word'].str.lower()
Fin_Pos = LM_dictionary[LM_dictionary.Positive > 0]['Word'].str.lower()
In [9]:
def neg_count(text):
    wordlist = word_tokenize(text)
    count = 0
    for word in wordlist:
        if word in Fin_Neg.tolist():
            count+=1
    return count
        
def pos_count(text):
    wordlist = word_tokenize(text)
    count = 0
    for word in wordlist:
        if word in Fin_Pos.tolist():
            count+=1
    return count
In [10]:
%%time
ddf = dd.from_pandas(complong, npartitions=(multiprocessing.cpu_count()-1)*1)
meta_df = pd.Series(dtype=int)
complong['pos_ct'] = ddf.apply(lambda x: pos_count(x['text']), axis=1, meta=meta_df).compute(scheduler="multiprocessing")
complong['neg_ct'] = ddf.apply(lambda x: neg_count(x['text']), axis=1, meta=meta_df).compute(scheduler="multiprocessing")

CPU times: total: 1min 57s
Wall time: 6h 34min 42s
In [11]:
complong['pos_pct'] = complong['pos_ct'] / complong['nwords'] 
complong['neg_pct'] = complong['neg_ct'] / complong['nwords']
complong['sentiment'] = (complong['pos_ct'] - complong['neg_ct']) / complong['nwords']
In [12]:
cols = ['transcriptid','section','seq', 
        'pos_ct', 'neg_ct', 'nwords',
        'pos_pct', 'neg_pct', 'sentiment']
complong2 = complong[cols].copy()
In [13]:
complong2.to_pickle(outputpath_compsent)
In [ ]: