from nltk.corpus import opinion_lexicon
import string
import re
from nltk.stem import PorterStemmer, WordNetLemmatizer
from nltk.corpus import stopwords
import csv


#nltk.download('opinion_lexicon')

# clean up the text

def remove_punctuation(text):
    no_urls = re.sub(r'(http|https)\S+', "", text)        #remove urls
    no_dots = re.sub(r'(\.|-){2,}', " ", no_urls)  # replace ... and --- with space
    no_enter = re.sub(r'\n', ' ', no_dots)
    no_s = re.sub(r"'s", "", no_enter)
    no_number = "".join([c for c in no_s if not c.isdigit()])
    no_punct = "".join([c for c in no_number if c not in string.punctuation])       #remove punctuations
    return no_punct

ps = PorterStemmer()

def word_lemmatizer(text):
    words = [w for w in text if w not in stopwords.words('english')]
#    stem_text = [ps.stem(w) for w in words]
#    lemma_text = [lemmatizer.lemmatize(w) for w in words]
    return words


pos_list = set(opinion_lexicon.positive())
neg_list = set(opinion_lexicon.negative())

pos_list_lm = set()
neg_list_lm = set()
with open('Neg_LM.csv', newline='') as csvfile:
    reader = csv.reader(csvfile, delimiter=' ')
    for row in reader:
        neg_list_lm.add(row[0].lower())

with open('Pos_LM.csv', newline='') as csvfile:
    reader = csv.reader(csvfile, delimiter=' ')
    for row in reader:
        pos_list_lm.add(row[0].lower())

# calculate sentiment
def sentiment(text):

    pos=0
    neg=0
    pos_lm=0
    neg_lm=0
    total = len(text)
    for word in text:
        if word in pos_list:
            pos += 1
        elif word in neg_list:
            neg += 1
        if word in pos_list_lm:
            pos_lm += 1
        elif word in neg_list_lm:
            neg_lm += 1
    return pos, neg, pos_lm, neg_lm, total