import pandas as pd
import re
import nltk
from nltk.tokenize import RegexpTokenizer
from sentiment import remove_punctuation, word_lemmatizer, sentiment

################ People's Daily_US #################
# import news
renmin = pd.read_csv("D:\\Dropbox\\China Foreign Share Discount\\People Daily\\All\\factiva0712.csv")

renmin['text'] = renmin['lp']+ '/n/n' + renmin['td']

renmin = renmin.dropna(subset=['text'])

renmin['text'] = [remove_punctuation(text) for text in renmin['text']]        # remove punctuations
tokenizer = RegexpTokenizer(r'\w+')
renmin['text'] = renmin['text'].apply(lambda x: tokenizer.tokenize(x.lower()))        # tokenize
renmin['text'] = renmin['text'].apply(lambda x: word_lemmatizer(x))       # remove stop words

positive = []
negative = []
positive_lm = []
negative_lm = []
wordcount = []

# calculate sentiment
for news in renmin['text']:
    x1,y1,x2,y2,z = sentiment(news)
    positive.append(x1)
    negative.append(y1)
    positive_lm.append(x2)
    negative_lm.append(y2)
    wordcount.append(z)

renmin['pos'] = positive
renmin['neg'] = negative
renmin['pos_lm'] = positive_lm
renmin['neg_lm'] = negative_lm
renmin['wordcount'] = wordcount

# political news flag
renmin['political'] = renmin['ns'].str.contains('gcat : Political/General News', flags=re.IGNORECASE, regex=True)
renmin['gpir'] = renmin['ns'].str.contains('gpir', flags=re.IGNORECASE, regex=True)

# other news flage
renmin['econ'] = renmin['ns'].str.contains('ecat : Economic News', flags=re.IGNORECASE, regex=True)
renmin['corp'] = renmin['ns'].str.contains('ccat : Corporate/Industrial News', flags=re.IGNORECASE, regex=True)
renmin['commodity'] = renmin['ns'].str.contains('mcat : Commodity/Financial Market News', flags=re.IGNORECASE, regex=True)

# mainly US flag
renmin['re_split'] = renmin['re'].str.split('|')
renmin.loc[:, 're_us'] = renmin.re_split.map(lambda x: 'usa' in x[0])
renmin.loc[:, 're_first'] = renmin.re_split.map(lambda x: x[0])


renmin1 = renmin[['pd', 'political', 'gpir', 'econ', 'corp', 'commodity', 're_us', 're_first', 'pos', 'neg', 'pos_lm', 'neg_lm', 'wordcount']]
renmin1.rename(columns={"pd": "date"})
renmin1.to_csv("sentiment_renmin0712.csv")


"""
################ People's Daily_Hong Kong #################
# import news
renmin = pd.read_csv("D:\\Dropbox\\China Foreign Share Discount\\People Daily HK\\factiva.csv")

renmin['text'] = renmin['lp']+ '/n/n' + renmin['td']

renmin = renmin.dropna(subset=['text'])

renmin['text'] = [remove_punctuation(text) for text in renmin['text']]        # remove punctuations
renmin['text'] = renmin['text'].apply(lambda x: tokenizer.tokenize(x.lower()))        # tokenize
renmin['text'] = renmin['text'].apply(lambda x: word_lemmatizer(x))       # remove stop words

positive = []
negative = []
positive_lm = []
negative_lm = []
wordcount = []

# calculate sentiment
for news in renmin['text']:
    x1,y1,x2,y2,z = sentiment(news)
    positive.append(x1)
    negative.append(y1)
    positive_lm.append(x2)
    negative_lm.append(y2)
    wordcount.append(z)

renmin['pos'] = positive
renmin['neg'] = negative
renmin['pos_lm'] = positive_lm
renmin['neg_lm'] = negative_lm
renmin['wordcount'] = wordcount

# political news flag
renmin['political'] = renmin['ns'].str.contains('gcat : Political/General News', flags=re.IGNORECASE, regex=True)
renmin['gpir'] = renmin['ns'].str.contains('gpir', flags=re.IGNORECASE, regex=True)

# region flag
renmin.loc[:, 're_us'] = ['usa' in region for region in renmin['re']]

renmin1 = renmin[['pd', 'political', 'gpir', 're_us', 'pos', 'neg', 'pos_lm', 'neg_lm', 'wordcount']]
renmin1.rename(columns={"pd": "date"})
renmin1.to_csv("sentiment_renminHK.csv")
"""