import os
os.chdir("D:\\Dropbox\\China Foreign Share Discount\\textual analysis")

import pandas as pd
import re
import nltk
from nltk.tokenize import RegexpTokenizer
from sentiment import remove_punctuation, word_lemmatizer, sentiment

####### press release #######
# clean up old data
calls = pd.read_pickle('D:\\Dropbox\\China Foreign Share Discount\\embassy\\embassy_eng\\embassy_eng.pkl')

# blank text
f = open("text874.txt", 'r', encoding="UTF-8")
calls["text"][874] = f.read()

# wrong dates
calls["dates"][24] = '2020-05-06'
calls["dates"][67] = '2020-03-18'
calls["dates"][243] = '2019-08-06'
calls["dates"][257] = '2019-07-22'
calls["dates"][285] = '2019-06-13'
calls["dates"][307] = '2019-05-13'
calls["dates"][326] = '2019-04-10'
calls["dates"][340] = '2019-03-20'
calls["dates"][342] = '2019-03-18'
calls["dates"][356] = '2019-02-26'
calls["dates"][381] = '2019-01-21'
calls["dates"][399] = '2018-12-24'
calls["dates"][414] = '2018-12-03'
calls["dates"][522] = '2018-07-11'
calls["dates"][582] = '2018-04-23'
calls["dates"][604] = '2018-03-28'
calls["dates"][613] = '2018-03-16'
calls["dates"][615] = '2018-03-14'
calls["dates"][658] = '2018-01-11'
calls["dates"][684] = '2017-12-07'
calls["dates"][700] = '2017-11-15'
calls["dates"][763] = '2017-08-14'
calls["dates"][795] = '2017-07-03'

# clean up the new dataset
calls2 = pd.read_pickle('D:\\Dropbox\\China Foreign Share Discount\\embassy\\embassy_eng\\embassy_eng_part2.pkl')

calls2["dates"][82] = '2021-03-26'
calls2["dates"][111] = '2021-02-23'
calls2["dates"][119] = '2021-02-05'
calls2["dates"][165] = '2020-12-04'
calls2["dates"][167] = '2020-12-02'
calls2 = calls2.drop([199])         # duplicate "2020-10-22"
calls2["dates"][216] = '2020-09-30'
calls2["dates"][228] = '2020-09-16'
calls2["dates"][276] = '2020-07-17'
calls2 = calls2.drop([285])         # duplicate '2020-07-07'
calls2["dates"][302] = '2020-06-15'
calls2 = calls2.loc[0:309]          # first record in calls is '2020-06-03'

calls = calls.append(calls2, ignore_index=True)

calls.to_pickle("embassy_eng0712.pkl")

# import press release data
calls = pd.read_pickle('embassy_eng0311.pkl')

# clean up the press release data
answerer_list = ["A", "Hua Chunying", "Geng Shuang", "Zhao Lijian", "Lu Kang", "Hong Lei", "Wang Wenbin"]
questioner_list = ["Q", "CCTV", "China Review News Agency", "China Daily", "China News Service", "TASS", "AFP", "CRI", "Shenzhen TV", "NIKKEI", "The Globe and Mail", "The Financial Times", "BBC", "Reuters", "Bloomberg", "PTI", "Xinhua News Agency", "The Paper", "Global Times", "EFE News Agency", "Follow-up", "CNN", "NHK", "RIA Novosti", "Associated Press of Pakistan", "Beijing Youth Daily", "Beijing Daily", "Tass News Agency", "Phoenix TV", "China Review News", "The Australian", "South China Morning Post", "Tass", "Beijing Media Network", "Yonhap News Agency", "CNR", "ABC Australia", "TBS", "Sky News", "Wall Street Journal", "Prensa Latina", "CNS", "De Telegraaf", "KBS", "JoongAng Ilbo", "Hong Kong Review News", "TASS News Agnecy", "Beijing Media Network", "Australian Broadcasting Corporation", "China Review News", "APP", "Kyodo News", "Sputnik", "Voice of Vietnam", "China Rational Radio", "Swedish TV", "Press Trust of India", "China Radio International", "SCMP"]

pattern_answer = ("(?:" + "|".join(answerer_list) + ")(?:\s*:)"  # answerer e.g. A:
                  + "(.+?)"  # the content of the answer
                  + "(?:" + "|".join(questioner_list) + ")(?:\s*:)"  # the next questioner e.g Q:
                  )

pattern_question = ("(?:" + "|".join(questioner_list) + ")(?:\s*:)"  # questioner e.g. Q:
                    + "(.+?)"  # the content of the answer
                    + "(?:" + "|".join(answerer_list) + ")(?:\s*:)"  # the next answerer e.g A:

                    )

answer_text = []
answer_us_text = []
QA_all = []
QA_us = []
for call in calls['text']:
    call += " Q:"
    start, _ = re.search("("+"|".join(questioner_list)+")"+r"\s*:", call).span()
    call = call[start:]
    answers = re.findall(pattern_answer, call, re.DOTALL)
    questions = re.findall(pattern_question, call, re.DOTALL)
    QA = tuple(zip(answers, questions))
    if len(answers) != len(questions):
        print(call)
    answer_all = ""
    answer_us = ""
    QA_all.append(len(QA))
    num_us = 0
    for answer, question in QA:
        answer_all += answer + " "
        if re.search(r'\bUS\b', answer) or re.search(r'\bUS\b', question) or re.search(r'\bUnite States\b', answer) or re.search(r'\bUnite States\b', question):
            answer_us += answer + " "
            num_us += 1
    QA_us.append(num_us)
    answer_text.append(answer_all)
    answer_us_text.append(answer_us)



calls['answer_text'] = answer_text
calls['answer_us_text'] = answer_us_text
calls['QA_all'] = QA_all
calls['QA_us'] = QA_us

calls = calls.loc[calls['answer_text'] != ""]

calls['answer_text'] = [remove_punctuation(text) for text in calls['answer_text']]  # remove punctuations
calls['answer_us_text'] = [remove_punctuation(text) for text in calls['answer_us_text']]  # remove punctuations
tokenizer = RegexpTokenizer(r'\w+')
calls['answer_text'] = calls['answer_text'].apply(lambda x: tokenizer.tokenize(x.lower()))  # tokenize
calls['answer_us_text'] = calls['answer_us_text'].apply(lambda x: tokenizer.tokenize(x.lower()))  # tokenize
calls['answer_text'] = calls['answer_text'].apply(lambda x: word_lemmatizer(x))  # remove stop words
calls['answer_us_text'] = calls['answer_us_text'].apply(lambda x: word_lemmatizer(x))  # remove stop words


# calculate sentiment
positive = []
negative = []
positive_lm = []
negative_lm = []
wordcount = []
for call in calls['answer_text']:
    x1,y1,x2,y2,z = sentiment(call)
    positive.append(x1)
    negative.append(y1)
    positive_lm.append(x2)
    negative_lm.append(y2)
    wordcount.append(z)

calls['pos'] = positive
calls['neg'] = negative
calls['pos_lm'] = positive_lm
calls['neg_lm'] = negative_lm
calls['wordcount'] = wordcount

positive = []
negative = []
positive_lm = []
negative_lm = []
wordcount = []
for call in calls['answer_us_text']:
    x1,y1,x2,y2,z = sentiment(call)
    positive.append(x1)
    negative.append(y1)
    positive_lm.append(x2)
    negative_lm.append(y2)
    wordcount.append(z)

calls['pos_us'] = positive
calls['neg_us'] = negative
calls['pos_lm_us'] = positive_lm
calls['neg_lm_us'] = negative_lm
calls['wordcount_us'] = wordcount


calls1 = calls[['dates', 'pos', 'neg', 'wordcount', 'pos_us', 'neg_us', 'pos_lm_us', 'neg_lm_us', 'wordcount_us', 'QA_all', 'QA_us']]
calls1.to_csv("sentiment_call0712.csv")
