from requests import get
import pandas as pd
import os
from bs4 import BeautifulSoup
import html
import re
import lxml.html

os.chdir(r'.\Data\Data_IPO')

hdr = {'User-Agent': 'LondonBusinessSchool jabraham@london.edu',
         'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
         'Referer': 'https://cssspritegenerator.com',
         'Accept-Charset': 'ISO-8859-1,utf-8;q=0.7,*;q=0.3',
         'Accept-Encoding': 'none',
         'Accept-Language': 'en-US,en;q=0.8',
         'Connection': 'keep-alive'}

def getWordCount(text):
    text = html.unescape(text)
    text = text.replace('\r', '')
    text = text.strip()

    # Split text into paragraphs and headings
    paragraphs = []
    lines = text.split('\n')
    paragraph = ''
    for line in lines:
        if line.strip() == '':
            if paragraph != '':
                paragraphs.append(paragraph)
                paragraph = ''
        elif line.isupper():
            if paragraph != '':
                paragraphs.append(paragraph)
                paragraph = ''
            paragraph += line + '\n'
        else:
            if paragraph != '' and not paragraph.endswith('\n'):
                paragraph += ' '
            paragraph += line

    # Add the last paragraph
    if paragraph != '':
        paragraphs.append(paragraph.strip())

    # Join the paragraphs with two line breaks and headings with three line breaks
    text = '\n'.join(paragraphs)
    # text = re.sub(r'\s', ' ', text).strip()

    text = text.encode('ascii', 'ignore').decode('utf-8')
    word_count = len(re.findall(r'\w+', text))
    return word_count

data8K = pd.read_stata('./raw_data/voluntary 8k/8KitemsforPy.dta')
data8K['wordCount'] = 0

for i in range(len(data8K)):
    print(i)
    curr_request = get(data8K.iloc[i]['index_url'], headers=hdr)
    text = curr_request.text
    curr_soup = BeautifulSoup(text, 'lxml')
    
    data = []
    table = curr_soup.find('table', class_='tableFile')
    for row in table.find_all('tr'):    
        columns = row.find_all('td')
        if (columns!=[]):
            url = columns[2].find('a').get('href')
            file_type = columns[3].text.strip()
            data.append([url,file_type])
    data = pd.DataFrame(data, columns=['url','filetype'])
    data = data[data['filetype'].str.lower().str.contains("8-k")|data['filetype'].str.lower().str.contains("8k")|data['filetype'].str.contains("EX")]
    data['url'] = 'https://www.sec.gov/' + data['url']
    
    wordCount = 0
    for j in range(len(data)):
        sub_request = get(data.iloc[j]['url'], headers=hdr)
        sub_text = sub_request.text
        sub_soup = BeautifulSoup(sub_text, 'lxml')
        sub_soup_text = sub_soup.body.get_text()
        wordCount = wordCount + getWordCount(sub_soup_text)

    data8K.loc[i,'wordCount']= wordCount

data8K.to_stata('./8KWordCount.dta', write_index=False)