from requests import get
import pandas as pd
import os

os.chdir(r'.\Data\Data_IPO')

hdr = {'User-Agent': 'LondonBusinessSchool jabraham@london.edu',
         'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
         'Referer': 'https://cssspritegenerator.com',
         'Accept-Charset': 'ISO-8859-1,utf-8;q=0.7,*;q=0.3',
         'Accept-Encoding': 'none',
         'Accept-Language': 'en-US,en;q=0.8',
         'Connection': 'keep-alive'}

data10K = pd.read_stata('./raw_data/10k gross size/10KGrossSizeforPy.dta')
data10K['txtFileSize'] = 0

for i in range(len(data10K)):
    print(i)
    url = data10K.iloc[i]['url']
    try:
        response = get(url, headers=hdr)
        response.raise_for_status()  # Raise HTTPError for bad responses
        data10K.loc[i,'txtFileSize']= len(response.content)
    except Exception as e:
        print(f"Error fetching {url}: {e}")

data10K.to_stata('./10KGrossSize.dta', write_index=False)