import csv
import concurrent.futures
from langdetect import detect as lang_detect
import requests
from bs4 import BeautifulSoup
import pandas as pd
import time
import pycld2 as cld2
import os

# Proxy IP rotation
def proxy_webshare():
    # proxy settings
    proxy = {
        "http": "xxx",
        "https": "xxx" #provide appropriate proxied if you have them
    }
    return proxy


def detect_language_nlp(text_content):
    # detected_language = cld2.detect(text_content, returnVectors=True)
    isReliable, textBytesFound, details, vectors = cld2.detect(text_content, returnVectors=True)
    # print(isReliable)
    # print(textBytesFound)
    # print(details)
    # print(vectors)
    return details

def check_language(website):
    # hardcoded headers for random header selection for each request
    headers = {
        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/89.0.4389.82 Safari/537.36'
    }

    website_dict = {
        'website available': False,
        'url': website,
        'len': None,
        'pycld2':None,
        'pycld2_confidence': None,
        'language_frm_source': None,
        'langdetect': None
    }

    # Formating Website
    # website = 'http://' + website.replace('https://', '').replace('http://', '').replace('www.', '')
    website = 'http://' + website.replace('https://', '').replace('http://', '').replace('www.', '')
    if website[-1] == '/' or website[-1] == '#':
        website = website[:-1]

    # Assigning formated URL
    website_dict['url'] = website
    print(f'Processing: {website}')

    try:
        response = requests.get(website, headers=headers, timeout=10)
        content = response.content
        soup = BeautifulSoup(content, 'lxml')

        if soup:
            text = soup.get_text()
            website_dict['len'] = len(text)
            # print(website_dict['len'])

            # Extract language from the <html> tag
            html_tag = soup.select_one('html[lang]')
            # print(html_tag)
            lang_from_source = html_tag.get('lang', None) if html_tag else None

            # print(lang_from_source)
            website_dict['language_frm_source'] = lang_from_source

            # Use langdetect package to detect language
            lang_detected = lang_detect(text)
            website_dict['langdetect'] = lang_detected

            lang = detect_language_nlp(text)
            # print(lang)
            if len(lang) > 0:
                website_dict['pycld2'] = lang[0][1]
                website_dict['pycld2_confidence'] = lang[0][2]

            website_dict['website available'] = True
            return website_dict
    except:
        website_dict['website available'] = False

    return website_dict

workingPath = r'.\Data\Data_website\Scrapping'
inputPath = workingPath + '\Inputs to the scraping process'
outputPath = workingPath + '\englishCheck'

if __name__ == '__main__':

    files = ['websites_EnglishIdentification']

    for file in files:
        # get the start time
        st = time.time()

        websites = []
        with open(os.path.join(inputPath,f'{file}.txt')) as f:
            websites = f.read().splitlines()

        # results = []
        # for website in websites:
        #     result = check_language(website)
        #     results.append(result)

        with concurrent.futures.ThreadPoolExecutor() as executor: #max_workers=10
            results = executor.map(check_language, websites)

        df = pd.DataFrame(results)

        df.to_csv(os.path.join(outputPath,f'{file}.csv'), index=False)

        print(f"Done - File: {file}")

        # get the end time
        et = time.time()

        # get the execution time
        elapsed_time = et - st
        print('Execution time:', elapsed_time, 'seconds')
