"""
-----------------------------------
# Instructions for Code Execution
-----------------------------------

## Overview
This script is designed to clean and process textual data from firm's web-pages.

## Folder Structure
- The script is located in the 'Code' folder.
- The 'Data' folder contains subfolders: 'Firmfiles' with the original `.txt` files and 'Firmfiles_cleaned' for storing processed files.
- The 'ESG_Dictionary.csv' file is also located within the 'Data' folder.

## Prerequisites
- Ensure that all `.txt` files to be processed are located in the 'Firmfiles' folder.
- The 'ESG_Dictionary.csv' file, used for identifying relevant ESG content is also present.

## Processing Steps
1. **Gibberish Removal**: The script first removes any gibberish or non-standard content from the text files.
2. **Duplicate Line Removal**: Subsequently, it eliminates duplicate lines to ensure uniqueness of content.
3. **ESG Content Extraction**: The script then extracts and retains only paragraphs containing ESG-related terms based on the provided ESG dictionary.
4. **Blank File Deletion**: Finally, any files that are left blank after the cleaning process will be automatically deleted.

## Output
- After applying the processing steps, the cleaned files are saved into the `"Firmfiles_cleaned"` folder.

"""
import os
import re
import csv
import io
import glob
#------------ Set up logging ------------ 
import log_config
log_config.setup_logging()
import logging
logging.info("Running script 'ESG File Cleaning Blocks.py'")

#------------ Folder locations here ------------ 
# Source directory (User-specified)
source_dir = '../Data/Firmfiles'
# Destination directory (User-specified)
destination_dir = '../Data/Firmfiles_cleaned'
# ESG Dictionary CSV (User-specified)
esg_dic = '../Data/ESG_Dictionary.csv'

# Ensure the destination directory exists
if not os.path.exists(destination_dir):
    os.makedirs(destination_dir)

#------------ List of functions to clean gibberish content, remove duplicate lines, extract ESG relevant paragraphs, and delete blank files ------------

# Function to clean gibberish content
def clean_gibberish(file_path, output_path):
    with open(file_path, 'r') as file:
        content = file.read()

    pattern = r'(\s*/[A-Za-z0-9+/]+={0,2}|\s\s+|\r)'
    cleaned_content = re.sub(pattern, '', content)

    with open(output_path, 'w') as file:
        file.write(cleaned_content)

# Function to remove duplicate lines
def remove_duplicates(file_path, output_path):
    unique_lines = set()
    ordered_lines = []

    with open(file_path, 'r') as file:
        for line in file:
            line = line.strip()
            if line not in unique_lines:
                unique_lines.add(line)
                ordered_lines.append(line)

    with open(output_path, 'w') as file:
        for line in ordered_lines:
            file.write(line + '\n')

# Function to extract relevant ESG paragraphs
def extract_esg_content(output_path, dictionary):
    with io.open(output_path, 'r', encoding='utf8') as file:
        text = file.read()

    paragraphs = text.split('\n')
    relevant_paragraphs = []
    for p in paragraphs:
        p_processed = re.sub(r'[^\w\s]', '', p.lower())
        if any(word in p_processed.split() for word in dictionary):
            relevant_paragraphs.append(p)

    with io.open(output_path, 'w', encoding='utf8') as file:
        file.write('\n'.join(relevant_paragraphs))

# Function to delete empty files
def delete_empty_files(directory):
    os.chdir(directory)
    for file_name in glob.glob('*.txt'):
        if os.path.getsize(file_name) == 0:
            print(f"Deleting {file_name}")
            os.remove(file_name)

#------------ Execution of main program ------------ 

# Load ESG dictionary
dictionary = set()
with open(esg_dic, 'r') as f:
    reader = csv.reader(f)
    for row in reader:
        dictionary.add(row[0].lower())

# Process each file (clean gibberish, remove duplicates and extract esg content)
for filename in os.listdir(source_dir):
    if filename.endswith(".txt"):
        file_path = os.path.join(source_dir, filename)
        output_path = os.path.join(destination_dir, filename)  # Define output path in the destination directory

        clean_gibberish(file_path, output_path)
        remove_duplicates(output_path, output_path)  # Input and output paths are the same
        extract_esg_content(output_path, dictionary)  # Continue processing on updated file

# Delete empty files in the destination directory
delete_empty_files(destination_dir)

print(f"Execution of cleaning of files is completed")

#------------ Log completion of the script ------------
logging.info("ESG File Cleaning Block.py'")