import subprocess
import os
import sys

# Set the working directory
os.chdir(r'../../Data/Data_website/Scrapping')

# Define the paths
script_path = r'../../../codes_websiteScraping/step3_scrapping/process_csv.py'
source_folderPE = r'.\scrapeList\PE Firms'
raw_data_folderPE = r'.\raw_data\PE Firms'
result_folderPE = r'.\ESGcounts\PE Firms'
source_folderHF = r'.\scrapeList\HF Firms'
raw_data_folderHF = r'.\raw_data\HF Firms'
result_folderHF = r'.\ESGcounts\HF Firms'

# Function to run the script and capture output
def run_script(script_path, source_folder, result_folder, raw_data_folder):
    result = subprocess.run(
        [sys.executable, script_path, source_folder, result_folder, raw_data_folder],
        capture_output=True, text=True
    )
    print(f"Processing {source_folder} complete.")
    print("Output:\n", result.stdout)
    print("Errors:\n", result.stderr)

# Execute the process_csv.py script with arguments for PE Firms
run_script(script_path, source_folderPE, result_folderPE, raw_data_folderPE)

# Execute the process_csv.py script with arguments for HF Firms
run_script(script_path, source_folderHF, result_folderHF, raw_data_folderHF)

print("Processing complete.")