In [1]:

    import subprocess
    import os
    import sys

    # Set the working directory
    os.chdir(r'../../Data/Data_website/Scrapping')

    # Define the paths
    script_path = r'../../../codes_websiteScraping/step3_scrapping/process_csv.py'
    source_folderPE = r'.\scrapeList\PE Firms'
    raw_data_folderPE = r'.\raw_data\PE Firms'
    result_folderPE = r'.\ESGcounts\PE Firms'
    source_folderHF = r'.\scrapeList\HF Firms'
    raw_data_folderHF = r'.\raw_data\HF Firms'
    result_folderHF = r'.\ESGcounts\HF Firms'

    # Function to run the script and capture output
    def run_script(script_path, source_folder, result_folder, raw_data_folder):
        result = subprocess.run(
            [sys.executable, script_path, source_folder, result_folder, raw_data_folder],
            capture_output=True, text=True
        )
        print(f"Processing {source_folder} complete.")
        print("Output:\n", result.stdout)
        print("Errors:\n", result.stderr)

    # Execute the process_csv.py script with arguments for PE Firms
    run_script(script_path, source_folderPE, result_folderPE, raw_data_folderPE)

    # Execute the process_csv.py script with arguments for HF Firms
    run_script(script_path, source_folderHF, result_folderHF, raw_data_folderHF)

    print("Processing complete.")

    Processing .\scrapeList\PE Firms complete.
    Output:

    Errors:
     C:\ProgramData\anaconda3\lib\site-packages\bs4\builder\__init__.py:545: XMLParsedAsHTMLWarning: It looks like you're parsing an XML document using an HTML parser. If this really is an HTML document (maybe it's XHTML?), you can ignore or filter this warning. If it's XML, you should know that using an XML parser will be more reliable. To parse this document as XML, make sure you have the lxml package installed, and pass the keyword argument `features="xml"` into the BeautifulSoup constructor.
      warnings.warn(

    Processing .\scrapeList\HF Firms complete.
    Output:

    Errors:

    Processing complete.

In [ ]: