# Gather Transcript Components¶

Extract information from the presentation and the QA section of transcripts. Organize into a dataframe at the component level.

- `component_long`: transcript-component level data
- `component_wide`: transcript level data

In [1]:

    import pandas as pd
    import numpy as np
    import os, openpyxl, re, nltk, multiprocessing
    import dask.dataframe as dd

In [2]:

    %matplotlib inline
    %load_ext autoreload
    %autoreload 2

In [3]:

    pd.set_option('display.memory_usage', 'deep')
    pd.set_option('display.precision', 2)
    pd.set_option('display.width', 200)
    pd.set_option('display.max_rows', 300)
    pd.set_option('display.max_columns', None)
    pd.set_option('display.max_colwidth', 80)
    pd.options.display.float_format = '{:,.4f}'.format

In [4]:

    from filepath_03 import * # change to reflect input and output file path

Extract Information from Transcripts

In [5]:

    transinfo = pd.read_pickle(filepath_trans)[['transcriptid','filepath','docname','cp_list','op_list']]

In [6]:

    PARinfo = pd.read_pickle(filepath_PAR1)[['transcriptid','filepath','par_name','par_firmtitle','par_role','par_seq']]

Extract the Presentation and Q&A Section Separately

In [7]:

    def import_text(txtfilepath):
        with open(txtfilepath, "r", encoding="utf-8") as f:
            doc = f.read()
        return doc

In [8]:

    def analyze_sections(row):
        doc = import_text(row['filepath']).lower()

        MD, QA = None, None
        if re.search(r"===\npresentation\n\-+(.*?)(?:\n\n={6,}|\n{4,}\-*\ndefinitions\n)", doc, flags=re.DOTALL):
            MD = re.search(r"===\npresentation\n\-+(.*?)(?:\n\n={6,}|\n{4,}\-*\ndefinitions\n)", doc, flags=re.DOTALL).group(1)
        if re.search(r"===\nquestions and answers\n\-+(.*?)(?:\n\n={6,}|\n{4,}\-*\ndefinitions\n)", doc, flags=re.DOTALL):
            QA = re.search(r"===\nquestions and answers\n\-+(.*?)(?:\n\n={6,}|\n{4,}\-*\ndefinitions\n)", doc, flags=re.DOTALL).group(1)

        speakerlist_MD, textlist_MD = [], []
        speakerlist_QA, textlist_QA = [], []
        if not MD is None:
            speakerlist_MD = re.findall(r"\n[^\n]*?\[\d+\](?:\n-{10,})", MD)
            MD2 = re.sub(r"\n[^\n]*?(\[\d+\])\n-",r"####################\1\n-", MD)+"####################"
            textlist_MD = re.findall(r"##########(\[\d+\].*?)##########", MD2, flags = re.DOTALL)
        if not QA is None:
            speakerlist_QA = re.findall(r"\n[^\n]*?\[\d+\](?:\n-{10,})", QA)
            QA2 = re.sub(r"\n[^\n]*?(\[\d+\])\n-",r"####################\1\n-", QA)+"####################"
            textlist_QA = re.findall(r"##########(\[\d+\].*?)##########", QA2, flags = re.DOTALL)

        return row['transcriptid'], len(speakerlist_MD), len(textlist_MD), speakerlist_MD, textlist_MD, len(speakerlist_QA), len(textlist_QA), speakerlist_QA, textlist_QA

In [10]:

    # transinfo = transinfo.sample(n=1000, random_state=1)

In [11]:

    ddf = dd.from_pandas(transinfo, npartitions=(multiprocessing.cpu_count()-1)*1)
    meta_df = pd.DataFrame(columns=[0,1,2,3,4,5,6,7,8], dtype=str)
    result = ddf.apply(lambda x: analyze_sections(x), axis=1, result_type='expand', meta=meta_df).compute(scheduler="multiprocessing")
    result.columns=['transcriptid',
                    'speakerlen_MD','textlen_MD','speakerlist_MD','textlist_MD',
                    'speakerlen_QA','textlen_QA','speakerlist_QA','textlist_QA']

In [13]:

    component_wide = result.merge(right=transinfo[['transcriptid','filepath']], on=['transcriptid'], how='inner')

Expand into long form with question and answer in sequence

In [14]:

    def expand_MD(df_in):
        df_out = pd.DataFrame()

        for index, row in df_in.iterrows():
            transcriptid = row['transcriptid']

            dict_names = {}
            dict_text = {}
            try:
                for item in row['speakerlist_MD']:
                    nametitle = re.search(r"\n(.*)\[(\d+)\]", item).group(1).strip()
                    seq1 = re.search(r"\n(.*)\[(\d+)\]", item).group(2)
                    dict_names[seq1] = nametitle

                for item in row['textlist_MD']:
                    seq2 = re.search(r"\[(\d+)\]", item).group(1)
                    text = re.search(r"\[\d+\]\n\-+\n(.*)", item, flags = re.DOTALL).group(1).strip() if re.search(r"\[\d+\]\n\-+\n(.*)", item, flags = re.DOTALL) else ""
                    dict_text[seq2] = text
            except:
                display(index)
                continue

            if not len(dict_names) == len(dict_text):
                display("LENGTH DOES NOT MATCH")
                display(index)

            s1 = pd.Series(dict_names, name="nametitle")
            s2 = pd.Series(dict_text, name = "text")
            tempdf = pd.concat([s1, s2], axis=1).reset_index().rename(columns={'index':'seq'})
            tempdf['transcriptid'] = transcriptid
            tempdf['section'] = "MD"

            df_out = pd.concat([df_out, tempdf], axis=0, ignore_index=True)
        return df_out

In [15]:

    MD_long= expand_MD(component_wide.query('speakerlen_MD>0'))

In [16]:

    def expand_QA(df_in):
        df_out = pd.DataFrame()

        for index, row in df_in.iterrows():
            transcriptid = row['transcriptid']

            dict_names = {}
            dict_text = {}
            try:
                for item in row['speakerlist_QA']:
                    nametitle = re.search(r"\n(.*)\[(\d+)\]", item).group(1).strip()
                    seq1 = re.search(r"\n(.*)\[(\d+)\]", item).group(2)
                    dict_names[seq1] = nametitle

                for item in row['textlist_QA']:
                    seq2 = re.search(r"\[(\d+)\]", item).group(1)
                    text = re.search(r"\[\d+\]\n\-+\n(.*)", item, flags = re.DOTALL).group(1).strip() if re.search(r"\[\d+\]\n\-+\n(.*)", item, flags = re.DOTALL) else ""

                    dict_text[seq2] = text
            except:
                display(index)
                continue

            if not len(dict_names) == len(dict_text):
                display("LENGTH DOES NOT MATCH")
                display(index)

            s1 = pd.Series(dict_names, name="nametitle")
            s2 = pd.Series(dict_text, name = "text")
            tempdf = pd.concat([s1, s2], axis=1).reset_index().rename(columns={'index':'seq'})
            tempdf['transcriptid'] = transcriptid
            tempdf['section'] = "QA"

            df_out = pd.concat([df_out, tempdf], axis=0, ignore_index=True)
        return df_out

In [17]:

    QA_long= expand_QA(component_wide.query('speakerlen_QA>0'))

Save

In [18]:

    component_long = pd.concat([MD_long, QA_long], ignore_index=True).sort_values(by=['transcriptid', 'section', 'seq'])
    component_long = component_long[['transcriptid','section','seq','nametitle','text']]
    component_long = component_long.merge(right=component_wide[['transcriptid','filepath']], how='left', on=['transcriptid'])

In [19]:

    component_long['nametitle'] = component_long['nametitle'].astype("string")
    component_long['seq'] = component_long['seq'].astype(int)

In [20]:

    component_long = component_long.sort_values(by=['transcriptid', 'section', 'seq']).reset_index(drop=True)

In [21]:

    component_long.to_pickle(outputpath_long)
    component_wide.to_pickle(outputpath_wide)

In [ ]: