Gather Transcript Components¶
Extract information from the presentation and the QA section of transcripts. Organize into a dataframe at the component level.
component_long: transcript-component level datacomponent_wide: transcript level data
In [1]:
import pandas as pd
import numpy as np
import os, openpyxl, re, nltk, multiprocessing
import dask.dataframe as dd
In [2]:
%matplotlib inline
%load_ext autoreload
%autoreload 2
In [3]:
pd.set_option('display.memory_usage', 'deep')
pd.set_option('display.precision', 2)
pd.set_option('display.width', 200)
pd.set_option('display.max_rows', 300)
pd.set_option('display.max_columns', None)
pd.set_option('display.max_colwidth', 80)
pd.options.display.float_format = '{:,.4f}'.format
In [4]:
from filepath_03 import * # change to reflect input and output file path
Extract Information from Transcripts
In [5]:
transinfo = pd.read_pickle(filepath_trans)[['transcriptid','filepath','docname','cp_list','op_list']]
In [6]:
PARinfo = pd.read_pickle(filepath_PAR1)[['transcriptid','filepath','par_name','par_firmtitle','par_role','par_seq']]
Extract the Presentation and Q&A Section Separately
In [7]:
def import_text(txtfilepath):
with open(txtfilepath, "r", encoding="utf-8") as f:
doc = f.read()
return doc
In [8]:
def analyze_sections(row):
doc = import_text(row['filepath']).lower()
MD, QA = None, None
if re.search(r"===\npresentation\n\-+(.*?)(?:\n\n={6,}|\n{4,}\-*\ndefinitions\n)", doc, flags=re.DOTALL):
MD = re.search(r"===\npresentation\n\-+(.*?)(?:\n\n={6,}|\n{4,}\-*\ndefinitions\n)", doc, flags=re.DOTALL).group(1)
if re.search(r"===\nquestions and answers\n\-+(.*?)(?:\n\n={6,}|\n{4,}\-*\ndefinitions\n)", doc, flags=re.DOTALL):
QA = re.search(r"===\nquestions and answers\n\-+(.*?)(?:\n\n={6,}|\n{4,}\-*\ndefinitions\n)", doc, flags=re.DOTALL).group(1)
speakerlist_MD, textlist_MD = [], []
speakerlist_QA, textlist_QA = [], []
if not MD is None:
speakerlist_MD = re.findall(r"\n[^\n]*?\[\d+\](?:\n-{10,})", MD)
MD2 = re.sub(r"\n[^\n]*?(\[\d+\])\n-",r"####################\1\n-", MD)+"####################"
textlist_MD = re.findall(r"##########(\[\d+\].*?)##########", MD2, flags = re.DOTALL)
if not QA is None:
speakerlist_QA = re.findall(r"\n[^\n]*?\[\d+\](?:\n-{10,})", QA)
QA2 = re.sub(r"\n[^\n]*?(\[\d+\])\n-",r"####################\1\n-", QA)+"####################"
textlist_QA = re.findall(r"##########(\[\d+\].*?)##########", QA2, flags = re.DOTALL)
return row['transcriptid'], len(speakerlist_MD), len(textlist_MD), speakerlist_MD, textlist_MD, len(speakerlist_QA), len(textlist_QA), speakerlist_QA, textlist_QA
In [10]:
# transinfo = transinfo.sample(n=1000, random_state=1)
In [11]:
ddf = dd.from_pandas(transinfo, npartitions=(multiprocessing.cpu_count()-1)*1)
meta_df = pd.DataFrame(columns=[0,1,2,3,4,5,6,7,8], dtype=str)
result = ddf.apply(lambda x: analyze_sections(x), axis=1, result_type='expand', meta=meta_df).compute(scheduler="multiprocessing")
result.columns=['transcriptid',
'speakerlen_MD','textlen_MD','speakerlist_MD','textlist_MD',
'speakerlen_QA','textlen_QA','speakerlist_QA','textlist_QA']
In [13]:
component_wide = result.merge(right=transinfo[['transcriptid','filepath']], on=['transcriptid'], how='inner')
Expand into long form with question and answer in sequence
In [14]:
def expand_MD(df_in):
df_out = pd.DataFrame()
for index, row in df_in.iterrows():
transcriptid = row['transcriptid']
dict_names = {}
dict_text = {}
try:
for item in row['speakerlist_MD']:
nametitle = re.search(r"\n(.*)\[(\d+)\]", item).group(1).strip()
seq1 = re.search(r"\n(.*)\[(\d+)\]", item).group(2)
dict_names[seq1] = nametitle
for item in row['textlist_MD']:
seq2 = re.search(r"\[(\d+)\]", item).group(1)
text = re.search(r"\[\d+\]\n\-+\n(.*)", item, flags = re.DOTALL).group(1).strip() if re.search(r"\[\d+\]\n\-+\n(.*)", item, flags = re.DOTALL) else ""
dict_text[seq2] = text
except:
display(index)
continue
if not len(dict_names) == len(dict_text):
display("LENGTH DOES NOT MATCH")
display(index)
s1 = pd.Series(dict_names, name="nametitle")
s2 = pd.Series(dict_text, name = "text")
tempdf = pd.concat([s1, s2], axis=1).reset_index().rename(columns={'index':'seq'})
tempdf['transcriptid'] = transcriptid
tempdf['section'] = "MD"
df_out = pd.concat([df_out, tempdf], axis=0, ignore_index=True)
return df_out
In [15]:
MD_long= expand_MD(component_wide.query('speakerlen_MD>0'))
In [16]:
def expand_QA(df_in):
df_out = pd.DataFrame()
for index, row in df_in.iterrows():
transcriptid = row['transcriptid']
dict_names = {}
dict_text = {}
try:
for item in row['speakerlist_QA']:
nametitle = re.search(r"\n(.*)\[(\d+)\]", item).group(1).strip()
seq1 = re.search(r"\n(.*)\[(\d+)\]", item).group(2)
dict_names[seq1] = nametitle
for item in row['textlist_QA']:
seq2 = re.search(r"\[(\d+)\]", item).group(1)
text = re.search(r"\[\d+\]\n\-+\n(.*)", item, flags = re.DOTALL).group(1).strip() if re.search(r"\[\d+\]\n\-+\n(.*)", item, flags = re.DOTALL) else ""
dict_text[seq2] = text
except:
display(index)
continue
if not len(dict_names) == len(dict_text):
display("LENGTH DOES NOT MATCH")
display(index)
s1 = pd.Series(dict_names, name="nametitle")
s2 = pd.Series(dict_text, name = "text")
tempdf = pd.concat([s1, s2], axis=1).reset_index().rename(columns={'index':'seq'})
tempdf['transcriptid'] = transcriptid
tempdf['section'] = "QA"
df_out = pd.concat([df_out, tempdf], axis=0, ignore_index=True)
return df_out
In [17]:
QA_long= expand_QA(component_wide.query('speakerlen_QA>0'))
Save
In [18]:
component_long = pd.concat([MD_long, QA_long], ignore_index=True).sort_values(by=['transcriptid', 'section', 'seq'])
component_long = component_long[['transcriptid','section','seq','nametitle','text']]
component_long = component_long.merge(right=component_wide[['transcriptid','filepath']], how='left', on=['transcriptid'])
In [19]:
component_long['nametitle'] = component_long['nametitle'].astype("string")
component_long['seq'] = component_long['seq'].astype(int)
In [20]:
component_long = component_long.sort_values(by=['transcriptid', 'section', 'seq']).reset_index(drop=True)
In [21]:
component_long.to_pickle(outputpath_long)
component_wide.to_pickle(outputpath_wide)
In [ ]: