Clean Transcript Components¶

In [1]:
import pandas as pd
import numpy as np
import os, re 
from fuzzywuzzy import fuzz
In [2]:
%matplotlib inline 
%load_ext autoreload
%autoreload 2
In [3]:
pd.set_option('display.memory_usage', 'deep')
pd.set_option('display.precision', 2) 
pd.set_option('display.width', 200)
pd.set_option('display.max_rows', 300)
pd.set_option('display.max_columns', None)
pd.set_option('display.max_colwidth', 80)
pd.options.display.float_format = '{:,.4f}'.format
In [4]:
from filepath_03 import * # change to reflect input and output file path

1. Identify Participants and Compute Basic Measures¶

In [5]:
transinfo = pd.read_pickle(filepath_trans)[['transcriptid','filepath','docname','cp_list','op_list']]
In [6]:
CP = pd.read_pickle(filepath_CP)
In [7]:
OP = pd.read_pickle(filepath_OP)
In [8]:
PARinfo = (pd.read_pickle(filepath_PAR2)[['transcriptid','filepath','par_name','par_firmtitle','par_seq','par_role']])
In [9]:
PARinfo = PARinfo.merge(right=CP[['transcriptid','par_name','execid','gender','CEO','CFO']], 
                        on=['transcriptid', 'par_name'], how='left')
PARinfo['efemale'] = np.where(PARinfo['gender']=='FEMALE', 1, np.nan)
PARinfo['efemale'] = np.where(PARinfo['gender']=='MALE', 0, PARinfo['efemale'])
PARinfo = PARinfo.drop(columns=['gender'])
In [10]:
PARinfo = PARinfo.merge(right=OP[['transcriptid','par_name','analys','afemale','is_coveringanalyst']], 
                        on=['transcriptid', 'par_name'], how='left')
In [11]:
DZpar = pd.read_csv(filepath_DZpar) # identify analysts in the regression sample
DZpar['transcriptid'] = DZpar['transcriptid'].astype(str) 
PARinfo = PARinfo.merge(right=DZpar[['transcriptid','par_name']], on=['transcriptid', 'par_name'], how='left', indicator=True)
PARinfo['in_DZ'] = (PARinfo['_merge']=="both")*1
PARinfo = PARinfo.drop(columns=['_merge'])
In [12]:
PARinfo['par_name_str'] = PARinfo['par_name'].astype("string")

Load Components

In [13]:
comp_long = pd.read_pickle(filepath_long)
comp_long['seq'] = comp_long['seq'].astype(int) 
comp_long = comp_long.sort_values(by=['transcriptid','section','seq'])

Compute basic textual measures

In [14]:
comp_long['nwords'] = comp_long['text'].str.split().str.len()
In [15]:
comp_long['nwords'] = comp_long['nwords'].astype(int, errors='ignore')
In [16]:
comp_long = comp_long.query('nwords>0 & text.notnull()')

Clean speak names in the component text and merge with participant identifiers

In [17]:
def one_way_freq(df, col, sort=True):
    return (pd.concat([df[col].value_counts(sort=sort).rename("count"), df[col].value_counts(normalize=True, sort=sort).rename("percent")], axis=1))
In [18]:
df_nametitle = one_way_freq(comp_long, 'nametitle').reset_index()
df_nametitle[['name','title']]=df_nametitle['nametitle'].str.split(",", n=1, expand=True)
In [19]:
comp_long = comp_long.merge(right=df_nametitle[['nametitle','name','title']], how='inner', on=['nametitle'])

Exaact and fuzzy match on names

In [20]:
matchcols = ['par_name_str','par_role','transcriptid','analys', 'afemale','is_coveringanalyst',
             'execid','CEO','CFO','efemale','in_DZ']

exact match

In [21]:
match1 = comp_long.query('name.notnull()')[['transcriptid','name']].drop_duplicates()
match1 = match1.merge(right=PARinfo[matchcols], 
                      left_on=['transcriptid','name'], right_on=['transcriptid','par_name_str'], how='inner')
match1 = match1.sort_values(by=['transcriptid','name'])
match1 = match1.query('name.notnull() & par_name_str.notnull()')

fuzzy match

In [22]:
lst_names = match1['name'].unique().tolist()
match2 = comp_long.query('name.notnull() & ~name.isin(@lst_names)')[['transcriptid','name']].drop_duplicates()
lst_parnames = match1['par_name_str'].unique().tolist()
temp = PARinfo.query('~par_name_str.isin(@lst_parnames)').copy()
In [23]:
match2 = match2.merge(right=PARinfo[matchcols], 
                      left_on=['transcriptid'], right_on=['transcriptid'], how='inner')
match2 = match2.sort_values(by=['transcriptid','name','par_name_str'])
match2 = match2.query('name.notnull() & par_name_str.notnull()')
In [24]:
match2['score1'] = match2.apply(lambda row: fuzz.partial_ratio(row['name'], row['par_name_str']), axis=1)
In [25]:
match2 = match2.sort_values(by=['score1'], ascending=[False])
match2 = match2.query('score1>=88') #upon manual inspection, set threshold to be 88
match2 = match2.sort_values(by=['transcriptid','name','score1'], ascending=[True, True, False]).drop_duplicates(subset=['transcriptid','name'])

Collect results and manual inspection (very little deletions, <=10)

In [26]:
cols=['transcriptid','name','par_name_str','par_role','analys', 'afemale','is_coveringanalyst',
      'execid','CEO','CFO','efemale','in_DZ']
matched = pd.concat([match1, match2], ignore_index=True)[cols]
In [27]:
# load back manually inspected data
matched = pd.read_pickle("../data_temp/matched.pkl")

2. Clean Component Text¶

Upon manual inspection, the classification of corporate participants and other participants still contains noise. Use information in the component text to clean classification.

In [28]:
comp_long2 = (comp_long[['transcriptid','section','seq','nametitle','name','text','nwords']]
              .merge(right=matched, on=['transcriptid','name'], how='left'))
comp_long2 = comp_long2.sort_values(by=['transcriptid','section','seq'])

Clean par_role

In [29]:
# indicate operator roles and unidentified audience  
def indicate_roles(nametitle):
    role = 'IDENTIFIED'
    if re.findall(r"thomson reuters media|thomson reuters journalist|operator", nametitle):
        role = "OPERATOR"
    if re.findall(r"unidentified", nametitle):
        role = "UNIDENTIFIED"
    if re.findall(r"unidentified", nametitle) and re.findall(r"company representative", nametitle):
        role = "COMP_REP"
    return role
comp_long2['role_special'] = comp_long2['nametitle'].apply(indicate_roles)
In [30]:
comp_long2['par_role'] = np.where((comp_long2['role_special']=="UNIDENTIFIED") & 
                                  (comp_long2['par_role']=="cp"), "others", comp_long2['par_role'])
In [31]:
comp_long2['par_role'] = np.where((comp_long2['role_special']=="OPERATOR") & 
                                  (comp_long2['par_role']=="cp"), "others", comp_long2['par_role'])
In [32]:
comp_long2['cp'] =(comp_long2['par_role']=="cp")*1 
comp_long2['op'] =(comp_long2['par_role']=="op")*1 
comp_long2['n_cp'] = comp_long2.groupby(['transcriptid'])['cp'].transform('sum')
comp_long2['n_op'] = comp_long2.groupby(['transcriptid'])['op'].transform('sum')
comp_long2['qa'] = np.where((comp_long2['par_role']!="cp")
                          & (comp_long2.groupby(['transcriptid','section'])['par_role'].shift(-1) =='cp'), 1, 0)
comp_long2['n_qa'] = comp_long2.groupby(['transcriptid','section'])['qa'].transform('sum')
In [33]:
lst1 = comp_long2.query('n_cp==0')['transcriptid'].unique().tolist()
lst2 = comp_long2.query('section=="QA"&n_qa==0&n_op==0')['transcriptid'].unique().tolist()
lst = list(set(lst1+lst2))
temp = comp_long2.query('transcriptid.isin(@lst)')[['transcriptid','section','seq','nametitle','par_role']]
In [34]:
def classify_role(nametitle):
    new_role = None
    if re.findall(r"\bceo\b|\bcfo\b|\bfounder\b|\bchairman\b|\bpresident\b", nametitle, flags = re.I):
        new_role = 'cp'
    elif re.findall(r'\banalyst\b', nametitle, flags = re.I):
        new_role = "op"
    return new_role
temp['new_role'] = temp['nametitle'].apply(classify_role)
In [35]:
comp_long2 = comp_long2.merge(right=temp[['transcriptid','section','seq','nametitle', 'new_role']], 
                              on=['transcriptid','section','seq','nametitle'], how='left')
In [36]:
comp_long2['par_role'] = np.where((comp_long2['new_role'].notnull()) & (comp_long2['par_role']!=comp_long2['new_role']), 
                                  comp_long2['new_role'], comp_long2['par_role'])
In [37]:
comp_long2 = comp_long2.drop(columns=['role_special', 'new_role',
                                      'cp','op','qa',
                                      'n_cp','n_op','n_qa'])

Upon manual inspection, management presentation and Q&A sessions could be wrongly labeled. Treat a session as a potential Q&A if 1) other participants speak for at least 3 times, 2) there are at least 3 exchanges between a corporate participant and another participant. Unidentified audience is not defined as "other participant" since there is no way to know for sure.

In [38]:
comp_long3 = comp_long2.copy()
comp_long3['op'] =(comp_long3['par_role']=="op")*1 
comp_long3['qa'] = np.where((comp_long3['par_role']!="cp")
                          & (comp_long3.groupby(['transcriptid','section'])['par_role'].shift(-1) =='cp'), 1, 0)
comp_long3['num_op'] = comp_long3.groupby(['transcriptid','section'])['op'].transform('sum')
comp_long3['num_qa'] = comp_long3.groupby(['transcriptid','section'])['qa'].transform('sum')
In [39]:
comp_long3['deemedQA'] = np.where((comp_long3['num_qa']>=3) & (comp_long3['num_op']>=3), 1, 0)

Save¶

In [40]:
comp_long3.to_pickle(outputpath_long2)
In [ ]:
In [ ]: