# Clean Transcript Components¶

In [1]:

    import pandas as pd
    import numpy as np
    import os, re
    from fuzzywuzzy import fuzz

In [2]:

    %matplotlib inline
    %load_ext autoreload
    %autoreload 2

In [3]:

    pd.set_option('display.memory_usage', 'deep')
    pd.set_option('display.precision', 2)
    pd.set_option('display.width', 200)
    pd.set_option('display.max_rows', 300)
    pd.set_option('display.max_columns', None)
    pd.set_option('display.max_colwidth', 80)
    pd.options.display.float_format = '{:,.4f}'.format

In [4]:

    from filepath_03 import * # change to reflect input and output file path

## 1. Identify Participants and Compute Basic Measures¶

In [5]:

    transinfo = pd.read_pickle(filepath_trans)[['transcriptid','filepath','docname','cp_list','op_list']]

In [6]:

    CP = pd.read_pickle(filepath_CP)

In [7]:

    OP = pd.read_pickle(filepath_OP)

In [8]:

    PARinfo = (pd.read_pickle(filepath_PAR2)[['transcriptid','filepath','par_name','par_firmtitle','par_seq','par_role']])

In [9]:

    PARinfo = PARinfo.merge(right=CP[['transcriptid','par_name','execid','gender','CEO','CFO']],
                            on=['transcriptid', 'par_name'], how='left')
    PARinfo['efemale'] = np.where(PARinfo['gender']=='FEMALE', 1, np.nan)
    PARinfo['efemale'] = np.where(PARinfo['gender']=='MALE', 0, PARinfo['efemale'])
    PARinfo = PARinfo.drop(columns=['gender'])

In [10]:

    PARinfo = PARinfo.merge(right=OP[['transcriptid','par_name','analys','afemale','is_coveringanalyst']],
                            on=['transcriptid', 'par_name'], how='left')

In [11]:

    DZpar = pd.read_csv(filepath_DZpar) # identify analysts in the regression sample
    DZpar['transcriptid'] = DZpar['transcriptid'].astype(str)
    PARinfo = PARinfo.merge(right=DZpar[['transcriptid','par_name']], on=['transcriptid', 'par_name'], how='left', indicator=True)
    PARinfo['in_DZ'] = (PARinfo['_merge']=="both")*1
    PARinfo = PARinfo.drop(columns=['_merge'])

In [12]:

    PARinfo['par_name_str'] = PARinfo['par_name'].astype("string")

Load Components

In [13]:

    comp_long = pd.read_pickle(filepath_long)
    comp_long['seq'] = comp_long['seq'].astype(int)
    comp_long = comp_long.sort_values(by=['transcriptid','section','seq'])

Compute basic textual measures

In [14]:

    comp_long['nwords'] = comp_long['text'].str.split().str.len()

In [15]:

    comp_long['nwords'] = comp_long['nwords'].astype(int, errors='ignore')

In [16]:

    comp_long = comp_long.query('nwords>0 & text.notnull()')

Clean speak names in the component text and merge with participant identifiers

In [17]:

    def one_way_freq(df, col, sort=True):
        return (pd.concat([df[col].value_counts(sort=sort).rename("count"), df[col].value_counts(normalize=True, sort=sort).rename("percent")], axis=1))

In [18]:

    df_nametitle = one_way_freq(comp_long, 'nametitle').reset_index()
    df_nametitle[['name','title']]=df_nametitle['nametitle'].str.split(",", n=1, expand=True)

In [19]:

    comp_long = comp_long.merge(right=df_nametitle[['nametitle','name','title']], how='inner', on=['nametitle'])

Exaact and fuzzy match on names

In [20]:

    matchcols = ['par_name_str','par_role','transcriptid','analys', 'afemale','is_coveringanalyst',
                 'execid','CEO','CFO','efemale','in_DZ']

exact match

In [21]:

    match1 = comp_long.query('name.notnull()')[['transcriptid','name']].drop_duplicates()
    match1 = match1.merge(right=PARinfo[matchcols],
                          left_on=['transcriptid','name'], right_on=['transcriptid','par_name_str'], how='inner')
    match1 = match1.sort_values(by=['transcriptid','name'])
    match1 = match1.query('name.notnull() & par_name_str.notnull()')

fuzzy match

In [22]:

    lst_names = match1['name'].unique().tolist()
    match2 = comp_long.query('name.notnull() & ~name.isin(@lst_names)')[['transcriptid','name']].drop_duplicates()
    lst_parnames = match1['par_name_str'].unique().tolist()
    temp = PARinfo.query('~par_name_str.isin(@lst_parnames)').copy()

In [23]:

    match2 = match2.merge(right=PARinfo[matchcols],
                          left_on=['transcriptid'], right_on=['transcriptid'], how='inner')
    match2 = match2.sort_values(by=['transcriptid','name','par_name_str'])
    match2 = match2.query('name.notnull() & par_name_str.notnull()')

In [24]:

    match2['score1'] = match2.apply(lambda row: fuzz.partial_ratio(row['name'], row['par_name_str']), axis=1)

In [25]:

    match2 = match2.sort_values(by=['score1'], ascending=[False])
    match2 = match2.query('score1>=88') #upon manual inspection, set threshold to be 88
    match2 = match2.sort_values(by=['transcriptid','name','score1'], ascending=[True, True, False]).drop_duplicates(subset=['transcriptid','name'])

Collect results and manual inspection (very little deletions, <=10)

In [26]:

    cols=['transcriptid','name','par_name_str','par_role','analys', 'afemale','is_coveringanalyst',
          'execid','CEO','CFO','efemale','in_DZ']
    matched = pd.concat([match1, match2], ignore_index=True)[cols]

In [27]:

    # load back manually inspected data
    matched = pd.read_pickle("../data_temp/matched.pkl")

## 2. Clean Component Text¶

Upon manual inspection, the classification of corporate participants and other participants still contains noise. Use information in the component text to clean classification.

In [28]:

    comp_long2 = (comp_long[['transcriptid','section','seq','nametitle','name','text','nwords']]
                  .merge(right=matched, on=['transcriptid','name'], how='left'))
    comp_long2 = comp_long2.sort_values(by=['transcriptid','section','seq'])

Clean `par_role`

In [29]:

    # indicate operator roles and unidentified audience
    def indicate_roles(nametitle):
        role = 'IDENTIFIED'
        if re.findall(r"thomson reuters media|thomson reuters journalist|operator", nametitle):
            role = "OPERATOR"
        if re.findall(r"unidentified", nametitle):
            role = "UNIDENTIFIED"
        if re.findall(r"unidentified", nametitle) and re.findall(r"company representative", nametitle):
            role = "COMP_REP"
        return role
    comp_long2['role_special'] = comp_long2['nametitle'].apply(indicate_roles)

In [30]:

    comp_long2['par_role'] = np.where((comp_long2['role_special']=="UNIDENTIFIED") &
                                      (comp_long2['par_role']=="cp"), "others", comp_long2['par_role'])

In [31]:

    comp_long2['par_role'] = np.where((comp_long2['role_special']=="OPERATOR") &
                                      (comp_long2['par_role']=="cp"), "others", comp_long2['par_role'])

In [32]:

    comp_long2['cp'] =(comp_long2['par_role']=="cp")*1
    comp_long2['op'] =(comp_long2['par_role']=="op")*1
    comp_long2['n_cp'] = comp_long2.groupby(['transcriptid'])['cp'].transform('sum')
    comp_long2['n_op'] = comp_long2.groupby(['transcriptid'])['op'].transform('sum')
    comp_long2['qa'] = np.where((comp_long2['par_role']!="cp")
                              & (comp_long2.groupby(['transcriptid','section'])['par_role'].shift(-1) =='cp'), 1, 0)
    comp_long2['n_qa'] = comp_long2.groupby(['transcriptid','section'])['qa'].transform('sum')

In [33]:

    lst1 = comp_long2.query('n_cp==0')['transcriptid'].unique().tolist()
    lst2 = comp_long2.query('section=="QA"&n_qa==0&n_op==0')['transcriptid'].unique().tolist()
    lst = list(set(lst1+lst2))
    temp = comp_long2.query('transcriptid.isin(@lst)')[['transcriptid','section','seq','nametitle','par_role']]

In [34]:

    def classify_role(nametitle):
        new_role = None
        if re.findall(r"\bceo\b|\bcfo\b|\bfounder\b|\bchairman\b|\bpresident\b", nametitle, flags = re.I):
            new_role = 'cp'
        elif re.findall(r'\banalyst\b', nametitle, flags = re.I):
            new_role = "op"
        return new_role
    temp['new_role'] = temp['nametitle'].apply(classify_role)

In [35]:

    comp_long2 = comp_long2.merge(right=temp[['transcriptid','section','seq','nametitle', 'new_role']],
                                  on=['transcriptid','section','seq','nametitle'], how='left')

In [36]:

    comp_long2['par_role'] = np.where((comp_long2['new_role'].notnull()) & (comp_long2['par_role']!=comp_long2['new_role']),
                                      comp_long2['new_role'], comp_long2['par_role'])

In [37]:

    comp_long2 = comp_long2.drop(columns=['role_special', 'new_role',
                                          'cp','op','qa',
                                          'n_cp','n_op','n_qa'])

Upon manual inspection, management presentation and Q&A sessions could be wrongly labeled. Treat a session as a potential Q&A if 1) other participants speak for at least 3 times, 2) there are at least 3 exchanges between a corporate participant and another participant. Unidentified audience is not defined as "other participant" since there is no way to know for sure.

In [38]:

    comp_long3 = comp_long2.copy()
    comp_long3['op'] =(comp_long3['par_role']=="op")*1
    comp_long3['qa'] = np.where((comp_long3['par_role']!="cp")
                              & (comp_long3.groupby(['transcriptid','section'])['par_role'].shift(-1) =='cp'), 1, 0)
    comp_long3['num_op'] = comp_long3.groupby(['transcriptid','section'])['op'].transform('sum')
    comp_long3['num_qa'] = comp_long3.groupby(['transcriptid','section'])['qa'].transform('sum')

In [39]:

    comp_long3['deemedQA'] = np.where((comp_long3['num_qa']>=3) & (comp_long3['num_op']>=3), 1, 0)

## Save¶

In [40]:

    comp_long3.to_pickle(outputpath_long2)

In [ ]:

In [ ]: