Construct Variables¶

Construct measures of analyst behaviors at conferences (Table 6)

In [1]:
import pandas as pd
import numpy as np
import os, re
In [2]:
import nltk, re
nltk.download('punkt')
from nltk.tokenize import sent_tokenize
from nltk.tokenize import word_tokenize

[nltk_data] Downloading package punkt to
[nltk_data]     C:\Users\rache\AppData\Roaming\nltk_data...
[nltk_data]   Package punkt is already up-to-date!
In [3]:
%matplotlib inline 
%load_ext autoreload
%autoreload 2
In [4]:
pd.set_option('display.memory_usage', 'deep')
pd.set_option('display.precision', 2) 
pd.set_option('display.width', 200)
pd.set_option('display.max_rows', 300)
pd.set_option('display.max_columns', None)
pd.set_option('display.max_colwidth', 80)
pd.options.display.float_format = '{:,.4f}'.format
In [5]:
from filepath_03 import * # change to reflect input and output file path

1. Load Files¶

In [6]:
transinfo = pd.read_pickle(filepath_trans)
In [7]:
complong = pd.read_pickle(filepath_complong)
In [8]:
complong_sent = pd.read_pickle(filepath_complong_sent)

2. Identify QA Pairs from Transcript Components¶

Clean component-level data to identify Q&A exchanges.

In [9]:
fullQA = (complong
          .query('deemedQA==1')
          [['transcriptid','section','seq','par_role','analys']]
          .sort_values(by=['transcriptid','section','seq']).copy())

Remove sections that are not Q&As based on various sanity checks.

In [10]:
fullQA['isAud'] = np.where(fullQA['par_role']!="cp", 1, 0)
fullQA['chk_Audfirst'] = fullQA.groupby(['transcriptid','section'])['isAud'].transform('first') # section starts with other participant speaking
fullQA['chk_hasAud'] = fullQA.groupby(['transcriptid','section'])['isAud'].transform('max')
fullQA['chk_continuous'] = np.where(fullQA.groupby(['transcriptid','section'])['seq'].shift(1)+1 == fullQA['seq'], 1, 0)
# deal with the first observation
fullQA['chk_continuous'] = np.where(fullQA['seq']==fullQA.groupby(['transcriptid','section'])['seq'].transform('first'), 1, fullQA['chk_continuous'])
In [11]:
fullQA = (fullQA.query('chk_hasAud==1 & chk_continuous==1') 
          .drop(columns=['chk_hasAud','chk_continuous']))
In [12]:
# Inspect sections that do not start with other-participant speaking. 
# For these cases, corporate participant speeches that occur before the first other participant speaking are usually introductory remarks. Remove them. 
fullQA['isAud_cumsum'] = fullQA.groupby(['transcriptid','section'])['isAud'].cumsum()
fullQA = fullQA.query('(chk_Audfirst==1) | (chk_Audfirst==0 & isAud_cumsum>0)')
In [13]:
fullQA = fullQA.drop(columns=['chk_Audfirst'])
fullQA['chk_Audfirst'] = fullQA.groupby(['transcriptid','section'])['isAud'].transform('first') # section starts with Aud speaking
assert(fullQA['chk_Audfirst'].mean()==1)
fullQA = fullQA.drop(columns=['chk_Audfirst','isAud_cumsum'])

Define groups of Q&A exchanges

In [14]:
fullQA['Q'] = fullQA['isAud']
fullQA['group'] = fullQA.groupby(['transcriptid','section'])['Q'].transform('cumsum')
fullQA['convo_per_group'] = fullQA.groupby(['transcriptid','section','group'])['seq'].transform('count')
fullQA['seq_in_group'] = fullQA.groupby(['transcriptid','section','group']).cumcount() + 1
fullQA['num_q_per_group'] = fullQA.groupby(['transcriptid','section','group'])['Q'].transform('sum')
In [15]:
fullQA['convo_per_group_sec'] = fullQA.groupby(['transcriptid','section'])['convo_per_group'].transform('max')
In [16]:
fullQA2 = fullQA.query('convo_per_group>=2 & convo_per_group_sec<=50 & num_q_per_group==1').copy()
fullQA2 = fullQA2[['transcriptid','section','seq','analys','Q','group','convo_per_group','seq_in_group']]
In [17]:
fullQA2['analys2'] = fullQA2.groupby(['transcriptid','section','group'])['analys'].transform('max') # Fill up analys to conversation level

Keep Q&A exchange initiated by an IBES analys

In [18]:
QA = fullQA2.query('analys2.notnull()').copy()

3. Compute Measures¶

In [19]:
cols = ['transcriptid','section','seq','nwords','afemale','efemale','CEO','CFO','execid','text']
QA_text = QA.copy()
QA_text = QA_text.merge(right=complong[cols], on=['transcriptid','section','seq'], how='inner')
QA_text = QA_text.merge(right=complong_sent.drop(columns=['nwords']), on=['transcriptid','section','seq'], how='inner')
QA_text['i_pos'] = (QA_text['sentiment']>0)*1
QA_text['i_neg'] = (QA_text['sentiment']<0)*1
In [20]:
def directness(text):
    direct = 1
    if re.findall(r"i wonder|i (?:am|was|have been) wondering|i.{1,5} like to ask|i would like to ask|(?:can|may) i ask|(?:can|could|will|would) you", text, flags=re.I):
        direct = 0
    return direct
def followup(text):
    '''
    Investor conferences typically involve a single hosting analyst, whereas earnings calls are attended by multiple analysts.
    It is common for the hosting analyst to speak right after the executive as the Q&A host 
    Define follow-up questions as those that explicitly contain the terms “follow up,” “followup,” “follow-up,” or related variants. 
    '''
    followup = 0
    if re.findall(r"followup|\bfollow\b.{0,3}\bup\b", text, flags=re.I):
        followup = 1
    return followup
def negativeq(text):
    negativeq = 0
    sentences= sent_tokenize(text)
    
    for sent in sentences:
        if re.findall(r"\?$", sent): 
            if re.findall(r"[wc]ouldn[\’\']t you|isn[\’\']t it|doesn[\’\']t it", sent, flags=re.I): 
                negativeq = 1
            
    return negativeq
def preface(text):
    preface = 0
    sentences= sent_tokenize(text)
    
    for i in range(1, len(sentences)): 
        sent_curr = sentences[i]
        sent_prev = sentences[i-1]
        if re.findall(r"\?$", sent_curr):
            if not re.findall(r"\?$", sent_prev) and not re.findall(r"\bhello\b|\bhi\b|\bhey\b|good morning|good afternoon|good evening", sent_prev):
                # require the preceding sentence to contain at least five words to avoid capturing brief transitional remarks (e.g., “all right” or “let’s move on”) made by the hosting analyst
                if len(sent_prev.split())>=5: 
                    preface = 1 
    if len(sentences) == 1:
        preface = 0 
    return preface
In [21]:
QA_text['directness'] = QA_text['text'].apply(directness)
QA_text['followup'] = QA_text['text'].apply(followup)
QA_text['negativeq'] = QA_text['text'].apply(negativeq)
QA_text['preface'] = QA_text['text'].apply(preface)
In [22]:
QA_text['aggrscore'] = QA_text[['directness','preface','followup','negativeq']].sum(axis=1)

Aggregate to transcriptid-analys level.

In [23]:
transANA = (QA_text.query('analys.notnull()')
            .groupby(['transcriptid','analys'])
            .agg(num_qn = ('seq','count'),
                 nwords_qn = ('nwords','mean'), 
                 sentiment = ('sentiment','mean'), 
                 aggr_score = ('aggrscore', 'mean'),                
                ))

4. Aggregate and Save¶

In [24]:
transANA2 = transANA.copy()
In [25]:
trans_private = pd.read_pickle(filepath_private)
In [26]:
transANA3 = transANA2.reset_index().merge(right=trans_private, how='left')
transANA3['i_private'] =transANA3[['i_1on1','i_breakout']].max(axis=1)
In [27]:
transANA3 = transANA3.merge(right=transinfo[['transcriptid', 'gvkey', 'permno', 'ticker', 'ncusip', 'date']], 
                            on=['transcriptid'], how='left')
In [28]:
transANA3.to_csv(outputpath_transANA_csv)
transANA3.to_pickle(outputpath_transANA_pkl)
In [ ]:
In [ ]:
In [ ]:
In [ ]: