Python 3.10.12 | packaged by Anaconda, Inc. | (main, Jul 5 2023, 19:09:20) [MSC v.1916 64 bit (AMD64)]

Type "copyright", "credits" or "license" for more information.

IPython 8.12.0 -- An enhanced Interactive Python.

Restarting kernel...

In [1]: """

...: Objective: calculate CEO speech properties during Q&A of conference calls

...:

...: """

...:

...: import re

...: import pandas as pd

...: import os

...: import numpy as np

...: from transformers import BertTokenizer, BertForSequenceClassification

...: from transformers import pipeline

...: import statistics

...:

...: def clean_string(input_string):

...: cleaned_string = re.sub(r'\s+', ' ', input_string).strip()

...: parts = cleaned_string.split(';')

...: cleaned_string2 = parts[0].strip() if len(parts) > 0 else cleaned_string

...: parts2 = cleaned_string2.split('–')

...: cleaned_string3 = parts2[0].strip() if len(parts2) > 0 else cleaned_string2

...: parts3 = cleaned_string3.split('-')

...: cleaned_string4 = parts3[0].strip() if len(parts3) > 0 else cleaned_string3

...: cleaned_string5 = re.sub(r'\s+', ' ', cleaned_string4).strip()

...:

...: if cleaned_string5=='Jen':

...: cleaned_string5='Jen-Hsun Huang'

...:

...: if cleaned_string5=='Lip':

...: cleaned_string5='Lip-Bu Tan'

...:

...: if cleaned_string5=='Hassane S. El':

...: cleaned_string5='Hassane S. El-Khoury'

...:

...: if cleaned_string5=='Debra Lynn Reed':

...: cleaned_string5='Debra Lynn Reed-Klages'

...:

...: if cleaned_string5=='Francois Locoh':

...: cleaned_string5='Francois Locoh-Donou'

...:

...: if cleaned_string5=='Allan Douglas':

...: cleaned_string5='Allan Douglas-David MacKay'

...: return cleaned_string5

...:

...: def count_words(input_string):

...: pattern = r'\b\w+\b'

...:

...: word_matches = re.findall(pattern, input_string)

...:

...: word_count = len(word_matches)

...:

...: return word_count

...:

...: def detect_absolutism(string):

...: absolutism_keywords = ['absolutely', 'all', 'always','complete', 'completely', 'constant', 'constantly', 'definitely', 'entire', 'ever', 'every','everyone', 'everything', 'full', 'must', 'never', 'nothing', 'totally', 'whole']

...:

...: pattern = r"\b" + r"\b|\b".join(re.escape(keyword) for keyword in absolutism_keywords) + r"\b"

...:

...: matches = re.findall(pattern, string, re.IGNORECASE)

...:

...: return len(matches)

...:

...: def detect_filler_words(string):

...: filler_keywords = ["Hmm", "Mhm", "Mm", "Uh", "Umm", "So", "No", "Yeah", "Okay", "You know"]

...:

...: pattern = r"\b" + r"\b|\b".join(re.escape(keyword) for keyword in filler_keywords) + r"\b"

...:

...: matches = re.findall(pattern, string, re.IGNORECASE)

...:

...: return len(matches)

...:

...: def detect_self_refrential(string):

...: self_refrential_keywords = ["I", "me", "myself", "my", "mine"]

...:

...: pattern = r"\b" + r"\b|\b".join(re.escape(keyword) for keyword in self_refrential_keywords) + r"\b"

...:

...: matches = re.findall(pattern, string, re.IGNORECASE)

...:

...: return len(matches)

...:

...: def detect_tone_finbert(string):

...: sentence_pattern = r'(?<=[.!?])\s+'

...:

...: sentences = re.split(sentence_pattern, string)

...:

...: finbert = BertForSequenceClassification.from_pretrained('yiyanghkust/finbert-tone',num_labels=3)

...: tokenizer = BertTokenizer.from_pretrained('yiyanghkust/finbert-tone')

...:

...: nlp = pipeline("sentiment-analysis", model=finbert, tokenizer=tokenizer)

...: results = nlp(sentences)

...: results_df=pd.DataFrame(results)

...: results_df=results_df[results_df['score']>=0.90]

...: results_df.reset_index(drop=True,inplace=True)

...: results_df['tone_score']=np.nan

...: for index,row in results_df.iterrows():

...: if row['label']=='Negative':

...: results_df.loc[index,'tone_score']=-1

...: if row['label']=='Positive':

...: results_df.loc[index,'tone_score']=1

...: if row['label']=='Neutral':

...: results_df.loc[index,'tone_score']=0

...:

...: return(results_df['tone_score'])

...:

...: transcripts_folder=r"D:\OneDrive - Indiana University\JAR Registered Report\Final_version_submisison\data\csv_conference_call_transcripts_2010_aug2022"

...: index_file=pd.read_csv(r"D:\OneDrive - Indiana University\JAR Registered Report\Final_version_submisison\data\ciq_w_audio_links_w_ceo_names_from_execucomp_and_ciq.csv",index_col=None)

...: index_file.drop(columns=['Unnamed: 0'],inplace=True)

...: index_file.reset_index(drop=True, inplace=True)

...:

...: ### this is to produce log file, the full data needs to be run on superr computers and slurm does not produce log file

...: index_file=index_file[0:10]

...:

...: for index, row in index_file.iterrows():

...: try:

...: file_name=str(row['keydev_id'])+'.csv'

...: ceo_name=row['ciq_fullname']

...: file_path=os.path.join(transcripts_folder, file_name)

...: df=pd.read_csv(file_path,index_col=None)

...: df.drop(columns=['Unnamed: 0'],inplace=True)

...: df['PERSON2'] = df['PERSON'].apply(clean_string)

...: df['SESSION2']=df['SESSION'].apply(clean_string)

...: df['CONTENT_CLEANED']=df['CONTENT'].apply(clean_string)

...: ceo_snippets=df[(df['PERSON2']==ceo_name) & (df['SESSION2']=='Question and Answer')].copy()

...: if len(ceo_snippets)>0:

...: ceo_snippets['CONTENT_CLEANED'] = ceo_snippets['CONTENT_CLEANED'].str.replace('[^a-zA-Z\s]', '', regex=True)

...: ceo_snippets['CONTENT_CLEANED'] = ceo_snippets['CONTENT_CLEANED'].str.encode('utf-8').str.decode('utf-8', 'ignore')

...: ceo_absolutim_list=[]

...: ceo_absolutim_list = ceo_snippets['CONTENT_CLEANED'].apply(detect_absolutism)

...: ceo_absolutim=ceo_absolutim_list.sum()

...: index_file.loc[index,'ceo_absolutim']=ceo_absolutim

...:

...: ceo_speech_length_list=[]

...: ceo_speech_length_list = ceo_snippets['CONTENT_CLEANED'].apply(count_words)

...: ceo_speech_length=ceo_speech_length_list.sum()

...: index_file.loc[index,'ceo_speech_length']=ceo_speech_length

...:

...: ceo_filler_words_list=[]

...: ceo_filler_words_list = ceo_snippets['CONTENT_CLEANED'].apply(detect_filler_words)

...: ceo_filler_words=ceo_filler_words_list.sum()

...: index_file.loc[index,'filler_words']=ceo_filler_words

...:

...: ceo_self_refrential_list=[]

...: ceo_self_refrential_list = ceo_snippets['CONTENT_CLEANED'].apply(detect_self_refrential)

...: ceo_self_refrential=ceo_self_refrential_list.sum()

...: index_file.loc[index,'self_refrential']=ceo_self_refrential

...:

...: ceo_tone_list=[]

...: ceo_tone_list = ceo_snippets['CONTENT'].apply(detect_tone_finbert)

...: ceo_tone_list = ceo_tone_list.stack()

...:

...: ceo_tone_list = ceo_tone_list.dropna()

...: ceo_tone=statistics.mean(ceo_tone_list)

...: index_file.loc[index,'tone']=ceo_tone

...: else:

...: index_file.loc[index,'ceo_absolutim']=np.nan

...: index_file.loc[index,'ceo_speech_length']=np.nan

...: index_file.loc[index,'filler_words']=np.nan

...: index_file.loc[index,'self_refrential']=np.nan

...: index_file.loc[index,'tone']=np.nan

...: except Exception as e:

...: with open(r'D:\OneDrive - Indiana University\JAR Registered Report\Final_version_submisison\data\speech_properties_error_log.txt', 'a') as error_file:

...: error_message = f"An error occurred when going through transcript with keydev_id {row['keydev_id']}: {str(e)}\n"

...: error_file.write(error_message)

...: continue

...:

...: index_file2=index_file[index_file['ceo_speech_length'].notna()]

...: index_file2.reset_index(drop=True,inplace=True)

...:

...: index_file2.to_csv(r"D:\OneDrive - Indiana University\JAR Registered Report\Final_version_submisison\data\index_file_w_ceo_speech_properties.csv",index=False)

C:\Users\nargolsh\AppData\Roaming\Python\Python310\site-packages\tqdm\auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html

from .autonotebook import tqdm as notebook_tqdm

C:\Users\nargolsh\AppData\Roaming\Python\Python310\site-packages\transformers\utils\generic.py:311: UserWarning: torch.utils._pytree._register_pytree_node is deprecated. Please use torch.utils._pytree.register_pytree_node instead.

torch.utils._pytree._register_pytree_node(

In [2]: