...:
...:
...: import os
...: import csv
...: import glob
...: import scipy
...: import sklearn
...: import numpy as np
...: import hmmlearn.hmm
...: import sklearn.cluster
...: import pickle as cpickle
...: import matplotlib.pyplot as plt
...: from scipy.spatial import distance
...: import sklearn.discriminant_analysis
...: import pandas as pd
...: from pyAudioAnalysis import audioBasicIO
...: from pyAudioAnalysis import audioTrainTest as at
...: from pyAudioAnalysis import MidTermFeatures as mtf
...: from pyAudioAnalysis import ShortTermFeatures as stf
...: from pyAudioAnalysis import audioSegmentation as aS
...: from urllib.parse import unquote
...: from pydub import AudioSegment
...: import operator
...: import wave
...: import traceback
...: import time
...: import re
...:
...:
...: start_time=time.time()
...: n1=0
...: n2=1
...:
...: #cleaning a few CEO names that were not clean
...:
...: def clean_string(input_string):
...: cleaned_string = re.sub(r'\s+', ' ', input_string).strip()
...: parts = cleaned_string.split(';')
...: cleaned_string2 = parts[0].strip() if len(parts) > 0 else cleaned_string
...: parts2 = cleaned_string2.split('–')
...: cleaned_string3 = parts2[0].strip() if len(parts2) > 0 else cleaned_string2
...: parts3 = cleaned_string3.split('-')
...: cleaned_string4 = parts3[0].strip() if len(parts3) > 0 else cleaned_string3
...: parts4 = cleaned_string4.split(',')
...: cleaned_string5 = parts4[0].strip() if len(parts4) > 0 else cleaned_string4
...: cleaned_string6 = re.sub(r'\s+', ' ', cleaned_string5).strip()
...:
...: if cleaned_string6=='Jen':
...: cleaned_string6='Jen-Hsun Huang'
...:
...: if cleaned_string6=='Lip':
...: cleaned_string6='Lip-Bu Tan'
...:
...: if cleaned_string6=='Hassane S. El':
...: cleaned_string6='Hassane S. El-Khoury'
...:
...: if cleaned_string6=='Debra Lynn Reed':
...: cleaned_string6='Debra Lynn Reed-Klages'
...:
...: if cleaned_string6=='Francois Locoh':
...: cleaned_string6='Francois Locoh-Donou'
...:
...: if cleaned_string6=='Allan Douglas':
...: cleaned_string6='Allan Douglas-David MacKay'
...: return cleaned_string6
...:
...:
...: def count_words_with_numbers_and_percentages(input_string):
...:
...: words = re.findall(r'\b(?:\w+|\d+)\b', input_string)
...: words2= re.findall(r'%', input_string)
...: words3= re.findall(r'&', input_string)
...: word_count = len(words)+len(words2)+len(words3)
...:
...: return word_count
...:
...:
...: def transc_word_cnt(transc_df):
...: word_cnt_dic={}
...: accum_word_cnt_dic={}
...: accum_word_cnt=0
...:
...:
...: for index,row in transc_df.iterrows():
...: if pd.isna(row['CONTENT']):
...: word_cnt=0
...: word_cnt_dic[index]=word_cnt
...:
...: accum_word_cnt+=word_cnt
...: accum_word_cnt_dic[index]=accum_word_cnt
...: else:
...: word_cnt=count_words_with_numbers_and_percentages(str(row['CONTENT']))
...: word_cnt_dic[index]=word_cnt
...:
...: accum_word_cnt+=word_cnt
...: accum_word_cnt_dic[index]=accum_word_cnt
...:
...: transc_df["word_cnt"] = pd.Series(word_cnt_dic)
...: transc_df["accum_word_end"] = pd.Series(accum_word_cnt_dic)
...: transc_df['accum_word_start'] = transc_df.accum_word_end.shift(1)
...: word_sum= transc_df["word_cnt"].sum()
...: transc_df['accum_word_start_p'] = transc_df['accum_word_start']/word_sum
...: transc_df["accum_word_end_p"] = transc_df['accum_word_end']/word_sum
...: transc_df.accum_word_start_p = transc_df.accum_word_start_p.round(4)
...: transc_df.accum_word_end_p = transc_df.accum_word_end_p.round(4)
...: return transc_df
...:
...:
...: def merge_intervals(intervals_list):
...: intervals_list.sort(key=lambda interval: interval[0])
...: merge = [intervals_list[0]]
...: for current in intervals_list:
...: previous = merge[-1]
...: if current[0] <= previous[1]:
...: previous[1] = max(previous[1], current[1])
...: else:
...: merge.append(current)
...: return merge
...:
...:
...: def cls_to_df(cls_obj):
...:
...: cls_df = pd.DataFrame(data=cls_obj,columns=["speaker"])
...: cls_df=cls_df.reset_index()
...: cls_df=cls_df.rename(columns={"index": 'time_0.2_s'})
...: cls_df['time_s']=cls_df['time_0.2_s']//5
...: cls_df.time_s=cls_df.time_s.round(1)
...: cls_df['time_end'] = cls_df.time_s.shift(-1)
...:
...: return cls_df
...:
...: def ceo_transc_word(ceo_name,transc_df):
...: ceo_word_p_list=[]
...: transc_df['SESSION']=transc_df['SESSION'].apply(clean_string)
...: transc_df['PERSON']=transc_df['PERSON'].apply(clean_string)
...: print("I entered ceo_transc_word function")
...: for index,row in transc_df.iterrows():
...:
...: if (row['SESSION']!='Presentation'):
...: continue
...: if (ceo_name in row['PERSON']) & (row['SESSION']=='Presentation'):
...:
...: ceo_word_p_list.append([row['accum_word_start_p'],row['accum_word_end_p']])
...:
...: m_ceo_word_p_list=merge_intervals(ceo_word_p_list)
...: test=pd.DataFrame(m_ceo_word_p_list)
...: test.to_csv(f"D:\OneDrive - Indiana University\JAR Registered Report\Final_version_submisison\data\diarization_test\ceo_part_{ceo_name}.csv")
...: return m_ceo_word_p_list
...:
...: def ceo_speak_time_est(m_ceo_word_p_list,cls_df):
...: total_s=cls_df.tail(1).time_s.iloc[0]+0.2
...:
...: ceo_speak_time_est_list = []
...: for ceo_word_p in m_ceo_word_p_list:
...: ceo_speak_time= [(i * total_s).round(0) for i in ceo_word_p]
...:
...: ceo_speak_time_est_list.append(ceo_speak_time)
...: return ceo_speak_time_est_list
...:
...: def find_ceo_speaker_num(ceo_speak_time_est_list,cls_df):
...: speaker_cnt_dic={}
...: for index, row in cls_df.iterrows():
...: for ceo_speak_time in ceo_speak_time_est_list:
...: if (row['time_s']>=ceo_speak_time[0])&((row['time_s']<=ceo_speak_time[1])): ##show the second one be time_end?
...: if row['speaker'] in speaker_cnt_dic:
...: speaker_cnt_dic[row['speaker']] += 1
...: else:
...: speaker_cnt_dic[row['speaker']] = 1
...: print(speaker_cnt_dic)
...: ceo_speaker_num=max(speaker_cnt_dic.items(), key=operator.itemgetter(1))[0]
...:
...: return ceo_speaker_num
...:
...: def ceo_speak_time(ceo_speaker_num,cls_df):
...: ceo_speak_time_list=[]
...: for index,row in cls_df.iterrows():
...: if row['speaker']==ceo_speaker_num:
...: ceo_speak_time_list.append([row['time_s'],row['time_end']])
...: m_ceo_speak_time_list=merge_intervals(ceo_speak_time_list)
...: return m_ceo_speak_time_list
...:
...: def pr_part_check(transc_df,total_s):
...: pr_part_end=0
...: transc_df['SESSION']=transc_df['SESSION'].apply(clean_string)
...:
...: for index,row in transc_df.iterrows():
...:
...: if (row['SESSION']=='Presentation')&(index>pr_part_end):
...: pr_part_end=index
...:
...: pr_part_end_p=transc_df['accum_word_end_p'].iloc[pr_part_end]
...:
...: pr_part_end_time=(pr_part_end_p*total_s).round(0)
...:
...: return pr_part_end_time
...:
...: def speak_time_check(m_ceo_speak_time_list,pr_part_end_time,interval_len=5,buffer_time=300):
...: m_ceo_speak_time_list_chk=[]
...: for ceo_speak_time in m_ceo_speak_time_list:
...: if (ceo_speak_time[1]-ceo_speak_time[0]>interval_len)&(ceo_speak_time[1]<pr_part_end_time+buffer_time):
...: m_ceo_speak_time_list_chk.append(ceo_speak_time)
...: return m_ceo_speak_time_list_chk
...:
...: def audio_slice_merge(audio_wav,speak_time_list,output_dir):
...:
...: audio_p_list=[]
...: for speak_time in speak_time_list:
...: speak_time_k=[i * 1000 for i in speak_time]
...: audio_p=audio_wav[speak_time_k[0]:speak_time_k[1]]
...: audio_p_list.append(audio_p)
...:
...: audio_p_m=sum(audio_p_list)
...: audio_p_m.export(output_dir, format="wav")
...:
...: def copy_transc(transc_input_dir,transc_output_dir,audio_transc_link_dir):
...: from shutil import copyfile
...: audio_transc_link_df=pd.read_csv(audio_transc_link_dir,low_memory=False)
...: transc_list=audio_transc_link_df['keydev_id'].tolist()
...: for transc in transc_list:
...: transc_fname=str(transc)+'.csv'
...: transc_fname_dir=os.path.join(transc_input_dir, transc_fname)
...: transc_fname_output_dir=os.path.join(transc_output_dir, transc_fname)
...: copyfile(transc_fname_dir, transc_fname_output_dir)
...:
...: def audio_fname_from_link(link_table_dir):
...:
...: df=pd.read_csv(link_table_dir,low_memory=False)
...: dir_fname={}
...: for index,row in df.iterrows():
...: id_ciq=row['keydev_id']
...: if pd.isnull(row['Downloaded Audio File Name']):
...: continue
...: else:
...: audio_fname=row['Downloaded Audio File Name']
...: dir_fname[id_ciq]=unquote(audio_fname)
...: fname_df=pd.DataFrame.from_dict(dir_fname, orient='index',columns=['audio_fname'])
...:
...: fname_df['keydev_id'] = fname_df.index
...: df_m=pd.merge(df, fname_df, on='keydev_id', how='outer')
...: df_m_dir=os.path.join(dir_input,'link_table_m.csv')
...: df_m.to_csv(df_m_dir)
...:
...: def find_ceo_speaker_num_simple(cls_df,cut=0.333):
...: speaker_cnt_dic={}
...:
...: for index, row in cls_df.iterrows():
...: if index>len(cls_df.index)*cut:
...:
...: break
...: if row['speaker'] in speaker_cnt_dic:
...: speaker_cnt_dic[row['speaker']] += 1
...: else:
...: speaker_cnt_dic[row['speaker']] = 1
...:
...: ceo_speaker_num=max(speaker_cnt_dic.items(), key=operator.itemgetter(1))[0]
...:
...: return ceo_speaker_num
...:
...: def audio_diarization_ceo_simple(audio_fname,audio_input_dir,cls_df,audio_output_dir,cut=0.333):
...:
...:
...: ceo_speaker_num=find_ceo_speaker_num_simple(cls_df=cls_df,cut=cut)
...: m_ceo_speak_time_list=ceo_speak_time(ceo_speaker_num=ceo_speaker_num,cls_df=cls_df)
...:
...: audio_fname_dir=os.path.join(audio_input_dir, audio_fname)
...: audio_wav = AudioSegment.from_wav(audio_fname_dir)
...:
...: audio_output_fname='2_'+audio_fname[:-4]+'.wav'
...:
...: audio_output_fname_dir=os.path.join(audio_output_dir, audio_output_fname)
...:
...: audio_slice_merge(audio_wav=audio_wav,speak_time_list=m_ceo_speak_time_list,output_dir=audio_output_fname_dir)
...:
...:
...:
...: def audio_diarization_ceo(audio_transc_link_dir,audio_input_dir,transc_dir,cls_output_dir,audio_output_dir,simple,start_r,end_r):
...: """
...: audio_transc_link: directory of the audio-transc link table csv file (with "ceo_lname" & "num_speakers" columns)
...: audio_input_dir=directory of the folder of the wav audio inputs
...: transc_dir=directory of the folder of the csv transcript inputs
...: cls_output_dir=directory of the folder to export the speaker diarization csv output
...: start_r=starting row of the link table to be processed
...: end_r=ending row of the link table to be processed (inclusive)
...:
...: """
...:
...:
...: error_dic={}
...: audio_list=os.listdir(audio_input_dir)
...:
...:
...: link_df=pd.read_csv(audio_transc_link_dir,low_memory=False)
...: n1=start_r
...: n2=min(end_r,len(link_df.index)-1)+1
...: link_df['ciq_fullname']=link_df['ciq_fullname'].apply(clean_string)
...: for index, row in link_df[n1:n2].iterrows():
...: transc_id=row['keydev_id']
...: transc_fname=str(transc_id)+".csv"
...: transc_file_dir=os.path.join(transc_dir, transc_fname)
...: print(f"working on this trascript: {row['keydev_id']}")
...: if pd.isnull(link_df.at[index,'Downloaded Audio File Name']):
...: continue
...: else:
...:
...: audio_fname=row['Downloaded Audio File Name']
...: if audio_fname[-3:]!='wav':
...: audio_fname=audio_fname[:-3]+'wav'
...: audio_fname=audio_fname.replace(',', '')
...: audio_fname=audio_fname.replace(':', '_')
...:
...: audio_fname=audio_fname.replace('NYSE ', 'NYSE_')
...: audio_fname=audio_fname.replace('NasdaqGS ', 'NasdaqGS_')
...: audio_fname=audio_fname.replace('NasdaqCM ', 'NasdaqCM_')
...: audio_fname=audio_fname.replace('LSE ', 'LSE_')
...: audio_fname=audio_fname.replace('OTCPK ', 'OTCPK_')
...:
...: if audio_fname not in audio_list:
...: #hier
...: print(audio_fname+" :this audio file was not in folder!")
...: continue
...:
...: audio_fname_dir=os.path.join(audio_input_dir, audio_fname)
...: print(' and this audio file at index:',index,'fname:',audio_fname)
...:
...: try:
...: transc_df=pd.read_csv(transc_file_dir,low_memory=False)
...: transc_df=transc_word_cnt(transc_df)
...: except Exception as ex:
...: error_dic[transc_id]:transc_id
...: print("error file:",ex)
...: continue
...:
...: if pd.isnull(link_df.at[index,'num_speakers']):
...: num_speakers=0
...:
...: else:
...: num_speakers=int(row['num_speakers'])
...:
...:
...: ceo_name=row['ciq_fullname']
...: print(f"The CEO name that I will be looking for in trascripts is {ceo_name}")
...:
...: """
...: in audioSegmentation.py
...:
...: def speaker_diarization(filename, n_speakers, mid_window=2.0, mid_step=0.2,
...: short_window=0.05, lda_dim=35, plot_res=False):
...:
...: ARGUMENTS:
...: - filename: the name of the WAV file to be analyzed
...: - n_speakers the number of speakers (clusters) in
...: the recording (<=0 for unknown)
...: - mid_window (opt) mid-term window size
...: - mid_step (opt) mid-term window step
...: - short_window (opt) short-term window size
...: - lda_dim (opt LDA dimension (0 for no LDA)
...: - plot_res (opt) 0 for not plotting the results 1 for plotting
...: """
...:
...: try:
...: cls_obj=aS.speaker_diarization(filename=audio_fname_dir,n_speakers=int(num_speakers),
...: mid_window=2,short_window=0.05,lda_dim=15, plot_res=0)
...: cls_df=cls_to_df(cls_obj=cls_obj)
...:
...: m_ceo_word_p_list=ceo_transc_word(ceo_name=ceo_name,transc_df=transc_df)
...: print("The code is doing the diarization process approach 1")
...:
...: cls_fname=audio_fname[:-4]+'.csv'
...: cls_file_dir=os.path.join(cls_output_dir,cls_fname)
...: cls_df.to_csv(cls_file_dir)
...: transc_df.to_csv(f"D:\OneDrive - Indiana University\JAR Registered Report\Final_version_submisison\data\diarization_test\examintion_transdf_{audio_fname}.csv")
...:
...:
...:
...: ceo_speak_time_est_list=ceo_speak_time_est(m_ceo_word_p_list=m_ceo_word_p_list,cls_df=cls_df)
...: print(f"This is ceo_speak_time_est_list: {ceo_speak_time_est_list}")
...:
...: ceo_speaker_num=find_ceo_speaker_num(ceo_speak_time_est_list=ceo_speak_time_est_list,cls_df=cls_df)
...:
...: print(f"This is the CEO speaker num: {ceo_speaker_num}")
...:
...: m_ceo_speak_time_list=ceo_speak_time(ceo_speaker_num=ceo_speaker_num,cls_df=cls_df)
...:
...: total_s=cls_df.tail(1).time_s.iloc[0]+0.2
...:
...: pr_part_end_time=pr_part_check(transc_df=transc_df,total_s=total_s)
...:
...: m_ceo_speak_time_list_chk=speak_time_check(m_ceo_speak_time_list=m_ceo_speak_time_list,
...: pr_part_end_time=pr_part_end_time,interval_len=5,buffer_time=300)
...:
...: audio_wav = AudioSegment.from_wav(audio_fname_dir)
...:
...: audio_output_fname='1_'+audio_fname[:-4]+'.wav'
...:
...: audio_output_fname_dir=os.path.join(audio_output_dir, audio_output_fname)
...:
...: audio_slice_merge(audio_wav=audio_wav,speak_time_list=m_ceo_speak_time_list_chk,
...: output_dir=audio_output_fname_dir)
...:
...:
...: except Exception as ex:
...: print("This is the exception that made the code do approach 2:", ex)
...: if simple==1:
...: try:
...: cls_obj=aS.speaker_diarization(filename=audio_fname_dir,n_speakers=int(num_speakers),
...: mid_window=2,short_window=0.05,lda_dim=15, plot_res=0)
...: cls_df_1=cls_to_df(cls_obj=cls_obj)
...: print(f"The code is doing the diarization in approach 2 on {audio_fname}")
...:
...: audio_diarization_ceo_simple(audio_fname=audio_fname,audio_input_dir=audio_input_dir,
...: cls_df=cls_df_1,audio_output_dir=audio_output_dir,cut=0.333)
...:
...:
...: except Exception as ex:
...: error_dic[transc_id]:transc_id
...: print("error file:",transc_id,ex)
...: print(traceback.print_exc())
...: continue
...:
...: else:
...: error_dic[transc_id]:transc_id
...: print("error file:",transc_id,ex)
...: print(traceback.print_exc())
...: continue
...:
...:
...:
...: error_df=pd.DataFrame.from_dict(error_dic, orient='index',columns=['error_fname'])
...: error_df_dir=os.path.join(dir_input,'error_fname_list.csv')
...: error_df.to_csv(error_df_dir)
...:
...: """
...: This code extracts CEO speaking parts from audio wav files with csv transcripts,
...: number of speakers, and CEO name data. It use the relative location of CEOs' words in the transcripts
...: to estimate CEOs' speaking part in the 'PR' session. With the cls 'speaker' output from
...: the speaker_diarization function of pyAudioAnalysis, it checks which 'speaker' in the cls speak most
...: in these estimated speaking parts, and it assumes that 'speaker' is the CEO. It then extracts and merges
...: the parts of this 'speaker' with the timestamp in the cls.
...:
...: The process above creates CEO audio output files starting with '1_'.
...: If the route above does not work, it uses a simpler process checking which 'speaker' speaks most
...: in the first 1/3 of the audio (As most CEO presents first in the PR session). The simplified process
...: create output files starting with '2_'.
...:
...: To run the code:
...: 1. Please place the audio input wav files in the "audio_inputs" folder
...: 2. Please place the transcript files in the "transc_inputs" folder
...: 3. Please set the directory to the 'pyAudioAnalysis_test' with the line below starting with os.chdir().
...:
...: """
...:
...:
...:
...: dir_input='D:\OneDrive - Indiana University\JAR Registered Report\Final_version_submisison\data\diarization_test'
...:
...:
...: os.chdir(dir_input)
...:
...:
...:
...: audio_transc_link_dir_default=os.path.join(os.getcwd(),'index_file_test.csv')
...: audio_input_dir_default=os.path.join(os.getcwd(),'audio_inputs')
...: transc_dir_default=os.path.join(os.getcwd(),'transcript_inputs')
...: cls_output_dir_default=os.path.join(os.getcwd(),'cls_outputs')
...: audio_output_dir_default=os.path.join(os.getcwd(),'ceo_audio_outputs')
...:
...: simple_default=1
...:
...: audio_transc_link_dir_default2=os.path.join(os.getcwd(),'index_file_test.csv')
...:
...: audio_diarization_ceo(audio_transc_link_dir_default2,audio_input_dir_default,transc_dir_default,cls_output_dir_default,audio_output_dir_default,simple_default,n1,n2)
...:
...: end_time=time.time()
...:
...: time=(end_time-start_time)/3600
...:
...: print(f"this code took {time} hours!")
working on this trascript: 108464683
and this audio file at index: 0 fname: CarMax Inc. (NYSE_KMX) Sep-22-2010 - Audio.wav
The CEO name that I will be looking for in trascripts is Thomas Joseph Folliard
C:\Users\nargolsh\AppData\Roaming\Python\Python310\site-packages\sklearn\cluster\_kmeans.py:1416: FutureWarning: The default value of `n_init` will change from 10 to 'auto' in 1.4. Set the value of `n_init` explicitly to suppress the warning
super()._check_params_vs_input(X, default_n_init=10)
I entered ceo_transc_word function
The code is doing the diarization process approach 1
This is ceo_speak_time_est_list: [[85.0, 150.0], [312.0, 357.0]]
{8.0: 142, 4.0: 9, 17.0: 91, 1.0: 25, 15.0: 8, 3.0: 73, 12.0: 200, 18.0: 12}
This is the CEO speaker num: 12.0
working on this trascript: 402970702
and this audio file at index: 1 fname: CarMax Inc. (NYSE_KMX) Sep-22-2017 - Audio.wav
The CEO name that I will be looking for in trascripts is William D. Nash
C:\Users\nargolsh\AppData\Roaming\Python\Python310\site-packages\sklearn\cluster\_kmeans.py:1416: FutureWarning: The default value of `n_init` will change from 10 to 'auto' in 1.4. Set the value of `n_init` explicitly to suppress the warning
super()._check_params_vs_input(X, default_n_init=10)
I entered ceo_transc_word function
The code is doing the diarization process approach 1
This is ceo_speak_time_est_list: [[74.0, 346.0], [488.0, 628.0]]
{16.0: 248, 14.0: 404, 3.0: 135, 15.0: 41, 9.0: 403, 17.0: 63, 4.0: 50, 5.0: 77, 2.0: 374, 1.0: 40, 11.0: 81, 8.0: 130, 13.0: 24}
This is the CEO speaker num: 14.0
this code took 0.1351375542084376 hours!
In [2]: