ds_fire_fighter/__pycache__/utils.cpython-311.pyc

§
cy¾fSJãóЗdd
lm
Z

ddlmZ
ddlZddlmZ
ddlmZ
ddl	m
Z

ddl	mZ
ddl	mZ
dd	l
mZ
dd
lmZ
ddlmZ
ddlmZ
dd
lmZ
ddlmZ
ddlZddlmZ
ddlZddlZddlmZ
ddl Z ddl!Z!ddl"Z"ddl#Z$ddl%m&Z&
ddl'Z'ddl(Z(ddl)m*Z*
ddl+Z+ddl,m-Z-
e-¦«
ej.d¦
«
a/ej.d¦
«
ej0d<eej.d¦
«
¬¦
«
Z1dZ2eddd¬¦«Z3d„Z4e4¦«Z5d@d„
Z6d„Z7d„Z8d „Z9d!„Z:d"„Z;d#„Z<dAd%„
Z=d&„Z>dBd(„
Z?dCd*„
Z@dDd,„
ZAd-eBd.eCfd/„ZDd0eEd1eBfd2„ZFdEd4„
ZGe5d3fd5„
ZHd6„ZId7eBfd8„ZJd9eEd:eEd;eEfd<„ZKd9eEd:eEd;eEfd=„ZLdFd?„
ZMdS)Gé©
ÚHuggingFaceBgeEmbeddings)
ÚRecursiveCharacterTextSplitterN)
ÚInMemoryDocstore)
ÚFAISS)
ÚPyPDFLoader)
Ú
TextLoader)
ÚDocx2txtLoader)
ÚChatGroq)
ÚPromptTemplate)
ÚStrOutputParser)
Úuuid4)
ÚDocument)
Ú
TextExtractor)
ÚThreadPoolExecutor)
ÚGroq)
ÚAudioSegment)
Ú
VideoFileClip)
Úload_dotenvÚOPENAI_API_KEYÚGROQ_API_KEY)
Úapi_keyúwhisper-large-v3zllama3-8b-8192éd)ÚtemperatureÚ
model_nameÚ
max_tokenscó>—d
}ddi
}
ddi
}t
||
|¬¦«}|S)NzBAAI/bge-small-enÚdeviceÚcudaÚnormalize_embeddingsT)rÚmodel_kwargsÚ
encode_kwargsr)rr"r#Ú
embeddingss    úHc:\Users\timmy_3aupohg\Downloads\Manaknight Projects\ds_aiindex\utils.pyÚload_embedding_modelr&-s?€Ø$€JؘfÐ%€LØ+¨TÐ2€MÝ)Ø%°LÐP]ðñô€JðÐóÚtextcóV
—|d
j}|d
j
}tddtd¬¦«}| |g
¦
«
}g}t|¦
«
D]N\}}| ¦«}	||	d<|
|	d<t|j|	¬¦«}
| |
¦
«

ŒO|S)	Nréèé
F)Ú
chunk_sizeÚ
chunk_overlapÚlength_functionÚis_separator_regexÚpageÚ	file_type©Úpage_contentÚmetadata)	r3r4rÚlenÚcreate_documentsÚ	enumerateÚcopyrÚappend)Údocr1r(r4Ú
text_splitterÚdocsÚ	documentsÚ
iÚchunkÚdoc_metadataÚdocuments           r%r6r6<sÁ€ØˆqŒ6Ô€DØ�1ŒvŒ€HÝ2ØØÝØ ð	ñô€Mð×)Ò)¨4¨&Ñ1Ô1€Dà€Iݘd‘O”Oð#ð#‰ˆ
ˆ5à—}’}‘”ˆØ ˆ�VÑØ$-ˆ�[Ñ!ݨÔ);ÀlÐSÑSÔSˆØ×Ò˜Ñ"Ô"Ð"Ð"ØÐr'c
óš—	t
|¦
«
}
|
 
¦«}t|¦
«
}|S#
td
|›�¦
«
‚
xYw
©NzError loading -- )r	Úloadr6Ú
ValueError)Ú
document_pathÚtxt_docr(r<s    r%Úload_txt_documentrHRsS€ð>ݘ]Ñ+Ô+ˆØ�|Š|‰~Œ~ˆå Ñ%Ô%ˆØˆøð
>ÝÐ<¨]Ð<Ð<Ñ=Ô=Ð=øøøó	‚36¶A
c
óš—	t
|¦
«
}
|
 
¦«}t|¦
«
}|S#
td
|›�¦
«
‚
xYw
rC)r
rDr6rE)rFÚdocx_docr(r<s    r%Úload_docx_documentrL]sS€ð>Ý! -Ñ0Ô0ˆØ�}Š}‰Œˆå Ñ%Ô%ˆØˆøð
>ÝÐ<¨]Ð<Ð<Ñ=Ô=Ð=øøørIc
ó|—	t
|¦
«
}
|
 
¦«}|S#
td
|›�¦
«
‚
xYw
rC)rÚload_and_splitrE)rFÚpdf_docÚpagess   r%Úload_pdf_documentrQisJ€ð>ݘmÑ,Ô,ˆØ×&Ò&Ñ(Ô(ˆØˆøð
>ÝÐ<¨]Ð<Ð<Ñ=Ô=Ð=øøøs‚$'§;c
óþ—| d
¦
«
rt|¦
«
S| d¦
«
rt|¦
«
S| d¦
«
rt|¦
«
St	d|›�¦
«
‚
)Nz.pdfz.txtz.docxzUnsupported document type for )ÚendswithrQrHrLrE)
rFs
 r%Ú
load_documentrTssˆ€Ø×Ò˜fÑ%Ô%ðK
Ý  Ñ/Ô/Ð/Ø	×	Ò	 Ñ	'Ô	'ðK
Ý  Ñ/Ô/Ð/Ø	×	Ò	 Ñ	(Ô	(ðK
Ý! -Ñ0Ô0Ð0åÐI¸-ÐIÐIÑJÔJÐJr'c
óÈ—t
|d
¦«5}
tj|
 ¦«¦
«
 d¦
«
cddd¦«
S#1swxYw

Y

dS)NÚrbzutf-8)ÚopenÚbase64Ú	b64encodeÚreadÚdecode)Ú
image_pathÚ
image_files  r%Úencode_imager^s™€ÝˆJ˜ÑÔð
? ÝÔ˜JŸOšOÑ-Ô-Ñ.Ô.×5Ò5°gÑ>Ô>ð
?ð
?ð
?ð
?ñ
?ô
?ð
?ð
?ð
?ð
?ð
?ð
?øøøð
?ð
?ð
?ð
?ð
?ð
?s‘9AÁAÁ
Ac
ó(
—t
|¦
«
}
d
dt›�dœ}	dddddœd	d
d|
›�i
dœgd
œg
ddœ}tjd||¬¦«}| ¦«dddd}n#t
$r}d}Yd}~nd}~w
wxYw
|S)Nzapplication/jsonzBearer )zContent-TypeÚ
Authorizationzgpt-4o-miniÚuserr(uWhat’s in this image?)Útyper(Ú	image_urlÚurlzdata:image/jpeg;base64,)rbrc)ÚroleÚcontenti,
)ÚmodelÚmessagesrz*https://api.openai.com/v1/chat/completions)ÚheadersÚjsonÚchoicesrÚmessagerfú$Image not good enough for processing)r^rÚrequestsÚpostrjÚ	Exception)r\Úbase64_imageriÚpayloadÚresponseÚ
es      r%Ú
process_imageru„s
€õ  
Ñ+Ô+€Lð+Ø,¥7Ð,Ð,ðð€Gð
:à"ð#ð%+Ø$=ððð
%0à %Ð'OÀÐ'OÐ'Oð*ððð ðððð"ð'
ð
ˆõ,”=Ð!MÐW^ÐelÐmÑmÔmˆà—=’=‘?”? 9Ô-¨aÔ0°Ô;¸IÔFˆˆøÝð
:ð
:ð
:Ø9ˆˆˆˆˆˆøøøøð
:øøøð€OsžAA9Á9
BÂB
Â
BÚimagecóZ
—| d
¦
«
d d¦
«
d}||
dœ}t¦«}| |¦
«
}d d„|D¦«¦
«
}|dkrt	|¦
«
}|dkrt||¬	¦«}|g
SdS)
Nú
/éÿÿÿÿú
.r)Úfilenamer1Úc
3óvK
—|]4}
|
 ¦«s|
 
¦«s|
dk¯0|
V—

Œ5d
S)ú

N)ÚisalnumÚisspace)Ú.0rts  r%ú	<genexpr>z(create_image_document.<locals>.<genexpr>¸s@èè€ÐNÐN˜ a§i¢i¡k¤kÐN°Q·Y²Y±[´[ÐNÀAÈÂIÀI�1ÀIÀIÀIÀIÐNÐNr'rmr2)ÚsplitrÚread_text_from_imageÚjoinrur)r\r1Ú
image_namer4Útext_extractorr(r:s       r%Úcreate_image_documentrˆ°sº€à×!Ò! #Ñ&Ô& rÔ*×0Ò0°Ñ5Ô5°aÔ8€Jà&°YÐ?Ð?€HÝ"‘_”_€NØ×.Ò.¨zÑ:Ô:€Dà
�7Š7ÐNÐN˜dÐNÑNÔNÑNÔN€Dðˆr‚z€zݘZÑ(Ô(ˆðÐ5Ò5Ð5å D°8Ð<Ñ<Ô<ˆàˆuˆàˆr'c
óÔ—t
|d
¦«5}
tjj ||
 ¦«fd¬¦«}ddd¦«
n#1swxYw

Y

|jS)NrVr)Úfilerg)rWÚclientÚaudioÚtranslationsÚcreaterZr()ÚfilepathrŠÚtranslations   r%Ú
audio_to_textr‘És¡€Ý	
ˆh˜Ñ	Ô	ð
 Ý”lÔ/×6Ò6ؘDŸIšI™KœKÐ(Ø$ð7ñ
ô
ˆð
ð
ð
ñ
ô
ð
ð
ð
ð
ð
ð
øøøð
ð
ð
ð
ð
ÔÐs‘;AÁAÁ
ATcó.—|
d
zdz}t
j
|¦
«
}t|¦
«
}tj |¦
«
 d¦
«
d}|›d�}tj |¦
«
stj|¦
«

g}||kr§tj
||z¦
«
}	tt|	¦
«
¦
«
D]r}
|
|z}t||z|¦«}|||…}
|›d|›d|
dz›d	�}|
 |d
¬¦«
| |¦
«

|rt!d|›�¦
«

ŒsnH|›d|›d
�}| |d
¬¦«
| |¦
«

|rt!d|›�¦
«

||fS)Né<r*rzrÚ_chunksrxÚ_chunké
ú.mp3Úmp3)
Úformatz
Exporting z_chunk1.mp3)rÚ	from_filer5ÚosÚpathÚbasenamerƒÚexistsÚmakedirsÚmathÚceilÚrangeÚintÚminÚexportr9Úprint)Úaudio_file_pathÚchunk_duration_minutesÚprint_outputÚchunk_length_msrŒÚaudio_duration_msÚ
base_filenameÚchunk_folderÚchunk_pathsÚ
num_chunksr>Ústart_msÚend_msr?Úchunk_filenames               r%Úsplit_audio_by_durationr³ÒsÙ
€à,¨rÑ1°DÑ8€Oõ
Ô" ?Ñ3Ô3€EݘE™
œ
Ðõ”G×$Ò$ _Ñ5Ô5×;Ò;¸CÑ@Ô@ÀÔC€MØ#Ð,Ð,Ð,€LÝ
Œ7�>Š>˜,Ñ'Ô'ð
"Ý
Œ�LÑ!Ô!Ð!à€Kà˜?Ò*Ð*å”YÐ0°?ÑBÑCÔCˆ
å•s˜:‘”Ñ'Ô'ð	5ð	5ˆAؘ?Ñ*ˆHݘ OÑ3Ð5FÑGÔGˆFؘ( 6˜/Ô*ˆEØ ,ÐMÐM¨}ÐMÐMÀAÀaÁCÐMÐMÐMˆNØ�LŠL˜°ˆLÑ6Ô6Ð6Ø×Ò˜~Ñ.Ô.Ð.Øð

5ÝÐ3 >Ð3Ð3Ñ4Ô4Ð4øð	5ð)ÐEÐE¨=ÐEÐEÐEˆØ
�Š�^¨EˆÑ2Ô2Ð2Ø×Ò˜>Ñ*Ô*Ð*Øð
	1ÝÐ/˜~Ð/Ð/Ñ0Ô0Ð0à˜Ð$Ð$r'rŒc	ó�—t
||
¦«\}}g}|D�
]}t|¦
«
}tj |¦
«
}tjd
|¦«}	|	r8|	 d¦
«
}
t|	 d¦
«
¦
«
}n'tj 	|¦
«
d}
d}|dz
|
z}||
z}
t|
ttj
|¦
«
¦
«
dz¦«}|
|›d|
›d�|dœ}t||¬	¦«}| |¦
«

�
Œt!j|¦
«

|S)
Nz(.*)_chunk(\d+)\.mp3$r–éri`êú
-z minutes)r{Údurationr1r2)r³r‘r›rœr�ÚreÚsearchÚgroupr£Úsplitextr¤r5rršrr9ÚshutilÚrmtree)r§r¨r1r­r®r=Ú
chunk_pathÚ
transcriptr²Úmatchr¬Úchunk_indexÚ	start_minÚend_minÚactual_end_minr4rAs                 r%Útranscribe_audio_chunksrÅùsa
€å 7¸ÐI_Ñ `Ô `Ñ€L�+à€IØ!ð#ñ#ˆ
å" :Ñ.Ô.ˆ
õœ×)Ò)¨*Ñ5Ô5ˆÝ”	Ð2°NÑCÔCˆØð	Ø!ŸKšK¨
™NœNˆMݘeŸkšk¨!™nœnÑ-Ô-ˆKˆKõœG×,Ò,¨^Ñ<Ô<¸QÔ?ˆM؈Kð! 1‘_Ð(>Ñ>ˆ	ØÐ 6Ñ6ˆÝ˜W¥s­<Ô+AÀ/Ñ+RÔ+RÑ'SÔ'SÐW\Ñ'\Ñ^Ô^ˆð*Ø(Ð<Ð<¨7Ð<Ð<Ð<Ø&ððˆõ
¨¸hÐGÑGÔGˆØ×Ò˜Ñ"Ô"Ð"Ñ"õ„M�,ÑÔÐàÐr'écó(—t
||
|¦«}|S©
N)
rÅ)r§r¨r1r=s    r%Úcreate_audio_documentrÉ#
s€Ý'¨Ð9OÐQZÑ[Ô[€IØÐr'Ú
video_pathÚ
time_intervalcóâ—t
|¦
«
}|j
}| d
d¦«}|j |¦
«
}t
j t
j |¦
«
¦
«
d}t
j 	t
j 
|¦
«
|›¦«}tj|d¬¦«
tj
|¦
«
}t|dd¦
«
}tdt!|¦
«
|
¦«D]‹}	|	}
t#|	|
zt!|¦
«
¦«}|
›d|›�}t
j 	||›d	�¦«}
tj||
¬
¦« |
d¬¦« ¦«
ŒŒt+d
|›d�¦
«

t-|dd¬¦«}tj|¦
«

|S)Nz.mp4r—rT)
Úexist_okr™r·r¶zs.png)
Ússr–)
ÚvframeszSnapshots saved in rzgà?Úvideo)r¨r1)rr·ÚreplacerŒÚwrite_audiofiler›rœr»r�r…ÚdirnamerŸÚffmpegÚprobeÚfloatr¢r£r¤ÚinputÚoutputÚrunr¦rÉÚremove)rÊrËrÐr·Ú
audio_pathÚ
_Ú
video_nameÚsnapshot_dirrÕr>Ú
start_timeÚend_timeÚinterval_strÚ	frame_imgr=s               r%Úpreprocess_video_datarã(
sÐ
€õ
˜*Ñ%Ô%€EðŒ~€Hð×#Ò# F¨FÑ3Ô3€JØ
Œ×#Ò# JÑ/Ô/€Aõ”×!Ò!¥"¤'×"2Ò"2°:Ñ">Ô">Ñ?Ô?À
ÔB€Jõ”7—<’<¥¤§¢°
Ñ ;Ô ;À
¸_ÑMÔM€LÝ„K� tÐ,Ñ,Ô,Ð,õ
ŒL˜Ñ$Ô$€EÝ�U˜8”_ ZÔ0Ñ1Ô1€Hõ�1•c˜(‘m”m ]Ñ
3Ô
3ð
ð
ˆ
؈
Ý�q˜=Ñ(­#¨h©-¬-Ñ8Ô8ˆð%Ð1Ð1 xÐ1Ð1ˆõ”G—L’L °,Ð/EÐ/EÐ/EÑFÔFˆ	õ
Ü
�: *ð

.ñ

.ô

.ç
ŠV�I qˆVÑ
)Ô
)ß
ŠS‰UŒUˆUˆUõ
Ð