diff --git a/__pycache__/search.cpython-311.pyc b/__pycache__/search.cpython-311.pyc index 48586e51..483315aa 100644 Binary files a/__pycache__/search.cpython-311.pyc and b/__pycache__/search.cpython-311.pyc differ diff --git a/__pycache__/utils.cpython-311.pyc b/__pycache__/utils.cpython-311.pyc index 0f48701f..543d75da 100644 Binary files a/__pycache__/utils.cpython-311.pyc and b/__pycache__/utils.cpython-311.pyc differ diff --git a/ai_index.py b/ai_index.py new file mode 100644 index 00000000..7a26ed5c --- /dev/null +++ b/ai_index.py @@ -0,0 +1,83 @@ +import sys, os +sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))) +from fastapi import FastAPI, File, UploadFile, BackgroundTasks, HTTPException +from fastapi.middleware.cors import CORSMiddleware +from pathlib import Path +from utils import load_documents_from_directory, create_vector_store, save_embedded_data, process_directory +from pydantic import BaseModel +from search import search_and_summarize +from typing import List + + +app = FastAPI() + + +# Define allowed origins for CORS +origins = [ + "http://localhost", + "http://localhost:8000", + "http://localhost:3000", + # Add other allowed origins here +] + +# Add CORS middleware +app.add_middleware( + CORSMiddleware, + allow_origins=origins, # Allows requests from listed origins + allow_credentials=True, + allow_methods=["*"], # Allows all HTTP methods + allow_headers=["*"], # Allows all headers +) + +# Define the directory where you want to save uploaded files +UPLOAD_DIR = Path("./uploads") + +# Ensure the directory exists +UPLOAD_DIR.mkdir(parents=True, exist_ok=True) + +class SearchRequest(BaseModel): + query: str + +def load_documents(directory: str): + + # loading the documents from the directory + documents, docs_id, num_pages = load_documents_from_directory(directory) + # embedding the documents + embed_db = create_vector_store(documents, docs_id, num_pages) + # saving the embedded data + status = save_embedded_data(embed_db) + # creating the thumbnails + status = process_directory(directory) + + return {"status": "Documents loaded successfully"} + +class SearchRequest(BaseModel): + query: str + +@app.post("/search/") +def search(request: SearchRequest): + + # Perform search using the utility function + results = search_and_summarize(request.query) + + return {"results": results} + +@app.post("/upload/") +async def upload_file(background_tasks: BackgroundTasks, file: UploadFile = File(...)): + + file_location = UPLOAD_DIR/file.filename + + # Save the uploaded file to the specified location + with open(file_location, "wb") as buffer: + buffer.write(await file.read()) + + # Add the load_documents function to the background tasks + background_tasks.add_task(load_documents, str(UPLOAD_DIR)) + + # Return the location of the saved file and inform about the successful upload + return {"message": "Upload successful. Document loading will begin shortly.", "file_location": str(UPLOAD_DIR)} + + +if __name__ == "__main__": + import uvicorn + uvicorn.run(app, host="0.0.0.0", port=8000) diff --git a/index/faiss_index_data/index.faiss b/index/faiss_index/index.faiss similarity index 87% rename from index/faiss_index_data/index.faiss rename to index/faiss_index/index.faiss index 1eeac1e8..22c33fac 100644 Binary files a/index/faiss_index_data/index.faiss and b/index/faiss_index/index.faiss differ diff --git a/index/faiss_index_data/index.pkl b/index/faiss_index/index.pkl similarity index 86% rename from index/faiss_index_data/index.pkl rename to index/faiss_index/index.pkl index 83b140db..299ae14a 100644 Binary files a/index/faiss_index_data/index.pkl and b/index/faiss_index/index.pkl differ diff --git a/loggings/app.log b/loggings/app.log index b0bdd084..5865e0ec 100644 --- a/loggings/app.log +++ b/loggings/app.log @@ -2808,3 +2808,336 @@ To avoid injury, hold up the wheel when unscrewing the bolts. 2024-08-16 17:37:05,637 - INFO - PyTorch version 2.4.0+cu124 available. 2024-08-16 17:37:06,226 - INFO - Load pretrained SentenceTransformer: BAAI/bge-small-en 2024-08-16 17:37:13,187 - INFO - Embedding model loaded +2024-08-16 17:38:12,435 - INFO - Searching for the query +2024-08-16 17:38:12,436 - INFO - Searching for Wirebrush +2024-08-16 17:38:12,436 - INFO - Loading embedded data +2024-08-16 17:38:14,222 - WARNING - C:\Users\timmy_3aupohg\anaconda3\envs\smog_env\Lib\site-packages\transformers\models\bert\modeling_bert.py:439: UserWarning: 1Torch was not compiled with flash attention. (Triggered internally at C:\cb\pytorch_1000000000000\work\aten\src\ATen\native\transformers\cuda\sdp_utils.cpp:555.) + attn_output = torch.nn.functional.scaled_dot_product_attention( + +2024-08-16 17:38:14,372 - INFO - Search completed +2024-08-16 17:38:14,373 - INFO - Search completed +2024-08-16 17:38:14,373 - INFO - Summarizing search results +2024-08-16 17:38:14,375 - INFO - Search results summarized +2024-08-16 20:07:32,804 - INFO - Loading the embedding model +2024-08-16 20:07:41,433 - INFO - PyTorch version 2.4.0+cu124 available. +2024-08-16 20:07:41,857 - INFO - Load pretrained SentenceTransformer: BAAI/bge-small-en +2024-08-16 20:07:49,114 - INFO - Embedding model loaded +2024-08-16 20:10:52,736 - INFO - Loading the embedding model +2024-08-16 20:10:58,470 - INFO - PyTorch version 2.4.0+cu124 available. +2024-08-16 20:10:58,831 - INFO - Load pretrained SentenceTransformer: BAAI/bge-small-en +2024-08-16 20:11:04,768 - INFO - Embedding model loaded +2024-08-16 20:11:44,178 - INFO - Loading the embedding model +2024-08-16 20:11:51,103 - INFO - PyTorch version 2.4.0+cu124 available. +2024-08-16 20:11:51,549 - INFO - Load pretrained SentenceTransformer: BAAI/bge-small-en +2024-08-16 20:11:56,825 - INFO - Embedding model loaded +2024-08-16 20:15:04,657 - INFO - Searching for the query +2024-08-16 20:15:04,657 - INFO - Searching for wirebrush +2024-08-16 20:15:04,657 - INFO - Loading embedded data +2024-08-16 20:16:13,332 - INFO - Loading the embedding model +2024-08-16 20:16:19,225 - INFO - PyTorch version 2.4.0+cu124 available. +2024-08-16 20:16:19,602 - INFO - Load pretrained SentenceTransformer: BAAI/bge-small-en +2024-08-16 20:16:24,953 - INFO - Embedding model loaded +2024-08-16 20:16:51,193 - INFO - Searching for the query +2024-08-16 20:16:51,193 - INFO - Searching for wirebrush +2024-08-16 20:16:51,193 - INFO - Loading embedded data +2024-08-16 20:16:53,308 - WARNING - C:\Users\timmy_3aupohg\anaconda3\envs\smog_env\Lib\site-packages\transformers\models\bert\modeling_bert.py:439: UserWarning: 1Torch was not compiled with flash attention. (Triggered internally at C:\cb\pytorch_1000000000000\work\aten\src\ATen\native\transformers\cuda\sdp_utils.cpp:555.) + attn_output = torch.nn.functional.scaled_dot_product_attention( + +2024-08-16 20:16:53,475 - INFO - Search completed +2024-08-16 20:16:53,475 - INFO - Search completed +2024-08-16 20:16:53,475 - INFO - Summarizing search results +2024-08-16 20:16:53,475 - INFO - Search results summarized +2024-08-16 20:17:37,427 - INFO - Searching for the query +2024-08-16 20:17:37,427 - INFO - Searching for + query IntrospectionQuery { + __schema { + + queryType { name } + mutationType { name } + subscriptionType { name } + types { + ...FullType + } + directives { + name + description + + locations + args { + ...InputValue + } + } + } + } + + fragment FullType on __Type { + kind + name + description + + fields(includeDeprecated: true) { + name + description + args { + ...InputValue + } + type { + ...TypeRef + } + isDeprecated + deprecationReason + } + inputFields { + ...InputValue + } + interfaces { + ...TypeRef + } + enumValues(includeDeprecated: true) { + name + description + isDeprecated + deprecationReason + } + possibleTypes { + ...TypeRef + } + } + + fragment InputValue on __InputValue { + name + description + type { ...TypeRef } + defaultValue + + + } + + fragment TypeRef on __Type { + kind + name + ofType { + kind + name + ofType { + kind + name + ofType { + kind + name + ofType { + kind + name + ofType { + kind + name + ofType { + kind + name + ofType { + kind + name + } + } + } + } + } + } + } + } + +2024-08-16 20:17:37,443 - INFO - Loading embedded data +2024-08-16 20:17:37,511 - INFO - Search completed +2024-08-16 20:17:37,511 - INFO - Search completed +2024-08-16 20:17:37,511 - INFO - Summarizing search results +2024-08-16 20:17:37,527 - INFO - Search results summarized +2024-08-16 20:18:42,832 - INFO - Searching for the query +2024-08-16 20:18:42,832 - INFO - Searching for wirebrush +2024-08-16 20:18:42,832 - INFO - Loading embedded data +2024-08-16 20:18:42,882 - INFO - Search completed +2024-08-16 20:18:42,899 - INFO - Search completed +2024-08-16 20:18:42,899 - INFO - Summarizing search results +2024-08-16 20:18:42,901 - INFO - Search results summarized +2024-08-16 20:22:46,791 - INFO - Loading the embedding model +2024-08-16 20:22:53,018 - INFO - PyTorch version 2.4.0+cu124 available. +2024-08-16 20:22:53,427 - INFO - Load pretrained SentenceTransformer: BAAI/bge-small-en +2024-08-16 20:22:59,566 - INFO - Embedding model loaded +2024-08-16 20:24:21,213 - INFO - Loading the embedding model +2024-08-16 20:24:26,903 - INFO - PyTorch version 2.4.0+cu124 available. +2024-08-16 20:24:27,281 - INFO - Load pretrained SentenceTransformer: BAAI/bge-small-en +2024-08-16 20:24:33,819 - INFO - Embedding model loaded +2024-08-16 20:24:42,819 - INFO - Loading pdf document from uploads\Employee_Attrition_Action_Plan.pdf +2024-08-16 20:24:43,532 - INFO - Summarizing document +2024-08-16 20:24:45,028 - INFO - HTTP Request: POST https://api.groq.com/openai/v1/chat/completions "HTTP/1.1 200 OK" +2024-08-16 20:24:45,559 - INFO - Creating vector store +2024-08-16 20:24:46,277 - WARNING - C:\Users\timmy_3aupohg\anaconda3\envs\smog_env\Lib\site-packages\transformers\models\bert\modeling_bert.py:439: UserWarning: 1Torch was not compiled with flash attention. (Triggered internally at C:\cb\pytorch_1000000000000\work\aten\src\ATen\native\transformers\cuda\sdp_utils.cpp:555.) + attn_output = torch.nn.functional.scaled_dot_product_attention( + +2024-08-16 20:24:46,583 - INFO - Vector store created +2024-08-16 20:24:46,583 - INFO - Saving the vector store +2024-08-16 20:26:45,690 - INFO - Loading the embedding model +2024-08-16 20:26:51,361 - INFO - PyTorch version 2.4.0+cu124 available. +2024-08-16 20:26:51,689 - INFO - Load pretrained SentenceTransformer: BAAI/bge-small-en +2024-08-16 20:26:57,758 - INFO - Embedding model loaded +2024-08-16 20:27:35,992 - INFO - Loading pdf document from uploads\Employee_Attrition_Action_Plan.pdf +2024-08-16 20:27:36,682 - INFO - Summarizing document +2024-08-16 20:27:37,718 - INFO - HTTP Request: POST https://api.groq.com/openai/v1/chat/completions "HTTP/1.1 200 OK" +2024-08-16 20:27:38,243 - INFO - Creating vector store +2024-08-16 20:27:39,588 - WARNING - C:\Users\timmy_3aupohg\anaconda3\envs\smog_env\Lib\site-packages\transformers\models\bert\modeling_bert.py:439: UserWarning: 1Torch was not compiled with flash attention. (Triggered internally at C:\cb\pytorch_1000000000000\work\aten\src\ATen\native\transformers\cuda\sdp_utils.cpp:555.) + attn_output = torch.nn.functional.scaled_dot_product_attention( + +2024-08-16 20:27:39,850 - INFO - Vector store created +2024-08-16 20:27:39,851 - INFO - Saving the vector store +2024-08-16 20:28:37,928 - INFO - Loading the embedding model +2024-08-16 20:28:43,166 - INFO - PyTorch version 2.4.0+cu124 available. +2024-08-16 20:28:43,498 - INFO - Load pretrained SentenceTransformer: BAAI/bge-small-en +2024-08-16 20:28:49,596 - INFO - Embedding model loaded +2024-08-16 20:30:50,050 - INFO - Loading the embedding model +2024-08-16 20:30:55,432 - INFO - PyTorch version 2.4.0+cu124 available. +2024-08-16 20:30:55,755 - INFO - Load pretrained SentenceTransformer: BAAI/bge-small-en +2024-08-16 20:31:01,549 - INFO - Embedding model loaded +2024-08-16 20:32:24,048 - INFO - Loading the embedding model +2024-08-16 20:32:29,504 - INFO - PyTorch version 2.4.0+cu124 available. +2024-08-16 20:32:29,834 - INFO - Load pretrained SentenceTransformer: BAAI/bge-small-en +2024-08-16 20:32:35,855 - INFO - Embedding model loaded +2024-08-16 20:32:49,136 - INFO - Loading pdf document from uploads\Employee_Attrition_Action_Plan.pdf +2024-08-16 20:32:49,964 - INFO - Summarizing document +2024-08-16 20:32:50,973 - INFO - HTTP Request: POST https://api.groq.com/openai/v1/chat/completions "HTTP/1.1 200 OK" +2024-08-16 20:32:51,487 - INFO - Creating vector store +2024-08-16 20:32:52,933 - WARNING - C:\Users\timmy_3aupohg\anaconda3\envs\smog_env\Lib\site-packages\transformers\models\bert\modeling_bert.py:439: UserWarning: 1Torch was not compiled with flash attention. (Triggered internally at C:\cb\pytorch_1000000000000\work\aten\src\ATen\native\transformers\cuda\sdp_utils.cpp:555.) + attn_output = torch.nn.functional.scaled_dot_product_attention( + +2024-08-16 20:32:53,278 - INFO - Vector store created +2024-08-16 20:32:53,278 - INFO - Saving the vector store +2024-08-16 20:32:53,278 - INFO - Saving embeddings +2024-08-16 20:32:53,278 - INFO - Vector store saved +2024-08-16 20:32:53,278 - INFO - Saving embeddings +2024-08-16 20:32:53,278 - INFO - Creating thumbnail for uploads\Employee_Attrition_Action_Plan.pdf +2024-08-16 20:36:58,132 - INFO - Searching for the query +2024-08-16 20:36:58,132 - INFO - Searching for Implement Career Development Programs +2024-08-16 20:36:58,133 - INFO - Loading embedded data +2024-08-16 20:36:59,422 - INFO - Search completed +2024-08-16 20:36:59,422 - INFO - Search completed +2024-08-16 20:36:59,422 - INFO - Summarizing search results +2024-08-16 20:36:59,422 - INFO - Search results summarized +2024-08-16 20:40:29,336 - INFO - Loading the embedding model +2024-08-16 20:40:37,718 - INFO - PyTorch version 2.4.0+cu124 available. +2024-08-16 20:40:38,253 - INFO - Load pretrained SentenceTransformer: BAAI/bge-small-en +2024-08-16 20:40:44,320 - INFO - Embedding model loaded +2024-08-16 20:40:59,981 - INFO - Searching for the query +2024-08-16 20:40:59,984 - INFO - Searching for Implement Career Development Programs +2024-08-16 20:40:59,985 - INFO - Loading embedded data +2024-08-16 20:41:01,671 - WARNING - C:\Users\timmy_3aupohg\anaconda3\envs\smog_env\Lib\site-packages\transformers\models\bert\modeling_bert.py:439: UserWarning: 1Torch was not compiled with flash attention. (Triggered internally at C:\cb\pytorch_1000000000000\work\aten\src\ATen\native\transformers\cuda\sdp_utils.cpp:555.) + attn_output = torch.nn.functional.scaled_dot_product_attention( + +2024-08-16 20:41:01,754 - INFO - Search completed +2024-08-16 20:41:01,754 - INFO - Search completed +2024-08-16 20:41:01,754 - INFO - Summarizing search results +2024-08-16 20:41:01,754 - INFO - Search results summarized +2024-08-16 20:42:39,142 - INFO - Loading pdf document from uploads\Employee_Attrition_Action_Plan.pdf +2024-08-16 20:42:39,142 - INFO - Loading docx document from uploads\How to change rear windshield wipers on TOYOTA Corolla.docx +2024-08-16 20:42:39,504 - INFO - Creating documents from text +2024-08-16 20:42:39,998 - INFO - Summarizing document +2024-08-16 20:42:41,011 - INFO - HTTP Request: POST https://api.groq.com/openai/v1/chat/completions "HTTP/1.1 200 OK" +2024-08-16 20:42:41,534 - INFO - Summarizing document +2024-08-16 20:42:42,011 - INFO - HTTP Request: POST https://api.groq.com/openai/v1/chat/completions "HTTP/1.1 200 OK" +2024-08-16 20:42:42,536 - INFO - Creating vector store +2024-08-16 20:42:44,194 - INFO - Vector store created +2024-08-16 20:42:44,194 - INFO - Saving the vector store +2024-08-16 20:42:44,194 - INFO - Saving embeddings +2024-08-16 20:42:44,194 - INFO - Vector store saved +2024-08-16 20:42:44,194 - INFO - Saving embeddings +2024-08-16 20:42:44,194 - INFO - Creating thumbnail for uploads\Employee_Attrition_Action_Plan.pdf +2024-08-16 20:42:44,212 - INFO - Creating thumbnail for uploads\How to change rear windshield wipers on TOYOTA Corolla.docx +2024-08-16 20:42:55,518 - INFO - Searching for the query +2024-08-16 20:42:55,518 - INFO - Searching for wirebrush +2024-08-16 20:42:55,518 - INFO - Loading embedded data +2024-08-16 20:42:55,581 - INFO - Search completed +2024-08-16 20:42:55,581 - INFO - Search completed +2024-08-16 20:42:55,581 - INFO - Summarizing search results +2024-08-16 20:42:55,581 - INFO - Search results summarized +2024-08-16 20:43:13,345 - INFO - Searching for the query +2024-08-16 20:43:13,345 - INFO - Searching for Implement Career Development Programs +2024-08-16 20:43:13,345 - INFO - Loading embedded data +2024-08-16 20:43:13,362 - INFO - Search completed +2024-08-16 20:43:13,362 - INFO - Search completed +2024-08-16 20:43:13,362 - INFO - Summarizing search results +2024-08-16 20:43:13,362 - INFO - Search results summarized +2024-08-16 20:43:37,481 - INFO - Searching for the query +2024-08-16 20:43:37,481 - INFO - Searching for toyota corolla +2024-08-16 20:43:37,481 - INFO - Loading embedded data +2024-08-16 20:43:37,497 - INFO - Search completed +2024-08-16 20:43:37,497 - INFO - Search completed +2024-08-16 20:43:37,497 - INFO - Summarizing search results +2024-08-16 20:43:37,497 - INFO - Search results summarized +2024-08-16 20:45:57,119 - INFO - Loading pdf document from uploads\corolla-2020-toyota-owners-manual.pdf +2024-08-16 20:45:57,119 - INFO - Loading pdf document from uploads\Employee_Attrition_Action_Plan.pdf +2024-08-16 20:45:57,136 - INFO - Loading docx document from uploads\How to change rear windshield wipers on TOYOTA Corolla.docx +2024-08-16 20:45:57,495 - INFO - Creating documents from text +2024-08-16 20:46:44,699 - INFO - Summarizing document +2024-08-16 20:46:45,723 - INFO - HTTP Request: POST https://api.groq.com/openai/v1/chat/completions "HTTP/1.1 200 OK" +2024-08-16 20:46:46,242 - INFO - Summarizing document +2024-08-16 20:46:46,879 - INFO - HTTP Request: POST https://api.groq.com/openai/v1/chat/completions "HTTP/1.1 200 OK" +2024-08-16 20:46:47,382 - INFO - Summarizing document +2024-08-16 20:46:47,888 - INFO - HTTP Request: POST https://api.groq.com/openai/v1/chat/completions "HTTP/1.1 200 OK" +2024-08-16 20:46:48,402 - INFO - Creating vector store +2024-08-16 20:46:58,675 - INFO - Searching for the query +2024-08-16 20:46:58,675 - INFO - Searching for Manual for toyota corolla +2024-08-16 20:46:58,675 - INFO - Loading embedded data +2024-08-16 20:46:59,733 - INFO - Search completed +2024-08-16 20:46:59,749 - INFO - Search completed +2024-08-16 20:46:59,749 - INFO - Summarizing search results +2024-08-16 20:46:59,749 - INFO - Search results summarized +2024-08-16 20:47:07,643 - INFO - Vector store created +2024-08-16 20:47:07,643 - INFO - Saving the vector store +2024-08-16 20:47:07,643 - INFO - Saving embeddings +2024-08-16 20:47:07,643 - INFO - Vector store saved +2024-08-16 20:47:07,643 - INFO - Saving embeddings +2024-08-16 20:47:07,659 - INFO - Creating thumbnail for uploads\corolla-2020-toyota-owners-manual.pdf +2024-08-16 20:47:07,674 - INFO - Creating thumbnail for uploads\Employee_Attrition_Action_Plan.pdf +2024-08-16 20:47:07,674 - INFO - Creating thumbnail for uploads\How to change rear windshield wipers on TOYOTA Corolla.docx +2024-08-16 20:48:03,817 - INFO - Searching for the query +2024-08-16 20:48:03,818 - INFO - Searching for Manual for toyota corolla +2024-08-16 20:48:03,818 - INFO - Loading embedded data +2024-08-16 20:48:04,276 - INFO - Search completed +2024-08-16 20:48:04,276 - INFO - Search completed +2024-08-16 20:48:04,276 - INFO - Summarizing search results +2024-08-16 20:48:04,276 - INFO - Search results summarized +2024-08-16 20:48:14,034 - INFO - Searching for the query +2024-08-16 20:48:14,044 - INFO - Searching for Manual for toyota corolla +2024-08-16 20:48:14,044 - INFO - Loading embedded data +2024-08-16 20:48:14,067 - INFO - Search completed +2024-08-16 20:48:14,067 - INFO - Search completed +2024-08-16 20:48:14,074 - INFO - Summarizing search results +2024-08-16 20:48:14,074 - INFO - Search results summarized +2024-08-16 21:32:02,645 - INFO - Searching for the query +2024-08-16 21:32:02,645 - INFO - Searching for Manual Of toyota corolla +2024-08-16 21:32:02,645 - INFO - Loading embedded data +2024-08-16 21:32:02,795 - INFO - Search completed +2024-08-16 21:32:02,799 - INFO - Search completed +2024-08-16 21:32:02,800 - INFO - Summarizing search results +2024-08-16 21:32:02,804 - INFO - Search results summarized +2024-08-16 21:33:39,365 - INFO - Transcribing audio chunks from uploads\audio-2.mp3 +2024-08-16 21:33:39,373 - INFO - Splitting audio file uploads\audio-2.mp3 by duration +2024-08-16 21:33:39,378 - INFO - Loading pdf document from uploads\corolla-2020-toyota-owners-manual.pdf +2024-08-16 21:33:39,404 - INFO - Loading pdf document from uploads\Employee_Attrition_Action_Plan.pdf +2024-08-16 21:33:39,411 - INFO - Loading docx document from uploads\How to change rear windshield wipers on TOYOTA Corolla.docx +2024-08-16 21:33:39,938 - INFO - Creating documents from text +2024-08-16 21:34:58,915 - INFO - Transcribing audio file audio-2_chunks/audio-2_chunk1.mp3 +2024-08-16 21:35:15,275 - INFO - HTTP Request: POST https://api.groq.com/openai/v1/audio/translations "HTTP/1.1 200 OK" +2024-08-16 21:35:18,193 - INFO - Transcribing audio file audio-2_chunks/audio-2_chunk2.mp3 +2024-08-16 21:35:28,536 - INFO - HTTP Request: POST https://api.groq.com/openai/v1/audio/translations "HTTP/1.1 200 OK" +2024-08-16 21:35:31,402 - INFO - Transcribing audio file audio-2_chunks/audio-2_chunk3.mp3 +2024-08-16 21:35:38,932 - INFO - HTTP Request: POST https://api.groq.com/openai/v1/audio/translations "HTTP/1.1 200 OK" +2024-08-16 21:35:43,996 - INFO - Transcribing audio file audio-2_chunks/audio-2_chunk4.mp3 +2024-08-16 21:36:00,061 - INFO - HTTP Request: POST https://api.groq.com/openai/v1/audio/translations "HTTP/1.1 200 OK" +2024-08-16 21:36:02,748 - INFO - Transcribing audio file audio-2_chunks/audio-2_chunk5.mp3 +2024-08-16 21:36:09,913 - INFO - HTTP Request: POST https://api.groq.com/openai/v1/audio/translations "HTTP/1.1 200 OK" +2024-08-16 21:36:12,705 - INFO - Transcribing audio file audio-2_chunks/audio-2_chunk6.mp3 +2024-08-16 21:36:21,767 - INFO - HTTP Request: POST https://api.groq.com/openai/v1/audio/translations "HTTP/1.1 200 OK" +2024-08-16 21:36:25,045 - INFO - Transcribing audio file audio-2_chunks/audio-2_chunk7.mp3 +2024-08-16 21:36:32,036 - INFO - HTTP Request: POST https://api.groq.com/openai/v1/audio/translations "HTTP/1.1 200 OK" +2024-08-16 21:36:37,503 - INFO - Summarizing document +2024-08-16 21:36:39,003 - INFO - HTTP Request: POST https://api.groq.com/openai/v1/chat/completions "HTTP/1.1 200 OK" +2024-08-16 21:36:39,512 - INFO - Summarizing document +2024-08-16 21:36:40,128 - INFO - HTTP Request: POST https://api.groq.com/openai/v1/chat/completions "HTTP/1.1 200 OK" diff --git a/main.py b/main.py index 0187639c..3cbb0f22 100644 --- a/main.py +++ b/main.py @@ -31,7 +31,7 @@ app.add_middleware( class SearchRequest(BaseModel): query: str -@app.get("/load_documents") +@app.post("/load_documents") def load_documents(directory: str): # loading the documents from the directory diff --git a/search.py b/search.py index 275e1eb8..76c3a994 100644 --- a/search.py +++ b/search.py @@ -12,7 +12,7 @@ def get_data_description(data_path): data_name = data_path.split('/')[-1].split('\\')[-1].split('.')[0] # print(data_name) # open the data.json file - with open('data/data.json') as f: + with open('uploads/data.json') as f: data = json.load(f) existing_data = data.keys() if data_name in existing_data: @@ -25,10 +25,10 @@ def get_data_thumbnail(data_path, timestamp = None): # ensuring no // or / or extension is present file_name = data_path.split('/')[-1].split('\\')[-1].split('.')[0] # first check is to see if the file_name has a .png image in the thumbnail folder - if os.path.exists(f'data/thumbnails/{file_name}.png'): - return f'data/thumbnails/{file_name}.png' + if os.path.exists(f'uploads/thumbnails/{file_name}.png'): + return f'uploads/thumbnails/{file_name}.png' # the second check is to see if we have a folder with this file_name - elif os.path.exists(f'data/{file_name}'): + elif os.path.exists(f'uploads/{file_name}'): # so now we want to access the first timestamp if timestamp: first = timestamp[0] @@ -40,8 +40,8 @@ def get_data_thumbnail(data_path, timestamp = None): # bringing them together image_file = f"{start}-{end}s.png" # niw checkin if the file exists - if os.path.exists(f'data/{file_name}/{image_file}'): - return f'data/{file_name}/{image_file}' + if os.path.exists(f'uploads/{file_name}/{image_file}'): + return f'uploads/{file_name}/{image_file}' def summarize_doc_search(data): summary = {} diff --git a/uploads/Employee_Attrition_Action_Plan.pdf b/uploads/Employee_Attrition_Action_Plan.pdf new file mode 100644 index 00000000..4e6e6056 Binary files /dev/null and b/uploads/Employee_Attrition_Action_Plan.pdf differ diff --git a/uploads/How to change rear windshield wipers on TOYOTA Corolla.docx b/uploads/How to change rear windshield wipers on TOYOTA Corolla.docx new file mode 100644 index 00000000..4b2dd1a2 Binary files /dev/null and b/uploads/How to change rear windshield wipers on TOYOTA Corolla.docx differ diff --git a/uploads/audio-2.mp3 b/uploads/audio-2.mp3 new file mode 100644 index 00000000..f31323ea Binary files /dev/null and b/uploads/audio-2.mp3 differ diff --git a/uploads/corolla-2020-toyota-owners-manual.pdf b/uploads/corolla-2020-toyota-owners-manual.pdf new file mode 100644 index 00000000..ec5d8568 Binary files /dev/null and b/uploads/corolla-2020-toyota-owners-manual.pdf differ diff --git a/uploads/data.json b/uploads/data.json new file mode 100644 index 00000000..c6c5fea2 --- /dev/null +++ b/uploads/data.json @@ -0,0 +1 @@ +{"Employee_Attrition_Action_Plan": {"doc_id": "18ab2b5127504921b6f316475dbf7c69", "num_pages": 3, "doc_summary": "This document is about an Action Plan to Prevent Employee Attrition, which outlines five strategies to reduce employee turnover and improve job satisfaction. The plan includes:\n\n* Conducting comprehensive exit interviews to identify patterns and common issues\n* Implementing career development programs to provide employees with a clear career path\n* Enhancing compensation and benefits to ensure they are competitive\n* Fostering a positive work environment through diversity, equity, and inclusion initiatives\n* Improving work-life balance by promoting flexibility and reducing burnout"}, "How to change rear windshield wipers on TOYOTA Corolla": {"doc_id": "ee5616e1611d43f3bc1896a2da892c64", "num_pages": 6, "doc_summary": "This document is about a replacement guide for the rear windshield wipers on a TOYOTA Corolla VI Liftback (E90) model. The guide provides instructions on how to change the rear windshield wipers on various trim levels of the vehicle, including 1.6, 1.8 D, 1.3 i, 1.6 GTI, 1.6 4WD, 1.3, 1.4, and 1.6 EFi"}, "corolla-2020-toyota-owners-manual": {"doc_id": "b3d96d6eb1ab452e9b2c8228b15cda13", "num_pages": 588, "doc_summary": "This document is about the user manual for a Toyota Corolla, providing information and instructions on various aspects of the vehicle, including safety and security, vehicle status, driving operations, interior features, maintenance, and troubleshooting. The manual covers topics such as child seat installation, theft deterrent systems, reading driving-related information, operating the Entune audio system, and caring for the vehicle's interior and exterior. It also includes information on reporting safety defects and provides instructions for Canadian owners on seat belt and SRS air"}, "audio-2": {"doc_id": "5216621677ff4c7ca7c9029907d41e81", "num_pages": 7, "doc_summary": "This document is about a tutorial on how to change the oil in a car or truck, specifically geared towards beginners who have never done an oil change before. The video covers the necessary tools and materials needed, including a socket set, gloves, paper towels, oil and filter, oil filter wrench, oil catch pan, funnel, and breaker bar. The tutorial also explains how to find the correct oil and filter for the vehicle, and how to jack up the vehicle to access the oil drain plug. The"}} \ No newline at end of file diff --git a/uploads/thumbnails/Employee_Attrition_Action_Plan.png b/uploads/thumbnails/Employee_Attrition_Action_Plan.png new file mode 100644 index 00000000..9bf177a9 Binary files /dev/null and b/uploads/thumbnails/Employee_Attrition_Action_Plan.png differ diff --git a/uploads/thumbnails/How to change rear windshield wipers on TOYOTA Corolla.png b/uploads/thumbnails/How to change rear windshield wipers on TOYOTA Corolla.png new file mode 100644 index 00000000..bae53732 Binary files /dev/null and b/uploads/thumbnails/How to change rear windshield wipers on TOYOTA Corolla.png differ diff --git a/uploads/thumbnails/audio-2.png b/uploads/thumbnails/audio-2.png new file mode 100644 index 00000000..654a9d9d Binary files /dev/null and b/uploads/thumbnails/audio-2.png differ diff --git a/uploads/thumbnails/corolla-2020-toyota-owners-manual.png b/uploads/thumbnails/corolla-2020-toyota-owners-manual.png new file mode 100644 index 00000000..f3548546 Binary files /dev/null and b/uploads/thumbnails/corolla-2020-toyota-owners-manual.png differ diff --git a/utils.py b/utils.py index b7250ab0..592029b4 100644 --- a/utils.py +++ b/utils.py @@ -405,15 +405,15 @@ def doc_summarizer(document_page: list) -> str: #-----------------------------------------------------OTHERS-------------------------------------------------------------- -def save_embedded_data(embeddings, key="data"): +def save_embedded_data(embeddings, path = "index/faiss_index",): logger.info(f"Saving embeddings") - embeddings.save_local(f"index/faiss_index_{key}") + embeddings.save_local(f"index/faiss_index") print("Embeddings saved") return 'saved' -def load_embedded_data(embeddings=embeddings, key="data"): +def load_embedded_data(embeddings=embeddings, path = "index/faiss_index"): logger.info(f"Loading embedded data") - embed_db = FAISS.load_local(f"index/faiss_index_{key}", embeddings, allow_dangerous_deserialization=True) + embed_db = FAISS.load_local(f"index/faiss_index", embeddings, allow_dangerous_deserialization=True) return embed_db #-----------------------------------------------------Data Loading Process---------------------------------------------------- @@ -522,7 +522,7 @@ def create_vector_store(documents: list, docs_id: list, num_pages: list): logger.info(f"Vector store created") logger.info(f"Saving the vector store") # saving the vector store automatically - save_embedded_data(vector_store, key="data") + save_embedded_data(vector_store) logger.info(f"Vector store saved") return vector_store