last commi

This commit is contained in:
timothyafolami
2024-08-16 21:39:28 +01:00
parent 83ecbded4b
commit ddd0dda276
18 changed files with 429 additions and 12 deletions
+5 -5
View File
@@ -405,15 +405,15 @@ def doc_summarizer(document_page: list) -> str:
#-----------------------------------------------------OTHERS--------------------------------------------------------------
def save_embedded_data(embeddings, key="data"):
def save_embedded_data(embeddings, path = "index/faiss_index",):
logger.info(f"Saving embeddings")
embeddings.save_local(f"index/faiss_index_{key}")
embeddings.save_local(f"index/faiss_index")
print("Embeddings saved")
return 'saved'
def load_embedded_data(embeddings=embeddings, key="data"):
def load_embedded_data(embeddings=embeddings, path = "index/faiss_index"):
logger.info(f"Loading embedded data")
embed_db = FAISS.load_local(f"index/faiss_index_{key}", embeddings, allow_dangerous_deserialization=True)
embed_db = FAISS.load_local(f"index/faiss_index", embeddings, allow_dangerous_deserialization=True)
return embed_db
#-----------------------------------------------------Data Loading Process----------------------------------------------------
@@ -522,7 +522,7 @@ def create_vector_store(documents: list, docs_id: list, num_pages: list):
logger.info(f"Vector store created")
logger.info(f"Saving the vector store")
# saving the vector store automatically
save_embedded_data(vector_store, key="data")
save_embedded_data(vector_store)
logger.info(f"Vector store saved")
return vector_store