timothyafolami
40 lines
1.4 KiB
Python
40 lines
1.4 KiB
Python
import sys, os
|
|
sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '..')))
|
|
from utils import create_vector_store, save_embedded_data, load_documents_from_directory, process_directory
|
|
from loggings.logging_config import logger
|
|
import time
|
|
|
|
|
|
# This module will load in the data, you only need to add the data path to it.
|
|
data_path = './data'
|
|
|
|
def load_data(data_path: str):
|
|
logger.info(f"Loading data from {data_path}")
|
|
start_time = time.time()
|
|
# logging the start time
|
|
logger.info(f"Start time: {start_time}")
|
|
documents, docs_id, num_pages = load_documents_from_directory(data_path)
|
|
logger.info(f"Data loaded")
|
|
logger.info(f"Creating vector store")
|
|
embed_db = create_vector_store(documents, docs_id, num_pages)
|
|
logger.info(f"Vector store created")
|
|
logger.info(f"Saving the vector store")
|
|
# saving the embedded data
|
|
save_embedded_data(embed_db)
|
|
logger.info(f"Vector store saved")
|
|
end_time = time.time()
|
|
logger.info(f"End time: {end_time}")
|
|
time_taken = end_time - start_time
|
|
logger.info(f"Time taken: {time_taken}")
|
|
|
|
print("Vector store created and saved")
|
|
# creating the thumbnails
|
|
logger.info(f"Creating thumbnails")
|
|
status = process_directory(data_path)
|
|
print(f"{status}: Thumbnails created.")
|
|
logger.info(f"Thumbnails created")
|
|
return embed_db
|
|
|
|
|
|
if __name__ == "__main__":
|
|
load_data(data_path) |