ds_erp_ai/data_ingest.py

import sys, os
sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '..')))
from utils import create_vector_store, save_embedded_data, load_documents_from_directory, process_directory
from loggings.logging_config import logger
import time 


# This module will load in the data, you only need to add the data path to it. 
data_path = './data'

def load_data(data_path: str):
    logger.info(f"Loading data from {data_path}")
    start_time = time.time()
    # logging the start time
    logger.info(f"Start time: {start_time}")
    documents, docs_id, num_pages = load_documents_from_directory(data_path)
    logger.info(f"Data loaded")
    logger.info(f"Creating vector store")
    embed_db = create_vector_store(documents, docs_id, num_pages)
    logger.info(f"Vector store created")
    logger.info(f"Saving the vector store") 
    # saving the embedded data
    save_embedded_data(embed_db)
    logger.info(f"Vector store saved")
    end_time = time.time()
    logger.info(f"End time: {end_time}")
    time_taken = end_time - start_time
    logger.info(f"Time taken: {time_taken}")
    
    print("Vector store created and saved")
    # creating the thumbnails
    logger.info(f"Creating thumbnails")
    status = process_directory(data_path)
    print(f"{status}: Thumbnails created.")
    logger.info(f"Thumbnails created")
    return embed_db


if __name__ == "__main__":
    load_data(data_path)