timothyafolami
597 lines
47 KiB
Plaintext
597 lines
47 KiB
Plaintext
{
|
||
"cells": [
|
||
{
|
||
"cell_type": "markdown",
|
||
"metadata": {},
|
||
"source": [
|
||
"## Libs import"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": 39,
|
||
"metadata": {},
|
||
"outputs": [],
|
||
"source": [
|
||
"from langchain_community.embeddings import HuggingFaceBgeEmbeddings\n",
|
||
"from langchain_text_splitters import RecursiveCharacterTextSplitter\n",
|
||
"import faiss\n",
|
||
"from langchain_community.docstore.in_memory import InMemoryDocstore\n",
|
||
"from langchain_community.vectorstores import FAISS\n",
|
||
"from langchain_community.document_loaders import PyPDFLoader\n",
|
||
"from langchain_community.document_loaders import TextLoader\n",
|
||
"from langchain_community.document_loaders import Docx2txtLoader\n",
|
||
"from langchain_community.document_loaders import UnstructuredWordDocumentLoader\n",
|
||
"from uuid import uuid4\n",
|
||
"import numpy as np\n",
|
||
"from langchain_core.documents import Document\n",
|
||
"import os\n",
|
||
"import json"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "markdown",
|
||
"metadata": {},
|
||
"source": [
|
||
"## Loading the embeddings model"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": 2,
|
||
"metadata": {},
|
||
"outputs": [
|
||
{
|
||
"name": "stderr",
|
||
"output_type": "stream",
|
||
"text": [
|
||
"c:\\Users\\timmy_3aupohg\\anaconda3\\envs\\smog_env\\Lib\\site-packages\\sentence_transformers\\cross_encoder\\CrossEncoder.py:11: TqdmExperimentalWarning: Using `tqdm.autonotebook.tqdm` in notebook mode. Use `tqdm.tqdm` instead to force console mode (e.g. in jupyter console)\n",
|
||
" from tqdm.autonotebook import tqdm, trange\n"
|
||
]
|
||
}
|
||
],
|
||
"source": [
|
||
"# Initialize embedding\n",
|
||
"model_name = \"BAAI/bge-small-en\"\n",
|
||
"model_kwargs = {\"device\": \"cuda\"} #can also be cpu\n",
|
||
"encode_kwargs = {\"normalize_embeddings\": True}\n",
|
||
"embeddings = HuggingFaceBgeEmbeddings(\n",
|
||
" model_name=model_name, model_kwargs=model_kwargs, encode_kwargs=encode_kwargs\n",
|
||
" )"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "markdown",
|
||
"metadata": {},
|
||
"source": [
|
||
"## Pdf loading"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": 38,
|
||
"metadata": {},
|
||
"outputs": [],
|
||
"source": [
|
||
"# this function is meant to create a document for non paginated text data\n",
|
||
"def create_documents(doc):\n",
|
||
" text = doc[0].page_content\n",
|
||
" metadata = doc[0].metadata\n",
|
||
" text_splitter = RecursiveCharacterTextSplitter(\n",
|
||
" chunk_size=1000,\n",
|
||
" chunk_overlap=10,\n",
|
||
" length_function=len,\n",
|
||
" is_separator_regex=False,\n",
|
||
" )\n",
|
||
" docs = text_splitter.create_documents([text])\n",
|
||
" # converting the text into documents\n",
|
||
" documents = []\n",
|
||
" for i, chunk in enumerate(docs):\n",
|
||
" # Increment page number based on the chunk index\n",
|
||
" doc_metadata = metadata.copy()\n",
|
||
" doc_metadata['page'] = i # Assign page number based on chunk index\n",
|
||
" document = Document(page_content=chunk.page_content, metadata=doc_metadata)\n",
|
||
" documents.append(document)\n",
|
||
" return documents\n",
|
||
"\n",
|
||
"\n",
|
||
"def load_txt_document(document_path):\n",
|
||
" try:\n",
|
||
" txt_doc = TextLoader(document_path)\n",
|
||
" text = txt_doc.load()\n",
|
||
" # implementig document splitting\n",
|
||
" docs = create_documents(text)\n",
|
||
" return docs\n",
|
||
" except:\n",
|
||
" raise ValueError(f\"Error loading -- {document_path}\")\n",
|
||
" \n",
|
||
" \n",
|
||
"def load_docx_document(document_path):\n",
|
||
" try:\n",
|
||
" docx_doc = Docx2txtLoader(document_path)\n",
|
||
" text = docx_doc.load()\n",
|
||
" # implementig document splitting\n",
|
||
" docs = create_documents(text)\n",
|
||
" return docs\n",
|
||
" except:\n",
|
||
" raise ValueError(f\"Error loading -- {document_path}\")\n",
|
||
"\n",
|
||
" \n",
|
||
"# creating a function that checks the document type and loads the document\n",
|
||
"def load_pdf_document(document_path):\n",
|
||
" try:\n",
|
||
" pdf_doc = PyPDFLoader(document_path)\n",
|
||
" pages = pdf_doc.load_and_split()\n",
|
||
" return pages\n",
|
||
" except:\n",
|
||
" raise ValueError(f\"Error loading -- {document_path}\")\n",
|
||
" \n",
|
||
"\n",
|
||
"\n",
|
||
"\n",
|
||
"# A general function that loads textual documents\n",
|
||
"def load_document(document_path):\n",
|
||
" if document_path.endswith(\".pdf\"):\n",
|
||
" return load_pdf_document(document_path)\n",
|
||
" elif document_path.endswith(\".txt\"):\n",
|
||
" return load_txt_document(document_path)\n",
|
||
" elif document_path.endswith(\".docx\"):\n",
|
||
" return load_docx_document(document_path)\n",
|
||
" else:\n",
|
||
" raise ValueError(f\"Unsupported document type for {document_path}\")\n",
|
||
"\n",
|
||
"\n",
|
||
"\n",
|
||
"def save_embedded_data(embeddings, key=\"data\"):\n",
|
||
" embeddings.save_local(f\"vec-db/index/faiss_index_{key}\")\n",
|
||
" print(\"Embeddings saved\")\n",
|
||
"\n",
|
||
"def load_embedded_data(embeddings, key=\"data\"):\n",
|
||
" embed_db = FAISS.load_local(f\"vec-db/index/faiss_index_{key}\", embeddings, allow_dangerous_deserialization=True)\n",
|
||
" return embed_db"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": 66,
|
||
"metadata": {},
|
||
"outputs": [],
|
||
"source": [
|
||
"# creating a function to load all documents from a directory. \n",
|
||
"def load_documents_from_directory(directory_path: str):\n",
|
||
" text_doc = ['pdf', 'txt', 'docx', 'doc', 'md']\n",
|
||
" image_doc = ['jpg', 'jpeg', 'png', 'gif', 'bmp']\n",
|
||
" audio_doc = ['mp3', 'wav', 'flac', 'ogg', 'm4a']\n",
|
||
" video_doc = ['mp4', 'avi', 'mkv', 'flv', 'mov']\n",
|
||
" \n",
|
||
" # accessing the name of the files in the directory\n",
|
||
" files = os.listdir(directory_path)\n",
|
||
" # creating a list to store the documents\n",
|
||
" documents = []\n",
|
||
" # another list for the document names \n",
|
||
" doc_names = []\n",
|
||
" # counting the number of pages in the document\n",
|
||
" num_pages= []\n",
|
||
" # iterating through the files in the directory\n",
|
||
" for file in files: \n",
|
||
" # updating the path\n",
|
||
" path = os.path.join(directory_path, file)\n",
|
||
" # getting the file extension and doc name\n",
|
||
" doc_name, extension = file.split('.')[0] , file.split('.')[-1]\n",
|
||
" # checking if the file is a text document\n",
|
||
" if extension in text_doc:\n",
|
||
" # loading the document\n",
|
||
" doc = load_document(path)\n",
|
||
" # appending the document to the documents list\n",
|
||
" documents.append(doc)\n",
|
||
" # appending the number of pages in the document\n",
|
||
" num_pages.append(len(doc))\n",
|
||
" # adding the document name to the doc_names list\n",
|
||
" doc_names.append(doc_name)\n",
|
||
" print(f\"Document {doc_name} loaded\")\n",
|
||
" \n",
|
||
" # so we need to create a document id for each document\n",
|
||
" docs_id = [uuid4().hex for i in range(len(documents))]\n",
|
||
" # creating a json file to store the documents, checking if it exists then open it, else create it\n",
|
||
" json_file = f\"{directory_path}/documents.json\"\n",
|
||
" if os.path.exists(json_file):\n",
|
||
" with open(json_file, 'r') as f:\n",
|
||
" data = json.load(f)\n",
|
||
" data['doc_names'] = doc_names\n",
|
||
" data['docs_id'] = docs_id\n",
|
||
" data['num_pages'] = num_pages\n",
|
||
" with open(json_file, 'w') as f:\n",
|
||
" json.dump(data, f)\n",
|
||
" else:\n",
|
||
" data = {'doc_names': doc_names, 'docs_id': docs_id, 'num_pages': num_pages}\n",
|
||
" with open(json_file, 'w') as f:\n",
|
||
" json.dump(data, f)\n",
|
||
" \n",
|
||
" # returning the documents, and doc ids\n",
|
||
" return documents, docs_id, num_pages"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": 67,
|
||
"metadata": {},
|
||
"outputs": [
|
||
{
|
||
"name": "stdout",
|
||
"output_type": "stream",
|
||
"text": [
|
||
"Document Car-Repair-Receipt-repair loaded\n",
|
||
"Document Car-Repair-Receipt-service loaded\n",
|
||
"Document Car-Repair-Receipt-tire loaded\n",
|
||
"Document Car-Repair-Receipt-tuning loaded\n",
|
||
"Document Car-Repair-Receipt-wash loaded\n",
|
||
"Document corolla-2020-toyota-owners-manual loaded\n",
|
||
"Document How to change engine oil and filter on TOYOTA Corolla loaded\n",
|
||
"Document How to change front brake pads on TOYOTA Corolla loaded\n",
|
||
"Document How to change rear windshield wipers on TOYOTA Corolla loaded\n",
|
||
"Document How to change spark plugs on TOYOTA COROLLA loaded\n"
|
||
]
|
||
}
|
||
],
|
||
"source": [
|
||
"documents, docs_id, num_pages = load_documents_from_directory('data') "
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": 68,
|
||
"metadata": {},
|
||
"outputs": [
|
||
{
|
||
"data": {
|
||
"text/plain": [
|
||
"['320bc9710952499baa9553d3f0d4e727',\n",
|
||
" '6ba07e1cf09a4ae6b54863040f901328',\n",
|
||
" 'dd067c452bd146e4becd61bde8602a3c',\n",
|
||
" '640493ad16b546d38851216917d3e82b',\n",
|
||
" '08cf1c3c8eab4efe9f81efcf8ce770be',\n",
|
||
" 'd8d6a3ca9a0a44e08cd4423ee3fb979d',\n",
|
||
" '2b6e45cd99ff46b08242282a423642d4',\n",
|
||
" '05524682d2e9425c83c9b57693182c50',\n",
|
||
" '4eb170648fbe47c3b87b2831a97f0dd8',\n",
|
||
" 'cec3e82f0432402e940a0299bfa086fe']"
|
||
]
|
||
},
|
||
"execution_count": 68,
|
||
"metadata": {},
|
||
"output_type": "execute_result"
|
||
}
|
||
],
|
||
"source": [
|
||
"docs_id"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": 81,
|
||
"metadata": {},
|
||
"outputs": [],
|
||
"source": [
|
||
"# A function to create vector store\n",
|
||
"def create_vector_store(embeddings, documents: list, docs_id: list, num_pages: list):\n",
|
||
" # index set up with the embedding dimension\n",
|
||
" index = faiss.IndexFlatL2(384)\n",
|
||
" # Initialize the FAISS vector store\n",
|
||
" vector_store = FAISS(\n",
|
||
" embedding_function=embeddings,\n",
|
||
" index=index,\n",
|
||
" docstore=InMemoryDocstore(),\n",
|
||
" index_to_docstore_id={},\n",
|
||
" )\n",
|
||
" # Now adding other documents to the store. \n",
|
||
" for i in range(len(documents)):\n",
|
||
" doc_id = docs_id[i]\n",
|
||
" page_ids = [doc_id+ str(i) for i in range(num_pages[i])]\n",
|
||
" vector_store.add_documents(documents=documents[i], ids=page_ids)\n",
|
||
" \n",
|
||
" # saving the vector store automatically\n",
|
||
" save_embedded_data(vector_store, key=\"data\")\n",
|
||
" \n",
|
||
" return vector_store"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": 82,
|
||
"metadata": {},
|
||
"outputs": [
|
||
{
|
||
"name": "stdout",
|
||
"output_type": "stream",
|
||
"text": [
|
||
"['320bc9710952499baa9553d3f0d4e7270']\n",
|
||
"['6ba07e1cf09a4ae6b54863040f9013280', '6ba07e1cf09a4ae6b54863040f9013281']\n",
|
||
"['dd067c452bd146e4becd61bde8602a3c0', 'dd067c452bd146e4becd61bde8602a3c1']\n",
|
||
"['640493ad16b546d38851216917d3e82b0', '640493ad16b546d38851216917d3e82b1']\n",
|
||
"['08cf1c3c8eab4efe9f81efcf8ce770be0']\n",
|
||
"['d8d6a3ca9a0a44e08cd4423ee3fb979d0', 'd8d6a3ca9a0a44e08cd4423ee3fb979d1', 'd8d6a3ca9a0a44e08cd4423ee3fb979d2', 'd8d6a3ca9a0a44e08cd4423ee3fb979d3', 'd8d6a3ca9a0a44e08cd4423ee3fb979d4', 'd8d6a3ca9a0a44e08cd4423ee3fb979d5', 'd8d6a3ca9a0a44e08cd4423ee3fb979d6', 'd8d6a3ca9a0a44e08cd4423ee3fb979d7', 'd8d6a3ca9a0a44e08cd4423ee3fb979d8', 'd8d6a3ca9a0a44e08cd4423ee3fb979d9', 'd8d6a3ca9a0a44e08cd4423ee3fb979d10', 'd8d6a3ca9a0a44e08cd4423ee3fb979d11', 'd8d6a3ca9a0a44e08cd4423ee3fb979d12', 'd8d6a3ca9a0a44e08cd4423ee3fb979d13', 'd8d6a3ca9a0a44e08cd4423ee3fb979d14', 'd8d6a3ca9a0a44e08cd4423ee3fb979d15', 'd8d6a3ca9a0a44e08cd4423ee3fb979d16', 'd8d6a3ca9a0a44e08cd4423ee3fb979d17', 'd8d6a3ca9a0a44e08cd4423ee3fb979d18', 'd8d6a3ca9a0a44e08cd4423ee3fb979d19', 'd8d6a3ca9a0a44e08cd4423ee3fb979d20', 'd8d6a3ca9a0a44e08cd4423ee3fb979d21', 'd8d6a3ca9a0a44e08cd4423ee3fb979d22', 'd8d6a3ca9a0a44e08cd4423ee3fb979d23', 'd8d6a3ca9a0a44e08cd4423ee3fb979d24', 'd8d6a3ca9a0a44e08cd4423ee3fb979d25', 'd8d6a3ca9a0a44e08cd4423ee3fb979d26', 'd8d6a3ca9a0a44e08cd4423ee3fb979d27', 'd8d6a3ca9a0a44e08cd4423ee3fb979d28', 'd8d6a3ca9a0a44e08cd4423ee3fb979d29', 'd8d6a3ca9a0a44e08cd4423ee3fb979d30', 'd8d6a3ca9a0a44e08cd4423ee3fb979d31', 'd8d6a3ca9a0a44e08cd4423ee3fb979d32', 'd8d6a3ca9a0a44e08cd4423ee3fb979d33', 'd8d6a3ca9a0a44e08cd4423ee3fb979d34', 'd8d6a3ca9a0a44e08cd4423ee3fb979d35', 'd8d6a3ca9a0a44e08cd4423ee3fb979d36', 'd8d6a3ca9a0a44e08cd4423ee3fb979d37', 'd8d6a3ca9a0a44e08cd4423ee3fb979d38', 'd8d6a3ca9a0a44e08cd4423ee3fb979d39', 'd8d6a3ca9a0a44e08cd4423ee3fb979d40', 'd8d6a3ca9a0a44e08cd4423ee3fb979d41', 'd8d6a3ca9a0a44e08cd4423ee3fb979d42', 'd8d6a3ca9a0a44e08cd4423ee3fb979d43', 'd8d6a3ca9a0a44e08cd4423ee3fb979d44', 'd8d6a3ca9a0a44e08cd4423ee3fb979d45', 'd8d6a3ca9a0a44e08cd4423ee3fb979d46', 'd8d6a3ca9a0a44e08cd4423ee3fb979d47', 'd8d6a3ca9a0a44e08cd4423ee3fb979d48', 'd8d6a3ca9a0a44e08cd4423ee3fb979d49', 'd8d6a3ca9a0a44e08cd4423ee3fb979d50', 'd8d6a3ca9a0a44e08cd4423ee3fb979d51', 'd8d6a3ca9a0a44e08cd4423ee3fb979d52', 'd8d6a3ca9a0a44e08cd4423ee3fb979d53', 'd8d6a3ca9a0a44e08cd4423ee3fb979d54', 'd8d6a3ca9a0a44e08cd4423ee3fb979d55', 'd8d6a3ca9a0a44e08cd4423ee3fb979d56', 'd8d6a3ca9a0a44e08cd4423ee3fb979d57', 'd8d6a3ca9a0a44e08cd4423ee3fb979d58', 'd8d6a3ca9a0a44e08cd4423ee3fb979d59', 'd8d6a3ca9a0a44e08cd4423ee3fb979d60', 'd8d6a3ca9a0a44e08cd4423ee3fb979d61', 'd8d6a3ca9a0a44e08cd4423ee3fb979d62', 'd8d6a3ca9a0a44e08cd4423ee3fb979d63', 'd8d6a3ca9a0a44e08cd4423ee3fb979d64', 'd8d6a3ca9a0a44e08cd4423ee3fb979d65', 'd8d6a3ca9a0a44e08cd4423ee3fb979d66', 'd8d6a3ca9a0a44e08cd4423ee3fb979d67', 'd8d6a3ca9a0a44e08cd4423ee3fb979d68', 'd8d6a3ca9a0a44e08cd4423ee3fb979d69', 'd8d6a3ca9a0a44e08cd4423ee3fb979d70', 'd8d6a3ca9a0a44e08cd4423ee3fb979d71', 'd8d6a3ca9a0a44e08cd4423ee3fb979d72', 'd8d6a3ca9a0a44e08cd4423ee3fb979d73', 'd8d6a3ca9a0a44e08cd4423ee3fb979d74', 'd8d6a3ca9a0a44e08cd4423ee3fb979d75', 'd8d6a3ca9a0a44e08cd4423ee3fb979d76', 'd8d6a3ca9a0a44e08cd4423ee3fb979d77', 'd8d6a3ca9a0a44e08cd4423ee3fb979d78', 'd8d6a3ca9a0a44e08cd4423ee3fb979d79', 'd8d6a3ca9a0a44e08cd4423ee3fb979d80', 'd8d6a3ca9a0a44e08cd4423ee3fb979d81', 'd8d6a3ca9a0a44e08cd4423ee3fb979d82', 'd8d6a3ca9a0a44e08cd4423ee3fb979d83', 'd8d6a3ca9a0a44e08cd4423ee3fb979d84', 'd8d6a3ca9a0a44e08cd4423ee3fb979d85', 'd8d6a3ca9a0a44e08cd4423ee3fb979d86', 'd8d6a3ca9a0a44e08cd4423ee3fb979d87', 'd8d6a3ca9a0a44e08cd4423ee3fb979d88', 'd8d6a3ca9a0a44e08cd4423ee3fb979d89', 'd8d6a3ca9a0a44e08cd4423ee3fb979d90', 'd8d6a3ca9a0a44e08cd4423ee3fb979d91', 'd8d6a3ca9a0a44e08cd4423ee3fb979d92', 'd8d6a3ca9a0a44e08cd4423ee3fb979d93', 'd8d6a3ca9a0a44e08cd4423ee3fb979d94', 'd8d6a3ca9a0a44e08cd4423ee3fb979d95', 'd8d6a3ca9a0a44e08cd4423ee3fb979d96', 'd8d6a3ca9a0a44e08cd4423ee3fb979d97', 'd8d6a3ca9a0a44e08cd4423ee3fb979d98', 'd8d6a3ca9a0a44e08cd4423ee3fb979d99', 'd8d6a3ca9a0a44e08cd4423ee3fb979d100', 'd8d6a3ca9a0a44e08cd4423ee3fb979d101', 'd8d6a3ca9a0a44e08cd4423ee3fb979d102', 'd8d6a3ca9a0a44e08cd4423ee3fb979d103', 'd8d6a3ca9a0a44e08cd4423ee3fb979d104', 'd8d6a3ca9a0a44e08cd4423ee3fb979d105', 'd8d6a3ca9a0a44e08cd4423ee3fb979d106', 'd8d6a3ca9a0a44e08cd4423ee3fb979d107', 'd8d6a3ca9a0a44e08cd4423ee3fb979d108', 'd8d6a3ca9a0a44e08cd4423ee3fb979d109', 'd8d6a3ca9a0a44e08cd4423ee3fb979d110', 'd8d6a3ca9a0a44e08cd4423ee3fb979d111', 'd8d6a3ca9a0a44e08cd4423ee3fb979d112', 'd8d6a3ca9a0a44e08cd4423ee3fb979d113', 'd8d6a3ca9a0a44e08cd4423ee3fb979d114', 'd8d6a3ca9a0a44e08cd4423ee3fb979d115', 'd8d6a3ca9a0a44e08cd4423ee3fb979d116', 'd8d6a3ca9a0a44e08cd4423ee3fb979d117', 'd8d6a3ca9a0a44e08cd4423ee3fb979d118', 'd8d6a3ca9a0a44e08cd4423ee3fb979d119', 'd8d6a3ca9a0a44e08cd4423ee3fb979d120', 'd8d6a3ca9a0a44e08cd4423ee3fb979d121', 'd8d6a3ca9a0a44e08cd4423ee3fb979d122', 'd8d6a3ca9a0a44e08cd4423ee3fb979d123', 'd8d6a3ca9a0a44e08cd4423ee3fb979d124', 'd8d6a3ca9a0a44e08cd4423ee3fb979d125', 'd8d6a3ca9a0a44e08cd4423ee3fb979d126', 'd8d6a3ca9a0a44e08cd4423ee3fb979d127', 'd8d6a3ca9a0a44e08cd4423ee3fb979d128', 'd8d6a3ca9a0a44e08cd4423ee3fb979d129', 'd8d6a3ca9a0a44e08cd4423ee3fb979d130', 'd8d6a3ca9a0a44e08cd4423ee3fb979d131', 'd8d6a3ca9a0a44e08cd4423ee3fb979d132', 'd8d6a3ca9a0a44e08cd4423ee3fb979d133', 'd8d6a3ca9a0a44e08cd4423ee3fb979d134', 'd8d6a3ca9a0a44e08cd4423ee3fb979d135', 'd8d6a3ca9a0a44e08cd4423ee3fb979d136', 'd8d6a3ca9a0a44e08cd4423ee3fb979d137', 'd8d6a3ca9a0a44e08cd4423ee3fb979d138', 'd8d6a3ca9a0a44e08cd4423ee3fb979d139', 'd8d6a3ca9a0a44e08cd4423ee3fb979d140', 'd8d6a3ca9a0a44e08cd4423ee3fb979d141', 'd8d6a3ca9a0a44e08cd4423ee3fb979d142', 'd8d6a3ca9a0a44e08cd4423ee3fb979d143', 'd8d6a3ca9a0a44e08cd4423ee3fb979d144', 'd8d6a3ca9a0a44e08cd4423ee3fb979d145', 'd8d6a3ca9a0a44e08cd4423ee3fb979d146', 'd8d6a3ca9a0a44e08cd4423ee3fb979d147', 'd8d6a3ca9a0a44e08cd4423ee3fb979d148', 'd8d6a3ca9a0a44e08cd4423ee3fb979d149', 'd8d6a3ca9a0a44e08cd4423ee3fb979d150', 'd8d6a3ca9a0a44e08cd4423ee3fb979d151', 'd8d6a3ca9a0a44e08cd4423ee3fb979d152', 'd8d6a3ca9a0a44e08cd4423ee3fb979d153', 'd8d6a3ca9a0a44e08cd4423ee3fb979d154', 'd8d6a3ca9a0a44e08cd4423ee3fb979d155', 'd8d6a3ca9a0a44e08cd4423ee3fb979d156', 'd8d6a3ca9a0a44e08cd4423ee3fb979d157', 'd8d6a3ca9a0a44e08cd4423ee3fb979d158', 'd8d6a3ca9a0a44e08cd4423ee3fb979d159', 'd8d6a3ca9a0a44e08cd4423ee3fb979d160', 'd8d6a3ca9a0a44e08cd4423ee3fb979d161', 'd8d6a3ca9a0a44e08cd4423ee3fb979d162', 'd8d6a3ca9a0a44e08cd4423ee3fb979d163', 'd8d6a3ca9a0a44e08cd4423ee3fb979d164', 'd8d6a3ca9a0a44e08cd4423ee3fb979d165', 'd8d6a3ca9a0a44e08cd4423ee3fb979d166', 'd8d6a3ca9a0a44e08cd4423ee3fb979d167', 'd8d6a3ca9a0a44e08cd4423ee3fb979d168', 'd8d6a3ca9a0a44e08cd4423ee3fb979d169', 'd8d6a3ca9a0a44e08cd4423ee3fb979d170', 'd8d6a3ca9a0a44e08cd4423ee3fb979d171', 'd8d6a3ca9a0a44e08cd4423ee3fb979d172', 'd8d6a3ca9a0a44e08cd4423ee3fb979d173', 'd8d6a3ca9a0a44e08cd4423ee3fb979d174', 'd8d6a3ca9a0a44e08cd4423ee3fb979d175', 'd8d6a3ca9a0a44e08cd4423ee3fb979d176', 'd8d6a3ca9a0a44e08cd4423ee3fb979d177', 'd8d6a3ca9a0a44e08cd4423ee3fb979d178', 'd8d6a3ca9a0a44e08cd4423ee3fb979d179', 'd8d6a3ca9a0a44e08cd4423ee3fb979d180', 'd8d6a3ca9a0a44e08cd4423ee3fb979d181', 'd8d6a3ca9a0a44e08cd4423ee3fb979d182', 'd8d6a3ca9a0a44e08cd4423ee3fb979d183', 'd8d6a3ca9a0a44e08cd4423ee3fb979d184', 'd8d6a3ca9a0a44e08cd4423ee3fb979d185', 'd8d6a3ca9a0a44e08cd4423ee3fb979d186', 'd8d6a3ca9a0a44e08cd4423ee3fb979d187', 'd8d6a3ca9a0a44e08cd4423ee3fb979d188', 'd8d6a3ca9a0a44e08cd4423ee3fb979d189', 'd8d6a3ca9a0a44e08cd4423ee3fb979d190', 'd8d6a3ca9a0a44e08cd4423ee3fb979d191', 'd8d6a3ca9a0a44e08cd4423ee3fb979d192', 'd8d6a3ca9a0a44e08cd4423ee3fb979d193', 'd8d6a3ca9a0a44e08cd4423ee3fb979d194', 'd8d6a3ca9a0a44e08cd4423ee3fb979d195', 'd8d6a3ca9a0a44e08cd4423ee3fb979d196', 'd8d6a3ca9a0a44e08cd4423ee3fb979d197', 'd8d6a3ca9a0a44e08cd4423ee3fb979d198', 'd8d6a3ca9a0a44e08cd4423ee3fb979d199', 'd8d6a3ca9a0a44e08cd4423ee3fb979d200', 'd8d6a3ca9a0a44e08cd4423ee3fb979d201', 'd8d6a3ca9a0a44e08cd4423ee3fb979d202', 'd8d6a3ca9a0a44e08cd4423ee3fb979d203', 'd8d6a3ca9a0a44e08cd4423ee3fb979d204', 'd8d6a3ca9a0a44e08cd4423ee3fb979d205', 'd8d6a3ca9a0a44e08cd4423ee3fb979d206', 'd8d6a3ca9a0a44e08cd4423ee3fb979d207', 'd8d6a3ca9a0a44e08cd4423ee3fb979d208', 'd8d6a3ca9a0a44e08cd4423ee3fb979d209', 'd8d6a3ca9a0a44e08cd4423ee3fb979d210', 'd8d6a3ca9a0a44e08cd4423ee3fb979d211', 'd8d6a3ca9a0a44e08cd4423ee3fb979d212', 'd8d6a3ca9a0a44e08cd4423ee3fb979d213', 'd8d6a3ca9a0a44e08cd4423ee3fb979d214', 'd8d6a3ca9a0a44e08cd4423ee3fb979d215', 'd8d6a3ca9a0a44e08cd4423ee3fb979d216', 'd8d6a3ca9a0a44e08cd4423ee3fb979d217', 'd8d6a3ca9a0a44e08cd4423ee3fb979d218', 'd8d6a3ca9a0a44e08cd4423ee3fb979d219', 'd8d6a3ca9a0a44e08cd4423ee3fb979d220', 'd8d6a3ca9a0a44e08cd4423ee3fb979d221', 'd8d6a3ca9a0a44e08cd4423ee3fb979d222', 'd8d6a3ca9a0a44e08cd4423ee3fb979d223', 'd8d6a3ca9a0a44e08cd4423ee3fb979d224', 'd8d6a3ca9a0a44e08cd4423ee3fb979d225', 'd8d6a3ca9a0a44e08cd4423ee3fb979d226', 'd8d6a3ca9a0a44e08cd4423ee3fb979d227', 'd8d6a3ca9a0a44e08cd4423ee3fb979d228', 'd8d6a3ca9a0a44e08cd4423ee3fb979d229', 'd8d6a3ca9a0a44e08cd4423ee3fb979d230', 'd8d6a3ca9a0a44e08cd4423ee3fb979d231', 'd8d6a3ca9a0a44e08cd4423ee3fb979d232', 'd8d6a3ca9a0a44e08cd4423ee3fb979d233', 'd8d6a3ca9a0a44e08cd4423ee3fb979d234', 'd8d6a3ca9a0a44e08cd4423ee3fb979d235', 'd8d6a3ca9a0a44e08cd4423ee3fb979d236', 'd8d6a3ca9a0a44e08cd4423ee3fb979d237', 'd8d6a3ca9a0a44e08cd4423ee3fb979d238', 'd8d6a3ca9a0a44e08cd4423ee3fb979d239', 'd8d6a3ca9a0a44e08cd4423ee3fb979d240', 'd8d6a3ca9a0a44e08cd4423ee3fb979d241', 'd8d6a3ca9a0a44e08cd4423ee3fb979d242', 'd8d6a3ca9a0a44e08cd4423ee3fb979d243', 'd8d6a3ca9a0a44e08cd4423ee3fb979d244', 'd8d6a3ca9a0a44e08cd4423ee3fb979d245', 'd8d6a3ca9a0a44e08cd4423ee3fb979d246', 'd8d6a3ca9a0a44e08cd4423ee3fb979d247', 'd8d6a3ca9a0a44e08cd4423ee3fb979d248', 'd8d6a3ca9a0a44e08cd4423ee3fb979d249', 'd8d6a3ca9a0a44e08cd4423ee3fb979d250', 'd8d6a3ca9a0a44e08cd4423ee3fb979d251', 'd8d6a3ca9a0a44e08cd4423ee3fb979d252', 'd8d6a3ca9a0a44e08cd4423ee3fb979d253', 'd8d6a3ca9a0a44e08cd4423ee3fb979d254', 'd8d6a3ca9a0a44e08cd4423ee3fb979d255', 'd8d6a3ca9a0a44e08cd4423ee3fb979d256', 'd8d6a3ca9a0a44e08cd4423ee3fb979d257', 'd8d6a3ca9a0a44e08cd4423ee3fb979d258', 'd8d6a3ca9a0a44e08cd4423ee3fb979d259', 'd8d6a3ca9a0a44e08cd4423ee3fb979d260', 'd8d6a3ca9a0a44e08cd4423ee3fb979d261', 'd8d6a3ca9a0a44e08cd4423ee3fb979d262', 'd8d6a3ca9a0a44e08cd4423ee3fb979d263', 'd8d6a3ca9a0a44e08cd4423ee3fb979d264', 'd8d6a3ca9a0a44e08cd4423ee3fb979d265', 'd8d6a3ca9a0a44e08cd4423ee3fb979d266', 'd8d6a3ca9a0a44e08cd4423ee3fb979d267', 'd8d6a3ca9a0a44e08cd4423ee3fb979d268', 'd8d6a3ca9a0a44e08cd4423ee3fb979d269', 'd8d6a3ca9a0a44e08cd4423ee3fb979d270', 'd8d6a3ca9a0a44e08cd4423ee3fb979d271', 'd8d6a3ca9a0a44e08cd4423ee3fb979d272', 'd8d6a3ca9a0a44e08cd4423ee3fb979d273', 'd8d6a3ca9a0a44e08cd4423ee3fb979d274', 'd8d6a3ca9a0a44e08cd4423ee3fb979d275', 'd8d6a3ca9a0a44e08cd4423ee3fb979d276', 'd8d6a3ca9a0a44e08cd4423ee3fb979d277', 'd8d6a3ca9a0a44e08cd4423ee3fb979d278', 'd8d6a3ca9a0a44e08cd4423ee3fb979d279', 'd8d6a3ca9a0a44e08cd4423ee3fb979d280', 'd8d6a3ca9a0a44e08cd4423ee3fb979d281', 'd8d6a3ca9a0a44e08cd4423ee3fb979d282', 'd8d6a3ca9a0a44e08cd4423ee3fb979d283', 'd8d6a3ca9a0a44e08cd4423ee3fb979d284', 'd8d6a3ca9a0a44e08cd4423ee3fb979d285', 'd8d6a3ca9a0a44e08cd4423ee3fb979d286', 'd8d6a3ca9a0a44e08cd4423ee3fb979d287', 'd8d6a3ca9a0a44e08cd4423ee3fb979d288', 'd8d6a3ca9a0a44e08cd4423ee3fb979d289', 'd8d6a3ca9a0a44e08cd4423ee3fb979d290', 'd8d6a3ca9a0a44e08cd4423ee3fb979d291', 'd8d6a3ca9a0a44e08cd4423ee3fb979d292', 'd8d6a3ca9a0a44e08cd4423ee3fb979d293', 'd8d6a3ca9a0a44e08cd4423ee3fb979d294', 'd8d6a3ca9a0a44e08cd4423ee3fb979d295', 'd8d6a3ca9a0a44e08cd4423ee3fb979d296', 'd8d6a3ca9a0a44e08cd4423ee3fb979d297', 'd8d6a3ca9a0a44e08cd4423ee3fb979d298', 'd8d6a3ca9a0a44e08cd4423ee3fb979d299', 'd8d6a3ca9a0a44e08cd4423ee3fb979d300', 'd8d6a3ca9a0a44e08cd4423ee3fb979d301', 'd8d6a3ca9a0a44e08cd4423ee3fb979d302', 'd8d6a3ca9a0a44e08cd4423ee3fb979d303', 'd8d6a3ca9a0a44e08cd4423ee3fb979d304', 'd8d6a3ca9a0a44e08cd4423ee3fb979d305', 'd8d6a3ca9a0a44e08cd4423ee3fb979d306', 'd8d6a3ca9a0a44e08cd4423ee3fb979d307', 'd8d6a3ca9a0a44e08cd4423ee3fb979d308', 'd8d6a3ca9a0a44e08cd4423ee3fb979d309', 'd8d6a3ca9a0a44e08cd4423ee3fb979d310', 'd8d6a3ca9a0a44e08cd4423ee3fb979d311', 'd8d6a3ca9a0a44e08cd4423ee3fb979d312', 'd8d6a3ca9a0a44e08cd4423ee3fb979d313', 'd8d6a3ca9a0a44e08cd4423ee3fb979d314', 'd8d6a3ca9a0a44e08cd4423ee3fb979d315', 'd8d6a3ca9a0a44e08cd4423ee3fb979d316', 'd8d6a3ca9a0a44e08cd4423ee3fb979d317', 'd8d6a3ca9a0a44e08cd4423ee3fb979d318', 'd8d6a3ca9a0a44e08cd4423ee3fb979d319', 'd8d6a3ca9a0a44e08cd4423ee3fb979d320', 'd8d6a3ca9a0a44e08cd4423ee3fb979d321', 'd8d6a3ca9a0a44e08cd4423ee3fb979d322', 'd8d6a3ca9a0a44e08cd4423ee3fb979d323', 'd8d6a3ca9a0a44e08cd4423ee3fb979d324', 'd8d6a3ca9a0a44e08cd4423ee3fb979d325', 'd8d6a3ca9a0a44e08cd4423ee3fb979d326', 'd8d6a3ca9a0a44e08cd4423ee3fb979d327', 'd8d6a3ca9a0a44e08cd4423ee3fb979d328', 'd8d6a3ca9a0a44e08cd4423ee3fb979d329', 'd8d6a3ca9a0a44e08cd4423ee3fb979d330', 'd8d6a3ca9a0a44e08cd4423ee3fb979d331', 'd8d6a3ca9a0a44e08cd4423ee3fb979d332', 'd8d6a3ca9a0a44e08cd4423ee3fb979d333', 'd8d6a3ca9a0a44e08cd4423ee3fb979d334', 'd8d6a3ca9a0a44e08cd4423ee3fb979d335', 'd8d6a3ca9a0a44e08cd4423ee3fb979d336', 'd8d6a3ca9a0a44e08cd4423ee3fb979d337', 'd8d6a3ca9a0a44e08cd4423ee3fb979d338', 'd8d6a3ca9a0a44e08cd4423ee3fb979d339', 'd8d6a3ca9a0a44e08cd4423ee3fb979d340', 'd8d6a3ca9a0a44e08cd4423ee3fb979d341', 'd8d6a3ca9a0a44e08cd4423ee3fb979d342', 'd8d6a3ca9a0a44e08cd4423ee3fb979d343', 'd8d6a3ca9a0a44e08cd4423ee3fb979d344', 'd8d6a3ca9a0a44e08cd4423ee3fb979d345', 'd8d6a3ca9a0a44e08cd4423ee3fb979d346', 'd8d6a3ca9a0a44e08cd4423ee3fb979d347', 'd8d6a3ca9a0a44e08cd4423ee3fb979d348', 'd8d6a3ca9a0a44e08cd4423ee3fb979d349', 'd8d6a3ca9a0a44e08cd4423ee3fb979d350', 'd8d6a3ca9a0a44e08cd4423ee3fb979d351', 'd8d6a3ca9a0a44e08cd4423ee3fb979d352', 'd8d6a3ca9a0a44e08cd4423ee3fb979d353', 'd8d6a3ca9a0a44e08cd4423ee3fb979d354', 'd8d6a3ca9a0a44e08cd4423ee3fb979d355', 'd8d6a3ca9a0a44e08cd4423ee3fb979d356', 'd8d6a3ca9a0a44e08cd4423ee3fb979d357', 'd8d6a3ca9a0a44e08cd4423ee3fb979d358', 'd8d6a3ca9a0a44e08cd4423ee3fb979d359', 'd8d6a3ca9a0a44e08cd4423ee3fb979d360', 'd8d6a3ca9a0a44e08cd4423ee3fb979d361', 'd8d6a3ca9a0a44e08cd4423ee3fb979d362', 'd8d6a3ca9a0a44e08cd4423ee3fb979d363', 'd8d6a3ca9a0a44e08cd4423ee3fb979d364', 'd8d6a3ca9a0a44e08cd4423ee3fb979d365', 'd8d6a3ca9a0a44e08cd4423ee3fb979d366', 'd8d6a3ca9a0a44e08cd4423ee3fb979d367', 'd8d6a3ca9a0a44e08cd4423ee3fb979d368', 'd8d6a3ca9a0a44e08cd4423ee3fb979d369', 'd8d6a3ca9a0a44e08cd4423ee3fb979d370', 'd8d6a3ca9a0a44e08cd4423ee3fb979d371', 'd8d6a3ca9a0a44e08cd4423ee3fb979d372', 'd8d6a3ca9a0a44e08cd4423ee3fb979d373', 'd8d6a3ca9a0a44e08cd4423ee3fb979d374', 'd8d6a3ca9a0a44e08cd4423ee3fb979d375', 'd8d6a3ca9a0a44e08cd4423ee3fb979d376', 'd8d6a3ca9a0a44e08cd4423ee3fb979d377', 'd8d6a3ca9a0a44e08cd4423ee3fb979d378', 'd8d6a3ca9a0a44e08cd4423ee3fb979d379', 'd8d6a3ca9a0a44e08cd4423ee3fb979d380', 'd8d6a3ca9a0a44e08cd4423ee3fb979d381', 'd8d6a3ca9a0a44e08cd4423ee3fb979d382', 'd8d6a3ca9a0a44e08cd4423ee3fb979d383', 'd8d6a3ca9a0a44e08cd4423ee3fb979d384', 'd8d6a3ca9a0a44e08cd4423ee3fb979d385', 'd8d6a3ca9a0a44e08cd4423ee3fb979d386', 'd8d6a3ca9a0a44e08cd4423ee3fb979d387', 'd8d6a3ca9a0a44e08cd4423ee3fb979d388', 'd8d6a3ca9a0a44e08cd4423ee3fb979d389', 'd8d6a3ca9a0a44e08cd4423ee3fb979d390', 'd8d6a3ca9a0a44e08cd4423ee3fb979d391', 'd8d6a3ca9a0a44e08cd4423ee3fb979d392', 'd8d6a3ca9a0a44e08cd4423ee3fb979d393', 'd8d6a3ca9a0a44e08cd4423ee3fb979d394', 'd8d6a3ca9a0a44e08cd4423ee3fb979d395', 'd8d6a3ca9a0a44e08cd4423ee3fb979d396', 'd8d6a3ca9a0a44e08cd4423ee3fb979d397', 'd8d6a3ca9a0a44e08cd4423ee3fb979d398', 'd8d6a3ca9a0a44e08cd4423ee3fb979d399', 'd8d6a3ca9a0a44e08cd4423ee3fb979d400', 'd8d6a3ca9a0a44e08cd4423ee3fb979d401', 'd8d6a3ca9a0a44e08cd4423ee3fb979d402', 'd8d6a3ca9a0a44e08cd4423ee3fb979d403', 'd8d6a3ca9a0a44e08cd4423ee3fb979d404', 'd8d6a3ca9a0a44e08cd4423ee3fb979d405', 'd8d6a3ca9a0a44e08cd4423ee3fb979d406', 'd8d6a3ca9a0a44e08cd4423ee3fb979d407', 'd8d6a3ca9a0a44e08cd4423ee3fb979d408', 'd8d6a3ca9a0a44e08cd4423ee3fb979d409', 'd8d6a3ca9a0a44e08cd4423ee3fb979d410', 'd8d6a3ca9a0a44e08cd4423ee3fb979d411', 'd8d6a3ca9a0a44e08cd4423ee3fb979d412', 'd8d6a3ca9a0a44e08cd4423ee3fb979d413', 'd8d6a3ca9a0a44e08cd4423ee3fb979d414', 'd8d6a3ca9a0a44e08cd4423ee3fb979d415', 'd8d6a3ca9a0a44e08cd4423ee3fb979d416', 'd8d6a3ca9a0a44e08cd4423ee3fb979d417', 'd8d6a3ca9a0a44e08cd4423ee3fb979d418', 'd8d6a3ca9a0a44e08cd4423ee3fb979d419', 'd8d6a3ca9a0a44e08cd4423ee3fb979d420', 'd8d6a3ca9a0a44e08cd4423ee3fb979d421', 'd8d6a3ca9a0a44e08cd4423ee3fb979d422', 'd8d6a3ca9a0a44e08cd4423ee3fb979d423', 'd8d6a3ca9a0a44e08cd4423ee3fb979d424', 'd8d6a3ca9a0a44e08cd4423ee3fb979d425', 'd8d6a3ca9a0a44e08cd4423ee3fb979d426', 'd8d6a3ca9a0a44e08cd4423ee3fb979d427', 'd8d6a3ca9a0a44e08cd4423ee3fb979d428', 'd8d6a3ca9a0a44e08cd4423ee3fb979d429', 'd8d6a3ca9a0a44e08cd4423ee3fb979d430', 'd8d6a3ca9a0a44e08cd4423ee3fb979d431', 'd8d6a3ca9a0a44e08cd4423ee3fb979d432', 'd8d6a3ca9a0a44e08cd4423ee3fb979d433', 'd8d6a3ca9a0a44e08cd4423ee3fb979d434', 'd8d6a3ca9a0a44e08cd4423ee3fb979d435', 'd8d6a3ca9a0a44e08cd4423ee3fb979d436', 'd8d6a3ca9a0a44e08cd4423ee3fb979d437', 'd8d6a3ca9a0a44e08cd4423ee3fb979d438', 'd8d6a3ca9a0a44e08cd4423ee3fb979d439', 'd8d6a3ca9a0a44e08cd4423ee3fb979d440', 'd8d6a3ca9a0a44e08cd4423ee3fb979d441', 'd8d6a3ca9a0a44e08cd4423ee3fb979d442', 'd8d6a3ca9a0a44e08cd4423ee3fb979d443', 'd8d6a3ca9a0a44e08cd4423ee3fb979d444', 'd8d6a3ca9a0a44e08cd4423ee3fb979d445', 'd8d6a3ca9a0a44e08cd4423ee3fb979d446', 'd8d6a3ca9a0a44e08cd4423ee3fb979d447', 'd8d6a3ca9a0a44e08cd4423ee3fb979d448', 'd8d6a3ca9a0a44e08cd4423ee3fb979d449', 'd8d6a3ca9a0a44e08cd4423ee3fb979d450', 'd8d6a3ca9a0a44e08cd4423ee3fb979d451', 'd8d6a3ca9a0a44e08cd4423ee3fb979d452', 'd8d6a3ca9a0a44e08cd4423ee3fb979d453', 'd8d6a3ca9a0a44e08cd4423ee3fb979d454', 'd8d6a3ca9a0a44e08cd4423ee3fb979d455', 'd8d6a3ca9a0a44e08cd4423ee3fb979d456', 'd8d6a3ca9a0a44e08cd4423ee3fb979d457', 'd8d6a3ca9a0a44e08cd4423ee3fb979d458', 'd8d6a3ca9a0a44e08cd4423ee3fb979d459', 'd8d6a3ca9a0a44e08cd4423ee3fb979d460', 'd8d6a3ca9a0a44e08cd4423ee3fb979d461', 'd8d6a3ca9a0a44e08cd4423ee3fb979d462', 'd8d6a3ca9a0a44e08cd4423ee3fb979d463', 'd8d6a3ca9a0a44e08cd4423ee3fb979d464', 'd8d6a3ca9a0a44e08cd4423ee3fb979d465', 'd8d6a3ca9a0a44e08cd4423ee3fb979d466', 'd8d6a3ca9a0a44e08cd4423ee3fb979d467', 'd8d6a3ca9a0a44e08cd4423ee3fb979d468', 'd8d6a3ca9a0a44e08cd4423ee3fb979d469', 'd8d6a3ca9a0a44e08cd4423ee3fb979d470', 'd8d6a3ca9a0a44e08cd4423ee3fb979d471', 'd8d6a3ca9a0a44e08cd4423ee3fb979d472', 'd8d6a3ca9a0a44e08cd4423ee3fb979d473', 'd8d6a3ca9a0a44e08cd4423ee3fb979d474', 'd8d6a3ca9a0a44e08cd4423ee3fb979d475', 'd8d6a3ca9a0a44e08cd4423ee3fb979d476', 'd8d6a3ca9a0a44e08cd4423ee3fb979d477', 'd8d6a3ca9a0a44e08cd4423ee3fb979d478', 'd8d6a3ca9a0a44e08cd4423ee3fb979d479', 'd8d6a3ca9a0a44e08cd4423ee3fb979d480', 'd8d6a3ca9a0a44e08cd4423ee3fb979d481', 'd8d6a3ca9a0a44e08cd4423ee3fb979d482', 'd8d6a3ca9a0a44e08cd4423ee3fb979d483', 'd8d6a3ca9a0a44e08cd4423ee3fb979d484', 'd8d6a3ca9a0a44e08cd4423ee3fb979d485', 'd8d6a3ca9a0a44e08cd4423ee3fb979d486', 'd8d6a3ca9a0a44e08cd4423ee3fb979d487', 'd8d6a3ca9a0a44e08cd4423ee3fb979d488', 'd8d6a3ca9a0a44e08cd4423ee3fb979d489', 'd8d6a3ca9a0a44e08cd4423ee3fb979d490', 'd8d6a3ca9a0a44e08cd4423ee3fb979d491', 'd8d6a3ca9a0a44e08cd4423ee3fb979d492', 'd8d6a3ca9a0a44e08cd4423ee3fb979d493', 'd8d6a3ca9a0a44e08cd4423ee3fb979d494', 'd8d6a3ca9a0a44e08cd4423ee3fb979d495', 'd8d6a3ca9a0a44e08cd4423ee3fb979d496', 'd8d6a3ca9a0a44e08cd4423ee3fb979d497', 'd8d6a3ca9a0a44e08cd4423ee3fb979d498', 'd8d6a3ca9a0a44e08cd4423ee3fb979d499', 'd8d6a3ca9a0a44e08cd4423ee3fb979d500', 'd8d6a3ca9a0a44e08cd4423ee3fb979d501', 'd8d6a3ca9a0a44e08cd4423ee3fb979d502', 'd8d6a3ca9a0a44e08cd4423ee3fb979d503', 'd8d6a3ca9a0a44e08cd4423ee3fb979d504', 'd8d6a3ca9a0a44e08cd4423ee3fb979d505', 'd8d6a3ca9a0a44e08cd4423ee3fb979d506', 'd8d6a3ca9a0a44e08cd4423ee3fb979d507', 'd8d6a3ca9a0a44e08cd4423ee3fb979d508', 'd8d6a3ca9a0a44e08cd4423ee3fb979d509', 'd8d6a3ca9a0a44e08cd4423ee3fb979d510', 'd8d6a3ca9a0a44e08cd4423ee3fb979d511', 'd8d6a3ca9a0a44e08cd4423ee3fb979d512', 'd8d6a3ca9a0a44e08cd4423ee3fb979d513', 'd8d6a3ca9a0a44e08cd4423ee3fb979d514', 'd8d6a3ca9a0a44e08cd4423ee3fb979d515', 'd8d6a3ca9a0a44e08cd4423ee3fb979d516', 'd8d6a3ca9a0a44e08cd4423ee3fb979d517', 'd8d6a3ca9a0a44e08cd4423ee3fb979d518', 'd8d6a3ca9a0a44e08cd4423ee3fb979d519', 'd8d6a3ca9a0a44e08cd4423ee3fb979d520', 'd8d6a3ca9a0a44e08cd4423ee3fb979d521', 'd8d6a3ca9a0a44e08cd4423ee3fb979d522', 'd8d6a3ca9a0a44e08cd4423ee3fb979d523', 'd8d6a3ca9a0a44e08cd4423ee3fb979d524', 'd8d6a3ca9a0a44e08cd4423ee3fb979d525', 'd8d6a3ca9a0a44e08cd4423ee3fb979d526', 'd8d6a3ca9a0a44e08cd4423ee3fb979d527', 'd8d6a3ca9a0a44e08cd4423ee3fb979d528', 'd8d6a3ca9a0a44e08cd4423ee3fb979d529', 'd8d6a3ca9a0a44e08cd4423ee3fb979d530', 'd8d6a3ca9a0a44e08cd4423ee3fb979d531', 'd8d6a3ca9a0a44e08cd4423ee3fb979d532', 'd8d6a3ca9a0a44e08cd4423ee3fb979d533', 'd8d6a3ca9a0a44e08cd4423ee3fb979d534', 'd8d6a3ca9a0a44e08cd4423ee3fb979d535', 'd8d6a3ca9a0a44e08cd4423ee3fb979d536', 'd8d6a3ca9a0a44e08cd4423ee3fb979d537', 'd8d6a3ca9a0a44e08cd4423ee3fb979d538', 'd8d6a3ca9a0a44e08cd4423ee3fb979d539', 'd8d6a3ca9a0a44e08cd4423ee3fb979d540', 'd8d6a3ca9a0a44e08cd4423ee3fb979d541', 'd8d6a3ca9a0a44e08cd4423ee3fb979d542', 'd8d6a3ca9a0a44e08cd4423ee3fb979d543', 'd8d6a3ca9a0a44e08cd4423ee3fb979d544', 'd8d6a3ca9a0a44e08cd4423ee3fb979d545', 'd8d6a3ca9a0a44e08cd4423ee3fb979d546', 'd8d6a3ca9a0a44e08cd4423ee3fb979d547', 'd8d6a3ca9a0a44e08cd4423ee3fb979d548', 'd8d6a3ca9a0a44e08cd4423ee3fb979d549', 'd8d6a3ca9a0a44e08cd4423ee3fb979d550', 'd8d6a3ca9a0a44e08cd4423ee3fb979d551', 'd8d6a3ca9a0a44e08cd4423ee3fb979d552', 'd8d6a3ca9a0a44e08cd4423ee3fb979d553', 'd8d6a3ca9a0a44e08cd4423ee3fb979d554', 'd8d6a3ca9a0a44e08cd4423ee3fb979d555', 'd8d6a3ca9a0a44e08cd4423ee3fb979d556', 'd8d6a3ca9a0a44e08cd4423ee3fb979d557', 'd8d6a3ca9a0a44e08cd4423ee3fb979d558', 'd8d6a3ca9a0a44e08cd4423ee3fb979d559', 'd8d6a3ca9a0a44e08cd4423ee3fb979d560', 'd8d6a3ca9a0a44e08cd4423ee3fb979d561', 'd8d6a3ca9a0a44e08cd4423ee3fb979d562', 'd8d6a3ca9a0a44e08cd4423ee3fb979d563', 'd8d6a3ca9a0a44e08cd4423ee3fb979d564', 'd8d6a3ca9a0a44e08cd4423ee3fb979d565', 'd8d6a3ca9a0a44e08cd4423ee3fb979d566', 'd8d6a3ca9a0a44e08cd4423ee3fb979d567', 'd8d6a3ca9a0a44e08cd4423ee3fb979d568', 'd8d6a3ca9a0a44e08cd4423ee3fb979d569', 'd8d6a3ca9a0a44e08cd4423ee3fb979d570', 'd8d6a3ca9a0a44e08cd4423ee3fb979d571', 'd8d6a3ca9a0a44e08cd4423ee3fb979d572', 'd8d6a3ca9a0a44e08cd4423ee3fb979d573', 'd8d6a3ca9a0a44e08cd4423ee3fb979d574', 'd8d6a3ca9a0a44e08cd4423ee3fb979d575', 'd8d6a3ca9a0a44e08cd4423ee3fb979d576', 'd8d6a3ca9a0a44e08cd4423ee3fb979d577', 'd8d6a3ca9a0a44e08cd4423ee3fb979d578', 'd8d6a3ca9a0a44e08cd4423ee3fb979d579', 'd8d6a3ca9a0a44e08cd4423ee3fb979d580', 'd8d6a3ca9a0a44e08cd4423ee3fb979d581', 'd8d6a3ca9a0a44e08cd4423ee3fb979d582', 'd8d6a3ca9a0a44e08cd4423ee3fb979d583', 'd8d6a3ca9a0a44e08cd4423ee3fb979d584', 'd8d6a3ca9a0a44e08cd4423ee3fb979d585', 'd8d6a3ca9a0a44e08cd4423ee3fb979d586', 'd8d6a3ca9a0a44e08cd4423ee3fb979d587']\n",
|
||
"['2b6e45cd99ff46b08242282a423642d40', '2b6e45cd99ff46b08242282a423642d41', '2b6e45cd99ff46b08242282a423642d42', '2b6e45cd99ff46b08242282a423642d43', '2b6e45cd99ff46b08242282a423642d44', '2b6e45cd99ff46b08242282a423642d45']\n",
|
||
"['05524682d2e9425c83c9b57693182c500', '05524682d2e9425c83c9b57693182c501', '05524682d2e9425c83c9b57693182c502', '05524682d2e9425c83c9b57693182c503', '05524682d2e9425c83c9b57693182c504', '05524682d2e9425c83c9b57693182c505', '05524682d2e9425c83c9b57693182c506']\n",
|
||
"['4eb170648fbe47c3b87b2831a97f0dd80', '4eb170648fbe47c3b87b2831a97f0dd81', '4eb170648fbe47c3b87b2831a97f0dd82', '4eb170648fbe47c3b87b2831a97f0dd83', '4eb170648fbe47c3b87b2831a97f0dd84', '4eb170648fbe47c3b87b2831a97f0dd85']\n",
|
||
"['cec3e82f0432402e940a0299bfa086fe0', 'cec3e82f0432402e940a0299bfa086fe1', 'cec3e82f0432402e940a0299bfa086fe2', 'cec3e82f0432402e940a0299bfa086fe3', 'cec3e82f0432402e940a0299bfa086fe4', 'cec3e82f0432402e940a0299bfa086fe5', 'cec3e82f0432402e940a0299bfa086fe6', 'cec3e82f0432402e940a0299bfa086fe7', 'cec3e82f0432402e940a0299bfa086fe8', 'cec3e82f0432402e940a0299bfa086fe9']\n",
|
||
"Embeddings saved\n"
|
||
]
|
||
}
|
||
],
|
||
"source": [
|
||
"# creating the vector store\n",
|
||
"vector_store = create_vector_store(embeddings, documents, docs_id, num_pages)"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": null,
|
||
"metadata": {},
|
||
"outputs": [],
|
||
"source": [
|
||
"# creating a function to add documents to the vector store\n",
|
||
"def add_documents_to_vector_store(embeddings, documents: list, docs_id: list, num_pages: list):\n",
|
||
" # loading the vector store\n",
|
||
" vector_store = load_embedded_data(embeddings)\n",
|
||
" for i in range(len(documents)):\n",
|
||
" doc_id = docs_id[i]\n",
|
||
" page_ids = [doc_id+ str(i) for i in range(num_pages[i])]\n",
|
||
" vector_store.add_documents(documents=documents[i], ids=page_ids)\n",
|
||
" print (\"Documents added to the vector store\")\n",
|
||
" "
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "markdown",
|
||
"metadata": {},
|
||
"source": [
|
||
"## Data Search"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": 83,
|
||
"metadata": {},
|
||
"outputs": [
|
||
{
|
||
"name": "stdout",
|
||
"output_type": "stream",
|
||
"text": [
|
||
"* [SIM=0.182989] 202 4-5. Using the driving support systems\n",
|
||
"COROLLA_UInside of displayed lines is \n",
|
||
"black\n",
|
||
"Indicates that the system is not able to recognize white (yellow) \n",
|
||
"lines or a course\n",
|
||
"* or is temporar-\n",
|
||
"ily canceled.\n",
|
||
"*: Boundary between asphalt and \n",
|
||
"the side of the road, such as \n",
|
||
"grass, soil, or a curb\n",
|
||
"Follow-up cruising display\n",
|
||
"Displayed when the multi-informa-tion display is switched to the driv-ing support system information screen.\n",
|
||
"Indicates that steering assist of the \n",
|
||
"lane centering function is operating by monitoring the position of a pre-ceding vehicle.\n",
|
||
"When the follow-up cruising display \n",
|
||
"is displayed, if the preceding vehi-cle moves, your vehicle may move in the same way. A lways pay care-\n",
|
||
"ful attention to your surroundings and operate the steering wheel as necessary to correct the path of the vehicle and ensure safety.\n",
|
||
"■Operation conditions of each \n",
|
||
"function\n",
|
||
"●Lane departure alert function\n",
|
||
"This function oper ates when all of \n",
|
||
"the following cond itions are met.\n",
|
||
"• LTA is turned on.• Vehicle speed is approximately 32 \n",
|
||
"mph (50 km/h) or more.*1\n",
|
||
"• System recognizes white (yellow) \n",
|
||
"lane lines or a course*2. (When a \n",
|
||
"white [yellow] line or course*2 is \n",
|
||
"recognized on only one side, the system will operate only for the \n",
|
||
"recognized side.)\n",
|
||
"• Width of traffic lane is approxi-\n",
|
||
"mately 9.8 ft. (3 m) or more.\n",
|
||
"• Turn signal lever is not operated.\n",
|
||
"(Vehicles with a Blind Spot Moni-\n",
|
||
"tor: Except when another vehicle \n",
|
||
"is in the lane on the side where the turn signal was operated)\n",
|
||
"• Vehicle is not being driven around \n",
|
||
"a sharp curve.\n",
|
||
"• No system malfunctions are \n",
|
||
"detected. ( P.204)\n",
|
||
"*1:The function oper ates even if the \n",
|
||
"vehicle speed is less than \n",
|
||
"approximately 32 mph (50 km/h) when the lane centering function is operating.\n",
|
||
"*2:Boundary between asphalt and \n",
|
||
"the side of the road, such as grass, soil, or a curb\n",
|
||
"●Steering assist function\n",
|
||
"This function operates when all of the following conditions are met in addition to the operation conditions for the lane departure alert function.\n",
|
||
"• Setting for “Steering Assist” in \n",
|
||
"of the multi-information display is \n",
|
||
"set to “ON”. ( P.548)\n",
|
||
"• Vehicle is not accelerated or \n",
|
||
"decelerated by a fixed amount or more.\n",
|
||
"• Steering wheel is not operated \n",
|
||
"with a steering force level suitable \n",
|
||
"for changing lanes.\n",
|
||
"• ABS, VSC, TRAC and PCS are \n",
|
||
"not operating.\n",
|
||
"• TRAC or VSC is not turned off.\n",
|
||
"• Hands off steering wheel warning \n",
|
||
"is not displayed. ( P.204)\n",
|
||
"●Vehicle sway warning function\n",
|
||
"This function operates when all of \n",
|
||
"https://www.MyCarManual.com [{'source': 'data\\\\corolla-2020-toyota-owners-manual.pdf', 'page': 201}]\n",
|
||
"* [SIM=0.184764] 199 4-5. Using the driving support systems\n",
|
||
"COROLLA_U4Drivingthat the vehicle might depart \n",
|
||
"from its lane or course*, the sys-\n",
|
||
"tem provides assistance as nec-\n",
|
||
"essary by operating the steering wheel in small amounts for a short period of time to keep the vehicle in its lane.\n",
|
||
"If the system detects that the steer-\n",
|
||
"ing wheel has not been operated \n",
|
||
"for a fixed amount of time or the steering wheel is not being firmly gripped, a warning is displayed on the multi-information display and the function is t emporarily can-\n",
|
||
"celed.\n",
|
||
"Vehicles with a Blind Spot Monitor: \n",
|
||
"When the system de termines that \n",
|
||
"the vehicle might depart from its lane and that the possibility of a col-lision with an over taking vehicle in \n",
|
||
"the adjacent lane is high, the steer-ing assist function will operate even if the turn signals are operating.\n",
|
||
"*: Boundary between asphalt and \n",
|
||
"the side of the road, such as grass, soil, or a curb■Vehicle sway warning func-\n",
|
||
"tion\n",
|
||
"When the vehicle is swaying within a lane, the warning buzzer will sound and a mes-sage will be displayed on the multi-information display to alert the driver.\n",
|
||
"https://www.MyCarManual.com [{'source': 'data\\\\corolla-2020-toyota-owners-manual.pdf', 'page': 198}]\n",
|
||
"* [SIM=0.188099] 200 4-5. Using the driving support systems\n",
|
||
"COROLLA_U■Lane centering function\n",
|
||
"This function is linked with \n",
|
||
"dynamic radar cruise control with full-speed range and pro-vides the required assistance by operating the steering wheel to keep the vehicle in its current lane.\n",
|
||
"When dynamic radar cruise control \n",
|
||
"with full-speed range is not operat-\n",
|
||
"ing, the lane centering function does not operate.\n",
|
||
"In situations where the white (yel-\n",
|
||
"low) lane lines are difficult to see or are not visible, such as when in a traffic jam, this function will operate to help follow a preceding vehicle by monitoring the position of the preceding vehicle.\n",
|
||
"If the system detects that the steer-\n",
|
||
"ing wheel has not been operated for a fixed amount of time or the steering wheel is not being firmly gripped, a warning is displayed on the multi-information display and the function is temporarily can-celed.\n",
|
||
"Press the LTA switch to turn the \n",
|
||
"LTA system on.\n",
|
||
"The LTA indicator illuminates and a \n",
|
||
"message is displayed on the multi-information display.\n",
|
||
"Press the LTA swit ch again to turn \n",
|
||
"the LTA system off.\n",
|
||
"When the LTA system is turned on \n",
|
||
"or off, operation of the LTA system continues in the same condition the next time the engine is started.Turning LTA system on\n",
|
||
"https://www.MyCarManual.com [{'source': 'data\\\\corolla-2020-toyota-owners-manual.pdf', 'page': 199}]\n",
|
||
"* [SIM=0.203011] 211 4-5. Using the driving support systems\n",
|
||
"COROLLA_U4Driving*: Boundary between asphalt and \n",
|
||
"the side of the road, such as \n",
|
||
"grass, soil, or a curb\n",
|
||
"■Operation conditions of each \n",
|
||
"function\n",
|
||
"●Lane departure alert function\n",
|
||
"This function oper ates when all of \n",
|
||
"the following cond itions are met.\n",
|
||
"• LDA is turned on.• Vehicle speed is approximately 32 \n",
|
||
"mph (50 km/h) or more.\n",
|
||
"• System recognizes white (yellow) \n",
|
||
"lane lines or a course*. (When a \n",
|
||
"white [yellow] line or course* is \n",
|
||
"recognized on only one side, the system will operat e only for the \n",
|
||
"recognized side.)\n",
|
||
"• Width of traffic lane is approxi-\n",
|
||
"mately 9.8 ft. ( 3 m) or more.\n",
|
||
"• Turn signal lever is not operated.\n",
|
||
"(Vehicles with a Blind Spot Moni-tor: Except when a nother vehicle \n",
|
||
"is in the lane on the side where \n",
|
||
"the turn signal was operated)\n",
|
||
"• Vehicle is not being driven around \n",
|
||
"a sharp curve.\n",
|
||
"• No system malfunctions are \n",
|
||
"detected. ( P.212)\n",
|
||
"*: Boundary between asphalt and \n",
|
||
"the side of the road, such as \n",
|
||
"grass, soil, or a curb\n",
|
||
"●Steering assist function\n",
|
||
"This function oper ates when all of \n",
|
||
"the following conditions are met in addition to the operation conditions for the lane departure alert function.\n",
|
||
"• Setting for “Steering Assist” in \n",
|
||
"of the multi-information display is \n",
|
||
"set to “ON”. ( P.548)\n",
|
||
"• Vehicle is not accelerated or \n",
|
||
"decelerated by a fixed amount or more.\n",
|
||
"• Steering wheel is not operated \n",
|
||
"with a steering force level suitable for changing lanes.\n",
|
||
"• ABS, VSC, TRAC and PCS are \n",
|
||
"not operating.• TRAC or VSC is not turned off.• Hands off steering wheel warning \n",
|
||
"is not displayed. ( P.212)\n",
|
||
"●Vehicle sway warning function\n",
|
||
"This function operates when all of \n",
|
||
"the following conditions are met.\n",
|
||
"• Setting for “Sway Warning” in \n",
|
||
"of the multi-information display is \n",
|
||
"set to “ON”. ( P.548)\n",
|
||
"• Vehicle speed is approximately 32 \n",
|
||
"mph (50 km/h) or more.\n",
|
||
"• Width of traffic lane is approxi-\n",
|
||
"mately 9.8 ft. (3 m) or more.\n",
|
||
"• No system malfunctions are \n",
|
||
"detected. ( P.212)\n",
|
||
"■Temporary cancelation of func-\n",
|
||
"tions\n",
|
||
"When operation conditions are no longer met, a func tion may be tem-\n",
|
||
"porarily canceled. However, when \n",
|
||
"the operation conditions are met \n",
|
||
"again, operation of the function is \n",
|
||
"automatically restored. ( P.211)\n",
|
||
"■Steering assist function\n",
|
||
"●Depending on the vehicle speed, lane departure situation, road con-ditions, etc., the driver may not \n",
|
||
"feel the function is operating or \n",
|
||
"the function may not operate at \n",
|
||
"all.\n",
|
||
"●The steering control of the func-tion is overridden by the driver’s \n",
|
||
"steering wheel operation.\n",
|
||
"●Do not attempt to test the opera-tion of the steering assist function.\n",
|
||
"■Lane departure alert function\n",
|
||
"●The warning buzzer may be diffi-cult to hear due to external noise, \n",
|
||
"audio playback, etc.\n",
|
||
"●If the edge of the course* is not \n",
|
||
"clear or straight, the lane depar-\n",
|
||
"ture alert function may not oper-\n",
|
||
"ate.\n",
|
||
"●Vehicles with a Blind Spot Monitor: It may not be possible for the sys-tem to determine if there is a dan-\n",
|
||
"ger of a collision with a vehicle in \n",
|
||
"an adjacent lane.\n",
|
||
"https://www.MyCarManual.com [{'source': 'data\\\\corolla-2020-toyota-owners-manual.pdf', 'page': 210}]\n"
|
||
]
|
||
}
|
||
],
|
||
"source": [
|
||
"# Example of searching\n",
|
||
"query = \"Steering assist function/lane centering function\"\n",
|
||
"results = vector_store.similarity_search_with_score(\n",
|
||
" query, k=4\n",
|
||
")\n",
|
||
"for res, score in results:\n",
|
||
" print(f\"* [SIM={score:3f}] {res.page_content} [{res.metadata}]\")"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": null,
|
||
"metadata": {},
|
||
"outputs": [],
|
||
"source": []
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": null,
|
||
"metadata": {},
|
||
"outputs": [],
|
||
"source": []
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": null,
|
||
"metadata": {},
|
||
"outputs": [],
|
||
"source": []
|
||
}
|
||
],
|
||
"metadata": {
|
||
"kernelspec": {
|
||
"display_name": "ai_index",
|
||
"language": "python",
|
||
"name": "python3"
|
||
},
|
||
"language_info": {
|
||
"codemirror_mode": {
|
||
"name": "ipython",
|
||
"version": 3
|
||
},
|
||
"file_extension": ".py",
|
||
"mimetype": "text/x-python",
|
||
"name": "python",
|
||
"nbconvert_exporter": "python",
|
||
"pygments_lexer": "ipython3",
|
||
"version": "3.11.9"
|
||
}
|
||
},
|
||
"nbformat": 4,
|
||
"nbformat_minor": 2
|
||
}
|