Skip to content

latest updates to .toml and uv.lock files #478

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
bjodom merged 1 commit into from
Jul 16, 2026
Merged

latest updates to .toml and uv.lock files #478

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
28 changes: 13 additions & 15 deletions LLM/09_rag_langchain.ipynb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -264,8 +264,8 @@
"Key Components:\n",
"- LangChain: Framework for developing applications powered by language models\n",
"- HuggingFace: Platform for accessing pre-trained models and embeddings\n",
"- Chroma: Vector store for efficient similarity search\n",
"- LlamaCpp: Efficient C++ implementation of Llama models"
"- FAISS: Vector store for efficient similarity search\n",
"- LlamaCpp: Efficient C++ implementation of Llama models\n"
]
},
{
Expand All @@ -284,15 +284,15 @@
"from langchain_community.document_loaders import DirectoryLoader, TextLoader\n",
"from langchain_community.document_loaders.pdf import PyPDFLoader\n",
"from langchain_huggingface import HuggingFaceEmbeddings\n",
"from langchain_chroma import Chroma\n",
"from langchain_community.vectorstores import FAISS\n",
"from langchain_community.llms import LlamaCpp\n",
"from langchain import chains, text_splitter, PromptTemplate\n",
"from huggingface_hub import hf_hub_download\n",
"from langchain_core.callbacks import BaseCallbackHandler\n",
"import sys\n",
"\n",
"# Setting constants and environment variables\n",
"VECTOR_DB_DIR = \"vector_dbs\""
"VECTOR_DB_DIR = \"vector_dbs\"\n"
]
},
{
Expand Down Expand Up @@ -508,34 +508,34 @@
" embedding generation, and storage.\n",
" \n",
" Args:\n",
" document_url (str): URL of the document to process\n",
" source (str): URL or path of the document to process\n",
" source_type (str): Type of the source ('url' or 'local')\n",
" embedding_fn: Function to generate embeddings\n",
" persist_dir (str): Directory for storing the vector database\n",
" \n",
" Returns:\n",
" Chroma: Initialized vector store with document embeddings\n",
" FAISS: Initialized vector store with document embeddings\n",
" \"\"\"\n",
" source_hash = get_source_hash(source, source_type)\n",
" persist_dir = os.path.join(VECTOR_DB_DIR, source_hash)\n",
"\n",
" # Check if embeddings already exist\n",
" if os.path.exists(persist_dir):\n",
" print(\"Loading existing embeddings...\")\n",
" return Chroma(persist_directory=persist_dir, embedding_function=embedding_fn)\n",
" return FAISS.load_local(persist_dir, embedding_fn, allow_dangerous_deserialization=True)\n",
"\n",
" # Create new embeddings if they don't exist\n",
" start_time = time.time()\n",
" print(\"Creating new embeddings...\")\n",
" document = load_document(source, source_type)\n",
" documents = split_document(document)\n",
" vector_store = Chroma.from_documents(\n",
" vector_store = FAISS.from_documents(\n",
" documents=documents,\n",
" embedding=embedding_fn,\n",
" persist_directory=persist_dir\n",
" embedding=embedding_fn\n",
" )\n",
" vector_store.save_local(persist_dir)\n",
" \n",
" print(f\"Embedding time: {time.time() - start_time:.2f} seconds\")\n",
" return vector_store"
" return vector_store\n"
]
},
{
Expand Down Expand Up @@ -678,8 +678,6 @@
},
"outputs": [],
"source": [
"#from langchain_community.vectorstores.chroma import Chroma\n",
"from langchain_chroma import Chroma\n",
"def main():\n",
" \"\"\"Main function to run the RAG system.\n",
" \n",
Expand Down Expand Up @@ -748,7 +746,7 @@
" except KeyboardInterrupt:\n",
" print(\"\\nProgram interrupted by user\")\n",
" except Exception as e:\n",
" print(f\"An error occurred: {str(e)}\")"
" print(f\"An error occurred: {str(e)}\")\n"
]
},
{
Expand Down
49 changes: 8 additions & 41 deletions LLM/10_rag_pytorch.ipynb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -131,7 +131,7 @@
"Key Components:\n",
"- LangChain: Framework for developing applications powered by language models\n",
"- HuggingFace: Platform for accessing pre-trained models and embeddings\n",
"- Chroma: Vector store for efficient similarity search"
"- FAISS: Vector store for efficient similarity search\n"
]
},
{
Expand All @@ -143,6 +143,7 @@
"from langchain import chains, text_splitter, PromptTemplate\n",
"from langchain_community.embeddings.fastembed import FastEmbedEmbeddings\n",
"from langchain_community import document_loaders, embeddings, vectorstores\n",
"from langchain_community.vectorstores import FAISS\n",
"from langchain.llms.base import LLM\n",
"from typing import Any, List, Optional, Dict\n",
"from pydantic import Field\n",
Expand All @@ -158,7 +159,7 @@
"\n",
"warnings.filterwarnings(\"ignore\")\n",
"\n",
"VECTOR_DB_DIR = \"vector_dbs\""
"VECTOR_DB_DIR = \"vector_dbs\"\n"
]
},
{
Expand Down Expand Up @@ -268,25 +269,11 @@
" raise ValueError(f\"Unsupported embedding type: {embedding_type}\")\n",
"\n",
"def clear_vector_store(persist_dir=VECTOR_DB_DIR):\n",
" import shutil\n",
" if os.path.exists(persist_dir):\n",
" try:\n",
" # Create a temporary Chroma instance to properly close any open connections\n",
" temp_db = vectorstores.Chroma(\n",
" persist_directory=persist_dir,\n",
" embedding_function=FastEmbedEmbeddings()\n",
" )\n",
" temp_db.persist()\n",
" del temp_db # Explicitly delete the instance\n",
" \n",
" # Add a small delay to ensure connections are closed\n",
" time.sleep(1)\n",
" \n",
" shutil.rmtree(persist_dir)\n",
" except Exception as e:\n",
" print(f\"Warning: Error while clearing vector store: {e}\")\n",
" # If regular deletion fails, try forcing it\n",
" os.system(f\"rm -rf {persist_dir}\")\n",
"\n"
]
},
Expand Down Expand Up @@ -329,15 +316,13 @@
" document = load_document(document_url)\n",
" documents = split_document(document)\n",
" \n",
" vector_store = vectorstores.Chroma.from_documents(\n",
" vector_store = FAISS.from_documents(\n",
" documents=documents,\n",
" embedding=embedding_fn,\n",
" persist_directory=unique_persist_dir,\n",
" collection_metadata={\"hnsw:space\": \"cosine\"}\n",
" embedding=embedding_fn\n",
" )\n",
" vector_store.persist()\n",
" vector_store.save_local(unique_persist_dir)\n",
" print(f\"Embedding time: {time.time() - start_time:.2f} seconds\")\n",
" return vector_store, unique_persist_dir"
" return vector_store, unique_persist_dir\n"
]
},
{
Expand Down Expand Up @@ -422,37 +407,19 @@
"metadata": {},
"outputs": [],
"source": [
"#from langchain_community.vectorstores.chroma import Chroma\n",
"def main(document_url, question, embedding_type=\"huggingface\", model_path=\"meta-llama/Llama-2-7b-chat-hf\"):\n",
" try:\n",
" llm = LoadLLM(model_path=model_path)\n",
" embedding_fn = initialize_embedding_fn(embedding_type)\n",
" vector_store, persist_dir = get_or_create_embeddings(document_url, embedding_fn)\n",
" answer = handle_user_interaction(vector_store, llm, question)\n",
" vector_store.persist()\n",
" del vector_store\n",
" # Cleanup old directories if needed\n",
" cleanup_old_directories(VECTOR_DB_DIR, keep_last=5)\n",
" return answer\n",
" except Exception as e:\n",
" print(f\"An error occurred: {e}\")\n",
" return None\n",
"\n",
"def cleanup_old_directories(base_dir, keep_last=5):\n",
" \"\"\"Clean up old vector store directories, keeping only the most recent ones.\"\"\"\n",
" try:\n",
" dirs = [os.path.join(base_dir, d) for d in os.listdir(base_dir)]\n",
" dirs = [d for d in dirs if os.path.isdir(d)]\n",
" dirs.sort(key=lambda x: os.path.getctime(x), reverse=True)\n",
" \n",
" # Remove all but the last keep_last directories\n",
" for dir_path in dirs[keep_last:]:\n",
" try:\n",
" shutil.rmtree(dir_path)\n",
" except Exception as e:\n",
" print(f\"Warning: Could not remove directory {dir_path}: {e}\")\n",
" except Exception as e:\n",
" print(f\"Warning: Error during cleanup: {e}\")"
" return None\n"
]
},
{
Expand Down
3 changes: 1 addition & 2 deletions LLM/rag/requirements.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,9 +4,8 @@ langchain-core==1.3.3
langchain-huggingface==0.1.2
huggingface-hub>=0.30.0
sentence-transformers==3.4.1
chromadb==1.5.9
faiss-cpu>=1.7.4
transformers>=5.5.0
pypdf==6.14.2
torch>=2.13.0
langchain-chroma==0.2.2
beautifulsoup4==4.13.3
Loading
Loading