:broom:

README.md

@@ -12,9 +12,6 @@ short_description: Human-Wildlife Conflict LLM
 license: bsd
 ---
 
-# Welcome to Streamlit!
+View app: <https://huggingface.co/spaces/boettiger-lab/hwc-llm>
 
-Edit `/src/streamlit_app.py` to customize this app to your heart's desire. :heart:
 
-If you have any questions, checkout our [documentation](https://docs.streamlit.io) and [community
-forums](https://discuss.streamlit.io).

requirements.txt

@@ -1,11 +1,6 @@
 streamlit
-langchain-chroma
-bs4
-langchain
-langchain-chroma
 langchain-community
 langchain-core
-langchain-core
 langchain_openai 
 langchain-text-splitters
 requests

src/app.py

@@ -115,3 +115,5 @@ if prompt := st.chat_input("What are the most cost-effective prevention methods
 # https://python.langchain.com/docs/tutorials/qa_chat_history/
 
 # Also see structured outputs.
+
+

tutorial.ipynb

@@ -3,19 +3,108 @@
   {
    "cell_type": "code",
    "execution_count": 1,
+   "id": "49d94364",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Requirement already satisfied: streamlit in /opt/conda/lib/python3.12/site-packages (1.45.1)\n",
+      "Requirement already satisfied: langchain-community in /opt/conda/lib/python3.12/site-packages (0.3.24)\n",
+      "Requirement already satisfied: langchain-openai in /opt/conda/lib/python3.12/site-packages (0.3.18)\n",
+      "Requirement already satisfied: langchain-text-splitters in /opt/conda/lib/python3.12/site-packages (0.3.8)\n",
+      "Requirement already satisfied: requests in /opt/conda/lib/python3.12/site-packages (2.32.3)\n",
+      "Requirement already satisfied: pathlib in /opt/conda/lib/python3.12/site-packages (1.0.1)\n",
+      "Requirement already satisfied: pypdf in /opt/conda/lib/python3.12/site-packages (5.5.0)\n",
+      "Requirement already satisfied: altair<6,>=4.0 in /opt/conda/lib/python3.12/site-packages (from streamlit) (5.5.0)\n",
+      "Requirement already satisfied: blinker<2,>=1.5.0 in /opt/conda/lib/python3.12/site-packages (from streamlit) (1.9.0)\n",
+      "Requirement already satisfied: cachetools<6,>=4.0 in /opt/conda/lib/python3.12/site-packages (from streamlit) (5.5.2)\n",
+      "Requirement already satisfied: click<9,>=7.0 in /opt/conda/lib/python3.12/site-packages (from streamlit) (8.2.0)\n",
+      "Requirement already satisfied: numpy<3,>=1.23 in /opt/conda/lib/python3.12/site-packages (from streamlit) (1.26.4)\n",
+      "Requirement already satisfied: packaging<25,>=20 in /opt/conda/lib/python3.12/site-packages (from streamlit) (24.2)\n",
+      "Requirement already satisfied: pandas<3,>=1.4.0 in /opt/conda/lib/python3.12/site-packages (from streamlit) (2.2.3)\n",
+      "Requirement already satisfied: pillow<12,>=7.1.0 in /opt/conda/lib/python3.12/site-packages (from streamlit) (11.2.1)\n",
+      "Requirement already satisfied: protobuf<7,>=3.20 in /opt/conda/lib/python3.12/site-packages (from streamlit) (5.29.3)\n",
+      "Requirement already satisfied: pyarrow>=7.0 in /opt/conda/lib/python3.12/site-packages (from streamlit) (19.0.1)\n",
+      "Requirement already satisfied: tenacity<10,>=8.1.0 in /opt/conda/lib/python3.12/site-packages (from streamlit) (9.1.2)\n",
+      "Requirement already satisfied: toml<2,>=0.10.1 in /opt/conda/lib/python3.12/site-packages (from streamlit) (0.10.2)\n",
+      "Requirement already satisfied: typing-extensions<5,>=4.4.0 in /opt/conda/lib/python3.12/site-packages (from streamlit) (4.13.2)\n",
+      "Requirement already satisfied: watchdog<7,>=2.1.5 in /opt/conda/lib/python3.12/site-packages (from streamlit) (6.0.0)\n",
+      "Requirement already satisfied: gitpython!=3.1.19,<4,>=3.0.7 in /opt/conda/lib/python3.12/site-packages (from streamlit) (3.1.44)\n",
+      "Requirement already satisfied: pydeck<1,>=0.8.0b4 in /opt/conda/lib/python3.12/site-packages (from streamlit) (0.9.1)\n",
+      "Requirement already satisfied: tornado<7,>=6.0.3 in /opt/conda/lib/python3.12/site-packages (from streamlit) (6.5)\n",
+      "Requirement already satisfied: charset_normalizer<4,>=2 in /opt/conda/lib/python3.12/site-packages (from requests) (3.4.2)\n",
+      "Requirement already satisfied: idna<4,>=2.5 in /opt/conda/lib/python3.12/site-packages (from requests) (3.10)\n",
+      "Requirement already satisfied: urllib3<3,>=1.21.1 in /opt/conda/lib/python3.12/site-packages (from requests) (2.4.0)\n",
+      "Requirement already satisfied: certifi>=2017.4.17 in /opt/conda/lib/python3.12/site-packages (from requests) (2025.4.26)\n",
+      "Requirement already satisfied: jinja2 in /opt/conda/lib/python3.12/site-packages (from altair<6,>=4.0->streamlit) (3.1.6)\n",
+      "Requirement already satisfied: jsonschema>=3.0 in /opt/conda/lib/python3.12/site-packages (from altair<6,>=4.0->streamlit) (4.23.0)\n",
+      "Requirement already satisfied: narwhals>=1.14.2 in /opt/conda/lib/python3.12/site-packages (from altair<6,>=4.0->streamlit) (1.39.1)\n",
+      "Requirement already satisfied: gitdb<5,>=4.0.1 in /opt/conda/lib/python3.12/site-packages (from gitpython!=3.1.19,<4,>=3.0.7->streamlit) (4.0.12)\n",
+      "Requirement already satisfied: smmap<6,>=3.0.1 in /opt/conda/lib/python3.12/site-packages (from gitdb<5,>=4.0.1->gitpython!=3.1.19,<4,>=3.0.7->streamlit) (5.0.2)\n",
+      "Requirement already satisfied: python-dateutil>=2.8.2 in /opt/conda/lib/python3.12/site-packages (from pandas<3,>=1.4.0->streamlit) (2.9.0.post0)\n",
+      "Requirement already satisfied: pytz>=2020.1 in /opt/conda/lib/python3.12/site-packages (from pandas<3,>=1.4.0->streamlit) (2025.2)\n",
+      "Requirement already satisfied: tzdata>=2022.7 in /opt/conda/lib/python3.12/site-packages (from pandas<3,>=1.4.0->streamlit) (2025.2)\n",
+      "Requirement already satisfied: langchain-core<1.0.0,>=0.3.59 in /opt/conda/lib/python3.12/site-packages (from langchain-community) (0.3.61)\n",
+      "Requirement already satisfied: langchain<1.0.0,>=0.3.25 in /opt/conda/lib/python3.12/site-packages (from langchain-community) (0.3.25)\n",
+      "Requirement already satisfied: SQLAlchemy<3,>=1.4 in /opt/conda/lib/python3.12/site-packages (from langchain-community) (2.0.41)\n",
+      "Requirement already satisfied: PyYAML>=5.3 in /opt/conda/lib/python3.12/site-packages (from langchain-community) (6.0.2)\n",
+      "Requirement already satisfied: aiohttp<4.0.0,>=3.8.3 in /opt/conda/lib/python3.12/site-packages (from langchain-community) (3.11.18)\n",
+      "Requirement already satisfied: dataclasses-json<0.7,>=0.5.7 in /opt/conda/lib/python3.12/site-packages (from langchain-community) (0.6.7)\n",
+      "Requirement already satisfied: pydantic-settings<3.0.0,>=2.4.0 in /opt/conda/lib/python3.12/site-packages (from langchain-community) (2.9.1)\n",
+      "Requirement already satisfied: langsmith<0.4,>=0.1.125 in /opt/conda/lib/python3.12/site-packages (from langchain-community) (0.2.11)\n",
+      "Requirement already satisfied: httpx-sse<1.0.0,>=0.4.0 in /opt/conda/lib/python3.12/site-packages (from langchain-community) (0.4.0)\n",
+      "Requirement already satisfied: aiohappyeyeballs>=2.3.0 in /opt/conda/lib/python3.12/site-packages (from aiohttp<4.0.0,>=3.8.3->langchain-community) (2.6.1)\n",
+      "Requirement already satisfied: aiosignal>=1.1.2 in /opt/conda/lib/python3.12/site-packages (from aiohttp<4.0.0,>=3.8.3->langchain-community) (1.3.2)\n",
+      "Requirement already satisfied: attrs>=17.3.0 in /opt/conda/lib/python3.12/site-packages (from aiohttp<4.0.0,>=3.8.3->langchain-community) (25.3.0)\n",
+      "Requirement already satisfied: frozenlist>=1.1.1 in /opt/conda/lib/python3.12/site-packages (from aiohttp<4.0.0,>=3.8.3->langchain-community) (1.6.0)\n",
+      "Requirement already satisfied: multidict<7.0,>=4.5 in /opt/conda/lib/python3.12/site-packages (from aiohttp<4.0.0,>=3.8.3->langchain-community) (6.4.4)\n",
+      "Requirement already satisfied: propcache>=0.2.0 in /opt/conda/lib/python3.12/site-packages (from aiohttp<4.0.0,>=3.8.3->langchain-community) (0.3.1)\n",
+      "Requirement already satisfied: yarl<2.0,>=1.17.0 in /opt/conda/lib/python3.12/site-packages (from aiohttp<4.0.0,>=3.8.3->langchain-community) (1.20.0)\n",
+      "Requirement already satisfied: marshmallow<4.0.0,>=3.18.0 in /opt/conda/lib/python3.12/site-packages (from dataclasses-json<0.7,>=0.5.7->langchain-community) (3.26.1)\n",
+      "Requirement already satisfied: typing-inspect<1,>=0.4.0 in /opt/conda/lib/python3.12/site-packages (from dataclasses-json<0.7,>=0.5.7->langchain-community) (0.9.0)\n",
+      "Requirement already satisfied: pydantic<3.0.0,>=2.7.4 in /opt/conda/lib/python3.12/site-packages (from langchain<1.0.0,>=0.3.25->langchain-community) (2.11.4)\n",
+      "Requirement already satisfied: jsonpatch<2.0,>=1.33 in /opt/conda/lib/python3.12/site-packages (from langchain-core<1.0.0,>=0.3.59->langchain-community) (1.33)\n",
+      "Requirement already satisfied: jsonpointer>=1.9 in /opt/conda/lib/python3.12/site-packages (from jsonpatch<2.0,>=1.33->langchain-core<1.0.0,>=0.3.59->langchain-community) (3.0.0)\n",
+      "Requirement already satisfied: httpx<1,>=0.23.0 in /opt/conda/lib/python3.12/site-packages (from langsmith<0.4,>=0.1.125->langchain-community) (0.28.1)\n",
+      "Requirement already satisfied: orjson<4.0.0,>=3.9.14 in /opt/conda/lib/python3.12/site-packages (from langsmith<0.4,>=0.1.125->langchain-community) (3.10.18)\n",
+      "Requirement already satisfied: requests-toolbelt<2.0.0,>=1.0.0 in /opt/conda/lib/python3.12/site-packages (from langsmith<0.4,>=0.1.125->langchain-community) (1.0.0)\n",
+      "Requirement already satisfied: anyio in /opt/conda/lib/python3.12/site-packages (from httpx<1,>=0.23.0->langsmith<0.4,>=0.1.125->langchain-community) (4.9.0)\n",
+      "Requirement already satisfied: httpcore==1.* in /opt/conda/lib/python3.12/site-packages (from httpx<1,>=0.23.0->langsmith<0.4,>=0.1.125->langchain-community) (1.0.9)\n",
+      "Requirement already satisfied: h11>=0.16 in /opt/conda/lib/python3.12/site-packages (from httpcore==1.*->httpx<1,>=0.23.0->langsmith<0.4,>=0.1.125->langchain-community) (0.16.0)\n",
+      "Requirement already satisfied: annotated-types>=0.6.0 in /opt/conda/lib/python3.12/site-packages (from pydantic<3.0.0,>=2.7.4->langchain<1.0.0,>=0.3.25->langchain-community) (0.7.0)\n",
+      "Requirement already satisfied: pydantic-core==2.33.2 in /opt/conda/lib/python3.12/site-packages (from pydantic<3.0.0,>=2.7.4->langchain<1.0.0,>=0.3.25->langchain-community) (2.33.2)\n",
+      "Requirement already satisfied: typing-inspection>=0.4.0 in /opt/conda/lib/python3.12/site-packages (from pydantic<3.0.0,>=2.7.4->langchain<1.0.0,>=0.3.25->langchain-community) (0.4.0)\n",
+      "Requirement already satisfied: python-dotenv>=0.21.0 in /opt/conda/lib/python3.12/site-packages (from pydantic-settings<3.0.0,>=2.4.0->langchain-community) (1.1.0)\n",
+      "Requirement already satisfied: greenlet>=1 in /opt/conda/lib/python3.12/site-packages (from SQLAlchemy<3,>=1.4->langchain-community) (3.2.2)\n",
+      "Requirement already satisfied: mypy_extensions>=0.3.0 in /opt/conda/lib/python3.12/site-packages (from typing-inspect<1,>=0.4.0->dataclasses-json<0.7,>=0.5.7->langchain-community) (1.1.0)\n",
+      "Requirement already satisfied: openai<2.0.0,>=1.68.2 in /opt/conda/lib/python3.12/site-packages (from langchain-openai) (1.82.0)\n",
+      "Requirement already satisfied: tiktoken<1,>=0.7 in /opt/conda/lib/python3.12/site-packages (from langchain-openai) (0.9.0)\n",
+      "Requirement already satisfied: distro<2,>=1.7.0 in /opt/conda/lib/python3.12/site-packages (from openai<2.0.0,>=1.68.2->langchain-openai) (1.9.0)\n",
+      "Requirement already satisfied: jiter<1,>=0.4.0 in /opt/conda/lib/python3.12/site-packages (from openai<2.0.0,>=1.68.2->langchain-openai) (0.10.0)\n",
+      "Requirement already satisfied: sniffio in /opt/conda/lib/python3.12/site-packages (from openai<2.0.0,>=1.68.2->langchain-openai) (1.3.1)\n",
+      "Requirement already satisfied: tqdm>4 in /opt/conda/lib/python3.12/site-packages (from openai<2.0.0,>=1.68.2->langchain-openai) (4.67.1)\n",
+      "Requirement already satisfied: regex>=2022.1.18 in /opt/conda/lib/python3.12/site-packages (from tiktoken<1,>=0.7->langchain-openai) (2024.11.6)\n",
+      "Requirement already satisfied: MarkupSafe>=2.0 in /opt/conda/lib/python3.12/site-packages (from jinja2->altair<6,>=4.0->streamlit) (3.0.2)\n",
+      "Requirement already satisfied: jsonschema-specifications>=2023.03.6 in /opt/conda/lib/python3.12/site-packages (from jsonschema>=3.0->altair<6,>=4.0->streamlit) (2025.4.1)\n",
+      "Requirement already satisfied: referencing>=0.28.4 in /opt/conda/lib/python3.12/site-packages (from jsonschema>=3.0->altair<6,>=4.0->streamlit) (0.36.2)\n",
+      "Requirement already satisfied: rpds-py>=0.7.1 in /opt/conda/lib/python3.12/site-packages (from jsonschema>=3.0->altair<6,>=4.0->streamlit) (0.25.0)\n",
+      "Requirement already satisfied: six>=1.5 in /opt/conda/lib/python3.12/site-packages (from python-dateutil>=2.8.2->pandas<3,>=1.4.0->streamlit) (1.17.0)\n"
+     ]
+    }
+   ],
+   "source": [
+    "!pip install streamlit langchain-community langchain-openai langchain-text-splitters requests pathlib pypdf"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
    "id": "d0bb4874-7f7b-40a9-88ea-922aaed0f3a3",
    "metadata": {},
    "outputs": [],
    "source": [
-    "\n",
-    "# +\n",
-    "from langchain_chroma import Chroma\n",
-    "from langchain_core.output_parsers import StrOutputParser\n",
-    "from langchain_core.runnables import RunnablePassthrough\n",
-    "from langchain_openai import OpenAIEmbeddings\n",
-    "from langchain_text_splitters import RecursiveCharacterTextSplitter\n",
-    "from langchain_community.document_loaders import PyPDFLoader\n",
-    "\n",
     "## dockerized streamlit app wants to read from os.getenv(), otherwise use st.secrets\n",
     "import streamlit as st\n",
     "import os\n",
@@ -29,7 +118,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 34,
+   "execution_count": 3,
    "id": "95ed10f3-5339-40cd-bf16-b0854f8b4b91",
    "metadata": {},
    "outputs": [],
@@ -39,47 +128,26 @@
     "import zipfile\n",
     "\n",
     "def download_and_unzip(url, output_dir):\n",
-    "    \"\"\"\n",
-    "    Downloads a ZIP file from a URL and unzips it to a specified directory.\n",
-    "    \n",
-    "    Args:\n",
-    "        url (str): The URL of the ZIP file.\n",
-    "        output_dir (str): The directory where the ZIP file will be unzipped.\n",
-    "    \"\"\"\n",
-    "    # Download the ZIP file\n",
     "    response = requests.get(url)\n",
     "    zip_file_path = os.path.basename(url)\n",
-    "\n",
-    "    # Save the ZIP file to the current directory\n",
     "    with open(zip_file_path, 'wb') as f:\n",
     "        f.write(response.content)\n",
-    "\n",
-    "    # Unzip the ZIP file\n",
     "    with zipfile.ZipFile(zip_file_path, 'r') as zip_ref:\n",
     "        zip_ref.extractall(output_dir)\n",
-    "\n",
-    "    # Remove the ZIP file\n",
     "    os.remove(zip_file_path)\n",
     "\n",
-    "# Example usage:\n",
-    "url = \"https://minio.carlboettiger.info/public-data/hwc.zip\"\n",
-    "output_dir = \"hwc\"\n",
-    "\n",
-    "# Create the output directory if it doesn't exist\n",
-    "if not os.path.exists(output_dir):\n",
-    "    os.makedirs(output_dir)\n",
-    "\n",
-    "download_and_unzip(url, output_dir)"
+    "download_and_unzip(\"https://minio.carlboettiger.info/public-data/hwc.zip\", \"hwc\")"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 13,
+   "execution_count": 5,
    "id": "2fbca6dc-a90b-4dd4-8225-8baac5c6622d",
    "metadata": {},
    "outputs": [],
    "source": [
     "import pathlib\n",
+    "from langchain_community.document_loaders import PyPDFLoader\n",
     "\n",
     "def pdf_loader(path):\n",
     "    all_documents = []\n",
@@ -90,12 +158,12 @@
     "        all_documents.extend(documents)\n",
     "    return all_documents\n",
     "\n",
-    "docs = pdf_loader('/home/rstudio/data/hwc/')\n"
+    "docs = pdf_loader('hwc/')\n"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 14,
+   "execution_count": 6,
    "id": "c6e99791-8f34-4722-9708-665e409c26bd",
    "metadata": {},
    "outputs": [],
@@ -115,31 +183,23 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 23,
+   "execution_count": 7,
    "id": "0a8a004d-bb80-42bf-b7e6-15a54e2dd804",
    "metadata": {},
    "outputs": [],
    "source": [
     "## Cirrus instead:\n",
+    "from langchain_openai import OpenAIEmbeddings\n",
     "embedding = OpenAIEmbeddings(\n",
     "                 model = \"cirrus\",\n",
     "                 api_key =  cirrus_key, \n",
     "                 base_url = \"https://llm.cirrus.carlboettiger.info/v1\",\n",
-    ")\n",
-    "\n",
-    "\n",
-    "\n",
-    "text = \"A test\"\n",
-    "\n",
-    "vectorstore = InMemoryVectorStore.from_texts(\n",
-    "    [text],\n",
-    "    embedding=embedding,\n",
-    ")\n"
+    ")"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 24,
+   "execution_count": 8,
    "id": "fd8bcc13-d06d-43dd-9e06-4f29da803133",
    "metadata": {},
    "outputs": [],
@@ -156,7 +216,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 25,
+   "execution_count": null,
    "id": "7f388394-5da2-4db8-8e48-e10436c8532d",
    "metadata": {},
    "outputs": [],
@@ -168,7 +228,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 26,
+   "execution_count": null,
    "id": "2bf50abf-5ccd-4de5-9fc4-c9043a66a108",
    "metadata": {},
    "outputs": [],
@@ -199,7 +259,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 27,
+   "execution_count": null,
    "id": "e15c64e7-0916-4042-8274-870e4fdb1af7",
    "metadata": {},
    "outputs": [
@@ -227,7 +287,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 28,
+   "execution_count": null,
    "id": "35613607-2c36-4761-a8ea-8c0889530f34",
    "metadata": {},
    "outputs": [
@@ -256,7 +316,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 29,
+   "execution_count": null,
    "id": "3dfc39f6-86e9-47c3-ab67-08f90ebbb823",
    "metadata": {},
    "outputs": [
@@ -284,7 +344,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 30,
+   "execution_count": null,
    "id": "56091874-0e41-4b35-be4f-08d8ec6faf56",
    "metadata": {},
    "outputs": [
@@ -310,7 +370,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 31,
+   "execution_count": null,
    "id": "918dc691-6c66-46b2-8930-01dbeb6f712b",
    "metadata": {},
    "outputs": [
@@ -336,7 +396,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 32,
+   "execution_count": null,
    "id": "07b9578c-9a89-4874-a34d-30a060ed3407",
    "metadata": {},
    "outputs": [
@@ -362,7 +422,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 33,
+   "execution_count": null,
    "id": "ba272b88-1622-4d06-9361-7f1e2ca89e73",
    "metadata": {},
    "outputs": [
@@ -398,7 +458,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 17,
+   "execution_count": null,
    "id": "d4bf2492-6852-43a7-8527-06ee4e9848c0",
    "metadata": {},
    "outputs": [
@@ -414,155 +474,7 @@
      ]
     }
    ],
-   "source": [
-    "import os\n",
-    "from langchain_community.vectorstores import FAISS\n",
-    "from langchain_community.vectorstores import Chroma\n",
-    "from langchain_community.vectorstores import Qdrant\n",
-    "from qdrant_client import QdrantClient\n",
-    "from qdrant_client.models import Distance, VectorParams\n",
-    "import gc\n",
-    "import torch\n",
-    "\n",
-    "# Option 1: FAISS (Facebook AI Similarity Search) - Most memory efficient\n",
-    "def create_faiss_vectorstore(splits, embedding, persist_directory=\"./faiss_db\", batch_size=100):\n",
-    "    \"\"\"\n",
-    "    Create FAISS vector store with batched processing to minimize GPU RAM usage\n",
-    "    \"\"\"\n",
-    "    os.makedirs(persist_directory, exist_ok=True)\n",
-    "    \n",
-    "    # Process documents in batches to avoid GPU memory overflow\n",
-    "    vectorstore = None\n",
-    "    \n",
-    "    for i in range(0, len(splits), batch_size):\n",
-    "        batch = splits[i:i + batch_size]\n",
-    "        print(f\"Processing batch {i//batch_size + 1}/{(len(splits) + batch_size - 1)//batch_size}\")\n",
-    "        \n",
-    "        if vectorstore is None:\n",
-    "            # Create initial vectorstore with first batch\n",
-    "            vectorstore = FAISS.from_documents(\n",
-    "                documents=batch,\n",
-    "                embedding=embedding\n",
-    "            )\n",
-    "        else:\n",
-    "            # Add subsequent batches to existing vectorstore\n",
-    "            batch_vectorstore = FAISS.from_documents(\n",
-    "                documents=batch,\n",
-    "                embedding=embedding\n",
-    "            )\n",
-    "            vectorstore.merge_from(batch_vectorstore)\n",
-    "            \n",
-    "            # Clean up temporary vectorstore\n",
-    "            del batch_vectorstore\n",
-    "        \n",
-    "        # Force garbage collection and clear GPU cache if using CUDA\n",
-    "        gc.collect()\n",
-    "        if torch.cuda.is_available():\n",
-    "            torch.cuda.empty_cache()\n",
-    "    \n",
-    "    # Save to disk\n",
-    "    vectorstore.save_local(persist_directory)\n",
-    "    print(f\"Vector store saved to {persist_directory}\")\n",
-    "    \n",
-    "    return vectorstore\n",
-    "\n",
-    "def load_faiss_vectorstore(embedding, persist_directory=\"./faiss_db\"):\n",
-    "    \"\"\"Load existing FAISS vector store from disk\"\"\"\n",
-    "    return FAISS.load_local(\n",
-    "        persist_directory,\n",
-    "        embedding,\n",
-    "        allow_dangerous_deserialization=True  # Only if you trust the source\n",
-    "    )\n",
-    "\n",
-    "# Option 2: Chroma - Persistent SQLite-based storage\n",
-    "def create_chroma_vectorstore(splits, embedding, persist_directory=\"./chroma_db\", batch_size=100):\n",
-    "    \"\"\"\n",
-    "    Create Chroma vector store with batched processing\n",
-    "    \"\"\"\n",
-    "    # Initialize Chroma with persistence\n",
-    "    vectorstore = Chroma(\n",
-    "        persist_directory=persist_directory,\n",
-    "        embedding_function=embedding\n",
-    "    )\n",
-    "    \n",
-    "    # Add documents in batches\n",
-    "    for i in range(0, len(splits), batch_size):\n",
-    "        batch = splits[i:i + batch_size]\n",
-    "        print(f\"Processing batch {i//batch_size + 1}/{(len(splits) + batch_size - 1)//batch_size}\")\n",
-    "        \n",
-    "        vectorstore.add_documents(batch)\n",
-    "        \n",
-    "        # Force garbage collection and clear GPU cache\n",
-    "        gc.collect()\n",
-    "        if torch.cuda.is_available():\n",
-    "            torch.cuda.empty_cache()\n",
-    "    \n",
-    "    # Persist to disk\n",
-    "    vectorstore.persist()\n",
-    "    print(f\"Vector store persisted to {persist_directory}\")\n",
-    "    \n",
-    "    return vectorstore\n",
-    "\n",
-    "def load_chroma_vectorstore(embedding, persist_directory=\"./chroma_db\"):\n",
-    "    \"\"\"Load existing Chroma vector store from disk\"\"\"\n",
-    "    return Chroma(\n",
-    "        persist_directory=persist_directory,\n",
-    "        embedding_function=embedding\n",
-    "    )\n",
-    "\n",
-    "# Option 3: Qdrant - High-performance vector database\n",
-    "def create_qdrant_vectorstore(splits, embedding, collection_name=\"documents\", \n",
-    "                            path=\"./qdrant_db\", batch_size=100):\n",
-    "    \"\"\"\n",
-    "    Create Qdrant vector store with local file-based storage\n",
-    "    \"\"\"\n",
-    "    # Initialize local Qdrant client\n",
-    "    client = QdrantClient(path=path)\n",
-    "    \n",
-    "    # Get embedding dimension (embed a sample text)\n",
-    "    sample_embedding = embedding.embed_query(\"sample text\")\n",
-    "    embedding_dim = len(sample_embedding)\n",
-    "    \n",
-    "    # Create collection if it doesn't exist\n",
-    "    try:\n",
-    "        client.create_collection(\n",
-    "            collection_name=collection_name,\n",
-    "            vectors_config=VectorParams(size=embedding_dim, distance=Distance.COSINE)\n",
-    "        )\n",
-    "    except Exception as e:\n",
-    "        print(f\"Collection might already exist: {e}\")\n",
-    "    \n",
-    "    # Create vectorstore\n",
-    "    vectorstore = Qdrant(\n",
-    "        client=client,\n",
-    "        collection_name=collection_name,\n",
-    "        embeddings=embedding\n",
-    "    )\n",
-    "    \n",
-    "    # Add documents in batches\n",
-    "    for i in range(0, len(splits), batch_size):\n",
-    "        batch = splits[i:i + batch_size]\n",
-    "        print(f\"Processing batch {i//batch_size + 1}/{(len(splits) + batch_size - 1)//batch_size}\")\n",
-    "        \n",
-    "        vectorstore.add_documents(batch)\n",
-    "        \n",
-    "        # Force garbage collection and clear GPU cache\n",
-    "        gc.collect()\n",
-    "        if torch.cuda.is_available():\n",
-    "            torch.cuda.empty_cache()\n",
-    "    \n",
-    "    print(f\"Vector store created in {path}\")\n",
-    "    return vectorstore\n",
-    "\n",
-    "def load_qdrant_vectorstore(embedding, collection_name=\"documents\", path=\"./qdrant_db\"):\n",
-    "    \"\"\"Load existing Qdrant vector store from disk\"\"\"\n",
-    "    client = QdrantClient(path=path)\n",
-    "    return Qdrant(\n",
-    "        client=client,\n",
-    "        collection_name=collection_name,\n",
-    "        embeddings=embedding\n",
-    "    )\n"
-   ]
+   "source": []
   },
   {
    "cell_type": "code",
@@ -622,7 +534,7 @@
  ],
  "metadata": {
   "kernelspec": {
-   "display_name": "Python 3 (ipykernel)",
+   "display_name": "base",
    "language": "python",
    "name": "python3"
   },