local gpu use

gpu-requirements.txt

@@ -0,0 +1 @@
+vllm

vllm-tutorial.ipynb

@@ -0,0 +1,641 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 10,
+   "id": "405bc169-e0b7-48e6-84b8-4e4a791cf61a",
+   "metadata": {
+    "scrolled": true
+   },
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "INFO 06-07 04:15:58 [__init__.py:243] Automatically detected platform cuda.\n",
+      "INFO 06-07 04:16:02 [__init__.py:31] Available plugins for group vllm.general_plugins:\n",
+      "INFO 06-07 04:16:02 [__init__.py:33] - lora_filesystem_resolver -> vllm.plugins.lora_resolvers.filesystem_resolver:register_filesystem_resolver\n",
+      "INFO 06-07 04:16:02 [__init__.py:36] All plugins in this group will be loaded. Set `VLLM_PLUGINS` to control which plugins to load.\n",
+      "INFO 06-07 04:16:03 [api_server.py:1289] vLLM API server version 0.9.0.1\n",
+      "INFO 06-07 04:16:03 [cli_args.py:300] non-default args: {'host': '0.0.0.0', 'task': 'embed', 'trust_remote_code': True, 'enforce_eager': True, 'tensor_parallel_size': 2, 'gpu_memory_utilization': 0.4}\n",
+      "WARNING 06-07 04:16:14 [config.py:907] awq quantization is not fully optimized yet. The speed can be slower than non-quantized models.\n",
+      "WARNING 06-07 04:16:14 [arg_utils.py:1583] Compute Capability < 8.0 is not supported by the V1 Engine. Falling back to V0. \n",
+      "WARNING 06-07 04:16:14 [arg_utils.py:1431] The model has a long context length (40960). This may causeOOM during the initial memory profiling phase, or result in low performance due to small KV cache size. Consider setting --max-model-len to a smaller value.\n",
+      "INFO 06-07 04:16:14 [config.py:1875] Defaulting to use mp for distributed inference\n",
+      "WARNING 06-07 04:16:14 [cuda.py:87] To see benefits of async output processing, enable CUDA graph. Since, enforce-eager is enabled, async output processor cannot be used\n",
+      "INFO 06-07 04:16:14 [api_server.py:257] Started engine process with PID 13896\n",
+      "INFO 06-07 04:16:18 [__init__.py:243] Automatically detected platform cuda.\n",
+      "INFO 06-07 04:16:21 [__init__.py:31] Available plugins for group vllm.general_plugins:\n",
+      "INFO 06-07 04:16:21 [__init__.py:33] - lora_filesystem_resolver -> vllm.plugins.lora_resolvers.filesystem_resolver:register_filesystem_resolver\n",
+      "INFO 06-07 04:16:21 [__init__.py:36] All plugins in this group will be loaded. Set `VLLM_PLUGINS` to control which plugins to load.\n",
+      "INFO 06-07 04:16:21 [llm_engine.py:230] Initializing a V0 LLM engine (v0.9.0.1) with config: model='Qwen/Qwen3-32B-AWQ', speculative_config=None, tokenizer='Qwen/Qwen3-32B-AWQ', skip_tokenizer_init=False, tokenizer_mode=auto, revision=None, override_neuron_config={}, tokenizer_revision=None, trust_remote_code=True, dtype=torch.float16, max_seq_len=40960, download_dir=None, load_format=LoadFormat.AUTO, tensor_parallel_size=2, pipeline_parallel_size=1, disable_custom_all_reduce=False, quantization=awq, enforce_eager=True, kv_cache_dtype=auto,  device_config=cuda, decoding_config=DecodingConfig(backend='auto', disable_fallback=False, disable_any_whitespace=False, disable_additional_properties=False, reasoning_backend=''), observability_config=ObservabilityConfig(show_hidden_metrics_for_version=None, otlp_traces_endpoint=None, collect_detailed_traces=None), seed=0, served_model_name=Qwen/Qwen3-32B-AWQ, num_scheduler_steps=1, multi_step_stream_outputs=True, enable_prefix_caching=None, chunked_prefill_enabled=False, use_async_output_proc=False, pooler_config=PoolerConfig(pooling_type=None, normalize=None, softmax=None, step_tag_id=None, returned_token_ids=None), compilation_config={\"compile_sizes\": [], \"inductor_compile_config\": {\"enable_auto_functionalized_v2\": false}, \"cudagraph_capture_sizes\": [], \"max_capture_size\": 0}, use_cached_outputs=True, \n",
+      "WARNING 06-07 04:16:22 [multiproc_worker_utils.py:306] Reducing Torch parallelism from 64 threads to 1 to avoid unnecessary CPU contention. Set OMP_NUM_THREADS in the external environment to tune this value as needed.\n",
+      "INFO 06-07 04:16:22 [cuda.py:240] Cannot use FlashAttention-2 backend for Volta and Turing GPUs.\n",
+      "INFO 06-07 04:16:22 [cuda.py:289] Using XFormers backend.\n",
+      "INFO 06-07 04:16:27 [__init__.py:243] Automatically detected platform cuda.\n",
+      "\u001b[1;36m(VllmWorkerProcess pid=14061)\u001b[0;0m INFO 06-07 04:16:30 [multiproc_worker_utils.py:225] Worker ready; awaiting tasks\n",
+      "\u001b[1;36m(VllmWorkerProcess pid=14061)\u001b[0;0m INFO 06-07 04:16:30 [__init__.py:31] Available plugins for group vllm.general_plugins:\n",
+      "\u001b[1;36m(VllmWorkerProcess pid=14061)\u001b[0;0m INFO 06-07 04:16:30 [__init__.py:33] - lora_filesystem_resolver -> vllm.plugins.lora_resolvers.filesystem_resolver:register_filesystem_resolver\n",
+      "\u001b[1;36m(VllmWorkerProcess pid=14061)\u001b[0;0m INFO 06-07 04:16:30 [__init__.py:36] All plugins in this group will be loaded. Set `VLLM_PLUGINS` to control which plugins to load.\n",
+      "\u001b[1;36m(VllmWorkerProcess pid=14061)\u001b[0;0m INFO 06-07 04:16:30 [cuda.py:240] Cannot use FlashAttention-2 backend for Volta and Turing GPUs.\n",
+      "\u001b[1;36m(VllmWorkerProcess pid=14061)\u001b[0;0m INFO 06-07 04:16:30 [cuda.py:289] Using XFormers backend.\n",
+      "INFO 06-07 04:16:31 [utils.py:1077] Found nccl from library libnccl.so.2\n",
+      "INFO 06-07 04:16:31 [pynccl.py:69] vLLM is using nccl==2.26.2\n",
+      "\u001b[1;36m(VllmWorkerProcess pid=14061)\u001b[0;0m INFO 06-07 04:16:31 [utils.py:1077] Found nccl from library libnccl.so.2\n",
+      "\u001b[1;36m(VllmWorkerProcess pid=14061)\u001b[0;0m INFO 06-07 04:16:31 [pynccl.py:69] vLLM is using nccl==2.26.2\n",
+      "\u001b[1;36m(VllmWorkerProcess pid=14061)\u001b[0;0m INFO 06-07 04:16:31 [custom_all_reduce_utils.py:245] reading GPU P2P access cache from /home/jovyan/.cache/vllm/gpu_p2p_access_cache_for_0,1.json\n",
+      "\u001b[1;36m(VllmWorkerProcess pid=14061)\u001b[0;0m WARNING 06-07 04:16:31 [custom_all_reduce.py:146] Custom allreduce is disabled because your platform lacks GPU P2P capability or P2P test failed. To silence this warning, specify disable_custom_all_reduce=True explicitly.\n",
+      "INFO 06-07 04:16:31 [custom_all_reduce_utils.py:245] reading GPU P2P access cache from /home/jovyan/.cache/vllm/gpu_p2p_access_cache_for_0,1.json\n",
+      "WARNING 06-07 04:16:31 [custom_all_reduce.py:146] Custom allreduce is disabled because your platform lacks GPU P2P capability or P2P test failed. To silence this warning, specify disable_custom_all_reduce=True explicitly.\n",
+      "INFO 06-07 04:16:31 [shm_broadcast.py:250] vLLM message queue communication handle: Handle(local_reader_ranks=[1], buffer_handle=(1, 4194304, 6, 'psm_8808788c'), local_subscribe_addr='ipc:///tmp/f2f3507a-b619-4382-897c-4059a5a27e80', remote_subscribe_addr=None, remote_addr_ipv6=False)\n",
+      "\u001b[1;36m(VllmWorkerProcess pid=14061)\u001b[0;0m INFO 06-07 04:16:31 [parallel_state.py:1064] rank 1 in world size 2 is assigned as DP rank 0, PP rank 0, TP rank 1, EP rank 1\n",
+      "INFO 06-07 04:16:31 [parallel_state.py:1064] rank 0 in world size 2 is assigned as DP rank 0, PP rank 0, TP rank 0, EP rank 0\n",
+      "INFO 06-07 04:16:31 [model_runner.py:1170] Starting to load model Qwen/Qwen3-32B-AWQ...\n",
+      "\u001b[1;36m(VllmWorkerProcess pid=14061)\u001b[0;0m INFO 06-07 04:16:31 [model_runner.py:1170] Starting to load model Qwen/Qwen3-32B-AWQ...\n",
+      "\u001b[1;36m(VllmWorkerProcess pid=14061)\u001b[0;0m INFO 06-07 04:16:32 [weight_utils.py:291] Using model weights format ['*.safetensors']\n",
+      "INFO 06-07 04:16:32 [weight_utils.py:291] Using model weights format ['*.safetensors']\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Loading safetensors checkpoint shards:   0% Completed | 0/4 [00:00<?, ?it/s]\n",
+      "Loading safetensors checkpoint shards:  25% Completed | 1/4 [00:04<00:13,  4.64s/it]\n",
+      "Loading safetensors checkpoint shards:  50% Completed | 2/4 [00:13<00:13,  6.86s/it]\n",
+      "Loading safetensors checkpoint shards:  75% Completed | 3/4 [00:21<00:07,  7.69s/it]\n",
+      "Loading safetensors checkpoint shards: 100% Completed | 4/4 [00:29<00:00,  7.84s/it]\n",
+      "Loading safetensors checkpoint shards: 100% Completed | 4/4 [00:29<00:00,  7.45s/it]\n",
+      "\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "INFO 06-07 04:17:02 [default_loader.py:280] Loading weights took 29.98 seconds\n",
+      "\u001b[1;36m(VllmWorkerProcess pid=14061)\u001b[0;0m INFO 06-07 04:17:02 [default_loader.py:280] Loading weights took 30.20 seconds\n",
+      "INFO 06-07 04:17:02 [model_runner.py:1202] Model loading took 8.3324 GiB and 31.055884 seconds\n",
+      "\u001b[1;36m(VllmWorkerProcess pid=14061)\u001b[0;0m INFO 06-07 04:17:02 [model_runner.py:1202] Model loading took 8.3324 GiB and 31.096897 seconds\n",
+      "INFO 06-07 04:17:03 [api_server.py:1336] Starting vLLM API server on http://0.0.0.0:8000\n",
+      "INFO 06-07 04:17:03 [launcher.py:28] Available routes are:\n",
+      "INFO 06-07 04:17:03 [launcher.py:36] Route: /openapi.json, Methods: GET, HEAD\n",
+      "INFO 06-07 04:17:03 [launcher.py:36] Route: /docs, Methods: GET, HEAD\n",
+      "INFO 06-07 04:17:03 [launcher.py:36] Route: /docs/oauth2-redirect, Methods: GET, HEAD\n",
+      "INFO 06-07 04:17:03 [launcher.py:36] Route: /redoc, Methods: GET, HEAD\n",
+      "INFO 06-07 04:17:03 [launcher.py:36] Route: /health, Methods: GET\n",
+      "INFO 06-07 04:17:03 [launcher.py:36] Route: /load, Methods: GET\n",
+      "INFO 06-07 04:17:03 [launcher.py:36] Route: /ping, Methods: POST\n",
+      "INFO 06-07 04:17:03 [launcher.py:36] Route: /ping, Methods: GET\n",
+      "INFO 06-07 04:17:03 [launcher.py:36] Route: /tokenize, Methods: POST\n",
+      "INFO 06-07 04:17:03 [launcher.py:36] Route: /detokenize, Methods: POST\n",
+      "INFO 06-07 04:17:03 [launcher.py:36] Route: /v1/models, Methods: GET\n",
+      "INFO 06-07 04:17:03 [launcher.py:36] Route: /version, Methods: GET\n",
+      "INFO 06-07 04:17:03 [launcher.py:36] Route: /v1/chat/completions, Methods: POST\n",
+      "INFO 06-07 04:17:03 [launcher.py:36] Route: /v1/completions, Methods: POST\n",
+      "INFO 06-07 04:17:03 [launcher.py:36] Route: /v1/embeddings, Methods: POST\n",
+      "INFO 06-07 04:17:03 [launcher.py:36] Route: /pooling, Methods: POST\n",
+      "INFO 06-07 04:17:03 [launcher.py:36] Route: /classify, Methods: POST\n",
+      "INFO 06-07 04:17:03 [launcher.py:36] Route: /score, Methods: POST\n",
+      "INFO 06-07 04:17:03 [launcher.py:36] Route: /v1/score, Methods: POST\n",
+      "INFO 06-07 04:17:03 [launcher.py:36] Route: /v1/audio/transcriptions, Methods: POST\n",
+      "INFO 06-07 04:17:03 [launcher.py:36] Route: /rerank, Methods: POST\n",
+      "INFO 06-07 04:17:03 [launcher.py:36] Route: /v1/rerank, Methods: POST\n",
+      "INFO 06-07 04:17:03 [launcher.py:36] Route: /v2/rerank, Methods: POST\n",
+      "INFO 06-07 04:17:03 [launcher.py:36] Route: /invocations, Methods: POST\n",
+      "INFO 06-07 04:17:03 [launcher.py:36] Route: /metrics, Methods: GET\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "INFO:     Started server process [13556]\n",
+      "INFO:     Waiting for application startup.\n",
+      "INFO:     Application startup complete.\n"
+     ]
+    }
+   ],
+   "source": [
+    "import os\n",
+    "import subprocess\n",
+    "import threading\n",
+    "import time\n",
+    "\n",
+    "# Set environment variable we need to support dual-GPU on Cirrus\n",
+    "os.environ[\"NCCL_P2P_LEVEL\"] = \"NVL\"\n",
+    "\n",
+    "def run_vllm_server():\n",
+    "    subprocess.run([\n",
+    "        \"vllm\", \"serve\", \"Qwen/Qwen3-32B-AWQ\",\n",
+    "        \"--host\", \"0.0.0.0\",\n",
+    "        \"--port\", \"8000\",\n",
+    "        \"--tensor-parallel-size\", \"2\",\n",
+    "        \"--trust-remote-code\",\n",
+    "        \"--gpu-memory-utilization\", \"0.4\",\n",
+    "        \"--enforce-eager\",\n",
+    "        \"--task\", \"embed\"\n",
+    "    ])\n",
+    "\n",
+    "# Start server in daemon thread\n",
+    "server_thread = threading.Thread(target=run_vllm_server, daemon=True)\n",
+    "server_thread.start()\n",
+    "\n",
+    "## give server time to start up.\n",
+    "\n",
+    "import time\n",
+    "# Pause execution for 100 seconds\n",
+    "time.sleep(200)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "9a8397fa-6896-40a5-97d9-1d0c98797b35",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "## wait for output above to print routes, ending with: \n",
+    "## INFO:     Application startup complete.\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "id": "24b64902-1305-43e7-9da8-e4d82d097cb5",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "INFO 06-07 04:02:50 [logger.py:42] Received request embd-32a68fa8f24a4855b090f66f426e61c4-0: prompt: ' product down', params: PoolingParams(dimensions=None, additional_metadata=None), prompt_token_ids: [1985, 1495], prompt_embeds shape: None, lora_request: None, prompt_adapter_request: None.\n",
+      "INFO 06-07 04:02:50 [engine.py:316] Added request embd-32a68fa8f24a4855b090f66f426e61c4-0.\n",
+      "INFO 06-07 04:02:52 [metrics.py:486] Avg prompt throughput: 0.2 tokens/s, Avg generation throughput: 0.1 tokens/s, Running: 0 reqs, Swapped: 0 reqs, Pending: 0 reqs, GPU KV cache usage: 0.0%, CPU KV cache usage: 0.0%.\n",
+      "INFO:     127.0.0.1:37090 - \"POST /v1/embeddings HTTP/1.1\" 200 OK\n"
+     ]
+    }
+   ],
+   "source": [
+    "## NOTE!  You must wait until the log above finishes and not just the cell.\n",
+    "## Connect to the local model\n",
+    "from langchain_openai import OpenAIEmbeddings\n",
+    "embedding = OpenAIEmbeddings(\n",
+    "                 model = \"Qwen/Qwen3-32B-AWQ\",\n",
+    "                 api_key = \"EMPTY\",\n",
+    "                 base_url = \"http://localhost:8000/v1\",\n",
+    ")\n",
+    "\n",
+    "## test that we can do embeddings\n",
+    "from langchain_core.vectorstores import InMemoryVectorStore\n",
+    "vectorstore = InMemoryVectorStore.from_texts([\"test text\"], embedding=embedding)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "id": "95ed10f3-5339-40cd-bf16-b0854f8b4b91",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import os\n",
+    "import requests\n",
+    "import zipfile\n",
+    "import pathlib\n",
+    "from langchain_community.document_loaders import PyPDFLoader\n",
+    "\n",
+    "def download_and_unzip(url, output_dir):\n",
+    "    response = requests.get(url)\n",
+    "    zip_file_path = os.path.basename(url)\n",
+    "    with open(zip_file_path, 'wb') as f:\n",
+    "        f.write(response.content)\n",
+    "    with zipfile.ZipFile(zip_file_path, 'r') as zip_ref:\n",
+    "        zip_ref.extractall(output_dir)\n",
+    "    os.remove(zip_file_path)\n",
+    "\n",
+    "def pdf_loader(path):\n",
+    "    all_documents = []\n",
+    "    docs_dir = pathlib.Path(path)\n",
+    "    for file in docs_dir.iterdir():\n",
+    "        loader = PyPDFLoader(file)\n",
+    "        documents = loader.load()\n",
+    "        all_documents.extend(documents)\n",
+    "    return all_documents\n",
+    "\n",
+    "\n",
+    "download_and_unzip(\"https://minio.carlboettiger.info/public-data/hwc.zip\", 'hwc')\n",
+    "docs = pdf_loader('hwc/')"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 9,
+   "id": "c6e99791-8f34-4722-9708-665e409c26bd",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Set up the Chat model from one of the NRP models\n",
+    "\n",
+    "import os\n",
+    "api_key = os.getenv(\"OPENAI_API_KEY\")\n",
+    "\n",
+    "# see `curl -H \"Authorization: Bearer $OPENAI_API_KEY\" https://llm.nrp-nautilus.io/v1/models`\n",
+    "models = {\"llama3\": \"llama3-sdsc\", \n",
+    "          \"deepseek-small\": \"DeepSeek-R1-Distill-Qwen-32B\",\n",
+    "          \"deepseek\": \"deepseek-r1-qwen-qualcomm\",\n",
+    "          \"gemma3\": \"gemma3\",\n",
+    "          \"phi3\": \"phi3\",\n",
+    "          \"olmo\": \"olmo\"\n",
+    "         }\n",
+    "\n",
+    "from langchain_openai import ChatOpenAI\n",
+    "llm = ChatOpenAI(model = models[\"gemma3\"], \n",
+    "                 api_key = api_key, \n",
+    "                 base_url = \"https://llm.nrp-nautilus.io\",  \n",
+    "                 temperature=0)\n",
+    "\n",
+    "# Embedding model from NRP usually times out.\n",
+    "#embedding = OpenAIEmbeddings(model = \"embed-mistral\", api_key = api_key, base_url = \"https://llm.nrp-nautilus.io\")\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "95d3e9a3-7334-44ba-a4bc-e7bfc4076358",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Build a retrival agent\n",
+    "from langchain_core.vectorstores import InMemoryVectorStore\n",
+    "from langchain_text_splitters import RecursiveCharacterTextSplitter\n",
+    "text_splitter = RecursiveCharacterTextSplitter(chunk_size=1000, chunk_overlap=200)\n",
+    "splits = text_splitter.split_documents(docs)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "fd8bcc13-d06d-43dd-9e06-4f29da803133",
+   "metadata": {
+    "scrolled": true
+   },
+   "outputs": [],
+   "source": [
+    "# slow part here, runs on remote GPU\n",
+    "from langchain_core.vectorstores import InMemoryVectorStore\n",
+    "vectorstore = InMemoryVectorStore.from_documents(documents=splits, embedding=embedding)\n",
+    "retriever = vectorstore.as_retriever()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "2bf50abf-5ccd-4de5-9fc4-c9043a66a108",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from langchain.chains import create_retrieval_chain\n",
+    "from langchain.chains.combine_documents import create_stuff_documents_chain\n",
+    "from langchain_core.prompts import ChatPromptTemplate\n",
+    "system_prompt = (\n",
+    "    \"You are an assistant for question-answering tasks. \"\n",
+    "    \"Use the following pieces of retrieved context to answer \"\n",
+    "    \"the question. If you don't know the answer, say that you \"\n",
+    "    \"don't know. Use three sentences maximum and keep the \"\n",
+    "    \"answer concise.\"\n",
+    "    \"\\n\\n\"\n",
+    "    \"{context}\"\n",
+    ")\n",
+    "prompt = ChatPromptTemplate.from_messages(\n",
+    "    [\n",
+    "        (\"system\", system_prompt),\n",
+    "        (\"human\", \"{input}\"),\n",
+    "    ]\n",
+    ")\n",
+    "question_answer_chain = create_stuff_documents_chain(llm, prompt)\n",
+    "rag_chain = create_retrieval_chain(retriever, question_answer_chain)\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "e15c64e7-0916-4042-8274-870e4fdb1af7",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "prompt = \"I live in Tanzania and am having issues with lions breaking into my boma and preying on cattle. What interventions might work best for me?\"\n",
+    "results = rag_chain.invoke({\"input\": prompt})\n",
+    "results"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "35613607-2c36-4761-a8ea-8c0889530f34",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "prompt = \"What are the most cost-effective prevention methods for elephants raiding my crops?\"\n",
+    "\n",
+    "results = rag_chain.invoke({\"input\": prompt})\n",
+    "results"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "3dfc39f6-86e9-47c3-ab67-08f90ebbb823",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "rag_chain.invoke({\"input\": \n",
+    "                  \"I have a small herd of goats and cattle and I am worried about jaguars preying on them. What preventative measures can I take?\"\n",
+    "                 })"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "56091874-0e41-4b35-be4f-08d8ec6faf56",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "rag_chain.invoke({\"input\": \"I am trying to prevent coyotes from eating the calves of my free-range cattle. What may work best?\"})"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "918dc691-6c66-46b2-8930-01dbeb6f712b",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "rag_chain.invoke({\"input\": \"We have major issues with deer raiding our large agricultural fields. Is there anything I can try to prevent this that won’t break the bank?\"})"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "07b9578c-9a89-4874-a34d-30a060ed3407",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "rag_chain.invoke({\"input\": \"We live in a suburban area and bears sometimes come into our town to eat from our fruit trees and trash. What are the best ways for us to prevent this as a community? We don’t want to have to get rid of our fruit trees…\"})"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "ba272b88-1622-4d06-9361-7f1e2ca89e73",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "prompt = \"What cattle husbandry strategies might be helpful to prevent conflict if we live in wolf country?\"\n",
+    "\n",
+    "rag_chain.invoke({\"input\": prompt})"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "9d4d1bf4-4084-430d-8b2d-39ce1d6815db",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "d4bf2492-6852-43a7-8527-06ee4e9848c0",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "## DRAFT exploring other embedding databases\n",
+    "\n",
+    "import os\n",
+    "from langchain_community.vectorstores import FAISS\n",
+    "from langchain_community.vectorstores import Chroma\n",
+    "from langchain_community.vectorstores import Qdrant\n",
+    "from qdrant_client import QdrantClient\n",
+    "from qdrant_client.models import Distance, VectorParams\n",
+    "import gc\n",
+    "import torch\n",
+    "\n",
+    "# Option 1: FAISS (Facebook AI Similarity Search) - Most memory efficient\n",
+    "def create_faiss_vectorstore(splits, embedding, persist_directory=\"./faiss_db\", batch_size=100):\n",
+    "    \"\"\"\n",
+    "    Create FAISS vector store with batched processing to minimize GPU RAM usage\n",
+    "    \"\"\"\n",
+    "    os.makedirs(persist_directory, exist_ok=True)\n",
+    "    \n",
+    "    # Process documents in batches to avoid GPU memory overflow\n",
+    "    vectorstore = None\n",
+    "    \n",
+    "    for i in range(0, len(splits), batch_size):\n",
+    "        batch = splits[i:i + batch_size]\n",
+    "        print(f\"Processing batch {i//batch_size + 1}/{(len(splits) + batch_size - 1)//batch_size}\")\n",
+    "        \n",
+    "        if vectorstore is None:\n",
+    "            # Create initial vectorstore with first batch\n",
+    "            vectorstore = FAISS.from_documents(\n",
+    "                documents=batch,\n",
+    "                embedding=embedding\n",
+    "            )\n",
+    "        else:\n",
+    "            # Add subsequent batches to existing vectorstore\n",
+    "            batch_vectorstore = FAISS.from_documents(\n",
+    "                documents=batch,\n",
+    "                embedding=embedding\n",
+    "            )\n",
+    "            vectorstore.merge_from(batch_vectorstore)\n",
+    "            \n",
+    "            # Clean up temporary vectorstore\n",
+    "            del batch_vectorstore\n",
+    "        \n",
+    "        # Force garbage collection and clear GPU cache if using CUDA\n",
+    "        gc.collect()\n",
+    "        if torch.cuda.is_available():\n",
+    "            torch.cuda.empty_cache()\n",
+    "    \n",
+    "    # Save to disk\n",
+    "    vectorstore.save_local(persist_directory)\n",
+    "    print(f\"Vector store saved to {persist_directory}\")\n",
+    "    \n",
+    "    return vectorstore\n",
+    "\n",
+    "def load_faiss_vectorstore(embedding, persist_directory=\"./faiss_db\"):\n",
+    "    \"\"\"Load existing FAISS vector store from disk\"\"\"\n",
+    "    return FAISS.load_local(\n",
+    "        persist_directory,\n",
+    "        embedding,\n",
+    "        allow_dangerous_deserialization=True  # Only if you trust the source\n",
+    "    )\n",
+    "\n",
+    "# Option 2: Chroma - Persistent SQLite-based storage\n",
+    "def create_chroma_vectorstore(splits, embedding, persist_directory=\"./chroma_db\", batch_size=100):\n",
+    "    \"\"\"\n",
+    "    Create Chroma vector store with batched processing\n",
+    "    \"\"\"\n",
+    "    # Initialize Chroma with persistence\n",
+    "    vectorstore = Chroma(\n",
+    "        persist_directory=persist_directory,\n",
+    "        embedding_function=embedding\n",
+    "    )\n",
+    "    \n",
+    "    # Add documents in batches\n",
+    "    for i in range(0, len(splits), batch_size):\n",
+    "        batch = splits[i:i + batch_size]\n",
+    "        print(f\"Processing batch {i//batch_size + 1}/{(len(splits) + batch_size - 1)//batch_size}\")\n",
+    "        \n",
+    "        vectorstore.add_documents(batch)\n",
+    "        \n",
+    "        # Force garbage collection and clear GPU cache\n",
+    "        gc.collect()\n",
+    "        if torch.cuda.is_available():\n",
+    "            torch.cuda.empty_cache()\n",
+    "    \n",
+    "    # Persist to disk\n",
+    "    vectorstore.persist()\n",
+    "    print(f\"Vector store persisted to {persist_directory}\")\n",
+    "    \n",
+    "    return vectorstore\n",
+    "\n",
+    "def load_chroma_vectorstore(embedding, persist_directory=\"./chroma_db\"):\n",
+    "    \"\"\"Load existing Chroma vector store from disk\"\"\"\n",
+    "    return Chroma(\n",
+    "        persist_directory=persist_directory,\n",
+    "        embedding_function=embedding\n",
+    "    )\n",
+    "\n",
+    "# Option 3: Qdrant - High-performance vector database\n",
+    "def create_qdrant_vectorstore(splits, embedding, collection_name=\"documents\", \n",
+    "                            path=\"./qdrant_db\", batch_size=100):\n",
+    "    \"\"\"\n",
+    "    Create Qdrant vector store with local file-based storage\n",
+    "    \"\"\"\n",
+    "    # Initialize local Qdrant client\n",
+    "    client = QdrantClient(path=path)\n",
+    "    \n",
+    "    # Get embedding dimension (embed a sample text)\n",
+    "    sample_embedding = embedding.embed_query(\"sample text\")\n",
+    "    embedding_dim = len(sample_embedding)\n",
+    "    \n",
+    "    # Create collection if it doesn't exist\n",
+    "    try:\n",
+    "        client.create_collection(\n",
+    "            collection_name=collection_name,\n",
+    "            vectors_config=VectorParams(size=embedding_dim, distance=Distance.COSINE)\n",
+    "        )\n",
+    "    except Exception as e:\n",
+    "        print(f\"Collection might already exist: {e}\")\n",
+    "    \n",
+    "    # Create vectorstore\n",
+    "    vectorstore = Qdrant(\n",
+    "        client=client,\n",
+    "        collection_name=collection_name,\n",
+    "        embeddings=embedding\n",
+    "    )\n",
+    "    \n",
+    "    # Add documents in batches\n",
+    "    for i in range(0, len(splits), batch_size):\n",
+    "        batch = splits[i:i + batch_size]\n",
+    "        print(f\"Processing batch {i//batch_size + 1}/{(len(splits) + batch_size - 1)//batch_size}\")\n",
+    "        \n",
+    "        vectorstore.add_documents(batch)\n",
+    "        \n",
+    "        # Force garbage collection and clear GPU cache\n",
+    "        gc.collect()\n",
+    "        if torch.cuda.is_available():\n",
+    "            torch.cuda.empty_cache()\n",
+    "    \n",
+    "    print(f\"Vector store created in {path}\")\n",
+    "    return vectorstore\n",
+    "\n",
+    "def load_qdrant_vectorstore(embedding, collection_name=\"documents\", path=\"./qdrant_db\"):\n",
+    "    \"\"\"Load existing Qdrant vector store from disk\"\"\"\n",
+    "    client = QdrantClient(path=path)\n",
+    "    return Qdrant(\n",
+    "        client=client,\n",
+    "        collection_name=collection_name,\n",
+    "        embeddings=embedding\n",
+    "    )\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "3cf725ad-69a3-4abd-9907-52427babf6d5",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "\n",
+    "# Usage examples:\n",
+    "\n",
+    "# Replace your original code with one of these options:\n",
+    "\n",
+    "# Option 1: FAISS (Recommended for most use cases)\n",
+    "vectorstore = create_faiss_vectorstore(\n",
+    "    splits=splits, \n",
+    "    embedding=embedding, \n",
+    "    persist_directory=\"./my_faiss_db\",\n",
+    "    batch_size=50  # Adjust based on your GPU memory\n",
+    ")\n",
+    "\n",
+    "# To load later:\n",
+    "# vectorstore = load_faiss_vectorstore(embedding, \"./my_faiss_db\")\n",
+    "\n",
+    "# Option 2: Chroma (Good for development and moderate scale)\n",
+    "# vectorstore = create_chroma_vectorstore(\n",
+    "#     splits=splits,\n",
+    "#     embedding=embedding,\n",
+    "#     persist_directory=\"./my_chroma_db\",\n",
+    "#     batch_size=50\n",
+    "# )\n",
+    "\n",
+    "# Option 3: Qdrant (Best for production and very large scale)\n",
+    "# vectorstore = create_qdrant_vectorstore(\n",
+    "#     splits=splits,\n",
+    "#     embedding=embedding,\n",
+    "#     collection_name=\"my_documents\",\n",
+    "#     path=\"./my_qdrant_db\",\n",
+    "#     batch_size=50\n",
+    "# )\n",
+    "\n",
+    "# Memory optimization settings\n",
+    "def optimize_gpu_memory():\n",
+    "    \"\"\"Additional GPU memory optimization\"\"\"\n",
+    "    if torch.cuda.is_available():\n",
+    "        # Set memory fraction if needed\n",
+    "        torch.cuda.set_per_process_memory_fraction(0.8)  # Use 80% of GPU memory\n",
+    "        \n",
+    "        # Enable memory mapping for large tensors\n",
+    "        torch.backends.cuda.matmul.allow_tf32 = True\n",
+    "        torch.backends.cudnn.allow_tf32 = True\n",
+    "\n",
+    "# Call before processing if you have GPU memory issues\n",
+    "# optimize_gpu_memory()"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.12.10"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}