Dockeriz optimized/complete

.dockerignore

@@ -1,36 +1,23 @@
-# Ignore virtual environments and cache
-__pycache__/
-*.pyc
-*.pyo
-*.pyd
-*.pyc
-.venv/
-venv/
-__pycache__/
-*.sqlite3-journal
+# Ignore version control
+.git
+.gitignore
 
-# Ignore IDE files
-.idea/
-.vscode/
+# Ignore notebooks
+Notebook/
 
-# Ignore development files
-*.swp
-*.swo
-*.swn
-*.swo
-*.swn
-*.swn
-*.swo
-*.swn
-*.swo
-*.swn
+# Ignore databases and logs
+**/*.db
+**/*.sqlite3
+**/chromadb.db
+**/error_log.txt
 
-# Ignore data files
-chromadb.db
-chroma.sqlite3
-Rag/chromadb.db
-# Ignore setup scripts
-setup.sh
+# Ignore cache
+**/__pycache__/
+**/*.pyc
+**/*.pyo
+**/*.pyd
 
 # Ignore environment files
-.env
+.env
+venv/
+.venv/

Dockerfile

@@ -1,17 +1,49 @@
-# Use the official Python 3.11.11 image
-FROM python:3.11.11-slim
+# Declare build arguments at the top (for initial stage)
+ARG USER_UID=1000
+ARG USER_GID=1000
 
-# Set the working directory in the container
+# Stage 1: Build dependencies
+FROM python:3.11-slim AS builder
 WORKDIR /app
-
-# Copy the requirements file into the container
+RUN apt-get update && \
+    apt-get install -y --no-install-recommends \
+    build-essential \
+    git && \
+    rm -rf /var/lib/apt/lists/*
+RUN python -m venv /opt/venv
+ENV PATH="/opt/venv/bin:$PATH"
 COPY requirements.txt .
-
-# Install Python dependencies
 RUN pip install --no-cache-dir -r requirements.txt
 
-# Copy the rest of the application code
+# Stage 2: Final image
+FROM python:3.11-slim
+
+# Re-declare build arguments for this stage
+ARG USER_UID=1000
+ARG USER_GID=1000
+
+COPY --from=builder /opt/venv /opt/venv
+ENV PATH="/opt/venv/bin:$PATH"
+WORKDIR /app
+RUN apt-get update && \
+    apt-get install -y --no-install-recommends \
+    libgomp1 && \
+    rm -rf /var/lib/apt/lists/*
+
 COPY . .
 
-# Set the main entry point
+# Create the group and user first
+RUN groupadd -g ${USER_GID} appuser && \
+    useradd -m -u ${USER_UID} -g appuser appuser
+
+# Create directories and set permissions
+RUN mkdir -p /app/Rag/chromadb.db && \
+    mkdir -p /app/Data && \
+    chown -R appuser:appuser /app
+
+USER appuser
+
+# Make sure your Python code uses this path for ChromaDB
+ENV CHROMA_PERSISTENCE_DIRECTORY=/app/Rag/chromadb.db
+
 CMD ["python", "-m", "Example.rag_example"]

Rag/rag_pipeline.py

@@ -13,13 +13,13 @@ API_KEY = os.getenv("GOOGLE_API_KEY")
 if API_KEY:
     genai.configure(api_key=API_KEY)
 
-chromadb_path = "Rag/chromadb.db"
+chromadb_path = "app/Rag/chromadb.db"
 # transcripts_folder_path = '/home/nightwing/Codes/Xyzbot/Data/transcripts'
 # processed_files_path = "/home/nightwing/Codes/Xyzbot/Rag/Processed_folder/processed_files.json"
 embedding_model = SentenceTransformer('all-MiniLM-L6-v2')
 
-client = chromadb.PersistentClient(path=chromadb_path)
-collection = client.get_or_create_collection(name="yt_transcript_collection")
+# client = chromadb.PersistentClient(path=chromadb_path)
+# collection = client.get_or_create_collection(name="yt_transcript_collection")
 
 # Logging
 logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(message)s')

requirements.txt

@@ -1,16 +1,18 @@
-pyarrow
-pandas
-pendulum
-google.generativeai
+# Core dependencies
 langchain>=0.3.16,<0.4.0
 langchain_openai
 langchain_chroma
 langchain-community>=0.3.16,<0.4.0
 chromadb>=0.4.14
-pypdf==4.2.0
 flask==3.0.1
 flask_cors==3.0.10
-sentence_transformers==3.3.1
-tqdm==4.67.1
-torch==2.5.1
-pydantic>=2.7.4,<3.0.0
+google.generativeai
+pydantic>=2.7.4,<3.0.0
+streamlit
+# PDF Processing
+pypdf==4.2.0
+
+# ML/AI Dependencies (with CPU-only versions)
+sentence_transformers==2.3.1
+--extra-index-url https://download.pytorch.org/whl/cpu
+torch==2.1.0+cpu