Upload 7 files

Old_Document.py

@@ -0,0 +1,175 @@
+import os
+import asyncio
+from dotenv import load_dotenv
+import gradio as gr
+from query_utils import process_query_for_rewrite, get_non_autism_response
+
+# helper functions
+GEMINI_API_KEY="AIzaSyCUCivstFpC9pq_jMHMYdlPrmh9Bx97dFo"
+
+TAVILY_API_KEY="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"
+
+OPENAI_API_KEY="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"
+
+QDRANT_API_KEY="eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJhY2Nlc3MiOiJtIiwiZXhwIjoxNzUxMDUxNzg4fQ.I9J-K7OM0BtcNKgj2d4uVM8QYAHYfFCVAyP4rlZkK2E"
+
+QDRANT_URL="https://6a3aade6-e8ad-4a6c-a579-21f5af90b7e8.us-east4-0.gcp.cloud.qdrant.io"
+
+OPENAI_API_KEY="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"
+
+WEAVIATE_URL="yorcqe2sqswhcaivxvt9a.c0.us-west3.gcp.weaviate.cloud"
+
+WEAVIATE_API_KEY="d2d0VGdZQTBmdTFlOWdDZl9tT2h3WDVWd1NpT1dQWHdGK0xjR1hYeWxicUxHVnFRazRUSjY2VlRUVlkwPV92MjAw"
+
+DEEPINFRA_API_KEY="285LUJulGIprqT6hcPhiXtcrphU04FG4"
+
+DEEPINFRA_BASE_URL="https://api.deepinfra.com/v1/openai"
+
+
+# if not (DEEPINFRA_TOKEN and WEAVIATE_URL and WEAVIATE_API_KEY):
+#     raise ValueError("Please set all required keys in .env")
+
+# DeepInfra client
+from openai import OpenAI
+openai = OpenAI(
+    api_key=DEEPINFRA_API_KEY,
+    base_url="https://api.deepinfra.com/v1/openai",
+)
+
+# Weaviate client
+import weaviate
+from weaviate.classes.init import Auth
+from contextlib import contextmanager
+
+@contextmanager
+def weaviate_client():
+    client = weaviate.connect_to_weaviate_cloud(
+        cluster_url=WEAVIATE_URL,
+        auth_credentials=Auth.api_key(WEAVIATE_API_KEY),
+        skip_init_checks=True,   # <-- This disables gRPC check
+
+    )
+    try:
+        yield client
+    finally:
+        client.close()
+
+# Global path tracker
+last_uploaded_path = None
+
+# Embed function
+def embed_texts(texts: list[str], batch_size: int = 50) -> list[list[float]]:
+    all_embeddings = []
+    for i in range(0, len(texts), batch_size):
+        batch = texts[i : i + batch_size]
+        try:
+            resp = openai.embeddings.create(
+                model="Qwen/Qwen3-Embedding-8B",
+                input=batch,
+                encoding_format="float"
+            )
+            batch_embs = [item.embedding for item in resp.data]
+            all_embeddings.extend(batch_embs)
+        except Exception as e:
+            print(f"Embedding error: {e}")
+            all_embeddings.extend([[] for _ in batch])
+    return all_embeddings
+
+def encode_query(query: str) -> list[float] | None:
+    embs = embed_texts([query], batch_size=1)
+    if embs and embs[0]:
+        return embs[0]
+    return None
+
+async def old_Document(query: str, top_k: int = 1) -> dict:
+    qe = encode_query(query)
+    if not qe:
+        return {"answer": []}
+
+    try:
+        with weaviate_client() as client:
+            coll = client.collections.get("user")
+            res = coll.query.near_vector(
+                near_vector=qe,
+                limit=top_k,
+                return_properties=["text"]
+            )
+        if not getattr(res, "objects", None):
+            return {"answer": []}
+        return {
+            "answer": [obj.properties.get("text", "[No Text]") for obj in res.objects]
+        }
+    except Exception as e:
+        print("RAG Error:", e)
+        return {"answer": []}
+
+# New functions to support Gradio app
+def ingest_file(path: str) -> str:
+    global last_uploaded_path
+    last_uploaded_path = path
+    return f"Old document ingested: {os.path.basename(path)}"
+
+def answer_question(query: str) -> str:
+    try:
+        # Process query for rewriting and relevance checking
+        corrected_query, is_autism_related, rewritten_query = process_query_for_rewrite(query)
+        
+        # If not autism-related, show direct rejection message
+        if not is_autism_related:
+            return get_non_autism_response()
+        
+        # Use the corrected query for retrieval
+        rag_resp = asyncio.run(old_Document(corrected_query))
+        chunks = rag_resp.get("answer", [])
+        if not chunks:
+            return "Sorry, I couldn't find relevant content in the old document."
+
+        # Combine chunks into a single answer for relevance checking
+        combined_answer = "\n".join(f"- {c}" for c in chunks)
+        
+        # NEW: Check if the retrieved content is sufficiently related to autism
+        from query_utils import check_answer_autism_relevance, get_non_autism_answer_response
+        
+        answer_relevance_score = check_answer_autism_relevance(combined_answer)
+        
+        # If answer relevance is below 50%, refuse the answer (updated threshold for enhanced scoring)
+        if answer_relevance_score < 50:
+            return get_non_autism_answer_response()
+        
+        # If sufficiently autism-related, return the answer
+        return combined_answer
+    except Exception as e:
+        return f"Error processing your request: {e}"
+
+# Gradio interface for Old Documents
+with gr.Blocks(title="Old Documents RAG") as demo:
+    gr.Markdown("## Old Documents RAG")
+    query = gr.Textbox(placeholder="Your question...", lines=2, label="Ask about Old Documents")
+    doc_file = gr.File(label="Upload Old Document (PDF, DOCX, TXT)")
+    btn = gr.Button("Submit")
+    out = gr.Textbox(label="Answer from Old Documents", lines=8, interactive=False)
+
+    def process_old_doc(query, doc_file):
+        if doc_file:
+            # Save and ingest the uploaded file
+            upload_dir = os.path.join(os.path.dirname(__file__), "uploaded_docs")
+            os.makedirs(upload_dir, exist_ok=True)
+            safe_filename = os.path.basename(doc_file.name)
+            save_path = os.path.join(upload_dir, safe_filename)
+            with open(save_path, "wb") as f:
+                f.write(doc_file.read())
+            status = ingest_file(save_path)
+            answer = answer_question(query)
+            return f"{status}\n\n{answer}"
+        else:
+            # Use last uploaded file or return error if none exists
+            if last_uploaded_path:
+                answer = answer_question(query)
+                return f"[Using previously uploaded document: {os.path.basename(last_uploaded_path)}]\n\n{answer}"
+            else:
+                return "No document uploaded. Please upload an old document to proceed."
+
+    btn.click(fn=process_old_doc, inputs=[query, doc_file], outputs=out)
+
+if __name__ == "__main__":
+    demo.launch(debug=True)

RAG.py

@@ -0,0 +1,123 @@
+import os
+import asyncio
+from dotenv import load_dotenv
+
+# helper functions
+GEMINI_API_KEY="AIzaSyCUCivstFpC9pq_jMHMYdlPrmh9Bx97dFo"
+
+TAVILY_API_KEY="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"
+
+OPENAI_API_KEY="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"
+
+QDRANT_API_KEY="eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJhY2Nlc3MiOiJtIiwiZXhwIjoxNzUxMDUxNzg4fQ.I9J-K7OM0BtcNKgj2d4uVM8QYAHYfFCVAyP4rlZkK2E"
+
+QDRANT_URL="https://6a3aade6-e8ad-4a6c-a579-21f5af90b7e8.us-east4-0.gcp.cloud.qdrant.io"
+
+OPENAI_API_KEY="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"
+
+WEAVIATE_URL="yorcqe2sqswhcaivxvt9a.c0.us-west3.gcp.weaviate.cloud"
+
+WEAVIATE_API_KEY="d2d0VGdZQTBmdTFlOWdDZl9tT2h3WDVWd1NpT1dQWHdGK0xjR1hYeWxicUxHVnFRazRUSjY2VlRUVlkwPV92MjAw"
+
+DEEPINFRA_API_KEY="285LUJulGIprqT6hcPhiXtcrphU04FG4"
+
+DEEPINFRA_BASE_URL="https://api.deepinfra.com/v1/openai"
+
+# Initialize DeepInfra-compatible OpenAI client
+from openai import OpenAI
+openai = OpenAI(
+    api_key=DEEPINFRA_API_KEY,
+    base_url="https://api.deepinfra.com/v1/openai",
+)
+
+# Weaviate imports
+import weaviate
+from weaviate.classes.init import Auth
+from contextlib import contextmanager
+
+@contextmanager
+def weaviate_client():
+    """
+    Context manager that yields a Weaviate client and
+    guarantees client.close() on exit.
+    """
+    client = weaviate.connect_to_weaviate_cloud(
+        cluster_url=WEAVIATE_URL,
+        auth_credentials=Auth.api_key(WEAVIATE_API_KEY),
+        skip_init_checks=True,   # <-- This disables gRPC check
+
+    )
+    try:
+        yield client
+    finally:
+        client.close()
+
+def embed_texts(texts: list[str], batch_size: int = 50) -> list[list[float]]:
+    """Embed texts in batches to avoid API limits."""
+    all_embeddings: list[list[float]] = []
+    for i in range(0, len(texts), batch_size):
+        batch = texts[i : i + batch_size]
+        try:
+            resp = openai.embeddings.create(
+                model="Qwen/Qwen3-Embedding-8B",
+                input=batch,
+                encoding_format="float"
+            )
+            batch_embs = [item.embedding for item in resp.data]
+            all_embeddings.extend(batch_embs)
+        except Exception as e:
+            print(f"Embedding batch error (items {i}–{i+len(batch)-1}): {e}")
+            all_embeddings.extend([[] for _ in batch])
+    return all_embeddings
+
+def encode_query(query: str) -> list[float] | None:
+    """Generate a single embedding vector for a query string."""
+    embs = embed_texts([query], batch_size=1)
+    if embs and embs[0]:
+        print("Query embedding (first 5 dims):", embs[0][:5])
+        return embs[0]
+    print("Failed to generate query embedding.")
+    return None
+
+async def rag_autism(query: str, top_k: int = 3) -> dict:
+    """
+    Run a RAG retrieval on the 'Books' collection in Weaviate.
+    Returns up to `top_k` matching text chunks.
+    """
+    qe = encode_query(query)
+    if not qe:
+        return {"answer": []}
+
+    try:
+        with weaviate_client() as client:
+            coll = client.collections.get("Books")
+            res  = coll.query.near_vector(
+                near_vector=qe,
+                limit=top_k,
+                return_properties=["text"]
+            )
+        if not getattr(res, "objects", None):
+            return {"answer": []}
+        return {
+            "answer": [
+                obj.properties.get("text", "[No Text]")
+                for obj in res.objects
+            ]
+        }
+    except Exception as e:
+        print("RAG Error:", e)
+        return {"answer": []}
+
+# Example test harness
+# if __name__ == "__main__":
+#     test_queries = [
+#         "What are the common early signs of autism in young children?",
+#         "What diagnostic criteria are used for autism spectrum disorder?",
+#         "What support strategies help improve communication skills in autistic individuals?"
+#     ]
+#     for q in test_queries:
+#         print(f"\nQuery: {q}")
+#         out = asyncio.run(rag_autism(q, top_k=3))
+#         print("Retrieved contexts:")
+#         for idx, ctx in enumerate(out["answer"], 1):
+#             print(f"{idx}. {ctx}")

RAG_Domain_know_doc.py

@@ -0,0 +1,165 @@
+import os
+import gradio as gr
+from openai import OpenAI
+import weaviate
+from weaviate.classes.init import Auth
+import pypdf  # Replaced PyPDF2
+import docx
+from langchain.text_splitter import RecursiveCharacterTextSplitter
+from dotenv import load_dotenv
+from prompt_template import (
+    Prompt_template_translation,
+    Prompt_template_LLM_Generation,
+    Prompt_template_Reranker,
+    Prompt_template_Wisal,
+    Prompt_template_Halluciations,
+    Prompt_template_paraphrasing,
+    Prompt_template_Translate_to_original,
+    Prompt_template_relevance
+)
+from query_utils import process_query_for_rewrite, get_non_autism_response
+# ─── Configuration ─────────────────────────────────────────────────────────────
+# helper functions
+GEMINI_API_KEY="AIzaSyCUCivstFpC9pq_jMHMYdlPrmh9Bx97dFo"
+
+TAVILY_API_KEY="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"
+
+OPENAI_API_KEY="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"
+
+QDRANT_API_KEY="eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJhY2Nlc3MiOiJtIiwiZXhwIjoxNzUxMDUxNzg4fQ.I9J-K7OM0BtcNKgj2d4uVM8QYAHYfFCVAyP4rlZkK2E"
+
+QDRANT_URL="https://6a3aade6-e8ad-4a6c-a579-21f5af90b7e8.us-east4-0.gcp.cloud.qdrant.io"
+
+OPENAI_API_KEY="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"
+
+WEAVIATE_URL="yorcqe2sqswhcaivxvt9a.c0.us-west3.gcp.weaviate.cloud"
+
+WEAVIATE_API_KEY="d2d0VGdZQTBmdTFlOWdDZl9tT2h3WDVWd1NpT1dQWHdGK0xjR1hYeWxicUxHVnFRazRUSjY2VlRUVlkwPV92MjAw"
+
+DEEPINFRA_API_KEY="285LUJulGIprqT6hcPhiXtcrphU04FG4"
+
+DEEPINFRA_BASE_URL="https://api.deepinfra.com/v1/openai"
+
+openai = OpenAI(
+    api_key=DEEPINFRA_API_KEY,
+    base_url="https://api.deepinfra.com/v1/openai",
+)
+# Initialize Weaviate client
+client = weaviate.connect_to_weaviate_cloud(
+    cluster_url=WEAVIATE_URL,
+    auth_credentials=Auth.api_key(WEAVIATE_API_KEY),
+    skip_init_checks=True,   # <-- This disables gRPC check
+
+)
+# ─── Utility: Extract raw text ──────────────────────────────────────────────────
+def extract_text(file_path: str) -> str:
+    ext = os.path.splitext(file_path)[1].lower()
+    if ext == ".pdf":
+        text = ""
+        with open(file_path, "rb") as f:
+            reader = pypdf.PdfReader(f)
+            for page in reader.pages:
+                page_text = page.extract_text() or ""
+                text += page_text + "\n"
+    elif ext == ".docx":
+        doc = docx.Document(file_path)
+        text = "\n".join(p.text for p in doc.paragraphs)
+    elif ext == ".txt":
+        with open(file_path, "r", encoding="utf-8") as f:
+            text = f.read()
+    else:
+        raise ValueError("Unsupported file format. Use PDF, DOCX, or TXT.")
+    return text
+# ─── Chunker & Embed ──────────────────────────────────────────────────────────
+splitter = RecursiveCharacterTextSplitter(
+    chunk_size=1000,
+    chunk_overlap=200,
+    separators=["\n\n", "\n", " "],
+)
+def embed_texts(texts: list[str], batch_size: int = 50) -> list[list[float]]:
+    """Embed texts in batches to avoid API limits."""
+    all_embeddings = []
+    for i in range(0, len(texts), batch_size):
+        batch = texts[i:i + batch_size]
+        resp = openai.embeddings.create(
+            model="Qwen/Qwen3-Embedding-8B",
+            input=batch,
+            encoding_format="float"
+        )
+        all_embeddings.extend([item.embedding for item in resp.data])
+    return all_embeddings
+# ─── Ingest & Index ───────────────────────────────────────────────────────────
+def ingest_file(file_path: str) -> str:
+    raw = extract_text(file_path)
+    docs = splitter.split_text(raw)
+    texts = [chunk for chunk in docs]
+    vectors = embed_texts(texts)
+    # Get the collection
+    documents = client.collections.get("Books")
+    # Batch insert with new API
+    with client.batch.dynamic() as batch:
+        for txt, vec in zip(texts, vectors):
+            batch.add_object(
+                collection="Books",
+                properties={"text": txt},
+                vector=vec
+            )
+    return f"Ingested {len(texts)} chunks from {os.path.basename(file_path)}"
+# ─── Query & Answer ───────────────────────────────────────────────────────────
+def answer_question(question: str) -> str:
+    # Process query for rewriting and relevance checking
+    corrected_query, is_autism_related, rewritten_query = process_query_for_rewrite(question)
+    
+    # If not autism-related, show direct rejection message
+    if not is_autism_related:
+        return get_non_autism_response()
+    
+    # Use the corrected query for retrieval
+    q_vec = embed_texts([corrected_query])[0]
+    documents = client.collections.get("Books")
+    response = documents.query.near_vector(
+        near_vector=q_vec,
+        limit=5,
+        return_metadata=["distance"]
+    )
+    hits = response.objects
+    context = "\n\n".join(hit.properties["text"] for hit in hits)
+    print(context)
+    wisal_prompt = Prompt_template_Wisal.format(new_query=corrected_query, document=context)
+    chat = openai.chat.completions.create(
+        model="Qwen/Qwen3-32B",
+        messages=[
+            {"role": "user", "content": wisal_prompt
+            }
+        ],
+        temperature=0,
+        reasoning_effort="none"
+    )
+    initial_answer = chat.choices[0].message.content
+    
+    # NEW: Check if the generated answer is sufficiently related to autism
+    from query_utils import check_answer_autism_relevance, get_non_autism_answer_response
+    
+    answer_relevance_score = check_answer_autism_relevance(initial_answer)
+    
+    # If answer relevance is below 50%, refuse the answer (updated threshold for enhanced scoring)
+    if answer_relevance_score < 50:
+        return get_non_autism_answer_response()
+    
+    # If sufficiently autism-related, return the answer
+    return initial_answer
+# ─── Gradio Interface ─────────────────────────────────────────────────────────
+with gr.Blocks(title="Document Q&A with Qwen & Weaviate") as demo:
+    gr.Markdown("## Upload a PDF, DOCX, or TXT and then ask away!")
+    with gr.Row():
+        up = gr.File(label="Select document")
+        btn = gr.Button("Ingest")
+        out = gr.Textbox(label="Status", interactive=False)
+    btn.click(fn=lambda f: ingest_file(f.name), inputs=up, outputs=out)
+    with gr.Row():
+        q = gr.Textbox(placeholder="Your question...", lines=2)
+        ask = gr.Button("Ask")
+        ans = gr.Textbox(label="Answer", lines=6, interactive=False)
+    ask.click(fn=answer_question, inputs=q, outputs=ans)
+if __name__ == "__main__":
+    demo.launch(debug=True)

User_Specific_Documents.py

@@ -0,0 +1,167 @@
+import os
+import gradio as gr
+from openai import OpenAI
+import weaviate
+from weaviate.classes.init import Auth
+import pypdf  # Replaced PyPDF2
+import docx
+from langchain.text_splitter import RecursiveCharacterTextSplitter
+from dotenv import load_dotenv
+from prompt_template import (
+    Prompt_template_translation,
+    Prompt_template_LLM_Generation,
+    Prompt_template_Reranker,
+    Prompt_template_Wisal,
+    Prompt_template_Halluciations,
+    Prompt_template_paraphrasing,
+    Prompt_template_Translate_to_original,
+    Prompt_template_relevance,
+    Prompt_template_User_document_prompt
+)
+from query_utils import process_query_for_rewrite, get_non_autism_response
+# ─── Configuration ─────────────────────────────────────────────────────────────
+# helper functions
+GEMINI_API_KEY="AIzaSyCUCivstFpC9pq_jMHMYdlPrmh9Bx97dFo"
+
+TAVILY_API_KEY="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"
+
+OPENAI_API_KEY="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"
+
+QDRANT_API_KEY="eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJhY2Nlc3MiOiJtIiwiZXhwIjoxNzUxMDUxNzg4fQ.I9J-K7OM0BtcNKgj2d4uVM8QYAHYfFCVAyP4rlZkK2E"
+
+QDRANT_URL="https://6a3aade6-e8ad-4a6c-a579-21f5af90b7e8.us-east4-0.gcp.cloud.qdrant.io"
+
+OPENAI_API_KEY="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"
+
+WEAVIATE_URL="yorcqe2sqswhcaivxvt9a.c0.us-west3.gcp.weaviate.cloud"
+
+WEAVIATE_API_KEY="d2d0VGdZQTBmdTFlOWdDZl9tT2h3WDVWd1NpT1dQWHdGK0xjR1hYeWxicUxHVnFRazRUSjY2VlRUVlkwPV92MjAw"
+
+DEEPINFRA_API_KEY="285LUJulGIprqT6hcPhiXtcrphU04FG4"
+
+DEEPINFRA_BASE_URL="https://api.deepinfra.com/v1/openai"
+
+openai = OpenAI(
+    api_key=DEEPINFRA_API_KEY,
+    base_url="https://api.deepinfra.com/v1/openai",
+)
+# Initialize Weaviate client
+client = weaviate.connect_to_weaviate_cloud(
+    cluster_url=WEAVIATE_URL,
+    auth_credentials=Auth.api_key(WEAVIATE_API_KEY),
+    skip_init_checks=True,   # <-- This disables gRPC check
+
+)
+# ─── Utility: Extract raw text ──────────────────────────────────────────────────
+def extract_text(file_path: str) -> str:
+    ext = os.path.splitext(file_path)[1].lower()
+    if ext == ".pdf":
+        text = ""
+        with open(file_path, "rb") as f:
+            reader = pypdf.PdfReader(f)
+            for page in reader.pages:
+                page_text = page.extract_text() or ""
+                text += page_text + "\n"
+    elif ext == ".docx":
+        doc = docx.Document(file_path)
+        text = "\n".join(p.text for p in doc.paragraphs)
+    elif ext == ".txt":
+        with open(file_path, "r", encoding="utf-8") as f:
+            text = f.read()
+    else:
+        raise ValueError("Unsupported file format. Use PDF, DOCX, or TXT.")
+    return text
+# ─── Chunker & Embed ──────────────────────────────────────────────────────────
+splitter = RecursiveCharacterTextSplitter(
+    chunk_size=1000,
+    chunk_overlap=200,
+    separators=["\n\n", "\n", " "],
+)
+def embed_texts(texts: list[str], batch_size: int = 70) -> list[list[float]]:
+    """Embed texts in batches to avoid API limits."""
+    all_embeddings = []
+    for i in range(0, len(texts), batch_size):
+        batch = texts[i:i + batch_size]
+        resp = openai.embeddings.create(
+            model="Qwen/Qwen3-Embedding-8B",
+            input=batch,
+            encoding_format="float"
+        )
+        all_embeddings.extend([item.embedding for item in resp.data])
+    return all_embeddings
+# ─── Ingest & Index ───────────────────────────────────────────────────────────
+def ingest_file(file_path: str) -> str:
+    raw = extract_text(file_path)
+    docs = splitter.split_text(raw)
+    texts = [chunk for chunk in docs]
+    vectors = embed_texts(texts)
+    # Get the collection
+    documents = client.collections.get("user")
+    # Batch insert with new API
+    with client.batch.dynamic() as batch:
+        for txt, vec in zip(texts, vectors):
+            batch.add_object(
+                collection="user",
+                properties={"text": txt},
+                vector=vec
+            )
+    return f"Ingested {len(texts)} chunks from {os.path.basename(file_path)}"
+# ───────────────────────────────────────────── Query & Answer ───────────────────────────────────────────────────────────
+def answer_question(question: str) -> str:
+    # Process query for rewriting and relevance checking
+    corrected_query, is_autism_related, rewritten_query = process_query_for_rewrite(question)
+    
+    # If not autism-related, show direct rejection message
+    if not is_autism_related:
+        return get_non_autism_response()
+    
+    # Use the corrected query for retrieval
+    q_vec = embed_texts([corrected_query])[0]
+    documents = client.collections.get("user")
+    response = documents.query.near_vector(
+        near_vector=q_vec,
+        limit=5,
+        return_metadata=["distance"]
+    )
+    hits = response.objects
+    context = "\n\n".join(hit.properties["text"] for hit in hits)
+    print(context)
+    
+    UserSpecificDocument_prompt = Prompt_template_User_document_prompt.format(new_query=corrected_query, document=context)
+    chat = openai.chat.completions.create(
+        model="Qwen/Qwen3-32B",
+        messages=[
+            {"role": "user", "content": UserSpecificDocument_prompt
+            }
+        ],
+        temperature=0,
+        reasoning_effort="none"
+    )
+    initial_answer = chat.choices[0].message.content
+    
+    # NEW: Check if the generated answer is sufficiently related to autism
+    from query_utils import check_answer_autism_relevance, get_non_autism_answer_response
+    
+    answer_relevance_score = check_answer_autism_relevance(initial_answer)
+    
+    # If answer relevance is below 50%, refuse the answer (updated threshold for enhanced scoring)
+    if answer_relevance_score < 50:
+        return get_non_autism_answer_response()
+    
+    # If sufficiently autism-related, return the answer
+    return initial_answer
+# ─── Gradio Interface ─────────────────────────────────────────────────────────
+with gr.Blocks(title="Document Q&A with Qwen & Weaviate") as demo:
+    gr.Markdown("## Upload a PDF, DOCX, or TXT and then ask away!")
+    with gr.Row():
+        up = gr.File(label="Select document")
+        btn = gr.Button("Ingest")
+        out = gr.Textbox(label="Status", interactive=False)
+    btn.click(fn=lambda f: ingest_file(f.name), inputs=up, outputs=out)
+    with gr.Row():
+        q = gr.Textbox(placeholder="Your question...", lines=2)
+        ask = gr.Button("Ask")
+        ans = gr.Textbox(label="Answer", lines=6, interactive=False)
+    ask.click(fn=answer_question, inputs=q, outputs=ans)
+if __name__ == "__main__":
+    demo.launch(debug=True)

prompt_template.py

@@ -0,0 +1,519 @@
+import os
+import asyncio
+import nest_asyncio
+from dotenv import load_dotenv
+# A prompt used to divide the part of the specified user experience document.
+Prompt_template_Chunking = """
+You are a specialized document processing agent tasked with meticulously cleaning, structuring, and chunking raw text content originating from structured book sources (e.g., medical or diagnostic manuals). Adhere to the following strict guidelines to prepare this content for downstream applications such as training data, search indexing, or diagnostic referencing, ensuring absolute preservation of original semantic meaning and formatting.
+
+INSTRUCTIONS:
+1. CONTENT CLEANING:
+* REMOVE: All headers, footers, and page numbers, code like F84.0 and References.
+* PRESERVE: All original content, including all section titles, sub-titles, bullet points, numbered lists, and tables. Do not omit or alter any part of the original text.
+* DO NOT: Summarize, rephrase, paraphrase, or alter any part of the content. Maintain the exact original wording.
+
+2. CONTENT STRUCTURING:
+* IDENTIFY HEADERS: Recognize and utilize natural section headers (e.g., "Diagnostic Criteria", "Level 1", "Level 2", "Symptoms", "Treatment", "Prognosis", "Introduction", "Summary", "Methodology") as primary paragraph separators or markers for new logical blocks.
+* LOGICAL BREAKS: If explicit headers are not present, use logical breaks between distinct topics or complete ideas to segment the content.
+
+3. CONTENT CHUNKING:
+* PARAGRAPH LENGTH: Divide the cleaned and structured content into paragraphs, aiming for each paragraph to be approximately 300 to 500 words.
+* SENTENCE INTEGRITY: Absolutely do not split sentences or separate parts of the same complete idea across different paragraphs. A paragraph must contain whole, coherent ideas.
+* SHORTER SECTIONS: If a logical section (identified by a header or a complete idea) is naturally shorter than 300 words but represents a complete and standalone piece of information, retain it as-is without trying to pad it or merge it with unrelated content.
+
+4. TABLE FORMATTING:
+* PRESERVE EXACTLY: All tables must be preserved in their entirety, including all rows and columns.
+* MARKDOWN SYNTAX: Format all tables using standard Markdown table syntax.
+Example:
+| Column Header A | Column Header B |
+|-----------------|-----------------|
+| Row 1 Value A   | Row 1 Value B   |
+| Row 2 Value A   | Row 2 Value B   |
+
+5. NO INTERPRETATION OR EXTERNAL INFORMATION:
+* STRICTLY CONTENT-BASED: Do not interpret, rephrase, summarize, infer, rewrite, or add any external information, comments, or your own insights.
+* OBJECTIVE PROCESSING: Base all decisions and transformations purely on the content provided to you.
+
+Your response should be the cleaned, structured, and chunked content. Do not include any conversational filler, introductions, or conclusions; just the processed text.
+
+{pdf_chunk_text}
+"""
+######################################################################################################
+
+Prompt_template_translation = """
+
+You are a friendly AI assistant. For each incoming user query, do **only** this:
+
+
+
+1. Detect the query’s language.
+
+2. If it isn’t English, translate it into English.
+
+3. If it *is* English (or once translated), check for clarity & grammar. If the phrasing is unclear or ungrammatical, rephrase it into a precise, professional English sentence that preserves the original meaning.
+
+
+
+**Output**: the final, corrected English query—nothing else.
+
+Query: {query}
+
+"""
+
+#############################################################################################
+
+Prompt_template_relevance = """
+
+You are Wisal, an AI assistant specialized in Autism Spectrum Disorders (ASD).  
+
+Given the **corrected English query** from step 1, decide if it’s about ASD (e.g. symptoms, diagnosis, therapy, behavior in ASD).
+
+
+
+- If **yes**, respond with: `RELATED`
+
+- If **no**, respond with exactly:
+
+  “Hello I’m Wisal, an AI assistant developed by Compumacy AI, and a knowledgeable Autism specialist.  
+
+   If you have any question related to autism please submit a question specifically about autism.”
+
+
+**Do not** include any other text.
+
+Query: {corrected_query}
+
+"""
+
+#############################################################################################
+# Prompt_template_relevance = """
+# You are Wisal, an AI assistant specialized in Autism Spectrum Disorders (ASD).
+
+# Given a **corrected English query**, your task is to determine if it is specifically related to ASD — such as symptoms, diagnosis, therapies, behaviors, or other autism-related topics.
+
+# Follow these steps:
+
+# 1. If the query is clearly about Autism, respond with: `RELATED`
+
+# 2. If the query is general or unclear, try to rephrase it to be Autism-specific.  
+#    Example:  
+#    - Original: “What are some ways that parents can reduce their stress?”  
+#    - Rephrased: “What are some ways that parents of children with Autism can reduce their stress?”
+
+# 3. If the query cannot be meaningfully rephrased in the context of Autism, return the polite redirection:  
+#    **“Hello I’m Wisal, an AI assistant developed by Compumacy AI, and a knowledgeable Autism specialist.  
+#    If you have any question related to autism please submit a question specifically about autism.”**
+
+# **Do not add or include any other text.**
+
+# Query: {corrected_query}
+# """
+
+#############################################################################################
+# LLM Generation 
+Prompt_template_LLM_Generation = """
+You are Wisal, an AI assistant developed by Compumacy AI , and a knowledgeable Autism .And Question-Answering assistant specializing in Autism.When I ask a question related to Autism, respond with a clear, concise, and accurate answer.
+Question:{new_query}
+your Answer here 
+"""
+######################################################################################################
+
+Prompt_template_Reranker= """
+You are an impartial evaluator tasked with sorting and outputting text passages based on their semantic relevance to a given query. Your goal is to determine which passages most directly address the core meaning of the query.
+
+Instructions:
+You will be given a query and a list of 5 passages, each with a number identifier.
+Sort and output the passages from most relevant [1] to least relevant [5].
+Only provide the sorted output using the number identifiers and corresponding passage text.
+Do not include explanations, rewritten content, or extra commentary.
+Focus solely on semantic relevance — how directly the passage answers or relates to the query.
+
+Input Format:
+Query: {new_query}
+Passages:
+{answers_list}
+
+Output Format:
+[1] <passage number> <passage text>
+[2] <passage number> <passage text>
+[3] <passage number> <passage text>
+[4] <passage number> <passage text>
+[5] <passage number> <passage text>
+"""
+
+#####################################################################################################
+
+Prompt_template_Wisal= """
+      You are Wisal, an AI assistant developed by Compumacy AI , and a knowledgeable Autism .
+      Your sole purpose is to provide helpful, respectful, and easy-to-understand answers about Autism Spectrum Disorder (ASD).
+      Always be clear, non-judgmental, and supportive.
+      Question: {new_query}
+      Answer the question based only on the provided context:
+      {document}
+
+"""
+######################################################################################################################
+Prompt_template_paraphrasing= """
+Rephrase the following passage using different words but keep the original meaning. Focus on directness and vary the phrasing for the cause.
+Only give one single rephrased version — no explanations, no options.
+Text : {document}
+
+"""
+
+#########################################################################################################
+Prompt_template_Halluciations= """
+Evaluate how confident you are that the given Answer is a good and accurate response to the Question.
+Please assign a Score using the following 5-point scale:
+1: You are not confident that the Answer addresses the Question at all, the Answer may be entirely off-topic or irrelevant to the Question.
+2: You have low confidence that the Answer addresses the Question, there are doubts and uncertainties about the accuracy of the Answer.
+3: You have moderate confidence that the Answer addresses the Question, the Answer seems reasonably accurate and on-topic, but with room for improvement.
+4: You have high confidence that the Answer addresses the Question, the Answer provides accurate information that addresses most of the Question.
+5: You are extremely confident that the Answer addresses the Question, the Answer is highly accurate, relevant, and effectively addresses the Question in its entirety.
+The output should strictly use the following template: Explanation: [provide a brief reasoning you used to derive the rating Score] and then write 'Score: <rating>' on the last line.
+Question: {new_query}
+Context:{document}
+Answer: {answer}
+"""
+############################################################################################################
+
+Prompt_template_Translate_to_original= """
+You are a translation assistant. Whenever you receive a user Question, determine its language. Then take your Answer (which is currently in English or any other language) and:
+If the Question is in Arabic, translate the Answer into Arabic.
+Otherwise, translate the Answer into the same language as the Question.
+Requirements:
+Preserve the original tone and style exactly.
+Don’t add, remove, or change any content beyond translating.
+Do not include any extra commentary or explanations—output only the translated text.
+Question: {query}  
+Answer : {document}
+"""
+
+############################################################################################################
+Prompt_template_User_document_prompt = """
+
+You are Wisal, an AI assistant developed by Compumacy AI, specialized in autism. When a user asks a question, you must respond only by quoting verbatim from the provided document(s). Do not add any of your own words, summaries, explanations, or interpretations. If the answer cannot be found in the documents, reply with exactly:
+“Answer not found in the document.”
+Question: {new_query}
+Answer the question based only on the provided context:
+{document}
+
+
+"""
+# Prompt_template_Reranker= """
+# You are an expert evaluator tasked with rating how well a given document matches a user query. Assess the document across three specific dimensions and provide a total relevance score out of 10.
+
+# Please consider the following criteria:
+
+# 1. Direct Answer Relevance (0–5 points):
+#    - Does the document directly address the core of the query?
+#    - Higher scores reflect more focused and pertinent content.
+#    - A score of 5 means the answer is highly aligned with the query.
+
+# 2. Information Completeness (0–3 points):
+#    - Does the document provide sufficient detail or context to fully answer the question?
+#    - Is the response thorough and informative, rather than partial or vague?
+
+# 3. Factual Accuracy (0–2 points):
+#    - Are the statements in the document factually correct and reliable?
+#    - Deduct points if any part of the document contains inaccuracies, outdated info, or misleading claims.
+# Query:{query}
+
+# Document:{document}
+
+# """
+
+# Prompt_template_relevant= """
+#         You are a grader assessing relevance of a retrieved document to a user question.
+#         Here is the retrieved document: {document}
+#         Here is the user question: {new_query}
+#         If the document contains keyword(s) or semantic meaning related to the user question, grade it as relevant.
+#         Give a binary score 'yes' or 'no' to indicate whether the document is relevant to the question.
+# """
+
+# Prompt_template_Reranker_relevant = """
+# You are given a user question and two responses from two AI assistants. Your task is to act as an impartial judge
+# and evaluate which response better follows the user's instructions and provides a higher-quality answer.
+# First, provide your reasoning within <think> and </think> tags. This should include your evaluation criteria for
+# a high-quality response, a detailed comparison of the two responses, and when helpful, a reference answer as
+# part of your evaluation. Be explicit in your thought process, referencing your criteria and explaining how each
+# response aligns with or deviates from them.
+# Avoid any position biases and ensure that the order in which the responses were presented does not influence your
+# decision. Do not allow the length of the responses to influence your evaluation. Do not favor certain names of
+# the assistants. Be as objective as possible.
+# Finally, assign the assistant's response a score from 0 to 10, using either an integer or a decimal with up
+# to 0.1 precision, with a higher score indicating a higher-quality response that better satisfies the criteria.
+# Enclose the scores within the tags <score_A> </score_A>, and <score_B> </score_B>.
+# Format your output like this:
+# <think> your_thinking_process </think>
+# <score_A> your_score_a </score_A> <score_B> your_score_b </score_B>
+# Below are the user's question and the two responses:
+# [User Question]
+# {instruction}
+# {new_query}
+# [The Start of Assistant A's Answer]
+# {web_answer}
+# [The End of Assistant A's Answer]
+# [The Start of Assistant B's Answer]
+# {generated_answer}
+# [The End of Assistant B's Answer]
+# """
+
+
+
+# Prompt_template_Evaluation= """
+# SYSTEM: You are a mental health concept knowledge evaluator. Your task is to assess how accurately, completely, and clearly the candidate's response defines the concept provided in the "Answer" field, taking into account the clinical context in the "History."
+#       USER:
+#       INSTRUCTIONS:
+
+#       1.  Read the "Answer" — this is the clinical concept or term to define (e.g., "Loss of interest or pleasure in activities…").
+#       2.  Read the "Candidate Response" — the model's definition/explanation of that concept.
+#       3.  Evaluate the response on:
+#           Definition Accuracy & Completeness: Are all core features of the concept present and correctly described?
+#           Clarity & Precision: Is the explanation clear, unambiguous, and clinically precise?
+#           Depth of Explanation: Does it include relevant examples or elaborations that demonstrate understanding?
+#           Relevance & Focus: Does it avoid irrelevant details and stick to the concept at hand?
+#       4.  Provide a single numeric score between 0 and 100:
+#           0:No meaningful overlap—incorrect or missing core elements.
+#           50:Some correct elements but major omissions or inaccuracies.
+#           75: Mostly correct with only minor gaps or imprecisions.
+#           90:Very close to a perfect definition; only small details missing.
+#           100:Perfectly accurate, complete, and clear.
+
+#       Do not justify or explain—output **only** the numeric score.
+
+#       Now, evaluate the following:
+#       Concept to Define (Correct_Answer):
+#       {answer}
+#       Candidate Response (Response_Answer):
+#       {final_answer}
+# """
+
+############################################################################################################
+# ENHANCED PROMPTS FOR AUTISM CONFIDENCE SCORING AND AUTOMATIC REWRITING
+############################################################################################################
+
+Prompt_template_autism_confidence = """
+You are an autism specialist AI evaluating how related a query is to autism or Autism Spectrum Disorders (ASD).
+
+Analyze the following query and provide a confidence score from 0 to 100 indicating how related it is to autism, ASD, or autism-related topics.
+
+ENHANCED SCORING GUIDELINES:
+
+**90-100: DIRECTLY AUTISM-RELATED**
+- Explicitly mentions autism, ASD, Asperger's, autistic individuals
+- Autism-specific therapies (ABA, TEACCH, social skills training)
+- Autism diagnostic criteria or screening tools
+- Autism-specific accommodations or support strategies
+
+**75-89: HIGHLY AUTISM-RELEVANT (Core Symptoms & Characteristics)**
+- Social communication difficulties, pragmatic language issues
+- Sensory processing disorders, sensory seeking/avoiding behaviors
+- Repetitive behaviors, stimming, self-regulation strategies
+- Special interests, restricted interests, hyperfocus
+- Executive functioning challenges in developmental context
+- Theory of mind, perspective-taking difficulties
+
+**60-74: SIGNIFICANTLY AUTISM-RELEVANT (Common Comorbidities & Related Issues)**
+- Depression in children/adolescents/adults (very common in autism)
+- Anxiety disorders, social anxiety, specific phobias
+- ADHD symptoms, attention and hyperactivity issues
+- Sleep disorders, sleep difficulties in neurodevelopmental context
+- Mood regulation, emotional dysregulation, meltdowns
+- Self-harm behaviors, aggression in developmental context
+- Eating difficulties, food selectivity, feeding issues
+- Gastrointestinal problems in neurodevelopmental context
+- Toileting issues, developmental delays in self-care
+
+**45-59: MODERATELY AUTISM-RELEVANT (Broader Developmental & Family Concerns)**
+- General child development questions (when could apply to autism)
+- Parent stress, family coping with special needs
+- School accommodations, IEP/504 plans
+- Transition planning, life skills development
+- Communication aids, assistive technology
+- Behavioral interventions, positive behavior support
+- Inclusion strategies, peer relationships
+
+**30-44: SOMEWHAT AUTISM-RELEVANT (General Topics with Potential Autism Applications)**
+- General behavioral challenges in children
+- Learning differences, cognitive development
+- Social skills development (general)
+- Mental health in children/adolescents
+- Developmental milestones, early childhood development
+- Family therapy, counseling approaches
+
+**0-29: NOT AUTISM-RELEVANT**
+- Unrelated medical conditions (unless neurological/developmental)
+- General adult topics without developmental context
+- Physical health unrelated to common autism comorbidities
+- Non-developmental behavioral issues
+- Completely unrelated topics (weather, cooking, sports, etc.)
+
+**SPECIAL CONSIDERATIONS:**
+- Questions about depression, anxiety, ADHD should score 60+ due to high comorbidity rates
+- Sensory issues, sleep problems, mood regulation should score 65+
+- Parent/caregiver stress and coping should score 50+
+- School and educational topics should score 45+ 
+- If query mentions children, adolescents, or developmental context, add 10-15 points
+- Behavioral questions in pediatric context should score 45+
+
+**EXAMPLES OF INDIRECT BUT HIGHLY RELEVANT QUERIES:**
+- "My child has frequent meltdowns" → 70-80 (common autism behavior)
+- "How to help with depression in teenagers" → 65-75 (very common in autism)
+- "Sleep problems in children" → 65 (extremely common autism comorbidity)
+- "ADHD and focus issues" → 65 (high comorbidity with autism)
+- "Anxiety in social situations" → 70 (core autism challenge)
+- "Eating problems in kids" → 60 (common autism issue)
+- "Parent stress with special needs child" → 55 (autism family context)
+
+Query: {query}
+
+Consider the context, age implications, and potential autism connections before scoring.
+Output only the numeric confidence score (0-100):
+"""
+
+############################################################################################################
+
+Prompt_template_autism_rewriter = """
+You are an autism specialist AI assistant. Your task is to rewrite queries to make them specifically about autism or Autism Spectrum Disorders (ASD) while preserving the original intent and recognizing common comorbidities.
+
+ENHANCED REWRITING GUIDELINES:
+
+**For Direct Autism Topics (85-100% relevance):**
+- Keep as-is, just ensure clarity and proper terminology
+
+**For Core Autism Symptoms (70-84% relevance):**
+- Frame within autism context while preserving specificity
+- Examples: "sensory issues" → "sensory processing challenges in autism"
+
+**For Comorbid Conditions (55-69% relevance):**
+- Explicitly connect to autism while maintaining the specific condition focus
+- Recognize high comorbidity rates and autism-specific aspects
+
+**For Developmental/Family Concerns (40-54% relevance):**
+- Frame within autism family/developmental context
+- Emphasize autism-specific challenges and considerations
+
+**COMORBIDITY-AWARE REWRITING EXAMPLES:**
+
+Depression/Mental Health:
+- "How to help with depression in teenagers?" → "How to support teenagers with autism who are experiencing depression?"
+- "Managing anxiety" → "Managing anxiety in individuals with autism spectrum disorders"
+
+ADHD/Attention Issues:
+- "ADHD symptoms in children" → "Understanding ADHD symptoms in children with autism (dual diagnosis)"
+- "Focus and attention problems" → "Addressing attention and focus challenges in autism"
+
+Sleep & Behavioral Issues:
+- "Sleep problems in kids" → "Managing sleep difficulties in children with autism"
+- "Child having meltdowns" → "Understanding and managing meltdowns in autism"
+- "Aggressive behavior" → "Addressing aggressive behaviors in individuals with autism"
+
+Sensory & Regulatory Issues:
+- "Sensory processing problems" → "Sensory processing disorders in autism spectrum conditions"
+- "Emotional regulation" → "Supporting emotional regulation in autism"
+
+Family & Educational:
+- "Parent stress with special needs child" → "Supporting parents of children with autism: managing stress and building resilience"
+- "School accommodations" → "Educational accommodations and supports for students with autism"
+- "Social skills development" → "Social skills training and development for individuals with autism"
+
+Feeding & Development:
+- "Eating problems in children" → "Addressing feeding difficulties and food selectivity in autism"
+- "Developmental delays" → "Understanding developmental patterns and delays in autism spectrum disorders"
+
+**REWRITING PRINCIPLES:**
+1. Always maintain the specific concern (depression, sleep, behavior, etc.)
+2. Explicitly connect to autism context
+3. Use person-first or identity-first language appropriately
+4. Preserve the question type and intent
+5. Add autism-specific considerations when relevant
+6. For comorbid conditions, acknowledge the dual nature
+
+**AVOID:**
+- Generic "autism-related" phrases
+- Losing the specific concern in overly broad rewriting
+- Ignoring the comorbidity aspect
+- Making assumptions about causation
+
+Original Query: {query}
+
+Rewritten autism-specific query:
+"""
+
+############################################################################################################
+
+Prompt_template_answer_autism_relevance = """
+You are an autism specialist evaluating whether an answer is sufficiently related to autism or Autism Spectrum Disorders (ASD).
+
+Analyze the following answer and determine if it is adequately focused on autism-related content, including common comorbidities and associated conditions.
+
+ENHANCED AUTISM-RELEVANCE CRITERIA:
+
+**85-100: HIGHLY AUTISM-RELEVANT**
+- Directly mentions autism, ASD, autistic individuals, or autism-specific terms
+- Discusses autism-specific interventions, therapies, or strategies
+- Addresses autism diagnostic criteria or assessment
+- Covers autism-specific accommodations or support systems
+
+**70-84: STRONGLY AUTISM-RELEVANT** 
+- Discusses core autism characteristics (social communication, sensory processing, repetitive behaviors)
+- Addresses autism-related developmental patterns
+- Covers autism-specific educational or therapeutic approaches
+- Discusses autism family dynamics or support strategies
+
+**55-69: SIGNIFICANTLY AUTISM-RELEVANT**
+- Addresses common autism comorbidities IN DEVELOPMENTAL CONTEXT:
+  * Depression, anxiety, or mood disorders in children/adolescents
+  * ADHD symptoms or attention challenges in developmental context
+  * Sleep disorders with sensory or behavioral components
+  * Feeding/eating difficulties with sensory aspects
+  * Self-regulation or emotional dysregulation issues
+- Discusses developmental delays or milestones with autism implications
+- Addresses sensory processing issues (even without explicit autism mention)
+- Covers behavioral challenges common in autism (meltdowns, aggression, self-harm)
+
+**40-54: MODERATELY AUTISM-RELEVANT**
+- Discusses general developmental topics that frequently apply to autism
+- Addresses parent/caregiver stress in special needs context
+- Covers educational accommodations or special needs support
+- Discusses social skills development or peer relationships in developmental context
+- Addresses communication challenges or assistive technology
+
+**25-39: SOMEWHAT AUTISM-RELEVANT**
+- General child development information that could apply to autism
+- Basic behavioral strategies that might be relevant
+- General mental health information in pediatric context
+- Family therapy or support approaches
+
+**0-24: NOT SUFFICIENTLY AUTISM-RELEVANT**
+- Purely general medical information without developmental context
+- Adult-focused content without autism or developmental relevance
+- Generic advice without special needs consideration
+- Completely unrelated topics
+
+**SPECIAL AUTISM-RELEVANCE INDICATORS:**
+✓ Mentions sensory processing, regulation, or sensory-seeking/avoiding
+✓ Discusses meltdowns, stimming, or repetitive behaviors
+✓ Addresses social communication or pragmatic language
+✓ Covers executive functioning in developmental context
+✓ Mentions special interests or restricted interests
+✓ Discusses transition planning or life skills development
+✓ Addresses inclusion, accommodations, or accessibility
+✓ Covers co-occurring conditions with autism-specific considerations
+
+**COMORBIDITY CONTEXT CLUES:**
+- Depression/anxiety discussed with developmental, social, or sensory factors
+- ADHD mentioned with autism-like symptoms or dual diagnosis considerations
+- Sleep issues connected to sensory processing or routine needs
+- Feeding problems involving texture, sensory, or routine aspects
+- Behavioral issues described with developmental or environmental triggers
+
+**THRESHOLD FOR ACCEPTANCE:**
+- 50-100: Adequately autism-relevant, should be provided
+- 0-49: Not sufficiently autism-focused, should be refused
+
+Answer to evaluate: {answer}
+
+Consider the developmental context, autism comorbidities, and indirect autism connections.
+Output only the numeric relevance score (0-100):
+"""

query_utils.py

@@ -0,0 +1,314 @@
+"""
+Utility functions for query processing and rewriting.
+"""
+import time
+import logging
+from openai import OpenAI
+from prompt_template import (
+    Prompt_template_translation, 
+    Prompt_template_relevance,
+    Prompt_template_autism_confidence,
+    Prompt_template_autism_rewriter,
+    Prompt_template_answer_autism_relevance
+)
+
+# Set up logging
+logging.basicConfig(level=logging.INFO)
+logger = logging.getLogger(__name__)
+
+# Initialize OpenAI client
+DEEPINFRA_API_KEY = "285LUJulGIprqT6hcPhiXtcrphU04FG4"
+openai = OpenAI(
+    api_key=DEEPINFRA_API_KEY,
+    base_url="https://api.deepinfra.com/v1/openai",
+)
+
+def call_llm(model: str, messages: list[dict], temperature: float = 0.0, timeout: int = 30, **kwargs) -> str:
+    """Call the LLM with given messages and return the response."""
+    try:
+        logger.info(f"Making API call to {model} with timeout {timeout}s")
+        start_time = time.time()
+        
+        resp = openai.chat.completions.create(
+            model=model,
+            messages=messages,
+            temperature=temperature,
+            timeout=timeout,
+            **kwargs
+        )
+        
+        elapsed = time.time() - start_time
+        logger.info(f"API call completed in {elapsed:.2f}s")
+        
+        return resp.choices[0].message.content.strip()
+        
+    except Exception as e:
+        logger.error(f"API call failed: {e}")
+        # Return fallback response
+        if "translation" in str(messages).lower():
+            # For translation, return the original query
+            return messages[0]["content"].split("Query: ")[-1] if "Query: " in messages[0]["content"] else "Error"
+        else:
+            # For relevance, assume not related
+            return "0"
+
+def enhanced_autism_relevance_check(query: str) -> dict:
+    """
+    Enhanced autism relevance checking with detailed analysis.
+    Returns a dictionary with score, category, and reasoning.
+    """
+    try:
+        logger.info(f"Enhanced autism relevance check for: '{query[:50]}...'")
+        
+        # Use the enhanced confidence prompt
+        confidence_prompt = Prompt_template_autism_confidence.format(query=query)
+        response = call_llm(
+            model="Qwen/Qwen3-32B",
+            messages=[{"role": "user", "content": confidence_prompt}],
+            reasoning_effort="none",
+            timeout=15
+        )
+        
+        # Extract numeric score
+        confidence_score = 0
+        try:
+            import re
+            numbers = re.findall(r'\d+', response)
+            if numbers:
+                confidence_score = int(numbers[0])
+                confidence_score = max(0, min(100, confidence_score))
+        except:
+            confidence_score = 0
+        
+        # Determine category and action based on enhanced scoring
+        if confidence_score >= 85:
+            category = "directly_autism_related"
+            action = "accept_as_is"
+            reasoning = "Directly mentions autism or autism-specific topics"
+        elif confidence_score >= 70:
+            category = "highly_autism_relevant"
+            action = "accept_as_is"
+            reasoning = "Core autism symptoms or characteristics"
+        elif confidence_score >= 55:
+            category = "significantly_autism_relevant"
+            action = "rewrite_for_autism"
+            reasoning = "Common comorbidity or autism-related issue"
+        elif confidence_score >= 40:
+            category = "moderately_autism_relevant"
+            action = "rewrite_for_autism"
+            reasoning = "Broader developmental or family concern related to autism"
+        elif confidence_score >= 25:
+            category = "somewhat_autism_relevant"
+            action = "conditional_rewrite"
+            reasoning = "General topic with potential autism applications"
+        else:
+            category = "not_autism_relevant"
+            action = "reject"
+            reasoning = "Not related to autism or autism care"
+        
+        result = {
+            "score": confidence_score,
+            "category": category,
+            "action": action,
+            "reasoning": reasoning
+        }
+        
+        logger.info(f"Enhanced relevance result: {result}")
+        return result
+        
+    except Exception as e:
+        logger.error(f"Error in enhanced_autism_relevance_check: {e}")
+        return {
+            "score": 0,
+            "category": "error",
+            "action": "reject",
+            "reasoning": "Error during processing"
+        }
+
+def check_autism_confidence(query: str) -> int:
+    """
+    Check autism relevance confidence score (0-100).
+    Returns the confidence score as an integer.
+    """
+    try:
+        logger.info(f"Checking autism confidence for query: '{query[:50]}...'")
+        
+        confidence_prompt = Prompt_template_autism_confidence.format(query=query)
+        response = call_llm(
+            model="Qwen/Qwen3-32B",
+            messages=[{"role": "user", "content": confidence_prompt}],
+            reasoning_effort="none",
+            timeout=15
+        )
+        
+        # Extract numeric score from response
+        confidence_score = 0
+        try:
+            # Try to extract number from response
+            import re
+            numbers = re.findall(r'\d+', response)
+            if numbers:
+                confidence_score = int(numbers[0])
+                # Ensure it's within valid range
+                confidence_score = max(0, min(100, confidence_score))
+            else:
+                logger.warning(f"No numeric score found in response: {response}")
+                confidence_score = 0
+        except:
+            logger.error(f"Failed to parse confidence score from: {response}")
+            confidence_score = 0
+            
+        logger.info(f"Autism confidence score: {confidence_score}")
+        return confidence_score
+        
+    except Exception as e:
+        logger.error(f"Error in check_autism_confidence: {e}")
+        return 0
+
+def rewrite_query_for_autism(query: str) -> str:
+    """
+    Automatically rewrite a query to be autism-specific.
+    """
+    try:
+        logger.info(f"Rewriting query for autism: '{query[:50]}...'")
+        
+        rewrite_prompt = Prompt_template_autism_rewriter.format(query=query)
+        rewritten_query = call_llm(
+            model="Qwen/Qwen3-32B",
+            messages=[{"role": "user", "content": rewrite_prompt}],
+            reasoning_effort="none",
+            timeout=15
+        )
+        
+        if rewritten_query == "Error" or len(rewritten_query.strip()) == 0:
+            logger.warning("Rewriting failed, using fallback")
+            rewritten_query = f"How does autism relate to {query.lower()}?"
+        else:
+            rewritten_query = rewritten_query.strip()
+            
+        logger.info(f"Query rewritten to: '{rewritten_query[:50]}...'")
+        return rewritten_query
+        
+    except Exception as e:
+        logger.error(f"Error in rewrite_query_for_autism: {e}")
+        return f"How does autism relate to {query.lower()}?"
+
+def check_answer_autism_relevance(answer: str) -> int:
+    """
+    Check if an answer is sufficiently related to autism (0-100 score).
+    Used for document-based queries to filter non-autism answers.
+    """
+    try:
+        logger.info(f"Checking answer autism relevance for: '{answer[:50]}...'")
+        
+        relevance_prompt = Prompt_template_answer_autism_relevance.format(answer=answer)
+        response = call_llm(
+            model="Qwen/Qwen3-32B",
+            messages=[{"role": "user", "content": relevance_prompt}],
+            reasoning_effort="none",
+            timeout=15
+        )
+        
+        # Extract numeric score from response
+        relevance_score = 0
+        try:
+            import re
+            numbers = re.findall(r'\d+', response)
+            if numbers:
+                relevance_score = int(numbers[0])
+                relevance_score = max(0, min(100, relevance_score))
+            else:
+                logger.warning(f"No numeric score found in response: {response}")
+                relevance_score = 0
+        except:
+            logger.error(f"Failed to parse relevance score from: {response}")
+            relevance_score = 0
+            
+        logger.info(f"Answer autism relevance score: {relevance_score}")
+        return relevance_score
+        
+    except Exception as e:
+        logger.error(f"Error in check_answer_autism_relevance: {e}")
+        return 0
+
+def process_query_for_rewrite(query: str) -> tuple[str, bool, str]:
+    """
+    Enhanced query processing with sophisticated autism relevance detection.
+    
+    NEW ENHANCED LOGIC:
+    1. Score 85-100 → Directly autism-related, use as-is
+    2. Score 70-84 → Highly autism-relevant (core symptoms), use as-is  
+    3. Score 55-69 → Significantly autism-relevant (comorbidities), rewrite for autism
+    4. Score 40-54 → Moderately autism-relevant, rewrite for autism
+    5. Score 25-39 → Somewhat relevant, conditional rewrite (ask user or auto-rewrite)
+    6. Score 0-24 → Not autism-related, reject
+    
+    Returns: (processed_query, is_autism_related, rewritten_query_if_needed)
+    """
+    try:
+        logger.info(f"Processing query with enhanced confidence logic: '{query[:50]}...'")
+        start_time = time.time()
+        
+        # Step 1: Translate and correct the query
+        logger.info("Step 1: Translating/correcting query")
+        corrected_query = call_llm(
+            model="Qwen/Qwen3-32B",
+            messages=[{"role": "user", "content": Prompt_template_translation.format(query=query)}],
+            reasoning_effort="none",
+            timeout=15
+        )
+        
+        if corrected_query == "Error":
+            logger.warning("Translation failed, using original query")
+            corrected_query = query
+        
+        # Step 2: Get enhanced autism relevance analysis
+        logger.info("Step 2: Enhanced autism relevance checking")
+        relevance_result = enhanced_autism_relevance_check(corrected_query)
+        
+        confidence_score = relevance_result["score"]
+        action = relevance_result["action"]
+        reasoning = relevance_result["reasoning"]
+        
+        logger.info(f"Relevance analysis: {confidence_score}% - {reasoning}")
+        
+        # Step 3: Take action based on enhanced analysis
+        if action == "accept_as_is":
+            logger.info(f"High relevance ({confidence_score}%) - accepting as-is: {reasoning}")
+            return corrected_query, True, ""
+            
+        elif action == "rewrite_for_autism":
+            logger.info(f"Moderate relevance ({confidence_score}%) - rewriting for autism: {reasoning}")
+            rewritten_query = rewrite_query_for_autism(corrected_query)
+            return rewritten_query, True, ""
+            
+        elif action == "conditional_rewrite":
+            # For somewhat relevant queries, automatically rewrite (could be enhanced with user confirmation)
+            logger.info(f"Low-moderate relevance ({confidence_score}%) - conditionally rewriting: {reasoning}")
+            rewritten_query = rewrite_query_for_autism(corrected_query)
+            return rewritten_query, True, ""
+            
+        else:  # action == "reject"
+            logger.info(f"Low relevance ({confidence_score}%) - rejecting: {reasoning}")
+            return corrected_query, False, ""
+        
+        elapsed = time.time() - start_time
+        logger.info(f"Enhanced query processing completed in {elapsed:.2f}s")
+        
+    except Exception as e:
+        logger.error(f"Error in process_query_for_rewrite: {e}")
+        # Fallback: return original query as not autism-related
+        return query, False, ""
+
+def get_non_autism_response() -> str:
+    """Return a more human-like response for non-autism queries."""
+    return ("Hi there! I appreciate you reaching out to me. I'm Wisal, and I specialize specifically in autism and Autism Spectrum Disorders. "
+            "I noticed your question isn't quite related to autism topics. I'd love to help you, but I'm most effective when answering "
+            "questions about autism, ASD, autism support strategies, therapies, or related concerns.\n\n"
+            "Could you try asking me something about autism instead? I'm here and ready to help with any autism-related questions you might have! 😊")
+
+def get_non_autism_answer_response() -> str:
+    """Return a more human-like response when document answers are not autism-related."""
+    return ("I'm sorry, but the information I found in the document doesn't seem to be related to autism or Autism Spectrum Disorders. "
+            "Since I'm Wisal, your autism specialist, I want to make sure I'm providing you with relevant, autism-focused information. "
+            "Could you try asking a question that's more specifically about autism? I'm here to help with any autism-related topics! 😊") 

web_search.py

@@ -0,0 +1,83 @@
+import os
+import asyncio
+import httpx
+import nest_asyncio
+from dotenv import load_dotenv
+
+# Apply nested asyncio patch
+nest_asyncio.apply()
+
+# helper functions
+GEMINI_API_KEY="AIzaSyCUCivstFpC9pq_jMHMYdlPrmh9Bx97dFo"
+
+TAVILY_API_KEY="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"
+
+OPENAI_API_KEY="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"
+
+QDRANT_API_KEY="eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJhY2Nlc3MiOiJtIiwiZXhwIjoxNzUxMDUxNzg4fQ.I9J-K7OM0BtcNKgj2d4uVM8QYAHYfFCVAyP4rlZkK2E"
+
+QDRANT_URL="https://6a3aade6-e8ad-4a6c-a579-21f5af90b7e8.us-east4-0.gcp.cloud.qdrant.io"
+
+OPENAI_API_KEY="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"
+
+WEAVIATE_URL="yorcqe2sqswhcaivxvt9a.c0.us-west3.gcp.weaviate.cloud"
+
+WEAVIATE_API_KEY="d2d0VGdZQTBmdTFlOWdDZl9tT2h3WDVWd1NpT1dQWHdGK0xjR1hYeWxicUxHVnFRazRUSjY2VlRUVlkwPV92MjAw"
+
+DEEPINFRA_API_KEY="285LUJulGIprqT6hcPhiXtcrphU04FG4"
+
+DEEPINFRA_BASE_URL="https://api.deepinfra.com/v1/openai"
+
+# Try to import tavily with fallback
+try:
+    from tavily import TavilyClient
+    tavily_client = TavilyClient(api_key=TAVILY_API_KEY)
+    TAVILY_AVAILABLE = True
+except ImportError:
+    print("Warning: Tavily package not found. Web search will use fallback mode.")
+    tavily_client = None
+    TAVILY_AVAILABLE = False
+
+async def search_autism(query: str) -> dict:
+    """Performs a web search for information about autism."""
+    
+    if not TAVILY_AVAILABLE:
+        print("Web search unavailable - tavily package not installed")
+        return {
+            "results": [],
+            "answer": "Web search functionality is currently unavailable. Please ensure all dependencies are installed."
+        }
+    
+    try:
+        # Execute a search query using Tavily
+        response = tavily_client.search(
+            query=query,
+            max_results=5,
+            search_depth="advanced",
+            topic="general",
+            include_answer=True
+        )
+        return {
+            "results": response.get("results", []),
+            "answer": response.get("answer", "")
+        }
+    except Exception as e:
+        print(f"Search error: {str(e)}")
+        return {
+            "results": [],
+            "answer": f"Unable to perform web search: {str(e)}"
+        }
+
+# Test function for development
+async def main():
+    query = "autism symptoms and treatments"
+    result = await search_autism(query)
+    print("Search Results:")
+    for res in result.get("results", []):
+        print(f"- {res.get('title')} ({res.get('url')})")
+    print("\nAnswer:")
+    print(result.get("answer", "No answer provided."))
+
+# Run the script
+if __name__ == "__main__":
+    asyncio.run(main())