Resolveed correference betwen query

Rag/rag_pipeline.py

@@ -6,7 +6,7 @@ import os
 import logging
 from Llm.llm_endpoints import get_llm_response
 from utils.get_link import get_source_link
-# from Rag.corefrence import resolve_coreference_in_query
+from utils.corefrence import resolve_corefrence
 # Configuration
 API_KEY = os.getenv("GOOGLE_API_KEY")
 if API_KEY:
@@ -134,7 +134,8 @@ def main_workflow(transcripts_folder_path, collection):
         if query_text.lower() == "exit":
             print("Ending the conversation. Goodbye")
             break
-        query_text_with_conversation_history = enhance_query_with_history(query_text, conversation_history)
+        resolved_query = resolve_corefrence(query_text, conversation_history)
+        query_text_with_conversation_history = enhance_query_with_history(resolved_query, conversation_history)
         # resolved_query = resolve_coreference_in_query(query_text_with_conversation_history, conversation_history)
         retrived_docs, metadatas = query_database(collection, query_text_with_conversation_history)
         print("-" * 50)

requirements.txt

@@ -6,11 +6,13 @@ langchain
 langchain_openai
 langchain_chroma
 langchain_community
-chromadb
+chromadb==0.4.8
 pypdf
 flask
 flask_cors
 sentence_transformers
 tqdm
 torch
-transformers
+transformers
+spacy==3.5.0
+coreferee==1.4.1

utils/corefrence.py

@@ -1,11 +1,14 @@
-from transformers import pipeline
-
-coref_pipeline = pipeline("coref-resolution", model="coref-roberta-large")
-
-
-def resolve_coreference_in_query(query_text, conversation_history):
-    context = "\n".join([f"User: {turn['user']}\nBot: {turn['bot']}" for turn in conversation_history])
-    full_text = f"{context}\nUser: {query_text}"
-    resolved_text = coref_pipeline(full_text)
-    resolved_query = resolved_text.split("User:")[-1].strip()
-    return resolved_query
+import spacy
+nlp = spacy.load('en_core_web_sm')
+nlp.add_pipe("coreferee")
+def resolve_corefrence(query_text, conversation_history):
+    combined_text = []
+    for turn in conversation_history:
+        combined_text.append(f"User:{turn['user']}")
+        combined_text.append(f"Bot:{turn['Bot']}")
+    combined_text.append(f"User:{query_text}")
+    combined_text = "\n".join(combined_text)
+    doc = nlp(combined_text)
+    resolved_text = doc._.corefrence_resolved
+    resolved_query = resolved_text.split('\n')[-1].replace("User: ", "")
+    return resolved_query.strip()