Initial commit

LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2024 Candid
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

README.md

@@ -1,14 +1,13 @@
 ---
 title: Ask Candid
-emoji: 🚀
-colorFrom: gray
-colorTo: yellow
+short_description: AI assistant for philanthropy and the social sector
+emoji: 💬
+colorFrom: blue
+colorTo: purple
 sdk: gradio
-sdk_version: 5.6.0
+sdk_version: 5.5.0
 app_file: app.py
-pinned: false
+pinned: true
 license: mit
-short_description: AI assistant for philanthropy and the social sector
 ---
-
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
+Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference

app.py

@@ -0,0 +1,109 @@
+from typing import List, Tuple, Dict, TypedDict, Optional, Any
+import os
+
+import gradio as gr
+
+from langchain_openai.chat_models import ChatOpenAI
+
+try:
+    from utils import format_chat_ag_response
+    from retrieval.config import ALL_INDICES
+    from chat import run_chat
+except ImportError:
+    from .utils import format_chat_ag_response
+    from .retrieval.config import ALL_INDICES
+    from .chat import run_chat
+
+ROOT = os.path.dirname(os.path.abspath(__file__))
+
+
+class LoggedComponents(TypedDict):
+    context: List[gr.components.Component]
+    found_helpful: gr.components.Component
+    will_recommend: gr.components.Component
+    comments: gr.components.Component
+    email: gr.components.Component
+
+
+def execute(
+    thread_id: str,
+    user_input: Dict[str, Any],
+    chatbot: List[Dict],
+    max_new_tokens: int,
+    indices: Optional[List[str]] = None,
+):
+    llm = ChatOpenAI(
+        model_name="gpt-4o",
+        max_tokens=max_new_tokens,
+        api_key=os.getenv("OPENAI_API_KEY"),
+        temperature=0.0,
+        streaming=True
+    )
+
+    return run_chat(
+        thread_id=thread_id,
+        user_input=user_input,
+        chatbot=chatbot,
+        llm=llm,
+        indices=indices
+    )
+
+
+def build_chat() -> Tuple[LoggedComponents, gr.Blocks]:
+    with gr.Blocks(theme=gr.themes.Soft(), title="Ask Candid") as demo:
+        with gr.Accordion(label="Advanced settings", open=False):
+            es_indices = gr.CheckboxGroup(
+                choices=list(ALL_INDICES),
+                value=list(ALL_INDICES),
+                label="Sources to include",
+                interactive=True
+            )
+            max_new_tokens = gr.Slider(
+                value=256 * 3, minimum=128, maximum=2048, step=128,
+                label="Max new tokens", interactive=True
+            )
+
+        with gr.Column():
+            chatbot = gr.Chatbot(
+                label="Candid Assistant",
+                elem_id="chatbot",
+                bubble_full_width=False,
+                avatar_images=(
+                    None,
+                    os.path.join(ROOT, "static", "candid_logo_yellow.png")
+                ),
+                height="45vh",
+                type="messages",
+                show_label=False,
+                show_copy_button=True,
+                show_share_button=True,
+                show_copy_all_button=True
+            )
+            msg = gr.MultimodalTextbox(label="Your message", interactive=True)
+            thread_id = gr.Text(visible=False, value="", label="thread_id")
+            gr.ClearButton(components=[msg, chatbot, thread_id], size="sm")
+
+        # pylint: disable=no-member
+        chat_msg = msg.submit(
+            fn=execute,
+            inputs=[thread_id, msg, chatbot, max_new_tokens, es_indices],
+            outputs=[msg, chatbot, thread_id]
+        )
+        chat_msg.then(format_chat_ag_response, chatbot, chatbot, api_name="bot_response")
+        logged = LoggedComponents(
+            context=[thread_id, chatbot]
+        )
+    return logged, demo
+
+
+if __name__ == '__main__':
+    _, app = build_chat()
+    app.queue(max_size=5).launch(
+        show_api=False,
+        auth=[
+            (os.getenv("APP_USERNAME"), os.getenv("APP_PASSWORD")),
+            (os.getenv("APP_PUBLIC_USERNAME"), os.getenv("APP_PUBLIC_PASSWORD")),
+        ],
+        auth_message="Login to Candid's AI assistant",
+        ssr_mode=False
+    )

chat.py

@@ -0,0 +1,252 @@
+from typing import List, Optional, Dict, Any, TypedDict, Annotated, Sequence
+from functools import partial
+import os
+
+import gradio as gr
+
+from langchain_core.messages import AIMessage, BaseMessage
+from langchain_core.output_parsers import StrOutputParser
+from langchain_core.prompts import ChatPromptTemplate
+from langchain_core.language_models.llms import LLM
+
+from langgraph.prebuilt import tools_condition, ToolNode
+from langgraph.checkpoint.memory import MemorySaver
+from langgraph.graph.state import StateGraph
+from langgraph.graph.message import add_messages
+from langgraph.constants import START, END
+
+try:
+    from utils import html_format_docs_chat, get_session_id
+    from tools.question_reformulation import reformulate_question_using_history
+    from tools.org_seach import (
+        extract_org_links_from_chatbot,
+        embed_org_links_in_text,
+        generate_org_link_dict,
+    )
+    from retrieval.elastic import retriever_tool
+except ImportError:
+    from .utils import html_format_docs_chat, get_session_id
+    from .tools.question_reformulation import reformulate_question_using_history
+    from .tools.org_seach import (
+        extract_org_links_from_chatbot,
+        embed_org_links_in_text,
+        generate_org_link_dict,
+    )
+    from .retrieval.elastic import retriever_tool
+
+ROOT = os.path.dirname(os.path.abspath(__file__))
+
+# TODO https://www.metadocs.co/2024/08/29/simple-domain-specific-corrective-rag-with-langchain-and-langgraph/
+
+
+class AgentState(TypedDict):
+    # The add_messages function defines how an update should be processed
+    # Default is to replace. add_messages says "append"
+    messages: Annotated[Sequence[BaseMessage], add_messages]
+    user_input: str
+    org_dict: Dict
+
+
+def search_agent(state, llm: LLM, tools) -> AgentState:
+    """Invokes the agent model to generate a response based on the current state. Given
+    the question, it will decide to retrieve using the retriever tool, or simply end.
+
+    Parameters
+    ----------
+    state : _type_
+        The current state
+    llm : LLM
+    tools : _type_
+        _description_
+
+    Returns
+    -------
+    AgentState
+        The updated state with the agent response appended to messages
+    """
+
+    print("---SEARCH AGENT---")
+    messages = state["messages"]
+    question = messages[-1].content
+
+    model = llm.bind_tools(tools)
+    response = model.invoke(messages)
+    # return a list, because this will get added to the existing list
+    return {"messages": [response], "user_input": question}
+
+
+def generate_with_context(state, llm: LLM) -> AgentState:
+    """Generate answer.
+
+    Parameters
+    ----------
+    state : _type_
+        The current state
+    llm : LLM
+    tools : _type_
+        _description_
+
+    Returns
+    -------
+    AgentState
+        The updated state with the agent response appended to messages
+    """
+
+    print("---GENERATE ANSWER---")
+    messages = state["messages"]
+    question = state["user_input"]
+    last_message = messages[-1]
+
+    sources_str = last_message.content
+    sources_list = last_message.artifact  # cannot use directly as list of Documents
+    # converting to html string
+    sources_html = html_format_docs_chat(sources_list)
+    if sources_list:
+        print("---ADD SOURCES---")
+    state["messages"].append(BaseMessage(content=sources_html, type="HTML"))
+
+    # Prompt
+    qa_system_prompt = """
+        You are an assistant for question-answering tasks in the social and philanthropic sector. \n
+        Use the following pieces of retrieved context to answer the question at the end. \n
+        If you don't know the answer, just say that you don't know. \n
+        Keep the response professional, friendly, and as concise as possible. \n
+        Question: {question}
+        Context: {context}
+        Answer:
+        """
+
+    qa_prompt = ChatPromptTemplate(
+        [
+            ("system", qa_system_prompt),
+            ("human", question),
+        ]
+    )
+
+    rag_chain = qa_prompt | llm | StrOutputParser()
+    response = rag_chain.invoke({"context": sources_str, "question": question})
+    # couldn't figure out why returning usual "response" was seen as HumanMessage
+    return {"messages": [AIMessage(content=response)], "user_input": question}
+
+
+def has_org_name(state: AgentState) -> AgentState:
+    """
+    Processes the latest message to extract organization links and determine the next step.
+
+    Args:
+        state (AgentState): The current state of the agent, including a list of messages.
+
+    Returns:
+        dict: A dictionary with the next agent action and, if available, a dictionary of organization links.
+    """
+    print("---HAS ORG NAMES?---")
+    messages = state["messages"]
+    last_message = messages[-1].content
+    output_list = extract_org_links_from_chatbot(last_message)
+    link_dict = generate_org_link_dict(output_list) if output_list else {}
+    if link_dict:
+        print("---FOUND ORG NAMES---")
+        return {"next": "insert_org_link", "org_dict": link_dict}
+    print("---NO ORG NAMES FOUND---")
+    return {"next": END, "messages": messages}
+
+
+def insert_org_link(state: AgentState) -> AgentState:
+    """
+    Embeds organization links in the latest message content and returns it as an AI message.
+
+    Args:
+        state (dict): The current state, including the organization links and latest message.
+
+    Returns:
+        dict: A dictionary with the updated message content as an AIMessage.
+    """
+    print("---INSERT ORG LINKS---")
+    messages = state["messages"]
+    last_message = messages[-1].content
+    messages.pop(-1) # Deleting the original message because we will append the same one but with links
+    link_dict = state["org_dict"]
+    last_message = embed_org_links_in_text(last_message, link_dict)
+    return {"messages": [AIMessage(content=last_message)]}
+
+
+def build_compute_graph(llm: LLM, indices: List[str]) -> StateGraph:
+    candid_retriever_tool = retriever_tool(indices=indices)
+    retrieve = ToolNode([candid_retriever_tool])
+    tools = [candid_retriever_tool]
+
+    G = StateGraph(AgentState)
+    # Add nodes
+    G.add_node("reformulate", partial(reformulate_question_using_history, llm=llm))
+    G.add_node("search_agent", partial(search_agent, llm=llm, tools=tools))
+    G.add_node("retrieve", retrieve)
+    G.add_node("generate_with_context", partial(generate_with_context, llm=llm))
+    G.add_node("has_org_name", has_org_name)
+    G.add_node("insert_org_link", insert_org_link)
+
+    # Add edges
+    G.add_edge(START, "reformulate")
+    G.add_edge("reformulate", "search_agent")
+    # Conditional edges from search_agent
+    G.add_conditional_edges(
+        source="search_agent",
+        path=tools_condition, # TODO just a conditional edge here?
+        path_map={
+            "tools": "retrieve",
+            "__end__": "has_org_name",
+        },
+    )
+    G.add_edge("retrieve", "generate_with_context")
+
+    # Add edges
+    G.add_edge("generate_with_context", "has_org_name")
+    # Use add_conditional_edges for has_org_name
+    G.add_conditional_edges(
+        "has_org_name",
+        lambda x: x["next"],  # Now we're accessing the 'next' key from the dict
+        {"insert_org_link": "insert_org_link", END: END},
+    )
+    G.add_edge("insert_org_link", END)
+
+    return G
+
+
+def run_chat(
+    thread_id: str,
+    user_input: Dict[str, Any],
+    chatbot: List[Dict],
+    llm: LLM,
+    indices: Optional[List[str]] = None,
+):
+    # https://langchain-ai.github.io/langgraph/tutorials/rag/langgraph_agentic_rag/#graph
+
+    chatbot.append({"role": "user", "content": user_input["text"]})
+    inputs = {"messages": chatbot}
+    # thread_id can be an email https://github.com/yurisasc/memory-enhanced-ai-assistant/blob/main/assistant.py
+    thread_id = get_session_id(thread_id)
+    config = {"configurable": {"thread_id": thread_id}}
+
+    workflow = build_compute_graph(llm=llm, indices=indices)
+
+    memory = MemorySaver()  # TODO: don't use for Prod
+    graph = workflow.compile(checkpointer=memory)
+    response = graph.invoke(inputs, config=config)
+    messages = response["messages"]
+    last_message = messages[-1]
+    ai_answer = last_message.content
+    sources_html = ""
+    for message in messages[-2:]:
+        if message.type == "HTML":
+            sources_html = message.content
+
+    chatbot.append({"role": "assistant", "content": ai_answer})
+    if sources_html:
+        chatbot.append(
+            {
+                "role": "assistant",
+                "content": sources_html,
+                "metadata": {"title": "Sources HTML"},
+            }
+        )
+
+    return gr.MultimodalTextbox(value=None, interactive=True), chatbot, thread_id

requirements.txt

@@ -0,0 +1,7 @@
+elasticsearch==7.17.6
+gradio
+langchain
+langchain-openai
+langgraph
+pydantic
+fuzzywuzzy

retrieval/candid_blog.py

@@ -0,0 +1,70 @@
+from typing import Dict, Tuple, Any
+
+
+def build_wp_candid_blog_knn_model_query(
+    query: str,
+    fields: Tuple[str] = (
+        "content",
+        "title"
+    ),
+    k: int = 10,
+    model_id: str = "sentence-transformers__all-mpnet-base-v2"
+):
+    output = []
+
+    for f in fields:
+        output.append({
+            "field": f"embeddings.{f}.chunks.vector.predicted_value",
+            "k": k,
+            "num_candidates": 100,
+            "query_vector_builder": {
+                "text_embedding": {
+                    "model_id": model_id,
+                    "model_text": query
+                }
+            },
+            "boost": 1 / len(fields)
+        })
+    return {"knn": output}
+
+
+def build_card_html(doc: Dict[str, Any], height_px: int = 200, show_chunks=False) -> str:
+    url = f"{doc['link']}"
+    fields = ["title", "excerpt"]
+
+    fields_dict = {}
+    fields_len = 0
+    for field in fields:
+        if doc.get(field, None) is not None:
+            fields_dict[field] = doc[field]
+            fields_dict[field + "_txt"] = f"<div>{doc[field]}</div>"
+
+            if (fields_len + len(doc[field])) > 999:
+                rest_text_len = 999 - fields_len
+                if rest_text_len > 0:
+                    fields_dict[field + "_txt"] = f"<div>{doc[field][:rest_text_len] + '[...]'}</div>"
+                else: fields_dict[field + "_txt"] = f"<span>{'[...]'}</span>"
+            fields_len = fields_len + len(doc[field])
+        else:
+            fields_dict[field] = ""
+            fields_dict[field + "_txt"] = ""
+    html = f"""
+    <div style='height: {height_px}px; padding: 5px;'>
+        <div style='height: {height_px}px; border: 1px solid #febe10;'>
+            <span style='padding-left: 10px; display: inline-block; width: 100%;'>
+                <div>
+                    <span>
+                        <b>Candid blog post:</b>
+                        <a href='{url}' target='_blank' style='text-decoration: none;'>
+                            {doc['title']} 
+                        </a>
+                    </span>
+                    <br>
+                    <br>
+                    {fields_dict["excerpt_txt"]}
+                </div>
+            </span>
+        </div>
+    </div>
+    """
+    return html

retrieval/candid_help.py

@@ -0,0 +1,41 @@
+from typing import Dict, Any
+
+
+def build_card_html(doc: Dict[str, Any], height_px: int = 200, show_chunks=False) -> str:
+    url = f"{doc['link']}"
+    fields = ["title", "summary"]
+
+    fields_dict = {}
+    fields_len = 0
+    for field in fields:
+        if doc.get(field, None) is not None:
+            fields_dict[field] = doc[field]
+            fields_dict[field + "_txt"] = f"<div>{doc[field]}</div>"
+
+            if (fields_len + len(doc[field])) > 999:
+                rest_text_len = 999 - fields_len
+                if rest_text_len > 0:
+                    fields_dict[field + "_txt"] = f"<div>{doc[field][:rest_text_len] + '[...]'}</div>"
+                else: fields_dict[field + "_txt"] = f"<span>{'[...]'}</span>"
+            fields_len = fields_len + len(doc[field])
+        else:
+            fields_dict[field] = ""
+            fields_dict[field + "_txt"] = ""
+    html = f"""
+    <div style='height: {height_px}px; padding: 5px;'>
+        <div style='height: {height_px}px; border: 1px solid #febe10;'>
+            <span style='padding-left: 10px; display: inline-block; width: 100%;'>
+                <div>
+                    <span>
+                        <b>Candid help article:</b>
+                        <a href='{url}' target='_blank' style='text-decoration: none;'>
+                            {doc['title']} 
+                        </a>
+                    </span>
+                    <br>
+                </div>
+            </span>
+        </div>
+    </div>
+    """
+    return html

retrieval/candid_learning.py

@@ -0,0 +1,41 @@
+from typing import Dict, Any
+
+
+def build_card_html(doc: Dict[str, Any], height_px: int = 200, show_chunks=False) -> str:
+    url = f"{doc['url']}"
+    fields = ["title", "excerpt"]
+
+    fields_dict = {}
+    fields_len = 0
+    for field in fields:
+        if doc.get(field, None) is not None:
+            fields_dict[field] = doc[field]
+            fields_dict[field + "_txt"] = f"<div>{doc[field]}</div>"
+
+            if (fields_len + len(doc[field])) > 999:
+                rest_text_len = 999 - fields_len
+                if rest_text_len > 0:
+                    fields_dict[field + "_txt"] = f"<div>{doc[field][:rest_text_len] + '[...]'}</div>"
+                else: fields_dict[field + "_txt"] = f"<span>{'[...]'}</span>"
+            fields_len = fields_len + len(doc[field])
+        else:
+            fields_dict[field] = ""
+            fields_dict[field + "_txt"] = ""
+    html = f"""
+    <div style='height: {height_px}px; padding: 5px;'>
+        <div style='height: {height_px}px; border: 1px solid #febe10;'>
+            <span style='padding-left: 10px; display: inline-block; width: 100%;'>
+                <div>
+                    <span>
+                        <b>Candid Learning resource:</b>
+                        <a href='{url}' target='_blank' style='text-decoration: none;'>
+                            {doc['title']} 
+                        </a>
+                    </span>
+                    <br>
+                </div>
+            </span>
+        </div>
+    </div>
+    """
+    return html

retrieval/config.py

@@ -0,0 +1,31 @@
+import os
+
+ONECANDID_QA = {
+    'ES_URL': os.getenv('ONECANDID_URL_QA'),
+    'ES_PWD':  os.getenv('ONECANDID_PWD_QA'),
+    'ES_UID': os.getenv('ONECANDID_UID_QA'),
+    "ES_CLOUD_ID": os.getenv("ONECANDID_CLOUD_ID"),
+    "ES_API_KEY": os.getenv("ONECANDID_API_KEY")
+}
+
+class Indices:
+    # NEWS_INDEX = "news-semantic-search-v1"
+    # TRANSACTION_INDEX = "search-semantic-cds-transactions_v1"
+    # ORGANIZATION_INDEX = "search-semantic-up-organizations_v2"
+    ISSUELAB_INDEX = "search-semantic-issuelab_v1"
+    ISSUELAB_INDEX_ELSER = "search-semantic-issuelab-elser_ve2"
+    YOUTUBE_INDEX = "search-semantic-youtube_v1"
+    YOUTUBE_INDEX_ELSER = "search-semantic-youtube-elser_ve1"
+    CANDID_BLOG_INDEX = "search-semantic-candid-blog_v1"
+    CANDID_BLOG_INDEX_ELSER = "search-semantic-candid-blog-elser_ve2"
+    CANDID_LEARNING_INDEX_ELSER = "search-semantic-candid-learning_ve1"
+    CANDID_HELP_INDEX_ELSER = "search-semantic-candid-help-elser_ve1"
+
+
+ALL_INDICES = (
+    "issuelab",
+    "youtube",
+    "candid_blog",
+    "candid_learning",
+    "candid_help"
+)

retrieval/elastic.py

@@ -0,0 +1,344 @@
+from typing import List, Tuple, Dict, Iterable, Iterator, Optional, Any
+from dataclasses import dataclass
+from functools import partial
+from itertools import groupby
+
+from pydantic import BaseModel, Field
+from langchain_core.documents import Document
+from langchain_core.tools import Tool
+
+from elasticsearch import Elasticsearch
+
+try:
+    # from news import build_knn_query as news_query
+    # from up_orgs import build_organizations_knn_model_query as org_query
+    # from cds import build_transactions_knn_query as transactions_query
+    from config import ONECANDID_QA, ALL_INDICES, Indices
+except ImportError:
+    # from .news import build_knn_query as news_query
+    # from .up_orgs import build_organizations_knn_model_query as org_query
+    # from .cds import build_transactions_knn_query as transactions_query
+    from .config import ONECANDID_QA, ALL_INDICES, Indices
+
+
+@dataclass
+class ElasticHitsResult:
+    """Dataclass for Elasticsearch hits results
+    """
+    index: str
+    id: Any
+    score: float
+    source: Dict[str, Any]
+    inner_hits: Dict[str, Any]
+
+
+class RetrieverInput(BaseModel):
+    """Input to the Elasticsearch retriever."""
+    user_input: str = Field(description="query to look up in retriever")
+
+
+def build_text_expansion_query(
+    query: str,
+    fields: Tuple[str],
+    model_id: str = ".elser_model_2_linux-x86_64"
+) -> Dict[str, Any]:
+
+    output = []
+
+    for f in fields:
+        output.append({
+            "nested": {
+                "path": f"embeddings.{f}.chunks",
+                "query": {
+                    "text_expansion": {
+                        f"embeddings.{f}.chunks.vector": {
+                            "model_id": model_id,
+                            "model_text": query,
+                            "boost": 1 / len(fields)
+                        }
+                    }
+                },
+                 "inner_hits": {
+                    "_source": False,
+                    "size": 2,
+                    "fields": [f"embeddings.{f}.chunks.chunk"]
+                }
+            }
+        })
+    return {"query": {"bool": {"should": output}}}
+
+
+def query_builder(query: str, indices: List[str], **kwargs):
+    queries = []
+    if indices is None:
+        indices = list(ALL_INDICES)
+
+    for index in indices:
+        if index == "news":
+            # q = news_query(query)
+            # q["_source"] = {"excludes": ["embeddings"]}
+            # q["size"] = 5
+            # queries.extend([{"index": Indices.NEWS_INDEX}, q])
+            pass
+        elif index == "organizations":
+            # q = org_query(query)
+            # q["_source"] = {"excludes": ["embeddings"]}
+            # q["size"] = 10
+            # queries.extend([{"index": Indices.ORGANIZATION_INDEX}, q])
+            pass
+        elif index == "grants":
+            # q = transactions_query(query)
+            # q["_source"] = {"excludes": ["embeddings"]}
+            # q["size"] = 10
+            # queries.extend([{"index": Indices.TRANSACTION_INDEX}, q])
+            pass
+        elif index == "issuelab":
+            q = build_text_expansion_query(
+                query=query,
+                fields=("description", "content", "combined_issuelab_findings", "combined_item_description")
+            )
+            q["_source"] = {"excludes": ["embeddings"]}
+            q["size"] = 1
+            queries.extend([{"index": Indices.ISSUELAB_INDEX_ELSER}, q])
+        elif index == "youtube":
+            q = build_text_expansion_query(
+                query=query,
+                fields=("captions_cleaned", "description_cleaned", "title")
+            )
+            # text_cleaned duplicates captions_cleaned
+            q["_source"] = {"excludes": ["embeddings", "captions", "description", "text_cleaned"]}
+            q["size"] = 2
+            queries.extend([{"index": Indices.YOUTUBE_INDEX_ELSER}, q])
+        elif index == "candid_blog":
+            q = build_text_expansion_query(
+                query=query,
+                fields=("content", "title")
+            )
+            q["_source"] = {"excludes": ["embeddings"]}
+            q["size"] = 2
+            queries.extend([{"index": Indices.CANDID_BLOG_INDEX_ELSER}, q])
+        elif index == "candid_learning":
+            q = build_text_expansion_query(
+                query=query,
+                fields=("content", "title", "training_topics", "staff_recommendations")
+            )
+            q["_source"] = {"excludes": ["embeddings"]}
+            q["size"] = 2
+            queries.extend([{"index": Indices.CANDID_LEARNING_INDEX_ELSER}, q])
+        elif index == "candid_help":
+            q = build_text_expansion_query(
+                query=query,
+                fields=("content", "combined_article_description")
+            )
+            q["_source"] = {"excludes": ["embeddings"]}
+            q["size"] = 2
+            queries.extend([{"index": Indices.CANDID_HELP_INDEX_ELSER}, q])
+
+    return queries
+
+
+def multi_search(queries: List[ElasticHitsResult]):
+    results = []
+    with Elasticsearch(
+        cloud_id=ONECANDID_QA["ES_CLOUD_ID"],
+        api_key=ONECANDID_QA["ES_API_KEY"],
+        verify_certs=False,
+        request_timeout=60 * 3
+    ) as es:
+        for query_group in es.msearch(body=queries).get("responses", []):
+            for hit in query_group.get("hits", {}).get("hits", []):
+                hit = ElasticHitsResult(
+                    index=hit["_index"],
+                    id=hit["_id"],
+                    score=hit["_score"],
+                    source=hit["_source"],
+                    inner_hits=hit.get("inner_hits", {})
+                )
+                results.append(hit)
+    return results
+
+
+def get_query_results(search_text: str, indices: Optional[List[str]] = None):
+    queries = query_builder(query=search_text, indices=indices)
+    return multi_search(queries)
+
+
+def reranker(query_results: Iterable[ElasticHitsResult]) -> Iterator[ElasticHitsResult]:
+    """Reranks Elasticsearch hits coming from multiple indicies/queries which may have scores on different scales.
+    This will shuffle results
+
+    Parameters
+    ----------
+    query_results : Iterable[ElasticHitsResult]
+
+    Yields
+    ------
+    Iterator[ElasticHitsResult]
+    """
+
+    results: List[ElasticHitsResult] = []
+    for _, data in groupby(query_results, key=lambda x: x.index):
+        data = list(data)
+        max_score = max(data, key=lambda x: x.score).score
+        min_score = min(data, key=lambda x: x.score).score
+
+        for d in data:
+            d.score = (d.score - min_score) / (max_score - min_score + 1e-9)
+            results.append(d)
+
+    yield from sorted(results, key=lambda x: x.score, reverse=True)
+
+
+def get_results(user_input: str, indices: List[str]) -> List[ElasticHitsResult]:
+    output = ["Search didn't return any Candid sources"]
+    page_content=[]
+    content = "Search didn't return any Candid sources"
+    results = get_query_results(search_text=user_input, indices=indices)
+    if results:
+        output = get_reranked_results(results)
+        for doc in output:
+            page_content.append(doc.page_content)
+        content = "/n/n".join(page_content)
+    # for the tool we need to return a tuple for content_and_artifact type
+    return content, output
+
+
+def get_context(field_name: str, hit: ElasticHitsResult, context_length: int = 1024) -> str:
+    """Pads the relevant chunk of text with context before and after
+
+    Parameters
+    ----------
+    field_name : str
+        a field with the long text that was chunked into pieces
+    hit : ElasticHitsResult
+    context_length : int, optional
+        length of text to add before and after the chunk, by default 1024
+
+    Returns
+    -------
+    str
+        longer chunks stuffed together
+    """
+
+    chunks_with_context = []
+    long_text = hit.source.get(f"{field_name}", "")
+    inner_hits_field = f"embeddings.{field_name}.chunks"
+    inner_hits = hit.inner_hits
+    found_chunks = inner_hits.get(inner_hits_field, {})
+    if found_chunks:
+        hits = found_chunks.get("hits", {}).get("hits", [])
+        for h in hits:
+            chunk = h.get("fields", {})[inner_hits_field][0]["chunk"][0]
+            chunk = chunk[3:-3] # cutting the middle because we may have tokenizing artefacts there
+            # Find the start and end indices of the chunk in the large text
+            start_index = long_text.find(chunk)
+            if start_index != -1: # Chunk is found
+                end_index = start_index + len(chunk)
+                pre_start_index = max(0, start_index - context_length)
+                post_end_index = min(len(long_text), end_index + context_length)
+                context = long_text[pre_start_index:post_end_index]
+                chunks_with_context.append(context)
+    chunks_with_context_txt = '\n\n'.join(chunks_with_context)
+
+    return chunks_with_context_txt
+
+
+def process_hit(hit: ElasticHitsResult) -> Document | None:
+    if "issuelab-elser" in hit.index:
+        combined_item_description = hit.source.get("combined_item_description", "") # title inside
+        description = hit.source.get("description", "")
+        combined_issuelab_findings = hit.source.get("combined_issuelab_findings", "")
+        # we only need to process long texts
+        chunks_with_context_txt = get_context("content", hit, context_length=12)
+        doc = Document(
+            page_content='\n\n'.join([
+                combined_item_description,
+                combined_issuelab_findings,
+                description,
+                chunks_with_context_txt
+            ]),
+            metadata={
+                "source": "IssueLab",
+                "source_id": hit.source["resource_id"],
+                "url": hit.source.get("permalink", "")
+            }
+        )
+    elif "youtube" in hit.index:
+        title = hit.source.get("title", "")
+        # we only need to process long texts
+        description_cleaned_with_context_txt = get_context("description_cleaned", hit, context_length=12)
+        captions_cleaned_with_context_txt = get_context("captions_cleaned", hit, context_length=12)
+        doc = Document(
+            page_content='\n\n'.join([title, description_cleaned_with_context_txt, captions_cleaned_with_context_txt]),
+            metadata={
+                "source": "Candid Youtube",
+                "source_id": hit.source['video_id'],
+                "url": f"https://www.youtube.com/watch?v={hit.source['video_id']}"
+            }
+        )
+    elif "candid-blog" in hit.index:
+        excerpt = hit.source.get("excerpt", "")
+        title = hit.source.get("title", "")
+        # we only need to process long texts
+        content_with_context_txt = get_context("content", hit, context_length=12)
+        doc = Document(
+            page_content='\n\n'.join([title, excerpt, content_with_context_txt]),
+            metadata={
+                "source": "Candid Blog",
+                "source_id": hit.source["id"],
+                "url": hit.source["link"]
+            }
+        )
+    elif "candid-learning" in hit.index:
+        title = hit.source.get("title", "")
+        content_with_context_txt = get_context("content", hit, context_length=12)
+        training_topics = hit.source.get("training_topics", "")
+        staff_recommendations = hit.source.get("staff_recommendations", "")
+
+        doc = Document(
+            page_content='\n\n'.join([title, staff_recommendations, training_topics, content_with_context_txt]),
+            metadata={
+                "source": "Candid Learning",
+                "source_id": hit.source["post_id"],
+                "url": hit.source.get("url", "")
+            }
+        )
+    elif "candid-help" in hit.index:
+        title = hit.source.get("title", "")
+        content_with_context_txt = get_context("content", hit, context_length=12)
+        combined_article_description = hit.source.get("combined_article_description", "")
+
+        doc = Document(
+            page_content='\n\n'.join([combined_article_description, content_with_context_txt]),
+            metadata={
+                "source": "Candid Help",
+                "source_id": hit.source["id"],
+                "url": hit.source.get("link", "")
+            }
+        )
+    else:
+        doc = None
+    return doc
+
+
+def get_reranked_results(results: List[ElasticHitsResult]) -> List[Document]:
+    output = []
+    for r in reranker(results):
+        hit = process_hit(r)
+        output.append(hit)
+    return output
+
+
+def retriever_tool(indices: List[str]) -> Tool:
+    # cannot use create_retriever_tool because it only provides content losing all metadata on the way
+    # https://python.langchain.com/docs/how_to/custom_tools/#returning-artifacts-of-tool-execution
+    return Tool(
+        name="retrieve_social_sector_information",
+        func=partial(get_results, indices=indices),
+        description=(
+            "Return additional information about social and philanthropic sector, "
+            "including nonprofits (NGO), grants, foundations, funding, RFP, LOI, Candid."
+        ),
+        args_schema=RetrieverInput,
+        response_format="content_and_artifact"
+    )

retrieval/elastic_qa.py

@@ -0,0 +1,194 @@
+from typing import List, Dict, Any, Optional
+import re
+import os
+
+import numpy as np
+
+from langchain_huggingface.embeddings import HuggingFaceEndpointEmbeddings
+from langchain_core.documents import Document
+
+from ...socialspark.retrieval_qa.elastic import ElasticHitsResult, ElasticsearchQABase
+
+from .elastic import query_builder
+from .config import ALL_INDICES
+
+timestamp_pattern = re.compile(r"^(\d:\d{2}:\d{2}\.\d{3}),(\d:\d{2}:\d{2}\.\d{3})\n(.*)$")
+
+
+def parse_youtube_captions(caption_text: str) -> List[str]:
+    timestamp_blocks = re.split(r'\n{2}', caption_text)
+    parsed_blocks = [m.groups() for block in timestamp_blocks if (m:=timestamp_pattern.match(block)) if not None]
+    video_length_mins = int(parsed_blocks[-1][0].split(':')[1])
+
+    # what is the block timing in minutes which gives 10 chunks? keep minimum 1 minute breaks to maintain context
+    minute_difference = min(video_length_mins // 10, 1)
+
+    prev_minute_start = 0
+    texts = []
+    text_block = []
+    for start, _, text in parsed_blocks:
+        text_block.append(text)
+
+        current_minute = int(start.split(':')[1])
+        if (current_minute - prev_minute_start) >= minute_difference:
+            texts.append(' '.join(text_block))
+            text_block.clear()
+            prev_minute_start = current_minute
+
+    return texts
+
+
+def parse_candid_learning(text: str) -> List[str]:
+    texts = []
+    for block in map(str.strip, re.split(r'\n{1,}', text, flags=re.I | re.M)):
+        if (
+            'back to top' in block.lower()
+            or 'table of contents' in block.lower()
+        ):
+            continue
+
+        texts.append(block)
+    return texts
+
+
+class ElasticsearchQA(ElasticsearchQABase):
+
+    indices: Optional[List[str]] = ALL_INDICES
+    embedding: Any = HuggingFaceEndpointEmbeddings(
+        huggingfacehub_api_token=os.getenv("HF_API_KEY"),
+        # model="sentence-transformers/all-mpnet-base-v2"
+        model="mixedbread-ai/mxbai-embed-large-v1"
+    )
+
+    def build_query(self, query: str, **kwargs) -> List[Dict[str, Any]]:
+        queries = query_builder(query=query, indices=self.indices)
+        return queries
+
+    def sub_section_alignment(self, query: str, document: List[str]) -> str:
+        question_vector = np.array(self.embedding.embed_query(query), dtype='float32')
+        vectors = np.array(self.embedding.embed_documents(document), dtype='float32')
+        vectors = np.array(vectors, dtype='float32')
+
+        size = max(5, int(0.1 * len(document)))
+
+        doc_vector = vectors.sum(axis=0, keepdims=True)
+        vectors /= np.linalg.norm(vectors, ord=2.0, axis=-1, keepdims=True)
+        doc_vector /= np.linalg.norm(doc_vector, ord=2.0, axis=-1, keepdims=True)
+        question_vector /= np.linalg.norm(question_vector, ord=2.0, axis=-1, keepdims=True)
+        # similarity = (doc_vector * vectors).sum(axis=-1)
+        similarity = (question_vector * vectors).sum(axis=-1)
+
+        return '\n'.join(text for text, _ in sorted(zip(document, similarity), key=lambda x: 1 - x[-1])[:size])
+
+    def process_hit(self, hit: ElasticHitsResult, q: str) -> Document | None:
+        if "news" in hit.index:
+            doc = Document(
+                page_content='\n\n'.join(v for k, v in hit.source["texts"].items() if v),
+                metadata={
+                    "source": "news",
+                    "source_id": hit.source['metadata']['link']
+                }
+            )
+        elif "transactions" in hit.index:
+            doc = Document(
+                page_content='\n\n'.join(v for k, v in hit.source["semantic_texts"].items() if v),
+                metadata={
+                    "source": "cds-transactions",
+                    "source_id": hit.source["id"]
+                }
+            )
+        elif "organizations" in hit.index:
+            source = hit.source
+            org_gen = source.get("combined_organization_description_general", "")
+            org_fin = source.get("combined_organization_description_financial", "")
+            org_contact = source.get("combined_organization_description_contacts", "")
+            mission = ""
+            if source.get("mission_statement", None) is not None:
+                mission = source.get("mission_statement", "")
+            keyword = ""
+            if source.get("keyword", None) is not None:
+                keyword = source.get("keyword", "")
+            programs = source.get("programs", "")
+            doc = Document(
+                page_content='\n\n'.join([org_gen, mission, keyword, org_fin, org_contact, programs]),
+                metadata={
+                    "source": "UP-organizations-QA",
+                    "source_id": hit.source["candid_entity_id"]
+                }
+            )
+        # elif "issuelab" in hit.index:
+        #     doc = Document(
+        #         page_content='\n\n'.join(v for k, v in hit.source["semantic_texts"].items() if v),
+        #         metadata={
+        #             "source": "IssueLab",
+        #             "source_id": hit.source["resource_id"]
+        #         }
+        #     )
+        elif "issuelab-elser" in hit.index:
+            title = hit.source.get("title", "")
+            description = hit.source.get("description", "")
+            doc = Document(
+                page_content='\n\n'.join([title, description]),
+                metadata={
+                    "source": "IssueLab",
+                    "source_id": hit.source["resource_id"],
+                    "url": hit.source.get("permalink", "")
+                }
+            )
+        elif "youtube" in hit.index:
+            title = hit.source.get("title", "")
+            summary = self.sub_section_alignment(
+                query=q,
+                document=parse_youtube_captions(hit.source.get("text"))
+            )
+            doc = Document(
+                # page_content='\n\n'.join([title]),
+                page_content=summary,
+                metadata={
+                    "source": "Candid's Youtube channel",
+                    "source_id": hit.source['video_id'],
+                    "url": f"https://www.youtube.com/watch?v={hit.source['video_id']}"
+                }
+            )
+        elif "candid-blog" in hit.index:
+            excerpt = hit.source.get("excerpt", "")
+            title = hit.source.get("title", "")
+            doc = Document(
+                page_content='\n\n'.join([title, excerpt]),
+                metadata={
+                    "source": "Candid Blog",
+                    "source_id": hit.source["id"],
+                    "url": hit.source["link"]
+                }
+            )
+        elif "candid-learning" in hit.index:
+            # content = hit.source.get("content", "")
+            title = hit.source.get("title", "")
+            summary = self.sub_section_alignment(
+                query=q,
+                document=parse_candid_learning(hit.source.get("content", ""))
+            )
+            doc = Document(
+                # page_content='\n\n'.join([title]),
+                page_content=summary,
+                metadata={
+                    "source": "Candid Learning",
+                    "source_id": hit.source["post_id"],
+                    "url": hit.source.get("url", "")
+                }
+            )
+        elif "candid-help" in hit.index:
+            title = hit.source.get("title", "")
+            content = hit.source.get("content", "")
+
+            doc = Document(
+                page_content='\n\n'.join([title, content]),
+                metadata={
+                    "source": "Candid Help",
+                    "source_id": hit.source["id"],
+                    "url": hit.source.get("link", "")
+                }
+            )
+        else:
+            doc = None
+        return doc

retrieval/issuelab.py

@@ -0,0 +1,108 @@
+from typing import List, Dict, Tuple, Union, Any
+
+import numpy as np
+
+
+def build_knn_query(
+    query: Union[str, np.ndarray, List[float]],
+    fields: Tuple[str] = (
+        "item_description_vector",
+        "description_vector",
+        "title_vector",
+        "content_vector",
+        "issuelab_key_findings_vector",
+    ),
+    subjects_vector: List[float] | None = None,
+    population_vector: List[float] | None = None,
+    k: int = 10,
+    model_id: str = "sentence-transformers__all-mpnet-base-v2"
+):
+    output = []
+
+    for f in fields:
+        if isinstance(query, str):
+            output.append({
+                "field": f"embeddings.{f}.predicted_value",
+                "k": k,
+                "num_candidates": 100,
+                "query_vector_builder": {
+                    "text_embedding": {
+                        "model_id": model_id,
+                        "model_text": query
+                    }
+                },
+                "boost": 1 / len(fields)
+            })
+        elif isinstance(query, (np.ndarray, list)):
+            output.append({
+                "field": f"embeddings.{f}.predicted_value",
+                "query_vector": list(query),
+                "k": k,
+                "num_candidates": 100,
+            })
+
+    if subjects_vector:
+        output.append({
+            "field": "embeddings.subjects_vector",
+            "query_vector": subjects_vector,
+            "k": k,
+            "num_candidates": 100,
+        })
+
+    if population_vector:
+        output.append({
+            "field": "embeddings.populations_vector",
+            "query_vector": population_vector,
+            "k": k,
+            "num_candidates": 100,
+        })
+    return {"knn": output}
+
+
+def issuelab_card_html(doc: Dict[str, Any], height_px: int = 200, show_chunks=False) -> str:
+    chunks_html = ""
+    if show_chunks:
+        cleaned_text = []
+        for k, v in doc["inner_hits"].items():
+            hits = v["hits"]["hits"]
+            for h in hits:
+                for k1, v1 in h["fields"].items():
+                    # we don't want other chunks
+                    if "content" in k1:
+                        cleaned_text.append(f"<div><p>{v1[0]['chunk'][0]}</p></div>")
+
+        chunks_html ="<span><b>Relevant parts of the content:</b></span>" + "<br>".join(cleaned_text)
+
+    html = f"""
+    <div style='height: auto; padding: 5px;'>
+        <div style='border: 1px solid #febe10;'>
+            <span style='display: inline-block; height: {height_px - 10}px; padding: 5px; vertical-align: top;'>
+                <img
+                    src='{doc['cover_graphic_small']}'
+                    style='max-height: 100%; overflow: hidden; border-radius: 3%;'
+                >
+            </span>
+
+            <span style='padding: 10px; display: inline-block; width: 70%;'>
+                <div>
+                    <span><b>Issuelab ID:</b> {doc['resource_id']}</span>
+                    <br>
+                    <span>
+                        <a href='{doc['issuelab_url']}' target='_blank' style='text-decoration: none;'>
+                            {doc['title']}
+                        </a>
+                    </span>
+                    <br>
+
+                    <span><b>Description:</b> {doc['description']}</span>
+                    <br>
+                    <div>{doc['combined_item_description']}</div>
+                    <br>
+                    <div>{chunks_html}</div>
+                    
+                </div>
+            </span>
+        </div>
+    </div>
+    """
+    return html

retrieval/youtube.py

@@ -0,0 +1,82 @@
+from typing import Dict, Tuple, Any
+
+
+def build_knn_model_query(
+    query: str,
+    fields: Tuple[str] = (
+        "captions_cleaned",
+        "description_cleaned",
+        "title"
+    ),
+    k: int = 10,
+    model_id: str = "sentence-transformers__all-mpnet-base-v2"
+):
+    output = []
+
+    for f in fields:
+        output.append({
+            "field": f"embeddings.{f}.chunks.vector.predicted_value",
+            "k": k,
+            "num_candidates": 100,
+            "query_vector_builder": {
+                "text_embedding": {
+                    "model_id": model_id,
+                    "model_text": query
+                }
+            },
+            "boost": 1 / len(fields)
+        })
+    return {"knn": output}
+
+
+def build_card_html(doc: Dict[str, Any], height_px: int = 200, show_chunks=False) -> str:
+    url = f"https://www.youtube.com/watch?v={doc['video_id']}"
+    fields = ["title", "description_cleaned"]
+
+    fields_dict = {}
+    fields_len = 0
+    for field in fields:
+        if doc.get(field, None) is not None:
+            fields_dict[field] = doc[field]
+            fields_dict[field + "_txt"] = f"<div>{doc[field]}</div>"
+
+            if (fields_len + len(doc[field])) > 999:
+                rest_text_len = 999 - fields_len
+                if rest_text_len > 0:
+                    fields_dict[field + "_txt"] = f"<div>{doc[field][:rest_text_len] + '[...]'}</div>"
+                else: fields_dict[field + "_txt"] = f"<span>{'[...]'}</span>"
+            fields_len = fields_len + len(doc[field])
+        else:
+            fields_dict[field] = ""
+            fields_dict[field + "_txt"] = ""
+    html = f"""
+    <div style='height: {height_px}px; padding: 5px;'>
+        <div style='height: {height_px}px; border: 1px solid #febe10;'>
+            <span style='padding-left: 10px; display: inline-block; width: 100%;'>
+                <div>
+                    <span>
+                        <b>Candid Youtube video:</b>
+                        <a href='{url}' target='_blank' style='text-decoration: none;'>
+                            {doc['title']} 
+                        </a>
+                    </span>
+                    <iframe 
+                        width="426"
+                        height="240"
+                        src="https://www.youtube.com/embed/{doc['video_id']}?si=0-y6eRrOzXTUSBDY"
+                        title="YouTube video player"
+                        frameborder="0"
+                        allow="accelerometer; autoplay; clipboard-write; encrypted-media; gyroscope; picture-in-picture; web-share"
+                        referrerpolicy="strict-origin-when-cross-origin"
+                        allowfullscreen
+                        style="display: inline-block; float: left;padding-right: 10px;padding-top: 5px;">
+                    </iframe>
+                    <br>
+                    <br>
+                    {fields_dict["description_cleaned_txt"]}
+                </div>
+            </span>
+        </div>
+    </div>
+    """
+    return html

search.py

@@ -0,0 +1,146 @@
+from typing import List, Optional
+import json
+
+import gradio as gr
+import requests
+
+from .utils import html_format_doc
+from .retrieval.up_orgs_keyword import organization_card_html
+from .retrieval.elastic import reranker, get_query_results
+from .retrieval.config import ALL_INDICES
+from . import UP_QA_SEARCH_API
+
+
+def run_search(search_text: str, indices: Optional[List[str]] = None):
+    results = get_query_results(search_text, indices=indices)
+
+    output = []
+    for result in reranker(results):
+        source_name = None
+        if "news" in result.index:
+            source_name = "news"
+        elif "transactions" in result.index:
+            source_name = "transactions"
+        elif "organizations" in result.index:
+            source_name = "organizations"
+        elif "issuelab-elser" in result.index:
+            source_name = "issuelab"
+        # elif "issuelab" in result.index:
+        #     source_name = "issuelab"
+        elif "youtube-elser" in result.index:
+            source_name = "youtube"
+        # elif "youtube" in result.index:
+        #     source_name = "youtube"
+        elif "candid-blog-elser" in result.index:
+            source_name = "candid_blog"
+        # elif "candid-blog" in result.index:
+        #     source_name = "candid_blog"
+        elif "candid-learning" in result.index:  # TODO fix that
+            source_name = "candid_learning"
+        elif "candid-help-elser" in result.index:
+            source_name = "candid_help"
+
+        doc = html_format_doc(doc=result.source, source=source_name)
+        output.append(doc)
+    return f"<div>{''.join(output)}</div>"
+
+
+def run_ks(search_text: str):
+    json_body = {"keyword": search_text, "rowCount": 10}
+
+    response = requests.post(
+        url=UP_QA_SEARCH_API["API_URL"],
+        json=json_body,
+        headers={
+            "accept": "application/json",
+            "content-type": "application/json",
+            "x-api-key": UP_QA_SEARCH_API["API_KEY"]
+        },
+        timeout=(5 * 60)
+    )
+
+    r_json = json.loads(response.text)
+    output_k = []
+    if r_json.get("returnedOrgs", None) is not None:
+        for doc in r_json["returnedOrgs"]:
+            org = {}
+            org["candid_entity_id"] = doc.get("candidEntityID", "")
+            org["main_name"] = doc.get("orgName", "")
+            org["logo"] = doc.get("logo", "")
+            org["seal"] = doc.get("seal", {})
+            org["city"] = doc.get("city", "")
+            org["admin1"] = doc.get("admin1", "")
+            org["country_name"] = doc.get("countryName", "")
+            org["taxonomy"] = doc.get("taxonomy", {})
+            highlights = doc.get("highlights", [])
+            if highlights:
+                for h in highlights:
+                    if h["field"] == "mission_statement":
+                        org["mission_statement"] = "; ".join(h["highlights"])
+
+            html = organization_card_html(org, 250)
+            output_k.append(html)
+
+    # Getting semantic results
+    output_s = run_search(search_text=search_text)
+
+    return f"<div>{''.join(output_k)}</div>", output_s
+
+
+def build_search_tab() -> gr.Blocks:
+    with gr.Blocks(theme=gr.themes.Soft(), title="Semantic search") as demo:
+        gr.Markdown(
+            "<h1>Alpha demo: Semantic search</h1>"
+            "Search and ask questions of Candid's data together with casual language"
+        )
+
+        query = gr.Text(placeholder="Search", show_label=False)
+
+        with gr.Accordion(label="Advanced settings", open=False):
+            es_indices = gr.CheckboxGroup(
+                choices=list(ALL_INDICES),
+                value=list(ALL_INDICES),
+                label="Sources to include",
+                interactive=True
+            )
+        search = gr.Button("Search")
+
+        feed = gr.HTML()
+
+        # pylint: disable=no-member
+        search.click(
+            fn=run_search,
+            inputs=[query, es_indices],
+            outputs=[feed],
+            api_name=False,
+            queue=True
+        )
+    return demo
+
+
+def build_ks_tab() -> gr.Blocks:
+    with gr.Blocks(theme=gr.themes.Soft(), title="Semantic search") as demo:
+        gr.Markdown(
+        "<h1>Alpha demo: Keyword versus Semantic search</h1>"
+        "Compare current search results versus semantic search results"
+        )
+        query = gr.TextArea(placeholder="Search", show_label=False, lines=1)
+        ask = gr.Button("Search Unified Platform organizations")
+        with gr.Row():
+            with gr.Column():
+                gr.Markdown("<h2>Keyword results</h2>")
+                feed_k = gr.HTML()
+            with gr.Column():
+                gr.Markdown("<h2>Semantic results</h2>")
+                feed_s = gr.HTML()
+
+        # pylint: disable=no-member
+        ask.click(
+            fn=run_ks,
+            inputs=[query],
+            outputs=[feed_k, feed_s],
+            api_name=False,
+            queue=True
+        )
+
+    return demo

static/css.py

@@ -0,0 +1,48 @@
+css_chat = """
+    .message-row img {
+        margin: 0px !important;
+    }
+    .avatar-container img {
+        padding: 0px !important;
+    }
+
+    #ssearch-sources {
+        display: flex;
+        gap: 10px;
+        min-width: 75vw;
+        padding-bottom: 5px;
+    }
+    
+    .ssearch-source-btn {
+        background-color: #febe10;
+        color: black;
+        padding: 5px;
+        text-align: center;
+        border-radius: 12px;
+        min-width: 70px;
+        max-width: 75px;
+        box-shadow: 0 2px 5px 0 rgba(0, 0, 0,0.2);
+        height: 45px; 
+        font-size:small;
+    }
+
+    .ssearch-source {
+        text-decoration: none;
+        display: block; 
+        box-sizing: border-box;
+    }
+
+    button.upload-button.svelte-1d7elt4 {
+        visibility: hidden !important;
+    }
+
+    .candid-org-link {
+        font-weight: bold;
+        text-decoration: none;
+    }
+
+    .candid-app-link {
+        font-size: small;
+    }
+
+"""

tools/config.py

@@ -0,0 +1,5 @@
+import os
+CDS_API = {
+    'CDS_API_URL': os.getenv('CDS_API_URL'),
+    'CDS_API_KEY': os.getenv('CDS_API_KEY')
+}

tools/org_seach.py

@@ -0,0 +1,190 @@
+from typing import List
+import re
+
+from fuzzywuzzy import fuzz
+
+from langchain.output_parsers.openai_tools import JsonOutputToolsParser
+from langchain_openai.chat_models import ChatOpenAI
+from langchain_core.runnables import RunnableSequence
+from langchain_core.prompts import ChatPromptTemplate
+from pydantic import BaseModel
+
+from dotenv import load_dotenv
+try:
+    from common.org_search_component import OrgSearch
+except ImportError:
+    from ...common.org_search_component import OrgSearch
+
+load_dotenv()
+search = OrgSearch()
+
+
+class OrganizationNames(BaseModel):
+    orgnames: List[str]
+
+
+def extract_org_links_from_chatbot(chatbot_output: str):
+    """
+    Extracts a list of organization names from the provided text.
+
+    Args:
+        chatbot_output (str):The chatbot output containing organization names and other content.
+
+    Returns:
+        list: A list of organization names extracted from the text.
+
+    Raises:
+        ValueError: If parsing fails or if an unexpected output format is received.
+    """
+    prompt = """Extract only the names of officially recognized organizations, foundations, and government entities from the text below. Do not include any entries that contain descriptions, regional identifiers, or explanations within parentheses or following the name. Strictly exclude databases, resources, crowdfunding platforms, and general terms. Provide the output only in the specified JSON format.
+
+    input text below:
+    
+        ```{chatbot_output}``
+
+    output format:
+    {{
+    'orgnames' : [list of organization names without any additional descriptions or identifiers]
+    }}
+
+    """
+
+    try:
+        parser = JsonOutputToolsParser()
+        llm = ChatOpenAI(model="gpt-4o").bind_tools([OrganizationNames])
+        prompt = ChatPromptTemplate.from_template(prompt)
+        chain = RunnableSequence(prompt, llm, parser)
+
+        # Run the chain with the input data
+        result = chain.invoke({"chatbot_output": chatbot_output})
+
+        # Extract the organization names from the output
+        output_list = result[0]["args"].get("orgnames", [])
+
+        # Validate output format
+        if not isinstance(output_list, list):
+            raise ValueError("Unexpected output format: 'orgnames' should be a list")
+
+        return output_list
+
+    except Exception as e:
+        # Log or print the error as needed for debugging
+        print(f"text does not have any organization: {e}")
+        return []
+
+
+def is_similar(name: str, list_of_dict: list, threshold: int = 80):
+    """
+    Returns True if `name` is similar to any names in `list_of_dict` based on a similarity threshold.
+    """
+    try:
+        for item in list_of_dict:
+            try:
+                # Attempt to calculate similarity score
+                similarity = fuzz.ratio(name.lower(), item["name"].lower())
+                if similarity >= threshold:
+                    return True
+            except KeyError:
+                # Handle cases where 'name' key might be missing in dictionary
+                print(f"KeyError: Missing 'name' key in dictionary item {item}")
+                continue
+            except AttributeError:
+                # Handle non-string name values in dictionary items
+                print(f"AttributeError: Non-string 'name' in dictionary item {item}")
+                continue
+    except TypeError as e:
+        # Handle cases where input types are incorrect
+        print(f"TypeError: {e}")
+        return False
+
+    return False
+
+
+def generate_org_link_dict(org_names_list: list):
+    """
+    Maps organization names to their Candid profile URLs if available.
+
+    For each organization in `output_list`, this function attempts to retrieve a matching profile
+    using `search_org`. If a similar name is found and a Candid entity ID is available, it constructs
+    a profile URL. If no ID or similar match is found, or if an error occurs, it assigns an empty string.
+
+    Args:
+        output_list (list): List of organization names (str) to retrieve Candid profile links for.
+
+    Returns:
+        dict: Dictionary with organization names as keys and Candid profile URLs or empty strings as values.
+
+    Example:
+        get_org_link(['New York-Presbyterian Hospital'])
+        # {'New York-Presbyterian Hospital': 'https://app.candid.org/profile/6915255'}
+    """
+    link_dict = {}
+
+    for org in org_names_list:
+        try:
+            # Attempt to retrieve organization data
+            response = search(org)
+
+            # Check if there is a valid response and if names are similar
+            if response and is_similar(org, response[0].get("names", "")):
+                # Try to get the Candid entity ID and construct the URL
+                candid_entity_id = response[0].get("candid_entity_id")
+                if candid_entity_id:
+                    link_dict[org] = (
+                        f"https://app.candid.org/profile/{candid_entity_id}"
+                    )
+                else:
+                    link_dict[org] = ""  # No ID found, set empty string
+            else:
+                link_dict[org] = ""  # No similar match found
+
+        except KeyError as e:
+            # Handle missing keys in the response dictionary
+            print(f"KeyError encountered for organization '{org}': {e}")
+            link_dict[org] = ""
+
+        except Exception as e:
+            # Catch any other unexpected errors
+
+            print(f"An error occurred for organization '{org}': {e}")
+            link_dict[org] = ""
+
+    return link_dict
+
+
+def embed_org_links_in_text(input_text: str, org_link_dict: dict):
+    """
+    Replaces organization names in `text` with links from `link_dict` and appends a Candid info message.
+
+    Args:
+        text (str): The text containing organization names.
+        link_dict (dict): Mapping of organization names to URLs.
+
+    Returns:
+        str: Updated text with linked organization names and an appended Candid message.
+    """
+    try:
+        for org_name, url in org_link_dict.items():
+            if url:  # Only proceed if the URL is not empty
+                regex_pattern = re.compile(re.escape(org_name))
+                input_text = regex_pattern.sub(
+                    repl=f"<a href={url} target='_blank' rel='noreferrer' class='candid-org-link'>{org_name}</a>",
+                    string=input_text
+                )
+
+        # Append Candid information message at the end
+        input_text += "<p class='candid-app-link'> Visit <a href=https://app.candid.org/ target='_blank' rel='noreferrer' class='candid-org-link'>Candid</a> to get nonprofit information you need.</p>"
+
+    except TypeError as e:
+        print(f"TypeError encountered: {e}")
+        return input_text
+
+    except re.error as e:
+        print(f"Regex error encountered for '{org_name}': {e}")
+        return input_text
+
+    except Exception as e:
+        print(f"Unexpected error: {e}")
+        return input_text
+
+    return input_text

tools/question_reformulation.py

@@ -0,0 +1,44 @@
+from langchain_core.prompts import ChatPromptTemplate
+from langchain_core.output_parsers import StrOutputParser
+
+
+def reformulate_question_using_history(state, llm):
+    """
+    Transform the query to produce a better query with details from previous messages.
+
+    Args:
+        state (messages): The current state
+        llm: LLM to use
+    Returns:
+        dict: The updated state with re-phrased question and original user_input for UI
+    """
+    print("---REFORMULATE THE USER INPUT---")
+    messages = state["messages"]
+    question = messages[-1].content
+
+    if len(messages) > 1:
+        contextualize_q_system_prompt = """Given a chat history and the latest user input \
+        which might reference context in the chat history, formulate a standalone input \
+        which can be understood without the chat history.
+        Chat history:
+        \n ------- \n
+        {chat_history} 
+        \n ------- \n
+        User input:
+        \n ------- \n
+        {question} 
+        \n ------- \n
+        Do NOT answer the question, \
+        just reformulate it if needed and otherwise return it as is.
+        """
+
+        contextualize_q_prompt = ChatPromptTemplate([
+            ("system", contextualize_q_system_prompt),
+            ("human", question),
+        ])
+
+        rag_chain = contextualize_q_prompt | llm | StrOutputParser()
+        new_question = rag_chain.invoke({"chat_history": messages, "question": question})
+        print(f"user asked: '{question}', agent reformulated the question basing on the chat history: {new_question}")
+        return {"messages": [new_question], "user_input" : question}
+    return {"messages": [question], "user_input" : question}

utils.py

@@ -0,0 +1,114 @@
+from typing import List, Dict, Any
+from uuid import uuid4
+
+import gradio as gr
+
+try:
+    from retrieval import (
+        candid_blog,
+        candid_help,
+        candid_learning,
+        # cds,
+        issuelab,
+        # news,
+        # up_orgs,
+        youtube
+    )
+except ImportError:
+    from .retrieval import (
+        candid_blog,
+        candid_help,
+        candid_learning,
+        # cds,
+        issuelab,
+        # news,
+        # up_orgs,
+        youtube
+    )
+
+
+# TODO summarize messages instead
+def filter_messages(messages, k=10):
+    return messages[-k:]
+
+
+def html_format_doc(doc: Dict[str, Any], source: str, show_chunks=False) -> str:
+    height_px = 200
+    html = ""
+
+    if source == "news":
+        # html = news.article_card_html(doc, height_px, show_chunks)
+        pass
+    elif source == "transactions":
+        # html = cds.transaction_card_html(doc, height_px, show_chunks)
+        pass
+    elif source == "organizations":
+        # html = up_orgs.organization_card_html(doc, 400, show_chunks)
+        pass
+    elif source == "issuelab":
+        html = issuelab.issuelab_card_html(doc, height_px, show_chunks)
+    elif source == "youtube":
+        html = youtube.build_card_html(doc, 400, show_chunks)
+    elif source == "candid_blog":
+        html = candid_blog.build_card_html(doc, height_px, show_chunks)
+    elif source == "candid_learning":
+        html = candid_learning.build_card_html(doc, height_px, show_chunks)
+    elif source == "candid_help":
+        html = candid_help.build_card_html(doc, height_px, show_chunks)
+    return html
+
+
+def html_format_docs_chat(docs):
+    """
+    Formats Candid sources into a line of buttons
+    """
+    html = ""
+    if docs:
+        docs_html = []
+        for doc in docs:
+            s_name = doc.metadata.get("source", "Source")
+            s_url = doc.metadata.get("url", "URL")
+            s_html = f"""<a href={s_url} target='_blank' rel='noreferrer' class='ssearch-source'> \
+<button class='ssearch-source-btn'>{s_name}</button></a>"""
+            docs_html.append(s_html)
+        docs_html_insert = "".join(s for s in docs_html)
+        html = f"""<div id='ssearch-sources'>{docs_html_insert}</div>"""
+    return html
+
+
+def format_chat_response(chatbot: List[Any]) -> List[Any]:
+    """We have sources appended as one more tuple. Here we concatinate HTML of sources
+        with the AI response
+    Returns:
+        _type_: updated chatbot message as HTML
+    """
+    if chatbot:
+        sources = chatbot[-1][1]
+        chatbot.pop(-1)
+        chatbot[-1][1] = chatbot[-1][1] + sources
+    return gr.HTML(chatbot)
+
+
+def format_chat_ag_response(chatbot: List[Any]) -> List[Any]:
+    """If we called retriever, we appended sources as as one more message. Here we concatinate HTML of sources
+        with the AI response
+    Returns:
+        _type_: updated chatbot message as HTML
+    """
+    sources = ""
+    if chatbot:
+        title = chatbot[-1]["metadata"].get("title", None)
+        if title == "Sources HTML":
+            sources = chatbot[-1]["content"]
+            chatbot.pop(-1)
+            chatbot[-1]["content"] = chatbot[-1]["content"] + sources
+    return gr.HTML(chatbot)
+
+
+def valid_inputs(*args) -> bool:
+    return any(a is not None or (isinstance(a, str) and a.strip() != '') for a in args)
+
+def get_session_id(thread_id: gr.components.Component) -> str:
+    if not thread_id:
+        thread_id = uuid4().hex
+    return thread_id