adding files

README.md

@@ -1,10 +1,59 @@
 ---
-title: Narrativ V4
-emoji: 🦀
-colorFrom: blue
-colorTo: pink
-sdk: docker
-pinned: false
+title: Narrativ
+emoji: 📉
+colorFrom: pink
+colorTo: yellow
+sdk: streamlit
+sdk_version: 1.37.1 # The latest supported version
 ---
 
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
+
+# Narrativ - Employment Trends and Employee Satisfaction Analyzer
+
+**Narrativ** aggregates and processes data related to employment trends and employee satisfaction from various sources, including internal HR databases and pulse surveys. The app analyzes the data, provides insights into employee satisfaction, and summarizes key trends in employment.
+
+## Features:
+
+- **Data Aggregation**: The app collects data from multiple sources, including internal HR databases and employee pulse surveys.
+- **Employee Satisfaction Analysis**: Each survey response is analyzed to gauge employee satisfaction, with an index indicating overall satisfaction levels.
+- **Trend Summarization**: The app provides concise summaries of employment trends, highlighting key information about workforce dynamics.
+- **Comprehensive Insights**: The app generates a report with satisfaction analysis and key employment trends for quick understanding.
+
+## Supported Sources:
+
+- **Internal HR Databases**: Data from company HR systems.
+- **Pulse Surveys**: Regular surveys conducted to assess employee satisfaction and engagement.
+
+## Tasks:
+
+### 1. Aggregate Data:
+   - The app collects data from the above sources.
+   - It processes the data for further analysis.
+
+### 2. Satisfaction Analysis:
+   - The app applies analysis algorithms to determine overall employee satisfaction levels.
+   - It generates a satisfaction index for each survey, reflecting the strength of employee sentiment.
+
+### 3. Summarize Trends:
+   - The app extracts and summarizes the most important employment trends or key points discussed in the data.
+   - The summary highlights major workforce dynamics and provides a quick overview of the trends.
+
+### 4. Display Insights:
+   - Each survey’s satisfaction index and trend summary are displayed, making it easy to analyze and understand workforce dynamics.
+
+## Output Format:
+
+- **Data Source:** [Name of source, e.g., Internal HR Database, Pulse Survey]
+- **Title:** [Title of the report]
+- **Satisfaction Index:** [Satisfaction index created by HR Assistant]
+- **Summary:** [Concise summary of key employment trends]
+- **Full Report:** [Link to the full report or the report body]
+
+## Installation:
+
+1. Clone the repository:
+   ```bash
+   git clone <repository-url>
+
+### 💬 Feedback
+Try it out!

app_social.py

@@ -0,0 +1,383 @@
+from openai import OpenAI
+import streamlit as st
+from langchain_openai import ChatOpenAI
+from langchain_openai.embeddings import OpenAIEmbeddings
+from langchain_text_splitters import RecursiveCharacterTextSplitter
+import markdown
+from operator import itemgetter
+from langchain.schema.runnable import RunnablePassthrough
+from langchain_core.prompts import ChatPromptTemplate
+from langchain.schema import Document
+from dotenv import load_dotenv
+from langchain_community.vectorstores import Qdrant
+from PIL import Image, ImageEnhance
+from tools import sentiment_analysis_util
+#from langchain_qdrant import Qdrant
+import os
+import pandas as pd
+import numpy as np
+import datetime
+
+
+# App config
+load_dotenv()
+OPENAI_API_KEY = os.environ["OPENAI_API_KEY"]
+base_llm = ChatOpenAI(model="gpt-4o")
+embedding_model = OpenAIEmbeddings(model="text-embedding-3-small")
+
+# Page config
+
+st.set_page_config(
+    page_title="Narrativ 📰",
+    layout="wide",
+    initial_sidebar_state="expanded",
+    page_icon="🔍",
+)
+
+
+# Load environment variables
+load_dotenv()
+OPENAI_API_KEY = os.environ["OPENAI_API_KEY"]
+base_llm = ChatOpenAI(model="gpt-4o")
+embedding_model = OpenAIEmbeddings(model="text-embedding-3-small")
+uploaded_file = None
+topic='employment'
+date='2025-02-15'
+
+# Custom CSS for centered content
+st.markdown("""
+    <style>
+    .main-container {
+        max-width: 800px;
+        margin: 0 auto;
+        padding: 20px;
+    }
+    
+    .stSelectbox {
+        max-width: 400px;
+        margin: 0 auto;
+    }
+    
+    /* Center all text elements */
+    .centered-text {
+        text-align: center;
+    }
+    </style>
+""", unsafe_allow_html=True)
+
+# Header
+col1, col2, col3, col4,col5 = st.columns([1, 1, 2, 1, 1])
+from PIL import Image, ImageEnhance
+
+with col3:
+    st.markdown("<h1 class='centered-text'>Search Narrativ</h1>", unsafe_allow_html=True)
+
+
+# Suggestions
+topic_suggestions = [
+    "employment",
+    "remote work",
+    "unemployment"
+]
+
+data=pd.read_csv('./data/sentiment_index_hr_index_final2.csv',
+                index_col='index',
+                parse_dates=True
+            )
+
+# Convert the index to datetime, if not already done
+data.index = pd.to_datetime(data.index)
+
+# Generate a sorted list of unique dates
+sorted_dates = sorted(pd.unique(data.index))
+
+# Format the sorted dates as string 'YYYY-MM-DD'
+date_suggestions = [pd.Timestamp(date).strftime('%Y-%m-%d') for date in sorted_dates]
+date_suggestions=np.append('',date_suggestions)
+# Create centered container for search
+# Define the allowed date range
+start_date = datetime.date(2025, 1, 15)
+end_date = datetime.date(2025, 1, 21)
+
+sidebar=st.sidebar
+
+with sidebar:
+    st.subheader("📰 News")
+    topic = st.selectbox(
+        "Topic:",
+        options=[""] + topic_suggestions,
+        index=0,
+        key="topic_select",
+        placeholder="Select or type a topic..."
+    ) 
+
+    date = st.selectbox(
+        "Date (optional):",
+        options=date_suggestions,
+        index=0,
+        key="date_select",
+        placeholder="Select or type a date..."
+    )
+    date=str(date)
+    prompt = st.button("Summarize News", key="chat_button", use_container_width=True)
+
+    st.subheader("📊 Survey")
+    uploaded_file = st.file_uploader("📂 Upload Pulse Survey (.txt)", type="txt")
+    prompt_survey = st.button("Survey results", key="chat_button1", use_container_width=True)
+
+# Handle search submission
+
+if 'messages' not in st.session_state:
+    st.session_state.messages = []
+
+st.session_state.messages.append({"role": "assistant", "content": f'{date} {prompt}'})
+
+
+if prompt:
+    image = Image.open('./data/Sentiment_index_hr.png')
+    enhancer = ImageEnhance.Brightness(image)
+    #darker_image = enhancer.enhance(0.5)  # Adjust the brightness factor as needed
+    st.image(image, output_format="PNG", clamp=True)
+
+    if date:
+        try:
+            data=pd.read_csv('./data/sentiment_index_hr_index_final2.csv',
+                index_col='index',
+                parse_dates=True,
+                infer_datetime_format=True
+            )
+
+            data = data.loc[data.index == date]
+            filtered_data = data[data.apply(lambda row: row.astype(str).str.contains(topic, na=False).any(), axis=1)]
+            data_all = filtered_data.values.flatten()
+            docs = data_all 
+            if len(docs)<1:
+                st.warning("No articles found that contain the prompt string.")
+        
+            # Create markdown formatted text from the matching articles.
+            # docs_text = "\n".join([f"- {article}" for article in data_prompt if article])
+            # docs = [Document(page_content=docs_text)]
+        except Exception as e:
+            st.write('Please, enter a topic into the side panel.')
+    
+    else:
+        try:
+            data = pd.read_csv(
+                './data/sentiment_index_hr_index_final2.csv',
+                index_col='index',
+                parse_dates=True,
+                infer_datetime_format=True
+            )
+            filtered_data = data[data.apply(lambda row: row.astype(str).str.contains(topic, na=False).any(), axis=1)]
+            if len(filtered_data)<1:
+                filtered_data=data[data.apply(lambda row: row.astype(str), axis=1)]
+            data_all = filtered_data.values.flatten()
+            docs = data_all 
+            if len(docs)<1:
+                st.warning("No articles found that contain the prompt string.")
+
+        except Exception as e:
+            st.write('Please, enter a topic into the side panel.')
+
+    # scrape in real time reddit news
+    reddit_news_articles=sentiment_analysis_util.fetch_reddit_news('')
+    docs_text = "\n".join([f"- {value}" for value in data_all if not pd.isna(value)])
+    docs_text_reddit = "\n".join([f"- {value}" for value in reddit_news_articles if not pd.isna(value)])
+    docs_text=docs_text+'\n'+'Reddit news:'+'\n'+docs_text_reddit
+    docs = [Document(page_content=docs_text)]
+
+    with open('./data/reddit.txt', 'w') as file:
+        file.write(docs_text_reddit)
+
+    split_documents = RecursiveCharacterTextSplitter.from_tiktoken_encoder(
+        chunk_size=1000,
+        chunk_overlap=20
+    ).split_documents(docs)
+
+    vectorstore = Qdrant.from_documents(
+        split_documents,
+        embedding_model,
+        location=":memory:",
+        collection_name="langchainblogs"
+    )
+
+    retriever = vectorstore.as_retriever()
+
+    print("Loaded Vectorstore")
+    
+    # Add user message to chat history
+    st.session_state.messages.append({"role": "user", "content": topic})
+    # Display user message in chat message container
+    with st.chat_message("user"):
+        st.markdown(topic)
+
+    # Generate summarized message rationalize dominant sentiment
+    RAG_PROMPT ="""You are an HR analyst specializing in employment trends, workforce dynamics, and remote work adoption. Your task is to analyze news articles provided by a client on a specific topic. You will receive the full text of relevant articles, along with key data points. Your goal is to evaluate labor market conditions and provide insights into workforce changes.
+                Your Tasks:
+                1. Summarize Opinions:
+
+                    Extract the key opinions and perspectives from the provided news articles, reddit posts and linkedin posts.
+                    The news articles will include: title, URL, date, text, article source, sentiment index created by the company, sentiment index using HF (Hugging Face) model, and confidence for the HF index.
+                    The reddit posts will include: title, URL, date, text.
+                    Highlight any significant patterns, agreements, or disagreements across sources regarding job trends, hiring, layoffs, wages, or remote work policies.
+                    Include sentiment from reddit articles! Explicitly mention the reddit source in the summary.
+
+                2. Analyze Sentiment:
+
+                    Determine the overall sentiment (positive, negative, neutral) about labor market conditions based on the extracted opinions.
+                    Provide a clear explanation of your sentiment conclusion, referencing specific points or trends from the articles.
+
+                3. Provide Chain-of-Thought Reasoning:
+
+                    Detail your reasoning process step by step. Explain how you interpreted the articles, derived insights, and reached your sentiment conclusion.
+                    Ensure the reasoning is logical, transparent, and grounded in the content provided.
+
+                4. Collect URL Sources:
+
+                    From the provided context, select 5 critical and recent URL sources related to labor market trends and remote work policies.
+
+                Output Format:
+
+                    Summary of Opinions: [Concise summary of key opinions]
+                    Sentiment Analysis:
+                        Sentiment: [Positive/Negative/Neutral]
+                        Reasoning: [Detailed explanation here]
+                    Chain-of-Thought Reasoning: [Step-by-step explanation]
+                    Sources: [URLs for 5 most critical and recent articles on this topic]
+
+                Guidelines:
+
+                    Maintain objectivity and precision in your analysis.
+                    Focus on labor market trends, job market shifts, and remote work dynamics.
+                    Use professional and analytical language suitable for client reports.
+                    Respond in the language of the article (mostly English).
+
+                CONTEXT:
+
+                {context}
+                QUERY:
+
+                {question}
+
+                Use the provided context to answer the user's question. Only use the provided context to answer the question. If you do not know the answer, respond with "I don't know."""
+    rag_prompt = ChatPromptTemplate.from_template(RAG_PROMPT)
+    # RAG CHAIN
+    lcel_rag_chain = (
+        {"context": itemgetter("question") | retriever, "question": itemgetter("question")}
+        | RunnablePassthrough.assign(context=itemgetter("context"))
+        | {"response": rag_prompt | base_llm, "context": itemgetter("context")}
+    )
+
+    try:
+        summary = lcel_rag_chain.invoke({"question": topic})
+        print(summary)
+        st.chat_message("assistant").write((summary['response'].content))
+    except Exception as e:
+        st.error(f"Error generating summary: {e}")
+
+    if date:
+        with open('./data/sentiment_index_hr_index_final_date.md', 'w') as file:
+                file.write(str(data_all))
+    else:
+        with open('./data/sentiment_index_hr_index_final1.md', 'w') as file:
+                file.write(str(data_all))
+
+if prompt_survey:
+    import survey_summary
+    st.session_state['uploaded_file'] = uploaded_file
+    analysis = survey_summary.survey_agent('',uploaded_file)
+    st.chat_message("assistant").write(str(analysis))
+
+client = OpenAI(api_key=OPENAI_API_KEY)
+
+if "openai_model" not in st.session_state:
+    st.session_state["openai_model"] = "gpt-4o"
+
+prompt1 = st.chat_input("Type your additional questions here...")
+
+# Suggested keywords with enhanced styling
+suggested_keywords = ["Latest News", "News on remote work", f"Survey sentiment", f"Employee satisfaction", f"How many employees?"]
+st.markdown("**Suggested Keywords:**")
+cols = st.columns(len(suggested_keywords))
+for idx, keyword in enumerate(suggested_keywords):
+    if cols[idx].button(keyword, key=keyword):
+        prompt1 = keyword
+
+if prompt1:
+    st.session_state.messages.append({"role": "user", "content": prompt1})
+    with open('./data/employee_pulse_survey.txt', 'r') as file:
+        survey_txt = file.read()
+    # Decide if call SQL agent, SURVEY agent or SENTIMENT agent
+    database_columns=pd.read_csv('./data/hr_data.csv').columns
+    response = base_llm.invoke(f"""You are the Supervisor of the company. In your team you have, general conversation analyst, data analyst, survey analyst and news article analyst. 
+                                    If the question {prompt1} can be answered from the history of the conversation:{st.session_state.messages[-10:]} or you can use your knowledge and do not need to call the team members, respond 'history'.
+                                    If not: decide if the question: '{prompt1}' is about data available in the database, based on the following columns: {database_columns}, it has information about all employees. If yes, respond 'data'.
+                                    If not: decide if the question is asking about the survey: {survey_txt}. If yes, respond 'survey'.
+                                    If not: decide if the question is asking about news articles on employment trends or remote work. If yes, respond 'news'.
+                                    Your response will be either 'history' or 'data' or 'survey' or 'news'. 
+                                    Don't answer anything else.
+                                    Survey: {survey_txt}""")
+    st.write(response.content)
+    if 'data' in response.content.lower():
+        # SQL AGENT
+        import sql_agent
+
+        response = sql_agent.sql_agent(f'the question is: {prompt1} and the history is: {st.session_state.messages[-10:]}')
+        st.session_state.messages.append({"role": "sql_agent", "content": response})
+
+    elif 'news' in response.content.lower():
+        # SENTIMENT AGENT
+        if date:
+            file_path = f'./data/sentiment_index_hr_index_final_date.md'
+        else:
+            file_path = f'./data/sentiment_index_hr_index_final1.md'
+            
+        try:
+            with open(file_path, "r", encoding="utf-8") as file_content:
+                docs = file_content.read()
+        except Exception as e:
+            st.error(f"Error loading context: {e}")
+            docs = ""
+
+        # Display user message in chat message container
+        response = base_llm.invoke(f"""You are a data analyst, the question is: {prompt1}, the conversation history is: {st.session_state.messages[-10:]} and the context is from {docs}""")
+        st.session_state.messages.append({"role": "news_agent", "content": response})
+        # st.chat_message("assistant").write(str(response))
+
+    elif 'survey' in response.content.lower():
+        # SURVEY AGENT
+        with open('./data/employee_pulse_survey.txt', 'r') as file:
+            survey_text = file.read()
+        import survey_agent1
+        response = survey_agent1.analyze_survey_document(survey_text, f'the question is: {prompt1} and the history is: {st.session_state.messages[-10:]}')
+        st.session_state.messages.append({"role": "survey_agent", "content": response})
+        # st.chat_message("assistant").write(str(response))
+        
+    # Go back to the MAIN SUPERVISOR
+        # Display user message in chat message container
+    print('History:',st.session_state.messages[-10:])
+    response = base_llm.invoke(f"""You are a supervisor, who collects the answers from the team and give the final answer to the user.
+                                      Take the last response, 'response', from your team member: SQL agent, SURVEY agent or SENTIMENT agent and give the final answer to the user.
+                                      The user's question is: {prompt1},
+                                      the responses from the team are: {st.session_state.messages[-10:]}""")
+
+    st.chat_message("supervisor").write(str(response.content))
+    st.session_state.messages.append({"role": "supervisor", "content": response.content})
+
+    # with st.chat_message("user"):
+    #     st.markdown(prompt1)
+    # # Display assistant response in chat message container
+    # with st.chat_message("assistant"):
+    #     try:
+    #         stream = client.chat.completions.create(
+    #             model=st.session_state["openai_model"],
+    #             messages=[
+    #                 {"role": m["role"], "content": m["content"]}
+    #                 for m in st.session_state.messages[:-10]
+    #             ],
+    #             stream=True,
+    #         )
+    #         response = st.write_stream(stream)
+    #         st.session_state.messages.append({"role": "supervisor", "content": response})
+    #     except Exception as e:
+    #         st.error(f"Error generating response: {e}")

requirements.txt

@@ -0,0 +1,12 @@
+streamlit==1.37.1
+cohere==4.37
+openai<2.0.0
+tiktoken<1
+python-dotenv==1.0.0
+langchain==0.2.5
+langchain_core==0.2.9
+langchain_community==0.2.5
+langchain-text-splitters==0.2.1
+langchain_openai==0.1.8
+qdrant-client==1.13.0
+Markdown==3.7

sql_agent.py

@@ -0,0 +1,34 @@
+from sqlalchemy import create_engine
+from langchain_openai import ChatOpenAI
+from langchain_community.utilities import SQLDatabase
+from langchain_community.agent_toolkits.sql.base import create_sql_agent  # Updated Import
+import streamlit as st
+import os
+from dotenv import load_dotenv
+
+load_dotenv()
+
+def sql_agent(prompt):
+    # Set OpenAI API key
+    OPENAI_API_KEY = os.getenv("OPENAI_API_KEY")
+
+    DATABASE_URL = "sqlite:///data/main.db"  # Change to your database URL
+    engine = create_engine(DATABASE_URL)
+    db = SQLDatabase(engine)
+
+    # Initialize OpenAI model
+    llm = ChatOpenAI(temperature=0, model_name="gpt-4o")
+
+    # Create SQL Agent
+    agent_executor = create_sql_agent(
+        llm=llm,
+        db=db,
+        agent_type="openai-tools",  # Updated agent type
+        verbose=True
+    )
+
+    # Example query
+    # query = "How many Senior Engineers are in the company?"
+    response = agent_executor.invoke({"input": prompt})['output']
+    
+    return response

survey_agent1.py

@@ -0,0 +1,125 @@
+import streamlit as st
+import openai
+import os
+from dotenv import load_dotenv
+from langchain_openai import ChatOpenAI
+from langchain_core.prompts import ChatPromptTemplate
+from langchain_core.runnables import RunnablePassthrough
+from langchain_openai.embeddings import OpenAIEmbeddings
+from langchain_text_splitters import RecursiveCharacterTextSplitter
+from langchain_community.vectorstores import Qdrant
+from langchain_core.documents import Document
+from operator import itemgetter
+
+
+# Load API key from .env file
+load_dotenv()
+base_llm = ChatOpenAI(model="gpt-4o")
+openai.api_key = os.getenv("OPENAI_API_KEY")
+embedding_model = OpenAIEmbeddings(model="text-embedding-3-small")
+
+
+def analyze_survey_document(survey_text, question):
+    """Send entire survey document to OpenAI for RAG-based analysis."""
+    prompt = """
+    You are an HR analyst answering questions about employee pulse survey responses. Below is the entire survey with questions and corresponding employee answers.
+
+    **context:**
+    {context}
+
+    **question:**
+    {question}
+
+    Answer all questions based on the context. If you don't know say I don't know.
+    """
+    rag_prompt = ChatPromptTemplate.from_template(prompt)
+
+# Token length function
+
+   # Text splitter for chunking per page
+    text_splitter =  RecursiveCharacterTextSplitter.from_tiktoken_encoder(
+        chunk_size=500,  # Adjust chunk size based on contract structure
+        chunk_overlap=50
+    )
+
+
+    chunks = text_splitter.split_text(survey_text)
+    
+    # Initialize Qdrant Vector Store for each page
+    qdrant_vectorstore = Qdrant.from_texts(
+        chunks,  # Pass as a Document object
+        embedding_model,  # Ensure embedding_model is defined
+        location=":memory:", 
+        collection_name="formatted_text"
+    )
+    retriever = qdrant_vectorstore.as_retriever()
+
+    # Define retrieval and response generation chain
+    lcel_rag_chain = (
+        {"context": itemgetter("question") | retriever, "question": itemgetter("question")}
+        | RunnablePassthrough.assign(context=itemgetter("context"))
+        | {"response": rag_prompt | base_llm, "context": itemgetter("context")}
+    )
+    response = lcel_rag_chain.invoke({"context": survey_text, "question": question})['response'].content
+
+    return response
+
+def chat_with_ai(chat_history):
+    """Chat interface with OpenAI model."""
+    try:
+        response = openai.chat.completions.create(
+            model="gpt-4o",
+            messages=chat_history
+        )
+        return response.choices[0].message.content
+    except Exception as e:
+        return f"Error: {str(e)}"
+
+def survey_agent(prompt,uploaded_file):
+    st.subheader("📊 Employee Pulse Survey Analysis & AI Chat")
+
+    with open('./data/employee_pulse_survey.txt', 'r') as file:
+        survey_text_base = file.read()
+
+    #uploaded_file = st.file_uploader("📂 Upload Pulse Survey (.txt)", type="txt")
+    if not uploaded_file:
+        uploaded_file1 = survey_text_base
+
+    #check1 = st.button(f"Generate summary")
+
+    #if check1:
+    if uploaded_file or uploaded_file1:
+        st.write("✅ File uploaded successfully! Analyzing responses...")
+        if uploaded_file:
+            survey_text = uploaded_file.read().decode("utf-8").strip()
+        else:
+            survey_text = uploaded_file1
+
+        with st.spinner("🔍 Analyzing entire survey..."):
+            analysis = analyze_survey_document(survey_text)
+            st.session_state["survey_summary"] = analysis
+            st.markdown(analysis)
+
+    # AI Chat with the survey analysis
+    st.header("💬 Chat with AI about the Survey")
+    st.write("Ask questions about the pulse survey insights.")
+
+    if "messages" not in st.session_state:
+        st.session_state["messages"] = [
+            {"role": "system", "content": "You are an HR expert analyzing a pulse survey. Provide insights based on the summary below."},
+            {"role": "user", "content": st.session_state.get("survey_summary", "No survey uploaded yet.")}
+        ]
+
+    user_input = st.text_input("🔍 Ask a question about the survey results:")
+    
+    if st.button("Ask AI"):
+        if user_input:
+            st.session_state["messages"].append({"role": "user", "content": user_input})
+            with st.spinner("🔍 Thinking..."):
+                ai_response = chat_with_ai(st.session_state["messages"])
+                st.session_state["messages"].append({"role": "assistant", "content": ai_response})
+                st.markdown(f"**AI:** {ai_response}")
+        else:
+            st.warning("⚠️ Please enter a question.")
+
+    return ai_response

survey_summary.py

@@ -0,0 +1,93 @@
+import streamlit as st
+import openai
+import os
+from dotenv import load_dotenv
+import pandas as pd
+# Load API key from .env file
+load_dotenv()
+openai.api_key = os.getenv("OPENAI_API_KEY")
+
+def analyze_survey_document(survey_text):
+    """Send entire survey document to OpenAI for RAG-based analysis."""
+    prompt = f"""
+    You are an HR analyst evaluating employee pulse survey responses. 
+    Below is the entire survey with questions and corresponding employee answers.
+    Also, you have access to LinkedIn posts for your employees Katerina Gawthorpe, Steven Gawthorpe, Deepali Khalkar, and Tyler Robinson. Look at her posts and include them in your analysis about what your employees is up to. Thier role is AI expert, notice if they are personally interested in AI and mention it.
+
+    **Your tasks:**
+    1. **Summarize responses for each question** - Capture overall themes and sentiments.
+    2. **Identify key areas of dissatisfaction** - Highlight the most frequent complaints.
+    3. **Provide recommendations** - Suggest how the company can address these concerns.
+    4. **Include LinkedIn posts** - Look at their posts and include them in your analysis about what your employees are up to. Their roles are data scinetists and AI experts, notice if they are personally interested in AI and mention it.
+    **Survey Responses:**
+    {survey_text}
+
+    **Your analysis should include:**
+    - **Summary for each question**
+    - **Overall themes across all questions**
+    - **Most common complaints**
+    - **Actionable recommendations**
+    """
+
+    try:
+        response = openai.chat.completions.create(
+            model="gpt-4o",
+            messages=[{"role": "system", "content": "You are an HR analyst summarizing an employee pulse survey."},
+                    {"role": "user", "content": prompt}]
+        )
+        return response.choices[0].message.content
+    except Exception as e:
+        return f"Error: {str(e)}"
+
+def chat_with_ai(chat_history):
+    """Chat interface with OpenAI model."""
+    try:
+        response = openai.chat.completions.create(
+            model="gpt-4o",
+            messages=chat_history
+        )
+        return response.choices[0].message.content
+    except Exception as e:
+        return f"Error: {str(e)}"
+
+def survey_agent(prompt,uploaded_file):
+    st.subheader("📊 Employee Pulse Survey Analysis & AI Chat")
+
+    with open('./data/employee_pulse_survey.txt', 'r') as file:
+        survey_text_base = file.read()
+
+    #uploaded_file = st.file_uploader("📂 Upload Pulse Survey (.txt)", type="txt")
+    if not uploaded_file:
+        uploaded_file1 = survey_text_base
+
+    # Get LinkedIn posts
+    df_linkedin=pd.read_csv('./data/linkedin_post_result.csv', usecols=['postTimestamp','postUrl','postContent','author'])
+    df_linkedin=df_linkedin.dropna()
+    df_linkedin_selected=pd.DataFrame()
+    df_linkedin_selected['text']=df_linkedin['postContent'].str.replace('\n', ' ')
+    df_linkedin_selected['author']=df_linkedin['author'].str.replace('\n', ' ')
+    df_linkedin_selected['date']=pd.to_datetime(df_linkedin['postTimestamp']).dt.strftime('%Y-%m-%d')
+    df_linkedin_selected['url']=df_linkedin['postUrl']
+
+    df_linkedin_selected=df_linkedin_selected.values.flatten()
+    docs_text_linkedin = "\n".join([f"- {value}" for value in df_linkedin_selected if not pd.isna(value)])
+
+    uploaded_file1=uploaded_file1+'\n'+'LinkedIn posts:'+'\n'+docs_text_linkedin
+    
+    #check1 = st.button(f"Generate summary")
+
+    #if check1:
+    if uploaded_file or uploaded_file1:
+        st.write("✅ File uploaded successfully! Analyzing responses...")
+        if uploaded_file:
+            survey_text = uploaded_file.read().decode("utf-8").strip()
+            survey_text=survey_text+'\n'+'LinkedIn posts:'+'\n'+docs_text_linkedin
+        else:
+            survey_text = uploaded_file1
+
+        with st.spinner("🔍 Analyzing entire survey..."):
+            analysis = analyze_survey_document(survey_text)
+            #st.session_state["survey_summary"] = analysis
+            #st.markdown(analysis)
+
+    return analysis

tools/sentiment_analysis_util.py

@@ -0,0 +1,260 @@
+
+import os
+from dotenv import load_dotenv
+from transformers import pipeline
+import pandas as pd
+#from langchain_openai import ChatOpenAI
+import praw
+from datetime import datetime
+import numpy as np
+#from tavily import TavilyClient
+
+load_dotenv()
+#TAVILY_API_KEY = os.environ["TAVILY_API_KEY"]
+
+# def fetch_news(topic):
+    
+#     """ Fetches news articles within a specified date range.
+
+#     Args:
+#     - topic (str): Topic of interest
+    
+#     Returns:
+#     - list: A list of dictionaries containing news. """
+    
+#     load_dotenv()
+#     days_to_fetch_news = os.environ["DAYS_TO_FETCH_NEWS"]
+
+#     googlenews = GoogleNews()
+#     googlenews.set_period(days_to_fetch_news)
+#     googlenews.get_news(topic)
+#     news_json=googlenews.get_texts()
+#     urls=googlenews.get_links()
+    
+#     no_of_news_articles_to_fetch = os.environ["NO_OF_NEWS_ARTICLES_TO_FETCH"]
+#     news_article_list = []
+#     counter = 0
+#     for article in news_json:
+        
+#         if(counter >= int(no_of_news_articles_to_fetch)):
+#             break
+
+#         relevant_info = {
+#             'News_Article': article,
+#             'URL': urls[counter]
+#         }
+#         news_article_list.append(relevant_info)
+#         counter+=1
+#     return news_article_list
+
+def fetch_tavily_news(topic):
+    """ Fetches news articles.
+
+    Args:
+    - topic (str): Topic of interest
+    
+    Returns:
+    - list: A list of dictionaries containing news. """
+    
+    # Step 1. Instantiating your TavilyClient
+    tavily_client = TavilyClient(api_key=TAVILY_API_KEY)
+
+    #response = tavily_client.search(topic)
+    # Step 2.1. Executing a context search query
+    answer = tavily_client.get_search_context(query=f"Give me news on {topic}")
+    
+    line=[]
+    tavily_news=[]
+
+    for i in range(len(answer.split("url")))[1:]:
+        https_link=(answer.split("url")[i].split("\\\\\\")[2]).split('"')[1]
+        topic_answer=answer.split("url")[i].split("\\\\\\")[-3]
+        tavily_news=np.append(tavily_news,{'https':https_link,'topic_answer':topic_answer})
+
+    return tavily_news
+
+def fetch_reddit_news(topic):
+    load_dotenv()
+    REDDIT_USER_AGENT= os.environ["REDDIT_USER_AGENT"]
+    REDDIT_CLIENT_ID= os.environ["REDDIT_CLIENT_ID"]
+    REDDIT_CLIENT_SECRET= os.environ["REDDIT_CLIENT_SECRET"]
+    #https://medium.com/geekculture/a-complete-guide-to-web-scraping-reddit-with-python-16e292317a52
+    user_agent = REDDIT_USER_AGENT
+    reddit = praw.Reddit (
+    client_id= REDDIT_CLIENT_ID,
+    client_secret= REDDIT_CLIENT_SECRET,
+    user_agent=user_agent
+    )
+
+    headlines = set ( )
+    for submission in reddit.subreddit('nova').search('job',time_filter='day'):
+        headlines.add(submission.title + ', Date: ' +datetime.utcfromtimestamp(int(submission.created_utc)).strftime('%Y-%m-%d %H:%M:%S') + ', URL:' +submission.url)
+    
+    for submission in reddit.subreddit('fednews').search('labor',time_filter='day'):
+        headlines.add(submission.title + ', Date: ' +datetime.utcfromtimestamp(int(submission.created_utc)).strftime('%Y-%m-%d %H:%M:%S') + ', URL:' +submission.url)
+    
+    for submission in reddit.subreddit('fednews').search('job',time_filter='day'):
+        headlines.add(submission.title + ', Date: ' +datetime.utcfromtimestamp(int(submission.created_utc)).strftime('%Y-%m-%d %H:%M:%S') + ', URL:' +submission.url)
+    
+    for submission in reddit.subreddit('fednews').search('employment',time_filter='day'):
+        headlines.add(submission.title + ', Date: ' +datetime.utcfromtimestamp(int(submission.created_utc)).strftime('%Y-%m-%d %H:%M:%S') + ', URL:' +submission.url)
+    
+    for submission in reddit.subreddit('fednews').search('layoff',time_filter='day'):
+        headlines.add(submission.title + ', Date: ' +datetime.utcfromtimestamp(int(submission.created_utc)).strftime('%Y-%m-%d %H:%M:%S') + ', URL:' +submission.url)
+    
+
+    for submission in reddit.subreddit('washingtondc').search('job',time_filter='day'):
+        headlines.add(submission.title + ', Date: ' +datetime.utcfromtimestamp(int(submission.created_utc)).strftime('%Y-%m-%d %H:%M:%S') + ', URL:' +submission.url)
+    
+    #if len(headlines)<10:
+    #    for submission in reddit.subreddit('washingtondc').search(topic,time_filter='year'):
+    #        headlines.add(submission.title + ', Date: ' +datetime.utcfromtimestamp(int(submission.created_utc)).strftime('%Y-%m-%d %H:%M:%S') + ', URL:' +submission.url)
+    #if len(headlines)<10:
+    #    for submission in reddit.subreddit('washingtondc').search(topic): #,time_filter='week'):
+    #        headlines.add(submission.title + ', Date: ' +datetime.utcfromtimestamp(int(submission.created_utc)).strftime('%Y-%m-%d %H:%M:%S') + ', URL:' +submission.url)
+
+    return headlines
+
+def analyze_sentiment(article):
+    """
+    Analyzes the sentiment of a given news article.
+
+    Args:
+    - news_article (dict): Dictionary containing 'summary', 'headline', and 'created_at' keys.
+
+    Returns:
+    - dict: A dictionary containing sentiment analysis results.
+    """
+
+    #Analyze sentiment using default model
+    #classifier = pipeline('sentiment-analysis')
+
+    #Analyze sentiment using specific model
+    classifier = pipeline(model='tabularisai/robust-sentiment-analysis') #mrm8488/distilroberta-finetuned-financial-news-sentiment-analysis')
+    sentiment_result = classifier(str(article))
+
+    analysis_result = {
+        'News_Article': article,
+        'Sentiment': sentiment_result
+    }
+
+    return analysis_result
+
+
+# def generate_summary_of_sentiment(sentiment_analysis_results): #, dominant_sentiment):
+    
+    
+#     news_article_sentiment = str(sentiment_analysis_results)
+#     print("News article sentiment : " + news_article_sentiment)
+    
+
+#     OPENAI_API_KEY = os.environ["OPENAI_API_KEY"]
+#     model = ChatOpenAI(
+#         model="gpt-4o",
+#         temperature=0,
+#         max_tokens=None,
+#         timeout=None,
+#         max_retries=2,
+#         api_key=OPENAI_API_KEY,  # if you prefer to pass api key in directly instaed of using env vars
+#         # base_url="...",
+#         # organization="...",
+#         # other params...
+#     )
+
+#     messages=[
+#             {"role": "system", "content": "You are a helpful assistant that looks at all news articles with their sentiment, hyperlink and date in front of the article text, the articles MUST be ordered by date!, and generate a summary rationalizing dominant sentiment. At the end of the summary, add URL links with dates for all the articles in the markdown format for streamlit. Make sure the articles as well as the links are ordered descending by Date!!!!!!! Example of adding the URLs: The Check out the links: [link](%s) % url, 2024-03-01. "},
+#             {"role": "user", "content": f"News articles and their sentiments: {news_article_sentiment}"} #, and dominant sentiment is: {dominant_sentiment}"}
+#     ]
+#     response = model.invoke(messages)
+    
+
+#     summary = response.content
+#     print ("+++++++++++++++++++++++++++++++++++++++++++++++")
+#     print(summary)
+#     print ("+++++++++++++++++++++++++++++++++++++++++++++++")
+#     return summary
+
+
+# def plot_sentiment_graph(sentiment_analysis_results):
+#     """
+#     Plots a sentiment analysis graph 
+
+#     Args:
+#     - sentiment_analysis_result): (dict): Dictionary containing 'Review Title : Summary', 'Rating', and 'Sentiment' keys.
+
+#     Returns:
+#     - dict: A dictionary containing sentiment analysis results.
+#     """
+#     df = pd.DataFrame(sentiment_analysis_results)
+#     print(df)
+
+#     #Group by Rating, sentiment value count
+#     grouped = df['Sentiment'].value_counts()
+
+#     sentiment_counts = df['Sentiment'].value_counts()
+
+#     # Plotting pie chart
+#     # fig = plt.figure(figsize=(5, 3))
+#     # plt.pie(sentiment_counts, labels=sentiment_counts.index, autopct='%1.1f%%', startangle=140)
+#     # plt.axis('equal')  # Equal aspect ratio ensures that pie is drawn as a circle.
+    
+#     #Open below when u running this program locally and c
+#     #plt.show()
+
+#     return sentiment_counts
+
+
+# def get_dominant_sentiment (sentiment_analysis_results):
+#     """
+#     Returns overall sentiment, negative or positive or neutral depending on the count of negative sentiment vs positive sentiment 
+
+#     Args:
+#     - sentiment_analysis_result): (dict): Dictionary containing 'summary', 'headline', and 'created_at' keys.
+
+#     Returns:
+#     - dict: A dictionary containing sentiment analysis results.
+#     """
+#     df = pd.DataFrame(sentiment_analysis_results)
+
+#     # Group by the 'sentiment' column and count the occurrences of each sentiment value
+#     print(df)
+#     print(df['Sentiment'])
+#     sentiment_counts = df['Sentiment'].value_counts().reset_index()
+#     sentiment_counts.columns = ['sentiment', 'count']
+#     print(sentiment_counts)
+
+#     # Find the sentiment with the highest count
+#     dominant_sentiment = sentiment_counts.loc[sentiment_counts['count'].idxmax()]
+
+#     return dominant_sentiment['sentiment']
+
+# #starting point of the program
+# if __name__ == '__main__':
+    
+#     #fetch news
+#     news_articles = fetch_news('AAPL')
+
+#     analysis_results = []
+    
+#     #Perform sentiment analysis for each product review
+#     for article in news_articles:
+#         sentiment_analysis_result = analyze_sentiment(article['News_Article'])
+
+#         # Display sentiment analysis results
+#         print(f'News Article: {sentiment_analysis_result["News_Article"]} : Sentiment: {sentiment_analysis_result["Sentiment"]}', '\n')
+
+#         result = {
+#                     'News_Article': sentiment_analysis_result["News_Article"],
+#                     'Sentiment': sentiment_analysis_result["Sentiment"][0]['label']
+#                 }
+        
+#         analysis_results.append(result)
+
+    
+#     #Graph dominant sentiment based on sentiment analysis data of reviews
+#     dominant_sentiment = get_dominant_sentiment(analysis_results)
+#     print(dominant_sentiment)
+    
+#     #Plot graph
+#     plot_sentiment_graph(analysis_results)
+