fix server issues

.gitignore

@@ -22,6 +22,7 @@ build/
 # Jupyter Notebook checkpoints
 .ipynb_checkpoints
 
+
 # Pytest cache
 .cache
 

.python-version

@@ -0,0 +1 @@
+Python 3.11.11

app.py

@@ -1,6 +1,6 @@
 import os
 from typing import Optional
-from fastapi import FastAPI, HTTPException, Security, Depends
+from fastapi import FastAPI, HTTPException, Security, Depends, Request
 from fastapi.security import APIKeyHeader
 from fastapi.middleware.cors import CORSMiddleware
 from fastapi.responses import JSONResponse
@@ -26,18 +26,33 @@ import requests
 import os
 from PyPDF2 import PdfReader
 from config import QUIZ_TYPES
+from config import Config
+import logging
+import time
 # Load environment variables
 load_dotenv()
-API_KEY = os.getenv("API_KEY_ACCESS")
+from config import Config
+API_KEY = Config.API_KEY_ACCESS
+
+# Configure logging
+logging.basicConfig(
+    level=logging.INFO,
+    format='%(asctime)s - %(name)s - %(levelname)s - %(message)s'
+)
+logger = logging.getLogger(__name__)
+
+# Also configure uvicorn logger
+uvicorn_logger = logging.getLogger("uvicorn.access")
+uvicorn_logger.setLevel(logging.INFO)
 
 base_path = os.path.join("data", "config_files")
 QUESTIONS_PATH = os.path.join(base_path, "questions.json")
 THEME_CONTEXT_PATH = os.path.join(base_path, "theme_context.json")
+backend_base_url = Config.BACKEND_BASE_URL
 
-# Load themes at module level
-with open(THEME_CONTEXT_PATH, "r") as f:
+
+with open(THEME_CONTEXT_PATH, "r", encoding="utf-8") as f:
     themes = json.load(f)
-
 # Initialize FastAPI app
 app = FastAPI(
     title="Fire Fighter Interview API",
@@ -45,6 +60,40 @@ app = FastAPI(
     version="1.0.0"
 )
 
+# Add request logging middleware
+@app.middleware("http")
+async def log_requests(request: Request, call_next):
+    start_time = time.time()
+
+    # Log incoming request (using both logger and print for visibility)
+    log_msg = f"🔥 INCOMING REQUEST: {request.method} {request.url}"
+    logger.info(log_msg)
+    print(log_msg)
+
+  
+
+    # Get request body for POST requests
+    if request.method == "POST":
+        body = await request.body()
+        body_msg = f"🔥 Request Body: {body.decode('utf-8') if body else 'Empty'}"
+        logger.info(body_msg)
+        print(body_msg)
+
+        # Re-create request with body for downstream processing
+        async def receive():
+            return {"type": "http.request", "body": body}
+        request._receive = receive
+
+    response = await call_next(request)
+
+    # Log response
+    process_time = time.time() - start_time
+    response_msg = f"🔥 RESPONSE: {response.status_code} - Time: {process_time:.4f}s"
+    logger.info(response_msg)
+    print(response_msg)
+
+    return response
+
 # Add CORS middleware
 app.add_middleware(
     CORSMiddleware,
@@ -58,7 +107,6 @@ app.add_middleware(
 api_key_header = APIKeyHeader(name="Authorization", auto_error=False)
 
 
-
 async def get_api_key(api_key_header: str = Security(api_key_header)) -> str:
     """Validate API key from header"""
     if not api_key_header or not api_key_header.startswith('Bearer '):
@@ -68,6 +116,7 @@ async def get_api_key(api_key_header: str = Security(api_key_header)) -> str:
         )
     
     token = api_key_header.split(' ')[1]
+    print(f"Token : {token}")
     if token != API_KEY:
         raise HTTPException(
             status_code=401,
@@ -79,23 +128,28 @@ async def get_api_key(api_key_header: str = Security(api_key_header)) -> str:
 class ChatRequest(BaseModel):
     resume_url: Optional[str] = None
     query: str=None
-    conversation_id: str
+    chat_id: int
     theme_id: Optional[int] = 1
+    full_history_url: Optional[str] = None
+    form_id:Optional[int] = None
+    feedback: Optional[str] = None
+    generate_theme:str="NO"
 
 class ChatResponse(BaseModel):
     message: str
     end: bool
+    pop_theme_generation:bool
     error: Optional[str] = None
 
 
 
 class GeneratePDFRequest(BaseModel):
-    conversation_id: str
-    feedback: Optional[str] = None
-    previous_results: Optional[Dict[str, Any]] = None
     resume_url: Optional[str] = None
+    chat_id: int
+    theme_id: Optional[int] = 1
     full_history_url: Optional[str] = None
     form_id:Optional[int] = None
+    generate_theme:str="YES"
 
 class QuizRequest(BaseModel):
     pdf_url: str
@@ -142,64 +196,98 @@ async def extract_pdf_text(pdf_url: str) -> Union[str, None]:
             status_code=500,
             detail=f"Error processing PDF: {str(e)}"
         )
-
-@app.post("/rescue-career/chat", response_model=ChatResponse)
+    
+@app.post("/rescue-career/chat")
 async def chat_endpoint(
     request: ChatRequest,
     api_key: str = Depends(get_api_key)
 ):
     try:
         # Validate theme
+        print(f"Received request with theme_id: {request.theme_id}")  # Debugging print
         matching_themes = [t for t in themes if t["id"] == request.theme_id]
         if not matching_themes:
+            print(f"No theme found with ID: {request.theme_id}")  # Debugging print
             raise HTTPException(
                 status_code=400,
                 detail=f"No theme found with ID {request.theme_id}"
             )
-
-        # Only try to load document if resume_url is provided
+        print(f"Validated theme ID: {request.theme_id}")  # Print statement added
+        
         resume_docs = ""
         if request.resume_url:
+            print(f"Loading resume from URL: {request.resume_url}")  # Debugging print
             docs = load_document(request.resume_url)
             if not docs:
+                print("Invalid resume URL: Unable to fetch document")  # Debugging print
                 raise HTTPException(
                     status_code=400, 
                     detail="Invalid resume URL: Unable to fetch document"
                 )
             resume_docs = "\n".join(f"- {doc.page_content}" for doc in docs)
+            print(f"Loaded resume documents: {resume_docs[:100]}")  # Debugging print
 
-        # Get AI chat response
-        response = ai_chat(
-            query=request.query,
-            conversation_id=request.conversation_id,
-            theme_id=request.theme_id,
-            resume=resume_docs
-        )
+        full_history_docs = ""
+        if request.full_history_url:
+            print(f"Loading full history from URL: {request.full_history_url}")  # Debugging print
+            docs = load_document(request.full_history_url)
+            if not docs:
+                print("Invalid full history URL: Unable to fetch document")  # Debugging print
+                raise HTTPException(
+                    status_code=400, 
+                    detail="Invalid full history URL: Unable to fetch document"
+                )
+            full_history_docs = "\n".join(f"- {doc.page_content}" for doc in docs)
+            print(f"Loaded full history documents: {full_history_docs[:100]}")  # Debugging print
 
+        form_response_docs = ""
+        if request.form_id:
+            print(f"Fetching form response for form_id: {request.form_id}")  # Debugging print
+            try:
+                x_api_key = os.getenv("BACKEND_XAPI_KEY")
+                url = f"{backend_base_url}/v3/api/custom/theme-document/answer/{request.form_id}?x-project={x_api_key}"
+                result = requests.get(url)
+                result.raise_for_status()  # Ensure we raise an error for bad responses
+                form_response = result.json()["data"]  # Return response in JSON format
+                form_response_docs = "\n".join(f"- {form_response}")
+                
+                print(f"Fetched form response: {form_response}")  # Debugging print
+            except requests.RequestException as e:
+                print(f"Error fetching onboarding data: {str(e)}")  # Debugging print
+                raise HTTPException(
+                    status_code=400, 
+                    detail="Unable to fetch onboarding data"
+                )
         # Parse response
-        try:
-            parsed_response = json.loads(response)
-            return ChatResponse(
-                message=parsed_response.get("message", ""),
-                end=parsed_response.get("end", "no") == "yes",
-                error=None
+        print("Parsing AI response...")  # Debugging print
+        query = request.query
+        if not query:
+            query = "Let's get started"
+        response = ai_chat(
+                query=query,
+                conversation_id=request.chat_id,
+                theme_id=request.theme_id,
+                resume=resume_docs,
+                full_history=full_history_docs,
+                form_response=form_response_docs
             )
-        except json.JSONDecodeError:
-            return ChatResponse(
-                message=response,
-                end=False,
-                error=None
-            )
-
-    except HTTPException as e:
-        # Re-raise HTTP exceptions
-        raise
+        print(response)
+        
+        return ChatResponse(
+            message=response.get("message", ""),
+            end=response.get("end", "no") == "yes",
+            pop_theme_generation=response.get("pop_theme_generation","no") == "yes",
+            error=None
+        )
+        
     except Exception as e:
+        print(f"Error processing chat request: {str(e)}")  # Print statement added
         raise HTTPException(
             status_code=500,
             detail=f"Error processing chat request: {str(e)}"
         )
-     
+
+
 
 @app.post("/rescue-career/generate-theme")
 async def generate_pdf_endpoint(
@@ -208,87 +296,119 @@ async def generate_pdf_endpoint(
 ):
     
     try:
-        # Here you would fetch the conversation data using the conversation_id
-        # This is a placeholder - replace with your actual conversation data fetching logic
-        conversation_data = await get_conversation_data(request.conversation_id)
-        
-        if not conversation_data:
+
+        print(f"Received request with theme_id: {request.theme_id}")  # Debugging print
+        matching_themes = [t for t in themes if t["id"] == request.theme_id]
+        if not matching_themes:
+            print(f"No theme found with ID: {request.theme_id}")  # Debugging print
             raise HTTPException(
-                status_code=404,
-                detail=f"No conversation found with ID {request.conversation_id}"
+                status_code=400,
+                detail=f"No theme found with ID {request.theme_id}"
             )
+        print(f"Validated theme ID: {request.theme_id}")  # Print statement added
         
         resume_docs = ""
         if request.resume_url:
+            print(f"Loading resume from URL: {request.resume_url}")  # Debugging print
             docs = load_document(request.resume_url)
             if not docs:
+                print("Invalid resume URL: Unable to fetch document")  # Debugging print
                 raise HTTPException(
                     status_code=400, 
                     detail="Invalid resume URL: Unable to fetch document"
                 )
             resume_docs = "\n".join(f"- {doc.page_content}" for doc in docs)
-
+            print(f"Loaded resume documents: {resume_docs[:100]}")  # Debugging print
 
         full_history_docs = ""
         if request.full_history_url:
+            print(f"Loading full history from URL: {request.full_history_url}")  # Debugging print
             docs = load_document(request.full_history_url)
             if not docs:
+                print("Invalid full history URL: Unable to fetch document")  # Debugging print
                 raise HTTPException(
                     status_code=400, 
-                    detail="Invalid full_history URL: Unable to fetch document"
+                    detail="Invalid full history URL: Unable to fetch document"
                 )
             full_history_docs = "\n".join(f"- {doc.page_content}" for doc in docs)
+            print(f"Loaded full history documents: {full_history_docs[:100]}")  # Debugging print
 
         form_response_docs = ""
         if request.form_id:
+            print(f"Fetching form response for form_id: {request.form_id}")  # Debugging print
             try:
-                x_api_key = os.getenv("BACKEND_XAPI_KEY")
-                url = f"{os.getenv('BACKEND_BASE_URL')}/v3/api/custom/theme-document/answer/{request.form_id}?x-project={x_api_key}"
+                x_api_key = Config.BACKEND_XAPI_KEY
+                url = f"{backend_base_url}/v3/api/custom/theme-document/answer/{request.form_id}?x-project={x_api_key}"
                 result = requests.get(url)
-                form_response = result.json()  # Return response in JSON format
-                form_response_docs =  "\n".join(f"- {form_response}")
-            except:
+                result.raise_for_status()  # Ensure we raise an error for bad responses
+                form_response = result.json()["data"]  # Return response in JSON format
+                form_response_docs = "\n".join(f"- {form_response}")
+                
+                print(f"Fetched form response: {form_response}")  # Debugging print
+            except requests.RequestException as e:
+                print(f"Error fetching onboarding data: {str(e)}")  # Debugging print
                 raise HTTPException(
                     status_code=400, 
-                    detail="Unable to fetch onborading data"
+                    detail="Unable to fetch onboarding data"
                 )
-        # Generate theme data using the generate_theme function
-        theme_data = generate_theme(
-            conversation_data=conversation_data,
-            feedback=request.feedback,
-            previous_result=request.previous_results,
-            resume = resume_docs,
+        # Here you would fetch the conversation data using the conversation_id
+        # This is a placeholder - replace with your actual conversation data fetching logic
+         # Get AI-generated theme content
+        # Get AI-generated theme content
+        response = ai_chat(
+            query="NOW GENERATE THE STARTPOP FRAMEWORK",
+            conversation_id=request.chat_id,
+            theme_id=request.theme_id,
+            resume=resume_docs,
+            full_history=full_history_docs,
             form_response=form_response_docs,
-            full_history = full_history_docs
-        )  
+            generate_theme="YES"
+        )
+        print(f"AI Response for theme: {response}")
         
-        if not theme_data:
-            raise HTTPException(
-                status_code=500,
-                detail="Failed to generate theme data"
-            )
+        # Ensure AI response is valid
+        if not isinstance(response, str):
+            raise HTTPException(status_code=500, detail="Invalid AI response format")
         
-        # Generate the PDF using the create_pdf function
-        pdf_content = create_pdf(theme_data)
-        
-        # Create filename with timestamp
+        # Generate PDF
+        response_data = json.loads(response)
+        pdf_content = create_pdf(response_data)
         timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
-        filename = f"theme_{timestamp}.pdf"
+        file_path = f"theme_{timestamp}.pdf"
         
-        # Return the PDF as a response
-        return Response(
-            content=pdf_content,
-            media_type="application/pdf",
-            headers={
-                "Content-Disposition": f'attachment; filename="{filename}"'
-            }
-        )
-    
+        # Save the PDF locally temporarily
+        with open(file_path, "wb") as file:
+            file.write(pdf_content)
+        
+        # Upload the PDF to S3 using the API
+        upload_url = f"{backend_base_url}/v3/api/custom/theme/doc-upload?x-project={x_api_key}"
+        with open(file_path, 'rb') as file:
+            files = {'file': file}
+            upload_response = requests.post(upload_url, files=files)
+        
+        # Check if the upload was successful
+        if upload_response.status_code != 200:
+            raise HTTPException(status_code=upload_response.status_code, detail="File upload to S3 failed: " + upload_response.text)
+        
+        upload_data = upload_response.json()  # Get the response in JSON format
+        
+        # Extract the uploaded file URL
+        theme_url = upload_data.get("url")  # Adjust this key based on the actual API response structure
+        if not theme_url:
+            raise HTTPException(status_code=500, detail="Failed to retrieve theme URL from upload response")
+        
+        # Clean up the temporary file
+        os.remove(file_path)
+        
+        # Return JSON response with theme URL and text
+        return {
+            "theme_url": theme_url,
+            "theme_text": response_data
+        }
     except Exception as e:
-        raise HTTPException(
-            status_code=500,
-            detail=f"Error generating PDF: {str(e)}"
-        )
+        print(f"Error generating theme: {str(e)}")
+        raise HTTPException(status_code=500, detail=f"Error: {str(e)}")
+
         
 @app.post("/rescue-career/generate-quiz", response_model=QuizResponse)
 async def generate_quiz_endpoint(
@@ -339,25 +459,13 @@ async def generate_quiz_endpoint(
             status_code=500,
             detail=f"Unexpected error during quiz generation: {str(e)}"
         )
+
+@app.get("/health")
+async def health_check():
+    """Health check endpoint to verify the service is running."""
+    return {"status": "healthy", "timestamp": datetime.now().isoformat()}
         
-        
-        
-async def get_conversation_data(conversation_id: str) -> dict:
-    """
-    Fetch conversation data using the conversation ID.
-    Replace this with your actual implementation to fetch conversation data.
-    """
-    try:
-        storage_path = "conversations.json"
-        with open(storage_path, 'r') as f:
-                convs =  json.load(f)
-        convs_id = convs[conversation_id]
-        return convs_id
-    except Exception as e:
-        print(f"Error fetching conversation data: {e}")
-        return None
-    
-    
+
 @app.on_event("startup")
 async def startup_event():
     """Initialize required components on startup"""
@@ -365,4 +473,4 @@ async def startup_event():
 
 if __name__ == "__main__":
     import uvicorn
-    uvicorn.run("app:app", host="0.0.0.0", port=5048, reload=True)
+    uvicorn.run("app:app", host="0.0.0.0", port=5042, reload=True)

config.py

@@ -1,3 +1,7 @@
+import os
+from dotenv import load_dotenv
+
+load_dotenv()
 QUIZ_TYPES = {
     1: {
         "name": "Single Line Text Inputs",
@@ -17,4 +21,16 @@ QUIZ_TYPES = {
         {"question": "Your question here", "options": ["True", "False"], "correct_answer": "True or False"}
         """
     }
-}
+}
+
+MODEL = "gpt-4o-mini"
+TEMPERATURE = 0.7
+
+
+class Config:
+    API_KEY_ACCESS = os.getenv("API_KEY_ACCESS")
+    OPENAI_API_KEY = os.getenv("OPENAI_API_KEY")
+    BACKEND_XAPI_KEY = os.getenv("BACKEND_XAPI_KEY")
+    BACKEND_BASE_URL = os.getenv("BACKEND_BASE_URL_")
+
+    

data/config_files/theme_context.json

@@ -41,7 +41,7 @@
     },
     {
         "id": 9,
-        "theme": "Successful and Unsuccessful Team Questions",
+        "theme": "Successful Team Questions",
         "context": "Teamwork is necessary to assist in overcoming emergencies. Especially expanding ones. I understand how chain of commands works and the importance of monitoring the span of control as well.\n- Great teamwork creates synergy in the workplace and makes environments happier, healthier, and safer.\n- There is no “I” in team.\n- All tasks are important no matter how big or small it may seem.\n- Great teams are built by people that are committed to training and knowing their roles well.\n- Need to train in order to create muscle memory and prevent “skills fade”\n- Potential Conclusions:\no I know that Firefighting is a team sport and that _____ Fire does a great job in fostering that camaraderie.\no I have seen the _____ Fire team play in _____ tournament and know that you guys applaud crews bonding over meals at the station.\no I really appreciate the fact that you allow crews to train together and share insights learned across the department so that everyone gets better all the time.\no Include stats and figures, following SOPs and safety protocols into the answer"
     },
     {
@@ -93,5 +93,10 @@
         "id": 19,
         "theme": "Challenge Questions",
         "context": "Life is full of challenges. It is important for firefighters to be the type of people that take those challenges head on and constantly work to find ways to overcome them.\n- Grit, persistence, creative problem solving, and strong mental health are the keys to overcoming challenges.\n- Life can change in a second. Its important to be adaptive and flexible.\n- Potential Conclusions:\no And I am certain that ______ Fire works very hard to meet every challenge and then overcome them.\no I think about how you guys handled the ________ situation\no or the recent fire at _______\no and as an outsider looking in, I was thoroughly impressed with the way the department handled it."
+    },
+    {
+        "id": 20,
+        "theme": "Unsuccessful Team Questions",
+        "context": "Unsuccessful teamwork can hinder the effectiveness of emergency responses. It is crucial to recognize the signs of dysfunction within a team and address them promptly.\n- Poor communication can lead to misunderstandings and mistakes, creating a chaotic environment.\n- Lack of trust among team members can result in a reluctance to share information or ask for help.\n- When roles are unclear, it can lead to confusion and inefficiency in operations.\n- Teams that do not prioritize training may experience skills fade, which can jeopardize safety and effectiveness.\n- Potential Conclusions:\no I understand that addressing team dysfunction is vital for the success of _____ Fire. Recognizing and resolving issues can lead to a more cohesive and effective team.\no I have observed that when teams struggle, it often stems from a lack of communication and trust. It's essential to foster an environment where everyone feels valued and heard.\no I believe that ongoing training and team-building activities are crucial in preventing issues that can lead to an unsuccessful team dynamic."
     }
 ]

docs/.python-version

@@ -0,0 +1 @@
+python=3.11

docs/prev.py

@@ -0,0 +1,304 @@
+import os
+import requests
+from dotenv import load_dotenv
+from langchain_openai import ChatOpenAI
+from langchain_core.prompts.prompt import PromptTemplate
+from langchain_core.output_parsers import StrOutputParser, JsonOutputParser
+from langchain_openai import OpenAIEmbeddings
+from langchain_core.documents import Document
+from uuid import uuid4
+import json
+import getpass
+import numpy as np
+from concurrent.futures import ThreadPoolExecutor, as_completed
+from sklearn.metrics.pairwise import cosine_similarity
+from typing import List
+import time
+from datetime import datetime
+import pytz
+import logging
+load_dotenv()
+
+os.environ["OPENAI_API_KEY"] = os.getenv("OPENAI_API_KEY")
+
+llm_temp = ChatOpenAI(model="gpt-4o-mini", temperature=0.7)
+def generate_theme(conversation_data, resume, full_history, form_response=None, feedback=None, previous_result=None) -> dict:
+    try:
+        # Define prompt for summarizing and extracting the required fields
+        theme_prompt = PromptTemplate(
+            template="""
+<|begin_of_text|><|start_header_id|>system<|end_header_id|>
+You are a Fire Fighter Interview preparation assistant that generates STARTPOP FORMAT based on user interaction with AI.
+Your responsibilities include carefully analyzing user interactions, themes, resumes,Onboarding questions and answers and work history to generate detailed STARTPOP formats for specific themes.
+
+### Context and Guidelines:
+1. **Purpose**: Generate a single behavioral question with a detailed STARTPOP format.
+2. **Input Sources**:
+   - Current theme
+   - User interaction with AI
+   - User resume
+   - Full work history
+   - Onboarding questions and answers for additional context
+3. **Output Format**: JSON object with the following fields:
+   - `theme_title`: Title of the theme provided.
+   - `question`: Behavioral question aligned with the theme.
+   - `Situation`: A bulleted list (75-100 words).
+   - `Task`: A bulleted list (50 words).
+   - `Action`: A bulleted list (2 negative actions and 2 positive actions).
+   - `Results and Transitions`: A bulleted list (25-50 words).
+   - `Personal Lessons`: A bulleted list (25-50 words).
+   - `Observations of Others`: A bulleted list (25 words).
+   - `Professional Connection`: A bulleted list (25-50 words). Additionally:
+     - Connect to the theme of the question.
+     - Creatively express why you should be part of their team.
+
+### Key Concepts in Firefighting:
+Throughout most Probationary Firefighter Interviews, evaluators assess alignment with the **7 Main Concepts of Firefighting**:
+- **High Performance Teams**
+- **Situational Awareness**
+- **Being a Great Problem Solver**
+- **Customer Service**
+- **Building Construction, Mechanical Aptitude**
+- **Emergency Medicine Experience**
+- **Mental and Physical Health**
+
+Additionally, they evaluate communication skills, competence, and likability.
+
+### 20 Important Themes:
+These themes are used for behavioral questions:
+- Customer Service
+- Conflict
+- Challenge
+- Leadership
+- Stress
+- Successful Team
+- Diversity
+- Mistake
+- Unsuccessful Team
+- Disagreement
+- Bent a Rule
+- Delivered a Difficult Message
+- Displayed Integrity
+- Took a Shortcut
+- Didn’t Follow the Rules
+- Emergency Response
+- Dealt with Disabilities
+- Solved a Big Problem
+- Continuous Improvement
+- Handled Sensitive Information
+
+### Behavioral Question Starters:
+Behavioral questions often begin with phrases like:
+- "Tell me a time when..."
+- "Can you tell me about a time when you..."
+- "Describe a situation where you had to..."
+- "Give me an example of how you..."
+- "Have you ever been in a position where you needed to..."
+- "Walk me through a time when you..."
+
+### STARTPOP Framework:
+The STARTPOP framework enhances the traditional STAR method. It includes:
+1. **Situation**: Set up the scenario concisely (include dates, ages, places, and circumstances).
+2. **Task**: Explain what needed to be done and why.
+3. **Actions**: Outline both negative and positive approaches.
+4. **Results and Transitions**: Share outcomes and ensure coherence.
+5. **Personal Lessons**: Reflect on what you learned.
+6. **Observations of Others**: Share insights about others involved.
+7. **Professional Connection**: Relate the experience to firefighting and express your desire to join the team.
+
+### Example STARTPOP:
+**Question**: Tell me a time when you made a mistake and how you fixed it?
+
+- **Situation**:
+  - In the Fall, my business, Tiger Building Services, does eavestrough cleaning.
+  - In 2019, we were working on a job late in the day, tired and running out of sunlight.
+  - I used handheld blowers without checking the wetness of debris, creating a muddy mess on the customer's deck.
+  - The customer was upset, and I realized my mistake.
+
+- **Task**:
+  - Defuse the situation and clean up the mess quickly.
+  - Protect my company's reputation and ensure good customer experiences.
+
+- **Actions**:
+  - Negative: Matching the customer's anger or ignoring the problem.
+  - Positive: Getting off the roof safely, apologizing, and switching strategies.
+  - Positive: Cleaning the gutters by hand and offering a free soft wash service.
+
+- **Results and Transitions**:
+  - The job took longer than expected, but we waived fees due to the inconvenience.
+  - The customer was satisfied after our resolution plan.
+
+- **Personal Lessons**:
+  - I learned to own up to mistakes, stay empathetic, and de-escalate tense situations.
+
+- **Observations of Others**:
+  - People are entitled to their emotions, and following SOPs prevents mistakes.
+
+- **Professional Connection**:
+  - Mistakes happen, but learning from them is crucial.
+  - I align with Markham Fire's values of transparency and accountability.
+
+### JSON Output Requirements:
+Generate a well-structured JSON output with the following fields:
+- `theme_title`
+- `question`
+- `Situation`
+- `Task`
+- `Action`
+- `Results and Transitions`
+- `Personal Lessons`
+- `Observations of Others`
+- `Professional Connection`
+
+### Review Process:
+1. Ensure all news items align with the specified theme and meet relevance criteria.
+2. Verify the JSON format is flawless, comprehensive, and well-structured.
+
+### Additional Notes:
+- You may be provided with feedback and previous results if the user is dissatisfied.
+- Use this feedback to refine and regenerate the STARTPOP.
+
+<|eot_id|><|start_header_id|>user<|end_header_id|>
+Rules for Generating Each Component:
+1. Situation: 75-100 words.
+2. Task: 50 words.
+3. Actions: 2 negative actions and 2 positive actions.
+4. Results: 25-50 words.
+5. Personal Lessons: 25-50 words.
+6. Observations of Others: 25 words.
+7. Professional Connection: 25-50 words + creative connection to the theme and team invitation.
+NOTE: MAKE SURE THE OUT IS WELL  DETAILED
+CONVERSATION DATA: {conversation_data}
+FEEDBACK: {feedback}
+PREVIOUS RESULT: {previous_result}
+USER RESUME: {resume}
+FULL WORK HISTORY: {full_history}
+Onboarding questions and answers for additional context: {form_response}
+<|start_header_id|>assistant<|end_header_id|>
+Return just the JSON output without any other explanation or comments.
+Thank you for your thorough and precise processing!
+
+""",
+            input_variables=["resume", "conversation_data", "feedback","form_response" "previous_result", "full_history"],
+        )
+
+        # Pipeline to process the prompt and parse output
+        theme_router = theme_prompt | llm_temp | JsonOutputParser()
+
+        # Call the pipeline and generate the cohesive output
+        output = theme_router.invoke({
+            "conversation_data": conversation_data,
+            "feedback": feedback,
+            "previous_result": previous_result,
+            "resume": resume,
+            "full_history": full_history,
+            "form_response":form_response
+        })
+
+        print(f"Output: {output}")
+        return output
+
+    except Exception as e:
+        print(f"Error: {e}")
+        return {}
+
+
+
+
+from fastapi import Response, HTTPException, Depends
+from typing import Optional
+import os
+import requests
+import datetime
+import base64  # For encoding the PDF content in Base64
+
+@app.post("/rescue-career/generate-theme")
+async def generate_pdf_endpoint(
+    request: GeneratePDFRequest,
+    api_key: str = Depends(get_api_key)
+):
+    
+    try:
+        # Fetch conversation data using the conversation_id
+        conversation_data = await get_conversation_data(request.conversation_id)
+        
+        if not conversation_data:
+            raise HTTPException(
+                status_code=404,
+                detail=f"No conversation found with ID {request.conversation_id}"
+            )
+        
+        resume_docs = ""
+        if request.resume_url:
+            docs = load_document(request.resume_url)
+            if not docs:
+                raise HTTPException(
+                    status_code=400, 
+                    detail="Invalid resume URL: Unable to fetch document"
+                )
+            resume_docs = "\n".join(f"- {doc.page_content}" for doc in docs)
+
+        full_history_docs = ""
+        if request.full_history_url:
+            docs = load_document(request.full_history_url)
+            if not docs:
+                raise HTTPException(
+                    status_code=400, 
+                    detail="Invalid full_history URL: Unable to fetch document"
+                )
+            full_history_docs = "\n".join(f"- {doc.page_content}" for doc in docs)
+
+        form_response_docs = ""
+        if request.form_id:
+            try:
+                x_api_key = os.getenv("BACKEND_XAPI_KEY")
+                url = f"{os.getenv('BACKEND_BASE_URL')}/v3/api/custom/theme-document/answer/{request.form_id}?x-project={x_api_key}"
+                result = requests.get(url)
+                form_response = result.json()  # Return response in JSON format
+                form_response_docs = "\n".join(f"- {form_response}")
+            except:
+                raise HTTPException(
+                    status_code=400, 
+                    detail="Unable to fetch onboarding data"
+                )
+
+        # Generate theme data using the generate_theme function
+        theme_data = generate_theme(
+            conversation_data=conversation_data,
+            feedback=request.feedback,
+            previous_result=request.previous_results,
+            resume=resume_docs,
+            form_response=form_response_docs,
+            full_history=full_history_docs
+        )  
+
+        if not theme_data:
+            raise HTTPException(
+                status_code=500,
+                detail="Failed to generate theme data"
+            )
+        
+        # Generate the PDF using the create_pdf function
+        pdf_content = create_pdf(theme_data)
+        
+        # Encode the PDF content in Base64
+        pdf_base64 = base64.b64encode(pdf_content).decode("utf-8")
+        
+        # Create filename with timestamp
+        timestamp = datetime.datetime.now().strftime("%Y%m%d_%H%M%S")
+        filename = f"theme_{timestamp}.pdf"
+        
+        # Return both the PDF (as Base64) and the theme data in a JSON response
+        return {
+            "theme_data": theme_data,
+            "pdf": {
+                "filename": filename,
+                "content": pdf_base64
+            }
+        }
+    
+    except Exception as e:
+        raise HTTPException(
+            status_code=500,
+            detail=f"Error generating PDF: {str(e)}"
+        )

docs/prev_app.py

@@ -0,0 +1,368 @@
+import os
+from typing import Optional
+from fastapi import FastAPI, HTTPException, Security, Depends
+from fastapi.security import APIKeyHeader
+from fastapi.middleware.cors import CORSMiddleware
+from fastapi.responses import JSONResponse
+from dotenv import load_dotenv
+from utils.document_loader import load_document
+import json
+from pydantic import BaseModel
+from src.llm import ai_chat
+from langchain_openai import ChatOpenAI
+import requests
+import tempfile
+from scripts.generate_pdf  import create_pdf
+from scripts.generate_theme import generate_theme
+from scripts.generate_quiz import generate_quiz
+from typing import Dict, Any
+from fastapi.responses import Response
+from datetime import datetime
+from fastapi import HTTPException
+from pydantic import BaseModel
+from typing import Optional, Union, Dict, Any
+import os
+import requests
+import os
+from PyPDF2 import PdfReader
+from config import QUIZ_TYPES
+# Load environment variables
+load_dotenv()
+API_KEY = os.getenv("API_KEY_ACCESS")
+
+base_path = os.path.join("data", "config_files")
+QUESTIONS_PATH = os.path.join(base_path, "questions.json")
+THEME_CONTEXT_PATH = os.path.join(base_path, "theme_context.json")
+
+# Load themes at module level
+with open(THEME_CONTEXT_PATH, "r") as f:
+    themes = json.load(f)
+
+# Initialize FastAPI app
+app = FastAPI(
+    title="Fire Fighter Interview API",
+    description="API For fire fighter",
+    version="1.0.0"
+)
+
+# Add CORS middleware
+app.add_middleware(
+    CORSMiddleware,
+    allow_origins=["*"],
+    allow_credentials=True,
+    allow_methods=["*"],
+    allow_headers=["*"],
+)
+
+# Setup API key authentication
+api_key_header = APIKeyHeader(name="Authorization", auto_error=False)
+
+
+
+async def get_api_key(api_key_header: str = Security(api_key_header)) -> str:
+    """Validate API key from header"""
+    if not api_key_header or not api_key_header.startswith('Bearer '):
+        raise HTTPException(
+            status_code=401,
+            detail={"error": "Unauthorized", "message": "API key is missing or invalid."}
+        )
+    
+    token = api_key_header.split(' ')[1]
+    if token != API_KEY:
+        raise HTTPException(
+            status_code=401,
+            detail={"error": "Unauthorized", "message": "API key does not match."}
+        )
+    
+    return token
+
+class ChatRequest(BaseModel):
+    resume_url: Optional[str] = None
+    query: str=None
+    conversation_id: str
+    theme_id: Optional[int] = 1
+
+class ChatResponse(BaseModel):
+    message: str
+    end: bool
+    error: Optional[str] = None
+
+
+
+class GeneratePDFRequest(BaseModel):
+    conversation_id: str
+    feedback: Optional[str] = None
+    previous_results: Optional[Dict[str, Any]] = None
+    resume_url: Optional[str] = None
+    full_history_url: Optional[str] = None
+    form_id:Optional[int] = None
+
+class QuizRequest(BaseModel):
+    pdf_url: str
+    quiz_type: int  # 1, 2, or 3 corresponding to QUIZ_TYPES
+
+class QuizResponse(BaseModel):
+    success: bool
+    message: str
+    quiz_data: Optional[Dict[str, Any]] = None
+    error: Optional[str] = None
+    
+
+async def extract_pdf_text(pdf_url: str) -> Union[str, None]:
+    """Extract text from PDF and handle potential errors."""
+    try:
+        response = requests.get(pdf_url)
+        response.raise_for_status()
+        
+        # Create a temporary file
+        with tempfile.NamedTemporaryFile(delete=False, suffix='.pdf') as temp_pdf:
+            temp_pdf.write(response.content)
+            temp_path = temp_pdf.name
+
+        # Extract text from PDF
+        reader = PdfReader(temp_path)
+        text = "\n\n".join(
+            page.extract_text() for page in reader.pages if page.extract_text()
+        )
+
+        # Clean up temporary file
+        os.unlink(temp_path)
+
+        if not text.strip():
+            return None
+        return text
+
+    except requests.RequestException as e:
+        raise HTTPException(
+            status_code=400,
+            detail=f"Error downloading PDF: {str(e)}"
+        )
+    except Exception as e:
+        raise HTTPException(
+            status_code=500,
+            detail=f"Error processing PDF: {str(e)}"
+        )
+
+@app.post("/rescue-career/chat", response_model=ChatResponse)
+async def chat_endpoint(
+    request: ChatRequest,
+    api_key: str = Depends(get_api_key)
+):
+    try:
+        # Validate theme
+        matching_themes = [t for t in themes if t["id"] == request.theme_id]
+        if not matching_themes:
+            raise HTTPException(
+                status_code=400,
+                detail=f"No theme found with ID {request.theme_id}"
+            )
+
+        # Only try to load document if resume_url is provided
+        resume_docs = ""
+        if request.resume_url:
+            docs = load_document(request.resume_url)
+            if not docs:
+                raise HTTPException(
+                    status_code=400, 
+                    detail="Invalid resume URL: Unable to fetch document"
+                )
+            resume_docs = "\n".join(f"- {doc.page_content}" for doc in docs)
+
+        # Get AI chat response
+        response = ai_chat(
+            query=request.query,
+            conversation_id=request.conversation_id,
+            theme_id=request.theme_id,
+            resume=resume_docs
+        )
+
+        # Parse response
+        try:
+            parsed_response = json.loads(response)
+            return ChatResponse(
+                message=parsed_response.get("message", ""),
+                end=parsed_response.get("end", "no") == "yes",
+                error=None
+            )
+        except json.JSONDecodeError:
+            return ChatResponse(
+                message=response,
+                end=False,
+                error=None
+            )
+
+    except HTTPException as e:
+        # Re-raise HTTP exceptions
+        raise
+    except Exception as e:
+        raise HTTPException(
+            status_code=500,
+            detail=f"Error processing chat request: {str(e)}"
+        )
+     
+
+@app.post("/rescue-career/generate-theme")
+async def generate_pdf_endpoint(
+    request: GeneratePDFRequest,
+    api_key: str = Depends(get_api_key)
+):
+    
+    try:
+        # Here you would fetch the conversation data using the conversation_id
+        # This is a placeholder - replace with your actual conversation data fetching logic
+        conversation_data = await get_conversation_data(request.conversation_id)
+        
+        if not conversation_data:
+            raise HTTPException(
+                status_code=404,
+                detail=f"No conversation found with ID {request.conversation_id}"
+            )
+        
+        resume_docs = ""
+        if request.resume_url:
+            docs = load_document(request.resume_url)
+            if not docs:
+                raise HTTPException(
+                    status_code=400, 
+                    detail="Invalid resume URL: Unable to fetch document"
+                )
+            resume_docs = "\n".join(f"- {doc.page_content}" for doc in docs)
+
+
+        full_history_docs = ""
+        if request.full_history_url:
+            docs = load_document(request.full_history_url)
+            if not docs:
+                raise HTTPException(
+                    status_code=400, 
+                    detail="Invalid full_history URL: Unable to fetch document"
+                )
+            full_history_docs = "\n".join(f"- {doc.page_content}" for doc in docs)
+
+        form_response_docs = ""
+        if request.form_id:
+            try:
+                x_api_key = os.getenv("BACKEND_XAPI_KEY")
+                url = f"{os.getenv('BACKEND_BASE_URL')}/v3/api/custom/theme-document/answer/{request.form_id}?x-project={x_api_key}"
+                result = requests.get(url)
+                form_response = result.json()  # Return response in JSON format
+                form_response_docs =  "\n".join(f"- {form_response}")
+            except:
+                raise HTTPException(
+                    status_code=400, 
+                    detail="Unable to fetch onborading data"
+                )
+        # Generate theme data using the generate_theme function
+        theme_data = generate_theme(
+            conversation_data=conversation_data,
+            feedback=request.feedback,
+            previous_result=request.previous_results,
+            resume = resume_docs,
+            form_response=form_response_docs,
+            full_history = full_history_docs
+        )  
+        
+        if not theme_data:
+            raise HTTPException(
+                status_code=500,
+                detail="Failed to generate theme data"
+            )
+        
+        # Generate the PDF using the create_pdf function
+        pdf_content = create_pdf(theme_data)
+        
+        # Create filename with timestamp
+        timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
+        filename = f"theme_{timestamp}.pdf"
+        
+        # Return the PDF as a response
+        return Response(
+            content=pdf_content,
+            media_type="application/pdf",
+            headers={
+                "Content-Disposition": f'attachment; filename="{filename}"'
+            }
+        )
+    
+    except Exception as e:
+        raise HTTPException(
+            status_code=500,
+            detail=f"Error generating PDF: {str(e)}"
+        )
+        
+@app.post("/rescue-career/generate-quiz", response_model=QuizResponse)
+async def generate_quiz_endpoint(
+    request: QuizRequest,
+    api_key: str = Depends(get_api_key)
+):
+    """Generate quiz based on PDF content and quiz type."""
+    # Validate quiz type
+    if request.quiz_type not in QUIZ_TYPES:
+        raise HTTPException(
+            status_code=400,
+            detail=f"Invalid quiz type. Must be one of: {list(QUIZ_TYPES)}"
+        )
+
+    try:
+        # Extract text from PDF
+        pdf_text = await extract_pdf_text(request.pdf_url)
+        if not pdf_text:
+            return QuizResponse(
+                success=False,
+                message="PDF extraction completed but no text content found",
+                error="Empty PDF content"
+            )
+            
+        # Generate quiz using the existing function
+        quiz_data = generate_quiz(
+            startpop_pdf=pdf_text,
+            quiz_type=request.quiz_type
+        )
+
+        if not quiz_data:
+            return QuizResponse(
+                success=False,
+                message="Quiz generation failed",
+                error="Unable to generate quiz from the provided content"
+            )
+
+        return QuizResponse(
+            success=True,
+            message="Quiz generated successfully",
+            quiz_data=quiz_data
+        )
+
+    except HTTPException as he:
+        raise he
+    except Exception as e:
+        raise HTTPException(
+            status_code=500,
+            detail=f"Unexpected error during quiz generation: {str(e)}"
+        )
+        
+        
+        
+async def get_conversation_data(conversation_id: str) -> dict:
+    """
+    Fetch conversation data using the conversation ID.
+    Replace this with your actual implementation to fetch conversation data.
+    """
+    try:
+        storage_path = "conversations.json"
+        with open(storage_path, 'r') as f:
+                convs =  json.load(f)
+        convs_id = convs[conversation_id]
+        return convs_id
+    except Exception as e:
+        print(f"Error fetching conversation data: {e}")
+        return None
+    
+    
+@app.on_event("startup")
+async def startup_event():
+    """Initialize required components on startup"""
+    pass
+
+if __name__ == "__main__":
+    import uvicorn
+    uvicorn.run("app:app", host="0.0.0.0", port=5048, reload=True)

gunicorn_config.py

@@ -0,0 +1,77 @@
+import multiprocessing
+import os
+
+# Server socket
+bind = "0.0.0.0:5042"  # Same port as in your app.py
+backlog = 2048
+
+# Worker processes
+workers = multiprocessing.cpu_count() * 2 + 1  # Recommended formula for worker count
+worker_class = "uvicorn.workers.UvicornWorker"  # Required for FastAPI
+worker_connections = 1000
+timeout = 30
+keepalive = 2
+
+# Logging
+accesslog = "logs/access.log"
+errorlog = "logs/error.log"
+loglevel = "info"
+
+# Process naming
+proc_name = "fire_fighter_api"
+
+# SSL (uncomment and configure if using HTTPS)
+# keyfile = "path/to/keyfile"
+# certfile = "path/to/certfile"
+
+# Server mechanics
+daemon = False
+pidfile = "gunicorn.pid"
+umask = 0
+user = None
+group = None
+tmp_upload_dir = None
+
+# Server hooks
+def on_starting(server):
+    """
+    Server startup hook
+    """
+    # Create logs directory if it doesn't exist
+    os.makedirs("logs", exist_ok=True)
+
+def post_fork(server, worker):
+    """
+    Worker initialization hook
+    """
+    server.log.info("Worker spawned (pid: %s)", worker.pid)
+
+def pre_fork(server, worker):
+    """
+    Pre-fork hook
+    """
+    pass
+
+def pre_exec(server):
+    """
+    Pre-exec hook
+    """
+    server.log.info("Forked child, re-executing.")
+
+def when_ready(server):
+    """
+    Server ready hook
+    """
+    server.log.info("Server is ready. Spawning workers")
+
+def worker_int(worker):
+    """
+    Worker interrupt hook
+    """
+    worker.log.info("worker received INT or QUIT signal")
+
+def worker_abort(worker):
+    """
+    Worker abort hook
+    """
+    worker.log.info("worker received SIGABRT signal") 

requirements.txt

@@ -1,23 +1,31 @@
-openai
-pandas
-python-dotenv
-fastapi
-uvicorn
-langchain-community
-langchain-openai
-pydantic
-pypdf
-pypandoc
-Spire.Doc
+openai==1.72.0
+pandas==2.2.3
+python-dotenv==1.1.0
+fastapi==0.115.9
+uvicorn==0.34.0
+langchain-community==0.3.21
+langchain-openai==0.3.12
+pydantic==2.11.3
+pypdf==5.4.0
+pypandoc==1.15
 plum-dispatch==1.7.4
-scikit-learn
-werkzeug
-python-multipart
-langgraph
-tiktoken
-langchainhub
-chromadb
-langchain
-langchain-text-splitters
-beautifulsoup4
-langchain-core
+scikit-learn==1.6.1
+Werkzeug==3.1.3
+python-multipart==0.0.20
+langgraph==0.3.27
+tiktoken==0.9.0
+langchainhub==0.1.21
+chromadb==1.0.3
+langchain==0.3.23
+langchain-text-splitters==0.3.8
+beautifulsoup4==4.13.3
+langchain-core==0.3.51
+PyPDF2==3.0.1
+reportlab==4.3.1
+python-docx==1.1.2
+unstructured==0.17.2
+pypdf==5.4.0
+gunicorn==23.0.0
+python-dotenv
+
+

scripts/generate_quiz.py

@@ -149,10 +149,18 @@ def generate_quiz(startpop_pdf, quiz_type=None) -> dict:
                 - Conclude strongly, avoiding phrases like “and so yeah…”.
                 ----END------
                                
-                NOTE: THE QUIZ FOCUES ON BULIDNG USER CONFIDENCE BY ANANLYZING  THE QUESTIONS AND FRAMEWORK FOR EACH QUESTION IN THE STARTPOP FRAMEWORK PDF,SOLELY USE THIS PDF PROVIDED BY THE USER 
-                BASED ON THIS FRAMEWORK , CREATE INTERVIEW BASED QUIZ FOR FIRE FIGHTING ROLE BY ANALYZING THIS DOCUMENT
-                NOTE : THE QUIZ SHOULD NOT BE  BASED ON STARTPOP FRAMEWORK ITSELF BUT ANALYZE THE STARTPOP FRAMEWORK PRESENTED TO GENERATE INTERVIEW BASED QUIZ
-                e.g  "The STARTPOP framework is specifically designed for firefighter interviews", THIS KIND OF QUESTION SHOULD NOT BE ASKED IN THE QUIZ....
+               ### Instructions:
+                - Analyze the provided STARTPOP PDF to extract relevant themes and concepts.
+                - Generate a quiz that builds user confidence by focusing on interview-based scenarios.
+                - Avoid questions directly about the STARTPOP framework itself (e.g., "What is STARTPOP?").
+                - Use the specified quiz type (`quiz_type`) to determine the output format.
+                - Generate at least 15 questions and above
+                
+                NOTE: The quiz focuses on building user confidence by analyzing the questions and framework presented in the STARTPOP PDF provided by the user. 
+                Based on this framework, create an interview-based quiz specifically for firefighting roles by thoroughly analyzing the document.
+                
+                IMPORTANT: The quiz should not directly reference the STARTPOP framework itself. Instead, it should generate interview-based questions derived from the insights of the STARTPOP framework.
+                For example, avoid questions like "The STARTPOP framework is specifically designed for firefighter interviews." Such questions should not be included in the quiz.
                 Thank you for your thorough and precise processing!
                             STARTPOP FULL PDF :{startpop_pdf}
                             question type : {quiz_type}

scripts/generate_theme.py

@@ -22,38 +22,23 @@ load_dotenv()
 os.environ["OPENAI_API_KEY"] = os.getenv("OPENAI_API_KEY")
 
 llm_temp = ChatOpenAI(model="gpt-4o-mini", temperature=0.7)
-def generate_theme(conversation_data, resume, full_history, form_response=None, feedback=None, previous_result=None) -> dict:
+
+def generate_theme(conversation_data,resume,full_history,feedback=None, previous_result=None) -> dict:
     try:
         # Define prompt for summarizing and extracting the required fields
         theme_prompt = PromptTemplate(
-            template="""
-<|begin_of_text|><|start_header_id|>system<|end_header_id|>
-You are a Fire Fighter Interview preparation assistant that generates STARTPOP FORMAT based on user interaction with AI.
-Your responsibilities include carefully analyzing user interactions, themes, resumes,Onboarding questions and answers and work history to generate detailed STARTPOP formats for specific themes.
+            template="""<|begin_of_text|><|start_header_id|>system<|end_header_id|>
+            You are a Fire Fighter Interview preparation assistant that generates STARTPOP FORMAT BASED on user interaction with AI.
+            You will be provided with the current theme, user interaction with AI (alongside user resume), and data.
 
-### Context and Guidelines:
-1. **Purpose**: Generate a single behavioral question with a detailed STARTPOP format.
-2. **Input Sources**:
-   - Current theme
-   - User interaction with AI
-   - User resume
-   - Full work history
-   - Onboarding questions and answers for additional context
-3. **Output Format**: JSON object with the following fields:
-   - `theme_title`: Title of the theme provided.
-   - `question`: Behavioral question aligned with the theme.
-   - `Situation`: A bulleted list (75-100 words).
-   - `Task`: A bulleted list (50 words).
-   - `Action`: A bulleted list (2 negative actions and 2 positive actions).
-   - `Results and Transitions`: A bulleted list (25-50 words).
-   - `Personal Lessons`: A bulleted list (25-50 words).
-   - `Observations of Others`: A bulleted list (25 words).
-   - `Professional Connection`: A bulleted list (25-50 words). Additionally:
-     - Connect to the theme of the question.
-     - Creatively express why you should be part of their team.
+Your responsibility is to carefully analyze user interaction with AI, the theme, and the user RESUME to generate a STARTPOP format for the theme.
+NOTE: A SINGLE QUESTION IS GENERATED WITH DETAILED STARTPOP FORMAT
+NOTE: For more Context, user full work history may also be provided
+TO KNOW MORE ABOUT THE PROJECT READ BELOW
+---START------
+Throughout most Probationary Firefighter Interviews, they will be evaluating a ton of things. Typically, they want to see how you align with the **7 Main Concepts of Firefighting**. They are also watching how nervous you are, your communication skills, and your overall general competence for the role. At the end of the day, you want them to like you.
 
-### Key Concepts in Firefighting:
-Throughout most Probationary Firefighter Interviews, evaluators assess alignment with the **7 Main Concepts of Firefighting**:
+### 7 Main Concepts:
 - **High Performance Teams**
 - **Situational Awareness**
 - **Being a Great Problem Solver**
@@ -62,10 +47,16 @@ Throughout most Probationary Firefighter Interviews, evaluators assess alignment
 - **Emergency Medicine Experience**
 - **Mental and Physical Health**
 
-Additionally, they evaluate communication skills, competence, and likability.
+Your crew of four firefighters is usually comprised of a Driver, a Captain, and two firefighters in the back. That is a High-Performance Team. 
 
-### 20 Important Themes:
-These themes are used for behavioral questions:
+We are frequently dispatched to calls that require using our understanding of Building Construction Concepts, Mechanical Aptitude, and Emergency Medical Experience. When you respond to an emergency event that is inherently dangerous (like a vehicle fire, a car accident in a slanted ditch, a person trapped under a machine, a house fire, or a chemical suicide), you need to use your Situational Awareness to keep that crew safe. 
+
+Sometimes the tools, training, and tactics that you have been taught work perfectly. Sometimes they don’t. Can you be a Good Problem Solver to quickly come up with something to make the situation better for the people, places, and environments that we protect?
+
+Ultimately, your crew will be serving the public, and the chiefs need to know that you can be trained to be above their desired standard so that you give the public great Customer Service.
+
+### 20 Important Themes
+Consider the 7 concepts to be the soil. All of your stories grow out of that soil. But not every story works for every question. You need to handpick the right one at the right times to give them. Sort of like how you handpick flowers out of the soil. You NEED to have **20 different flowers** so that you are fully prepared for whatever behavioral question they throw at you. These are the **20 Themes** that you would use for behavioral questions:
 - Customer Service
 - Conflict
 - Challenge
@@ -74,7 +65,7 @@ These themes are used for behavioral questions:
 - Successful Team
 - Diversity
 - Mistake
-- Unsuccessful Team
+- Unsuccessful Team     
 - Disagreement
 - Bent a Rule
 - Delivered a Difficult Message
@@ -87,117 +78,162 @@ These themes are used for behavioral questions:
 - Continuous Improvement
 - Handled Sensitive Information
 
-### Behavioral Question Starters:
-Behavioral questions often begin with phrases like:
-- "Tell me a time when..."
-- "Can you tell me about a time when you..."
-- "Describe a situation where you had to..."
-- "Give me an example of how you..."
-- "Have you ever been in a position where you needed to..."
-- "Walk me through a time when you..."
+### Behavioral Question Starters
+Behavioral questions usually start with phrases like:
+- “Tell me a time when…”
+- “Can you tell me about a time when you…”
+- "Describe a situation where you had to…"
+- "Give me an example of how you…"
+- "Have you ever been in a position where you needed to…"
+- "Walk me through a time when you…"
 
-### STARTPOP Framework:
-The STARTPOP framework enhances the traditional STAR method. It includes:
-1. **Situation**: Set up the scenario concisely (include dates, ages, places, and circumstances).
-2. **Task**: Explain what needed to be done and why.
-3. **Actions**: Outline both negative and positive approaches.
-4. **Results and Transitions**: Share outcomes and ensure coherence.
-5. **Personal Lessons**: Reflect on what you learned.
-6. **Observations of Others**: Share insights about others involved.
-7. **Professional Connection**: Relate the experience to firefighting and express your desire to join the team.
+### STARTPOP Framework
+The STAR Format is what most people tell you to do in order to answer a firefighter interview question. It’s a great framework. I highly recommend it. I just advise that you pump it up even further. I call it **STARTPOP**. 
 
-### Example STARTPOP:
-**Question**: Tell me a time when you made a mistake and how you fixed it?
+Try and pull from different parts of your life. My Chief Training Officer told me that he enjoys candidates that are able to use different experiences to answer the questions. Listening to someone drone on and on about a singular time or type of event in their life is a massive turn-off to the interview panel. That’s a bad thing. Just like most things, variety is the spice of life.
 
-- **Situation**:
-  - In the Fall, my business, Tiger Building Services, does eavestrough cleaning.
-  - In 2019, we were working on a job late in the day, tired and running out of sunlight.
-  - I used handheld blowers without checking the wetness of debris, creating a muddy mess on the customer's deck.
-  - The customer was upset, and I realized my mistake.
+#### Components of STARTPOP:
+1. **Situation**: 
+   - Set up the answer in the mind of the question asker. 
+   - Your storytelling skills matter here. It has to be concise and impactful (no more than 25 seconds long).
+   - Include dates, ages, places, and circumstances.
 
-- **Task**:
-  - Defuse the situation and clean up the mess quickly.
-  - Protect my company's reputation and ensure good customer experiences.
+2. **Task**: 
+   - Explain what you needed to do and why you needed to do it.
+   - Recap the situation quickly from a different angle.
 
-- **Actions**:
-  - Negative: Matching the customer's anger or ignoring the problem.
-  - Positive: Getting off the roof safely, apologizing, and switching strategies.
-  - Positive: Cleaning the gutters by hand and offering a free soft wash service.
+3. **Actions**: 
+   - Outline both the negative and the positive way of doing things.
+   - Show high moral character in every question.
 
-- **Results and Transitions**:
-  - The job took longer than expected, but we waived fees due to the inconvenience.
-  - The customer was satisfied after our resolution plan.
+4. **Results**: 
+   - Explain what happened as a result of your actions.
+   - Share results in a time-specific manner (e.g., “5 months later X happened”).
 
-- **Personal Lessons**:
-  - I learned to own up to mistakes, stay empathetic, and de-escalate tense situations.
+5. **Transitions**: 
+   - Speak in a way that aligns with professional expectations.
+   - Ensure coherence in your responses.
 
-- **Observations of Others**:
-  - People are entitled to their emotions, and following SOPs prevents mistakes.
+6. **Personal Lessons**: 
+   - Discuss what you learned about yourself.
+   - Address any concerns the interviewers might have about hiring you.
 
-- **Professional Connection**:
-  - Mistakes happen, but learning from them is crucial.
-  - I align with Markham Fire's values of transparency and accountability.
+7. **Other People Observations**: 
+   - Share insights about others in the situation.
+   - Keep it short and to the point.
 
-### JSON Output Requirements:
-Generate a well-structured JSON output with the following fields:
-- `theme_title`
-- `question`
-- `Situation`
-- `Task`
-- `Action`
-- `Results and Transitions`
-- `Personal Lessons`
-- `Observations of Others`
-- `Professional Connection`
+8. **Professional Connection**: 
+   - Relate your experience directly to the fire service.
+   - Conclude strongly, avoiding phrases like “and so yeah…”.
 
-### Review Process:
-1. Ensure all news items align with the specified theme and meet relevance criteria.
-2. Verify the JSON format is flawless, comprehensive, and well-structured.
+EXAMPLE STARTPOP
 
-### Additional Notes:
-- You may be provided with feedback and previous results if the user is dissatisfied.
-- Use this feedback to refine and regenerate the STARTPOP.
+question: Tell me a time when you made a MISTAKE how did you fix it? (Eaves Cleaning Mistake)
+Situation:
+• In the Fall my business, Tiger Building Services, does a lot of eavestrough cleaning.
+• Back in 2019 I was working with an employee in my truck. We were working nicely to hit my daily revenue target.
+• We got to the last job of the day; we were tired and running out of sunlight. But I really wanted to squeeze it in.
+• We have procedures to follow in order to work safely and effectively. My goal is to be as low impact as possible.
+• I made a mistake when we used the handheld blowers on their eaves to blow out the debris without checking how
+wet the debris was or the ground around the back of the house. It made a muddy mess all over their white deck.
+• They were livid. Swearing and completely unhappy with how we were doing the work. I take ownership of my
+mistakes and realized I screwed up by using blowers instead of hand bombing it.
+Task:
+• My task was to defuse the situation and clean up the mess as quickly as possible.
+• I had to do it because as the owner of the company it was my reputation on the line. We got the job through one
+of the apps that we use to fill out our schedule and it is imperative that I make sure their customers have good
+experiences with us so that we keep our top position on the app.
+• I am also a man of integrity and try to be always empathetic, so I felt obligated to correct the mistake.
+Action:
+• The wrong approach would have been to match the customers energy and just as belligerent and abrasive. It
+would have escalated the situation to a point where things could have gotten ugly and pretty physical.
+• It would have also been wrong to just ignore or make fun of the customer and the problem we created, or to just
+pack our ladders and tools and run away as quickly as possible.
+• The correct approach was to get off the roof safely and speak with the customer on the ground eye to eye.
+• I made sure to do that and then apologized for the mess that we made. I empathized with them and the way they
+were feeling. I told them that it was our mistake, and we will work to correct it immediately.
+• I switched our strategy. Told the employee to clean use their hand for the gutters while I cleaned the deck.
+Results and Transitions:
+• It was a losing situation for me in the short run. The job ended up taking a bit longer than expected and I actually
+told them that we would waive the fees due to the inconveniences we created.
+• After we finished up, I gave her a plan of action. She would get the eaves cleaning for free, and we would return
+the following day with our soft wash system to make sure that she had a sparkling clean deck also free of charge.
+• The next morning when we finished the free soft wash, she was happy with the resolution plan and Jiffy was
+impressed with our ability to correct the mistake and alleviate the situation.
+Personal Lessons:
+• What I learned about myself was that I do make mistakes, but I am the type of person that owns up to it.
+• I am also honest and empathetic, and I can perform in stressful situations and that I could de-escalate tense
+situations, to be adaptable and think quickly on the fly.
+• I used the LAST tactic for good customer service: Listened, Apologized, Solved the problem, then thanked them.
+• I took the full brunt of their anger, made an action plan that instantly calmed the situation and then acted on it to
+make them happy with the service.
+Observations of Others:
+• What I learned about other people is that people are entitled to their reactions, emotions, and feelings.
+• I respect those emotions and have learned that following actionable game plans will help avoid or resolve issues.
+• I know the term proper planning prevents poor performance is applicable here.
+• There is a reason organization’s have SOPs and SOGs. They are there to be followed in order to avoid mistakes.
+Professional Connection:
+• My biggest takeaway was it is okay to make mistakes, but it is not okay to not learn from them.
+• I know that the team on Markham Fire sometimes makes mistakes on the firegrounds, but they are also the type
+of people that own up to their mistake and learn from them.
+• I also know that Chief Grant promotes having an open and transparent organization that is not afraid from
+admitting an error or correcting it.
 
-<|eot_id|><|start_header_id|>user<|end_header_id|>
-Rules for Generating Each Component:
-1. Situation: 100 - 120 words.
-2. Task: 100 words.
-3. Actions: 2 negative actions and 2 positive actions.
-4. Results: 50-70 words.
-5. Personal Lessons: 50-70 words.
-6. Observations of Others: 40 words.
-7. Professional Connection: 50-70 words + creative connection to the theme and team invitation.
-NOTE: MAKE SURE THE OUT IS WELL  DETAILED
-CONVERSATION DATA: {conversation_data}
-FEEDBACK: {feedback}
-PREVIOUS RESULT: {previous_result}
-USER RESUME: {resume}
-FULL WORK HISTORY: {full_history}
-Onboarding questions and answers for additional context: {form_response}
-<|start_header_id|>assistant<|end_header_id|>
+---END------
+
+
+JSON Output Requirements: Generate a list of well-structured JSON output STARTPOP with question and correcpoding STARTPPOP with the following fields:
+                - theme_title: The title the theme provided
+                - question: The question 
+                - Situation: A bulleted list of texts as seen in examples
+                - Task: A bulleted list of texts as seen in examples
+                - Action: A bulleted list of texts as seen in examples
+                - Personal Lessons: A bulleted list of texts as seen in examples
+                - Results and Transitions: A bulleted list of texts as seen in examples
+                - Observations of Others: A bulleted list of texts as seen in examples
+                - Professional Connection: A bulleted list of texts as seen in examples
+
+Review Process: 
+                - Carefully review all news items to confirm they align with the specified theme and meet relevance criteria.
+                - Ensure the JSON format is flawless, comprehensive, and well-structured, with all fields included and correctly formatted.
+
+NOTE: 1. you MAY BE PROVIDED WITH FEEDBACK AND PREVIOUS RESULT, MEANING AI HAS GENERATED STARTPOP BEFORE AND MAYBE USER IS NOT SATISFIED WITH THE RESULT THEN YOU GENERATE A NEW ONE BASED ON THE FEEDBACK
+NOTE: Each question will have a correpoding STARTPOP feilds
 Return just the JSON output without any other explanation or comments.
-Thank you for your thorough and precise processing!
 
-""",
-            input_variables=["resume", "conversation_data", "feedback","form_response" "previous_result", "full_history"],
+Thank you for your thorough and precise processing!
+            CONVERSATION DATA :{conversation_data}
+            FEEDBACK: {feedback}
+            PREVIOUS RESULT: {previous_result}
+            USER RESUME : {resume}
+            FULL WORK HISTORY : {full_history}
+            <|eot_id|><|start_header_id|>user<|end_header_id|>
+            
+            RULES FOR GENERATING EACH COMPONENT - FOLLOW THESE RULES THOROUGHLY MAKE SURE YOUR OUTPUT IS WELL  DETAILED
+            
+            THE FRAME WORK MUST BE DETAILED WITH THE FOLLWWING RULES
+            1. Situation : 75 - 100 words  
+            2. Task: 50 words
+            3. Actions: 2 Negative actions and 2 positive actions
+            4. Results: 25 - 5o words
+            5. Personal Lessons : 25 - 50 words
+            6. Observation of others: 25 words
+            7. Professional connections: 25 - 50 words and in addition to the 25-50 words:
+                - Connect to the theme of questions (Be creative here)
+                - Ask to be part of their team(be creattive here)  
+            """,
+            input_variables=["resume","conversation_data", "feedback", "previous_result","full_history"],
         )
 
         # Pipeline to process the prompt and parse output
         theme_router = theme_prompt | llm_temp | JsonOutputParser()
 
         # Call the pipeline and generate the cohesive output
-        output = theme_router.invoke({
-            "conversation_data": conversation_data,
-            "feedback": feedback,
-            "previous_result": previous_result,
-            "resume": resume,
-            "full_history": full_history,
-            "form_response":form_response
-        })
-
-        print(f"Output: {output}")
+        output = theme_router.invoke({"conversation_data": conversation_data, "feedback": feedback, "previous_result": previous_result,"resume":resume,"full_history":full_history})
+        
+        print(f"Output : {output}")
         return output
-
+     
     except Exception as e:
-        print(f"Error: {e}")
-        return {}
+        print(f"Error:{e}")
+        return {} 

server_deploy.sh

@@ -0,0 +1,226 @@
+#!/bin/bash
+
+# Configuration
+REPO_URL="http://owusu:890eccfcea010beb94a0adba246aaf9258330b70@23.29.118.76:3000/owusu/ds-fire-fighter.git"
+APP_DIR="/home/ec2-user/ds-fire-fighter"
+BRANCH="main"
+PYTHON_VERSION="3.11"
+WORKERS=4
+THREADS=2
+TIMEOUT=120
+MAX_REQUESTS=1000
+MAX_REQUESTS_JITTER=50
+DEBUG_MODE=true
+
+# Colors for output
+GREEN='\033[0;32m'
+RED='\033[0;31m'
+YELLOW='\033[1;33m'
+BLUE='\033[0;34m'
+NC='\033[0m' # No Color
+
+# Logging function
+log() {
+    local level=$1
+    local message=$2
+    local timestamp=$(date '+%Y-%m-%d %H:%M:%S')
+    
+    case $level in
+        "INFO")
+            echo -e "${BLUE}[$timestamp] INFO: $message${NC}"
+            ;;
+        "SUCCESS")
+            echo -e "${GREEN}[$timestamp] SUCCESS: $message${NC}"
+            ;;
+        "WARNING")
+            echo -e "${YELLOW}[$timestamp] WARNING: $message${NC}"
+            ;;
+        "ERROR")
+            echo -e "${RED}[$timestamp] ERROR: $message${NC}"
+            ;;
+    esac
+}
+
+# Debug logging function
+debug_log() {
+    if [ "$DEBUG_MODE" = true ]; then
+        echo -e "${YELLOW}[DEBUG] $1${NC}"
+    fi
+}
+
+# Error handling
+set -e
+trap 'last_command=$current_command; current_command=$BASH_COMMAND' DEBUG
+trap 'if [ $? -ne 0 ]; then log "ERROR" "Command failed: $last_command"; exit 1; fi' EXIT
+
+# Function to check if a command exists
+command_exists() {
+    command -v "$1" >/dev/null 2>&1
+}
+
+# Function to install Python 3.11
+install_python() {
+    log "INFO" "Installing Python ${PYTHON_VERSION}..."
+    sudo yum update -y
+    sudo yum groupinstall -y "Development Tools"
+    sudo yum install -y openssl-devel bzip2-devel libffi-devel xz-devel
+    
+    # Install Python 3.11 from source
+    cd /tmp
+    wget https://www.python.org/ftp/python/3.11.0/Python-3.11.0.tgz
+    tar xzf Python-3.11.0.tgz
+    cd Python-3.11.0
+    ./configure --enable-optimizations
+    make -j $(nproc)
+    sudo make altinstall
+    
+    # Create symlink for python3.11
+    sudo ln -sf /usr/local/bin/python3.11 /usr/bin/python3.11
+    sudo ln -sf /usr/local/bin/pip3.11 /usr/bin/pip3.11
+}
+
+# Function to setup git repository
+setup_repo() {
+    if [ ! -d "$APP_DIR" ]; then
+        log "INFO" "Cloning repository..."
+        git clone $REPO_URL $APP_DIR
+    else
+        log "INFO" "Updating repository..."
+        cd $APP_DIR
+        git fetch origin
+        git reset --hard origin/$BRANCH
+        git clean -fd
+    fi
+}
+
+# Function to setup virtual environment
+setup_venv() {
+    log "INFO" "Setting up virtual environment..."
+    if [ ! -d "$APP_DIR/venv" ]; then
+        python3.11 -m venv $APP_DIR/venv
+    fi
+    source $APP_DIR/venv/bin/activate
+    pip install --upgrade pip
+    pip install -r $APP_DIR/requirements.txt
+}
+
+# Function to create gunicorn config
+create_gunicorn_config() {
+    log "INFO" "Creating Gunicorn configuration..."
+    cat > $APP_DIR/gunicorn_config.py << EOL
+import multiprocessing
+import os
+
+# Server socket
+bind = "0.0.0.0:5042"
+backlog = 2048
+
+# Worker processes
+workers = ${WORKERS}
+worker_class = "uvicorn.workers.UvicornWorker"
+worker_connections = 1000
+timeout = ${TIMEOUT}
+keepalive = 2
+
+# Process naming
+proc_name = "firefighter"
+pythonpath = "."
+
+# Logging
+accesslog = "logs/access.log"
+errorlog = "logs/error.log"
+loglevel = "info"
+
+# Server mechanics
+daemon = False
+pidfile = "gunicorn.pid"
+umask = 0
+user = None
+group = None
+tmp_upload_dir = None
+
+# Worker lifecycle
+max_requests = ${MAX_REQUESTS}
+max_requests_jitter = ${MAX_REQUESTS_JITTER}
+graceful_timeout = 30
+preload_app = True
+
+# Debug
+reload = False
+reload_engine = "auto"
+spew = False
+
+# Server mechanics
+check_config = False
+preload_app = True
+EOL
+}
+
+# Function to setup systemd service
+setup_service() {
+    log "INFO" "Setting up systemd service..."
+    sudo tee /etc/systemd/system/firefighter.service << EOL
+[Unit]
+Description=Fire Fighter Interview API
+After=network.target
+
+[Service]
+User=ec2-user
+WorkingDirectory=$APP_DIR
+Environment="PATH=$APP_DIR/venv/bin"
+Environment="PYTHONPATH=$APP_DIR"
+ExecStart=$APP_DIR/start.sh
+Restart=always
+RestartSec=5
+StartLimitInterval=0
+
+[Install]
+WantedBy=multi-user.target
+EOL
+
+    sudo systemctl daemon-reload
+    sudo systemctl enable firefighter
+}
+
+# Main deployment process
+main() {
+    log "INFO" "Starting deployment process..."
+
+    # Check and install Python if needed
+    if ! command_exists python3.11; then
+        install_python
+    fi
+
+    # Setup repository
+    setup_repo
+
+    # Create logs directory
+    mkdir -p $APP_DIR/logs
+
+    # Setup virtual environment and install dependencies
+    setup_venv
+
+    # Create gunicorn config
+    create_gunicorn_config
+
+    # Setup and start service
+    setup_service
+    sudo systemctl restart firefighter
+
+    log "SUCCESS" "Deployment completed!"
+    log "INFO" "Your application should now be running at http://localhost:5042"
+    
+    # Print helpful commands
+    echo -e "\n${BLUE}Useful commands:${NC}"
+    echo "View service status: sudo systemctl status firefighter"
+    echo "View logs: sudo journalctl -u firefighter -f"
+    echo "View application logs: tail -f $APP_DIR/logs/access.log"
+    echo "View error logs: tail -f $APP_DIR/logs/error.log"
+    echo "Restart service: sudo systemctl restart firefighter"
+    echo "Stop service: sudo systemctl stop firefighter"
+    echo "Start service: sudo systemctl start firefighter"
+    echo "Deploy new version: ./server_deploy.sh"
+}
+
+# Run main function
+main 

src/llm.py

@@ -7,21 +7,40 @@ from langchain_core.prompts import ChatPromptTemplate, MessagesPlaceholder
 from langchain_core.messages import HumanMessage, AIMessage, BaseMessage
 from langgraph.checkpoint.memory import MemorySaver
 from langgraph.graph import START, MessagesState, StateGraph
-from utils.utils import format_questions_text
 from src.prompts import chat_prompt
 from langchain_openai import ChatOpenAI
+from src.models import Phase2Generation,Phase1Generation
+from config  import TEMPERATURE
+import os
+import requests
+import json
+from dotenv import load_dotenv
+from typing import List, Dict, Optional
+from dataclasses import dataclass
+from datetime import datetime
+from langchain_core.messages import HumanMessage, AIMessage
+
+# Load environment variables
+load_dotenv()
+
+
 @dataclass
 class Message:
     role: str  # 'human' or 'ai'
     content: str
-    timestamp: str
+    timestamp: str=None
 
 QUESTIONS_PATH = "./data/config_files/questions.json"
 with open(QUESTIONS_PATH, "r") as f:
     questions = json.load(f)
 
+from config import Config
+
+
 prompt_template = None
 MODEL = "gpt-4o-mini"
+
+
 def initialize_workflow(model) -> StateGraph:
     """Initialize LangGraph workflow"""
     workflow = StateGraph(state_schema=MessagesState)
@@ -37,70 +56,76 @@ def initialize_workflow(model) -> StateGraph:
     return workflow.compile(checkpointer=memory)
 
 
-def setup_prompt_template(theme: int, resume: str) -> ChatPromptTemplate:
+def setup_prompt_template(theme: int, resume: str,full_history=None,form_response=None,generate_theme="NO") -> ChatPromptTemplate:
     """Set up the prompt template"""
     return ChatPromptTemplate.from_messages([
-        ("system", chat_prompt(theme, resume)),
+        ("system", chat_prompt(theme, resume,full_history,form_response,generate_theme)),
         MessagesPlaceholder(variable_name="messages")
     ])
 
-def parse_ai_response(content: str) -> Dict:
-    """Parse AI response content into expected format"""
+
+def fetch_conversation_history(conversation_id: str) -> List[Message]:
+    """
+    Fetch conversation history from the API using the conversation ID.
+    """
+    x_api_key = Config.BACKEND_XAPI_KEY
+    base_url = Config.BACKEND_BASE_URL
+    url = f"{base_url}/v3/api/custom/jordan/ai-chat/get-messages/{conversation_id}?x-project={x_api_key}"
+    
     try:
-        response = json.loads(content)
-        return {
-            "message": response.get("message", ""),
-            "end": response.get("end", "no") == "yes"
-        }
-    except json.JSONDecodeError:
-        return {
-            "message": content,
-            "end": False
-        }
+        response = requests.get(url)
+        response.raise_for_status()  # Raise an error for bad responses
+        data = response.json()["data"]  # First JSON parse
 
-def add_message(storage_path: Path, conversation_id: str, role: str, content: str) -> None:
-    """Add a message to the conversation history"""
-    message_data = {
-        "role": role,
-        "content": content,
-        "timestamp": datetime.now().isoformat()
-    }
+        if isinstance(data, str):
+            print("Data is a string, parsing as JSON...")
+            data = json.loads(data)
+           
 
-    conversations = load_conversations(storage_path)
-    if conversation_id not in conversations:
-        conversations[conversation_id] = {"messages": []}
-    conversations[conversation_id]["messages"].append(message_data)
-    save_conversations(storage_path, conversations)
+        # Parse the API response into Message objects
+        messages = []
+        
+        # Check if data exists and is a list
+        if data and isinstance(data, list):
+            for item in data:
+               
+                
+                # Check if item is a dictionary
+                if isinstance(item, dict):
+                    role = item.get("role", "unknown")
+                    content = item.get("content", "")
+                    timestamp = datetime.now().isoformat()  # Use current timestamp if not provided
+                    messages.append(Message(role=role, content=content))
+                elif isinstance(item, str):
+                    # If item is a string, it might be JSON that needs parsing
+                    try:
+                        parsed_item = json.loads(item)
+                        if isinstance(parsed_item, dict):
+                            role = parsed_item.get("role", "unknown")
+                            content = parsed_item.get("content", "")
+                            messages.append(Message(role=role, content=content))
+                        else:
+                            print(f"Parsed item is not a dict: {parsed_item}")
+                    except json.JSONDecodeError as json_err:
+                        print(f"Failed to parse JSON string: {item}, error: {json_err}")
+                else:
+                    print(f"Unexpected item type: {type(item)} for item: {item}")
+        else:
+            print(f"No data or data is not a list. Data: {data}")
+            
+        return messages
+        
+    except requests.RequestException as e:
+        print(f"Error fetching conversation history: {e}")
+        return []
+    except KeyError as e:
+        print(f"Expected key not found in response: {e}")
+        return []
+    except Exception as e:
+        print(f"Unexpected error: {e}")
+        return []
 
 
-def get_conversation_history(conversation_id: str, storage_path: Path) -> List[Message]:
-    """Get the conversation history"""
-    conversations = load_conversations(storage_path)
-    if conversation_id not in conversations:
-        return None
-
-    return [
-        Message(
-            role=msg["role"],
-            content=msg["content"],
-            timestamp=msg["timestamp"]
-        )
-        for msg in conversations[conversation_id]["messages"]
-    ]
-
-def load_conversations(storage_path: Path) -> Dict:
-    """Load conversations from storage file"""
-    try:
-        with open(storage_path, 'r') as f:
-            return json.load(f)
-    except FileNotFoundError:
-        return {}
-
-def save_conversations(storage_path: Path, conversations: Dict) -> None:
-    """Save conversations to storage file"""
-    with open(storage_path, 'w') as f:
-        json.dump(conversations, f, indent=2)
-
 def convert_to_langchain_messages(messages: List[Message]) -> List[HumanMessage | AIMessage]:
     """Convert our Message objects to LangChain message objects"""
     converted_messages = []
@@ -111,87 +136,78 @@ def convert_to_langchain_messages(messages: List[Message]) -> List[HumanMessage
             converted_messages.append(AIMessage(content=msg.content))
     return converted_messages
 
-
-def ai_chat(query: str, conversation_id: str, theme_id: int, resume: str) -> str:
+def ai_chat(query: str, conversation_id: str, theme_id: int, resume: str, full_history=None, form_response=None, generate_theme="NO") -> str:
     """Main chat function that processes queries and manages conversation"""
-    storage_path = Path("conversations.json")
-    
     class State(TypedDict):
-        messages: Annotated[Sequence[BaseMessage], "The messages in the conversation"]
+        messages: List[HumanMessage | AIMessage]
         language: str
+
     # Initialize model and workflow
-    model = ChatOpenAI(model=MODEL)
+    model = ChatOpenAI(model=MODEL, temperature=TEMPERATURE)
+    if generate_theme == "YES":
+        model = model.with_structured_output(Phase2Generation)
+    else:
+        model = model.with_structured_output(Phase1Generation)
+
     workflow = StateGraph(state_schema=State)
     
-    
-    
     def call_model(state: State):
-        prompt_template = setup_prompt_template(theme_id, resume)
+        prompt_template = setup_prompt_template(theme_id, resume, full_history, form_response, generate_theme)
         prompt = prompt_template.invoke({
             "messages": state["messages"], 
             "language": state["language"]
         })
         response = model.invoke(prompt)
         return {"messages": [response]}
-
+    
     workflow.add_edge(START, "model")
     workflow.add_node("model", call_model)
     
     memory = MemorySaver()
     app = workflow.compile(checkpointer=memory)
+    
+    # Fetch conversation history from the API
+    history = fetch_conversation_history(conversation_id)
 
-    # Get conversation history
-    history = get_conversation_history(conversation_id, storage_path)
+    print(history)
     
     config = {"configurable": {"thread_id": conversation_id}}
     language = "English"
-
+    
     if not history:
         # New conversation
         input_messages = [HumanMessage(content=query)] if query else [HumanMessage(content="Let's get started")]
-        output = app.invoke(
-            {"messages": input_messages, "language": language},
-            config
-        )
     else:
         # Existing conversation
         history = convert_to_langchain_messages(history)
         input_messages = history + [HumanMessage(content=query)] if query else history
-        output = app.invoke(
-            {"messages": input_messages, "language": language},
-            config
-        )
+    
+    output = app.invoke(
+        {"messages": input_messages, "language": language},
+        config
+    )
+    
+    if generate_theme == "YES":
+        structured_message = output["messages"][0]
+        output = structured_message.json(by_alias=True)  # This returns a JSON string.
+    else:
+        structured_message = output["messages"][0]
+        output = structured_message.json()  # This returns a JSON string.
+        output = json.loads(output)
 
-    # Store messages
-    if query:
-        add_message(storage_path, conversation_id, "human", query)
-    add_message(storage_path, conversation_id, "ai", output["messages"][-1].content)
+        message = output.get("message")
+        print(output)
+    
+    return output
 
-    return output["messages"][-1].content
 
 
 # Example usage:
 if __name__ == "__main__":
-    # Sample resume
-    sample_resume = """
-    John Doe
-    EMT-B Certified
-    5 years experience as volunteer firefighter
-    Bachelor's in Fire Science
-    """
-    
-    # Sample conversation
-    conversation_id = "12345"
-    theme_id = 1  # Customer Service theme
-    
-    # Start conversation
-    
-    
-    # Continue conversation
-    follow_up = ai_chat(
-        query="What was my last questions?",
-        conversation_id=conversation_id,
-        theme_id=theme_id,
-        resume=sample_resume
-    )
-    print("AI:", follow_up)
+    #conversation_id = "12345"  # Replace with the actual conversation ID
+    query = "Hello let us continue"
+    theme_id = 1
+    resume = "Emergency Response Specialist"
+    conversation_id = 1
+    response = ai_chat(query, conversation_id, theme_id, resume)
+    print(response)

src/models.py

@@ -0,0 +1,22 @@
+from pydantic import BaseModel, Field
+from typing import List
+
+class Phase2Generation(BaseModel):
+    theme_title: str
+    question: str
+    Situation: List[str]
+    Task: List[str]
+    Action: List[str]
+    Results_and_Transitions: List[str] = Field(..., alias="Results and Transitions")
+    Personal_Lessons: List[str] = Field(..., alias="Personal Lessons")
+    Observations_of_Others: List[str] = Field(..., alias="Observations of Others")
+    Professional_Connection: List[str] = Field(..., alias="Professional Connection")
+
+    class Config:
+        allow_population_by_field_name = True
+
+
+class Phase1Generation(BaseModel):
+    end:str
+    message:str
+    pop_theme_generation:str

src/prompts.py

@@ -1,7 +1,7 @@
 import json
 from typing import List, Dict, Optional
 from dataclasses import dataclass
-from utils.utils import format_questions_text, format_theme_text
+from utils.utils import format_questions_text, format_theme_text,format_qna_json_text
 @dataclass
 class Message:
     role: str  # 'human' or 'ai'
@@ -14,121 +14,341 @@ QUESTIONS_PATH = "./data/config_files/questions.json"
 with open(QUESTIONS_PATH, "r") as f:
             questions = json.load(f)
 
-def chat_prompt(theme,resume):
-    return f"""  
-                You are a Fire Fighter Interview preparation assistant.
+def chat_prompt(theme,resume,full_history=None, form_response=None,generate_theme="NO"):
+    
+    if form_response:
+           form_response = format_qna_json_text(form_response)
+    if generate_theme=="YES":
+        
 
-Throughout most Probationary Firefighter Interviews, they will be evaluating a ton of things. Typically, they want to see how you align with the **7 Main Concepts of Firefighting**. They are also watching how nervous you are, your communication skills, and your overall general competence for the role. At the end of the day, you want them to like you.
-
-### 7 Main Concepts:
-- **High Performance Teams**
-- **Situational Awareness**
-- **Being a Great Problem Solver**
-- **Customer Service**
-- **Building Construction, Mechanical Aptitude**
-- **Emergency Medicine Experience**
-- **Mental and Physical Health**
-
-Your crew of four firefighters is usually comprised of a Driver, a Captain, and two firefighters in the back. That is a High-Performance Team. 
-
-We are frequently dispatched to calls that require using our understanding of Building Construction Concepts, Mechanical Aptitude, and Emergency Medical Experience. When you respond to an emergency event that is inherently dangerous (like a vehicle fire, a car accident in a slanted ditch, a person trapped under a machine, a house fire, or a chemical suicide), you need to use your Situational Awareness to keep that crew safe. 
-
-Sometimes the tools, training, and tactics that you have been taught work perfectly. Sometimes they don’t. Can you be a Good Problem Solver to quickly come up with something to make the situation better for the people, places, and environments that we protect?
-
-Ultimately, your crew will be serving the public, and the chiefs need to know that you can be trained to be above their desired standard so that you give the public great Customer Service.
-
-### 20 Important Themes
-Consider the 7 concepts to be the soil. All of your stories grow out of that soil. But not every story works for every question. You need to handpick the right one at the right times to give them. Sort of like how you handpick flowers out of the soil. You NEED to have **20 different flowers** so that you are fully prepared for whatever behavioral question they throw at you. These are the **20 Themes** that you would use for behavioral questions:
-- Customer Service
-- Conflict
-- Challenge
-- Leadership
-- Stress
-- Successful Team
-- Diversity
-- Mistake
-- Unsuccessful Team     
-- Disagreement
-- Bent a Rule
-- Delivered a Difficult Message
-- Displayed Integrity
-- Took a Shortcut
-- Didn’t Follow the Rules
-- Emergency Response
-- Dealt with Disabilities
-- Solved a Big Problem
-- Continuous Improvement
-- Handled Sensitive Information
-
-### Behavioral Question Starters
-Behavioral questions usually start with phrases like:
-- “Tell me a time when…”
-- “Can you tell me about a time when you…”
-- "Describe a situation where you had to…"
-- "Give me an example of how you…"
-- "Have you ever been in a position where you needed to…"
-- "Walk me through a time when you…"
-
-Your goal is to engage in conversation with the user. You will be provided with the current theme, the resume of the user, and example general competency questions and behavioral questions.
-USER_RESUME FROM START TO END : 
---- START ---
-{resume}
---- END --- 
-
-### STARTPOP Framework
-The STAR Format is what most people tell you to do in order to answer a firefighter interview question. It’s a great framework. I highly recommend it. I just advise that you pump it up even further. I call it **STARTPOP**. 
-
-Try and pull from different parts of your life. My Chief Training Officer told me that he enjoys candidates that are able to use different experiences to answer the questions. Listening to someone drone on and on about a singular time or type of event in their life is a massive turn-off to the interview panel. That’s a bad thing. Just like most things, variety is the spice of life.
-
-#### Components of STARTPOP:
-1. **Situation**: 
-   - Set up the answer in the mind of the question asker. 
-   - Your storytelling skills matter here. It has to be concise and impactful (no more than 25 seconds long).
-   - Include dates, ages, places, and circumstances.
-
-2. **Task**: 
-   - Explain what you needed to do and why you needed to do it.
-   - Recap the situation quickly from a different angle.
-
-3. **Actions**: 
-   - Outline both the negative and the positive way of doing things.
-   - Show high moral character in every question.
-
-4. **Results**: 
-   - Explain what happened as a result of your actions.
-   - Share results in a time-specific manner (e.g., “5 months later X happened”).
-
-5. **Transitions**: 
-   - Speak in a way that aligns with professional expectations.
-   - Ensure coherence in your responses.
-
-6. **Personal Lessons**: 
-   - Discuss what you learned about yourself.
-   - Address any concerns the interviewers might have about hiring you.
-
-7. **Other People Observations**: 
-   - Share insights about others in the situation.
-   - Keep it short and to the point.
-
-8. **Professional Connection**: 
-   - Relate your experience directly to the fire service.
-   - Conclude strongly, avoiding phrases like “and so yeah…”.
-                Current theme with  More context about the theme for Creating The Professional Connection (Lessons Learned): {format_theme_text(theme)}
-                
-Sample General Competency QUESTIONS and Situational Questions: {format_questions_text(questions,'General Competency Questions')}
-Sample Situational Questions: {format_questions_text(questions,'Situational Questions')}
-
-Your task is to engage the user in conversation, ask relevant questions, that will ultimately help them prepare a strong STARTPOP response based on their experiences and the current theme.
-YOU WILL BE PROVIDED WITH THE USER RESUME, ASK 1 QUESTION AT A TIME AND MAKE IT CONVERSATIONAL AND INTERESTING.
-These responses will be saved and later used to generate a STARTPOP framework by US (DO NOT WORRY ABOUT THAT, WE WILL BE THE ONE TO GENERATE, JUST ENGAGE USER WITH QUESTION AND ANSWER).
-Output format
-CUURENT TEHEME USER IS INTERESTED IN {format_theme_text(theme)}
-NOTE: !!! EXPLICITLY FOCUS ON THE CURRENT THEME SPECIFIED
-WILL BE IN JSON, avoid puttting ```json, before or after , return the excat json with nothing else
-   message:
-   end: "yes" or "no" if you are done with asking questions and confident the responses are okay enough to prepare STARTPOP by us
-NOTE: DO NOT KEEP THE CONVERSATION , CAREFULL ANALYZE USER RESUME AND THE PROVIDED EXAMPLES QUESTIONS AND ALL CONTEXT , ASK RELEVANT QUESTION BASED ON THE THEME AND THAT IS ALL 
-"""
+            prompt = f"""
+                <|begin_of_text|><|start_header_id|>system<|end_header_id|>
+                You are a Fire Fighter Interview preparation assistant that generates STARTPOP FORMAT based on user interaction with AI.
+                Your responsibilities include carefully analyzing user interactions, themes, resumes,Onboarding questions and answers and work history to generate detailed STARTPOP formats for specific themes.
 
 
 
+                ────────────────────────────────────────
+                **Input Sources**::
+                ────────────────────────────────────────
+                Current theme with  More context about the theme for Creating The Professional Connection (Lessons Learned)::{format_theme_text(theme)}
+                USER RESUME: {resume}
+                FULL WORK HISTORY: {full_history}
+                Onboarding questions and answers for additional context: {form_response}
+                2. **Input Sources**:
+                  - Current theme
+                  - User interaction with AI
+                  - User resume
+                  - Full work history
+                  - Onboarding questions and answers for additional context
+
+
+                ### Context and Guidelines:
+                1. **Purpose**: Generate a single behavioral question with a detailed STARTPOP format.
+                2. **Input Sources**:
+                  - Current theme
+                  - User interaction with AI
+                  - User resume
+                  - Full work history
+                  - Onboarding questions and answers for additional context
+
+                  
+                3. **Output Format**: JSON object with the following fields:
+                  - `theme_title`: Title of the theme provided.
+                  - `question`: Behavioral question aligned with the theme.
+                  - `Situation`: A bulleted list (75-100 words).
+                  - `Task`: A bulleted list (50 words).
+                  - `Action`: A bulleted list (2 negative actions and 2 positive actions).
+                  - `Results and Transitions`: A bulleted list (25-50 words).
+                  - `Personal Lessons`: A bulleted list (25-50 words).
+                  - `Observations of Others`: A bulleted list (25 words).
+                  - `Professional Connection`: A bulleted list (25-50 words). Additionally:
+                    - Connect to the theme of the question.
+                    - Creatively express why you should be part of their team.
+
+                ### Key Concepts in Firefighting:
+                Throughout most Probationary Firefighter Interviews, evaluators assess alignment with the **7 Main Concepts of Firefighting**:
+                - **High Performance Teams**
+                - **Situational Awareness**
+                - **Being a Great Problem Solver**
+                - **Customer Service**
+                - **Building Construction, Mechanical Aptitude**
+                - **Emergency Medicine Experience**
+                - **Mental and Physical Health**
+
+                Additionally, they evaluate communication skills, competence, and likability.
+
+                ### 20 Important Themes:
+                These themes are used for behavioral questions:
+                - Customer Service
+                - Conflict
+                - Challenge
+                - Leadership
+                - Stress
+                - Successful Team
+                - Diversity
+                - Mistake
+                - Unsuccessful Team
+                - Disagreement
+                - Bent a Rule
+                - Delivered a Difficult Message
+                - Displayed Integrity
+                - Took a Shortcut
+                - Didn’t Follow the Rules
+                - Emergency Response
+                - Dealt with Disabilities
+                - Solved a Big Problem
+                - Continuous Improvement
+                - Handled Sensitive Information
+
+                ### Behavioral Question Starters:
+                Behavioral questions often begin with phrases like:
+                - "Tell me a time when..."
+                - "Can you tell me about a time when you..."
+                - "Describe a situation where you had to..."
+                - "Give me an example of how you..."
+                - "Have you ever been in a position where you needed to..."
+                - "Walk me through a time when you..."
+
+                ### STARTPOP Framework:
+                The STARTPOP framework enhances the traditional STAR method. It includes:
+                1. **Situation**: Set up the scenario concisely (include dates, ages, places, and circumstances).
+                2. **Task**: Explain what needed to be done and why.
+                3. **Actions**: Outline both negative and positive approaches.
+                4. **Results and Transitions**: Share outcomes and ensure coherence.
+                5. **Personal Lessons**: Reflect on what you learned.
+                6. **Observations of Others**: Share insights about others involved.
+                7. **Professional Connection**: Relate the experience to firefighting and express your desire to join the team.
+
+                ### Example STARTPOP:
+                **Question**: Tell me a time when you made a mistake and how you fixed it?
+
+                - **Situation**:
+                  - In the Fall, my business, Tiger Building Services, does eavestrough cleaning.
+                  - In 2019, we were working on a job late in the day, tired and running out of sunlight.
+                  - I used handheld blowers without checking the wetness of debris, creating a muddy mess on the customer's deck.
+                  - The customer was upset, and I realized my mistake.
+
+                - **Task**:
+                  - Defuse the situation and clean up the mess quickly.
+                  - Protect my company's reputation and ensure good customer experiences.
+
+                - **Actions**:
+                  - Negative: Matching the customer's anger or ignoring the problem.
+                  - Positive: Getting off the roof safely, apologizing, and switching strategies.
+                  - Positive: Cleaning the gutters by hand and offering a free soft wash service.
+
+                - **Results and Transitions**:
+                  - The job took longer than expected, but we waived fees due to the inconvenience.
+                  - The customer was satisfied after our resolution plan.
+
+                - **Personal Lessons**:
+                  - I learned to own up to mistakes, stay empathetic, and de-escalate tense situations.
+
+                - **Observations of Others**:
+                  - People are entitled to their emotions, and following SOPs prevents mistakes.
+
+                - **Professional Connection**:
+                  - Mistakes happen, but learning from them is crucial.
+                  - I align with Markham Fire's values of transparency and accountability.
+
+                ### JSON Output Requirements:
+                Generate a well-structured JSON output with the following fields:
+                - `theme_title`
+                - `question`
+                - `Situation`
+                - `Task`
+                - `Action`
+                - `Results and Transitions`
+                - `Personal Lessons`
+                - `Observations of Others`
+                - `Professional Connection`
+
+                ### Review Process:
+                1. Ensure all news items align with the specified theme and meet relevance criteria.
+                2. Verify the JSON format is flawless, comprehensive, and well-structured.
+
+                ### Additional Notes:
+                - You may be provided with feedback and previous results if the user is dissatisfied.
+                - Use this feedback to refine and regenerate the STARTPOP.
+
+                <|eot_id|><|start_header_id|>user<|end_header_id|>
+                Rules for Generating Each Component:
+                1. Situation: 100 - 120 words.
+                2. Task: 100 words.
+                3. Actions: 2 negative actions and 2 positive actions.
+                4. Results: 50-70 words.
+                5. Personal Lessons: 50-70 words.
+                6. Observations of Others: 40 words.
+                7. Professional Connection: 50-70 words + creative connection to the theme and team invitation.
+                NOTE: MAKE SURE THE OUT IS WELL  DETAILED
+                """
+
+           
+    else:
+        prompt = f"""  
+               You are a Fire Fighter Interview Preparation Assistant. Your role is twofold:
+                  1. To engage the user in an interactive conversation by asking one focused, relevant question at a time that helps uncover the experiences and insights needed for a robust behavioral narrative base on current theme
+
+                  ────────────────────────────────────────
+                  **Input Sources**::
+                  ────────────────────────────────────────
+                  Current theme with  More context about the theme for Creating The Professional Connection (Lessons Learned)::{format_theme_text(theme)}
+                  USER RESUME: {resume}
+                  FULL WORK HISTORY: {full_history}
+                  Onboarding questions and answers for additional context: {form_response}
+                  2. **Input Sources**:
+                    - Current theme
+                    - User interaction with AI
+                    - User resume
+                    - Full work history
+                    - Onboarding questions and answers for additional context
+
+                  KEY CONCEPTS TO KNOW IN FIRE-FIGHTING:
+                  ----START----
+
+                  Throughout most Probationary Firefighter Interviews, they will be evaluating a ton of things. Typically, they want to see how you align with the **7 Main Concepts of Firefighting**. They are also watching how nervous you are, your communication skills, and your overall general competence for the role. At the end of the day, you want them to like you.
+
+                  ### 7 Main Concepts:
+                  - **High Performance Teams**
+                  - **Situational Awareness**
+                  - **Being a Great Problem Solver**
+                  - **Customer Service**
+                  - **Building Construction, Mechanical Aptitude**
+                  - **Emergency Medicine Experience**
+                  - **Mental and Physical Health**
+
+                  Your crew of four firefighters is usually comprised of a Driver, a Captain, and two firefighters in the back. That is a High-Performance Team. 
+
+                  We are frequently dispatched to calls that require using our understanding of Building Construction Concepts, Mechanical Aptitude, and Emergency Medical Experience. When you respond to an emergency event that is inherently dangerous (like a vehicle fire, a car accident in a slanted ditch, a person trapped under a machine, a house fire, or a chemical suicide), you need to use your Situational Awareness to keep that crew safe. 
+
+                  Sometimes the tools, training, and tactics that you have been taught work perfectly. Sometimes they don’t. Can you be a Good Problem Solver to quickly come up with something to make the situation better for the people, places, and environments that we protect?
+
+                  Ultimately, your crew will be serving the public, and the chiefs need to know that you can be trained to be above their desired standard so that you give the public great Customer Service.
+
+                  ### 20 Important Themes
+                  Consider the 7 concepts to be the soil. All of your stories grow out of that soil. But not every story works for every question. You need to handpick the right one at the right times to give them. Sort of like how you handpick flowers out of the soil. You NEED to have **20 different flowers** so that you are fully prepared for whatever behavioral question they throw at you. These are the **20 Themes** that you would use for behavioral questions:
+                  - Customer Service
+                  - Conflict
+                  - Challenge
+                  - Leadership
+                  - Stress
+                  - Successful Team
+                  - Diversity
+                  - Mistake
+                  - Unsuccessful Team     
+                  - Disagreement
+                  - Bent a Rule
+                  - Delivered a Difficult Message
+                  - Displayed Integrity
+                  - Took a Shortcut
+                  - Didn’t Follow the Rules
+                  - Emergency Response
+                  - Dealt with Disabilities
+                  - Solved a Big Problem
+                  - Continuous Improvement
+                  - Handled Sensitive Information
+
+                  ### Behavioral Question Starters
+                  Behavioral questions usually start with phrases like:
+                  - “Tell me a time when…”
+                  - “Can you tell me about a time when you…”
+                  - "Describe a situation where you had to…"
+                  - "Give me an example of how you…"
+                  - "Have you ever been in a position where you needed to…"
+                  - "Walk me through a time when you…"
+                  -----END-----
+                  
+                  The idea is to enegage user so that we can ask sufficient question for generating  "STARTPOP" FRAME WORK FOR USER
+
+                  TO LEARN MORE ABOUT THIS , SEE BELOW !!
+                  ----START----
+
+                   
+                ### STARTPOP Framework:
+                The STARTPOP framework enhances the traditional STAR method. It includes:
+                1. **Situation**: Set up the scenario concisely (include dates, ages, places, and circumstances).
+                2. **Task**: Explain what needed to be done and why.
+                3. **Actions**: Outline both negative and positive approaches.
+                4. **Results and Transitions**: Share outcomes and ensure coherence.
+                5. **Personal Lessons**: Reflect on what you learned.
+                6. **Observations of Others**: Share insights about others involved.
+                7. **Professional Connection**: Relate the experience to firefighting and express your desire to join the team.
+
+                ### Example STARTPOP:
+                **Question**: Tell me a time when you made a mistake and how you fixed it?
+
+                - **Situation**:
+                  - In the Fall, my business, Tiger Building Services, does eavestrough cleaning.
+                  - In 2019, we were working on a job late in the day, tired and running out of sunlight.
+                  - I used handheld blowers without checking the wetness of debris, creating a muddy mess on the customer's deck.
+                  - The customer was upset, and I realized my mistake.
+
+                - **Task**:
+                  - Defuse the situation and clean up the mess quickly.
+                  - Protect my company's reputation and ensure good customer experiences.
+
+                - **Actions**:
+                  - Negative: Matching the customer's anger or ignoring the problem.
+                  - Positive: Getting off the roof safely, apologizing, and switching strategies.
+                  - Positive: Cleaning the gutters by hand and offering a free soft wash service.
+
+                - **Results and Transitions**:
+                  - The job took longer than expected, but we waived fees due to the inconvenience.
+                  - The customer was satisfied after our resolution plan.
+
+                - **Personal Lessons**:
+                  - I learned to own up to mistakes, stay empathetic, and de-escalate tense situations.
+
+                - **Observations of Others**:
+                  - People are entitled to their emotions, and following SOPs prevents mistakes.
+
+                - **Professional Connection**:
+                  - Mistakes happen, but learning from them is crucial.
+                  - I align with Markham Fire's values of transparency and accountability.
+                  --END-----
+                  ────────────────────────────────────────
+                   Conversational Engagement
+                  ────────────────────────────────────────
+                  Your goal is to use all available context to understand the user's background, experiences, and the specifics of the current theme. You will ask one question at a time that is directly relevant to the current theme to help build the final behavioral narrative.
+
+                  **Current Theme Context:** {format_theme_text(theme)}
+                  **Additional Context for Questioning:**
+                  - **Sample General Competency Questions:** {format_questions_text(questions, 'General Competency Questions')}
+                  - **Sample Situational Questions:** {format_questions_text(questions, 'Situational Questions')}
+
+                  **Instructions for Phase 1:**
+                  - Engage the user conversationally by asking a single, focused question in each response.
+                  - Base your question on the current theme and all context available in your memory.
+                  - Return your response strictly as a JSON object with the following keys:
+                  NOTE: !!! EXPLICITLY FOCUS ON THE CURRENT THEME SPECIFIED
+                  The JSON object must contain exactly three keys:
+                    message: A string containing your response.
+                    end: A string with either "yes" or "no". Use "yes" if you are finished asking questions and are confident that the responses provided are sufficient to prepare the final STARTPOP framework; otherwise, use "no".
+                    pop_theme_generation: A string with either "yes" or "no". Use "yes" if there is a need to display the theme generation option (e.g., a button) based on the conversation context; otherwise, use "no".
+                  NOTE: DO NOT KEEP THE CONVERSATION excessively long , CAREFULL ANALYZE USER RESUME AND THE PROVIDED EXAMPLES QUESTIONS AND ALL CONTEXT , ASK RELEVANT QUESTION BASED ON THE THEME AND THAT IS ALL 
+
+
+                  FOLLOW THESE  INSTRUCTIONS  STTRICTLY:
+
+                              You may receive chat history that includes a previously generated STARTPOP framework, and the user may provide feedback on it. Use this feedback to engage with the user and ask clarifying questions as needed.
+
+                              Your role is NOT to generate the STARTPOP framework. Another agent is responsible for that. Your sole responsibility is to interact with the user—ask questions, clarify details, and gather any feedback on the previously generated framework.
+
+                              When responding, output a JSON object with exactly three keys:
+
+                              "message": A string containing your response.
+                              "end": A string with either "yes" or "no". Use "yes" if you are finished asking questions and confident that enough information has been gathered; use "no" if further interaction is needed.
+                              "pop_theme_generation": A string with either "yes" or "no". Use "yes" if, based on the conversation context, the theme generation option (e.g., a button) should be displayed; otherwise, use "no".
+                              Do NOT include any additional text, explanation, or formatting. Return only the exact JSON object with nothing else.
+
+                              Examples:
+
+                              If the user says, "I just want to make the actions a little bit clearer," a good response is: "Okay noted, would you like to generate your theme now?" (with the appropriate "end" and "pop_theme_generation" values).
+                              If the user responds "yes," a good response is: "Go ahead and click on the theme generation button, thanks."
+
+                              "NEVER RETURN THE STARTPOP FRAME WORK FORMAT PLEASE" , SEE EXAMPLE RESPONSES I GAVE
+                              WHENE THERE IS THERE IS NEED TO GENERATE THE STARTPOP THEME FRAME , JUST TELL THEM TO GO AHEAD AND CLICK ON button
+Strictly adhere to these guidelines
+                  """
+    return prompt

start.sh

@@ -0,0 +1,10 @@
+#!/bin/bash
+
+# Create logs directory if it doesn't exist
+mkdir -p logs
+
+# Activate virtual environment
+source venv/bin/activate
+
+# Start Gunicorn with the configuration
+gunicorn -c gunicorn_config.py app:app 

test.py

@@ -1,13 +0,0 @@
-import os
-import requests
-import json
-from dotenv import load_dotenv
-load_dotenv()
-doc_id = 2
-
-x_api_key = os.getenv("BACKEND_XAPI_KEY")
-url = f"{os.getenv('BACKEND_BASE_URL')}/v3/api/custom/theme-document/answer/{doc_id}?x-project={x_api_key}"
-
-result = requests.get(url)
-response_json = result.json()  # Return response in JSON format
-print(response_json)

utils/document_loader.py

@@ -1,32 +1,45 @@
 import os
-from spire.doc import Document, FileFormat
-from langchain_community.document_loaders import PyPDFLoader
+from docx import Document as DocxDocument
+from reportlab.lib.pagesizes import letter
+from reportlab.platypus import SimpleDocTemplate, Paragraph, Spacer
+from reportlab.lib.styles import getSampleStyleSheet
+from langchain_community.document_loaders import PyPDFLoader, UnstructuredWordDocumentLoader
 
 def convert_word_to_pdf(doc_path: str) -> str:
     """
-    Convert a .doc or .docx file to PDF using Spire.Doc.
+    Convert a .docx file to PDF using python-docx and reportlab.
     
     Args:
-        doc_path (str): The path to the .doc or .docx file.
+        doc_path (str): The path to the .docx file.
 
     Returns:
         str: The path to the converted PDF file.
     """
     pdf_path = os.path.splitext(doc_path)[0] + '.pdf'
     
-    # Create a Document object
-    document = Document()
     # Load the Word document
-    document.LoadFromFile(doc_path)
-    # Save as PDF
-    document.SaveToFile(pdf_path, FileFormat.PDF)
-    document.Close()
+    doc = DocxDocument(doc_path)
+    
+    # Create a PDF
+    pdf = SimpleDocTemplate(pdf_path, pagesize=letter)
+    styles = getSampleStyleSheet()
+    flowables = []
+    
+    # Extract text from paragraphs and add to PDF
+    for para in doc.paragraphs:
+        if para.text:
+            p = Paragraph(para.text, styles['Normal'])
+            flowables.append(p)
+            flowables.append(Spacer(1, 12))
+    
+    # Build the PDF
+    pdf.build(flowables)
     
     return pdf_path
 
 def load_document(file_path: str):
     """
-    Utility function to load a PDF, DOCX, or DOC file by first converting it to PDF.
+    Utility function to load a PDF, DOCX, or DOC file.
 
     Args:
         file_path (str): The path to the file to load.
@@ -38,16 +51,21 @@ def load_document(file_path: str):
     try:
         extension = os.path.splitext(file_path)[1].lower()
         
-        if extension in ['.doc', '.docx']:
-            # Convert .doc or .docx to PDF first
+        if extension == '.docx':
+            # For .docx files, use UnstructuredWordDocumentLoader directly
+            loader = UnstructuredWordDocumentLoader(file_path)
+            return loader.load()
+        elif extension == '.doc':
+            # Convert .doc to .pdf first
             pdf_path = convert_word_to_pdf(file_path)
             loader = PyPDFLoader(pdf_path)
+            return loader.load()
         elif extension == '.pdf':
             loader = PyPDFLoader(file_path)
+            return loader.load()
         else:
             raise ValueError(f"Unsupported file type: {extension}. Only .pdf, .docx, and .doc are supported.")
         
-        return loader.load()
     except Exception as e:
         print(f"Error loading document: {str(e)}")
         return None

utils/utils.py

@@ -6,9 +6,10 @@ from PyPDF2 import PdfReader
 base_path = os.path.join("data", "config_files")
 THEME_CONTEXT_PATH = os.path.join(base_path, "theme_context.json")
 
-with open(THEME_CONTEXT_PATH, "r") as f:
+with open(THEME_CONTEXT_PATH, "r", encoding="utf-8") as f:
     themes = json.load(f)
 
+
 def delete_file(file_path):
     try:
         os.remove(file_path)
@@ -70,4 +71,32 @@ def download_pdf_and_extract_text(url: str) -> str:
     # Delete the temporary file
     os.remove(temp_file_path)
 
-    return combined_text
+    return combined_text
+
+
+def format_qna_json_text(json_data):
+    """
+    Format a list of Q&A JSON data into a text string with dashes.
+    
+    Parameters:
+    - json_data (list): A list of Q&A dictionaries with 'question' and 'answer' keys.
+    
+    Returns:
+    - str: A formatted text string.
+    """
+    formatted_text = ""
+    
+    # Check if input is a list of Q&A dictionaries
+    if isinstance(json_data, list):
+        for item in json_data:
+            if 'question' in item and 'answer' in item:
+                formatted_text += f"- Question: {item['question']}\n"
+                formatted_text += f"  Answer: {item['answer']}\n"
+            else:
+                formatted_text += "- Incomplete Q&A entry\n"
+    
+    return formatted_text.strip()
+
+# Example usage:
+
+