all setup reinitialized

assessment_data.csv

@@ -0,0 +1,13 @@
+assessment_id,assessment_name,start_date,user_name,total_assigned_items,completed_items,area
+1,Test Assesment,2024-10-14T00:00:00.000Z,Saba JoeJoe,36,1,Agile Methodologies
+1,Test Assesment,2024-10-14T00:00:00.000Z,Saba JoeJoe,36,1,API Developement
+1,Test Assesment,2024-10-14T00:00:00.000Z,Saba JoeJoe,36,1,Code Review
+1,Test Assesment,2024-10-14T00:00:00.000Z,Saba JoeJoe,36,1,Collaboration with Cross-Functional Teams
+1,Test Assesment,2024-10-14T00:00:00.000Z,Saba JoeJoe,36,1,Continuous Integration/Continuous Deployment (CI/CD)
+1,Test Assesment,2024-10-14T00:00:00.000Z,Saba JoeJoe,36,1,Debugging Techniques
+1,Test Assesment,2024-10-14T00:00:00.000Z,Saba JoeJoe,36,1,Documentation
+1,Test Assesment,2024-10-14T00:00:00.000Z,Saba JoeJoe,36,1,Performance Optimization
+1,Test Assesment,2024-10-14T00:00:00.000Z,Saba JoeJoe,36,1,System Design
+1,Test Assesment,2024-10-14T00:00:00.000Z,Saba JoeJoe,36,1,Unit Testing
+1,Test Assesment,2024-10-14T00:00:00.000Z,Saba JoeJoe,36,1,Version Control
+1,Test Assesment,2024-10-14T00:00:00.000Z,Tahsin Protik,1,0,Performance Optimization

data/processed/assessment_prediction/company_id/output.csv

@@ -0,0 +1,6 @@
+open_items,red_flags,num_employees,duration,assessment_type_biweekly,assessment_type_quarterly,assessment_type_weekly,open_items_assessment_type_weekly_lag_1,open_items_assessment_type_biweekly_lag_1,open_items_assessment_type_quarterly_lag_1,open_items_weekly_ma_3,open_items_biweekly_ma_3,open_items_quarterly_ma_3,percentage_change_open_items
+10,2,30,1,0,0,1,0.0,0.0,0.0,10.0,0.0,0.0,0.0
+12,1,25,1,1,0,0,10.0,0.0,0.0,10.0,12.0,0.0,19.999999999999996
+11,3,28,1,0,1,0,0.0,12.0,0.0,10.0,12.0,11.0,-8.333333333333337
+9,1,30,1,0,0,1,0.0,0.0,11.0,9.0,12.0,11.0,-18.181818181818176
+13,4,27,1,1,0,0,9.0,0.0,0.0,9.0,13.0,11.0,44.44444444444444

data/processed/assessment_prediction/testid/output.csv

@@ -0,0 +1,13 @@
+open_items,red_flags,num_employees,duration,assessment_type_biweekly,assessment_type_quarterly,assessment_type_weekly,open_items_assessment_type_weekly_lag_1,open_items_assessment_type_biweekly_lag_1,open_items_assessment_type_quarterly_lag_1,open_items_weekly_ma_3,open_items_biweekly_ma_3,open_items_quarterly_ma_3,percentage_change_open_items
+10,2,30,1,0,0,1,0.0,0.0,0.0,10.0,0.0,0.0,0.0
+12,1,25,1,1,0,0,10.0,0.0,0.0,10.0,12.0,0.0,19.999999999999996
+11,3,28,1,0,1,0,0.0,12.0,0.0,10.0,12.0,11.0,-8.333333333333337
+9,1,30,1,0,0,1,0.0,0.0,11.0,9.0,12.0,11.0,-18.181818181818176
+13,4,27,1,1,0,0,9.0,0.0,0.0,9.0,13.0,11.0,44.44444444444444
+14,2,26,1,0,0,1,0.0,13.0,0.0,11.5,13.0,0.0,7.692307692307687
+15,1,31,1,0,1,0,14.0,0.0,0.0,14.0,13.0,15.0,7.14285714285714
+16,3,29,1,1,0,0,0.0,0.0,15.0,14.0,16.0,15.0,6.666666666666665
+12,2,25,1,0,0,1,0.0,16.0,0.0,12.0,16.0,15.0,-25.0
+11,1,30,1,0,1,0,12.0,0.0,0.0,12.0,16.0,11.0,-8.333333333333337
+10,4,27,1,0,0,1,0.0,0.0,11.0,11.0,0.0,11.0,-9.090909090909093
+9,3,26,1,1,0,0,10.0,0.0,0.0,10.0,9.0,11.0,-9.999999999999998

data/raw/dummy_assessment_data.csv

@@ -0,0 +1,6 @@
+start_date,end_date,open_items,red_flags,num_employees,assessment_type
+2023-01-01,2023-01-02,10,2,30,weekly
+2023-01-08,2023-01-09,12,1,25,biweekly
+2023-01-15,2023-01-16,11,3,28,quarterly
+2023-01-22,2023-01-23,9,1,30,weekly
+2023-01-29,2023-01-30,13,4,27,biweekly

data/raw/erp_assessment_prediction/testid_raw_data.csv

@@ -0,0 +1,13 @@
+start_date,end_date,open_items,red_flags,num_employees,assessment_type
+2023-01-01,2023-01-02,10,2,30,weekly
+2023-01-08,2023-01-09,12,1,25,biweekly
+2023-01-15,2023-01-16,11,3,28,quarterly
+2023-01-22,2023-01-23,9,1,30,weekly
+2023-01-29,2023-01-30,13,4,27,biweekly
+2023-02-05,2023-02-06,14,2,26,weekly
+2023-02-12,2023-02-13,15,1,31,quarterly
+2023-02-19,2023-02-20,16,3,29,biweekly
+2023-02-26,2023-02-27,12,2,25,weekly
+2023-03-05,2023-03-06,11,1,30,quarterly
+2023-03-12,2023-03-13,10,4,27,weekly
+2023-03-19,2023-03-20,9,3,26,biweekly

data/raw/erp_company_assessment/testid_raw_data.2csv

@@ -0,0 +1,11 @@
+Assessment_ID,Open_Items,Red_Flags,Assessment_Frequency,Assessment_Start_Date,Assessment_End_Date,Assessment_Area,Assessment_Status,Assessment_Admin
+1,3,1,Weekly,2023-01-01,2023-01-07,Deployment,Completed,Admin A
+2,4,2,Bi-Weekly,2023-01-16,2023-01-22,Communication,Completed,Admin B
+3,2,0,Weekly,2023-01-31,2023-02-06,Deployment,Completed,Admin A
+4,5,1,Quarterly,2023-02-15,2023-02-21,Communication,In Progress,Admin B
+5,1,0,Bi-Weekly,2023-03-02,2023-03-08,Deployment,Completed,Admin A
+6,3,3,Weekly,2023-03-17,2023-03-23,Deployment,Completed,Admin A
+7,2,2,Quarterly,2023-04-01,2023-04-07,Communication,Incomplete,Admin B
+8,4,1,Bi-Weekly,2023-04-16,2023-04-22,Deployment,Completed,Admin A
+9,5,1,Weekly,2023-05-01,2023-05-07,Communication,In Progress,Admin B
+10,3,2,Quarterly,2023-05-16,2023-05-22,Deployment,Completed,Admin A

data/raw/erp_company_assessment/testid_raw_data.csv

@@ -0,0 +1,11 @@
+Assessment_ID,Open_Items,Red_Flags,Assessment_Frequency,Assessment_Start_Date,Assessment_End_Date,Assessment_Area,Assessment_Status,Assessment_Admin,Department
+1,3,1,Weekly,2023-01-01,2023-01-07,Deployment,Completed,Admin A,IT
+2,4,2,Bi-Weekly,2023-01-16,2023-01-22,Communication,Completed,Admin B,HR
+3,2,0,Weekly,2023-01-31,2023-02-06,Deployment,Completed,Admin A,Finance
+4,5,1,Quarterly,2023-02-15,2023-02-21,Communication,In Progress,Admin B,IT
+5,1,0,Bi-Weekly,2023-03-02,2023-03-08,Deployment,Completed,Admin A,HR
+6,3,3,Weekly,2023-03-17,2023-03-23,Deployment,Completed,Admin A,Finance
+7,2,2,Quarterly,2023-04-01,2023-04-07,Communication,Incomplete,Admin B,IT
+8,4,1,Bi-Weekly,2023-04-16,2023-04-22,Deployment,Completed,Admin A,HR
+9,5,1,Weekly,2023-05-01,2023-05-07,Communication,In Progress,Admin B,Finance
+10,3,2,Quarterly,2023-05-16,2023-05-22,Deployment,Completed,Admin A,IT

models/assessment_prediction/testid/testid_latest_data.csv

@@ -0,0 +1,2 @@
+open_items,red_flags,num_employees,duration,assessment_type_biweekly,assessment_type_quarterly,assessment_type_weekly,open_items_assessment_type_weekly_lag_1,open_items_assessment_type_biweekly_lag_1,open_items_assessment_type_quarterly_lag_1,open_items_weekly_ma_3,open_items_biweekly_ma_3,open_items_quarterly_ma_3,percentage_change_open_items
+9,3,26,1,1,0,0,10.0,0.0,0.0,10.0,9.0,11.0,-9.999999999999998

notebooks/data/processed/assessment_prediction/testid/output.csv

@@ -0,0 +1,6 @@
+open_items,red_flags,num_employees,duration,assessment_type_biweekly,assessment_type_quarterly,assessment_type_weekly,open_items_assessment_type_weekly_lag_1,open_items_assessment_type_biweekly_lag_1,open_items_assessment_type_quarterly_lag_1,open_items_weekly_ma_3,open_items_biweekly_ma_3,open_items_quarterly_ma_3,percentage_change_open_items
+10,2,30,1,0,0,1,0.0,0.0,0.0,10.0,0.0,0.0,0.0
+12,1,25,1,1,0,0,10.0,0.0,0.0,10.0,12.0,0.0,19.999999999999996
+11,3,28,1,0,1,0,0.0,12.0,0.0,10.0,12.0,11.0,-8.333333333333337
+9,1,30,1,0,0,1,0.0,0.0,11.0,9.0,12.0,11.0,-18.181818181818176
+13,4,27,1,1,0,0,9.0,0.0,0.0,9.0,13.0,11.0,44.44444444444444

notebooks/dummy_assessment_data.csv

@@ -0,0 +1,11 @@
+Assessment_ID,Open_Items,Red_Flags,Assessment_Frequency,Assessment_Start_Date,Assessment_End_Date,Assessment_Area,Assessment_Status,Assessment_Admin
+1,3,1,Weekly,2023-01-01,2023-01-07,Deployment,Completed,Admin A
+2,4,2,Bi-Weekly,2023-01-16,2023-01-22,Communication,Completed,Admin B
+3,2,0,Weekly,2023-01-31,2023-02-06,Deployment,Completed,Admin A
+4,5,1,Quarterly,2023-02-15,2023-02-21,Communication,In Progress,Admin B
+5,1,0,Bi-Weekly,2023-03-02,2023-03-08,Deployment,Completed,Admin A
+6,3,3,Weekly,2023-03-17,2023-03-23,Deployment,Completed,Admin A
+7,2,2,Quarterly,2023-04-01,2023-04-07,Communication,Incomplete,Admin B
+8,4,1,Bi-Weekly,2023-04-16,2023-04-22,Deployment,Completed,Admin A
+9,5,1,Weekly,2023-05-01,2023-05-07,Communication,In Progress,Admin B
+10,3,2,Quarterly,2023-05-16,2023-05-22,Deployment,Completed,Admin A

notebooks/models/assessment_prediction/testid/testid_latest_data.csv

@@ -0,0 +1,2 @@
+open_items,red_flags,num_employees,duration,assessment_type_biweekly,assessment_type_quarterly,assessment_type_weekly,open_items_assessment_type_weekly_lag_1,open_items_assessment_type_biweekly_lag_1,open_items_assessment_type_quarterly_lag_1,open_items_weekly_ma_3,open_items_biweekly_ma_3,open_items_quarterly_ma_3,percentage_change_open_items
+13,4,27,1,1,0,0,9.0,0.0,0.0,9.0,13.0,11.0,44.44444444444444

notebooks/test

@@ -0,0 +1,13 @@
+,start_date,end_date,open_items,red_flags,num_employees,assessment_type
+0,2023-01-01,2023-01-02,10,2,30,weekly
+1,2023-01-08,2023-01-09,12,1,25,biweekly
+2,2023-01-15,2023-01-16,11,3,28,quarterly
+3,2023-01-22,2023-01-23,9,1,30,weekly
+4,2023-01-29,2023-01-30,13,4,27,biweekly
+5,2023-02-05,2023-02-06,14,2,26,weekly
+6,2023-02-12,2023-02-13,15,1,31,quarterly
+7,2023-02-19,2023-02-20,16,3,29,biweekly
+8,2023-02-26,2023-02-27,12,2,25,weekly
+9,2023-03-05,2023-03-06,11,1,30,quarterly
+10,2023-03-12,2023-03-13,10,4,27,weekly
+11,2023-03-19,2023-03-20,9,3,26,biweekly

notebooks/test_sop_generator.ipynb

@@ -7,57 +7,28 @@
    "outputs": [],
    "source": [
     "from langchain_community.document_loaders import PyPDFLoader\n",
-    "loader = PyPDFLoader(\"/root/ds_erp_ai/data/raw/test_sop.pdf\")\n",
+    "loader = PyPDFLoader(\"/content/Example SOP (1) (1).pdf\")\n",
     "docs = loader.load()"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 3,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "from langchain_community.document_loaders import PyPDFLoader\n",
-    "loader = PyPDFLoader(\"/root/ds_erp_ai/data/raw/test_sop.pdf\")\n",
-    "docs = loader.load()"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 4,
+   "execution_count": 1,
    "metadata": {},
    "outputs": [
     {
-     "data": {
-      "text/plain": [
-       "[Document(metadata={'source': '/root/ds_erp_ai/data/raw/test_sop.pdf', 'page': 0}, page_content='ERGO  \\nERGO 202 4 VISION  \\nDEFINITION  \\nA standard operating procedure (SOP) is a step -by-step, repeatable process for any critical \\nand routine task. It’s  a kind of documentation that prevents stress, mistakes, and \\nmiscommunication. SOPs ensure reliability, efficiency, and consistently hitting quality \\nstandards in regular work activities.  The SOPs represent the what, when, where and how of \\nevery role in the organization as it aligns to it’s vision and goals.  \\n \\nVISION:  \\nERGO envisions itself as the leading platform for renting spaces, equipment, and services, \\ntransforming the way people access and share resources. By the end of the next year, we \\naim to significantly increase our user base and participation rates, fosterin g a vibrant and \\nactive community that thrives on mutual benefit and shared success.  \\n \\nWe are committed to creating an intuitive, reliable, and user -friendly app that empowers \\nindividuals and businesses to rent what they need when they need it, contributing to a \\nmore efficient and sustainable use of resources. We believe in the power of tech nology to \\nsimplify transactions, build connections, and unlock new possibilities.  \\n \\nOur strategic vision is to continuously innovate and improve our platform, ensuring it \\nmeets the evolving needs of our users. We will invest in marketing and user engagement \\nstrategies to attract new users and retain existing ones, with a focus on providin g \\nexceptional customer service and a seamless user experience.  \\n \\nIn the long term, we aspire to expand our services to new markets and regions, establishing \\nERGO as a global leader in the sharing economy. We are dedicated to creating value for our \\nusers, stakeholders, and the broader community, contributing to a more co nnected and \\nresource -efficient world.\"   \\n \\nRemember, a strategic vision is a guiding light for a company’s future direction. It will be \\nrevisited and potentially revised as the company grows and the market landscape \\nchanges. It’s also important to ensure that all stakeholders understand and align with the \\nvision. This includes employees, investors, and customers. They all play a crucia l role in \\nthe company’s journey towards its vision.  \\n '),\n",
-       " Document(metadata={'source': '/root/ds_erp_ai/data/raw/test_sop.pdf', 'page': 1}, page_content='  '),\n",
-       " Document(metadata={'source': '/root/ds_erp_ai/data/raw/test_sop.pdf', 'page': 2}, page_content='GOALS;  \\n1. MARKETING  \\na. Brand Awareness : Increase brand awareness by 50% over the next year \\nthrough targeted marketing campaigns.  \\nb. User Acquisition:  Acquire 100,000 new users over the next year through \\ndigital marketing strategies.  \\nc. User Engagement : Improve user engagement by 30% by implementing a \\ncontent marketing strategy that educates users about the benefits and uses \\nof ERGO.  \\nd. Market Expansion : Enter two new geographic markets by the end of the year \\nthrough localized marketing campaigns.  \\n2. IT INFRASTRUCTURE : \\na. Platform Stability: Achieve 99.9% uptime for the ERGO app to ensure a \\nseamless user experience.  \\nb. Security: Implement robust security measures to protect user data and build \\ntrust with the user base.  \\nc. Scalability: Enhance the IT infrastructure to support a 50% increase in user \\ntraffic over the next year.  \\nd. Feature Development: Develop and launch three new features based on user \\nfeedback to improve the functionality and usability of the ERGO app.  \\n3. SALES TEAM GOALS:  \\na. New User Sales: Sign up 50,000 new users over the next year through direct \\nsales efforts.  \\nb. Partnership Development: Establish partnerships with 20 new businesses to \\noffer their spaces, equipment, or services on ERGO.  \\nc. Customer Retention: Achieve a customer retention rate of 90% by providing \\nexcellent customer service and follow -up. \\nd. Revenue Growth: Increase sales revenue by 30% over the next year by \\nupselling premium features and services.  \\n  '),\n",
-       " Document(metadata={'source': '/root/ds_erp_ai/data/raw/test_sop.pdf', 'page': 3}, page_content='ERGO 2024 Standard Operating Procedures : \\n \\nAPPLICATION  \\nThis document applies to Information technology specialists  and contractors. Marketing \\nteams and Contracting Specialist inv olved with Marketing contracts. Current and future \\nsales associates employed or contracted by ERGO.  \\n \\nVISION STATEMENT  \\n \\nERGO’s strategic goals for the next year encompass marketing, IT infrastructure, and sales. \\nIn marketing, we aim to increase brand awareness by 50% through targeted campaigns, \\nacquire 100,000 new users via digital strategies, improve user engagement by 30%  with a \\ncontent marketing strategy, and expand into two new markets. For IT infrastructure, our \\ngoals are to achieve 99.9% app uptime, implement robust security measures, enhance \\ninfrastructure to support a 50% increase in user traffic, and develop three n ew features \\nbased on user feedback. The sales team aims to sign up 50,000 new users, establish \\npartnerships with 20 new businesses, achieve a 90% customer retention rate, and increase \\nsales revenue by 30% through upselling.  \\n \\nRESPONSIBILITIES  \\n \\nCHIEF OPERATIONS OFFICER : \\nCross -Functional Coordination: As a key member of the executive team, the COO will \\nensure alignment and coordination across different departments to achieve these goals.  \\nPerformance Monitoring: The COO will establish key performance indicators (KPIs) and \\nregularly review progress towards these goals, making necessary adjustments to strategies \\nand plans.  \\n \\nSALES MANAGER;  \\nUnder the oversight and control of the COO,  the sales manager will guide the sales team to \\nsign up 50,000 new users, establish partnerships with 20 new businesses, achieve a 90% \\ncustomer retention rate, and increase sales revenue by 30% through upselling.  \\n \\nCAMPAIGN MANAGER:  '),\n",
-       " Document(metadata={'source': '/root/ds_erp_ai/data/raw/test_sop.pdf', 'page': 4}, page_content='Under the oversight and control of the COO  the campaign manager  will oversee the \\nmarketing team’s efforts to increase brand awareness and acquire new users. They will \\nalso ensure the implementation of a content marketing strategy to improve user \\nengagement and  spearhead efforts to expand into two new geographic markets.  The \\ncampaign  manager will exercise operational oversight and control over marketing  \\nspecialist and ad contracts.   \\n \\nCONTENT MARKETING SPECIALIST:  \\nUnder the oversight and control of the campaign manager the content marketing specialist \\nwill;  \\n1. Create fun and informative quizzes related to your brand or services . Share them on \\nsocial media and encourage users to participate and share their results.  \\n2. Share behind -the-scenes glimpses of your company culture, product development, \\nor team members. People love to connect with the human side of brands.  \\n3. User -Generated Content Contests: Run a contest where users submit photos, \\nvideos, or stories related to your brand. Feature the best entries on your website or \\nsocial media.  \\n4. Host online webinars or workshops that provide value to potential users. Topics \\ncould include product demos, services  trends, or problem -solving sessions.  \\n5. Create comprehensive guides or e -books that address common pain points your \\nproduct solves. Promote these through targeted ads and email campaigns.  \\n6. Share success stories of existing users. Highlight how your product transformed \\ntheir lives or businesses. Authentic testimonials resonate well with potential users.  \\n7. Develop a series of short videos explaining different aspects of your product or \\nservices . Keep them engaging and easy to understand.  \\n8. Create visually appealing infographics that educate users about ERGO’s benefits. \\nUse interactive elements like clickable sections or animations.  \\n9. Set up a community forum where users can ask questions, share tips, and interact \\nwith each other. Host live Q&A sessions on social media.  \\n10. Write blog posts specifically tailored to the culture, language, and interests of the \\nnew markets. Address local challenges and showcase how ERGO can solve them.  \\n11. Create landing pages in the local language, highlighting the unique value \\nproposition of ERGO for each market.  \\n12. Partner with influencers or thought leaders from the new markets. Have them \\ncreate content or host events related to ERGO.  \\n \\nDIGITAL MARKETING SPECIALIST:  '),\n",
-       " Document(metadata={'source': '/root/ds_erp_ai/data/raw/test_sop.pdf', 'page': 5}, page_content='Under the oversight and control of the campaign manager the digital marketing specialist \\nwill;  \\n1. Conduct market research to understand the target audience, their preferences, and \\nwhere they spend their time online..  \\n2. React to media inquiries: engage with journalists and media outlets to kickstart your \\nPR efforts1.  \\n3. Start a referral program to encourage existing users to refer new ones.  \\n4. Regularly publish informative blog posts to attract organic traffic and showcase \\nyour expertise . \\n5. Offer limited -time promotions or exclusive deals to entice new users.  \\n6. Personalize content: tailor content to user interests and needs. Use personalized \\npush notifications, emails, and SEO strategies.  \\n7. Measure metrics: track page views, time spent on site, bounce rate, and social \\nmedia interactions. Tools like Full Session  can provide insights into user behavior4.  \\n8. Engage with audience: respond to comments, likes, and shares. Regularly publish \\nrelevant content to keep users engaged.  \\n9. Conduct market research: understand local nuances, preferences, and behaviors. \\nDefine demographics, languages, and dialects.  \\n10. Adapt content: translate and culturally adapt content. Consider local holidays, \\nsymbols, and traditions.  \\n11. Localize marketing channels: choose platforms popular in the new markets. Adjust \\nadvertising campaigns accordingly.  \\n12. Test and analyze pilot localized campaigns, analyze results, and optimize based on \\ndata.  \\nUnder the oversight and control of the campaign manager  and in coordination with other \\nspecialist  the digital marketing specialist must;  \\n1. Leverage social media: use platforms where your audience is most active. Share \\nengaging content, run targeted ads, and interact with followers  \\n2. Implement a search -focused content marketing strategy, create valuable content \\nthat aligns with what your audience searches for. Optimize it for relevant keywords \\nto improve organic visibility  \\n3. Run targeted ads on cloud -related platforms or industry -specific websites in the \\nnew markets. Emphasize ERGO’s cloud advantages for local businesses.  \\nSEARCH ENGINE OPTIMIZATION SPECIALIST:  \\nUnder the oversight and control of the campaign manager the search engine optimization \\nspecialist  will;  \\n1. Create valuable content that aligns with what your audience searches for. Optimize \\nit for relevant keywords to improve organic visibility  '),\n",
-       " Document(metadata={'source': '/root/ds_erp_ai/data/raw/test_sop.pdf', 'page': 6}, page_content='2. Personalize Content: Tailor content to user interests and needs. Use personalized \\npush notifications, emails, and SEO strategies.  \\n3. Measure Metrics: Track page views, time spent on site, bounce rate, and social \\nmedia interactions. Tools like Full  Session can provide insights into user behavior4.  \\n4. Under the oversight and control of the campaign manager and in coordination with \\nother specialist the search engine optimization specialist must;  \\n5. Engage with journalists and media outlets to kickstart your PR efforts.  \\n6. Use platforms where your audience is most active. Share engaging content, run \\ntargeted ads, and interact with followers.  \\n \\nINFORMATION  TECHNOLOGY OFFICER  \\nIn coordination cooperation with the Chief Operations Officer  the Information Technology \\nOfficer will  work closely with the it team to achieve 99.9% uptime for the ergo app, \\nimplement robust security measures, shall enhance the it infrastructure to support \\nincreased user traffic, and must  develop and launch new features based on user feedback.  \\nThe Information Technology officer will h ave a well -defined plan in place to handle security \\nincidents  in local and new markets . \\nCLOUD ENGINEER:  \\nunder the oversight and control of the Information Technology Officer the Cloud Engineer \\nshall;  \\n1. Select cloud service providers with strong security practices. Ensure they comply \\nwith industry standards.  \\n2. Clearly communicate to users how their data will be collected, used, and \\nprotected. Maintain a comprehensive privacy policy on your website.  \\n3. Obtain explicit consent from users before collecting any personal data. Implement \\ncookie consent banners and ensure compliance with regulations like GDPR or \\nCCPA.  \\n4. Isolate different components of your infrastructure to prevent lateral movement in \\ncase of a breach  \\n5. Conduct periodic security audits to identify vulnerabilities and address them \\npromptly.  \\nunder the oversight and control of the Information Technology Officer the Cloud Engineer \\nwill;  \\n1. Design , implement, and manag e cloud computing solutions, such as public, \\nprivate, and hybrid cloud environments.  \\n2. Create step -by-step video tutorials on how new users can set up their accounts, \\naccess features, and utilize cloud -based services within ERGO.  \\n3. Deploy cloud infrastructure  '),\n",
-       " Document(metadata={'source': '/root/ds_erp_ai/data/raw/test_sop.pdf', 'page': 7}, page_content=\"4. Migrate on -premises systems to the cloud  \\n1. Continuously monitor and optimize ERGO’s cloud resources for scalability and \\ncost -effectiveness. Share success stories of how cloud improvements directly \\nbenefit users.  Ensure  cloud security and compliance   \\n2. Host webinars on cloud -related topics, emphasizing how ERGO leverages cloud \\ntechnology. Invite industry experts and influencers to participate.  \\nunder the oversight and control of the Information Technology Officer the Cloud Engineer \\nmust ; \\n3. Establish an online community forum where users can share tips, ask questions, \\nand discuss cloud -related topics. Cloud engineers can actively participate and \\nprovide insights.  \\n4. Create visually appealing case studies or blog posts highlighting how ERGO’s cloud \\ninfrastructure enables seamless user experiences. Explain how scalability, \\nreliability, and security are achieved through cloud services.  \\n5. Develop an interactive web page or app feature that takes users on a virtual tour of \\nERGO’s cloud architecture. Explain key components, data flow, and benefits.  \\n6. Train cloud support teams to assist users in the new markets. Ensure they \\nunderstand the cultural context and can address cloud -related queries effectively.  \\n \\nDATABASE ADMINISTRATOR (DBA):  \\nDatabase administrators are responsible for the design, implementation, and \\nmaintenance of databases that store and organize an organization's data. They install and \\nconfigure database management systems, optimize database performance, backup and \\nrestore d ata, and enforce data security policies.   \\nunder the oversight and control of the Information Technology Officer the DBA will;  \\n1. Continuously update and expand the user database. Capture user data through \\nsign -ups, app usage, and interactions. Ensure data accuracy and completeness.  \\n2. Leverage lookalike audience targeting on platforms like Facebook and Google Ads. \\nUse existing user profiles to find similar potential users.  \\n3. Set up automated triggers based on user behavior (e.g., abandoned carts, frequent \\nvisits). Send personalized emails or notifications to re -engage users.  \\n4. Regularly clean and optimize the database. Remove inactive or irrelevant users to \\nimprove engagement metrics.  \\n5. Gather data on market trends, cultural nuances, and user preferences in the new \\nmarkets. Understand how ERGO’s brand can align with local values.  \\nunder the oversight and control of the Information Technology Officer the DBA must;  \"),\n",
-       " Document(metadata={'source': '/root/ds_erp_ai/data/raw/test_sop.pdf', 'page': 8}, page_content='1. Collaborate with the marketing team to collect data on brand awareness metrics \\n(e.g., social media mentions, website traffic, search volume). Use this data to \\nidentify trends and areas for improvement.  \\n2. Create segments within the database based on user interactions with ERGO’s \\nbrand. Tailor marketing messages to each segment, emphasizing brand values, \\nunique selling points, and success stories.  \\n3. Experiment with different marketing campaigns and track their impact on brand \\nawareness. Optimize based on the most effective strategies.  \\n6. Work closely with data analysts to profile existing users. Understand their \\ndemographics, behavior, and preferences. Use this information to target similar \\naudiences in digital marketing campaigns.  \\n7. Collaborate with content creators to personalize marketing content. Use database \\ninsights to tailor messages that resonate with users’ interests and pain points.  \\n8. Collaborate with content teams to create region -specific marketing materials. \\nTranslate content, adapt visuals, and customize messaging.  \\nIT SECURITY ANALYST:  \\nUnder the oversight and control of the Information Technology Officer the IT Security \\nAnalyst shall;  \\n1. Research and understand the data protection laws and regulations in the new \\nmarkets. Ensure compliance with GDPR, CCPA, or other relevant standards.  \\n2. Train local teams on incident response procedures  \\nUnder the oversight and control of the Information Technology Officer the IT Security \\nAnalyst will;  \\n1. Restrict access to marketing databases. Only authorized personnel should handle \\nsensitive customer information.  \\n2. Implement CAPTCHA or other anti -bot measures during user registration to prevent \\nfraudulent accounts.  \\n3. If ERGO uses apis for user registration or authentication, ensure they are properly \\nsecured with authentication tokens and rate limiting.  \\n4. Set up anomaly detection to identify suspicious user activity (e.g., sudden spikes in \\nregistrations).  \\n5. Regularly update CMS platforms and plugins to patch security vulnerabilities. Use \\nstrong authentication for CMS access.  \\n6. Ensure that content marketing materials (blogs, videos, etc.) Are free from \\nmalicious code (e.g., cross -site scripting). Scan for vulnerabilities.  \\nUnder the oversight and control of the Information Technology Officer the IT Security \\nAnalyst  must;  '),\n",
-       " Document(metadata={'source': '/root/ds_erp_ai/data/raw/test_sop.pdf', 'page': 9}, page_content=\"1. Collaborate with the marketing team to ensure that all customer data collected for \\ntargeted campaigns is securely stored and transmitted. Implement encryption for \\ndata in transit and at rest.  \\n2. Educate marketing staff about security best practices. Ensure they understand the \\nimportance of protecting customer data.  \\nIT OPERATIONS MANAGER:  \\nIT operations managers oversee the day -to-day operations of an organization's IT \\ninfrastructure and support teams. They develop and implement IT policies and procedures, \\nmanage IT resources and budgets, coordinate IT projects and initiatives, and ensure th at IT \\nsystems meet business requirements and objectives.  \\nIT SUPPORT SPECIALIST:  \\nUnder the oversight and control of the IT OPERATIONS MANAGER the IT Security Analyst \\nwill;  \\n1. Ensure that the onboarding process is seamless for new users. Provide clear \\ninstructions, troubleshoot any issues, and guide them through setup.  \\n2. Set up efficient user support channels (chat, email, or phone). Respond promptly to \\ninquiries and resolve any technical hurdles.  \\n3. Reach out to users who haven’t engaged recently. Offer personalized assistance, \\nask for feedback, and provide tips on using ERGO effectively.  \\n4. Gather user feedback on their experiences. Use this information to improve the \\nproduct and enhance brand perception.  \\n5. Conduct surveys to understand pain points and areas for improvement. Use this \\ndata to enhance user engagement strategies.  \\n6. Continuously update the knowledge base with troubleshooting guides, FAQs, and \\nbest practices. Make it easily accessible to users.  \\n7. Collaborate with localization teams to ensure that ERGO’s software and \\ndocumentation are available in the local languages of the new markets.  \\n8. Anticipate region -specific technical challenges. Prepare troubleshooting guides \\ntailored to the new markets.  \\n9. Host webinars or virtual training sessions for existing and potential users. Teach \\nthem how to make the most of ERGO’s services.  \\nUnder the oversight and control of the IT OPERATIONS MANAGER the IT Security Analyst \\nmust;  \\n1. Collaborate with the marketing team to create engaging technical content. Write \\nblog posts, FAQs, or video tutorials about ERGO’s unique features and benefits.  \\n2. Encourage existing users to refer new users. Offer incentives or discounts for \\nsuccessful referrals.  \"),\n",
-       " Document(metadata={'source': '/root/ds_erp_ai/data/raw/test_sop.pdf', 'page': 10}, page_content='3. Understand cultural nuances and adapt communication accordingly. Provide user \\nsupport that aligns with local customs and preferences.  \\nDEVOPS ENGINEER:  \\nUnder the oversight and control of the IT OPERATIONS MANAGER the DEVOPS engineer  \\nshall ; \\n1. Collaborate with legal and compliance teams to understand data protection laws \\nand ensure adherence in each region.  \\nUnder the oversight and control of the IT OPERATIONS MANAGER the DEVOPS engineer \\nwill;  \\n1. Implement monitoring tools to track website performance, uptime, and user \\ninteractions. Set up alerts for any anomalies or downtime.  \\n2. Ensure that ERGO’s infrastructure can handle increased traffic during marketing \\ncampaigns. Autoscale servers and optimize load balancers.  \\n3. Design robust and fault -tolerant systems to handle user registrations, logins, and \\ndata processing. Use redundancy and failover mechanisms.  \\n4. optimize content delivery networks (CDNs). Reduce latency for users accessing \\nERGO’s content.  \\n5. Set up infrastructure to support A/B testing for different content variations. Monitor \\nuser engagement metrics for each variant.  \\n6. Customize deployment scripts for each new market. Consider regional cloud \\nproviders and data centers.  \\nUnder the oversight and control of the IT OPERATIONS MANAGER the DEVOPS engineer \\nmust;  \\n1. Set up CI/CD pipelines to streamline marketing campaign deployments. Automate \\nthe rollout of new landing pages, banners, and promotional content.  \\n2. Collaborate with DBA  to simulate heavy user loads during testing. Optimize \\ndatabase queries and API endpoints for scalability.  \\n3. Create a feedback loop between DevOps and content teams. Rapidly deploy \\nchanges based on user feedback to improve engagement.  \\n4. Set up databases with geo -replication to ensure data availability and low latency in \\nnew markets.  \\n \\n \\n \\n \\n \\n \\n '),\n",
-       " Document(metadata={'source': '/root/ds_erp_ai/data/raw/test_sop.pdf', 'page': 11}, page_content=' \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n ')]"
-      ]
-     },
-     "execution_count": 4,
-     "metadata": {},
-     "output_type": "execute_result"
+     "ename": "NameError",
+     "evalue": "name 'SOPsResponse' is not defined",
+     "output_type": "error",
+     "traceback": [
+      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
+      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
+      "Cell \u001b[0;32mIn[1], line 22\u001b[0m\n\u001b[1;32m     19\u001b[0m \u001b[38;5;28;01mclass\u001b[39;00m \u001b[38;5;21;01mRoles_response\u001b[39;00m(BaseModel):\n\u001b[1;32m     20\u001b[0m     roles: \u001b[38;5;28mlist\u001b[39m[\u001b[38;5;28mstr\u001b[39m]\n\u001b[0;32m---> 22\u001b[0m \u001b[38;5;28;01mclass\u001b[39;00m \u001b[38;5;21;01mSopGenerator\u001b[39;00m:\n\u001b[1;32m     23\u001b[0m     \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21m__init__\u001b[39m(\u001b[38;5;28mself\u001b[39m):\n\u001b[1;32m     24\u001b[0m         \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mapi_key \u001b[38;5;241m=\u001b[39m os\u001b[38;5;241m.\u001b[39mgetenv(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mOPENAI_API_KEY\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n",
+      "Cell \u001b[0;32mIn[1], line 89\u001b[0m, in \u001b[0;36mSopGenerator\u001b[0;34m()\u001b[0m\n\u001b[1;32m     60\u001b[0m   response \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mclient\u001b[38;5;241m.\u001b[39mbeta\u001b[38;5;241m.\u001b[39mchat\u001b[38;5;241m.\u001b[39mcompletions\u001b[38;5;241m.\u001b[39mparse(\n\u001b[1;32m     61\u001b[0m             model\u001b[38;5;241m=\u001b[39m\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mmodel,\n\u001b[1;32m     62\u001b[0m             messages\u001b[38;5;241m=\u001b[39m[\n\u001b[0;32m   (...)\u001b[0m\n\u001b[1;32m     83\u001b[0m     temperature\u001b[38;5;241m=\u001b[39m\u001b[38;5;241m0.1\u001b[39m\n\u001b[1;32m     84\u001b[0m     )\n\u001b[1;32m     86\u001b[0m   \u001b[38;5;28;01mreturn\u001b[39;00m json\u001b[38;5;241m.\u001b[39mloads(response\u001b[38;5;241m.\u001b[39mchoices[\u001b[38;5;241m0\u001b[39m]\u001b[38;5;241m.\u001b[39mmessage\u001b[38;5;241m.\u001b[39mcontent)\n\u001b[0;32m---> 89\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mgenerate_sops\u001b[39m(\u001b[38;5;28mself\u001b[39m, roles, docs_text) \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m \u001b[43mSOPsResponse\u001b[49m:\n\u001b[1;32m     90\u001b[0m     roles_sops_all \u001b[38;5;241m=\u001b[39m {}\n\u001b[1;32m     92\u001b[0m     \u001b[38;5;28;01mfor\u001b[39;00m role \u001b[38;5;129;01min\u001b[39;00m roles:\n",
+      "\u001b[0;31mNameError\u001b[0m: name 'SOPsResponse' is not defined"
+     ]
     }
    ],
-   "source": [
-    "docs"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 5,
-   "metadata": {},
-   "outputs": [],
    "source": [
     "import os\n",
     "import json\n",
@@ -187,31 +158,11 @@
     "roles = [\"Devops engineers\"]\n",
     "sops_response = service.check_role_sop(roles,docs)"
    ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 6,
-   "metadata": {},
-   "outputs": [
-    {
-     "data": {
-      "text/plain": [
-       "{'message': 'SOPs found for the roles: Devops engineers', 'status': True}"
-      ]
-     },
-     "execution_count": 6,
-     "metadata": {},
-     "output_type": "execute_result"
-    }
-   ],
-   "source": [
-    "sops_response"
-   ]
   }
  ],
  "metadata": {
   "kernelspec": {
-   "display_name": "Python 3 (ipykernel)",
+   "display_name": "erp",
    "language": "python",
    "name": "python3"
   },

requirements.txt

@@ -3,8 +3,4 @@ langchain-community
 langchain-openai
 pydantic
 flask
-python-dotenv
-pypdf
-pypandoc
-Spire.Doc
-plum-dispatch==1.7.4
+python-dotenv

run.py

@@ -4,4 +4,4 @@ app = create_app()
 
 
 if __name__ == '__main__':
-    app.run(debug=True, port=5401)
+    app.run(host='0.0.0.0', port=5402, debug=True)

scripts/assessment_data.py

@@ -0,0 +1,70 @@
+import pandas as pd
+
+def generate_summary_stats_v2(file_path):
+    # Load the DataFrame from the provided file path
+    df = pd.read_csv(file_path)
+    
+    # Ensure date columns are correctly parsed
+    df['Assessment_Start_Date'] = pd.to_datetime(df['Assessment_Start_Date'])
+    df['Assessment_End_Date'] = pd.to_datetime(df['Assessment_End_Date'])
+    
+    # Add completion rate calculation
+    completed_status = df['Assessment_Status'] == 'Completed'
+    completion_rate_by_frequency = df[completed_status].groupby('Assessment_Frequency').size() / df.groupby('Assessment_Frequency').size()
+    
+    in_progress_status = df['Assessment_Status'] == 'In Progress'
+    incomplete_status = df['Assessment_Status'] == 'Incomplete'
+    
+    # Calculate in-progress and incomplete rates by frequency
+    in_progress_rate_by_frequency = df[in_progress_status].groupby('Assessment_Frequency').size() / df.groupby('Assessment_Frequency').size()
+    incomplete_rate_by_frequency = df[incomplete_status].groupby('Assessment_Frequency').size() / df.groupby('Assessment_Frequency').size()
+    
+    # Fill NaN values (where no assessments are in-progress or incomplete for certain frequencies)
+    completion_rate_by_frequency = completion_rate_by_frequency.fillna(0)
+    in_progress_rate_by_frequency = in_progress_rate_by_frequency.fillna(0)
+    incomplete_rate_by_frequency = incomplete_rate_by_frequency.fillna(0)
+    
+    # Round all numerical values to 2 decimal places
+    completion_rate_by_frequency = completion_rate_by_frequency.round(2)
+    in_progress_rate_by_frequency = in_progress_rate_by_frequency.round(2)
+    incomplete_rate_by_frequency = incomplete_rate_by_frequency.round(2)
+
+    summary_stats = {
+        'Open Items and Red Flags': {
+            'Total Open Items': round(df['Open_Items'].sum(), 2),
+            'Average Open Items per Assessment': round(df['Open_Items'].mean(), 2),
+            'Total Red Flags': round(df['Red_Flags'].sum(), 2),
+            'Average Red Flags per Assessment': round(df['Red_Flags'].mean(), 2),
+            'Max Red Flags in a Single Assessment': round(df['Red_Flags'].max(), 2),
+            'Most Common Area with Red Flags': df[df['Red_Flags'] > 0]['Assessment_Area'].mode()[0]
+        },
+        'Assessment Frequency': {
+            'Assessment Type Breakdown': df['Assessment_Frequency'].value_counts(normalize=True).round(2).to_dict(),
+            'Average Time Between Assessments': round((df['Assessment_End_Date'] - df['Assessment_Start_Date']).dt.days.mean(), 2),
+            'Average Assessment Duration': round(df['Assessment_End_Date'].sub(df['Assessment_Start_Date']).dt.days.mean(), 2),
+            'Completion Rate by Frequency': completion_rate_by_frequency.to_dict(),
+            'In Progress Rate by Frequency': in_progress_rate_by_frequency.to_dict(),
+            'Incomplete Rate by Frequency': incomplete_rate_by_frequency.to_dict()
+        },
+        'Assessment Start and End Dates': {
+            'Longest Assessment Duration (days)': round(df['Assessment_End_Date'].sub(df['Assessment_Start_Date']).dt.days.max(), 2),
+            'Shortest Assessment Duration (days)': round(df['Assessment_End_Date'].sub(df['Assessment_Start_Date']).dt.days.min(), 2),
+        },
+        'Assessment Areas': {
+            'Most Assessed Area': df['Assessment_Area'].value_counts().idxmax(),
+            'Most Open Items in Area': df.groupby('Assessment_Area')['Open_Items'].sum().idxmax(),
+            'Area with Most Red Flags': df.groupby('Assessment_Area')['Red_Flags'].sum().idxmax()
+        },
+        'Assessment Status': {
+            'Assessment Status Distribution': df['Assessment_Status'].value_counts(normalize=True).round(2).to_dict(),
+            'Incomplete Assessments': round(df[df['Assessment_Status'] == 'Incomplete'].shape[0], 2),
+            'In Progress Assessments': round(df[df['Assessment_Status'] == 'In Progress'].shape[0], 2)
+        },
+        'Assessment Admin': {
+            'Most Frequent Admin': df['Assessment_Admin'].mode()[0],
+            'Admin with Fewest Red Flags': df.groupby('Assessment_Admin')['Red_Flags'].sum().idxmin(),
+            'Admin with Most Open Items': df.groupby('Assessment_Admin')['Open_Items'].mean().idxmax()
+        }
+    }
+    
+    return summary_stats

scripts/run_assessment_prediction_trainer.py

@@ -0,0 +1,45 @@
+import os
+import logging
+from logging.handlers import RotatingFileHandler
+from sklearn.ensemble import RandomForestRegressor
+from sklearn.multioutput import MultiOutputRegressor
+from src.pipeline.data_preprocessor import DataPreprocessor
+from src.pipeline.model_trainer import ModelTrainer
+
+# Set up logging
+handler = RotatingFileHandler('/root/ds_erp_ai/logs/prediction_pipeline.log', maxBytes=100000, backupCount=3)
+logger = logging.getLogger(__name__)
+logger.setLevel(logging.INFO)
+logger.addHandler(handler)
+
+# Example of DataPreprocessor and ModelTrainer classes from the previous steps
+class CompanyModelPipeline:
+    def __init__(self, company_ids, input_base_path):
+        self.company_ids = company_ids
+        self.input_base_path = input_base_path
+
+    def run_pipeline(self):
+        for company_id in self.company_ids:
+            try:
+                # Define paths for the company
+                input_path = os.path.join(self.input_base_path, f'{company_id}_raw_data.csv')
+
+                logger.info(f"Starting preprocessing for company {company_id}.")
+                
+                # Step 1 : Preprocess the data
+                preprocessor = DataPreprocessor(input_path=input_path, company_id=company_id)
+                processed_data_path = preprocessor.run()
+                logger.info(f"Data preprocessing completed for company {company_id}. Processed data saved to {processed_data_path}.")
+
+                # Step 2: Train and save the model
+                model = MultiOutputRegressor(RandomForestRegressor(n_estimators=100, random_state=42))
+                trainer = ModelTrainer(preprocessed_data_path=processed_data_path, company_id=company_id, model=model)
+                model_path, latest_data_path, evaluation_results = trainer.run()
+
+                logger.info(f"Model training and evaluation completed for company {company_id}.")
+                logger.info(f"Model saved to {model_path} and latest data saved to {latest_data_path}.")
+                logger.info(f"Evaluation Results for company {company_id}: {evaluation_results}")
+
+            except Exception as e:
+                logger.error(f"An error occurred while processing company {company_id}: {e}")
+

scripts/statistics_data.py

@@ -0,0 +1,170 @@
+import requests
+import pandas as pd
+from dotenv import load_dotenv
+load_dotenv()
+import os
+
+DATA_KEY = os.getenv("AI_DATA_KEY")
+# Constants for API requests
+URL = "https://erpai.mkdlabs.com/v3/api/custom/erpai/common/get-data-ai"
+HEADERS = {
+    "x-project": DATA_KEY # Replace with your actual key
+}
+
+# JSON bodies for API requests
+def create_json_body(area_type, company_id):
+    return {
+        "type": area_type,
+        "options": {
+            "company_id": company_id
+        }
+    }
+
+# Function to fetch data from the API
+def fetch_data(json_body):
+    json_body["options"]["company_id"] = json_body["options"].get("company_id")  # Ensure company_id is included
+    response = requests.post(URL, headers=HEADERS, json=json_body)
+    response.raise_for_status()  # Raise an error for bad responses
+    return response.json()
+
+def convert_assessment_data_to_dataframe(assessment_data):
+    if not assessment_data or "data" not in assessment_data or not assessment_data["data"]:
+        # Return empty DataFrame with expected columns
+        return pd.DataFrame(columns=[
+            "assessment_id", "assessment_name", "start_date", "open_items_overall",
+            "completed_items_overall", "total_assigned_items_overall", "user_name",
+            "user_total_assigned_items", "user_completed_items", "area", "red_flags"
+        ])
+    
+    df_assessment = []
+    for assessment in assessment_data.get("data", []):
+        assessment_id = assessment.get("assessment_id", "")
+        assessment_name = assessment.get("assessment_name", "")
+        start_date = assessment.get("start_date", "")
+        open_items = assessment.get("open_items", 0)
+        completed_items = assessment.get("completed_items", 0)
+        total_assigned_items = assessment.get("total_assigned_items", 0)
+        red_flags = assessment.get("red_flags", 0)
+
+        for user in assessment.get("user_details", []):
+            user_name = user.get("name", "")
+            user_total_items = user.get("total_assigned_items", 0)
+            user_completed_items = user.get("completed_items", 0)
+            
+            for area in user.get("area_list", []):
+                df_assessment.append({
+                    "assessment_id": assessment_id,
+                    "assessment_name": assessment_name,
+                    "start_date": start_date,
+                    "open_items_overall": open_items,
+                    "completed_items_overall": completed_items,
+                    "total_assigned_items_overall": total_assigned_items,
+                    "user_name": user_name,
+                    "user_total_assigned_items": user_total_items,
+                    "user_completed_items": user_completed_items,
+                    "area": area,
+                    "red_flags": red_flags
+                })
+    return pd.DataFrame(df_assessment)
+
+def generate_summary_statistics(df):
+    if df.empty:
+        return {
+            "total_assessments": 0,
+            "avg_open_items_per_assessment": 0,
+            "avg_completed_items_per_assessment": 0,
+            "avg_total_assigned_items_per_assessment": 0,
+            "avg_red_flags": 0,
+            "total_users": 0,
+            "avg_user_total_assigned_items": 0,
+            "avg_user_completed_items": 0,
+            "completion_rate_per_user": 0,
+            "area_summary": {}
+        }
+
+    total_assessments = df['assessment_id'].nunique()
+    avg_open_items = df.groupby('assessment_id')['open_items_overall'].mean().mean()
+    avg_completed_items = df.groupby('assessment_id')['completed_items_overall'].mean().mean()
+    avg_total_assigned_items = df.groupby('assessment_id')['total_assigned_items_overall'].mean().mean()
+    avg_red_flags = df['red_flags'].mean()
+
+    total_users = df['user_name'].nunique()
+    avg_user_total_items = df.groupby('user_name')['user_total_assigned_items'].mean().mean()
+    avg_user_completed_items = df.groupby('user_name')['user_completed_items'].mean().mean()
+    completion_rate_per_user = (df['user_completed_items'].sum() / df['user_total_assigned_items'].sum()) * 100 if df['user_total_assigned_items'].sum() > 0 else 0
+    
+    area_summary = df['area'].value_counts()
+
+    return {
+        "total_assessments": total_assessments,
+        "avg_open_items_per_assessment": avg_open_items,
+        "avg_completed_items_per_assessment": avg_completed_items,
+        "avg_total_assigned_items_per_assessment": avg_total_assigned_items,
+        "avg_red_flags": avg_red_flags,
+        "total_users": total_users,
+        "avg_user_total_assigned_items": avg_user_total_items,
+        "avg_user_completed_items": avg_user_completed_items,
+        "completion_rate_per_user": completion_rate_per_user,
+        "area_summary": area_summary.to_dict()
+    }
+
+def generate_extended_statistics(df):
+    if df.empty:
+        return {
+            "top_5_efficient_users": {},
+            "bottom_5_least_efficient_users": {},
+            "areas_with_most_uncompleted_items": {}
+        }
+
+    df['user_completion_rate'] = (df['user_completed_items'] / df['user_total_assigned_items']).fillna(0) * 100
+    top_5_efficient_users = df.groupby('user_name')['user_completion_rate'].mean().nlargest(5).to_dict()
+    bottom_5_least_efficient_users = df.groupby('user_name')['user_completion_rate'].mean().nsmallest(5).to_dict()
+
+    df['uncompleted_items'] = df['user_total_assigned_items'] - df['user_completed_items']
+    areas_with_most_uncompleted_items = df.groupby('area')['uncompleted_items'].sum().nlargest(5).to_dict()
+
+    return {
+        "top_5_efficient_users": top_5_efficient_users,
+        "bottom_5_least_efficient_users": bottom_5_least_efficient_users,
+        "areas_with_most_uncompleted_items": areas_with_most_uncompleted_items
+    }
+
+def generate_problematic_area_statistics(df):
+    if df.empty:
+        return pd.DataFrame(columns=["total_open_items", "total_red_flags"])
+
+    total_open_items = df.groupby('name')['open_items'].sum().sort_values(ascending=False)
+    total_red_flags = df.groupby('name')['red_flags'].sum().sort_values(ascending=False)
+
+    return pd.DataFrame({
+        "total_open_items": total_open_items,
+        "total_red_flags": total_red_flags
+    }).fillna(0)
+
+def generate_summary_stats(assessment_data, area_data):
+    # Handle empty or invalid assessment data
+    assessment_df = convert_assessment_data_to_dataframe(assessment_data)
+    
+    # Handle empty or invalid area data
+    problematic_area_df = pd.DataFrame(area_data.get("data", []) if area_data and "data" in area_data else [])
+    
+    summary_stats = generate_summary_statistics(assessment_df)
+    extended_stats = generate_extended_statistics(assessment_df)
+    summary_stats["users(Workers) based stats"] = extended_stats
+    
+    problematic_stats = generate_problematic_area_statistics(problematic_area_df)
+    summary_stats["Area based stats"] = problematic_stats.to_dict(orient='index')
+
+    return summary_stats
+
+
+if __name__ == "__main__":
+    json_body_area = create_json_body("problematic-areas", 106)
+    json_body_assessment = create_json_body("user-stats-by-assessment", 106)
+
+    # Fetching data
+    problematic_areas_data = fetch_data(json_body_area)
+    assessment_data = fetch_data(json_body_assessment)
+
+    example_result = generate_summary_stats(assessment_data, problematic_areas_data)
+    print(example_result)

src/api/routes/chatbot.py

@@ -4,6 +4,8 @@ from werkzeug.utils import secure_filename
 from src.services.chatbot import Chatbot
 from src.utils.utils import delete_all_files_in_directory
 from src.utils.document_loader import load_document  
+from src.services.chatbot import Chatbot
+from src.utils.auth import auth_check
 
 
 # Initialize the Blueprint
@@ -17,6 +19,7 @@ def allowed_file(filename):
     return '.' in filename and filename.rsplit('.', 1)[1].lower() in ALLOWED_EXTENSIONS
 
 @bot.route('/validate_worker_document', methods=['POST'])
+@auth_check
 def validate_worker_document():
     try:
         # Retrieve form data
@@ -59,3 +62,157 @@ def validate_worker_document():
 
     except Exception as e:
         return jsonify({"error": "Internal Server Error", "message": str(e)}), 500
+
+
+@bot.route('/predict_next_n_assessments', methods=['POST'])
+@auth_check
+def predict_next_n_assessments():
+    try:
+        # Retrieve JSON data from the request
+        data = request.get_json()
+        #company_info = data.get('company_info')
+        companyid = data.get('companyid')
+        N = data.get('N')
+
+        if not companyid or N is None:
+            return jsonify({"error": "Missing data", "message": "Company info, company ID, or N value not provided."}), 400
+
+        # Instantiate the chatbot service
+        chatbot = Chatbot()
+
+        # Call the prediction method
+        response = chatbot.predict_next_n_assessment(
+            #company_info=company_info,
+            companyid=companyid,
+            N=N
+        )
+        if not response:
+            return jsonify({"error": "No predictions available", "message": "Prediction process failed."}), 400
+        
+        return jsonify({"predictions": response}), 200
+
+    except Exception as e:
+        return jsonify({"error": "Internal Server Error", "message": str(e)}), 500
+
+
+
+@bot.route('/direct-prompt-bot', methods=['POST'])
+@auth_check
+def use_bot_predict_assessments():
+    try:
+        # Retrieve JSON data from the request
+        data = request.get_json()
+        company_info = data.get('company_info')
+        companyid = data.get('companyid')
+        query = data.get('query')
+
+        if  not companyid or query is None:
+            return jsonify({"error": "Missing data", "message": "company ID, or query value not provided."}), 400
+
+        # Instantiate the chatbot service
+        chatbot = Chatbot()
+
+        # Call the prediction method
+        response = chatbot.predict_based_on_past_assessment(
+            company_info=company_info,
+            companyid=companyid,
+            query=query
+        )
+        if not response:
+            return jsonify({"error": "No predictions available", "message": "Prediction process failed."}), 400
+        
+        return jsonify({"predictions": response}), 200
+
+    except Exception as e:
+        return jsonify({"error": "Internal Server Error", "message": str(e)}), 500
+
+
+@bot.route('/suggest_assessment_frequencies', methods=['POST'])
+@auth_check
+def use_bot_suggest_frequencies():
+    try:
+        # Retrieve JSON data from the request
+        data = request.get_json()
+        
+        # Check if 'sops' is present in the request data
+        sops = data.get('sops')
+        options = data.get('options')
+        if not sops and not options :
+            return jsonify({"error": "Invalid data", "message": "The 'sops' or 'options' field is missing or not formatted correctly."}), 400
+
+        # Instantiate the chatbot service
+        chatbot = Chatbot()
+
+        # Call the prediction method
+        response = chatbot.recommend_assessment_frequencies(sops,options)
+        
+        if not response:
+            return jsonify({"error": "Prediction Failed", "message": "The prediction process returned no results."}), 400
+        
+        # Return the successful response with predictions
+        return jsonify({"response": response}), 200
+
+    except Exception as e:
+        # Log the error (optional, but helpful for debugging)
+       print(f"Error in /suggest_assessment_frequencies: {e}")
+        
+       return jsonify({"error": "Internal Server Error", "message": str(e)}), 500
+
+
+
+@bot.route('/predict_goal_achievment_proba', methods=['POST'])
+@auth_check
+def predict_goal_achievement():
+    try:
+        # Retrieve JSON data from the request
+        data = request.get_json()
+        company_info = data.get('company_info')
+        companyid = data.get('companyid')
+   
+
+        if not company_info or not companyid:
+            return jsonify({"error": "Missing data", "message": "Company info, company ID, or N value not provided."}), 400
+
+        # Instantiate the chatbot service
+        chatbot = Chatbot()
+
+        # Call the prediction method
+        response = chatbot.predict_goal_achievement_probability(
+            company_info=company_info,
+            companyid=companyid
+        )
+        if not response:
+            return jsonify({"error": "No predictions available", "message": "Prediction process failed."}), 400
+        
+        return jsonify({"predictions": response}), 200
+
+    except Exception as e:
+        return jsonify({"error": "Internal Server Error", "message": str(e)}), 500
+
+@bot.route('/suggest_more_areas', methods=['POST'])
+@auth_check
+def suggest_more_areas():
+    try:
+        # Retrieve JSON data from the request
+        data = request.get_json()
+        position = data.get('position')
+        existing_areas = data.get('existing_areas')
+
+        if not position or not existing_areas:
+            return jsonify({"error": "Missing data", "message": "Position or existing areas not provided."}), 400
+
+        # Instantiate the chatbot service
+        chatbot = Chatbot()
+
+        # Call the prediction method
+        response = chatbot.suggest_more_areas(
+            position=position,
+            existing_areas=existing_areas
+        )
+        if not response:
+            return jsonify({"error": "No predictions available", "message": "Prediction process failed."}), 400
+
+        return jsonify({"areas": response}), 200
+
+    except Exception as e:
+        return jsonify({"error": "Internal Server Error", "message": str(e)}), 500

src/api/routes/questions.py

@@ -4,11 +4,12 @@ from flask import Blueprint, request, jsonify, current_app
 from werkzeug.utils import secure_filename
 from src.services.sop_generator import (SopPersonalAssessment,SopGeneratorExecutive)
 from src.services.sop_document_parser import DocumentParser
-from src.services.questions_generator import QuestionsGenerator
+from src.services.questions_generator import QuestionsGenerator,QuestionsGeneratorV2
 
 from src.utils.utils import delete_all_files_in_directory
 from src.utils.document_loader import load_document  
 import json
+from src.utils.auth import auth_check
 # Initialize the Blueprint
 qs_b = Blueprint('questions', __name__)
 
@@ -22,46 +23,75 @@ def allowed_file(filename):
     return '.' in filename and filename.rsplit('.', 1)[1].lower() in ALLOWED_EXTENSIONS
 
 
+
 @qs_b.route('/generate_questions_from_sop', methods=['POST'])
-def generate_questions_from_sop():
-    # Check if the request contains data
+@auth_check
+def generate_questions_from_sop_v2():
     if not request.is_json:
         return jsonify({"error": "Invalid input", "message": "Input data must be in JSON format."}), 400
 
-    # Parse the incoming JSON data
     input_data = request.get_json()
 
-    # Validate required fields
-    required_fields = ['sops', 'assessment_type', 'frequency_type', 'duration']
+    required_fields = ['sops', 'assessment_type', 'duration']
     for field in required_fields:
         if field not in input_data:
             return jsonify({"error": "Missing data", "message": f"'{field}' is required."}), 400
 
     try:
-        # Extract fields from input_data
-        sops = input_data['sops']
-        assessment_type = input_data['assessment_type']
-        frequency_type = input_data['frequency_type']
-        duration = input_data['duration']
-
         # Prepare the data for the generator
         generator_input = {
-            "sops": json.dumps(sops),  # Convert SOPs to JSON string
-            "assessment_type": assessment_type,
-            "frequency_type": frequency_type,
-            "duration": duration
+            "sops": input_data['sops'],
+            "assessment_type": input_data['assessment_type'],
+            "duration": input_data['duration']
         }
 
-        # Call the generator to create questions
-        generator = QuestionsGenerator()
+        # Generate questions using the QuestionGenerator
+        generator = QuestionsGeneratorV2()
         questions_response = generator.generate_questions(generator_input)
 
-        # Check if the response is valid
         if not questions_response:
             return jsonify({"error": "Question generation failed", "message": "Could not generate questions from the provided data."}), 500
 
-        # Return the generated questions
-        return jsonify({"questions": questions_response}), 200
+        # Convert the Pydantic model to a dictionary and return as JSON
+        return jsonify({"questions": questions_response.dict()}), 200
+
 
     except Exception as e:
-        return jsonify({"error": "Internal Server Error", "message": str(e)}), 500
+        return jsonify({"error": "Internal Server Error", "message": str(e)}), 500
+
+
+@qs_b.route('/generate_questions_from_sop-latest', methods=['POST'])
+@auth_check
+def generate_questions_from_sop_v3():
+    if not request.is_json:
+        return jsonify({"error": "Invalid input", "message": "Input data must be in JSON format."}), 400
+
+    input_data = request.get_json()
+
+    required_fields = ['sops', 'assessment_type', 'duration']
+    for field in required_fields:
+        if field not in input_data:
+            return jsonify({"error": "Missing data", "message": f"'{field}' is required."}), 400
+
+    try:
+        # Prepare the data for the generator
+        generator_input = {
+            "sops": input_data['sops'],
+            "assessment_type": input_data['assessment_type'],
+            "duration": input_data['duration']
+        }
+
+        # Generate questions using the QuestionGenerator
+        generator = QuestionsGeneratorV2()
+        questions_response = generator.generate_questions_for_all(generator_input)
+
+        if not questions_response:
+            return jsonify({"error": "Question generation failed", "message": "Could not generate questions from the provided data."}), 500
+
+        # Convert the Pydantic model to a dictionary and return as JSON
+        return jsonify({"questions": questions_response}), 200
+
+   
+    except Exception as e:
+        return jsonify({"error": "Internal Server Error", "message": str(e)}), 500
+

src/api/routes/sops.py

@@ -3,9 +3,11 @@ from flask import Blueprint, request, jsonify, current_app
 from werkzeug.utils import secure_filename
 from src.services.sop_generator import (SopPersonalAssessment,SopGeneratorExecutive)
 from src.services.sop_document_parser import DocumentParser
+from src.utils.auth import auth_check
 
 from src.utils.utils import delete_all_files_in_directory
 from src.utils.document_loader import load_document  
+from flask import Blueprint, jsonify, request, make_response
 import json
 # Initialize the Blueprint
 sops_bp = Blueprint('sops', __name__)
@@ -19,7 +21,8 @@ def allowed_file(filename):
     """Check if the file has an allowed extension."""
     return '.' in filename and filename.rsplit('.', 1)[1].lower() in ALLOWED_EXTENSIONS
 
-@sops_bp.route('/personal_assessment/get_roles', methods=['POST'])
+@sops_bp.route('/personal_assessment/get_roles_from_doc', methods=['POST'])
+@auth_check
 def get_roles():
     # Check if the post request has the file part
     if 'document' not in request.files:
@@ -62,8 +65,33 @@ def get_roles():
 
 
 
+@sops_bp.route('/personal_assessment/get_roles_from_questionnaire', methods=['POST'])
+@auth_check
+def get_roles_questionnaire():
+    # Check if the post request has the file part
+    questionnaire_data = request.json
+
+    # Validate the required fields in the questionnaire data
+    if not questionnaire_data.get('questionnaire_response'):
+        return jsonify({
+            "error": "Missing required fields",
+            "message": "Please provide questionnaire_response in the request body"
+        }), 400
+    
+    generator = SopPersonalAssessment()
+
+    roles = generator.generate_roles_from_questionnaire(questionnaire_data)
+
+    if not roles:
+        return jsonify({"error": "No roles found", "message": "No roles were extracted from the questionnaire."}), 404
+
+    return jsonify({"roles": roles, "message": "Roles successfully extracted from the questionnaire."}), 200
+
+
+
 
 @sops_bp.route('/personal_assessment/generate_sops_from_doc', methods=['POST'])
+@auth_check
 def generate_sops():
     # Check if the POST request has the file part
     if 'document' not in request.files:
@@ -77,6 +105,16 @@ def generate_sops():
     if file.filename == '':
         return jsonify({"error": "No selected file", "message": "A file was not selected for upload. Please select a valid file."}), 400
 
+    if 'workers_info' not in request.form:
+        return jsonify({"error": "No roles provided", "message": "Please provide roles as a JSON array."}), 400
+    
+    try:
+        roles_data = request.form.get('workers_info')
+        # Manually load roles from the string to JSON
+        roles_data = json.loads(roles_data)
+        
+    except json.JSONDecodeError:
+        return jsonify({"error": "Invalid JSON", "message": "The roles must be a valid JSON array."}), 400
     if file and allowed_file(file.filename):
         filename = secure_filename(file.filename)
         upload_folder = current_app.config['UPLOAD_FOLDER']
@@ -91,7 +129,7 @@ def generate_sops():
             
             # Generate SOPs based on the roles provided
             sop_generator = DocumentParser()
-            sops = sop_generator.extract_sops_from_doc(docs)
+            sops = sop_generator.extract_sops_from_doc(docs,roles_data)
             # Cleanup: Delete all files in the upload directory after processing
             delete_all_files_in_directory(upload_folder)
             if not sops:
@@ -110,6 +148,7 @@ def generate_sops():
 
 
 @sops_bp.route('/personal_assessment/generate_sops_from_questionnaire', methods=['POST'])
+@auth_check
 def generate_sops_from_questionnaire_per():
     """
     Generate SOPs based on the questionnaire data provided in the request body.
@@ -150,37 +189,45 @@ def generate_sops_from_questionnaire_per():
 
 
 
-@sops_bp.route('/personal_assessment/generate_sops_by_roles_and_areas', methods=['POST'])
-def generate_sops_by_roles_and_areas():
-    """
-    Generate SOPs based on the roles, SOP types (will, shall, must), and areas provided in the request body.
-    """
-    try:
-        # Get the roles data from the request body
-        roles = request.json.get('roles', None)
-        sop_generator = SopPersonalAssessment()
+@sops_bp.route('/personal_assessment/generate_sops_by_roles_and_areas', methods=['POST']) 
+@auth_check 
+def generate_sops_by_roles_and_areas(): 
+    """ 
+    Generate SOPs based on the roles, SOP types (will, shall, must), and areas provided in the request body. 
+    """ 
+    try: 
 
-        # Validate the presence of roles data
-        if not roles or not isinstance(roles, list):
-            return jsonify({"error": "Invalid input", "message": "The 'roles' field should be a non-empty list."}), 400
+        # Get the roles and qna data from the request body 
+        roles = request.json.get('roles', None) 
+        qna = request.json.get('qna', None)
+        sop_generator = SopPersonalAssessment() 
+        
+        # Validate the presence of roles and qna data 
+        if not roles or not isinstance(roles, list): 
+            return make_response(jsonify({"error": "Invalid input", "message": "The 'roles' field should be a non-empty list."}), 400) 
+       
+ 
+        # Generate SOPs for all roles at once 
+        sops_response = sop_generator.generate_sops_by_role_and_area(roles=roles,qna=qna) 
+        if not sops_response: 
+            return make_response(jsonify({ 
+                "error": True, 
+                "message": "Error in generating sops, please try again" 
+            }), 200) 
+        # Return the generated SOPs
+        return make_response(jsonify(sops_response), 200)
+ 
+    except Exception as e: 
+        return make_response(jsonify({ 
+            "error": "Processing error", 
+            "message": f"An error occurred while generating SOPs: {str(e)}" 
+        }), 500)
 
-        # Generate SOPs for all roles at once
-        sops_response = sop_generator.generate_sops_by_role_and_area(roles=roles)
-
-        return jsonify({
-            "sops": sops_response,
-            "message": "SOPs successfully generated for all provided roles."
-        }), 200
-
-    except Exception as e:
-        return jsonify({
-            "error": "Processing error",
-            "message": f"An error occurred while generating SOPs: {str(e)}"
-        }), 500
 
 
 
 @sops_bp.route('/executive/generate_sop_mission_from_vision', methods=['POST'])
+@auth_check
 def generate_executive_sops_from_doc():
     """
     Generate SOPs for executives based on a document containing vision and mission.
@@ -255,8 +302,79 @@ def generate_executive_sops_from_doc():
     return jsonify({"error": "File type not allowed", "message": "The uploaded file type is not allowed. Please upload a PDF, DOC, or DOCX file."}), 400
 
 
+@sops_bp.route('/executive/generate_mission_vision_doc', methods=['POST'])
+@auth_check
+def generate_executive_goals_from_doc():
+    """
+    Generate SOPs for executives based on a document containing vision and mission.
+    """
+    # Check if the POST request has the file part and roles
+    if 'document' not in request.files:
+        return jsonify({"error": "No file part", "message": "Please upload a file with the key 'document'."}), 400
+    
+    if 'departments' not in request.form:
+        return jsonify({"error": "department missing", "message": "Please provide departments'."}), 400
+    
+    try:
+        departments = request.form.get('departments')
+        # Manually load roles from the string to JSON
+        departments = json.loads(departments)
+        
+    except json.JSONDecodeError:
+        return jsonify({"error": "Invalid JSON", "message": "The departments must be a valid JSON array."}), 400
+    except ValueError as e:
+        return jsonify({"error": "Invalid roles format", "message": str(e)}), 400
+
+
+    file = request.files['document']
+
+    # If the user does not select a file, the browser may also submit an empty part without filename
+    if file.filename == '':
+        return jsonify({"error": "No selected file", "message": "A file was not selected for upload. Please select a valid file."}), 400
+
+    if file and allowed_file(file.filename):
+        filename = secure_filename(file.filename)
+        upload_folder = current_app.config['UPLOAD_FOLDER']
+        file_path = os.path.join(upload_folder, filename)
+        
+        # Save the file to the upload folder
+        file.save(file_path)
+
+        try:
+            # Use the utility function to generate docs from the file
+            docs = load_document(file_path)
+            
+            sop_doc = DocumentParser()
+            
+            vision_mission = sop_doc.extract_vision_mission2(docs,departments)
+
+            if not vision_mission:
+                return jsonify({"error": "Vision and Mission generation error ", "message": "Error in generating mssion and viso."}), 400
+            
+            # Check if both vision and mission are empty
+            if not vision_mission.get('vision') and not vision_mission.get('mission'):
+                # Cleanup: Delete all files in the upload directory if parsing fails
+                delete_all_files_in_directory(upload_folder)
+                return jsonify({"vision": [], "mission": [], "message": "The document does not contain mission and vision."}), 200
+
+            print(f"Vision and mission: {vision_mission}")
+            vission = vision_mission.get('vision')
+            mission = vision_mission.get('mission')
+            
+            return jsonify({"mission": mission, "vission": vission, "message": "vision and mission generated successfully"}), 200
+
+        except Exception as e:
+            # Cleanup: Delete all files in the upload directory if an error occurs
+            delete_all_files_in_directory(upload_folder)
+            return jsonify({"error": "Processing error", "message": f"An error occurred while processing the document: {str(e)}"}), 500
+
+    return jsonify({"error": "File type not allowed", "message": "The uploaded file type is not allowed. Please upload a PDF, DOC, or DOCX file."}), 400
+
+
+
 
 @sops_bp.route('/executive/generate_sop_managers_doc', methods=['POST'])
+@auth_check
 def generate_sop_managers_doc():
     if 'document' not in request.files:
         return jsonify({"error": "No file part", "message": "Please upload a file with the key 'document'."}), 400
@@ -276,15 +394,19 @@ def generate_sop_managers_doc():
         try:
             docs = load_document(file_path)
             
-            sop_generator = SopGeneratorExecutive()
-            result = sop_generator.generate_sops_for_department_managers(docs)
+            # Load department managers from form data as a JSON string
+            department_managers_json = request.form.get('departments_managers', '[]')
+            department_managers = json.loads(department_managers_json)
+    
+            sop_generator = DocumentParser()
+            result = sop_generator.extract_sops_for_workers_by_department(docs, department_managers)
             
             delete_all_files_in_directory(upload_folder)
             
             if not result:
                 return jsonify({"error": "Processing error", "message": "Failed to generate SOPs for department managers."}), 500
             
-            return jsonify({"sops": result.dict(), "message": "SOPs successfully generated for department managers."}), 200
+            return jsonify({"sops": result, "message": "SOPs successfully generated for department managers."}), 200
 
         except Exception as e:
             delete_all_files_in_directory(upload_folder)
@@ -293,42 +415,69 @@ def generate_sop_managers_doc():
     return jsonify({"error": "File type not allowed", "message": "The uploaded file type is not allowed. Please upload a PDF, DOC, or DOCX file."}), 400
 
 
+@sops_bp.route('/executive/generate_dept_sops_from_questionnaire', methods=['POST'])
+@auth_check
+def generate_sops_dept_from_questionnaire():
+    try:
+        data = request.json
+        questionnaire_data = data.get('questionnaire')
+        
+        sop_generator = SopGeneratorExecutive()
+        result = sop_generator.generate_sops_from_questionnaire2(questionnaire_data)
 
+        if not result:
+            return make_response(jsonify({"error": "Processing error", "message": "Failed to generate SOPs from questionnaire."}), 500)
+
+        return make_response(jsonify({"sops": result, "message": "SOPs successfully generated from questionnaire."}), 200)
+
+    except Exception as e:
+        return make_response(jsonify({"error": "Processing error", "message": f"An error occurred while processing the request: {str(e)}"}), 500)
 
 
 @sops_bp.route('/executive/generate_sops_from_questionnaire', methods=['POST'])
+@auth_check
 def generate_sops_from_questionnaire():
     try:
         data = request.json
         questionnaire_data = data.get('questionnaire')
         executives = data.get('executives', [])
-        managers = data.get('managers', [])
-        departments = data.get('departments', [])
 
-        if not questionnaire_data or not executives or not managers or not departments:
-            return jsonify({"error": "Missing data", "message": "Please provide questionnaire data, executives, managers, and departments."}), 400
+        if not questionnaire_data or not executives:
+            return make_response(jsonify({"error": "Missing data", "message": "Please provide questionnaire data, executives, managers, and departments."}), 400)
 
         sop_generator = SopGeneratorExecutive()
-        result = sop_generator.generate_sops_from_questionnaire(questionnaire_data, executives, managers, departments)
+        result = sop_generator.generate_sops_from_questionnaire(questionnaire_data, executives)
 
         if not result:
-            return jsonify({"error": "Processing error", "message": "Failed to generate SOPs from questionnaire."}), 500
+            return make_response(jsonify({"error": "Processing error", "message": "Failed to generate SOPs from questionnaire."}), 500)
 
-        # Convert Pydantic models to dictionaries
-        serializable_result = {
-            "executive_sops": {exec: sops.dict() for exec, sops in result["executive_sops"].items()},
-            "department_sops": result["department_sops"].dict()
-        }
-
-        return jsonify({"sops": serializable_result, "message": "SOPs successfully generated from questionnaire."}), 200
+        return make_response(jsonify({"sops": result, "message": "SOPs successfully generated from questionnaire."}), 200)
 
     except Exception as e:
-        return jsonify({"error": "Processing error", "message": f"An error occurred while processing the request: {str(e)}"}), 500
-    
+        return make_response(jsonify({"error": "Processing error", "message": f"An error occurred while processing the request: {str(e)}"}), 500)
+
+
+@sops_bp.route('/executive/generate_mission_goals_from_questionnaire', methods=['POST'])
+@auth_check
+def generate_vision_goals_quest():
+    try:
+        data = request.json
+        questionnaire_data = data.get('questionnaire')
+        
+        sop_generator = SopGeneratorExecutive()
+        vision_mission = sop_generator.generate_vision_mission_from_questionnaire(questionnaire_data)
+        vission = vision_mission.get('vision')
+        mission = vision_mission.get('mission')
+        
+        return jsonify({"mission": mission, "vission": vission, "message": "vision and mission generated successfully"}), 200
+
+    except Exception as e:
+        return make_response(jsonify({"error": "Processing error", "message": f"An error occurred while processing the request: {str(e)}"}), 500)
 
 
 
 @sops_bp.route('/executive/get_roles_for_reference_managers', methods=['POST'])
+@auth_check
 def get_roles_for_reference_managers():
     try:
         # Retrieve form data
@@ -369,9 +518,70 @@ def get_roles_for_reference_managers():
     
 
 
+@sops_bp.route('/executive/extract_vision_dept_goals_doc', methods=['POST'])
+@auth_check
+def get_vision_dpt_goals():
+    try:
+        # Check Content-Type
+        if 'multipart/form-data' not in request.content_type:
+            return jsonify({"error": "Invalid Content-Type", "message": "Request must have Content-Type 'multipart/form-data'."}), 415
 
+        # Retrieve uploaded file
+        file = request.files.get('document')
+
+        if not file:
+            return jsonify({"error": "Missing data", "message": "Document not provided."}), 400
+
+        if file.filename == '':
+            return jsonify({"error": "No selected file", "message": "No file selected for upload. Please select a valid file."}), 400
+
+        if file and allowed_file(file.filename):
+            filename = secure_filename(file.filename)
+            upload_folder = current_app.config['UPLOAD_FOLDER']
+            file_path = os.path.join(upload_folder, filename)
+            
+            file.save(file_path)
+            docs = load_document(file_path)
+
+            # Use extractor to extract roles from the document
+            extractor = DocumentParser()
+            dept_goals = extractor.extract_dept_goals(docs=docs)
+
+            if not dept_goals:
+                return jsonify({"error": "Processing error", "message": "Error generating dept goals"}), 404
+
+            return jsonify({"response": dept_goals, "message": "Vision and dept goals successfully generated."}), 200
+
+    except Exception as e:
+        return jsonify({"error": "Processing error", "message": f"An error occurred while processing the request: {str(e)}"}), 500
+
+
+
+@sops_bp.route('/executive/extract_vision_dept_goals-questionnaire', methods=['POST'])
+@auth_check
+def generate_dept_goals_quest():
+    try:
+        data = request.json
+        questionnaire_data = data.get('questionnaire')
+        
+
+        if not questionnaire_data:
+            return make_response(jsonify({"error": "Missing data", "message": "Please provide questionnaire data"}), 400)
+
+        sop_generator = SopGeneratorExecutive()
+        result = sop_generator.generate_dept_goals_from_questionnaire(questionnaire_data)
+
+        if not result:
+            return make_response(jsonify({"error": "Processing error", "message": "Failed to generate SOPs from questionnaire."}), 500)
+
+        return make_response(jsonify({"sops": result, "message": "SOPs successfully generated from questionnaire."}), 200)
+
+    except Exception as e:
+        return make_response(jsonify({"error": "Processing error", "message": f"An error occurred while processing the request: {str(e)}"}), 500)
+    
 
 @sops_bp.route('/manager/get_roles_for_reference_workers', methods=['POST'])
+@auth_check
 def get_roles_for_reference_workers():
     try:
         # Retrieve form data
@@ -411,7 +621,9 @@ def get_roles_for_reference_workers():
         return jsonify({"error": "Processing error", "message": f"An error occurred while processing the request: {str(e)}"}), 500
     
 
+
 @sops_bp.route('/manager/generate_sop_workers_doc', methods=['POST'])
+@auth_check
 def generate_sop_workers_doc():
     try:
         # Check if the document is provided
@@ -462,3 +674,4 @@ def generate_sop_workers_doc():
     except Exception as e:
         delete_all_files_in_directory(upload_folder)
         return jsonify({"error": "Processing error", "message": f"An error occurred while processing the document: {str(e)}"}), 500
+

src/models/bot_response_schema.py

@@ -4,4 +4,43 @@ from typing import List, Dict
 
 class ValidateWorker(BaseModel):
     result:str
-    
+    
+class Result(BaseModel):
+    response:str
+
+class Cases(BaseModel):
+    open_items: int
+    red_flags: int
+
+class AssessmentsFrequency(BaseModel):
+    weekly: Cases
+    biweekly: Cases
+    quarterly: Cases
+
+class AssessmentPrediction(BaseModel):
+    AssessmentN: AssessmentsFrequency
+
+class AssessmentPredictionsResponse(BaseModel):
+    predictions: List[AssessmentPrediction]
+
+
+class AssessmentSuggestion(BaseModel):
+    assessment_type:str
+    duration:int
+    Justification:str
+
+
+
+
+class GoalAchivementPrediction(BaseModel):
+    prediction:str
+    reasoning:str
+
+
+class AchievementPrediction(BaseModel):
+    goal_achievement:GoalAchivementPrediction
+    goal_achievement_new_assessment:str
+
+
+class Areas(BaseModel):
+    areas:List[str]

src/models/questions_response.py

@@ -1,12 +1,37 @@
-from pydantic import BaseModel
+'''from pydantic import BaseModel
 from typing import List, Dict
 
 class Question(BaseModel):
     assigned_to: str
     role: str
     questions: str
+    area_tag:str
 
 class AssementQuestion(BaseModel):
     number: int
-    questions: List[Question]
+    questions: List[Question]'''
 
+
+from pydantic import BaseModel
+from typing import List, Dict
+from typing import Optional
+
+class Question(BaseModel):
+    area_tag: Optional[int]
+    area_name: str
+    assigned_to: int
+    questions: str
+    role: int
+
+class FrequencyQuestions(BaseModel):
+    frequency_number: int
+    items: List[Question]
+
+class Questions(BaseModel):
+    questions: List[FrequencyQuestions]
+
+class AssessmentQuestions(BaseModel):
+    questions: Questions
+
+class AllQuestions(BaseModel):
+    questions : List[Question]

src/models/sop_response_schemas.py

@@ -3,6 +3,7 @@ from typing import List, Optional
 
 
 class Categories(BaseModel):
+
     must: Optional[List[str]] = Field(default_factory=list)
     shall: Optional[List[str]] = Field(default_factory=list)
     will: Optional[List[str]] = Field(default_factory=list)
@@ -11,6 +12,9 @@ class RoleSops(BaseModel):
     role:str
     sops:Categories
 
+class RoleSopssLists(BaseModel):
+    sops:List[RoleSops]
+
 #class RoleSOPs(BaseModel):
 #    sops: SOPs
 class Roles_response(BaseModel):
@@ -27,6 +31,14 @@ class SOPsResponse(BaseModel):
     mission: List[str]
     vission:List[str]
 
+class DeptMisiion(BaseModel):
+    departments: str
+    goals: List[str]
+
+class VisionMissionResponse2(BaseModel):
+    vision: List[str]
+    mission: List[str]
+
 class VisionMissionResponse(BaseModel):
     vision: List[str]
     mission: List[str]
@@ -116,3 +128,10 @@ class RolesComparisonResponse(BaseModel):
 
 
 
+class DeptGoal(BaseModel):
+    name :str
+    goals: List[str]
+class DeptGoalsVisssion(BaseModel):
+    vision: List[str]
+    mission: List[str]
+    department_goals: List[DeptGoal]

src/pipeline/data_preprocessor.py

@@ -0,0 +1,71 @@
+import pandas as pd
+import os
+import logging
+from logging.handlers import RotatingFileHandler
+
+
+class DataPreprocessor:
+    def __init__(self, input_path, company_id):
+        self.input_path = input_path
+        self.output_dir = os.path.join('data', 'processed', 'assessment_prediction', company_id)
+        self.company_id = company_id
+        self.df = None
+
+    def load_data(self):
+        self.df = pd.read_csv(self.input_path)
+
+    def preprocess(self):
+        # Convert 'start_date' and 'end_date' to datetime
+        self.df['start_date'] = pd.to_datetime(self.df['start_date'])
+        self.df['end_date'] = pd.to_datetime(self.df['end_date'])
+
+        # Add duration (in days) by subtracting start_date from end_date
+        self.df['duration'] = (self.df['end_date'] - self.df['start_date']).dt.days
+
+        # Drop the 'start_date' and 'end_date' columns as they are not needed for training
+        self.df.drop(columns=['start_date', 'end_date'], inplace=True)
+
+        # Convert 'assessment_type' to categorical (one-hot encoding)
+        self.df = pd.get_dummies(self.df, columns=['assessment_type'], drop_first=False)
+
+        # Convert boolean columns to 1s and 0s
+        self.df['assessment_type_weekly'] = self.df['assessment_type_weekly'].astype(int)
+        self.df['assessment_type_biweekly'] = self.df['assessment_type_biweekly'].astype(int)
+        self.df['assessment_type_quarterly'] = self.df['assessment_type_quarterly'].astype(int)
+
+        # Function to create lagged features based on assessment type
+        def create_lagged_features(df, col, assessment_col):
+            lagged_col = f"{col}_{assessment_col}_lag_1"
+            df[lagged_col] = df[col].where(df[assessment_col] == 1).shift(1)
+            return df
+
+        # Create lagged features for each assessment type
+        self.df = create_lagged_features(self.df, 'open_items', 'assessment_type_weekly')
+        self.df = create_lagged_features(self.df, 'open_items', 'assessment_type_biweekly')
+        self.df = create_lagged_features(self.df, 'open_items', 'assessment_type_quarterly')
+
+        # Fill NaNs with 0 instead of dropping rows
+        self.df.fillna(0, inplace=True)
+
+        # Create moving averages for each assessment type
+        self.df['open_items_weekly_ma_3'] = self.df['open_items'].where(self.df['assessment_type_weekly'] == 1).rolling(window=3, min_periods=1).mean().fillna(0)
+        self.df['open_items_biweekly_ma_3'] = self.df['open_items'].where(self.df['assessment_type_biweekly'] == 1).rolling(window=3, min_periods=1).mean().fillna(0)
+        self.df['open_items_quarterly_ma_3'] = self.df['open_items'].where(self.df['assessment_type_quarterly'] == 1).rolling(window=3, min_periods=1).mean().fillna(0)
+
+        # Add percentage change in open items
+        self.df['percentage_change_open_items'] = self.df['open_items'].pct_change().fillna(0) * 100
+
+    def save_data(self):
+        os.makedirs(self.output_dir, exist_ok=True)  # Ensure output directory exists
+        output_path = os.path.join(self.output_dir, 'output.csv')
+        self.df.to_csv(output_path, index=False)
+        return output_path
+
+    def run(self):
+        self.load_data()
+        self.preprocess()
+        return self.save_data()
+
+# Example usage:
+# preprocessor = DataPreprocessor(input_path='path_to_raw_data.csv', company_id='company_123')
+# processed_data_path = preprocessor.run()

src/pipeline/inference.py

@@ -0,0 +1,85 @@
+import pandas as pd
+import joblib
+import os
+
+class AssessmentInference:
+    def __init__(self, company_id, num_assessments, model_dir='models'):
+        self.company_id = company_id
+        self.num_assessments = num_assessments
+        self.model_dir = model_dir
+        self.model = None
+        self.latest_data = None
+
+    def load_model(self):
+        # Load the trained model
+        model_path = os.path.join(self.model_dir, 'assessment_prediction', self.company_id, f'{self.company_id}_model.pkl')
+        self.model = joblib.load(model_path)
+        print(f"Model loaded from {model_path}")
+
+    def load_latest_data(self):
+        # Load the latest assessment data
+        latest_data_path = os.path.join(self.model_dir, 'assessment_prediction', self.company_id, f'{self.company_id}_latest_data.csv')
+        self.latest_data = pd.read_csv(latest_data_path)
+        print(f"Latest data loaded from {latest_data_path}")
+
+    def predict_next_assessment(self, current_data, assessment_type):
+        # Update assessment type (weekly, biweekly, quarterly) in the data for prediction
+        current_data['assessment_type_weekly'] = 1 if assessment_type == 'weekly' else 0
+        current_data['assessment_type_biweekly'] = 1 if assessment_type == 'biweekly' else 0
+        current_data['assessment_type_quarterly'] = 1 if assessment_type == 'quarterly' else 0
+
+        # Exclude target variables (open_items, red_flags) from the feature set
+        features = current_data.drop(columns=['open_items', 'red_flags'])
+
+        # Predict the next open items and red flags
+        prediction = self.model.predict(features)
+        open_items_pred, red_flags_pred = prediction[0]
+
+        # Ensure the predictions are integers by rounding
+        open_items_pred = int(round(open_items_pred))
+        red_flags_pred = int(round(red_flags_pred))
+
+        return {
+            'assessment_type': assessment_type,
+            'open_items': open_items_pred,
+            'red_flags': red_flags_pred
+        }
+
+    def predict_next_assessments(self):
+        predictions = []
+        current_data = self.latest_data.copy()
+
+        # Iteratively forecast the next assessments
+        for i in range(self.num_assessments):
+            print(f"\nForecasting assessment {i + 1}/{self.num_assessments}")
+
+            # Predict for weekly, biweekly, and quarterly for the same forecast step
+            weekly_prediction = self.predict_next_assessment(current_data, 'weekly')
+            biweekly_prediction = self.predict_next_assessment(current_data, 'biweekly')
+            quarterly_prediction = self.predict_next_assessment(current_data, 'quarterly')
+
+            # Append predictions for all types in one forecast step
+            predictions.append({
+                'forecast_step': i + 1,
+                'weekly': weekly_prediction,
+                'biweekly': biweekly_prediction,
+                'quarterly': quarterly_prediction
+            })
+
+            # Update the current data with the weekly prediction (or any of the predictions) for the next step
+            current_data['open_items'] = weekly_prediction['open_items']
+            current_data['red_flags'] = weekly_prediction['red_flags']
+
+        return predictions
+
+    def run(self):
+        self.load_model()
+        self.load_latest_data()
+        predictions = self.predict_next_assessments()
+        return predictions
+
+
+# Example usage
+#inference = AssessmentInference(company_id='testid', num_assessments=5)
+#predictions = inference.run()
+#print(predictions)

src/pipeline/model_trainer.py

@@ -0,0 +1,89 @@
+import pandas as pd
+import os
+from sklearn.model_selection import train_test_split
+from sklearn.ensemble import RandomForestRegressor
+from sklearn.multioutput import MultiOutputRegressor
+import joblib
+from sklearn.metrics import mean_absolute_error, mean_squared_error, r2_score
+import logging
+from logging.handlers import RotatingFileHandler
+
+
+handler = RotatingFileHandler('/root/ds_erp_ai/logs/prediction_pipeline.log', maxBytes=100000, backupCount=3)
+logger = logging.getLogger(__name__)
+logger.setLevel(logging.INFO)
+logger.addHandler(handler)
+
+class ModelTrainer:
+    def __init__(self, preprocessed_data_path, company_id, model):
+        self.preprocessed_data_path = preprocessed_data_path
+        self.output_dir = os.path.join('models', 'assessment_prediction', company_id)
+        self.company_id = company_id
+        self.df = None
+        self.model = model  # Model passed as an argument
+        self.X_test = None
+        self.y_test = None
+
+    def load_data(self):
+        self.df = pd.read_csv(self.preprocessed_data_path)
+
+    def train_model(self):
+        # Split data into features (X) and target variables (y)
+        X = self.df.drop(columns=['open_items', 'red_flags'])
+        y = self.df[['open_items', 'red_flags']]  # Multi-target for open items and red flags
+
+        # Split into training and test sets with 10% as test size
+        X_train, self.X_test, y_train, self.y_test = train_test_split(X, y, test_size=0.1, random_state=42, shuffle=False)
+
+
+        # Train the model
+        self.model.fit(X_train, y_train)
+
+        # Save the trained model
+        os.makedirs(self.output_dir, exist_ok=True)
+        model_path = os.path.join(self.output_dir, f'{self.company_id}_model.pkl')
+        joblib.dump(self.model, model_path)
+        print(f"Model saved to {model_path}")
+
+        # Save the latest row (last assessment data) for inference
+        latest_data_path = os.path.join(self.output_dir, f'{self.company_id}_latest_data.csv')
+        self.df.tail(1).to_csv(latest_data_path, index=False)
+        print(f"Latest assessment data saved to {latest_data_path}")
+
+        # Return the model path and latest data path
+        return model_path, latest_data_path
+
+    def evaluate_model(self):
+        # Predict using the test data
+        y_pred = self.model.predict(self.X_test)
+
+        # Calculate evaluation metrics
+        mae = mean_absolute_error(self.y_test, y_pred)
+        mse = mean_squared_error(self.y_test, y_pred)
+        r2 = r2_score(self.y_test, y_pred)
+
+        print("Model Evaluation Metrics:")
+        print(f"Mean Absolute Error (MAE): {mae}")
+        print(f"Mean Squared Error (MSE): {mse}")
+        print(f"R-squared (R²): {r2}")
+
+        # Return evaluation results
+        return {'mae': mae, 'mse': mse, 'r2': r2}
+
+    def run(self):
+        # Load data and train the model
+        self.load_data()
+        model_path, latest_data_path = self.train_model()
+
+        # Evaluate the model immediately after training
+        evaluation_results = self.evaluate_model()
+
+        return model_path, latest_data_path, evaluation_results
+
+# Example usage
+'''model = MultiOutputRegressor(RandomForestRegressor(n_estimators=100, random_state=42))
+trainer = ModelTrainer(preprocessed_data_path=res, company_id='testid', model=model)
+model_path, latest_data_path, evaluation_results = trainer.run()
+print(f"The model was saved at: {model_path}")
+print(f"The latest data was saved at: {latest_data_path}")
+print(f"Evaluation Results: {evaluation_results}")'''

src/prompts/chatbot.py

@@ -1,3 +1,5 @@
+
+
 def validate_worker_prompt() -> str:
    return """
     You are a worker in the company "Validate" where you are asked a specific yes or no question:
@@ -15,4 +17,257 @@ def validate_worker_prompt() -> str:
       result:"validated" 
     }
     """
-    
+
+def predict_based_past_assessment_prompt(query,company_info, summary_stats):
+    # Extract company information from the dictionary
+   
+    
+    # Create the prompt with the provided company info and summary statistics
+    prompt = f"""
+    **Prompt for the Chatbot:**
+
+    **Context:**
+    You are an AI assistant and work as assessment anaylyst, and your primary responsibility is to provide **insights**, **predictions**, and **recommendations** based on the company's past assessment data and organizational structure. You are not allowed to respond to any queries outside of this domain.
+
+    **General Company Information:**
+     Company info is {company_info}
+
+    **Assessment Summary**:
+    The following is a detailed summary of past assessments will be provided. Use this information to provide predictions and recommendations based on trends and data points.
+    
+     Past assessment summary : {summary_stats}
+
+
+    **Instructions:**
+    Use the above information to answer user queries. You should:
+    - Analyze historical data to identify trends and problem areas.
+    - Predict potential outcomes for future assessments based on past performance (e.g., meeting deadlines, reducing red flags).
+    - Provide **actionable recommendations** that can help improve performance in future assessments.
+
+    **User Query**:
+    "{query}"
+
+    **Your Response**:
+    Predict and provide recommendations based on the company’s historical data, focusing on the areas most relevant to the query. Ensure the response is based on past trends and performance issues.
+
+    **Examples of Insightful Responses**:
+    - "To improve your performance in the next assessment, you should focus on reducing red flags in the Communication department, as it has had the most issues."
+    - "Based on the company's past performance, there is a 70% chance that you will meet the deadline for the next weekly assessment. To ensure success, focus on completing open items in the IT department."
+    - "The data indicates that quarterly assessments have the highest rate of incomplete tasks. I recommend prioritizing quarterly assessment tasks to avoid falling behind."
+    """
+    
+    return prompt
+
+
+
+def predict_next_n_assessments_prompt():
+    
+    # Create the prompt with provided company info, summary statistics, and number of assessments (n)
+    prompt = """
+    **Prompt for the Chatbot:**
+
+    **Context:**
+    You are an AI assistant responsible for thoroughly analyzing the past assessment data of a company. Your primary responsibility is to provide **dynamic predictions** for the next {n} assessments, considering different scenarios based on the frequency of assessments: **weekly**, **bi-weekly**, or **quarterly**. Use the company's past performance to predict the following for each of the next {n} assessments:
+    - **Number of Open Items**.
+    - **Number of Red Flags**.
+    - **Predictions for Weekly, Bi-Weekly, and Quarterly assessments**.
+
+    **Input:**
+    - Company basic info
+    - Past assessment statistics
+    - N - number of next assessments to be predicted 
+
+    **General Company Information:**
+    
+
+    **Assessment Summary (Past Data)**:
+    Detailed information on past assessments will be provided. Use this information to make informed predictions for the next {n} assessments, taking into account the frequency of assessments.
+
+    **Instructions**:
+    - Analyze the historical data to predict the number of open items and red flags for the next n assessments based on whether they are conducted weekly, bi-weekly, or quarterly.
+    - Consider the implications of each frequency on the predictions, and ensure your analysis reflects the potential outcomes for each scenario.
+    - Return the response in the following JSON format:
+
+    **Response Format**:
+
+    {
+      "assessment_1": [
+        {
+          "weekly": {"open_items": X, "red_flags": Y},
+          "biweekly": {"open_items": X, "red_flags": Y},
+          "quarterly": {"open_items": X, "red_flags": Y}
+        }
+      ],
+      "assessment_2": [
+        {
+          "weekly": {"open_items": X, "red_flags": Y},
+          "biweekly": {"open_items": X, "red_flags": Y},
+          "quarterly": {"open_items": X, "red_flags": Y}
+        }
+      ]
+      // assuming N is 2
+    }
+    ```
+    Ensure each assessment is provided with three predictions: one for Weekly, one for Bi-Weekly, and one for Quarterly assessments, and analyze the data dynamically to reflect the different scenarios.
+    """
+    
+    return prompt
+
+
+def predict_next_n_assessments_prompt_v2():
+    prompt = """
+    You are an AI analyst specializing in company  team assessments. Using the provided summary statistics, predict the next N assessments.
+
+    **Input:**
+    1. Summary Statistics containing:
+        - Total assessments completed
+        - Average open/completed items per assessment
+        - User efficiency metrics
+        - Area-based statistics
+        - Red flag distributions
+        - Team completion rates
+    2. N: number of future assessments to predict
+
+    **Validation Rules:**
+    1. Ensure summary statistics are complete and consistent
+    2. Check number of historical assessments:
+        - If only 1 assessment: Mark as "invalid" with reason explaining need for more data
+        - If 2+ assessments: Proceed with predictions
+    3. Verify that area-based stats align with overall metrics
+
+    **Analysis Instructions:**
+    1. Use the average metrics as baseline predictions
+    2. Adjust predictions based on:
+        - Current completion rates per user
+        - Area-specific trends in open items and red flags
+        - Distribution of work across different areas
+        - Historical red flag patterns
+    3. Consider frequency impacts:
+        - Weekly: More frequent checks = faster issue detection and resolution
+        - Biweekly: Moderate item accumulation
+        - Quarterly: Longer accumulation period but more time for resolution
+
+    **Output Format:**
+    {
+        "state": "valid" | "invalid",
+        "reason": string | null,  // Required if state is "invalid"
+        "predictions": [
+            {
+                "assessment_1": {
+                    "weekly": {
+                        "open_items": number,
+                        "red_flags": number
+                    },
+                    "biweekly": {
+                        "open_items": number,
+                        "red_flags": number
+                    },
+                    "quarterly": {
+                        "open_items": number,
+                        "red_flags": number
+                    }
+                }
+            },
+            // ... N predictions
+        ]
+    }
+
+    NOTE: Do not add extra anything also any ``JSON`` formatter tags. Only output JSON.
+
+    Use the following guidelines for predictions:
+    1. Weekly predictions should be based on:
+       - avg_open_items_per_assessment
+       - completion_rate_per_user
+       - current area-based distribution of open items
+
+    2. Biweekly predictions should consider:
+       - Accumulation over two weeks
+       - Historical completion patterns
+       - Area-specific red flag frequencies
+
+    3. Quarterly predictions should account for:
+       - Longer-term accumulation patterns
+       - Team efficiency metrics
+       - Historical area-based completion rates
+
+    Make predictions that reflect realistic patterns based on the team's performance metrics and area-specific challenges.
+    """
+    return prompt
+
+
+
+
+def recommend_assessment_frequency_prompt():
+    return '''
+    You are provided with the Standard Operating Procedures (SOPs) for various roles within a company, along with options (e.g., ['weekly', 'monthly', 'biweekly']) for how frequently assessments should be performed. Your task is to recommend the best assessment type and frequency (weekly, biweekly, or quarterly) for all employees, based on the overall nature of the SOPs.
+
+    When making your recommendation, consider the following:
+    - The complexity and criticality of the tasks outlined in the SOPs.
+    - The frequency of updates or changes to the roles or procedures.
+    - The urgency of the responsibilities and their potential impact on the company's performance.
+    - The balance between the need for regular evaluations and avoiding employee overload.
+    - The need for continuous monitoring of key goals, such as compliance, safety, or mission-critical objectives.
+   
+    Provide a concise and collective assessment recommendation including:
+    - Assessment Type: Select from the provided options (e.g., if the options are ['weekly', 'monthly'], choose the most appropriate based on your analysis of the SOP).
+    - Duration: An integer indicating how long the assessment cycle should last (e.g., if you recommend weekly assessments for two weeks, duration = 2).
+    - Justification: Explain the rationale for choosing the recommended assessment type and frequency, based on the overall content of the SOPs.
+    
+    Note: Predict the duration properly based the assessment type pick and the sop analysis
+    Example Response:
+    "Based on the complexity and critical nature of the tasks described in the SOPs, it is recommended that assessments be conducted biweekly for 6 weeks. This frequency will allow for regular progress reviews without overwhelming employees, while ensuring that critical objectives such as compliance and security are met."
+    '''
+
+
+
+def predict_goal_achievement_probability_prompt():
+    
+    return  '''
+
+    You are provided with summary stats of past assessments and general basic information about the company.
+
+    Your goal is to predict two things after analyzing the summary statistics and the company info:
+
+    1. Goal achievement probability: Predict if the company can meet its goals and vision within the original deadline and the reasoning behind it.
+    2. Goal achievement probability for new assessment: Predict whether a worker will meet the deadline, and if not, specify the extension needed in weeks or days to complete the assessments.
+
+    Example response:  --> 
+    {
+    "goal_achievement": {
+        "predictions": "Based on the analysis of previous assessments and current data, it is anticipated that the company may not meet its goals by the original deadline.",
+        "reasoning": "The data indicates that workers are currently falling behind on completing their assessments. With the current rate of progress, it is expected that an extension of approximately 2 weeks beyond the original due date may be required to complete all assessments."
+    },
+    "goal_achievement_new_assessment": "Based on previous assessments, it is anticipated that the workers will not be able to complete the assessments on time. They may require an extension of approximately 2 weeks beyond the due date to complete the assessment."
+    }
+
+    Be dynamic in your response and you do not need to write as exactly as it is the example response , just keep the format but your response after the analysis should be dynamic and professional
+    '''
+def suggest_more_areas_prompt():
+    return '''
+    You are provided a company position and a set of areas where the employee will be performing their tasks.
+
+    Your role is to analyze the position and suggest additional areas in the range 5 - 10 that the employee should be working on to improve their performance.
+
+    Example:
+
+    Position: Data Scientist
+    Existing areas: Development, Testing, Communication
+
+    Suggested areas: [
+        "Data Exploration",
+        "Machine Learning",
+        "Data Visualization",
+        "Reporting"
+    ]
+
+    Your task is to fill in the suggested areas list with relevant areas based on the position and existing areas.
+
+    Please provide the suggested areas in the following format:
+
+    Suggested_areas: [
+        "Area 1",
+        "Area 2",
+        ...,
+        "Area N"
+    ]
+    '''

src/prompts/questions.py

@@ -6,20 +6,26 @@ def get_questions_prompt():
     Each question will be based on the SOPs of specific workers in different departments, and the questions should vary depending on the assessment frequency type and frequency number.
     The goal of the assessment is to focus on the progress of the tasks outlined in each worker's SOPs.
     Make sure each question is relevant to the worker's SOP, and attach a tag to each question indicating the topic area (e.g., communication, timeline, development).
-    The questions should become more detailed or challenging as the assessment progresses over time.
+    The questions should become more detailed or challenging as the assessment progresses over time and the question is 
     If either the name or role of the assigned person is available in the SOP, use it to formulate the questions.
 
     Input:
     assessment type: (e.g., daily, weekly, biweekly)
     frequency type: (e.g., daily, weekly, biweekly)
-    frequency number: (e.g., day 3, week 2, biweekly 1)
+    frequency number(the current week or frequency e.g if assessment is weekly and frequcny number is 2 , it means week 2): (e.g., day 3, week 2, biweekly 1)
     total duration: (e.g., 6 weeks, 12 days)
     SOPs of the assessment:
+    roles_data e.g [{"position""test position","mame":"name of staff"}]
 
     Instructions:
 
-    1. Review the SOPs of the assessment and generate questions for the workers based on the frequency type, frequency number, and topic areas.
+    1. Review the SOPs of the assessment and generate questions for the workers based on the frequency type, frequency number, and topic areas and roles data provided
     2. Regardless of the assement type, always use 1,2,3 for the frequency numbering, nothing else
+    3. All questions are "yes" or "no" questions nothing extra and precise ,not long
+    4. Generate a total of at least 20  questions all rounda  based on the sops and roles for  each frequency number 
+    5. make sure the questions are up to 20  for the current frequency
+
+     NOTE: !!! MAKE SURE YOU CORRECTLY ATTACH "assigned_to" AS THE ID OF THE MEMBER OF THE ROLE AS STATED IN THE SOP. CHECK MEMBERS UNDER THE ROLE IN THE PROVIDED SOP AND USE THE CORRECT ID OF THE MEMBER, DO NOT USE MEMBER iD THAT IS NOT PROVIDED AS "assigned_to" pls !!!
     Example response:
 
     questions 
@@ -30,24 +36,209 @@ def get_questions_prompt():
                 "frequency_number": "2",
                 "questions": [
                     {
-                        "assigned_to": "person name or role",
-                        "role": "person role or name",
+                        "assigned_to": "id",----id of the member attached to the role, check the member unde the role and attacj the id here 
+                        "role": "person role",
                         "question": "e.g., Is the internal project team being followed according to the SOP?"
+                        "area_tag":"timeline",
+                        "postion":"person position"
                     }
-                ]
+                ]   ## up to at least 20 questions
             },
             {
                 
                 "frequency_number": "3",
                 "questions": [
                     {
-                        "assigned_to": "person name or role",
-                        "role": "person role or name",
-                        "question": "e.g., Have communication protocols been followed for the task at hand?"
-                    }
+                        "assigned_to": "id",----id of the member attached to the role, check the member unde the role and attacj the id here 
+                        "role": "person role",
+                        "question": "e.g., Have communication protocols been followed for the task at hand?".
+                        "area_tag":"communication",
+                        "position":"person position"
+                    }   ## up to at least 20 questions
                 ]
             }
         ]
     }
     """
     return prompt
+
+
+def get_questions_prompt_v2():
+    prompt = """
+    You are tasked with generating assessment questions for workers based on their SOPs. These questions should vary by assessment type (daily, weekly, biweekly), frequency number (e.g., day 2, week 3), and total duration (e.g., 6 weeks).
+
+    You are generating a variable number of assessment questions for different assessment types, which can be daily, weekly, biweekly, etc. The assessment frequency and the specific frequency number should be taken into account when generating the questions.
+    For a particular assessment, based on the type (e.g., daily, weekly, biweekly) and total duration (e.g., if it is weekly and the total duration is 6 weeks, generate at least 20 questions weekly for up to six weeks).
+    Each question will be based on the SOPs of specific workers in different departments, and the questions should vary depending on the assessment frequency type and frequency number.
+    The goal of the assessment is to focus on the progress of the tasks outlined in each worker's SOPs.
+    Make sure each question is relevant to the worker's SOP, and attach an area tag ID, aissgned_to ID, role ID, beacuse each role sop will the provided with area tags and member for that role, so questions should be generated based on that area
+    The questions should become more detailed or challenging as the assessment progresses over time and the question is 
+  
+
+    
+    Guidelines:
+    1. Generate yes/no questions relevant to each worker's SOP based on their role.
+    2. Use frequency numbers as 1, 2, 3, etc., regardless of the type.
+    3. Tag each question with the provided topic area ID (e.g., 1, 2, 3, 4, 5).
+    4. Tag each question with the provided role ID (e.g., 1, 2, 3, 4, 5).
+    5. Tag each question with the provided assigned_to ID (the ID of the person in charge) (e.g., 1, 2, 3, 4, 5).
+    4. The questions should evolve in detail as assessments progress over time.
+    6. For example if ans assement_type is weekly and the duration is 4, this means we generate questions for 4 weeks , frequency_number 1 to  top 4 using the sop and all data 
+    7 For each frequency_number, generate at least 15 - 20 questions.
+
+    Example response:
+
+
+   {
+    "questions": {
+        "questions": [
+        {
+             frequency_number: 1,
+             items: [
+             {
+                    "area_tag": 5,
+                    "assigned_to": 8,
+                    "questions": "Has the content calendar been developed and shared with the team?",
+                    "role": 4
+                },
+                {
+                
+                }
+             ]
+        },
+        {
+        
+        }
+        ]
+
+    """
+    return prompt
+
+
+def get_questions_prompt_v3():
+    prompt = """
+    You are tasked with generating assessment questions for workers based on their SOPs. These questions should vary by assessment type (daily, weekly, biweekly), frequency number (e.g., day 2, week 3), and total duration (e.g., 6 weeks).
+    
+    INPUT: SOPS
+
+    Guidelines:
+    1. Generate yes/no questions relevant to each worker's SOP based on their role.
+    2. Use frequency numbers as 1, 2, 3, etc., regardless of the type.
+    3. Tag each question with the provided topic area ID (e.g., 1, 2, 3, 4, 5).
+    4. Tag each question with the provided role ID (e.g., 1, 2, 3, 4, 5).
+    5. Tag each question with the provided assigned_to ID (the ID of the person in charge) (e.g., 1, 2, 3, 4, 5).
+    6. The questions should evolve in detail as assessments progress over time.
+    7. For each frequency_number, generate at least 15 - 20 questions.
+     NOTE: !!! MAKE SURE YOU CORRECTLY ATTACH "assigned_to" AS THE ID OF THE MEMBER OF THE ROLE AS STATED IN THE SOP. CHECK MEMBERS UNDER THE ROLE IN THE PROVIDED SOP AND USE THE CORRECT ID OF THE MEMBER, DO NOT USE MEMBER iD THAT IS NOT PROVIDED AS "assigned_to" pls !!!
+    Provide the response in the following JSON format:
+    {
+        "questions": {
+            "questions": [
+                {
+                    "frequency_number": 1,
+                    "items": [
+                        {
+                            "area_tag": id of the rea tag,
+                            "assigned_to": "id",----id of the member attached to the role, check the member unde the role and attacj the id here ,
+                            "questions": "Has the content calendar been developed and shared with the team?",
+                            "role": 4"role id" in SOP 
+                        },
+                        ...
+                    ]
+                },
+                ...
+            ]
+        }
+    }
+    """
+    return prompt
+
+
+def get_questions_prompt_v5():
+    prompt = """
+    Comprehensive Assessment Question Generation Methodology
+
+    Objective:
+    Generate a robust, long-term assessment framework for organizational performance evaluation, focusing on systematic and consistent monitoring of operational procedures.
+
+    Strategic Design Principles:
+    1. Question Generation Framework:
+       - Create a holistic set of assessment questions
+       - Designed for sustained use across multiple assessment periods
+       - Provides comprehensive organizational insight
+       - Enables continuous performance tracking and improvement
+
+    2. Structural Requirements:
+       a. Question Attributes:
+          - Binary (Yes/No) response format
+          - Directly mapped to specific organizational roles
+          - Aligned with Standard Operating Procedures (SOPs)
+          - Covers multiple performance dimensions
+
+       b. Taxonomical Tagging:
+          - area_tag: Precise topic/domain identifier
+          - role: Specific organizational role classification
+          - assigned_to: Exact responsible team member identifier
+
+    3. Question Design Criteria:
+       a. Content Depth:
+          - Reveal operational effectiveness
+          - Uncover potential process improvements
+          - Highlight compliance and procedural adherence
+          - Capture nuanced performance indicators
+
+       b. Evaluation Dimensions:
+          - Operational efficiency
+          - Quality control
+          - Safety protocols
+          - Procedural compliance
+          - Resource utilization
+          - Knowledge application
+          - Risk management
+
+    4. Contextual Considerations:
+       - Reflect current organizational structure
+       - Adaptable to evolving operational landscapes
+       - Maintain consistency in assessment approach
+       - Support data-driven decision-making
+       Tagging Requirements:
+       - area_tag: Unique identifier for topic area
+       - role: Corresponding role ID from SOP
+       - assigned_to: Exact ID of specific team member responsible
+    Output Specification:
+    {
+        "questions": {
+            "items": [
+                {
+                    "area_tag": int,          # Domain/Topic Identifier
+                    "area_name":str           # Name of the area     
+                    "assigned_to": str,       # Responsible Member ID
+                    "questions": str,         # Assessment Query
+                    "role": int               # Organizational Role Identifier
+                }
+            ]
+        }
+    }
+
+    Implementation Guidelines:
+    - Develop questions that transcend mere compliance
+    - Encourage reflective and proactive organizational learning
+    - Ensure questions are clear, unambiguous, and actionable
+    - Prioritize questions that drive continuous improvement
+
+    Critical Mandate:
+    Craft questions that are:
+    - Precise and targeted
+    - Aligned with organizational SOPs
+    - Capable of revealing substantive operational insights
+    - Supportive of strategic organizational objectives
+     NOTE: Generate at least 25 questions and can be more depending on the structure of the sop
+     NOTE: !!! MAKE SURE YOU CORRECTLY ATTACH "assigned_to" AS THE ID OF THE MEMBER OF THE ROLE AS STATED IN THE SOP. CHECK MEMBERS UNDER THE ROLE IN THE PROVIDED SOP AND USE THE CORRECT ID OF THE MEMBER, DO NOT USE MEMBER iD THAT IS NOT PROVIDED AS "assigned_to" pls FOLLOW THIS STRICTLY!!!
+     NOTE: CHECK THE "role_id" UNDER THE PROVIDED SOP AND USE THE CORRECT ID OF THE ROLE, DO NOT USE OR FORMULATE "role_id" THAT IS NOT PROVIDED AS "role" pls FOLLOW THIS STRICTLY!!!
+
+     NOTE: IF area tags is not provided for specicfic role SOPS, kindly formulate an area name based on the sop and make the area_tag null but forumlate rea name, only do this if area tags info is not provided for specific role sops
+     NOTE: Use exactly the area names provided if available unless area tags is missing and you need to forumalate one
+
+    """
+    return prompt
+    

src/prompts/sops.py

@@ -1,18 +1,34 @@
 def get_sop_extraction_from_doc():
-    return '''Your task is to extract the "Vision", "Mission", and role-specific Standard Operating Procedures (SOPs) from the provided document.
+    return '''Your task is to generate Standard Operating Procedures (SOPs) for the verified workers based on the provided document and their positions.
     You must extract and categorize the SOPs into three categories: "must", "shall", and "will."
     
     Instructions:
     1. **Vision**: Extract the vision of the company or organization.
     2. **Mission**: Extract the mission of the company or organization. If not explicitly mentioned, consider the mission as the company's goals.
-    3. **Role-specific SOPs**: 
-        - Categorize the questions under three categories: "must," "shall," and "will."
-        - The questions should be directly addressed to the person in the role. Do not reference the role itself in the question.
-        - If SOPs for the role are not explicitly stated, infer them from the context, but only if there is clear evidence within the document. Do not generate or assume SOPs that are not directly supported by the document.
-        - If no SOPs are found for the role, return an empty list for each category.
-    Provide the extracted sections exactly as they appear in the document.'''
+    3. **Position-specific SOPs**: 
+        - Categorize the SOPs under three categories: "must," "shall," and "will."
+        - The SOPs should be directly relevant to the verified workers' positions and responsibilities, ensuring that the position names remain exactly the same as provided.
+        - If multiple workers share the same position, generate a single set of SOPs for that position., Do not repeat it for the same position
+        - If SOPs for the position are not explicitly stated, infer them from the context, but only if there is clear evidence within the document. Do not generate or assume SOPs that are not directly supported by the document.
+        - If no SOPs are found for the position, return an empty list for each category.
+    Use the provided workers' information to ensure the SOPs are tailored to the verified workers.'''
 
 
+def get_roles_extraction_from_questionnaire():
+    return '''Your task is to extract the "Roles" from the provided questionnaire responses.
+    You must identify and categorize the roles based on the information provided.
+    
+    
+    Instructions:
+    1. **Roles**: Extract the roles mentioned in the questionnaire.
+    2. **Vision**: If applicable, extract the vision of the company or organization as it relates to the roles.
+    3. **Mission**: If applicable, extract the mission of the company or organization as it relates to the roles.
+    4. **Role-specific SOPs**: 
+        - Identify any role-specific Standard Operating Procedures (SOPs) mentioned in the questionnaire.
+        - If SOPs for the role are not explicitly stated, infer them from the context, but only if there is clear evidence within the questionnaire. Do not generate or assume SOPs that are not directly supported by the information provided.
+        - If no roles or SOPs are found, return an empty list for each category.
+    Provide the extracted roles and any relevant sections exactly as they appear in the questionnaire.'''
+
 
 def get_sop_personalassessment_from_questionnaire():
     return '''Your task is to generate Standard Operating Procedures (SOPs) based on the responses to the questionnaire provided.
@@ -33,36 +49,78 @@ def get_sop_personalassessment_from_questionnaire():
 
     Provide the generated SOPs based on the questionnaire responses.'''
 
-def get_sop_personalassessment_from_area_role(role,areas,sop_types):
-    return f"""Your job is to generate Standard Operating Procedures (SOPs) for the role of "{role}" with a focus on the areas "{areas}" based on the following instructions:
-
+def get_sop_personalassessment_from_area_role(role,sop_types,qna=None,areas=None):
+    if not areas:
+        areas = "Not provided"
+    return f"""Your job is to generate Standard Operating Procedures (SOPs) for the role of "{role}" with a focus on the prvided area
+           "{areas}" based on the following instructions:
+     
             Instructions:
             Categorization: Organize the SOPs under the selected categories: a checkboxex of the three categories "must" , "shall" and "will"
-            So use the selected sop types categories: {sop_types}
+            So use the selected sop types categories: {sop_types} #adhere strictly to these sop types alone and  make sure they are not missing
             Direct Instructions: The SOPs should directly address responsibilities, objectives, and challenges related to the area of "{areas}" for the role of "{role}".
+            Questions and Anwer context: If extra questions and answers answered based on role is  the provided, use that as more context(related to this role)
             Contextual Inference: If SOPs for the area are not explicitly stated, infer them from the role and area context provided.
-            Empty Lists: If no SOPs are generated, return an empty list for each category.
             Format: The SOPs should be direct and concise.
 
-                                             """
+            provided questions and answer: {qna}
+            INSTRUCTIONS : STRICTLY ADHERE TO THESE INSTRUCTIONS
+            NOTE: IF AREAS ARE NOT PROVIDED (AREA IS "NOT PROVIDED"), INTUITIVELY PROVIDE THE SOP BASED ON THE ROLE NAME.
+            NOTE: MAKE SURE SOPS ARE NOT MISSING FOR THE PROVIDED TYPES.
+            NOTE: FOR SOP TYPES NOT SELECTED RETURN AN EMPTY LIST and not "null"  E.G IF "SHALL" AND "WILL" ARE SELECTED BUT "MUST" IS NOT AMONG, MUST WILL BE AN EMPTY LIST
+
+      
+
+                                                                                       """
+
+
+def get_sop_personalassessment_from_area_rolev2():
+   
+    return f"""Your job is to generate Standard Operating Procedures (SOPs) for the information provided on different roles role of  with a focus on the prvided areas
+         based on the following instructions:
+            Instructions:
+            Categorization: Organize the SOPs under the selected categories: a checkboxex of the three categories "must" , "shall" and "will"
+            So use the selected sop types categories for the specified role : #adhere strictly to these sop types alone and  make sure they are not missing
+            Direct Instructions: The SOPs should directly address responsibilities, objectives, and challenges related to the area of specified to the specific role".
+            Questions and Anwer context: If extra questions and answers answered based on roles is  the provided, use that as more context to generate the sop
+            Contextual Inference: If SOPs for the area are not explicitly stated, infer them from the role and area context provided.
+            Format: The SOPs should be direct and concise.(5-7) bullet point per sop type is okay for each role but 3-4 on average is very good 
+            INSTRUCTIONS : STRICTLY ADHERE TO THESE INSTRUCTIONS
+            NOTE: IF AREAS ARE NOT PROVIDED (AREA IS "NOT PROVIDED"), INTUITIVELY PROVIDE THE SOP BASED ON THE ROLE NAME.
+            NOTE: MAKE SURE SOPS ARE NOT MISSING FOR THE PROVIDED TYPES.
+            NOTE: FOR SOP TYPES NOT SELECTED RETURN AN EMPTY LIST and not "null"  E.G IF "SHALL" AND "WILL" ARE SELECTED BUT "MUST" IS NOT AMONG, MUST WILL BE AN EMPTY LIST
+
+           NOTE !!!: IF A ROLE POINTS TO A SPECIFIC SOP TYPE (E.G., "SHALL" AND "MUST"), THESE TWO MUST NEVER BE EMPTY FOR THAT ROLE.
+                   : FORMAT: SOPS SHOULD BE CLEAR, DIRECT, AND CONCISE. EACH ROLE SHOULD HAVE 5-7 BULLET POINTS PER SOP TYPE ("WILL," "SHALL," OR "MUST"). FOR COMPLEX ROLES, EACH SOP TYPE MAY HAVE A MAXIMUM OF 7-10 BULLET POINTS, NOT TOTAL ACROSS ALL TYPES, BUT PER SOP TYPE.
+      
+
+                                                                                       """
+
 def get_sop_executive_from_vision_goals(executive):
-    return f"""Your task is to generate Standard Operating Procedures (SOPs) for executives namely:{executive} based on the provided vision and goals/mission.
+    return f"""Your task is to generate Standard Operating Procedures (SOPs) for the executive namely: {executive}, based on the provided vision and goals/mission.
 
     You must extract and categorize the SOPs into three categories: "must," "shall," and "will."
     Instructions:
     1. **Vision**: Use the provided vision to align SOPs with the overall direction of the organization.
     2. **Goals**: Utilize the specified goals to create SOPs that support their achievement.
     3. **Executive-level SOPs**: 
-        - Categorize the SOPs under three categories: "must," "shall," and "will."
+        - Categorize the SOPs under three categories: "must," "shall," and "will."  
         - The SOPs should be directly aligned with executive responsibilities and decision-making.
         - Ensure the SOPs reflect high-level strategic thinking and leadership expectations.
         - If SOPs are not explicitly mentioned, infer them from the context of the vision and goals, but only if there is clear evidence. Do not generate or assume SOPs that are not directly supported by the information provided.
         - If no SOPs can be generated, return an empty list for each category.
+       
+    4. If the executive is specified as "Admin," meaning there is a single executive managing all workers and projects, use the vision and mission to generate SOPs.
+    5. Ensure that the generated SOPs can mention the company name if known from the document for better clarity.
+    6. For any exceutive provided , make sure sops is generated if mission and vision is provided
+    
+    Provide the generated SOPs based on the vision and goals, focusing on executive-level responsibilities and actions"""
 
-    Provide the generated SOPs based on the vision and goals, focusing on executive-level responsibilities and actions."""
 
 def get_vision_mission_extraction_from_doc():
     return """Extract vision and mission statements from the document:
+    
+    You are provided with an organization document , your goal is to extract vision and mission(or goals) from the document
 
     1. Analyze for explicit or implicit statements.
     2. Vision: Long-term aspirations or ideal future state.
@@ -71,10 +129,32 @@ def get_vision_mission_extraction_from_doc():
     5. Infer from context if not explicit.
     6. Format as two lists: vision and mission.
     7. Return empty list if none found for either category.
-
-    Provide extracted or inferred vision and mission statements."""
+    8. If vision and mission is found in the document , extract them as it is ,no changes
+    NOTE: MAKE SURE YOU EXTRACT EVERY INFORMATION FOUND FOR VISION AND GOALS FROM THE DOCUMENT.DO NOT OMIT ANY 
+   PROVIDE EXTRACTED OR INFERRED VISION AND MISSION STATEMENTS."""
 
 
+def get_vision_mission_extraction_from_doc2():
+    return """
+
+    You are provided with a organization document and the departments in the organization and your role is to extract the vision and mission(alo called goals) from the document
+    If the vision and mission are clearly stated in the document
+        - Extract the vision of the organization as they are 
+        - Extract the goals(mission) of each the company based on the provided goals of the department and overall questionairre response
+        
+    if the vision and mission are not clearly stated in the document
+        - Infer the vision and mission from the context of the document
+        -  Analyze for explicit or implicit statements.
+            2. Vision: Long-term aspirations or ideal future state.
+            3. Mission: Organization's purpose, core functions, or primary objectives.
+            4. Include multiple statements if found.
+            5. Infer from context if not explicit.
+            7. Return empty list if none found for either category.
+            8. If vision and mission is found in the document , extract them as it is ,no changes
+        NOTED: if the goal(mission) and vision cant not be found at all, make it empty please
+        NOTE: MAKE SURE YOU EXTRACT EVERY INFORMATION FOUND FOR VISION AND GOALS FROM THE DOCUMENT.DO NOT OMIT ANY
+       """
+        
 
 ''' def get_sop_executive_for_managers():
     return Your task is to extract the "Vision", "Mission", and executive-generated Standard Operating Procedures (SOPs) specifically for managers from the provided document.
@@ -88,9 +168,30 @@ def get_vision_mission_extraction_from_doc():
         - If SOPs for managers are not explicitly stated, infer them from the context provided by the executives, but only if there is clear evidence within the document.
         - Do not generate or assume SOPs that are not directly supported by the document or the executive directives.
         - If no SOPs are found for managers, return an empty list for each category.
+        - add header as the starting statment of the sop depending on who the sop is pointing too e.g "Under the inmsights amd control of the develpopment the admin:"  
     Provide the extracted sections exactly as they appear in the document.
     '''
 
+def get_vision_mission_extraction_from_questionnaire_executive():
+    return """
+    
+    You are provided with an organization's response from a questionnaire, and your role is to extract the vision and mission (also called goals) from the questionnaire response:
+    
+    - Generate the vision(at least one paragraph)of the organization based on the questionnaire and 
+    - Generate the goals (mission) of the company based on the provided departmental goals and overall questionairre response
+    
+    If the vision and mission are not clearly stated in the questionnaire:
+        - Infer the vision and mission from the context of the questionnaire.
+        - Analyze for explicit or implicit statements.
+            1. Vision: Long-term aspirations or ideal future state.
+            2. Mission: Organization's purpose, core functions, or primary objectives.
+            3. Include multiple statements if found.
+            4. Infer from context if not explicitly stated.
+            5. Return an empty list if none are found for either category.
+    
+    NOTE: If the goal and mission of a can not be gotten from the questionaire response, make it empty.
+    NOTE: Ensure you extract every piece of information found for the vision and goals from the questionnaire. DO NOT OMIT ANYTHING.
+    """
 
 
 def get_departments_managers_workers_extraction_prompt():
@@ -192,49 +293,132 @@ def get_sop_for_department_managers():
     '''
 
 
+def get_sop_executive_from_questionnaire():
+    return '''Generate Standard Operating Procedures (SOPs) for specific executives and department managers based on the provided questionnaire responses.
+
+    
+
+    Format the response as a JSON object with the following structure:
+                {
+                    "executives": {
+                        "must": ["SOP 1", "SOP 2", ...],
+                        "shall": ["SOP 1", "SOP 2", ...],
+                        "will": ["SOP 1", "SOP 2", ...]
+                    },
+                    "departments": [
+                        {
+                            "name": "Department Name",
+                            "must": ["SOP 1", "SOP 2", ...],
+                            "shall": ["SOP 1", "SOP 2", ...],
+                            "will": ["SOP 1", "SOP 2", ...]
+                        },
+                        {
+                            "name": "Department Name",
+                            "must": ["SOP 1", "SOP 2", ...],
+                            "shall": ["SOP 1", "SOP 2", ...],
+                            "will": ["SOP 1", "SOP 2", ...]
+                        }
+                    ]
+                }
+
+        Ensure that each specified department has its own set of SOPs and if managers are empty arrays- make sop empty for that department
+    '''
+
 
 def get_sop_executive_from_questionnaire():
     return '''Generate Standard Operating Procedures (SOPs) for specific executives and department managers based on the provided questionnaire responses.
 
     Instructions:
-    1. Use the organizational vision and strategic goals to create overarching SOPs for each executive.
-    2. Use the departmental strategic goals to create specific SOPs for each department's managers.
+    1. Use the organizational vision and strategic goals and department strategic goals to create overarching SOPs for the executives.
+    2. Use the departmental strategic goals and team questionnaire  to create specific SOPs for each department.
     3. Categorize all SOPs into "must," "shall," and "will" categories.
     4. Ensure SOPs are actionable, clear, and directly related to the provided information.
     5. For executives, focus on high-level, strategic SOPs that align with the overall vision and goals.
-    6. For department managers, create department-specific SOPs based on their strategic goals.
-    7. Only generate SOPs for the specified departments.
+    6. For departments, create SOPs based on their department strategic goals and team questionaiire response
 
     Format the response as a JSON object with the following structure:
     {
-        "executives": {
-            "Executive Name 1": {
+        "executives": [{
+             "role":"exceutive role name":
+            "sops": {
                 "must": ["SOP 1", "SOP 2", ...],
                 "shall": ["SOP 1", "SOP 2", ...],
                 "will": ["SOP 1", "SOP 2", ...]
             },
-            "Executive Name 2": {
+            {"role":"exceutitve role 2 name"
+            "sops": {
                 "must": ["SOP 1", "SOP 2", ...],
                 "shall": ["SOP 1", "SOP 2", ...],
-                "will": ["SOP 1", "SOP 2", ...]
-            },
+                "will": ["SOP 1", "SOP 2", ...]}
+            ],
             ...
         },
         "departments": [
             {
                 "name": "Department Name",
-                "managers": {
+                "managers": [
+                "role":"manager role name",
+                "sops":{
                     "must": ["SOP 1", "SOP 2", ...],
                     "shall": ["SOP 1", "SOP 2", ...],
                     "will": ["SOP 1", "SOP 2", ...]
-                }
+                },
+                {
+                "name": "Department Name 2",
+                "managers": [
+                "role":"manager role name",
+                "sops":{
+                    "must": ["SOP 1", "SOP 2", ...],
+                    "shall": ["SOP 1", "SOP 2", ...],
+                    "will": ["SOP 1", "SOP 2", ...]
+                }]
             },
             ...
         ]
     }
-    Ensure that each specified department has its own set of SOPs.
+    Ensure that each specified department has its own set of SOPs.if managers are not provided or empty array for a department, make sop for that department empty--> do not form managers and create sop
     '''
 
+
+
+
+
+def get_sop_executive_from_questionnaire2():
+    return '''Generate Standard Operating Procedures (SOPs) for specific  department managers based on the provided questionnaire responses.
+
+    Instructions:
+    2. Use the departmental strategic goals and team questionnaire and role bases answers if provided to create specific SOPs for each department.
+    3. Categorize all SOPs into "must," "shall," and "will" categories.
+    4. Ensure SOPs are actionable, clear, and directly related to the provided information.
+    5. For executives, focus on high-level, strategic SOPs that align with the overall vision and goals.
+    6. For departments, create SOPs based on their department strategic goals and team questionaiire response
+    
+    Format the response as a JSON object with the following structure:
+    {"departments": [
+            {
+                "name": "Department Name",
+                "managers": [
+                "role":"manager role name",
+                "sops":{
+                    "must": ["SOP 1", "SOP 2", ...],
+                    "shall": ["SOP 1", "SOP 2", ...],
+                    "will": ["SOP 1", "SOP 2", ...]
+                },
+                {
+                "name": "Department Name 2",
+                "managers": [
+                "role":"manager role name",
+                "sops":{
+                    "must": ["SOP 1", "SOP 2", ...],
+                    "shall": ["SOP 1", "SOP 2", ...],
+                    "will": ["SOP 1", "SOP 2", ...]
+                }]
+            },
+            ...
+        ]
+    }
+    Ensure that each specified department has its own set of SOPs.if managers are not provided or empty array for a department, make sop for that department empty--> do not form managers and create sop
+    '''
 def get_roles_reference_comparison():
     prompt = """
     You are tasked with comparing a list of reference roles with the extracted roles from a document.
@@ -293,3 +477,81 @@ def get_sop_for_department_workers():
         ]
     }
     '''
+
+def get_sop_for_department_managers():
+    return '''Generate SOPs for each manager under the unique department based on the information the managers info provided
+
+    Instructions:
+    1. Focus on the provided department and manager role.
+    2. Categorize SOPs into "must," "shall," and "will."
+    3. SOPs should be actionable and relevant to the manager's duties.
+    4. If no SOPs can be generated, return empty lists for each category.
+    5. Use the provided document and the managers and department information to generate the SOP.
+    6. If the provided document cannot provide SOPs for a specific manager stated, then return an empty list for the SOP for that worker.
+    
+    Example forma
+    {
+        "departments": [
+            {
+                "name": "Department A",
+                "managers": [
+                    {
+                        "name": "manager  A",
+                        "must": ["Conduct weekly meetings"],
+                        "shall": ["Submit monthly reports"],
+                        "will": ["Improve efficiency"]
+                    }
+
+                ]
+            }
+        ]
+    }
+    '''
+
+
+
+def get_dept_vision_mission_extraction_from_doc2():
+    return """
+
+    You are provided with an organizational document and a list of departments within the organization. Your role is to extract the vision of the organization and the goals/mission of each department from the document. Follow these guidelines:
+
+        If the vision and mission are clearly stated in the document:
+
+        Extract the vision of the organization exactly as stated, without modification.
+        Extract the goals/mission of each department exactly as provided in the document, ensuring no information is omitted.
+        If the vision and mission are not clearly stated:
+
+        Infer the vision and mission from the context of the document by analyzing explicit or implicit statements.
+        Vision: Identify the long-term aspirations or ideal future state of the organization.
+        Mission: Identify the organization's purpose, core functions, or primary objectives, including departmental goals where applicable.
+        Additional Instructions:
+
+        If multiple statements for vision or mission are found, include all relevant statements.
+        If vision and mission are entirely absent, return an empty list for the missing category.
+        Ensure all extracted or inferred information is complete and accurately reflects the document's content.
+        Do not omit any relevant details for vision and mission found in the document.
+        Note: If the vision or mission is found in the document, extract it exactly as it appears without making any changes.
+       """
+
+def get_dept_vision_mission_extraction_from_questionaiire():
+        return """
+
+        You are provided with an questionaiire response  within the organization. Your role is to extract the vision of the organization and the goals/mission of each department from the document. Follow these guidelines:
+
+            If the vision and mission are clearly stated in the document:
+
+            Extract the vision of the organization exactly as stated, without modification.
+            Extract the goals/mission of each department exactly as provided in the document, ensuring no information is omitted.
+            If the vision and mission are not clearly stated:
+
+            Infer the vision and mission from the context of the document by analyzing explicit or implicit statements.
+            Vision: Identify the long-term aspirations or ideal future state of the organization.
+            Mission: Identify the organization's purpose, core functions, or primary objectives, including departmental goals where applicable.
+            Additional Instructions:
+
+            If multiple statements for vision or mission are found, include all relevant statements.
+            If vision and mission are entirely absent, return an empty list for the missing category.
+            Ensure all extracted or inferred information is complete and accurately reflects the respone's content.
+            Do not omit any relevant details for vision and mission found in the document.
+          
+        """

src/services/chatbot.py

@@ -7,10 +7,72 @@ from src.prompts.sops import *
 from src.prompts.chatbot import *
 from src.models.sop_response_schemas import *
 from src.models.bot_response_schema import *
+from scripts.assessment_data import generate_summary_stats_v2
 from dotenv import load_dotenv
+from scripts.statistics_data import (generate_summary_stats,create_json_body,fetch_data)
 load_dotenv()
 
+import random
+import json
+from datetime import datetime, timedelta
+import pandas as pd
 
+# Define possible areas
+areas = [
+    "Agile Methodologies", "API Development", "Code Review", "Collaboration with Cross-Functional Teams",
+    "Continuous Integration/Continuous Deployment (CI/CD)", "Debugging Techniques", "Documentation",
+    "Performance Optimization", "System Design", "Unit Testing", "Version Control"
+]
+
+# Generate random dates
+def random_date():
+    start_date = datetime(2024, 1, 1)
+    return (start_date + timedelta(days=random.randint(1, 300))).strftime('%Y-%m-%dT00:00:00.000Z')
+
+# Generate dummy assessments
+def generate_dummy_data(num_assessments=50, max_users_per_assessment=5):
+    data = []
+    for i in range(1, num_assessments + 1):
+        assessment_id = str(i)
+        assessment_name = f"Assessment {i}"
+        start_date = random_date()
+        red_flags = random.randint(0, 5)
+        open_items = random.randint(0, 50)
+        completed_items = random.randint(0, 50)
+        total_assigned_items = open_items + completed_items
+
+        # Generate random users for each assessment
+        user_details = []
+        num_users = random.randint(1, max_users_per_assessment)
+        for _ in range(num_users):
+            user_name = f"User_{random.randint(1, 100)}"
+            user_total_items = random.randint(1, 50)
+            user_completed_items = random.randint(0, user_total_items)
+            area_list = random.sample(areas, random.randint(1, 5))
+
+            user_details.append({
+                "name": user_name,
+                "total_assigned_items": user_total_items,
+                "completed_items": user_completed_items,
+                "area_list": area_list
+            })
+
+        data.append({
+            "assessment_id": assessment_id,
+            "red_flags": red_flags,
+            "open_items": open_items,
+            "completed_items": completed_items,
+            "total_assigned_items": total_assigned_items,
+            "assessment_name": assessment_name,
+            "start_date": start_date,
+            "user_details": user_details
+        })
+
+    return {"error": False, "data": data}
+
+# Generate dummy data
+dummy_data = generate_dummy_data(num_assessments=100, max_users_per_assessment=5)
+#print(dummy_data)
 #SopGeneratorDocument
 class Chatbot:
     def __init__(self):
@@ -52,7 +114,7 @@ class Chatbot:
                     }
                 ],
                 response_format=ValidateWorker,
-                max_tokens=4096,
+                max_tokens=1024,
                 temperature=0.1
             )
 
@@ -64,3 +126,281 @@ class Chatbot:
         except Exception as e:
             print(f"An error occurred: {e}")
             return None
+        
+
+    def suggest_more_areas(self, position, existing_areas) -> VisionMissionResponse:
+        """
+        This method is responsible for suggesting more areas based on the worker's position and existing areas.
+        The system generates a prompt, and then uses the GPT-4 model to return more areas based on the query.
+
+        :param position: The worker's position.
+        :param existing areas: The existing areas.
+        :return: VisionMissionResponse containing the suggested areas or None if an error occurs.
+        """
+        try:
+        
+            prompt = suggest_more_areas_prompt()
+            response = self.client.beta.chat.completions.parse(
+                model=self.model,
+                messages=[
+                    {
+                        "role": "system",
+                        "content": f'''{prompt} '''
+                    },
+                    {
+                        "role": "user",
+                        "content": f"position :{position}",
+                    },
+                    {
+                        "role": "user",
+                        "content": f"existing areas :{existing_areas}",
+                    }
+                ],
+                response_format=Areas,
+                max_tokens=1024,
+                temperature=0.1
+            )
+
+            # Parse the response from the LLM
+            extracted_text = json.loads(response.choices[0].message.content)
+
+            return extracted_text
+
+        except Exception as e:
+            print(f"An error occurred: {e}")
+            return None
+
+
+
+    def predict_based_on_past_assessment(self, query, company_info, companyid) -> Result:
+        """
+        This method generates predictions based on past assessment data of a company. It queries the backend for the 
+        company's assessment data, generates a prompt, and then uses the GPT-4 model to return predictions based on the query.
+
+        :param query: The question or query asked by the user.
+        :param company_info: General information about the company (name, size, departments, etc.).
+        :param companyid: Unique identifier of the company to fetch its specific data.
+        :return: Result containing the prediction result or None if an error occurs.
+        """
+        try:
+            # Define the path to the company's assessment data (stored as a CSV)
+            json_body_area = create_json_body("problematic-areas", companyid)
+            json_body_assessment = create_json_body("user-stats-by-assessment", companyid)
+            # Fetching data
+            problematic_areas_data = fetch_data(json_body_area)
+            assessment_data = fetch_data(json_body_assessment)
+
+            summary_stats = generate_summary_stats(assessment_data, problematic_areas_data)
+            print(summary_stats)
+           
+
+            # Generate the prompt using the company info and the summary statistics
+            prompt = predict_based_past_assessment_prompt(
+                query=query,
+                company_info=company_info,
+                summary_stats=summary_stats
+            )
+   
+            # Interact with GPT-4 model to get a response
+            response = self.client.beta.chat.completions.parse(
+                model=self.model,
+                messages=[
+                    {
+                        "role": "system",
+                        "content": f"{prompt}"
+                    },
+                    {
+                        "role": "user",
+                        "content": f"{query}",
+                    }
+                ],
+                response_format=Result,
+                max_tokens=1024,
+                temperature=0.1
+            )
+
+            # Extract and return the response from the GPT-4 model
+            extracted_text = json.loads(response.choices[0].message.content)
+
+            return extracted_text
+
+        except Exception as e:
+            print(f"An error occurred: {e}")
+            return None
+
+    
+    def predict_next_n_assessment(self,companyid, N) -> AssessmentPredictionsResponse:
+        """
+        This method generates predictions based on past assessment data of a company. It queries the backend for the 
+        company's assessment data, generates a prompt, and then uses the GPT-4 model to return predictions based on the query.
+
+        :param query: The question or query asked by the user.
+        :param company_info: General information about the company (name, size, departments, etc.).
+        :param companyid: Unique identifier of the company to fetch its specific data.
+        :param N: Number of assessments to predict.
+        :return: Result containing the prediction result or None if an error occurs.
+        """
+        try:
+
+            json_body_area = create_json_body("problematic-areas", companyid)
+            json_body_assessment = create_json_body("user-stats-by-assessment", companyid)
+            # Fetching data
+            problematic_areas_data = fetch_data(json_body_area)
+            assessment_data = fetch_data(json_body_assessment)
+
+            summary_stats = generate_summary_stats(assessment_data, problematic_areas_data)
+            print(summary_stats)
+            # Define the path to the company's assessment data (stored as a CSV)
+            #data_path = os.path.join('data', 'raw', 'erp_company_assessment', f'{companyid}_raw_data.csv')
+            
+            # Generate summary statistics from the company's assessment data
+            #summary_stats = generate_summary_stats_v2(file_path=data_path)
+
+            # Generate the prompt using the company info and the summary statistics
+            prompt = predict_next_n_assessments_prompt_v2()
+          
+            
+            # Interact with GPT-4 model to get a response
+            MODEL = "gpt-4o"
+            response = self.client.beta.chat.completions.parse(
+                model=MODEL,
+                messages=[
+                    {
+                        "role": "system",
+                        "content": f"{prompt}"
+                    },
+
+                    {
+                        "role": "user",
+                        "content": f"Summary stattistics of company past assessment: {summary_stats}",
+                    },
+                     {
+                        "role": "user",
+                        "content": f"Number of next assessments to predict: {N}",
+                    }
+                ],
+                #response_format=AssessmentPredictionsResponse,
+                max_tokens=1024,
+                temperature=0.1
+            )
+
+            # Extract the response from the GPT-4 model
+            preds = json.loads(response.choices[0].message.content)
+
+            # Initialize dictionary to store assessments with dynamic names
+            #predictions = {}
+
+            # Loop through the predicted assessments and rename them dynamically
+            #for i in range(N):
+                #assessment_key = f"assessment_{i + 1}"
+                #predictions[assessment_key] = extracted_text["predictions"][i]['AssessmentN']
+
+            # Return the dynamically named assessments
+            return preds
+
+        except Exception as e:
+            print(f"An error occurred: {e}")
+            return None
+
+
+    def recommend_assessment_frequencies(self,sops,options) -> AssessmentSuggestion:
+        """
+    
+        Process the SOPs data and return assessment frequencies.
+        
+
+        """
+        try:
+
+            chunk_size = 1000  # Define your chunk size
+        # Convert the 'sops' dictionary values into a single text string for chunking
+            sops_text = '\n'.join([str(item) for sublist in sops.values() for item in sublist])
+            
+            # Break the text into chunks of the defined size
+            docs_text = [sops_text[i:i + chunk_size] for i in range(0, len(sops_text), chunk_size)]
+
+            # Create a list of documents
+            docs = [{"type": "text", "text": text} for text in docs_text]
+            # Generate the prompt using the company info and the summary statistics
+            
+            prompt = recommend_assessment_frequency_prompt()  # Update your prompt to handle managers and workers
+                    
+            response = self.client.beta.chat.completions.parse(
+                model=self.model,
+                messages=[
+                    {"role": "system", "content": prompt},
+                    {"role": "user", "content": f'provided sops {docs}'},
+                    {"role": "user", "content": f'options: {options}'}
+                ],
+                response_format=AssessmentSuggestion,  # Use the updated response schema
+                max_tokens=4096,
+                temperature=0.1
+            )
+            
+            return json.loads(response.choices[0].message.content)
+
+        except Exception as e:
+            print(f"An error occurred: {e}")
+            return None
+
+
+    def predict_goal_achievement_probability(self, company_info, companyid) -> AchievementPrediction:
+        """
+        This method generates predictions based on past assessment data of a company. It queries the backend for the 
+        company's assessment data, generates a prompt, and then uses the GPT-4 model to return predictions based on the query.
+
+        :param company_info: General information about the company (name, size, departments, etc.).
+        :param companyid: Unique identifier of the company to fetch its specific data.
+        :return: Result containing the prediction result or None if an error occurs.
+        """
+        try:
+            # Define the path to the company's assessment data (stored as a CSV)
+            #data_path = os.path.join('data', 'raw', 'erp_company_assessment', f'{companyid}_raw_data.csv')
+            
+            # Generate summary statistics from the company's assessment data
+            #summary_stats = generate_summary_stats_v2(file_path=data_path)
+            json_body_area = create_json_body("problematic-areas", companyid)
+            json_body_assessment = create_json_body("user-stats-by-assessment", companyid)
+            # Fetching data
+            problematic_areas_data = fetch_data(json_body_area)
+            assessment_data = fetch_data(json_body_assessment)
+
+            summary_stats = generate_summary_stats(assessment_data, problematic_areas_data)
+           
+
+            # Generate the prompt using the company info and the summary statistics
+            prompt = predict_goal_achievement_probability_prompt()
+          
+            # Interact with GPT-4 model to get a response
+            response = self.client.beta.chat.completions.parse(
+                model=self.model,
+                messages=[
+                    {
+                        "role": "system",
+                        "content": prompt
+                    },
+                    {
+                        "role": "user",
+                        "content": f"company info: {company_info}",
+                    },
+                    {
+                        "role": "user",
+                        "content": f"Summary stats of past assement: {summary_stats}",
+                    }
+                ],
+                response_format=AchievementPrediction,
+                max_tokens=1024,
+                temperature=0.1
+            )
+
+            # Extract the response from the GPT-4 model
+            predictions = json.loads(response.choices[0].message.content)
+            # Return the dynamically named assessments
+            return predictions
+
+        except Exception as e:
+            print(f"An error occurred: {e}")
+            return None
+
+
+

src/services/document_parser.py

@@ -59,7 +59,7 @@ class DocumentParser:
 
     def extract_vision_mission(self, docs) -> VisionMissionResponse:
         """
-        Extracts Vision, Mission, and SOPs categorized into 'must,' 'shall,' and 'will' from the document.
+        Extracts Vision, Mission' from the document.
         
         :param docs: The document(s) from which to extract information.
         :return: VisionMissionResponse containing the vision, mission, and role-specific SOPs.
@@ -92,6 +92,42 @@ class DocumentParser:
 
         except:
             return False
+
+    def extract_vision_mission2(self, docs) -> VisionMissionResponse2:
+        """
+        Extracts Vision, Mission, and SOPs categorized into 'must,' 'shall,' and 'will' from the document.
+        
+        :param docs: The document(s) from which to extract information.
+        :return: VisionMissionResponse containing the vision, mission, and role-specific SOPs.
+        """
+
+        try:
+                docs_text = self._extract_text_from_docs(docs)
+                prompt = get_vision_mission_extraction_from_doc2()
+                response = self.client.beta.chat.completions.parse(
+                    model=self.model,
+                    messages=[
+                        {
+                            "role": "system",
+                            "content": f'''{prompt}'''
+                        },
+                        {
+                            "role": "user",
+                            "content": [{"type": "text", "text": text} for text in docs_text],
+                        }
+                    ],
+                    response_format=VisionMissionResponse2,
+                    max_tokens=10000,
+                    temperature=0.1
+                )
+
+                # Parse the response from the LLM
+                extracted_text = json.loads(response.choices[0].message.content)
+
+                return extracted_text
+
+        except:
+            return False
     
     '''def extract_departments_and_managers(self, docs):
         """

src/services/questions_generator.py

@@ -6,23 +6,74 @@ from typing import List, Dict, Optional
 from src.prompts.sops import *
 from src.models.questions_response import *
 from src.services.sop_document_parser import DocumentParser
-from src.prompts.questions import get_questions_prompt
+from src.prompts.questions import *
 from dotenv import load_dotenv
 load_dotenv()
 
+
+
+
 class QuestionsGenerator:
     def __init__(self):
         self.api_key = os.getenv("OPENAI_API_KEY")
         self.client = OpenAI(api_key=self.api_key)
         self.model = "gpt-4o-mini"
     
-    def generate_questions(self, input_data):
+    def generate_questions(self, input_data: Dict) -> AssessmentQuestions:
+        try:
+            sops = input_data['sops']
+            assessment_type = input_data['assessment_type']
+            total_duration = input_data['duration']
+            
+            # Chunk the SOPs into smaller pieces
+            chunk_size = 1000
+            docs_text = [sops[i:i + chunk_size] for i in range(0, len(sops), chunk_size)]
+            docs = [{"type": "text", "text": text} for text in docs_text]
+
+            prompt = get_questions_prompt_v3()
+
+            all_questions = []
+
+            for frequency_number in range(1, total_duration + 1):
+                frequency_label = f"{assessment_type} number : {frequency_number}"
+
+                response = self.client.beta.chat.completions.parse(
+                    model=self.model,
+                    messages=[
+                        {"role": "system", "content": prompt},
+                        {"role": "user", "content": f"The SOPs are provided below."},
+                        {"role": "user", "content": json.dumps(docs)},
+                        {"role": "user", "content": f"Assessment Type: {assessment_type}"},
+                        {"role": "user", "content": f"Current Frequency Number to generate: {frequency_label}"},
+                        {"role": "user", "content": f"Duration: {total_duration}"}
+                    ],
+                    temperature=0.1,
+                    response_format=FrequencyQuestions,  # Ensure you specify the correct format
+                    max_tokens=10000
+                    
+
+                )
+                
+                questions_json = json.loads(response.choices[0].message.content)
+                
+                all_questions.append(questions_json)
+            
+            return AssessmentQuestions(questions=Questions(questions=all_questions))
+        
+        except Exception as e:
+            print(f"An error occurred: {e}")
+            return None
+
+
+    def generate_questions_v2(self, input_data):
         try:
             sops = input_data['sops']
             assessment_type = input_data['assessment_type']
             frequency_type = input_data['frequency_type']
+            frequency_number = input_data['frequency_number']  # Specific week for which to generate questions
             total_duration = input_data['duration']
-            
+            roles_data = input_data["roles_data"]
+
             # Chunk the SOPs into smaller pieces
             chunk_size = 1000  # Define your chunk size
             docs_text = [sops[i:i + chunk_size] for i in range(0, len(sops), chunk_size)]
@@ -31,36 +82,154 @@ class QuestionsGenerator:
 
             prompt = get_questions_prompt()  # Get the questions prompt for the SOP
 
+            # Prepare the frequency label
+            frequency_label = f"{frequency_type} {frequency_number}"  # e.g., week 2
+
+            # Generate questions for the current frequency number only
+            response = self.client.beta.chat.completions.parse(
+                model=self.model,
+                messages=[
+                    {"role": "system", "content": prompt},
+                    {"role": "user", "content": f"The SOPs are provided below."},
+                    {"role": "user", "content": json.dumps(docs)},  # Use the chunked documents
+                    {"role": "user", "content": f"Assessment Type: {assessment_type}"},
+                    {"role": "user", "content": f"Frequency Type: {frequency_type}"},
+                    {"role": "user", "content": f"Current Frequency Number to generate: {frequency_label}"},
+                    {"role": "user", "content": f"Duration: {total_duration}"},
+                    {"role": "user", "content": f"Roles Data: {roles_data}"}
+                ],
+                response_format=AssementQuestion,  # Ensure you specify the correct format
+                max_tokens=10000,
+                temperature=0.1
+            )
+            
+            # Parse and format the response
+            questions = json.loads(response.choices[0].message.content)
+            
+            return {
+                "frequency_number": frequency_label,
+                "questions": questions
+            }
+
+        except json.JSONDecodeError:
+  
+  
+  
+            return False
+
+
+import os
+import json
+import threading
+from openai import OpenAI
+from pydantic import BaseModel, Field
+from typing import List, Dict, Optional
+from src.prompts.sops import *
+from src.models.questions_response import *
+from src.services.sop_document_parser import DocumentParser
+from src.prompts.questions import *
+from dotenv import load_dotenv
+load_dotenv()
+
+class QuestionsGeneratorV2:
+    def __init__(self):
+        self.api_key = os.getenv("OPENAI_API_KEY")
+        self.client = OpenAI(api_key=self.api_key)
+        self.model = "gpt-4o-mini"
+    
+    def _generate_questions_for_frequency(self, frequency_number: int, assessment_type: str, docs: List[Dict], total_duration: int, all_questions: List[Dict]):
+        try:
+            frequency_label = f"{assessment_type} number : {frequency_number}"
+
+            response = self.client.beta.chat.completions.parse(
+                model=self.model,
+                messages=[
+                    {"role": "system", "content": get_questions_prompt_v3()},
+                    {"role": "user", "content": f"The SOPs are provided below."},
+                    {"role": "user", "content": json.dumps(docs)},
+                    {"role": "user", "content": f"Assessment Type: {assessment_type}"},
+                    {"role": "user", "content": f"Current Frequency Number to generate: {frequency_label}"},
+                    {"role": "user", "content": f"Duration: {total_duration}"}
+                ],
+                temperature=0.1,
+                response_format=FrequencyQuestions,  # Ensure you specify the correct format
+                max_tokens=10000
+            )
+
+            questions_json = json.loads(response.choices[0].message.content)
+            
+            # Add the questions to the shared list (using lock to avoid race conditions)
+            all_questions.append(questions_json)
+        except Exception as e:
+            print(f"An error occurred while generating questions for frequency {frequency_number}: {e}")
+
+    def generate_questions(self, input_data: Dict) -> AssessmentQuestions:
+        try:
+            sops = input_data['sops']
+            assessment_type = input_data['assessment_type']
+            total_duration = input_data['duration']
+            
+            # Chunk the SOPs into smaller pieces
+            chunk_size = 1000
+            docs_text = [sops[i:i + chunk_size] for i in range(0, len(sops), chunk_size)]
+            docs = [{"type": "text", "text": text} for text in docs_text]
+
             all_questions = []
 
-            # Iterate through each frequency number (e.g., week 1, week 2, etc.)
-            for frequency_number in range(1, total_duration + 1):
-                frequency_label = f"{frequency_type} {frequency_number}"  # e.g., week 1, daily 3
+            # Create a list to store threads
+            threads = []
 
-                # Generate questions for the current frequency number
-                response = self.client.beta.chat.completions.parse(
-                    model=self.model,
-                    messages=[
-                        {"role": "system", "content": prompt},
-                        {"role": "user", "content": f"The SOPs are provided below."},
-                        {"role": "user", "content": docs},  # Use the chunked documents
-                        {"role": "user", "content": f"Assessment Type: {assessment_type}"},
-                        {"role": "user", "content": f"Frequency Type: {frequency_type}"},
-                        {"role": "user", "content": f"Current Frequency Number to generate : {frequency_label}"},
-                        {"role": "user", "content": f"Duration: {total_duration}"}
-                    ],
-                    response_format=AssementQuestion,  # Use the updated response schema
-                    max_tokens=4096,
-                    temperature=0.1
-                )
-                
-                questions = json.loads(response.choices[0].message.content)
-                all_questions.append({
-                    "frequency_number": frequency_label,
-                    "questions": questions
-                })
-            
-            return all_questions
+            # Create a lock to ensure thread-safe modifications of all_questions
+            lock = threading.Lock()
+
+            # Launch threads for each frequency
+            for frequency_number in range(1, total_duration + 1):
+                thread = threading.Thread(target=self._generate_questions_for_frequency, args=(frequency_number, assessment_type, docs, total_duration, all_questions))
+                threads.append(thread)
+                thread.start()
+
+            # Wait for all threads to complete
+            for thread in threads:
+                thread.join()
+
+            return AssessmentQuestions(questions=Questions(questions=all_questions))
         
-        except json.JSONDecodeError:
-            return False
+        except Exception as e:
+            print(f"An error occurred: {e}")
+            return None
+
+
+    def generate_questions_for_all(self,input_data, all_questions: List[Dict]=[]):
+        try:
+            sops = input_data['sops']
+            assessment_type = input_data['assessment_type']
+            total_duration = input_data['duration']
+            
+            
+            # Chunk the SOPs into smaller pieces
+            chunk_size = 1000
+            docs_text = [sops[i:i + chunk_size] for i in range(0, len(sops), chunk_size)]
+            docs = [{"type": "text", "text": text} for text in docs_text]
+
+
+            response = self.client.beta.chat.completions.parse(
+                model=self.model,
+                messages=[
+                    {"role": "system", "content": get_questions_prompt_v5()},
+                    {"role": "user", "content": f"The SOPs are provided below."},
+                    {"role": "user", "content": json.dumps(docs)},
+                    {"role": "user", "content": f"Assessment Type: {assessment_type}"},
+                    {"role": "user", "content": f"Duration: {total_duration}"}
+                ],
+                temperature=0.1,
+                response_format=AllQuestions,  # Ensure you specify the correct format
+                max_tokens=6000
+            )
+
+            questions_json = json.loads(response.choices[0].message.content)
+            
+            # Add the questions to the shared list (using lock to avoid race conditions
+            return questions_json
+        except Exception as e:
+            print(f"An error occurred while generating questions: {e}")
+            return None

src/services/sop_document_parser.py

@@ -21,7 +21,7 @@ class DocumentParser:
         return [doc.page_content for doc in docs]
     # Existing methods...
 
-    def extract_sops_from_doc(self, docs) -> VisionMissionResponse:
+    def extract_sops_from_doc(self, docs,roles_data) -> SOPsResponse:
         """
         Extracts Vision, Mission, and SOPs categorized into 'must,' 'shall,' and 'will' from the document.
         
@@ -39,6 +39,10 @@ class DocumentParser:
                             "role": "system",
                             "content": f'''{prompt}'''
                         },
+                        {
+                             "role":"user",
+                             "content":f"Verfied workers data:{roles_data}"
+                        },
                         {
                             "role": "user",
                             "content": [{"type": "text", "text": text} for text in docs_text],
@@ -81,7 +85,7 @@ class DocumentParser:
                         }
                     ],
                     response_format=VisionMissionResponse,
-                    max_tokens=4096,
+                    max_tokens=1600,
                     temperature=0.1
                 )
 
@@ -92,7 +96,83 @@ class DocumentParser:
 
         except:
             return False
-    
+        
+    def extract_vision_mission2(self, docs,departments) -> VisionMissionResponse2:
+        """
+        Extracts Vision, Mission, and SOPs categorized into 'must,' 'shall,' and 'will' from the document.
+        
+        :param docs: The document(s) from which to extract information.
+        :return: VisionMissionResponse containing the vision, mission, and role-specific SOPs.
+        """
+
+        try:
+                docs_text = self._extract_text_from_docs(docs)
+                prompt = get_vision_mission_extraction_from_doc2()
+                response = self.client.beta.chat.completions.parse(
+                    model=self.model,
+                    messages=[
+                        {
+                            "role": "system",
+                            "content": f'''{prompt}'''
+                        },
+                         {
+                            "role": "user",
+                            "content": f"Department to consider for the company goals generation{departments}"
+                        },
+                        {
+                            "role": "user",
+                            "content": [{"type": "text", "text": text} for text in docs_text],
+                        }
+                    ],
+                    response_format=VisionMissionResponse2,
+                    max_tokens=10000,
+                    temperature=0.1
+                )
+
+                # Parse the response from the LLM
+                extracted_text = json.loads(response.choices[0].message.content)
+
+                return extracted_text
+
+        except:
+            return False
+
+    def extract_dept_goals(self, docs) -> VisionMissionResponse2:
+        """
+        Extracts Vision, Mission, and SOPs categorized into 'must,' 'shall,' and 'will' from the document.
+        
+        :param docs: The document(s) from which to extract information.
+        :return: VisionMissionResponse containing the vision, mission, and role-specific SOPs.
+        """
+
+        try:
+                docs_text = self._extract_text_from_docs(docs)
+                prompt = get_vision_mission_extraction_from_doc2()
+                response = self.client.beta.chat.completions.parse(
+                    model=self.model,
+                    messages=[
+                        {
+                            "role": "system",
+                            "content": f'''{prompt}'''
+                        },
+                        {
+                            "role": "user",
+                            "content": [{"type": "text", "text": text} for text in docs_text],
+                        }
+                    ],
+                    response_format=DeptGoalsVisssion,
+                    max_tokens=10000,
+                    temperature=0.1
+                )
+
+                # Parse the response from the LLM
+                extracted_text = json.loads(response.choices[0].message.content)
+
+                return extracted_text
+
+        except:
+            return False
+        
     def get_roles(self, docs):
         # Extract the text content from the Document objects
         docs_text = [doc.page_content for doc in docs] 
@@ -116,7 +196,7 @@ class DocumentParser:
             }
         ],
         response_format=Roles_response,
-        max_tokens=1024,
+        max_tokens=4096,
         temperature=0.1
         )
     
@@ -168,7 +248,7 @@ class DocumentParser:
                         {"role": "user", "content": [{"type": "text", "text": text} for text in docs_text]}
                     ],
                     response_format=DepartmentsAndWorkersResponse,  # Use the updated response schema
-                    max_tokens=4096,
+                    max_tokens=16000,
                     temperature=0.1
                 )
                 
@@ -208,7 +288,7 @@ class DocumentParser:
                     {"role": "system", "content": f"The reference roles are{reference_roles} while the extracted roles are {extracted_managers}"},
                     {"role": "user", "content": prompt}
                 ],
-                max_tokens=1024,
+                max_tokens=16000,
                 temperature=0.1,
                 response_format=RolesComparisonResponse
             )
@@ -253,7 +333,7 @@ class DocumentParser:
                     {"role": "system", "content": f"The reference roles are{reference_roles} while the extracted roles are {extracted_workers}"},
                     {"role": "user", "content": prompt}
                 ],
-                max_tokens=1024,
+                max_tokens=16000,
                 temperature=0.1,
                 response_format=RolesComparisonResponse
             )
@@ -284,7 +364,7 @@ class DocumentParser:
                         {"role": "user", "content": [{"type": "text", "text": text} for text in docs_text]}
                     ],
                     response_format=DepartmentMembers,  # Use the updated response schema
-                    max_tokens=4096,
+                    max_tokens=16000,
                     temperature=0.1
                 )
                 
@@ -312,7 +392,36 @@ class DocumentParser:
                         {"role": "user", "content": [{"type": "text", "text": text} for text in docs_text]}
                     ],
                     response_format=WorkerSOPsResponse,  # Use the updated response schema
-                    max_tokens=4096,
+                    max_tokens=16000,
+                    temperature=0.1
+                )
+                
+                return json.loads(response.choices[0].message.content)
+                
+            except json.JSONDecodeError:
+                return False
+
+         
+    def extract_sops_for_managers_by_department(self, docs,depts_managers):
+            """
+            Extract departments, managers, and workers from the document.
+            
+            :param docs: List of document chunks
+            :return: Dictionary containing departments, their managers, and workers.
+            """
+            try:
+                docs_text = self._extract_text_from_docs(docs)
+                prompt = get_sop_for_department_managers()  # Update your prompt to handle managers and workers
+                        
+                response = self.client.beta.chat.completions.parse(
+                    model=self.model,
+                    messages=[
+                        {"role": "system", "content": prompt},
+                        {"role": "user", "content": f"Workers information: {depts_managers}"},
+                        {"role": "user", "content": [{"type": "text", "text": text} for text in docs_text]}
+                    ],
+                    response_format=WorkerSOPsResponse,  # Use the updated response schema
+                    max_tokens=16000,
                     temperature=0.1
                 )
                 

src/services/sop_generator.py

@@ -7,6 +7,13 @@ from src.prompts.sops import *
 from src.models.sop_response_schemas import *
 from src.services.sop_document_parser import DocumentParser
 from dotenv import load_dotenv
+from concurrent.futures import ThreadPoolExecutor, as_completed
+from typing import List, Dict
+import json
+from openai import OpenAI
+import os
+import time
+from collections import deque
 load_dotenv()
 
 
@@ -16,6 +23,9 @@ class SopPersonalAssessment:
         self.api_key = os.getenv("OPENAI_API_KEY")
         self.client = OpenAI(api_key=self.api_key)
         self.model = "gpt-4o-mini"
+        self.rpm_limit = 3  # Requests per minute limit
+        self.batch_size = 3  # Process 3 roles at a time
+        self.retry_delays = [4, 8, 16, 32]
 
     # Existing methods...
 
@@ -59,7 +69,7 @@ class SopPersonalAssessment:
                     }
                 ],
                 response_format=SOPsResponse,
-                max_tokens=2048,
+                max_tokens=16000,
                 temperature=0.1
             )
 
@@ -72,40 +82,76 @@ class SopPersonalAssessment:
             print(f"Error occurred: {str(e)}")
             return False
 
-
-    def generate_sops_by_role_and_area(self, roles: List[dict]) -> RoleSops:
+   
+    def generate_sops_by_role_and_area(self, roles,qna) -> RoleSops:
 
         try:
-            sops_by_role = []
-            for role_info in roles:
-                role = role_info['role']
-                sop_types = role_info['sop_types']  # List of SOP types: ["will", "shall", "must"]
-                areas = role_info['areas']  # List of areas: ["communication", "development", etc.]
-
-
-                prompt = get_sop_personalassessment_from_area_role(role,areas,sop_types)
+       
+            
+                MODEL = "gpt-4o-mini"
+                prompt = get_sop_personalassessment_from_area_rolev2()
                 response = self.client.beta.chat.completions.parse(
-                model=self.model,
+                model=MODEL,
                 messages=[
                     {
                         "role": "system",
                         "content": f'''{prompt}
                                     ''',
+                    },
+                    {
+                        "role": "user",
+                        "content": f'''povided Roles Data {roles}
+                                    ''',
+                    },
+                    {
+                        "role": "user",
+                        "content": f'''Extra question and answers for more context {qna}
+                                    ''',
                     }
                 ],
-                response_format=RoleSops,
-                max_tokens=1024,
+                response_format=RoleSopssLists,
+                max_tokens=6000,
                 temperature=0.1
             )   
                 extracted_text = json.loads(response.choices[0].message.content)
                 # You can customize this to generate SOPs based on the role, SOP types, and areas
-                sops_by_role.append(extracted_text)
-
-            
-
-            return sops_by_role
+               
+                return extracted_text["sops"]
         
-        except:
+        except Exception as e:
+            print(f"Error occurred: {str(e)}")
+            return False
+
+    
+    def generate_roles_from_questionnaire(self, questionnaire_data: List[dict]) -> Roles_response:
+
+        try:
+            # List of areas: ["communication", "development", etc.]
+
+            prompt = get_roles_extraction_from_questionnaire()
+            response = self.client.beta.chat.completions.parse(
+                model=self.model,
+                messages=[
+                    {
+                        "role": "system",
+                        "content": f'''{prompt}''',
+                    },
+                    {
+                        "role": "user",
+                        "content": f'''Questionairre data : {questionnaire_data}''',
+                    }
+                ],
+                response_format=Roles_response,
+                max_tokens=16000,
+                temperature=0.1
+            )   
+            extracted_text = json.loads(response.choices[0].message.content)
+            # You can customize this to generate roles based on the questionnaire data
+
+            return extracted_text
+                
+        except Exception as e:
+            print(f"Error occurred: {str(e)}")
             return False
         
 
@@ -116,7 +162,7 @@ class SopGeneratorExecutive:
         self.client = OpenAI(api_key=self.api_key)
         self.model = "gpt-4o-mini"
 
-    def extract_sops_from_executive_vision_goals_doc(self, data: dict,executives:List) -> SOPsResponse:
+    def extract_sops_from_executive_vision_goals_doc(self, data: dict,executive:str) -> SOPsResponse:
         """
         Extracts SOPs categorized into 'must,' 'shall,' and 'will' based on executive vision and goals.
 
@@ -126,13 +172,15 @@ class SopGeneratorExecutive:
         try:
              
             vision = data.get("vision", "No vision provided")
-            goals = data.get("goals", "No goals provided")
+            goals = data.get("mission", "No goals provided")
             
-            prompt = get_sop_executive_from_vision_goals(executives)
+            prompt = get_sop_executive_from_vision_goals(executive)
 
             user_content = f'''
+            Find the mission and vision provided below
             Vision: {vision}
             Goals: {goals}
+            
             '''
 
             response = self.client.beta.chat.completions.parse(
@@ -148,7 +196,7 @@ class SopGeneratorExecutive:
                     }
                 ],
                 response_format=Categories,
-                max_tokens=2048,
+                max_tokens=16000,
                 temperature=0.1
             )
 
@@ -182,7 +230,7 @@ class SopGeneratorExecutive:
                             {"role": "user", "content": f"Generate SOPs for {role['position']} in {department['name']} department."}
                         ],
                         response_format=ManagerSOPs,
-                        max_tokens=1024,
+                        max_tokens=16000,
                         temperature=0.1
                     )
                     manager_sops = json.loads(response.choices[0].message.content)
@@ -200,56 +248,35 @@ class SopGeneratorExecutive:
             return False
 
 
-    def generate_sops_from_questionnaire(self, questionnaire_data: dict, executives: List[str], managers: List[str], departments: List[str]):
+    def generate_sops_from_questionnaire(self, questionnaire_data: dict, executives: List[str]):
         try:
+            print("Generating SOPs from questionnaire...")
             prompt = get_sop_executive_from_questionnaire()
+            print(f"Prompt: {prompt}")
 
             # Prepare the questionnaire data for the prompt
             user_content = json.dumps(questionnaire_data, indent=2)
+            print(f"User content: {user_content}")
 
             response = self.client.beta.chat.completions.parse(
                 model=self.model,
                 messages=[
                     {"role": "system", "content": prompt},
-                    {"role": "user", "content": f"Generate SOPs based on this questionnaire:\n{user_content}\n\nExecutives to consider: {', '.join(executives)}\nManagers to consider: {', '.join(managers)}\nDepartments to consider: {', '.join(departments)}"}
+                    {"role": "user", "content": f"Generate SOPs based on this questionnaire response:\n{user_content}\n\nExecutives to consider: {', '.join(executives)}"}
                 ],
                 response_format={"type": "json_object"},
-                max_tokens=4096,
+                max_tokens=16000,
                 temperature=0.1
             )
 
+            print("Response received from API.")
             sops_data = json.loads(response.choices[0].message.content)
+            print(f"SOPs data: {sops_data}")
 
             # Process executive SOPs
-            executive_sops = {}
-            for executive in executives:
-                if executive in sops_data['executives']:
-                    executive_sops[executive] = Categories(**sops_data['executives'][executive])
-                else:
-                    executive_sops[executive] = Categories()
-
-            # Process department manager SOPs
-            departments_with_sops = []
-            for dept_name in departments:
-                dept_data = next((d for d in sops_data['departments'] if d['name'].lower() == dept_name.lower()), None)
-                if dept_data:
-                    managers_with_sops = [
-                        ManagerWithSOPs(
-                            title=manager,
-                            sops=ManagerSOPs(**dept_data['managers'])
-                        )
-                        for manager in managers
-                    
-                    ]
-                    if managers_with_sops:
-                        departments_with_sops.append(DepartmentManagerSOPs(
-                            name=dept_name,
-                            managers=managers_with_sops
-                        ))
 
             return {
-                "executive_sops": executive_sops,
-                "department_sops": ExecutiveManagerSOPsResponse(departments=departments_with_sops)
+                "response": sops_data
             }
 
         except Exception as e:
@@ -257,6 +284,108 @@ class SopGeneratorExecutive:
             return False
 
 
+    def generate_vision_mission_from_questionnaire(self, questionnaire_data: dict):
+        """
+        Generate vision and mission based on the questionnaire data provided in the request body.
+        The request body is expected to contain plain-text information for vision and mission."""
+        try:
+            print("Generating vsion and mission from questionnaire...")
+            prompt = get_vision_mission_extraction_from_questionnaire_executive()
+            print(f"Prompt: {prompt}")
+
+            # Prepare the questionnaire data for the prompt
+            user_content = json.dumps(questionnaire_data, indent=2)
+            print(f"User content: {user_content}")
+
+            response = self.client.beta.chat.completions.parse(
+                model=self.model,
+                messages=[
+                    {"role": "system", "content": prompt},
+                    {"role": "user", "content": f"questionnaire response:\n{user_content}"}
+                ],
+                response_format=VisionMissionResponse2,
+                max_tokens=16000,
+                temperature=0.1
+            )
+
+            print("Response received from API.")
+            response  = json.loads(response.choices[0].message.content)
+       
+
+            return response
+
+        except Exception as e:
+            print(f"Error in generate_sops_from_questionnaire: {str(e)}")
+            return False
+
+    
+    def generate_dept_goals_from_questionnaire(self, questionnaire_data):
+        try:
+            print("Generating SOPs from questionnaire...")
+            prompt = get_dept_vision_mission_extraction_from_questionaiire()
+            print(f"Prompt: {prompt}")
+
+            # Prepare the questionnaire data for the prompt
+            user_content = json.dumps(questionnaire_data, indent=2)
+    
+            response = self.client.beta.chat.completions.parse(
+                model=self.model,
+                messages=[
+                    {"role": "system", "content": prompt},
+                    {"role": "user", "content": f"Generate  based on this questionnaire response:\n{user_content}\n"}
+                ],
+                response_format=DeptGoalsVisssion,
+                max_tokens=16000,
+                temperature=0.1
+            )
+
+            print("Response received from API.")
+            sops_data = json.loads(response.choices[0].message.content)
+
+
+            # Process executive SOPs
+
+            return sops_data
+
+        except Exception as e:
+            print(f"Error in generate_sops_from_questionnaire: {str(e)}")
+            return False
+        
+    def generate_sops_from_questionnaire2(self, questionnaire_data: dict):
+        try:
+            print("Generating SOPs from questionnaire...")
+            prompt = get_sop_executive_from_questionnaire2()
+            print(f"Prompt: {prompt}")
+
+            # Prepare the questionnaire data for the prompt
+            user_content = json.dumps(questionnaire_data, indent=2)
+            print(f"User content: {user_content}")
+
+            response = self.client.beta.chat.completions.parse(
+                model=self.model,
+                messages=[
+                    {"role": "system", "content": prompt},
+                    {"role": "user", "content": f"Generate SOPs based on this questionnaire response:\n{user_content}\n"}
+                ],
+                response_format={"type": "json_object"},
+                max_tokens=16000,
+                temperature=0.1
+            )
+
+            print("Response received from API.")
+            sops_data = json.loads(response.choices[0].message.content)
+            print(f"SOPs data: {sops_data}")
+
+            # Process executive SOPs
+
+            return {
+                "response": sops_data
+            }
+
+        except Exception as e:
+            print(f"Error in generate_sops_from_questionnaire: {str(e)}")
+            return False
+
 
 class SopGeneratorManager:
     def __init__(self):

src/utils/auth.py

@@ -0,0 +1,24 @@
+import os
+from functools import wraps
+from flask import Flask, session, redirect, url_for, request, g, jsonify
+from dotenv import load_dotenv
+load_dotenv()
+
+API_KEY = os.getenv("API_KEY")
+
+def auth_check(func):
+    @wraps(func)
+    def decorated_function(*args, **kwargs):
+        auth_header = request.headers.get('Authorization')
+
+        if not auth_header or not auth_header.startswith('Bearer '):
+            return jsonify({"error": "Unauthorized", "message": "API key is missing or invalid."}), 401
+
+        token = auth_header.split(' ')[1]
+        if token != API_KEY:
+            return jsonify({"error": "Unauthorized", "message": "API key does not match."}), 401
+
+        g.authenticated = True
+        return func(*args, **kwargs)
+
+    return decorated_function

test.py

@@ -1,16 +1,140 @@
-# Example usage of the Chatbot class:
-from src.services.chatbot import Chatbot
-from src.utils.document_loader import load_document
+import requests
+import pandas as pd
+from dotenv import load_dotenv
+load_dotenv()
+import os
+
+DATA_KEY = os.getenv("AI_DATA_KEY")
+# Constants for API requests
+URL = "https://erpai.mkdlabs.com//v3/api/custom/erpai/common/get-data-ai"
+HEADERS = {
+    "x-project": DATA_KEY # Replace with your actual key
+}
+
+# JSON bodies for API requests
+def create_json_body(area_type, company_id):
+    return {
+        "type": area_type,
+        "options": {
+            "company_id": company_id
+        }
+    }
+
+# Function to fetch data from the API
+def fetch_data(json_body):
+    json_body["options"]["company_id"] = json_body["options"].get("company_id")  # Ensure company_id is included
+    response = requests.post(URL, headers=HEADERS, json=json_body)
+    response.raise_for_status()  # Raise an error for bad responses
+    return response.json()
+
+
+
+def convert_assessment_data_to_dataframe(assessment_data):
+    df_assessment = []
+    for assessment in assessment_data.get("data", []):
+        assessment_id = assessment["assessment_id"]
+        assessment_name = assessment["assessment_name"]
+        start_date = assessment["start_date"]
+        open_items = assessment["open_items"]
+        completed_items = assessment["completed_items"]
+        total_assigned_items = assessment["total_assigned_items"]
+        red_flags = assessment["red_flags"]
+
+        for user in assessment.get("user_details", []):
+            user_name = user["name"]
+            user_total_items = user["total_assigned_items"]
+            user_completed_items = user["completed_items"]
+            
+            for area in user.get("area_list", []):
+                df_assessment.append({
+                    "assessment_id": assessment_id,
+                    "assessment_name": assessment_name,
+                    "start_date": start_date,
+                    "open_items_overall": open_items,
+                    "completed_items_overall": completed_items,
+                    "total_assigned_items_overall": total_assigned_items,
+                    "user_name": user_name,
+                    "user_total_assigned_items": user_total_items,
+                    "user_completed_items": user_completed_items,
+                    "area": area,
+                    "red_flags": red_flags
+                })
+    return pd.DataFrame(df_assessment)
+
+# Convert to DataFrame
+
+
+# Summary statistics for overall assessment level
+def generate_summary_statistics(df):
+    total_assessments = df['assessment_id'].nunique()
+    avg_open_items = df.groupby('assessment_id')['open_items_overall'].mean().mean()
+    avg_completed_items = df.groupby('assessment_id')['completed_items_overall'].mean().mean()
+    avg_total_assigned_items = df.groupby('assessment_id')['total_assigned_items_overall'].mean().mean()
+    avg_red_flags = df['red_flags'].mean()
+
+    total_users = df['user_name'].nunique()
+    avg_user_total_items = df.groupby('user_name')['user_total_assigned_items'].mean().mean()
+    avg_user_completed_items = df.groupby('user_name')['user_completed_items'].mean().mean()
+    completion_rate_per_user = (df['user_completed_items'].sum() / df['user_total_assigned_items'].sum()) * 100 if df['user_total_assigned_items'].sum() > 0 else 0
+    
+    area_summary = df['area'].value_counts()
+
+    return {
+        "total_assessments": total_assessments,
+        "avg_open_items_per_assessment": avg_open_items,
+        "avg_completed_items_per_assessment": avg_completed_items,
+        "avg_total_assigned_items_per_assessment": avg_total_assigned_items,
+        "avg_red_flags": avg_red_flags,
+        "total_users": total_users,
+        "avg_user_total_assigned_items": avg_user_total_items,
+        "avg_user_completed_items": avg_user_completed_items,
+        "completion_rate_per_user": completion_rate_per_user,
+        "area_summary": area_summary.to_dict()
+    }
+
+# Additional statistics for efficiency and areas
+def generate_extended_statistics(df):
+    df['user_completion_rate'] = (df['user_completed_items'] / df['user_total_assigned_items']).fillna(0) * 100
+
+    top_5_efficient_users = df.groupby('user_name')['user_completion_rate'].mean().nlargest(5).to_dict()
+    bottom_5_least_efficient_users = df.groupby('user_name')['user_completion_rate'].mean().nsmallest(5).to_dict()
+
+    df['uncompleted_items'] = df['user_total_assigned_items'] - df['user_completed_items']
+    areas_with_most_uncompleted_items = df.groupby('area')['uncompleted_items'].sum().nlargest(5).to_dict()
+
+    return {
+        "top_5_efficient_users": top_5_efficient_users,
+        "bottom_5_least_efficient_users": bottom_5_least_efficient_users,
+        "areas_with_most_uncompleted_items": areas_with_most_uncompleted_items
+    }
+
+# Generate statistics for problematic areas
+def generate_problematic_area_statistics(df):
+    total_open_items = df.groupby('name')['open_items'].sum().sort_values(ascending=False)
+    total_red_flags = df.groupby('name')['red_flags'].sum().sort_values(ascending=False)
+
+    return pd.DataFrame({
+        "total_open_items": total_open_items,
+        "total_red_flags": total_red_flags
+    }).fillna(0)
+
+def generate_summary_stats(assessment_data, area_data):
+    assessment_df = convert_assessment_data_to_dataframe(assessment_data)
+    problematic_area_df = pd.DataFrame(area_data.get("data", []))
+    
+    summary_stats = generate_summary_statistics(assessment_df)
+    extended_stats = generate_extended_statistics(assessment_df)
+    summary_stats["users(Workers) based stats"] = extended_stats
+    
+    problematic_stats = generate_problematic_area_statistics(problematic_area_df)
+    summary_stats["Area based stats"] = problematic_stats.to_dict(orient='index')
+
+    return summary_stats
+
+
 if __name__ == "__main__":
-    chatbot = Chatbot()
+    from src.services.chatbot import Chatbot
+    bot = Chatbot()
+    res = bot.predict_next_n_assessment(companyid=12,N=3)
 
-    # Example inputs
-    path = r"C:\Users\User\Desktop\Blessing_AI\MKD\test_erp_ai\erp_ai\test\erp_ai\data\raw\coding_task_completion_document.pdf"
-
-    question = "Have you completed Task X?"
-    user_input = "Yes"
-    docs = load_document(path)
-
-    # Validate the worker's answer using the provided document
-    validation_result = chatbot.validate_worker(question, docs)
-    print(validation_result)
+    print(res)

tests/test_pipeline/test_data_preprocessor.py

@@ -0,0 +1,18 @@
+import unittest
+from src.pipeline.data_preprocessor import DataPreprocessor
+import os
+class TestDataPreprocessor(unittest.TestCase):
+
+    def setUp(self):
+        self.dp = DataPreprocessor(
+            input_path="/root/ds_erp_ai/data/raw/dummy_assessment_data.csv",
+            company_id="company_id"
+        )
+
+    def test_run(self):
+        res = self.dp.run()
+        self.assertIsNotNone(res)  # Check that the result is not None
+        self.assertTrue(os.path.exists(res))  # Check that the output file exists
+
+if __name__ == '__main__':
+    unittest.main()