Files
email_alerts_v2/src/app.py
T
bolade 75a0a3fde7 feat: Implement async AI analysis for email threads
- Added `get_latest_email_date()` function in `database.py` to retrieve the most recent email date for a given account and folder.
- Enhanced `fetch_folder_emails()` in `zoho_client.py` to intelligently determine the start date for fetching emails based on the latest email date in the database.
- Introduced `analyze_and_update_threads_async()` for asynchronous analysis of email threads, allowing concurrent processing.
- Created a synchronous wrapper `analyze_and_update_threads()` for easier integration.
- Updated `fetch_emails()` to support database session and account email parameters.
- Added comprehensive documentation in `AI_ANALYSIS_GUIDE.md` detailing the new AI analysis functionality.
- Implemented tests for the new features, including `test_fetch_with_db.py`, `test_ai_analysis.py`, and `test_single_analysis.py`.
- Added error handling and logging improvements throughout the codebase.
2025-08-11 23:20:20 +01:00

489 lines
15 KiB
Python

import asyncio
import json
import logging
import os
from contextlib import suppress
from typing import List
from fastapi import BackgroundTasks, Depends, FastAPI, HTTPException, Request
from fastapi.responses import HTMLResponse, RedirectResponse
from fastapi.staticfiles import StaticFiles
from fastapi.templating import Jinja2Templates
from sqlalchemy.orm import Session
from src.ai import analyze_thread
from src.alerts import process_alerts
from src.database import (
Message,
SessionLocal,
Thread,
create_db_tables,
get_thread_messages,
ingest_emails,
)
from src.zoho_client import ZohoClient
from dotenv import load_dotenv
load_dotenv()
def get_db():
db = SessionLocal()
try:
yield db
finally:
db.close()
logging.basicConfig(level=logging.INFO, filename="email_alerts.log")
create_db_tables()
app = FastAPI(title="Email Alerts UI")
# Static and templates
os.makedirs("templates", exist_ok=True)
os.makedirs("static", exist_ok=True)
templates = Jinja2Templates(directory="templates")
app.mount("/static", StaticFiles(directory="static"), name="static")
@app.get("/", response_class=HTMLResponse)
def home(request: Request, db: Session = Depends(get_db), account: str | None = None):
q = db.query(Thread).order_by(Thread.updated_at.desc())
if account:
q = q.filter(Thread.account_email == account.lower())
threads = q.limit(100).all()
return templates.TemplateResponse(
"threads.html",
{
"request": request,
"threads": threads,
"account": account or "",
"status": _status_for_templates(),
},
)
@app.get("/thread/{thread_id}", response_class=HTMLResponse)
def show_thread(thread_id: int, request: Request, db: Session = Depends(get_db)):
thread = db.query(Thread).filter(Thread.id == thread_id).one_or_none()
if not thread:
raise HTTPException(status_code=404, detail="Thread not found")
messages: List[Message] = get_thread_messages(db, thread_id)
# Convert for AI analyzer and template
msg_dicts = [
{
"id": m.id,
"date_sent": m.date_sent,
"subject": m.subject,
"from_email": m.from_email,
"to_email": m.to_email,
"body": m.body,
"is_incoming": m.is_incoming,
}
for m in messages
]
ai = analyze_thread(thread.subject or "", msg_dicts)
# Save AI info on the thread for listing and downstream alerts
try:
from datetime import datetime, timezone
thread.actionable = bool(ai.get("actionable", False))
thread.ai_summary = ai.get("summary")
thread.ai_confidence = ai.get("confidence")
thread.last_analyzed_at = datetime.now(timezone.utc)
db.commit()
except Exception:
pass
return templates.TemplateResponse(
"thread_detail.html",
{
"request": request,
"thread": thread,
"messages": messages,
"ai": ai,
"status": _status_for_templates(),
},
)
@app.get("/health")
def health():
return {"status": "ok"}
# ---------------------
# Config editor routes
# ---------------------
CONFIG_PATH = os.path.join(os.path.dirname(__file__), "..", "config.json")
def load_config() -> dict:
if not os.path.exists(CONFIG_PATH):
return {
"email_address": "",
"time_frames": [
{"name": "1-24 hours", "hours": 24, "alert_level": 1},
{"name": "24-48 hours", "hours": 48, "alert_level": 2},
{"name": "48+ hours", "hours": 72, "alert_level": 3},
],
"email_days_back": 7,
"agency_domains": [],
"zoho_email": "",
"zoho_app_password": "",
"whatsapp_to": "",
"auto_process": False,
"auto_process_interval": 30,
# Sync status
"sync_in_progress": False,
"last_sync_at": None,
"last_sync_count": 0,
"last_sync_status": "idle",
"last_sync_error": None,
}
with open(CONFIG_PATH, "r", encoding="utf-8") as f:
return json.load(f)
def save_config(cfg: dict) -> None:
with open(CONFIG_PATH, "w", encoding="utf-8") as f:
json.dump(cfg, f, indent=2, ensure_ascii=False)
def _status_for_templates() -> dict:
cfg = load_config()
return {
"auto_process": bool(cfg.get("auto_process")),
"interval": int(cfg.get("auto_process_interval", 30) or 30),
"sync_in_progress": bool(cfg.get("sync_in_progress")),
"last_sync_at": cfg.get("last_sync_at"),
"last_sync_status": cfg.get("last_sync_status", "idle"),
"last_sync_count": int(cfg.get("last_sync_count") or 0),
"last_sync_error": cfg.get("last_sync_error"),
}
@app.get("/config", response_class=HTMLResponse)
def config_form(request: Request, saved: int | None = None):
cfg = load_config()
# Render up to existing frames or at least 3 rows
frames = cfg.get("time_frames") or []
min_rows = max(len(frames), 3)
return templates.TemplateResponse(
"config.html",
{
"request": request,
"cfg": cfg,
"rows": list(range(min_rows)),
"saved": bool(saved),
"status": _status_for_templates(),
},
)
@app.post("/config")
async def config_save(request: Request):
form = await request.form()
cfg = load_config()
# Basic fields
cfg["email_address"] = (form.get("email_address") or "").strip()
# email_days_back
try:
cfg["email_days_back"] = int(
form.get("email_days_back") or cfg.get("email_days_back", 7)
)
except Exception:
cfg["email_days_back"] = 7
# agency domains (comma or newline separated)
domains_raw = (form.get("agency_domains") or "").replace("\r", "")
parts = [p.strip() for p in domains_raw.replace(",", "\n").split("\n") if p.strip()]
cfg["agency_domains"] = parts
# auto_process
cfg["auto_process"] = form.get("auto_process") == "on"
# auto_process_interval
try:
cfg["auto_process_interval"] = int(
form.get("auto_process_interval") or cfg.get("auto_process_interval", 30)
)
except Exception:
cfg["auto_process_interval"] = 30
# Zoho (optional - note: current client reads env vars)
cfg["zoho_email"] = (form.get("zoho_email") or cfg.get("zoho_email", "")).strip()
cfg["zoho_app_password"] = (
form.get("zoho_app_password") or cfg.get("zoho_app_password", "")
).strip()
# WhatsApp destination
cfg["whatsapp_to"] = (form.get("whatsapp_to") or cfg.get("whatsapp_to", "")).strip()
# Time frames: collect indexed rows
frames: list[dict] = []
# find indices present
indices = set()
for k in form.keys():
if k.startswith("time_name_"):
try:
indices.add(int(k.split("_")[-1]))
except Exception:
pass
for i in sorted(indices):
name = (form.get(f"time_name_{i}") or "").strip()
hrs_raw = form.get(f"time_hours_{i}") or ""
lvl_raw = form.get(f"time_alert_{i}") or ""
if not name and not hrs_raw and not lvl_raw:
continue
try:
hours = int(hrs_raw)
except Exception:
hours = 0
try:
level = int(lvl_raw)
except Exception:
level = 0
if name:
frames.append({"name": name, "hours": hours, "alert_level": level})
if frames:
cfg["time_frames"] = frames
save_config(cfg)
return RedirectResponse(url="/config?saved=1", status_code=303)
@app.post("/process")
def process(db: Session = Depends(get_db)):
cfg = load_config()
alerted = process_alerts(db, cfg)
return {"alerted_threads": alerted}
def _sync_emails_once(cfg: dict) -> int:
"""Fetch INBOX and Sent from Zoho and ingest into DB. Returns threads requiring reply count."""
from datetime import datetime, timezone
account_email = cfg.get("email_address") or cfg.get("zoho_email") or ""
if not account_email:
raise ValueError("Configure email_address or zoho_email in /config")
# Incremental lookback by last_sync_at
days_back = int(cfg.get("email_days_back", 7) or 7)
last_sync_at = cfg.get("last_sync_at")
if last_sync_at:
try:
last_dt = datetime.fromisoformat(last_sync_at)
now = datetime.now(timezone.utc)
delta_days = int(((now - last_dt).total_seconds() + 86399) // 86400)
days_back = max(1, delta_days)
except Exception:
pass
max_results = 100
client = ZohoClient(
email=cfg.get("zoho_email") or account_email,
app_password=cfg.get("zoho_app_password"),
)
db = SessionLocal()
try:
inbox = client.fetch_folder_emails(
folder="INBOX",
max_results=max_results,
days_back=days_back,
db_session=db,
account_email=account_email,
)
sent = client.fetch_folder_emails(
folder="Sent",
max_results=max_results,
days_back=days_back,
db_session=db,
account_email=account_email,
)
finally:
client.close()
try:
ingest_emails(
db, account_email=account_email, emails=inbox, default_folder="INBOX"
)
ingest_emails(
db, account_email=account_email, emails=sent, default_folder="Sent"
)
# Return count for UX/debug
count = (
db.query(Thread)
.filter(Thread.account_email == account_email.lower())
.count()
)
return count
finally:
db.close()
def _sync_emails_background_task():
"""Background task to sync emails and update config status."""
from datetime import datetime, timezone
cfg = load_config()
try:
count = _sync_emails_once(cfg)
# Update success status
cfg = load_config() # Reload in case it was modified
cfg.update(
{
"sync_in_progress": False,
"last_sync_status": "success",
"last_sync_at": datetime.now(timezone.utc).isoformat(),
"last_sync_count": count,
"last_sync_error": None,
}
)
save_config(cfg)
except Exception as e:
# Update error status
cfg = load_config() # Reload in case it was modified
cfg.update(
{
"sync_in_progress": False,
"last_sync_status": "error",
"last_sync_error": str(e),
}
)
save_config(cfg)
logging.error(f"Email sync failed: {e}")
@app.post("/sync_emails")
async def sync_emails(background_tasks: BackgroundTasks):
cfg = load_config()
if cfg.get("sync_in_progress"):
return RedirectResponse(url="/?sync=busy", status_code=303)
# Mark sync as starting
cfg["sync_in_progress"] = True
cfg["last_sync_status"] = "running"
cfg["last_sync_error"] = None
save_config(cfg)
# Add the background task
background_tasks.add_task(_sync_emails_background_task)
return RedirectResponse(url="/?sync=started", status_code=303)
# ---------------------
# Auto-processing loop
# ---------------------
_auto_task = None
_stop_event = asyncio.Event()
async def _auto_runner():
# Delay a bit on startup
await asyncio.sleep(2)
while not _stop_event.is_set():
cfg = load_config()
interval_min = int(cfg.get("auto_process_interval", 30) or 30)
if cfg.get("auto_process"):
# Sync emails then run alerts
try:
if not cfg.get("sync_in_progress"):
from datetime import datetime, timezone
# Mark sync as starting
cfg["sync_in_progress"] = True
cfg["last_sync_status"] = "running"
cfg["last_sync_error"] = None
save_config(cfg)
try:
count = _sync_emails_once(cfg)
# Update success status
cfg = load_config() # Reload in case it was modified
cfg.update(
{
"sync_in_progress": False,
"last_sync_status": "success",
"last_sync_at": datetime.now(timezone.utc).isoformat(),
"last_sync_count": count,
"last_sync_error": None,
}
)
save_config(cfg)
except Exception as e:
# Update error status
cfg = load_config() # Reload in case it was modified
cfg.update(
{
"sync_in_progress": False,
"last_sync_status": "error",
"last_sync_error": str(e),
}
)
save_config(cfg)
logging.error(f"Auto email sync failed: {e}")
except Exception:
# keep loop alive
pass
db = SessionLocal()
try:
process_alerts(db, cfg)
finally:
db.close()
try:
await asyncio.wait_for(
_stop_event.wait(), timeout=max(5, interval_min * 60)
)
except asyncio.TimeoutError:
continue
@app.on_event("startup")
async def on_startup():
# Reset any stale sync status from previous session
try:
cfg = load_config()
if cfg.get("sync_in_progress"):
cfg.update(
{
"sync_in_progress": False,
"last_sync_status": "interrupted",
"last_sync_error": "Previous session ended while sync was in progress",
}
)
save_config(cfg)
logging.info("Reset stale sync status from previous session")
except Exception as e:
logging.error(f"Failed to reset stale sync status on startup: {e}")
# Start the auto-processing task
global _auto_task
_stop_event.clear()
_auto_task = asyncio.create_task(_auto_runner())
@app.on_event("shutdown")
async def on_shutdown():
# Reset sync status if it's currently in progress
try:
cfg = load_config()
if cfg.get("sync_in_progress"):
cfg.update(
{
"sync_in_progress": False,
"last_sync_status": "interrupted",
"last_sync_error": "Server shutdown while sync was in progress",
}
)
save_config(cfg)
logging.info("Reset sync status due to server shutdown")
except Exception as e:
logging.error(f"Failed to reset sync status on shutdown: {e}")
# Stop the auto-processing task
if _auto_task:
_stop_event.set()
with suppress(Exception):
await _auto_task