75a0a3fde7
- Added `get_latest_email_date()` function in `database.py` to retrieve the most recent email date for a given account and folder. - Enhanced `fetch_folder_emails()` in `zoho_client.py` to intelligently determine the start date for fetching emails based on the latest email date in the database. - Introduced `analyze_and_update_threads_async()` for asynchronous analysis of email threads, allowing concurrent processing. - Created a synchronous wrapper `analyze_and_update_threads()` for easier integration. - Updated `fetch_emails()` to support database session and account email parameters. - Added comprehensive documentation in `AI_ANALYSIS_GUIDE.md` detailing the new AI analysis functionality. - Implemented tests for the new features, including `test_fetch_with_db.py`, `test_ai_analysis.py`, and `test_single_analysis.py`. - Added error handling and logging improvements throughout the codebase.
489 lines
15 KiB
Python
489 lines
15 KiB
Python
import asyncio
|
|
import json
|
|
import logging
|
|
import os
|
|
from contextlib import suppress
|
|
from typing import List
|
|
|
|
from fastapi import BackgroundTasks, Depends, FastAPI, HTTPException, Request
|
|
from fastapi.responses import HTMLResponse, RedirectResponse
|
|
from fastapi.staticfiles import StaticFiles
|
|
from fastapi.templating import Jinja2Templates
|
|
from sqlalchemy.orm import Session
|
|
|
|
from src.ai import analyze_thread
|
|
from src.alerts import process_alerts
|
|
from src.database import (
|
|
Message,
|
|
SessionLocal,
|
|
Thread,
|
|
create_db_tables,
|
|
get_thread_messages,
|
|
ingest_emails,
|
|
)
|
|
from src.zoho_client import ZohoClient
|
|
from dotenv import load_dotenv
|
|
|
|
load_dotenv()
|
|
|
|
|
|
def get_db():
|
|
db = SessionLocal()
|
|
try:
|
|
yield db
|
|
finally:
|
|
db.close()
|
|
|
|
|
|
logging.basicConfig(level=logging.INFO, filename="email_alerts.log")
|
|
|
|
|
|
create_db_tables()
|
|
app = FastAPI(title="Email Alerts UI")
|
|
|
|
# Static and templates
|
|
os.makedirs("templates", exist_ok=True)
|
|
os.makedirs("static", exist_ok=True)
|
|
templates = Jinja2Templates(directory="templates")
|
|
app.mount("/static", StaticFiles(directory="static"), name="static")
|
|
|
|
|
|
@app.get("/", response_class=HTMLResponse)
|
|
def home(request: Request, db: Session = Depends(get_db), account: str | None = None):
|
|
q = db.query(Thread).order_by(Thread.updated_at.desc())
|
|
if account:
|
|
q = q.filter(Thread.account_email == account.lower())
|
|
threads = q.limit(100).all()
|
|
return templates.TemplateResponse(
|
|
"threads.html",
|
|
{
|
|
"request": request,
|
|
"threads": threads,
|
|
"account": account or "",
|
|
"status": _status_for_templates(),
|
|
},
|
|
)
|
|
|
|
|
|
@app.get("/thread/{thread_id}", response_class=HTMLResponse)
|
|
def show_thread(thread_id: int, request: Request, db: Session = Depends(get_db)):
|
|
thread = db.query(Thread).filter(Thread.id == thread_id).one_or_none()
|
|
if not thread:
|
|
raise HTTPException(status_code=404, detail="Thread not found")
|
|
messages: List[Message] = get_thread_messages(db, thread_id)
|
|
# Convert for AI analyzer and template
|
|
msg_dicts = [
|
|
{
|
|
"id": m.id,
|
|
"date_sent": m.date_sent,
|
|
"subject": m.subject,
|
|
"from_email": m.from_email,
|
|
"to_email": m.to_email,
|
|
"body": m.body,
|
|
"is_incoming": m.is_incoming,
|
|
}
|
|
for m in messages
|
|
]
|
|
ai = analyze_thread(thread.subject or "", msg_dicts)
|
|
# Save AI info on the thread for listing and downstream alerts
|
|
try:
|
|
from datetime import datetime, timezone
|
|
|
|
thread.actionable = bool(ai.get("actionable", False))
|
|
thread.ai_summary = ai.get("summary")
|
|
thread.ai_confidence = ai.get("confidence")
|
|
thread.last_analyzed_at = datetime.now(timezone.utc)
|
|
db.commit()
|
|
except Exception:
|
|
pass
|
|
return templates.TemplateResponse(
|
|
"thread_detail.html",
|
|
{
|
|
"request": request,
|
|
"thread": thread,
|
|
"messages": messages,
|
|
"ai": ai,
|
|
"status": _status_for_templates(),
|
|
},
|
|
)
|
|
|
|
|
|
@app.get("/health")
|
|
def health():
|
|
return {"status": "ok"}
|
|
|
|
|
|
# ---------------------
|
|
# Config editor routes
|
|
# ---------------------
|
|
|
|
CONFIG_PATH = os.path.join(os.path.dirname(__file__), "..", "config.json")
|
|
|
|
|
|
def load_config() -> dict:
|
|
if not os.path.exists(CONFIG_PATH):
|
|
return {
|
|
"email_address": "",
|
|
"time_frames": [
|
|
{"name": "1-24 hours", "hours": 24, "alert_level": 1},
|
|
{"name": "24-48 hours", "hours": 48, "alert_level": 2},
|
|
{"name": "48+ hours", "hours": 72, "alert_level": 3},
|
|
],
|
|
"email_days_back": 7,
|
|
"agency_domains": [],
|
|
"zoho_email": "",
|
|
"zoho_app_password": "",
|
|
"whatsapp_to": "",
|
|
"auto_process": False,
|
|
"auto_process_interval": 30,
|
|
# Sync status
|
|
"sync_in_progress": False,
|
|
"last_sync_at": None,
|
|
"last_sync_count": 0,
|
|
"last_sync_status": "idle",
|
|
"last_sync_error": None,
|
|
}
|
|
with open(CONFIG_PATH, "r", encoding="utf-8") as f:
|
|
return json.load(f)
|
|
|
|
|
|
def save_config(cfg: dict) -> None:
|
|
with open(CONFIG_PATH, "w", encoding="utf-8") as f:
|
|
json.dump(cfg, f, indent=2, ensure_ascii=False)
|
|
|
|
|
|
def _status_for_templates() -> dict:
|
|
cfg = load_config()
|
|
return {
|
|
"auto_process": bool(cfg.get("auto_process")),
|
|
"interval": int(cfg.get("auto_process_interval", 30) or 30),
|
|
"sync_in_progress": bool(cfg.get("sync_in_progress")),
|
|
"last_sync_at": cfg.get("last_sync_at"),
|
|
"last_sync_status": cfg.get("last_sync_status", "idle"),
|
|
"last_sync_count": int(cfg.get("last_sync_count") or 0),
|
|
"last_sync_error": cfg.get("last_sync_error"),
|
|
}
|
|
|
|
|
|
@app.get("/config", response_class=HTMLResponse)
|
|
def config_form(request: Request, saved: int | None = None):
|
|
cfg = load_config()
|
|
# Render up to existing frames or at least 3 rows
|
|
frames = cfg.get("time_frames") or []
|
|
min_rows = max(len(frames), 3)
|
|
return templates.TemplateResponse(
|
|
"config.html",
|
|
{
|
|
"request": request,
|
|
"cfg": cfg,
|
|
"rows": list(range(min_rows)),
|
|
"saved": bool(saved),
|
|
"status": _status_for_templates(),
|
|
},
|
|
)
|
|
|
|
|
|
@app.post("/config")
|
|
async def config_save(request: Request):
|
|
form = await request.form()
|
|
cfg = load_config()
|
|
|
|
# Basic fields
|
|
cfg["email_address"] = (form.get("email_address") or "").strip()
|
|
# email_days_back
|
|
try:
|
|
cfg["email_days_back"] = int(
|
|
form.get("email_days_back") or cfg.get("email_days_back", 7)
|
|
)
|
|
except Exception:
|
|
cfg["email_days_back"] = 7
|
|
# agency domains (comma or newline separated)
|
|
domains_raw = (form.get("agency_domains") or "").replace("\r", "")
|
|
parts = [p.strip() for p in domains_raw.replace(",", "\n").split("\n") if p.strip()]
|
|
cfg["agency_domains"] = parts
|
|
# auto_process
|
|
cfg["auto_process"] = form.get("auto_process") == "on"
|
|
# auto_process_interval
|
|
try:
|
|
cfg["auto_process_interval"] = int(
|
|
form.get("auto_process_interval") or cfg.get("auto_process_interval", 30)
|
|
)
|
|
except Exception:
|
|
cfg["auto_process_interval"] = 30
|
|
|
|
# Zoho (optional - note: current client reads env vars)
|
|
cfg["zoho_email"] = (form.get("zoho_email") or cfg.get("zoho_email", "")).strip()
|
|
cfg["zoho_app_password"] = (
|
|
form.get("zoho_app_password") or cfg.get("zoho_app_password", "")
|
|
).strip()
|
|
# WhatsApp destination
|
|
cfg["whatsapp_to"] = (form.get("whatsapp_to") or cfg.get("whatsapp_to", "")).strip()
|
|
|
|
# Time frames: collect indexed rows
|
|
frames: list[dict] = []
|
|
# find indices present
|
|
indices = set()
|
|
for k in form.keys():
|
|
if k.startswith("time_name_"):
|
|
try:
|
|
indices.add(int(k.split("_")[-1]))
|
|
except Exception:
|
|
pass
|
|
for i in sorted(indices):
|
|
name = (form.get(f"time_name_{i}") or "").strip()
|
|
hrs_raw = form.get(f"time_hours_{i}") or ""
|
|
lvl_raw = form.get(f"time_alert_{i}") or ""
|
|
if not name and not hrs_raw and not lvl_raw:
|
|
continue
|
|
try:
|
|
hours = int(hrs_raw)
|
|
except Exception:
|
|
hours = 0
|
|
try:
|
|
level = int(lvl_raw)
|
|
except Exception:
|
|
level = 0
|
|
if name:
|
|
frames.append({"name": name, "hours": hours, "alert_level": level})
|
|
if frames:
|
|
cfg["time_frames"] = frames
|
|
|
|
save_config(cfg)
|
|
return RedirectResponse(url="/config?saved=1", status_code=303)
|
|
|
|
|
|
@app.post("/process")
|
|
def process(db: Session = Depends(get_db)):
|
|
cfg = load_config()
|
|
alerted = process_alerts(db, cfg)
|
|
return {"alerted_threads": alerted}
|
|
|
|
|
|
def _sync_emails_once(cfg: dict) -> int:
|
|
"""Fetch INBOX and Sent from Zoho and ingest into DB. Returns threads requiring reply count."""
|
|
from datetime import datetime, timezone
|
|
|
|
account_email = cfg.get("email_address") or cfg.get("zoho_email") or ""
|
|
if not account_email:
|
|
raise ValueError("Configure email_address or zoho_email in /config")
|
|
|
|
# Incremental lookback by last_sync_at
|
|
days_back = int(cfg.get("email_days_back", 7) or 7)
|
|
last_sync_at = cfg.get("last_sync_at")
|
|
if last_sync_at:
|
|
try:
|
|
last_dt = datetime.fromisoformat(last_sync_at)
|
|
now = datetime.now(timezone.utc)
|
|
delta_days = int(((now - last_dt).total_seconds() + 86399) // 86400)
|
|
days_back = max(1, delta_days)
|
|
except Exception:
|
|
pass
|
|
max_results = 100
|
|
client = ZohoClient(
|
|
email=cfg.get("zoho_email") or account_email,
|
|
app_password=cfg.get("zoho_app_password"),
|
|
)
|
|
|
|
db = SessionLocal()
|
|
try:
|
|
inbox = client.fetch_folder_emails(
|
|
folder="INBOX",
|
|
max_results=max_results,
|
|
days_back=days_back,
|
|
db_session=db,
|
|
account_email=account_email,
|
|
)
|
|
sent = client.fetch_folder_emails(
|
|
folder="Sent",
|
|
max_results=max_results,
|
|
days_back=days_back,
|
|
db_session=db,
|
|
account_email=account_email,
|
|
)
|
|
finally:
|
|
client.close()
|
|
|
|
try:
|
|
ingest_emails(
|
|
db, account_email=account_email, emails=inbox, default_folder="INBOX"
|
|
)
|
|
ingest_emails(
|
|
db, account_email=account_email, emails=sent, default_folder="Sent"
|
|
)
|
|
# Return count for UX/debug
|
|
count = (
|
|
db.query(Thread)
|
|
.filter(Thread.account_email == account_email.lower())
|
|
.count()
|
|
)
|
|
return count
|
|
finally:
|
|
db.close()
|
|
|
|
|
|
def _sync_emails_background_task():
|
|
"""Background task to sync emails and update config status."""
|
|
from datetime import datetime, timezone
|
|
|
|
cfg = load_config()
|
|
try:
|
|
count = _sync_emails_once(cfg)
|
|
# Update success status
|
|
cfg = load_config() # Reload in case it was modified
|
|
cfg.update(
|
|
{
|
|
"sync_in_progress": False,
|
|
"last_sync_status": "success",
|
|
"last_sync_at": datetime.now(timezone.utc).isoformat(),
|
|
"last_sync_count": count,
|
|
"last_sync_error": None,
|
|
}
|
|
)
|
|
save_config(cfg)
|
|
except Exception as e:
|
|
# Update error status
|
|
cfg = load_config() # Reload in case it was modified
|
|
cfg.update(
|
|
{
|
|
"sync_in_progress": False,
|
|
"last_sync_status": "error",
|
|
"last_sync_error": str(e),
|
|
}
|
|
)
|
|
save_config(cfg)
|
|
logging.error(f"Email sync failed: {e}")
|
|
|
|
|
|
@app.post("/sync_emails")
|
|
async def sync_emails(background_tasks: BackgroundTasks):
|
|
cfg = load_config()
|
|
if cfg.get("sync_in_progress"):
|
|
return RedirectResponse(url="/?sync=busy", status_code=303)
|
|
|
|
# Mark sync as starting
|
|
cfg["sync_in_progress"] = True
|
|
cfg["last_sync_status"] = "running"
|
|
cfg["last_sync_error"] = None
|
|
save_config(cfg)
|
|
|
|
# Add the background task
|
|
background_tasks.add_task(_sync_emails_background_task)
|
|
|
|
return RedirectResponse(url="/?sync=started", status_code=303)
|
|
|
|
|
|
# ---------------------
|
|
# Auto-processing loop
|
|
# ---------------------
|
|
_auto_task = None
|
|
_stop_event = asyncio.Event()
|
|
|
|
|
|
async def _auto_runner():
|
|
# Delay a bit on startup
|
|
await asyncio.sleep(2)
|
|
while not _stop_event.is_set():
|
|
cfg = load_config()
|
|
interval_min = int(cfg.get("auto_process_interval", 30) or 30)
|
|
if cfg.get("auto_process"):
|
|
# Sync emails then run alerts
|
|
try:
|
|
if not cfg.get("sync_in_progress"):
|
|
from datetime import datetime, timezone
|
|
|
|
# Mark sync as starting
|
|
cfg["sync_in_progress"] = True
|
|
cfg["last_sync_status"] = "running"
|
|
cfg["last_sync_error"] = None
|
|
save_config(cfg)
|
|
|
|
try:
|
|
count = _sync_emails_once(cfg)
|
|
# Update success status
|
|
cfg = load_config() # Reload in case it was modified
|
|
cfg.update(
|
|
{
|
|
"sync_in_progress": False,
|
|
"last_sync_status": "success",
|
|
"last_sync_at": datetime.now(timezone.utc).isoformat(),
|
|
"last_sync_count": count,
|
|
"last_sync_error": None,
|
|
}
|
|
)
|
|
save_config(cfg)
|
|
except Exception as e:
|
|
# Update error status
|
|
cfg = load_config() # Reload in case it was modified
|
|
cfg.update(
|
|
{
|
|
"sync_in_progress": False,
|
|
"last_sync_status": "error",
|
|
"last_sync_error": str(e),
|
|
}
|
|
)
|
|
save_config(cfg)
|
|
logging.error(f"Auto email sync failed: {e}")
|
|
except Exception:
|
|
# keep loop alive
|
|
pass
|
|
db = SessionLocal()
|
|
try:
|
|
process_alerts(db, cfg)
|
|
finally:
|
|
db.close()
|
|
try:
|
|
await asyncio.wait_for(
|
|
_stop_event.wait(), timeout=max(5, interval_min * 60)
|
|
)
|
|
except asyncio.TimeoutError:
|
|
continue
|
|
|
|
|
|
@app.on_event("startup")
|
|
async def on_startup():
|
|
# Reset any stale sync status from previous session
|
|
try:
|
|
cfg = load_config()
|
|
if cfg.get("sync_in_progress"):
|
|
cfg.update(
|
|
{
|
|
"sync_in_progress": False,
|
|
"last_sync_status": "interrupted",
|
|
"last_sync_error": "Previous session ended while sync was in progress",
|
|
}
|
|
)
|
|
save_config(cfg)
|
|
logging.info("Reset stale sync status from previous session")
|
|
except Exception as e:
|
|
logging.error(f"Failed to reset stale sync status on startup: {e}")
|
|
|
|
# Start the auto-processing task
|
|
global _auto_task
|
|
_stop_event.clear()
|
|
_auto_task = asyncio.create_task(_auto_runner())
|
|
|
|
|
|
@app.on_event("shutdown")
|
|
async def on_shutdown():
|
|
# Reset sync status if it's currently in progress
|
|
try:
|
|
cfg = load_config()
|
|
if cfg.get("sync_in_progress"):
|
|
cfg.update(
|
|
{
|
|
"sync_in_progress": False,
|
|
"last_sync_status": "interrupted",
|
|
"last_sync_error": "Server shutdown while sync was in progress",
|
|
}
|
|
)
|
|
save_config(cfg)
|
|
logging.info("Reset sync status due to server shutdown")
|
|
except Exception as e:
|
|
logging.error(f"Failed to reset sync status on shutdown: {e}")
|
|
|
|
# Stop the auto-processing task
|
|
if _auto_task:
|
|
_stop_event.set()
|
|
with suppress(Exception):
|
|
await _auto_task
|