human hand off

backend/app/services/handoff_delivery.py

@@ -0,0 +1,215 @@
+import asyncio
+import email.utils
+import logging
+from datetime import datetime, timezone
+
+from bson import ObjectId
+
+from app.database import mongo_db
+
+logger = logging.getLogger(__name__)
+
+JOB_MIRROR = "MIRROR_TO_HUMAN"
+JOB_RELAY = "RELAY_TO_EXTERNAL"
+
+STATUS_PENDING = "PENDING"
+STATUS_SENDING = "SENDING"
+STATUS_SENT = "SENT"
+STATUS_FAILED = "FAILED"
+STATUS_AMBIGUOUS = "AMBIGUOUS"
+
+
+class HandoffPreflightError(RuntimeError):
+    """A deterministic error that happened before SMTP submission."""
+
+
+def _now() -> datetime:
+    return datetime.now(timezone.utc)
+
+
+async def enqueue_handoff_delivery(
+    database,
+    *,
+    kind: str,
+    account_id: str,
+    handoff_id: ObjectId,
+    source_message_id: ObjectId,
+    agent_email: str,
+) -> dict:
+    idempotency_key = f"{kind}:{account_id}:{handoff_id}:{source_message_id}"
+    now = _now()
+    document = {
+        "idempotency_key": idempotency_key,
+        "kind": kind,
+        "account_id": account_id,
+        "handoff_id": handoff_id,
+        "source_message_id": source_message_id,
+        "agent_email": agent_email.lower(),
+        "generated_message_id": email.utils.make_msgid(
+            domain=agent_email.rsplit("@", 1)[-1].lower()
+        ),
+        "status": STATUS_PENDING,
+        "attempt_count": 0,
+        "next_attempt_at": now,
+        "created_at": now,
+        "updated_at": now,
+    }
+    await database.handoff_delivery_jobs.update_one(
+        {"idempotency_key": idempotency_key},
+        {"$setOnInsert": document},
+        upsert=True,
+    )
+    return await database.handoff_delivery_jobs.find_one(
+        {"idempotency_key": idempotency_key}
+    )
+
+
+async def process_handoff_delivery_once(database=mongo_db) -> bool:
+    now = _now()
+    job = await database.handoff_delivery_jobs.find_one_and_update(
+        {
+            "status": STATUS_PENDING,
+            "next_attempt_at": {"$lte": now},
+        },
+        {
+            "$set": {
+                "status": STATUS_SENDING,
+                "started_at": now,
+                "updated_at": now,
+            },
+            "$inc": {"attempt_count": 1},
+        },
+        sort=[("created_at", 1), ("_id", 1)],
+        return_document=True,
+    )
+    if not job:
+        return False
+
+    try:
+        from app.services.human_handoff_service import deliver_handoff_job
+
+        target_message_id = await deliver_handoff_job(database, job)
+    except HandoffPreflightError as error:
+        logger.error(
+            "handoff_delivery_preflight_failed job=%s kind=%s error=%s",
+            job["_id"],
+            job["kind"],
+            error,
+        )
+        await database.handoff_delivery_jobs.update_one(
+            {"_id": job["_id"], "status": STATUS_SENDING},
+            {
+                "$set": {
+                    "status": STATUS_FAILED,
+                    "error": str(error),
+                    "failed_at": _now(),
+                    "updated_at": _now(),
+                }
+            },
+        )
+        if job["kind"] == JOB_RELAY:
+            await database.messages.update_one(
+                {"_id": job["source_message_id"]},
+                {
+                    "$set": {
+                        "relay_status": STATUS_FAILED,
+                        "relay_error": str(error),
+                        "relay_failed_at": _now(),
+                    }
+                },
+            )
+    except Exception as error:
+        # SMTP exceptions can be ambiguous: the remote MTA may have accepted the
+        # message before the acknowledgement was lost. Never auto-retry these.
+        logger.exception(
+            "handoff_delivery_ambiguous job=%s kind=%s",
+            job["_id"],
+            job["kind"],
+        )
+        await database.handoff_delivery_jobs.update_one(
+            {"_id": job["_id"], "status": STATUS_SENDING},
+            {
+                "$set": {
+                    "status": STATUS_AMBIGUOUS,
+                    "error": str(error),
+                    "ambiguous_at": _now(),
+                    "updated_at": _now(),
+                }
+            },
+        )
+        if job["kind"] == JOB_RELAY:
+            await database.messages.update_one(
+                {"_id": job["source_message_id"]},
+                {
+                    "$set": {
+                        "relay_status": STATUS_AMBIGUOUS,
+                        "relay_error": str(error),
+                        "relay_failed_at": _now(),
+                    }
+                },
+            )
+    else:
+        await database.handoff_delivery_jobs.update_one(
+            {"_id": job["_id"], "status": STATUS_SENDING},
+            {
+                "$set": {
+                    "status": STATUS_SENT,
+                    "target_message_id": target_message_id,
+                    "sent_at": _now(),
+                    "updated_at": _now(),
+                }
+            },
+        )
+    return True
+
+
+class HandoffDeliveryWorker:
+    def __init__(self, poll_seconds: float = 1.0):
+        self.poll_seconds = poll_seconds
+        self._task: asyncio.Task | None = None
+        self._stop = asyncio.Event()
+
+    async def start(self) -> None:
+        if self._task and not self._task.done():
+            return
+        # A process that died during SMTP submission cannot know whether the
+        # remote MTA accepted it. Quarantine instead of risking a duplicate.
+        await mongo_db.handoff_delivery_jobs.update_many(
+            {"status": STATUS_SENDING},
+            {
+                "$set": {
+                    "status": STATUS_AMBIGUOUS,
+                    "error": "Worker restarted during SMTP submission.",
+                    "ambiguous_at": _now(),
+                    "updated_at": _now(),
+                }
+            },
+        )
+        self._stop.clear()
+        self._task = asyncio.create_task(self._run())
+
+    async def stop(self) -> None:
+        self._stop.set()
+        if self._task:
+            await self._task
+            self._task = None
+
+    async def _run(self) -> None:
+        while not self._stop.is_set():
+            processed = False
+            try:
+                processed = await process_handoff_delivery_once()
+            except Exception:
+                logger.exception("handoff_delivery_worker_iteration_failed")
+            if processed:
+                continue
+            try:
+                await asyncio.wait_for(
+                    self._stop.wait(),
+                    timeout=self.poll_seconds,
+                )
+            except TimeoutError:
+                pass
+
+
+handoff_delivery_worker = HandoffDeliveryWorker()