import logging import re from datetime import datetime, timezone from bson import ObjectId from app.services.external_reply_service import ( ExternalReplyThreadingError, build_external_reply_plan, internet_message_id_of, log_original_external_message, resolve_external_reply_parent, resolve_internet_message_id, send_external_reply, ) from app.services.handoff_email_templates import ( external_reply_subject, handoff_email_subject, ) from app.services.handoff_mirror_policy import ( GENERATED_BY_HANDOFF_BRIDGE, GENERATED_BY_HUMAN, GENERATED_BY_SYSTEM, MESSAGE_ORIGIN_HANDOFF_MIRROR, MESSAGE_ORIGIN_HUMAN_OWNER, RELAY_STATUS_FAILED, RELAY_STATUS_SENT, should_mirror_to_human, ) from app.services.handoff_mirror import load_mirror_payload, load_mirror_update_payload from app.services.handoff_evaluator import evaluate_handoff from app.services.mail_sender import deliver_email from app.services.message_ids import ( canonical_message_id, folded_message_id, message_id_fields, parse_message_id_list, ) from app.services.reply_extractor import extract_new_reply_content, validate_external_reply_body from app.services.thread_service import ( attach_message_to_thread, message_event_at, threading_fields_from_parent, ) logger = logging.getLogger(__name__) MESSAGE_SCOPE_EXTERNAL = "EXTERNAL" MESSAGE_SCOPE_HANDOFF = "HANDOFF" THREAD_TYPE_EXTERNAL = "EXTERNAL" THREAD_TYPE_HUMAN_HANDOFF = "HUMAN_HANDOFF" HANDOFF_STATUS_ACTIVE = "ACTIVE" HANDOFF_STATUS_WAITING = "WAITING_FOR_HUMAN" HANDOFF_STATUS_HUMAN_REPLIED = "HUMAN_REPLIED" HANDOFF_STATUS_AGENT_REPLIED = "AGENT_REPLIED" OWNER_AGENT = "AGENT" OWNER_HUMAN = "HUMAN" LINK_EXTERNAL_TRIGGERED = "EXTERNAL_TRIGGERED_HANDOFF" LINK_EXTERNAL_RELAYED = "EXTERNAL_RELAYED_TO_HUMAN" LINK_HUMAN_TRIGGERED_EXTERNAL = "HUMAN_TRIGGERED_EXTERNAL_REPLY" LINK_EXTERNAL_UPDATE_RELAYED = "EXTERNAL_UPDATE_RELAYED_TO_HUMAN" def _now() -> datetime: return datetime.now(timezone.utc) def _snippet(text: str, limit: int = 240) -> str: return re.sub(r"\s+", " ", (text or "")).strip()[:limit] def _handoff_reply_subject(external_subject: str) -> str: return f"Re: {handoff_email_subject(external_subject)}" async def create_empty_handoff_thread( database, *, account_id: str, subject: str, ) -> ObjectId: now = _now() document = { "account_id": account_id, "thread_type": THREAD_TYPE_HUMAN_HANDOFF, "normalized_subject": subject, "created_at": now, "updated_at": now, "last_message_at": now, "latest_message_id": None, "message_count": 0, } result = await database.email_threads.insert_one(document) return result.inserted_id async def _record_message_link( database, *, handoff_id: ObjectId, external_message_id: ObjectId | None, handoff_message_id: ObjectId, relationship: str, ) -> None: await database.handoff_message_links.insert_one( { "handoff_id": handoff_id, "external_message_id": external_message_id, "handoff_message_id": handoff_message_id, "relationship": relationship, "created_at": _now(), } ) async def _latest_external_customer_message( database, account_id: str, external_thread_id: ObjectId, customer_address: str, ) -> dict | None: customer = customer_address.lower() pipeline = [ { "$match": { "account_id": account_id, "thread_id": external_thread_id, "message_scope": MESSAGE_SCOPE_EXTERNAL, "deletion_state": {"$in": [None, ""]}, } }, {"$sort": {"event_at": -1, "_id": -1}}, { "$lookup": { "from": "messages", "localField": "message_id", "foreignField": "_id", "as": "message", } }, {"$unwind": "$message"}, { "$match": { "$or": [ {"message.direction": "inbound"}, {"message.sender.address": customer}, ] } }, {"$limit": 1}, ] cursor = await database.mailbox_entries.aggregate(pipeline) rows = [row async for row in cursor] return rows[0]["message"] if rows else None async def _latest_external_message(database, account_id: str, external_thread_id: ObjectId) -> dict | None: pipeline = [ { "$match": { "account_id": account_id, "thread_id": external_thread_id, "message_scope": MESSAGE_SCOPE_EXTERNAL, "deletion_state": {"$in": [None, ""]}, } }, {"$sort": {"event_at": -1, "_id": -1}}, {"$limit": 1}, { "$lookup": { "from": "messages", "localField": "message_id", "foreignField": "_id", "as": "message", } }, {"$unwind": "$message"}, ] cursor = await database.mailbox_entries.aggregate(pipeline) rows = [row async for row in cursor] return rows[0]["message"] if rows else None async def _latest_handoff_message(database, account_id: str, handoff_thread_id: ObjectId) -> dict | None: pipeline = [ { "$match": { "account_id": account_id, "thread_id": handoff_thread_id, "message_scope": MESSAGE_SCOPE_HANDOFF, } }, {"$sort": {"event_at": -1, "_id": -1}}, {"$limit": 1}, { "$lookup": { "from": "messages", "localField": "message_id", "foreignField": "_id", "as": "message", } }, {"$unwind": "$message"}, ] cursor = await database.mailbox_entries.aggregate(pipeline) rows = [row async for row in cursor] return rows[0]["message"] if rows else None async def find_active_handoff_for_external_thread( database, account_id: str, external_thread_id: ObjectId, ) -> dict | None: return await database.human_handoffs.find_one( { "account_id": account_id, "external_thread_id": external_thread_id, "status": {"$in": [HANDOFF_STATUS_ACTIVE, HANDOFF_STATUS_WAITING, HANDOFF_STATUS_HUMAN_REPLIED, HANDOFF_STATUS_AGENT_REPLIED]}, } ) async def resolve_handoff_from_inbound( database, *, account_id: str, agent_email: str, sender_address: str, in_reply_to: str | None, references: list[str], ) -> dict | None: sender = sender_address.lower() rules = await database.human_handoff_rules.find( {"account_id": account_id, "owner_email": sender, "active": True} ).to_list(length=10) if not rules: return None search_ids = [*([in_reply_to] if in_reply_to else []), *references] if not search_ids: return None for internet_id in reversed(parse_message_id_list(" ".join(search_ids))): folded = folded_message_id(internet_id) message = await database.messages.find_one( { "$or": [ {"message_id_folded": folded}, {"message_id": internet_id}, ], "message_scope": MESSAGE_SCOPE_HANDOFF, } ) if message is None: continue entry = await database.mailbox_entries.find_one( { "account_id": account_id, "message_id": message["_id"], "message_scope": MESSAGE_SCOPE_HANDOFF, } ) if entry is None: continue handoff = await database.human_handoffs.find_one( { "account_id": account_id, "handoff_thread_id": entry["thread_id"], "human_email": sender, "status": {"$ne": "CLOSED"}, } ) if handoff: return handoff return None async def _handoff_mirror_exists(database, external_message_id: ObjectId) -> bool: existing = await database.messages.find_one( { "mirrored_from_external_message_id": external_message_id, "message_scope": MESSAGE_SCOPE_HANDOFF, }, {"_id": 1}, ) return existing is not None async def _bridged_external_exists(database, handoff_message_id: ObjectId) -> dict | None: return await database.messages.find_one( { "generated_from_handoff_message_id": handoff_message_id, "message_scope": MESSAGE_SCOPE_EXTERNAL, } ) async def store_handoff_message( database, *, account_id: str, agent_email: str, handoff_thread_id: ObjectId, direction: str, sender: dict, recipients: list[dict], subject: str, body_text: str, internet_message_id: str | None = None, in_reply_to: str | None = None, references: list[str] | None = None, handoff_id: ObjectId | None = None, link_relationship: str | None = None, external_message_id: ObjectId | None = None, event_at: datetime | None = None, raw_body: str | None = None, new_reply_content: str | None = None, body_html: str | None = None, message_origin: str | None = None, generated_by: str | None = None, mirrored_from_external_message_id: ObjectId | None = None, relay_status: str | None = None, generated_external_message_id: ObjectId | None = None, ) -> dict: now = event_at or _now() display_body = new_reply_content if new_reply_content is not None else body_text document = { "direction": direction, "message_scope": MESSAGE_SCOPE_HANDOFF, "account_id": account_id, "sender": sender, "to": recipients, "subject": subject, "snippet": _snippet(display_body), "body_text": display_body, "raw_body": raw_body, "new_reply_content": new_reply_content, "body_html": body_html, "attachments": [], "created_at": now, "sent_at": now if direction == "outbound" else None, "received_at": now if direction == "inbound" else None, "status": "received" if direction == "inbound" else "accepted", **message_id_fields(internet_message_id), "in_reply_to": canonical_message_id(in_reply_to), "references": references or [], } if message_origin: document["message_origin"] = message_origin if generated_by: document["generated_by"] = generated_by if mirrored_from_external_message_id: document["mirrored_from_external_message_id"] = mirrored_from_external_message_id if relay_status: document["relay_status"] = relay_status if generated_external_message_id: document["generated_external_message_id"] = generated_external_message_id result = await database.messages.insert_one(document) document["_id"] = result.inserted_id await attach_message_to_thread( database, account_id=account_id, thread_id=handoff_thread_id, message_id=result.inserted_id, subject=subject, event_at=now, ) await database.mailbox_entries.insert_one( { "account_id": account_id, "message_id": result.inserted_id, "thread_id": handoff_thread_id, "mailbox": "handoff", "message_scope": MESSAGE_SCOPE_HANDOFF, "direction": direction, "sender": sender, "recipients": recipients, "subject": subject, "snippet": document["snippet"], "attachments": [], "status": document["status"], "is_read": direction == "outbound", "is_starred": False, "event_at": now, "created_at": now, "trashed_at": None, "deletion_state": None, } ) if handoff_id and link_relationship: await _record_message_link( database, handoff_id=handoff_id, external_message_id=external_message_id, handoff_message_id=result.inserted_id, relationship=link_relationship, ) return document async def send_handoff_to_human( database, *, account_id: str, agent_email: str, human_email: str, external_message: dict, external_thread_id: ObjectId, handoff_reason: str, domain, existing_handoff: dict | None = None, ) -> dict: if await _handoff_mirror_exists(database, external_message["_id"]): logger.info( "handoff_mirror_skipped_duplicate external_message=%s", external_message["_id"], ) if existing_handoff: return existing_handoff existing = await database.human_handoffs.find_one( { "account_id": account_id, "external_thread_id": external_thread_id, } ) if existing: return existing raise RuntimeError("Handoff mirror duplicate without handoff record") external_subject = external_message.get("subject", "") stable_subject = handoff_email_subject(external_subject) mirror_source = {**external_message, "_agent_email": agent_email} if existing_handoff: handoff_id = existing_handoff["_id"] handoff_thread_id = existing_handoff["handoff_thread_id"] parent = await _latest_handoff_message(database, account_id, handoff_thread_id) threading = threading_fields_from_parent(parent) if parent else {} subject = _handoff_reply_subject(external_subject) mirror = await load_mirror_update_payload(mirror_source, agent_email=agent_email) relationship = LINK_EXTERNAL_UPDATE_RELAYED else: handoff_thread_id = await create_empty_handoff_thread( database, account_id=account_id, subject=stable_subject, ) threading = {} subject = stable_subject mirror = await load_mirror_payload(mirror_source) external_sender_text = (external_message.get("sender") or {}).get("address") or "unknown sender" handoff_doc = { "account_id": account_id, "agent_email": agent_email.lower(), "human_email": human_email.lower(), "external_thread_id": external_thread_id, "handoff_thread_id": handoff_thread_id, "external_subject": external_subject, "external_sender": external_sender_text, "handoff_reason": handoff_reason, "status": HANDOFF_STATUS_WAITING, "current_owner": OWNER_HUMAN, "created_at": _now(), "updated_at": _now(), "last_external_message_at": message_event_at(external_message), "last_human_message_at": None, "trigger_external_message_id": external_message["_id"], "trigger_internet_message_id": internet_message_id_of(external_message), } insert = await database.human_handoffs.insert_one(handoff_doc) handoff_id = insert.inserted_id relationship = LINK_EXTERNAL_TRIGGERED internet_message_id = await deliver_email( sender=agent_email, recipients=[human_email], cc=[], bcc=[], subject=subject, body_text=mirror.body_text, body_html=mirror.body_html, domain=domain, in_reply_to=threading.get("in_reply_to"), references=threading.get("references"), inline_parts=mirror.inline_parts, file_attachments=mirror.file_attachments, ) handoff_message = await store_handoff_message( database, account_id=account_id, agent_email=agent_email, handoff_thread_id=handoff_thread_id, direction="outbound", sender={"name": "Gentic Agent", "address": agent_email}, recipients=[{"name": human_email.split("@")[0], "address": human_email}], subject=subject, body_text=mirror.body_text, body_html=mirror.body_html, internet_message_id=internet_message_id, in_reply_to=threading.get("in_reply_to"), references=threading.get("references"), handoff_id=handoff_id, link_relationship=relationship, external_message_id=external_message["_id"], message_origin=MESSAGE_ORIGIN_HANDOFF_MIRROR, generated_by=GENERATED_BY_SYSTEM, mirrored_from_external_message_id=external_message["_id"], ) await database.human_handoffs.update_one( {"_id": handoff_id}, { "$set": { "status": HANDOFF_STATUS_WAITING, "current_owner": OWNER_HUMAN, "updated_at": _now(), "last_external_message_at": message_event_at(external_message), "latest_handoff_message_id": handoff_message["_id"], } }, ) logger.info( "handoff_sent handoff_id=%s external_thread=%s handoff_thread=%s", handoff_id, external_thread_id, handoff_thread_id, ) return await database.human_handoffs.find_one({"_id": handoff_id}) async def _handoff_rule_for_human(database, account_id: str, human_email: str) -> dict | None: return await database.human_handoff_rules.find_one( { "account_id": account_id, "owner_email": human_email.lower(), "active": True, } ) async def process_human_reply( database, *, account_id: str, agent_email: str, handoff: dict, inbound_parsed: dict, inbound_body: str, domain, ) -> None: handoff_thread_id = handoff["handoff_thread_id"] external_thread_id = handoff["external_thread_id"] human_email = handoff["human_email"] rule = await _handoff_rule_for_human(database, account_id, human_email) allow_replies = rule.get("allow_replies", True) if rule else True extracted = extract_new_reply_content( body_text=inbound_body, body_html=inbound_parsed.get("body_html"), ) new_reply_content = extracted["new_reply_content"].strip() raw_body = extracted["raw_body"] human_message = await store_handoff_message( database, account_id=account_id, agent_email=agent_email, handoff_thread_id=handoff_thread_id, direction="inbound", sender={"name": human_email.split("@")[0], "address": human_email}, recipients=[{"name": "Gentic Agent", "address": agent_email}], subject=inbound_parsed.get("subject", ""), body_text=raw_body, raw_body=raw_body, new_reply_content=new_reply_content, body_html=inbound_parsed.get("body_html"), internet_message_id=inbound_parsed.get("message_id"), in_reply_to=inbound_parsed.get("in_reply_to"), references=inbound_parsed.get("references", []), handoff_id=handoff["_id"], message_origin=MESSAGE_ORIGIN_HUMAN_OWNER, generated_by=GENERATED_BY_HUMAN, ) now = _now() if not allow_replies: await database.human_handoffs.update_one( {"_id": handoff["_id"]}, { "$set": { "status": HANDOFF_STATUS_HUMAN_REPLIED, "current_owner": OWNER_AGENT, "updated_at": now, "last_human_message_at": now, } }, ) return if not new_reply_content: logger.warning( "handoff_reply_empty_extract handoff_id=%s account=%s", handoff["_id"], account_id, ) await database.human_handoffs.update_one( {"_id": handoff["_id"]}, { "$set": { "status": HANDOFF_STATUS_HUMAN_REPLIED, "current_owner": OWNER_AGENT, "updated_at": now, "last_human_message_at": now, } }, ) return if not validate_external_reply_body(new_reply_content): logger.error( "handoff_reply_leakage_guard handoff_id=%s account=%s", handoff["_id"], account_id, ) return external_recipient = handoff.get("external_sender") or "" external_parent = await resolve_external_reply_parent( database, account_id=account_id, external_thread_id=external_thread_id, customer_address=external_recipient, handoff=handoff, ) if external_parent is None: logger.warning( "handoff_reply_missing_external_parent handoff_id=%s external_thread=%s " "trigger_internet_message_id=%s", handoff["_id"], external_thread_id, handoff.get("trigger_internet_message_id"), ) return parent_sender = (external_parent.get("sender") or {}).get("address") log_original_external_message( internal_id=external_parent["_id"], thread_id=external_thread_id, internet_message_id=internet_message_id_of(external_parent), sender=parent_sender, recipient=agent_email, subject=external_parent.get("subject"), in_reply_to=external_parent.get("in_reply_to"), references=external_parent.get("references"), raw_message_available=bool(external_parent.get("raw_mime")), ) external_recipient = external_recipient or parent_sender if not external_recipient: return parent_internet_id = await resolve_internet_message_id(database, external_parent) if not parent_internet_id: logger.error( "handoff_reply_missing_parent_message_id handoff_id=%s parent_internal_id=%s", handoff["_id"], external_parent["_id"], ) return logger.info( "handoff_external_reply_threading handoff_id=%s parent_internal_id=%s " "parent_internet_message_id=%s human_in_reply_to=%s human_references=%s", handoff["_id"], external_parent["_id"], parent_internet_id, human_message.get("in_reply_to"), human_message.get("references"), ) existing_bridge = await _bridged_external_exists(database, human_message["_id"]) if existing_bridge: logger.info( "handoff_bridge_already_processed handoff_message=%s external=%s", human_message["_id"], existing_bridge["_id"], ) return customer_body = new_reply_content reply_plan = build_external_reply_plan( parent_external_message=external_parent, body_text=customer_body, subject_override=external_reply_subject( handoff.get("external_subject") or external_parent.get("subject", "") ), recipients=[external_recipient], parent_internet_id_override=parent_internet_id, ) logger.info( "handoff_external_reply_headers handoff_id=%s b_in_reply_to=%s b_references=%s", handoff["_id"], reply_plan.threading.in_reply_to, reply_plan.threading.references, ) try: external_doc = await send_external_reply( database, account_id=account_id, sender=agent_email, sender_name="Gentic Agent", external_thread_id=external_thread_id, plan=reply_plan, domain=domain, resolution_method="human_handoff_bridge", generated_by=GENERATED_BY_HANDOFF_BRIDGE, generated_from_handoff_message_id=human_message["_id"], ) except ExternalReplyThreadingError: await database.messages.update_one( {"_id": human_message["_id"]}, {"$set": {"relay_status": RELAY_STATUS_FAILED, "relay_failed_at": now}}, ) logger.error( "handoff_bridge_threading_rejected handoff_id=%s handoff_message=%s " "parent_internet_message_id=%s", handoff["_id"], human_message["_id"], parent_internet_id, ) return except Exception: await database.messages.update_one( {"_id": human_message["_id"]}, {"$set": {"relay_status": RELAY_STATUS_FAILED, "relay_failed_at": now}}, ) logger.exception( "handoff_bridge_send_failed handoff_id=%s handoff_message=%s", handoff["_id"], human_message["_id"], ) raise await database.messages.update_one( {"_id": human_message["_id"]}, { "$set": { "relay_status": RELAY_STATUS_SENT, "generated_external_message_id": external_doc["_id"], "relay_delivered_at": now, } }, ) await _record_message_link( database, handoff_id=handoff["_id"], external_message_id=external_doc["_id"], handoff_message_id=human_message["_id"], relationship=LINK_HUMAN_TRIGGERED_EXTERNAL, ) await database.human_handoffs.update_one( {"_id": handoff["_id"]}, { "$set": { "status": HANDOFF_STATUS_WAITING, "current_owner": OWNER_HUMAN, "updated_at": now, "last_human_message_at": now, } }, ) logger.info( "handoff_bridge_complete handoff_id=%s handoff_message=%s external=%s", handoff["_id"], human_message["_id"], external_doc["_id"], ) async def maybe_trigger_handoff( database, *, account_id: str, agent_email: str, external_message: dict, external_thread_id: ObjectId, domain, ) -> None: if not should_mirror_to_human(external_message): logger.debug( "handoff_mirror_skipped message_id=%s generated_by=%s origin=%s direction=%s", external_message.get("_id"), external_message.get("generated_by"), external_message.get("message_origin"), external_message.get("direction"), ) return existing = await find_active_handoff_for_external_thread( database, account_id, external_thread_id ) if existing: if await _handoff_mirror_exists(database, external_message["_id"]): return rule = await _handoff_rule_for_human( database, account_id, existing.get("human_email", "") ) if rule and not rule.get("allow_replies", True): return await send_handoff_to_human( database, account_id=account_id, agent_email=agent_email, human_email=existing["human_email"], external_message=external_message, external_thread_id=external_thread_id, handoff_reason=existing.get("handoff_reason", ""), domain=domain, existing_handoff=existing, ) return rules = await database.human_handoff_rules.find( {"account_id": account_id, "active": True} ).to_list(length=20) if not rules: return body_text = external_message.get("body_text") or external_message.get("snippet") or "" email_payload = { "subject": external_message.get("subject", ""), "snippet": external_message.get("snippet", ""), "body_text": body_text, "is_starred": False, } for rule in rules: evaluation = evaluate_handoff( email=email_payload, labels=rule.get("labels", []), email_types=rule.get("email_types", []), instructions=rule.get("instructions", ""), ) if not evaluation["would_handoff"]: continue if not rule.get("allow_replies", True): continue reason = (rule.get("instructions") or "").strip() or "This conversation requires human attention." await send_handoff_to_human( database, account_id=account_id, agent_email=agent_email, human_email=rule["owner_email"], external_message=external_message, external_thread_id=external_thread_id, handoff_reason=reason, domain=domain, existing_handoff=None, ) return