Suyash091/agentic.email
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

human handoffs

Browse source
This commit is contained in:
debu-2 2026-10-06 01:51:23 +05:30
parent 79188e0f59
commit 36a55930f9
35 changed files with 4039 additions and 139 deletions
Download patch file Download diff file Expand all files Collapse all files

117
backend/app/services/handoff_evaluator.py Normal file
View file

@ -0,0 +1,117 @@
import re
HANDOFF_LABELS = ["Important", "Finance", "Clients", "Projects", "Personal", "Support"]
EMAIL_TYPE_META = {
"utility": {
"title": "Utility",
"patterns": r"account update|alert|notice|security|verification|password|service information|noreply|no-reply",
},
"transactional": {
"title": "Transactional",
"patterns": r"order|payment|invoice|receipt|billing|subscription|shipped|confirmation|refund|approval",
},
"promotional": {
"title": "Promotional",
"patterns": r"offer|deal|discount|promo|sale|limited time|save \d+%",
},
"marketing": {
"title": "Marketing",
"patterns": r"newsletter|campaign|marketing|outreach|webinar|subscribe|unsubscribe",
},
}
def _email_text(email: dict) -> str:
parts = [
email.get("subject", ""),
email.get("snippet", ""),
email.get("body_text", ""),
]
return " ".join(parts).lower()
def infer_labels(email: dict) -> list[str]:
text = _email_text(email)
matched: list[str] = []
if re.search(r"urgent|important|asap|priority", text) or email.get("is_starred"):
matched.append("Important")
if re.search(r"invoice|bill|payment|finance|receipt", text):
matched.append("Finance")
if re.search(r"client|customer", text):
matched.append("Clients")
if re.search(r"project|proposal|contract", text):
matched.append("Projects")
if re.search(r"personal|family", text):
matched.append("Personal")
if re.search(r"support|help desk|ticket|assistance", text):
matched.append("Support")
return matched
def infer_email_types(email: dict) -> list[str]:
text = _email_text(email)
matched: list[str] = []
for key, meta in EMAIL_TYPE_META.items():
if re.search(meta["patterns"], text, flags=re.IGNORECASE):
matched.append(key)
return matched
def instruction_matches(instructions: str, email: dict) -> tuple[bool, str]:
text = _email_text(email)
instructions = (instructions or "").strip()
if not instructions:
return True, "No handoff instructions were set, so this step passes."
keywords = re.findall(r"[a-z]{4,}", instructions.lower())
topic_hits = [word for word in keywords if word in text and len(word) > 4]
if topic_hits:
return True, f"Mentions topics related to your instructions ({', '.join(topic_hits[:3])})."
intent_patterns = [
(r"refund|chargeback", "This email appears to involve a refund or payment issue."),
(r"approval|authorize|sign off", "This email appears to require approval."),
(r"human|speak with|talk to|call me", "The sender appears to be asking for a person."),
(r"not confident|unsure|escalat", "Your instructions mention uncertain situations."),
(r"urgent|asap|immediately", "The message appears urgent."),
]
for pattern, reason in intent_patterns:
if re.search(pattern, instructions, flags=re.IGNORECASE) and re.search(pattern, text, flags=re.IGNORECASE):
return True, reason
if re.search(r"refund|approval|human|speak with|person|payment", instructions, flags=re.IGNORECASE):
if re.search(r"refund|approval|human|speak|person|payment|decision", text, flags=re.IGNORECASE):
return True, "The email situation matches your handoff instructions."
return False, "This email does not clearly match your handoff instructions."
def evaluate_handoff(
*,
email: dict,
labels: list[str],
email_types: list[str],
instructions: str,
) -> dict:
inferred_labels = infer_labels(email)
inferred_types = infer_email_types(email)
label_ok = not labels or any(label in inferred_labels for label in labels)
matched_labels = [label for label in labels if label in inferred_labels] if labels else inferred_labels[:1]
type_ok = not email_types or any(item in inferred_types for item in email_types)
matched_types = [item for item in email_types if item in inferred_types] if email_types else inferred_types[:1]
instruction_ok, instruction_reason = instruction_matches(instructions, email)
would_handoff = label_ok and type_ok and instruction_ok
return {
"label_matched": label_ok,
"matched_labels": matched_labels,
"type_matched": type_ok,
"matched_types": matched_types,
"instruction_matched": instruction_ok,
"instruction_reason": instruction_reason,
"would_handoff": would_handoff,
}

669
backend/app/services/human_handoff_service.py Normal file
View file

@ -0,0 +1,669 @@
import logging
import re
from datetime import datetime, timezone
from bson import ObjectId
from app.services.handoff_evaluator import evaluate_handoff
from app.services.mail_sender import deliver_email
from app.services.message_ids import build_references_header, normalize_message_id, parse_message_id_list
from app.services.thread_service import (
attach_message_to_thread,
message_event_at,
threading_fields_from_parent,
)
logger = logging.getLogger(__name__)
MESSAGE_SCOPE_EXTERNAL = "EXTERNAL"
MESSAGE_SCOPE_HANDOFF = "HANDOFF"
THREAD_TYPE_EXTERNAL = "EXTERNAL"
THREAD_TYPE_HUMAN_HANDOFF = "HUMAN_HANDOFF"
HANDOFF_STATUS_ACTIVE = "ACTIVE"
HANDOFF_STATUS_WAITING = "WAITING_FOR_HUMAN"
HANDOFF_STATUS_HUMAN_REPLIED = "HUMAN_REPLIED"
HANDOFF_STATUS_AGENT_REPLIED = "AGENT_REPLIED"
OWNER_AGENT = "AGENT"
OWNER_HUMAN = "HUMAN"
LINK_EXTERNAL_TRIGGERED = "EXTERNAL_TRIGGERED_HANDOFF"
LINK_EXTERNAL_RELAYED = "EXTERNAL_RELAYED_TO_HUMAN"
LINK_HUMAN_TRIGGERED_EXTERNAL = "HUMAN_TRIGGERED_EXTERNAL_REPLY"
LINK_EXTERNAL_UPDATE_RELAYED = "EXTERNAL_UPDATE_RELAYED_TO_HUMAN"
def _now() -> datetime:
return datetime.now(timezone.utc)
def _snippet(text: str, limit: int = 240) -> str:
return re.sub(r"\s+", " ", (text or "")).strip()[:limit]
def _handoff_subject(external_subject: str) -> str:
subject = (external_subject or "(No subject)").strip()
if subject.lower().startswith("human handoff:"):
return subject
return f"Human Handoff: {subject}"
def _build_handoff_body(
*,
external_sender: str,
external_subject: str,
external_body: str,
handoff_reason: str,
agent_email: str,
) -> str:
return (
"Human Handoff\n\n"
"A conversation needs your attention.\n\n"
f"From\n{external_sender}\n\n"
f"Subject\n{external_subject}\n\n"
"Latest message\n\n"
f"\"{external_body.strip()}\"\n\n"
"Why this was handed off\n\n"
f"{handoff_reason}\n\n"
"────────────────────────────────────────\n\n"
f"Reply to this email to respond to the original sender through {agent_email}.\n\n"
"Your reply will be sent to the customer.\n\n"
"────────────────────────────────────────\n\n"
"Gentic Human Handoff"
)
async def create_empty_handoff_thread(
database,
*,
account_id: str,
subject: str,
) -> ObjectId:
now = _now()
document = {
"account_id": account_id,
"thread_type": THREAD_TYPE_HUMAN_HANDOFF,
"normalized_subject": subject,
"created_at": now,
"updated_at": now,
"last_message_at": now,
"latest_message_id": None,
"message_count": 0,
}
result = await database.email_threads.insert_one(document)
return result.inserted_id
async def _record_message_link(
database,
*,
handoff_id: ObjectId,
external_message_id: ObjectId | None,
handoff_message_id: ObjectId,
relationship: str,
) -> None:
await database.handoff_message_links.insert_one(
{
"handoff_id": handoff_id,
"external_message_id": external_message_id,
"handoff_message_id": handoff_message_id,
"relationship": relationship,
"created_at": _now(),
}
)
async def _latest_external_message(database, account_id: str, external_thread_id: ObjectId) -> dict | None:
pipeline = [
{
"$match": {
"account_id": account_id,
"thread_id": external_thread_id,
"message_scope": MESSAGE_SCOPE_EXTERNAL,
"deletion_state": {"$in": [None, ""]},
}
},
{"$sort": {"event_at": -1, "_id": -1}},
{"$limit": 1},
{
"$lookup": {
"from": "messages",
"localField": "message_id",
"foreignField": "_id",
"as": "message",
}
},
{"$unwind": "$message"},
]
rows = [row async for row in database.mailbox_entries.aggregate(pipeline)]
return rows[0]["message"] if rows else None
async def _latest_handoff_message(database, account_id: str, handoff_thread_id: ObjectId) -> dict | None:
pipeline = [
{
"$match": {
"account_id": account_id,
"thread_id": handoff_thread_id,
"message_scope": MESSAGE_SCOPE_HANDOFF,
}
},
{"$sort": {"event_at": -1, "_id": -1}},
{"$limit": 1},
{
"$lookup": {
"from": "messages",
"localField": "message_id",
"foreignField": "_id",
"as": "message",
}
},
{"$unwind": "$message"},
]
rows = [row async for row in database.mailbox_entries.aggregate(pipeline)]
return rows[0]["message"] if rows else None
async def find_active_handoff_for_external_thread(
database,
account_id: str,
external_thread_id: ObjectId,
) -> dict | None:
return await database.human_handoffs.find_one(
{
"account_id": account_id,
"external_thread_id": external_thread_id,
"status": {"$in": [HANDOFF_STATUS_ACTIVE, HANDOFF_STATUS_WAITING, HANDOFF_STATUS_HUMAN_REPLIED, HANDOFF_STATUS_AGENT_REPLIED]},
}
)
async def resolve_handoff_from_inbound(
database,
*,
account_id: str,
agent_email: str,
sender_address: str,
in_reply_to: str | None,
references: list[str],
) -> dict | None:
sender = sender_address.lower()
rules = await database.human_handoff_rules.find(
{"account_id": account_id, "owner_email": sender, "active": True}
).to_list(length=10)
if not rules:
return None
search_ids = [*([in_reply_to] if in_reply_to else []), *references]
if not search_ids:
return None
for internet_id in reversed(parse_message_id_list(" ".join(search_ids))):
message = await database.messages.find_one(
{
"message_id": internet_id,
"message_scope": MESSAGE_SCOPE_HANDOFF,
}
)
if message is None:
continue
entry = await database.mailbox_entries.find_one(
{
"account_id": account_id,
"message_id": message["_id"],
"message_scope": MESSAGE_SCOPE_HANDOFF,
}
)
if entry is None:
continue
handoff = await database.human_handoffs.find_one(
{
"account_id": account_id,
"handoff_thread_id": entry["thread_id"],
"human_email": sender,
"status": {"$ne": "CLOSED"},
}
)
if handoff:
return handoff
return None
async def store_handoff_message(
database,
*,
account_id: str,
agent_email: str,
handoff_thread_id: ObjectId,
direction: str,
sender: dict,
recipients: list[dict],
subject: str,
body_text: str,
internet_message_id: str | None = None,
in_reply_to: str | None = None,
references: list[str] | None = None,
handoff_id: ObjectId | None = None,
link_relationship: str | None = None,
external_message_id: ObjectId | None = None,
event_at: datetime | None = None,
) -> dict:
now = event_at or _now()
document = {
"direction": direction,
"message_scope": MESSAGE_SCOPE_HANDOFF,
"account_id": account_id,
"sender": sender,
"to": recipients,
"subject": subject,
"snippet": _snippet(body_text),
"body_text": body_text,
"body_html": None,
"attachments": [],
"created_at": now,
"sent_at": now if direction == "outbound" else None,
"received_at": now if direction == "inbound" else None,
"status": "received" if direction == "inbound" else "accepted",
"message_id": normalize_message_id(internet_message_id),
"in_reply_to": normalize_message_id(in_reply_to),
"references": references or [],
}
result = await database.messages.insert_one(document)
document["_id"] = result.inserted_id
await attach_message_to_thread(
database,
account_id=account_id,
thread_id=handoff_thread_id,
message_id=result.inserted_id,
subject=subject,
event_at=now,
)
await database.mailbox_entries.insert_one(
{
"account_id": account_id,
"message_id": result.inserted_id,
"thread_id": handoff_thread_id,
"mailbox": "handoff",
"message_scope": MESSAGE_SCOPE_HANDOFF,
"direction": direction,
"sender": sender,
"recipients": recipients,
"subject": subject,
"snippet": document["snippet"],
"attachments": [],
"status": document["status"],
"is_read": direction == "outbound",
"is_starred": False,
"event_at": now,
"created_at": now,
"trashed_at": None,
"deletion_state": None,
}
)
if handoff_id and link_relationship:
await _record_message_link(
database,
handoff_id=handoff_id,
external_message_id=external_message_id,
handoff_message_id=result.inserted_id,
relationship=link_relationship,
)
return document
async def send_handoff_to_human(
database,
*,
account_id: str,
agent_email: str,
human_email: str,
external_message: dict,
external_thread_id: ObjectId,
handoff_reason: str,
domain,
existing_handoff: dict | None = None,
) -> dict:
external_sender = external_message.get("sender", {})
external_sender_text = external_sender.get("address") or "unknown sender"
external_subject = external_message.get("subject", "")
external_body = external_message.get("body_text") or external_message.get("snippet") or ""
if existing_handoff:
handoff_id = existing_handoff["_id"]
handoff_thread_id = existing_handoff["handoff_thread_id"]
parent = await _latest_handoff_message(database, account_id, handoff_thread_id)
threading = threading_fields_from_parent(parent) if parent else {}
subject = _handoff_subject(external_subject)
body = _build_handoff_body(
external_sender=external_sender_text,
external_subject=external_subject,
external_body=external_body,
handoff_reason="New message in the linked conversation.",
agent_email=agent_email,
)
relationship = LINK_EXTERNAL_UPDATE_RELAYED
else:
handoff_thread_id = await create_empty_handoff_thread(
database,
account_id=account_id,
subject=_handoff_subject(external_subject),
)
threading = {}
subject = _handoff_subject(external_subject)
body = _build_handoff_body(
external_sender=external_sender_text,
external_subject=external_subject,
external_body=external_body,
handoff_reason=handoff_reason,
agent_email=agent_email,
)
handoff_doc = {
"account_id": account_id,
"agent_email": agent_email.lower(),
"human_email": human_email.lower(),
"external_thread_id": external_thread_id,
"handoff_thread_id": handoff_thread_id,
"external_subject": external_subject,
"external_sender": external_sender_text,
"handoff_reason": handoff_reason,
"status": HANDOFF_STATUS_WAITING,
"current_owner": OWNER_HUMAN,
"created_at": _now(),
"updated_at": _now(),
"last_external_message_at": message_event_at(external_message),
"last_human_message_at": None,
}
insert = await database.human_handoffs.insert_one(handoff_doc)
handoff_id = insert.inserted_id
relationship = LINK_EXTERNAL_TRIGGERED
internet_message_id = await deliver_email(
sender=agent_email,
recipients=[human_email],
cc=[],
bcc=[],
subject=subject,
body_text=body,
body_html=None,
domain=domain,
in_reply_to=threading.get("in_reply_to"),
references=threading.get("references"),
)
handoff_message = await store_handoff_message(
database,
account_id=account_id,
agent_email=agent_email,
handoff_thread_id=handoff_thread_id,
direction="outbound",
sender={"name": "Gentic Agent", "address": agent_email},
recipients=[{"name": human_email.split("@")[0], "address": human_email}],
subject=subject,
body_text=body,
internet_message_id=internet_message_id,
in_reply_to=threading.get("in_reply_to"),
references=threading.get("references"),
handoff_id=handoff_id,
link_relationship=relationship,
external_message_id=external_message["_id"],
)
await database.human_handoffs.update_one(
{"_id": handoff_id},
{
"$set": {
"status": HANDOFF_STATUS_WAITING,
"current_owner": OWNER_HUMAN,
"updated_at": _now(),
"last_external_message_at": message_event_at(external_message),
"latest_handoff_message_id": handoff_message["_id"],
}
},
)
logger.info(
"handoff_sent handoff_id=%s external_thread=%s handoff_thread=%s",
handoff_id,
external_thread_id,
handoff_thread_id,
)
return await database.human_handoffs.find_one({"_id": handoff_id})
async def _handoff_rule_for_human(database, account_id: str, human_email: str) -> dict | None:
return await database.human_handoff_rules.find_one(
{
"account_id": account_id,
"owner_email": human_email.lower(),
"active": True,
}
)
async def process_human_reply(
database,
*,
account_id: str,
agent_email: str,
handoff: dict,
inbound_parsed: dict,
inbound_body: str,
domain,
) -> None:
handoff_thread_id = handoff["handoff_thread_id"]
external_thread_id = handoff["external_thread_id"]
human_email = handoff["human_email"]
rule = await _handoff_rule_for_human(database, account_id, human_email)
allow_replies = rule.get("allow_replies", True) if rule else True
human_message = await store_handoff_message(
database,
account_id=account_id,
agent_email=agent_email,
handoff_thread_id=handoff_thread_id,
direction="inbound",
sender={"name": human_email.split("@")[0], "address": human_email},
recipients=[{"name": "Gentic Agent", "address": agent_email}],
subject=inbound_parsed.get("subject", ""),
body_text=inbound_body,
internet_message_id=inbound_parsed.get("message_id"),
in_reply_to=inbound_parsed.get("in_reply_to"),
references=inbound_parsed.get("references", []),
handoff_id=handoff["_id"],
link_relationship=LINK_HUMAN_TRIGGERED_EXTERNAL,
)
now = _now()
if not allow_replies:
await database.human_handoffs.update_one(
{"_id": handoff["_id"]},
{
"$set": {
"status": HANDOFF_STATUS_HUMAN_REPLIED,
"current_owner": OWNER_AGENT,
"updated_at": now,
"last_human_message_at": now,
}
},
)
return
external_parent = await _latest_external_message(database, account_id, external_thread_id)
if external_parent is None:
logger.warning("handoff_reply_missing_external_parent handoff_id=%s", handoff["_id"])
return
threading = threading_fields_from_parent(external_parent)
external_subject = handoff.get("external_subject") or external_parent.get("subject", "")
if not re.match(r"^re:\s*", external_subject, flags=re.IGNORECASE):
external_subject = f"Re: {external_subject}"
external_recipient = handoff.get("external_sender") or external_parent.get("sender", {}).get("address")
if not external_recipient:
return
customer_body = inbound_body.strip()
internet_message_id = await deliver_email(
sender=agent_email,
recipients=[external_recipient],
cc=[],
bcc=[],
subject=external_subject,
body_text=customer_body,
body_html=None,
domain=domain,
in_reply_to=threading.get("in_reply_to"),
references=threading.get("references"),
)
external_doc = {
"direction": "outbound",
"message_scope": MESSAGE_SCOPE_EXTERNAL,
"account_id": account_id,
"sender": {"name": "Gentic Agent", "address": agent_email},
"to": [{"name": external_recipient.split("@")[0], "address": external_recipient}],
"subject": external_subject,
"snippet": _snippet(customer_body),
"body_text": customer_body,
"body_html": None,
"attachments": [],
"created_at": now,
"sent_at": now,
"status": "accepted",
"message_id": normalize_message_id(internet_message_id),
"in_reply_to": threading.get("in_reply_to"),
"references": threading.get("references", []),
"parent_message_id": external_parent["_id"],
"thread_relation": "reply",
"thread_resolution_method": "human_handoff_bridge",
}
ext_result = await database.messages.insert_one(external_doc)
external_doc["_id"] = ext_result.inserted_id
await attach_message_to_thread(
database,
account_id=account_id,
thread_id=external_thread_id,
message_id=ext_result.inserted_id,
subject=external_subject,
event_at=now,
)
await database.mailbox_entries.insert_one(
{
"account_id": account_id,
"message_id": ext_result.inserted_id,
"thread_id": external_thread_id,
"mailbox": "sent",
"message_scope": MESSAGE_SCOPE_EXTERNAL,
"direction": "outbound",
"sender": external_doc["sender"],
"recipients": external_doc["to"],
"subject": external_subject,
"snippet": external_doc["snippet"],
"attachments": [],
"status": "accepted",
"is_read": True,
"is_starred": False,
"event_at": now,
"created_at": now,
"trashed_at": None,
"deletion_state": None,
}
)
relay_body = f"Sent to customer:\n\n\"{customer_body}\""
relay_subject = f"Re: {_handoff_subject(handoff.get('external_subject', ''))}"
relay_threading = threading_fields_from_parent(human_message)
relay_internet_id = await deliver_email(
sender=agent_email,
recipients=[human_email],
cc=[],
bcc=[],
subject=relay_subject,
body_text=relay_body,
body_html=None,
domain=domain,
in_reply_to=relay_threading.get("in_reply_to"),
references=relay_threading.get("references"),
)
await store_handoff_message(
database,
account_id=account_id,
agent_email=agent_email,
handoff_thread_id=handoff_thread_id,
direction="outbound",
sender={"name": "Gentic Agent", "address": agent_email},
recipients=[{"name": human_email.split("@")[0], "address": human_email}],
subject=relay_subject,
body_text=relay_body,
internet_message_id=relay_internet_id,
in_reply_to=relay_threading.get("in_reply_to"),
references=relay_threading.get("references"),
handoff_id=handoff["_id"],
link_relationship=LINK_EXTERNAL_RELAYED,
external_message_id=ext_result.inserted_id,
)
await database.human_handoffs.update_one(
{"_id": handoff["_id"]},
{
"$set": {
"status": HANDOFF_STATUS_AGENT_REPLIED,
"current_owner": OWNER_AGENT,
"updated_at": now,
"last_human_message_at": now,
}
},
)
async def maybe_trigger_handoff(
database,
*,
account_id: str,
agent_email: str,
external_message: dict,
external_thread_id: ObjectId,
domain,
) -> None:
rules = await database.human_handoff_rules.find(
{"account_id": account_id, "active": True}
).to_list(length=20)
if not rules:
return
body_text = external_message.get("body_text") or external_message.get("snippet") or ""
email_payload = {
"subject": external_message.get("subject", ""),
"snippet": external_message.get("snippet", ""),
"body_text": body_text,
"is_starred": False,
}
for rule in rules:
evaluation = evaluate_handoff(
email=email_payload,
labels=rule.get("labels", []),
email_types=rule.get("email_types", []),
instructions=rule.get("instructions", ""),
)
if not evaluation["would_handoff"]:
continue
if not rule.get("allow_replies", True):
continue
existing = await find_active_handoff_for_external_thread(
database, account_id, external_thread_id
)
reason = evaluation.get("instruction_reason") or rule.get("instructions") or "Matched handoff rules."
await send_handoff_to_human(
database,
account_id=account_id,
agent_email=agent_email,
human_email=rule["owner_email"],
external_message=external_message,
external_thread_id=external_thread_id,
handoff_reason=reason,
domain=domain,
existing_handoff=existing,
)
return

133
backend/app/services/inbound_smtp.py
View file

@ -9,8 +9,18 @@ from sqlalchemy import func, select
from app.config import get_settings
from app.database import SessionFactory, mongo_db
from app.models import Account
from app.services.domain_provisioning import get_domain_by_name
from app.services.human_handoff_service import (
MESSAGE_SCOPE_EXTERNAL,
MESSAGE_SCOPE_HANDOFF,
process_human_reply,
resolve_handoff_from_inbound,
maybe_trigger_handoff,
)
from app.services.mime_parser import parse_mime
from app.services.raw_mime_storage import delivery_digest, get_raw_mime_storage
from app.services.thread_resolver import participants_from_parsed, resolve_thread
from app.services.thread_service import ensure_thread_for_message, message_event_at
logger = logging.getLogger(__name__)
@ -69,9 +79,59 @@ class InboundHandler:
body_html = parsed.pop("body_html")
stored = await get_raw_mime_storage().put(raw, delivery_key)
now = datetime.now(timezone.utc)
sender_address = (parsed.get("sender") or {}).get("address", "").lower()
for recipient in recipients:
account_id = envelope.account_ids[recipient]
agent_email = recipient
handoff = await resolve_handoff_from_inbound(
mongo_db,
account_id=account_id,
agent_email=agent_email,
sender_address=sender_address,
in_reply_to=parsed.get("in_reply_to"),
references=parsed.get("references", []),
)
if handoff:
async with SessionFactory() as database_session:
domain = await get_domain_by_name(
database_session,
agent_email.split("@", 1)[1],
)
if domain is None:
logger.warning("handoff_reply_no_domain account=%s", account_id)
continue
message = {
**parsed,
"body_text": body_text,
"body_html": body_html,
"direction": "inbound",
"message_scope": MESSAGE_SCOPE_HANDOFF,
"delivery_key": delivery_key,
"received_at": now,
"created_at": now,
}
await process_human_reply(
mongo_db,
account_id=account_id,
agent_email=agent_email,
handoff=handoff,
inbound_parsed=message,
inbound_body=body_text,
domain=domain,
)
logger.info(
"handoff_human_reply handoff_id=%s account=%s",
handoff["_id"],
account_id,
)
return f"250 2.0.0 Handoff reply accepted as {delivery_key}"
message = {
**parsed,
"direction": "inbound",
"message_scope": MESSAGE_SCOPE_EXTERNAL,
"envelope_sender": envelope.mail_from.lower(),
"envelope_recipients": recipients,
"delivery_key": delivery_key,
@ -92,14 +152,54 @@ class InboundHandler:
{"$setOnInsert": message},
upsert=True,
)
persisted = await mongo_db.messages.find_one(
{"delivery_key": delivery_key},
{"_id": 1},
)
persisted = await mongo_db.messages.find_one({"delivery_key": delivery_key})
if persisted is None:
raise RuntimeError("Inbound message was not persisted.")
if body_text:
await mongo_db.messages.update_one(
{"_id": persisted["_id"]},
{"$set": {"body_text": body_text, "body_html": body_html}},
)
persisted["body_text"] = body_text
persisted["body_html"] = body_html
participants = participants_from_parsed(parsed)
for recipient in recipients:
account_id = envelope.account_ids[recipient]
agent_email = recipient
resolution = await resolve_thread(
mongo_db,
account_id=account_id,
in_reply_to=parsed.get("in_reply_to"),
references=parsed.get("references", []),
subject=parsed.get("subject", ""),
participants=participants,
)
thread_id = await ensure_thread_for_message(
mongo_db,
account_id=account_id,
message=persisted,
resolution=resolution,
)
await mongo_db.email_threads.update_one(
{"_id": thread_id},
{"$set": {"thread_type": "EXTERNAL"}},
)
await mongo_db.messages.update_one(
{"_id": persisted["_id"]},
{
"$set": {
"message_scope": MESSAGE_SCOPE_EXTERNAL,
"in_reply_to": parsed.get("in_reply_to"),
"references": parsed.get("references", []),
"raw_headers": parsed.get("raw_headers", {}),
"parent_message_id": resolution.parent_message_id,
"thread_relation": resolution.thread_relation,
"thread_resolution_method": resolution.thread_resolution_method,
}
},
)
await mongo_db.mailbox_entries.update_one(
{
"account_id": account_id,
@ -111,6 +211,7 @@ class InboundHandler:
"account_id": account_id,
"message_id": persisted["_id"],
"mailbox": "inbox",
"message_scope": MESSAGE_SCOPE_EXTERNAL,
"direction": "inbound",
"sender": parsed["sender"],
"recipients": parsed["to"],
@ -120,14 +221,34 @@ class InboundHandler:
"status": "received",
"is_read": False,
"is_starred": False,
"event_at": now,
"event_at": message_event_at(persisted),
"created_at": now,
"trashed_at": None,
"deletion_state": None,
}
},
"$set": {"thread_id": thread_id},
},
upsert=True,
)
async with SessionFactory() as database_session:
domain = await get_domain_by_name(
database_session,
agent_email.split("@", 1)[1],
)
if domain is not None:
await maybe_trigger_handoff(
mongo_db,
account_id=account_id,
agent_email=agent_email,
external_message=persisted,
external_thread_id=thread_id,
domain=domain,
)
logger.info(
"message_received message_id=%s thread_id=%s scope=EXTERNAL",
persisted["_id"],
thread_id,
)
except Exception:
logger.exception("Temporary failure accepting inbound message")
return "451 4.3.0 Temporary server failure; please retry"

19
backend/app/services/mail_sender.py
View file

@ -20,6 +20,9 @@ def build_message(
body_text: str,
body_html: str | None,
reply_to: str | None = None,
in_reply_to: str | None = None,
references: list[str] | None = None,
message_id: str | None = None,
) -> EmailMessage:
message = EmailMessage(policy=SMTP_POLICY)
message["From"] = sender
@ -30,7 +33,12 @@ def build_message(
message["Reply-To"] = reply_to
message["Subject"] = subject
message["Date"] = email.utils.formatdate(localtime=True)
message["Message-ID"] = email.utils.make_msgid(domain=sender.split("@")[-1])
generated_id = message_id or email.utils.make_msgid(domain=sender.split("@")[-1])
message["Message-ID"] = generated_id
if in_reply_to:
message["In-Reply-To"] = in_reply_to
if references:
message["References"] = " ".join(references)
message.set_content(body_text)
if body_html:
message.add_alternative(body_html, subtype="html")
@ -95,7 +103,10 @@ async def deliver_email(
body_html: str | None,
domain: MailDomain,
reply_to: str | None = None,
) -> None:
in_reply_to: str | None = None,
references: list[str] | None = None,
message_id: str | None = None,
) -> str:
message = build_message(
sender=sender,
recipients=recipients,
@ -104,6 +115,9 @@ async def deliver_email(
body_text=body_text,
body_html=body_html,
reply_to=reply_to,
in_reply_to=in_reply_to,
references=references,
message_id=message_id,
)
signed = sign_message(message, domain)
await send_via_smtp(
@ -111,3 +125,4 @@ async def deliver_email(
sender=sender,
recipients=[*recipients, *cc, *bcc],
)
return str(message["Message-ID"])

18
backend/app/services/mailbox_scope.py Normal file
View file

@ -0,0 +1,18 @@
EXTERNAL_SCOPE_FILTER = {
"$or": [
{"message_scope": "EXTERNAL"},
{"message_scope": {"$exists": False}},
]
}
def apply_external_scope(entry_match: dict) -> dict:
if "$and" in entry_match:
entry_match["$and"].append(EXTERNAL_SCOPE_FILTER)
return entry_match
if "$or" in entry_match:
existing = entry_match.pop("$or")
entry_match["$and"] = [{"$or": existing}, EXTERNAL_SCOPE_FILTER]
return entry_match
entry_match.update(EXTERNAL_SCOPE_FILTER)
return entry_match

51
backend/app/services/message_ids.py Normal file
View file

@ -0,0 +1,51 @@
import re
_MESSAGE_ID_PATTERN = re.compile(r"<[^>]+>")
def normalize_message_id(value: str | None) -> str | None:
if value is None:
return None
cleaned = value.strip()
if not cleaned:
return None
if not cleaned.startswith("<"):
cleaned = f"<{cleaned.strip('<>')}>"
return cleaned.lower()
def parse_message_id_list(header_value: str | None) -> list[str]:
if not header_value:
return []
folded = re.sub(r"\s+", " ", header_value.strip())
seen: set[str] = set()
ordered: list[str] = []
for match in _MESSAGE_ID_PATTERN.findall(folded):
normalized = normalize_message_id(match)
if normalized and normalized not in seen:
seen.add(normalized)
ordered.append(normalized)
if not ordered:
single = normalize_message_id(folded)
if single:
ordered.append(single)
return ordered
def first_message_id(header_value: str | None) -> str | None:
values = parse_message_id_list(header_value)
return values[0] if values else None
def build_references_header(references: list[str], parent_message_id: str) -> str:
chain: list[str] = []
seen: set[str] = set()
for value in references:
normalized = normalize_message_id(value)
if normalized and normalized not in seen:
seen.add(normalized)
chain.append(normalized)
parent = normalize_message_id(parent_message_id)
if parent and parent not in seen:
chain.append(parent)
return " ".join(chain)

21
backend/app/services/mime_parser.py
View file

@ -5,6 +5,8 @@ from email.parser import BytesParser
from email.utils import getaddresses, parsedate_to_datetime
from html import unescape
from app.services.message_ids import first_message_id, parse_message_id_list
def _addresses(values: list[str]) -> list[dict[str, str]]:
return [
@ -75,8 +77,25 @@ def parse_mime(raw: bytes) -> dict:
pass
body_text = "\n".join(text_parts).strip()
body_html = "\n".join(html_parts).strip() or None
internet_message_id = first_message_id(message.get("Message-ID"))
in_reply_to = first_message_id(message.get("In-Reply-To"))
references = parse_message_id_list(message.get("References"))
raw_headers = {
"message_id": message.get("Message-ID"),
"in_reply_to": message.get("In-Reply-To"),
"references": message.get("References"),
"subject": message.get("Subject"),
"date": message.get("Date"),
"from": message.get("From"),
"to": message.get("To"),
"cc": message.get("Cc"),
"reply_to": message.get("Reply-To"),
}
return {
"message_id": (message.get("Message-ID") or "").strip() or None,
"message_id": internet_message_id,
"in_reply_to": in_reply_to,
"references": references,
"raw_headers": raw_headers,
"sender": from_value,
"to": _addresses(message.get_all("to", [])),
"cc": _addresses(message.get_all("cc", [])),

49
backend/app/services/reply_recipients.py Normal file
View file

@ -0,0 +1,49 @@
from app.schemas import MailAddress
def _addresses(items: list[dict] | list[MailAddress] | None) -> list[str]:
if not items:
return []
values: list[str] = []
for item in items:
if isinstance(item, dict):
address = item.get("address", "")
else:
address = item.address
if address:
values.append(address.lower())
return values
def reply_recipients(
*,
message: dict,
viewer_email: str,
reply_all: bool = False,
) -> tuple[list[str], list[str]]:
viewer = viewer_email.lower()
own_addresses = {viewer}
reply_to = _addresses(message.get("reply_to"))
sender = _addresses([message.get("sender", {})])
to = _addresses(message.get("to") or message.get("recipients"))
cc = _addresses(message.get("cc"))
primary = reply_to or sender
primary = [address for address in primary if address not in own_addresses]
if not reply_all:
return primary, []
recipients = []
for address in [*primary, *to, *cc]:
if address in own_addresses or address in recipients:
continue
recipients.append(address)
if not recipients and primary:
recipients = primary
to_out = recipients[:1] if recipients else []
cc_out = recipients[1:] if len(recipients) > 1 else []
return to_out, cc_out

16
backend/app/services/subject_normalize.py Normal file
View file

@ -0,0 +1,16 @@
import re
_PREFIX_PATTERN = re.compile(
r"^(?:(?:re|fw|fwd)\s*(?:\[\d+\])?\s*:\s*)+",
flags=re.IGNORECASE,
)
def normalize_subject(subject: str | None) -> str:
value = (subject or "").strip()
while True:
next_value = _PREFIX_PATTERN.sub("", value, count=1).strip()
if next_value == value:
break
value = next_value
return re.sub(r"\s+", " ", value).strip().lower()

252
backend/app/services/thread_resolver.py Normal file
View file

@ -0,0 +1,252 @@
import logging
from dataclasses import dataclass
from datetime import datetime, timedelta, timezone
from bson import ObjectId
from app.services.message_ids import (
first_message_id,
normalize_message_id,
parse_message_id_list,
)
from app.services.subject_normalize import normalize_subject
logger = logging.getLogger(__name__)
FALLBACK_WINDOW = timedelta(days=7)
@dataclass(frozen=True)
class ThreadResolution:
thread_id: ObjectId | None
parent_message_id: ObjectId | None
thread_relation: str
thread_resolution_method: str
@dataclass(frozen=True)
class KnownMessage:
id: ObjectId
thread_id: ObjectId
internet_message_id: str
account_id: str
sent_at: datetime | None
received_at: datetime | None
created_at: datetime
participants: frozenset[str]
normalized_subject: str
def _message_timestamp(message: KnownMessage) -> datetime:
return message.sent_at or message.received_at or message.created_at
def resolve_thread_from_known(
*,
account_id: str,
in_reply_to: str | None,
references: list[str],
subject: str,
participants: set[str],
known_by_internet_id: dict[str, KnownMessage],
known_threads: dict[ObjectId, list[KnownMessage]] | None = None,
now: datetime | None = None,
) -> ThreadResolution:
scoped = {
key: value
for key, value in known_by_internet_id.items()
if value.account_id == account_id
}
in_reply = normalize_message_id(in_reply_to)
if in_reply and in_reply in scoped:
parent = scoped[in_reply]
return ThreadResolution(
thread_id=parent.thread_id,
parent_message_id=parent.id,
thread_relation="reply",
thread_resolution_method="in_reply_to",
)
reference_ids = [
normalize_message_id(value)
for value in references
if normalize_message_id(value)
]
for reference in reversed(reference_ids):
if reference in scoped:
parent = scoped[reference]
return ThreadResolution(
thread_id=parent.thread_id,
parent_message_id=parent.id,
thread_relation="reply",
thread_resolution_method="references",
)
fallback = _conservative_fallback(
account_id=account_id,
subject=subject,
participants=participants,
known_by_internet_id=scoped,
known_threads=known_threads or {},
now=now or datetime.now(timezone.utc),
)
if fallback is not None:
return fallback
return ThreadResolution(
thread_id=None,
parent_message_id=None,
thread_relation="new_thread",
thread_resolution_method="new_thread",
)
def _conservative_fallback(
*,
account_id: str,
subject: str,
participants: set[str],
known_by_internet_id: dict[str, KnownMessage],
known_threads: dict[ObjectId, list[KnownMessage]],
now: datetime,
) -> ThreadResolution | None:
normalized = normalize_subject(subject)
if not normalized or not participants:
return None
candidates: dict[ObjectId, KnownMessage] = {}
for message in known_by_internet_id.values():
if message.normalized_subject != normalized:
continue
if not participants.intersection(message.participants):
continue
timestamp = _message_timestamp(message)
if timestamp < now - FALLBACK_WINDOW:
continue
candidates[message.thread_id] = message
if len(candidates) != 1:
return None
parent = next(iter(candidates.values()))
return ThreadResolution(
thread_id=parent.thread_id,
parent_message_id=parent.id,
thread_relation="unknown",
thread_resolution_method="fallback",
)
def participants_from_parsed(parsed: dict) -> set[str]:
addresses: set[str] = set()
for key in ("sender",):
sender = parsed.get(key)
if isinstance(sender, dict) and sender.get("address"):
addresses.add(sender["address"].lower())
for key in ("to", "cc", "reply_to"):
for item in parsed.get(key, []) or []:
if isinstance(item, dict) and item.get("address"):
addresses.add(item["address"].lower())
return addresses
async def lookup_known_messages(
database,
account_id: str,
internet_ids: list[str],
) -> dict[str, KnownMessage]:
normalized_ids = [
value
for value in {normalize_message_id(item) for item in internet_ids}
if value
]
if not normalized_ids:
return {}
pipeline = [
{
"$match": {
"message_id": {"$in": normalized_ids},
"$or": [
{"message_scope": "EXTERNAL"},
{"message_scope": {"$exists": False}},
],
}
},
{
"$lookup": {
"from": "mailbox_entries",
"let": {"message_id": "$_id"},
"pipeline": [
{
"$match": {
"$expr": {
"$and": [
{"$eq": ["$message_id", "$$message_id"]},
{"$eq": ["$account_id", account_id]},
]
}
}
},
{"$limit": 1},
],
"as": "entry",
}
},
{"$match": {"entry.0": {"$exists": True}}},
]
known: dict[str, KnownMessage] = {}
cursor = database.messages.aggregate(pipeline)
async for document in cursor:
internet_id = normalize_message_id(document.get("message_id"))
if not internet_id:
continue
entry = document["entry"][0]
thread_id = entry.get("thread_id") or document.get("thread_id")
if thread_id is None:
continue
participants = participants_from_parsed(document)
known[internet_id] = KnownMessage(
id=document["_id"],
thread_id=thread_id,
internet_message_id=internet_id,
account_id=account_id,
sent_at=document.get("sent_at"),
received_at=document.get("received_at"),
created_at=document.get("created_at"),
participants=frozenset(participants),
normalized_subject=normalize_subject(document.get("subject")),
)
return known
async def resolve_thread(
database,
*,
account_id: str,
in_reply_to: str | None,
references: list[str],
subject: str,
participants: set[str],
) -> ThreadResolution:
search_ids = [
*([in_reply_to] if in_reply_to else []),
*references,
]
known = await lookup_known_messages(database, account_id, search_ids)
resolution = resolve_thread_from_known(
account_id=account_id,
in_reply_to=in_reply_to,
references=references,
subject=subject,
participants=participants,
known_by_internet_id=known,
)
logger.info(
"thread_resolved account_id=%s method=%s relation=%s parent_found=%s",
account_id,
resolution.thread_resolution_method,
resolution.thread_relation,
resolution.parent_message_id is not None,
)
return resolution

137
backend/app/services/thread_service.py Normal file
View file

@ -0,0 +1,137 @@
from datetime import datetime, timezone
from bson import ObjectId
from app.services.message_ids import build_references_header, normalize_message_id
from app.services.subject_normalize import normalize_subject
from app.services.thread_resolver import ThreadResolution
def message_event_at(message: dict) -> datetime:
return (
message.get("received_at")
or message.get("sent_at")
or message.get("created_at")
or datetime.now(timezone.utc)
)
async def create_thread(
database,
*,
account_id: str,
subject: str,
message_id: ObjectId,
event_at: datetime,
thread_type: str = "EXTERNAL",
) -> ObjectId:
now = datetime.now(timezone.utc)
document = {
"account_id": account_id,
"thread_type": thread_type,
"normalized_subject": normalize_subject(subject),
"created_at": now,
"updated_at": now,
"last_message_at": event_at,
"latest_message_id": message_id,
"message_count": 1,
}
result = await database.email_threads.insert_one(document)
return result.inserted_id
async def message_already_in_thread(
database,
*,
account_id: str,
thread_id: ObjectId,
message_id: ObjectId,
) -> bool:
existing = await database.mailbox_entries.find_one(
{
"account_id": account_id,
"thread_id": thread_id,
"message_id": message_id,
},
{"_id": 1},
)
return existing is not None
async def attach_message_to_thread(
database,
*,
account_id: str,
thread_id: ObjectId,
message_id: ObjectId,
subject: str,
event_at: datetime,
) -> None:
if await message_already_in_thread(
database,
account_id=account_id,
thread_id=thread_id,
message_id=message_id,
):
return
now = datetime.now(timezone.utc)
await database.email_threads.update_one(
{"_id": thread_id, "account_id": account_id},
{
"$set": {
"updated_at": now,
"last_message_at": event_at,
"latest_message_id": message_id,
"normalized_subject": normalize_subject(subject),
},
"$inc": {"message_count": 1},
},
)
async def ensure_thread_for_message(
database,
*,
account_id: str,
message: dict,
resolution: ThreadResolution,
) -> ObjectId:
message_id = message["_id"]
event_at = message_event_at(message)
subject = message.get("subject", "")
if resolution.thread_id is not None:
await attach_message_to_thread(
database,
account_id=account_id,
thread_id=resolution.thread_id,
message_id=message_id,
subject=subject,
event_at=event_at,
)
return resolution.thread_id
return await create_thread(
database,
account_id=account_id,
subject=subject,
message_id=message_id,
event_at=event_at,
)
def threading_fields_from_parent(parent: dict) -> dict:
parent_internet_id = normalize_message_id(parent.get("message_id"))
parent_references = parent.get("references") or []
if isinstance(parent_references, str):
parent_references = [parent_references]
references = []
if parent_references:
references = list(parent_references)
if parent_internet_id:
references = build_references_header(references, parent_internet_id).split()
return {
"in_reply_to": parent_internet_id,
"references": references,
"parent_message_id": parent["_id"],
}