thread fix

This commit is contained in:
debu-2 2026-10-06 04:42:47 +05:30
parent d58fb8841b
commit a58ba11566
11 changed files with 177 additions and 30 deletions

View file

@ -27,6 +27,7 @@ from app.services.mail_sender import deliver_email
from app.services.message_ids import canonical_message_id, message_id_fields
from app.services.mime_parser import parse_mime
from app.services.raw_mime_storage import get_raw_mime_storage
from app.services.reply_extractor import build_reply_display, reply_fields_for_message
from app.services.reply_recipients import reply_recipients
from app.services.thread_service import (
create_thread,
@ -98,7 +99,12 @@ def summarize(message: dict, entry: dict) -> MessageSummary:
sender=MailAddress(**sender),
recipients=[MailAddress(**item) for item in recipients],
subject=message.get("subject", ""),
snippet=message.get("snippet", message.get("body_text", "")[:240]),
snippet=message.get("snippet")
or re.sub(
r"\s+",
" ",
message.get("new_reply_content") or message.get("body_text", ""),
).strip()[:240],
status=message.get("status", "received"),
created_at=message.get("received_at") or message["created_at"],
is_read=entry.get("is_read", True),
@ -200,6 +206,10 @@ async def create_email(
}
if not subject.strip():
subject = reply_plan.subject
reply_display = build_reply_display(
body_text=payload.body_text,
body_html=payload.body_html,
)
document = {
"direction": "outbound",
"message_scope": MESSAGE_SCOPE_EXTERNAL,
@ -212,7 +222,10 @@ async def create_email(
"cc": [address(value) for value in cc],
"bcc": [address(value) for value in bcc],
"subject": subject,
"snippet": re.sub(r"\s+", " ", payload.body_text).strip()[:240],
"snippet": reply_display["snippet"],
"new_reply_content": reply_display["new_reply_content"],
"quoted_content": reply_display["quoted_content"],
"has_quoted_text": reply_display["has_quoted_text"],
"body_text": payload.body_text,
"body_html": payload.body_html,
"attachments": [],
@ -371,6 +384,11 @@ async def get_email(
body_text = parsed["body_text"]
body_html = parsed["body_html"]
summary = summarize(document, entry)
reply_fields = reply_fields_for_message(
document,
body_text=body_text,
body_html=body_html,
)
return MessageRead(
**summary.model_dump(),
cc=document.get("cc", []),
@ -378,6 +396,7 @@ async def get_email(
reply_to=document.get("reply_to", []),
body_text=body_text,
body_html=body_html,
**reply_fields,
message_id=document.get("message_id"),
in_reply_to=document.get("in_reply_to"),
references=document.get("references", []),

View file

@ -14,6 +14,7 @@ from app.routers.emails import decode_cursor, encode_cursor, summarize
from app.schemas import MessageRead, ThreadPage, ThreadRead, ThreadSummary
from app.services.mailbox_scope import apply_external_scope
from app.services.mime_parser import parse_mime
from app.services.reply_extractor import reply_fields_for_message
from app.services.raw_mime_storage import get_raw_mime_storage
router = APIRouter(prefix="/threads", tags=["threads"])
@ -241,6 +242,11 @@ async def message_read(
body_text = parsed["body_text"]
body_html = parsed["body_html"]
summary = summarize(message, entry)
reply_fields = reply_fields_for_message(
message,
body_text=body_text,
body_html=body_html,
)
return MessageRead(
**summary.model_dump(),
cc=message.get("cc", []),
@ -248,6 +254,7 @@ async def message_read(
reply_to=message.get("reply_to", []),
body_text=body_text,
body_html=body_html,
**reply_fields,
message_id=message.get("message_id"),
in_reply_to=message.get("in_reply_to"),
references=message.get("references", []),

View file

@ -406,6 +406,9 @@ class MessageRead(MessageSummary):
reply_to: list[MailAddress] = Field(default_factory=list)
body_text: str = ""
body_html: str | None = None
new_reply_content: str = ""
quoted_content: str = ""
has_quoted_text: bool = False
message_id: str | None = None
in_reply_to: str | None = None
references: list[str] = Field(default_factory=list)

View file

@ -17,6 +17,7 @@ from app.services.message_ids import (
message_id_fields,
)
from app.services.mime_parser import parse_mime
from app.services.reply_extractor import build_reply_display
from app.services.raw_mime_storage import get_raw_mime_storage
from app.services.thread_service import threading_fields_from_parent
@ -410,6 +411,10 @@ async def send_external_reply(
)
now = datetime.now(timezone.utc)
reply_display = build_reply_display(
body_text=plan.body_text,
body_html=plan.body_html,
)
document = {
"direction": "outbound",
"message_scope": MESSAGE_SCOPE_EXTERNAL,
@ -417,7 +422,10 @@ async def send_external_reply(
"sender": {"name": sender_name, "address": sender},
"to": [{"name": value.split("@")[0], "address": value} for value in plan.recipients],
"subject": plan.subject,
"snippet": re.sub(r"\s+", " ", plan.body_text).strip()[:240],
"snippet": reply_display["snippet"],
"new_reply_content": reply_display["new_reply_content"],
"quoted_content": reply_display["quoted_content"],
"has_quoted_text": reply_display["has_quoted_text"],
"body_text": plan.body_text,
"body_html": plan.body_html,
"attachments": [],

View file

@ -13,6 +13,7 @@ from app.services.external_reply_service import log_original_external_message
from app.services.human_handoff_service import MESSAGE_SCOPE_EXTERNAL
from app.services.message_ids import canonical_message_id, message_id_fields
from app.services.mime_parser import parse_mime
from app.services.reply_extractor import build_reply_display
from app.services.raw_mime_storage import delivery_digest, get_raw_mime_storage
from app.services.thread_resolver import participants_from_parsed, resolve_thread
from app.services.thread_service import ensure_thread_for_message, message_event_at
@ -155,12 +156,26 @@ class InboundHandler:
raise RuntimeError("Inbound message was not persisted.")
if body_text or body_html:
reply_display = build_reply_display(body_text=body_text, body_html=body_html)
await mongo_db.messages.update_one(
{"_id": persisted["_id"]},
{"$set": {"body_text": body_text, "body_html": body_html}},
{
"$set": {
"body_text": body_text,
"body_html": body_html,
"new_reply_content": reply_display["new_reply_content"],
"quoted_content": reply_display["quoted_content"],
"has_quoted_text": reply_display["has_quoted_text"],
"snippet": reply_display["snippet"],
}
},
)
persisted["body_text"] = body_text
persisted["body_html"] = body_html
persisted["snippet"] = reply_display["snippet"]
persisted["new_reply_content"] = reply_display["new_reply_content"]
persisted["quoted_content"] = reply_display["quoted_content"]
persisted["has_quoted_text"] = reply_display["has_quoted_text"]
participants = participants_from_parsed(parsed)
for recipient in recipients:
@ -214,7 +229,7 @@ class InboundHandler:
"sender": parsed["sender"],
"recipients": parsed["to"],
"subject": parsed["subject"],
"snippet": parsed["snippet"],
"snippet": persisted.get("snippet", parsed["snippet"]),
"attachments": parsed["attachments"],
"status": "received",
"is_read": False,

View file

@ -6,6 +6,7 @@ from email.utils import getaddresses, parsedate_to_datetime
from html import unescape
from app.services.message_ids import first_message_id, parse_message_id_list
from app.services.reply_extractor import build_reply_display
def _addresses(values: list[str]) -> list[dict[str, str]]:
@ -36,11 +37,7 @@ def _header_str(value) -> str | None:
def _snippet(text: str, html: str | None, limit: int = 240) -> str:
source = text
if not source and html:
source = re.sub(r"<[^>]+>", " ", html)
source = unescape(source)
return re.sub(r"\s+", " ", source).strip()[:limit]
return build_reply_display(body_text=text, body_html=html, limit=limit)["snippet"]
def parse_mime(raw: bytes) -> dict:

View file

@ -4,19 +4,13 @@ from html import unescape
# Patterns that mark the start of quoted previous-message content.
TEXT_QUOTE_MARKERS = [
re.compile(r"\n-{2,}\s*Original Message\s*-{2,}", re.IGNORECASE),
re.compile(
r"\nOn .+? wrote:\s*\n",
re.IGNORECASE,
),
re.compile(
r"\nOn .+?, .+? at .+? .+? wrote:\s*\n",
re.IGNORECASE,
),
re.compile(r"(?:^|\n)\s*On .+? wrote:\s*", re.IGNORECASE),
re.compile(
r"\nFrom:\s*.+\nSent:\s*.+\nTo:\s*.+\nSubject:\s*.+",
re.IGNORECASE,
),
re.compile(r"\n_{5,}\n"),
re.compile(r"\n\s*> ?"),
]
HTML_QUOTE_PATTERNS = [
@ -110,6 +104,46 @@ def extract_new_reply_content(
}
def reply_fields_for_message(
message: dict,
*,
body_text: str | None,
body_html: str | None = None,
) -> dict[str, str | bool]:
if "new_reply_content" in message:
return {
"new_reply_content": message.get("new_reply_content", ""),
"quoted_content": message.get("quoted_content", ""),
"has_quoted_text": bool(message.get("has_quoted_text") or message.get("quoted_content")),
}
built = build_reply_display(body_text=body_text, body_html=body_html)
return {
"new_reply_content": built["new_reply_content"],
"quoted_content": built["quoted_content"],
"has_quoted_text": built["has_quoted_text"],
}
def build_reply_display(
*,
body_text: str | None,
body_html: str | None = None,
limit: int = 240,
) -> dict[str, str | bool]:
extracted = extract_new_reply_content(body_text=body_text, body_html=body_html)
new_content = extracted["new_reply_content"]
quoted = extracted["quoted_content"]
fallback = re.sub(r"\s+", " ", (body_text or "")).strip()
snippet_source = new_content or fallback
return {
"new_reply_content": new_content,
"quoted_content": quoted,
"has_quoted_text": bool(quoted),
"raw_body": extracted["raw_body"],
"snippet": re.sub(r"\s+", " ", snippet_source).strip()[:limit],
}
def validate_external_reply_body(body: str) -> bool:
normalized = (body or "").strip()
if not normalized:

View file

@ -49,6 +49,35 @@ def test_external_reply_subject_uses_customer_subject():
assert external_reply_subject("Human Handoff · are you ready?") == "Re: are you ready?"
def test_extracts_reply_before_blockquote_lines():
body = """responder: B
> sender: A"""
result = extract_new_reply_content(body_text=body)
assert result["new_reply_content"] == "responder: B"
assert "sender: A" in result["quoted_content"]
def test_extracts_reply_from_gmail_on_wrote_without_blank_line():
body = """sender: C
On Tue, Oct 6, 2026 at 4:29 AM <suyash@gentic.email> wrote:
> Responder: B
> sender: A"""
result = extract_new_reply_content(body_text=body)
assert result["new_reply_content"] == "sender: C"
assert "Responder: B" in result["quoted_content"]
def test_build_reply_display_uses_clean_snippet():
from app.services.reply_extractor import build_reply_display
body = """responder: D
On Tue, Oct 6, 2026 at 4:29 AM wrote:
> sender: C"""
result = build_reply_display(body_text=body)
assert result["snippet"] == "responder: D"
assert result["has_quoted_text"] is True
def test_outlook_original_message_split():
body = """Thanks for the update.