From d0fee5c8f76d4f53a25c8b9b1ec67a55ed8a2460 Mon Sep 17 00:00:00 2001 From: debu-2 Date: Tue, 6 Oct 2026 05:03:33 +0530 Subject: [PATCH] thread fix --- backend/app/services/reply_extractor.py | 101 ++++++++++++---- backend/tests/test_reply_extractor.py | 21 ++++ frontend/src/components/ComposeModal.jsx | 19 ++- frontend/src/components/MailboxView.jsx | 8 +- frontend/src/styles.css | 3 + frontend/src/utils/emailReply.js | 144 +++++++++++++++++++++++ 6 files changed, 262 insertions(+), 34 deletions(-) create mode 100644 frontend/src/utils/emailReply.js diff --git a/backend/app/services/reply_extractor.py b/backend/app/services/reply_extractor.py index 65f1187..4ab8b87 100644 --- a/backend/app/services/reply_extractor.py +++ b/backend/app/services/reply_extractor.py @@ -1,5 +1,6 @@ import re from html import unescape +from html.parser import HTMLParser # Patterns that mark the start of quoted previous-message content. TEXT_QUOTE_MARKERS = [ @@ -13,12 +14,6 @@ TEXT_QUOTE_MARKERS = [ re.compile(r"\n\s*> ?"), ] -HTML_QUOTE_PATTERNS = [ - re.compile(r"]*>.*?", re.IGNORECASE | re.DOTALL), - re.compile(r']+class="[^"]*gmail_quote[^"]*"[^>]*>.*?', re.IGNORECASE | re.DOTALL), - re.compile(r']+class="[^"]*gmail_attr[^"]*"[^>]*>.*?', re.IGNORECASE | re.DOTALL), -] - PRIVATE_LEAK_MARKERS = [ "human handoff", "a conversation needs your attention", @@ -39,14 +34,75 @@ def _normalize_newlines(text: str) -> str: return text.replace("\r\n", "\n").replace("\r", "\n") +class _ReplyHTMLTextParser(HTMLParser): + BLOCK_TAGS = {"div", "p", "pre", "li", "ul", "ol", "table", "tr"} + IGNORED_TAGS = {"script", "style", "iframe", "object"} + + def __init__(self) -> None: + super().__init__(convert_charrefs=True) + self.parts: list[str] = [] + self.quote_depth = 0 + self.ignored_depth = 0 + self.at_line_start = True + + def _newline(self) -> None: + if self.parts and not self.parts[-1].endswith("\n"): + self.parts.append("\n") + self.at_line_start = True + + def _text(self, value: str) -> None: + if self.ignored_depth: + return + normalized = re.sub(r"\s+", " ", value) + if not normalized.strip(): + return + if self.at_line_start: + normalized = normalized.lstrip() + if self.quote_depth: + self.parts.append("> " * self.quote_depth) + self.parts.append(normalized) + self.at_line_start = False + + def handle_starttag(self, tag: str, attrs) -> None: + tag = tag.lower() + if tag in self.IGNORED_TAGS: + self.ignored_depth += 1 + return + if self.ignored_depth: + return + if tag == "br": + self._newline() + elif tag == "blockquote": + self._newline() + self.quote_depth += 1 + elif tag in self.BLOCK_TAGS: + self._newline() + + def handle_endtag(self, tag: str) -> None: + tag = tag.lower() + if tag in self.IGNORED_TAGS: + self.ignored_depth = max(0, self.ignored_depth - 1) + return + if self.ignored_depth: + return + if tag == "blockquote": + self._newline() + self.quote_depth = max(0, self.quote_depth - 1) + elif tag in self.BLOCK_TAGS: + self._newline() + + def handle_data(self, data: str) -> None: + self._text(data) + + def text(self) -> str: + return "".join(self.parts).strip() + + def _html_to_text(html: str) -> str: - cleaned = html - for pattern in HTML_QUOTE_PATTERNS: - cleaned = pattern.sub("", cleaned) - cleaned = re.sub(r"", "\n", cleaned, flags=re.IGNORECASE) - cleaned = re.sub(r"", "\n", cleaned, flags=re.IGNORECASE) - cleaned = re.sub(r"<[^>]+>", "", cleaned) - return unescape(cleaned) + parser = _ReplyHTMLTextParser() + parser.feed(html) + parser.close() + return unescape(parser.text()) def _strip_trailing_quoted_lines(text: str) -> str: @@ -83,10 +139,7 @@ def extract_new_reply_content( raw_text = _html_to_text(body_html).strip() if body_html: - html_stripped = body_html - for pattern in HTML_QUOTE_PATTERNS: - html_stripped = pattern.sub("", html_stripped) - html_text = _html_to_text(html_stripped).strip() + html_text = _html_to_text(body_html).strip() if html_text: new_from_html, quoted_from_html = _split_at_quote_markers(html_text) if new_from_html: @@ -110,17 +163,13 @@ def reply_fields_for_message( body_text: str | None, body_html: str | None = None, ) -> dict[str, str | bool]: - if "new_reply_content" in message: - return { - "new_reply_content": message.get("new_reply_content", ""), - "quoted_content": message.get("quoted_content", ""), - "has_quoted_text": bool(message.get("has_quoted_text") or message.get("quoted_content")), - } built = build_reply_display(body_text=body_text, body_html=body_html) + new_content = built["new_reply_content"] or message.get("new_reply_content", "") + quoted_content = built["quoted_content"] or message.get("quoted_content", "") return { - "new_reply_content": built["new_reply_content"], - "quoted_content": built["quoted_content"], - "has_quoted_text": built["has_quoted_text"], + "new_reply_content": new_content, + "quoted_content": quoted_content, + "has_quoted_text": bool(quoted_content), } diff --git a/backend/tests/test_reply_extractor.py b/backend/tests/test_reply_extractor.py index 4c1bba0..72bd678 100644 --- a/backend/tests/test_reply_extractor.py +++ b/backend/tests/test_reply_extractor.py @@ -78,6 +78,27 @@ On Tue, Oct 6, 2026 at 4:29 AM wrote: assert result["has_quoted_text"] is True +def test_html_reply_preserves_nested_quoted_history(): + body = """ +
sender: E
+
+
+ On Tue, Oct 6, 2026 at 4:48 AM <suyash@gentic.email> wrote: +
+
+
responder: D
+
sender: C
+
+
+ """ + result = extract_new_reply_content(body_text="", body_html=body) + + assert result["new_reply_content"] == "sender: E" + assert "On Tue, Oct 6, 2026 at 4:48 AM" in result["quoted_content"] + assert "> responder: D" in result["quoted_content"] + assert "> > sender: C" in result["quoted_content"] + + def test_outlook_original_message_split(): body = """Thanks for the update. diff --git a/frontend/src/components/ComposeModal.jsx b/frontend/src/components/ComposeModal.jsx index af0b9fd..96e2c64 100644 --- a/frontend/src/components/ComposeModal.jsx +++ b/frontend/src/components/ComposeModal.jsx @@ -34,12 +34,12 @@ import RecipientField, { recipientsToPayload } from "./compose/RecipientField"; import { useCallback, useEffect, useMemo, useRef, useState } from "react"; import { listMailAliases, listOrgMembers, sendEmail } from "../api"; import { clearDraft, draftStorageKey } from "../utils/composeDraft"; +import { plainTextFromEmailHtml } from "../utils/emailReply"; const EMOJI_SAMPLES = ["๐Ÿ˜€", "๐Ÿ˜Š", "๐Ÿ‘", "๐Ÿ™", "โœ…", "๐ŸŽ‰", "๐Ÿ“Ž", "๐Ÿ“…", "๐Ÿ’ก", "๐Ÿ”ฅ", "โค๏ธ", "๐Ÿ˜…"]; function plainTextFromHtml(value) { - const doc = new DOMParser().parseFromString(value, "text/html"); - return (doc.body.textContent || "").replace(/\u00a0/g, " ").trim(); + return plainTextFromEmailHtml(value); } function formatFileSize(bytes) { @@ -195,6 +195,7 @@ export default function ComposeModal({ const [notice, setNotice] = useState(""); const [warnings, setWarnings] = useState([]); const [showEmoji, setShowEmoji] = useState(false); + const [showQuoted, setShowQuoted] = useState(false); const [status, setStatus] = useState({ sending: false, error: "" }); const [replyToMessageId, setReplyToMessageId] = useState(null); const [replyAll, setReplyAll] = useState(false); @@ -202,6 +203,7 @@ export default function ComposeModal({ const orgDomain = account.company_domain || account.email.split("@")[1]; const draftKey = draftStorageKey(windowId || account.id); const bodyEmpty = !plainTextFromHtml(bodyHtml); + const hasQuotedHistory = bodyHtml.includes('class="gmail_quote"'); const signatures = useMemo(() => [ { @@ -586,7 +588,7 @@ export default function ComposeModal({ )}
+ {hasQuotedHistory && ( + + )}
diff --git a/frontend/src/components/MailboxView.jsx b/frontend/src/components/MailboxView.jsx index 8da37c4..6fbc3a3 100644 --- a/frontend/src/components/MailboxView.jsx +++ b/frontend/src/components/MailboxView.jsx @@ -44,6 +44,7 @@ import { labelCounts, matchesLens, } from "../utils/inboxSignals"; +import { buildReplyBodyHtml } from "../utils/emailReply"; const PAGE_SIZE = 25; const CATEGORY_TABS = [ @@ -86,15 +87,12 @@ function buildReplyContext(message, replyAll = false) { const sender = message?.sender || { name: "", address: "" }; const subject = (message?.subject || "").trim(); const replySubject = /^re:/i.test(subject) ? subject : `Re: ${subject || "(No subject)"}`; - const quote = (message?.new_reply_content || message?.body_text || message?.snippet || "").trim(); return { replyToMessageId: message.id, replyAll, to: replyAll ? [] : [{ email: sender.address, name: sender.name }], subject: replySubject, - bodyHtml: quote - ? `


${quote.replace(/\n/g, "
")}
` - : "


", + bodyHtml: buildReplyBodyHtml(message), }; } @@ -781,7 +779,7 @@ export default function MailboxView({ const target = threadDetail?.messages?.[threadDetail.messages.length - 1] || detail; onCompose?.({ subject: `Fwd: ${target?.subject || ""}`, - bodyHtml: `


${(target?.new_reply_content || target?.body_text || target?.snippet || "").replace(/\n/g, "
")}
`, + bodyHtml: buildReplyBodyHtml(target), }); }} onExtractActions={() => detail && extractActions(detail)} diff --git a/frontend/src/styles.css b/frontend/src/styles.css index b541321..cdaefb4 100644 --- a/frontend/src/styles.css +++ b/frontend/src/styles.css @@ -945,6 +945,9 @@ h1 em { color:var(--coral); font-style:normal; font-weight:500; } .compose-editor:focus { outline:none; box-shadow:none; } .compose-editor:empty:before { content:attr(data-placeholder); color:#aaa4ac; pointer-events:none; } .compose-editor blockquote { margin:8px 0; padding-left:12px; border-left:3px solid #e5dfd9; color:#67677a; } +.compose-editor.quotes-collapsed .gmail_quote { display:none; } +.compose-quoted-toggle { align-self:flex-start; margin:0 0 10px 16px; padding:1px 8px 3px; border:1px solid #dedbe2; border-radius:5px; background:#f3f1f5; color:#77717d; font:700 12px/1 "DM Sans"; letter-spacing:1px; cursor:pointer; } +.compose-quoted-toggle:hover { background:#eae7ed; color:#45404a; } .compose-bottom-stack { flex-shrink:0; border-top:1px solid #EEECEF; background:#fff; } .compose-agent-bar { height:42px; padding:0 var(--compose-pad); display:grid; grid-template-columns:auto 1fr auto; align-items:center; gap:10px; border-top:1px solid #EEEAF8; border-bottom:1px solid #EEEAF8; background:#FBF9FF; color:#77728C; font-size:11px; } .compose-agent-bar .agent-spark { color:#7357F6; } diff --git a/frontend/src/utils/emailReply.js b/frontend/src/utils/emailReply.js new file mode 100644 index 0000000..54ce2aa --- /dev/null +++ b/frontend/src/utils/emailReply.js @@ -0,0 +1,144 @@ +const BLOCK_TAGS = new Set(["DIV", "P", "PRE", "LI", "UL", "OL"]); +const SAFE_TAGS = new Set([ + "A", + "B", + "BLOCKQUOTE", + "BR", + "CODE", + "DIV", + "EM", + "I", + "LI", + "OL", + "P", + "PRE", + "SPAN", + "STRONG", + "U", + "UL", +]); + +export function escapeHtml(value = "") { + return value + .replaceAll("&", "&") + .replaceAll("<", "<") + .replaceAll(">", ">") + .replaceAll('"', """) + .replaceAll("'", "'"); +} + +function cleanMessageHtml(value) { + if (!value) return ""; + const doc = new DOMParser().parseFromString(value, "text/html"); + + function clean(node) { + [...node.childNodes].forEach((child) => { + if (child.nodeType !== Node.ELEMENT_NODE) return; + if (!SAFE_TAGS.has(child.tagName)) { + if (["SCRIPT", "STYLE", "IFRAME", "OBJECT", "EMBED"].includes(child.tagName)) { + child.remove(); + return; + } + clean(child); + child.replaceWith(...child.childNodes); + return; + } + + [...child.attributes].forEach((attribute) => child.removeAttribute(attribute.name)); + if (child.tagName === "BLOCKQUOTE") { + child.setAttribute( + "style", + "margin:0 0 0 .8ex;border-left:1px solid #ccc;padding-left:1ex", + ); + } + clean(child); + }); + } + + clean(doc.body); + return doc.body.innerHTML; +} + +function textBodyAsHtml(value) { + return `
${escapeHtml(value || "")}
`; +} + +function senderIdentity(message) { + const sender = message?.sender || message?.from || {}; + const name = typeof sender === "string" ? "" : (sender.name || "").trim(); + const address = typeof sender === "string" + ? sender.trim() + : (sender.address || sender.email || "").trim(); + if (name && address) return `${name} <${address}>`; + return address || name || "Unknown sender"; +} + +function attributionDate(message) { + const value = message?.received_at + || message?.sent_at + || message?.created_at + || message?.updated_at; + const date = value ? new Date(value) : null; + if (!date || Number.isNaN(date.getTime())) return ""; + + const datePart = new Intl.DateTimeFormat(undefined, { + weekday: "short", + month: "short", + day: "numeric", + year: "numeric", + }).format(date); + const timePart = new Intl.DateTimeFormat(undefined, { + hour: "numeric", + minute: "2-digit", + }).format(date); + return `${datePart} at ${timePart}`; +} + +export function buildReplyBodyHtml(message) { + const original = cleanMessageHtml(message?.body_html) + || textBodyAsHtml(message?.body_text || message?.snippet || ""); + const date = attributionDate(message); + const identity = senderIdentity(message); + const attribution = `On ${date ? `${date} ` : ""}${identity} wrote:`; + + return [ + "

", + '
', + `
${escapeHtml(attribution)}
`, + '
', + original, + "
", + "
", + ].join(""); +} + +export function plainTextFromEmailHtml(value) { + if (!value) return ""; + const doc = new DOMParser().parseFromString(value, "text/html"); + + function render(node) { + if (node.nodeType === Node.TEXT_NODE) return node.textContent || ""; + if (node.nodeType !== Node.ELEMENT_NODE) return ""; + if (node.tagName === "BR") return "\n"; + + if (node.tagName === "BLOCKQUOTE") { + const content = [...node.childNodes] + .map((child) => render(child)) + .join("") + .replace(/^\n+|\n+$/g, ""); + return `\n${content.split("\n").map((line) => `> ${line}`).join("\n")}\n`; + } + + const content = [...node.childNodes] + .map((child) => render(child)) + .join(""); + return BLOCK_TAGS.has(node.tagName) ? `\n${content}\n` : content; + } + + return render(doc.body) + .replace(/\u00a0/g, " ") + .replace(/[ \t]+\n/g, "\n") + .replace(/\n{3,}/g, "\n\n") + .trim(); +}