thread fix

2026-10-06 05:03:33 +05:30 · 2026-10-06 05:03:33 +05:30 · d0fee5c8f7
commit d0fee5c8f7
parent a58ba11566
6 changed files with 262 additions and 34 deletions
--- a/backend/app/services/reply_extractor.py
+++ b/backend/app/services/reply_extractor.py
@ -1,5 +1,6 @@
 import re
 from html import unescape
+from html.parser import HTMLParser

 # Patterns that mark the start of quoted previous-message content.
 TEXT_QUOTE_MARKERS = [
@ -13,12 +14,6 @@ TEXT_QUOTE_MARKERS = [
    re.compile(r"\n\s*> ?"),
 ]

-HTML_QUOTE_PATTERNS = [
-    re.compile(r"<blockquote\b[^>]*>.*?</blockquote>", re.IGNORECASE | re.DOTALL),
-    re.compile(r'<div[^>]+class="[^"]*gmail_quote[^"]*"[^>]*>.*?</div>', re.IGNORECASE | re.DOTALL),
-    re.compile(r'<div[^>]+class="[^"]*gmail_attr[^"]*"[^>]*>.*?</div>', re.IGNORECASE | re.DOTALL),
-]
-
 PRIVATE_LEAK_MARKERS = [
    "human handoff",
    "a conversation needs your attention",
@ -39,14 +34,75 @@ def _normalize_newlines(text: str) -> str:
    return text.replace("\r\n", "\n").replace("\r", "\n")


+class _ReplyHTMLTextParser(HTMLParser):
+    BLOCK_TAGS = {"div", "p", "pre", "li", "ul", "ol", "table", "tr"}
+    IGNORED_TAGS = {"script", "style", "iframe", "object"}
+
+    def __init__(self) -> None:
+        super().__init__(convert_charrefs=True)
+        self.parts: list[str] = []
+        self.quote_depth = 0
+        self.ignored_depth = 0
+        self.at_line_start = True
+
+    def _newline(self) -> None:
+        if self.parts and not self.parts[-1].endswith("\n"):
+            self.parts.append("\n")
+        self.at_line_start = True
+
+    def _text(self, value: str) -> None:
+        if self.ignored_depth:
+            return
+        normalized = re.sub(r"\s+", " ", value)
+        if not normalized.strip():
+            return
+        if self.at_line_start:
+            normalized = normalized.lstrip()
+            if self.quote_depth:
+                self.parts.append("> " * self.quote_depth)
+        self.parts.append(normalized)
+        self.at_line_start = False
+
+    def handle_starttag(self, tag: str, attrs) -> None:
+        tag = tag.lower()
+        if tag in self.IGNORED_TAGS:
+            self.ignored_depth += 1
+            return
+        if self.ignored_depth:
+            return
+        if tag == "br":
+            self._newline()
+        elif tag == "blockquote":
+            self._newline()
+            self.quote_depth += 1
+        elif tag in self.BLOCK_TAGS:
+            self._newline()
+
+    def handle_endtag(self, tag: str) -> None:
+        tag = tag.lower()
+        if tag in self.IGNORED_TAGS:
+            self.ignored_depth = max(0, self.ignored_depth - 1)
+            return
+        if self.ignored_depth:
+            return
+        if tag == "blockquote":
+            self._newline()
+            self.quote_depth = max(0, self.quote_depth - 1)
+        elif tag in self.BLOCK_TAGS:
+            self._newline()
+
+    def handle_data(self, data: str) -> None:
+        self._text(data)
+
+    def text(self) -> str:
+        return "".join(self.parts).strip()
+
+
 def _html_to_text(html: str) -> str:
-    cleaned = html
-    for pattern in HTML_QUOTE_PATTERNS:
-        cleaned = pattern.sub("", cleaned)
-    cleaned = re.sub(r"<br\s*/?>", "\n", cleaned, flags=re.IGNORECASE)
-    cleaned = re.sub(r"</p\s*>", "\n", cleaned, flags=re.IGNORECASE)
-    cleaned = re.sub(r"<[^>]+>", "", cleaned)
-    return unescape(cleaned)
+    parser = _ReplyHTMLTextParser()
+    parser.feed(html)
+    parser.close()
+    return unescape(parser.text())


 def _strip_trailing_quoted_lines(text: str) -> str:
@ -83,10 +139,7 @@ def extract_new_reply_content(
        raw_text = _html_to_text(body_html).strip()

    if body_html:
-        html_stripped = body_html
-        for pattern in HTML_QUOTE_PATTERNS:
-            html_stripped = pattern.sub("", html_stripped)
-        html_text = _html_to_text(html_stripped).strip()
+        html_text = _html_to_text(body_html).strip()
        if html_text:
            new_from_html, quoted_from_html = _split_at_quote_markers(html_text)
            if new_from_html:
@ -110,17 +163,13 @@ def reply_fields_for_message(
    body_text: str | None,
    body_html: str | None = None,
 ) -> dict[str, str | bool]:
-    if "new_reply_content" in message:
-        return {
-            "new_reply_content": message.get("new_reply_content", ""),
-            "quoted_content": message.get("quoted_content", ""),
-            "has_quoted_text": bool(message.get("has_quoted_text") or message.get("quoted_content")),
-        }
    built = build_reply_display(body_text=body_text, body_html=body_html)
+    new_content = built["new_reply_content"] or message.get("new_reply_content", "")
+    quoted_content = built["quoted_content"] or message.get("quoted_content", "")
    return {
-        "new_reply_content": built["new_reply_content"],
-        "quoted_content": built["quoted_content"],
-        "has_quoted_text": built["has_quoted_text"],
+        "new_reply_content": new_content,
+        "quoted_content": quoted_content,
+        "has_quoted_text": bool(quoted_content),
    }


--- a/backend/tests/test_reply_extractor.py
+++ b/backend/tests/test_reply_extractor.py
@ -78,6 +78,27 @@ On Tue, Oct 6, 2026 at 4:29 AM wrote:
    assert result["has_quoted_text"] is True


+def test_html_reply_preserves_nested_quoted_history():
+    body = """
+    <div>sender: E</div>
+    <div class="gmail_quote">
+      <div class="gmail_attr">
+        On Tue, Oct 6, 2026 at 4:48 AM &lt;suyash@gentic.email&gt; wrote:
+      </div>
+      <blockquote>
+        <div>responder: D</div>
+        <blockquote><div>sender: C</div></blockquote>
+      </blockquote>
+    </div>
+    """
+    result = extract_new_reply_content(body_text="", body_html=body)
+
+    assert result["new_reply_content"] == "sender: E"
+    assert "On Tue, Oct 6, 2026 at 4:48 AM" in result["quoted_content"]
+    assert "> responder: D" in result["quoted_content"]
+    assert "> > sender: C" in result["quoted_content"]
+
+
 def test_outlook_original_message_split():
    body = """Thanks for the update.

--- a/frontend/src/components/ComposeModal.jsx
+++ b/frontend/src/components/ComposeModal.jsx
@ -34,12 +34,12 @@ import RecipientField, { recipientsToPayload } from "./compose/RecipientField";
 import { useCallback, useEffect, useMemo, useRef, useState } from "react";
 import { listMailAliases, listOrgMembers, sendEmail } from "../api";
 import { clearDraft, draftStorageKey } from "../utils/composeDraft";
+import { plainTextFromEmailHtml } from "../utils/emailReply";

 const EMOJI_SAMPLES = ["😀", "😊", "👍", "🙏", "✅", "🎉", "📎", "📅", "💡", "🔥", "❤️", "😅"];

 function plainTextFromHtml(value) {
-  const doc = new DOMParser().parseFromString(value, "text/html");
-  return (doc.body.textContent || "").replace(/\u00a0/g, " ").trim();
+  return plainTextFromEmailHtml(value);
 }

 function formatFileSize(bytes) {
@ -195,6 +195,7 @@ export default function ComposeModal({
  const [notice, setNotice] = useState("");
  const [warnings, setWarnings] = useState([]);
  const [showEmoji, setShowEmoji] = useState(false);
+  const [showQuoted, setShowQuoted] = useState(false);
  const [status, setStatus] = useState({ sending: false, error: "" });
  const [replyToMessageId, setReplyToMessageId] = useState(null);
  const [replyAll, setReplyAll] = useState(false);
@ -202,6 +203,7 @@ export default function ComposeModal({
  const orgDomain = account.company_domain || account.email.split("@")[1];
  const draftKey = draftStorageKey(windowId || account.id);
  const bodyEmpty = !plainTextFromHtml(bodyHtml);
+  const hasQuotedHistory = bodyHtml.includes('class="gmail_quote"');

  const signatures = useMemo(() => [
    {
@ -586,7 +588,7 @@ export default function ComposeModal({
        )}

        <div
-          className="compose-editor"
+          className={`compose-editor${hasQuotedHistory && !showQuoted ? " quotes-collapsed" : ""}`}
          ref={editorRef}
          contentEditable
          suppressContentEditableWarning
@ -600,6 +602,17 @@ export default function ComposeModal({
            addFiles(event.dataTransfer.files);
          }}
        />
+        {hasQuotedHistory && (
+          <button
+            type="button"
+            className="compose-quoted-toggle"
+            onClick={() => setShowQuoted((visible) => !visible)}
+            aria-expanded={showQuoted}
+            title={showQuoted ? "Hide trimmed content" : "Show trimmed content"}
+          >
+            ···
+          </button>
+        )}

        <div className="compose-bottom-stack">
          <div className="compose-agent-bar compose-agent-bar-disabled" title="AG Agent drafting — coming soon">
--- a/frontend/src/components/MailboxView.jsx
+++ b/frontend/src/components/MailboxView.jsx
@ -44,6 +44,7 @@ import {
  labelCounts,
  matchesLens,
 } from "../utils/inboxSignals";
+import { buildReplyBodyHtml } from "../utils/emailReply";

 const PAGE_SIZE = 25;
 const CATEGORY_TABS = [
@ -86,15 +87,12 @@ function buildReplyContext(message, replyAll = false) {
  const sender = message?.sender || { name: "", address: "" };
  const subject = (message?.subject || "").trim();
  const replySubject = /^re:/i.test(subject) ? subject : `Re: ${subject || "(No subject)"}`;
-  const quote = (message?.new_reply_content || message?.body_text || message?.snippet || "").trim();
  return {
    replyToMessageId: message.id,
    replyAll,
    to: replyAll ? [] : [{ email: sender.address, name: sender.name }],
    subject: replySubject,
-    bodyHtml: quote
-      ? `<p><br></p><blockquote>${quote.replace(/\n/g, "<br>")}</blockquote>`
-      : "<p><br></p>",
+    bodyHtml: buildReplyBodyHtml(message),
  };
 }

@ -781,7 +779,7 @@ export default function MailboxView({
                  const target = threadDetail?.messages?.[threadDetail.messages.length - 1] || detail;
                  onCompose?.({
                    subject: `Fwd: ${target?.subject || ""}`,
-                    bodyHtml: `<p><br></p><blockquote>${(target?.new_reply_content || target?.body_text || target?.snippet || "").replace(/\n/g, "<br>")}</blockquote>`,
+                    bodyHtml: buildReplyBodyHtml(target),
                  });
                }}
                onExtractActions={() => detail && extractActions(detail)}
--- a/frontend/src/styles.css
+++ b/frontend/src/styles.css
@ -945,6 +945,9 @@ h1 em { color:var(--coral); font-style:normal; font-weight:500; }
 .compose-editor:focus { outline:none; box-shadow:none; }
 .compose-editor:empty:before { content:attr(data-placeholder); color:#aaa4ac; pointer-events:none; }
 .compose-editor blockquote { margin:8px 0; padding-left:12px; border-left:3px solid #e5dfd9; color:#67677a; }
+.compose-editor.quotes-collapsed .gmail_quote { display:none; }
+.compose-quoted-toggle { align-self:flex-start; margin:0 0 10px 16px; padding:1px 8px 3px; border:1px solid #dedbe2; border-radius:5px; background:#f3f1f5; color:#77717d; font:700 12px/1 "DM Sans"; letter-spacing:1px; cursor:pointer; }
+.compose-quoted-toggle:hover { background:#eae7ed; color:#45404a; }
 .compose-bottom-stack { flex-shrink:0; border-top:1px solid #EEECEF; background:#fff; }
 .compose-agent-bar { height:42px; padding:0 var(--compose-pad); display:grid; grid-template-columns:auto 1fr auto; align-items:center; gap:10px; border-top:1px solid #EEEAF8; border-bottom:1px solid #EEEAF8; background:#FBF9FF; color:#77728C; font-size:11px; }
 .compose-agent-bar .agent-spark { color:#7357F6; }
--- a/frontend/src/utils/emailReply.js
+++ b/frontend/src/utils/emailReply.js
@ -0,0 +1,144 @@
+const BLOCK_TAGS = new Set(["DIV", "P", "PRE", "LI", "UL", "OL"]);
+const SAFE_TAGS = new Set([
+  "A",
+  "B",
+  "BLOCKQUOTE",
+  "BR",
+  "CODE",
+  "DIV",
+  "EM",
+  "I",
+  "LI",
+  "OL",
+  "P",
+  "PRE",
+  "SPAN",
+  "STRONG",
+  "U",
+  "UL",
+]);
+
+export function escapeHtml(value = "") {
+  return value
+    .replaceAll("&", "&amp;")
+    .replaceAll("<", "&lt;")
+    .replaceAll(">", "&gt;")
+    .replaceAll('"', "&quot;")
+    .replaceAll("'", "&#39;");
+}
+
+function cleanMessageHtml(value) {
+  if (!value) return "";
+  const doc = new DOMParser().parseFromString(value, "text/html");
+
+  function clean(node) {
+    [...node.childNodes].forEach((child) => {
+      if (child.nodeType !== Node.ELEMENT_NODE) return;
+      if (!SAFE_TAGS.has(child.tagName)) {
+        if (["SCRIPT", "STYLE", "IFRAME", "OBJECT", "EMBED"].includes(child.tagName)) {
+          child.remove();
+          return;
+        }
+        clean(child);
+        child.replaceWith(...child.childNodes);
+        return;
+      }
+
+      [...child.attributes].forEach((attribute) => child.removeAttribute(attribute.name));
+      if (child.tagName === "BLOCKQUOTE") {
+        child.setAttribute(
+          "style",
+          "margin:0 0 0 .8ex;border-left:1px solid #ccc;padding-left:1ex",
+        );
+      }
+      clean(child);
+    });
+  }
+
+  clean(doc.body);
+  return doc.body.innerHTML;
+}
+
+function textBodyAsHtml(value) {
+  return `<div style="white-space:pre-wrap">${escapeHtml(value || "")}</div>`;
+}
+
+function senderIdentity(message) {
+  const sender = message?.sender || message?.from || {};
+  const name = typeof sender === "string" ? "" : (sender.name || "").trim();
+  const address = typeof sender === "string"
+    ? sender.trim()
+    : (sender.address || sender.email || "").trim();
+  if (name && address) return `${name} <${address}>`;
+  return address || name || "Unknown sender";
+}
+
+function attributionDate(message) {
+  const value = message?.received_at
+    || message?.sent_at
+    || message?.created_at
+    || message?.updated_at;
+  const date = value ? new Date(value) : null;
+  if (!date || Number.isNaN(date.getTime())) return "";
+
+  const datePart = new Intl.DateTimeFormat(undefined, {
+    weekday: "short",
+    month: "short",
+    day: "numeric",
+    year: "numeric",
+  }).format(date);
+  const timePart = new Intl.DateTimeFormat(undefined, {
+    hour: "numeric",
+    minute: "2-digit",
+  }).format(date);
+  return `${datePart} at ${timePart}`;
+}
+
+export function buildReplyBodyHtml(message) {
+  const original = cleanMessageHtml(message?.body_html)
+    || textBodyAsHtml(message?.body_text || message?.snippet || "");
+  const date = attributionDate(message);
+  const identity = senderIdentity(message);
+  const attribution = `On ${date ? `${date} ` : ""}${identity} wrote:`;
+
+  return [
+    "<div><br></div>",
+    '<div class="gmail_quote">',
+    `<div class="gmail_attr">${escapeHtml(attribution)}<br></div>`,
+    '<blockquote class="gmail_quote" ',
+    'style="margin:0 0 0 .8ex;border-left:1px solid #ccc;padding-left:1ex">',
+    original,
+    "</blockquote>",
+    "</div>",
+  ].join("");
+}
+
+export function plainTextFromEmailHtml(value) {
+  if (!value) return "";
+  const doc = new DOMParser().parseFromString(value, "text/html");
+
+  function render(node) {
+    if (node.nodeType === Node.TEXT_NODE) return node.textContent || "";
+    if (node.nodeType !== Node.ELEMENT_NODE) return "";
+    if (node.tagName === "BR") return "\n";
+
+    if (node.tagName === "BLOCKQUOTE") {
+      const content = [...node.childNodes]
+        .map((child) => render(child))
+        .join("")
+        .replace(/^\n+|\n+$/g, "");
+      return `\n${content.split("\n").map((line) => `> ${line}`).join("\n")}\n`;
+    }
+
+    const content = [...node.childNodes]
+      .map((child) => render(child))
+      .join("");
+    return BLOCK_TAGS.has(node.tagName) ? `\n${content}\n` : content;
+  }
+
+  return render(doc.body)
+    .replace(/\u00a0/g, " ")
+    .replace(/[ \t]+\n/g, "\n")
+    .replace(/\n{3,}/g, "\n\n")
+    .trim();
+}