thread fix

backend/app/services/reply_extractor.py

@@ -1,5 +1,6 @@
 import re
 from html import unescape
+from html.parser import HTMLParser
 
 # Patterns that mark the start of quoted previous-message content.
 TEXT_QUOTE_MARKERS = [
@@ -13,12 +14,6 @@ TEXT_QUOTE_MARKERS = [
     re.compile(r"\n\s*> ?"),
 ]
 
-HTML_QUOTE_PATTERNS = [
-    re.compile(r"<blockquote\b[^>]*>.*?</blockquote>", re.IGNORECASE | re.DOTALL),
-    re.compile(r'<div[^>]+class="[^"]*gmail_quote[^"]*"[^>]*>.*?</div>', re.IGNORECASE | re.DOTALL),
-    re.compile(r'<div[^>]+class="[^"]*gmail_attr[^"]*"[^>]*>.*?</div>', re.IGNORECASE | re.DOTALL),
-]
-
 PRIVATE_LEAK_MARKERS = [
     "human handoff",
     "a conversation needs your attention",
@@ -39,14 +34,75 @@ def _normalize_newlines(text: str) -> str:
     return text.replace("\r\n", "\n").replace("\r", "\n")
 
 
+class _ReplyHTMLTextParser(HTMLParser):
+    BLOCK_TAGS = {"div", "p", "pre", "li", "ul", "ol", "table", "tr"}
+    IGNORED_TAGS = {"script", "style", "iframe", "object"}
+
+    def __init__(self) -> None:
+        super().__init__(convert_charrefs=True)
+        self.parts: list[str] = []
+        self.quote_depth = 0
+        self.ignored_depth = 0
+        self.at_line_start = True
+
+    def _newline(self) -> None:
+        if self.parts and not self.parts[-1].endswith("\n"):
+            self.parts.append("\n")
+        self.at_line_start = True
+
+    def _text(self, value: str) -> None:
+        if self.ignored_depth:
+            return
+        normalized = re.sub(r"\s+", " ", value)
+        if not normalized.strip():
+            return
+        if self.at_line_start:
+            normalized = normalized.lstrip()
+            if self.quote_depth:
+                self.parts.append("> " * self.quote_depth)
+        self.parts.append(normalized)
+        self.at_line_start = False
+
+    def handle_starttag(self, tag: str, attrs) -> None:
+        tag = tag.lower()
+        if tag in self.IGNORED_TAGS:
+            self.ignored_depth += 1
+            return
+        if self.ignored_depth:
+            return
+        if tag == "br":
+            self._newline()
+        elif tag == "blockquote":
+            self._newline()
+            self.quote_depth += 1
+        elif tag in self.BLOCK_TAGS:
+            self._newline()
+
+    def handle_endtag(self, tag: str) -> None:
+        tag = tag.lower()
+        if tag in self.IGNORED_TAGS:
+            self.ignored_depth = max(0, self.ignored_depth - 1)
+            return
+        if self.ignored_depth:
+            return
+        if tag == "blockquote":
+            self._newline()
+            self.quote_depth = max(0, self.quote_depth - 1)
+        elif tag in self.BLOCK_TAGS:
+            self._newline()
+
+    def handle_data(self, data: str) -> None:
+        self._text(data)
+
+    def text(self) -> str:
+        return "".join(self.parts).strip()
+
+
 def _html_to_text(html: str) -> str:
-    cleaned = html
-    for pattern in HTML_QUOTE_PATTERNS:
-        cleaned = pattern.sub("", cleaned)
-    cleaned = re.sub(r"<br\s*/?>", "\n", cleaned, flags=re.IGNORECASE)
-    cleaned = re.sub(r"</p\s*>", "\n", cleaned, flags=re.IGNORECASE)
-    cleaned = re.sub(r"<[^>]+>", "", cleaned)
-    return unescape(cleaned)
+    parser = _ReplyHTMLTextParser()
+    parser.feed(html)
+    parser.close()
+    return unescape(parser.text())
 
 
 def _strip_trailing_quoted_lines(text: str) -> str:
@@ -83,10 +139,7 @@ def extract_new_reply_content(
         raw_text = _html_to_text(body_html).strip()
 
     if body_html:
-        html_stripped = body_html
-        for pattern in HTML_QUOTE_PATTERNS:
-            html_stripped = pattern.sub("", html_stripped)
-        html_text = _html_to_text(html_stripped).strip()
+        html_text = _html_to_text(body_html).strip()
         if html_text:
             new_from_html, quoted_from_html = _split_at_quote_markers(html_text)
             if new_from_html:
@@ -110,17 +163,13 @@ def reply_fields_for_message(
     body_text: str | None,
     body_html: str | None = None,
 ) -> dict[str, str | bool]:
-    if "new_reply_content" in message:
-        return {
-            "new_reply_content": message.get("new_reply_content", ""),
-            "quoted_content": message.get("quoted_content", ""),
-            "has_quoted_text": bool(message.get("has_quoted_text") or message.get("quoted_content")),
-        }
     built = build_reply_display(body_text=body_text, body_html=body_html)
+    new_content = built["new_reply_content"] or message.get("new_reply_content", "")
+    quoted_content = built["quoted_content"] or message.get("quoted_content", "")
     return {
-        "new_reply_content": built["new_reply_content"],
-        "quoted_content": built["quoted_content"],
-        "has_quoted_text": built["has_quoted_text"],
+        "new_reply_content": new_content,
+        "quoted_content": quoted_content,
+        "has_quoted_text": bool(quoted_content),
     }