import re from html import unescape from html.parser import HTMLParser # Patterns that mark the start of quoted previous-message content. TEXT_QUOTE_MARKERS = [ re.compile(r"\n-{2,}\s*Original Message\s*-{2,}", re.IGNORECASE), re.compile(r"(?:^|\n)\s*On .+? wrote:\s*", re.IGNORECASE), re.compile( r"\nFrom:\s*.+\nSent:\s*.+\nTo:\s*.+\nSubject:\s*.+", re.IGNORECASE, ), re.compile(r"\n_{5,}\n"), re.compile(r"\n\s*> ?"), ] PRIVATE_LEAK_MARKERS = [ "human handoff", " · handoff · ", "a conversation needs your attention", "why this was handed off", "why you're receiving this", "gentic human handoff", "no handoff instructions were set", "step passes", "classifier returned", "rule evaluation succeeded", "reply normally to respond through", "latest message", "private agent ↔ human conversation", ] def _normalize_newlines(text: str) -> str: return text.replace("\r\n", "\n").replace("\r", "\n") class _ReplyHTMLTextParser(HTMLParser): BLOCK_TAGS = {"div", "p", "pre", "li", "ul", "ol", "table", "tr"} IGNORED_TAGS = {"script", "style", "iframe", "object"} QUOTE_CLASSES = { "gmail_attr", "gmail_quote", "protonmail_quote", "yahoo_quoted", "moz-cite-prefix", } def __init__(self) -> None: super().__init__(convert_charrefs=True) self.parts: list[str] = [] self.quote_depth = 0 self.quote_tags: list[str] = [] self.ignored_depth = 0 self.at_line_start = True def _newline(self) -> None: if self.parts and not self.parts[-1].endswith("\n"): self.parts.append("\n") self.at_line_start = True def _text(self, value: str) -> None: if self.ignored_depth: return normalized = re.sub(r"\s+", " ", value) if not normalized.strip(): return if self.at_line_start: normalized = normalized.lstrip() if self.quote_depth: self.parts.append("> " * self.quote_depth) self.parts.append(normalized) self.at_line_start = False def handle_starttag(self, tag: str, attrs) -> None: tag = tag.lower() if tag in self.IGNORED_TAGS: self.ignored_depth += 1 return if self.ignored_depth: return classes = { value for key, value in attrs if key.lower() == "class" for value in (value or "").split() } is_quote = tag == "blockquote" or bool(classes.intersection(self.QUOTE_CLASSES)) if tag == "br": self._newline() elif is_quote: self._newline() self.quote_depth += 1 self.quote_tags.append(tag) elif tag in self.BLOCK_TAGS: self._newline() def handle_endtag(self, tag: str) -> None: tag = tag.lower() if tag in self.IGNORED_TAGS: self.ignored_depth = max(0, self.ignored_depth - 1) return if self.ignored_depth: return if self.quote_tags and self.quote_tags[-1] == tag: self.quote_tags.pop() self.quote_depth = max(0, self.quote_depth - 1) self._newline() elif tag in self.BLOCK_TAGS: self._newline() def handle_data(self, data: str) -> None: self._text(data) def text(self) -> str: return "".join(self.parts).strip() def _html_to_text(html: str) -> str: parser = _ReplyHTMLTextParser() parser.feed(html) parser.close() return unescape(parser.text()) def _strip_trailing_quoted_lines(text: str) -> str: lines = text.split("\n") kept: list[str] = [] for line in lines: if re.match(r"^\s*>", line): break kept.append(line) return "\n".join(kept).strip() def _split_at_quote_markers(text: str) -> tuple[str, str]: normalized = _normalize_newlines(text) earliest_index = None for pattern in TEXT_QUOTE_MARKERS: match = pattern.search(normalized) if match and (earliest_index is None or match.start() < earliest_index): earliest_index = match.start() if earliest_index is not None: new_content = normalized[:earliest_index].strip() quoted = normalized[earliest_index:].strip() return new_content, quoted return _strip_trailing_quoted_lines(normalized), "" def extract_new_reply_content( *, body_text: str | None, body_html: str | None = None, ) -> dict[str, str]: raw_text = _normalize_newlines((body_text or "").strip()) if body_html and not raw_text: raw_text = _html_to_text(body_html).strip() if body_html: html_text = _html_to_text(body_html).strip() if html_text: new_from_html, quoted_from_html = _split_at_quote_markers(html_text) if new_from_html: return { "raw_body": raw_text or html_text, "new_reply_content": new_from_html, "quoted_content": quoted_from_html, } new_content, quoted_content = _split_at_quote_markers(raw_text) return { "raw_body": raw_text, "new_reply_content": new_content, "quoted_content": quoted_content, } def reply_fields_for_message( message: dict, *, body_text: str | None, body_html: str | None = None, ) -> dict[str, str | bool]: built = build_reply_display(body_text=body_text, body_html=body_html) new_content = built["new_reply_content"] or message.get("new_reply_content", "") quoted_content = built["quoted_content"] or message.get("quoted_content", "") return { "new_reply_content": new_content, "quoted_content": quoted_content, "has_quoted_text": bool(quoted_content), } def build_reply_display( *, body_text: str | None, body_html: str | None = None, limit: int = 240, ) -> dict[str, str | bool]: extracted = extract_new_reply_content(body_text=body_text, body_html=body_html) new_content = extracted["new_reply_content"] quoted = extracted["quoted_content"] fallback = re.sub(r"\s+", " ", (body_text or "")).strip() snippet_source = new_content or fallback return { "new_reply_content": new_content, "quoted_content": quoted, "has_quoted_text": bool(quoted), "raw_body": extracted["raw_body"], "snippet": re.sub(r"\s+", " ", snippet_source).strip()[:limit], } def validate_external_reply_body(body: str) -> bool: normalized = (body or "").strip() if not normalized: return False lower = normalized.lower() hits = sum(1 for marker in PRIVATE_LEAK_MARKERS if marker in lower) return hits == 0