diff --git a/backend/app/main.py b/backend/app/main.py index 48dbd75..bc5202c 100644 --- a/backend/app/main.py +++ b/backend/app/main.py @@ -33,11 +33,13 @@ from .routes_agent_messages import router as agent_messages_router from .routes_urmiport import router as urmiport_router from .routes_urmiport_api import router as urmiport_api_router from .agent_templates_seed import seed_agent_message_templates +from .whatsapp_broadcast_jobs import resume_broadcast_jobs @asynccontextmanager async def lifespan(_: FastAPI): Base.metadata.create_all(bind=engine) + resume_broadcast_jobs() with Session(engine) as db: seed_agent_message_templates(db) with engine.begin() as conn: diff --git a/backend/app/models.py b/backend/app/models.py index 6a334fd..44afe6f 100644 --- a/backend/app/models.py +++ b/backend/app/models.py @@ -644,6 +644,34 @@ class WhatsAppMessageTemplate(Base): ) +class WhatsAppBroadcastJob(Base): + """A CSV broadcast that sends in paced batches and reports progress while it runs.""" + + __tablename__ = "whatsapp_broadcast_jobs" + + id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + user_id: Mapped[uuid.UUID] = mapped_column(ForeignKey("user_accounts.id", ondelete="CASCADE"), index=True) + account_id: Mapped[uuid.UUID] = mapped_column( + ForeignKey("whatsapp_business_accounts.id", ondelete="CASCADE"), index=True + ) + filename: Mapped[str] = mapped_column(String(200)) + country_code: Mapped[str] = mapped_column(String(8), default="91") + template_name: Mapped[str] = mapped_column(String(512)) + template_language: Mapped[str] = mapped_column(String(16), default="en") + product_policy: Mapped[str] = mapped_column(String(32), default="STRICT") + use_sample_header: Mapped[bool] = mapped_column(Boolean, default=False) + body_parameters: Mapped[list] = mapped_column(JSON, default=list) + recipients: Mapped[list] = mapped_column(JSON, default=list) + status: Mapped[str] = mapped_column(String(24), default="processing", index=True) + total: Mapped[int] = mapped_column(default=0) + sent_count: Mapped[int] = mapped_column(default=0) + failed_count: Mapped[int] = mapped_column(default=0) + next_index: Mapped[int] = mapped_column(default=0) + last_error: Mapped[str | None] = mapped_column(String(500), nullable=True) + created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now(), index=True) + completed_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True) + + class WhatsAppConnectionAudit(Base): __tablename__ = "whatsapp_connection_audits" diff --git a/backend/app/routes_whatsapp.py b/backend/app/routes_whatsapp.py index 3718a70..2999206 100644 --- a/backend/app/routes_whatsapp.py +++ b/backend/app/routes_whatsapp.py @@ -1,7 +1,7 @@ import json import logging -from fastapi import APIRouter, Depends, Header, HTTPException, Query, Request, Response, status +from fastapi import APIRouter, Depends, File, Form, Header, HTTPException, Query, Request, Response, UploadFile, status from pydantic import BaseModel, Field from sqlalchemy import or_, select from sqlalchemy.orm import Session @@ -16,6 +16,7 @@ from .models import ( WhatsAppPhoneAsset, WhatsAppWebhookEvent, ) +from .whatsapp_broadcast_jobs import create_csv_broadcast, list_broadcast_jobs from .whatsapp_graph_client import WhatsAppGraphClient, WhatsAppGraphError from .whatsapp_config import resolved_whatsapp_app_secret from .whatsapp_onboarding import ( @@ -359,6 +360,47 @@ async def whatsapp_account_broadcast( ) +@router.get("/accounts/{account_id}/broadcasts/jobs") +def whatsapp_broadcast_jobs( + account_id: str, + db: Session = Depends(get_db), + user: UserAccount = Depends(current_user), +) -> list[dict]: + return list_broadcast_jobs(db, user, account_id) + + +@router.post("/accounts/{account_id}/broadcasts/csv") +async def whatsapp_broadcast_csv( + account_id: str, + file: UploadFile = File(), + country_code: str = Form(default="91"), + template_name: str = Form(), + template_language: str = Form(default="en"), + product_policy: str = Form(default="STRICT"), + use_sample_header: str = Form(default="false"), + body_parameters: str = Form(default=""), + db: Session = Depends(get_db), + user: UserAccount = Depends(current_user), +) -> dict: + raw = await file.read() + if len(raw) > 1_000_000: + raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail="The CSV must be under 1 MB.") + parameters = [line.strip() for line in body_parameters.splitlines() if line.strip()] + return await create_csv_broadcast( + db, + user, + account_id, + file.filename or "broadcast.csv", + raw, + country_code=country_code, + template_name=template_name, + template_language=template_language, + body_parameters=parameters, + product_policy=product_policy, + use_sample_header=use_sample_header.strip().lower() in {"1", "true", "yes", "on"}, + ) + + @router.get("/accounts/{account_id}/media/{media_id}") async def whatsapp_account_media( account_id: str, diff --git a/backend/app/whatsapp_broadcast_jobs.py b/backend/app/whatsapp_broadcast_jobs.py new file mode 100644 index 0000000..d80969e --- /dev/null +++ b/backend/app/whatsapp_broadcast_jobs.py @@ -0,0 +1,368 @@ +"""CSV broadcasts paced under Meta's per-number throughput limit.""" + +from __future__ import annotations + +import asyncio +import csv +import io +import logging +import re +from datetime import datetime, timezone +from pathlib import Path +from typing import Any +from uuid import UUID + +from fastapi import HTTPException, status +from sqlalchemy import select +from sqlalchemy.orm import Session + +from .database import SessionLocal +from .models import UserAccount, WhatsAppBroadcastJob +from .whatsapp_graph_client import WhatsAppGraphClient, WhatsAppGraphError +from .whatsapp_messages import extract_wa_message_id, record_outbound_message +from .whatsapp_onboarding import _messaging_phone, _owned_account + +logger = logging.getLogger(__name__) + +# Meta Cloud API allows 80 messages per second on a Cloud API number, and 20 per +# second when that number is also signed into the WhatsApp Business app. +# Ten sequential sends, then a two second pause, stays under the lower limit. +BATCH_SIZE = 10 +COOLDOWN_SECONDS = 2 +RATE_LIMIT_PAUSE_SECONDS = 10 +MAX_NUMBERS = 1000 + +COUNTRIES = ( + {"code": "91", "label": "India"}, + {"code": "1", "label": "United States"}, + {"code": "44", "label": "United Kingdom"}, + {"code": "971", "label": "United Arab Emirates"}, + {"code": "65", "label": "Singapore"}, +) +_COUNTRY_CODES = {item["code"] for item in COUNTRIES} +_running: set[UUID] = set() + + +def country_options() -> list[dict[str, str]]: + return [{"code": item["code"], "label": item["label"]} for item in COUNTRIES] + + +def prepare_phone_number(raw: str, country_code: str) -> tuple[str | None, str | None]: + """Strip symbols, add the country code to a 10-digit number, then check the prefix. + + Returns (number, None) when it can be sent, (None, None) for a blank cell, + or (None, reason) when the row should count as failed. + """ + digits = re.sub(r"\D", "", raw or "") + if not digits: + return None, None + full_length = 10 + len(country_code) + if len(digits) == 10: + digits = f"{country_code}{digits}" + elif len(digits) != full_length: + return None, "This number does not match the selected country code." + if not digits.startswith(country_code): + return None, f"After cleanup this number must start with {country_code}." + return digits, None + + +def _parse_csv(raw: bytes, country_code: str) -> tuple[list[str], int]: + try: + text = raw.decode("utf-8-sig") + except UnicodeDecodeError as exc: + raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail="The CSV must be UTF-8.") from exc + reader = csv.DictReader(io.StringIO(text)) + if not reader.fieldnames: + raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail="The CSV needs a header row.") + fields = [(name or "").strip().lower() for name in reader.fieldnames] + if "phone_number" not in fields: + raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail="The CSV needs a phone_number column.") + reader.fieldnames = fields + rows = list(reader) + filled = [row for row in rows if str((row or {}).get("phone_number") or "").strip()] + if not filled: + raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail="Add at least one phone number.") + if len(filled) > MAX_NUMBERS: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail=f"A CSV can include at most {MAX_NUMBERS} numbers.", + ) + ready: list[str] = [] + seen: set[str] = set() + failed = 0 + for row in filled: + number, reason = prepare_phone_number(str(row.get("phone_number") or ""), country_code) + if reason or not number: + failed += 1 + continue + if number in seen: + failed += 1 + continue + seen.add(number) + ready.append(number) + return ready, failed + + +def _serialize_job(job: WhatsAppBroadcastJob) -> dict[str, Any]: + return { + "id": str(job.id), + "account_id": str(job.account_id), + "filename": job.filename, + "country_code": job.country_code, + "template_name": job.template_name, + "template_language": job.template_language, + "status": job.status, + "total": job.total, + "sent_count": job.sent_count, + "failed_count": job.failed_count, + "last_error": job.last_error, + "created_at": job.created_at, + "completed_at": job.completed_at, + } + + +def list_broadcast_jobs(db: Session, user: UserAccount, account_id: Any) -> list[dict[str, Any]]: + account = _owned_account(db, user, account_id) + rows = db.scalars( + select(WhatsAppBroadcastJob) + .where(WhatsAppBroadcastJob.user_id == user.id, WhatsAppBroadcastJob.account_id == account.id) + .order_by(WhatsAppBroadcastJob.created_at.desc()) + .limit(50) + ).all() + return [_serialize_job(row) for row in rows] + + +def _finish_if_idle(db: Session, job: WhatsAppBroadcastJob) -> None: + recipients = job.recipients if isinstance(job.recipients, list) else [] + if job.next_index < len(recipients): + return + job.status = "completed" + job.completed_at = datetime.now(timezone.utc) + db.commit() + + +async def create_csv_broadcast( + db: Session, + user: UserAccount, + account_id: Any, + filename: str, + raw: bytes, + *, + country_code: str, + template_name: str, + template_language: str, + body_parameters: list[str], + product_policy: str, + use_sample_header: bool, +) -> dict[str, Any]: + code = "".join(ch for ch in str(country_code or "") if ch.isdigit()) + if code not in _COUNTRY_CODES: + raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail="Choose a country code from the list.") + name = template_name.strip() + if not name: + raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail="Choose a marketing template.") + account = _owned_account(db, user, account_id) + _messaging_phone(account) + if use_sample_header: + from .whatsapp_template_service import _find_sample, _header_dict, _template_for_send + + template = _template_for_send(db, user, account, name, template_language.strip() or "en") + header = _header_dict(template.definition if template else None) + if template is None or _find_sample(user.id, str(header.get("sample_id") or "")) is None: + raise HTTPException( + status_code=status.HTTP_409_CONFLICT, + detail="Add the sample file on the template page before this broadcast can attach it.", + ) + ready, failed = _parse_csv(raw, code) + safe_name = Path(filename or "broadcast.csv").name[:200] or "broadcast.csv" + job = WhatsAppBroadcastJob( + user_id=user.id, + account_id=account.id, + filename=safe_name, + country_code=code, + template_name=name, + template_language=template_language.strip() or "en", + product_policy=product_policy if product_policy in {"STRICT", "CLOUD_API_FALLBACK"} else "STRICT", + use_sample_header=use_sample_header, + body_parameters=[str(item).strip() for item in body_parameters if str(item).strip()], + recipients=ready, + status="processing", + total=len(ready) + failed, + sent_count=0, + failed_count=failed, + next_index=0, + ) + db.add(job) + db.commit() + db.refresh(job) + _finish_if_idle(db, job) + db.refresh(job) + if job.status == "processing": + schedule_broadcast_job(job.id) + return _serialize_job(job) + + +def schedule_broadcast_job(job_id: UUID) -> None: + if job_id in _running: + return + try: + loop = asyncio.get_running_loop() + except RuntimeError: + return + _running.add(job_id) + loop.create_task(run_broadcast_job(job_id)) + + +def resume_broadcast_jobs() -> None: + with SessionLocal() as db: + ids = db.scalars( + select(WhatsAppBroadcastJob.id).where(WhatsAppBroadcastJob.status == "processing") + ).all() + for job_id in ids: + schedule_broadcast_job(job_id) + + +def _rate_limited(detail: str) -> bool: + text = detail.lower() + return "130429" in text or "rate limit" in text or "throughput" in text + + +async def _send_one(client: WhatsAppGraphClient, number: str, name: str, language: str, components: list[dict[str, Any]] | None, policy: str) -> tuple[str, dict[str, Any] | None, str | None]: + try: + result = await client.send_marketing_template( + number, + name, + language_code=language, + components=components, + product_policy=policy, + ) + return "sent", result if isinstance(result, dict) else None, None + except WhatsAppGraphError as exc: + if not _rate_limited(exc.detail): + return "failed", None, exc.detail + await asyncio.sleep(RATE_LIMIT_PAUSE_SECONDS) + try: + result = await client.send_marketing_template( + number, + name, + language_code=language, + components=components, + product_policy=policy, + ) + return "sent", result if isinstance(result, dict) else None, None + except WhatsAppGraphError as retry_exc: + return "failed", None, retry_exc.detail + + +async def run_broadcast_job(job_id: UUID) -> None: + client: WhatsAppGraphClient | None = None + components: list[dict[str, Any]] | None = None + from_number = "" + phone_number_id = "" + try: + while True: + with SessionLocal() as db: + job = db.get(WhatsAppBroadcastJob, job_id) + if job is None or job.status != "processing": + return + recipients = list(job.recipients or []) + if job.next_index >= len(recipients): + _finish_if_idle(db, job) + return + batch_start = job.next_index + batch = recipients[batch_start:batch_start + BATCH_SIZE] + user = db.get(UserAccount, job.user_id) + if user is None: + job.status = "failed" + job.last_error = "This broadcast has no workspace." + job.completed_at = datetime.now(timezone.utc) + db.commit() + return + if client is None: + account = _owned_account(db, user, job.account_id) + phone, token = _messaging_phone(account) + phone_number_id = phone.phone_number_id + from_number = phone.display_phone or phone.phone_number_id + pieces: list[dict[str, Any]] = [] + if job.use_sample_header: + from .whatsapp_template_service import header_parameter_for_send + + pieces.append( + await header_parameter_for_send( + db, + user, + account, + token, + phone.phone_number_id, + job.template_name, + job.template_language, + ) + ) + parameters = job.body_parameters if isinstance(job.body_parameters, list) else [] + if parameters: + pieces.append( + { + "type": "body", + "parameters": [{"type": "text", "text": str(item)} for item in parameters], + } + ) + components = pieces or None + client = WhatsAppGraphClient( + access_token=token, + phone_number_id=phone.phone_number_id, + business_account_id=account.waba_id, + ) + template_name = job.template_name + template_language = job.template_language + policy = job.product_policy + user_id = job.user_id + if client is None: + return + for offset, number in enumerate(batch): + outcome, result, error = await _send_one(client, number, template_name, template_language, components, policy) + with SessionLocal() as db: + job = db.get(WhatsAppBroadcastJob, job_id) + if job is None or job.status != "processing": + return + expected = batch_start + offset + if job.next_index != expected: + return + job.next_index = expected + 1 + if outcome == "sent": + job.sent_count += 1 + record_outbound_message( + db, + to_number=number, + body=f"Broadcast: {template_name}", + message_type="template", + wa_message_id=extract_wa_message_id(result), + status="sent", + payload=result, + from_number=from_number, + user_id=user_id, + phone_number_id=phone_number_id, + ) + else: + job.failed_count += 1 + job.last_error = (error or "Meta did not accept this number.")[:500] + db.commit() + if len(batch) == BATCH_SIZE: + await asyncio.sleep(COOLDOWN_SECONDS) + except HTTPException as exc: + _mark_failed(job_id, str(exc.detail)) + except Exception: + logger.exception("CSV broadcast %s stopped", job_id) + _mark_failed(job_id, "The broadcast stopped before it finished.") + finally: + _running.discard(job_id) + + +def _mark_failed(job_id: UUID, message: str) -> None: + with SessionLocal() as db: + job = db.get(WhatsAppBroadcastJob, job_id) + if job is None or job.status != "processing": + return + job.status = "failed" + job.last_error = message[:500] + job.completed_at = datetime.now(timezone.utc) + db.commit() diff --git a/frontend/src/features/whatsapp/WhatsAppBroadcastPage.tsx b/frontend/src/features/whatsapp/WhatsAppBroadcastPage.tsx index 240522e..ffc1e6e 100644 --- a/frontend/src/features/whatsapp/WhatsAppBroadcastPage.tsx +++ b/frontend/src/features/whatsapp/WhatsAppBroadcastPage.tsx @@ -1,7 +1,16 @@ import { Loader2 } from 'lucide-react' import { useEffect, useState, type FormEvent } from 'react' import { Link, useSearchParams } from 'react-router-dom' -import { waTemplateApi, whatsappApi, type WaManagedTemplate, type WhatsAppAccount, type WhatsAppTemplateRow } from '../../lib/api' +import { Bar, BarChart, CartesianGrid, Cell, ResponsiveContainer, Tooltip, XAxis, YAxis } from 'recharts' +import { waTemplateApi, whatsappApi, type WaManagedTemplate, type WhatsAppAccount, type WhatsAppBroadcastJob, type WhatsAppTemplateRow } from '../../lib/api' + +const COUNTRIES = [ + { code: '91', label: 'India' }, + { code: '1', label: 'United States' }, + { code: '44', label: 'United Kingdom' }, + { code: '971', label: 'United Arab Emirates' }, + { code: '65', label: 'Singapore' }, +] const TEAM_TEMPLATE_NAME = 'urmi_team_aircheck' const TEAM_TEMPLATE_BODY = "Team, the broadcast desk is on air. This is the first note Urmi is sending through Meta's marketing line, and it is written for the people building it. Read it on your phone, then bring it to standup so we can close the test." @@ -34,6 +43,13 @@ export function WhatsAppBroadcastPage() { const [created, setCreated] = useState(null) const [templateTick, setTemplateTick] = useState(0) const [recipients, setRecipients] = useState('') + const [countryCode, setCountryCode] = useState('91') + const [csvFile, setCsvFile] = useState(null) + const [csvKey, setCsvKey] = useState(0) + const [csvBusy, setCsvBusy] = useState(false) + const [jobs, setJobs] = useState([]) + const [selectedJobId, setSelectedJobId] = useState('') + const [jobTick, setJobTick] = useState(0) const [bodyValues, setBodyValues] = useState('') const [strict, setStrict] = useState(true) const [loading, setLoading] = useState(true) @@ -79,6 +95,29 @@ export function WhatsAppBroadcastPage() { .catch(() => setManaged([])) }, [accountId, templateTick]) + useEffect(() => { + if (!accountId) return + let cancelled = false + let timer = 0 + async function load() { + try { + const rows = await whatsappApi.broadcastJobs(accountId) + if (cancelled) return + setJobs(rows) + if (rows.some((job) => job.status === 'processing')) { + timer = window.setTimeout(() => { void load() }, 3000) + } + } catch { + if (!cancelled) setJobs([]) + } + } + void load() + return () => { + cancelled = true + window.clearTimeout(timer) + } + }, [accountId, jobTick]) + async function createTemplate(event: FormEvent) { event.preventDefault() setCreating(true) @@ -108,6 +147,11 @@ export function WhatsAppBroadcastPage() { setResult(null) const numbers = recipients.split(/[\n,]/).map((item) => item.trim()).filter(Boolean) const parameters = bodyValues.split('\n').map((item) => item.trim()).filter(Boolean) + if (!numbers.length) { + setError('Enter at least one number, or start a CSV broadcast.') + setBusy(false) + return + } if (mediaHeader && !savedSample) { setError(`Add the sample ${headerLabel} on the template page. Meta rejects this send without it.`) setBusy(false) @@ -142,6 +186,47 @@ export function WhatsAppBroadcastPage() { const mediaHeader = ['IMAGE', 'VIDEO', 'DOCUMENT'].includes(selectedTemplate?.header_format || '') const savedSample = Boolean(selectedTemplate?.sample_stored) const headerLabel = selectedTemplate?.header_format === 'VIDEO' ? 'video' : selectedTemplate?.header_format === 'DOCUMENT' ? 'document' : 'image' + const selectedJob = jobs.find((job) => job.id === selectedJobId) || null + + async function startCsv() { + if (!csvFile) { + setError('Choose a CSV with a phone_number column.') + return + } + if (!templateName.trim()) { + setError('Choose a marketing template.') + return + } + if (mediaHeader && !savedSample) { + setError(`Add the sample ${headerLabel} on the template page. Meta rejects this send without it.`) + return + } + if (mediaHeader && !useSample) { + setError(`Turn on the saved sample ${headerLabel}. This template’s header is a ${headerLabel}, and Meta rejects the send without it.`) + return + } + setCsvBusy(true) + setError('') + try { + const job = await whatsappApi.uploadBroadcastCsv(accountId, csvFile, { + country_code: countryCode, + template_name: templateName.trim(), + template_language: templateLanguage.trim() || 'en', + product_policy: strict ? 'STRICT' : 'CLOUD_API_FALLBACK', + use_sample_header: Boolean(mediaHeader && useSample && savedSample), + body_parameters: bodyValues, + }) + setJobs((current) => [job, ...current.filter((item) => item.id !== job.id)]) + setSelectedJobId(job.id) + setCsvFile(null) + setCsvKey((value) => value + 1) + setJobTick((value) => value + 1) + } catch (reason) { + setError(reason instanceof Error ? reason.message : 'The CSV broadcast could not be started') + } finally { + setCsvBusy(false) + } + } return (
@@ -239,9 +324,32 @@ export function WhatsAppBroadcastPage() {