first commit

monitor/__init__.py

@@ -0,0 +1,3 @@
+"""Browser monitoring package."""
+
+__version__ = "0.1.0"

monitor/browser.py

@@ -0,0 +1,310 @@
+"""Browser discovery, launch, and page capture using Playwright."""
+
+from __future__ import annotations
+
+import logging
+import shutil
+import time
+from datetime import datetime
+from pathlib import Path
+from typing import Optional
+
+from playwright.sync_api import (
+    BrowserContext,
+    Error as PlaywrightError,
+    Page,
+    Playwright,
+    sync_playwright,
+)
+
+from monitor.models import AppConfig, PageSnapshot, WebsiteConfig
+from monitor.snapshot import capture_page_snapshot
+from storage.snapshots import SnapshotStore
+
+logger = logging.getLogger(__name__)
+
+FATAL_BROWSER_ERRORS = ("TargetClosedError", "BrowserClosedError", "TargetCrashedError")
+
+
+def _is_fatal_browser_error(exc: PlaywrightError) -> bool:
+    """Return True if the error indicates the browser or page is no longer usable."""
+    return type(exc).__name__ in FATAL_BROWSER_ERRORS
+
+# Common browser executable locations on Linux
+BROWSER_CANDIDATES: dict[str, list[str]] = {
+    "chrome": [
+        "/usr/bin/google-chrome",
+        "/usr/bin/google-chrome-stable",
+        "/snap/bin/chromium",  # some systems
+    ],
+    "chromium": [
+        "/usr/bin/chromium",
+        "/usr/bin/chromium-browser",
+        "/snap/bin/chromium",
+    ],
+    "edge": [
+        "/usr/bin/microsoft-edge",
+        "/usr/bin/microsoft-edge-stable",
+    ],
+    "firefox": [
+        "/usr/bin/firefox",
+        "/snap/bin/firefox",
+    ],
+}
+
+# Playwright channel names for installed browsers
+PLAYWRIGHT_CHANNELS: dict[str, str] = {
+    "chrome": "chrome",
+    "edge": "msedge",
+    "firefox": "firefox",
+}
+
+
+def discover_browser_executable(browser_type: str) -> Optional[str]:
+    """Find an installed browser executable for the given type."""
+    candidates = BROWSER_CANDIDATES.get(browser_type.lower(), [])
+    for candidate in candidates:
+        path = Path(candidate)
+        if path.is_file() and shutil.which(str(path)):
+            return str(path.resolve())
+    return None
+
+
+def resolve_browser_launch_options(config: AppConfig) -> dict:
+    """Build Playwright launch options from configuration."""
+    browser_type = config.browser.type.lower()
+    options: dict = {
+        "headless": config.browser.headless,
+        "args": ["--disable-blink-features=AutomationControlled"],
+    }
+
+    if config.browser.executable_path:
+        options["executable_path"] = config.browser.executable_path
+        logger.info("Using configured browser executable: %s", config.browser.executable_path)
+    else:
+        discovered = discover_browser_executable(browser_type)
+        if discovered:
+            options["executable_path"] = discovered
+            logger.info("Browser detected: %s at %s", browser_type.title(), discovered)
+        elif browser_type in PLAYWRIGHT_CHANNELS:
+            options["channel"] = PLAYWRIGHT_CHANNELS[browser_type]
+            logger.info("Using Playwright channel: %s", options["channel"])
+        else:
+            logger.warning(
+                "Could not auto-detect %s; Playwright will use bundled Chromium",
+                browser_type,
+            )
+
+    return options
+
+
+class BrowserManager:
+    """Manages a persistent browser context with recovery support."""
+
+    def __init__(self, config: AppConfig, snapshot_store: SnapshotStore) -> None:
+        self.config = config
+        self.snapshot_store = snapshot_store
+        self._playwright: Optional[Playwright] = None
+        self._context: Optional[BrowserContext] = None
+        self._page: Optional[Page] = None
+
+    @property
+    def is_running(self) -> bool:
+        return self._context is not None
+
+    def start(self) -> None:
+        """Launch the browser with a persistent profile."""
+        self._playwright = sync_playwright().start()
+        profile_dir = Path(self.config.browser.profile_directory)
+        profile_dir.mkdir(parents=True, exist_ok=True)
+
+        launch_options = resolve_browser_launch_options(self.config)
+        browser_type = self.config.browser.type.lower()
+
+        try:
+            if browser_type == "firefox":
+                self._context = self._playwright.firefox.launch_persistent_context(
+                    user_data_dir=str(profile_dir.resolve()),
+                    headless=self.config.browser.headless,
+                    executable_path=launch_options.get("executable_path"),
+                )
+            else:
+                chromium_options = {
+                    "user_data_dir": str(profile_dir.resolve()),
+                    "headless": self.config.browser.headless,
+                    "args": launch_options.get("args", []),
+                }
+                if "executable_path" in launch_options:
+                    chromium_options["executable_path"] = launch_options["executable_path"]
+                if "channel" in launch_options:
+                    chromium_options["channel"] = launch_options["channel"]
+
+                self._context = self._playwright.chromium.launch_persistent_context(
+                    **chromium_options
+                )
+
+            self._page = self._context.pages[0] if self._context.pages else self._context.new_page()
+            logger.info("Browser started with profile: %s", profile_dir)
+        except PlaywrightError as exc:
+            logger.error("Failed to start browser: %s", exc)
+            self.stop()
+            raise
+
+    def stop(self) -> None:
+        """Close browser and Playwright resources."""
+        if self._context:
+            try:
+                self._context.close()
+            except PlaywrightError as exc:
+                logger.warning("Error closing browser context: %s", exc)
+            self._context = None
+            self._page = None
+
+        if self._playwright:
+            try:
+                self._playwright.stop()
+            except PlaywrightError as exc:
+                logger.warning("Error stopping Playwright: %s", exc)
+            self._playwright = None
+
+        logger.info("Browser stopped")
+
+    def restart(self) -> None:
+        """Restart the browser after a crash or failure."""
+        logger.warning("Restarting browser...")
+        self.stop()
+        self.start()
+
+    def ensure_running(self) -> None:
+        """Ensure browser is running, restarting if needed."""
+        if not self.is_running:
+            self.start()
+
+    def visit_and_snapshot(self, website: WebsiteConfig) -> PageSnapshot:
+        """Navigate to a website and capture a page snapshot."""
+        self.ensure_running()
+        assert self._page is not None
+
+        start_time = time.monotonic()
+        timestamp = time.time()
+
+        try:
+            timeout_ms = int(self.config.monitor.page_load_timeout_seconds * 1000)
+            render_wait_ms = int(self.config.monitor.render_wait_seconds * 1000)
+
+            logger.info("Opening %s (%s)", website.name, website.url)
+            self._page.goto(
+                website.url,
+                wait_until="domcontentloaded",
+                timeout=timeout_ms,
+            )
+            try:
+                self._page.wait_for_load_state("networkidle", timeout=timeout_ms)
+            except PlaywrightError:
+                logger.debug(
+                    "networkidle timeout for %s; continuing after render wait",
+                    website.id,
+                )
+            self._page.wait_for_timeout(render_wait_ms)
+
+            load_duration = time.monotonic() - start_time
+            snapshot = capture_page_snapshot(
+                page=self._page,
+                website=website,
+                load_duration=load_duration,
+                snapshot_store=self.snapshot_store,
+                timestamp=timestamp,
+            )
+            logger.info("Snapshot captured: %s", website.name)
+            return snapshot
+
+        except PlaywrightError as exc:
+            load_duration = time.monotonic() - start_time
+            error_type = type(exc).__name__
+
+            if _is_fatal_browser_error(exc):
+                logger.error(
+                    "Browser crash detected on %s [%s]: %s",
+                    website.id,
+                    error_type,
+                    exc,
+                )
+                self._context = None
+                self._page = None
+                raise
+
+            logger.error(
+                "Website %s navigation failed [%s]: %s",
+                website.id,
+                error_type,
+                exc,
+            )
+            return PageSnapshot(
+                website_id=website.id,
+                website_name=website.name,
+                requested_url=website.url,
+                final_url=self._page.url if self._page else website.url,
+                timestamp=datetime.fromtimestamp(timestamp),
+                title="",
+                visible_text="",
+                html="",
+                screenshot_path=None,
+                load_duration=load_duration,
+                success=False,
+                error=f"{error_type}: {exc}",
+            )
+
+    def open_for_login(self, urls: list[str]) -> None:
+        """
+        Open the monitor profile for manual login.
+
+        Keeps the browser open until the user closes it or presses Ctrl+C.
+        Always runs with a visible window regardless of headless config.
+        """
+        original_headless = self.config.browser.headless
+        self.config.browser.headless = False
+
+        try:
+            self.start()
+            assert self._context is not None
+
+            if urls:
+                for index, url in enumerate(urls):
+                    page = self._page if index == 0 else self._context.new_page()
+                    logger.info("Opening tab %d: %s", index + 1, url)
+                    page.goto(url, wait_until="domcontentloaded", timeout=60000)
+            else:
+                logger.info("No URLs provided — browser opened with default tab")
+
+            profile_dir = Path(self.config.browser.profile_directory).resolve()
+            logger.info("Profile directory: %s", profile_dir)
+            logger.info(
+                "Log in to your websites in the browser window. "
+                "Close the browser or press Ctrl+C when finished."
+            )
+
+            while self._context and self._context.pages:
+                time.sleep(0.5)
+
+            logger.info("Browser window closed")
+        except KeyboardInterrupt:
+            logger.info("Login session ended by user")
+        finally:
+            self.config.browser.headless = original_headless
+            self.stop()
+
+    def test_browser(self, test_url: str = "https://example.com") -> bool:
+        """Open browser and visit a test URL to verify configuration."""
+        try:
+            self.start()
+            assert self._page is not None
+            logger.info("Test browser: navigating to %s", test_url)
+            self._page.goto(test_url, wait_until="domcontentloaded", timeout=30000)
+            title = self._page.title()
+            logger.info("Test browser success. Page title: %s", title)
+            return True
+        except PlaywrightError as exc:
+            logger.error("Test browser failed: %s", exc)
+            return False
+        finally:
+            self.stop()

monitor/config.py

@@ -0,0 +1,101 @@
+"""Configuration loading and validation."""
+
+from __future__ import annotations
+
+import os
+from pathlib import Path
+from typing import Any
+
+import yaml
+
+from monitor.models import (
+    AppConfig,
+    BrowserConfig,
+    MonitorConfig,
+    NotificationChannelConfig,
+    NotificationsConfig,
+    WebsiteConfig,
+)
+
+
+def _channel_config(data: dict[str, Any] | None) -> NotificationChannelConfig:
+    if not data:
+        return NotificationChannelConfig()
+    return NotificationChannelConfig(enabled=bool(data.get("enabled", False)))
+
+
+def load_config(path: str | Path) -> AppConfig:
+    """Load application configuration from a YAML file."""
+    config_path = Path(path)
+    if not config_path.exists():
+        raise FileNotFoundError(f"Configuration file not found: {config_path}")
+
+    with config_path.open("r", encoding="utf-8") as handle:
+        raw: dict[str, Any] = yaml.safe_load(handle) or {}
+
+    browser_raw = raw.get("browser", {})
+    monitor_raw = raw.get("monitor", {})
+    notifications_raw = raw.get("notifications", {})
+
+    browser = BrowserConfig(
+        type=str(browser_raw.get("type", "chrome")).lower(),
+        executable_path=browser_raw.get("executable_path"),
+        headless=bool(browser_raw.get("headless", False)),
+        profile_directory=str(browser_raw.get("profile_directory", "./profiles/default")),
+    )
+
+    monitor = MonitorConfig(
+        round_robin_delay_seconds=float(monitor_raw.get("round_robin_delay_seconds", 5)),
+        page_load_timeout_seconds=float(monitor_raw.get("page_load_timeout_seconds", 30)),
+        render_wait_seconds=float(monitor_raw.get("render_wait_seconds", 3)),
+        snapshot_directory=str(monitor_raw.get("snapshot_directory", "./snapshots")),
+        snapshot_retention_count=int(monitor_raw.get("snapshot_retention_count", 50)),
+        browser_restart_max_retries=int(monitor_raw.get("browser_restart_max_retries", 3)),
+        browser_restart_backoff_seconds=float(
+            monitor_raw.get("browser_restart_backoff_seconds", 5)
+        ),
+    )
+
+    websites: list[WebsiteConfig] = []
+    for entry in raw.get("websites", []):
+        known_keys = {"id", "name", "url", "enabled"}
+        extra = {k: v for k, v in entry.items() if k not in known_keys}
+        websites.append(
+            WebsiteConfig(
+                id=str(entry["id"]),
+                name=str(entry.get("name", entry["id"])),
+                url=str(entry["url"]),
+                enabled=bool(entry.get("enabled", True)),
+                extra=extra,
+            )
+        )
+
+    notifications = NotificationsConfig(
+        sms=_channel_config(notifications_raw.get("sms")),
+        email=_channel_config(notifications_raw.get("email")),
+    )
+
+    return AppConfig(
+        browser=browser,
+        monitor=monitor,
+        websites=websites,
+        notifications=notifications,
+    )
+
+
+def load_env_file(path: str | Path = ".env") -> None:
+    """Load simple KEY=VALUE pairs from a .env file into os.environ."""
+    env_path = Path(path)
+    if not env_path.exists():
+        return
+
+    with env_path.open("r", encoding="utf-8") as handle:
+        for line in handle:
+            line = line.strip()
+            if not line or line.startswith("#") or "=" not in line:
+                continue
+            key, value = line.split("=", 1)
+            key = key.strip()
+            value = value.strip().strip('"').strip("'")
+            if key and key not in os.environ:
+                os.environ[key] = value

monitor/models.py

@@ -0,0 +1,97 @@
+"""Data models for the browser monitoring application."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from datetime import datetime
+from typing import Any, Optional
+
+
+@dataclass
+class PageSnapshot:
+    """Structured representation of a captured page state."""
+
+    website_id: str
+    website_name: str
+    requested_url: str
+    final_url: str
+    timestamp: datetime
+    title: str
+    visible_text: str
+    html: str
+    screenshot_path: Optional[str]
+    load_duration: float
+    success: bool
+    error: Optional[str] = None
+
+
+@dataclass
+class ProcessingResult:
+    """Result from OpenAI processing (or stub)."""
+
+    success: bool
+    value: Optional[Any] = None
+    condition_met: bool = False
+    message: str = ""
+    email_subject: str = ""
+    email_body: str = ""
+    reason: str = ""
+
+
+@dataclass
+class WebsiteConfig:
+    """Configuration for a single monitored website."""
+
+    id: str
+    name: str
+    url: str
+    enabled: bool = True
+    extra: dict[str, Any] = field(default_factory=dict)
+
+
+@dataclass
+class BrowserConfig:
+    """Browser automation settings."""
+
+    type: str = "chrome"
+    executable_path: Optional[str] = None
+    headless: bool = False
+    profile_directory: str = "./profiles/default"
+
+
+@dataclass
+class MonitorConfig:
+    """Monitoring loop settings."""
+
+    round_robin_delay_seconds: float = 5.0
+    page_load_timeout_seconds: float = 30.0
+    render_wait_seconds: float = 3.0
+    snapshot_directory: str = "./snapshots"
+    snapshot_retention_count: int = 50
+    browser_restart_max_retries: int = 3
+    browser_restart_backoff_seconds: float = 5.0
+
+
+@dataclass
+class NotificationChannelConfig:
+    """Per-channel notification settings."""
+
+    enabled: bool = False
+
+
+@dataclass
+class NotificationsConfig:
+    """Notification settings."""
+
+    sms: NotificationChannelConfig = field(default_factory=NotificationChannelConfig)
+    email: NotificationChannelConfig = field(default_factory=NotificationChannelConfig)
+
+
+@dataclass
+class AppConfig:
+    """Root application configuration."""
+
+    browser: BrowserConfig = field(default_factory=BrowserConfig)
+    monitor: MonitorConfig = field(default_factory=MonitorConfig)
+    websites: list[WebsiteConfig] = field(default_factory=list)
+    notifications: NotificationsConfig = field(default_factory=NotificationsConfig)

monitor/processor.py

@@ -0,0 +1,85 @@
+"""OpenAI processing stub and condition evaluation."""
+
+from __future__ import annotations
+
+import logging
+import operator
+from typing import Any, Callable, Optional
+
+from monitor.models import PageSnapshot, ProcessingResult, WebsiteConfig
+
+logger = logging.getLogger(__name__)
+
+# Supported operators for deterministic condition evaluation (future use)
+OPERATORS: dict[str, Callable[[Any, Any], bool]] = {
+    ">": operator.gt,
+    ">=": operator.ge,
+    "<": operator.lt,
+    "<=": operator.le,
+    "==": operator.eq,
+    "!=": operator.ne,
+}
+
+
+def process_with_openai(
+    snapshot: PageSnapshot,
+    website_config: WebsiteConfig,
+) -> ProcessingResult:
+    """
+    Process a page snapshot with OpenAI.
+
+    STUB: This function is intentionally not implemented yet.
+    It will eventually send the snapshot and website instructions to OpenAI
+    for page understanding and value extraction.
+
+    Condition evaluation will remain in Python (see evaluate_condition).
+    """
+    logger.info("Processing %s (OpenAI stub — not implemented)", website_config.name)
+
+    return ProcessingResult(
+        success=False,
+        value=None,
+        condition_met=False,
+        message="OpenAI processing not yet implemented",
+        email_subject="",
+        email_body="",
+        reason="stub",
+    )
+
+
+def evaluate_condition(
+    value: Any,
+    operator_name: str,
+    threshold: Any,
+) -> bool:
+    """
+    Deterministically evaluate a condition in Python.
+
+    Example: evaluate_condition(105, ">", 100) -> True
+
+    This is separated from OpenAI so the LLM only extracts values,
+    while Python performs reliable comparisons.
+    """
+    op = OPERATORS.get(operator_name)
+    if op is None:
+        raise ValueError(f"Unsupported operator: {operator_name}")
+    return op(value, threshold)
+
+
+def handle_processing_result(
+    result: ProcessingResult,
+    notification_service: Any,
+    notifications_config: Any,
+) -> None:
+    """Route notifications when a condition is met."""
+    if not result.condition_met:
+        return
+
+    message = result.message or "Monitoring condition met."
+    if notifications_config.sms.enabled:
+        notification_service.send_sms(message)
+
+    if notifications_config.email.enabled:
+        subject = result.email_subject or "Website Monitor Alert"
+        body = result.email_body or message
+        notification_service.send_email(subject, body)

monitor/scheduler.py

@@ -0,0 +1,154 @@
+"""Round-robin monitoring scheduler."""
+
+from __future__ import annotations
+
+import logging
+import time
+from typing import Optional
+
+from monitor.browser import BrowserManager
+from monitor.models import AppConfig, WebsiteConfig
+from monitor.processor import handle_processing_result, process_with_openai
+from notifications.base import NotificationService
+from notifications.email import EmailNotificationService
+from notifications.sms import SmsNotificationService
+
+logger = logging.getLogger(__name__)
+
+
+class MonitorScheduler:
+    """Runs continuous round-robin website monitoring."""
+
+    def __init__(
+        self,
+        config: AppConfig,
+        browser_manager: BrowserManager,
+        notification_service: Optional[NotificationService] = None,
+    ) -> None:
+        self.config = config
+        self.browser_manager = browser_manager
+        self.notification_service = notification_service or _build_notification_service()
+        self._running = False
+        self._current_index = 0
+
+    @property
+    def enabled_websites(self) -> list[WebsiteConfig]:
+        return [site for site in self.config.websites if site.enabled]
+
+    def start(self) -> None:
+        """Start the round-robin monitoring loop."""
+        sites = self.enabled_websites
+        if not sites:
+            logger.error("No enabled websites configured. Exiting.")
+            return
+
+        self._running = True
+        logger.info("Starting monitor with %d website(s)", len(sites))
+
+        try:
+            self.browser_manager.start()
+        except Exception as exc:
+            logger.error("Failed to start browser: %s", exc)
+            return
+
+        while self._running:
+            website = sites[self._current_index % len(sites)]
+            self._current_index += 1
+
+            if not self._running:
+                break
+
+            self._process_website(website)
+
+            if not self._running:
+                break
+
+            delay = self.config.monitor.round_robin_delay_seconds
+            if delay > 0:
+                self._interruptible_sleep(delay)
+
+    def stop(self) -> None:
+        """Signal the scheduler to stop."""
+        logger.info("Stopping monitor...")
+        self._running = False
+
+    def _interruptible_sleep(self, seconds: float) -> None:
+        """Sleep in small increments so shutdown is responsive."""
+        end = time.monotonic() + seconds
+        while self._running and time.monotonic() < end:
+            time.sleep(min(0.5, end - time.monotonic()))
+
+    def _process_website(self, website: WebsiteConfig) -> None:
+        """Visit, snapshot, process, and notify for a single website."""
+        retries = 0
+        max_retries = self.config.monitor.browser_restart_max_retries
+        backoff = self.config.monitor.browser_restart_backoff_seconds
+
+        snapshot = None
+        while retries <= max_retries:
+            try:
+                snapshot = self.browser_manager.visit_and_snapshot(website)
+                break
+            except Exception as exc:
+                retries += 1
+                logger.error(
+                    "Browser error on %s (attempt %d/%d): %s",
+                    website.id,
+                    retries,
+                    max_retries,
+                    exc,
+                )
+                if retries > max_retries:
+                    logger.error("Max browser restart retries reached for %s", website.id)
+                    return
+                try:
+                    self.browser_manager.restart()
+                except Exception as restart_exc:
+                    logger.error("Browser restart failed: %s", restart_exc)
+                    time.sleep(backoff * retries)
+                    continue
+                time.sleep(backoff)
+
+        if snapshot is None:
+            return
+
+        try:
+            result = process_with_openai(snapshot, website)
+        except Exception as exc:
+            logger.error(
+                "OpenAI processing failed for %s: %s",
+                website.id,
+                exc,
+            )
+            return
+
+        try:
+            handle_processing_result(
+                result,
+                self.notification_service,
+                self.config.notifications,
+            )
+        except Exception as exc:
+            logger.error(
+                "Notification failed for %s: %s",
+                website.id,
+                exc,
+            )
+
+
+class CompositeNotificationService(NotificationService):
+    """Delegates to SMS and email notification services."""
+
+    def __init__(self) -> None:
+        self._sms = SmsNotificationService()
+        self._email = EmailNotificationService()
+
+    def send_sms(self, message: str) -> bool:
+        return self._sms.send_sms(message)
+
+    def send_email(self, subject: str, body: str) -> bool:
+        return self._email.send_email(subject, body)
+
+
+def _build_notification_service() -> NotificationService:
+    return CompositeNotificationService()

monitor/snapshot.py

@@ -0,0 +1,65 @@
+"""Page snapshot capture logic."""
+
+from __future__ import annotations
+
+from datetime import datetime
+
+from playwright.sync_api import Error as PlaywrightError, Page
+
+from monitor.models import PageSnapshot, WebsiteConfig
+from storage.snapshots import SnapshotStore
+
+
+def capture_page_snapshot(
+    page: Page,
+    website: WebsiteConfig,
+    load_duration: float,
+    snapshot_store: SnapshotStore,
+    timestamp: float,
+) -> PageSnapshot:
+    """Capture page metadata, HTML, text, and screenshot."""
+    captured_at = datetime.fromtimestamp(timestamp)
+    screenshot_path = None
+    title = ""
+    visible_text = ""
+    html = ""
+    error_parts: list[str] = []
+
+    try:
+        title = page.title()
+    except PlaywrightError as exc:
+        error_parts.append(f"title: {exc}")
+
+    try:
+        visible_text = page.inner_text("body")
+    except PlaywrightError as exc:
+        error_parts.append(f"visible_text: {exc}")
+
+    try:
+        html = page.content()
+    except PlaywrightError as exc:
+        error_parts.append(f"html: {exc}")
+
+    try:
+        screenshot_path = snapshot_store.save_screenshot(
+            website_id=website.id,
+            timestamp=captured_at,
+            image_bytes=page.screenshot(full_page=True),
+        )
+    except PlaywrightError as exc:
+        error_parts.append(f"screenshot: {exc}")
+
+    return PageSnapshot(
+        website_id=website.id,
+        website_name=website.name,
+        requested_url=website.url,
+        final_url=page.url,
+        timestamp=captured_at,
+        title=title,
+        visible_text=visible_text,
+        html=html,
+        screenshot_path=screenshot_path,
+        load_duration=load_duration,
+        success=len(error_parts) == 0,
+        error="; ".join(error_parts) if error_parts else None,
+    )