From dc1245f164fb65de6c24518b9295b0380e348e60 Mon Sep 17 00:00:00 2001 From: debu-2 Date: Wed, 30 Sep 2026 15:48:03 +0530 Subject: [PATCH] first commit --- .env.example | 17 +++ .gitignore | 99 ++++++++++++ README.md | 258 +++++++++++++++++++++++++++++++ config.yaml | 37 +++++ login.py | 130 ++++++++++++++++ main.py | 104 +++++++++++++ monitor/__init__.py | 3 + monitor/browser.py | 310 ++++++++++++++++++++++++++++++++++++++ monitor/config.py | 101 +++++++++++++ monitor/models.py | 97 ++++++++++++ monitor/processor.py | 85 +++++++++++ monitor/scheduler.py | 154 +++++++++++++++++++ monitor/snapshot.py | 65 ++++++++ notifications/__init__.py | 13 ++ notifications/base.py | 32 ++++ notifications/email.py | 57 +++++++ notifications/sms.py | 41 +++++ profiles/.gitkeep | 0 requirements.txt | 2 + snapshots/.gitkeep | 0 storage/__init__.py | 5 + storage/snapshots.py | 56 +++++++ 22 files changed, 1666 insertions(+) create mode 100644 .env.example create mode 100644 .gitignore create mode 100644 README.md create mode 100644 config.yaml create mode 100755 login.py create mode 100644 main.py create mode 100644 monitor/__init__.py create mode 100644 monitor/browser.py create mode 100644 monitor/config.py create mode 100644 monitor/models.py create mode 100644 monitor/processor.py create mode 100644 monitor/scheduler.py create mode 100644 monitor/snapshot.py create mode 100644 notifications/__init__.py create mode 100644 notifications/base.py create mode 100644 notifications/email.py create mode 100644 notifications/sms.py create mode 100644 profiles/.gitkeep create mode 100644 requirements.txt create mode 100644 snapshots/.gitkeep create mode 100644 storage/__init__.py create mode 100644 storage/snapshots.py diff --git a/.env.example b/.env.example new file mode 100644 index 0000000..4da295b --- /dev/null +++ b/.env.example @@ -0,0 +1,17 @@ +# SMS provider configuration (never commit real credentials) +SMS_PROVIDER=logging +SMS_API_KEY= +SMS_FROM_NUMBER= +SMS_TO_NUMBER= + +# SMTP email configuration +SMTP_HOST= +SMTP_PORT=587 +SMTP_USER= +SMTP_PASSWORD= +SMTP_USE_TLS=true +EMAIL_FROM= +EMAIL_TO= + +# Future OpenAI configuration +OPENAI_API_KEY= diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..5534b23 --- /dev/null +++ b/.gitignore @@ -0,0 +1,99 @@ +# Environment and secrets +.env +.env.* +!.env.example + +# Virtual environments +.venv/ +venv/ +env/ +ENV/ +.envrc + +# Python bytecode and packaging +__pycache__/ +*.py[cod] +*$py.class +*.so +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST +pip-log.txt +pip-delete-this-directory.txt + +# Testing and type checking +.pytest_cache/ +.mypy_cache/ +.ruff_cache/ +.coverage +.coverage.* +htmlcov/ +.tox/ +.nox/ +coverage.xml +*.cover +*.py,cover +.hypothesis/ + +# Runtime data (browser profiles and captured snapshots) +profiles/* +!profiles/.gitkeep +snapshots/* +!snapshots/.gitkeep + +# Logs and local databases +*.log +logs/ +*.sqlite +*.sqlite3 +*.db + +# Temporary files +tmp/ +temp/ +*.tmp +*.temp +*.swp +*.swo +*~ +.cache/ + +# Playwright +playwright-report/ +test-results/ +playwright/.cache/ + +# IDE and editor +.idea/ +.vscode/ +*.sublime-project +*.sublime-workspace +*.code-workspace +.project +.pydevproject +.settings/ + +# OS files +.DS_Store +.DS_Store? +._* +.Spotlight-V100 +.Trashes +ehthumbs.db +Thumbs.db +Desktop.ini diff --git a/README.md b/README.md new file mode 100644 index 0000000..1f6e50a --- /dev/null +++ b/README.md @@ -0,0 +1,258 @@ +# Browser Monitoring & Notification Agent + +A Python application that continuously monitors websites using a **real browser** installed on your machine. It visits configured sites in round-robin order, captures snapshots (screenshots + metadata), and is designed for future OpenAI-based page understanding and alert notifications. + +## Features + +- **Browser-first monitoring** — Uses Playwright with your installed Chrome, Chromium, Edge, or Firefox +- **Persistent browser profile** — Reuses cookies, sessions, and login state across runs +- **Round-robin scheduling** — Visits one website at a time, sequentially +- **Snapshots** — Screenshots, page title, visible text, HTML, and metadata +- **Snapshot retention** — Configurable cleanup to prevent unbounded disk growth +- **OpenAI stub** — Ready for future page understanding integration +- **Notification framework** — SMS and email abstractions (logging by default) +- **Error resilience** — One site failing does not stop the monitor +- **Browser recovery** — Automatic restart on browser crashes +- **Graceful shutdown** — Clean exit on Ctrl+C + +## Requirements + +- Python 3.10+ +- A supported browser installed locally (Chrome recommended) + +## Installation + +```bash +cd browser_automation +python -m venv .venv +source .venv/bin/activate +pip install -r requirements.txt +``` + +Playwright is used with your **installed browser**, not a separately downloaded Playwright browser bundle. If auto-detection fails, set `browser.executable_path` in `config.yaml`. + +Copy environment variable template: + +```bash +cp .env.example .env +``` + +Never commit `.env` — it is listed in `.gitignore`. + +## Browser Configuration + +Edit `config.yaml`: + +```yaml +browser: + type: chrome # chrome | chromium | edge | firefox + executable_path: null # set if auto-detection fails + headless: false # false = visible browser window + profile_directory: "./profiles/default" +``` + +### Persistent Profile / Login Sessions + +1. Set `profile_directory` to a dedicated folder (e.g. `./profiles/default`). +2. Run the **login launcher** to open that profile and sign in manually. +3. Close the browser or press Ctrl+C when finished. +4. Start the monitor — sessions and cookies are preserved. + +Do **not** store passwords in `config.yaml`. Use the browser profile for authentication. + +### Login launcher + +Use `login.py` to open the monitor profile without starting the monitoring loop: + +```bash +# Open profile with a blank tab +python login.py + +# Open BookMyShow (or any URL) +python login.py --url "https://in.bookmyshow.com" + +# Open all enabled websites from config.yaml as tabs +python login.py --websites + +# Open a specific configured site by id +python login.py --website-id website_1 +``` + +The launcher always opens a **visible** browser window, even if `headless: true` in config. + +### Auto-Detection Paths (Linux) + +| Browser | Typical paths | +|----------|---------------| +| Chrome | `/usr/bin/google-chrome`, `/usr/bin/google-chrome-stable` | +| Chromium | `/usr/bin/chromium`, `/usr/bin/chromium-browser` | +| Edge | `/usr/bin/microsoft-edge` | +| Firefox | `/usr/bin/firefox` | + +## Configuration + +`config.yaml` controls monitoring behaviour: + +```yaml +monitor: + round_robin_delay_seconds: 5 # pause between website visits + page_load_timeout_seconds: 30 + render_wait_seconds: 3 # extra wait for JavaScript rendering + snapshot_directory: "./snapshots" + snapshot_retention_count: 50 # max screenshots per website + +websites: + - id: website_1 + name: Website One + url: "https://example.com" + enabled: true + +notifications: + sms: + enabled: false + email: + enabled: false +``` + +Add as many websites as needed. Disabled sites are skipped. + +## Usage + +### Start monitoring + +```bash +python main.py +``` + +### Custom config file + +```bash +python main.py --config config.yaml +``` + +### Debug logging + +```bash +python main.py --debug +``` + +### Test browser setup + +```bash +python main.py --test-browser +python main.py --test-browser --test-url "https://example.com" +``` + +### Graceful shutdown + +Press `Ctrl+C`. The monitor stops scheduling new checks, closes the browser, and exits cleanly. + +## Architecture + +``` +Browser → Snapshot → OpenAI Processor (stub) → Condition Engine → Notifications +``` + +| Layer | Responsibility | +|-------|----------------| +| OpenAI (future) | Page understanding, value extraction | +| Python | Scheduling, condition evaluation, notifications | +| Playwright | Real browser automation | + +### Project Structure + +``` +browser_automation/ +├── main.py +├── config.yaml +├── monitor/ +│ ├── browser.py # Browser discovery, launch, navigation +│ ├── scheduler.py # Round-robin loop +│ ├── snapshot.py # Page capture +│ ├── processor.py # OpenAI stub + condition evaluation +│ ├── models.py # PageSnapshot, ProcessingResult, etc. +│ └── config.py # YAML loading +├── notifications/ +│ ├── base.py # NotificationService abstraction +│ ├── sms.py # SMS provider stub +│ └── email.py # SMTP / logging email +├── storage/ +│ └── snapshots.py # Screenshot storage + retention +├── profiles/ # Persistent browser profiles +└── snapshots/ # Captured screenshots +``` + +## Snapshots + +Screenshots are saved per website: + +``` +snapshots/ + website_1/ + 2026-09-30T13-20-00.png + website_2/ + 2026-09-30T13-20-05.png +``` + +Older snapshots are removed automatically when `snapshot_retention_count` is exceeded. + +## Notifications + +By default, notifications are **logged** to the console: + +``` +SMS WOULD BE SENT: +... + +EMAIL WOULD BE SENT: +Subject: ... +``` + +### Email (SMTP) + +Set environment variables in `.env`: + +``` +SMTP_HOST=smtp.example.com +SMTP_PORT=587 +SMTP_USER=your-user +SMTP_PASSWORD=your-password +EMAIL_FROM=monitor@example.com +EMAIL_TO=you@example.com +``` + +Enable in `config.yaml`: + +```yaml +notifications: + email: + enabled: true +``` + +### SMS + +Configure provider credentials via environment variables (`SMS_PROVIDER`, `SMS_API_KEY`, etc.). Enable in config when a provider is implemented. + +## Error Handling + +- Navigation timeouts, DNS failures, and page errors are logged per website +- The monitor continues to the next site +- Browser crashes trigger automatic restart with backoff +- Bot/automation blocks are logged and skipped — no CAPTCHA or bypass logic + +## Security + +- No credentials in source code or `config.yaml` +- Use `.env` for secrets (not committed) +- No CAPTCHA bypass, Cloudflare bypass, or fingerprint spoofing + +## Future Extensions + +The architecture supports adding: + +- OpenAI page understanding and condition triggers +- Per-site instructions and check intervals +- Multiple browser profiles +- Alert deduplication and cooldown +- Urmi, Twilio, WhatsApp integrations +- Dashboard and database storage diff --git a/config.yaml b/config.yaml new file mode 100644 index 0000000..dbf8649 --- /dev/null +++ b/config.yaml @@ -0,0 +1,37 @@ +browser: + type: chrome + executable_path: null + headless: false + profile_directory: "./profiles/default" + +monitor: + round_robin_delay_seconds: 5 + page_load_timeout_seconds: 30 + render_wait_seconds: 3 + snapshot_directory: "./snapshots" + snapshot_retention_count: 50 + browser_restart_max_retries: 3 + browser_restart_backoff_seconds: 5 + +websites: + - id: website_1 + name: Website One + url: "https://in.bookmyshow.com/my-profile/purchase-history" + enabled: true + + - id: website_2 + name: Website Two + url: "https://example.org" + enabled: true + + - id: website_3 + name: Website Three + url: "https://example.net" + enabled: true + +notifications: + sms: + enabled: false + + email: + enabled: false diff --git a/login.py b/login.py new file mode 100755 index 0000000..3e81b8e --- /dev/null +++ b/login.py @@ -0,0 +1,130 @@ +#!/usr/bin/env python3 +"""Open the monitor browser profile for manual login.""" + +from __future__ import annotations + +import argparse +import logging +import sys + +from monitor.browser import BrowserManager +from monitor.config import load_config, load_env_file +from storage.snapshots import SnapshotStore + + +def setup_logging(debug: bool = False) -> None: + level = logging.DEBUG if debug else logging.INFO + logging.basicConfig( + level=level, + format="%(asctime)s %(levelname)-5s %(message)s", + datefmt="%H:%M:%S", + ) + logging.getLogger("playwright").setLevel(logging.WARNING) + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser( + description=( + "Open the monitor's persistent browser profile so you can log in " + "to websites manually. Sessions are saved for the main monitor." + ), + ) + parser.add_argument( + "--config", + default="config.yaml", + help="Path to configuration file (default: config.yaml)", + ) + parser.add_argument( + "--url", + action="append", + dest="urls", + metavar="URL", + help="URL to open (can be repeated for multiple tabs)", + ) + parser.add_argument( + "--websites", + action="store_true", + help="Open all enabled websites from config.yaml", + ) + parser.add_argument( + "--website-id", + action="append", + dest="website_ids", + metavar="ID", + help="Open a specific configured website by id (can be repeated)", + ) + parser.add_argument( + "--debug", + action="store_true", + help="Enable debug logging", + ) + return parser.parse_args() + + +def resolve_urls(args: argparse.Namespace, config) -> list[str]: + """Build the list of URLs to open from CLI flags.""" + urls: list[str] = [] + + if args.urls: + urls.extend(args.urls) + + if args.websites: + urls.extend(site.url for site in config.websites if site.enabled) + + if args.website_ids: + site_map = {site.id: site for site in config.websites} + for site_id in args.website_ids: + site = site_map.get(site_id) + if site is None: + raise ValueError(f"Unknown website id: {site_id}") + if not site.enabled: + logging.getLogger(__name__).warning( + "Website '%s' is disabled in config but opening anyway", + site_id, + ) + urls.append(site.url) + + # Preserve order, remove duplicates + seen: set[str] = set() + unique_urls: list[str] = [] + for url in urls: + if url not in seen: + seen.add(url) + unique_urls.append(url) + + return unique_urls + + +def main() -> int: + args = parse_args() + setup_logging(debug=args.debug) + load_env_file() + + logger = logging.getLogger(__name__) + + try: + config = load_config(args.config) + except FileNotFoundError as exc: + logger.error("%s", exc) + return 1 + + try: + urls = resolve_urls(args, config) + except ValueError as exc: + logger.error("%s", exc) + return 1 + + snapshot_store = SnapshotStore( + base_directory=config.monitor.snapshot_directory, + retention_count=config.monitor.snapshot_retention_count, + ) + browser_manager = BrowserManager(config=config, snapshot_store=snapshot_store) + + logger.info("Starting login launcher") + browser_manager.open_for_login(urls=urls) + logger.info("Login launcher finished — profile saved") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/main.py b/main.py new file mode 100644 index 0000000..e2fe01c --- /dev/null +++ b/main.py @@ -0,0 +1,104 @@ +#!/usr/bin/env python3 +"""Browser Monitoring & Notification Agent — CLI entry point.""" + +from __future__ import annotations + +import argparse +import logging +import signal +import sys +from pathlib import Path + +from monitor.browser import BrowserManager +from monitor.config import load_config, load_env_file +from monitor.scheduler import MonitorScheduler +from storage.snapshots import SnapshotStore + + +def setup_logging(debug: bool = False) -> None: + """Configure standard logging.""" + level = logging.DEBUG if debug else logging.INFO + logging.basicConfig( + level=level, + format="%(asctime)s %(levelname)-5s %(message)s", + datefmt="%H:%M:%S", + ) + logging.getLogger("playwright").setLevel(logging.WARNING) + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser( + description="Browser Monitoring & Notification Agent", + ) + parser.add_argument( + "--config", + default="config.yaml", + help="Path to configuration file (default: config.yaml)", + ) + parser.add_argument( + "--debug", + action="store_true", + help="Enable debug logging", + ) + parser.add_argument( + "--test-browser", + action="store_true", + help="Open browser and visit a test URL, then exit", + ) + parser.add_argument( + "--test-url", + default="https://example.com", + help="URL for --test-browser (default: https://example.com)", + ) + return parser.parse_args() + + +def main() -> int: + args = parse_args() + setup_logging(debug=args.debug) + load_env_file() + + logger = logging.getLogger(__name__) + + try: + config = load_config(args.config) + except FileNotFoundError as exc: + logger.error("%s", exc) + return 1 + + snapshot_store = SnapshotStore( + base_directory=config.monitor.snapshot_directory, + retention_count=config.monitor.snapshot_retention_count, + ) + browser_manager = BrowserManager(config=config, snapshot_store=snapshot_store) + + if args.test_browser: + success = browser_manager.test_browser(test_url=args.test_url) + return 0 if success else 1 + + scheduler = MonitorScheduler( + config=config, + browser_manager=browser_manager, + ) + + def shutdown_handler(signum: int, frame: object) -> None: + logger.info("Shutdown signal received (%s)", signum) + scheduler.stop() + + signal.signal(signal.SIGINT, shutdown_handler) + signal.signal(signal.SIGTERM, shutdown_handler) + + try: + scheduler.start() + except KeyboardInterrupt: + logger.info("Interrupted by user") + finally: + scheduler.stop() + browser_manager.stop() + logging.shutdown() + + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/monitor/__init__.py b/monitor/__init__.py new file mode 100644 index 0000000..87c7c54 --- /dev/null +++ b/monitor/__init__.py @@ -0,0 +1,3 @@ +"""Browser monitoring package.""" + +__version__ = "0.1.0" diff --git a/monitor/browser.py b/monitor/browser.py new file mode 100644 index 0000000..29721da --- /dev/null +++ b/monitor/browser.py @@ -0,0 +1,310 @@ +"""Browser discovery, launch, and page capture using Playwright.""" + +from __future__ import annotations + +import logging +import shutil +import time +from datetime import datetime +from pathlib import Path +from typing import Optional + +from playwright.sync_api import ( + BrowserContext, + Error as PlaywrightError, + Page, + Playwright, + sync_playwright, +) + +from monitor.models import AppConfig, PageSnapshot, WebsiteConfig +from monitor.snapshot import capture_page_snapshot +from storage.snapshots import SnapshotStore + +logger = logging.getLogger(__name__) + +FATAL_BROWSER_ERRORS = ("TargetClosedError", "BrowserClosedError", "TargetCrashedError") + + +def _is_fatal_browser_error(exc: PlaywrightError) -> bool: + """Return True if the error indicates the browser or page is no longer usable.""" + return type(exc).__name__ in FATAL_BROWSER_ERRORS + +# Common browser executable locations on Linux +BROWSER_CANDIDATES: dict[str, list[str]] = { + "chrome": [ + "/usr/bin/google-chrome", + "/usr/bin/google-chrome-stable", + "/snap/bin/chromium", # some systems + ], + "chromium": [ + "/usr/bin/chromium", + "/usr/bin/chromium-browser", + "/snap/bin/chromium", + ], + "edge": [ + "/usr/bin/microsoft-edge", + "/usr/bin/microsoft-edge-stable", + ], + "firefox": [ + "/usr/bin/firefox", + "/snap/bin/firefox", + ], +} + +# Playwright channel names for installed browsers +PLAYWRIGHT_CHANNELS: dict[str, str] = { + "chrome": "chrome", + "edge": "msedge", + "firefox": "firefox", +} + + +def discover_browser_executable(browser_type: str) -> Optional[str]: + """Find an installed browser executable for the given type.""" + candidates = BROWSER_CANDIDATES.get(browser_type.lower(), []) + for candidate in candidates: + path = Path(candidate) + if path.is_file() and shutil.which(str(path)): + return str(path.resolve()) + return None + + +def resolve_browser_launch_options(config: AppConfig) -> dict: + """Build Playwright launch options from configuration.""" + browser_type = config.browser.type.lower() + options: dict = { + "headless": config.browser.headless, + "args": ["--disable-blink-features=AutomationControlled"], + } + + if config.browser.executable_path: + options["executable_path"] = config.browser.executable_path + logger.info("Using configured browser executable: %s", config.browser.executable_path) + else: + discovered = discover_browser_executable(browser_type) + if discovered: + options["executable_path"] = discovered + logger.info("Browser detected: %s at %s", browser_type.title(), discovered) + elif browser_type in PLAYWRIGHT_CHANNELS: + options["channel"] = PLAYWRIGHT_CHANNELS[browser_type] + logger.info("Using Playwright channel: %s", options["channel"]) + else: + logger.warning( + "Could not auto-detect %s; Playwright will use bundled Chromium", + browser_type, + ) + + return options + + +class BrowserManager: + """Manages a persistent browser context with recovery support.""" + + def __init__(self, config: AppConfig, snapshot_store: SnapshotStore) -> None: + self.config = config + self.snapshot_store = snapshot_store + self._playwright: Optional[Playwright] = None + self._context: Optional[BrowserContext] = None + self._page: Optional[Page] = None + + @property + def is_running(self) -> bool: + return self._context is not None + + def start(self) -> None: + """Launch the browser with a persistent profile.""" + self._playwright = sync_playwright().start() + profile_dir = Path(self.config.browser.profile_directory) + profile_dir.mkdir(parents=True, exist_ok=True) + + launch_options = resolve_browser_launch_options(self.config) + browser_type = self.config.browser.type.lower() + + try: + if browser_type == "firefox": + self._context = self._playwright.firefox.launch_persistent_context( + user_data_dir=str(profile_dir.resolve()), + headless=self.config.browser.headless, + executable_path=launch_options.get("executable_path"), + ) + else: + chromium_options = { + "user_data_dir": str(profile_dir.resolve()), + "headless": self.config.browser.headless, + "args": launch_options.get("args", []), + } + if "executable_path" in launch_options: + chromium_options["executable_path"] = launch_options["executable_path"] + if "channel" in launch_options: + chromium_options["channel"] = launch_options["channel"] + + self._context = self._playwright.chromium.launch_persistent_context( + **chromium_options + ) + + self._page = self._context.pages[0] if self._context.pages else self._context.new_page() + logger.info("Browser started with profile: %s", profile_dir) + except PlaywrightError as exc: + logger.error("Failed to start browser: %s", exc) + self.stop() + raise + + def stop(self) -> None: + """Close browser and Playwright resources.""" + if self._context: + try: + self._context.close() + except PlaywrightError as exc: + logger.warning("Error closing browser context: %s", exc) + self._context = None + self._page = None + + if self._playwright: + try: + self._playwright.stop() + except PlaywrightError as exc: + logger.warning("Error stopping Playwright: %s", exc) + self._playwright = None + + logger.info("Browser stopped") + + def restart(self) -> None: + """Restart the browser after a crash or failure.""" + logger.warning("Restarting browser...") + self.stop() + self.start() + + def ensure_running(self) -> None: + """Ensure browser is running, restarting if needed.""" + if not self.is_running: + self.start() + + def visit_and_snapshot(self, website: WebsiteConfig) -> PageSnapshot: + """Navigate to a website and capture a page snapshot.""" + self.ensure_running() + assert self._page is not None + + start_time = time.monotonic() + timestamp = time.time() + + try: + timeout_ms = int(self.config.monitor.page_load_timeout_seconds * 1000) + render_wait_ms = int(self.config.monitor.render_wait_seconds * 1000) + + logger.info("Opening %s (%s)", website.name, website.url) + self._page.goto( + website.url, + wait_until="domcontentloaded", + timeout=timeout_ms, + ) + try: + self._page.wait_for_load_state("networkidle", timeout=timeout_ms) + except PlaywrightError: + logger.debug( + "networkidle timeout for %s; continuing after render wait", + website.id, + ) + self._page.wait_for_timeout(render_wait_ms) + + load_duration = time.monotonic() - start_time + snapshot = capture_page_snapshot( + page=self._page, + website=website, + load_duration=load_duration, + snapshot_store=self.snapshot_store, + timestamp=timestamp, + ) + logger.info("Snapshot captured: %s", website.name) + return snapshot + + except PlaywrightError as exc: + load_duration = time.monotonic() - start_time + error_type = type(exc).__name__ + + if _is_fatal_browser_error(exc): + logger.error( + "Browser crash detected on %s [%s]: %s", + website.id, + error_type, + exc, + ) + self._context = None + self._page = None + raise + + logger.error( + "Website %s navigation failed [%s]: %s", + website.id, + error_type, + exc, + ) + return PageSnapshot( + website_id=website.id, + website_name=website.name, + requested_url=website.url, + final_url=self._page.url if self._page else website.url, + timestamp=datetime.fromtimestamp(timestamp), + title="", + visible_text="", + html="", + screenshot_path=None, + load_duration=load_duration, + success=False, + error=f"{error_type}: {exc}", + ) + + def open_for_login(self, urls: list[str]) -> None: + """ + Open the monitor profile for manual login. + + Keeps the browser open until the user closes it or presses Ctrl+C. + Always runs with a visible window regardless of headless config. + """ + original_headless = self.config.browser.headless + self.config.browser.headless = False + + try: + self.start() + assert self._context is not None + + if urls: + for index, url in enumerate(urls): + page = self._page if index == 0 else self._context.new_page() + logger.info("Opening tab %d: %s", index + 1, url) + page.goto(url, wait_until="domcontentloaded", timeout=60000) + else: + logger.info("No URLs provided — browser opened with default tab") + + profile_dir = Path(self.config.browser.profile_directory).resolve() + logger.info("Profile directory: %s", profile_dir) + logger.info( + "Log in to your websites in the browser window. " + "Close the browser or press Ctrl+C when finished." + ) + + while self._context and self._context.pages: + time.sleep(0.5) + + logger.info("Browser window closed") + except KeyboardInterrupt: + logger.info("Login session ended by user") + finally: + self.config.browser.headless = original_headless + self.stop() + + def test_browser(self, test_url: str = "https://example.com") -> bool: + """Open browser and visit a test URL to verify configuration.""" + try: + self.start() + assert self._page is not None + logger.info("Test browser: navigating to %s", test_url) + self._page.goto(test_url, wait_until="domcontentloaded", timeout=30000) + title = self._page.title() + logger.info("Test browser success. Page title: %s", title) + return True + except PlaywrightError as exc: + logger.error("Test browser failed: %s", exc) + return False + finally: + self.stop() diff --git a/monitor/config.py b/monitor/config.py new file mode 100644 index 0000000..6072270 --- /dev/null +++ b/monitor/config.py @@ -0,0 +1,101 @@ +"""Configuration loading and validation.""" + +from __future__ import annotations + +import os +from pathlib import Path +from typing import Any + +import yaml + +from monitor.models import ( + AppConfig, + BrowserConfig, + MonitorConfig, + NotificationChannelConfig, + NotificationsConfig, + WebsiteConfig, +) + + +def _channel_config(data: dict[str, Any] | None) -> NotificationChannelConfig: + if not data: + return NotificationChannelConfig() + return NotificationChannelConfig(enabled=bool(data.get("enabled", False))) + + +def load_config(path: str | Path) -> AppConfig: + """Load application configuration from a YAML file.""" + config_path = Path(path) + if not config_path.exists(): + raise FileNotFoundError(f"Configuration file not found: {config_path}") + + with config_path.open("r", encoding="utf-8") as handle: + raw: dict[str, Any] = yaml.safe_load(handle) or {} + + browser_raw = raw.get("browser", {}) + monitor_raw = raw.get("monitor", {}) + notifications_raw = raw.get("notifications", {}) + + browser = BrowserConfig( + type=str(browser_raw.get("type", "chrome")).lower(), + executable_path=browser_raw.get("executable_path"), + headless=bool(browser_raw.get("headless", False)), + profile_directory=str(browser_raw.get("profile_directory", "./profiles/default")), + ) + + monitor = MonitorConfig( + round_robin_delay_seconds=float(monitor_raw.get("round_robin_delay_seconds", 5)), + page_load_timeout_seconds=float(monitor_raw.get("page_load_timeout_seconds", 30)), + render_wait_seconds=float(monitor_raw.get("render_wait_seconds", 3)), + snapshot_directory=str(monitor_raw.get("snapshot_directory", "./snapshots")), + snapshot_retention_count=int(monitor_raw.get("snapshot_retention_count", 50)), + browser_restart_max_retries=int(monitor_raw.get("browser_restart_max_retries", 3)), + browser_restart_backoff_seconds=float( + monitor_raw.get("browser_restart_backoff_seconds", 5) + ), + ) + + websites: list[WebsiteConfig] = [] + for entry in raw.get("websites", []): + known_keys = {"id", "name", "url", "enabled"} + extra = {k: v for k, v in entry.items() if k not in known_keys} + websites.append( + WebsiteConfig( + id=str(entry["id"]), + name=str(entry.get("name", entry["id"])), + url=str(entry["url"]), + enabled=bool(entry.get("enabled", True)), + extra=extra, + ) + ) + + notifications = NotificationsConfig( + sms=_channel_config(notifications_raw.get("sms")), + email=_channel_config(notifications_raw.get("email")), + ) + + return AppConfig( + browser=browser, + monitor=monitor, + websites=websites, + notifications=notifications, + ) + + +def load_env_file(path: str | Path = ".env") -> None: + """Load simple KEY=VALUE pairs from a .env file into os.environ.""" + env_path = Path(path) + if not env_path.exists(): + return + + with env_path.open("r", encoding="utf-8") as handle: + for line in handle: + line = line.strip() + if not line or line.startswith("#") or "=" not in line: + continue + key, value = line.split("=", 1) + key = key.strip() + value = value.strip().strip('"').strip("'") + if key and key not in os.environ: + os.environ[key] = value diff --git a/monitor/models.py b/monitor/models.py new file mode 100644 index 0000000..09d0a4d --- /dev/null +++ b/monitor/models.py @@ -0,0 +1,97 @@ +"""Data models for the browser monitoring application.""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from datetime import datetime +from typing import Any, Optional + + +@dataclass +class PageSnapshot: + """Structured representation of a captured page state.""" + + website_id: str + website_name: str + requested_url: str + final_url: str + timestamp: datetime + title: str + visible_text: str + html: str + screenshot_path: Optional[str] + load_duration: float + success: bool + error: Optional[str] = None + + +@dataclass +class ProcessingResult: + """Result from OpenAI processing (or stub).""" + + success: bool + value: Optional[Any] = None + condition_met: bool = False + message: str = "" + email_subject: str = "" + email_body: str = "" + reason: str = "" + + +@dataclass +class WebsiteConfig: + """Configuration for a single monitored website.""" + + id: str + name: str + url: str + enabled: bool = True + extra: dict[str, Any] = field(default_factory=dict) + + +@dataclass +class BrowserConfig: + """Browser automation settings.""" + + type: str = "chrome" + executable_path: Optional[str] = None + headless: bool = False + profile_directory: str = "./profiles/default" + + +@dataclass +class MonitorConfig: + """Monitoring loop settings.""" + + round_robin_delay_seconds: float = 5.0 + page_load_timeout_seconds: float = 30.0 + render_wait_seconds: float = 3.0 + snapshot_directory: str = "./snapshots" + snapshot_retention_count: int = 50 + browser_restart_max_retries: int = 3 + browser_restart_backoff_seconds: float = 5.0 + + +@dataclass +class NotificationChannelConfig: + """Per-channel notification settings.""" + + enabled: bool = False + + +@dataclass +class NotificationsConfig: + """Notification settings.""" + + sms: NotificationChannelConfig = field(default_factory=NotificationChannelConfig) + email: NotificationChannelConfig = field(default_factory=NotificationChannelConfig) + + +@dataclass +class AppConfig: + """Root application configuration.""" + + browser: BrowserConfig = field(default_factory=BrowserConfig) + monitor: MonitorConfig = field(default_factory=MonitorConfig) + websites: list[WebsiteConfig] = field(default_factory=list) + notifications: NotificationsConfig = field(default_factory=NotificationsConfig) diff --git a/monitor/processor.py b/monitor/processor.py new file mode 100644 index 0000000..036e253 --- /dev/null +++ b/monitor/processor.py @@ -0,0 +1,85 @@ +"""OpenAI processing stub and condition evaluation.""" + +from __future__ import annotations + +import logging +import operator +from typing import Any, Callable, Optional + +from monitor.models import PageSnapshot, ProcessingResult, WebsiteConfig + +logger = logging.getLogger(__name__) + +# Supported operators for deterministic condition evaluation (future use) +OPERATORS: dict[str, Callable[[Any, Any], bool]] = { + ">": operator.gt, + ">=": operator.ge, + "<": operator.lt, + "<=": operator.le, + "==": operator.eq, + "!=": operator.ne, +} + + +def process_with_openai( + snapshot: PageSnapshot, + website_config: WebsiteConfig, +) -> ProcessingResult: + """ + Process a page snapshot with OpenAI. + + STUB: This function is intentionally not implemented yet. + It will eventually send the snapshot and website instructions to OpenAI + for page understanding and value extraction. + + Condition evaluation will remain in Python (see evaluate_condition). + """ + logger.info("Processing %s (OpenAI stub — not implemented)", website_config.name) + + return ProcessingResult( + success=False, + value=None, + condition_met=False, + message="OpenAI processing not yet implemented", + email_subject="", + email_body="", + reason="stub", + ) + + +def evaluate_condition( + value: Any, + operator_name: str, + threshold: Any, +) -> bool: + """ + Deterministically evaluate a condition in Python. + + Example: evaluate_condition(105, ">", 100) -> True + + This is separated from OpenAI so the LLM only extracts values, + while Python performs reliable comparisons. + """ + op = OPERATORS.get(operator_name) + if op is None: + raise ValueError(f"Unsupported operator: {operator_name}") + return op(value, threshold) + + +def handle_processing_result( + result: ProcessingResult, + notification_service: Any, + notifications_config: Any, +) -> None: + """Route notifications when a condition is met.""" + if not result.condition_met: + return + + message = result.message or "Monitoring condition met." + if notifications_config.sms.enabled: + notification_service.send_sms(message) + + if notifications_config.email.enabled: + subject = result.email_subject or "Website Monitor Alert" + body = result.email_body or message + notification_service.send_email(subject, body) diff --git a/monitor/scheduler.py b/monitor/scheduler.py new file mode 100644 index 0000000..73b8757 --- /dev/null +++ b/monitor/scheduler.py @@ -0,0 +1,154 @@ +"""Round-robin monitoring scheduler.""" + +from __future__ import annotations + +import logging +import time +from typing import Optional + +from monitor.browser import BrowserManager +from monitor.models import AppConfig, WebsiteConfig +from monitor.processor import handle_processing_result, process_with_openai +from notifications.base import NotificationService +from notifications.email import EmailNotificationService +from notifications.sms import SmsNotificationService + +logger = logging.getLogger(__name__) + + +class MonitorScheduler: + """Runs continuous round-robin website monitoring.""" + + def __init__( + self, + config: AppConfig, + browser_manager: BrowserManager, + notification_service: Optional[NotificationService] = None, + ) -> None: + self.config = config + self.browser_manager = browser_manager + self.notification_service = notification_service or _build_notification_service() + self._running = False + self._current_index = 0 + + @property + def enabled_websites(self) -> list[WebsiteConfig]: + return [site for site in self.config.websites if site.enabled] + + def start(self) -> None: + """Start the round-robin monitoring loop.""" + sites = self.enabled_websites + if not sites: + logger.error("No enabled websites configured. Exiting.") + return + + self._running = True + logger.info("Starting monitor with %d website(s)", len(sites)) + + try: + self.browser_manager.start() + except Exception as exc: + logger.error("Failed to start browser: %s", exc) + return + + while self._running: + website = sites[self._current_index % len(sites)] + self._current_index += 1 + + if not self._running: + break + + self._process_website(website) + + if not self._running: + break + + delay = self.config.monitor.round_robin_delay_seconds + if delay > 0: + self._interruptible_sleep(delay) + + def stop(self) -> None: + """Signal the scheduler to stop.""" + logger.info("Stopping monitor...") + self._running = False + + def _interruptible_sleep(self, seconds: float) -> None: + """Sleep in small increments so shutdown is responsive.""" + end = time.monotonic() + seconds + while self._running and time.monotonic() < end: + time.sleep(min(0.5, end - time.monotonic())) + + def _process_website(self, website: WebsiteConfig) -> None: + """Visit, snapshot, process, and notify for a single website.""" + retries = 0 + max_retries = self.config.monitor.browser_restart_max_retries + backoff = self.config.monitor.browser_restart_backoff_seconds + + snapshot = None + while retries <= max_retries: + try: + snapshot = self.browser_manager.visit_and_snapshot(website) + break + except Exception as exc: + retries += 1 + logger.error( + "Browser error on %s (attempt %d/%d): %s", + website.id, + retries, + max_retries, + exc, + ) + if retries > max_retries: + logger.error("Max browser restart retries reached for %s", website.id) + return + try: + self.browser_manager.restart() + except Exception as restart_exc: + logger.error("Browser restart failed: %s", restart_exc) + time.sleep(backoff * retries) + continue + time.sleep(backoff) + + if snapshot is None: + return + + try: + result = process_with_openai(snapshot, website) + except Exception as exc: + logger.error( + "OpenAI processing failed for %s: %s", + website.id, + exc, + ) + return + + try: + handle_processing_result( + result, + self.notification_service, + self.config.notifications, + ) + except Exception as exc: + logger.error( + "Notification failed for %s: %s", + website.id, + exc, + ) + + +class CompositeNotificationService(NotificationService): + """Delegates to SMS and email notification services.""" + + def __init__(self) -> None: + self._sms = SmsNotificationService() + self._email = EmailNotificationService() + + def send_sms(self, message: str) -> bool: + return self._sms.send_sms(message) + + def send_email(self, subject: str, body: str) -> bool: + return self._email.send_email(subject, body) + + +def _build_notification_service() -> NotificationService: + return CompositeNotificationService() diff --git a/monitor/snapshot.py b/monitor/snapshot.py new file mode 100644 index 0000000..b7bd135 --- /dev/null +++ b/monitor/snapshot.py @@ -0,0 +1,65 @@ +"""Page snapshot capture logic.""" + +from __future__ import annotations + +from datetime import datetime + +from playwright.sync_api import Error as PlaywrightError, Page + +from monitor.models import PageSnapshot, WebsiteConfig +from storage.snapshots import SnapshotStore + + +def capture_page_snapshot( + page: Page, + website: WebsiteConfig, + load_duration: float, + snapshot_store: SnapshotStore, + timestamp: float, +) -> PageSnapshot: + """Capture page metadata, HTML, text, and screenshot.""" + captured_at = datetime.fromtimestamp(timestamp) + screenshot_path = None + title = "" + visible_text = "" + html = "" + error_parts: list[str] = [] + + try: + title = page.title() + except PlaywrightError as exc: + error_parts.append(f"title: {exc}") + + try: + visible_text = page.inner_text("body") + except PlaywrightError as exc: + error_parts.append(f"visible_text: {exc}") + + try: + html = page.content() + except PlaywrightError as exc: + error_parts.append(f"html: {exc}") + + try: + screenshot_path = snapshot_store.save_screenshot( + website_id=website.id, + timestamp=captured_at, + image_bytes=page.screenshot(full_page=True), + ) + except PlaywrightError as exc: + error_parts.append(f"screenshot: {exc}") + + return PageSnapshot( + website_id=website.id, + website_name=website.name, + requested_url=website.url, + final_url=page.url, + timestamp=captured_at, + title=title, + visible_text=visible_text, + html=html, + screenshot_path=screenshot_path, + load_duration=load_duration, + success=len(error_parts) == 0, + error="; ".join(error_parts) if error_parts else None, + ) diff --git a/notifications/__init__.py b/notifications/__init__.py new file mode 100644 index 0000000..6da7c9f --- /dev/null +++ b/notifications/__init__.py @@ -0,0 +1,13 @@ +"""Notifications package.""" + +from notifications.base import LoggingNotificationService, NotificationService +from notifications.email import EmailNotificationService +from notifications.sms import SmsNotificationService, create_sms_service + +__all__ = [ + "NotificationService", + "LoggingNotificationService", + "SmsNotificationService", + "EmailNotificationService", + "create_sms_service", +] diff --git a/notifications/base.py b/notifications/base.py new file mode 100644 index 0000000..f9f7c06 --- /dev/null +++ b/notifications/base.py @@ -0,0 +1,32 @@ +"""Notification service abstraction.""" + +from __future__ import annotations + +import logging +from abc import ABC, abstractmethod + +logger = logging.getLogger(__name__) + + +class NotificationService(ABC): + """Abstract notification service.""" + + @abstractmethod + def send_sms(self, message: str) -> bool: + """Send an SMS message.""" + + @abstractmethod + def send_email(self, subject: str, body: str) -> bool: + """Send an email message.""" + + +class LoggingNotificationService(NotificationService): + """Default implementation that logs notifications instead of sending them.""" + + def send_sms(self, message: str) -> bool: + logger.info("SMS WOULD BE SENT:\n%s", message) + return True + + def send_email(self, subject: str, body: str) -> bool: + logger.info("EMAIL WOULD BE SENT:\nSubject: %s\n%s", subject, body) + return True diff --git a/notifications/email.py b/notifications/email.py new file mode 100644 index 0000000..c758db0 --- /dev/null +++ b/notifications/email.py @@ -0,0 +1,57 @@ +"""Email notification provider.""" + +from __future__ import annotations + +import logging +import os +import smtplib +from email.mime.multipart import MIMEMultipart +from email.mime.text import MIMEText + +from notifications.base import LoggingNotificationService + +logger = logging.getLogger(__name__) + + +class EmailNotificationService(LoggingNotificationService): + """ + Email notification service. + + Uses SMTP credentials from environment variables when configured. + Falls back to logging when SMTP is not set up. + """ + + def __init__(self) -> None: + self.smtp_host = os.environ.get("SMTP_HOST", "") + self.smtp_port = int(os.environ.get("SMTP_PORT", "587")) + self.smtp_user = os.environ.get("SMTP_USER", "") + self.smtp_password = os.environ.get("SMTP_PASSWORD", "") + self.email_from = os.environ.get("EMAIL_FROM", "") + self.email_to = os.environ.get("EMAIL_TO", "") + self.use_tls = os.environ.get("SMTP_USE_TLS", "true").lower() == "true" + + def _smtp_configured(self) -> bool: + return all([self.smtp_host, self.smtp_user, self.smtp_password, self.email_from, self.email_to]) + + def send_email(self, subject: str, body: str) -> bool: + if not self._smtp_configured(): + return super().send_email(subject, body) + + try: + msg = MIMEMultipart() + msg["From"] = self.email_from + msg["To"] = self.email_to + msg["Subject"] = subject + msg.attach(MIMEText(body, "plain")) + + with smtplib.SMTP(self.smtp_host, self.smtp_port, timeout=30) as server: + if self.use_tls: + server.starttls() + server.login(self.smtp_user, self.smtp_password) + server.sendmail(self.email_from, [self.email_to], msg.as_string()) + + logger.info("Email sent to %s", self.email_to) + return True + except Exception as exc: + logger.error("Failed to send email: %s", exc) + return False diff --git a/notifications/sms.py b/notifications/sms.py new file mode 100644 index 0000000..bb2f98b --- /dev/null +++ b/notifications/sms.py @@ -0,0 +1,41 @@ +"""SMS notification provider stubs.""" + +from __future__ import annotations + +import logging +import os + +from notifications.base import LoggingNotificationService, NotificationService + +logger = logging.getLogger(__name__) + + +class SmsNotificationService(LoggingNotificationService): + """ + SMS notification service. + + Currently logs messages. Future providers (Twilio, Urmi, etc.) can be + plugged in without changing the monitoring engine. + """ + + def __init__(self) -> None: + self.provider = os.environ.get("SMS_PROVIDER", "logging") + self.api_key = os.environ.get("SMS_API_KEY", "") + self.from_number = os.environ.get("SMS_FROM_NUMBER", "") + self.to_number = os.environ.get("SMS_TO_NUMBER", "") + + def send_sms(self, message: str) -> bool: + if self.provider == "logging" or not self.api_key: + return super().send_sms(message) + + # Placeholder for future provider integration + logger.warning( + "SMS provider '%s' is not yet implemented; logging instead", + self.provider, + ) + return super().send_sms(message) + + +def create_sms_service() -> NotificationService: + """Factory for SMS-capable notification service.""" + return SmsNotificationService() diff --git a/profiles/.gitkeep b/profiles/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..187f5d1 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,2 @@ +playwright>=1.40.0 +PyYAML>=6.0 diff --git a/snapshots/.gitkeep b/snapshots/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/storage/__init__.py b/storage/__init__.py new file mode 100644 index 0000000..9417b35 --- /dev/null +++ b/storage/__init__.py @@ -0,0 +1,5 @@ +"""Storage package.""" + +from storage.snapshots import SnapshotStore + +__all__ = ["SnapshotStore"] diff --git a/storage/snapshots.py b/storage/snapshots.py new file mode 100644 index 0000000..16fc138 --- /dev/null +++ b/storage/snapshots.py @@ -0,0 +1,56 @@ +"""Snapshot file storage and retention management.""" + +from __future__ import annotations + +import logging +from datetime import datetime +from pathlib import Path + +logger = logging.getLogger(__name__) + + +class SnapshotStore: + """Manages screenshot storage and cleanup.""" + + def __init__(self, base_directory: str, retention_count: int = 50) -> None: + self.base_directory = Path(base_directory) + self.retention_count = retention_count + self.base_directory.mkdir(parents=True, exist_ok=True) + + def _website_directory(self, website_id: str) -> Path: + directory = self.base_directory / website_id + directory.mkdir(parents=True, exist_ok=True) + return directory + + def _timestamp_filename(self, timestamp: datetime) -> str: + return timestamp.strftime("%Y-%m-%dT%H-%M-%S") + ".png" + + def save_screenshot( + self, + website_id: str, + timestamp: datetime, + image_bytes: bytes, + ) -> str: + """Save a screenshot and enforce retention limits.""" + directory = self._website_directory(website_id) + filename = self._timestamp_filename(timestamp) + filepath = directory / filename + filepath.write_bytes(image_bytes) + logger.debug("Screenshot saved: %s", filepath) + + self._cleanup_old_snapshots(directory) + return str(filepath) + + def _cleanup_old_snapshots(self, directory: Path) -> None: + """Remove oldest snapshots beyond retention_count.""" + screenshots = sorted(directory.glob("*.png"), key=lambda p: p.stat().st_mtime) + excess = len(screenshots) - self.retention_count + if excess <= 0: + return + + for old_file in screenshots[:excess]: + try: + old_file.unlink() + logger.debug("Removed old snapshot: %s", old_file) + except OSError as exc: + logger.warning("Failed to remove snapshot %s: %s", old_file, exc)