diff --git a/.gitignore b/.gitignore index 3f5a00f4..5265bf30 100644 --- a/.gitignore +++ b/.gitignore @@ -40,3 +40,10 @@ github_deploy_key* dist/ .vscode/ .idea/ +# Local test config (contains credentials) +up.discover-test.json +# Generated discovery report (re-run with --discover to recreate) +api_endpoints_discovery.md +# Python endpoint-discovery tool (optional) +tools/endpoint-discovery/__pycache__ +tools/endpoint-discovery/.venv diff --git a/tools/endpoint-discovery/.env.example b/tools/endpoint-discovery/.env.example new file mode 100644 index 00000000..925e6534 --- /dev/null +++ b/tools/endpoint-discovery/.env.example @@ -0,0 +1,10 @@ +# UniFi controller URL (same URL you use for unpoller) +UNIFI_URL=https://192.168.1.1 +UNIFI_USER=admin +UNIFI_PASS=your-password + +# Optional: run with visible browser +# HEADLESS=false + +# Optional: custom output directory (default: this directory) +# OUTPUT_DIR=. diff --git a/tools/endpoint-discovery/README.md b/tools/endpoint-discovery/README.md new file mode 100644 index 00000000..615cccf6 --- /dev/null +++ b/tools/endpoint-discovery/README.md @@ -0,0 +1,35 @@ +# UniFi endpoint discovery (headless) + +Runs a headless browser against your UniFi controller, logs in, and records all API requests the UI makes. Use this to see which endpoints your controller exposes (e.g. when debugging 404s like [device-tags #935](https://github.com/unpoller/unpoller/issues/935)). + +## What it does + +1. Launches Chromium (headless by default). +2. Navigates to your UniFi controller URL. +3. If it sees a login form, fills username/password and submits. +4. Visits common UI paths to trigger API calls. +5. Captures every XHR/fetch request to `/api` or `/proxy/` (same origin). +6. Writes a markdown file: `API_ENDPOINTS_HEADLESS_YYYY-MM-DD.md` in this directory. + +## Quick start + +```bash +cd tools/endpoint-discovery +python3 -m venv .venv +.venv/bin/pip install playwright +.venv/bin/playwright install chromium +UNIFI_URL=https://YOUR_CONTROLLER UNIFI_USER=admin UNIFI_PASS=yourpassword .venv/bin/python discover.py +``` + +The script writes `API_ENDPOINTS_HEADLESS_.md` in the same directory. Paste that file (or its contents) in an issue or comment when reporting which endpoints your controller supports. + +## Options + +- **Headed (see the browser):** `HEADLESS=false` before the command. +- **Output elsewhere:** `OUTPUT_DIR=/path/to/dir` before the command. +- **Use .env:** Copy `.env.example` to `.env`, set your values, then run `python discover.py` (install `python-dotenv` for .env support). + +## Requirements + +- Python 3.8+ +- Direct controller URL (e.g. `https://192.168.1.1`) works best; same-origin requests are captured. diff --git a/tools/endpoint-discovery/discover.py b/tools/endpoint-discovery/discover.py new file mode 100644 index 00000000..d72f8c8b --- /dev/null +++ b/tools/endpoint-discovery/discover.py @@ -0,0 +1,198 @@ +#!/usr/bin/env python3 +""" +Headless endpoint discovery: navigate the UniFi controller UI and capture +all XHR/fetch requests (method, URL, optional headers), similar to manually +clicking around in Chrome and logging network traffic. + +Usage: + UNIFI_URL=https://192.168.1.1 UNIFI_USER=admin UNIFI_PASS=... python discover.py + Or copy .env.example to .env and run (with python-dotenv): python discover.py + +Output: API_ENDPOINTS_HEADLESS_.md in current dir (or set OUTPUT_DIR) + +Requires: pip install playwright && playwright install chromium +Optional: pip install python-dotenv (to load .env) +""" + +import os +from datetime import date +from pathlib import Path +from urllib.parse import urlparse + +try: + from dotenv import load_dotenv + load_dotenv(Path(__file__).resolve().parent / ".env") +except ImportError: + pass + +from playwright.sync_api import sync_playwright + +SCRIPT_DIR = Path(__file__).resolve().parent +# Default: write output in the same directory as the script (easy for users) +OUTPUT_DIR = Path(os.environ.get("OUTPUT_DIR", str(SCRIPT_DIR))) +HEADLESS = os.environ.get("HEADLESS", "true").lower() != "false" + +GROUP_ORDER = ["api", "proxy-network-api", "proxy-network-v2", "proxy-users", "proxy-other", "other"] +GROUP_TITLES = { + "api": "API (legacy)", + "proxy-network-api": "Proxy /network API (v1)", + "proxy-network-v2": "Proxy /network v2 API", + "proxy-users": "Proxy /users API", + "proxy-other": "Proxy (other)", + "other": "Other", +} + + +def is_api_like(pathname: str) -> bool: + return "/api" in pathname or "/proxy/" in pathname + + +def normalize_url(url_str: str) -> str: + try: + u = urlparse(url_str) + return f"{u.scheme}://{u.netloc}{u.path or ''}{u.query and '?' + u.query or ''}" + except Exception: + return url_str + + +def run_group(pathname: str) -> str: + if pathname.startswith("/api/"): + return "api" + if pathname.startswith("/proxy/network/api/"): + return "proxy-network-api" + if pathname.startswith("/proxy/network/v2/"): + return "proxy-network-v2" + if pathname.startswith("/proxy/users/"): + return "proxy-users" + if pathname.startswith("/proxy/"): + return "proxy-other" + return "other" + + +def main() -> None: + base_url = (os.environ.get("UNIFI_URL") or "").rstrip("/") + user = os.environ.get("UNIFI_USER") or "" + password = os.environ.get("UNIFI_PASS") or "" + + if not base_url or not user or not password: + print("Set UNIFI_URL, UNIFI_USER, and UNIFI_PASS (env or .env).", file=__import__("sys").stderr) + raise SystemExit(1) + + try: + origin = urlparse(base_url) + our_origin = f"{origin.scheme}://{origin.netloc}" + except Exception: + our_origin = "" + + captured: dict[str, dict] = {} + + def on_request(request): + if request.resource_type not in ("xhr", "fetch"): + return + url = request.url + try: + u = urlparse(url) + req_origin = f"{u.scheme}://{u.netloc}" + if req_origin != our_origin: + return + pathname = u.path or "" + if not is_api_like(pathname): + return + key = f"{request.method} {normalize_url(url)}" + if key not in captured: + captured[key] = { + "method": request.method, + "url": normalize_url(url), + "pathname": pathname, + "request_headers": request.headers, + } + except Exception: + pass + + with sync_playwright() as p: + browser = p.chromium.launch( + headless=HEADLESS, + args=["--ignore-certificate-errors"], + ) + context = browser.new_context( + ignore_https_errors=True, + user_agent="Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36", + ) + page = context.new_page() + page.on("request", on_request) + + print("Navigating to", base_url) + try: + page.goto(base_url, wait_until="networkidle", timeout=30000) + except Exception: + pass + + # Login + if page.locator('input[type="password"]').count() > 0: + print("Login form detected, submitting credentials...") + page.locator('input[name="username"], input[name="email"], input[type="text"]').first.fill(user) + page.locator('input[type="password"]').fill(password) + page.locator('button[type="submit"], input[type="submit"], button:has-text("Login"), button:has-text("Sign in")').first.click() + try: + page.wait_for_load_state("networkidle") + except Exception: + pass + page.wait_for_timeout(3000) + + # Visit common paths to trigger more API calls + for path in ["/", "/devices", "/clients", "/settings", "/insights", "/topology", "/dashboard"]: + try: + page.goto(base_url + path, wait_until="domcontentloaded", timeout=10000) + page.wait_for_timeout(2000) + except Exception: + pass + + browser.close() + + # Build markdown + today = date.today().isoformat() + entries = sorted(captured.values(), key=lambda e: e["url"]) + by_group: dict[str, list] = {} + for e in entries: + g = run_group(e["pathname"]) + by_group.setdefault(g, []).append(e) + + lines = [ + "# API Endpoints (headless discovery)", + "", + f"- **Date**: {today}", + f"- **Controller**: {base_url}", + f"- **Total unique requests**: {len(entries)}", + "", + "---", + "", + ] + for g in GROUP_ORDER: + list_ = by_group.get(g) + if not list_: + continue + lines.append(f"## {GROUP_TITLES.get(g, g)}") + lines.append("") + for e in list_: + u = urlparse(e["url"]) + path_only = (u.path or "") + (("?" + u.query) if u.query else "") + lines.append(f"- `{e['method']} {path_only}`") + lines.append("") + + lines.extend(["---", "", "## Sample request headers (first request)", ""]) + if entries and entries[0].get("request_headers"): + lines.append("```") + for k, v in entries[0]["request_headers"].items(): + kl = k.lower() + if kl.startswith("x-") or kl in ("accept", "authorization"): + lines.append(f"{k}: {v}") + lines.append("```") + + out_path = OUTPUT_DIR / f"API_ENDPOINTS_HEADLESS_{today}.md" + OUTPUT_DIR.mkdir(parents=True, exist_ok=True) + out_path.write_text("\n".join(lines) + "\n", encoding="utf-8") + print("Wrote", out_path, f"({len(entries)} unique endpoints)") + + +if __name__ == "__main__": + main() diff --git a/tools/endpoint-discovery/requirements.txt b/tools/endpoint-discovery/requirements.txt new file mode 100644 index 00000000..49711953 --- /dev/null +++ b/tools/endpoint-discovery/requirements.txt @@ -0,0 +1,3 @@ +playwright>=1.49.0 +# Optional: for .env support +# python-dotenv>=1.0.0