Merge pull request #936 from brngates98/feat/endpoint-discovery-tool

.gitignore

@@ -40,3 +40,6 @@ github_deploy_key*
 dist/
 .vscode/
 .idea/
+# Endpoint discovery tool
+tools/endpoint-discovery/__pycache__
+tools/endpoint-discovery/.venv

tools/endpoint-discovery/.env.example

@@ -0,0 +1,10 @@
+# UniFi controller URL (same URL you use for unpoller)
+UNIFI_URL=https://192.168.1.1
+UNIFI_USER=admin
+UNIFI_PASS=your-password
+
+# Optional: run with visible browser
+# HEADLESS=false
+
+# Optional: custom output directory (default: this directory)
+# OUTPUT_DIR=.

tools/endpoint-discovery/README.md

@@ -0,0 +1,35 @@
+# UniFi endpoint discovery (headless)
+
+Runs a headless browser against your UniFi controller, logs in, and records all API requests the UI makes. Use this to see which endpoints your controller exposes (e.g. when debugging 404s like [device-tags #935](https://github.com/unpoller/unpoller/issues/935)).
+
+## What it does
+
+1. Launches Chromium (headless by default).
+2. Navigates to your UniFi controller URL.
+3. If it sees a login form, fills username/password and submits.
+4. Visits common UI paths to trigger API calls.
+5. Captures every XHR/fetch request to `/api` or `/proxy/` (same origin).
+6. Writes a markdown file: `API_ENDPOINTS_HEADLESS_YYYY-MM-DD.md` in this directory.
+
+## Quick start
+
+```bash
+cd tools/endpoint-discovery
+python3 -m venv .venv
+.venv/bin/pip install playwright
+.venv/bin/playwright install chromium
+UNIFI_URL=https://YOUR_CONTROLLER UNIFI_USER=admin UNIFI_PASS=yourpassword .venv/bin/python discover.py
+```
+
+The script writes `API_ENDPOINTS_HEADLESS_<date>.md` in the same directory. Paste that file (or its contents) in an issue or comment when reporting which endpoints your controller supports.
+
+## Options
+
+- **Headed (see the browser):** `HEADLESS=false` before the command.
+- **Output elsewhere:** `OUTPUT_DIR=/path/to/dir` before the command.
+- **Use .env:** Copy `.env.example` to `.env`, set your values, then run `python discover.py` (install `python-dotenv` for .env support).
+
+## Requirements
+
+- Python 3.8+
+- Direct controller URL (e.g. `https://192.168.1.1`) works best; same-origin requests are captured.

tools/endpoint-discovery/discover.py

@@ -0,0 +1,198 @@
+#!/usr/bin/env python3
+"""
+Headless endpoint discovery: navigate the UniFi controller UI and capture
+all XHR/fetch requests (method, URL, optional headers), similar to manually
+clicking around in Chrome and logging network traffic.
+
+Usage:
+  UNIFI_URL=https://192.168.1.1 UNIFI_USER=admin UNIFI_PASS=... python discover.py
+  Or copy .env.example to .env and run (with python-dotenv): python discover.py
+
+Output: API_ENDPOINTS_HEADLESS_<date>.md in current dir (or set OUTPUT_DIR)
+
+Requires: pip install playwright && playwright install chromium
+Optional: pip install python-dotenv  (to load .env)
+"""
+
+import os
+from datetime import date
+from pathlib import Path
+from urllib.parse import urlparse
+
+try:
+    from dotenv import load_dotenv
+    load_dotenv(Path(__file__).resolve().parent / ".env")
+except ImportError:
+    pass
+
+from playwright.sync_api import sync_playwright
+
+SCRIPT_DIR = Path(__file__).resolve().parent
+# Default: write output in the same directory as the script (easy for users)
+OUTPUT_DIR = Path(os.environ.get("OUTPUT_DIR", str(SCRIPT_DIR)))
+HEADLESS = os.environ.get("HEADLESS", "true").lower() != "false"
+
+GROUP_ORDER = ["api", "proxy-network-api", "proxy-network-v2", "proxy-users", "proxy-other", "other"]
+GROUP_TITLES = {
+    "api": "API (legacy)",
+    "proxy-network-api": "Proxy /network API (v1)",
+    "proxy-network-v2": "Proxy /network v2 API",
+    "proxy-users": "Proxy /users API",
+    "proxy-other": "Proxy (other)",
+    "other": "Other",
+}
+
+
+def is_api_like(pathname: str) -> bool:
+    return "/api" in pathname or "/proxy/" in pathname
+
+
+def normalize_url(url_str: str) -> str:
+    try:
+        u = urlparse(url_str)
+        return f"{u.scheme}://{u.netloc}{u.path or ''}{u.query and '?' + u.query or ''}"
+    except Exception:
+        return url_str
+
+
+def run_group(pathname: str) -> str:
+    if pathname.startswith("/api/"):
+        return "api"
+    if pathname.startswith("/proxy/network/api/"):
+        return "proxy-network-api"
+    if pathname.startswith("/proxy/network/v2/"):
+        return "proxy-network-v2"
+    if pathname.startswith("/proxy/users/"):
+        return "proxy-users"
+    if pathname.startswith("/proxy/"):
+        return "proxy-other"
+    return "other"
+
+
+def main() -> None:
+    base_url = (os.environ.get("UNIFI_URL") or "").rstrip("/")
+    user = os.environ.get("UNIFI_USER") or ""
+    password = os.environ.get("UNIFI_PASS") or ""
+
+    if not base_url or not user or not password:
+        print("Set UNIFI_URL, UNIFI_USER, and UNIFI_PASS (env or .env).", file=__import__("sys").stderr)
+        raise SystemExit(1)
+
+    try:
+        origin = urlparse(base_url)
+        our_origin = f"{origin.scheme}://{origin.netloc}"
+    except Exception:
+        our_origin = ""
+
+    captured: dict[str, dict] = {}
+
+    def on_request(request):
+        if request.resource_type not in ("xhr", "fetch"):
+            return
+        url = request.url
+        try:
+            u = urlparse(url)
+            req_origin = f"{u.scheme}://{u.netloc}"
+            if req_origin != our_origin:
+                return
+            pathname = u.path or ""
+            if not is_api_like(pathname):
+                return
+            key = f"{request.method} {normalize_url(url)}"
+            if key not in captured:
+                captured[key] = {
+                    "method": request.method,
+                    "url": normalize_url(url),
+                    "pathname": pathname,
+                    "request_headers": request.headers,
+                }
+        except Exception:
+            pass
+
+    with sync_playwright() as p:
+        browser = p.chromium.launch(
+            headless=HEADLESS,
+            args=["--ignore-certificate-errors"],
+        )
+        context = browser.new_context(
+            ignore_https_errors=True,
+            user_agent="Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
+        )
+        page = context.new_page()
+        page.on("request", on_request)
+
+        print("Navigating to", base_url)
+        try:
+            page.goto(base_url, wait_until="networkidle", timeout=30000)
+        except Exception:
+            pass
+
+        # Login
+        if page.locator('input[type="password"]').count() > 0:
+            print("Login form detected, submitting credentials...")
+            page.locator('input[name="username"], input[name="email"], input[type="text"]').first.fill(user)
+            page.locator('input[type="password"]').fill(password)
+            page.locator('button[type="submit"], input[type="submit"], button:has-text("Login"), button:has-text("Sign in")').first.click()
+            try:
+                page.wait_for_load_state("networkidle")
+            except Exception:
+                pass
+            page.wait_for_timeout(3000)
+
+        # Visit common paths to trigger more API calls
+        for path in ["/", "/devices", "/clients", "/settings", "/insights", "/topology", "/dashboard"]:
+            try:
+                page.goto(base_url + path, wait_until="domcontentloaded", timeout=10000)
+                page.wait_for_timeout(2000)
+            except Exception:
+                pass
+
+        browser.close()
+
+    # Build markdown
+    today = date.today().isoformat()
+    entries = sorted(captured.values(), key=lambda e: e["url"])
+    by_group: dict[str, list] = {}
+    for e in entries:
+        g = run_group(e["pathname"])
+        by_group.setdefault(g, []).append(e)
+
+    lines = [
+        "# API Endpoints (headless discovery)",
+        "",
+        f"- **Date**: {today}",
+        f"- **Controller**: {base_url}",
+        f"- **Total unique requests**: {len(entries)}",
+        "",
+        "---",
+        "",
+    ]
+    for g in GROUP_ORDER:
+        list_ = by_group.get(g)
+        if not list_:
+            continue
+        lines.append(f"## {GROUP_TITLES.get(g, g)}")
+        lines.append("")
+        for e in list_:
+            u = urlparse(e["url"])
+            path_only = (u.path or "") + (("?" + u.query) if u.query else "")
+            lines.append(f"- `{e['method']} {path_only}`")
+        lines.append("")
+
+    lines.extend(["---", "", "## Sample request headers (first request)", ""])
+    if entries and entries[0].get("request_headers"):
+        lines.append("```")
+        for k, v in entries[0]["request_headers"].items():
+            kl = k.lower()
+            if kl.startswith("x-") or kl in ("accept", "authorization"):
+                lines.append(f"{k}: {v}")
+        lines.append("```")
+
+    out_path = OUTPUT_DIR / f"API_ENDPOINTS_HEADLESS_{today}.md"
+    OUTPUT_DIR.mkdir(parents=True, exist_ok=True)
+    out_path.write_text("\n".join(lines) + "\n", encoding="utf-8")
+    print("Wrote", out_path, f"({len(entries)} unique endpoints)")
+
+
+if __name__ == "__main__":
+    main()

tools/endpoint-discovery/requirements.txt

@@ -0,0 +1,3 @@
+playwright>=1.49.0
+# Optional: for .env support
+# python-dotenv>=1.0.0