Merge pull request #937 from brngates98/feat/discover-flag

.gitignore

@@ -40,6 +40,7 @@ github_deploy_key*
 dist/
 .vscode/
 .idea/
-# Endpoint discovery tool
-tools/endpoint-discovery/__pycache__
-tools/endpoint-discovery/.venv
+# Local test config (contains credentials)
+up.discover-test.json
+# Generated discovery report (re-run with --discover to recreate)
+api_endpoints_discovery.md

docs/PR_936_REPLACEMENT.md

@@ -0,0 +1,66 @@
+# Endpoint discovery via `--discover` (Go)
+
+The [unpoller](https://github.com/unpoller/unpoller) repo has a Python/Playwright endpoint-discovery tool in `tools/endpoint-discovery/` (browser-based; discovers unknown endpoints; more support currently). This doc describes the Go-based discovery: it lives in [unpoller/unifi](https://github.com/unpoller/unifi) and unpoller, probes a fixed list of known API paths, and discovers/confirms known endpoints only.
+
+---
+
+## Summary
+
+- **Feature:** `--discover` flag on unpoller that probes known API endpoints on the controller and writes a shareable markdown report.
+- **Credentials:** Uses the same config file (and first unifi controller) as normal unpoller runs.
+- **Output:** Markdown file (default `api_endpoints_discovery.md`) with method, path, and HTTP status for each endpoint. Users can share this when reporting API/404 issues (e.g. [issue #935](https://github.com/unpoller/unpoller/issues/935)).
+- **Dependency:** Requires [unpoller/unifi](https://github.com/unpoller/unifi) with `DiscoverEndpoints` and `Probe` merged.
+
+---
+
+## Changes in unpoller (this PR)
+
+| File | Change |
+|------|--------|
+| `pkg/poller/config.go` | Add `Discover bool`, `DiscoverOutput string` to `Flags`. |
+| `pkg/poller/start.go` | Register `--discover` and `--discover-output`; when `--discover`, call `RunDiscover()` and exit. |
+| `pkg/poller/commands.go` | Add `RunDiscover()`: load config, init inputs, find input implementing `Discoverer`, call `Discover(outputPath)`. |
+| `pkg/poller/inputs.go` | Add optional interface `Discoverer` with `Discover(outputPath string) error`. |
+| `pkg/inputunifi/discover.go` | **New file.** Implement `Discoverer`: first controller, authenticate, get sites, call `c.Unifi.DiscoverEndpoints(site, outputPath)`. |
+| `.gitignore` | Add `up.discover-test.json`, `api_endpoints_discovery.md`. |
+
+---
+
+## Usage
+
+```bash
+# Same config as normal unpoller (first unifi controller is used)
+unpoller --discover --config /path/to/up.conf --discover-output api_endpoints_discovery.md
+```
+
+If config is in the default search path:
+
+```bash
+unpoller --discover --discover-output api_endpoints_discovery.md
+```
+
+---
+
+## PR title
+
+**Add `--discover` to probe API endpoints and write shareable report**
+
+---
+
+## PR description (suggested)
+
+Go-based endpoint discovery: probes known API paths on the controller and writes a shareable report. Uses the [unifi](https://github.com/unpoller/unifi) library; same config as normal polling. The Python tool in `tools/endpoint-discovery/` remains for browser-based discovery (more coverage).
+
+**What it does**
+- `unpoller --discover` uses the first unifi controller from your config, authenticates, and probes a set of known API paths.
+- Writes a markdown report (default `api_endpoints_discovery.md`) with method, path, and HTTP status for each endpoint.
+- Same credentials as normal polling; users can share the file when reporting API/404 issues (e.g. #935).
+
+**Usage**
+```bash
+unpoller --discover --config /path/to/up.conf --discover-output api_endpoints_discovery.md
+```
+
+**Requires** [unpoller/unifi](https://github.com/unpoller/unifi) with `DiscoverEndpoints` (and `Probe`) merged.
+
+**CI:** Merge unifi first, then this PR (or update go.mod to require the new unifi release). For local testing, use `replace github.com/unpoller/unifi/v5 => ../unifi` with the unifi repo checked out.

go.mod

@@ -12,7 +12,7 @@ require (
 	github.com/prometheus/common v0.67.5
 	github.com/spf13/pflag v1.0.10
 	github.com/stretchr/testify v1.11.1
-	github.com/unpoller/unifi/v5 v5.12.0
+	github.com/unpoller/unifi/v5 v5.13.0
 	golang.org/x/crypto v0.47.0
 	golang.org/x/term v0.39.0
 	golift.io/cnfg v0.2.3
@@ -46,5 +46,3 @@ require (
 	golang.org/x/sys v0.40.0 // indirect
 	google.golang.org/protobuf v1.36.11 // indirect
 )
-
-// replace github.com/unpoller/unifi/v5 => ../unifi

go.sum

@@ -77,8 +77,8 @@ github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO
 github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
 github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U=
 github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U=
-github.com/unpoller/unifi/v5 v5.12.0 h1:wdC7HpASWXUL7iEHkmaAeKMFpZ8UazWDIpuijY43saE=
-github.com/unpoller/unifi/v5 v5.12.0/go.mod h1:vSIXIclPG9dpKxUp+pavfgENHWaTZXvDg7F036R1YCo=
+github.com/unpoller/unifi/v5 v5.13.0 h1:0pdMckXzGoH0Lg8+WGV4p6ZJeMQsmqmfkdZLtumdIao=
+github.com/unpoller/unifi/v5 v5.13.0/go.mod h1:vSIXIclPG9dpKxUp+pavfgENHWaTZXvDg7F036R1YCo=
 github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
 go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto=
 go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE=

pkg/inputunifi/discover.go

@@ -0,0 +1,49 @@
+package inputunifi
+
+import (
+	"fmt"
+)
+
+// Discover implements poller.Discoverer. It uses the first configured controller
+// to probe known API endpoints and write a shareable report to outputPath.
+// Uses the same credentials as normal polling (from config file).
+func (u *InputUnifi) Discover(outputPath string) error {
+	if u.Config == nil || u.Disable {
+		return fmt.Errorf("unifi input disabled or not configured")
+	}
+
+	u.setDefaults(&u.Default)
+
+	if len(u.Controllers) == 0 && !u.Dynamic {
+		u.Controllers = []*Controller{&u.Default}
+	}
+
+	if len(u.Controllers) == 0 {
+		return fmt.Errorf("no unifi controller configured")
+	}
+
+	c := u.setControllerDefaults(u.Controllers[0])
+	if c.URL == "" {
+		return fmt.Errorf("first controller has no URL")
+	}
+
+	if err := u.getUnifi(c); err != nil {
+		return fmt.Errorf("authenticating to controller: %w", err)
+	}
+
+	sites, err := c.Unifi.GetSites()
+	if err != nil {
+		return fmt.Errorf("getting sites: %w", err)
+	}
+
+	site := "default"
+	if len(sites) > 0 && sites[0].Name != "" {
+		site = sites[0].Name
+	}
+
+	if err := c.Unifi.DiscoverEndpoints(site, outputPath); err != nil {
+		return fmt.Errorf("writing discovery report: %w", err)
+	}
+
+	return nil
+}

pkg/poller/commands.go

@@ -191,3 +191,45 @@ func (u *UnifiPoller) HealthCheck() error {
 	// All checks passed, application is healthy.
 	return nil
 }
+
+// RunDiscover loads config, initializes inputs, finds an input that implements
+// Discoverer, and runs Discover(outputPath). Uses the same config as normal polling.
+func (u *UnifiPoller) RunDiscover() error {
+	cfile, err := getFirstFile(strings.Split(u.Flags.ConfigFile, ","))
+	if err != nil {
+		return fmt.Errorf("discover: config file not found: %w", err)
+	}
+
+	u.Flags.ConfigFile = cfile
+	u.Logf("Loading Configuration File: %s", u.Flags.ConfigFile)
+
+	if err := u.ParseConfigs(); err != nil {
+		return fmt.Errorf("discover: parse config: %w", err)
+	}
+
+	if err := u.InitializeInputs(); err != nil {
+		return fmt.Errorf("discover: initialize inputs: %w", err)
+	}
+
+	outputPath := u.Flags.DiscoverOutput
+	if outputPath == "" {
+		outputPath = "api_endpoints_discovery.md"
+	}
+
+	inputSync.RLock()
+	defer inputSync.RUnlock()
+
+	for _, input := range inputs {
+		if d, ok := input.Input.(Discoverer); ok {
+			if err := d.Discover(outputPath); err != nil {
+				return fmt.Errorf("discover: %w", err)
+			}
+
+			u.Logf("Discovery report written to %s (share with maintainers for API/404 issues).", outputPath)
+
+			return nil
+		}
+	}
+
+	return fmt.Errorf("discover: no input plugin supports discovery (unifi input required)")
+}

pkg/poller/config.go

@@ -68,12 +68,14 @@ type UnifiPoller struct {
 
 // Flags represents the CLI args available and their settings.
 type Flags struct {
-	ConfigFile string
-	DumpJSON   string
-	HashPW     string
-	ShowVer    bool
-	DebugIO    bool
-	Health     bool
+	ConfigFile     string
+	DumpJSON       string
+	HashPW         string
+	ShowVer        bool
+	DebugIO        bool
+	Health         bool
+	Discover       bool
+	DiscoverOutput string
 	*pflag.FlagSet
 }
 

pkg/poller/inputs.go

@@ -24,6 +24,12 @@ type Input interface {
 	DebugInput() (bool, error)
 }
 
+// Discoverer is an optional interface for inputs that can discover API endpoints
+// on a controller and write a shareable report (e.g. for support/debugging).
+type Discoverer interface {
+	Discover(outputPath string) error
+}
+
 // InputPlugin describes an input plugin's consumable interface.
 type InputPlugin struct {
 	Name   string

pkg/poller/start.go

@@ -38,6 +38,10 @@ func (u *UnifiPoller) Start() error {
 		return u.HealthCheck()
 	}
 
+	if u.Flags.Discover {
+		return u.RunDiscover()
+	}
+
 	cfile, err := getFirstFile(strings.Split(u.Flags.ConfigFile, ","))
 	if err != nil {
 		return err
@@ -81,6 +85,9 @@ func (f *Flags) Parse(args []string) {
 		"This debug option prints a json payload and exits. See man page for more info.")
 	f.BoolVarP(&f.DebugIO, "debugio", "d", false, "Debug the Inputs and Outputs configured and exit.")
 	f.BoolVarP(&f.Health, "health", "", false, "Run health check and exit with status 0 (healthy) or 1 (unhealthy).")
+	f.BoolVarP(&f.Discover, "discover", "", false, "Discover API endpoints on the controller and write a shareable report, then exit.")
+	f.StringVarP(&f.DiscoverOutput, "discover-output", "", "api_endpoints_discovery.md",
+		"Path for the discovery report when using --discover.")
 	f.StringVarP(&f.ConfigFile, "config", "c", DefaultConfFile(),
 		"Poller config file path. Separating multiple paths with a comma will load the first config file found.")
 	f.BoolVarP(&f.ShowVer, "version", "v", false, "Print the version and exit.")

tools/endpoint-discovery/.env.example

@@ -1,10 +0,0 @@
-# UniFi controller URL (same URL you use for unpoller)
-UNIFI_URL=https://192.168.1.1
-UNIFI_USER=admin
-UNIFI_PASS=your-password
-
-# Optional: run with visible browser
-# HEADLESS=false
-
-# Optional: custom output directory (default: this directory)
-# OUTPUT_DIR=.

tools/endpoint-discovery/README.md

@@ -1,35 +0,0 @@
-# UniFi endpoint discovery (headless)
-
-Runs a headless browser against your UniFi controller, logs in, and records all API requests the UI makes. Use this to see which endpoints your controller exposes (e.g. when debugging 404s like [device-tags #935](https://github.com/unpoller/unpoller/issues/935)).
-
-## What it does
-
-1. Launches Chromium (headless by default).
-2. Navigates to your UniFi controller URL.
-3. If it sees a login form, fills username/password and submits.
-4. Visits common UI paths to trigger API calls.
-5. Captures every XHR/fetch request to `/api` or `/proxy/` (same origin).
-6. Writes a markdown file: `API_ENDPOINTS_HEADLESS_YYYY-MM-DD.md` in this directory.
-
-## Quick start
-
-```bash
-cd tools/endpoint-discovery
-python3 -m venv .venv
-.venv/bin/pip install playwright
-.venv/bin/playwright install chromium
-UNIFI_URL=https://YOUR_CONTROLLER UNIFI_USER=admin UNIFI_PASS=yourpassword .venv/bin/python discover.py
-```
-
-The script writes `API_ENDPOINTS_HEADLESS_<date>.md` in the same directory. Paste that file (or its contents) in an issue or comment when reporting which endpoints your controller supports.
-
-## Options
-
-- **Headed (see the browser):** `HEADLESS=false` before the command.
-- **Output elsewhere:** `OUTPUT_DIR=/path/to/dir` before the command.
-- **Use .env:** Copy `.env.example` to `.env`, set your values, then run `python discover.py` (install `python-dotenv` for .env support).
-
-## Requirements
-
-- Python 3.8+
-- Direct controller URL (e.g. `https://192.168.1.1`) works best; same-origin requests are captured.

tools/endpoint-discovery/discover.py

@@ -1,198 +0,0 @@
-#!/usr/bin/env python3
-"""
-Headless endpoint discovery: navigate the UniFi controller UI and capture
-all XHR/fetch requests (method, URL, optional headers), similar to manually
-clicking around in Chrome and logging network traffic.
-
-Usage:
-  UNIFI_URL=https://192.168.1.1 UNIFI_USER=admin UNIFI_PASS=... python discover.py
-  Or copy .env.example to .env and run (with python-dotenv): python discover.py
-
-Output: API_ENDPOINTS_HEADLESS_<date>.md in current dir (or set OUTPUT_DIR)
-
-Requires: pip install playwright && playwright install chromium
-Optional: pip install python-dotenv  (to load .env)
-"""
-
-import os
-from datetime import date
-from pathlib import Path
-from urllib.parse import urlparse
-
-try:
-    from dotenv import load_dotenv
-    load_dotenv(Path(__file__).resolve().parent / ".env")
-except ImportError:
-    pass
-
-from playwright.sync_api import sync_playwright
-
-SCRIPT_DIR = Path(__file__).resolve().parent
-# Default: write output in the same directory as the script (easy for users)
-OUTPUT_DIR = Path(os.environ.get("OUTPUT_DIR", str(SCRIPT_DIR)))
-HEADLESS = os.environ.get("HEADLESS", "true").lower() != "false"
-
-GROUP_ORDER = ["api", "proxy-network-api", "proxy-network-v2", "proxy-users", "proxy-other", "other"]
-GROUP_TITLES = {
-    "api": "API (legacy)",
-    "proxy-network-api": "Proxy /network API (v1)",
-    "proxy-network-v2": "Proxy /network v2 API",
-    "proxy-users": "Proxy /users API",
-    "proxy-other": "Proxy (other)",
-    "other": "Other",
-}
-
-
-def is_api_like(pathname: str) -> bool:
-    return "/api" in pathname or "/proxy/" in pathname
-
-
-def normalize_url(url_str: str) -> str:
-    try:
-        u = urlparse(url_str)
-        return f"{u.scheme}://{u.netloc}{u.path or ''}{u.query and '?' + u.query or ''}"
-    except Exception:
-        return url_str
-
-
-def run_group(pathname: str) -> str:
-    if pathname.startswith("/api/"):
-        return "api"
-    if pathname.startswith("/proxy/network/api/"):
-        return "proxy-network-api"
-    if pathname.startswith("/proxy/network/v2/"):
-        return "proxy-network-v2"
-    if pathname.startswith("/proxy/users/"):
-        return "proxy-users"
-    if pathname.startswith("/proxy/"):
-        return "proxy-other"
-    return "other"
-
-
-def main() -> None:
-    base_url = (os.environ.get("UNIFI_URL") or "").rstrip("/")
-    user = os.environ.get("UNIFI_USER") or ""
-    password = os.environ.get("UNIFI_PASS") or ""
-
-    if not base_url or not user or not password:
-        print("Set UNIFI_URL, UNIFI_USER, and UNIFI_PASS (env or .env).", file=__import__("sys").stderr)
-        raise SystemExit(1)
-
-    try:
-        origin = urlparse(base_url)
-        our_origin = f"{origin.scheme}://{origin.netloc}"
-    except Exception:
-        our_origin = ""
-
-    captured: dict[str, dict] = {}
-
-    def on_request(request):
-        if request.resource_type not in ("xhr", "fetch"):
-            return
-        url = request.url
-        try:
-            u = urlparse(url)
-            req_origin = f"{u.scheme}://{u.netloc}"
-            if req_origin != our_origin:
-                return
-            pathname = u.path or ""
-            if not is_api_like(pathname):
-                return
-            key = f"{request.method} {normalize_url(url)}"
-            if key not in captured:
-                captured[key] = {
-                    "method": request.method,
-                    "url": normalize_url(url),
-                    "pathname": pathname,
-                    "request_headers": request.headers,
-                }
-        except Exception:
-            pass
-
-    with sync_playwright() as p:
-        browser = p.chromium.launch(
-            headless=HEADLESS,
-            args=["--ignore-certificate-errors"],
-        )
-        context = browser.new_context(
-            ignore_https_errors=True,
-            user_agent="Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
-        )
-        page = context.new_page()
-        page.on("request", on_request)
-
-        print("Navigating to", base_url)
-        try:
-            page.goto(base_url, wait_until="networkidle", timeout=30000)
-        except Exception:
-            pass
-
-        # Login
-        if page.locator('input[type="password"]').count() > 0:
-            print("Login form detected, submitting credentials...")
-            page.locator('input[name="username"], input[name="email"], input[type="text"]').first.fill(user)
-            page.locator('input[type="password"]').fill(password)
-            page.locator('button[type="submit"], input[type="submit"], button:has-text("Login"), button:has-text("Sign in")').first.click()
-            try:
-                page.wait_for_load_state("networkidle")
-            except Exception:
-                pass
-            page.wait_for_timeout(3000)
-
-        # Visit common paths to trigger more API calls
-        for path in ["/", "/devices", "/clients", "/settings", "/insights", "/topology", "/dashboard"]:
-            try:
-                page.goto(base_url + path, wait_until="domcontentloaded", timeout=10000)
-                page.wait_for_timeout(2000)
-            except Exception:
-                pass
-
-        browser.close()
-
-    # Build markdown
-    today = date.today().isoformat()
-    entries = sorted(captured.values(), key=lambda e: e["url"])
-    by_group: dict[str, list] = {}
-    for e in entries:
-        g = run_group(e["pathname"])
-        by_group.setdefault(g, []).append(e)
-
-    lines = [
-        "# API Endpoints (headless discovery)",
-        "",
-        f"- **Date**: {today}",
-        f"- **Controller**: {base_url}",
-        f"- **Total unique requests**: {len(entries)}",
-        "",
-        "---",
-        "",
-    ]
-    for g in GROUP_ORDER:
-        list_ = by_group.get(g)
-        if not list_:
-            continue
-        lines.append(f"## {GROUP_TITLES.get(g, g)}")
-        lines.append("")
-        for e in list_:
-            u = urlparse(e["url"])
-            path_only = (u.path or "") + (("?" + u.query) if u.query else "")
-            lines.append(f"- `{e['method']} {path_only}`")
-        lines.append("")
-
-    lines.extend(["---", "", "## Sample request headers (first request)", ""])
-    if entries and entries[0].get("request_headers"):
-        lines.append("```")
-        for k, v in entries[0]["request_headers"].items():
-            kl = k.lower()
-            if kl.startswith("x-") or kl in ("accept", "authorization"):
-                lines.append(f"{k}: {v}")
-        lines.append("```")
-
-    out_path = OUTPUT_DIR / f"API_ENDPOINTS_HEADLESS_{today}.md"
-    OUTPUT_DIR.mkdir(parents=True, exist_ok=True)
-    out_path.write_text("\n".join(lines) + "\n", encoding="utf-8")
-    print("Wrote", out_path, f"({len(entries)} unique endpoints)")
-
-
-if __name__ == "__main__":
-    main()

tools/endpoint-discovery/requirements.txt

@@ -1,3 +0,0 @@
-playwright>=1.49.0
-# Optional: for .env support
-# python-dotenv>=1.0.0