From 7a0aec7b60c5226fd050d072b5712d75b03657d6 Mon Sep 17 00:00:00 2001 From: dmduran12 Date: Thu, 14 May 2026 15:14:10 -0700 Subject: [PATCH] feat(presets): expose bundled broker presets via GET /api/broker_presets MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds a new read-only endpoint that serves the bundled `repeater/presets/*.yaml` catalogue so the admin UI can render a network picker without bundling its own copy of the broker dicts. The UI side of this is paired with pyMC-dev/pyMC-RepeaterUI#TBD which retires src/assets/broker-templates.json in favour of authClient.get('/api/broker_presets'). Why The UI previously shipped a separate JSON snapshot of every supported MC2MQTT network. The JSON and these YAML files drifted: the Waev entry on the UI side pointed at mqtt-a.waev.app with audience mqtt.waev.app (single primary, no failover) while the YAML side here listed two brokers (A + B). The result was that operators picking 'Waev' from the dropdown silently lost the redundancy this preset is meant to provide. What changes repeater/presets/*.yaml - Add optional top-level `display_name` and `website` fields. The loader treats them as advisory metadata for the UI; the runtime connection code never reads them. `display_name` falls back to the titlecased filename stem if absent so existing third-party presets keep rendering. repeater/presets/waev.yaml - Collapse from two broker entries (waev-a, waev-b) to a single broker on `mqtt.waev.app`. The Waev edge Worker (see waev/src/router.ts: MQTT_PRIMARY_FAILOVER_TIMEOUT_MS) already does server-side A/B failover on the alias host with a 1500 ms timeout. Two independent client connections would defeat the dedup-on-pubkey-hash contract on the waev ingest side. Operators who want to pin to a specific container can edit host/audience after import. repeater/presets/meshmapper.yaml (new) - Port of the historical MeshMapper entry from the UI's deprecated JSON. Single broker on mqtt.meshmapper.cc, format: letsmesh (matches the published wire contract; bump to a dedicated value if/when wire-level differentiation lands). repeater/web/api_endpoints.py - New `broker_presets` CherryPy handler at `GET /api/broker_presets`. Unauthenticated to match the existing `mqtt_status` precedent — the response carries only public hostnames + TLS hints, no PII. Imports the presets module lazily so a broken YAML never blocks process startup. Response shape: { success: true, data: [{ id, name, website?, brokers: [ ... raw YAML dicts ... ] }, …] } tests/test_presets.py - Locks the new metadata fields (display_name, website) on all three presets. - Locks the Waev single-alias-broker design with an explicit comment tying the test to the waev Worker failover code. - Adds MeshMapper coverage parallel to the other public-network presets. - Adds a stub-instance test that drives the new `broker_presets` method on an APIEndpoints stand-in (bypassing the heavyweight `__init__`) and asserts the UI-ready response shape. Verification - New endpoint serves the expected three presets (letsmesh: 2 brokers, meshmapper: 1, waev: 1) when exercised end-to-end against a local mock that imports the real preset loader. - Existing legacy-config migration tests (broker_index 0/1/-1 → preset + overrides) still pass — the override pipeline is untouched. Co-Authored-By: Oz --- repeater/presets/letsmesh.yaml | 7 ++ repeater/presets/meshmapper.yaml | 33 ++++++ repeater/presets/waev.yaml | 35 ++++--- repeater/web/api_endpoints.py | 55 ++++++++++ tests/test_presets.py | 174 ++++++++++++++++++++++++++----- 5 files changed, 263 insertions(+), 41 deletions(-) create mode 100644 repeater/presets/meshmapper.yaml diff --git a/repeater/presets/letsmesh.yaml b/repeater/presets/letsmesh.yaml index f35ba8e..ed0ee70 100644 --- a/repeater/presets/letsmesh.yaml +++ b/repeater/presets/letsmesh.yaml @@ -9,6 +9,13 @@ # # Note: order matters for backward compatibility with the legacy # letsmesh.broker_index field. Index 0 is Europe, index 1 is US West. +# +# Optional UI metadata. Consumed by the GET /api/broker_presets endpoint +# so the React/Vue admin can render this preset in the "From Template" +# dropdown. `display_name` and `website` are advisory only — they are +# never read by the runtime broker connection code. +display_name: "LetsMesh" +website: "https://letsmesh.net" brokers: - name: "Europe (LetsMesh v1)" enabled: true diff --git a/repeater/presets/meshmapper.yaml b/repeater/presets/meshmapper.yaml new file mode 100644 index 0000000..c3cc295 --- /dev/null +++ b/repeater/presets/meshmapper.yaml @@ -0,0 +1,33 @@ +# MeshMapper MC2MQTT broker preset. +# +# MeshMapper (https://meshmapper.net) is a community MeshCore visualization +# and analytics platform. Their single ingest broker speaks the standard +# MeshCoreToMQTT (MC2MQTT) protocol with the canonical +# `meshcore/{IATA}/{PUBLIC_KEY}/...` topic structure. Today the operator +# does not differentiate from the LetsMesh format flavor, so this preset +# uses `format: letsmesh` to match the published wire contract; bump to +# a dedicated `meshmapper` format value (and the corresponding entry in +# MC2MQTT_FORMATS in `repeater/data_acquisition/mqtt_handler.py`) only +# when there's a real wire-level deviation to express. +# +# Reference all MeshMapper endpoints with: brokers: [{preset: meshmapper}] +# +# Optional UI metadata. Consumed by the GET /api/broker_presets endpoint +# so the React/Vue admin can render this preset in the "From Template" +# dropdown. `display_name` and `website` are advisory only — they are +# never read by the runtime broker connection code. +display_name: "MeshMapper" +website: "https://meshmapper.net" +brokers: + - name: "MeshMapper" + enabled: true + host: mqtt.meshmapper.cc + port: 443 + transport: "websockets" + audience: "mqtt.meshmapper.cc" + use_jwt_auth: true + format: letsmesh + retain_status: false + tls: + enabled: true + insecure: false diff --git a/repeater/presets/waev.yaml b/repeater/presets/waev.yaml index fe4f7a4..3c61ee1 100644 --- a/repeater/presets/waev.yaml +++ b/repeater/presets/waev.yaml @@ -7,26 +7,29 @@ # and is reserved for future Waev-specific deviations. # # Reference all Waev endpoints with: brokers: [{preset: waev}] +# +# Redundancy model: a single client connection to `mqtt.waev.app`. The +# Waev edge Worker (see waev/src/router.ts: MQTT_PRIMARY_FAILOVER_TIMEOUT_MS) +# fans the WebSocket upgrade to broker A first with a 1500 ms timeout, +# then transparently retries against broker B on failure. Operators who +# want to pin to a single container can edit the broker after import and +# set `host`/`audience` to one of: +# - mqtt-a.waev.app (force primary, no failover) +# - mqtt-b.waev.app (force backup) +# +# Optional UI metadata. Consumed by the GET /api/broker_presets endpoint +# so the React/Vue admin can render this preset in the "From Template" +# dropdown. `display_name` and `website` are advisory only — they are +# never read by the runtime broker connection code. +display_name: "Waev" +website: "https://waev.app" brokers: - - name: "waev-a" + - name: "Waev" enabled: true - host: mqtt-a.waev.app + host: mqtt.waev.app port: 443 transport: "websockets" - audience: "mqtt-a.waev.app" - use_jwt_auth: true - format: waev - retain_status: true - tls: - enabled: true - insecure: false - - - name: "waev-b" - enabled: true - host: mqtt-b.waev.app - port: 443 - transport: "websockets" - audience: "mqtt-b.waev.app" + audience: "mqtt.waev.app" use_jwt_auth: true format: waev retain_status: true diff --git a/repeater/web/api_endpoints.py b/repeater/web/api_endpoints.py index ab40a5c..a3dd603 100644 --- a/repeater/web/api_endpoints.py +++ b/repeater/web/api_endpoints.py @@ -59,6 +59,7 @@ logger = logging.getLogger("HTTPServer") # POST /api/update_advert_rate_limit_config - Update advert rate limiting settings # GET /api/mqtt_status - Get MQTT Observer connection status # POST /api/update_mqtt_config - Update MQTT Observer configuration +# GET /api/broker_presets - List bundled MC2MQTT broker presets (waev, letsmesh, …) # Packets # GET /api/packet_stats?hours=24 - Get packet statistics @@ -1159,6 +1160,60 @@ class APIEndpoints: logger.error(f"Error getting MQTT status: {e}") return self._error(str(e)) + @cherrypy.expose + @cherrypy.tools.json_out() + def broker_presets(self): + """List bundled MC2MQTT broker presets. + + GET /api/broker_presets + + Returns the sorted list of ``repeater/presets/*.yaml`` packaged + with this build, in a UI-ready shape so the admin frontend's + "From Template" dropdown does not need to bundle its own copy + of the broker catalogue. + + Response: + { + "success": true, + "data": [ + { + "id": "waev", # preset filename stem + "name": "Waev", # YAML display_name, or titlecased id + "website": "https://waev.app", # optional, omitted if absent + "brokers": [ ... raw broker dicts from the YAML ... ] + }, + ... + ] + } + + Unauthenticated by design - the response contains only public + broker hostnames and TLS hints, mirroring the access policy on + ``mqtt_status``. + """ + self._set_cors_headers() + try: + # Imported lazily so a broken/missing yaml in the presets + # package never blocks process startup; the loader logs and + # skips bad files. + from repeater.presets import get_preset, list_presets + + data = [] + for preset_id in list_presets(): + preset = get_preset(preset_id) or {} + entry = { + "id": preset_id, + "name": preset.get("display_name") or preset_id.title(), + "brokers": list(preset.get("brokers", [])), + } + website = preset.get("website") + if website: + entry["website"] = website + data.append(entry) + return self._success(data) + except Exception as e: + logger.error(f"Error listing broker presets: {e}") + return self._error(str(e)) + @cherrypy.expose @cherrypy.tools.json_out() @cherrypy.tools.json_in() diff --git a/tests/test_presets.py b/tests/test_presets.py index 4253bff..8f4c7e8 100644 --- a/tests/test_presets.py +++ b/tests/test_presets.py @@ -25,21 +25,30 @@ from repeater.presets import get_preset, list_presets # Preset loader contract # -------------------------------------------------------------------- def test_list_presets_returns_bundled_names(): - """The shipped wheel must contain at least 'waev' and 'letsmesh'.""" + """The shipped wheel must contain the public-network presets.""" names = list_presets() assert "waev" in names assert "letsmesh" in names + assert "meshmapper" in names -def test_get_preset_waev_has_two_brokers(): - """Waev preset shape: top-level 'brokers' list with two MC2MQTT entries.""" +def test_get_preset_waev_uses_alias_for_server_side_failover(): + """Waev preset ships ONE broker pointing at the alias host. + + The Waev edge Worker (waev/src/router.ts: + MQTT_PRIMARY_FAILOVER_TIMEOUT_MS) does server-side A/B failover on + `mqtt.waev.app`. Repeaters connect once and let the Worker handle + redundancy - we explicitly do NOT want to materialize two independent + client connections, because that would defeat the dedup-on-pubkey-hash + contract on the waev ingest side. + """ preset = get_preset("waev") brokers = preset.get("brokers", []) - assert len(brokers) == 2 - for b in brokers: - assert "name" in b - assert "host" in b - assert b.get("format") == "waev" + assert len(brokers) == 1, "Waev preset should be a single alias broker" + broker = brokers[0] + assert broker["host"] == "mqtt.waev.app" + assert broker["audience"] == "mqtt.waev.app" + assert broker.get("format") == "waev" def test_get_preset_unknown_returns_empty_dict(): @@ -47,16 +56,125 @@ def test_get_preset_unknown_returns_empty_dict(): assert get_preset("definitely-not-a-real-preset") == {} +def test_get_preset_waev_carries_ui_metadata(): + """Waev preset exposes top-level display_name + website for the UI. + + These optional top-level fields are consumed by + ``GET /api/broker_presets`` so the admin frontend's "From Template" + dropdown does not need to bundle its own copy of the broker catalogue. + """ + preset = get_preset("waev") + assert preset.get("display_name") == "Waev" + assert preset.get("website") == "https://waev.app" + + +def test_get_preset_letsmesh_carries_ui_metadata(): + """LetsMesh preset exposes the same top-level UI metadata as Waev.""" + preset = get_preset("letsmesh") + assert preset.get("display_name") == "LetsMesh" + assert preset.get("website") == "https://letsmesh.net" + + +def test_get_preset_meshmapper_is_single_broker_mc2mqtt(): + """MeshMapper preset is a single MC2MQTT broker on mqtt.meshmapper.cc. + + The preset intentionally re-uses the `letsmesh` format value because + MeshMapper today speaks the standard MC2MQTT wire format with no + documented deviations. A dedicated `meshmapper` format value can be + introduced later if/when wire-level differentiation lands. + """ + preset = get_preset("meshmapper") + assert preset.get("display_name") == "MeshMapper" + assert preset.get("website") == "https://meshmapper.net" + brokers = preset.get("brokers", []) + assert len(brokers) == 1 + broker = brokers[0] + assert broker["host"] == "mqtt.meshmapper.cc" + assert broker["audience"] == "mqtt.meshmapper.cc" + assert broker.get("format") == "letsmesh" + + +# -------------------------------------------------------------------- +# GET /api/broker_presets - UI-facing shape +# -------------------------------------------------------------------- +class _StubEndpoint: + """Minimal stand-in for APIEndpoints to exercise the broker_presets method. + + The full APIEndpoints.__init__ pulls in ConfigManager, AuthAPIEndpoints, + CompanionAPIEndpoints, UpdateAPIEndpoints, and CADCalibrationEngine, none + of which are relevant to this read-only handler. The stub satisfies just + the four protocol points the method actually touches. + """ + + config = {} + + def _is_cors_enabled(self): + return False + + def _set_cors_headers(self): + pass + + def _success(self, data, **kwargs): + result = {"success": True, "data": data} + result.update(kwargs) + return result + + def _error(self, error): + return {"success": False, "error": str(error)} + + +def _call_broker_presets(): + """Bind the unbound method onto a stub instance and invoke it.""" + from repeater.web.api_endpoints import APIEndpoints + + return APIEndpoints.broker_presets(_StubEndpoint()) + + +def test_broker_presets_returns_success_with_list_payload(): + """Happy path: response wraps a list of preset entries.""" + response = _call_broker_presets() + assert response["success"] is True + assert isinstance(response["data"], list) + # At least waev + letsmesh + meshmapper ship in the public catalogue. + assert len(response["data"]) >= 3 + + +def test_broker_presets_waev_entry_is_ui_ready(): + """Waev entry carries id, display name, website, and a single broker. + + The single broker points at the alias `mqtt.waev.app`, which is where + Waev's edge Worker provides server-side A/B failover. Audience equals + host so JWT verification stays consistent if/when the Worker turns + on aud enforcement. + """ + response = _call_broker_presets() + waev = next(p for p in response["data"] if p["id"] == "waev") + assert waev["name"] == "Waev" + assert waev["website"] == "https://waev.app" + assert len(waev["brokers"]) == 1 + broker = waev["brokers"][0] + assert broker["host"] == "mqtt.waev.app" + assert broker["audience"] == "mqtt.waev.app" + + +def test_broker_presets_letsmesh_entry_is_ui_ready(): + """LetsMesh entry mirrors the Waev contract.""" + response = _call_broker_presets() + letsmesh = next(p for p in response["data"] if p["id"] == "letsmesh") + assert letsmesh["name"] == "LetsMesh" + assert letsmesh["website"] == "https://letsmesh.net" + assert len(letsmesh["brokers"]) == 2 + + # -------------------------------------------------------------------- # Pass 1: preset expansion # -------------------------------------------------------------------- def test_expand_preset_entries_inlines_bundled_brokers(): - """A {preset: waev} entry expands to the two Waev broker dicts.""" + """A {preset: waev} entry expands to the single Waev alias broker.""" expanded = _expand_preset_entries([{"preset": "waev"}]) - assert len(expanded) == 2 - names = [b["name"] for b in expanded] - assert "waev-a" in names - assert "waev-b" in names + assert len(expanded) == 1 + assert expanded[0]["name"] == "Waev" + assert expanded[0]["host"] == "mqtt.waev.app" def test_expand_preset_entries_drops_unknown_preset_with_warning(caplog): @@ -70,29 +188,35 @@ def test_expand_preset_entries_drops_unknown_preset_with_warning(caplog): # -------------------------------------------------------------------- # Pass 2: override-by-name merge # -------------------------------------------------------------------- -def test_merge_overrides_by_name_disables_one_preset_broker(): - """Override AFTER preset wins: documented happy-path.""" +def test_merge_overrides_by_name_pins_waev_to_primary(): + """Override AFTER preset wins: an operator can pin to broker A only. + + Use case: an operator wants to bypass the server-side failover and + target broker A directly (e.g. while debugging a B-specific issue). + They re-point the single Waev broker's host/audience to mqtt-a.waev.app + via an override after the preset expansion. + """ pre_expanded = _expand_preset_entries([{"preset": "waev"}]) - merged = _merge_overrides_by_name(pre_expanded + [{"name": "waev-b", "enabled": False}]) - assert len(merged) == 2 - by_name = {b["name"]: b for b in merged} - assert by_name["waev-a"]["enabled"] is True - assert by_name["waev-b"]["enabled"] is False + merged = _merge_overrides_by_name( + pre_expanded + [{"name": "Waev", "host": "mqtt-a.waev.app", "audience": "mqtt-a.waev.app"}] + ) + assert len(merged) == 1 + assert merged[0]["host"] == "mqtt-a.waev.app" + assert merged[0]["audience"] == "mqtt-a.waev.app" def test_merge_overrides_by_name_later_wins_documented_rule(): """Override BEFORE preset is overwritten - locks the documented rule. The preset-expanded entry comes after the user's override in this case, - so the preset wins and the user's `enabled: False` is silently lost. This + so the preset wins and the user's host override is silently lost. This is the published rule ("place override entries AFTER preset entries"); this test exists so a future refactor can't quietly flip it. """ - user_first = [{"name": "waev-b", "enabled": False}] + user_first = [{"name": "Waev", "host": "mqtt-a.waev.app"}] pipeline = _merge_overrides_by_name(user_first + _expand_preset_entries([{"preset": "waev"}])) - by_name = {b["name"]: b for b in pipeline} - # Preset wins - waev-b is enabled despite the user trying to disable it earlier. - assert by_name["waev-b"]["enabled"] is True + # Preset wins - host is reset to the alias. + assert pipeline[0]["host"] == "mqtt.waev.app" # --------------------------------------------------------------------