diff --git a/.agentmap.yaml b/.agentmap.yaml deleted file mode 100644 index e8347df..0000000 --- a/.agentmap.yaml +++ /dev/null @@ -1,103 +0,0 @@ -# MeshCore Hub — codebase orientation map -# See: https://github.com/anthropics/agentmap - -meta: - project: meshcore-hub - version: 1 - updated: "2026-02-27" - stack: - - python 3.13 - - fastapi - - sqlalchemy (async) - - paho-mqtt - - click - - lit-html SPA - - tailwind + daisyui - - sqlite - -tasks: - install: "pip install -e '.[dev]'" - test: "pytest" - run: "meshcore-hub api --reload" - lint: "pre-commit run --all-files" - -tree: - src/meshcore_hub/: - __main__.py: "Click CLI entry point, registers subcommands" - common/: - config.py: "pydantic-settings, all env vars [config]" - database.py: "async SQLAlchemy session management" - mqtt.py: "MQTT client helpers" - i18n.py: "translation loader, t() function" - models/: - base.py: "Base, UUIDMixin, TimestampMixin" - node.py: null - member.py: null - advertisement.py: null - message.py: null - telemetry.py: null - node_tag.py: null - schemas/: - events.py: "inbound MQTT event schemas" - commands.py: "outbound command schemas" - nodes.py: "API request/response schemas" - members.py: null - messages.py: null - interface/: - receiver.py: "reads device events, publishes to MQTT" - sender.py: "subscribes MQTT commands, writes to device" - device.py: "meshcore library wrapper" - mock_device.py: "fake device for testing" - collector/: - subscriber.py: "MQTT subscriber, routes events to handlers" - handlers/: "per-event-type DB persistence" - cleanup.py: "data retention and node cleanup" - webhook.py: "forward events to HTTP endpoints" - tag_import.py: "seed node tags from YAML" - member_import.py: "seed members from YAML" - api/: - app.py: "FastAPI app factory" - auth.py: "API key authentication" - dependencies.py: "DI for db session and auth" - metrics.py: "Prometheus /metrics endpoint" - routes/: "REST endpoints per resource" - web/: - app.py: "FastAPI app factory, SPA shell" - pages.py: "custom markdown page loader" - middleware.py: null - templates/: - spa.html: "single Jinja2 shell template" - static/js/spa/: - app.js: "SPA entry, route registration" - router.js: "History API client-side router" - api.js: "fetch wrapper for API calls" - components.js: "shared lit-html helpers, t() re-export" - icons.js: "SVG icon functions" - pages/: "lazy-loaded page modules" - alembic/: "DB migrations" - etc/: - prometheus/: "Prometheus scrape + alert rules" - alertmanager/: null - seed/: "YAML seed data (node_tags, members)" - tests/: - -key_symbols: - - src/meshcore_hub/__main__.py::cli — Click root group [entry-point] - - src/meshcore_hub/common/config.py::CommonSettings — shared env config base - - src/meshcore_hub/common/database.py::DatabaseManager — async session factory - - src/meshcore_hub/common/models/base.py::Base — declarative base for all models - - src/meshcore_hub/api/app.py::create_app — API FastAPI factory - - src/meshcore_hub/web/app.py::create_app — Web FastAPI factory - - src/meshcore_hub/api/auth.py::require_read — read-key auth dependency - - src/meshcore_hub/api/auth.py::require_admin — admin-key auth dependency - - src/meshcore_hub/collector/subscriber.py::MQTTSubscriber — event ingestion loop - - src/meshcore_hub/interface/receiver.py::Receiver — device→MQTT bridge - - src/meshcore_hub/interface/sender.py::Sender — MQTT→device bridge - -conventions: - - four Click subcommands: interface, collector, api, web - - "MQTT topic pattern: {prefix}/{pubkey}/event/{name} and .../command/{name}" - - env config via pydantic-settings, no manual os.environ - - web SPA: ES modules + lit-html, pages export async render() - - i18n via t() with JSON locale files in static/locales/ - - node tags are freeform key-value pairs, standard keys in AGENTS.md diff --git a/.env.example b/.env.example index f9e46e0..eadfff8 100644 --- a/.env.example +++ b/.env.example @@ -60,16 +60,24 @@ SEED_HOME=./seed # MQTT SETTINGS # ============================================================================= # MQTT broker connection settings for collector and API services +# Uses the MeshCore MQTT broker (WebSocket transport with subscriber auth) +# See: https://github.com/michaelhart/meshcore-mqtt-broker # MQTT Broker host # When using the local MQTT broker (--profile mqtt), use "mqtt" # When using an external broker, set the hostname/IP MQTT_HOST=mqtt -# MQTT Broker port (default: 1883, or 8883 for TLS) +# MQTT Broker port +# Default: 1883 (local, plain WebSocket) +# Production behind reverse proxy: 8883 (TLS) MQTT_PORT=1883 -# MQTT authentication (optional) +# MQTT subscriber authentication +# The broker uses subscriber accounts with roles: +# Role 1 (admin): full access including /internal topics +# Role 2 (full_access): all public topics, no filtering +# Role 3 (limited): filtered data (SNR, RSSI, etc. removed) MQTT_USERNAME= MQTT_PASSWORD= @@ -77,20 +85,23 @@ MQTT_PASSWORD= MQTT_PREFIX=meshcore # Enable TLS/SSL for MQTT connection -# When enabled, uses TLS with system CA certificates (e.g., for Let's Encrypt) +# Set to true when connecting via wss:// (e.g., behind a reverse proxy) MQTT_TLS=false # MQTT transport protocol -# Options: tcp, websockets -MQTT_TRANSPORT=tcp +# The MeshCore MQTT broker uses WebSockets exclusively +MQTT_TRANSPORT=websockets -# MQTT WebSocket path (used only when MQTT_TRANSPORT=websockets) -# Common values: /mqtt, / -MQTT_WS_PATH=/mqtt +# MQTT WebSocket path (used when MQTT_TRANSPORT=websockets) +# Default: / (broker accepts any path) +# Production: can be set to /mqtt if reverse proxy rewrites paths +MQTT_WS_PATH=/ -# External port mappings for local MQTT broker (--profile mqtt only) -MQTT_EXTERNAL_PORT=1883 -MQTT_WS_PORT=9001 +# JWT audience claim for packet capture authentication tokens +# Must match AUTH_EXPECTED_AUDIENCE on the broker +# Local default: mqtt.localhost +# Production: set to your broker's domain (e.g., mqtt.example.com) +MQTT_TOKEN_AUDIENCE=mqtt.localhost # ============================================================================= # PACKET CAPTURE SETTINGS @@ -149,6 +160,7 @@ PACKETCAPTURE_MQTT2_TOKEN_AUDIENCE=mqtt-eu-v1.letsmesh.net PACKETCAPTURE_MQTT2_KEEPALIVE=120 # Broker 3 - Local MQTT (enabled by default, wired to hub's MQTT_* settings) +# Uses websockets and auth tokens by default (set in docker-compose.yml) PACKETCAPTURE_MQTT3_ENABLED=true PACKETCAPTURE_MQTT3_KEEPALIVE=60 diff --git a/.github/workflows/docker-mqtt-broker.yml b/.github/workflows/docker-mqtt-broker.yml new file mode 100644 index 0000000..0a98a4a --- /dev/null +++ b/.github/workflows/docker-mqtt-broker.yml @@ -0,0 +1,75 @@ +name: Build MQTT Broker Image + +on: + schedule: + - cron: "0 4 * * 0" + workflow_dispatch: + inputs: + ref: + description: "Upstream ref to build (branch, tag, or SHA)" + required: false + default: "main" + +env: + REGISTRY: ghcr.io + UPSTREAM_REPO: michaelhart/meshcore-mqtt-broker + IMAGE_NAME: ipnet-mesh/meshcore-mqtt-broker + +jobs: + build: + name: Build and Push MQTT Broker + runs-on: ubuntu-latest + permissions: + contents: read + packages: write + + steps: + - name: Checkout this repo (sparse) + uses: actions/checkout@v6 + with: + sparse-checkout: | + etc/docker/meshcore-mqtt-broker + + - name: Checkout upstream source + uses: actions/checkout@v6 + with: + repository: ${{ env.UPSTREAM_REPO }} + ref: ${{ inputs.ref || 'main' }} + path: upstream + + - name: Copy Dockerfile into upstream source + run: cp etc/docker/meshcore-mqtt-broker/Dockerfile upstream/Dockerfile + + - name: Set up QEMU + uses: docker/setup-qemu-action@v4 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v4 + + - name: Log in to Container Registry + uses: docker/login-action@v4 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Extract metadata + id: meta + uses: docker/metadata-action@v6 + with: + images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} + tags: | + type=raw,value=latest,enable={{is_default_branch}} + type=sha + + - name: Build and push Docker image + uses: docker/build-push-action@v7 + with: + context: ./upstream + file: ./upstream/Dockerfile + platforms: linux/amd64,linux/arm64 + push: true + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + cache-from: type=gha + cache-to: type=gha,mode=max diff --git a/AGENTS.md b/AGENTS.md index d9c12aa..cb5be95 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -24,25 +24,23 @@ This document provides context and guidelines for AI coding assistants working o ## Project Overview -MeshCore Hub is a Python 3.13+ monorepo for managing and orchestrating MeshCore mesh networks. Data ingestion is done via [meshcore-packet-capture](https://github.com/agessaman/meshcore-packet-capture), which captures MeshCore mesh traffic and publishes events to MQTT. MeshCore Hub then collects, stores, and presents this data. It consists of four main components: +MeshCore Hub is a Python 3.14+ monorepo for managing and orchestrating MeshCore mesh networks. Data ingestion is done via [meshcore-packet-capture](https://github.com/agessaman/meshcore-packet-capture), which captures MeshCore mesh traffic and publishes events to MQTT. MeshCore Hub then collects, stores, and presents this data. It consists of four main components: - **meshcore_collector**: Collects MeshCore events from MQTT and stores them in a database -- **meshcore_api**: REST API for querying data and sending commands via MQTT +- **meshcore_api**: REST API for querying data - **meshcore_web**: Web dashboard for visualizing network status - **meshcore_common**: Shared utilities, models, and configurations ## Key Documentation -- [PROMPT.md](PROMPT.md) - Original project specification and requirements - [SCHEMAS.md](SCHEMAS.md) - MeshCore event JSON schemas and database mappings -- [PLAN.md](PLAN.md) - Implementation plan and architecture decisions -- [TASKS.md](TASKS.md) - Detailed task breakdown with checkboxes for progress tracking +- [UPGRADING.md](UPGRADING.md) - Upgrade guide for breaking changes ## Technology Stack | Category | Technology | |----------|------------| -| Language | Python 3.13+ | +| Language | Python 3.14+ | | Package Management | pip with pyproject.toml | | CLI Framework | Click | | Configuration | Pydantic Settings | @@ -50,6 +48,7 @@ MeshCore Hub is a Python 3.13+ monorepo for managing and orchestrating MeshCore | Migrations | Alembic | | REST API | FastAPI | | MQTT Client | paho-mqtt | +| MQTT Broker | [meshcore-mqtt-broker](https://github.com/michaelhart/meshcore-mqtt-broker) (WebSocket + JWT auth) | | Templates | Jinja2 (server), lit-html (SPA) | | Frontend | ES Modules SPA with client-side routing | | CSS Framework | Tailwind CSS + DaisyUI | @@ -265,6 +264,8 @@ meshcore-hub/ │ │ ├── cli.py # Collector CLI with seed commands │ │ ├── subscriber.py # MQTT subscriber │ │ ├── cleanup.py # Data retention/cleanup service +│ │ ├── letsmesh_decoder.py # Native Python packet decoder +│ │ ├── letsmesh_normalizer.py # LetsMesh upload topic normalizer │ │ ├── tag_import.py # Tag import from YAML │ │ ├── member_import.py # Member import from YAML │ │ ├── handlers/ # Event handlers @@ -304,7 +305,6 @@ meshcore-hub/ │ ├── env.py │ └── versions/ ├── etc/ -│ ├── mosquitto.conf # MQTT broker configuration │ ├── prometheus/ # Prometheus configuration │ │ ├── prometheus.yml # Scrape and alerting config │ │ └── alerts.yml # Alert rules @@ -329,15 +329,25 @@ meshcore-hub/ ## MQTT Topic Structure -### Events +The MQTT broker ([meshcore-mqtt-broker](https://github.com/michaelhart/meshcore-mqtt-broker)) uses WebSocket transport with MeshCore public key authentication for publishers and subscriber accounts for consumers. + +### Upload Topics (published by packet capture) ``` -//event/ +/// ``` Examples: -- `meshcore/abc123.../event/advertisement` -- `meshcore/abc123.../event/contact_msg_recv` -- `meshcore/abc123.../event/channel_msg_recv` +- `meshcore/STN/abc123.../packets` +- `meshcore/STN/abc123.../status` +- `meshcore/STN/abc123.../internal` + +The `` segment is a 3-letter airport code (e.g., `STN`, `SEA`) or `test`, validated by the MQTT broker. The hub ignores this segment during parsing. + +### Subscriber Subscriptions +The collector subscribes to: +- `{prefix}/+/+/packets` +- `{prefix}/+/+/status` +- `{prefix}/+/+/internal` ## Database Conventions @@ -579,7 +589,10 @@ Key variables: - `SEED_HOME` - Directory containing seed data files (default: `./seed`) - `CONTENT_HOME` - Directory containing custom content (pages, media) (default: `./content`) - `MQTT_HOST`, `MQTT_PORT`, `MQTT_PREFIX` - MQTT broker connection -- `MQTT_TLS` - Enable TLS/SSL for MQTT (default: `false`) +- `MQTT_TRANSPORT` - MQTT transport protocol (default: `websockets`) +- `MQTT_WS_PATH` - WebSocket path (default: `/`) +- `MQTT_TLS` - Enable TLS/SSL for MQTT (default: `false`, set `true` for `wss://`) +- `MQTT_TOKEN_AUDIENCE` - JWT audience claim for packet capture auth tokens (default: `mqtt.localhost`) - `API_READ_KEY`, `API_ADMIN_KEY` - API authentication keys - `WEB_ADMIN_ENABLED` - Enable admin interface at /a/ (default: `false`, requires auth proxy) - `WEB_TRUSTED_PROXY_HOSTS` - Comma-separated list of trusted proxy hosts for admin authentication headers. Default: `*` (all hosts). Recommended: set to your reverse proxy IP in production. A startup warning is emitted when using the default `*` with admin enabled. diff --git a/CLAUDE.md b/CLAUDE.md deleted file mode 100644 index 48b2af0..0000000 --- a/CLAUDE.md +++ /dev/null @@ -1,5 +0,0 @@ -# Claude Code Instructions - -Refer to @AGENTS.md for coding instructions. - -DO NOT MODIFY "CLAUDE.md", UPDATE "AGENTS.md". diff --git a/README.md b/README.md index 72831b1..dd7d623 100644 --- a/README.md +++ b/README.md @@ -5,10 +5,13 @@ [![codecov](https://codecov.io/github/ipnet-mesh/meshcore-hub/graph/badge.svg?token=DO4F82DLKS)](https://codecov.io/github/ipnet-mesh/meshcore-hub) [![BuyMeACoffee](https://raw.githubusercontent.com/pachadotdev/buymeacoffee-badges/main/bmc-donate-yellow.svg)](https://www.buymeacoffee.com/jinglemansweep) -Python 3.13+ platform for managing and orchestrating MeshCore mesh networks. +Python 3.14+ platform for managing and orchestrating MeshCore mesh networks. ![MeshCore Hub Web Dashboard](docs/images/web.png) +> [!WARNING] +> **Breaking Changes** — The latest release replaces Mosquitto with a JWT-based MQTT broker, removes the proprietary receiver service in favor of [meshcore-packet-capture](https://github.com/agessaman/meshcore-packet-capture), and renames `receiver_node_id` to `observer_node_id` in the database. If upgrading from a previous version, see [UPGRADING.md](UPGRADING.md) for migration steps. + > [!IMPORTANT] > **Help Translate MeshCore Hub** 🌍 > @@ -21,7 +24,7 @@ MeshCore Hub provides a complete solution for monitoring, collecting, and intera | Component | Description | |-----------|-------------| | **Collector** | Subscribes to MQTT events and persists them to a database | -| **API** | REST API for querying data and sending commands to the network | +| **API** | REST API for querying data | | **Web Dashboard** | Single Page Application (SPA) for visualizing network status | ## Architecture @@ -63,7 +66,6 @@ flowchart LR - **Event Persistence**: Store messages, advertisements, telemetry, and trace data - **REST API**: Query historical data with filtering and pagination -- **Command Dispatch**: Send messages and advertisements via the API - **Node Tagging**: Add custom metadata to nodes for organization - **Web Dashboard**: Visualize network status, node locations, and message history - **Internationalization**: Full i18n support with composable translation patterns @@ -111,7 +113,7 @@ Docker Compose uses **profiles** to select which services to run: |---------|----------|----------| | `all` | db-migrate, collector, api, web | Everything on one host | | `core` | db-migrate, collector, api, web | Central server infrastructure | -| `mqtt` | mosquitto broker | Local MQTT broker (optional) | +| `mqtt` | meshcore-mqtt-broker | Local MQTT broker (optional) | | `receiver` | packet capture observer | Observes RF traffic and publishes to MQTT | | `metrics` | prometheus, alertmanager | Prometheus metrics and alerting | | `seed` | seed | One-time seed data import | @@ -178,8 +180,8 @@ All components are configured via environment variables. Create a `.env` file or | `MQTT_PASSWORD` | *(none)* | MQTT password (optional) | | `MQTT_PREFIX` | `meshcore` | Topic prefix for all MQTT messages | | `MQTT_TLS` | `false` | Enable TLS/SSL for MQTT connection | -| `MQTT_TRANSPORT` | `tcp` | MQTT transport (`tcp` or `websockets`) | -| `MQTT_WS_PATH` | `/mqtt` | MQTT WebSocket path (used when `MQTT_TRANSPORT=websockets`) | +| `MQTT_TRANSPORT` | `websockets` | MQTT transport (`tcp` or `websockets`) | +| `MQTT_WS_PATH` | `/` | MQTT WebSocket path (used when `MQTT_TRANSPORT=websockets`) | ### Collector Settings @@ -191,9 +193,9 @@ All components are configured via environment variables. Create a `.env` file or The collector subscribes to packets published by [meshcore-packet-capture](https://github.com/agessaman/meshcore-packet-capture): -- `/+/packets` -- `/+/status` -- `/+/internal` +- `/+/+/packets` +- `/+/+/status` +- `/+/+/internal` Normalization behavior: @@ -261,7 +263,7 @@ The collector automatically cleans up old event data and inactive nodes: | `API_HOST` | `0.0.0.0` | API bind address | | `API_PORT` | `8000` | API port | | `API_READ_KEY` | *(none)* | Read-only API key | -| `API_ADMIN_KEY` | *(none)* | Admin API key (required for commands) | +| `API_ADMIN_KEY` | *(none)* | Admin API key | | `METRICS_ENABLED` | `true` | Enable Prometheus metrics endpoint at `/metrics` | | `METRICS_CACHE_TTL` | `60` | Seconds to cache metrics output (reduces database load) | @@ -537,12 +539,8 @@ The API supports optional bearer token authentication: # Read-only access curl -H "Authorization: Bearer " http://localhost:8000/api/v1/nodes -# Admin access (required for commands) -curl -X POST \ - -H "Authorization: Bearer " \ - -H "Content-Type: application/json" \ - -d '{"destination": "abc123...", "text": "Hello!"}' \ - http://localhost:8000/api/v1/commands/send-message +# Admin access +curl -H "Authorization: Bearer " http://localhost:8000/api/v1/members ``` ### Example Endpoints @@ -559,9 +557,6 @@ curl -X POST \ | GET | `/api/v1/telemetry` | List telemetry data | | GET | `/api/v1/trace-paths` | List trace paths | | GET | `/api/v1/members` | List network members | -| POST | `/api/v1/commands/send-message` | Send direct message | -| POST | `/api/v1/commands/send-channel-message` | Send channel message | -| POST | `/api/v1/commands/send-advertisement` | Send advertisement | | GET | `/api/v1/dashboard/stats` | Get network statistics | | GET | `/api/v1/dashboard/activity` | Get daily advertisement activity | | GET | `/api/v1/dashboard/message-activity` | Get daily message activity | diff --git a/UPGRADING.md b/UPGRADING.md new file mode 100644 index 0000000..d0505d1 --- /dev/null +++ b/UPGRADING.md @@ -0,0 +1,253 @@ +# Upgrading MeshCore Hub + +This guide covers upgrading from a previous MeshCore Hub release to the current version. The latest release includes **breaking changes** to the MQTT broker, packet capture service, and data ingestion pipeline. + +## Overview of Changes + +| Area | Before | After | +|------|--------|-------| +| MQTT broker | Eclipse Mosquitto (TCP) | [meshcore-mqtt-broker](https://github.com/michaelhart/meshcore-mqtt-broker) (WebSocket, JWT auth) | +| Packet capture | Proprietary `interface-receiver` service | [meshcore-packet-capture](https://github.com/agessaman/meshcore-packet-capture) (LetsMesh Observer model) | +| Auth model | MQTT username/password for publishing | JWT signed by device hardware public key | +| Collector MQTT | Anonymous subscriber | Subscriber account (admin-level) with credentials | +| Decoder | Node.js `meshcore-decoder` CLI subprocess | Native Python `meshcoredecoder` library | +| Python | 3.13 | 3.14 | +| DB columns | `receiver_node_id` | `observer_node_id` | +| DB table | `event_receivers` | `event_observers` | +| API commands | `/api/v1/commands/*` | Removed | +| Compose profiles | `receiver`, `sender`, `mock` | `receiver` (packet-capture) | + +## Step 1: Backup the Database + +**Do not skip this step.** The database migration renames columns and tables, and while it has been tested, you should always have a backup. + +```bash +# Create a timestamped backup of the database volume +docker run --rm \ + -v meshcore_hub_data:/data \ + -v $(pwd):/backup \ + alpine tar czf /backup/meshcore-hub-db-$(date +%Y%m%d-%H%M%S).tar.gz -C / data + +# Verify the backup was created +ls -lh meshcore-hub-db-*.tar.gz +``` + +To restore from backup if needed: + +```bash +docker run --rm \ + -v meshcore_hub_data:/data \ + -v $(pwd):/backup \ + alpine sh -c "cd / && tar xzf /backup/meshcore-hub-db-YYYYMMDD-HHMMSS.tar.gz" +``` + +## Step 2: Stop and Remove Containers + +Stop all services and remove orphaned containers from the old configuration: + +```bash +docker compose down --remove-orphans +``` + +> **Important:** Do NOT use `--volumes` / `-v`. That would delete your database. The `--remove-orphans` flag cleans up old services (like `interface-receiver`, `interface-sender`) that no longer exist in the new compose file. + +## Step 3: Update Configuration Files + +Download the latest configuration files: + +```bash +# Download the new docker-compose.yml +wget -O docker-compose.yml https://raw.githubusercontent.com/ipnet-mesh/meshcore-hub/main/docker-compose.yml + +# Download the new .env.example for reference +wget -O .env.example https://raw.githubusercontent.com/ipnet-mesh/meshcore-hub/main/.env.example +``` + +Then compare your existing `.env` against the new `.env.example` and update it (see Step 4). + +## Step 4: Migrate Your `.env` File + +### Variables to Remove + +These variables no longer exist and should be removed from your `.env`: + +```bash +# Removed: ingest mode is now always LetsMesh upload +COLLECTOR_INGEST_MODE=native + +# Removed: decoder is now a native Python library, always enabled +COLLECTOR_LETSMESH_DECODER_ENABLED=true +COLLECTOR_LETSMESH_DECODER_COMMAND=meshcore-decoder +COLLECTOR_LETSMESH_DECODER_TIMEOUT_SECONDS=2.0 + +# Removed: serial baud is handled by meshcore-packet-capture +SERIAL_BAUD=115200 + +# Removed: sender service no longer exists +SERIAL_PORT_SENDER=/dev/ttyUSB1 +NODE_ADDRESS_SENDER= + +# Removed: device name/address now handled by meshcore-packet-capture +MESHCORE_DEVICE_NAME= +NODE_ADDRESS= + +# Removed: contact cleanup was specific to the proprietary receiver +CONTACT_CLEANUP_ENABLED=true +CONTACT_CLEANUP_DAYS=7 + +# Removed: Mosquitto-specific ports +MQTT_EXTERNAL_PORT=1883 +MQTT_WS_PORT=9001 +``` + +### Variables to Update + +| Variable | Old Value | New Value | Notes | +|----------|-----------|-----------|-------| +| `MQTT_TRANSPORT` | `tcp` | `websockets` | Required by the new JWT-based broker | +| `MQTT_WS_PATH` | `/mqtt` | `/` | New broker accepts connections on `/` | + +### Variables to Add + +```bash +# MQTT subscriber authentication for the collector +# The collector connects as a subscriber to read all published topics +# including /internal. Set these to match your broker's SUBSCRIBER_1 config. +MQTT_USERNAME=subscriber + +# Generate a secure password (do not use a simple password in production): +# openssl rand -base64 32 +MQTT_PASSWORD= + +# JWT audience claim for packet capture authentication tokens +# Must match AUTH_EXPECTED_AUDIENCE on the broker +MQTT_TOKEN_AUDIENCE=mqtt.localhost + +# IATA airport code for your observer location (required for packet capture) +# Use the 3-letter code for the nearest airport. +# Look up your code: https://www.iata.org/en/publications/directories/code-search/ +PACKETCAPTURE_IATA=LOC +``` + +All other `PACKETCAPTURE_*` variables have sensible defaults in `docker-compose.yml` and only need to be set in `.env` if you want to override them. See `.env.example` for the full list. + +## Step 5: Run Database Migration + +The migration renames `receiver_node_id` → `observer_node_id` across all event tables and `event_receivers` → `event_observers`: + +```bash +docker compose --profile core run --rm db-migrate +``` + +This runs automatically as part of the `core` profile, but can also be run standalone with the `migrate` profile: + +```bash +docker compose --profile migrate run --rm db-migrate +``` + +## Step 6: Start Services + +### With local MQTT broker (single-host deployment) + +```bash +# Start everything including the MQTT broker +docker compose --profile mqtt --profile core up -d + +# Or include packet capture on the same host +docker compose --profile mqtt --profile core --profile receiver up -d +``` + +### With external MQTT broker + +```bash +# Start core services only (broker runs elsewhere) +docker compose --profile core up -d +``` + +### Verify + +```bash +# Check all containers are running +docker compose ps + +# Check collector connected to MQTT +docker compose logs collector | grep -i "connected to mqtt" + +# Check the web dashboard +open http://localhost:8080 +``` + +## Notes + +### JWT-Based Packet Capture Authentication + +The new packet capture service ([meshcore-packet-capture](https://github.com/agessaman/meshcore-packet-capture)) uses the LetsMesh Observer model: + +- **No custom MQTT credentials needed for publishing.** Authentication is handled via JWT tokens signed by the capture device's hardware public key. The MQTT broker validates the JWT and authorizes publishing automatically. +- The collector connects as a **subscriber** to read all published events, including `/internal` topics. Configure `MQTT_USERNAME` and `MQTT_PASSWORD` to match the broker's subscriber account. + +### Production MQTT Configuration + +In production, the MQTT WebSocket server should be hosted behind a TLS/SSL-terminated reverse proxy (e.g., Nginx Proxy Manager, Caddy, Traefik) under the `/mqtt` path. The proxy handles TLS termination and forwards plain WebSocket connections to the broker on port 1883. + +**Local / development (default):** +```bash +MQTT_PORT=1883 +MQTT_TRANSPORT=websockets +MQTT_WS_PATH=/ +MQTT_TLS=false +MQTT_TOKEN_AUDIENCE=mqtt.localhost +``` + +**Production (behind reverse proxy):** +```bash +MQTT_PORT=443 +MQTT_TRANSPORT=websockets +MQTT_WS_PATH=/mqtt +MQTT_TLS=true +MQTT_TOKEN_AUDIENCE=mqtt.example.com # your public domain +``` + +### Existing LetsMesh Observer Installs + +If you already run [meshcore-packet-capture](https://github.com/agessaman/meshcore-packet-capture) separately, configure **MQTT server #3** to point at your MeshCore Hub MQTT broker. Servers #1 and #2 are reserved for Let's Mesh US (`mqtt-us-v1.letsmesh.net`) and Let's Mesh EU (`mqtt-eu-v1.letsmesh.net`) respectively. + +```bash +# In your packet-capture .env or docker-compose environment: +PACKETCAPTURE_MQTT3_ENABLED=true +PACKETCAPTURE_MQTT3_SERVER=your-meshcore-hub-host +PACKETCAPTURE_MQTT3_PORT=1883 +PACKETCAPTURE_MQTT3_TRANSPORT=websockets +PACKETCAPTURE_MQTT3_USE_TLS=false +PACKETCAPTURE_MQTT3_USE_AUTH_TOKEN=true +PACKETCAPTURE_MQTT3_TOKEN_AUDIENCE=mqtt.localhost +``` + +### Removed Services + +The following Docker Compose services have been removed: + +| Old Service | Replacement | +|-------------|-------------| +| `interface-receiver` | `packet-capture` (profile: `receiver`) | +| `interface-sender` | None (removed) | +| `interface-mock-receiver` | None (removed) | + +The `packet-capture` service uses the [meshcore-packet-capture](https://github.com/agessaman/meshcore-packet-capture) image and is included in `docker-compose.yml` under the `receiver` profile for an easy transition. + +### Removed API Endpoints + +The command dispatch API endpoints have been removed: + +- `POST /api/v1/commands/send-message` +- `POST /api/v1/commands/send-channel-message` +- `POST /api/v1/commands/send-advertisement` + +### Native Python Decoder + +The Node.js `meshcore-decoder` CLI tool has been replaced by the native Python `meshcoredecoder` library. This means: + +- No Node.js runtime is needed in the Docker image +- The decoder is always enabled (no toggle) +- The `COLLECTOR_LETSMESH_DECODER_*` configuration variables have been removed +- `COLLECTOR_LETSMESH_DECODER_KEYS` is still supported for providing additional channel decryption keys diff --git a/docker-compose.yml b/docker-compose.yml index 951172f..f0eaca2 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,35 +1,55 @@ services: # ========================================================================== - # MQTT Broker - Eclipse Mosquitto (optional, use --profile mqtt) + # MQTT Broker - MeshCore MQTT Broker (optional, use --profile mqtt) + # WebSocket-only broker with MeshCore public key authentication # Most users will connect to an external MQTT broker instead + # See: https://github.com/michaelhart/meshcore-mqtt-broker # ========================================================================== mqtt: - image: eclipse-mosquitto:2 + image: ghcr.io/ipnet-mesh/meshcore-mqtt-broker:latest container_name: meshcore-mqtt profiles: - all - mqtt restart: unless-stopped ports: - - "${MQTT_EXTERNAL_PORT:-1883}:1883" - - "${MQTT_WS_PORT:-9001}:9001" + - "${MQTT_PORT:-1883}:${MQTT_PORT:-1883}" volumes: - # - ./etc/mosquitto.conf:/mosquitto/config/mosquitto.conf:ro - - mosquitto_data:/mosquitto/data - - mosquitto_log:/mosquitto/log + - mqtt_broker_data:/data + environment: + # Broker listener + - MQTT_WS_PORT=${MQTT_PORT:-1883} + - MQTT_HOST=0.0.0.0 + # JWT audience validation + - AUTH_EXPECTED_AUDIENCE=${MQTT_TOKEN_AUDIENCE:-mqtt.localhost} + # Subscriber accounts (hub connects as role 1 = admin for /internal access) + - SUBSCRIBER_MAX_CONNECTIONS_DEFAULT=5 + - SUBSCRIBER_1=${MQTT_USERNAME:-admin}:${MQTT_PASSWORD:-admin}:1 + # Abuse detection + - ABUSE_ENFORCEMENT_ENABLED=false + - ABUSE_DUPLICATE_WINDOW_SIZE=100 + - ABUSE_DUPLICATE_WINDOW_MS=300000 + - ABUSE_DUPLICATE_THRESHOLD=10 + - ABUSE_MAX_DUPLICATES_PER_PACKET=5 + - ABUSE_DUPLICATE_RATE_THRESHOLD=0.3 + - ABUSE_DUPLICATE_RATE_WINDOW_MS=300000 + - ABUSE_BUCKET_CAPACITY=20 + - ABUSE_BUCKET_REFILL_RATE=3 + - ABUSE_MAX_PACKET_SIZE=255 + - ABUSE_MAX_TOPICS_PER_DAY=3 + - ABUSE_ANOMALY_THRESHOLD=10 + - ABUSE_MAX_IATA_CHANGES_24H=3 + - ABUSE_TOPIC_HISTORY_SIZE=50 + - ABUSE_TOPIC_HISTORY_WINDOW_MS=86400000 + - ABUSE_PERSISTENCE_PATH=/data/abuse-detection.db + - ABUSE_PERSISTENCE_INTERVAL_MS=300000 healthcheck: test: [ "CMD", - "mosquitto_sub", - "-t", - "$$SYS/#", - "-C", - "1", - "-i", - "healthcheck", - "-W", - "3", + "node", + "-e", + "const net=require('net');const s=net.createConnection(process.env.MQTT_WS_PORT||1883,'127.0.0.1',()=>{s.end();process.exit(0)});s.on('error',()=>process.exit(1));setTimeout(()=>process.exit(1),3000)", ] interval: 30s timeout: 10s @@ -49,6 +69,9 @@ services: profiles: - all - receiver + depends_on: + mqtt: + condition: service_healthy restart: unless-stopped devices: - "${SERIAL_PORT:-/dev/ttyUSB0}:${SERIAL_PORT:-/dev/ttyUSB0}" @@ -91,11 +114,14 @@ services: - PACKETCAPTURE_MQTT3_PORT=${MQTT_PORT} - PACKETCAPTURE_MQTT3_USERNAME=${MQTT_USERNAME:-} - PACKETCAPTURE_MQTT3_PASSWORD=${MQTT_PASSWORD:-} + - PACKETCAPTURE_MQTT3_TRANSPORT=websockets - PACKETCAPTURE_MQTT3_USE_TLS=${MQTT_TLS:-false} + - PACKETCAPTURE_MQTT3_USE_AUTH_TOKEN=true + - PACKETCAPTURE_MQTT3_TOKEN_AUDIENCE=${MQTT_TOKEN_AUDIENCE:-mqtt.localhost} - PACKETCAPTURE_MQTT3_KEEPALIVE=${PACKETCAPTURE_MQTT3_KEEPALIVE:-60} - # Topics match collector's expected // format - - PACKETCAPTURE_MQTT3_TOPIC_STATUS=meshcore/{PUBLIC_KEY}/status - - PACKETCAPTURE_MQTT3_TOPIC_PACKETS=meshcore/{PUBLIC_KEY}/packets + # Topics match broker's /// format + - PACKETCAPTURE_MQTT3_TOPIC_STATUS=meshcore/{IATA}/{PUBLIC_KEY}/status + - PACKETCAPTURE_MQTT3_TOPIC_PACKETS=meshcore/{IATA}/{PUBLIC_KEY}/packets # MQTT reconnection - PACKETCAPTURE_MAX_MQTT_RETRIES=${PACKETCAPTURE_MAX_MQTT_RETRIES:-5} - PACKETCAPTURE_MQTT_RETRY_DELAY=${PACKETCAPTURE_MQTT_RETRY_DELAY:-5} @@ -115,10 +141,12 @@ services: profiles: - all - core - restart: unless-stopped depends_on: db-migrate: condition: service_completed_successfully + mqtt: + condition: service_healthy + restart: unless-stopped volumes: - hub_data:/data - ${SEED_HOME:-./seed}:/seed @@ -130,8 +158,8 @@ services: - MQTT_PASSWORD=${MQTT_PASSWORD:-} - MQTT_PREFIX=${MQTT_PREFIX:-meshcore} - MQTT_TLS=${MQTT_TLS:-false} - - MQTT_TRANSPORT=${MQTT_TRANSPORT:-tcp} - - MQTT_WS_PATH=${MQTT_WS_PATH:-/mqtt} + - MQTT_TRANSPORT=${MQTT_TRANSPORT:-websockets} + - MQTT_WS_PATH=${MQTT_WS_PATH:-/} - COLLECTOR_LETSMESH_DECODER_KEYS=${COLLECTOR_LETSMESH_DECODER_KEYS:-} - DATA_HOME=/data - SEED_HOME=/seed @@ -191,8 +219,8 @@ services: - MQTT_PASSWORD=${MQTT_PASSWORD:-} - MQTT_PREFIX=${MQTT_PREFIX:-meshcore} - MQTT_TLS=${MQTT_TLS:-false} - - MQTT_TRANSPORT=${MQTT_TRANSPORT:-tcp} - - MQTT_WS_PATH=${MQTT_WS_PATH:-/mqtt} + - MQTT_TRANSPORT=${MQTT_TRANSPORT:-websockets} + - MQTT_WS_PATH=${MQTT_WS_PATH:-/} - DATA_HOME=/data - API_HOST=0.0.0.0 - API_PORT=8000 @@ -373,10 +401,8 @@ services: volumes: hub_data: name: meshcore_hub_data - mosquitto_data: - name: meshcore_mosquitto_data - mosquitto_log: - name: meshcore_mosquitto_log + mqtt_broker_data: + name: meshcore_mqtt_broker_data prometheus_data: name: meshcore_prometheus_data alertmanager_data: diff --git a/etc/docker/meshcore-mqtt-broker/Dockerfile b/etc/docker/meshcore-mqtt-broker/Dockerfile new file mode 100644 index 0000000..dd4189d --- /dev/null +++ b/etc/docker/meshcore-mqtt-broker/Dockerfile @@ -0,0 +1,16 @@ +FROM node:22-alpine + +RUN apk add --no-cache python3 make g++ + +WORKDIR /app + +COPY package*.json ./ +RUN npm ci + +COPY . . + +RUN mkdir -p /data + +EXPOSE 1883 + +CMD ["npx", "tsx", "src/server.ts"] diff --git a/etc/docker/meshcore-mqtt-broker/build.sh b/etc/docker/meshcore-mqtt-broker/build.sh new file mode 100755 index 0000000..6481e02 --- /dev/null +++ b/etc/docker/meshcore-mqtt-broker/build.sh @@ -0,0 +1,45 @@ +#!/usr/bin/env bash +set -euo pipefail + +UPSTREAM_REPO="michaelhart/meshcore-mqtt-broker" +UPSTREAM_REF="main" +IMAGE_NAME="ghcr.io/ipnet-mesh/meshcore-mqtt-broker" +PLATFORM="linux/amd64,linux/arm64" +WORKDIR="$(mktemp -d)" +BUILD_ARGS=() + +cleanup() { rm -rf "$WORKDIR"; } +trap cleanup EXIT + +while [[ $# -gt 0 ]]; do + case "$1" in + --ref) + UPSTREAM_REF="$2" + shift 2 + ;; + --platform) + PLATFORM="$2" + shift 2 + ;; + *) + BUILD_ARGS+=("$1") + shift + ;; + esac +done + +echo "==> Cloning ${UPSTREAM_REPO} @ ${UPSTREAM_REF} into ${WORKDIR}" +git clone --depth 1 --branch "${UPSTREAM_REF}" "https://github.com/${UPSTREAM_REPO}.git" "${WORKDIR}/source" + +echo "==> Copying custom Dockerfile" +cp "$(dirname "$0")/Dockerfile" "${WORKDIR}/source/Dockerfile" + +echo "==> Building ${IMAGE_NAME}:latest" +docker buildx build \ + --platform "${PLATFORM}" \ + -t "${IMAGE_NAME}:latest" \ + -t "${IMAGE_NAME}:sha-$(git -C "${WORKDIR}/source" rev-parse --short HEAD)" \ + "${WORKDIR}/source" \ + "${BUILD_ARGS[@]}" + +echo "==> Done" diff --git a/etc/mosquitto.conf b/etc/mosquitto.conf deleted file mode 100644 index 5ceba1f..0000000 --- a/etc/mosquitto.conf +++ /dev/null @@ -1,94 +0,0 @@ -# MeshCore Hub - Mosquitto MQTT Broker Configuration -# Eclipse Mosquitto 2.x configuration file - -# ============================================================================= -# General Settings -# ============================================================================= - -# Persistence for retained messages and subscriptions -persistence true -persistence_location /mosquitto/data/ - -# Logging configuration -log_dest file /mosquitto/log/mosquitto.log -log_dest stdout -log_type error -log_type warning -log_type notice -log_type information -log_timestamp true -log_timestamp_format %Y-%m-%dT%H:%M:%S - -# Connection messages -connection_messages true - -# ============================================================================= -# Default Listener (MQTT over TCP) -# ============================================================================= - -# Listen on all interfaces, port 1883 -listener 1883 -protocol mqtt - -# Allow anonymous connections (for development/testing) -# For production, set to false and configure authentication -allow_anonymous true - -# ============================================================================= -# WebSocket Listener (optional, for browser clients) -# ============================================================================= - -listener 9001 -protocol websockets - -# ============================================================================= -# Security Settings -# ============================================================================= - -# Maximum packet size (default 268435455 bytes) -# Uncomment to limit: -# message_size_limit 1048576 - -# Maximum number of client connections (0 = unlimited) -max_connections 100 - -# Maximum queued messages per client -max_queued_messages 1000 - -# Maximum number of QoS 1 and 2 messages in flight -max_inflight_messages 20 - -# ============================================================================= -# Authentication (uncomment for production) -# ============================================================================= - -# Password file authentication -# password_file /mosquitto/config/passwd - -# To create password file: -# mosquitto_passwd -c /mosquitto/config/passwd username - -# ============================================================================= -# Access Control (uncomment for production) -# ============================================================================= - -# ACL file for topic-level access control -# acl_file /mosquitto/config/acl - -# Example ACL file content: -# user readonly -# topic read meshcore/# -# -# user admin -# topic readwrite meshcore/# - -# ============================================================================= -# TLS/SSL (uncomment for production) -# ============================================================================= - -# listener 8883 -# protocol mqtt -# cafile /mosquitto/config/ca.crt -# certfile /mosquitto/config/server.crt -# keyfile /mosquitto/config/server.key -# require_certificate false diff --git a/src/meshcore_hub/collector/subscriber.py b/src/meshcore_hub/collector/subscriber.py index 2424a1a..d275b5e 100644 --- a/src/meshcore_hub/collector/subscriber.py +++ b/src/meshcore_hub/collector/subscriber.py @@ -364,22 +364,41 @@ class Subscriber(LetsMeshNormalizer): logger.error(f"Failed to connect to database: {e}") raise - # Connect to MQTT broker - try: - self.mqtt.connect() - self.mqtt.start_background() - self._mqtt_connected = True - logger.info("Connected to MQTT broker") - except Exception as e: - self._mqtt_connected = False - logger.error(f"Failed to connect to MQTT broker: {e}") - raise + # Connect to MQTT broker with retry + max_retries = 10 + retry_delay = 2.0 + for attempt in range(1, max_retries + 1): + try: + self.mqtt.connect() + self.mqtt.start_background() + self._mqtt_connected = True + logger.info("Connected to MQTT broker") + break + except Exception as e: + self._mqtt_connected = False + if attempt < max_retries: + logger.warning( + "MQTT connection attempt %d/%d failed: %s. Retrying in %.1fs...", + attempt, + max_retries, + e, + retry_delay, + ) + time.sleep(retry_delay) + retry_delay = min(retry_delay * 2, 30.0) + else: + logger.error( + "Failed to connect to MQTT broker after %d attempts: %s", + max_retries, + e, + ) + raise # Subscribe to LetsMesh upload topics letsmesh_topics = [ - f"{self.mqtt.topic_builder.prefix}/+/packets", - f"{self.mqtt.topic_builder.prefix}/+/status", - f"{self.mqtt.topic_builder.prefix}/+/internal", + f"{self.mqtt.topic_builder.prefix}/+/+/packets", + f"{self.mqtt.topic_builder.prefix}/+/+/status", + f"{self.mqtt.topic_builder.prefix}/+/+/internal", ] for letsmesh_topic in letsmesh_topics: self.mqtt.subscribe(letsmesh_topic, self._handle_mqtt_message) diff --git a/src/meshcore_hub/common/mqtt.py b/src/meshcore_hub/common/mqtt.py index 532df52..fcb9794 100644 --- a/src/meshcore_hub/common/mqtt.py +++ b/src/meshcore_hub/common/mqtt.py @@ -131,17 +131,17 @@ class TopicBuilder: """Parse a LetsMesh upload topic to extract public key and feed type. LetsMesh upload topics are expected in this form: - //(packets|status|internal) + ///(packets|status|internal) """ parts = [part for part in topic.strip("/").split("/") if part] prefix_parts = self._prefix_parts() prefix_len = len(prefix_parts) - if len(parts) != prefix_len + 2 or parts[:prefix_len] != prefix_parts: + if len(parts) != prefix_len + 3 or parts[:prefix_len] != prefix_parts: return None - public_key = parts[prefix_len] - feed_type = parts[prefix_len + 1] + public_key = parts[prefix_len + 1] + feed_type = parts[prefix_len + 2] if feed_type not in {"packets", "status", "internal"}: return None @@ -399,6 +399,8 @@ def create_mqtt_client( prefix: str = "meshcore", client_id: Optional[str] = None, tls: bool = False, + transport: str = "tcp", + ws_path: str = "/mqtt", ) -> MQTTClient: """Create and configure an MQTT client. @@ -410,6 +412,8 @@ def create_mqtt_client( prefix: Topic prefix client_id: Client identifier (optional) tls: Enable TLS/SSL connection (optional) + transport: Transport protocol (tcp or websockets) + ws_path: WebSocket path (used when transport=websockets) Returns: Configured MQTTClient instance @@ -422,5 +426,7 @@ def create_mqtt_client( prefix=prefix, client_id=client_id, tls=tls, + transport=transport, + ws_path=ws_path, ) return MQTTClient(config) diff --git a/tests/e2e/docker-compose.test.yml b/tests/e2e/docker-compose.test.yml index e95e8f2..c036c6d 100644 --- a/tests/e2e/docker-compose.test.yml +++ b/tests/e2e/docker-compose.test.yml @@ -10,14 +10,37 @@ services: # MQTT Broker mqtt: - image: eclipse-mosquitto:2 + image: ghcr.io/ipnet-mesh/meshcore-mqtt-broker:latest container_name: meshcore-test-mqtt ports: - "11883:1883" volumes: - - ../../etc/mosquitto.conf:/mosquitto/config/mosquitto.conf:ro + - test_mqtt_data:/data + environment: + - MQTT_WS_PORT=1883 + - MQTT_HOST=0.0.0.0 + - AUTH_EXPECTED_AUDIENCE=mqtt.localhost + - SUBSCRIBER_MAX_CONNECTIONS_DEFAULT=5 + - SUBSCRIBER_1=test-admin:test-password:1 + - ABUSE_ENFORCEMENT_ENABLED=false + - ABUSE_DUPLICATE_WINDOW_SIZE=100 + - ABUSE_DUPLICATE_WINDOW_MS=300000 + - ABUSE_DUPLICATE_THRESHOLD=10 + - ABUSE_MAX_DUPLICATES_PER_PACKET=5 + - ABUSE_DUPLICATE_RATE_THRESHOLD=0.3 + - ABUSE_DUPLICATE_RATE_WINDOW_MS=300000 + - ABUSE_BUCKET_CAPACITY=20 + - ABUSE_BUCKET_REFILL_RATE=3 + - ABUSE_MAX_PACKET_SIZE=255 + - ABUSE_MAX_TOPICS_PER_DAY=3 + - ABUSE_ANOMALY_THRESHOLD=10 + - ABUSE_MAX_IATA_CHANGES_24H=3 + - ABUSE_TOPIC_HISTORY_SIZE=50 + - ABUSE_TOPIC_HISTORY_WINDOW_MS=86400000 + - ABUSE_PERSISTENCE_PATH=/data/abuse-detection.db + - ABUSE_PERSISTENCE_INTERVAL_MS=300000 healthcheck: - test: ["CMD", "mosquitto_sub", "-t", "$$SYS/#", "-C", "1", "-i", "healthcheck", "-W", "3"] + test: ["CMD", "node", "-e", "const net=require('net');const s=net.createConnection(1883,'127.0.0.1',()=>{s.end();process.exit(0)});s.on('error',()=>process.exit(1));setTimeout(()=>process.exit(1),3000)"] interval: 5s timeout: 5s retries: 3 @@ -63,6 +86,10 @@ services: - MQTT_HOST=mqtt - MQTT_PORT=1883 - MQTT_PREFIX=test + - MQTT_TRANSPORT=websockets + - MQTT_WS_PATH=/ + - MQTT_USERNAME=test-admin + - MQTT_PASSWORD=test-password - DATABASE_URL=sqlite:////data/test.db command: ["collector"] healthcheck: @@ -92,6 +119,10 @@ services: - MQTT_HOST=mqtt - MQTT_PORT=1883 - MQTT_PREFIX=test + - MQTT_TRANSPORT=websockets + - MQTT_WS_PATH=/ + - MQTT_USERNAME=test-admin + - MQTT_PASSWORD=test-password - DATABASE_URL=sqlite:////data/test.db - API_HOST=0.0.0.0 - API_PORT=8000 @@ -134,3 +165,5 @@ services: volumes: test_data: name: meshcore_test_data + test_mqtt_data: + name: meshcore_test_mqtt_data diff --git a/tests/test_collector/test_letsmesh_normalizer_integration.py b/tests/test_collector/test_letsmesh_normalizer_integration.py index 20f2e7d..7871a75 100644 --- a/tests/test_collector/test_letsmesh_normalizer_integration.py +++ b/tests/test_collector/test_letsmesh_normalizer_integration.py @@ -55,7 +55,7 @@ class TestStatusFeed: "status", ) payload = {"uptime": 3600, "nodes_seen": 5} - result = norm._normalize_letsmesh_event("meshcore/BB.../status", payload) + result = norm._normalize_letsmesh_event("meshcore/STN/BB.../status", payload) assert result is not None pk, event_type, pl = result assert pk == OBSERVER_KEY @@ -72,7 +72,7 @@ class TestInternalFeed: "internal", ) payload = {"info": "restart"} - result = norm._normalize_letsmesh_event("meshcore/BB.../internal", payload) + result = norm._normalize_letsmesh_event("meshcore/STN/BB.../internal", payload) assert result is not None pk, event_type, pl = result assert pk == OBSERVER_KEY diff --git a/tests/test_collector/test_subscriber.py b/tests/test_collector/test_subscriber.py index 1852d11..afbaa2e 100644 --- a/tests/test_collector/test_subscriber.py +++ b/tests/test_collector/test_subscriber.py @@ -14,7 +14,7 @@ class TestSubscriber: """Create a mock MQTT client.""" client = MagicMock() client.topic_builder = MagicMock() - client.topic_builder.prefix = "meshcore/BOS" + client.topic_builder.prefix = "meshcore" client.topic_builder.all_events_topic.return_value = "meshcore/+/event/#" client.topic_builder.parse_event_topic.return_value = ( "a" * 64, @@ -76,8 +76,8 @@ class TestSubscriber: ) subscriber._handle_mqtt_message( - topic="meshcore/abc/status", - pattern="meshcore/+/status", + topic="meshcore/STN/abc/status", + pattern="meshcore/+/+/status", payload={"public_key": "b" * 64, "name": "Test"}, ) @@ -93,9 +93,9 @@ class TestSubscriber: subscriber.start() expected_calls = [ - call("meshcore/BOS/+/packets", subscriber._handle_mqtt_message), - call("meshcore/BOS/+/status", subscriber._handle_mqtt_message), - call("meshcore/BOS/+/internal", subscriber._handle_mqtt_message), + call("meshcore/+/+/packets", subscriber._handle_mqtt_message), + call("meshcore/+/+/status", subscriber._handle_mqtt_message), + call("meshcore/+/+/internal", subscriber._handle_mqtt_message), ] mock_mqtt_client.subscribe.assert_has_calls(expected_calls, any_order=False) assert mock_mqtt_client.subscribe.call_count == 3 @@ -115,8 +115,8 @@ class TestSubscriber: subscriber.start() subscriber._handle_mqtt_message( - topic=f"meshcore/BOS/{'a' * 64}/status", - pattern="meshcore/BOS/+/status", + topic=f"meshcore/STN/{'a' * 64}/status", + pattern="meshcore/+/+/status", payload={ "origin": "Observer Node", "origin_id": "b" * 64, @@ -150,8 +150,8 @@ class TestSubscriber: subscriber.start() subscriber._handle_mqtt_message( - topic=f"meshcore/BOS/{'a' * 64}/status", - pattern="meshcore/BOS/+/status", + topic=f"meshcore/STN/{'a' * 64}/status", + pattern="meshcore/+/+/status", payload={ "origin": "Observer Node", "origin_id": "b" * 64, @@ -180,8 +180,8 @@ class TestSubscriber: subscriber.start() subscriber._handle_mqtt_message( - topic=f"meshcore/BOS/{'a' * 64}/status", - pattern="meshcore/BOS/+/status", + topic=f"meshcore/STN/{'a' * 64}/status", + pattern="meshcore/+/+/status", payload={ "origin_id": "b" * 64, "stats": {"cpu": 27, "mem": 91, "debug_flags": 7}, @@ -222,8 +222,8 @@ class TestSubscriber: }, ): subscriber._handle_mqtt_message( - topic=f"meshcore/BOS/{'a' * 64}/packets", - pattern="meshcore/BOS/+/packets", + topic=f"meshcore/STN/{'a' * 64}/packets", + pattern="meshcore/+/+/packets", payload={ "packet_type": "5", "hash": "ABCDEF1234", @@ -267,8 +267,8 @@ class TestSubscriber: return_value=None, ): subscriber._handle_mqtt_message( - topic=f"meshcore/BOS/{'a' * 64}/packets", - pattern="meshcore/BOS/+/packets", + topic=f"meshcore/STN/{'a' * 64}/packets", + pattern="meshcore/+/+/packets", payload={ "packet_type": "5", "hash": "ABCDEF1234", @@ -321,8 +321,8 @@ class TestSubscriber: ), ): subscriber._handle_mqtt_message( - topic=f"meshcore/BOS/{'a' * 64}/packets", - pattern="meshcore/BOS/+/packets", + topic=f"meshcore/STN/{'a' * 64}/packets", + pattern="meshcore/+/+/packets", payload={ "packet_type": "5", "hash": "ABCDEF1234", @@ -375,8 +375,8 @@ class TestSubscriber: }, ): subscriber._handle_mqtt_message( - topic=f"meshcore/BOS/{'a' * 64}/packets", - pattern="meshcore/BOS/+/packets", + topic=f"meshcore/STN/{'a' * 64}/packets", + pattern="meshcore/+/+/packets", payload={ "packet_type": "1", "hash": "ABABAB1234", @@ -424,8 +424,8 @@ class TestSubscriber: }, ): subscriber._handle_mqtt_message( - topic=f"meshcore/BOS/{'a' * 64}/packets", - pattern="meshcore/BOS/+/packets", + topic=f"meshcore/STN/{'a' * 64}/packets", + pattern="meshcore/+/+/packets", payload={ "packet_type": "5", "hash": "FEEDC0DE", @@ -479,8 +479,8 @@ class TestSubscriber: }, ): subscriber._handle_mqtt_message( - topic=f"meshcore/BOS/{'a' * 64}/packets", - pattern="meshcore/BOS/+/packets", + topic=f"meshcore/STN/{'a' * 64}/packets", + pattern="meshcore/+/+/packets", payload={ "packet_type": "4", "hash": "A1B2C3D4", @@ -534,8 +534,8 @@ class TestSubscriber: }, ): subscriber._handle_mqtt_message( - topic=f"meshcore/BOS/{'a' * 64}/packets", - pattern="meshcore/BOS/+/packets", + topic=f"meshcore/STN/{'a' * 64}/packets", + pattern="meshcore/+/+/packets", payload={ "packet_type": "11", "hash": "E5F6A7B8", @@ -586,8 +586,8 @@ class TestSubscriber: }, ): subscriber._handle_mqtt_message( - topic=f"meshcore/BOS/{'a' * 64}/packets", - pattern="meshcore/BOS/+/packets", + topic=f"meshcore/STN/{'a' * 64}/packets", + pattern="meshcore/+/+/packets", payload={ "packet_type": "9", "hash": "99887766", @@ -638,8 +638,8 @@ class TestSubscriber: }, ): subscriber._handle_mqtt_message( - topic=f"meshcore/BOS/{'a' * 64}/packets", - pattern="meshcore/BOS/+/packets", + topic=f"meshcore/STN/{'a' * 64}/packets", + pattern="meshcore/+/+/packets", payload={ "packet_type": "9", "hash": "99887766", @@ -687,8 +687,8 @@ class TestSubscriber: }, ): subscriber._handle_mqtt_message( - topic=f"meshcore/BOS/{'a' * 64}/packets", - pattern="meshcore/BOS/+/packets", + topic=f"meshcore/STN/{'a' * 64}/packets", + pattern="meshcore/+/+/packets", payload={ "packet_type": "8", "hash": "99887766", @@ -738,8 +738,8 @@ class TestSubscriber: }, ): subscriber._handle_mqtt_message( - topic=f"meshcore/BOS/{'a' * 64}/packets", - pattern="meshcore/BOS/+/packets", + topic=f"meshcore/STN/{'a' * 64}/packets", + pattern="meshcore/+/+/packets", payload={ "packet_type": "8", "hash": "99887766", @@ -787,8 +787,8 @@ class TestSubscriber: return_value=decoded_packet, ): subscriber._handle_mqtt_message( - topic=f"meshcore/BOS/{'a' * 64}/packets", - pattern="meshcore/BOS/+/packets", + topic=f"meshcore/STN/{'a' * 64}/packets", + pattern="meshcore/+/+/packets", payload={ "packet_type": "10", "hash": "99887766", @@ -833,8 +833,8 @@ class TestSubscriber: }, ): subscriber._handle_mqtt_message( - topic=f"meshcore/BOS/{'a' * 64}/packets", - pattern="meshcore/BOS/+/packets", + topic=f"meshcore/STN/{'a' * 64}/packets", + pattern="meshcore/+/+/packets", payload={ "packet_type": "5", "hash": "ABABAB1234", diff --git a/tests/test_common/test_mqtt.py b/tests/test_common/test_mqtt.py index 2a33c69..73d2405 100644 --- a/tests/test_common/test_mqtt.py +++ b/tests/test_common/test_mqtt.py @@ -38,20 +38,20 @@ class TestTopicBuilder: def test_parse_letsmesh_upload_topic(self) -> None: """LetsMesh upload topics map to public key and feed type.""" - builder = TopicBuilder(prefix="meshcore/BOS") + builder = TopicBuilder(prefix="meshcore") parsed = builder.parse_letsmesh_upload_topic( - "meshcore/BOS/ABCDEF1234567890/status" + "meshcore/STN/ABCDEF1234567890/status" ) assert parsed == ("ABCDEF1234567890", "status") def test_parse_letsmesh_upload_topic_rejects_unknown_feed(self) -> None: """Unknown LetsMesh feed topics are rejected.""" - builder = TopicBuilder(prefix="meshcore/BOS") + builder = TopicBuilder(prefix="meshcore") parsed = builder.parse_letsmesh_upload_topic( - "meshcore/BOS/ABCDEF1234567890/something_else" + "meshcore/STN/ABCDEF1234567890/something_else" ) assert parsed is None