diff --git a/.env.example b/.env.example index b8624db..41416ad 100644 --- a/.env.example +++ b/.env.example @@ -192,7 +192,11 @@ PACKETCAPTURE_EXIT_ON_RECONNECT_FAIL=true # Note: Public + #test keys are built into the collector code by default. # To show friendly channel names in the web feed, use label=hex (example: bot=ABCDEF...). # Without keys, encrypted packets cannot be shown as plaintext. -# COLLECTOR_LETSMESH_DECODER_KEYS= +# COLLECTOR_CHANNEL_KEYS= + +# Include built-in 'test' channel messages (channel_idx 217) +# Default: false (test channel messages are discarded) +# COLLECTOR_INCLUDE_TEST_CHANNEL=false # ------------------- # Webhook Settings diff --git a/AGENTS.md b/AGENTS.md index 4b650a4..611d074 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -4,6 +4,9 @@ This document provides context and guidelines for AI coding assistants working o ## Agent Rules +### Critical Rules (MUST follow) + +* **Always use parenthesized exception tuples** — `except (ValueError, TypeError):` not `except ValueError, TypeError:`. The comma form is Python 2 syntax and will fail at import time in Python 3. This is the most common error that passes visual review but breaks the application. * You MUST use Python (version in `.python-version` file) * You MUST activate a Python virtual environment in the `.venv` directory or create one if it does not exist: - `ls ./.venv` to check if it exists @@ -66,7 +69,6 @@ MeshCore Hub is a Python 3.14+ monorepo for managing and orchestrating MeshCore - Use type hints for all function signatures - Write docstrings for public modules, classes, and functions - Keep functions focused and under 50 lines where possible -- **Always use parenthesized exception tuples** — `except (ValueError, TypeError):` not `except ValueError, TypeError:`. The comma form is Python 2 syntax and will fail at import time in Python 3 ### Imports @@ -598,6 +600,7 @@ Key variables: - `MQTT_WS_PATH` - WebSocket path (default: `/`) - `MQTT_TLS` - Enable TLS/SSL for MQTT (default: `false`, set `true` for `wss://`) - `MQTT_TOKEN_AUDIENCE` - JWT audience claim for packet capture auth tokens (default: `mqtt.localhost`) +- `COLLECTOR_INCLUDE_TEST_CHANNEL` - Include built-in 'test' channel messages (default: `false`) - `API_READ_KEY`, `API_ADMIN_KEY` - API authentication keys - `WEB_ADMIN_ENABLED` - Enable admin interface at /a/ (default: `false`, requires auth proxy) - `WEB_TRUSTED_PROXY_HOSTS` - Comma-separated list of trusted proxy hosts for admin authentication headers. Default: `*` (all hosts). Recommended: set to your reverse proxy IP in production. A startup warning is emitted when using the default `*` with admin enabled. diff --git a/README.md b/README.md index 2963daf..399dd20 100644 --- a/README.md +++ b/README.md @@ -295,7 +295,8 @@ All components are configured via environment variables. Create a `.env` file or | Variable | Default | Description | |----------|---------|-------------| -| `COLLECTOR_LETSMESH_DECODER_KEYS` | *(none)* | Additional decoder channel keys (`label=hex`, `label:hex`, or `hex`) | +| `COLLECTOR_CHANNEL_KEYS` | *(none)* | Additional decoder channel keys (`label=hex`, `label:hex`, or `hex`) | +| `COLLECTOR_INCLUDE_TEST_CHANNEL` | `false` | Include built-in 'test' channel messages | #### LetsMesh Packet Decoding @@ -317,12 +318,12 @@ Normalization behavior: - `packet_type=1`, `2`, and `7` packets are mapped to `contact_msg_recv` when decryptable text is available. - For channel packets, if a channel key is available, a channel label is attached (for example `Public` or `#test`) for UI display. - In the messages feed and dashboard channel sections, known channel indexes are preferred for labels (`17 -> Public`, `217 -> #test`) to avoid stale channel-name mismatches. -- Additional channel names are loaded from `COLLECTOR_LETSMESH_DECODER_KEYS` when entries are provided as `label=hex` (for example `bot=`). +- Additional channel names are loaded from `COLLECTOR_CHANNEL_KEYS` when entries are provided as `label=hex` (for example `bot=`). - Decoder-advertisement packets with location metadata update node GPS (`lat/lon`) for map display. - This keeps advertisement listings focused on node advert traffic only, not observer status telemetry. - Packets without decryptable message text are kept as informational `letsmesh_packet` events and are not shown in the messages feed; when decode succeeds the decoded JSON is attached to those packet log events. - When decoder output includes a human sender (`payload.decoded.decrypted.sender`), message text is normalized to `Name: Message` before storage; receiver/observer names are never used as sender fallback. -- The collector keeps built-in keys for `Public` and `#test`, and merges any additional keys from `COLLECTOR_LETSMESH_DECODER_KEYS`. +- The collector keeps built-in keys for `Public` and `#test`, and merges any additional keys from `COLLECTOR_CHANNEL_KEYS`. - Docker runtime uses the native Python `meshcoredecoder` library (no external Node.js dependency). ### Webhooks diff --git a/SCHEMAS.md b/SCHEMAS.md index 0276dd7..90a5478 100644 --- a/SCHEMAS.md +++ b/SCHEMAS.md @@ -180,7 +180,7 @@ Group/broadcast messages on specific channels. - In LetsMesh upload compatibility mode, packet type `5` is normalized to `CHANNEL_MSG_RECV` and packet types `1`, `2`, and `7` are normalized to `CONTACT_MSG_RECV` when decryptable text is available. - LetsMesh packets without decryptable message text are treated as informational `letsmesh_packet` events instead of message events. - For UI labels, known channel indexes are mapped (`17 -> Public`, `217 -> #test`) and preferred over ambiguous/stale channel-name hints. -- Additional channel labels can be provided through `COLLECTOR_LETSMESH_DECODER_KEYS` using `label=hex` entries. +- Additional channel labels can be provided through `COLLECTOR_CHANNEL_KEYS` using `label=hex` entries. - When decoder output includes a human sender (`payload.decoded.decrypted.sender`), message text is normalized to `Name: Message`; sender identity remains unknown when only hash/prefix metadata is available. **Compatibility ingest note (advertisements)**: diff --git a/UPGRADING.md b/UPGRADING.md index 66505c7..ab2b7c3 100644 --- a/UPGRADING.md +++ b/UPGRADING.md @@ -377,4 +377,5 @@ The Node.js `meshcore-decoder` CLI tool has been replaced by the native Python ` - No Node.js runtime is needed in the Docker image - The decoder is always enabled (no toggle) - The `COLLECTOR_LETSMESH_DECODER_*` configuration variables have been removed -- `COLLECTOR_LETSMESH_DECODER_KEYS` is still supported for providing additional channel decryption keys +- `COLLECTOR_LETSMESH_DECODER_KEYS` has been renamed to `COLLECTOR_CHANNEL_KEYS` +- New `COLLECTOR_INCLUDE_TEST_CHANNEL` variable controls whether built-in test channel messages are collected (default: `false`) diff --git a/docker-compose.yml b/docker-compose.yml index 3c10a76..8318e22 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -157,7 +157,8 @@ services: - MQTT_TLS=${MQTT_TLS:-false} - MQTT_TRANSPORT=${MQTT_TRANSPORT:-websockets} - MQTT_WS_PATH=${MQTT_WS_PATH:-/} - - COLLECTOR_LETSMESH_DECODER_KEYS=${COLLECTOR_LETSMESH_DECODER_KEYS:-} + - COLLECTOR_CHANNEL_KEYS=${COLLECTOR_CHANNEL_KEYS:-} + - COLLECTOR_INCLUDE_TEST_CHANNEL=${COLLECTOR_INCLUDE_TEST_CHANNEL:-false} - DATA_HOME=/data - SEED_HOME=/seed # Webhook configuration @@ -278,7 +279,8 @@ services: - NETWORK_WELCOME_TEXT=${NETWORK_WELCOME_TEXT:-} - CONTENT_HOME=/content - TZ=${TZ:-UTC} - - COLLECTOR_LETSMESH_DECODER_KEYS=${COLLECTOR_LETSMESH_DECODER_KEYS:-} + - COLLECTOR_CHANNEL_KEYS=${COLLECTOR_CHANNEL_KEYS:-} + - COLLECTOR_INCLUDE_TEST_CHANNEL=${COLLECTOR_INCLUDE_TEST_CHANNEL:-false} # Feature flags (set to false to disable specific pages) - FEATURE_DASHBOARD=${FEATURE_DASHBOARD:-true} - FEATURE_NODES=${FEATURE_NODES:-true} diff --git a/src/meshcore_hub/collector/cli.py b/src/meshcore_hub/collector/cli.py index 03a95e4..3bf95e0 100644 --- a/src/meshcore_hub/collector/cli.py +++ b/src/meshcore_hub/collector/cli.py @@ -260,7 +260,7 @@ def _run_collector_service( click.echo("") builtin_keys = len(LetsMeshPacketDecoder.BUILTIN_CHANNEL_KEYS) - env_keys = len(settings.collector_letsmesh_decoder_keys_list) + env_keys = len(settings.collector_channel_keys_list) click.echo(f"Packet decoder: {builtin_keys} built-in keys, {env_keys} from .env") click.echo("") @@ -281,7 +281,8 @@ def _run_collector_service( cleanup_interval_hours=settings.data_retention_interval_hours, node_cleanup_enabled=settings.node_cleanup_enabled, node_cleanup_days=settings.node_cleanup_days, - letsmesh_decoder_channel_keys=settings.collector_letsmesh_decoder_keys_list, + channel_keys=settings.collector_channel_keys_list, + include_test_channel=settings.collector_include_test_channel, ) diff --git a/src/meshcore_hub/collector/letsmesh_decoder.py b/src/meshcore_hub/collector/letsmesh_decoder.py index 36e3085..237b91d 100644 --- a/src/meshcore_hub/collector/letsmesh_decoder.py +++ b/src/meshcore_hub/collector/letsmesh_decoder.py @@ -34,6 +34,9 @@ class LetsMeshPacketDecoder: ("test", "9CD8FCF22A47333B591D96A2B848B73F"), ) + TEST_CHANNEL_HASH: str = "D9" + TEST_CHANNEL_IDX: int = 217 + def __init__( self, channel_keys: list[str] | None = None, diff --git a/src/meshcore_hub/collector/letsmesh_normalizer.py b/src/meshcore_hub/collector/letsmesh_normalizer.py index 6f82b31..8fa32b2 100644 --- a/src/meshcore_hub/collector/letsmesh_normalizer.py +++ b/src/meshcore_hub/collector/letsmesh_normalizer.py @@ -18,6 +18,7 @@ class LetsMeshNormalizer: # Attributes are provided by Subscriber at runtime. mqtt: Any _letsmesh_decoder: LetsMeshPacketDecoder + _include_test_channel: bool def _normalize_letsmesh_event( self, @@ -100,6 +101,20 @@ class LetsMeshNormalizer: if decoded_packet is None: decoded_packet = self._letsmesh_decoder.decode_payload(payload) + # Filter test channel messages when not explicitly included. + if packet_type == 5 and not self._include_test_channel: + channel_hash = self._extract_letsmesh_decoder_channel_hash(decoded_packet) + if ( + channel_hash + and channel_hash.upper() == LetsMeshPacketDecoder.TEST_CHANNEL_HASH + ): + logger.debug( + "Skipping LetsMesh packet %s (type=%s): test channel excluded", + packet_hash_text or "unknown", + packet_type, + ) + return None + # In LetsMesh compatibility mode, only show messages that decrypt. text = self._extract_letsmesh_decoder_text(decoded_packet) if not text: diff --git a/src/meshcore_hub/collector/subscriber.py b/src/meshcore_hub/collector/subscriber.py index d275b5e..e555ad9 100644 --- a/src/meshcore_hub/collector/subscriber.py +++ b/src/meshcore_hub/collector/subscriber.py @@ -47,7 +47,8 @@ class Subscriber(LetsMeshNormalizer): cleanup_interval_hours: int = 24, node_cleanup_enabled: bool = False, node_cleanup_days: int = 90, - letsmesh_decoder_channel_keys: list[str] | None = None, + channel_keys: list[str] | None = None, + include_test_channel: bool = False, ): """Initialize subscriber. @@ -60,7 +61,8 @@ class Subscriber(LetsMeshNormalizer): cleanup_interval_hours: Hours between cleanup runs node_cleanup_enabled: Enable automatic cleanup of inactive nodes node_cleanup_days: Remove nodes not seen for this many days - letsmesh_decoder_channel_keys: Optional channel keys for decrypting group text + channel_keys: Optional channel keys for decrypting group text + include_test_channel: Include built-in test channel messages """ self.mqtt = mqtt_client self.db = db_manager @@ -84,8 +86,9 @@ class Subscriber(LetsMeshNormalizer): self._cleanup_thread: Optional[threading.Thread] = None self._last_cleanup: Optional[datetime] = None self._letsmesh_decoder = LetsMeshPacketDecoder( - channel_keys=letsmesh_decoder_channel_keys, + channel_keys=channel_keys, ) + self._include_test_channel = include_test_channel @property def is_healthy(self) -> bool: @@ -479,7 +482,8 @@ def create_subscriber( cleanup_interval_hours: int = 24, node_cleanup_enabled: bool = False, node_cleanup_days: int = 90, - letsmesh_decoder_channel_keys: list[str] | None = None, + channel_keys: list[str] | None = None, + include_test_channel: bool = False, ) -> Subscriber: """Create a configured subscriber instance. @@ -499,7 +503,8 @@ def create_subscriber( cleanup_interval_hours: Hours between cleanup runs node_cleanup_enabled: Enable automatic cleanup of inactive nodes node_cleanup_days: Remove nodes not seen for this many days - letsmesh_decoder_channel_keys: Optional channel keys for decrypting group text + channel_keys: Optional channel keys for decrypting group text + include_test_channel: Include built-in test channel messages Returns: Configured Subscriber instance @@ -532,7 +537,8 @@ def create_subscriber( cleanup_interval_hours=cleanup_interval_hours, node_cleanup_enabled=node_cleanup_enabled, node_cleanup_days=node_cleanup_days, - letsmesh_decoder_channel_keys=letsmesh_decoder_channel_keys, + channel_keys=channel_keys, + include_test_channel=include_test_channel, ) # Register handlers @@ -559,7 +565,8 @@ def run_collector( cleanup_interval_hours: int = 24, node_cleanup_enabled: bool = False, node_cleanup_days: int = 90, - letsmesh_decoder_channel_keys: list[str] | None = None, + channel_keys: list[str] | None = None, + include_test_channel: bool = False, ) -> None: """Run the collector (blocking). @@ -579,7 +586,8 @@ def run_collector( cleanup_interval_hours: Hours between cleanup runs node_cleanup_enabled: Enable automatic cleanup of inactive nodes node_cleanup_days: Remove nodes not seen for this many days - letsmesh_decoder_channel_keys: Optional channel keys for decrypting group text + channel_keys: Optional channel keys for decrypting group text + include_test_channel: Include built-in test channel messages """ subscriber = create_subscriber( mqtt_host=mqtt_host, @@ -597,7 +605,8 @@ def run_collector( cleanup_interval_hours=cleanup_interval_hours, node_cleanup_enabled=node_cleanup_enabled, node_cleanup_days=node_cleanup_days, - letsmesh_decoder_channel_keys=letsmesh_decoder_channel_keys, + channel_keys=channel_keys, + include_test_channel=include_test_channel, ) # Set up signal handlers diff --git a/src/meshcore_hub/common/config.py b/src/meshcore_hub/common/config.py index 9e5f464..ab95bde 100644 --- a/src/meshcore_hub/common/config.py +++ b/src/meshcore_hub/common/config.py @@ -138,13 +138,17 @@ class CollectorSettings(CommonSettings): description="Remove nodes not seen for this many days (last_seen)", ge=1, ) - collector_letsmesh_decoder_keys: Optional[str] = Field( + collector_channel_keys: Optional[str] = Field( default=None, description=( - "Optional channel secret keys for LetsMesh message decryption. " + "Optional channel secret keys for message decryption. " "Provide as comma/space separated hex values." ), ) + collector_include_test_channel: bool = Field( + default=False, + description="Include built-in 'test' channel messages (channel_idx 217).", + ) @property def collector_data_dir(self) -> str: @@ -185,13 +189,13 @@ class CollectorSettings(CommonSettings): return str(Path(self.effective_seed_home) / "members.yaml") @property - def collector_letsmesh_decoder_keys_list(self) -> list[str]: - """Parse configured LetsMesh decoder keys into a normalized list.""" - if not self.collector_letsmesh_decoder_keys: + def collector_channel_keys_list(self) -> list[str]: + """Parse configured channel keys into a normalized list.""" + if not self.collector_channel_keys: return [] return [ part.strip() - for part in re.split(r"[,\s]+", self.collector_letsmesh_decoder_keys) + for part in re.split(r"[,\s]+", self.collector_channel_keys) if part.strip() ] diff --git a/src/meshcore_hub/web/app.py b/src/meshcore_hub/web/app.py index af8a7ac..3aa99be 100644 --- a/src/meshcore_hub/web/app.py +++ b/src/meshcore_hub/web/app.py @@ -33,7 +33,7 @@ STATIC_DIR = PACKAGE_DIR / "static" def _parse_decoder_key_entries(raw: str | None) -> list[str]: - """Parse COLLECTOR_LETSMESH_DECODER_KEYS into key entries.""" + """Parse COLLECTOR_CHANNEL_KEYS into key entries.""" if not raw: return [] return [part.strip() for part in re.split(r"[,\s]+", raw) if part.strip()] @@ -41,11 +41,18 @@ def _parse_decoder_key_entries(raw: str | None) -> list[str]: def _build_channel_labels() -> dict[str, str]: """Build UI channel labels from built-in + configured decoder keys.""" - raw_keys = os.getenv("COLLECTOR_LETSMESH_DECODER_KEYS") + raw_keys = os.getenv("COLLECTOR_CHANNEL_KEYS") + include_test = os.getenv("COLLECTOR_INCLUDE_TEST_CHANNEL", "false").lower() in ( + "true", + "1", + "yes", + ) decoder = LetsMeshPacketDecoder( channel_keys=_parse_decoder_key_entries(raw_keys), ) labels = decoder.channel_labels_by_index() + if not include_test: + labels.pop(LetsMeshPacketDecoder.TEST_CHANNEL_IDX, None) return {str(idx): label for idx, label in sorted(labels.items())} diff --git a/tests/test_collector/test_letsmesh_normalizer_integration.py b/tests/test_collector/test_letsmesh_normalizer_integration.py index 7871a75..50b1206 100644 --- a/tests/test_collector/test_letsmesh_normalizer_integration.py +++ b/tests/test_collector/test_letsmesh_normalizer_integration.py @@ -18,6 +18,7 @@ class _TestNormalizer(LetsMeshNormalizer): def __init__(self, decoder: LetsMeshPacketDecoder) -> None: self._letsmesh_decoder = decoder + self._include_test_channel = True self.mqtt = MagicMock() self.mqtt.topic_builder.parse_letsmesh_upload_topic = MagicMock( return_value=None, diff --git a/tests/test_collector/test_subscriber.py b/tests/test_collector/test_subscriber.py index 4378dba..e25beb1 100644 --- a/tests/test_collector/test_subscriber.py +++ b/tests/test_collector/test_subscriber.py @@ -402,6 +402,7 @@ class TestSubscriber: subscriber = Subscriber( mock_mqtt_client, db_manager, + include_test_channel=True, ) handler = MagicMock() subscriber.register_handler("channel_msg_recv", handler) diff --git a/tests/test_common/test_config.py b/tests/test_common/test_config.py index 19e0ae4..bd6c64d 100644 --- a/tests/test_common/test_config.py +++ b/tests/test_common/test_config.py @@ -60,14 +60,14 @@ class TestCollectorSettings: assert settings.node_tags_file == "/seed/data/node_tags.yaml" assert settings.members_file == "/seed/data/members.yaml" - def test_collector_letsmesh_decoder_keys_list(self) -> None: - """LetsMesh decoder keys are parsed from comma/space-separated env values.""" + def test_collector_channel_keys_list(self) -> None: + """Channel keys are parsed from comma/space-separated env values.""" settings = CollectorSettings( _env_file=None, - collector_letsmesh_decoder_keys="aa11, bb22 cc33", + collector_channel_keys="aa11, bb22 cc33", ) - assert settings.collector_letsmesh_decoder_keys_list == [ + assert settings.collector_channel_keys_list == [ "aa11", "bb22", "cc33", diff --git a/tests/test_web/conftest.py b/tests/test_web/conftest.py index de2ec76..3eb3ade 100644 --- a/tests/test_web/conftest.py +++ b/tests/test_web/conftest.py @@ -320,6 +320,7 @@ def web_app(mock_http_client: MockHttpClient, monkeypatch: pytest.MonkeyPatch) - """Create a web app with mocked HTTP client.""" # Ensure tests use a consistent locale regardless of local .env monkeypatch.setenv("WEB_DATETIME_LOCALE", "en-US") + monkeypatch.setenv("COLLECTOR_INCLUDE_TEST_CHANNEL", "true") app = create_app( api_url="http://localhost:8000", api_key="test-api-key",