diff --git a/.env.example b/.env.example index 5ff0335..3255bd9 100644 --- a/.env.example +++ b/.env.example @@ -238,6 +238,20 @@ DATA_RETENTION_DAYS=30 # Applies to both event data and node cleanup DATA_RETENTION_INTERVAL_HOURS=24 +# ------------------- +# Raw Packet Capture +# ------------------- +# Capture every inbound packets-feed packet into the raw_packets table. +# Off by default. In Docker Compose this is derived from FEATURE_PACKETS, so +# setting FEATURE_PACKETS=true enables both capture and the Packets page. +# RAW_PACKET_CAPTURE_ENABLED=false + +# Days to retain raw packets. Defaults to DATA_RETENTION_DAYS when unset. +# The raw_packets table grows fastest of all; lower this on busy meshes or +# constrained storage. Retention runs regardless of capture being enabled, so +# disabling capture lets existing rows drain. +# RAW_PACKET_RETENTION_DAYS=30 + # ------------------- # Node Cleanup Settings # ------------------- @@ -497,6 +511,9 @@ NETWORK_ANNOUNCEMENT= # FEATURE_PAGES=true # FEATURE_CHANNELS=true # FEATURE_RADIO_CONFIG=true +# Packets page is OFF by default. Enabling it also drives raw-packet capture +# on the collector via Compose (RAW_PACKET_CAPTURE_ENABLED=${FEATURE_PACKETS}). +# FEATURE_PACKETS=false # ------------------- # Contact Information diff --git a/AGENTS.md b/AGENTS.md index d7d5844..b00afae 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -681,7 +681,7 @@ Key variables: - `WEB_AUTO_REFRESH_SECONDS` - Auto-refresh interval in seconds for list pages (default: `30`, `0` to disable) - `WEB_DEBUG` - Enable debug mode in the web dashboard (default: `false`) - `TZ` - Timezone for web dashboard date/time display (default: `UTC`, e.g., `America/New_York`, `Europe/London`) -- `FEATURE_DASHBOARD`, `FEATURE_NODES`, `FEATURE_ADVERTISEMENTS`, `FEATURE_MESSAGES`, `FEATURE_MAP`, `FEATURE_MEMBERS`, `FEATURE_PAGES`, `FEATURE_CHANNELS`, `FEATURE_RADIO_CONFIG` - Feature flags to enable/disable specific web dashboard pages (default: all `true`). Dependencies: Dashboard auto-disables when all of Nodes/Advertisements/Messages are disabled. Map auto-disables when Nodes is disabled. +- `FEATURE_DASHBOARD`, `FEATURE_NODES`, `FEATURE_ADVERTISEMENTS`, `FEATURE_MESSAGES`, `FEATURE_MAP`, `FEATURE_MEMBERS`, `FEATURE_PAGES`, `FEATURE_CHANNELS`, `FEATURE_RADIO_CONFIG` - Feature flags to enable/disable specific web dashboard pages (default: all `true`). Dependencies: Dashboard auto-disables when all of Nodes/Advertisements/Messages are disabled. Map auto-disables when Nodes is disabled. `FEATURE_PACKETS` - enables the Packets page (default: `false`); in Compose it also drives `RAW_PACKET_CAPTURE_ENABLED` on the collector. - `NETWORK_DOMAIN` - Network domain name (default: none) - `NETWORK_NAME` - Network display name (default: `MeshCore Network`) - `NETWORK_CITY` - Network city location (default: none) @@ -763,6 +763,16 @@ When enabled, the collector automatically deletes event data older than the rete - Telemetry - Trace paths - Event logs +- Raw packets (using `RAW_PACKET_RETENTION_DAYS`) + +**Raw Packet Capture:** + +| Variable | Description | +|----------|-------------| +| `RAW_PACKET_CAPTURE_ENABLED` | Capture every packets-feed packet into `raw_packets` (default: false). In Compose, derived from `FEATURE_PACKETS`. | +| `RAW_PACKET_RETENTION_DAYS` | Days to retain raw packets (default: falls back to `DATA_RETENTION_DAYS`). | + +When enabled, the collector writes one `RawPacket` row per observer reception from the LetsMesh `packets` feed, reusing the decode the normalizer already performs (decoder per-hex cache). The `/packets` API serves these with channel-visibility redaction (restricted-channel packets returned metadata-only) and a role-aware Redis cache key. The web Packets page is gated behind `FEATURE_PACKETS` (default off). Raw-packet retention cleanup runs whenever data cleanup runs, regardless of the capture flag. **Node Cleanup:** diff --git a/SCHEMAS.md b/SCHEMAS.md index 7b77d1e..a5b9d06 100644 --- a/SCHEMAS.md +++ b/SCHEMAS.md @@ -461,6 +461,28 @@ Notification that routing path to a node has changed. --- +### Per-Payload-Type Classifications + +Packets on the LetsMesh `packets` feed that no structured handler claims are classified by their MeshCore payload type rather than the generic `letsmesh_packet`. These are informational (logged to `events_log`, captured to `raw_packets`), with no dedicated table: + +| Payload type | `event_type` | Notes | +|---|---|---| +| `0x00 REQ` | `req` | Encrypted request, metadata only | +| `0x01 RESPONSE` | `response` | Response with no structured content | +| `0x02 TXT_MSG` | `encrypted_direct` | Direct text that did not decrypt | +| `0x03 ACK` | `ack` | Acknowledgment | +| `0x04 ADVERT` | `advert` | Advert lacking identity metadata | +| `0x05 GRP_TXT` | `encrypted_channel` | Channel text with unknown key | +| `0x06 GRP_DATA` | `grp_data` | Group datagram | +| `0x07 ANON_REQ` | `anon_req` | Anonymous request | +| `0x08 PATH` | `path` | Returned path (unparsed) | +| `0x09 TRACE` | `trace` | Trace (unparsed) | +| `0x0A MULTIPART` | `multipart` | Fragment of a sequence (no reassembly) | +| `0x0B CONTROL` | `control` | Control packet not matched to contact/status | +| `0x0F RAW_CUSTOM` | `raw_custom` | Custom-encrypted, opaque | + +`letsmesh_packet` remains as the safety-net label when the payload type cannot be resolved. + ### Other Informational Events The following events are logged but have varying or device-specific payloads: @@ -478,6 +500,39 @@ The following events are logged but have varying or device-specific payloads: --- +## Raw Packets + +When `RAW_PACKET_CAPTURE_ENABLED` is set (Compose derives it from `FEATURE_PACKETS`), the collector stores **every** inbound packet from the LetsMesh `packets` feed exactly as received, independent of how it is classified. One row is written per observer reception (no deduplication), reusing the decode the normalizer already performs. + +**Database Table**: `raw_packets` + +| Column | Type | Description | +|--------|------|-------------| +| `id` | UUID | Primary key | +| `observer_node_id` | FK `nodes.id` (SET NULL) | Receiving interface, from the MQTT topic public key | +| `packet_hash` | string(32) | LetsMesh packet hash; links to structured records, groups multi-observer receptions | +| `raw_hex` | text | On-air bytes from `payload["raw"]` | +| `packet_type` | integer | Wire packet type | +| `payload_type` | integer | Decoder payload type | +| `event_type` | string(50) | How the collector classified the packet (`advertisement`, `channel_msg_recv`, `letsmesh_packet`, …) | +| `channel_idx` | integer | `int(channelHash, 16)` for channel-message packets, else NULL; drives visibility redaction | +| `source_pubkey_prefix` | string(12) | Sender prefix from decoder `sourceHash` / `senderPublicKey` | +| `route_type` | string(20) | `flood`, `transport_flood`, `direct`, `transport_direct` | +| `path_len` | integer | Hop count | +| `snr` | float | Signal-to-noise ratio reported by the observer | +| `decoded` | JSON | Decoder summary (so detail views need no re-decode) | +| `received_at` | datetime | When received by the observing interface | + +**Linking from structured tables**: `advertisements` and `messages` carry a nullable `packet_hash` (the LetsMesh wire hash) populated at ingest. Because the structured tables are deduplicated across observers, one row links to *all* the per-observer `raw_packets` sharing that hash — reachable via `GET /api/v1/packets?packet_hash=`. Only rows ingested while capture was enabled have a populated `packet_hash` (no backfill). + +**REST API**: `GET /api/v1/packets`, `GET /api/v1/packets/{id}` + +**`RawPacketRead` shape** mirrors the columns above plus a `redacted: bool` field and observer details (`observed_by`, `observer_name`, `observer_tag_name`). Channel-message packets on a channel above the requesting role's visibility level are returned with `redacted=true` and `raw_hex`/`decoded`/`source_pubkey_prefix` nulled; non-channel and visible-channel packets are returned in full. Responses are cached in Redis with a role-aware key. + +Filters: `search` (hash/raw_hex/observer, bounded to a recent window when no `since` given), `event_type`, `packet_type`, `channel_idx`, `route_type`, `public_key`/`pubkey_prefix`, `observed_by`, `decryptable`, `min_snr`/`max_snr`, `min_path_len`/`max_path_len`, `redacted`, `since`/`until`, `sort`/`order`, `limit`/`offset`. + +--- + ## Webhook Payload Format All webhook events are wrapped in a standard envelope before being sent: diff --git a/alembic/versions/20260612_1947_e9f0c4079540_add_raw_packets_table.py b/alembic/versions/20260612_1947_e9f0c4079540_add_raw_packets_table.py new file mode 100644 index 0000000..d74188f --- /dev/null +++ b/alembic/versions/20260612_1947_e9f0c4079540_add_raw_packets_table.py @@ -0,0 +1,117 @@ +"""add raw_packets table + +Revision ID: e9f0c4079540 +Revises: 20260610_1200 +Create Date: 2026-06-12 19:47:12.406423+00:00 + +""" + +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa +from sqlalchemy.dialects import sqlite + +# revision identifiers, used by Alembic. +revision: str = "e9f0c4079540" +down_revision: Union[str, None] = "20260610_1200" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.create_table( + "raw_packets", + sa.Column("observer_node_id", sa.String(), nullable=True), + sa.Column("packet_hash", sa.String(length=32), nullable=True), + sa.Column("raw_hex", sa.Text(), nullable=True), + sa.Column("packet_type", sa.Integer(), nullable=True), + sa.Column("payload_type", sa.Integer(), nullable=True), + sa.Column("event_type", sa.String(length=50), nullable=True), + sa.Column("channel_idx", sa.Integer(), nullable=True), + sa.Column("source_pubkey_prefix", sa.String(length=12), nullable=True), + sa.Column("route_type", sa.String(length=20), nullable=True), + sa.Column("path_len", sa.Integer(), nullable=True), + sa.Column("snr", sa.Float(), nullable=True), + sa.Column("decoded", sqlite.JSON(), nullable=True), + sa.Column("received_at", sa.DateTime(timezone=True), nullable=False), + sa.Column("id", sa.String(), nullable=False), + sa.Column( + "created_at", + sa.DateTime(timezone=True), + server_default=sa.text("(CURRENT_TIMESTAMP)"), + nullable=False, + ), + sa.Column( + "updated_at", + sa.DateTime(timezone=True), + server_default=sa.text("(CURRENT_TIMESTAMP)"), + nullable=False, + ), + sa.ForeignKeyConstraint( + ["observer_node_id"], ["nodes.id"], ondelete="SET NULL" + ), + sa.PrimaryKeyConstraint("id"), + ) + with op.batch_alter_table("raw_packets", schema=None) as batch_op: + batch_op.create_index( + batch_op.f("ix_raw_packets_channel_idx"), ["channel_idx"], unique=False + ) + batch_op.create_index( + "ix_raw_packets_channel_idx_received_at", + ["channel_idx", "received_at"], + unique=False, + ) + batch_op.create_index( + batch_op.f("ix_raw_packets_event_type"), ["event_type"], unique=False + ) + batch_op.create_index( + "ix_raw_packets_event_type_received_at", + ["event_type", "received_at"], + unique=False, + ) + batch_op.create_index( + batch_op.f("ix_raw_packets_observer_node_id"), + ["observer_node_id"], + unique=False, + ) + batch_op.create_index( + batch_op.f("ix_raw_packets_packet_hash"), ["packet_hash"], unique=False + ) + batch_op.create_index( + batch_op.f("ix_raw_packets_packet_type"), ["packet_type"], unique=False + ) + batch_op.create_index( + "ix_raw_packets_received_at", ["received_at"], unique=False + ) + batch_op.create_index( + batch_op.f("ix_raw_packets_source_pubkey_prefix"), + ["source_pubkey_prefix"], + unique=False, + ) + batch_op.create_index( + "ix_raw_packets_source_pubkey_prefix_received_at", + ["source_pubkey_prefix", "received_at"], + unique=False, + ) + + # ### end Alembic commands ### + + +def downgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + with op.batch_alter_table("raw_packets", schema=None) as batch_op: + batch_op.drop_index("ix_raw_packets_source_pubkey_prefix_received_at") + batch_op.drop_index(batch_op.f("ix_raw_packets_source_pubkey_prefix")) + batch_op.drop_index("ix_raw_packets_received_at") + batch_op.drop_index(batch_op.f("ix_raw_packets_packet_type")) + batch_op.drop_index(batch_op.f("ix_raw_packets_packet_hash")) + batch_op.drop_index(batch_op.f("ix_raw_packets_observer_node_id")) + batch_op.drop_index("ix_raw_packets_event_type_received_at") + batch_op.drop_index(batch_op.f("ix_raw_packets_event_type")) + batch_op.drop_index("ix_raw_packets_channel_idx_received_at") + batch_op.drop_index(batch_op.f("ix_raw_packets_channel_idx")) + + op.drop_table("raw_packets") + # ### end Alembic commands ### diff --git a/alembic/versions/20260612_2111_e8eb47c49062_add_packet_hash_to_messages_and_.py b/alembic/versions/20260612_2111_e8eb47c49062_add_packet_hash_to_messages_and_.py new file mode 100644 index 0000000..1ddcfd9 --- /dev/null +++ b/alembic/versions/20260612_2111_e8eb47c49062_add_packet_hash_to_messages_and_.py @@ -0,0 +1,52 @@ +"""add packet_hash to messages and advertisements + +Revision ID: e8eb47c49062 +Revises: e9f0c4079540 +Create Date: 2026-06-12 21:11:17.664733+00:00 + +""" + +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa + +# revision identifiers, used by Alembic. +revision: str = "e8eb47c49062" +down_revision: Union[str, None] = "e9f0c4079540" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + with op.batch_alter_table("advertisements", schema=None) as batch_op: + batch_op.add_column( + sa.Column("packet_hash", sa.String(length=32), nullable=True) + ) + batch_op.create_index( + batch_op.f("ix_advertisements_packet_hash"), ["packet_hash"], unique=False + ) + + with op.batch_alter_table("messages", schema=None) as batch_op: + batch_op.add_column( + sa.Column("packet_hash", sa.String(length=32), nullable=True) + ) + batch_op.create_index( + batch_op.f("ix_messages_packet_hash"), ["packet_hash"], unique=False + ) + + # ### end Alembic commands ### + + +def downgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + with op.batch_alter_table("messages", schema=None) as batch_op: + batch_op.drop_index(batch_op.f("ix_messages_packet_hash")) + batch_op.drop_column("packet_hash") + + with op.batch_alter_table("advertisements", schema=None) as batch_op: + batch_op.drop_index(batch_op.f("ix_advertisements_packet_hash")) + batch_op.drop_column("packet_hash") + + # ### end Alembic commands ### diff --git a/docker-compose.yml b/docker-compose.yml index f4b8341..f714755 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -197,6 +197,10 @@ services: - DATA_RETENTION_INTERVAL_HOURS=${DATA_RETENTION_INTERVAL_HOURS:-24} - NODE_CLEANUP_ENABLED=${NODE_CLEANUP_ENABLED:-true} - NODE_CLEANUP_DAYS=${NODE_CLEANUP_DAYS:-30} + # Raw packet capture (derived from FEATURE_PACKETS so one var drives both + # capture and the web Packets page). Retention defaults to DATA_RETENTION_DAYS. + - RAW_PACKET_CAPTURE_ENABLED=${FEATURE_PACKETS:-false} + - RAW_PACKET_RETENTION_DAYS=${RAW_PACKET_RETENTION_DAYS:-${DATA_RETENTION_DAYS:-30}} command: ["collector"] healthcheck: test: ["CMD", "meshcore-hub", "health", "collector"] @@ -340,6 +344,8 @@ services: - FEATURE_PAGES=${FEATURE_PAGES:-true} - FEATURE_CHANNELS=${FEATURE_CHANNELS:-true} - FEATURE_RADIO_CONFIG=${FEATURE_RADIO_CONFIG:-true} + # Packets page is off by default; enabling it also turns on collector capture. + - FEATURE_PACKETS=${FEATURE_PACKETS:-false} command: ["web"] healthcheck: test: diff --git a/docs/letsmesh.md b/docs/letsmesh.md index 1900bd8..5dc081b 100644 --- a/docs/letsmesh.md +++ b/docs/letsmesh.md @@ -16,6 +16,31 @@ The collector subscribes to packets published by [meshcore-packet-capture](https - Decoder payload type `1` can map to native response events (`telemetry_response`, `battery`, `path_updated`, `status_response`) when decrypted structured content is available. - `packet_type=5` packets are mapped to `channel_msg_recv`. - `packet_type=1`, `2`, and `7` packets are mapped to `contact_msg_recv` when decryptable text is available. +- Packets that no structured handler claims are no longer all labelled `letsmesh_packet`. They are classified by their MeshCore payload type so the `event_type` is specific: + + | Payload type | `event_type` | + |---|---| + | `0x00 REQ` | `req` | + | `0x01 RESPONSE` | `response` | + | `0x02 TXT_MSG` (undecryptable) | `encrypted_direct` | + | `0x03 ACK` | `ack` | + | `0x04 ADVERT` (no identity) | `advert` | + | `0x05 GRP_TXT` (unknown key) | `encrypted_channel` | + | `0x06 GRP_DATA` | `grp_data` | + | `0x07 ANON_REQ` | `anon_req` | + | `0x08 PATH` | `path` | + | `0x09 TRACE` | `trace` | + | `0x0A MULTIPART` | `multipart` | + | `0x0B CONTROL` | `control` | + | `0x0F RAW_CUSTOM` | `raw_custom` | + + `letsmesh_packet` is retained only as a safety net for packets whose payload type cannot be resolved. Reaching these fallbacks for `TXT_MSG`/`GRP_TXT` means the payload did not decrypt, hence the `encrypted_*` labels (decryptable ones become `contact_msg_recv` / `channel_msg_recv`). + +## Raw Packet Capture + +- When `RAW_PACKET_CAPTURE_ENABLED` is set (Compose derives it from `FEATURE_PACKETS`), every packet on the `packets` feed is also stored verbatim in the `raw_packets` table — one row per observer reception, independent of structured classification. The `status` and `internal` feeds carry no on-air `raw` hex and are **not** captured as raw packets. +- Capture reuses the single decode the normalizer already performs (the decoder caches per raw hex), so it adds only an insert plus an observer upsert to the ingest path. +- The `/packets` API and Packets page apply channel-visibility rules: channel-message packets on a channel above the viewer's role are returned **metadata-only with the payload redacted** (`redacted=true`, `raw_hex`/`decoded` nulled), not hidden. Non-channel and visible-channel packets are returned in full. ## Channel Keys diff --git a/docs/plans/20260612-2014-raw-packets-feature/plan.md b/docs/plans/20260612-2014-raw-packets-feature/plan.md new file mode 100644 index 0000000..36c65e8 --- /dev/null +++ b/docs/plans/20260612-2014-raw-packets-feature/plan.md @@ -0,0 +1,509 @@ +# Raw Packets: Capture, Store, and Browse Decoded Wire Packets + +## Summary + +Add a first-class **Raw Packets** feature that captures every inbound MeshCore +packet exactly as it arrives over the LetsMesh `packets` feed, independent of how +the collector later classifies it. Today the collector decodes each packet and +routes it to a structured handler (advertisement, message, trace, telemetry, +contact); the original on-air bytes are then discarded for everything that +matches a structured handler, and only unclassified leftovers reach the +`events_log` table. There is no complete, searchable record of raw traffic. + +This plan introduces a `raw_packets` table populated **unconditionally** at +ingest from the already-decoded packet, a `/packets` API with rich filtering and +channel-visibility-aware redaction, and a new SPA **Packets** page (disabled by +default) for display/search/filter. The structured tables remain derived views, +linkable back to raw packets by `packet_hash`. + +## Background & Motivation + +### Current State + +1. The collector subscribes to three LetsMesh upload feeds + (`subscriber.py:512`): `/+/+/packets`, `/status`, `/internal`. +2. Each `packets` payload carries the on-air packet as hex in `payload["raw"]`. +3. `LetsMeshNormalizer._normalize_letsmesh_event` (`letsmesh_normalizer.py:23`) + decodes the packet once via `LetsMeshPacketDecoder.decode_payload` + (`letsmesh_decoder.py:219`) and classifies it into an event type: message, + advertisement, trace, contact, telemetry, path/status, or the catch-all + `letsmesh_packet`. +4. The normalized event is dispatched to a handler (`subscriber.py:213`). + Structured handlers (`handlers/__init__.py:26-31`) extract specific columns + into their own tables and **drop** the raw hex. Only unmatched/informational + events fall through to `handle_event_log`, which stores the full payload JSON + (raw hex included) in `events_log`. + +### Problems + +- **Lossy capture**: The most interesting packets (adverts, channel messages, + traces, telemetry) are parsed and their raw form is not retained. The raw + record only survives for the packets that *don't* map to a structured handler. +- **No raw browse/search**: Operators debugging the mesh (decode failures, + unexpected packet types, path/SNR anomalies) have no way to view or filter the + actual wire packets. +- **No per-observer record**: Structured tables dedupe by event hash across + observers; there is no one-row-per-reception log of what each observer heard. + +### Why Now + +The decoder already runs on every packet and already attempts decryption with the +DB-backed channel key store (`letsmesh_decoder.py:240`). The channel-visibility +machinery (`channel_visibility.py`) and API-key auth (`auth.py`) already exist and +are used by the messages feed. Capturing the raw packet is therefore a cheap +addition that reuses the existing decode, key store, and visibility model — no new +crypto and no second decode. + +## Goals + +- Introduce a `RawPacket` SQLAlchemy model capturing every `packets`-feed packet, + one row per observer reception (no dedup). +- Persist the raw hex, classification, decoded summary, channel index, source + identity, and link metadata (`packet_hash`) at ingest, reusing the decode the + normalizer already performs. +- Add a separate `RAW_PACKET_RETENTION_DAYS` knob (defaulting to the global + retention default) and wire `raw_packets` into the existing cleanup scheduler. +- Provide a `/packets` API with filtering, sorting, and pagination consistent + with `/advertisements` and `/messages`. +- Apply channel-visibility rules to channel-message packets: packets on channels + above the user's role are returned as **metadata-only with the payload + redacted** (not hidden); `raw_hex` is returned in full for any non-redacted + packet. +- Provide a SPA **Packets** page behind a feature flag that is **off by default**. + +## Non-Goals + +- Capturing the `status` / `internal` feeds (they carry station housekeeping, not + on-air packets — see Requirements). These already land in `events_log`. +- Deduplicating raw packets across observers (explicitly one row per reception). +- Changing the `meshcoredecoder` library or any decryption logic. +- Decrypting direct/contact (node-to-node ECDH) messages — the hub holds no node + private keys; those remain ciphertext with no decoded text. +- Encrypting `raw_hex` at rest, or per-user (non-role) access control. +- Backfilling raw packets for traffic received before this feature ships. + +## Requirements + +### Functional Requirements + +- **FR-1**: A `RawPacket` database model with fields: + - `id` (UUID primary key) + - `observer_node_id` (FK `nodes.id`, `ondelete=SET NULL`, nullable, indexed) — + the receiving interface, resolved from the MQTT topic public key + - `packet_hash` (String(32), nullable, indexed) — LetsMesh `hash`; links rows to + structured-table records and groups multi-observer receptions + - `raw_hex` (Text, nullable) — the on-air bytes from `payload["raw"]` + - `packet_type` (Integer, nullable, indexed) — wire packet type + - `payload_type` (Integer, nullable) — decoder payload type + - `event_type` (String(50), nullable, indexed) — how the collector classified + the packet (`advertisement`, `channel_msg_recv`, `contact_msg_recv`, + `trace_data`, `telemetry_response`, `letsmesh_packet`, ...) + - `channel_idx` (Integer, nullable, indexed) — `int(channelHash, 16)` for + channel-message packets; drives visibility filtering. `NULL` for non-channel + packets. + - `source_pubkey_prefix` (String(12), nullable, indexed) — sender prefix derived + from the decoder `sourceHash` / `senderPublicKey`, for efficient + "packets from this node" filtering + - `route_type` (String(20), nullable) + - `path_len` (Integer, nullable) + - `snr` (Float, nullable) + - `decoded` (JSON, nullable) — decoder summary, so the detail view needs no + re-decode + - `received_at` (DateTime(tz), default `utc_now`, indexed) + - `created_at`, `updated_at` (timestamps via `TimestampMixin`) +- **FR-2**: When capture is enabled (FR-13), for **every** message on the + `packets` feed the collector writes exactly one `RawPacket` row, regardless of + how the packet is subsequently classified or whether a structured handler also + persists it. Capture happens before/independent of structured dispatch so the + table is complete. When capture is disabled, no `RawPacket` rows are written and + the structured handlers continue unchanged. +- **FR-3**: The `status` and `internal` feeds are **not** captured as raw packets + (they carry no `raw` hex). No change to their existing `letsmesh_status` / + `letsmesh_internal` handling. +- **FR-4**: No deduplication. If N observers report the same packet, N rows are + stored (grouped by `packet_hash`). +- **FR-5**: `RawPacket` rows are subject to data-retention cleanup using a + dedicated `RAW_PACKET_RETENTION_DAYS` setting, defaulting to the existing global + `DATA_RETENTION_DAYS` value. Cleanup runs in the existing scheduler. +- **FR-6**: `GET /packets` lists raw packets with the following filters, all + optional and combinable: + - `search` (string) — `ilike` across `packet_hash`, `raw_hex`, and observer + name/public key + - `event_type` (string, comma-separated allowed) — classification filter + - `packet_type` (int, comma-separated allowed) — wire type filter + - `channel_idx` (int) — single channel filter + - `route_type` (string, comma-separated; `all`/`none`/`""` disables) — same + semantics as `/advertisements` + - `public_key` / `pubkey_prefix` (string) — filter by `source_pubkey_prefix` + - `observed_by` (list[str]) — filter by receiver node public keys + - `decryptable` (bool) — only packets whose `decoded` contains decrypted text + (vs ciphertext-only) + - `min_snr` / `max_snr` (float) — SNR range + - `min_path_len` / `max_path_len` (int) — hop-count range + - `redacted` (bool) — include only / exclude redacted (metadata-only) rows + - `since` / `until` (datetime) — `received_at` window + - `sort` / `order` — `sort` in `VALID_PACKET_SORT_COLUMNS`, `order` in + `asc`/`desc` + - `limit` (1–100, default 50) / `offset` (>= 0) +- **FR-7**: `GET /packets/{id}` returns a single raw packet with its decoded + summary, subject to the same visibility/redaction rules. +- **FR-8**: **Channel-visibility redaction**. Using the user's resolved role + (`channel_visibility.resolve_user_role` + `get_visible_channel_indices`): + - Non-channel packets (`channel_idx IS NULL`): returned in full. + - Channel packets whose `channel_idx` is visible to the role: returned in full, + including `raw_hex` and `decoded`. + - Channel packets whose `channel_idx` is **above** the role's level: returned + with `redacted=true` and `raw_hex`, `decoded`, and any decoded text nulled + out. Non-sensitive metadata is retained: `packet_hash`, `packet_type`, + `event_type`, `channel_idx`, `route_type`, `path_len`, `snr`, + `observer`, `received_at`. + - The built-in Public channel (idx 17) is always visible. +- **FR-9**: `raw_hex` is **always included** in the response for any non-redacted + packet the user is permitted to see (no extra role gate beyond visibility). +- **FR-10**: A SPA **Packets** page at `/packets`, controlled by a + `FEATURE_PACKETS` flag that defaults to **off**. The page provides: + - text search box (`search`) + - event-type chips / dropdown (`event_type`) + - channel dropdown (`channel_idx`) showing only channels the role can see + - observer multi-select (`observed_by`) + - route-type toggle group (`route_type`) + - time-range picker (`since` / `until`) + - a collapsible "advanced" panel for `packet_type`, SNR range, path-len range, + `decryptable`, `redacted` + - **responsive layout matching the other list pages**: a zebra `` on + desktop (`hidden lg:block`) with `sortableTableHeader` columns, and a stacked + **card** list on mobile (`lg:hidden`) with a `mobileSortSelect` + - pagination + - a per-row lock badge when `redacted=true` +- **FR-11**: A node-detail "packets from this node" view can reuse `/packets` + with `pubkey_prefix` (out of scope to build the node-detail UI here, but the + filter and index must support it). +- **FR-12**: Navigation entry points for the Packets page, all gated on + `features.packets` (so they stay hidden while the flag is off by default), + inserted **immediately after Messages** on every surface (i.e. `... → Messages → + Packets → Map → Members ...`), consistently across all three: + - **Homepage hero nav card** in `home.js` (`renderNavCard`) with a + `--color-packets` accent + - **Sidebar / mobile menu**: the dynamic nav in `app.js` and the static nav in + `spa.html` + - a matching page title in `updatePageTitle` +- **FR-13**: A **collector-side capture flag** `RAW_PACKET_CAPTURE_ENABLED` + (default `false`) controls whether the collector writes `RawPacket` rows. It is + independent of the web `FEATURE_PACKETS` flag (the collector and web are separate + processes with separate settings), but Compose links them by default so a single + source var drives both: + `RAW_PACKET_CAPTURE_ENABLED=${FEATURE_PACKETS:-false}`. When disabled, the + collector performs no raw-packet inserts (eliminating the write-amplification + cost) while normalization/structured handling is unaffected. Capture has no + backfill, so the web page only shows packets captured while the flag was on. + Retention cleanup of `raw_packets` runs regardless of this flag, so turning + capture off lets existing rows drain via `RAW_PACKET_RETENTION_DAYS`. + +### Technical Requirements + +- **TR-1**: New model `RawPacket` in + `src/meshcore_hub/common/models/raw_packet.py`, exported from + `models/__init__.py`. Follows the `Advertisement` pattern (`UUIDMixin`, + `TimestampMixin`). Single-column indexes: `received_at`, `event_type`, + `packet_hash`, `channel_idx`, `source_pubkey_prefix`, `observer_node_id`. + Composite indexes to serve the common "filter then sort by newest" pattern + without a full scan + filesort: `(event_type, received_at)`, + `(channel_idx, received_at)`, `(source_pubkey_prefix, received_at)`. See TR-17 + for the write-cost trade-off. +- **TR-2**: Alembic migration to create the `raw_packets` table, datestamped per + convention (`alembic/versions/2026XXXX_XXXX_add_raw_packets.py`). +- **TR-3**: New handler `src/meshcore_hub/collector/handlers/raw_packet.py` + exposing `store_raw_packet(...)`. It performs the find-or-create observer node + pattern used by `handle_event_log` (`handlers/event_log.py:34-50`) and inserts a + `RawPacket` row. It does not dedup. +- **TR-4**: Capture hook in `Subscriber._handle_mqtt_message` + (`subscriber.py:189`), guarded by `self._raw_packet_capture_enabled` (a cheap + boolean short-circuit when disabled — note it saves the insert, not the decode, + which normalization performs regardless). When capture is enabled and the parsed + feed is `packets`, call + `store_raw_packet` with the raw payload, the decoder output already produced by + the normalizer, the resolved `event_type`, and the observer public key. To avoid + decoding twice, `_normalize_letsmesh_event` is refactored to also return (or the + subscriber to reuse) the `decoded_packet` and `event_type`; capture uses that + single decode. Capture failures are logged and never block event dispatch. +- **TR-5**: Channel index and source identity are derived at capture time: + `channel_idx = int(channelHash, 16)` from + `_extract_letsmesh_decoder_channel_hash`; `source_pubkey_prefix` from the + decoder `sourceHash` / `senderPublicKey` via the existing + `_normalize_pubkey_prefix` helper. +- **TR-6**: `RAW_PACKET_RETENTION_DAYS` config field added to `CollectorSettings` + in `config.py` (alongside `data_retention_*` at `config.py:118`), defaulting to + the value of `data_retention_days`. Threaded through `create_subscriber` / + `run_collector` like the other cleanup params. +- **TR-6a**: `raw_packet_capture_enabled` (`RAW_PACKET_CAPTURE_ENABLED`) bool field + added to `CollectorSettings`, default `false`, following the + `data_retention_enabled` / `node_cleanup_enabled` convention. Threaded through + `create_subscriber` / `run_collector` to `Subscriber.__init__` and stored as + `self._raw_packet_capture_enabled` for the TR-4 guard. The collector logs the + capture state at startup. Compose wires it on the **collector** service as + `RAW_PACKET_CAPTURE_ENABLED=${FEATURE_PACKETS:-false}` (in `docker-compose.yml` + and any prod/traefik overrides); `FEATURE_PACKETS=${FEATURE_PACKETS:-false}` + stays on the web service. Both env vars added to `.env.example`. +- **TR-7**: `collector/cleanup.py` gains a `raw_packets_deleted` stat and a cleanup + step for `RawPacket`. Because `_cleanup_table` (`cleanup.py:185`) deletes by + `created_at < cutoff` with a single retention value, add support for a per-table + retention override (or a dedicated call) so `raw_packets` can use + `RAW_PACKET_RETENTION_DAYS` independently of `DATA_RETENTION_DAYS`. The override + falls back to `DATA_RETENTION_DAYS` when `RAW_PACKET_RETENTION_DAYS` is unset. + Add `RawPacket.observer_node_id` to the observer union in + `_observer_node_id_union()` (`cleanup.py:124`) so a node that appears only as a + raw-packet observer still counts as an observer, and `recompute_observer_flags` + does not clear its `is_observer` flag while it has live raw packets. +- **TR-8**: Pydantic schemas `RawPacketRead` / `RawPacketList` in + `src/meshcore_hub/common/schemas/` (mirroring `messages.py` + `AdvertisementRead`/`AdvertisementList`). `RawPacketRead` includes a + `redacted: bool` field. +- **TR-9**: New route module `src/meshcore_hub/api/routes/raw_packets.py`, + registered in `api/routes/__init__.py` with `prefix="/packets"`, `tags=["Packets"]`. + Uses `RequireRead`, `DbSession`, and `@cached("packets", + key_builder=_packets_key_builder)`. `_packets_key_builder` includes the resolved + role (`resolve_user_role(request) or "anonymous"`) in the cache key — exactly + like `messages.py:30` — so role-redacted responses are **never** served across + roles. TTL uses the default `redis_cache_ttl` setting (`REDIS_CACHE_TTL`, default + 30s); no dedicated TTL var. `VALID_PACKET_SORT_COLUMNS = + {"time", "event_type", "packet_type", "snr", "path_len"}`, default `time`/`desc`. + Observer hydration follows the `aliased(Node)` + `selectinload(Node.tags)` pattern + in `advertisements.py`. +- **TR-10**: The route applies redaction (FR-8) after fetching rows: compute the + visible channel-index set once via `get_visible_channel_indices`, then for each + channel-message row above the level, null the sensitive fields and set + `redacted=true`. The SQL query is **not** pre-filtered by visibility (rows are + returned but redacted), so pagination counts are stable across roles. +- **TR-11**: Web proxy access in `web/app.py` `_build_endpoint_access()`: + `"v1/packets": { "GET": _OPEN }` (read gated only by `RequireRead` / + channel-visibility redaction, consistent with `v1/messages` and + `v1/advertisements`). +- **TR-12**: `FEATURE_PACKETS` feature flag in `WebSettings` (`config.py:397`), + **default `False`**, registered in the `features` property (`config.py:428`) as + `"packets": self.feature_packets` (no OIDC gate). +- **TR-13**: New SPA page module + `src/meshcore_hub/web/static/js/spa/pages/packets.js` (lit-html), modelled on + `pages/advertisements.js`, including its **responsive layout**: a `hidden + lg:block` zebra `
` with `sortableTableHeader` on desktop and a + `lg:hidden` stacked-card list with `mobileSortSelect` on mobile. Wire into + `app.js`: lazy import entry, route + `router.addRoute('/packets', pageHandler(pages.packets))` guarded by + `features.packets`, dynamic nav item (`items.push`), and page title + (`updatePageTitle`). Renders the `redacted` lock badge. +- **TR-13a**: Add the Packets entry to **all three nav surfaces**, each guarded by + `features.packets` and inserted immediately **after the Messages entry** (before + Map): the static `spa.html` sidebar/mobile `
  • ` (`{% if features.packets %}`), + the dynamic `app.js` nav (`items.push`), and the `home.js` hero card grid + (`renderNavCard`). +- **TR-13b**: Correct the `home.js` hero-card order to match the sidebar/dynamic + nav: it currently renders **Members before Map**; reorder to **Map before + Members** so every surface follows the canonical order `Dashboard → Nodes → + Advertisements → Channels → Messages → Packets → Map → Members`. Add a packets + icon function to `icons.js` (imported by `app.js`, `home.js`, and `packets.js`). + Add `--color-packets` (light + dark blocks), a `.nav-icon-packets` rule, and the + `.navbar .menu li:has(.nav-icon-packets)` accent in `app.css`. +- **TR-14**: i18n keys for packets UI strings (including `entities.packets`, + `entities.packet`, filter labels, and the redacted badge) added to **both** + locale files — `en.json` and `nl.json` — and documented per the existing i18n + docs. +- **TR-15**: `node build.js` rebuild of the SPA bundle after frontend changes. +- **TR-16**: Documentation updates: + - `AGENTS.md` — new model, API route, `FEATURE_PACKETS`, + `RAW_PACKET_RETENTION_DAYS`. + - `SCHEMAS.md` — the `raw_packets` table and the `RawPacketRead` shape. + - `.env.example` — add `RAW_PACKET_RETENTION_DAYS` and + `RAW_PACKET_CAPTURE_ENABLED` near `DATA_RETENTION_DAYS` (commented, noting the + retention default = `DATA_RETENTION_DAYS` and capture default = `false`), and + `# FEATURE_PACKETS=false` in the feature-flags block, with a comment that + Compose derives `RAW_PACKET_CAPTURE_ENABLED` from `FEATURE_PACKETS`. + - `docker-compose.yml` (+ prod/traefik overrides) — `RAW_PACKET_CAPTURE_ENABLED=${FEATURE_PACKETS:-false}` + on the collector service; `FEATURE_PACKETS=${FEATURE_PACKETS:-false}` on the + web service. + - `docs/upgrading.md` — add a new **`## v0.13.0`** section at the top (above + `v0.12.0`) with the upgrade steps (see TR-18). + - `docs/letsmesh.md` — already covered in Phase 6. +- **TR-17**: **Performance / indexing.** The `raw_packets` table is the + highest-volume table in the system (one row per packet per observer), so: + - The composite indexes in TR-1 cover the dominant query shapes (filter by + `event_type` / `channel_idx` / `source_pubkey_prefix`, sorted by + `received_at DESC`). Do **not** add further composite indexes speculatively — + each one is paid on every insert in the hot ingest path. + - `search` is an unanchored `ilike '%term%'` over `raw_hex` / `packet_hash` and + **cannot use an index** (full scan). The route should require it to combine + with a narrowing filter (a `since` default window) so substring search runs + against a bounded row set; document this limitation. Hash lookups should use a + prefix (`ilike 'abc%'`) where possible so the `packet_hash` index applies. + - Pagination uses an exact `COUNT(*)` over the filtered subquery (matching the + other list routes); on a large table this is the most expensive part of the + request. Accept it for parity now; note approximate/capped counts as a possible + follow-up if it becomes a hotspot. + - **Write amplification:** capture adds one `INSERT` per packet to the ingest + path. SQLite is single-writer, so this competes with the structured handlers' + writes already happening per packet. The capture handler must keep its + transaction minimal (single insert + observer upsert, no extra reads) and must + not hold the session longer than needed. The separate (and typically shorter) + `RAW_PACKET_RETENTION_DAYS` is the primary mechanism keeping the table — and + its index size — bounded. +- **TR-18**: `docs/upgrading.md` **v0.13.0** section content: + - Run `meshcore-hub db upgrade` to create the `raw_packets` table. + - New optional env vars, all safe to omit: `FEATURE_PACKETS` (defaults to + `false` — Packets page and nav hidden until enabled), `RAW_PACKET_CAPTURE_ENABLED` + (collector capture; Compose derives it from `FEATURE_PACKETS`, default `false`), + and `RAW_PACKET_RETENTION_DAYS` (defaults to `DATA_RETENTION_DAYS`). + - Explain the capture↔page split: setting `FEATURE_PACKETS=true` enables both + capture and the page via the Compose wiring; advanced operators can set the two + independently. No backfill — only packets captured after enabling appear. + - Note the `raw_packets` table grows fastest of all; recommend tuning + `RAW_PACKET_RETENTION_DAYS` for busy meshes / constrained storage, and that + disabling capture lets it drain via retention. + - Note Redis caching of `/packets` is role-aware and honours the existing + `REDIS_CACHE_TTL`. + +## Implementation Plan + +### Phase 1: Model & Migration + +- Create `src/meshcore_hub/common/models/raw_packet.py` with the fields in FR-1 / + TR-1, following the `Advertisement` model. +- Export `RawPacket` from `models/__init__.py`. +- Generate the Alembic migration for the `raw_packets` table with all indexes. +- Unit tests for the model (column presence, defaults, indexes). + +### Phase 2: Capture at Ingest + +- Add `store_raw_packet(...)` in + `src/meshcore_hub/collector/handlers/raw_packet.py` (find-or-create observer + node, derive `channel_idx` and `source_pubkey_prefix`, insert one row, no dedup). +- Refactor `_normalize_letsmesh_event` / `_handle_mqtt_message` so the single + decode performed during normalization is reused for capture (return or expose + `decoded_packet` + resolved `event_type`). +- Add `raw_packet_capture_enabled` to `CollectorSettings`, thread it to + `Subscriber` as `self._raw_packet_capture_enabled`, and guard the capture hook + on it (log the state at startup). +- Call `store_raw_packet` for the `packets` feed only (when capture is enabled), + before/independent of structured dispatch. Wrap in try/except so capture never + blocks event handling. +- Tests: capture writes one row per packet; one row per observer for the same + hash; `channel_idx` / `source_pubkey_prefix` derivation; status/internal feeds + produce no raw rows; **capture disabled writes no rows while structured handlers + still run**. + +### Phase 3: Retention & Config + +- Add `RAW_PACKET_RETENTION_DAYS` to `CollectorSettings`, defaulting to + `data_retention_days`. +- Thread the value through `create_subscriber` / `run_collector` and the cleanup + scheduler. +- Extend `cleanup.py` with a `raw_packets_deleted` stat and a `RawPacket` cleanup + step using the dedicated retention value (per-table retention override on + `_cleanup_table`, or a dedicated call). +- Add `RawPacket.observer_node_id` to `_observer_node_id_union()` so raw-packet-only + observers retain their `is_observer` flag in `recompute_observer_flags`. +- Tests: raw packets older than `RAW_PACKET_RETENTION_DAYS` are deleted; default + falls back to the global retention value; stat is reported; a node observing + only raw packets keeps `is_observer=true` until those packets are pruned. + +### Phase 4: Schemas & API + +- Add `RawPacketRead` / `RawPacketList` schemas (with `redacted: bool`). +- Create `api/routes/raw_packets.py`: + - `GET /packets` with all filters in FR-6, sort whitelist, pagination, observer + hydration, and `@cached("packets", key_builder=_packets_key_builder)` — the + key builder folds the resolved role into the cache key (TR-9) so redacted + responses don't leak across roles; default TTL. + - `GET /packets/{id}`. + - Redaction pass (FR-8) using `get_visible_channel_indices`; `raw_hex` always + present for non-redacted rows (FR-9). +- Register the router in `api/routes/__init__.py` (`prefix="/packets"`). +- Add `"v1/packets": { "GET": _OPEN }` to `_build_endpoint_access()`. +- Tests: each filter; sort/pagination; redaction for a low role viewing a + member-channel packet (`redacted=true`, null `raw_hex`/`decoded`); non-channel + packets never redacted; Public (17) always visible. + +### Phase 5: Web Packets Page + +- Add `FEATURE_PACKETS` to `WebSettings` (default `False`) and the `features` map. +- Create `pages/packets.js` (lit-html) modelled on `advertisements.js`, with the + **desktop zebra table (`hidden lg:block`) + mobile stacked cards (`lg:hidden`)** + responsive split using `sortableTableHeader` / `mobileSortSelect`: search box, + event-type and channel filters, observer multi-select, route-type toggle, + time-range picker, collapsible advanced filters (`packet_type`, SNR range, + path-len range, `decryptable`, `redacted`), pagination, and a `redacted` lock + badge. +- Wire `app.js`: lazy import, route guarded by `features.packets`, page title, + i18n keys; add a packets icon function in `icons.js`. +- Add the Packets nav entry after Messages on **all three surfaces** (guarded by + `features.packets`): `spa.html` sidebar/mobile `
  • `, `app.js` dynamic nav, and + the `home.js` hero card grid (`renderNavCard`). +- Reorder the `home.js` hero cards so **Map precedes Members**, matching the + sidebar/dynamic nav (TR-13b). +- Add `--color-packets` (light + dark), `.nav-icon-packets`, and the navbar + `:has(.nav-icon-packets)` accent in `app.css`. +- Add i18n strings to `en.json` (incl. `entities.packets`, `entities.packet`) and + locale stubs. +- `node build.js` to rebuild the bundle. +- Tests in the web test suite (nav/route hidden when the flag is off, list render, + redacted badge). + +### Phase 6: Docs + +- Update `AGENTS.md` (model, route, `FEATURE_PACKETS`, `RAW_PACKET_RETENTION_DAYS`). +- Update `SCHEMAS.md` with the `raw_packets` table and `RawPacketRead`. +- Update `.env.example`: add `RAW_PACKET_RETENTION_DAYS` (commented, near + `DATA_RETENTION_DAYS`, noting the `DATA_RETENTION_DAYS` default) and + `# FEATURE_PACKETS=false` in the feature-flags block. +- Add a new `## v0.13.0` section to `docs/upgrading.md` (above `v0.12.0`) with the + TR-18 content: `db upgrade`, the two new optional env vars and their defaults, + the no-backfill / table-growth note, and the role-aware cache note. +- Update `docs/letsmesh.md` to note raw-packet capture is `packets`-feed only and + that channel-restricted packets are returned metadata-only/redacted. +- Confirm `entities.packets` / `entities.packet` and filter labels exist in both + `en.json` and `nl.json`. + +## Open Decisions (Resolved) + +- **Retention**: separate `RAW_PACKET_RETENTION_DAYS`, default = global + `DATA_RETENTION_DAYS`. (Resolved.) +- **Feeds**: `packets` only; `status`/`internal` excluded (no `raw` hex). + (Resolved.) +- **Dedup**: none — one row per observer reception. (Resolved.) +- **Feature flag**: `FEATURE_PACKETS` defaults to **off**. (Resolved.) +- **Restricted channel packets**: shown as metadata-only with payload redacted + (`redacted=true`, null `raw_hex`/`decoded`), not hidden. (Resolved.) +- **`raw_hex` exposure**: always returned for non-redacted, visible packets. + (Resolved.) + +## Review + +**Status**: Draft — pending review. + +## References + +- `src/meshcore_hub/collector/subscriber.py:189` — `_handle_mqtt_message` (capture + hook site); `:512` — feed subscriptions +- `src/meshcore_hub/collector/letsmesh_normalizer.py:23` — + `_normalize_letsmesh_event` (single-decode + classification to reuse) +- `src/meshcore_hub/collector/letsmesh_decoder.py:219` — `decode_payload`; + `:174` — channel hash computation +- `src/meshcore_hub/collector/handlers/event_log.py:34-50` — find-or-create + observer node pattern to follow in the raw-packet handler +- `src/meshcore_hub/collector/handlers/__init__.py:26-37` — handler registration / + structured vs informational split +- `src/meshcore_hub/common/models/advertisement.py` — model pattern (UUIDMixin, + TimestampMixin, indexes) +- `src/meshcore_hub/collector/cleanup.py:54,185` — `cleanup_old_data` / + `_cleanup_table` (retention to extend for per-table override) +- `src/meshcore_hub/common/config.py:118-135` — data-retention settings; `:397-448` + — feature flags and the `features` map +- `src/meshcore_hub/api/routes/advertisements.py` — list/detail route pattern + (aliased node joins, observer hydration, sort whitelist, pagination, `@cached`) +- `src/meshcore_hub/api/routes/messages.py:53,83-106` — `channel_idx` filter and + channel-visibility filtering to mirror +- `src/meshcore_hub/api/channel_visibility.py` — `resolve_user_role`, + `get_visible_channel_indices`, `get_all_known_channel_indices` +- `src/meshcore_hub/api/auth.py:52,145` — `require_read` / `RequireRead` +- `src/meshcore_hub/web/app.py:68-129` — `_build_endpoint_access()` proxy access map +- `src/meshcore_hub/web/static/js/spa/pages/advertisements.js` — SPA list page to + model `packets.js` on +- `src/meshcore_hub/web/static/js/spa/app.js:21,64-99,161` — lazy import map, route + registration, nav, page titles diff --git a/docs/plans/20260612-2014-raw-packets-feature/tasks.md b/docs/plans/20260612-2014-raw-packets-feature/tasks.md new file mode 100644 index 0000000..04005b1 --- /dev/null +++ b/docs/plans/20260612-2014-raw-packets-feature/tasks.md @@ -0,0 +1,192 @@ +# Tasks: Raw Packets — Capture, Store, and Browse Decoded Wire Packets + +> Generated from `plan.md` on 2026-06-12 + +## 1. Database Model & Migration + +- [x] 1.1 Create `RawPacket` SQLAlchemy model + - [x] 1.1.1 Create `RawPacket` class in `src/meshcore_hub/common/models/raw_packet.py` following the `Advertisement` pattern (`Base`, `UUIDMixin`, `TimestampMixin`) + - [x] 1.1.2 Fields: `observer_node_id` (FK `nodes.id`, `ondelete="SET NULL"`, nullable), `packet_hash` (String(32), nullable), `raw_hex` (Text, nullable), `packet_type` (Integer, nullable), `payload_type` (Integer, nullable), `event_type` (String(50), nullable), `channel_idx` (Integer, nullable), `source_pubkey_prefix` (String(12), nullable), `route_type` (String(20), nullable), `path_len` (Integer, nullable), `snr` (Float, nullable), `decoded` (JSON, nullable), `received_at` (DateTime(tz), default `utc_now`) + - [x] 1.1.3 Single-column indexes on `received_at`, `event_type`, `packet_hash`, `channel_idx`, `source_pubkey_prefix`, `observer_node_id` + - [x] 1.1.4 Composite indexes `(event_type, received_at)`, `(channel_idx, received_at)`, `(source_pubkey_prefix, received_at)` for the "filter then sort by newest" pattern + - [x] 1.1.5 Add `__repr__` + - [x] 1.1.6 Export `RawPacket` from `models/__init__.py` + +- [x] 1.2 Generate Alembic migration + - [x] 1.2.1 Run `meshcore-hub db revision --autogenerate -m "add raw_packets table"` + - [x] 1.2.2 Review generated migration: all columns, FK, and every index (single + composite) present + - [x] 1.2.3 Test migration: `meshcore-hub db upgrade` and verify table + indexes created + +- [x] 1.3 Write unit tests for the model + - [x] 1.3.1 Test `RawPacket` instantiation and defaults (`received_at`, timestamps) + - [x] 1.3.2 Test nullable columns accept `None` + - [x] 1.3.3 Test index presence via table metadata + +## 2. Collector Capture & Capture Flag + +- [x] 2.1 Add `raw_packet_capture_enabled` setting + - [x] 2.1.1 Add `raw_packet_capture_enabled: bool` to `CollectorSettings` in `config.py` (env `RAW_PACKET_CAPTURE_ENABLED`, default `false`), following the `data_retention_enabled` convention + - [x] 2.1.2 Thread through `create_subscriber()` and `run_collector()` to `Subscriber.__init__` + - [x] 2.1.3 Store as `self._raw_packet_capture_enabled`; log capture state at startup + +- [x] 2.2 Create the raw-packet handler + - [x] 2.2.1 Create `src/meshcore_hub/collector/handlers/raw_packet.py` with `store_raw_packet(...)` + - [x] 2.2.2 Find-or-create observer node from the topic public key (follow `handlers/event_log.py:34-50`), update `last_seen` + - [x] 2.2.3 Derive `channel_idx = int(channelHash, 16)` via `_extract_letsmesh_decoder_channel_hash` + - [x] 2.2.4 Derive `source_pubkey_prefix` from decoder `sourceHash` / `senderPublicKey` via `_normalize_pubkey_prefix` + - [x] 2.2.5 Insert exactly one `RawPacket` row (no dedup); keep the transaction minimal (single insert + observer upsert, no extra reads) + +- [x] 2.3 Wire the capture hook into the subscriber + - [x] 2.3.1 Refactor `_normalize_letsmesh_event` / `_handle_mqtt_message` so the decode already performed during normalization is reused for capture (return/expose `decoded_packet` + resolved `event_type`) + - [x] 2.3.2 In `_handle_mqtt_message`, short-circuit on `self._raw_packet_capture_enabled` (saves the insert, not the decode) + - [x] 2.3.3 When enabled and feed is `packets`, call `store_raw_packet` before/independent of structured dispatch + - [x] 2.3.4 Wrap capture in try/except so it never blocks event dispatch + - [x] 2.3.5 Ensure `status` / `internal` feeds produce no raw rows + +- [x] 2.4 Write collector tests + - [x] 2.4.1 Capture writes one row per packet (enabled) + - [x] 2.4.2 One row per observer for the same `packet_hash` (no dedup) + - [x] 2.4.3 `channel_idx` / `source_pubkey_prefix` derivation correct + - [x] 2.4.4 `status` / `internal` feeds write no raw rows + - [x] 2.4.5 Capture disabled writes no rows while structured handlers still run + - [x] 2.4.6 Capture failure is logged and does not interrupt event handling + +## 3. Retention & Config + +- [x] 3.1 Add `RAW_PACKET_RETENTION_DAYS` setting + - [x] 3.1.1 Add `raw_packet_retention_days: int` to `CollectorSettings` near `data_retention_*`, defaulting to `data_retention_days` when unset + - [x] 3.1.2 Thread through `create_subscriber()` / `run_collector()` and the cleanup scheduler + +- [x] 3.2 Extend cleanup for raw packets + - [x] 3.2.1 Add a `raw_packets_deleted` counter to `CleanupStats` (`__init__` + `__repr__`) + - [x] 3.2.2 Add per-table retention override to `_cleanup_table` (or a dedicated call) so `raw_packets` uses `RAW_PACKET_RETENTION_DAYS` independently + - [x] 3.2.3 Add the `RawPacket` cleanup step to `cleanup_old_data`; include in `total_deleted` + - [x] 3.2.4 Ensure cleanup runs regardless of `raw_packet_capture_enabled` (so existing rows drain when capture is off) + - [x] 3.2.5 Add `RawPacket.observer_node_id` to `_observer_node_id_union()` so raw-packet-only observers keep `is_observer` + +- [x] 3.3 Write retention tests + - [x] 3.3.1 Raw packets older than `RAW_PACKET_RETENTION_DAYS` are deleted; stat reported + - [x] 3.3.2 Default falls back to global `DATA_RETENTION_DAYS` when unset + - [x] 3.3.3 A node observing only raw packets keeps `is_observer=true` until those packets are pruned + +## 4. Schemas & API + +- [x] 4.1 Create Pydantic schemas + - [x] 4.1.1 Add `RawPacketRead` / `RawPacketList` (mirror `AdvertisementRead` / `AdvertisementList`) + - [x] 4.1.2 `RawPacketRead` includes a `redacted: bool` field + +- [x] 4.2 Create the packets route module + - [x] 4.2.1 Create `src/meshcore_hub/api/routes/raw_packets.py` with `RequireRead`, `DbSession` + - [x] 4.2.2 Add `_packets_key_builder(request)` folding `resolve_user_role(request) or "anonymous"` into the key (per `messages.py:30`) + - [x] 4.2.3 Decorate list with `@cached("packets", key_builder=_packets_key_builder)` using the default `redis_cache_ttl` + - [x] 4.2.4 Define `VALID_PACKET_SORT_COLUMNS = {"time", "event_type", "packet_type", "snr", "path_len"}`, default `time`/`desc` + - [x] 4.2.5 Register router in `api/routes/__init__.py` with `prefix="/packets"`, `tags=["Packets"]` + +- [x] 4.3 Implement `GET /packets` filtering + - [x] 4.3.1 `search` — `ilike` over `packet_hash`, `raw_hex`, observer name/public key + - [x] 4.3.2 `event_type` (comma-separated), `packet_type` (comma-separated) + - [x] 4.3.3 `channel_idx` (single) + - [x] 4.3.4 `route_type` (comma-separated; `all`/`none`/`""` disables — match `/advertisements`) + - [x] 4.3.5 `public_key` / `pubkey_prefix` → `source_pubkey_prefix` + - [x] 4.3.6 `observed_by` (list) → observer public keys + - [x] 4.3.7 `decryptable` (bool) — only packets whose `decoded` has decrypted text + - [x] 4.3.8 `min_snr` / `max_snr`, `min_path_len` / `max_path_len` + - [x] 4.3.9 `redacted` (bool) — include only / exclude redacted rows + - [x] 4.3.10 `since` / `until` on `received_at`; `sort` / `order`; `limit` (1–100, default 50) / `offset` + - [x] 4.3.11 Observer hydration via `aliased(Node)` + `selectinload(Node.tags)` (per `advertisements.py`) + - [x] 4.3.12 Require `search` to combine with a narrowing window (default `since`) so the unindexed substring scan runs against a bounded set + +- [x] 4.4 Implement channel-visibility redaction + - [x] 4.4.1 Compute the visible channel-index set once via `get_visible_channel_indices` (Public idx 17 always visible) + - [x] 4.4.2 Non-channel packets (`channel_idx IS NULL`) returned in full + - [x] 4.4.3 Visible channel packets returned in full incl. `raw_hex` and `decoded` + - [x] 4.4.4 Channel packets above the role: set `redacted=true`, null `raw_hex` / `decoded` / decoded text; keep hash/type/channel/path/snr/observer/timing + - [x] 4.4.5 Do not pre-filter the SQL by visibility (rows returned but redacted) so pagination counts stay stable across roles + +- [x] 4.5 Implement `GET /packets/{id}` + - [x] 4.5.1 Single-row fetch with observer hydration + - [x] 4.5.2 Apply the same redaction rules; 404 when not found + +- [x] 4.6 Web proxy access + - [x] 4.6.1 Add `"v1/packets": { "GET": _OPEN }` to `_build_endpoint_access()` in `web/app.py` + +- [x] 4.7 Write API tests + - [x] 4.7.1 Each filter narrows results correctly + - [x] 4.7.2 Sort whitelist + pagination (count stable across roles) + - [x] 4.7.3 Low role viewing a member-channel packet → `redacted=true`, null `raw_hex`/`decoded` + - [x] 4.7.4 Non-channel packets never redacted; Public (17) always visible + - [x] 4.7.5 Cache key differs by role (no cross-role leakage of redacted content) + +## 5. Web Packets Page + +- [x] 5.1 Add `FEATURE_PACKETS` feature flag + - [x] 5.1.1 Add `feature_packets: bool` to `WebSettings` (default `False`, env `FEATURE_PACKETS`) + - [x] 5.1.2 Add `"packets": self.feature_packets` to the `features` property (no OIDC gate) + +- [x] 5.2 Create the Packets page module + - [x] 5.2.1 Create `src/meshcore_hub/web/static/js/spa/pages/packets.js` (lit-html), modelled on `advertisements.js` + - [x] 5.2.2 Desktop layout: `hidden lg:block` zebra `
    • ` with `sortableTableHeader` columns + - [x] 5.2.3 Mobile layout: `lg:hidden` stacked-card list with `mobileSortSelect` + - [x] 5.2.4 Filter controls: search box, event-type chips/dropdown, channel dropdown (visible channels only), observer multi-select, route-type toggle, time-range picker + - [x] 5.2.5 Collapsible "advanced" panel: `packet_type`, SNR range, path-len range, `decryptable`, `redacted` + - [x] 5.2.6 Pagination + per-row lock badge when `redacted=true` + +- [x] 5.3 Wire routing, icon, and nav + - [x] 5.3.1 Add lazy import entry and `router.addRoute('/packets', pageHandler(pages.packets))` guarded by `features.packets` in `app.js` + - [x] 5.3.2 Add a packets `updatePageTitle` case + - [x] 5.3.3 Add a packets icon function to `icons.js` (imported by `app.js`, `home.js`, `packets.js`) + - [x] 5.3.4 Add nav entry **immediately after Messages** on all three surfaces (guarded by `features.packets`): `spa.html` sidebar/mobile `
  • `, `app.js` dynamic nav, `home.js` hero card grid + - [x] 5.3.5 Reorder `home.js` hero cards so **Map precedes Members** (canonical order `Dashboard → Nodes → Advertisements → Channels → Messages → Packets → Map → Members`) + - [x] 5.3.6 Add `--color-packets` (light + dark), `.nav-icon-packets`, and the `.navbar .menu li:has(.nav-icon-packets)` accent in `app.css` + +- [x] 5.4 i18n + - [x] 5.4.1 Add keys to `en.json`: `entities.packets`, `entities.packet`, filter labels, redacted-badge label + - [x] 5.4.2 Add the same keys to `nl.json` + - [x] 5.4.3 Add i18n key tests if the suite covers them + +- [x] 5.5 Build & web tests + - [x] 5.5.1 Run `node build.js` to rebuild the SPA bundle + - [x] 5.5.2 Test nav/route hidden when `FEATURE_PACKETS` is off + - [x] 5.5.3 Test list render and redacted lock badge + +## 6. Docs, Env & Compose + +- [x] 6.1 Config & compose wiring + - [x] 6.1.1 `.env.example`: add `RAW_PACKET_RETENTION_DAYS` and `RAW_PACKET_CAPTURE_ENABLED` near `DATA_RETENTION_DAYS` (commented; note retention default = `DATA_RETENTION_DAYS`, capture default = `false`) + - [x] 6.1.2 `.env.example`: add `# FEATURE_PACKETS=false` to the feature-flags block with a note that Compose derives capture from it + - [x] 6.1.3 `docker-compose.yml`: add `RAW_PACKET_CAPTURE_ENABLED=${FEATURE_PACKETS:-false}` to the collector service env + - [x] 6.1.4 `docker-compose.yml`: add `FEATURE_PACKETS=${FEATURE_PACKETS:-false}` to the web service env + - [x] 6.1.5 Apply the same env additions to prod/traefik compose overrides + +- [x] 6.2 Reference docs + - [x] 6.2.1 `AGENTS.md`: add `RawPacket` model, `/packets` route, `FEATURE_PACKETS`, `RAW_PACKET_CAPTURE_ENABLED`, `RAW_PACKET_RETENTION_DAYS` + - [x] 6.2.2 `SCHEMAS.md`: add the `raw_packets` table and the `RawPacketRead` shape + - [x] 6.2.3 `docs/letsmesh.md`: note raw capture is `packets`-feed only and channel-restricted packets are returned metadata-only/redacted + +- [x] 6.3 `docs/upgrading.md` v0.13.0 section + - [x] 6.3.1 Add a new `## v0.13.0` section at the top (above `v0.12.0`) + - [x] 6.3.2 Step: run `meshcore-hub db upgrade` to create `raw_packets` + - [x] 6.3.3 Document new optional env vars and defaults: `FEATURE_PACKETS` (false), `RAW_PACKET_CAPTURE_ENABLED` (Compose-derived from `FEATURE_PACKETS`, false), `RAW_PACKET_RETENTION_DAYS` (= `DATA_RETENTION_DAYS`) + - [x] 6.3.4 Explain the capture↔page split and no-backfill behaviour + - [x] 6.3.5 Note `raw_packets` grows fastest; recommend tuning `RAW_PACKET_RETENTION_DAYS`; disabling capture drains via retention + - [x] 6.3.6 Note `/packets` Redis caching is role-aware and honours `REDIS_CACHE_TTL` + +## 7. Verification + +- [x] 7.1 Code quality + - [x] 7.1.1 Run `pre-commit run --all-files` and fix all issues + +- [x] 7.2 Component tests + - [x] 7.2.1 `pytest tests/test_common/` (model) + - [x] 7.2.2 `pytest tests/test_collector/` (capture, flag, retention) + - [x] 7.2.3 `pytest tests/test_api/` (filters, redaction, role-aware cache) + - [x] 7.2.4 `pytest tests/test_web/` (flag-gated nav/route, render) + - [x] 7.2.5 Full `pytest` to verify no regressions + +- [x] 7.3 Manual verification + - [x] 7.3.1 With `RAW_PACKET_CAPTURE_ENABLED=false` (default), confirm no `raw_packets` rows are written and structured handlers still work + - [x] 7.3.2 Enable capture, confirm one row per packet per observer with correct `event_type` / `channel_idx` / `source_pubkey_prefix` + - [x] 7.3.3 With `FEATURE_PACKETS=true`, confirm the Packets page and nav appear after Messages on sidebar, mobile menu, and home page; Map precedes Members + - [x] 7.3.4 Verify desktop table / mobile card layouts and all filters + - [x] 7.3.5 Verify redaction: a low-privilege role sees restricted-channel packets as metadata-only with a lock badge; raw_hex absent + - [x] 7.3.6 Verify retention prunes old raw packets per `RAW_PACKET_RETENTION_DAYS` and the table drains after disabling capture diff --git a/docs/upgrading.md b/docs/upgrading.md index 2023780..25ef689 100644 --- a/docs/upgrading.md +++ b/docs/upgrading.md @@ -2,6 +2,46 @@ This guide covers upgrading from a previous MeshCore Hub release to the current version. Check the relevant version section below before upgrading. +## v0.13.0 + +### Raw Packets (capture, browse, and search wire packets) + +A new **Raw Packets** feature captures every inbound MeshCore packet exactly as it arrives over the LetsMesh `packets` feed into a dedicated `raw_packets` table, independent of how the collector later classifies it. A new `/packets` API and a SPA **Packets** page (table on desktop, cards on mobile) let operators browse, filter, and search the raw traffic. + +**Database migration required:** + +``` +meshcore-hub db upgrade +``` + +This creates the `raw_packets` table and its indexes. On Docker deployments the migration runs automatically on startup. + +**New optional environment variables (all safe to omit):** + +| Variable | Default | Description | +| ---------------------------- | ---------------------- | -------------------------------------------------------------------------------------------- | +| `FEATURE_PACKETS` | `false` | Show the Packets page and nav entry. Off by default. | +| `RAW_PACKET_CAPTURE_ENABLED` | `false` | Collector-side capture of raw packets. In Compose this is **derived from `FEATURE_PACKETS`**. | +| `RAW_PACKET_RETENTION_DAYS` | = `DATA_RETENTION_DAYS` | Days to retain raw packets, independent of the global retention window. | + +**Capture ↔ page split:** capture runs in the collector while the page is served by the web app — two separate processes with separate settings. Docker Compose links them: setting `FEATURE_PACKETS=true` enables **both** capture (`RAW_PACKET_CAPTURE_ENABLED=${FEATURE_PACKETS}` on the collector) and the page. Advanced operators running the processes directly can set the two flags independently. + +**No backfill:** only packets captured *after* enabling appear — historical traffic is not reconstructed. + +**Storage:** `raw_packets` grows fastest of all tables (one row per packet per observer). On busy meshes or constrained storage, lower `RAW_PACKET_RETENTION_DAYS`. Retention cleanup runs regardless of whether capture is currently enabled, so turning capture off lets existing rows drain. Restricted-channel packets are stored in full but returned **metadata-only (redacted)** to roles that cannot see the channel. + +**Caching:** `/packets` responses are cached in Redis (when enabled) using a **role-aware** cache key and honour the existing `REDIS_CACHE_TTL`, so redacted responses are never served across roles. + +The `advertisements` and `messages` tables gain a nullable `packet_hash` column (added by the same `db upgrade`) so each event can link to its captured raw packets. When `FEATURE_PACKETS` is on, the Adverts and Messages list pages show a packet icon linking to the raw packets for that transmission. Only events ingested while capture was enabled carry the hash (no backfill), so the link is hidden for older rows. + +**No action required** to keep current behaviour — the feature is off by default. + +### Finer-Grained Packet Classification + +Packets the collector previously could not categorise were all emitted as a single `letsmesh_packet` event. They are now classified by their MeshCore payload type — `req`, `response`, `ack`, `encrypted_direct`, `encrypted_channel`, `grp_data`, `anon_req`, `multipart`, `control`, `raw_custom`, plus `advert`/`path`/`trace` for malformed variants. `letsmesh_packet` remains only as a safety net for packets whose payload type can't be resolved. + +**Action only if you consume `event_type`:** any external webhook filter, saved query, or dashboard keyed on `letsmesh_packet` should be updated to the specific type(s) it cares about. No database migration or config change is involved. + ## v0.12.0 ### Multi-Worker API (`API_WORKERS`) diff --git a/src/meshcore_hub/api/routes/__init__.py b/src/meshcore_hub/api/routes/__init__.py index fd997d9..301ac10 100644 --- a/src/meshcore_hub/api/routes/__init__.py +++ b/src/meshcore_hub/api/routes/__init__.py @@ -12,6 +12,7 @@ from meshcore_hub.api.routes.dashboard import router as dashboard_router from meshcore_hub.api.routes.user_profiles import router as user_profiles_router from meshcore_hub.api.routes.adoptions import router as adoptions_router from meshcore_hub.api.routes.channels import router as channels_router +from meshcore_hub.api.routes.raw_packets import router as raw_packets_router api_router = APIRouter() @@ -30,3 +31,4 @@ api_router.include_router(dashboard_router, prefix="/dashboard", tags=["Dashboar api_router.include_router(user_profiles_router, prefix="/user", tags=["User"]) api_router.include_router(adoptions_router, prefix="/adoptions", tags=["Adoptions"]) api_router.include_router(channels_router, prefix="/channels", tags=["Channels"]) +api_router.include_router(raw_packets_router, prefix="/packets", tags=["Packets"]) diff --git a/src/meshcore_hub/api/routes/advertisements.py b/src/meshcore_hub/api/routes/advertisements.py index aa78183..b2fe519 100644 --- a/src/meshcore_hub/api/routes/advertisements.py +++ b/src/meshcore_hub/api/routes/advertisements.py @@ -228,6 +228,7 @@ def list_advertisements( "advert_timestamp": adv.advert_timestamp, "received_at": adv.received_at, "created_at": adv.created_at, + "packet_hash": adv.packet_hash, "observers": ( observers_by_hash.get(adv.event_hash, []) if adv.event_hash else [] ), @@ -313,6 +314,7 @@ def get_advertisement( "advert_timestamp": adv.advert_timestamp, "received_at": adv.received_at, "created_at": adv.created_at, + "packet_hash": adv.packet_hash, "observers": observers, } return AdvertisementRead(**data) diff --git a/src/meshcore_hub/api/routes/messages.py b/src/meshcore_hub/api/routes/messages.py index 40580e0..811bd23 100644 --- a/src/meshcore_hub/api/routes/messages.py +++ b/src/meshcore_hub/api/routes/messages.py @@ -200,6 +200,7 @@ def list_messages( "sender_timestamp": m.sender_timestamp, "received_at": m.received_at, "created_at": m.created_at, + "packet_hash": m.packet_hash, "observers": ( observers_by_hash.get(m.event_hash, []) if m.event_hash else [] ), @@ -266,6 +267,7 @@ def get_message( "sender_timestamp": message.sender_timestamp, "received_at": message.received_at, "created_at": message.created_at, + "packet_hash": message.packet_hash, "observers": observers, } return MessageRead(**data) diff --git a/src/meshcore_hub/api/routes/raw_packets.py b/src/meshcore_hub/api/routes/raw_packets.py new file mode 100644 index 0000000..7e8d5ef --- /dev/null +++ b/src/meshcore_hub/api/routes/raw_packets.py @@ -0,0 +1,320 @@ +"""Raw packet API routes.""" + +from datetime import datetime, timedelta, timezone +from typing import Optional + +from fastapi import APIRouter, HTTPException, Query, Request +from sqlalchemy import String, cast, func, or_, select +from sqlalchemy.orm import aliased, selectinload + +from meshcore_hub.api.auth import RequireRead +from meshcore_hub.api.cache import cached, sorted_query_string +from meshcore_hub.api.channel_visibility import ( + get_max_visibility_level, + get_visible_channel_indices, + resolve_user_role, +) +from meshcore_hub.api.dependencies import DbSession +from meshcore_hub.common.models import Node, RawPacket +from meshcore_hub.common.schemas.raw_packets import RawPacketList, RawPacketRead + +router = APIRouter() + +VALID_PACKET_SORT_COLUMNS = {"time", "event_type", "packet_type", "snr", "path_len"} +DISABLE_FILTER_VALUES = {"all", "none", ""} + +# Unanchored substring search over raw_hex cannot use an index (full scan). When +# the caller supplies no explicit time window we bound the scan to this many days +# so search runs against a recent slice rather than the whole table (see TR-17). +SEARCH_DEFAULT_WINDOW_DAYS = 7 + + +def _packets_key_builder(request: Request) -> str: + """Role-aware cache key so redacted responses never leak across roles.""" + role = resolve_user_role(request) or "anonymous" + return f"packets:role={role}:{sorted_query_string(request)}" + + +def _get_tag_name(node: Optional[Node]) -> Optional[str]: + """Extract name tag from a node's tags.""" + if not node or not node.tags: + return None + for tag in node.tags: + if tag.key == "name": + return tag.value + return None + + +def _csv_set(value: Optional[str]) -> set[str]: + """Split a comma-separated filter value into a normalized set.""" + if not value: + return set() + return {t.strip() for t in value.split(",") if t.strip()} + + +@router.get("", response_model=RawPacketList) +@cached("packets", key_builder=_packets_key_builder) +def list_raw_packets( + _: RequireRead, + session: DbSession, + request: Request, + search: Optional[str] = Query( + None, description="Search in packet hash, raw hex, or observer name/key" + ), + event_type: Optional[str] = Query( + None, description="Comma-separated classification filter" + ), + packet_type: Optional[str] = Query( + None, description="Comma-separated wire packet type filter" + ), + packet_hash: Optional[str] = Query( + None, + description="Filter by exact wire packet hash (links from adverts/messages)", + ), + channel_idx: Optional[int] = Query(None, description="Filter by channel index"), + route_type: Optional[str] = Query( + None, + description="Comma-separated route types. Use 'all'/'none' to disable.", + ), + public_key: Optional[str] = Query( + None, description="Filter by source public key prefix" + ), + pubkey_prefix: Optional[str] = Query( + None, description="Filter by source public key prefix" + ), + observed_by: Optional[list[str]] = Query( + None, description="Filter by receiver node public keys" + ), + decryptable: Optional[bool] = Query( + None, description="Only packets whose payload decrypted (or the inverse)" + ), + min_snr: Optional[float] = Query(None, description="Minimum SNR"), + max_snr: Optional[float] = Query(None, description="Maximum SNR"), + min_path_len: Optional[int] = Query(None, description="Minimum hop count"), + max_path_len: Optional[int] = Query(None, description="Maximum hop count"), + redacted: Optional[bool] = Query( + None, description="Include only / exclude redacted (metadata-only) rows" + ), + since: Optional[datetime] = Query(None, description="Start timestamp"), + until: Optional[datetime] = Query(None, description="End timestamp"), + sort: Optional[str] = Query(None, description="Sort column"), + order: Optional[str] = Query(None, description="Sort direction: asc or desc"), + limit: int = Query(50, ge=1, le=100, description="Page size"), + offset: int = Query(0, ge=0, description="Page offset"), +) -> RawPacketList: + """List raw packets with filtering, redaction, and pagination.""" + ObserverNode = aliased(Node) + + query = select( + RawPacket, + ObserverNode.public_key.label("observer_pk"), + ObserverNode.name.label("observer_name"), + ObserverNode.id.label("observer_id"), + ).outerjoin(ObserverNode, RawPacket.observer_node_id == ObserverNode.id) + + # Bound an unindexed substring search to a recent window when unconstrained. + if search and since is None: + since = datetime.now(timezone.utc) - timedelta(days=SEARCH_DEFAULT_WINDOW_DAYS) + + if search: + pattern = f"%{search}%" + query = query.where( + or_( + RawPacket.packet_hash.ilike(pattern), + RawPacket.raw_hex.ilike(pattern), + ObserverNode.name.ilike(pattern), + ObserverNode.public_key.ilike(pattern), + ) + ) + + event_types = _csv_set(event_type) + if event_types: + query = query.where(RawPacket.event_type.in_(event_types)) + + packet_types = {int(t) for t in _csv_set(packet_type) if t.lstrip("-").isdigit()} + if packet_types: + query = query.where(RawPacket.packet_type.in_(packet_types)) + + if packet_hash: + query = query.where(RawPacket.packet_hash == packet_hash) + + if channel_idx is not None: + query = query.where(RawPacket.channel_idx == channel_idx) + + if route_type and route_type.strip().lower() not in DISABLE_FILTER_VALUES: + requested = {t.lower() for t in _csv_set(route_type)} + if requested: + query = query.where(RawPacket.route_type.in_(requested)) + + source_prefix = public_key or pubkey_prefix + if source_prefix: + query = query.where(RawPacket.source_pubkey_prefix == source_prefix) + + if observed_by: + query = query.where(ObserverNode.public_key.in_(observed_by)) + + if decryptable is not None: + # Best-effort: the decoder only emits a "decrypted" object when + # decryption succeeds. Match on the serialized JSON text. + decoded_text = cast(RawPacket.decoded, String) + has_decrypted = decoded_text.ilike('%"decrypted":%') & ~decoded_text.ilike( + '%"decrypted": null%' + ) + query = query.where(has_decrypted if decryptable else ~has_decrypted) + + if min_snr is not None: + query = query.where(RawPacket.snr >= min_snr) + if max_snr is not None: + query = query.where(RawPacket.snr <= max_snr) + if min_path_len is not None: + query = query.where(RawPacket.path_len >= min_path_len) + if max_path_len is not None: + query = query.where(RawPacket.path_len <= max_path_len) + + if since: + query = query.where(RawPacket.received_at >= since) + if until: + query = query.where(RawPacket.received_at <= until) + + # Channel-visibility: a row is "redacted" when it is a channel packet on a + # channel the role cannot see. Apply the `redacted` filter at SQL level (not + # post-fetch) so pagination counts stay stable. + role = resolve_user_role(request) + max_level = get_max_visibility_level(role) + visible_indices = get_visible_channel_indices(session, max_level) + + is_redacted = RawPacket.channel_idx.is_not(None) & RawPacket.channel_idx.not_in( + visible_indices + ) + if redacted is True: + query = query.where(is_redacted) + elif redacted is False: + query = query.where(~is_redacted) + + # Total count over the filtered subquery + count_query = select(func.count()).select_from(query.subquery()) + total = session.execute(count_query).scalar() or 0 + + # Sorting + sort = sort if sort in VALID_PACKET_SORT_COLUMNS else "time" + order = order if order in ("asc", "desc") else "desc" + sort_column = { + "time": RawPacket.received_at, + "event_type": RawPacket.event_type, + "packet_type": RawPacket.packet_type, + "snr": RawPacket.snr, + "path_len": RawPacket.path_len, + }[sort] + query = query.order_by(sort_column.desc() if order == "desc" else sort_column.asc()) + + query = query.offset(offset).limit(limit) + results = session.execute(query).all() + + # Hydrate observer tags + observer_ids = {row.observer_id for row in results if row.observer_id} + nodes_by_id: dict[str, Node] = {} + if observer_ids: + nodes = ( + session.execute( + select(Node) + .where(Node.id.in_(observer_ids)) + .options(selectinload(Node.tags)) + ) + .scalars() + .all() + ) + nodes_by_id = {n.id: n for n in nodes} + + items = [ + _build_read( + row[0], + observer_pk=row.observer_pk, + observer_name=row.observer_name, + observer_node=nodes_by_id.get(row.observer_id) if row.observer_id else None, + visible_indices=visible_indices, + ) + for row in results + ] + + return RawPacketList(items=items, total=total, limit=limit, offset=offset) + + +def _build_read( + packet: RawPacket, + observer_pk: Optional[str], + observer_name: Optional[str], + observer_node: Optional[Node], + visible_indices: set[int], +) -> RawPacketRead: + """Build a RawPacketRead, applying channel-visibility redaction.""" + redacted = ( + packet.channel_idx is not None and packet.channel_idx not in visible_indices + ) + return RawPacketRead( + id=packet.id, + observed_by=observer_pk, + observer_name=observer_name, + observer_tag_name=_get_tag_name(observer_node), + packet_hash=packet.packet_hash, + raw_hex=None if redacted else packet.raw_hex, + packet_type=packet.packet_type, + payload_type=packet.payload_type, + event_type=packet.event_type, + channel_idx=packet.channel_idx, + source_pubkey_prefix=None if redacted else packet.source_pubkey_prefix, + route_type=packet.route_type, + path_len=packet.path_len, + snr=packet.snr, + decoded=None if redacted else packet.decoded, + redacted=redacted, + received_at=packet.received_at, + created_at=packet.created_at, + ) + + +@router.get("/{packet_id}", response_model=RawPacketRead) +def get_raw_packet( + _: RequireRead, + session: DbSession, + request: Request, + packet_id: str, +) -> RawPacketRead: + """Get a single raw packet by ID, subject to redaction rules.""" + ObserverNode = aliased(Node) + result = session.execute( + select( + RawPacket, + ObserverNode.public_key.label("observer_pk"), + ObserverNode.name.label("observer_name"), + ObserverNode.id.label("observer_id"), + ) + .outerjoin(ObserverNode, RawPacket.observer_node_id == ObserverNode.id) + .where(RawPacket.id == packet_id) + ).one_or_none() + + if not result: + raise HTTPException(status_code=404, detail="Raw packet not found") + + role = resolve_user_role(request) + max_level = get_max_visibility_level(role) + visible_indices = get_visible_channel_indices(session, max_level) + + observer_node = None + if result.observer_id: + observer_node = ( + session.execute( + select(Node) + .where(Node.id == result.observer_id) + .options(selectinload(Node.tags)) + ) + .scalars() + .one_or_none() + ) + + return _build_read( + result[0], + observer_pk=result.observer_pk, + observer_name=result.observer_name, + observer_node=observer_node, + visible_indices=visible_indices, + ) diff --git a/src/meshcore_hub/collector/cleanup.py b/src/meshcore_hub/collector/cleanup.py index fdcd05c..1274b80 100644 --- a/src/meshcore_hub/collector/cleanup.py +++ b/src/meshcore_hub/collector/cleanup.py @@ -17,6 +17,7 @@ from meshcore_hub.common.models import ( Message, Node, NodeTag, + RawPacket, Telemetry, TracePath, ) @@ -34,6 +35,7 @@ class CleanupStats: self.telemetry_deleted: int = 0 self.trace_paths_deleted: int = 0 self.event_logs_deleted: int = 0 + self.raw_packets_deleted: int = 0 self.nodes_deleted: int = 0 self.observers_cleared: int = 0 self.total_deleted: int = 0 @@ -46,6 +48,7 @@ class CleanupStats: f"telemetry={self.telemetry_deleted}, " f"trace_paths={self.trace_paths_deleted}, " f"event_logs={self.event_logs_deleted}, " + f"raw_packets={self.raw_packets_deleted}, " f"nodes={self.nodes_deleted}, " f"observers_cleared={self.observers_cleared})" ) @@ -55,6 +58,7 @@ async def cleanup_old_data( db: AsyncSession, retention_days: int, dry_run: bool = False, + raw_packet_retention_days: int | None = None, ) -> CleanupStats: """Delete event data older than the retention period. @@ -62,12 +66,22 @@ async def cleanup_old_data( db: Database session retention_days: Number of days to retain data dry_run: If True, only count records without deleting + raw_packet_retention_days: Days to retain raw packets (falls back to + ``retention_days`` when None). Raw-packet cleanup runs whenever data + cleanup runs, independent of whether capture is currently enabled. Returns: CleanupStats object with deletion counts """ stats = CleanupStats() cutoff_date = datetime.now(timezone.utc) - timedelta(days=retention_days) + raw_packet_cutoff = datetime.now(timezone.utc) - timedelta( + days=( + raw_packet_retention_days + if raw_packet_retention_days is not None + else retention_days + ) + ) logger.info( "Starting data cleanup (dry_run=%s, retention_days=%d, cutoff=%s)", @@ -101,12 +115,18 @@ async def cleanup_old_data( db, EventLog, cutoff_date, "event_logs", dry_run ) + # Clean up raw packets (independent retention window) + stats.raw_packets_deleted = await _cleanup_table( + db, RawPacket, raw_packet_cutoff, "raw_packets", dry_run + ) + stats.total_deleted = ( stats.advertisements_deleted + stats.messages_deleted + stats.telemetry_deleted + stats.trace_paths_deleted + stats.event_logs_deleted + + stats.raw_packets_deleted ) # Clear the is_observer flag for nodes whose events were all pruned above. @@ -134,6 +154,9 @@ def _observer_node_id_union() -> CompoundSelect: select(TracePath.observer_node_id).where( TracePath.observer_node_id.is_not(None) ), + select(RawPacket.observer_node_id).where( + RawPacket.observer_node_id.is_not(None) + ), select(EventObserver.observer_node_id), ) diff --git a/src/meshcore_hub/collector/cli.py b/src/meshcore_hub/collector/cli.py index a17b85b..8fc0793 100644 --- a/src/meshcore_hub/collector/cli.py +++ b/src/meshcore_hub/collector/cli.py @@ -281,6 +281,8 @@ def _run_collector_service( node_cleanup_enabled=settings.node_cleanup_enabled, node_cleanup_days=settings.node_cleanup_days, channel_refresh_interval_seconds=settings.channel_refresh_interval_seconds, + raw_packet_capture_enabled=settings.raw_packet_capture_enabled, + raw_packet_retention_days=settings.effective_raw_packet_retention_days, ) diff --git a/src/meshcore_hub/collector/handlers/advertisement.py b/src/meshcore_hub/collector/handlers/advertisement.py index 4b9088f..d198732 100644 --- a/src/meshcore_hub/collector/handlers/advertisement.py +++ b/src/meshcore_hub/collector/handlers/advertisement.py @@ -91,6 +91,9 @@ def handle_advertisement( if delta <= timedelta(hours=4): advert_timestamp_for_hash = advert_timestamp_dt + # LetsMesh wire packet hash, for linking to raw_packets + packet_hash = payload.get("packet_hash") or payload.get("hash") + # Compute event hash for deduplication (5-minute time bucket) event_hash = compute_advertisement_hash( public_key=adv_public_key, @@ -195,6 +198,7 @@ def handle_advertisement( flags=flags, received_at=now, event_hash=event_hash, + packet_hash=packet_hash, route_type=route_type, advert_timestamp=advert_timestamp_dt, ) diff --git a/src/meshcore_hub/collector/handlers/message.py b/src/meshcore_hub/collector/handlers/message.py index 8cb3ae5..20b25c0 100644 --- a/src/meshcore_hub/collector/handlers/message.py +++ b/src/meshcore_hub/collector/handlers/message.py @@ -86,6 +86,9 @@ def _handle_message( except (ValueError, OSError): pass + # LetsMesh wire packet hash, for linking to raw_packets + packet_hash = payload.get("packet_hash") or payload.get("hash") + # Compute event hash for deduplication event_hash = compute_message_hash( text=text, @@ -151,6 +154,7 @@ def _handle_message( sender_timestamp=sender_timestamp, received_at=now, event_hash=event_hash, + packet_hash=packet_hash, ) session.add(message) diff --git a/src/meshcore_hub/collector/handlers/raw_packet.py b/src/meshcore_hub/collector/handlers/raw_packet.py new file mode 100644 index 0000000..07a5a3c --- /dev/null +++ b/src/meshcore_hub/collector/handlers/raw_packet.py @@ -0,0 +1,140 @@ +"""Raw packet capture handler. + +Stores one ``RawPacket`` row per observer reception from the LetsMesh +``packets`` feed, independent of how the packet is later classified. The decode +is reused from the normalizer's per-hex decode cache (a second ``decode_payload`` +call is a cache hit, not a re-decode), so capture adds only a single insert plus +the observer upsert to the ingest path. +""" + +import logging +from datetime import datetime, timezone +from typing import Any, Optional + +from sqlalchemy import select + +from meshcore_hub.collector.letsmesh_normalizer import LetsMeshNormalizer +from meshcore_hub.common.database import DatabaseManager +from meshcore_hub.common.models import Node, RawPacket + +logger = logging.getLogger(__name__) + +# Wire route-type codes -> labels (mirrors LetsMeshNormalizer._ROUTE_TYPE_MAP). +_ROUTE_TYPE_MAP: dict[int, str] = { + 0: "transport_flood", + 1: "flood", + 2: "direct", + 3: "transport_direct", +} + + +def _extract_source_pubkey_prefix( + decoded_packet: Optional[dict[str, Any]], +) -> Optional[str]: + """Derive a 12-char sender prefix from the decoder sourceHash/senderPublicKey.""" + decoded = LetsMeshNormalizer._extract_letsmesh_decoder_payload(decoded_packet) + if not decoded: + return None + for key in ("sourceHash", "senderPublicKey"): + prefix = LetsMeshNormalizer._normalize_pubkey_prefix(decoded.get(key)) + if prefix: + return prefix + return None + + +def store_raw_packet( + public_key: str, + payload: dict[str, Any], + decoded_packet: Optional[dict[str, Any]], + event_type: str, + db: DatabaseManager, +) -> None: + """Capture a single raw packet from the LetsMesh ``packets`` feed. + + Args: + public_key: Observer (receiver) node public key from the MQTT topic + payload: Original LetsMesh upload payload (carries ``raw``, ``hash``, SNR) + decoded_packet: Decoder output already produced during normalization + event_type: How the collector classified the packet + db: Database manager + """ + now = datetime.now(timezone.utc) + + raw_value = payload.get("raw") + raw_hex = raw_value.strip() if isinstance(raw_value, str) else None + + hash_value = payload.get("hash") + packet_hash = hash_value if isinstance(hash_value, str) else None + + payload_type = LetsMeshNormalizer._extract_letsmesh_decoder_payload_type( + decoded_packet + ) + packet_type = LetsMeshNormalizer._parse_int(payload.get("packet_type")) + if packet_type is None: + packet_type = payload_type + + channel_idx = LetsMeshNormalizer._parse_int(payload.get("channel_idx")) + if channel_idx is None: + channel_hash = LetsMeshNormalizer._extract_letsmesh_decoder_channel_hash( + decoded_packet + ) + if channel_hash: + channel_idx = LetsMeshNormalizer._parse_channel_hash_idx(channel_hash) + + source_pubkey_prefix = _extract_source_pubkey_prefix(decoded_packet) + + route_type: Optional[str] = None + if isinstance(decoded_packet, dict): + route_raw = LetsMeshNormalizer._parse_int(decoded_packet.get("routeType")) + if route_raw in _ROUTE_TYPE_MAP: + route_type = _ROUTE_TYPE_MAP[route_raw] + + path_len = LetsMeshNormalizer._parse_path_length(payload.get("path")) + if path_len is None: + path_len = LetsMeshNormalizer._extract_letsmesh_decoder_path_length( + decoded_packet + ) + + snr = LetsMeshNormalizer._parse_float(payload.get("SNR")) + if snr is None: + snr = LetsMeshNormalizer._parse_float(payload.get("snr")) + + with db.session_scope() as session: + observer_node = None + if public_key: + observer_node = session.execute( + select(Node).where(Node.public_key == public_key) + ).scalar_one_or_none() + if not observer_node: + observer_node = Node( + public_key=public_key, + first_seen=now, + last_seen=now, + is_observer=True, + ) + session.add(observer_node) + session.flush() + else: + observer_node.last_seen = now + if not observer_node.is_observer: + observer_node.is_observer = True + + session.add( + RawPacket( + observer_node_id=observer_node.id if observer_node else None, + packet_hash=packet_hash, + raw_hex=raw_hex, + packet_type=packet_type, + payload_type=payload_type, + event_type=event_type, + channel_idx=channel_idx, + source_pubkey_prefix=source_pubkey_prefix, + route_type=route_type, + path_len=path_len, + snr=snr, + decoded=decoded_packet, + received_at=now, + ) + ) + + logger.debug("Captured raw packet: %s (%s)", packet_hash or "unknown", event_type) diff --git a/src/meshcore_hub/collector/letsmesh_normalizer.py b/src/meshcore_hub/collector/letsmesh_normalizer.py index 5cb37ce..f91e909 100644 --- a/src/meshcore_hub/collector/letsmesh_normalizer.py +++ b/src/meshcore_hub/collector/letsmesh_normalizer.py @@ -74,7 +74,10 @@ class LetsMeshNormalizer: normalized_packet_payload["decoded_payload_type"] = ( decoded_payload_type ) - return observer_public_key, "letsmesh_packet", normalized_packet_payload + fallback_event_type = self._classify_fallback_event_type( + payload, decoded_packet + ) + return observer_public_key, fallback_event_type, normalized_packet_payload if feed_type == "internal": return observer_public_key, "letsmesh_internal", payload @@ -140,6 +143,8 @@ class LetsMeshNormalizer: txt_type if txt_type is not None else packet_type ) normalized_payload["signature"] = payload.get("signature") or packet_hash + if packet_hash_text: + normalized_payload["packet_hash"] = packet_hash_text path_len = self._parse_path_length(payload.get("path")) if path_len is None: path_len = self._extract_letsmesh_decoder_path_length(decoded_packet) @@ -549,6 +554,44 @@ class LetsMeshNormalizer: 3: "transport_direct", } + # MeshCore payload type -> event_type for packets that reach the fallback + # (i.e. the structured/message/advert builders declined). Reaching this path + # for TXT_MSG (2) / GRP_TXT (5) means the payload did not decrypt, so they + # are labelled as encrypted rather than as messages. Unknown/unresolvable + # types keep the generic ``letsmesh_packet`` safety-net label. + _FALLBACK_EVENT_TYPES: dict[int, str] = { + 0: "req", # PAYLOAD_TYPE_REQ + 1: "response", # PAYLOAD_TYPE_RESPONSE + 2: "encrypted_direct", # PAYLOAD_TYPE_TXT_MSG (undecryptable) + 3: "ack", # PAYLOAD_TYPE_ACK + 4: "advert", # PAYLOAD_TYPE_ADVERT (no identity metadata) + 5: "encrypted_channel", # PAYLOAD_TYPE_GRP_TXT (unknown channel key) + 6: "grp_data", # PAYLOAD_TYPE_GRP_DATA + 7: "anon_req", # PAYLOAD_TYPE_ANON_REQ + 8: "path", # PAYLOAD_TYPE_PATH + 9: "trace", # PAYLOAD_TYPE_TRACE + 10: "multipart", # PAYLOAD_TYPE_MULTIPART + 11: "control", # PAYLOAD_TYPE_CONTROL + 15: "raw_custom", # PAYLOAD_TYPE_RAW_CUSTOM + } + + @classmethod + def _classify_fallback_event_type( + cls, + payload: dict[str, Any], + decoded_packet: dict[str, Any] | None, + ) -> str: + """Resolve a specific event type for an otherwise-unhandled packet. + + Maps the MeshCore payload type to a precise label so uncategorised + packets are no longer all lumped under ``letsmesh_packet``. Falls back + to ``letsmesh_packet`` only when the payload type cannot be resolved. + """ + packet_type = cls._resolve_letsmesh_packet_type(payload, decoded_packet) + if packet_type is None: + return "letsmesh_packet" + return cls._FALLBACK_EVENT_TYPES.get(packet_type, "letsmesh_packet") + def _build_letsmesh_advertisement_payload( self, payload: dict[str, Any], @@ -583,6 +626,10 @@ class LetsMeshNormalizer: "public_key": public_key, } + advert_hash = payload.get("hash") + if isinstance(advert_hash, str) and advert_hash: + normalized_payload["packet_hash"] = advert_hash + route_type_raw = self._parse_int(decoded_packet.get("routeType")) if route_type_raw is not None and route_type_raw in self._ROUTE_TYPE_MAP: normalized_payload["route_type"] = self._ROUTE_TYPE_MAP[route_type_raw] diff --git a/src/meshcore_hub/collector/subscriber.py b/src/meshcore_hub/collector/subscriber.py index ad0a2ce..cf79450 100644 --- a/src/meshcore_hub/collector/subscriber.py +++ b/src/meshcore_hub/collector/subscriber.py @@ -48,6 +48,8 @@ class Subscriber(LetsMeshNormalizer): node_cleanup_enabled: bool = False, node_cleanup_days: int = 90, channel_refresh_interval_seconds: int = 300, + raw_packet_capture_enabled: bool = False, + raw_packet_retention_days: int = 30, ): """Initialize subscriber. @@ -61,6 +63,8 @@ class Subscriber(LetsMeshNormalizer): node_cleanup_enabled: Enable automatic cleanup of inactive nodes node_cleanup_days: Remove nodes not seen for this many days channel_refresh_interval_seconds: Seconds between channel key refresh + raw_packet_capture_enabled: Capture every packets-feed packet to raw_packets + raw_packet_retention_days: Days to retain raw packets """ self.mqtt = mqtt_client self.db = db_manager @@ -83,6 +87,14 @@ class Subscriber(LetsMeshNormalizer): self._node_cleanup_days = node_cleanup_days self._cleanup_thread: Optional[threading.Thread] = None self._last_cleanup: Optional[datetime] = None + # Raw packet capture + self._raw_packet_capture_enabled = raw_packet_capture_enabled + self._raw_packet_retention_days = raw_packet_retention_days + logger.info( + "Raw packet capture %s (retention=%d days)", + "enabled" if raw_packet_capture_enabled else "disabled", + raw_packet_retention_days, + ) # Channel key refresh self._channel_refresh_interval_seconds = channel_refresh_interval_seconds self._channel_refresh_thread: Optional[threading.Thread] = None @@ -208,8 +220,43 @@ class Subscriber(LetsMeshNormalizer): public_key, event_type, normalized_payload = parsed logger.debug("Received event: %s from %s...", event_type, public_key[:12]) + + # Capture the raw packet (packets feed only) independent of, and before, + # structured dispatch so the raw_packets table is complete. The boolean + # short-circuit avoids the insert entirely when capture is disabled. + if self._raw_packet_capture_enabled: + self._maybe_capture_raw_packet(topic, public_key, event_type, payload) + self._dispatch_event(public_key, event_type, normalized_payload) + def _maybe_capture_raw_packet( + self, + topic: str, + public_key: str, + event_type: str, + payload: dict[str, Any], + ) -> None: + """Persist a raw_packets row for a packets-feed reception. + + Reuses the decode the normalizer already performed (the decoder caches + per raw hex, so this ``decode_payload`` call is a cache hit). Capture + failures are logged and never block event dispatch. + """ + try: + parsed_topic = self.mqtt.topic_builder.parse_letsmesh_upload_topic(topic) + if not parsed_topic: + return + _, feed_type = parsed_topic + if feed_type != "packets": + return + + from meshcore_hub.collector.handlers.raw_packet import store_raw_packet + + decoded_packet = self._letsmesh_decoder.decode_payload(payload) + store_raw_packet(public_key, payload, decoded_packet, event_type, self.db) + except Exception as e: + logger.error("Error capturing raw packet: %s", e) + def _dispatch_event( self, public_key: str, @@ -370,6 +417,9 @@ class Subscriber(LetsMeshNormalizer): session, self._cleanup_retention_days, dry_run=False, + raw_packet_retention_days=( + self._raw_packet_retention_days + ), ) logger.info( "Event cleanup completed: %s", stats @@ -600,6 +650,8 @@ def create_subscriber( node_cleanup_enabled: bool = False, node_cleanup_days: int = 90, channel_refresh_interval_seconds: int = 300, + raw_packet_capture_enabled: bool = False, + raw_packet_retention_days: int = 30, ) -> Subscriber: """Create a configured subscriber instance. @@ -653,6 +705,8 @@ def create_subscriber( node_cleanup_enabled=node_cleanup_enabled, node_cleanup_days=node_cleanup_days, channel_refresh_interval_seconds=channel_refresh_interval_seconds, + raw_packet_capture_enabled=raw_packet_capture_enabled, + raw_packet_retention_days=raw_packet_retention_days, ) # Register handlers @@ -680,6 +734,8 @@ def run_collector( node_cleanup_enabled: bool = False, node_cleanup_days: int = 90, channel_refresh_interval_seconds: int = 300, + raw_packet_capture_enabled: bool = False, + raw_packet_retention_days: int = 30, ) -> None: """Run the collector (blocking). @@ -718,6 +774,8 @@ def run_collector( node_cleanup_enabled=node_cleanup_enabled, node_cleanup_days=node_cleanup_days, channel_refresh_interval_seconds=channel_refresh_interval_seconds, + raw_packet_capture_enabled=raw_packet_capture_enabled, + raw_packet_retention_days=raw_packet_retention_days, ) # Set up signal handlers diff --git a/src/meshcore_hub/common/config.py b/src/meshcore_hub/common/config.py index 192111d..586a782 100644 --- a/src/meshcore_hub/common/config.py +++ b/src/meshcore_hub/common/config.py @@ -143,6 +143,26 @@ class CollectorSettings(CommonSettings): ge=10, ) + # Raw packet capture settings + raw_packet_capture_enabled: bool = Field( + default=False, + description="Capture every inbound packets-feed packet into raw_packets", + ) + raw_packet_retention_days: Optional[int] = Field( + default=None, + description=( + "Days to retain raw packets (defaults to DATA_RETENTION_DAYS when unset)" + ), + ge=1, + ) + + @property + def effective_raw_packet_retention_days(self) -> int: + """Resolve raw-packet retention, falling back to the global default.""" + if self.raw_packet_retention_days is not None: + return self.raw_packet_retention_days + return self.data_retention_days + @property def collector_data_dir(self) -> str: """Get the collector data directory path.""" @@ -411,6 +431,9 @@ class WebSettings(CommonSettings): feature_channels: bool = Field( default=True, description="Enable the /channels page" ) + feature_packets: bool = Field( + default=False, description="Enable the /packets page (off by default)" + ) feature_pages: bool = Field( default=True, description="Enable custom markdown pages" ) @@ -444,6 +467,7 @@ class WebSettings(CommonSettings): "map": self.feature_map and self.feature_nodes, "members": self.feature_members and self.oidc_enabled, "channels": self.feature_channels, + "packets": self.feature_packets, "pages": self.feature_pages, "radio_config": self.feature_radio_config, } diff --git a/src/meshcore_hub/common/models/__init__.py b/src/meshcore_hub/common/models/__init__.py index 7b340c2..73650b9 100644 --- a/src/meshcore_hub/common/models/__init__.py +++ b/src/meshcore_hub/common/models/__init__.py @@ -8,6 +8,7 @@ from meshcore_hub.common.models.advertisement import Advertisement from meshcore_hub.common.models.trace_path import TracePath from meshcore_hub.common.models.telemetry import Telemetry from meshcore_hub.common.models.event_log import EventLog +from meshcore_hub.common.models.raw_packet import RawPacket from meshcore_hub.common.models.user_profile import UserProfile from meshcore_hub.common.models.user_profile_node import UserProfileNode from meshcore_hub.common.models.event_observer import EventObserver, add_event_observer @@ -23,6 +24,7 @@ __all__ = [ "TracePath", "Telemetry", "EventLog", + "RawPacket", "UserProfile", "UserProfileNode", "EventObserver", diff --git a/src/meshcore_hub/common/models/advertisement.py b/src/meshcore_hub/common/models/advertisement.py index 6ddc901..ea5ef33 100644 --- a/src/meshcore_hub/common/models/advertisement.py +++ b/src/meshcore_hub/common/models/advertisement.py @@ -63,6 +63,12 @@ class Advertisement(Base, UUIDMixin, TimestampMixin): nullable=True, unique=True, ) + # LetsMesh wire packet hash, links this advert to its raw_packets rows. + packet_hash: Mapped[Optional[str]] = mapped_column( + String(32), + nullable=True, + index=True, + ) route_type: Mapped[Optional[str]] = mapped_column( String(20), nullable=True, diff --git a/src/meshcore_hub/common/models/message.py b/src/meshcore_hub/common/models/message.py index ca2c01e..5b0813b 100644 --- a/src/meshcore_hub/common/models/message.py +++ b/src/meshcore_hub/common/models/message.py @@ -81,6 +81,12 @@ class Message(Base, UUIDMixin, TimestampMixin): nullable=True, unique=True, ) + # LetsMesh wire packet hash, links this event to its raw_packets rows. + packet_hash: Mapped[Optional[str]] = mapped_column( + String(32), + nullable=True, + index=True, + ) __table_args__ = ( Index("ix_messages_message_type", "message_type"), diff --git a/src/meshcore_hub/common/models/raw_packet.py b/src/meshcore_hub/common/models/raw_packet.py new file mode 100644 index 0000000..9abc813 --- /dev/null +++ b/src/meshcore_hub/common/models/raw_packet.py @@ -0,0 +1,122 @@ +"""RawPacket model for storing raw decoded wire packets.""" + +from datetime import datetime +from typing import Any, Optional + +from sqlalchemy import DateTime, Float, ForeignKey, Index, Integer, String, Text +from sqlalchemy.dialects.sqlite import JSON +from sqlalchemy.orm import Mapped, mapped_column + +from meshcore_hub.common.models.base import Base, TimestampMixin, UUIDMixin, utc_now + + +class RawPacket(Base, UUIDMixin, TimestampMixin): + """Raw on-air packet captured from the LetsMesh ``packets`` feed. + + One row is stored per observer reception (no deduplication). Rows are + linkable back to the structured tables (messages, advertisements, ...) by + ``packet_hash``. The raw hex and decoded summary are retained so the API can + serve a complete searchable record of wire traffic without re-decoding. + + Attributes: + id: UUID primary key + observer_node_id: FK to nodes (the receiving interface) + packet_hash: LetsMesh packet hash, links rows to structured records and + groups multi-observer receptions + raw_hex: The on-air bytes from ``payload["raw"]`` + packet_type: Wire packet type + payload_type: Decoder payload type + event_type: How the collector classified the packet + channel_idx: ``int(channelHash, 16)`` for channel-message packets, else + NULL; drives channel-visibility redaction + source_pubkey_prefix: Sender prefix derived from the decoder sourceHash / + senderPublicKey, for efficient "packets from this node" filtering + route_type: Mapped route type (flood, transport_flood, direct, ...) + path_len: Hop count + snr: Signal-to-noise ratio as reported by the observer + decoded: Decoder summary JSON, so the detail view needs no re-decode + received_at: When received by the observing interface + created_at: Record creation timestamp + """ + + __tablename__ = "raw_packets" + + observer_node_id: Mapped[Optional[str]] = mapped_column( + ForeignKey("nodes.id", ondelete="SET NULL"), + nullable=True, + index=True, + ) + packet_hash: Mapped[Optional[str]] = mapped_column( + String(32), + nullable=True, + index=True, + ) + raw_hex: Mapped[Optional[str]] = mapped_column( + Text, + nullable=True, + ) + packet_type: Mapped[Optional[int]] = mapped_column( + Integer, + nullable=True, + index=True, + ) + payload_type: Mapped[Optional[int]] = mapped_column( + Integer, + nullable=True, + ) + event_type: Mapped[Optional[str]] = mapped_column( + String(50), + nullable=True, + index=True, + ) + channel_idx: Mapped[Optional[int]] = mapped_column( + Integer, + nullable=True, + index=True, + ) + source_pubkey_prefix: Mapped[Optional[str]] = mapped_column( + String(12), + nullable=True, + index=True, + ) + route_type: Mapped[Optional[str]] = mapped_column( + String(20), + nullable=True, + ) + path_len: Mapped[Optional[int]] = mapped_column( + Integer, + nullable=True, + ) + snr: Mapped[Optional[float]] = mapped_column( + Float, + nullable=True, + ) + decoded: Mapped[Optional[dict[str, Any]]] = mapped_column( + JSON, + nullable=True, + ) + received_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), + default=utc_now, + nullable=False, + ) + + __table_args__ = ( + Index("ix_raw_packets_received_at", "received_at"), + # Composite indexes serve the common "filter then sort by newest" shape + # without a full scan + filesort. See plan TR-17 for the write-cost + # trade-off; do not add further composites speculatively. + Index("ix_raw_packets_event_type_received_at", "event_type", "received_at"), + Index("ix_raw_packets_channel_idx_received_at", "channel_idx", "received_at"), + Index( + "ix_raw_packets_source_pubkey_prefix_received_at", + "source_pubkey_prefix", + "received_at", + ), + ) + + def __repr__(self) -> str: + return ( + f"" + ) diff --git a/src/meshcore_hub/common/schemas/__init__.py b/src/meshcore_hub/common/schemas/__init__.py index 9664c35..b93eba7 100644 --- a/src/meshcore_hub/common/schemas/__init__.py +++ b/src/meshcore_hub/common/schemas/__init__.py @@ -28,6 +28,10 @@ from meshcore_hub.common.schemas.messages import ( from meshcore_hub.common.schemas.network import ( RadioConfig, ) +from meshcore_hub.common.schemas.raw_packets import ( + RawPacketRead, + RawPacketList, +) __all__ = [ # Events @@ -54,4 +58,7 @@ __all__ = [ "MessageFilters", # Network "RadioConfig", + # Raw packets + "RawPacketRead", + "RawPacketList", ] diff --git a/src/meshcore_hub/common/schemas/messages.py b/src/meshcore_hub/common/schemas/messages.py index c86e02a..13f57ce 100644 --- a/src/meshcore_hub/common/schemas/messages.py +++ b/src/meshcore_hub/common/schemas/messages.py @@ -58,6 +58,9 @@ class MessageRead(BaseModel): ) received_at: datetime = Field(..., description="When received by interface") created_at: datetime = Field(..., description="Record creation timestamp") + packet_hash: Optional[str] = Field( + default=None, description="LetsMesh wire packet hash, links to raw packets" + ) observers: list[ObserverInfo] = Field( default_factory=list, description="All observers that captured this message" ) @@ -138,6 +141,9 @@ class AdvertisementRead(BaseModel): ) received_at: datetime = Field(..., description="When received") created_at: datetime = Field(..., description="Record creation timestamp") + packet_hash: Optional[str] = Field( + default=None, description="LetsMesh wire packet hash, links to raw packets" + ) observers: list[ObserverInfo] = Field( default_factory=list, description="All observers that captured this advertisement", diff --git a/src/meshcore_hub/common/schemas/raw_packets.py b/src/meshcore_hub/common/schemas/raw_packets.py new file mode 100644 index 0000000..46d9291 --- /dev/null +++ b/src/meshcore_hub/common/schemas/raw_packets.py @@ -0,0 +1,67 @@ +"""Pydantic schemas for raw packet API endpoints.""" + +from datetime import datetime +from typing import Any, Optional + +from pydantic import BaseModel, Field + + +class RawPacketRead(BaseModel): + """Schema for reading a raw packet. + + When ``redacted`` is true the packet was observed on a channel above the + requesting role's visibility level: ``raw_hex`` and ``decoded`` are nulled + out and only non-sensitive metadata is retained. + """ + + id: str = Field(..., description="Raw packet UUID") + observed_by: Optional[str] = Field( + default=None, description="Observing interface node public key" + ) + observer_name: Optional[str] = Field(default=None, description="Observer node name") + observer_tag_name: Optional[str] = Field( + default=None, description="Observer name from tags" + ) + packet_hash: Optional[str] = Field( + default=None, description="LetsMesh packet hash (links to structured records)" + ) + raw_hex: Optional[str] = Field( + default=None, description="On-air packet bytes as hex (null when redacted)" + ) + packet_type: Optional[int] = Field(default=None, description="Wire packet type") + payload_type: Optional[int] = Field( + default=None, description="Decoder payload type" + ) + event_type: Optional[str] = Field( + default=None, description="How the collector classified the packet" + ) + channel_idx: Optional[int] = Field( + default=None, description="Channel index for channel-message packets" + ) + source_pubkey_prefix: Optional[str] = Field( + default=None, description="Sender public key prefix" + ) + route_type: Optional[str] = Field(default=None, description="Route type") + path_len: Optional[int] = Field(default=None, description="Hop count") + snr: Optional[float] = Field(default=None, description="Signal-to-noise ratio") + decoded: Optional[dict[str, Any]] = Field( + default=None, description="Decoder summary (null when redacted)" + ) + redacted: bool = Field( + default=False, + description="True when the payload was redacted for channel visibility", + ) + received_at: datetime = Field(..., description="When received") + created_at: datetime = Field(..., description="Record creation timestamp") + + class Config: + from_attributes = True + + +class RawPacketList(BaseModel): + """Schema for paginated raw packet list response.""" + + items: list[RawPacketRead] = Field(..., description="List of raw packets") + total: int = Field(..., description="Total number of raw packets") + limit: int = Field(..., description="Page size limit") + offset: int = Field(..., description="Page offset") diff --git a/src/meshcore_hub/web/app.py b/src/meshcore_hub/web/app.py index 41e4b84..2719ad2 100644 --- a/src/meshcore_hub/web/app.py +++ b/src/meshcore_hub/web/app.py @@ -105,6 +105,9 @@ def _build_endpoint_access( "v1/advertisements": { "GET": _OPEN, }, + "v1/packets": { + "GET": _OPEN, + }, "v1/dashboard": { "GET": _OPEN, }, diff --git a/src/meshcore_hub/web/static/css/app.css b/src/meshcore_hub/web/static/css/app.css index 71782a6..433a620 100644 --- a/src/meshcore_hub/web/static/css/app.css +++ b/src/meshcore_hub/web/static/css/app.css @@ -24,6 +24,7 @@ --color-adverts: oklch(0.7 0.17 330); /* magenta */ --color-messages: oklch(0.75 0.18 180); /* teal */ --color-channels: oklch(0.72 0.15 300); /* purple */ + --color-packets: oklch(0.72 0.17 145); /* green */ --color-map: oklch(0.8471 0.199 83.87); /* yellow (matches btn-warning) */ --color-members: oklch(0.72 0.17 50); /* orange */ --color-neutral: oklch(0.3 0.01 250); /* subtle dark grey */ @@ -37,6 +38,7 @@ --color-adverts: oklch(0.55 0.17 330); --color-messages: oklch(0.55 0.18 180); --color-channels: oklch(0.55 0.15 300); + --color-packets: oklch(0.52 0.17 145); --color-map: oklch(0.58 0.16 45); --color-members: oklch(0.55 0.18 25); --color-neutral: oklch(0.85 0.01 250); @@ -68,6 +70,7 @@ .nav-icon-nodes { color: var(--color-nodes); } .nav-icon-adverts { color: var(--color-adverts); } .nav-icon-messages { color: var(--color-messages); } +.nav-icon-packets { color: var(--color-packets); } .nav-icon-map { color: var(--color-map); } .nav-icon-members { color: var(--color-members); } @@ -76,6 +79,7 @@ .navbar .menu li:has(.nav-icon-nodes) { --nav-color: var(--color-nodes); } .navbar .menu li:has(.nav-icon-adverts) { --nav-color: var(--color-adverts); } .navbar .menu li:has(.nav-icon-messages) { --nav-color: var(--color-messages); } +.navbar .menu li:has(.nav-icon-packets) { --nav-color: var(--color-packets); } .navbar .menu li:has(.nav-icon-map) { --nav-color: var(--color-map); } .navbar .menu li:has(.nav-icon-members) { --nav-color: var(--color-members); } diff --git a/src/meshcore_hub/web/static/js/spa/app.js b/src/meshcore_hub/web/static/js/spa/app.js index eedc1e9..cbb27ad 100644 --- a/src/meshcore_hub/web/static/js/spa/app.js +++ b/src/meshcore_hub/web/static/js/spa/app.js @@ -9,7 +9,7 @@ import { Router } from './router.js'; import { isAbortError } from './api.js'; import { html, litRender, getConfig, hasRole, renderAuthSection } from './components.js'; import { loadLocale, t } from './i18n.js'; -import { iconHome, iconDashboard, iconNodes, iconAdvertisements, iconMessages, iconMap, iconMembers, iconPage, iconChannel } from './icons.js'; +import { iconHome, iconDashboard, iconNodes, iconAdvertisements, iconMessages, iconPackets, iconMap, iconMembers, iconPage, iconChannel } from './icons.js'; // Page modules (lazy-loaded) const pages = { @@ -19,6 +19,8 @@ const pages = { nodeDetail: () => import('./pages/node-detail.js'), messages: () => import('./pages/messages.js'), advertisements: () => import('./pages/advertisements.js'), + packets: () => import('./pages/packets.js'), + packetDetail: () => import('./pages/packet-detail.js'), map: () => import('./pages/map.js'), members: () => import('./pages/members.js'), channels: () => import('./pages/channels.js'), @@ -83,6 +85,10 @@ if (features.messages !== false) { if (features.advertisements !== false) { router.addRoute('/advertisements', pageHandler(pages.advertisements)); } +if (features.packets === true) { + router.addRoute('/packets', pageHandler(pages.packets)); + router.addRoute('/packets/:id', pageHandler(pages.packetDetail)); +} if (features.map !== false) { router.addRoute('/map', pageHandler(pages.map)); } @@ -159,6 +165,7 @@ function updatePageTitle(pathname) { if (features.channels !== false) titles['/channels'] = composePageTitle('entities.channels'); if (features.messages !== false) titles['/messages'] = composePageTitle('entities.messages'); if (features.advertisements !== false) titles['/advertisements'] = composePageTitle('entities.advertisements'); + if (features.packets === true) titles['/packets'] = composePageTitle('entities.packets'); if (features.map !== false) titles['/map'] = composePageTitle('entities.map'); if (features.members !== false) titles['/members'] = composePageTitle('entities.members'); titles['/profile'] = composePageTitle('links.profile'); @@ -212,6 +219,9 @@ function renderMobileNav(config) { if (features.messages !== false) { items.push(html`
  • ${iconMessages('h-5 w-5 nav-icon-messages')} ${t('entities.messages')}
    • `); } + if (features.packets === true) { + items.push(html`
  • ${iconPackets('h-5 w-5 nav-icon-packets')} ${t('entities.packets')}
    • `); + } if (features.map !== false) { items.push(html`
  • ${iconMap('h-5 w-5 nav-icon-map')} ${t('entities.map')}
    • `); } diff --git a/src/meshcore_hub/web/static/js/spa/icons.js b/src/meshcore_hub/web/static/js/spa/icons.js index dc47109..13e8b7f 100644 --- a/src/meshcore_hub/web/static/js/spa/icons.js +++ b/src/meshcore_hub/web/static/js/spa/icons.js @@ -26,6 +26,10 @@ export function iconMessages(cls = 'h-5 w-5') { return html``; } +export function iconPackets(cls = 'h-5 w-5') { + return html``; +} + export function iconHome(cls = 'h-5 w-5') { return html``; } diff --git a/src/meshcore_hub/web/static/js/spa/pages/advertisements.js b/src/meshcore_hub/web/static/js/spa/pages/advertisements.js index 53a60b6..07a812a 100644 --- a/src/meshcore_hub/web/static/js/spa/pages/advertisements.js +++ b/src/meshcore_hub/web/static/js/spa/pages/advertisements.js @@ -8,6 +8,23 @@ import { observerIcons, observerDetailRow, toggleObserverDetail, toggleCardObserverDetail } from '../components.js'; import { createAutoRefresh } from '../auto-refresh.js'; +import { iconPackets } from '../icons.js'; + +function packetLink(packetHash, navigate) { + if (!packetHash) { + return nothing; + } + const icon = iconPackets('h-5 w-5 nav-icon-packets'); + const title = t('packets.view_raw'); + const url = `/packets?packet_hash=${packetHash}`; + if (navigate) { + // Inside a card that is itself an : use a span to avoid nested anchors. + return html` { e.preventDefault(); e.stopPropagation(); navigate(url); }}>${icon}`; + } + return html` e.stopPropagation()}>${icon}`; +} function routeTypeBadge(routeType) { if (!routeType) { @@ -38,6 +55,8 @@ export async function render(container, params, router) { const order = query.order || 'desc'; const config = getConfig(); + const features = config.features || {}; + const packetsEnabled = features.packets === true; const tz = config.timezone || ''; const tzBadge = tz && tz !== 'UTC' ? html`${tz}` : nothing; const navigate = (url) => router.navigate(url); @@ -143,6 +162,7 @@ ${displayContent}`, container);
    ${routeTypeBadge(ad.route_type)} ${receiversBlock} + ${packetsEnabled ? packetLink(ad.packet_hash, navigate) : nothing}
    @@ -170,7 +190,7 @@ ${displayContent}`, container); }); const tableRows = advertisements.length === 0 - ? html`` + ? html`` : advertisements.map(ad => { const adName = ad.node_tag_name || ad.node_name || ad.name; const adDescription = ad.node_tag_description; @@ -202,6 +222,7 @@ ${displayContent}`, container); + ${packetsEnabled ? html`` : nothing} ${observerDetailRow(ad.observers || [], null, { hidePath: true })}`; }); @@ -299,6 +320,7 @@ ${mobileSortSelect({ ${sortable(t('common.time'), 'time')} + ${packetsEnabled ? html`` : nothing} diff --git a/src/meshcore_hub/web/static/js/spa/pages/home.js b/src/meshcore_hub/web/static/js/spa/pages/home.js index a20e819..1b8addb 100644 --- a/src/meshcore_hub/web/static/js/spa/pages/home.js +++ b/src/meshcore_hub/web/static/js/spa/pages/home.js @@ -4,7 +4,7 @@ import { getConfig, errorAlert, pageColors, renderStatCard, t, } from '../components.js'; import { - iconDashboard, iconNodes, iconAdvertisements, iconMessages, iconMembers, iconMap, + iconDashboard, iconNodes, iconAdvertisements, iconMessages, iconPackets, iconMembers, iconMap, iconPage, iconInfo, iconChart, iconAntenna, iconUsers, iconChannel, iconSettings, iconFrequency, iconBandwidth, iconSpreadingFactor, iconCodingRate, iconTxPower, } from '../icons.js'; @@ -105,11 +105,11 @@ function renderHeroSection({ networkName, logoUrl, logoInvertLight, networkCity, label: t('entities.messages'), colorVar: '--color-messages', }) : nothing} - ${features.members !== false ? renderNavCard({ - href: '/members', - icon: iconMembers('w-full h-full'), - label: t('entities.members'), - colorVar: '--color-members', + ${features.packets === true ? renderNavCard({ + href: '/packets', + icon: iconPackets('w-full h-full'), + label: t('entities.packets'), + colorVar: '--color-packets', }) : nothing} ${features.map !== false ? renderNavCard({ href: '/map', @@ -117,6 +117,12 @@ function renderHeroSection({ networkName, logoUrl, logoInvertLight, networkCity, label: t('entities.map'), colorVar: '--color-map', }) : nothing} + ${features.members !== false ? renderNavCard({ + href: '/members', + icon: iconMembers('w-full h-full'), + label: t('entities.members'), + colorVar: '--color-members', + }) : nothing} ${features.pages !== false && customPages.length > 0 ? html`
    diff --git a/src/meshcore_hub/web/static/js/spa/pages/messages.js b/src/meshcore_hub/web/static/js/spa/pages/messages.js index 7d9b609..38abc74 100644 --- a/src/meshcore_hub/web/static/js/spa/pages/messages.js +++ b/src/meshcore_hub/web/static/js/spa/pages/messages.js @@ -9,6 +9,22 @@ import { observerIcons, observerDetailRow, toggleObserverDetail, toggleCardObserverDetail } from '../components.js'; import { createAutoRefresh } from '../auto-refresh.js'; +import { iconPackets } from '../icons.js'; + +function packetLink(packetHash, navigate) { + if (!packetHash) { + return nothing; + } + const icon = iconPackets('h-5 w-5 nav-icon-packets'); + const title = t('packets.view_raw'); + const url = `/packets?packet_hash=${packetHash}`; + if (navigate) { + return html` { e.preventDefault(); e.stopPropagation(); navigate(url); }}>${icon}`; + } + return html` e.stopPropagation()}>${icon}`; +} export async function render(container, params, router) { const { signal } = params || {}; @@ -25,6 +41,8 @@ export async function render(container, params, router) { const order = query.order || 'desc'; const config = getConfig(); + const features = config.features || {}; + const packetsEnabled = features.packets === true; let channelLabels = new Map(); const tz = config.timezone || ''; const tzBadge = tz && tz !== 'UTC' ? html`${tz}` : nothing; @@ -265,6 +283,7 @@ ${displayContent}`, container);
    ${receiversBlock} + ${packetsEnabled ? packetLink(msg.packet_hash) : nothing}

    ${displayMessage}

    @@ -294,7 +313,7 @@ ${displayContent}`, container); }); const tableRows = messages.length === 0 - ? html`` + ? html`` : messages.map(msg => { const isChannel = msg.message_type === 'channel'; const typeIcon = isChannel ? '\u{1F4FB}' : '\u{1F464}'; @@ -321,6 +340,7 @@ ${displayContent}`, container); + ${packetsEnabled ? html`` : nothing} ${observerDetailRow(msg.observers || [])}`; }); @@ -427,6 +447,7 @@ ${mobileSortSelect({ ${sortable(t('common.from'), 'from')} ${sortable(t('entities.message'), 'message')} + ${packetsEnabled ? html`` : nothing} diff --git a/src/meshcore_hub/web/static/js/spa/pages/packet-detail.js b/src/meshcore_hub/web/static/js/spa/pages/packet-detail.js new file mode 100644 index 0000000..025b7a4 --- /dev/null +++ b/src/meshcore_hub/web/static/js/spa/pages/packet-detail.js @@ -0,0 +1,119 @@ +import { apiGet, isAbortError } from '../api.js'; +import { + html, litRender, nothing, t, + getConfig, formatDateTime, warningBadge, copyToClipboard +} from '../components.js'; + +function field(label, value) { + return html` +
    + ${label} + ${value} +
    `; +} + +export async function render(container, params, router) { + const { signal } = params || {}; + const id = params.id; + const config = getConfig(); + const tz = config.timezone || ''; + const tzBadge = tz && tz !== 'UTC' ? html`${tz}` : nothing; + + function shell(content, leaf) { + litRender(html` +
    + +
    + +
    +

    ${t('packets.detail_title')}

    + ${tzBadge} +
    +${content}`, container); + } + + shell(html`
    ${t('common.loading')}
    `); + + try { + const [p, channelsData] = await Promise.all([ + apiGet(`/api/v1/packets/${id}`, {}, { signal }), + apiGet('/api/v1/channels', { limit: 200 }, { signal }).catch(() => ({ items: [] })), + ]); + + const channelNames = new Map( + (channelsData.items || []) + .map(c => [parseInt(c.channel_hash, 16), c.name]) + .filter(([idx]) => !Number.isNaN(idx)) + ); + + let channelDisplay = html`—`; + if (p.channel_idx != null) { + const name = channelNames.get(p.channel_idx); + channelDisplay = html`${name ? `${name} (${p.channel_idx})` : `${p.channel_idx}`}`; + } + + const observerDisplay = p.observed_by + ? html`${p.observer_tag_name || p.observer_name || p.observed_by}` + : html`—`; + + const redactedNotice = p.redacted + ? html`
    \u{1F512} ${t('packets.redacted_notice')}
    ` + : nothing; + + const rawBlock = p.redacted + ? nothing + : html` +
    +
    + ${t('packets.col_raw')} + ${p.raw_hex ? html`` : nothing} +
    + 
    ${p.raw_hex || '—'}
    +
    `; + + const decodedBlock = (!p.redacted && p.decoded) + ? html` +
    + ${t('packets.decoded')} + 
    ${JSON.stringify(p.decoded, null, 2)}
    +
    ` + : nothing; + + shell(html` +${redactedNotice} +
    +
    +
    + ${field(t('common.time'), formatDateTime(p.received_at))} + ${field(t('common.observers'), observerDisplay)} + ${field(t('packets.col_event_type'), p.event_type || '—')} + ${field(t('entities.channel'), channelDisplay)} + ${field(t('packets.col_source'), p.source_pubkey_prefix + ? html`${p.source_pubkey_prefix}` + : html`—`)} + ${field(t('packets.packet_hash'), p.packet_hash + ? html`${p.packet_hash}` + : html`—`)} + ${field(t('packets.packet_type'), p.packet_type != null ? p.packet_type : '—')} + ${field(t('packets.payload_type'), p.payload_type != null ? p.payload_type : '—')} + ${field(t('packets.col_route_type'), p.route_type || '—')} + ${field(t('common.snr_db'), p.snr != null ? Number(p.snr).toFixed(1) : '—')} + ${field(t('common.hops'), p.path_len != null ? p.path_len : '—')} +
    + ${rawBlock} + ${decodedBlock} +
    +
    `, p.packet_hash || p.event_type); + } catch (e) { + if (isAbortError(e)) return; + if (e.status === 404) { + shell(html`
    ${t('common.entity_not_found_details', { entity: t('entities.packet').toLowerCase() })}
    `); + return; + } + shell(warningBadge(e.message)); + } +} diff --git a/src/meshcore_hub/web/static/js/spa/pages/packets.js b/src/meshcore_hub/web/static/js/spa/pages/packets.js new file mode 100644 index 0000000..a74030f --- /dev/null +++ b/src/meshcore_hub/web/static/js/spa/pages/packets.js @@ -0,0 +1,292 @@ +import { apiGet, isAbortError } from '../api.js'; +import { + html, litRender, nothing, t, + getConfig, formatDateTime, formatDateTimeShort, + warningBadge, + pagination, sortableTableHeader, mobileSortSelect, + renderFilterCard, autoSubmit, submitOnEnter +} from '../components.js'; +import { createAutoRefresh } from '../auto-refresh.js'; + +const EVENT_TYPES = [ + // Structured classifications + 'advertisement', 'channel_msg_recv', 'contact_msg_recv', + 'trace_data', 'telemetry_response', 'path_updated', 'status_response', + // Per-payload-type classifications for otherwise-unhandled packets + 'req', 'response', 'ack', 'encrypted_direct', 'encrypted_channel', + 'grp_data', 'anon_req', 'multipart', 'control', 'raw_custom', + 'advert', 'path', 'trace', 'letsmesh_packet', +]; + +function lockBadge() { + return html`\u{1F512}`; +} + +function channelLabel(packet, channelNames) { + if (packet.channel_idx == null) { + return html`—`; + } + const name = channelNames.get(packet.channel_idx); + const text = name ? `${name} (${packet.channel_idx})` : `${packet.channel_idx}`; + return html`${text}${packet.redacted ? html` ${lockBadge()}` : nothing}`; +} + +function observerCell(packet) { + const name = packet.observer_tag_name || packet.observer_name; + if (!packet.observed_by) { + return html`—`; + } + const label = name || (packet.observed_by.slice(0, 12) + '…'); + return html` e.stopPropagation()}>${label}`; +} + +export async function render(container, params, router) { + const { signal } = params || {}; + const query = params.query || {}; + const search = query.search || ''; + const packet_hash = query.packet_hash || ''; + const event_type = query.event_type || ''; + const channel_idx = query.channel_idx || ''; + const route_type = query.route_type || 'all'; + const observed_by = query.observed_by + ? (Array.isArray(query.observed_by) ? query.observed_by : [query.observed_by]) + : []; + const min_snr = query.min_snr || ''; + const max_snr = query.max_snr || ''; + const page = parseInt(query.page, 10) || 1; + const limit = parseInt(query.limit, 10) || 20; + const offset = (page - 1) * limit; + const sort = query.sort || 'time'; + const order = query.order || 'desc'; + + const config = getConfig(); + const tz = config.timezone || ''; + const tzBadge = tz && tz !== 'UTC' ? html`${tz}` : nothing; + const navigate = (url) => router.navigate(url); + + let lastContent = nothing; + let lastTotal = null; + + function renderPage(content, { total = null, error = null } = {}) { + if (!error) { + lastContent = content; + lastTotal = total; + } + const displayContent = error ? lastContent : content; + const displayTotal = error ? lastTotal : total; + litRender(html` +
    +

    ${t('entities.packets')}

    + ${tzBadge} +
    +
    + ${displayTotal !== null + ? html`${t('common.total', { count: displayTotal })}` + : nothing} + + ${error ? warningBadge(error) : nothing} +
    +${displayContent}`, container); + } + + renderPage(nothing); + + async function fetchAndRenderData() { + try { + const apiParams = { limit, offset, search, sort, order }; + if (packet_hash) apiParams.packet_hash = packet_hash; + if (event_type) apiParams.event_type = event_type; + if (channel_idx !== '') apiParams.channel_idx = channel_idx; + if (route_type && route_type !== 'all') apiParams.route_type = route_type; + if (observed_by.length > 0) apiParams.observed_by = observed_by; + if (min_snr !== '') apiParams.min_snr = min_snr; + if (max_snr !== '') apiParams.max_snr = max_snr; + + const [data, nodesData, channelsData] = await Promise.all([ + apiGet('/api/v1/packets', apiParams, { signal }), + apiGet('/api/v1/nodes', { limit: 500, observer: true }, { signal }), + apiGet('/api/v1/channels', { limit: 200 }, { signal }).catch(() => ({ items: [] })), + ]); + + const packets = data.items || []; + const total = data.total || 0; + const totalPages = Math.ceil(total / limit); + + const sortedNodes = (nodesData.items || []).map(n => { + const tagName = n.tags?.find(tg => tg.key === 'name')?.value; + return { ...n, _sortName: (tagName || n.name || '').toLowerCase(), _displayName: tagName || n.name || n.public_key.slice(0, 12) + '...' }; + }).sort((a, b) => a._sortName.localeCompare(b._sortName)); + + const channelList = (channelsData.items || []).map(c => ({ + name: c.name, + idx: parseInt(c.channel_hash, 16), + })).filter(c => !Number.isNaN(c.idx)); + const channelNames = new Map(channelList.map(c => [c.idx, c.name])); + + const noneFound = html`
    ${t('common.no_entity_found', { entity: t('entities.packets').toLowerCase() })}
    `; + + const mobileCards = packets.length === 0 + ? noneFound + : packets.map(p => html` + +
    +
    +
    +
    ${p.event_type || '—'}
    +
    ${channelLabel(p, channelNames)}
    +
    +
    +
    ${formatDateTimeShort(p.received_at)}
    +
    ${observerCell(p)}
    +
    +
    +
    + ${p.snr != null ? html`${Number(p.snr).toFixed(1)} dB` : nothing} + ${p.path_len != null ? html`${p.path_len} ${t('common.hops').toLowerCase()}` : nothing} +
    +
    +     `); + + const tableRows = packets.length === 0 + ? html`` + : packets.map(p => html` navigate(`/packets/${p.id}`)}> + + + + + + + + `); + + const paginationBlock = pagination(page, totalPages, '/packets', { + search, packet_hash, event_type, channel_idx, route_type, observed_by, + min_snr, max_snr, limit, sort, order, + }); + + const filterFields = [ + () => html` +
    + + +
    `, + () => html` +
    + + +
    `, + () => html` +
    + + +
    `, + ]; + if (sortedNodes.length > 0) { + filterFields.push(() => html` +
    + + +
    `); + } + filterFields.push(() => html` +
    + + +
    `); + filterFields.push(() => html` +
    + + +
    `); + + const hasActiveFilters = search !== '' || event_type !== '' || channel_idx !== '' || observed_by.length > 0 || min_snr !== '' || max_snr !== ''; + const existingDetails = container.querySelector('details.collapse'); + const isFilterOpen = existingDetails ? existingDetails.open : hasActiveFilters; + + const filterCard = renderFilterCard({ + fields: filterFields, + basePath: '/packets', + navigate, + collapsible: true, + defaultOpen: isFilterOpen, + }); + + const headerParams = { search, packet_hash, event_type, channel_idx, route_type, observed_by, min_snr, max_snr, limit }; + const sortable = (label, sortKey) => sortableTableHeader(label, { + sortKey, currentSort: sort, currentOrder: order, + navigate, basePath: '/packets', params: headerParams, + }); + + const packetHashChip = packet_hash + ? html`
    + + ${packet_hash} + + ${t('common.clear_filters')} +
    ` + : nothing; + + renderPage(html`${filterCard} +${packetHashChip} + +${mobileSortSelect({ + currentSort: sort, currentOrder: order, + navigate, basePath: '/packets', + params: headerParams, + options: [ + { value: 'time:desc', label: t('packets.sort.newest') }, + { value: 'time:asc', label: t('packets.sort.oldest') }, + { value: 'event_type:asc', label: t('packets.sort.event_az') }, + { value: 'snr:desc', label: t('packets.sort.snr_high') }, + { value: 'path_len:desc', label: t('packets.sort.path_high') }, + ], +})} + +
    + ${mobileCards} +
    + +
    ${t('common.no_entity_found', { entity: t('entities.advertisements').toLowerCase() })}
    ${t('common.no_entity_found', { entity: t('entities.advertisements').toLowerCase() })}
    ${routeTypeBadge(ad.route_type)} ${formatDateTime(ad.received_at)} ${receiversBlock}${packetLink(ad.packet_hash)}
    ${t('advertisements.col_route_type')}${t('common.observers')}
    ${t('common.no_entity_found', { entity: t('entities.messages').toLowerCase() })}
    ${t('common.no_entity_found', { entity: t('entities.messages').toLowerCase() })}
    ${displayMessage} ${receiversBlock}${packetLink(msg.packet_hash)}
    ${t('common.observers')}
    ${t('common.no_entity_found', { entity: t('entities.packets').toLowerCase() })}
    ${formatDateTime(p.received_at)}${p.packet_hash ? html`${p.packet_hash}` : html`—`}${observerCell(p)}${p.event_type || '—'}${channelLabel(p, channelNames)}${p.snr != null ? Number(p.snr).toFixed(1) : '—'}${p.path_len != null ? p.path_len : '—'}
    + + + ${sortable(t('common.time'), 'time')} + + + ${sortable(t('packets.col_event_type'), 'event_type')} + + ${sortable(t('common.snr_db'), 'snr')} + ${sortable(t('common.hops'), 'path_len')} + + + + ${tableRows} + +
    ${t('packets.packet_hash')}${t('common.observers')}${t('entities.channel')}
    + + +${paginationBlock}`, { total }); + + } catch (e) { + if (isAbortError(e)) return; + renderPage(nothing, { error: e.message }); + } + } + + await fetchAndRenderData(); + + const toggleEl = container.querySelector('#auto-refresh-toggle'); + const { cleanup } = createAutoRefresh({ + fetchAndRender: fetchAndRenderData, + toggleContainer: toggleEl, + }); + return cleanup; +} diff --git a/src/meshcore_hub/web/static/locales/en.json b/src/meshcore_hub/web/static/locales/en.json index b1c545b..95c771d 100644 --- a/src/meshcore_hub/web/static/locales/en.json +++ b/src/meshcore_hub/web/static/locales/en.json @@ -1,301 +1,329 @@ { - "entities": { - "home": "Home", - "dashboard": "Dashboard", - "nodes": "Nodes", - "node": "Node", - "node_detail": "Node Detail", - "advertisements": "Adverts", - "advertisement": "Advert", - "messages": "Messages", - "message": "Message", - "map": "Map", - "members": "Members", - "member": "Member", - "tags": "Tags", - "tag": "Tag", - "channel": "Channel", - "channels": "Channels" - }, - "common": { - "filter": "Filter", - "filters": "Filters", - "clear": "Clear", - "clear_filters": "Clear Filters", - "search": "Search", - "cancel": "Cancel", - "delete": "Delete", - "edit": "Edit", - "move": "Move", - "save": "Save", - "save_changes": "Save Changes", - "add": "Add", - "add_entity": "Add {{entity}}", - "add_new_entity": "Add New {{entity}}", - "edit_entity": "Edit {{entity}}", - "delete_entity": "Delete {{entity}}", - "delete_all_entity": "Delete All {{entity}}", - "move_entity": "Move {{entity}}", - "move_entity_to_another_node": "Move {{entity}} to Another Node", - "copy_entity": "Copy {{entity}}", - "copy_all_entity_to_another_node": "Copy All {{entity}} to Another Node", - "view_entity": "View {{entity}}", - "recent_entity": "Recent {{entity}}", - "total_entity": "Total {{entity}}", - "all_entity": "All {{entity}}", - "no_entity_found": "No {{entity}} found", - "no_entity_recorded": "No {{entity}} recorded", - "no_entity_defined": "No {{entity}} defined", - "no_entity_in_database": "No {{entity}} in database", - "no_entity_configured": "No {{entity}} configured", - "no_entity_yet": "No {{entity}} yet", - "entity_not_found_details": "{{entity}} not found: {{details}}", - "page_not_found": "Page not found", - "delete_entity_confirm": "Are you sure you want to delete {{entity}} {{name}}?", - "delete_all_entity_confirm": "Are you sure you want to delete all {{count}} {{entity}} from {{name}}?", - "cannot_be_undone": "This action cannot be undone.", - "entity_added_success": "{{entity}} added successfully", - "entity_updated_success": "{{entity}} updated successfully", - "entity_deleted_success": "{{entity}} deleted successfully", - "entity_moved_success": "{{entity}} moved successfully", - "all_entity_deleted_success": "All {{entity}} deleted successfully", - "copy_all_entity_description": "Copy all {{count}} {{entity}} from {{name}} to another node.", - "previous": "Previous", - "next": "Next", - "go_home": "Go Home", - "loading": "Loading...", - "error": "Error", - "failed_to_load_page": "Failed to load page", - "total": "{{count}} total", - "shown": "{{count}} shown", - "count_entity": "{{count}} {{entity}}", - "type": "Type", - "name": "Name", - "key": "Key", - "value": "Value", - "time": "Time", - "actions": "Actions", - "updated": "Updated", - "sign_in": "Sign In", - "sign_out": "Sign Out", - "view_details": "View Details", - "all_types": "All Types", - "all_channels": "All Channels", - "all_members": "All Members", - "all_operators": "All Operators", - "filter_member_label": "Member", - "filter_operator_label": "Operator", - "filter_observer_label": "Observer", - "node_type": "Node Type", - "show": "Show", - "search_placeholder": "Search by name, ID, or public key...", - "contact": "Contact", - "description": "Description", - "callsign": "Callsign", - "tags": "Tags", - "last_seen": "Last Seen", - "first_seen_label": "First seen:", - "last_seen_label": "Last seen:", - "location": "Location", - "public_key": "Public Key", - "received": "Received", - "received_by": "Received By", - "observers": "Observers", - "snr_db": "SNR (dB)", - "hops": "Hops", - "from": "From", - "close": "close", - "unnamed": "Unnamed", - "unnamed_node": "Unnamed Node", - "validation_invalid_number": "Value must be a valid number", - "validation_invalid_boolean": "Value must be true, false, yes, no, 1, or 0", - "sort_by": "Sort by" - }, - "links": { - "website": "Website", - "github": "GitHub", - "discord": "Discord", - "youtube": "YouTube", - "profile": "Profile" - }, - "auto_refresh": { - "pause": "Pause auto-refresh", - "resume": "Resume auto-refresh" - }, - "time": { - "days_ago": "{{count}}d ago", - "hours_ago": "{{count}}h ago", - "minutes_ago": "{{count}}m ago", - "less_than_minute": "<1m ago", - "last_7_days": "Last 7 days", - "per_day_last_7_days": "Per day (last 7 days)", - "over_time_last_7_days": "Over time (last 7 days)", - "activity_per_day_last_7_days": "Activity per day (last 7 days)" - }, - "node_types": { - "chat": "Chat", - "repeater": "Repeater", - "companion": "Companion", - "room": "Room Server", - "unknown": "Unknown" - }, - "home": { - "welcome_default": "Welcome to the {{network_name}} mesh network dashboard. Monitor network activity, view connected nodes, and explore message history.", - "all_discovered_nodes": "All discovered nodes", - "network_info": "Network Info", - "network_activity": "Network Activity", - "frequency": "Frequency", - "bandwidth": "Bandwidth", - "spreading_factor": "Spreading Factor", - "coding_rate": "Coding Rate", - "tx_power": "TX Power" - }, - "dashboard": { - "all_discovered_nodes": "All discovered nodes", - "recent_channel_messages": "Recent Channel Messages", - "channel": "Channel {{number}}" - }, - "nodes": { - "scan_to_add": "Scan to add as contact", - "ownership": "Ownership", - "adopt": "Adopt", - "release": "Release", - "adopted_by": "Adopted by {{name}}", - "adopted_by_prefix": "Adopted by", - "not_adopted": "This node has not been adopted by any operator.", - "adopt_success": "Node adopted successfully", - "release_success": "Node released successfully", - "release_confirm": "Are you sure you want to release this node?", - "sort": { - "last_seen_newest": "Last Seen (newest)", - "last_seen_oldest": "Last Seen (oldest)", - "name_az": "Name (A\u2013Z)", - "name_za": "Name (Z\u2013A)", - "key_asc": "Public Key (ascending)", - "key_desc": "Public Key (descending)" - } - }, - "advertisements": { - "filter_route_type_label": "Advert Type", - "route_type_all": "All", - "route_type_flood": "Flood & Relay", - "route_type_direct": "Zero-hop only", - "route_type_unknown": "Unknown", - "col_route_type": "Type", - "sort": { - "newest": "Time (newest)", - "oldest": "Time (oldest)", - "node_az": "Node (A\u2013Z)", - "node_za": "Node (Z\u2013A)", - "key_asc": "Public Key (ascending)", - "key_desc": "Public Key (descending)" - } - }, - "messages": { - "type_direct": "Direct", - "type_channel": "Channel", - "type_contact": "Contact", - "type_public": "Public", - "sort": { - "newest": "Time (newest)", - "oldest": "Time (oldest)", - "type_az": "Type (A\u2013Z)", - "type_za": "Type (Z\u2013A)", - "from_az": "From (A\u2013Z)", - "from_za": "From (Z\u2013A)", - "message_az": "Message (A\u2013Z)", - "message_za": "Message (Z\u2013A)" - } - }, - "map": { - "show_labels": "Show Labels", - "infrastructure_only": "Infrastructure Only", - "legend": "Legend:", - "infrastructure": "Infrastructure", - "public": "Public", - "nodes_on_map": "{{count}} nodes on map", - "nodes_none_have_coordinates": "{{count}} nodes (none have coordinates)", - "gps_description": "Nodes are placed on the map based on GPS coordinates from node reports or manual tags.", - "owner": "Owner:", - "role": "Role:", - "select_destination_node": "-- Select destination node --" - }, - "members": { - "empty_state_description": "No members yet.", - "empty_description": "Members will appear here once users log in and adopt nodes." - }, - "channels": { - "title": "Channels", - "add_channel": "Add Channel", - "edit_channel": "Edit Channel", - "delete_channel": "Delete Channel", - "delete_confirm": "Are you sure you want to delete channel {{name}}?", - "name_label": "Channel Name", - "key_label": "Channel Key (hex)", - "visibility_label": "Visibility", - "visibility_community": "Community", - "visibility_member": "Member", - "visibility_operator": "Operator", - "visibility_admin": "Admin", - "enabled_label": "Enabled", - "channel_hash_label": "Hash", - "disabled": "Disabled", - "optgroup_standard": "Standard", - "optgroup_custom": "Custom" - }, - "not_found": { - "description": "The page you're looking for doesn't exist or has been moved." - }, - "custom_page": { - "failed_to_load": "Failed to load page" - }, - "auth": { - "login": "Login", - "logout": "Logout", - "login_required": "Login required", - "admin_required": "Admin access required", - "login_hint": "Log in to access admin features", - "logged_in_as": "Logged in as {{name}}", - "session_expired": "Session expired, please log in again", - "role_admin": "admin", - "role_member": "member" - }, - "user_profile": { - "title": "Profile", - "your_profile": "Your Profile", - "profile_updated": "Profile updated successfully", - "save_profile": "Save Profile", - "name_label": "Display Name", - "name_placeholder": "Your name or preferred name", - "callsign_label": "Callsign", - "callsign_placeholder": "Amateur radio callsign (e.g., W1ABC)", - "description_label": "Description", - "description_placeholder": "A short bio or description", - "url_label": "Website / Profile Link", - "url_placeholder": "https://...", - "adopted_nodes": "Adopted Nodes", - "no_adopted_nodes": "No adopted nodes", - "login_to_view": "Log in to view your profile", - "view_profile": "View Profile", - "edit_profile": "Edit Profile", - "role_operator": "Operator", - "role_member": "Member", - "member_since": "Member since {{date}}" - }, - "members_page": { - "operators": "Operators", - "members": "Members", - "node_count": "{{count}} nodes", - "empty_state": "No members yet", - "empty_description": "Members will appear here once users log in and adopt nodes." - }, - "footer": { - "powered_by": "Powered by" - }, - "errors": { - "go_home": "Go Home", - "not_found": "Page not found", - "server_error": "Internal server error", - "unexpected": "An unexpected error occurred" + "entities": { + "home": "Home", + "dashboard": "Dashboard", + "nodes": "Nodes", + "node": "Node", + "node_detail": "Node Detail", + "advertisements": "Adverts", + "advertisement": "Advert", + "messages": "Messages", + "packets": "Packets", + "message": "Message", + "packet": "Packet", + "map": "Map", + "members": "Members", + "member": "Member", + "tags": "Tags", + "tag": "Tag", + "channel": "Channel", + "channels": "Channels" + }, + "common": { + "filter": "Filter", + "filters": "Filters", + "clear": "Clear", + "clear_filters": "Clear Filters", + "search": "Search", + "cancel": "Cancel", + "delete": "Delete", + "edit": "Edit", + "move": "Move", + "save": "Save", + "save_changes": "Save Changes", + "add": "Add", + "add_entity": "Add {{entity}}", + "add_new_entity": "Add New {{entity}}", + "edit_entity": "Edit {{entity}}", + "delete_entity": "Delete {{entity}}", + "delete_all_entity": "Delete All {{entity}}", + "move_entity": "Move {{entity}}", + "move_entity_to_another_node": "Move {{entity}} to Another Node", + "copy_entity": "Copy {{entity}}", + "copy_all_entity_to_another_node": "Copy All {{entity}} to Another Node", + "view_entity": "View {{entity}}", + "recent_entity": "Recent {{entity}}", + "total_entity": "Total {{entity}}", + "all_entity": "All {{entity}}", + "no_entity_found": "No {{entity}} found", + "no_entity_recorded": "No {{entity}} recorded", + "no_entity_defined": "No {{entity}} defined", + "no_entity_in_database": "No {{entity}} in database", + "no_entity_configured": "No {{entity}} configured", + "no_entity_yet": "No {{entity}} yet", + "entity_not_found_details": "{{entity}} not found: {{details}}", + "page_not_found": "Page not found", + "delete_entity_confirm": "Are you sure you want to delete {{entity}} {{name}}?", + "delete_all_entity_confirm": "Are you sure you want to delete all {{count}} {{entity}} from {{name}}?", + "cannot_be_undone": "This action cannot be undone.", + "entity_added_success": "{{entity}} added successfully", + "entity_updated_success": "{{entity}} updated successfully", + "entity_deleted_success": "{{entity}} deleted successfully", + "entity_moved_success": "{{entity}} moved successfully", + "all_entity_deleted_success": "All {{entity}} deleted successfully", + "copy_all_entity_description": "Copy all {{count}} {{entity}} from {{name}} to another node.", + "previous": "Previous", + "next": "Next", + "go_home": "Go Home", + "loading": "Loading...", + "error": "Error", + "failed_to_load_page": "Failed to load page", + "total": "{{count}} total", + "shown": "{{count}} shown", + "count_entity": "{{count}} {{entity}}", + "type": "Type", + "name": "Name", + "key": "Key", + "value": "Value", + "time": "Time", + "actions": "Actions", + "updated": "Updated", + "sign_in": "Sign In", + "sign_out": "Sign Out", + "view_details": "View Details", + "all_types": "All Types", + "all_channels": "All Channels", + "all_members": "All Members", + "all_operators": "All Operators", + "filter_member_label": "Member", + "filter_operator_label": "Operator", + "filter_observer_label": "Observer", + "node_type": "Node Type", + "show": "Show", + "search_placeholder": "Search by name, ID, or public key...", + "contact": "Contact", + "description": "Description", + "callsign": "Callsign", + "tags": "Tags", + "last_seen": "Last Seen", + "first_seen_label": "First seen:", + "last_seen_label": "Last seen:", + "location": "Location", + "public_key": "Public Key", + "received": "Received", + "received_by": "Received By", + "observers": "Observers", + "snr_db": "SNR (dB)", + "hops": "Hops", + "from": "From", + "close": "close", + "unnamed": "Unnamed", + "unnamed_node": "Unnamed Node", + "validation_invalid_number": "Value must be a valid number", + "validation_invalid_boolean": "Value must be true, false, yes, no, 1, or 0", + "sort_by": "Sort by" + }, + "links": { + "website": "Website", + "github": "GitHub", + "discord": "Discord", + "youtube": "YouTube", + "profile": "Profile" + }, + "auto_refresh": { + "pause": "Pause auto-refresh", + "resume": "Resume auto-refresh" + }, + "time": { + "days_ago": "{{count}}d ago", + "hours_ago": "{{count}}h ago", + "minutes_ago": "{{count}}m ago", + "less_than_minute": "<1m ago", + "last_7_days": "Last 7 days", + "per_day_last_7_days": "Per day (last 7 days)", + "over_time_last_7_days": "Over time (last 7 days)", + "activity_per_day_last_7_days": "Activity per day (last 7 days)" + }, + "node_types": { + "chat": "Chat", + "repeater": "Repeater", + "companion": "Companion", + "room": "Room Server", + "unknown": "Unknown" + }, + "home": { + "welcome_default": "Welcome to the {{network_name}} mesh network dashboard. Monitor network activity, view connected nodes, and explore message history.", + "all_discovered_nodes": "All discovered nodes", + "network_info": "Network Info", + "network_activity": "Network Activity", + "frequency": "Frequency", + "bandwidth": "Bandwidth", + "spreading_factor": "Spreading Factor", + "coding_rate": "Coding Rate", + "tx_power": "TX Power" + }, + "dashboard": { + "all_discovered_nodes": "All discovered nodes", + "recent_channel_messages": "Recent Channel Messages", + "channel": "Channel {{number}}" + }, + "nodes": { + "scan_to_add": "Scan to add as contact", + "ownership": "Ownership", + "adopt": "Adopt", + "release": "Release", + "adopted_by": "Adopted by {{name}}", + "adopted_by_prefix": "Adopted by", + "not_adopted": "This node has not been adopted by any operator.", + "adopt_success": "Node adopted successfully", + "release_success": "Node released successfully", + "release_confirm": "Are you sure you want to release this node?", + "sort": { + "last_seen_newest": "Last Seen (newest)", + "last_seen_oldest": "Last Seen (oldest)", + "name_az": "Name (A–Z)", + "name_za": "Name (Z–A)", + "key_asc": "Public Key (ascending)", + "key_desc": "Public Key (descending)" } + }, + "advertisements": { + "filter_route_type_label": "Advert Type", + "route_type_all": "All", + "route_type_flood": "Flood & Relay", + "route_type_direct": "Zero-hop only", + "route_type_unknown": "Unknown", + "col_route_type": "Type", + "sort": { + "newest": "Time (newest)", + "oldest": "Time (oldest)", + "node_az": "Node (A–Z)", + "node_za": "Node (Z–A)", + "key_asc": "Public Key (ascending)", + "key_desc": "Public Key (descending)" + } + }, + "messages": { + "type_direct": "Direct", + "type_channel": "Channel", + "type_contact": "Contact", + "type_public": "Public", + "sort": { + "newest": "Time (newest)", + "oldest": "Time (oldest)", + "type_az": "Type (A–Z)", + "type_za": "Type (Z–A)", + "from_az": "From (A–Z)", + "from_za": "From (Z–A)", + "message_az": "Message (A–Z)", + "message_za": "Message (Z–A)" + } + }, + "packets": { + "filter_event_type": "Event Type", + "filter_min_snr": "Min SNR", + "filter_max_snr": "Max SNR", + "redacted": "Restricted", + "redacted_title": "Hidden by channel visibility", + "redacted_notice": "This packet was observed on a channel above your access level. The payload is hidden.", + "detail_title": "Packet Detail", + "decoded": "Decoded", + "copy_raw": "Copy raw hex", + "view_raw": "View raw packets", + "packet_hash": "Packet Hash", + "packet_type": "Packet Type", + "payload_type": "Payload Type", + "col_event_type": "Event", + "col_source": "Source", + "col_route_type": "Route Type", + "col_raw": "Raw", + "sort": { + "newest": "Time (newest)", + "oldest": "Time (oldest)", + "event_az": "Event (A–Z)", + "snr_high": "SNR (highest)", + "path_high": "Hops (most)" + } + }, + "map": { + "show_labels": "Show Labels", + "infrastructure_only": "Infrastructure Only", + "legend": "Legend:", + "infrastructure": "Infrastructure", + "public": "Public", + "nodes_on_map": "{{count}} nodes on map", + "nodes_none_have_coordinates": "{{count}} nodes (none have coordinates)", + "gps_description": "Nodes are placed on the map based on GPS coordinates from node reports or manual tags.", + "owner": "Owner:", + "role": "Role:", + "select_destination_node": "-- Select destination node --" + }, + "members": { + "empty_state_description": "No members yet.", + "empty_description": "Members will appear here once users log in and adopt nodes." + }, + "channels": { + "title": "Channels", + "add_channel": "Add Channel", + "edit_channel": "Edit Channel", + "delete_channel": "Delete Channel", + "delete_confirm": "Are you sure you want to delete channel {{name}}?", + "name_label": "Channel Name", + "key_label": "Channel Key (hex)", + "visibility_label": "Visibility", + "visibility_community": "Community", + "visibility_member": "Member", + "visibility_operator": "Operator", + "visibility_admin": "Admin", + "enabled_label": "Enabled", + "channel_hash_label": "Hash", + "disabled": "Disabled", + "optgroup_standard": "Standard", + "optgroup_custom": "Custom" + }, + "not_found": { + "description": "The page you're looking for doesn't exist or has been moved." + }, + "custom_page": { + "failed_to_load": "Failed to load page" + }, + "auth": { + "login": "Login", + "logout": "Logout", + "login_required": "Login required", + "admin_required": "Admin access required", + "login_hint": "Log in to access admin features", + "logged_in_as": "Logged in as {{name}}", + "session_expired": "Session expired, please log in again", + "role_admin": "admin", + "role_member": "member" + }, + "user_profile": { + "title": "Profile", + "your_profile": "Your Profile", + "profile_updated": "Profile updated successfully", + "save_profile": "Save Profile", + "name_label": "Display Name", + "name_placeholder": "Your name or preferred name", + "callsign_label": "Callsign", + "callsign_placeholder": "Amateur radio callsign (e.g., W1ABC)", + "description_label": "Description", + "description_placeholder": "A short bio or description", + "url_label": "Website / Profile Link", + "url_placeholder": "https://...", + "adopted_nodes": "Adopted Nodes", + "no_adopted_nodes": "No adopted nodes", + "login_to_view": "Log in to view your profile", + "view_profile": "View Profile", + "edit_profile": "Edit Profile", + "role_operator": "Operator", + "role_member": "Member", + "member_since": "Member since {{date}}" + }, + "members_page": { + "operators": "Operators", + "members": "Members", + "node_count": "{{count}} nodes", + "empty_state": "No members yet", + "empty_description": "Members will appear here once users log in and adopt nodes." + }, + "footer": { + "powered_by": "Powered by" + }, + "errors": { + "go_home": "Go Home", + "not_found": "Page not found", + "server_error": "Internal server error", + "unexpected": "An unexpected error occurred" + } } diff --git a/src/meshcore_hub/web/static/locales/nl.json b/src/meshcore_hub/web/static/locales/nl.json index e7490d0..4ab8120 100644 --- a/src/meshcore_hub/web/static/locales/nl.json +++ b/src/meshcore_hub/web/static/locales/nl.json @@ -1,219 +1,247 @@ { - "entities": { - "home": "Startpagina", - "dashboard": "Dashboard", - "nodes": "Knooppunten", - "node": "Knooppunt", - "node_detail": "Knooppuntdetails", - "advertisements": "Advertenties", - "advertisement": "Advertentie", - "messages": "Berichten", - "message": "Bericht", - "map": "Kaart", - "members": "Leden", - "member": "Lid", - "admin": "Beheer", - "tags": "Labels", - "tag": "Label" - }, - "common": { - "filter": "Filter", - "clear": "Wissen", - "clear_filters": "Filters wissen", - "search": "Zoeken", - "cancel": "Annuleren", - "delete": "Verwijderen", - "edit": "Bewerken", - "move": "Verplaatsen", - "save": "Opslaan", - "save_changes": "Wijzigingen opslaan", - "add": "Toevoegen", - "add_entity": "{{entity}} toevoegen", - "add_new_entity": "Nieuwe {{entity}} toevoegen", - "edit_entity": "{{entity}} bewerken", - "delete_entity": "{{entity}} verwijderen", - "delete_all_entity": "Alle {{entity}} verwijderen", - "move_entity": "{{entity}} verplaatsen", - "move_entity_to_another_node": "{{entity}} naar ander knooppunt verplaatsen", - "copy_entity": "{{entity}} kopiëren", - "copy_all_entity_to_another_node": "Alle {{entity}} naar ander knooppunt kopiëren", - "view_entity": "{{entity}} bekijken", - "recent_entity": "Recente {{entity}}", - "total_entity": "Totaal {{entity}}", - "all_entity": "Alle {{entity}}", - "no_entity_found": "Geen {{entity}} gevonden", - "no_entity_recorded": "Geen {{entity}} geregistreerd", - "no_entity_defined": "Geen {{entity}} gedefinieerd", - "no_entity_in_database": "Geen {{entity}} in database", - "no_entity_configured": "Geen {{entity}} geconfigureerd", - "no_entity_yet": "Nog geen {{entity}}", - "entity_not_found_details": "{{entity}} niet gevonden: {{details}}", - "page_not_found": "Pagina niet gevonden", - "delete_entity_confirm": "Weet u zeker dat u {{entity}} {{name}} wilt verwijderen?", - "delete_all_entity_confirm": "Weet u zeker dat u alle {{count}} {{entity}} van {{name}} wilt verwijderen?", - "cannot_be_undone": "Deze actie kan niet ongedaan worden gemaakt.", - "entity_added_success": "{{entity}} succesvol toegevoegd", - "entity_updated_success": "{{entity}} succesvol bijgewerkt", - "entity_deleted_success": "{{entity}} succesvol verwijderd", - "entity_moved_success": "{{entity}} succesvol verplaatst", - "all_entity_deleted_success": "Alle {{entity}} succesvol verwijderd", - "copy_all_entity_description": "Kopieer alle {{count}} {{entity}} van {{name}} naar een ander knooppunt.", - "previous": "Vorige", - "next": "Volgende", - "go_home": "Naar startpagina", - "loading": "Laden...", - "error": "Fout", - "failed_to_load_page": "Pagina laden mislukt", - "total": "{{count}} totaal", - "shown": "{{count}} weergegeven", - "count_entity": "{{count}} {{entity}}", - "type": "Type", - "name": "Naam", - "key": "Sleutel", - "value": "Waarde", - "time": "Tijd", - "actions": "Acties", - "updated": "Bijgewerkt", - "sign_in": "Inloggen", - "sign_out": "Uitloggen", - "view_details": "Details bekijken", - "all_types": "Alle types", - "node_type": "Knooppunttype", - "show": "Toon", - "search_placeholder": "Zoek op naam, ID of publieke sleutel...", - "contact": "Contact", - "description": "Beschrijving", - "callsign": "Roepnaam", - "tags": "Labels", - "last_seen": "Laatst gezien", - "first_seen_label": "Eerst gezien:", - "last_seen_label": "Laatst gezien:", - "location": "Locatie", - "public_key": "Publieke sleutel", - "received": "Ontvangen", - "received_by": "Ontvangen door", - "receivers": "Ontvangers", - "from": "Van", - "close": "sluiten", - "unnamed": "Naamloos", - "unnamed_node": "Naamloos knooppunt", - "all_operators": "Alle Operators", - "filter_operator_label": "Operator" - }, - "links": { - "website": "Website", - "github": "GitHub", - "discord": "Discord", - "youtube": "YouTube", - "profile": "Profiel" - }, - "auto_refresh": { - "pause": "Pauzeer verversen", - "resume": "Hervat verversen" - }, - "time": { - "days_ago": "{{count}}d geleden", - "hours_ago": "{{count}}u geleden", - "minutes_ago": "{{count}}m geleden", - "less_than_minute": "<1m geleden", - "last_7_days": "Laatste 7 dagen", - "per_day_last_7_days": "Per dag (laatste 7 dagen)", - "over_time_last_7_days": "In de tijd (laatste 7 dagen)", - "activity_per_day_last_7_days": "Activiteit per dag (laatste 7 dagen)" - }, - "node_types": { - "chat": "Chat", - "repeater": "Repeater", - "room": "Ruimte", - "unknown": "Onbekend" - }, - "home": { - "welcome_default": "Welkom bij het {{network_name}} mesh-netwerk dashboard. Monitor netwerkactiviteit, bekijk verbonden knooppunten en verken berichtgeschiedenis.", - "all_discovered_nodes": "Alle ontdekte knooppunten", - "network_info": "Netwerkinfo", - "network_activity": "Netwerkactiviteit", - "frequency": "Frequentie", - "bandwidth": "Bandbreedte", - "spreading_factor": "Spreading Factor", - "coding_rate": "Coderingssnelheid", - "tx_power": "TX Vermogen" - }, - "dashboard": { - "all_discovered_nodes": "Alle ontdekte knooppunten", - "recent_channel_messages": "Recente kanaalberichten", - "channel": "Kanaal {{number}}" - }, - "nodes": { - "scan_to_add": "Scan om als contact toe te voegen" - }, - "advertisements": {}, - "messages": { - "type_direct": "Direct", - "type_channel": "Kanaal", - "type_contact": "Contact", - "type_public": "Publiek" - }, - "map": { - "show_labels": "Toon labels", - "infrastructure_only": "Alleen infrastructuur", - "legend": "Legenda:", - "infrastructure": "Infrastructuur", - "public": "Publiek", - "nodes_on_map": "{{count}} knooppunten op kaart", - "nodes_none_have_coordinates": "{{count}} knooppunten (geen met coördinaten)", - "gps_description": "Knooppunten worden op de kaart geplaatst op basis van GPS-coördinaten uit knooppuntrapporten of handmatige labels.", - "owner": "Eigenaar:", - "role": "Rol:", - "select_destination_node": "-- Selecteer bestemmingsknooppunt --" - }, - "members": { - "empty_state_description": "Nog geen leden.", - "empty_description": "Leden verschijnen hier zodra gebruikers inloggen en knooppunten adopteren." - }, - "channels": { - "visibility_community": "Community", - "visibility_member": "Lid", - "visibility_operator": "Operator", - "visibility_admin": "Beheerder" - }, - "not_found": { - "description": "De pagina die u zoekt bestaat niet of is verplaatst." - }, - "custom_page": { - "failed_to_load": "Pagina laden mislukt" - }, - "admin": { - "access_denied": "Toegang geweigerd", - "admin_not_enabled": "De beheerinterface is niet ingeschakeld.", - "admin_enable_hint": "Stel WEB_ADMIN_ENABLED=true in om beheerfuncties in te schakelen.", - "welcome": "Welkom bij het beheerpaneel.", - "members_description": "Beheer netwerkleden en operators.", - "tags_description": "Beheer aangepaste labels en metadata voor netwerkknooppunten." - }, - "admin_members": { - "network_members": "Netwerkleden ({{count}})", - "member_id": "Lid-ID", - "member_id_hint": "Unieke identificatie (letters, cijfers, underscore)", - "empty_state_hint": "Klik op \"Lid toevoegen\" om het eerste lid aan te maken." - }, - "admin_node_tags": { - "select_node": "Selecteer knooppunt", - "select_node_placeholder": "-- Selecteer een knooppunt --", - "load_tags": "Labels laden", - "move_warning": "Dit verplaatst het label van het huidige knooppunt naar het bestemmingsknooppunt.", - "copy_all": "Alles kopiëren", - "copy_all_info": "Labels die al bestaan op het bestemmingsknooppunt worden overgeslagen. Originele labels blijven op dit knooppunt.", - "delete_all": "Alles verwijderen", - "delete_all_warning": "Alle labels worden permanent verwijderd.", - "destination_node": "Bestemmingsknooppunt", - "tag_key": "Label sleutel", - "for_this_node": "voor dit knooppunt", - "empty_state_hint": "Voeg hieronder een nieuw label toe.", - "select_a_node": "Selecteer een knooppunt", - "select_a_node_description": "Kies een knooppunt uit de vervolgkeuzelijst hierboven om de labels te bekijken en beheren.", - "copied_entities": "{{copied}} label(s) gekopieerd, {{skipped}} overgeslagen" - }, - "footer": { - "powered_by": "Mogelijk gemaakt door" + "entities": { + "home": "Startpagina", + "dashboard": "Dashboard", + "nodes": "Knooppunten", + "node": "Knooppunt", + "node_detail": "Knooppuntdetails", + "advertisements": "Advertenties", + "advertisement": "Advertentie", + "messages": "Berichten", + "packets": "Pakketten", + "message": "Bericht", + "packet": "Pakket", + "map": "Kaart", + "members": "Leden", + "member": "Lid", + "admin": "Beheer", + "tags": "Labels", + "tag": "Label" + }, + "common": { + "filter": "Filter", + "clear": "Wissen", + "clear_filters": "Filters wissen", + "search": "Zoeken", + "cancel": "Annuleren", + "delete": "Verwijderen", + "edit": "Bewerken", + "move": "Verplaatsen", + "save": "Opslaan", + "save_changes": "Wijzigingen opslaan", + "add": "Toevoegen", + "add_entity": "{{entity}} toevoegen", + "add_new_entity": "Nieuwe {{entity}} toevoegen", + "edit_entity": "{{entity}} bewerken", + "delete_entity": "{{entity}} verwijderen", + "delete_all_entity": "Alle {{entity}} verwijderen", + "move_entity": "{{entity}} verplaatsen", + "move_entity_to_another_node": "{{entity}} naar ander knooppunt verplaatsen", + "copy_entity": "{{entity}} kopiëren", + "copy_all_entity_to_another_node": "Alle {{entity}} naar ander knooppunt kopiëren", + "view_entity": "{{entity}} bekijken", + "recent_entity": "Recente {{entity}}", + "total_entity": "Totaal {{entity}}", + "all_entity": "Alle {{entity}}", + "no_entity_found": "Geen {{entity}} gevonden", + "no_entity_recorded": "Geen {{entity}} geregistreerd", + "no_entity_defined": "Geen {{entity}} gedefinieerd", + "no_entity_in_database": "Geen {{entity}} in database", + "no_entity_configured": "Geen {{entity}} geconfigureerd", + "no_entity_yet": "Nog geen {{entity}}", + "entity_not_found_details": "{{entity}} niet gevonden: {{details}}", + "page_not_found": "Pagina niet gevonden", + "delete_entity_confirm": "Weet u zeker dat u {{entity}} {{name}} wilt verwijderen?", + "delete_all_entity_confirm": "Weet u zeker dat u alle {{count}} {{entity}} van {{name}} wilt verwijderen?", + "cannot_be_undone": "Deze actie kan niet ongedaan worden gemaakt.", + "entity_added_success": "{{entity}} succesvol toegevoegd", + "entity_updated_success": "{{entity}} succesvol bijgewerkt", + "entity_deleted_success": "{{entity}} succesvol verwijderd", + "entity_moved_success": "{{entity}} succesvol verplaatst", + "all_entity_deleted_success": "Alle {{entity}} succesvol verwijderd", + "copy_all_entity_description": "Kopieer alle {{count}} {{entity}} van {{name}} naar een ander knooppunt.", + "previous": "Vorige", + "next": "Volgende", + "go_home": "Naar startpagina", + "loading": "Laden...", + "error": "Fout", + "failed_to_load_page": "Pagina laden mislukt", + "total": "{{count}} totaal", + "shown": "{{count}} weergegeven", + "count_entity": "{{count}} {{entity}}", + "type": "Type", + "name": "Naam", + "key": "Sleutel", + "value": "Waarde", + "time": "Tijd", + "actions": "Acties", + "updated": "Bijgewerkt", + "sign_in": "Inloggen", + "sign_out": "Uitloggen", + "view_details": "Details bekijken", + "all_types": "Alle types", + "node_type": "Knooppunttype", + "show": "Toon", + "search_placeholder": "Zoek op naam, ID of publieke sleutel...", + "contact": "Contact", + "description": "Beschrijving", + "callsign": "Roepnaam", + "tags": "Labels", + "last_seen": "Laatst gezien", + "first_seen_label": "Eerst gezien:", + "last_seen_label": "Laatst gezien:", + "location": "Locatie", + "public_key": "Publieke sleutel", + "received": "Ontvangen", + "received_by": "Ontvangen door", + "receivers": "Ontvangers", + "from": "Van", + "close": "sluiten", + "unnamed": "Naamloos", + "unnamed_node": "Naamloos knooppunt", + "all_operators": "Alle Operators", + "filter_operator_label": "Operator" + }, + "links": { + "website": "Website", + "github": "GitHub", + "discord": "Discord", + "youtube": "YouTube", + "profile": "Profiel" + }, + "auto_refresh": { + "pause": "Pauzeer verversen", + "resume": "Hervat verversen" + }, + "time": { + "days_ago": "{{count}}d geleden", + "hours_ago": "{{count}}u geleden", + "minutes_ago": "{{count}}m geleden", + "less_than_minute": "<1m geleden", + "last_7_days": "Laatste 7 dagen", + "per_day_last_7_days": "Per dag (laatste 7 dagen)", + "over_time_last_7_days": "In de tijd (laatste 7 dagen)", + "activity_per_day_last_7_days": "Activiteit per dag (laatste 7 dagen)" + }, + "node_types": { + "chat": "Chat", + "repeater": "Repeater", + "room": "Ruimte", + "unknown": "Onbekend" + }, + "home": { + "welcome_default": "Welkom bij het {{network_name}} mesh-netwerk dashboard. Monitor netwerkactiviteit, bekijk verbonden knooppunten en verken berichtgeschiedenis.", + "all_discovered_nodes": "Alle ontdekte knooppunten", + "network_info": "Netwerkinfo", + "network_activity": "Netwerkactiviteit", + "frequency": "Frequentie", + "bandwidth": "Bandbreedte", + "spreading_factor": "Spreading Factor", + "coding_rate": "Coderingssnelheid", + "tx_power": "TX Vermogen" + }, + "dashboard": { + "all_discovered_nodes": "Alle ontdekte knooppunten", + "recent_channel_messages": "Recente kanaalberichten", + "channel": "Kanaal {{number}}" + }, + "nodes": { + "scan_to_add": "Scan om als contact toe te voegen" + }, + "advertisements": {}, + "messages": { + "type_direct": "Direct", + "type_channel": "Kanaal", + "type_contact": "Contact", + "type_public": "Publiek" + }, + "packets": { + "filter_event_type": "Gebeurtenistype", + "filter_min_snr": "Min. SNR", + "filter_max_snr": "Max. SNR", + "redacted": "Beperkt", + "redacted_title": "Verborgen door kanaalzichtbaarheid", + "redacted_notice": "Dit pakket is waargenomen op een kanaal boven uw toegangsniveau. De inhoud is verborgen.", + "detail_title": "Pakketdetail", + "decoded": "Gedecodeerd", + "copy_raw": "Ruwe hex kopiëren", + "view_raw": "Ruwe pakketten bekijken", + "packet_hash": "Pakkethash", + "packet_type": "Pakkettype", + "payload_type": "Payloadtype", + "col_event_type": "Gebeurtenis", + "col_source": "Bron", + "col_route_type": "Routetype", + "col_raw": "Ruw", + "sort": { + "newest": "Tijd (nieuwste)", + "oldest": "Tijd (oudste)", + "event_az": "Gebeurtenis (A–Z)", + "snr_high": "SNR (hoogste)", + "path_high": "Hops (meeste)" } + }, + "map": { + "show_labels": "Toon labels", + "infrastructure_only": "Alleen infrastructuur", + "legend": "Legenda:", + "infrastructure": "Infrastructuur", + "public": "Publiek", + "nodes_on_map": "{{count}} knooppunten op kaart", + "nodes_none_have_coordinates": "{{count}} knooppunten (geen met coördinaten)", + "gps_description": "Knooppunten worden op de kaart geplaatst op basis van GPS-coördinaten uit knooppuntrapporten of handmatige labels.", + "owner": "Eigenaar:", + "role": "Rol:", + "select_destination_node": "-- Selecteer bestemmingsknooppunt --" + }, + "members": { + "empty_state_description": "Nog geen leden.", + "empty_description": "Leden verschijnen hier zodra gebruikers inloggen en knooppunten adopteren." + }, + "channels": { + "visibility_community": "Community", + "visibility_member": "Lid", + "visibility_operator": "Operator", + "visibility_admin": "Beheerder" + }, + "not_found": { + "description": "De pagina die u zoekt bestaat niet of is verplaatst." + }, + "custom_page": { + "failed_to_load": "Pagina laden mislukt" + }, + "admin": { + "access_denied": "Toegang geweigerd", + "admin_not_enabled": "De beheerinterface is niet ingeschakeld.", + "admin_enable_hint": "Stel WEB_ADMIN_ENABLED=true in om beheerfuncties in te schakelen.", + "welcome": "Welkom bij het beheerpaneel.", + "members_description": "Beheer netwerkleden en operators.", + "tags_description": "Beheer aangepaste labels en metadata voor netwerkknooppunten." + }, + "admin_members": { + "network_members": "Netwerkleden ({{count}})", + "member_id": "Lid-ID", + "member_id_hint": "Unieke identificatie (letters, cijfers, underscore)", + "empty_state_hint": "Klik op \"Lid toevoegen\" om het eerste lid aan te maken." + }, + "admin_node_tags": { + "select_node": "Selecteer knooppunt", + "select_node_placeholder": "-- Selecteer een knooppunt --", + "load_tags": "Labels laden", + "move_warning": "Dit verplaatst het label van het huidige knooppunt naar het bestemmingsknooppunt.", + "copy_all": "Alles kopiëren", + "copy_all_info": "Labels die al bestaan op het bestemmingsknooppunt worden overgeslagen. Originele labels blijven op dit knooppunt.", + "delete_all": "Alles verwijderen", + "delete_all_warning": "Alle labels worden permanent verwijderd.", + "destination_node": "Bestemmingsknooppunt", + "tag_key": "Label sleutel", + "for_this_node": "voor dit knooppunt", + "empty_state_hint": "Voeg hieronder een nieuw label toe.", + "select_a_node": "Selecteer een knooppunt", + "select_a_node_description": "Kies een knooppunt uit de vervolgkeuzelijst hierboven om de labels te bekijken en beheren.", + "copied_entities": "{{copied}} label(s) gekopieerd, {{skipped}} overgeslagen" + }, + "footer": { + "powered_by": "Mogelijk gemaakt door" + } } diff --git a/src/meshcore_hub/web/templates/spa.html b/src/meshcore_hub/web/templates/spa.html index f65b41d..7b6d69c 100644 --- a/src/meshcore_hub/web/templates/spa.html +++ b/src/meshcore_hub/web/templates/spa.html @@ -73,6 +73,9 @@ {% if features.messages %}
  • {{ t('entities.messages') }}
    • {% endif %} + {% if features.packets %} +
  • {{ t('entities.packets') }}
    • + {% endif %} {% if features.map %}
  • {{ t('entities.map') }}
    • {% endif %} diff --git a/tests/test_api/test_raw_packets.py b/tests/test_api/test_raw_packets.py new file mode 100644 index 0000000..95ba923 --- /dev/null +++ b/tests/test_api/test_raw_packets.py @@ -0,0 +1,228 @@ +"""Tests for raw packet API routes.""" + +from datetime import datetime, timezone + +import pytest + +from meshcore_hub.common.models import Channel, RawPacket + + +class TestListRawPackets: + """Tests for GET /packets endpoint.""" + + def test_list_empty(self, client_no_auth): + response = client_no_auth.get("/api/v1/packets") + assert response.status_code == 200 + data = response.json() + assert data["items"] == [] + assert data["total"] == 0 + + def test_list_with_data(self, client_no_auth, api_db_session): + api_db_session.add( + RawPacket( + raw_hex="0011", + packet_hash="h1", + event_type="letsmesh_packet", + received_at=datetime.now(timezone.utc), + ) + ) + api_db_session.commit() + + response = client_no_auth.get("/api/v1/packets") + assert response.status_code == 200 + data = response.json() + assert data["total"] == 1 + item = data["items"][0] + assert item["raw_hex"] == "0011" + assert item["redacted"] is False + + def test_filter_event_type(self, client_no_auth, api_db_session): + now = datetime.now(timezone.utc) + api_db_session.add_all( + [ + RawPacket(raw_hex="00", event_type="advertisement", received_at=now), + RawPacket(raw_hex="11", event_type="channel_msg_recv", received_at=now), + ] + ) + api_db_session.commit() + + response = client_no_auth.get("/api/v1/packets?event_type=advertisement") + assert response.status_code == 200 + data = response.json() + assert data["total"] == 1 + assert data["items"][0]["event_type"] == "advertisement" + + def test_filter_source_prefix_and_snr(self, client_no_auth, api_db_session): + now = datetime.now(timezone.utc) + api_db_session.add_all( + [ + RawPacket( + raw_hex="00", + source_pubkey_prefix="AABBCCDDEEFF", + snr=10.0, + received_at=now, + ), + RawPacket( + raw_hex="11", + source_pubkey_prefix="112233445566", + snr=2.0, + received_at=now, + ), + ] + ) + api_db_session.commit() + + response = client_no_auth.get( + "/api/v1/packets?pubkey_prefix=AABBCCDDEEFF&min_snr=5" + ) + assert response.status_code == 200 + data = response.json() + assert data["total"] == 1 + assert data["items"][0]["source_pubkey_prefix"] == "AABBCCDDEEFF" + + def test_pagination_params_echoed(self, client_no_auth): + response = client_no_auth.get("/api/v1/packets?limit=25&offset=5") + assert response.status_code == 200 + data = response.json() + assert data["limit"] == 25 + assert data["offset"] == 5 + + def test_filter_by_exact_packet_hash(self, client_no_auth, api_db_session): + now = datetime.now(timezone.utc) + api_db_session.add_all( + [ + RawPacket(raw_hex="00", packet_hash="HASH_AAA", received_at=now), + RawPacket(raw_hex="11", packet_hash="HASH_AAA", received_at=now), + RawPacket(raw_hex="22", packet_hash="HASH_BBB", received_at=now), + ] + ) + api_db_session.commit() + + response = client_no_auth.get("/api/v1/packets?packet_hash=HASH_AAA") + assert response.status_code == 200 + data = response.json() + assert data["total"] == 2 + assert {i["packet_hash"] for i in data["items"]} == {"HASH_AAA"} + + +class TestRawPacketRedaction: + """Tests for channel-visibility redaction of raw packets.""" + + @pytest.fixture + def packets_with_visibility(self, api_db_session): + """Community-channel, admin-channel, and non-channel raw packets.""" + pub_key = "AABBCCDDEEFF00112233445566778899" + adm_key = "FFEEDDCCBBAA99887766554433221100" + pub_idx = int(Channel.compute_channel_hash(pub_key), 16) + adm_idx = int(Channel.compute_channel_hash(adm_key), 16) + + api_db_session.add_all( + [ + Channel( + name="CommunityCh", + key_hex=pub_key, + channel_hash=Channel.compute_channel_hash(pub_key), + visibility="community", + enabled=True, + ), + Channel( + name="AdminCh", + key_hex=adm_key, + channel_hash=Channel.compute_channel_hash(adm_key), + visibility="admin", + enabled=True, + ), + ] + ) + now = datetime.now(timezone.utc) + pub_pkt = RawPacket( + raw_hex="C0FFEE", + event_type="channel_msg_recv", + channel_idx=pub_idx, + decoded={"payload": {"decoded": {"channelHash": "x"}}}, + received_at=now, + ) + adm_pkt = RawPacket( + raw_hex="ADADAD", + event_type="channel_msg_recv", + channel_idx=adm_idx, + decoded={"payload": {"decoded": {"channelHash": "y"}}}, + received_at=now, + ) + plain_pkt = RawPacket( + raw_hex="DEADBE", + event_type="advertisement", + channel_idx=None, + received_at=now, + ) + api_db_session.add_all([pub_pkt, adm_pkt, plain_pkt]) + api_db_session.commit() + return pub_pkt, adm_pkt, plain_pkt + + def test_anonymous_redacts_admin_channel( + self, client_no_auth, packets_with_visibility + ): + """Anonymous sees the admin-channel packet metadata-only (redacted).""" + response = client_no_auth.get("/api/v1/packets") + assert response.status_code == 200 + data = response.json() + # All three rows are returned (count stable), but the admin one is redacted. + assert data["total"] == 3 + adm = next(i for i in data["items"] if i["raw_hex"] is None) + assert adm["redacted"] is True + assert adm["decoded"] is None + # Non-channel and community packets are not redacted. + assert any( + i["raw_hex"] == "DEADBE" and not i["redacted"] for i in data["items"] + ) + assert any( + i["raw_hex"] == "C0FFEE" and not i["redacted"] for i in data["items"] + ) + + def test_admin_sees_full(self, client_no_auth, packets_with_visibility): + """Admin role sees the admin-channel packet in full.""" + response = client_no_auth.get( + "/api/v1/packets", headers={"X-User-Roles": "admin"} + ) + assert response.status_code == 200 + data = response.json() + assert data["total"] == 3 + assert all(not i["redacted"] for i in data["items"]) + assert any(i["raw_hex"] == "ADADAD" for i in data["items"]) + + def test_redacted_filter(self, client_no_auth, packets_with_visibility): + """The redacted filter narrows to (only|excluding) redacted rows.""" + only = client_no_auth.get("/api/v1/packets?redacted=true").json() + assert only["total"] == 1 + assert only["items"][0]["redacted"] is True + + without = client_no_auth.get("/api/v1/packets?redacted=false").json() + assert without["total"] == 2 + assert all(not i["redacted"] for i in without["items"]) + + def test_count_stable_across_roles(self, client_no_auth, packets_with_visibility): + """Pagination count is the same regardless of role (rows redacted not hidden).""" + anon = client_no_auth.get("/api/v1/packets").json() + admin = client_no_auth.get( + "/api/v1/packets", headers={"X-User-Roles": "admin"} + ).json() + assert anon["total"] == admin["total"] == 3 + + def test_get_single_redacted(self, client_no_auth, packets_with_visibility): + """GET /packets/{id} redacts an above-role channel packet (not 404).""" + _, adm_pkt, _ = packets_with_visibility + response = client_no_auth.get(f"/api/v1/packets/{adm_pkt.id}") + assert response.status_code == 200 + data = response.json() + assert data["redacted"] is True + assert data["raw_hex"] is None + + def test_get_single_visible_to_admin(self, client_no_auth, packets_with_visibility): + _, adm_pkt, _ = packets_with_visibility + response = client_no_auth.get( + f"/api/v1/packets/{adm_pkt.id}", headers={"X-User-Roles": "admin"} + ) + assert response.status_code == 200 + data = response.json() + assert data["redacted"] is False + assert data["raw_hex"] == "ADADAD" diff --git a/tests/test_collector/test_cleanup.py b/tests/test_collector/test_cleanup.py index 6ca08ec..a2de490 100644 --- a/tests/test_collector/test_cleanup.py +++ b/tests/test_collector/test_cleanup.py @@ -18,6 +18,7 @@ from meshcore_hub.common.models import ( Message, Node, NodeTag, + RawPacket, Telemetry, TracePath, ) @@ -466,3 +467,84 @@ async def test_cleanup_clears_stale_observer_flags( ) assert stale_flag is False assert active_flag is True + + +@pytest.mark.asyncio +async def test_raw_packets_use_dedicated_retention( + async_db_session: AsyncSession, +) -> None: + """Raw packets older than RAW_PACKET_RETENTION_DAYS are deleted.""" + old_date = datetime.now(timezone.utc) - timedelta(days=20) + recent_date = datetime.now(timezone.utc) - timedelta(days=3) + async_db_session.add( + RawPacket(raw_hex="00", created_at=old_date, updated_at=old_date) + ) + async_db_session.add( + RawPacket(raw_hex="11", created_at=recent_date, updated_at=recent_date) + ) + await async_db_session.commit() + + # Global retention 30 days would keep both; raw retention of 7 prunes the old one. + stats = await cleanup_old_data( + async_db_session, + retention_days=30, + dry_run=False, + raw_packet_retention_days=7, + ) + + assert stats.raw_packets_deleted == 1 + remaining = await async_db_session.scalar( + select(func.count()).select_from(RawPacket) + ) + assert remaining == 1 + + +@pytest.mark.asyncio +async def test_raw_packet_retention_defaults_to_global( + async_db_session: AsyncSession, +) -> None: + """When raw retention is None, the global retention value is used.""" + old_date = datetime.now(timezone.utc) - timedelta(days=40) + async_db_session.add( + RawPacket(raw_hex="00", created_at=old_date, updated_at=old_date) + ) + await async_db_session.commit() + + stats = await cleanup_old_data( + async_db_session, + retention_days=30, + dry_run=False, + raw_packet_retention_days=None, + ) + + assert stats.raw_packets_deleted == 1 + + +@pytest.mark.asyncio +async def test_raw_packet_only_observer_keeps_flag( + async_db_session: AsyncSession, +) -> None: + """A node observing only (live) raw packets keeps is_observer=True.""" + node = Node(public_key="d" * 64, is_observer=True) + async_db_session.add(node) + await async_db_session.flush() + + recent_date = datetime.now(timezone.utc) - timedelta(days=2) + async_db_session.add( + RawPacket( + raw_hex="00", + observer_node_id=node.id, + created_at=recent_date, + updated_at=recent_date, + ) + ) + await async_db_session.commit() + + stats = await cleanup_old_data(async_db_session, retention_days=30, dry_run=False) + + assert stats.observers_cleared == 0 + await async_db_session.rollback() + flag = await async_db_session.scalar( + select(Node.is_observer).where(Node.id == node.id) + ) + assert flag is True diff --git a/tests/test_collector/test_handlers/test_advertisement.py b/tests/test_collector/test_handlers/test_advertisement.py index a3baa1d..a44fa1c 100644 --- a/tests/test_collector/test_handlers/test_advertisement.py +++ b/tests/test_collector/test_handlers/test_advertisement.py @@ -276,3 +276,19 @@ class TestHandleAdvertisement: ads = db_session.execute(select(Advertisement)).scalars().all() assert len(ads) == 1 + + +class TestAdvertisementPacketHash: + """packet_hash links an advertisement to its raw_packets rows.""" + + def test_stores_packet_hash(self, db_manager, db_session): + payload = { + "public_key": "a" * 64, + "name": "TestNode", + "adv_type": "chat", + "packet_hash": "FEEDFACE9999", + } + handle_advertisement("b" * 64, "advertisement", payload, db_manager) + + ad = db_session.execute(select(Advertisement)).scalar_one() + assert ad.packet_hash == "FEEDFACE9999" diff --git a/tests/test_collector/test_handlers/test_message.py b/tests/test_collector/test_handlers/test_message.py index ddb1407..2d34474 100644 --- a/tests/test_collector/test_handlers/test_message.py +++ b/tests/test_collector/test_handlers/test_message.py @@ -174,3 +174,29 @@ class TestHandleChannelMessage: assert observer is not None assert observer.path_len == 5 assert observer.snr == 10.0 + + +class TestMessagePacketHash: + """packet_hash links a message to its raw_packets rows.""" + + def test_stores_packet_hash_from_payload(self, db_manager, db_session): + payload = { + "pubkey_prefix": "01ab2186c4d5", + "text": "Hello!", + "packet_hash": "DEADBEEF1234", + } + handle_contact_message("a" * 64, "contact_msg_recv", payload, db_manager) + + msg = db_session.execute(select(Message)).scalar_one() + assert msg.packet_hash == "DEADBEEF1234" + + def test_falls_back_to_hash_field(self, db_manager, db_session): + payload = { + "pubkey_prefix": "01ab2186c4d5", + "text": "Hello again!", + "hash": "CAFEBABE5678", + } + handle_contact_message("a" * 64, "contact_msg_recv", payload, db_manager) + + msg = db_session.execute(select(Message)).scalar_one() + assert msg.packet_hash == "CAFEBABE5678" diff --git a/tests/test_collector/test_handlers/test_raw_packet.py b/tests/test_collector/test_handlers/test_raw_packet.py new file mode 100644 index 0000000..597c947 --- /dev/null +++ b/tests/test_collector/test_handlers/test_raw_packet.py @@ -0,0 +1,98 @@ +"""Tests for the raw packet capture handler.""" + +from sqlalchemy import select + +from meshcore_hub.collector.handlers.raw_packet import store_raw_packet +from meshcore_hub.common.models import Node, RawPacket + + +def _channel_decoded() -> dict: + """Decoder output resembling a channel-message packet on channel 0x2A.""" + return { + "payloadType": 4, + "routeType": 1, + "pathLength": 2, + "payload": { + "decoded": { + "channelHash": "2A", + "sourceHash": "01AB2186C4D5EE", + } + }, + } + + +class TestStoreRawPacket: + """Tests for store_raw_packet.""" + + def test_captures_one_row(self, db_manager, db_session): + """A single packet writes exactly one raw_packets row.""" + payload = {"raw": "0011223344", "hash": "deadbeef", "SNR": 9.5, "path": "0102"} + + store_raw_packet( + "a" * 64, payload, _channel_decoded(), "channel_msg_recv", db_manager + ) + + rows = db_session.execute(select(RawPacket)).scalars().all() + assert len(rows) == 1 + rp = rows[0] + assert rp.raw_hex == "0011223344" + assert rp.packet_hash == "deadbeef" + assert rp.event_type == "channel_msg_recv" + assert rp.snr == 9.5 + + def test_derives_channel_idx_and_source_prefix(self, db_manager, db_session): + """channel_idx from channelHash; source prefix from sourceHash.""" + payload = {"raw": "00", "hash": "h1"} + + store_raw_packet( + "a" * 64, payload, _channel_decoded(), "channel_msg_recv", db_manager + ) + + rp = db_session.execute(select(RawPacket)).scalar_one() + assert rp.channel_idx == 0x2A # 42 + assert rp.source_pubkey_prefix == "01AB2186C4D5" + assert rp.route_type == "flood" + assert rp.path_len == 2 + assert rp.payload_type == 4 + + def test_no_dedup_one_row_per_observer(self, db_manager, db_session): + """The same packet hash from two observers stores two rows.""" + payload = {"raw": "00", "hash": "shared"} + + store_raw_packet( + "a" * 64, payload, _channel_decoded(), "channel_msg_recv", db_manager + ) + store_raw_packet( + "b" * 64, payload, _channel_decoded(), "channel_msg_recv", db_manager + ) + + rows = db_session.execute(select(RawPacket)).scalars().all() + assert len(rows) == 2 + assert {r.packet_hash for r in rows} == {"shared"} + + def test_marks_observer_node(self, db_manager, db_session): + """The observing node is created and flagged is_observer.""" + payload = {"raw": "00", "hash": "h1"} + + store_raw_packet( + "c" * 64, payload, _channel_decoded(), "channel_msg_recv", db_manager + ) + + node = db_session.execute( + select(Node).where(Node.public_key == "c" * 64) + ).scalar_one() + assert node.is_observer is True + + rp = db_session.execute(select(RawPacket)).scalar_one() + assert rp.observer_node_id == node.id + + def test_non_channel_packet_has_null_channel_idx(self, db_manager, db_session): + """A packet with no channelHash stores channel_idx = None.""" + payload = {"raw": "00", "hash": "h1"} + decoded = {"payloadType": 1, "payload": {"decoded": {"sourceHash": "AABBCCDD"}}} + + store_raw_packet("a" * 64, payload, decoded, "letsmesh_packet", db_manager) + + rp = db_session.execute(select(RawPacket)).scalar_one() + assert rp.channel_idx is None + assert rp.source_pubkey_prefix == "AABBCCDD"[:12] diff --git a/tests/test_collector/test_letsmesh_normalizer_integration.py b/tests/test_collector/test_letsmesh_normalizer_integration.py index 3b303b0..b1d2e93 100644 --- a/tests/test_collector/test_letsmesh_normalizer_integration.py +++ b/tests/test_collector/test_letsmesh_normalizer_integration.py @@ -212,7 +212,7 @@ class TestGroupTextPacket: result = norm._normalize_letsmesh_event("t", {"raw": raw}) assert result is not None _, event_type, pl = result - assert event_type == "letsmesh_packet" + assert event_type == "encrypted_channel" assert pl.get("decoded_payload_type") == 5 @@ -333,7 +333,7 @@ class TestTracePacket: result = norm._normalize_letsmesh_event("t", {"raw": raw}) assert result is not None _, event_type, pl = result - assert event_type == "letsmesh_packet" + assert event_type == "trace" class TestControlPacket: @@ -547,7 +547,7 @@ class TestResponsePacket: class TestAckPacket: - def test_ack_falls_through_to_letsmesh_packet(self) -> None: + def test_ack_classified_as_ack(self) -> None: raw = "ack1" decoded = { "payloadType": 3, @@ -569,13 +569,13 @@ class TestAckPacket: result = norm._normalize_letsmesh_event("t", {"raw": raw}) assert result is not None _, event_type, pl = result - assert event_type == "letsmesh_packet" + assert event_type == "ack" assert pl["decoded_payload_type"] == 3 assert pl["decoded_packet"]["payloadType"] == 3 class TestRequestPacket: - def test_request_without_decrypted_content_falls_through(self) -> None: + def test_request_without_decrypted_content_classified_as_req(self) -> None: raw = "req1" decoded = { "payloadType": 0, @@ -602,7 +602,7 @@ class TestRequestPacket: result = norm._normalize_letsmesh_event("t", {"raw": raw}) assert result is not None _, event_type, pl = result - assert event_type == "letsmesh_packet" + assert event_type == "req" assert pl["decoded_payload_type"] == 0 @@ -625,3 +625,40 @@ class TestUnrecognizedFeed: result = norm._normalize_letsmesh_event("garbage", {}) assert result is None + + +class TestFallbackClassification: + """The fallback classifier maps payload types to specific event types.""" + + def test_payload_type_to_event_type_map(self) -> None: + cases = { + 0: "req", + 1: "response", + 2: "encrypted_direct", + 3: "ack", + 4: "advert", + 5: "encrypted_channel", + 6: "grp_data", + 7: "anon_req", + 8: "path", + 9: "trace", + 10: "multipart", + 11: "control", + 15: "raw_custom", + } + for packet_type, expected in cases.items(): + event_type = LetsMeshNormalizer._classify_fallback_event_type( + {"packet_type": packet_type}, None + ) + assert event_type == expected, f"type {packet_type}" + + def test_unknown_type_keeps_letsmesh_packet(self) -> None: + # Unresolvable type falls back to the generic safety-net label. + assert ( + LetsMeshNormalizer._classify_fallback_event_type({}, None) + == "letsmesh_packet" + ) + assert ( + LetsMeshNormalizer._classify_fallback_event_type({"packet_type": 99}, None) + == "letsmesh_packet" + ) diff --git a/tests/test_collector/test_subscriber.py b/tests/test_collector/test_subscriber.py index 0f01e49..af6c2de 100644 --- a/tests/test_collector/test_subscriber.py +++ b/tests/test_collector/test_subscriber.py @@ -83,6 +83,105 @@ class TestSubscriber: handler.assert_called_once() + def test_raw_capture_disabled_writes_no_rows(self, mock_mqtt_client, db_manager): + """With capture disabled, no raw_packets rows are written but the + structured handler still runs.""" + from sqlalchemy import select + + from meshcore_hub.common.models import RawPacket + + mock_mqtt_client.topic_builder.parse_letsmesh_upload_topic.return_value = ( + "a" * 64, + "packets", + ) + subscriber = Subscriber( + mock_mqtt_client, db_manager, raw_packet_capture_enabled=False + ) + handler = MagicMock() + subscriber.register_handler("channel_msg_recv", handler) + subscriber._normalize_letsmesh_event = MagicMock( # type: ignore[method-assign] + return_value=("a" * 64, "channel_msg_recv", {"text": "hi"}) + ) + + subscriber._handle_mqtt_message( + topic="meshcore/STN/abc/packets", + pattern="meshcore/+/+/packets", + payload={"raw": "00", "hash": "h1"}, + ) + + handler.assert_called_once() + session = db_manager.get_session() + try: + rows = session.execute(select(RawPacket)).scalars().all() + assert len(rows) == 0 + finally: + session.close() + + def test_raw_capture_enabled_writes_row(self, mock_mqtt_client, db_manager): + """With capture enabled, a packets-feed message writes one raw row.""" + from sqlalchemy import select + + from meshcore_hub.common.models import RawPacket + + mock_mqtt_client.topic_builder.parse_letsmesh_upload_topic.return_value = ( + "a" * 64, + "packets", + ) + subscriber = Subscriber( + mock_mqtt_client, db_manager, raw_packet_capture_enabled=True + ) + subscriber._normalize_letsmesh_event = MagicMock( # type: ignore[method-assign] + return_value=("a" * 64, "channel_msg_recv", {"text": "hi"}) + ) + subscriber._letsmesh_decoder.decode_payload = MagicMock( # type: ignore[method-assign] + return_value={"payloadType": 4, "payload": {"decoded": {}}} + ) + + subscriber._handle_mqtt_message( + topic="meshcore/STN/abc/packets", + pattern="meshcore/+/+/packets", + payload={"raw": "0011", "hash": "h1"}, + ) + + session = db_manager.get_session() + try: + rows = session.execute(select(RawPacket)).scalars().all() + assert len(rows) == 1 + assert rows[0].raw_hex == "0011" + assert rows[0].event_type == "channel_msg_recv" + finally: + session.close() + + def test_raw_capture_skips_status_feed(self, mock_mqtt_client, db_manager): + """Capture is packets-feed only; status feed writes no raw rows.""" + from sqlalchemy import select + + from meshcore_hub.common.models import RawPacket + + mock_mqtt_client.topic_builder.parse_letsmesh_upload_topic.return_value = ( + "a" * 64, + "status", + ) + subscriber = Subscriber( + mock_mqtt_client, db_manager, raw_packet_capture_enabled=True + ) + subscriber._normalize_letsmesh_event = MagicMock( # type: ignore[method-assign] + return_value=("a" * 64, "letsmesh_status", {}) + ) + + subscriber._handle_mqtt_message( + topic="meshcore/STN/abc/status", + pattern="meshcore/+/+/status", + payload={"some": "status"}, + ) + + session = db_manager.get_session() + try: + rows = session.execute(select(RawPacket)).scalars().all() + assert len(rows) == 0 + finally: + session.close() + def test_start_subscribes_to_letsmesh_topics(self, mock_mqtt_client, db_manager): """LetsMesh ingest mode subscribes to packets/status/internal feeds.""" subscriber = Subscriber( @@ -246,7 +345,7 @@ class TestSubscriber: def test_letsmesh_packet_without_decrypted_text_is_not_shown_as_message( self, mock_mqtt_client, db_manager ) -> None: - """Undecodable LetsMesh packets are kept as informational events, not messages.""" + """Undecodable channel packets are kept as informational events, not messages.""" mock_mqtt_client.topic_builder.parse_letsmesh_upload_topic.return_value = ( "a" * 64, "packets", @@ -255,9 +354,11 @@ class TestSubscriber: mock_mqtt_client, db_manager, ) - letsmesh_packet_handler = MagicMock() + # A GRP_TXT (type 5) that does not decrypt is classified as + # encrypted_channel, not routed to the channel message handler. + encrypted_handler = MagicMock() channel_handler = MagicMock() - subscriber.register_handler("letsmesh_packet", letsmesh_packet_handler) + subscriber.register_handler("encrypted_channel", encrypted_handler) subscriber.register_handler("channel_msg_recv", channel_handler) subscriber.start() @@ -276,7 +377,7 @@ class TestSubscriber: }, ) - letsmesh_packet_handler.assert_called_once() + encrypted_handler.assert_called_once() channel_handler.assert_not_called() def test_letsmesh_packet_uses_decoder_text_when_available( @@ -760,7 +861,7 @@ class TestSubscriber: def test_letsmesh_packet_fallback_logs_decoded_payload( self, mock_mqtt_client, db_manager ) -> None: - """Unmapped packets include decoder output in letsmesh_packet payload.""" + """Unmapped packets are classified by payload type and keep decoder output.""" mock_mqtt_client.topic_builder.parse_letsmesh_upload_topic.return_value = ( "a" * 64, "packets", @@ -769,8 +870,9 @@ class TestSubscriber: mock_mqtt_client, db_manager, ) + # MULTIPART (type 10) has no structured handler -> classified as multipart. packet_handler = MagicMock() - subscriber.register_handler("letsmesh_packet", packet_handler) + subscriber.register_handler("multipart", packet_handler) subscriber.start() decoded_packet = { @@ -799,7 +901,7 @@ class TestSubscriber: packet_handler.assert_called_once() _public_key, event_type, payload, _db = packet_handler.call_args.args - assert event_type == "letsmesh_packet" + assert event_type == "multipart" assert payload["decoded_payload_type"] == 10 assert payload["decoded_packet"] == decoded_packet diff --git a/tests/test_common/test_models.py b/tests/test_common/test_models.py index 50b11fd..8b19dc3 100644 --- a/tests/test_common/test_models.py +++ b/tests/test_common/test_models.py @@ -14,6 +14,7 @@ from meshcore_hub.common.models import ( Telemetry, EventLog, EventObserver, + RawPacket, add_event_observer, ) @@ -67,6 +68,66 @@ class TestNodeModel: assert node.tags[0].key == "altitude" +class TestRawPacketModel: + """Tests for RawPacket model.""" + + def test_create_raw_packet(self, db_session) -> None: + """Test creating a raw packet with defaults.""" + packet = RawPacket( + packet_hash="abc123", + raw_hex="0011223344", + packet_type=5, + payload_type=4, + event_type="channel_msg_recv", + channel_idx=42, + source_pubkey_prefix="01AB2186C4D5", + route_type="flood", + path_len=2, + snr=12.5, + decoded={"payloadType": 4}, + ) + db_session.add(packet) + db_session.commit() + + assert packet.id is not None + assert packet.received_at is not None + assert packet.created_at is not None + assert packet.updated_at is not None + assert packet.event_type == "channel_msg_recv" + assert packet.channel_idx == 42 + assert packet.decoded == {"payloadType": 4} + + def test_raw_packet_nullable_columns(self, db_session) -> None: + """Test that the optional columns accept None.""" + packet = RawPacket() + db_session.add(packet) + db_session.commit() + + assert packet.id is not None + assert packet.observer_node_id is None + assert packet.packet_hash is None + assert packet.raw_hex is None + assert packet.channel_idx is None + assert packet.source_pubkey_prefix is None + assert packet.decoded is None + + def test_raw_packet_indexes(self) -> None: + """Test that the expected indexes are declared on the table.""" + index_names = {idx.name for idx in RawPacket.__table__.indexes} # type: ignore[attr-defined] + for expected in ( + "ix_raw_packets_received_at", + "ix_raw_packets_event_type", + "ix_raw_packets_packet_hash", + "ix_raw_packets_channel_idx", + "ix_raw_packets_source_pubkey_prefix", + "ix_raw_packets_observer_node_id", + "ix_raw_packets_event_type_received_at", + "ix_raw_packets_channel_idx_received_at", + "ix_raw_packets_source_pubkey_prefix_received_at", + ): + assert expected in index_names + + class TestMessageModel: """Tests for Message model.""" diff --git a/tests/test_web/test_features.py b/tests/test_web/test_features.py index 7a3d9c9..639cc15 100644 --- a/tests/test_web/test_features.py +++ b/tests/test_web/test_features.py @@ -6,7 +6,7 @@ import pytest from fastapi.testclient import TestClient from meshcore_hub.web.app import create_app -from tests.test_web.conftest import MockHttpClient +from tests.test_web.conftest import ALL_FEATURES_ENABLED, MockHttpClient class TestFeatureFlagsConfig: @@ -193,6 +193,47 @@ class TestFeatureFlagsSEO: ) +class TestPacketsFeatureFlag: + """Test the packets feature flag (off by default).""" + + def _make_app(self, mock_http_client: MockHttpClient, packets: bool): + features = dict(ALL_FEATURES_ENABLED) + features["packets"] = packets + app = create_app( + api_url="http://localhost:8000", + api_key="test-api-key", + network_name="Test Network", + features=features, + ) + app.state.http_client = mock_http_client + return TestClient(app, raise_server_exceptions=True) + + def test_packets_nav_hidden_when_disabled( + self, mock_http_client: MockHttpClient + ) -> None: + """The packets nav link is absent when the feature is off.""" + client = self._make_app(mock_http_client, packets=False) + html = client.get("/").text + assert 'href="/packets"' not in html + # Messages still shows (ordering sanity) + assert 'href="/messages"' in html + + def test_packets_nav_shown_when_enabled( + self, mock_http_client: MockHttpClient + ) -> None: + """The packets nav link appears when the feature is on.""" + client = self._make_app(mock_http_client, packets=True) + html = client.get("/").text + assert 'href="/packets"' in html + + def test_packets_disabled_by_default_in_settings(self) -> None: + """The declared default for feature_packets is False (env-independent).""" + from meshcore_hub.common.config import WebSettings + + # Check the field default directly so a local .env cannot mask it. + assert WebSettings.model_fields["feature_packets"].default is False + + class TestFeatureFlagsIndividual: """Test individual feature flags."""