From c029eae5244d3c1fda3c313e73e4ebad37e95fc6 Mon Sep 17 00:00:00 2001 From: Louis King Date: Sat, 4 Jul 2026 00:02:18 +0100 Subject: [PATCH] feat: persist path_hash_bytes column and add packet list filter Persist the path-hash byte width (1/2/3) as a nullable Integer column on RawPacket, computed at ingest by the collector and backfilled for historical rows via a self-contained Python migration. Replace the per-request Python decode loop in the grouped-list route with a SQL MAX() aggregate + HAVING filter, and add a discrete `** filter + ("Path width": Any / 1B / 2B / 3B) that auto-submits and round-trips through + the URL query string, surviving pagination and sort changes. + +### Technical Requirements + +- **TR-1** — Model (`common/models/raw_packet.py`): add + `path_hash_bytes: Mapped[Optional[int]] = mapped_column(Integer, nullable=True)` + immediately after `path_len`. Unindexed (mirrors `path_len`). +- **TR-2** — Collector (`collector/handlers/raw_packet.py`): in + `store_raw_packet`, after `path_len` is resolved (~line 96), compute + `path_hash_bytes` from `decoded_packet` via the two-tier lookup + (`decoded.path` → `payload.decoded.pathHashes`) using the already-imported + `LetsMeshNormalizer._normalize_hash_list` staticmethod, then + `max(len(h) // 2 for h in hashes)`. Add a small module-level + `_path_hash_byte_width(hashes)` helper (mirrors the API route's helper; the + API copy is deleted in Phase 3 so no naming collision persists). Pass + `path_hash_bytes=` into the `RawPacket(...)` kwargs. Must be null-safe (no + path → `None`). +- **TR-3** — Migration (`alembic/versions/`, `down_revision = "38abdf4651fc"`): + author via `meshcore-hub db revision --autogenerate -m "add path_hash_bytes to + raw_packets"` (produces a `batch_alter_table` `add_column`), then **manually + append a Python batched backfill** (~1000 rows/batch) that reads each row's + `decoded` JSON, applies the self-contained dual-path extraction + + `max(len//2)`, and `UPDATE`s the column. Python (not SQL JSON) so it is + portable across SQLite and Postgres. Migration logic must be a frozen + self-contained snapshot (no imports of app code that may change). Downgrade + drops the column. **Read mechanism**: the `decoded` column must be read via a + Core `select()` on a `sa.Table` declared with a `sa.JSON`-typed column (or + explicitly `json.loads()` the fetched scalar), **not** raw + `text("SELECT decoded ...")`. Raw `text()` bypasses SQLAlchemy's JSON type + adapter, so deserialization becomes driver-dependent (SQLite returns TEXT, + Postgres/psycopg2 may return a parsed object or string). Core `select()` + routes through the type adapter and yields a Python `dict` consistently on + both backends. +- **TR-4** — API route (`api/routes/packet_groups.py`): + - Phase 1 `group_query` select: add + `func.max(RawPacket.path_hash_bytes).label("path_hash_bytes")`. + - New param: `path_hash_bytes: Optional[int] = Query(None, ge=1, le=3, + description="Filter by path-hash byte width (1/2/3)")`. + - Apply filter: + `if path_hash_bytes is not None: group_query = group_query.having(func.max(RawPacket.path_hash_bytes) == path_hash_bytes)`. + - **Delete Phase 3** (lines 198-216, the `decoded_rows` fetch + `width_by_hash` + loop). Read `grp.path_hash_bytes` from the group row instead. + - Build item with + `path_hash_bytes=(None if is_redacted else grp.path_hash_bytes)`. + - Remove the now-dead `_path_hash_byte_width` helper (lines 54-59) from this + file. **Keep** `_extract_path_hashes` — the detail route `get_packet_group` + still uses it (`:311`). +- **TR-5** — Frontend (`packets.js`): + - Parse `const path_hash_bytes = query.path_hash_bytes || '';` alongside the + other filter reads (~line 57). + - Extend `hasActiveFilters` (~line 72) with `|| path_hash_bytes !== ''`. + - Add a 4th entry to the `filterFields` array (~line 170): a + `` filter field to `packets.js` (TR-5). +- Wire the query parse, `hasActiveFilters`, `apiParams`, `pagination` params, and + `headerParams` (TR-5). +- Add `packets.filter_path_width` to both locale files (TR-6). +- Rebuild the SPA bundle via the Docker build pipeline (local `node build.js` + fails on a missing fontsource asset — build through compose). + +### Phase 5: Verify + +- `pytest --no-cov tests/test_api/test_packet_groups.py tests/test_collector/` +- `pre-commit run --all-files` +- Apply the migration on the stack and manually verify the filter on the packet + list (all three known widths; "Any" returns null-width rows too). + +## Open Questions + +- None outstanding. Metric (byte width), approach (persisted column), filter + shape (discrete select), and unknown-option exclusion (known widths only) are + all resolved. +- **Phase-ordering window** (minor, not blocking): Between migration (Phase 1) + and collector ingester deploy (Phase 2), new RawPacket rows are inserted with + `path_hash_bytes = NULL`. The backfill fills historical rows, but a brief gap + exists for rows ingested during the deploy window. Once Phase 2 is active, new + rows carry the correct value and Phase 3 reads it. In practice the window is + seconds in a single `docker compose up -d` rebuild; any rows missed still render + `?B` (unchanged UX), and the next backfill run would catch them. +- Minor follow-up (not blocking): whether to also surface `path_hash_bytes` on + the raw `/packets` endpoint's `RawPacketRead` schema, now that the column + exists. + +## References + +- `docs/plans/20260612-2014-raw-packets-feature/plan.md` — the Raw Packets + feature this builds on (model, collector chokepoint, channel-visibility + redaction, `/packets` SPA page). +- `docs/plans/20260426-1137-improve-snr-path-visibility/plan.md` — prior + `path_len` (hop-count) column work and the "no backfill for per-observer + fields" decision (here we *do* backfill, since the value is derivable from the + already-stored `decoded` JSON). +- `src/meshcore_hub/api/routes/packet_groups.py:54-59,198-216,240` — current + runtime `_path_hash_byte_width` + Phase 3 aggregation + redaction nulling + (to be replaced/deleted). +- `src/meshcore_hub/collector/handlers/raw_packet.py:92-96,122-138` — `path_len` + precedent and the single `RawPacket(...)` construction site. +- `src/meshcore_hub/collector/letsmesh_normalizer.py:826-867` — + `_extract_message_path_hashes` + `_normalize_hash_list` (staticmethod) to reuse. +- `src/meshcore_hub/api/routes/raw_packets.py:165-172` — `min_snr`/`max_snr` and + `min_path_len`/`max_path_len` range-filter precedent. +- `alembic/versions/20260622_2243_38abdf4651fc_add_spam_scoring_columns_to_messages.py` + — `batch_alter_table` add-column + index precedent (current migration head). +- `alembic/versions/20260421_0001_normalize_public_key_case.py` — Python data + backfill precedent (`op.get_bind()`, dialect-aware raw SQL). +- Recent commits: `f845830` (JSON tree / path flow / chart polish, PR #292), + `2c25a9c` (locale-aware numbers + filter panel redesign, PR #291) — the + reception-widget + filter-panel work this plan extends. + +## Review + +**Status**: Approved with Changes + +**Reviewed**: 2026-07-03 + +### Resolutions + +- **Factual correction — aggregation fix is committed, not uncommitted**: The + Phase 3 MAX-across-all-receptions fix described as "uncommitted" was committed + in PR #292 (`f845830`). Updated to "recently committed (PR #292)". + +- **Gap — self-contained migration backfill**: Clarified that the backfill loop + must include a frozen, inline copy of the dual-path extraction + + `max(len//2)` logic. Rows with `decoded = NULL` are left as NULL (the + `WHERE path_hash_bytes IS NULL` clause naturally skips them). + +- **Gap — collector / API helper naming collision**: Noted that the collector's + `_path_hash_byte_width` helper mirrors the API route's (deleted in Phase 3), + so no persistent naming conflict exists. + +- **Risk — rows ingested between migration and collector deploy**: Added a brief + note under Open Questions that the Phase 1→2 deploy window can leave + `path_hash_bytes = NULL` for new rows, but the window is seconds in a compose + rebuild, and the UX is unchanged (`?B` label) for any rows missed. + +- **Postgres JSON type & backfill read mechanism**: Confirmed the `decoded` + column is SQLAlchemy `JSON` (not `JSONB`). The Python backfill reads decoded + as a Python `dict` via a Core `select()` on a `sa.Table` with a `sa.JSON`-typed + column — this routes through SQLAlchemy's JSON type adapter, which deserializes + consistently across SQLite (TEXT storage) and Postgres. Raw + `text("SELECT decoded ...")` must **not** be used: it bypasses the type adapter, + making deserialization driver-dependent (SQLite returns TEXT; Postgres/psycopg2 + may return a parsed object or string). No SQL JSON operators are needed — + consistent with TR-3. + +### Remaining Action Items + +- **Migration down_revision**: Before generating the migration, confirm that + `38abdf4651fc` is still the head revision (new migrations may land between + plan approval and implementation). The `meshcore-hub db revision` command + auto-resolves this — just verify the `down_revision` value in the generated + file. + +- **Phase 2 collector test**: The existing `test_store_raw_packet` test in + `tests/test_collector/` does not appear to cover `path_len` persistence; + verify whether a collector test for the new column can sit alongside or + replace the `path_len` test, and whether the test fixture accepts + `decoded_packet=` with path hashes. + +- **Frontend bundle rebuild**: Must be built via Docker compose (local + `node build.js` fails on the fontsource asset). The plan notes this in + Phase 4. diff --git a/docs/plans/20260703-2338-path-hash-bytes-filter/tasks.md b/docs/plans/20260703-2338-path-hash-bytes-filter/tasks.md new file mode 100644 index 0000000..d2b0537 --- /dev/null +++ b/docs/plans/20260703-2338-path-hash-bytes-filter/tasks.md @@ -0,0 +1,112 @@ +# Tasks: Persist & Filter by Path-Hash Byte Width + +> Generated from `plan.md` on 2026-07-03 + +## 1. Model & Migration (Phase 1) + +- [x] 1.1 Add `path_hash_bytes` column to `RawPacket` model + - [x] 1.1.1 Add `Mapped[Optional[int]] = mapped_column(Integer, nullable=True)` after `path_len` in `src/meshcore_hub/common/models/raw_packet.py` + - [x] 1.1.2 Verify model test reflects new column (presence + default None) + +- [x] 1.2 Generate and customize alembic migration + - [x] 1.2.1 Run `meshcore-hub db revision --autogenerate -m "add path_hash_bytes to raw_packets"` against migration head `38abdf4651fc` + - [x] 1.2.2 Verify `down_revision` is `38abdf4651fc` in generated file + - [x] 1.2.3 Append self-contained Python batched backfill loop (~1000 rows/batch): + - Read `decoded` via Core `select()` on a `sa.Table` declared with a `sa.JSON`-typed column (routes through SQLAlchemy's JSON type adapter → `dict` on both SQLite and Postgres); do **not** use raw `text("SELECT decoded ...")` (bypasses the adapter → driver-dependent deserialization) + - Inline frozen dual-path extraction: `decoded["path"]` fallback `decoded.get("payload", {}).get("decoded", {}).get("pathHashes")` + - Normalize with inline `_normalize_hash_list` logic (split on `","`, strip) + - Compute `max(len(h) // 2 for h in hashes)` for each row + - `UPDATE raw_packets SET path_hash_bytes = ? WHERE id = ?` + - Skip NULL decoded rows (leave NULL) + - No imports of app code + - [x] 1.2.4 Verify downgrade drops the column + - [x] 1.2.5 Test migration with `meshcore-hub db upgrade` on both SQLite and Postgres backends + +## 2. Collector Ingest (Phase 2) + +- [x] 2.1 Add `_path_hash_byte_width` helper + - [x] 2.1.1 Add module-level `_path_hash_byte_width(hashes: list[str] | None) -> int | None` in `src/meshcore_hub/collector/handlers/raw_packet.py` + - [x] 2.1.2 Implement as `max(len(h) // 2 for h in hashes) if hashes else None` + +- [x] 2.2 Compute `path_hash_bytes` at ingest + - [x] 2.2.1 In `store_raw_packet`, after `path_len` block (~line 96), extract path hashes from `decoded_packet`: + - `LetsMeshNormalizer._normalize_hash_list(decoded_packet.get("path"))` + - Fallback: `_normalize_hash_list(decoded_packet.get("payload", {}).get("decoded", {}).get("pathHashes"))` + - [x] 2.2.2 Compute `path_hash_bytes = _path_hash_byte_width(hashes)` (None-safe) + - [x] 2.2.3 Pass `path_hash_bytes=path_hash_bytes` into `RawPacket(...)` kwargs (~line 122) + - [x] 2.2.4 Verify null-safe: no path / no decoded → `None` + +- [x] 2.3 Add collector unit test + - [x] 2.3.1 Create test in `tests/test_collector/` (or extend existing `test_store_raw_packet`) covering: + - `decoded.path` with hashes of mixed widths → correct MAX byte-width persisted + - Trace fallback `decoded.payload.decoded.pathHashes` → correct width + - No path present → `None` + - Verify `path_len` persistence test still passes alongside + +## 3. API Route (Phase 3) + +- [x] 3.1 Add SQL aggregate to group query + - [x] 3.1.1 In Phase 1 `group_query` select (`src/meshcore_hub/api/routes/packet_groups.py`), add: + `func.max(RawPacket.path_hash_bytes).label("path_hash_bytes")` + - [x] 3.1.2 Add `path_hash_bytes: Optional[int] = Query(None, ge=1, le=3)` parameter to route function + - [x] 3.1.3 Apply `HAVING` filter: `if path_hash_bytes is not None: group_query = group_query.having(func.max(RawPacket.path_hash_bytes) == path_hash_bytes)` + - [x] 3.1.4 Verify HAVING works with existing WHERE clauses and `func.count(RawPacket.id)` + +- [x] 3.2 Delete Phase 3 Python decode loop + - [x] 3.2.1 Remove the `decoded_rows` fetch + `width_by_hash` loop (lines 198-216) + - [x] 3.2.2 Replace with direct read: `path_hash_bytes=(None if is_redacted else grp.path_hash_bytes)` + - [x] 3.2.3 Remove dead `_path_hash_byte_width` helper (lines 54-59) + - [x] 3.2.4 Confirm `_extract_path_hashes` (line 36) is kept — still used by detail route at line 311 + +- [x] 3.3 Update existing tests + - [x] 3.3.1 In `tests/test_api/test_packet_groups.py::TestPathHashBytes`, update insert fixtures to set `path_hash_bytes=` explicitly on `RawPacket` rows (tests bypass collector) + - [x] 3.3.2 Keep coverage for 1/2/3/None/redacted paths + - [x] 3.3.3 Verify all existing tests pass without Phase 3 + +- [x] 3.4 Add filter tests + - [x] 3.4.1 `?path_hash_bytes=2` returns only width-2 groups + - [x] 3.4.2 `?path_hash_bytes=1` and `3` return correct subsets + - [x] 3.4.3 Missing parameter returns all groups (including null-width) + - [x] 3.4.4 Redacted group response field is still null despite persisted value + - [x] 3.4.5 Verify `total` count is correct when filtered vs unfiltered + +## 4. Frontend & i18n (Phase 4) + +- [x] 4.1 Add filter `` + with options: Any (empty value) / 1B / 2B / 3B, mirroring `event_type` select style + - [x] 4.1.7 Option labels use `packets.path_width_bytes` (`{{count}}B`) and `common.all` + +- [x] 4.2 Add i18n keys + - [x] 4.2.1 Add `"packets.filter_path_width"` to `src/meshcore_hub/web/static/locales/en.json` ("Path width") + - [x] 4.2.2 Add `"packets.filter_path_width"` to `src/meshcore_hub/web/static/locales/nl.json` ("Padbreedte") + +- [x] 4.3 Rebuild SPA bundle + - [x] 4.3.1 Build via Docker compose pipeline (not local `node build.js` — fontsource asset requires compose context) + +## 5. Verification (Phase 5) + +- [x] 5.1 Run targeted tests + - [x] 5.1.1 `pytest --no-cov tests/test_api/test_packet_groups.py` + - [x] 5.1.2 `pytest --no-cov tests/test_collector/` + - [x] 5.1.3 `pytest --no-cov tests/test_common/` (model column test) + +- [x] 5.2 Run full test suite + - [x] 5.2.1 `pytest --no-cov 2>&1 | grep -iE "passed|failed" | tail -3` + +- [x] 5.3 Run pre-commit + - [x] 5.3.1 `pre-commit run --all-files` + +- [x] 5.4 Manual smoke test on Docker stack + - [x] 5.4.1 Apply migration: `docker compose run --rm migrate db upgrade` + - [x] 5.4.2 Rebuild and restart stack + - [x] 5.4.3 Verify packet list loads with all three width filters (1B/2B/3B) + - [x] 5.4.4 Verify "Any" filter returns all rows including null-width + - [x] 5.4.5 Verify filter survives pagination and sort changes via URL query string + - [x] 5.4.6 Verify redacted groups show `?B` on detail page diff --git a/src/meshcore_hub/api/routes/packet_groups.py b/src/meshcore_hub/api/routes/packet_groups.py index f5ac8b3..18fefdb 100644 --- a/src/meshcore_hub/api/routes/packet_groups.py +++ b/src/meshcore_hub/api/routes/packet_groups.py @@ -51,14 +51,6 @@ def _extract_path_hashes(decoded: dict[str, Any] | None) -> list[str] | None: return hashes if isinstance(hashes, list) else None -def _path_hash_byte_width(path_hashes: Optional[list[str]]) -> Optional[int]: - """Widest path-hash prefix width in bytes (each hash is hex: 2/4/6 chars).""" - if not path_hashes: - return None - widths = [len(h) // 2 for h in path_hashes if isinstance(h, str) and h] - return max(widths) if widths else None - - def _get_tag_name(node: Optional[Node]) -> Optional[str]: if not node or not node.tags: return None @@ -79,6 +71,9 @@ def list_packet_groups( ), event_type: Optional[str] = Query(None, description="Filter by event type"), channel_idx: Optional[int] = Query(None, description="Filter by channel index"), + path_hash_bytes: Optional[int] = Query( + None, ge=1, le=3, description="Filter by path-hash byte width (1/2/3)" + ), since: Optional[datetime] = Query(None, description="Start timestamp"), until: Optional[datetime] = Query(None, description="End timestamp"), sort: Optional[str] = Query( @@ -120,6 +115,7 @@ def list_packet_groups( func.count(RawPacket.id).label("reception_count"), func.count(RawPacket.observer_node_id.distinct()).label("observer_count"), func.min(RawPacket.received_at).label("first_seen"), + func.max(RawPacket.path_hash_bytes).label("path_hash_bytes"), ) .outerjoin(ObserverNode, RawPacket.observer_node_id == ObserverNode.id) .where(RawPacket.packet_hash.is_not(None)) @@ -146,6 +142,11 @@ def list_packet_groups( group_query = group_query.group_by(RawPacket.packet_hash) + if path_hash_bytes is not None: + group_query = group_query.having( + func.max(RawPacket.path_hash_bytes) == path_hash_bytes + ) + count_query = select(func.count()).select_from(group_query.subquery()) total = session.execute(count_query).scalar() or 0 @@ -195,20 +196,6 @@ def list_packet_groups( group_counts = {r.packet_hash: r for r in group_rows} - # ── Phase 3: path-hash width — load decoded only for representative rows ─── - rep_id_to_hash = {rep.id: h for h, rep in representative.items()} - width_by_hash: dict[str, Optional[int]] = {} - if rep_id_to_hash: - decoded_rows = session.execute( - select(RawPacket.id, RawPacket.decoded).where( - RawPacket.id.in_(rep_id_to_hash.keys()) - ) - ).all() - for rid, decoded in decoded_rows: - h = rep_id_to_hash.get(rid) - if h is not None: - width_by_hash[h] = _path_hash_byte_width(_extract_path_hashes(decoded)) - items = [] for h in hashes: rep = representative.get(h) @@ -231,7 +218,7 @@ def list_packet_groups( ), reception_count=grp.reception_count, observer_count=grp.observer_count, - path_hash_bytes=(None if is_redacted else width_by_hash.get(h)), + path_hash_bytes=(None if is_redacted else grp.path_hash_bytes), receptions=[], first_seen=grp.first_seen, redacted=is_redacted, diff --git a/src/meshcore_hub/collector/handlers/raw_packet.py b/src/meshcore_hub/collector/handlers/raw_packet.py index 07a5a3c..a763e42 100644 --- a/src/meshcore_hub/collector/handlers/raw_packet.py +++ b/src/meshcore_hub/collector/handlers/raw_packet.py @@ -42,6 +42,14 @@ def _extract_source_pubkey_prefix( return None +def _path_hash_byte_width(hashes: list[str] | None) -> int | None: + """Widest path-hash prefix width in bytes (each hash is hex: 2/4/6 chars).""" + if not hashes: + return None + widths = [len(h) // 2 for h in hashes if isinstance(h, str) and h] + return max(widths) if widths else None + + def store_raw_packet( public_key: str, payload: dict[str, Any], @@ -95,6 +103,14 @@ def store_raw_packet( decoded_packet ) + path_hashes = LetsMeshNormalizer._normalize_hash_list( + decoded_packet.get("path") if isinstance(decoded_packet, dict) else None + ) + if not path_hashes and isinstance(decoded_packet, dict): + inner = (decoded_packet.get("payload") or {}).get("decoded") or {} + path_hashes = LetsMeshNormalizer._normalize_hash_list(inner.get("pathHashes")) + path_hash_bytes = _path_hash_byte_width(path_hashes) + snr = LetsMeshNormalizer._parse_float(payload.get("SNR")) if snr is None: snr = LetsMeshNormalizer._parse_float(payload.get("snr")) @@ -131,6 +147,7 @@ def store_raw_packet( source_pubkey_prefix=source_pubkey_prefix, route_type=route_type, path_len=path_len, + path_hash_bytes=path_hash_bytes, snr=snr, decoded=decoded_packet, received_at=now, diff --git a/src/meshcore_hub/common/models/raw_packet.py b/src/meshcore_hub/common/models/raw_packet.py index 835c9f7..7247077 100644 --- a/src/meshcore_hub/common/models/raw_packet.py +++ b/src/meshcore_hub/common/models/raw_packet.py @@ -32,6 +32,7 @@ class RawPacket(Base, UUIDMixin, TimestampMixin): senderPublicKey, for efficient "packets from this node" filtering route_type: Mapped route type (flood, transport_flood, direct, ...) path_len: Hop count + path_hash_bytes: Widest path-hash prefix width in bytes (1/2/3) snr: Signal-to-noise ratio as reported by the observer decoded: Decoder summary JSON, so the detail view needs no re-decode received_at: When received by the observing interface @@ -86,6 +87,10 @@ class RawPacket(Base, UUIDMixin, TimestampMixin): Integer, nullable=True, ) + path_hash_bytes: Mapped[Optional[int]] = mapped_column( + Integer, + nullable=True, + ) snr: Mapped[Optional[float]] = mapped_column( Float, nullable=True, diff --git a/src/meshcore_hub/web/static/js/spa/icons.js b/src/meshcore_hub/web/static/js/spa/icons.js index cf2c66c..2676b6c 100644 --- a/src/meshcore_hub/web/static/js/spa/icons.js +++ b/src/meshcore_hub/web/static/js/spa/icons.js @@ -170,6 +170,10 @@ export function iconTxPower(cls = 'h-5 w-5') { return html``; } +export function iconRuler(cls = 'h-5 w-5') { + return html``; +} + export function iconFilter(cls = 'h-5 w-5') { return html``; } diff --git a/src/meshcore_hub/web/static/js/spa/pages/packets.js b/src/meshcore_hub/web/static/js/spa/pages/packets.js index 59b1199..25ec70b 100644 --- a/src/meshcore_hub/web/static/js/spa/pages/packets.js +++ b/src/meshcore_hub/web/static/js/spa/pages/packets.js @@ -7,7 +7,7 @@ import { renderFilterForm, renderFilterToggle, autoSubmit, submitOnEnter } from '../components.js'; import { createAutoRefresh } from '../auto-refresh.js'; -import { iconSatelliteDish, iconPath } from '../icons.js'; +import { iconSatelliteDish, iconPath, iconRuler } from '../icons.js'; const EVENT_TYPES = [ 'advertisement', 'channel_msg_recv', 'contact_msg_recv', @@ -34,12 +34,19 @@ function receptionBadge(packet) { const rc = packet.reception_count ?? 1; const oc = packet.observer_count ?? 1; const pb = packet.path_hash_bytes; + const knownWidth = pb != null && pb > 0; + const widthLabel = knownWidth + ? t('packets.path_width_bytes', { count: pb }) + : t('packets.path_width_unknown'); return html` ${iconSatelliteDish('h-4 w-4 opacity-70')} ${formatNumber(oc)} + ${iconPath('h-4 w-4 opacity-70')} ${formatNumber(rc)} - ${pb ? html`${t('packets.path_width_bytes', { count: pb })}` : nothing} + + ${iconRuler('h-4 w-4 opacity-70')} + ${widthLabel} `; } @@ -49,6 +56,7 @@ export async function render(container, params, router) { const search = query.search || ''; const event_type = query.event_type || ''; const channel_idx = query.channel_idx || ''; + const path_hash_bytes = query.path_hash_bytes || ''; const page = parseInt(query.page, 10) || 1; const limit = parseInt(query.limit, 10) || 20; const offset = (page - 1) * limit; @@ -63,7 +71,7 @@ export async function render(container, params, router) { let lastContent = nothing; let lastTotal = null; let currentFilterFields = []; - const hasActiveFilters = search !== '' || event_type !== '' || channel_idx !== ''; + const hasActiveFilters = search !== '' || event_type !== '' || channel_idx !== '' || path_hash_bytes !== ''; function onFilterToggle() { renderPage(lastContent, { total: lastTotal }); } @@ -104,6 +112,7 @@ ${displayContent}`, container); const apiParams = { limit, offset, search, sort, order }; if (event_type) apiParams.event_type = event_type; if (channel_idx !== '') apiParams.channel_idx = channel_idx; + if (path_hash_bytes !== '') apiParams.path_hash_bytes = path_hash_bytes; const [data, channelsData] = await Promise.all([ apiGet('/api/v1/packet-groups', apiParams, { signal }), @@ -157,7 +166,7 @@ ${displayContent}`, container); `); const paginationBlock = pagination(page, totalPages, '/packets', { - search, event_type, channel_idx, limit, sort, order, + search, event_type, channel_idx, path_hash_bytes, limit, sort, order, }); const filterFields = [ @@ -181,10 +190,18 @@ ${displayContent}`, container); ${channelList.map(c => html``)} + `, + () => html` +
+ +
`, ]; - const headerParams = { search, event_type, channel_idx, limit }; + const headerParams = { search, event_type, channel_idx, path_hash_bytes, limit }; const sortable = (label, sortKey) => sortableTableHeader(label, { sortKey, currentSort: sort, currentOrder: order, navigate, basePath: '/packets', params: headerParams, diff --git a/src/meshcore_hub/web/static/locales/en.json b/src/meshcore_hub/web/static/locales/en.json index e6a4105..b97d5e0 100644 --- a/src/meshcore_hub/web/static/locales/en.json +++ b/src/meshcore_hub/web/static/locales/en.json @@ -82,6 +82,7 @@ "sign_out": "Sign Out", "view_details": "View Details", "all_types": "All Types", + "all": "Any", "all_channels": "All Channels", "all_members": "All Members", "all_operators": "All Operators", @@ -218,6 +219,7 @@ }, "packets": { "filter_event_type": "Event Type", + "filter_path_width": "Path Width", "filter_min_snr": "Min SNR", "filter_max_snr": "Max SNR", "redacted": "Restricted", @@ -248,6 +250,7 @@ "path_nodes_more": "+{{count}} more — view all →", "not_found_retention": "Packet not found — it may have been cleaned up due to data retention.", "path_width_bytes": "{{count}}B", + "path_width_unknown": "?B", "path_width_title": "Path hash width (bytes)", "sort": { "newest": "Time (newest)", diff --git a/src/meshcore_hub/web/static/locales/nl.json b/src/meshcore_hub/web/static/locales/nl.json index e46d29e..469c19e 100644 --- a/src/meshcore_hub/web/static/locales/nl.json +++ b/src/meshcore_hub/web/static/locales/nl.json @@ -80,6 +80,7 @@ "sign_out": "Uitloggen", "view_details": "Details bekijken", "all_types": "Alle types", + "all": "Alle", "node_type": "Knooppunttype", "show": "Toon", "search_placeholder": "Zoek op naam, ID of publieke sleutel...", @@ -160,6 +161,7 @@ }, "packets": { "filter_event_type": "Gebeurtenistype", + "filter_path_width": "Padbreedte", "filter_min_snr": "Min. SNR", "filter_max_snr": "Max. SNR", "redacted": "Beperkt", @@ -184,6 +186,7 @@ "path_nodes_more": "+{{count}} meer — alle bekijken →", "not_found_retention": "Pakket niet gevonden — mogelijk opgeschoond door dataretentie.", "path_width_bytes": "{{count}}B", + "path_width_unknown": "?B", "path_width_title": "Breedte pad-hash (bytes)", "sort": { "newest": "Tijd (nieuwste)", diff --git a/tests/test_api/test_packet_groups.py b/tests/test_api/test_packet_groups.py index 23b0f3e..199d630 100644 --- a/tests/test_api/test_packet_groups.py +++ b/tests/test_api/test_packet_groups.py @@ -300,14 +300,14 @@ class TestListPacketGroups: class TestPathHashBytes: - """Tests for the derived path_hash_bytes field on the list endpoint.""" + """Tests for the persisted path_hash_bytes field on the list endpoint.""" def test_one_byte_path(self, client_no_auth, api_db_session): api_db_session.add( RawPacket( raw_hex="AA", packet_hash="H1", - decoded={"path": ["aa", "bb"]}, + path_hash_bytes=1, received_at=_now(), ) ) @@ -320,7 +320,7 @@ class TestPathHashBytes: RawPacket( raw_hex="AA", packet_hash="H1", - decoded={"path": ["aabb"]}, + path_hash_bytes=2, received_at=_now(), ) ) @@ -328,17 +328,45 @@ class TestPathHashBytes: item = client_no_auth.get("/api/v1/packet-groups").json()["items"][0] assert item["path_hash_bytes"] == 2 - def test_mixed_path_uses_max(self, client_no_auth, api_db_session): + def test_three_byte_path(self, client_no_auth, api_db_session): api_db_session.add( RawPacket( raw_hex="AA", packet_hash="H1", - decoded={"path": ["aa", "aabb"]}, + path_hash_bytes=3, received_at=_now(), ) ) api_db_session.commit() item = client_no_auth.get("/api/v1/packet-groups").json()["items"][0] + assert item["path_hash_bytes"] == 3 + + def test_width_aggregates_across_receptions(self, client_no_auth, api_db_session): + """Width is observer-relative; take the widest across all receptions. + + The oldest reception (representative) carries no path, while a newer + reception of the same packet saw a 2-byte path. The badge must reflect + the widest path seen, not just the representative row. + """ + base = _now() + api_db_session.add( + RawPacket( + raw_hex="AA", + packet_hash="H1", + path_hash_bytes=None, + received_at=base, + ) + ) + api_db_session.add( + RawPacket( + raw_hex="BB", + packet_hash="H1", + path_hash_bytes=2, + received_at=base.replace(microsecond=base.microsecond + 1), + ) + ) + api_db_session.commit() + item = client_no_auth.get("/api/v1/packet-groups").json()["items"][0] assert item["path_hash_bytes"] == 2 def test_no_path_returns_none(self, client_no_auth, api_db_session): @@ -366,7 +394,7 @@ class TestPathHashBytes: raw_hex="SECRET", packet_hash="ADM_HASH", channel_idx=adm_idx, - decoded={"path": ["aabb"]}, + path_hash_bytes=2, received_at=_now(), ) ) @@ -376,6 +404,142 @@ class TestPathHashBytes: assert item["path_hash_bytes"] is None +class TestPathHashBytesFilter: + """Tests for the ?path_hash_bytes= query filter.""" + + def test_filter_returns_only_matching_width(self, client_no_auth, api_db_session): + now = _now() + api_db_session.add_all( + [ + RawPacket( + raw_hex="A", packet_hash="H1", path_hash_bytes=1, received_at=now + ), + RawPacket( + raw_hex="B", packet_hash="H2", path_hash_bytes=2, received_at=now + ), + RawPacket( + raw_hex="C", packet_hash="H3", path_hash_bytes=3, received_at=now + ), + ] + ) + api_db_session.commit() + + data = client_no_auth.get("/api/v1/packet-groups?path_hash_bytes=2").json() + assert data["total"] == 1 + assert data["items"][0]["packet_hash"] == "H2" + + def test_filter_width_1(self, client_no_auth, api_db_session): + now = _now() + api_db_session.add_all( + [ + RawPacket( + raw_hex="A", packet_hash="H1", path_hash_bytes=1, received_at=now + ), + RawPacket( + raw_hex="B", packet_hash="H2", path_hash_bytes=3, received_at=now + ), + ] + ) + api_db_session.commit() + + data = client_no_auth.get("/api/v1/packet-groups?path_hash_bytes=1").json() + assert data["total"] == 1 + assert data["items"][0]["packet_hash"] == "H1" + + def test_filter_width_3(self, client_no_auth, api_db_session): + now = _now() + api_db_session.add_all( + [ + RawPacket( + raw_hex="A", packet_hash="H1", path_hash_bytes=1, received_at=now + ), + RawPacket( + raw_hex="B", packet_hash="H2", path_hash_bytes=3, received_at=now + ), + ] + ) + api_db_session.commit() + + data = client_no_auth.get("/api/v1/packet-groups?path_hash_bytes=3").json() + assert data["total"] == 1 + assert data["items"][0]["packet_hash"] == "H2" + + def test_no_filter_returns_all_including_null(self, client_no_auth, api_db_session): + now = _now() + api_db_session.add_all( + [ + RawPacket( + raw_hex="A", packet_hash="H1", path_hash_bytes=1, received_at=now + ), + RawPacket( + raw_hex="B", packet_hash="H2", path_hash_bytes=None, received_at=now + ), + ] + ) + api_db_session.commit() + + data = client_no_auth.get("/api/v1/packet-groups").json() + assert data["total"] == 2 + widths = {i["path_hash_bytes"] for i in data["items"]} + assert widths == {1, None} + + def test_filter_excludes_null_width(self, client_no_auth, api_db_session): + now = _now() + api_db_session.add_all( + [ + RawPacket( + raw_hex="A", packet_hash="H1", path_hash_bytes=None, received_at=now + ), + RawPacket( + raw_hex="B", packet_hash="H2", path_hash_bytes=2, received_at=now + ), + ] + ) + api_db_session.commit() + + data = client_no_auth.get("/api/v1/packet-groups?path_hash_bytes=2").json() + assert data["total"] == 1 + assert data["items"][0]["path_hash_bytes"] == 2 + + def test_filter_redacted_group_still_null(self, client_no_auth, api_db_session): + adm_key = "FFEEDDCCBBAA99887766554433221100" + adm_idx = int(Channel.compute_channel_hash(adm_key), 16) + now = _now() + api_db_session.add( + Channel( + name="Adm", + key_hex=adm_key, + channel_hash=Channel.compute_channel_hash(adm_key), + visibility="admin", + enabled=True, + ) + ) + api_db_session.add_all( + [ + RawPacket( + raw_hex="SECRET", + packet_hash="ADM", + channel_idx=adm_idx, + path_hash_bytes=2, + received_at=now, + ), + RawPacket( + raw_hex="OK", + packet_hash="PUB", + path_hash_bytes=2, + received_at=now, + ), + ] + ) + api_db_session.commit() + + data = client_no_auth.get("/api/v1/packet-groups?path_hash_bytes=2").json() + assert data["total"] == 2 + adm = next(i for i in data["items"] if i["packet_hash"] == "ADM") + assert adm["redacted"] is True + assert adm["path_hash_bytes"] is None + + class TestGetPacketGroup: """Tests for GET /packet-groups/{hash} (detail).""" diff --git a/tests/test_collector/test_handlers/test_raw_packet.py b/tests/test_collector/test_handlers/test_raw_packet.py index 58bb69d..d77b1e7 100644 --- a/tests/test_collector/test_handlers/test_raw_packet.py +++ b/tests/test_collector/test_handlers/test_raw_packet.py @@ -144,3 +144,84 @@ class TestStoreRawPacketEdges: rp = db_session.execute(select(RawPacket)).scalar_one() assert rp.source_pubkey_prefix is None + + +class TestStoreRawPacketPathHashBytes: + """Tests for path_hash_bytes computation at ingest.""" + + def test_top_level_path_max_byte_width(self, db_manager, db_session): + """Mixed-width hashes in decoded.path persist the widest byte width.""" + decoded = { + "payloadType": 1, + "path": ["aa", "aabbcc"], + "pathLength": 2, + "payload": {"decoded": {"sourceHash": "AABBCCDD"}}, + } + store_raw_packet( + "a" * 64, {"raw": "00", "hash": "h1"}, decoded, "flood", db_manager + ) + + rp = db_session.execute(select(RawPacket)).scalar_one() + assert rp.path_hash_bytes == 3 + + def test_trace_fallback_path_hashes(self, db_manager, db_session): + """Trace-style pathHashes in payload.decoded are used as fallback.""" + decoded = { + "payloadType": 1, + "payload": {"decoded": {"pathHashes": ["aabb", "ccdd"]}}, + } + store_raw_packet( + "a" * 64, {"raw": "00", "hash": "h1"}, decoded, "trace", db_manager + ) + + rp = db_session.execute(select(RawPacket)).scalar_one() + assert rp.path_hash_bytes == 2 + + def test_one_byte_path(self, db_manager, db_session): + """Single-byte hashes persist width 1.""" + decoded = { + "payloadType": 1, + "path": ["aa", "bb"], + "payload": {"decoded": {}}, + } + store_raw_packet( + "a" * 64, {"raw": "00", "hash": "h1"}, decoded, "flood", db_manager + ) + + rp = db_session.execute(select(RawPacket)).scalar_one() + assert rp.path_hash_bytes == 1 + + def test_no_path_returns_none(self, db_manager, db_session): + """No path hashes at all persist path_hash_bytes = None.""" + decoded = {"payloadType": 3, "payload": {"decoded": {}}} + store_raw_packet( + "a" * 64, {"raw": "00", "hash": "h1"}, decoded, "ack", db_manager + ) + + rp = db_session.execute(select(RawPacket)).scalar_one() + assert rp.path_hash_bytes is None + + def test_null_decoded_returns_none(self, db_manager, db_session): + """A None decoded_packet leaves path_hash_bytes as None.""" + store_raw_packet( + "a" * 64, {"raw": "00", "hash": "h1"}, None, "unknown", db_manager + ) + + rp = db_session.execute(select(RawPacket)).scalar_one() + assert rp.path_hash_bytes is None + + def test_path_len_still_persisted(self, db_manager, db_session): + """path_len is still persisted alongside path_hash_bytes.""" + decoded = { + "payloadType": 1, + "path": ["aa", "bb", "cc"], + "pathLength": 3, + "payload": {"decoded": {"sourceHash": "AABBCCDD"}}, + } + store_raw_packet( + "a" * 64, {"raw": "00", "hash": "h1"}, decoded, "flood", db_manager + ) + + rp = db_session.execute(select(RawPacket)).scalar_one() + assert rp.path_len == 3 + assert rp.path_hash_bytes == 1 diff --git a/tests/test_common/test_models.py b/tests/test_common/test_models.py index 8b19dc3..382fa21 100644 --- a/tests/test_common/test_models.py +++ b/tests/test_common/test_models.py @@ -109,6 +109,7 @@ class TestRawPacketModel: assert packet.raw_hex is None assert packet.channel_idx is None assert packet.source_pubkey_prefix is None + assert packet.path_hash_bytes is None assert packet.decoded is None def test_raw_packet_indexes(self) -> None: