feat: persist path_hash_bytes column and add packet list filter

Persist the path-hash byte width (1/2/3) as a nullable Integer column on
RawPacket, computed at ingest by the collector and backfilled for historical
rows via a self-contained Python migration. Replace the per-request Python
decode loop in the grouped-list route with a SQL MAX() aggregate + HAVING
filter, and add a discrete <select> filter (Any/1B/2B/3B) to the /packets
SPA page.

- Model: add path_hash_bytes column to RawPacket (nullable Integer)
- Migration: batch_alter_table add_column + keyset-paginated Python backfill
  reading decoded via Core select() on sa.JSON column (portable across
  SQLite and Postgres)
- Collector: compute path_hash_bytes at ingest via two-tier path-hash
  extraction (decoded.path -> payload.decoded.pathHashes)
- API: add func.max() aggregate to group query, HAVING filter on
  ?path_hash_bytes=1|2|3 param; delete Phase 3 decode loop and dead helper
- Frontend: add path-width select filter wired through query/apiParams/
  pagination/headerParams; add i18n keys (en/nl)
- Tests: 1186 passed, 22 skipped; collector + API + model coverage
This commit is contained in:
Louis King
2026-07-04 00:02:18 +01:00
parent 1615c71e18
commit c029eae524
13 changed files with 875 additions and 34 deletions
@@ -0,0 +1,117 @@
"""add path_hash_bytes to raw_packets
Revision ID: 57bb65130b97
Revises: 38abdf4651fc
Create Date: 2026-07-03 22:50:33.598167+00:00
Adds a nullable ``path_hash_bytes`` column to ``raw_packets`` (widest
path-hash prefix width: 1/2/3, or NULL when no path hashes are present).
The collector computes this at ingest going forward; this migration
backfills historical rows from their stored ``decoded`` JSON using a
self-contained, frozen copy of the dual-path extraction logic.
"""
from typing import Any, Sequence, Union
from alembic import op
import sqlalchemy as sa
# revision identifiers, used by Alembic.
revision: str = "57bb65130b97"
down_revision: Union[str, None] = "38abdf4651fc"
branch_labels: Union[str, Sequence[str], None] = None
depends_on: Union[str, Sequence[str], None] = None
_BATCH_SIZE = 1000
# Declared with a sa.JSON-typed column so the SQLAlchemy type adapter
# deserializes ``decoded`` to a Python dict consistently on both SQLite
# (TEXT storage) and Postgres. Using raw text("SELECT decoded ...") would
# bypass the adapter and make deserialization driver-dependent.
_raw_packets = sa.table(
"raw_packets",
sa.Column("id", sa.String, primary_key=True),
sa.Column("decoded", sa.JSON),
sa.Column("path_hash_bytes", sa.Integer),
)
def _normalize_hash_list(value: Any) -> list[str] | None:
"""Frozen copy of LetsMeshNormalizer._normalize_hash_list.
Accepts even-length hex strings of 2 or more characters.
Each string is uppercased and validated as hexadecimal.
"""
if not isinstance(value, list):
return None
normalized: list[str] = []
for item in value:
if not isinstance(item, str):
continue
token = item.strip().upper()
if len(token) < 2 or len(token) % 2 != 0:
continue
if any(ch not in "0123456789ABCDEF" for ch in token):
continue
normalized.append(token)
return normalized or None
def _compute_path_hash_byte_width(decoded: Any) -> int | None:
"""Frozen copy of the dual-path extraction + max(len//2) logic.
Path hashes live at ``decoded.path`` for normal packets, with
``decoded.payload.decoded.pathHashes`` as the trace-style fallback.
Returns the widest prefix width in bytes, or None when no path hashes
are present/decodable.
"""
if not isinstance(decoded, dict):
return None
hashes = _normalize_hash_list(decoded.get("path"))
if not hashes:
payload = decoded.get("payload") or {}
inner = payload.get("decoded") or {}
hashes = _normalize_hash_list(inner.get("pathHashes"))
if not hashes:
return None
return max(len(h) // 2 for h in hashes)
def upgrade() -> None:
with op.batch_alter_table("raw_packets", schema=None) as batch_op:
batch_op.add_column(sa.Column("path_hash_bytes", sa.Integer(), nullable=True))
conn = op.get_bind()
# Keyset-paginated backfill over all rows so rows whose ``decoded`` is
# NULL (uncomputable) are visited once and left NULL without causing an
# infinite loop.
last_id: str | None = None
while True:
query = (
sa.select(_raw_packets.c.id, _raw_packets.c.decoded)
.order_by(_raw_packets.c.id)
.limit(_BATCH_SIZE)
)
if last_id is not None:
query = query.where(_raw_packets.c.id > last_id)
batch = conn.execute(query).all()
if not batch:
break
for row in batch:
last_id = row.id
width = _compute_path_hash_byte_width(row.decoded)
if width is not None:
conn.execute(
sa.update(_raw_packets)
.where(_raw_packets.c.id == row.id)
.values(path_hash_bytes=width)
)
def downgrade() -> None:
with op.batch_alter_table("raw_packets", schema=None) as batch_op:
batch_op.drop_column("path_hash_bytes")
@@ -0,0 +1,330 @@
# Persist & Filter by Path-Hash Byte Width
## Summary
The packet list page (`/packets`, served by `GET /api/v1/packet-groups`) shows a
"path-hash byte width" metric per row — the ruler badge (`1B`/`2B`/`3B`/`?B`) we
recently added to the reception widget. That value is currently **derived at
request time** in Python by decoding each group's `decoded` JSON, which makes it
unfilterable at the database level and forces an expensive extra query on every
list load.
This plan **persists `path_hash_bytes` as a real column on `RawPacket`**, computed
once at ingest by the collector and backfilled for historical rows. With the value
persisted, the grouped-list route's Python decode loop collapses into a SQL
`MAX()` aggregate, and we add a clean **discrete-select filter** (Any / 1B / 2B /
3B) to the packet list page via a `HAVING` clause — fully consistent pagination and
counts, with no per-request JSON decoding.
## Background & Motivation
### Current State
- `path_hash_bytes` is **not a column**. It is computed at runtime in
`api/routes/packet_groups.py:198-216` ("Phase 3"), which fetches the `decoded`
JSON for every reception of the current page's packet hashes, extracts the path
arrays, and aggregates the widest byte-width (1/2/3 from 2/4/6 hex chars). The
aggregation was recently fixed (PR #292, commit `f845830`) to take `MAX`
across **all** receptions because the decoded `path` is observer-relative — a
single representative row often misses a multibyte path that other receptions
carry.
- The packet list widget renders this as `[Dish][oc] × [Path][rc] @ [Ruler][pb]`,
the most prominent path metric on the list.
- The collector is the **sole writer** of `RawPacket`, through a single chokepoint
`store_raw_packet` (`collector/handlers/raw_packet.py:122-138`). It already
computes `path_len` (hop count) there with a two-tier fallback
(`raw_packet.py:92-96`), and it already has a collector-side mirror of the
path-hash extraction: `_extract_message_path_hashes`
(`letsmesh_normalizer.py:826-844`) + the `_normalize_hash_list` staticmethod
(`:846-867`).
- The raw `/packets` endpoint already supports `min_path_len`/`max_path_len`
range filters on the real `path_len` column — the closest filtering precedent.
### Problems
- **Unfilterable.** Because the value is computed post-query in Python, it cannot
be filtered at the SQL level without either persisting a column or pushing
brittle, backend-specific JSON extraction into SQL (dual-path: `decoded.path`
and the trace fallback `decoded.payload.decoded.pathHashes`).
- **Pagination inconsistency.** A post-query Python filter (the only no-migration
hack) would make `total` and page sizes inconsistent — bad UX.
- **Request-time cost.** Every list load performs an extra decoded-JSON fetch +
per-row Python decode just to render the badge.
### Why Persist a Column (chosen approach)
The user confirmed the metric is **byte width** (the ruler badge) and the
approach is **persisted column (A)**. This is the only solution that is correct,
maintainable, and gives consistent pagination. The collector already extracts
path hashes at ingest, so computing the byte-width there is a trivial addition
covering 100% of new rows; a one-time Python backfill covers historical rows.
## Goals
- Persist `path_hash_bytes` (nullable `Integer`, values 1/2/3 or NULL) on
`RawPacket`, computed at ingest from the decoded path hashes.
- Backfill the column for all existing `raw_packets` rows from their `decoded`
JSON, portably across SQLite and Postgres.
- Add a **discrete-select filter** (Any / 1B / 2B / 3B) to the `/packets` list
page, applied as a SQL `HAVING` on the per-group `MAX(path_hash_bytes)`.
- Simplify the grouped-list route: replace the Python decode loop (Phase 3) with
a SQL `MAX()` aggregate, removing the per-request JSON fetch.
## Non-Goals
- Filtering the **raw** `/packets` endpoint (not the list page) by byte width.
The column will exist on the model and DB, but surfacing it on the raw endpoint
/ `RawPacketRead` schema is out of scope (easy follow-up).
- An "Unknown" (`?B` / null) filter option — the user chose **known widths only**
(1/2/3). Null-width packets are simply never selected by the filter; they
remain visible under "Any".
- A numeric range (min/max) or multi-select filter shape — the user chose a
**discrete single-select**.
- Indexing `path_hash_bytes` (mirrors `path_len`, which is also unindexed). Add
an index later only if query plans show a need.
- Changing the per-reception path extraction in the **detail** route
(`_extract_path_hashes` stays; only the list-path width computation is removed).
## Requirements
### Functional Requirements
- **FR-1**: A new nullable `Integer` column `path_hash_bytes` on `raw_packets`,
holding the widest path-hash prefix width (1/2/3) for that reception, or NULL
when no path hashes are present/decodable.
- **FR-2**: Every newly inserted `RawPacket` row has `path_hash_bytes` computed
from its `decoded` dict at ingest (collector is the sole writer).
- **FR-3**: Historical `raw_packets` rows are backfilled from their `decoded`
JSON during migration, using the same dual-path extraction and `max(len//2)`
semantics as the runtime computation.
- **FR-4**: `GET /api/v1/packet-groups` accepts an optional
`path_hash_bytes` query parameter (integer 1/2/3). When set, only groups whose
`MAX(path_hash_bytes)` equals the value are returned. Omitting it returns all
groups (including null-width ones) — unchanged behavior.
- **FR-5**: The list response still nulls `path_hash_bytes` for redacted groups
(channel-visibility), exactly as today (`None if is_redacted else ...`).
- **FR-6**: The `/packets` SPA page offers a **discrete `<select>`** filter
("Path width": Any / 1B / 2B / 3B) that auto-submits and round-trips through
the URL query string, surviving pagination and sort changes.
### Technical Requirements
- **TR-1** — Model (`common/models/raw_packet.py`): add
`path_hash_bytes: Mapped[Optional[int]] = mapped_column(Integer, nullable=True)`
immediately after `path_len`. Unindexed (mirrors `path_len`).
- **TR-2** — Collector (`collector/handlers/raw_packet.py`): in
`store_raw_packet`, after `path_len` is resolved (~line 96), compute
`path_hash_bytes` from `decoded_packet` via the two-tier lookup
(`decoded.path``payload.decoded.pathHashes`) using the already-imported
`LetsMeshNormalizer._normalize_hash_list` staticmethod, then
`max(len(h) // 2 for h in hashes)`. Add a small module-level
`_path_hash_byte_width(hashes)` helper (mirrors the API route's helper; the
API copy is deleted in Phase 3 so no naming collision persists). Pass
`path_hash_bytes=` into the `RawPacket(...)` kwargs. Must be null-safe (no
path → `None`).
- **TR-3** — Migration (`alembic/versions/`, `down_revision = "38abdf4651fc"`):
author via `meshcore-hub db revision --autogenerate -m "add path_hash_bytes to
raw_packets"` (produces a `batch_alter_table` `add_column`), then **manually
append a Python batched backfill** (~1000 rows/batch) that reads each row's
`decoded` JSON, applies the self-contained dual-path extraction +
`max(len//2)`, and `UPDATE`s the column. Python (not SQL JSON) so it is
portable across SQLite and Postgres. Migration logic must be a frozen
self-contained snapshot (no imports of app code that may change). Downgrade
drops the column. **Read mechanism**: the `decoded` column must be read via a
Core `select()` on a `sa.Table` declared with a `sa.JSON`-typed column (or
explicitly `json.loads()` the fetched scalar), **not** raw
`text("SELECT decoded ...")`. Raw `text()` bypasses SQLAlchemy's JSON type
adapter, so deserialization becomes driver-dependent (SQLite returns TEXT,
Postgres/psycopg2 may return a parsed object or string). Core `select()`
routes through the type adapter and yields a Python `dict` consistently on
both backends.
- **TR-4** — API route (`api/routes/packet_groups.py`):
- Phase 1 `group_query` select: add
`func.max(RawPacket.path_hash_bytes).label("path_hash_bytes")`.
- New param: `path_hash_bytes: Optional[int] = Query(None, ge=1, le=3,
description="Filter by path-hash byte width (1/2/3)")`.
- Apply filter:
`if path_hash_bytes is not None: group_query = group_query.having(func.max(RawPacket.path_hash_bytes) == path_hash_bytes)`.
- **Delete Phase 3** (lines 198-216, the `decoded_rows` fetch + `width_by_hash`
loop). Read `grp.path_hash_bytes` from the group row instead.
- Build item with
`path_hash_bytes=(None if is_redacted else grp.path_hash_bytes)`.
- Remove the now-dead `_path_hash_byte_width` helper (lines 54-59) from this
file. **Keep** `_extract_path_hashes` — the detail route `get_packet_group`
still uses it (`:311`).
- **TR-5** — Frontend (`packets.js`):
- Parse `const path_hash_bytes = query.path_hash_bytes || '';` alongside the
other filter reads (~line 57).
- Extend `hasActiveFilters` (~line 72) with `|| path_hash_bytes !== ''`.
- Add a 4th entry to the `filterFields` array (~line 170): a
`<select name="path_hash_bytes" class="select select-sm" @change=${autoSubmit}>`
with options Any / 1B / 2B / 3B, mirroring the `event_type` select style.
Option labels reuse `packets.path_width_bytes` (`{{count}}B`) and
`common.all` / a new `packets.filter_path_width` label.
- Add `if (path_hash_bytes !== '') apiParams.path_hash_bytes = path_hash_bytes;`
(~line 111).
- Add `path_hash_bytes` to the `pagination(...)` params (~line 166) and
`headerParams` (~line 194) so it survives pagination and sort.
- **TR-6** — i18n (`locales/en.json`, `locales/nl.json`): add
`packets.filter_path_width` ("Path width" / "Padbreedte"). Option labels reuse
existing `packets.path_width_bytes` and `common.all` — no new keys needed for
them.
- **TR-7** — `apiGet` (`api.js`) and `createFilterHandler` (`components.js`)
already skip empty/falsy values, so no changes there.
## Implementation Plan
### Phase 1: Model & Migration
- Add `path_hash_bytes` column to `RawPacket` (TR-1).
- Generate the migration with `--autogenerate` against head `38abdf4651fc`
(TR-3), producing `batch_alter_table` `add_column`.
- Manually append the Python batched backfill loop (TR-3): read
`id, decoded WHERE path_hash_bytes IS NULL` in batches (~1000 rows/batch)
via a Core `select()` on a `sa.Table` with a `sa.JSON`-typed `decoded` column
(so the type adapter deserializes to a `dict` on both SQLite and Postgres —
do **not** use raw `text("SELECT decoded ...")`), compute width, `UPDATE`.
The backfill must include a **self-contained**, frozen copy of the dual-path
extraction + `max(len//2)` logic — no imports of app code that may change in
future versions. Rows where `decoded` is `None` are left as `NULL` (the
column is nullable; `WHERE path_hash_bytes IS NULL` naturally skips
already-filled rows and leaves truly uncomputable ones alone).
- Verify the model test (column presence/default) and that `db upgrade` applies
cleanly on both backends.
### Phase 2: Collector Ingest Compute
- Add the module-level `_path_hash_byte_width(hashes)` helper in
`collector/handlers/raw_packet.py`.
- In `store_raw_packet`, compute `path_hash_bytes` from `decoded_packet` after
`path_len`, using `LetsMeshNormalizer._normalize_hash_list` + the two-tier
lookup (TR-2). Pass it into the `RawPacket(...)` kwargs.
- Add a collector test: `store_raw_packet` persists the expected width for a
decoded packet carrying `decoded.path`, for the trace fallback path, and
`None` when no path is present (mirror the existing `path_len` test).
### Phase 3: API Route Simplification & Filter
- In `packet_groups.py`, add the `func.max(RawPacket.path_hash_bytes)` aggregate
to Phase 1 and the new `path_hash_bytes` query param + `HAVING` filter (TR-4).
- Delete Phase 3 (the Python decode loop); read `grp.path_hash_bytes`. Remove the
now-dead `_path_hash_byte_width` helper. Keep `_extract_path_hashes`.
- Preserve redaction nulling in the item construction (TR-4 / FR-5).
- Update `tests/test_api/test_packet_groups.py::TestPathHashBytes` to set
`path_hash_bytes=` directly on inserted `RawPacket` rows (tests bypass the
collector). Keep 1/2/3/None/redacted coverage.
- Add filter tests: `?path_hash_bytes=2` returns only width-2 groups;
`?path_hash_bytes=1`/`3` likewise; omitted returns all; a redacted group's
response field is still null.
### Phase 4: Frontend Filter & i18n
- Add the discrete `<select>` filter field to `packets.js` (TR-5).
- Wire the query parse, `hasActiveFilters`, `apiParams`, `pagination` params, and
`headerParams` (TR-5).
- Add `packets.filter_path_width` to both locale files (TR-6).
- Rebuild the SPA bundle via the Docker build pipeline (local `node build.js`
fails on a missing fontsource asset — build through compose).
### Phase 5: Verify
- `pytest --no-cov tests/test_api/test_packet_groups.py tests/test_collector/`
- `pre-commit run --all-files`
- Apply the migration on the stack and manually verify the filter on the packet
list (all three known widths; "Any" returns null-width rows too).
## Open Questions
- None outstanding. Metric (byte width), approach (persisted column), filter
shape (discrete select), and unknown-option exclusion (known widths only) are
all resolved.
- **Phase-ordering window** (minor, not blocking): Between migration (Phase 1)
and collector ingester deploy (Phase 2), new RawPacket rows are inserted with
`path_hash_bytes = NULL`. The backfill fills historical rows, but a brief gap
exists for rows ingested during the deploy window. Once Phase 2 is active, new
rows carry the correct value and Phase 3 reads it. In practice the window is
seconds in a single `docker compose up -d` rebuild; any rows missed still render
`?B` (unchanged UX), and the next backfill run would catch them.
- Minor follow-up (not blocking): whether to also surface `path_hash_bytes` on
the raw `/packets` endpoint's `RawPacketRead` schema, now that the column
exists.
## References
- `docs/plans/20260612-2014-raw-packets-feature/plan.md` — the Raw Packets
feature this builds on (model, collector chokepoint, channel-visibility
redaction, `/packets` SPA page).
- `docs/plans/20260426-1137-improve-snr-path-visibility/plan.md` — prior
`path_len` (hop-count) column work and the "no backfill for per-observer
fields" decision (here we *do* backfill, since the value is derivable from the
already-stored `decoded` JSON).
- `src/meshcore_hub/api/routes/packet_groups.py:54-59,198-216,240` — current
runtime `_path_hash_byte_width` + Phase 3 aggregation + redaction nulling
(to be replaced/deleted).
- `src/meshcore_hub/collector/handlers/raw_packet.py:92-96,122-138` — `path_len`
precedent and the single `RawPacket(...)` construction site.
- `src/meshcore_hub/collector/letsmesh_normalizer.py:826-867` —
`_extract_message_path_hashes` + `_normalize_hash_list` (staticmethod) to reuse.
- `src/meshcore_hub/api/routes/raw_packets.py:165-172` — `min_snr`/`max_snr` and
`min_path_len`/`max_path_len` range-filter precedent.
- `alembic/versions/20260622_2243_38abdf4651fc_add_spam_scoring_columns_to_messages.py`
— `batch_alter_table` add-column + index precedent (current migration head).
- `alembic/versions/20260421_0001_normalize_public_key_case.py` — Python data
backfill precedent (`op.get_bind()`, dialect-aware raw SQL).
- Recent commits: `f845830` (JSON tree / path flow / chart polish, PR #292),
`2c25a9c` (locale-aware numbers + filter panel redesign, PR #291) — the
reception-widget + filter-panel work this plan extends.
## Review
**Status**: Approved with Changes
**Reviewed**: 2026-07-03
### Resolutions
- **Factual correction — aggregation fix is committed, not uncommitted**: The
Phase 3 MAX-across-all-receptions fix described as "uncommitted" was committed
in PR #292 (`f845830`). Updated to "recently committed (PR #292)".
- **Gap — self-contained migration backfill**: Clarified that the backfill loop
must include a frozen, inline copy of the dual-path extraction +
`max(len//2)` logic. Rows with `decoded = NULL` are left as NULL (the
`WHERE path_hash_bytes IS NULL` clause naturally skips them).
- **Gap — collector / API helper naming collision**: Noted that the collector's
`_path_hash_byte_width` helper mirrors the API route's (deleted in Phase 3),
so no persistent naming conflict exists.
- **Risk — rows ingested between migration and collector deploy**: Added a brief
note under Open Questions that the Phase 1→2 deploy window can leave
`path_hash_bytes = NULL` for new rows, but the window is seconds in a compose
rebuild, and the UX is unchanged (`?B` label) for any rows missed.
- **Postgres JSON type & backfill read mechanism**: Confirmed the `decoded`
column is SQLAlchemy `JSON` (not `JSONB`). The Python backfill reads decoded
as a Python `dict` via a Core `select()` on a `sa.Table` with a `sa.JSON`-typed
column — this routes through SQLAlchemy's JSON type adapter, which deserializes
consistently across SQLite (TEXT storage) and Postgres. Raw
`text("SELECT decoded ...")` must **not** be used: it bypasses the type adapter,
making deserialization driver-dependent (SQLite returns TEXT; Postgres/psycopg2
may return a parsed object or string). No SQL JSON operators are needed —
consistent with TR-3.
### Remaining Action Items
- **Migration down_revision**: Before generating the migration, confirm that
`38abdf4651fc` is still the head revision (new migrations may land between
plan approval and implementation). The `meshcore-hub db revision` command
auto-resolves this — just verify the `down_revision` value in the generated
file.
- **Phase 2 collector test**: The existing `test_store_raw_packet` test in
`tests/test_collector/` does not appear to cover `path_len` persistence;
verify whether a collector test for the new column can sit alongside or
replace the `path_len` test, and whether the test fixture accepts
`decoded_packet=` with path hashes.
- **Frontend bundle rebuild**: Must be built via Docker compose (local
`node build.js` fails on the fontsource asset). The plan notes this in
Phase 4.
@@ -0,0 +1,112 @@
# Tasks: Persist & Filter by Path-Hash Byte Width
> Generated from `plan.md` on 2026-07-03
## 1. Model & Migration (Phase 1)
- [x] 1.1 Add `path_hash_bytes` column to `RawPacket` model
- [x] 1.1.1 Add `Mapped[Optional[int]] = mapped_column(Integer, nullable=True)` after `path_len` in `src/meshcore_hub/common/models/raw_packet.py`
- [x] 1.1.2 Verify model test reflects new column (presence + default None)
- [x] 1.2 Generate and customize alembic migration
- [x] 1.2.1 Run `meshcore-hub db revision --autogenerate -m "add path_hash_bytes to raw_packets"` against migration head `38abdf4651fc`
- [x] 1.2.2 Verify `down_revision` is `38abdf4651fc` in generated file
- [x] 1.2.3 Append self-contained Python batched backfill loop (~1000 rows/batch):
- Read `decoded` via Core `select()` on a `sa.Table` declared with a `sa.JSON`-typed column (routes through SQLAlchemy's JSON type adapter → `dict` on both SQLite and Postgres); do **not** use raw `text("SELECT decoded ...")` (bypasses the adapter → driver-dependent deserialization)
- Inline frozen dual-path extraction: `decoded["path"]` fallback `decoded.get("payload", {}).get("decoded", {}).get("pathHashes")`
- Normalize with inline `_normalize_hash_list` logic (split on `","`, strip)
- Compute `max(len(h) // 2 for h in hashes)` for each row
- `UPDATE raw_packets SET path_hash_bytes = ? WHERE id = ?`
- Skip NULL decoded rows (leave NULL)
- No imports of app code
- [x] 1.2.4 Verify downgrade drops the column
- [x] 1.2.5 Test migration with `meshcore-hub db upgrade` on both SQLite and Postgres backends
## 2. Collector Ingest (Phase 2)
- [x] 2.1 Add `_path_hash_byte_width` helper
- [x] 2.1.1 Add module-level `_path_hash_byte_width(hashes: list[str] | None) -> int | None` in `src/meshcore_hub/collector/handlers/raw_packet.py`
- [x] 2.1.2 Implement as `max(len(h) // 2 for h in hashes) if hashes else None`
- [x] 2.2 Compute `path_hash_bytes` at ingest
- [x] 2.2.1 In `store_raw_packet`, after `path_len` block (~line 96), extract path hashes from `decoded_packet`:
- `LetsMeshNormalizer._normalize_hash_list(decoded_packet.get("path"))`
- Fallback: `_normalize_hash_list(decoded_packet.get("payload", {}).get("decoded", {}).get("pathHashes"))`
- [x] 2.2.2 Compute `path_hash_bytes = _path_hash_byte_width(hashes)` (None-safe)
- [x] 2.2.3 Pass `path_hash_bytes=path_hash_bytes` into `RawPacket(...)` kwargs (~line 122)
- [x] 2.2.4 Verify null-safe: no path / no decoded → `None`
- [x] 2.3 Add collector unit test
- [x] 2.3.1 Create test in `tests/test_collector/` (or extend existing `test_store_raw_packet`) covering:
- `decoded.path` with hashes of mixed widths → correct MAX byte-width persisted
- Trace fallback `decoded.payload.decoded.pathHashes` → correct width
- No path present → `None`
- Verify `path_len` persistence test still passes alongside
## 3. API Route (Phase 3)
- [x] 3.1 Add SQL aggregate to group query
- [x] 3.1.1 In Phase 1 `group_query` select (`src/meshcore_hub/api/routes/packet_groups.py`), add:
`func.max(RawPacket.path_hash_bytes).label("path_hash_bytes")`
- [x] 3.1.2 Add `path_hash_bytes: Optional[int] = Query(None, ge=1, le=3)` parameter to route function
- [x] 3.1.3 Apply `HAVING` filter: `if path_hash_bytes is not None: group_query = group_query.having(func.max(RawPacket.path_hash_bytes) == path_hash_bytes)`
- [x] 3.1.4 Verify HAVING works with existing WHERE clauses and `func.count(RawPacket.id)`
- [x] 3.2 Delete Phase 3 Python decode loop
- [x] 3.2.1 Remove the `decoded_rows` fetch + `width_by_hash` loop (lines 198-216)
- [x] 3.2.2 Replace with direct read: `path_hash_bytes=(None if is_redacted else grp.path_hash_bytes)`
- [x] 3.2.3 Remove dead `_path_hash_byte_width` helper (lines 54-59)
- [x] 3.2.4 Confirm `_extract_path_hashes` (line 36) is kept — still used by detail route at line 311
- [x] 3.3 Update existing tests
- [x] 3.3.1 In `tests/test_api/test_packet_groups.py::TestPathHashBytes`, update insert fixtures to set `path_hash_bytes=` explicitly on `RawPacket` rows (tests bypass collector)
- [x] 3.3.2 Keep coverage for 1/2/3/None/redacted paths
- [x] 3.3.3 Verify all existing tests pass without Phase 3
- [x] 3.4 Add filter tests
- [x] 3.4.1 `?path_hash_bytes=2` returns only width-2 groups
- [x] 3.4.2 `?path_hash_bytes=1` and `3` return correct subsets
- [x] 3.4.3 Missing parameter returns all groups (including null-width)
- [x] 3.4.4 Redacted group response field is still null despite persisted value
- [x] 3.4.5 Verify `total` count is correct when filtered vs unfiltered
## 4. Frontend & i18n (Phase 4)
- [x] 4.1 Add filter `<select>` to packets.js
- [x] 4.1.1 Parse `const path_hash_bytes = query.path_hash_bytes || '';` (~line 57)
- [x] 4.1.2 Extend `hasActiveFilters` with `|| path_hash_bytes !== ''` (~line 72)
- [x] 4.1.3 Add to `apiParams`: `if (path_hash_bytes !== '') apiParams.path_hash_bytes = path_hash_bytes;` (~line 111)
- [x] 4.1.4 Add `path_hash_bytes` to `pagination(...)` params (~line 166)
- [x] 4.1.5 Add `path_hash_bytes` to `headerParams` (~line 194)
- [x] 4.1.6 Add 4th entry to `filterFields` array (~line 170):
`<select name="path_hash_bytes" class="select select-sm" @change=${autoSubmit}>`
with options: Any (empty value) / 1B / 2B / 3B, mirroring `event_type` select style
- [x] 4.1.7 Option labels use `packets.path_width_bytes` (`{{count}}B`) and `common.all`
- [x] 4.2 Add i18n keys
- [x] 4.2.1 Add `"packets.filter_path_width"` to `src/meshcore_hub/web/static/locales/en.json` ("Path width")
- [x] 4.2.2 Add `"packets.filter_path_width"` to `src/meshcore_hub/web/static/locales/nl.json` ("Padbreedte")
- [x] 4.3 Rebuild SPA bundle
- [x] 4.3.1 Build via Docker compose pipeline (not local `node build.js` — fontsource asset requires compose context)
## 5. Verification (Phase 5)
- [x] 5.1 Run targeted tests
- [x] 5.1.1 `pytest --no-cov tests/test_api/test_packet_groups.py`
- [x] 5.1.2 `pytest --no-cov tests/test_collector/`
- [x] 5.1.3 `pytest --no-cov tests/test_common/` (model column test)
- [x] 5.2 Run full test suite
- [x] 5.2.1 `pytest --no-cov 2>&1 | grep -iE "passed|failed" | tail -3`
- [x] 5.3 Run pre-commit
- [x] 5.3.1 `pre-commit run --all-files`
- [x] 5.4 Manual smoke test on Docker stack
- [x] 5.4.1 Apply migration: `docker compose run --rm migrate db upgrade`
- [x] 5.4.2 Rebuild and restart stack
- [x] 5.4.3 Verify packet list loads with all three width filters (1B/2B/3B)
- [x] 5.4.4 Verify "Any" filter returns all rows including null-width
- [x] 5.4.5 Verify filter survives pagination and sort changes via URL query string
- [x] 5.4.6 Verify redacted groups show `?B` on detail page
+10 -23
View File
@@ -51,14 +51,6 @@ def _extract_path_hashes(decoded: dict[str, Any] | None) -> list[str] | None:
return hashes if isinstance(hashes, list) else None
def _path_hash_byte_width(path_hashes: Optional[list[str]]) -> Optional[int]:
"""Widest path-hash prefix width in bytes (each hash is hex: 2/4/6 chars)."""
if not path_hashes:
return None
widths = [len(h) // 2 for h in path_hashes if isinstance(h, str) and h]
return max(widths) if widths else None
def _get_tag_name(node: Optional[Node]) -> Optional[str]:
if not node or not node.tags:
return None
@@ -79,6 +71,9 @@ def list_packet_groups(
),
event_type: Optional[str] = Query(None, description="Filter by event type"),
channel_idx: Optional[int] = Query(None, description="Filter by channel index"),
path_hash_bytes: Optional[int] = Query(
None, ge=1, le=3, description="Filter by path-hash byte width (1/2/3)"
),
since: Optional[datetime] = Query(None, description="Start timestamp"),
until: Optional[datetime] = Query(None, description="End timestamp"),
sort: Optional[str] = Query(
@@ -120,6 +115,7 @@ def list_packet_groups(
func.count(RawPacket.id).label("reception_count"),
func.count(RawPacket.observer_node_id.distinct()).label("observer_count"),
func.min(RawPacket.received_at).label("first_seen"),
func.max(RawPacket.path_hash_bytes).label("path_hash_bytes"),
)
.outerjoin(ObserverNode, RawPacket.observer_node_id == ObserverNode.id)
.where(RawPacket.packet_hash.is_not(None))
@@ -146,6 +142,11 @@ def list_packet_groups(
group_query = group_query.group_by(RawPacket.packet_hash)
if path_hash_bytes is not None:
group_query = group_query.having(
func.max(RawPacket.path_hash_bytes) == path_hash_bytes
)
count_query = select(func.count()).select_from(group_query.subquery())
total = session.execute(count_query).scalar() or 0
@@ -195,20 +196,6 @@ def list_packet_groups(
group_counts = {r.packet_hash: r for r in group_rows}
# ── Phase 3: path-hash width — load decoded only for representative rows ───
rep_id_to_hash = {rep.id: h for h, rep in representative.items()}
width_by_hash: dict[str, Optional[int]] = {}
if rep_id_to_hash:
decoded_rows = session.execute(
select(RawPacket.id, RawPacket.decoded).where(
RawPacket.id.in_(rep_id_to_hash.keys())
)
).all()
for rid, decoded in decoded_rows:
h = rep_id_to_hash.get(rid)
if h is not None:
width_by_hash[h] = _path_hash_byte_width(_extract_path_hashes(decoded))
items = []
for h in hashes:
rep = representative.get(h)
@@ -231,7 +218,7 @@ def list_packet_groups(
),
reception_count=grp.reception_count,
observer_count=grp.observer_count,
path_hash_bytes=(None if is_redacted else width_by_hash.get(h)),
path_hash_bytes=(None if is_redacted else grp.path_hash_bytes),
receptions=[],
first_seen=grp.first_seen,
redacted=is_redacted,
@@ -42,6 +42,14 @@ def _extract_source_pubkey_prefix(
return None
def _path_hash_byte_width(hashes: list[str] | None) -> int | None:
"""Widest path-hash prefix width in bytes (each hash is hex: 2/4/6 chars)."""
if not hashes:
return None
widths = [len(h) // 2 for h in hashes if isinstance(h, str) and h]
return max(widths) if widths else None
def store_raw_packet(
public_key: str,
payload: dict[str, Any],
@@ -95,6 +103,14 @@ def store_raw_packet(
decoded_packet
)
path_hashes = LetsMeshNormalizer._normalize_hash_list(
decoded_packet.get("path") if isinstance(decoded_packet, dict) else None
)
if not path_hashes and isinstance(decoded_packet, dict):
inner = (decoded_packet.get("payload") or {}).get("decoded") or {}
path_hashes = LetsMeshNormalizer._normalize_hash_list(inner.get("pathHashes"))
path_hash_bytes = _path_hash_byte_width(path_hashes)
snr = LetsMeshNormalizer._parse_float(payload.get("SNR"))
if snr is None:
snr = LetsMeshNormalizer._parse_float(payload.get("snr"))
@@ -131,6 +147,7 @@ def store_raw_packet(
source_pubkey_prefix=source_pubkey_prefix,
route_type=route_type,
path_len=path_len,
path_hash_bytes=path_hash_bytes,
snr=snr,
decoded=decoded_packet,
received_at=now,
@@ -32,6 +32,7 @@ class RawPacket(Base, UUIDMixin, TimestampMixin):
senderPublicKey, for efficient "packets from this node" filtering
route_type: Mapped route type (flood, transport_flood, direct, ...)
path_len: Hop count
path_hash_bytes: Widest path-hash prefix width in bytes (1/2/3)
snr: Signal-to-noise ratio as reported by the observer
decoded: Decoder summary JSON, so the detail view needs no re-decode
received_at: When received by the observing interface
@@ -86,6 +87,10 @@ class RawPacket(Base, UUIDMixin, TimestampMixin):
Integer,
nullable=True,
)
path_hash_bytes: Mapped[Optional[int]] = mapped_column(
Integer,
nullable=True,
)
snr: Mapped[Optional[float]] = mapped_column(
Float,
nullable=True,
@@ -170,6 +170,10 @@ export function iconTxPower(cls = 'h-5 w-5') {
return html`<svg xmlns="http://www.w3.org/2000/svg" class=${cls} fill="none" viewBox="0 0 24 24" stroke="currentColor"><path stroke-linecap="round" stroke-linejoin="round" stroke-width="2" d="M3.75 13.5l10.5-11.25L12 10.5h8.25L9.75 21.75 12 13.5H3.75z" /></svg>`;
}
export function iconRuler(cls = 'h-5 w-5') {
return html`<svg xmlns="http://www.w3.org/2000/svg" class=${cls} fill="none" viewBox="0 0 24 24" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><path d="M21.3 8.7 8.7 21.3a1 1 0 0 1-1.4 0L2.7 16.7a1 1 0 0 1 0-1.4L15.3 2.7a1 1 0 0 1 1.4 0l4.6 4.6a1 1 0 0 1 0 1.4Z" /><path d="m7.5 10.5 2 2" /><path d="m10.5 7.5 2 2" /><path d="m13.5 4.5 2 2" /><path d="m4.5 13.5 2 2" /></svg>`;
}
export function iconFilter(cls = 'h-5 w-5') {
return html`<svg xmlns="http://www.w3.org/2000/svg" class=${cls} fill="none" viewBox="0 0 24 24" stroke="currentColor"><path stroke-linecap="round" stroke-linejoin="round" stroke-width="2" d="M12 3c2.755 0 5.455.232 8.083.678.533.09.917.556.917 1.096v1.044a2.25 2.25 0 01-.659 1.591l-5.432 5.432a2.25 2.25 0 00-.659 1.591v2.927a2.25 2.25 0 01-1.244 2.013L9.75 21v-6.568a2.25 2.25 0 00-.659-1.591L3.659 7.409A2.25 2.25 0 013 5.818V4.774c0-.54.384-1.006.917-1.096A48.32 48.32 0 0112 3z" /></svg>`;
}
@@ -7,7 +7,7 @@ import {
renderFilterForm, renderFilterToggle, autoSubmit, submitOnEnter
} from '../components.js';
import { createAutoRefresh } from '../auto-refresh.js';
import { iconSatelliteDish, iconPath } from '../icons.js';
import { iconSatelliteDish, iconPath, iconRuler } from '../icons.js';
const EVENT_TYPES = [
'advertisement', 'channel_msg_recv', 'contact_msg_recv',
@@ -34,12 +34,19 @@ function receptionBadge(packet) {
const rc = packet.reception_count ?? 1;
const oc = packet.observer_count ?? 1;
const pb = packet.path_hash_bytes;
const knownWidth = pb != null && pb > 0;
const widthLabel = knownWidth
? t('packets.path_width_bytes', { count: pb })
: t('packets.path_width_unknown');
return html`<span class="inline-flex items-center gap-1">
${iconSatelliteDish('h-4 w-4 opacity-70')}
<span class="badge badge-sm badge-primary" title=${t('common.observers')}>${formatNumber(oc)}</span>
<span class="opacity-40" aria-hidden="true">×</span>
${iconPath('h-4 w-4 opacity-70')}
<span class="badge badge-sm badge-primary" title=${t('packets.reception_plural')}>${formatNumber(rc)}</span>
${pb ? html`<span class="badge badge-sm badge-ghost" title=${t('packets.path_width_title')}>${t('packets.path_width_bytes', { count: pb })}</span>` : nothing}
<span class="opacity-40" aria-hidden="true">@</span>
${iconRuler('h-4 w-4 opacity-70')}
<span class="badge badge-sm badge-primary ${knownWidth ? '' : 'opacity-60'}" title=${t('packets.path_width_title')}>${widthLabel}</span>
</span>`;
}
@@ -49,6 +56,7 @@ export async function render(container, params, router) {
const search = query.search || '';
const event_type = query.event_type || '';
const channel_idx = query.channel_idx || '';
const path_hash_bytes = query.path_hash_bytes || '';
const page = parseInt(query.page, 10) || 1;
const limit = parseInt(query.limit, 10) || 20;
const offset = (page - 1) * limit;
@@ -63,7 +71,7 @@ export async function render(container, params, router) {
let lastContent = nothing;
let lastTotal = null;
let currentFilterFields = [];
const hasActiveFilters = search !== '' || event_type !== '' || channel_idx !== '';
const hasActiveFilters = search !== '' || event_type !== '' || channel_idx !== '' || path_hash_bytes !== '';
function onFilterToggle() { renderPage(lastContent, { total: lastTotal }); }
@@ -104,6 +112,7 @@ ${displayContent}`, container);
const apiParams = { limit, offset, search, sort, order };
if (event_type) apiParams.event_type = event_type;
if (channel_idx !== '') apiParams.channel_idx = channel_idx;
if (path_hash_bytes !== '') apiParams.path_hash_bytes = path_hash_bytes;
const [data, channelsData] = await Promise.all([
apiGet('/api/v1/packet-groups', apiParams, { signal }),
@@ -157,7 +166,7 @@ ${displayContent}`, container);
</tr>`);
const paginationBlock = pagination(page, totalPages, '/packets', {
search, event_type, channel_idx, limit, sort, order,
search, event_type, channel_idx, path_hash_bytes, limit, sort, order,
});
const filterFields = [
@@ -181,10 +190,18 @@ ${displayContent}`, container);
<option value="" ?selected=${channel_idx === ''}>${t('common.all_channels')}</option>
${channelList.map(c => html`<option value=${c.idx} ?selected=${String(channel_idx) === String(c.idx)}>${c.name} (${c.idx})</option>`)}
</select>
</div>`,
() => html`
<div class="flex flex-col gap-1 max-w-48">
<label class="flex items-center py-1"><span class="opacity-80 text-sm">${t('packets.filter_path_width')}</span></label>
<select name="path_hash_bytes" class="select select-sm" @change=${autoSubmit}>
<option value="" ?selected=${path_hash_bytes === ''}>${t('common.all')}</option>
${[1, 2, 3].map(w => html`<option value=${w} ?selected=${path_hash_bytes === String(w)}>${t('packets.path_width_bytes', { count: w })}</option>`)}
</select>
</div>`,
];
const headerParams = { search, event_type, channel_idx, limit };
const headerParams = { search, event_type, channel_idx, path_hash_bytes, limit };
const sortable = (label, sortKey) => sortableTableHeader(label, {
sortKey, currentSort: sort, currentOrder: order,
navigate, basePath: '/packets', params: headerParams,
@@ -82,6 +82,7 @@
"sign_out": "Sign Out",
"view_details": "View Details",
"all_types": "All Types",
"all": "Any",
"all_channels": "All Channels",
"all_members": "All Members",
"all_operators": "All Operators",
@@ -218,6 +219,7 @@
},
"packets": {
"filter_event_type": "Event Type",
"filter_path_width": "Path Width",
"filter_min_snr": "Min SNR",
"filter_max_snr": "Max SNR",
"redacted": "Restricted",
@@ -248,6 +250,7 @@
"path_nodes_more": "+{{count}} more — view all →",
"not_found_retention": "Packet not found — it may have been cleaned up due to data retention.",
"path_width_bytes": "{{count}}B",
"path_width_unknown": "?B",
"path_width_title": "Path hash width (bytes)",
"sort": {
"newest": "Time (newest)",
@@ -80,6 +80,7 @@
"sign_out": "Uitloggen",
"view_details": "Details bekijken",
"all_types": "Alle types",
"all": "Alle",
"node_type": "Knooppunttype",
"show": "Toon",
"search_placeholder": "Zoek op naam, ID of publieke sleutel...",
@@ -160,6 +161,7 @@
},
"packets": {
"filter_event_type": "Gebeurtenistype",
"filter_path_width": "Padbreedte",
"filter_min_snr": "Min. SNR",
"filter_max_snr": "Max. SNR",
"redacted": "Beperkt",
@@ -184,6 +186,7 @@
"path_nodes_more": "+{{count}} meer — alle bekijken →",
"not_found_retention": "Pakket niet gevonden — mogelijk opgeschoond door dataretentie.",
"path_width_bytes": "{{count}}B",
"path_width_unknown": "?B",
"path_width_title": "Breedte pad-hash (bytes)",
"sort": {
"newest": "Tijd (nieuwste)",
+170 -6
View File
@@ -300,14 +300,14 @@ class TestListPacketGroups:
class TestPathHashBytes:
"""Tests for the derived path_hash_bytes field on the list endpoint."""
"""Tests for the persisted path_hash_bytes field on the list endpoint."""
def test_one_byte_path(self, client_no_auth, api_db_session):
api_db_session.add(
RawPacket(
raw_hex="AA",
packet_hash="H1",
decoded={"path": ["aa", "bb"]},
path_hash_bytes=1,
received_at=_now(),
)
)
@@ -320,7 +320,7 @@ class TestPathHashBytes:
RawPacket(
raw_hex="AA",
packet_hash="H1",
decoded={"path": ["aabb"]},
path_hash_bytes=2,
received_at=_now(),
)
)
@@ -328,17 +328,45 @@ class TestPathHashBytes:
item = client_no_auth.get("/api/v1/packet-groups").json()["items"][0]
assert item["path_hash_bytes"] == 2
def test_mixed_path_uses_max(self, client_no_auth, api_db_session):
def test_three_byte_path(self, client_no_auth, api_db_session):
api_db_session.add(
RawPacket(
raw_hex="AA",
packet_hash="H1",
decoded={"path": ["aa", "aabb"]},
path_hash_bytes=3,
received_at=_now(),
)
)
api_db_session.commit()
item = client_no_auth.get("/api/v1/packet-groups").json()["items"][0]
assert item["path_hash_bytes"] == 3
def test_width_aggregates_across_receptions(self, client_no_auth, api_db_session):
"""Width is observer-relative; take the widest across all receptions.
The oldest reception (representative) carries no path, while a newer
reception of the same packet saw a 2-byte path. The badge must reflect
the widest path seen, not just the representative row.
"""
base = _now()
api_db_session.add(
RawPacket(
raw_hex="AA",
packet_hash="H1",
path_hash_bytes=None,
received_at=base,
)
)
api_db_session.add(
RawPacket(
raw_hex="BB",
packet_hash="H1",
path_hash_bytes=2,
received_at=base.replace(microsecond=base.microsecond + 1),
)
)
api_db_session.commit()
item = client_no_auth.get("/api/v1/packet-groups").json()["items"][0]
assert item["path_hash_bytes"] == 2
def test_no_path_returns_none(self, client_no_auth, api_db_session):
@@ -366,7 +394,7 @@ class TestPathHashBytes:
raw_hex="SECRET",
packet_hash="ADM_HASH",
channel_idx=adm_idx,
decoded={"path": ["aabb"]},
path_hash_bytes=2,
received_at=_now(),
)
)
@@ -376,6 +404,142 @@ class TestPathHashBytes:
assert item["path_hash_bytes"] is None
class TestPathHashBytesFilter:
"""Tests for the ?path_hash_bytes= query filter."""
def test_filter_returns_only_matching_width(self, client_no_auth, api_db_session):
now = _now()
api_db_session.add_all(
[
RawPacket(
raw_hex="A", packet_hash="H1", path_hash_bytes=1, received_at=now
),
RawPacket(
raw_hex="B", packet_hash="H2", path_hash_bytes=2, received_at=now
),
RawPacket(
raw_hex="C", packet_hash="H3", path_hash_bytes=3, received_at=now
),
]
)
api_db_session.commit()
data = client_no_auth.get("/api/v1/packet-groups?path_hash_bytes=2").json()
assert data["total"] == 1
assert data["items"][0]["packet_hash"] == "H2"
def test_filter_width_1(self, client_no_auth, api_db_session):
now = _now()
api_db_session.add_all(
[
RawPacket(
raw_hex="A", packet_hash="H1", path_hash_bytes=1, received_at=now
),
RawPacket(
raw_hex="B", packet_hash="H2", path_hash_bytes=3, received_at=now
),
]
)
api_db_session.commit()
data = client_no_auth.get("/api/v1/packet-groups?path_hash_bytes=1").json()
assert data["total"] == 1
assert data["items"][0]["packet_hash"] == "H1"
def test_filter_width_3(self, client_no_auth, api_db_session):
now = _now()
api_db_session.add_all(
[
RawPacket(
raw_hex="A", packet_hash="H1", path_hash_bytes=1, received_at=now
),
RawPacket(
raw_hex="B", packet_hash="H2", path_hash_bytes=3, received_at=now
),
]
)
api_db_session.commit()
data = client_no_auth.get("/api/v1/packet-groups?path_hash_bytes=3").json()
assert data["total"] == 1
assert data["items"][0]["packet_hash"] == "H2"
def test_no_filter_returns_all_including_null(self, client_no_auth, api_db_session):
now = _now()
api_db_session.add_all(
[
RawPacket(
raw_hex="A", packet_hash="H1", path_hash_bytes=1, received_at=now
),
RawPacket(
raw_hex="B", packet_hash="H2", path_hash_bytes=None, received_at=now
),
]
)
api_db_session.commit()
data = client_no_auth.get("/api/v1/packet-groups").json()
assert data["total"] == 2
widths = {i["path_hash_bytes"] for i in data["items"]}
assert widths == {1, None}
def test_filter_excludes_null_width(self, client_no_auth, api_db_session):
now = _now()
api_db_session.add_all(
[
RawPacket(
raw_hex="A", packet_hash="H1", path_hash_bytes=None, received_at=now
),
RawPacket(
raw_hex="B", packet_hash="H2", path_hash_bytes=2, received_at=now
),
]
)
api_db_session.commit()
data = client_no_auth.get("/api/v1/packet-groups?path_hash_bytes=2").json()
assert data["total"] == 1
assert data["items"][0]["path_hash_bytes"] == 2
def test_filter_redacted_group_still_null(self, client_no_auth, api_db_session):
adm_key = "FFEEDDCCBBAA99887766554433221100"
adm_idx = int(Channel.compute_channel_hash(adm_key), 16)
now = _now()
api_db_session.add(
Channel(
name="Adm",
key_hex=adm_key,
channel_hash=Channel.compute_channel_hash(adm_key),
visibility="admin",
enabled=True,
)
)
api_db_session.add_all(
[
RawPacket(
raw_hex="SECRET",
packet_hash="ADM",
channel_idx=adm_idx,
path_hash_bytes=2,
received_at=now,
),
RawPacket(
raw_hex="OK",
packet_hash="PUB",
path_hash_bytes=2,
received_at=now,
),
]
)
api_db_session.commit()
data = client_no_auth.get("/api/v1/packet-groups?path_hash_bytes=2").json()
assert data["total"] == 2
adm = next(i for i in data["items"] if i["packet_hash"] == "ADM")
assert adm["redacted"] is True
assert adm["path_hash_bytes"] is None
class TestGetPacketGroup:
"""Tests for GET /packet-groups/{hash} (detail)."""
@@ -144,3 +144,84 @@ class TestStoreRawPacketEdges:
rp = db_session.execute(select(RawPacket)).scalar_one()
assert rp.source_pubkey_prefix is None
class TestStoreRawPacketPathHashBytes:
"""Tests for path_hash_bytes computation at ingest."""
def test_top_level_path_max_byte_width(self, db_manager, db_session):
"""Mixed-width hashes in decoded.path persist the widest byte width."""
decoded = {
"payloadType": 1,
"path": ["aa", "aabbcc"],
"pathLength": 2,
"payload": {"decoded": {"sourceHash": "AABBCCDD"}},
}
store_raw_packet(
"a" * 64, {"raw": "00", "hash": "h1"}, decoded, "flood", db_manager
)
rp = db_session.execute(select(RawPacket)).scalar_one()
assert rp.path_hash_bytes == 3
def test_trace_fallback_path_hashes(self, db_manager, db_session):
"""Trace-style pathHashes in payload.decoded are used as fallback."""
decoded = {
"payloadType": 1,
"payload": {"decoded": {"pathHashes": ["aabb", "ccdd"]}},
}
store_raw_packet(
"a" * 64, {"raw": "00", "hash": "h1"}, decoded, "trace", db_manager
)
rp = db_session.execute(select(RawPacket)).scalar_one()
assert rp.path_hash_bytes == 2
def test_one_byte_path(self, db_manager, db_session):
"""Single-byte hashes persist width 1."""
decoded = {
"payloadType": 1,
"path": ["aa", "bb"],
"payload": {"decoded": {}},
}
store_raw_packet(
"a" * 64, {"raw": "00", "hash": "h1"}, decoded, "flood", db_manager
)
rp = db_session.execute(select(RawPacket)).scalar_one()
assert rp.path_hash_bytes == 1
def test_no_path_returns_none(self, db_manager, db_session):
"""No path hashes at all persist path_hash_bytes = None."""
decoded = {"payloadType": 3, "payload": {"decoded": {}}}
store_raw_packet(
"a" * 64, {"raw": "00", "hash": "h1"}, decoded, "ack", db_manager
)
rp = db_session.execute(select(RawPacket)).scalar_one()
assert rp.path_hash_bytes is None
def test_null_decoded_returns_none(self, db_manager, db_session):
"""A None decoded_packet leaves path_hash_bytes as None."""
store_raw_packet(
"a" * 64, {"raw": "00", "hash": "h1"}, None, "unknown", db_manager
)
rp = db_session.execute(select(RawPacket)).scalar_one()
assert rp.path_hash_bytes is None
def test_path_len_still_persisted(self, db_manager, db_session):
"""path_len is still persisted alongside path_hash_bytes."""
decoded = {
"payloadType": 1,
"path": ["aa", "bb", "cc"],
"pathLength": 3,
"payload": {"decoded": {"sourceHash": "AABBCCDD"}},
}
store_raw_packet(
"a" * 64, {"raw": "00", "hash": "h1"}, decoded, "flood", db_manager
)
rp = db_session.execute(select(RawPacket)).scalar_one()
assert rp.path_len == 3
assert rp.path_hash_bytes == 1
+1
View File
@@ -109,6 +109,7 @@ class TestRawPacketModel:
assert packet.raw_hex is None
assert packet.channel_idx is None
assert packet.source_pubkey_prefix is None
assert packet.path_hash_bytes is None
assert packet.decoded is None
def test_raw_packet_indexes(self) -> None: