mirror of
https://github.com/ipnet-mesh/meshcore-hub.git
synced 2026-08-02 15:03:05 +02:00
feat: persist path_hash_bytes column and add packet list filter
Persist the path-hash byte width (1/2/3) as a nullable Integer column on RawPacket, computed at ingest by the collector and backfilled for historical rows via a self-contained Python migration. Replace the per-request Python decode loop in the grouped-list route with a SQL MAX() aggregate + HAVING filter, and add a discrete <select> filter (Any/1B/2B/3B) to the /packets SPA page. - Model: add path_hash_bytes column to RawPacket (nullable Integer) - Migration: batch_alter_table add_column + keyset-paginated Python backfill reading decoded via Core select() on sa.JSON column (portable across SQLite and Postgres) - Collector: compute path_hash_bytes at ingest via two-tier path-hash extraction (decoded.path -> payload.decoded.pathHashes) - API: add func.max() aggregate to group query, HAVING filter on ?path_hash_bytes=1|2|3 param; delete Phase 3 decode loop and dead helper - Frontend: add path-width select filter wired through query/apiParams/ pagination/headerParams; add i18n keys (en/nl) - Tests: 1186 passed, 22 skipped; collector + API + model coverage
This commit is contained in:
@@ -0,0 +1,117 @@
|
||||
"""add path_hash_bytes to raw_packets
|
||||
|
||||
Revision ID: 57bb65130b97
|
||||
Revises: 38abdf4651fc
|
||||
Create Date: 2026-07-03 22:50:33.598167+00:00
|
||||
|
||||
Adds a nullable ``path_hash_bytes`` column to ``raw_packets`` (widest
|
||||
path-hash prefix width: 1/2/3, or NULL when no path hashes are present).
|
||||
The collector computes this at ingest going forward; this migration
|
||||
backfills historical rows from their stored ``decoded`` JSON using a
|
||||
self-contained, frozen copy of the dual-path extraction logic.
|
||||
|
||||
"""
|
||||
|
||||
from typing import Any, Sequence, Union
|
||||
|
||||
from alembic import op
|
||||
import sqlalchemy as sa
|
||||
|
||||
# revision identifiers, used by Alembic.
|
||||
revision: str = "57bb65130b97"
|
||||
down_revision: Union[str, None] = "38abdf4651fc"
|
||||
branch_labels: Union[str, Sequence[str], None] = None
|
||||
depends_on: Union[str, Sequence[str], None] = None
|
||||
|
||||
_BATCH_SIZE = 1000
|
||||
|
||||
# Declared with a sa.JSON-typed column so the SQLAlchemy type adapter
|
||||
# deserializes ``decoded`` to a Python dict consistently on both SQLite
|
||||
# (TEXT storage) and Postgres. Using raw text("SELECT decoded ...") would
|
||||
# bypass the adapter and make deserialization driver-dependent.
|
||||
_raw_packets = sa.table(
|
||||
"raw_packets",
|
||||
sa.Column("id", sa.String, primary_key=True),
|
||||
sa.Column("decoded", sa.JSON),
|
||||
sa.Column("path_hash_bytes", sa.Integer),
|
||||
)
|
||||
|
||||
|
||||
def _normalize_hash_list(value: Any) -> list[str] | None:
|
||||
"""Frozen copy of LetsMeshNormalizer._normalize_hash_list.
|
||||
|
||||
Accepts even-length hex strings of 2 or more characters.
|
||||
Each string is uppercased and validated as hexadecimal.
|
||||
"""
|
||||
if not isinstance(value, list):
|
||||
return None
|
||||
normalized: list[str] = []
|
||||
for item in value:
|
||||
if not isinstance(item, str):
|
||||
continue
|
||||
token = item.strip().upper()
|
||||
if len(token) < 2 or len(token) % 2 != 0:
|
||||
continue
|
||||
if any(ch not in "0123456789ABCDEF" for ch in token):
|
||||
continue
|
||||
normalized.append(token)
|
||||
return normalized or None
|
||||
|
||||
|
||||
def _compute_path_hash_byte_width(decoded: Any) -> int | None:
|
||||
"""Frozen copy of the dual-path extraction + max(len//2) logic.
|
||||
|
||||
Path hashes live at ``decoded.path`` for normal packets, with
|
||||
``decoded.payload.decoded.pathHashes`` as the trace-style fallback.
|
||||
Returns the widest prefix width in bytes, or None when no path hashes
|
||||
are present/decodable.
|
||||
"""
|
||||
if not isinstance(decoded, dict):
|
||||
return None
|
||||
hashes = _normalize_hash_list(decoded.get("path"))
|
||||
if not hashes:
|
||||
payload = decoded.get("payload") or {}
|
||||
inner = payload.get("decoded") or {}
|
||||
hashes = _normalize_hash_list(inner.get("pathHashes"))
|
||||
if not hashes:
|
||||
return None
|
||||
return max(len(h) // 2 for h in hashes)
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
with op.batch_alter_table("raw_packets", schema=None) as batch_op:
|
||||
batch_op.add_column(sa.Column("path_hash_bytes", sa.Integer(), nullable=True))
|
||||
|
||||
conn = op.get_bind()
|
||||
|
||||
# Keyset-paginated backfill over all rows so rows whose ``decoded`` is
|
||||
# NULL (uncomputable) are visited once and left NULL without causing an
|
||||
# infinite loop.
|
||||
last_id: str | None = None
|
||||
while True:
|
||||
query = (
|
||||
sa.select(_raw_packets.c.id, _raw_packets.c.decoded)
|
||||
.order_by(_raw_packets.c.id)
|
||||
.limit(_BATCH_SIZE)
|
||||
)
|
||||
if last_id is not None:
|
||||
query = query.where(_raw_packets.c.id > last_id)
|
||||
|
||||
batch = conn.execute(query).all()
|
||||
if not batch:
|
||||
break
|
||||
|
||||
for row in batch:
|
||||
last_id = row.id
|
||||
width = _compute_path_hash_byte_width(row.decoded)
|
||||
if width is not None:
|
||||
conn.execute(
|
||||
sa.update(_raw_packets)
|
||||
.where(_raw_packets.c.id == row.id)
|
||||
.values(path_hash_bytes=width)
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
with op.batch_alter_table("raw_packets", schema=None) as batch_op:
|
||||
batch_op.drop_column("path_hash_bytes")
|
||||
@@ -0,0 +1,330 @@
|
||||
# Persist & Filter by Path-Hash Byte Width
|
||||
|
||||
## Summary
|
||||
|
||||
The packet list page (`/packets`, served by `GET /api/v1/packet-groups`) shows a
|
||||
"path-hash byte width" metric per row — the ruler badge (`1B`/`2B`/`3B`/`?B`) we
|
||||
recently added to the reception widget. That value is currently **derived at
|
||||
request time** in Python by decoding each group's `decoded` JSON, which makes it
|
||||
unfilterable at the database level and forces an expensive extra query on every
|
||||
list load.
|
||||
|
||||
This plan **persists `path_hash_bytes` as a real column on `RawPacket`**, computed
|
||||
once at ingest by the collector and backfilled for historical rows. With the value
|
||||
persisted, the grouped-list route's Python decode loop collapses into a SQL
|
||||
`MAX()` aggregate, and we add a clean **discrete-select filter** (Any / 1B / 2B /
|
||||
3B) to the packet list page via a `HAVING` clause — fully consistent pagination and
|
||||
counts, with no per-request JSON decoding.
|
||||
|
||||
## Background & Motivation
|
||||
|
||||
### Current State
|
||||
|
||||
- `path_hash_bytes` is **not a column**. It is computed at runtime in
|
||||
`api/routes/packet_groups.py:198-216` ("Phase 3"), which fetches the `decoded`
|
||||
JSON for every reception of the current page's packet hashes, extracts the path
|
||||
arrays, and aggregates the widest byte-width (1/2/3 from 2/4/6 hex chars). The
|
||||
aggregation was recently fixed (PR #292, commit `f845830`) to take `MAX`
|
||||
across **all** receptions because the decoded `path` is observer-relative — a
|
||||
single representative row often misses a multibyte path that other receptions
|
||||
carry.
|
||||
- The packet list widget renders this as `[Dish][oc] × [Path][rc] @ [Ruler][pb]`,
|
||||
the most prominent path metric on the list.
|
||||
- The collector is the **sole writer** of `RawPacket`, through a single chokepoint
|
||||
`store_raw_packet` (`collector/handlers/raw_packet.py:122-138`). It already
|
||||
computes `path_len` (hop count) there with a two-tier fallback
|
||||
(`raw_packet.py:92-96`), and it already has a collector-side mirror of the
|
||||
path-hash extraction: `_extract_message_path_hashes`
|
||||
(`letsmesh_normalizer.py:826-844`) + the `_normalize_hash_list` staticmethod
|
||||
(`:846-867`).
|
||||
- The raw `/packets` endpoint already supports `min_path_len`/`max_path_len`
|
||||
range filters on the real `path_len` column — the closest filtering precedent.
|
||||
|
||||
### Problems
|
||||
|
||||
- **Unfilterable.** Because the value is computed post-query in Python, it cannot
|
||||
be filtered at the SQL level without either persisting a column or pushing
|
||||
brittle, backend-specific JSON extraction into SQL (dual-path: `decoded.path`
|
||||
and the trace fallback `decoded.payload.decoded.pathHashes`).
|
||||
- **Pagination inconsistency.** A post-query Python filter (the only no-migration
|
||||
hack) would make `total` and page sizes inconsistent — bad UX.
|
||||
- **Request-time cost.** Every list load performs an extra decoded-JSON fetch +
|
||||
per-row Python decode just to render the badge.
|
||||
|
||||
### Why Persist a Column (chosen approach)
|
||||
|
||||
The user confirmed the metric is **byte width** (the ruler badge) and the
|
||||
approach is **persisted column (A)**. This is the only solution that is correct,
|
||||
maintainable, and gives consistent pagination. The collector already extracts
|
||||
path hashes at ingest, so computing the byte-width there is a trivial addition
|
||||
covering 100% of new rows; a one-time Python backfill covers historical rows.
|
||||
|
||||
## Goals
|
||||
|
||||
- Persist `path_hash_bytes` (nullable `Integer`, values 1/2/3 or NULL) on
|
||||
`RawPacket`, computed at ingest from the decoded path hashes.
|
||||
- Backfill the column for all existing `raw_packets` rows from their `decoded`
|
||||
JSON, portably across SQLite and Postgres.
|
||||
- Add a **discrete-select filter** (Any / 1B / 2B / 3B) to the `/packets` list
|
||||
page, applied as a SQL `HAVING` on the per-group `MAX(path_hash_bytes)`.
|
||||
- Simplify the grouped-list route: replace the Python decode loop (Phase 3) with
|
||||
a SQL `MAX()` aggregate, removing the per-request JSON fetch.
|
||||
|
||||
## Non-Goals
|
||||
|
||||
- Filtering the **raw** `/packets` endpoint (not the list page) by byte width.
|
||||
The column will exist on the model and DB, but surfacing it on the raw endpoint
|
||||
/ `RawPacketRead` schema is out of scope (easy follow-up).
|
||||
- An "Unknown" (`?B` / null) filter option — the user chose **known widths only**
|
||||
(1/2/3). Null-width packets are simply never selected by the filter; they
|
||||
remain visible under "Any".
|
||||
- A numeric range (min/max) or multi-select filter shape — the user chose a
|
||||
**discrete single-select**.
|
||||
- Indexing `path_hash_bytes` (mirrors `path_len`, which is also unindexed). Add
|
||||
an index later only if query plans show a need.
|
||||
- Changing the per-reception path extraction in the **detail** route
|
||||
(`_extract_path_hashes` stays; only the list-path width computation is removed).
|
||||
|
||||
## Requirements
|
||||
|
||||
### Functional Requirements
|
||||
|
||||
- **FR-1**: A new nullable `Integer` column `path_hash_bytes` on `raw_packets`,
|
||||
holding the widest path-hash prefix width (1/2/3) for that reception, or NULL
|
||||
when no path hashes are present/decodable.
|
||||
- **FR-2**: Every newly inserted `RawPacket` row has `path_hash_bytes` computed
|
||||
from its `decoded` dict at ingest (collector is the sole writer).
|
||||
- **FR-3**: Historical `raw_packets` rows are backfilled from their `decoded`
|
||||
JSON during migration, using the same dual-path extraction and `max(len//2)`
|
||||
semantics as the runtime computation.
|
||||
- **FR-4**: `GET /api/v1/packet-groups` accepts an optional
|
||||
`path_hash_bytes` query parameter (integer 1/2/3). When set, only groups whose
|
||||
`MAX(path_hash_bytes)` equals the value are returned. Omitting it returns all
|
||||
groups (including null-width ones) — unchanged behavior.
|
||||
- **FR-5**: The list response still nulls `path_hash_bytes` for redacted groups
|
||||
(channel-visibility), exactly as today (`None if is_redacted else ...`).
|
||||
- **FR-6**: The `/packets` SPA page offers a **discrete `<select>`** filter
|
||||
("Path width": Any / 1B / 2B / 3B) that auto-submits and round-trips through
|
||||
the URL query string, surviving pagination and sort changes.
|
||||
|
||||
### Technical Requirements
|
||||
|
||||
- **TR-1** — Model (`common/models/raw_packet.py`): add
|
||||
`path_hash_bytes: Mapped[Optional[int]] = mapped_column(Integer, nullable=True)`
|
||||
immediately after `path_len`. Unindexed (mirrors `path_len`).
|
||||
- **TR-2** — Collector (`collector/handlers/raw_packet.py`): in
|
||||
`store_raw_packet`, after `path_len` is resolved (~line 96), compute
|
||||
`path_hash_bytes` from `decoded_packet` via the two-tier lookup
|
||||
(`decoded.path` → `payload.decoded.pathHashes`) using the already-imported
|
||||
`LetsMeshNormalizer._normalize_hash_list` staticmethod, then
|
||||
`max(len(h) // 2 for h in hashes)`. Add a small module-level
|
||||
`_path_hash_byte_width(hashes)` helper (mirrors the API route's helper; the
|
||||
API copy is deleted in Phase 3 so no naming collision persists). Pass
|
||||
`path_hash_bytes=` into the `RawPacket(...)` kwargs. Must be null-safe (no
|
||||
path → `None`).
|
||||
- **TR-3** — Migration (`alembic/versions/`, `down_revision = "38abdf4651fc"`):
|
||||
author via `meshcore-hub db revision --autogenerate -m "add path_hash_bytes to
|
||||
raw_packets"` (produces a `batch_alter_table` `add_column`), then **manually
|
||||
append a Python batched backfill** (~1000 rows/batch) that reads each row's
|
||||
`decoded` JSON, applies the self-contained dual-path extraction +
|
||||
`max(len//2)`, and `UPDATE`s the column. Python (not SQL JSON) so it is
|
||||
portable across SQLite and Postgres. Migration logic must be a frozen
|
||||
self-contained snapshot (no imports of app code that may change). Downgrade
|
||||
drops the column. **Read mechanism**: the `decoded` column must be read via a
|
||||
Core `select()` on a `sa.Table` declared with a `sa.JSON`-typed column (or
|
||||
explicitly `json.loads()` the fetched scalar), **not** raw
|
||||
`text("SELECT decoded ...")`. Raw `text()` bypasses SQLAlchemy's JSON type
|
||||
adapter, so deserialization becomes driver-dependent (SQLite returns TEXT,
|
||||
Postgres/psycopg2 may return a parsed object or string). Core `select()`
|
||||
routes through the type adapter and yields a Python `dict` consistently on
|
||||
both backends.
|
||||
- **TR-4** — API route (`api/routes/packet_groups.py`):
|
||||
- Phase 1 `group_query` select: add
|
||||
`func.max(RawPacket.path_hash_bytes).label("path_hash_bytes")`.
|
||||
- New param: `path_hash_bytes: Optional[int] = Query(None, ge=1, le=3,
|
||||
description="Filter by path-hash byte width (1/2/3)")`.
|
||||
- Apply filter:
|
||||
`if path_hash_bytes is not None: group_query = group_query.having(func.max(RawPacket.path_hash_bytes) == path_hash_bytes)`.
|
||||
- **Delete Phase 3** (lines 198-216, the `decoded_rows` fetch + `width_by_hash`
|
||||
loop). Read `grp.path_hash_bytes` from the group row instead.
|
||||
- Build item with
|
||||
`path_hash_bytes=(None if is_redacted else grp.path_hash_bytes)`.
|
||||
- Remove the now-dead `_path_hash_byte_width` helper (lines 54-59) from this
|
||||
file. **Keep** `_extract_path_hashes` — the detail route `get_packet_group`
|
||||
still uses it (`:311`).
|
||||
- **TR-5** — Frontend (`packets.js`):
|
||||
- Parse `const path_hash_bytes = query.path_hash_bytes || '';` alongside the
|
||||
other filter reads (~line 57).
|
||||
- Extend `hasActiveFilters` (~line 72) with `|| path_hash_bytes !== ''`.
|
||||
- Add a 4th entry to the `filterFields` array (~line 170): a
|
||||
`<select name="path_hash_bytes" class="select select-sm" @change=${autoSubmit}>`
|
||||
with options Any / 1B / 2B / 3B, mirroring the `event_type` select style.
|
||||
Option labels reuse `packets.path_width_bytes` (`{{count}}B`) and
|
||||
`common.all` / a new `packets.filter_path_width` label.
|
||||
- Add `if (path_hash_bytes !== '') apiParams.path_hash_bytes = path_hash_bytes;`
|
||||
(~line 111).
|
||||
- Add `path_hash_bytes` to the `pagination(...)` params (~line 166) and
|
||||
`headerParams` (~line 194) so it survives pagination and sort.
|
||||
- **TR-6** — i18n (`locales/en.json`, `locales/nl.json`): add
|
||||
`packets.filter_path_width` ("Path width" / "Padbreedte"). Option labels reuse
|
||||
existing `packets.path_width_bytes` and `common.all` — no new keys needed for
|
||||
them.
|
||||
- **TR-7** — `apiGet` (`api.js`) and `createFilterHandler` (`components.js`)
|
||||
already skip empty/falsy values, so no changes there.
|
||||
|
||||
## Implementation Plan
|
||||
|
||||
### Phase 1: Model & Migration
|
||||
|
||||
- Add `path_hash_bytes` column to `RawPacket` (TR-1).
|
||||
- Generate the migration with `--autogenerate` against head `38abdf4651fc`
|
||||
(TR-3), producing `batch_alter_table` `add_column`.
|
||||
- Manually append the Python batched backfill loop (TR-3): read
|
||||
`id, decoded WHERE path_hash_bytes IS NULL` in batches (~1000 rows/batch)
|
||||
via a Core `select()` on a `sa.Table` with a `sa.JSON`-typed `decoded` column
|
||||
(so the type adapter deserializes to a `dict` on both SQLite and Postgres —
|
||||
do **not** use raw `text("SELECT decoded ...")`), compute width, `UPDATE`.
|
||||
The backfill must include a **self-contained**, frozen copy of the dual-path
|
||||
extraction + `max(len//2)` logic — no imports of app code that may change in
|
||||
future versions. Rows where `decoded` is `None` are left as `NULL` (the
|
||||
column is nullable; `WHERE path_hash_bytes IS NULL` naturally skips
|
||||
already-filled rows and leaves truly uncomputable ones alone).
|
||||
- Verify the model test (column presence/default) and that `db upgrade` applies
|
||||
cleanly on both backends.
|
||||
|
||||
### Phase 2: Collector Ingest Compute
|
||||
|
||||
- Add the module-level `_path_hash_byte_width(hashes)` helper in
|
||||
`collector/handlers/raw_packet.py`.
|
||||
- In `store_raw_packet`, compute `path_hash_bytes` from `decoded_packet` after
|
||||
`path_len`, using `LetsMeshNormalizer._normalize_hash_list` + the two-tier
|
||||
lookup (TR-2). Pass it into the `RawPacket(...)` kwargs.
|
||||
- Add a collector test: `store_raw_packet` persists the expected width for a
|
||||
decoded packet carrying `decoded.path`, for the trace fallback path, and
|
||||
`None` when no path is present (mirror the existing `path_len` test).
|
||||
|
||||
### Phase 3: API Route Simplification & Filter
|
||||
|
||||
- In `packet_groups.py`, add the `func.max(RawPacket.path_hash_bytes)` aggregate
|
||||
to Phase 1 and the new `path_hash_bytes` query param + `HAVING` filter (TR-4).
|
||||
- Delete Phase 3 (the Python decode loop); read `grp.path_hash_bytes`. Remove the
|
||||
now-dead `_path_hash_byte_width` helper. Keep `_extract_path_hashes`.
|
||||
- Preserve redaction nulling in the item construction (TR-4 / FR-5).
|
||||
- Update `tests/test_api/test_packet_groups.py::TestPathHashBytes` to set
|
||||
`path_hash_bytes=` directly on inserted `RawPacket` rows (tests bypass the
|
||||
collector). Keep 1/2/3/None/redacted coverage.
|
||||
- Add filter tests: `?path_hash_bytes=2` returns only width-2 groups;
|
||||
`?path_hash_bytes=1`/`3` likewise; omitted returns all; a redacted group's
|
||||
response field is still null.
|
||||
|
||||
### Phase 4: Frontend Filter & i18n
|
||||
|
||||
- Add the discrete `<select>` filter field to `packets.js` (TR-5).
|
||||
- Wire the query parse, `hasActiveFilters`, `apiParams`, `pagination` params, and
|
||||
`headerParams` (TR-5).
|
||||
- Add `packets.filter_path_width` to both locale files (TR-6).
|
||||
- Rebuild the SPA bundle via the Docker build pipeline (local `node build.js`
|
||||
fails on a missing fontsource asset — build through compose).
|
||||
|
||||
### Phase 5: Verify
|
||||
|
||||
- `pytest --no-cov tests/test_api/test_packet_groups.py tests/test_collector/`
|
||||
- `pre-commit run --all-files`
|
||||
- Apply the migration on the stack and manually verify the filter on the packet
|
||||
list (all three known widths; "Any" returns null-width rows too).
|
||||
|
||||
## Open Questions
|
||||
|
||||
- None outstanding. Metric (byte width), approach (persisted column), filter
|
||||
shape (discrete select), and unknown-option exclusion (known widths only) are
|
||||
all resolved.
|
||||
- **Phase-ordering window** (minor, not blocking): Between migration (Phase 1)
|
||||
and collector ingester deploy (Phase 2), new RawPacket rows are inserted with
|
||||
`path_hash_bytes = NULL`. The backfill fills historical rows, but a brief gap
|
||||
exists for rows ingested during the deploy window. Once Phase 2 is active, new
|
||||
rows carry the correct value and Phase 3 reads it. In practice the window is
|
||||
seconds in a single `docker compose up -d` rebuild; any rows missed still render
|
||||
`?B` (unchanged UX), and the next backfill run would catch them.
|
||||
- Minor follow-up (not blocking): whether to also surface `path_hash_bytes` on
|
||||
the raw `/packets` endpoint's `RawPacketRead` schema, now that the column
|
||||
exists.
|
||||
|
||||
## References
|
||||
|
||||
- `docs/plans/20260612-2014-raw-packets-feature/plan.md` — the Raw Packets
|
||||
feature this builds on (model, collector chokepoint, channel-visibility
|
||||
redaction, `/packets` SPA page).
|
||||
- `docs/plans/20260426-1137-improve-snr-path-visibility/plan.md` — prior
|
||||
`path_len` (hop-count) column work and the "no backfill for per-observer
|
||||
fields" decision (here we *do* backfill, since the value is derivable from the
|
||||
already-stored `decoded` JSON).
|
||||
- `src/meshcore_hub/api/routes/packet_groups.py:54-59,198-216,240` — current
|
||||
runtime `_path_hash_byte_width` + Phase 3 aggregation + redaction nulling
|
||||
(to be replaced/deleted).
|
||||
- `src/meshcore_hub/collector/handlers/raw_packet.py:92-96,122-138` — `path_len`
|
||||
precedent and the single `RawPacket(...)` construction site.
|
||||
- `src/meshcore_hub/collector/letsmesh_normalizer.py:826-867` —
|
||||
`_extract_message_path_hashes` + `_normalize_hash_list` (staticmethod) to reuse.
|
||||
- `src/meshcore_hub/api/routes/raw_packets.py:165-172` — `min_snr`/`max_snr` and
|
||||
`min_path_len`/`max_path_len` range-filter precedent.
|
||||
- `alembic/versions/20260622_2243_38abdf4651fc_add_spam_scoring_columns_to_messages.py`
|
||||
— `batch_alter_table` add-column + index precedent (current migration head).
|
||||
- `alembic/versions/20260421_0001_normalize_public_key_case.py` — Python data
|
||||
backfill precedent (`op.get_bind()`, dialect-aware raw SQL).
|
||||
- Recent commits: `f845830` (JSON tree / path flow / chart polish, PR #292),
|
||||
`2c25a9c` (locale-aware numbers + filter panel redesign, PR #291) — the
|
||||
reception-widget + filter-panel work this plan extends.
|
||||
|
||||
## Review
|
||||
|
||||
**Status**: Approved with Changes
|
||||
|
||||
**Reviewed**: 2026-07-03
|
||||
|
||||
### Resolutions
|
||||
|
||||
- **Factual correction — aggregation fix is committed, not uncommitted**: The
|
||||
Phase 3 MAX-across-all-receptions fix described as "uncommitted" was committed
|
||||
in PR #292 (`f845830`). Updated to "recently committed (PR #292)".
|
||||
|
||||
- **Gap — self-contained migration backfill**: Clarified that the backfill loop
|
||||
must include a frozen, inline copy of the dual-path extraction +
|
||||
`max(len//2)` logic. Rows with `decoded = NULL` are left as NULL (the
|
||||
`WHERE path_hash_bytes IS NULL` clause naturally skips them).
|
||||
|
||||
- **Gap — collector / API helper naming collision**: Noted that the collector's
|
||||
`_path_hash_byte_width` helper mirrors the API route's (deleted in Phase 3),
|
||||
so no persistent naming conflict exists.
|
||||
|
||||
- **Risk — rows ingested between migration and collector deploy**: Added a brief
|
||||
note under Open Questions that the Phase 1→2 deploy window can leave
|
||||
`path_hash_bytes = NULL` for new rows, but the window is seconds in a compose
|
||||
rebuild, and the UX is unchanged (`?B` label) for any rows missed.
|
||||
|
||||
- **Postgres JSON type & backfill read mechanism**: Confirmed the `decoded`
|
||||
column is SQLAlchemy `JSON` (not `JSONB`). The Python backfill reads decoded
|
||||
as a Python `dict` via a Core `select()` on a `sa.Table` with a `sa.JSON`-typed
|
||||
column — this routes through SQLAlchemy's JSON type adapter, which deserializes
|
||||
consistently across SQLite (TEXT storage) and Postgres. Raw
|
||||
`text("SELECT decoded ...")` must **not** be used: it bypasses the type adapter,
|
||||
making deserialization driver-dependent (SQLite returns TEXT; Postgres/psycopg2
|
||||
may return a parsed object or string). No SQL JSON operators are needed —
|
||||
consistent with TR-3.
|
||||
|
||||
### Remaining Action Items
|
||||
|
||||
- **Migration down_revision**: Before generating the migration, confirm that
|
||||
`38abdf4651fc` is still the head revision (new migrations may land between
|
||||
plan approval and implementation). The `meshcore-hub db revision` command
|
||||
auto-resolves this — just verify the `down_revision` value in the generated
|
||||
file.
|
||||
|
||||
- **Phase 2 collector test**: The existing `test_store_raw_packet` test in
|
||||
`tests/test_collector/` does not appear to cover `path_len` persistence;
|
||||
verify whether a collector test for the new column can sit alongside or
|
||||
replace the `path_len` test, and whether the test fixture accepts
|
||||
`decoded_packet=` with path hashes.
|
||||
|
||||
- **Frontend bundle rebuild**: Must be built via Docker compose (local
|
||||
`node build.js` fails on the fontsource asset). The plan notes this in
|
||||
Phase 4.
|
||||
@@ -0,0 +1,112 @@
|
||||
# Tasks: Persist & Filter by Path-Hash Byte Width
|
||||
|
||||
> Generated from `plan.md` on 2026-07-03
|
||||
|
||||
## 1. Model & Migration (Phase 1)
|
||||
|
||||
- [x] 1.1 Add `path_hash_bytes` column to `RawPacket` model
|
||||
- [x] 1.1.1 Add `Mapped[Optional[int]] = mapped_column(Integer, nullable=True)` after `path_len` in `src/meshcore_hub/common/models/raw_packet.py`
|
||||
- [x] 1.1.2 Verify model test reflects new column (presence + default None)
|
||||
|
||||
- [x] 1.2 Generate and customize alembic migration
|
||||
- [x] 1.2.1 Run `meshcore-hub db revision --autogenerate -m "add path_hash_bytes to raw_packets"` against migration head `38abdf4651fc`
|
||||
- [x] 1.2.2 Verify `down_revision` is `38abdf4651fc` in generated file
|
||||
- [x] 1.2.3 Append self-contained Python batched backfill loop (~1000 rows/batch):
|
||||
- Read `decoded` via Core `select()` on a `sa.Table` declared with a `sa.JSON`-typed column (routes through SQLAlchemy's JSON type adapter → `dict` on both SQLite and Postgres); do **not** use raw `text("SELECT decoded ...")` (bypasses the adapter → driver-dependent deserialization)
|
||||
- Inline frozen dual-path extraction: `decoded["path"]` fallback `decoded.get("payload", {}).get("decoded", {}).get("pathHashes")`
|
||||
- Normalize with inline `_normalize_hash_list` logic (split on `","`, strip)
|
||||
- Compute `max(len(h) // 2 for h in hashes)` for each row
|
||||
- `UPDATE raw_packets SET path_hash_bytes = ? WHERE id = ?`
|
||||
- Skip NULL decoded rows (leave NULL)
|
||||
- No imports of app code
|
||||
- [x] 1.2.4 Verify downgrade drops the column
|
||||
- [x] 1.2.5 Test migration with `meshcore-hub db upgrade` on both SQLite and Postgres backends
|
||||
|
||||
## 2. Collector Ingest (Phase 2)
|
||||
|
||||
- [x] 2.1 Add `_path_hash_byte_width` helper
|
||||
- [x] 2.1.1 Add module-level `_path_hash_byte_width(hashes: list[str] | None) -> int | None` in `src/meshcore_hub/collector/handlers/raw_packet.py`
|
||||
- [x] 2.1.2 Implement as `max(len(h) // 2 for h in hashes) if hashes else None`
|
||||
|
||||
- [x] 2.2 Compute `path_hash_bytes` at ingest
|
||||
- [x] 2.2.1 In `store_raw_packet`, after `path_len` block (~line 96), extract path hashes from `decoded_packet`:
|
||||
- `LetsMeshNormalizer._normalize_hash_list(decoded_packet.get("path"))`
|
||||
- Fallback: `_normalize_hash_list(decoded_packet.get("payload", {}).get("decoded", {}).get("pathHashes"))`
|
||||
- [x] 2.2.2 Compute `path_hash_bytes = _path_hash_byte_width(hashes)` (None-safe)
|
||||
- [x] 2.2.3 Pass `path_hash_bytes=path_hash_bytes` into `RawPacket(...)` kwargs (~line 122)
|
||||
- [x] 2.2.4 Verify null-safe: no path / no decoded → `None`
|
||||
|
||||
- [x] 2.3 Add collector unit test
|
||||
- [x] 2.3.1 Create test in `tests/test_collector/` (or extend existing `test_store_raw_packet`) covering:
|
||||
- `decoded.path` with hashes of mixed widths → correct MAX byte-width persisted
|
||||
- Trace fallback `decoded.payload.decoded.pathHashes` → correct width
|
||||
- No path present → `None`
|
||||
- Verify `path_len` persistence test still passes alongside
|
||||
|
||||
## 3. API Route (Phase 3)
|
||||
|
||||
- [x] 3.1 Add SQL aggregate to group query
|
||||
- [x] 3.1.1 In Phase 1 `group_query` select (`src/meshcore_hub/api/routes/packet_groups.py`), add:
|
||||
`func.max(RawPacket.path_hash_bytes).label("path_hash_bytes")`
|
||||
- [x] 3.1.2 Add `path_hash_bytes: Optional[int] = Query(None, ge=1, le=3)` parameter to route function
|
||||
- [x] 3.1.3 Apply `HAVING` filter: `if path_hash_bytes is not None: group_query = group_query.having(func.max(RawPacket.path_hash_bytes) == path_hash_bytes)`
|
||||
- [x] 3.1.4 Verify HAVING works with existing WHERE clauses and `func.count(RawPacket.id)`
|
||||
|
||||
- [x] 3.2 Delete Phase 3 Python decode loop
|
||||
- [x] 3.2.1 Remove the `decoded_rows` fetch + `width_by_hash` loop (lines 198-216)
|
||||
- [x] 3.2.2 Replace with direct read: `path_hash_bytes=(None if is_redacted else grp.path_hash_bytes)`
|
||||
- [x] 3.2.3 Remove dead `_path_hash_byte_width` helper (lines 54-59)
|
||||
- [x] 3.2.4 Confirm `_extract_path_hashes` (line 36) is kept — still used by detail route at line 311
|
||||
|
||||
- [x] 3.3 Update existing tests
|
||||
- [x] 3.3.1 In `tests/test_api/test_packet_groups.py::TestPathHashBytes`, update insert fixtures to set `path_hash_bytes=` explicitly on `RawPacket` rows (tests bypass collector)
|
||||
- [x] 3.3.2 Keep coverage for 1/2/3/None/redacted paths
|
||||
- [x] 3.3.3 Verify all existing tests pass without Phase 3
|
||||
|
||||
- [x] 3.4 Add filter tests
|
||||
- [x] 3.4.1 `?path_hash_bytes=2` returns only width-2 groups
|
||||
- [x] 3.4.2 `?path_hash_bytes=1` and `3` return correct subsets
|
||||
- [x] 3.4.3 Missing parameter returns all groups (including null-width)
|
||||
- [x] 3.4.4 Redacted group response field is still null despite persisted value
|
||||
- [x] 3.4.5 Verify `total` count is correct when filtered vs unfiltered
|
||||
|
||||
## 4. Frontend & i18n (Phase 4)
|
||||
|
||||
- [x] 4.1 Add filter `<select>` to packets.js
|
||||
- [x] 4.1.1 Parse `const path_hash_bytes = query.path_hash_bytes || '';` (~line 57)
|
||||
- [x] 4.1.2 Extend `hasActiveFilters` with `|| path_hash_bytes !== ''` (~line 72)
|
||||
- [x] 4.1.3 Add to `apiParams`: `if (path_hash_bytes !== '') apiParams.path_hash_bytes = path_hash_bytes;` (~line 111)
|
||||
- [x] 4.1.4 Add `path_hash_bytes` to `pagination(...)` params (~line 166)
|
||||
- [x] 4.1.5 Add `path_hash_bytes` to `headerParams` (~line 194)
|
||||
- [x] 4.1.6 Add 4th entry to `filterFields` array (~line 170):
|
||||
`<select name="path_hash_bytes" class="select select-sm" @change=${autoSubmit}>`
|
||||
with options: Any (empty value) / 1B / 2B / 3B, mirroring `event_type` select style
|
||||
- [x] 4.1.7 Option labels use `packets.path_width_bytes` (`{{count}}B`) and `common.all`
|
||||
|
||||
- [x] 4.2 Add i18n keys
|
||||
- [x] 4.2.1 Add `"packets.filter_path_width"` to `src/meshcore_hub/web/static/locales/en.json` ("Path width")
|
||||
- [x] 4.2.2 Add `"packets.filter_path_width"` to `src/meshcore_hub/web/static/locales/nl.json` ("Padbreedte")
|
||||
|
||||
- [x] 4.3 Rebuild SPA bundle
|
||||
- [x] 4.3.1 Build via Docker compose pipeline (not local `node build.js` — fontsource asset requires compose context)
|
||||
|
||||
## 5. Verification (Phase 5)
|
||||
|
||||
- [x] 5.1 Run targeted tests
|
||||
- [x] 5.1.1 `pytest --no-cov tests/test_api/test_packet_groups.py`
|
||||
- [x] 5.1.2 `pytest --no-cov tests/test_collector/`
|
||||
- [x] 5.1.3 `pytest --no-cov tests/test_common/` (model column test)
|
||||
|
||||
- [x] 5.2 Run full test suite
|
||||
- [x] 5.2.1 `pytest --no-cov 2>&1 | grep -iE "passed|failed" | tail -3`
|
||||
|
||||
- [x] 5.3 Run pre-commit
|
||||
- [x] 5.3.1 `pre-commit run --all-files`
|
||||
|
||||
- [x] 5.4 Manual smoke test on Docker stack
|
||||
- [x] 5.4.1 Apply migration: `docker compose run --rm migrate db upgrade`
|
||||
- [x] 5.4.2 Rebuild and restart stack
|
||||
- [x] 5.4.3 Verify packet list loads with all three width filters (1B/2B/3B)
|
||||
- [x] 5.4.4 Verify "Any" filter returns all rows including null-width
|
||||
- [x] 5.4.5 Verify filter survives pagination and sort changes via URL query string
|
||||
- [x] 5.4.6 Verify redacted groups show `?B` on detail page
|
||||
@@ -51,14 +51,6 @@ def _extract_path_hashes(decoded: dict[str, Any] | None) -> list[str] | None:
|
||||
return hashes if isinstance(hashes, list) else None
|
||||
|
||||
|
||||
def _path_hash_byte_width(path_hashes: Optional[list[str]]) -> Optional[int]:
|
||||
"""Widest path-hash prefix width in bytes (each hash is hex: 2/4/6 chars)."""
|
||||
if not path_hashes:
|
||||
return None
|
||||
widths = [len(h) // 2 for h in path_hashes if isinstance(h, str) and h]
|
||||
return max(widths) if widths else None
|
||||
|
||||
|
||||
def _get_tag_name(node: Optional[Node]) -> Optional[str]:
|
||||
if not node or not node.tags:
|
||||
return None
|
||||
@@ -79,6 +71,9 @@ def list_packet_groups(
|
||||
),
|
||||
event_type: Optional[str] = Query(None, description="Filter by event type"),
|
||||
channel_idx: Optional[int] = Query(None, description="Filter by channel index"),
|
||||
path_hash_bytes: Optional[int] = Query(
|
||||
None, ge=1, le=3, description="Filter by path-hash byte width (1/2/3)"
|
||||
),
|
||||
since: Optional[datetime] = Query(None, description="Start timestamp"),
|
||||
until: Optional[datetime] = Query(None, description="End timestamp"),
|
||||
sort: Optional[str] = Query(
|
||||
@@ -120,6 +115,7 @@ def list_packet_groups(
|
||||
func.count(RawPacket.id).label("reception_count"),
|
||||
func.count(RawPacket.observer_node_id.distinct()).label("observer_count"),
|
||||
func.min(RawPacket.received_at).label("first_seen"),
|
||||
func.max(RawPacket.path_hash_bytes).label("path_hash_bytes"),
|
||||
)
|
||||
.outerjoin(ObserverNode, RawPacket.observer_node_id == ObserverNode.id)
|
||||
.where(RawPacket.packet_hash.is_not(None))
|
||||
@@ -146,6 +142,11 @@ def list_packet_groups(
|
||||
|
||||
group_query = group_query.group_by(RawPacket.packet_hash)
|
||||
|
||||
if path_hash_bytes is not None:
|
||||
group_query = group_query.having(
|
||||
func.max(RawPacket.path_hash_bytes) == path_hash_bytes
|
||||
)
|
||||
|
||||
count_query = select(func.count()).select_from(group_query.subquery())
|
||||
total = session.execute(count_query).scalar() or 0
|
||||
|
||||
@@ -195,20 +196,6 @@ def list_packet_groups(
|
||||
|
||||
group_counts = {r.packet_hash: r for r in group_rows}
|
||||
|
||||
# ── Phase 3: path-hash width — load decoded only for representative rows ───
|
||||
rep_id_to_hash = {rep.id: h for h, rep in representative.items()}
|
||||
width_by_hash: dict[str, Optional[int]] = {}
|
||||
if rep_id_to_hash:
|
||||
decoded_rows = session.execute(
|
||||
select(RawPacket.id, RawPacket.decoded).where(
|
||||
RawPacket.id.in_(rep_id_to_hash.keys())
|
||||
)
|
||||
).all()
|
||||
for rid, decoded in decoded_rows:
|
||||
h = rep_id_to_hash.get(rid)
|
||||
if h is not None:
|
||||
width_by_hash[h] = _path_hash_byte_width(_extract_path_hashes(decoded))
|
||||
|
||||
items = []
|
||||
for h in hashes:
|
||||
rep = representative.get(h)
|
||||
@@ -231,7 +218,7 @@ def list_packet_groups(
|
||||
),
|
||||
reception_count=grp.reception_count,
|
||||
observer_count=grp.observer_count,
|
||||
path_hash_bytes=(None if is_redacted else width_by_hash.get(h)),
|
||||
path_hash_bytes=(None if is_redacted else grp.path_hash_bytes),
|
||||
receptions=[],
|
||||
first_seen=grp.first_seen,
|
||||
redacted=is_redacted,
|
||||
|
||||
@@ -42,6 +42,14 @@ def _extract_source_pubkey_prefix(
|
||||
return None
|
||||
|
||||
|
||||
def _path_hash_byte_width(hashes: list[str] | None) -> int | None:
|
||||
"""Widest path-hash prefix width in bytes (each hash is hex: 2/4/6 chars)."""
|
||||
if not hashes:
|
||||
return None
|
||||
widths = [len(h) // 2 for h in hashes if isinstance(h, str) and h]
|
||||
return max(widths) if widths else None
|
||||
|
||||
|
||||
def store_raw_packet(
|
||||
public_key: str,
|
||||
payload: dict[str, Any],
|
||||
@@ -95,6 +103,14 @@ def store_raw_packet(
|
||||
decoded_packet
|
||||
)
|
||||
|
||||
path_hashes = LetsMeshNormalizer._normalize_hash_list(
|
||||
decoded_packet.get("path") if isinstance(decoded_packet, dict) else None
|
||||
)
|
||||
if not path_hashes and isinstance(decoded_packet, dict):
|
||||
inner = (decoded_packet.get("payload") or {}).get("decoded") or {}
|
||||
path_hashes = LetsMeshNormalizer._normalize_hash_list(inner.get("pathHashes"))
|
||||
path_hash_bytes = _path_hash_byte_width(path_hashes)
|
||||
|
||||
snr = LetsMeshNormalizer._parse_float(payload.get("SNR"))
|
||||
if snr is None:
|
||||
snr = LetsMeshNormalizer._parse_float(payload.get("snr"))
|
||||
@@ -131,6 +147,7 @@ def store_raw_packet(
|
||||
source_pubkey_prefix=source_pubkey_prefix,
|
||||
route_type=route_type,
|
||||
path_len=path_len,
|
||||
path_hash_bytes=path_hash_bytes,
|
||||
snr=snr,
|
||||
decoded=decoded_packet,
|
||||
received_at=now,
|
||||
|
||||
@@ -32,6 +32,7 @@ class RawPacket(Base, UUIDMixin, TimestampMixin):
|
||||
senderPublicKey, for efficient "packets from this node" filtering
|
||||
route_type: Mapped route type (flood, transport_flood, direct, ...)
|
||||
path_len: Hop count
|
||||
path_hash_bytes: Widest path-hash prefix width in bytes (1/2/3)
|
||||
snr: Signal-to-noise ratio as reported by the observer
|
||||
decoded: Decoder summary JSON, so the detail view needs no re-decode
|
||||
received_at: When received by the observing interface
|
||||
@@ -86,6 +87,10 @@ class RawPacket(Base, UUIDMixin, TimestampMixin):
|
||||
Integer,
|
||||
nullable=True,
|
||||
)
|
||||
path_hash_bytes: Mapped[Optional[int]] = mapped_column(
|
||||
Integer,
|
||||
nullable=True,
|
||||
)
|
||||
snr: Mapped[Optional[float]] = mapped_column(
|
||||
Float,
|
||||
nullable=True,
|
||||
|
||||
@@ -170,6 +170,10 @@ export function iconTxPower(cls = 'h-5 w-5') {
|
||||
return html`<svg xmlns="http://www.w3.org/2000/svg" class=${cls} fill="none" viewBox="0 0 24 24" stroke="currentColor"><path stroke-linecap="round" stroke-linejoin="round" stroke-width="2" d="M3.75 13.5l10.5-11.25L12 10.5h8.25L9.75 21.75 12 13.5H3.75z" /></svg>`;
|
||||
}
|
||||
|
||||
export function iconRuler(cls = 'h-5 w-5') {
|
||||
return html`<svg xmlns="http://www.w3.org/2000/svg" class=${cls} fill="none" viewBox="0 0 24 24" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><path d="M21.3 8.7 8.7 21.3a1 1 0 0 1-1.4 0L2.7 16.7a1 1 0 0 1 0-1.4L15.3 2.7a1 1 0 0 1 1.4 0l4.6 4.6a1 1 0 0 1 0 1.4Z" /><path d="m7.5 10.5 2 2" /><path d="m10.5 7.5 2 2" /><path d="m13.5 4.5 2 2" /><path d="m4.5 13.5 2 2" /></svg>`;
|
||||
}
|
||||
|
||||
export function iconFilter(cls = 'h-5 w-5') {
|
||||
return html`<svg xmlns="http://www.w3.org/2000/svg" class=${cls} fill="none" viewBox="0 0 24 24" stroke="currentColor"><path stroke-linecap="round" stroke-linejoin="round" stroke-width="2" d="M12 3c2.755 0 5.455.232 8.083.678.533.09.917.556.917 1.096v1.044a2.25 2.25 0 01-.659 1.591l-5.432 5.432a2.25 2.25 0 00-.659 1.591v2.927a2.25 2.25 0 01-1.244 2.013L9.75 21v-6.568a2.25 2.25 0 00-.659-1.591L3.659 7.409A2.25 2.25 0 013 5.818V4.774c0-.54.384-1.006.917-1.096A48.32 48.32 0 0112 3z" /></svg>`;
|
||||
}
|
||||
|
||||
@@ -7,7 +7,7 @@ import {
|
||||
renderFilterForm, renderFilterToggle, autoSubmit, submitOnEnter
|
||||
} from '../components.js';
|
||||
import { createAutoRefresh } from '../auto-refresh.js';
|
||||
import { iconSatelliteDish, iconPath } from '../icons.js';
|
||||
import { iconSatelliteDish, iconPath, iconRuler } from '../icons.js';
|
||||
|
||||
const EVENT_TYPES = [
|
||||
'advertisement', 'channel_msg_recv', 'contact_msg_recv',
|
||||
@@ -34,12 +34,19 @@ function receptionBadge(packet) {
|
||||
const rc = packet.reception_count ?? 1;
|
||||
const oc = packet.observer_count ?? 1;
|
||||
const pb = packet.path_hash_bytes;
|
||||
const knownWidth = pb != null && pb > 0;
|
||||
const widthLabel = knownWidth
|
||||
? t('packets.path_width_bytes', { count: pb })
|
||||
: t('packets.path_width_unknown');
|
||||
return html`<span class="inline-flex items-center gap-1">
|
||||
${iconSatelliteDish('h-4 w-4 opacity-70')}
|
||||
<span class="badge badge-sm badge-primary" title=${t('common.observers')}>${formatNumber(oc)}</span>
|
||||
<span class="opacity-40" aria-hidden="true">×</span>
|
||||
${iconPath('h-4 w-4 opacity-70')}
|
||||
<span class="badge badge-sm badge-primary" title=${t('packets.reception_plural')}>${formatNumber(rc)}</span>
|
||||
${pb ? html`<span class="badge badge-sm badge-ghost" title=${t('packets.path_width_title')}>${t('packets.path_width_bytes', { count: pb })}</span>` : nothing}
|
||||
<span class="opacity-40" aria-hidden="true">@</span>
|
||||
${iconRuler('h-4 w-4 opacity-70')}
|
||||
<span class="badge badge-sm badge-primary ${knownWidth ? '' : 'opacity-60'}" title=${t('packets.path_width_title')}>${widthLabel}</span>
|
||||
</span>`;
|
||||
}
|
||||
|
||||
@@ -49,6 +56,7 @@ export async function render(container, params, router) {
|
||||
const search = query.search || '';
|
||||
const event_type = query.event_type || '';
|
||||
const channel_idx = query.channel_idx || '';
|
||||
const path_hash_bytes = query.path_hash_bytes || '';
|
||||
const page = parseInt(query.page, 10) || 1;
|
||||
const limit = parseInt(query.limit, 10) || 20;
|
||||
const offset = (page - 1) * limit;
|
||||
@@ -63,7 +71,7 @@ export async function render(container, params, router) {
|
||||
let lastContent = nothing;
|
||||
let lastTotal = null;
|
||||
let currentFilterFields = [];
|
||||
const hasActiveFilters = search !== '' || event_type !== '' || channel_idx !== '';
|
||||
const hasActiveFilters = search !== '' || event_type !== '' || channel_idx !== '' || path_hash_bytes !== '';
|
||||
|
||||
function onFilterToggle() { renderPage(lastContent, { total: lastTotal }); }
|
||||
|
||||
@@ -104,6 +112,7 @@ ${displayContent}`, container);
|
||||
const apiParams = { limit, offset, search, sort, order };
|
||||
if (event_type) apiParams.event_type = event_type;
|
||||
if (channel_idx !== '') apiParams.channel_idx = channel_idx;
|
||||
if (path_hash_bytes !== '') apiParams.path_hash_bytes = path_hash_bytes;
|
||||
|
||||
const [data, channelsData] = await Promise.all([
|
||||
apiGet('/api/v1/packet-groups', apiParams, { signal }),
|
||||
@@ -157,7 +166,7 @@ ${displayContent}`, container);
|
||||
</tr>`);
|
||||
|
||||
const paginationBlock = pagination(page, totalPages, '/packets', {
|
||||
search, event_type, channel_idx, limit, sort, order,
|
||||
search, event_type, channel_idx, path_hash_bytes, limit, sort, order,
|
||||
});
|
||||
|
||||
const filterFields = [
|
||||
@@ -181,10 +190,18 @@ ${displayContent}`, container);
|
||||
<option value="" ?selected=${channel_idx === ''}>${t('common.all_channels')}</option>
|
||||
${channelList.map(c => html`<option value=${c.idx} ?selected=${String(channel_idx) === String(c.idx)}>${c.name} (${c.idx})</option>`)}
|
||||
</select>
|
||||
</div>`,
|
||||
() => html`
|
||||
<div class="flex flex-col gap-1 max-w-48">
|
||||
<label class="flex items-center py-1"><span class="opacity-80 text-sm">${t('packets.filter_path_width')}</span></label>
|
||||
<select name="path_hash_bytes" class="select select-sm" @change=${autoSubmit}>
|
||||
<option value="" ?selected=${path_hash_bytes === ''}>${t('common.all')}</option>
|
||||
${[1, 2, 3].map(w => html`<option value=${w} ?selected=${path_hash_bytes === String(w)}>${t('packets.path_width_bytes', { count: w })}</option>`)}
|
||||
</select>
|
||||
</div>`,
|
||||
];
|
||||
|
||||
const headerParams = { search, event_type, channel_idx, limit };
|
||||
const headerParams = { search, event_type, channel_idx, path_hash_bytes, limit };
|
||||
const sortable = (label, sortKey) => sortableTableHeader(label, {
|
||||
sortKey, currentSort: sort, currentOrder: order,
|
||||
navigate, basePath: '/packets', params: headerParams,
|
||||
|
||||
@@ -82,6 +82,7 @@
|
||||
"sign_out": "Sign Out",
|
||||
"view_details": "View Details",
|
||||
"all_types": "All Types",
|
||||
"all": "Any",
|
||||
"all_channels": "All Channels",
|
||||
"all_members": "All Members",
|
||||
"all_operators": "All Operators",
|
||||
@@ -218,6 +219,7 @@
|
||||
},
|
||||
"packets": {
|
||||
"filter_event_type": "Event Type",
|
||||
"filter_path_width": "Path Width",
|
||||
"filter_min_snr": "Min SNR",
|
||||
"filter_max_snr": "Max SNR",
|
||||
"redacted": "Restricted",
|
||||
@@ -248,6 +250,7 @@
|
||||
"path_nodes_more": "+{{count}} more — view all →",
|
||||
"not_found_retention": "Packet not found — it may have been cleaned up due to data retention.",
|
||||
"path_width_bytes": "{{count}}B",
|
||||
"path_width_unknown": "?B",
|
||||
"path_width_title": "Path hash width (bytes)",
|
||||
"sort": {
|
||||
"newest": "Time (newest)",
|
||||
|
||||
@@ -80,6 +80,7 @@
|
||||
"sign_out": "Uitloggen",
|
||||
"view_details": "Details bekijken",
|
||||
"all_types": "Alle types",
|
||||
"all": "Alle",
|
||||
"node_type": "Knooppunttype",
|
||||
"show": "Toon",
|
||||
"search_placeholder": "Zoek op naam, ID of publieke sleutel...",
|
||||
@@ -160,6 +161,7 @@
|
||||
},
|
||||
"packets": {
|
||||
"filter_event_type": "Gebeurtenistype",
|
||||
"filter_path_width": "Padbreedte",
|
||||
"filter_min_snr": "Min. SNR",
|
||||
"filter_max_snr": "Max. SNR",
|
||||
"redacted": "Beperkt",
|
||||
@@ -184,6 +186,7 @@
|
||||
"path_nodes_more": "+{{count}} meer — alle bekijken →",
|
||||
"not_found_retention": "Pakket niet gevonden — mogelijk opgeschoond door dataretentie.",
|
||||
"path_width_bytes": "{{count}}B",
|
||||
"path_width_unknown": "?B",
|
||||
"path_width_title": "Breedte pad-hash (bytes)",
|
||||
"sort": {
|
||||
"newest": "Tijd (nieuwste)",
|
||||
|
||||
@@ -300,14 +300,14 @@ class TestListPacketGroups:
|
||||
|
||||
|
||||
class TestPathHashBytes:
|
||||
"""Tests for the derived path_hash_bytes field on the list endpoint."""
|
||||
"""Tests for the persisted path_hash_bytes field on the list endpoint."""
|
||||
|
||||
def test_one_byte_path(self, client_no_auth, api_db_session):
|
||||
api_db_session.add(
|
||||
RawPacket(
|
||||
raw_hex="AA",
|
||||
packet_hash="H1",
|
||||
decoded={"path": ["aa", "bb"]},
|
||||
path_hash_bytes=1,
|
||||
received_at=_now(),
|
||||
)
|
||||
)
|
||||
@@ -320,7 +320,7 @@ class TestPathHashBytes:
|
||||
RawPacket(
|
||||
raw_hex="AA",
|
||||
packet_hash="H1",
|
||||
decoded={"path": ["aabb"]},
|
||||
path_hash_bytes=2,
|
||||
received_at=_now(),
|
||||
)
|
||||
)
|
||||
@@ -328,17 +328,45 @@ class TestPathHashBytes:
|
||||
item = client_no_auth.get("/api/v1/packet-groups").json()["items"][0]
|
||||
assert item["path_hash_bytes"] == 2
|
||||
|
||||
def test_mixed_path_uses_max(self, client_no_auth, api_db_session):
|
||||
def test_three_byte_path(self, client_no_auth, api_db_session):
|
||||
api_db_session.add(
|
||||
RawPacket(
|
||||
raw_hex="AA",
|
||||
packet_hash="H1",
|
||||
decoded={"path": ["aa", "aabb"]},
|
||||
path_hash_bytes=3,
|
||||
received_at=_now(),
|
||||
)
|
||||
)
|
||||
api_db_session.commit()
|
||||
item = client_no_auth.get("/api/v1/packet-groups").json()["items"][0]
|
||||
assert item["path_hash_bytes"] == 3
|
||||
|
||||
def test_width_aggregates_across_receptions(self, client_no_auth, api_db_session):
|
||||
"""Width is observer-relative; take the widest across all receptions.
|
||||
|
||||
The oldest reception (representative) carries no path, while a newer
|
||||
reception of the same packet saw a 2-byte path. The badge must reflect
|
||||
the widest path seen, not just the representative row.
|
||||
"""
|
||||
base = _now()
|
||||
api_db_session.add(
|
||||
RawPacket(
|
||||
raw_hex="AA",
|
||||
packet_hash="H1",
|
||||
path_hash_bytes=None,
|
||||
received_at=base,
|
||||
)
|
||||
)
|
||||
api_db_session.add(
|
||||
RawPacket(
|
||||
raw_hex="BB",
|
||||
packet_hash="H1",
|
||||
path_hash_bytes=2,
|
||||
received_at=base.replace(microsecond=base.microsecond + 1),
|
||||
)
|
||||
)
|
||||
api_db_session.commit()
|
||||
item = client_no_auth.get("/api/v1/packet-groups").json()["items"][0]
|
||||
assert item["path_hash_bytes"] == 2
|
||||
|
||||
def test_no_path_returns_none(self, client_no_auth, api_db_session):
|
||||
@@ -366,7 +394,7 @@ class TestPathHashBytes:
|
||||
raw_hex="SECRET",
|
||||
packet_hash="ADM_HASH",
|
||||
channel_idx=adm_idx,
|
||||
decoded={"path": ["aabb"]},
|
||||
path_hash_bytes=2,
|
||||
received_at=_now(),
|
||||
)
|
||||
)
|
||||
@@ -376,6 +404,142 @@ class TestPathHashBytes:
|
||||
assert item["path_hash_bytes"] is None
|
||||
|
||||
|
||||
class TestPathHashBytesFilter:
|
||||
"""Tests for the ?path_hash_bytes= query filter."""
|
||||
|
||||
def test_filter_returns_only_matching_width(self, client_no_auth, api_db_session):
|
||||
now = _now()
|
||||
api_db_session.add_all(
|
||||
[
|
||||
RawPacket(
|
||||
raw_hex="A", packet_hash="H1", path_hash_bytes=1, received_at=now
|
||||
),
|
||||
RawPacket(
|
||||
raw_hex="B", packet_hash="H2", path_hash_bytes=2, received_at=now
|
||||
),
|
||||
RawPacket(
|
||||
raw_hex="C", packet_hash="H3", path_hash_bytes=3, received_at=now
|
||||
),
|
||||
]
|
||||
)
|
||||
api_db_session.commit()
|
||||
|
||||
data = client_no_auth.get("/api/v1/packet-groups?path_hash_bytes=2").json()
|
||||
assert data["total"] == 1
|
||||
assert data["items"][0]["packet_hash"] == "H2"
|
||||
|
||||
def test_filter_width_1(self, client_no_auth, api_db_session):
|
||||
now = _now()
|
||||
api_db_session.add_all(
|
||||
[
|
||||
RawPacket(
|
||||
raw_hex="A", packet_hash="H1", path_hash_bytes=1, received_at=now
|
||||
),
|
||||
RawPacket(
|
||||
raw_hex="B", packet_hash="H2", path_hash_bytes=3, received_at=now
|
||||
),
|
||||
]
|
||||
)
|
||||
api_db_session.commit()
|
||||
|
||||
data = client_no_auth.get("/api/v1/packet-groups?path_hash_bytes=1").json()
|
||||
assert data["total"] == 1
|
||||
assert data["items"][0]["packet_hash"] == "H1"
|
||||
|
||||
def test_filter_width_3(self, client_no_auth, api_db_session):
|
||||
now = _now()
|
||||
api_db_session.add_all(
|
||||
[
|
||||
RawPacket(
|
||||
raw_hex="A", packet_hash="H1", path_hash_bytes=1, received_at=now
|
||||
),
|
||||
RawPacket(
|
||||
raw_hex="B", packet_hash="H2", path_hash_bytes=3, received_at=now
|
||||
),
|
||||
]
|
||||
)
|
||||
api_db_session.commit()
|
||||
|
||||
data = client_no_auth.get("/api/v1/packet-groups?path_hash_bytes=3").json()
|
||||
assert data["total"] == 1
|
||||
assert data["items"][0]["packet_hash"] == "H2"
|
||||
|
||||
def test_no_filter_returns_all_including_null(self, client_no_auth, api_db_session):
|
||||
now = _now()
|
||||
api_db_session.add_all(
|
||||
[
|
||||
RawPacket(
|
||||
raw_hex="A", packet_hash="H1", path_hash_bytes=1, received_at=now
|
||||
),
|
||||
RawPacket(
|
||||
raw_hex="B", packet_hash="H2", path_hash_bytes=None, received_at=now
|
||||
),
|
||||
]
|
||||
)
|
||||
api_db_session.commit()
|
||||
|
||||
data = client_no_auth.get("/api/v1/packet-groups").json()
|
||||
assert data["total"] == 2
|
||||
widths = {i["path_hash_bytes"] for i in data["items"]}
|
||||
assert widths == {1, None}
|
||||
|
||||
def test_filter_excludes_null_width(self, client_no_auth, api_db_session):
|
||||
now = _now()
|
||||
api_db_session.add_all(
|
||||
[
|
||||
RawPacket(
|
||||
raw_hex="A", packet_hash="H1", path_hash_bytes=None, received_at=now
|
||||
),
|
||||
RawPacket(
|
||||
raw_hex="B", packet_hash="H2", path_hash_bytes=2, received_at=now
|
||||
),
|
||||
]
|
||||
)
|
||||
api_db_session.commit()
|
||||
|
||||
data = client_no_auth.get("/api/v1/packet-groups?path_hash_bytes=2").json()
|
||||
assert data["total"] == 1
|
||||
assert data["items"][0]["path_hash_bytes"] == 2
|
||||
|
||||
def test_filter_redacted_group_still_null(self, client_no_auth, api_db_session):
|
||||
adm_key = "FFEEDDCCBBAA99887766554433221100"
|
||||
adm_idx = int(Channel.compute_channel_hash(adm_key), 16)
|
||||
now = _now()
|
||||
api_db_session.add(
|
||||
Channel(
|
||||
name="Adm",
|
||||
key_hex=adm_key,
|
||||
channel_hash=Channel.compute_channel_hash(adm_key),
|
||||
visibility="admin",
|
||||
enabled=True,
|
||||
)
|
||||
)
|
||||
api_db_session.add_all(
|
||||
[
|
||||
RawPacket(
|
||||
raw_hex="SECRET",
|
||||
packet_hash="ADM",
|
||||
channel_idx=adm_idx,
|
||||
path_hash_bytes=2,
|
||||
received_at=now,
|
||||
),
|
||||
RawPacket(
|
||||
raw_hex="OK",
|
||||
packet_hash="PUB",
|
||||
path_hash_bytes=2,
|
||||
received_at=now,
|
||||
),
|
||||
]
|
||||
)
|
||||
api_db_session.commit()
|
||||
|
||||
data = client_no_auth.get("/api/v1/packet-groups?path_hash_bytes=2").json()
|
||||
assert data["total"] == 2
|
||||
adm = next(i for i in data["items"] if i["packet_hash"] == "ADM")
|
||||
assert adm["redacted"] is True
|
||||
assert adm["path_hash_bytes"] is None
|
||||
|
||||
|
||||
class TestGetPacketGroup:
|
||||
"""Tests for GET /packet-groups/{hash} (detail)."""
|
||||
|
||||
|
||||
@@ -144,3 +144,84 @@ class TestStoreRawPacketEdges:
|
||||
|
||||
rp = db_session.execute(select(RawPacket)).scalar_one()
|
||||
assert rp.source_pubkey_prefix is None
|
||||
|
||||
|
||||
class TestStoreRawPacketPathHashBytes:
|
||||
"""Tests for path_hash_bytes computation at ingest."""
|
||||
|
||||
def test_top_level_path_max_byte_width(self, db_manager, db_session):
|
||||
"""Mixed-width hashes in decoded.path persist the widest byte width."""
|
||||
decoded = {
|
||||
"payloadType": 1,
|
||||
"path": ["aa", "aabbcc"],
|
||||
"pathLength": 2,
|
||||
"payload": {"decoded": {"sourceHash": "AABBCCDD"}},
|
||||
}
|
||||
store_raw_packet(
|
||||
"a" * 64, {"raw": "00", "hash": "h1"}, decoded, "flood", db_manager
|
||||
)
|
||||
|
||||
rp = db_session.execute(select(RawPacket)).scalar_one()
|
||||
assert rp.path_hash_bytes == 3
|
||||
|
||||
def test_trace_fallback_path_hashes(self, db_manager, db_session):
|
||||
"""Trace-style pathHashes in payload.decoded are used as fallback."""
|
||||
decoded = {
|
||||
"payloadType": 1,
|
||||
"payload": {"decoded": {"pathHashes": ["aabb", "ccdd"]}},
|
||||
}
|
||||
store_raw_packet(
|
||||
"a" * 64, {"raw": "00", "hash": "h1"}, decoded, "trace", db_manager
|
||||
)
|
||||
|
||||
rp = db_session.execute(select(RawPacket)).scalar_one()
|
||||
assert rp.path_hash_bytes == 2
|
||||
|
||||
def test_one_byte_path(self, db_manager, db_session):
|
||||
"""Single-byte hashes persist width 1."""
|
||||
decoded = {
|
||||
"payloadType": 1,
|
||||
"path": ["aa", "bb"],
|
||||
"payload": {"decoded": {}},
|
||||
}
|
||||
store_raw_packet(
|
||||
"a" * 64, {"raw": "00", "hash": "h1"}, decoded, "flood", db_manager
|
||||
)
|
||||
|
||||
rp = db_session.execute(select(RawPacket)).scalar_one()
|
||||
assert rp.path_hash_bytes == 1
|
||||
|
||||
def test_no_path_returns_none(self, db_manager, db_session):
|
||||
"""No path hashes at all persist path_hash_bytes = None."""
|
||||
decoded = {"payloadType": 3, "payload": {"decoded": {}}}
|
||||
store_raw_packet(
|
||||
"a" * 64, {"raw": "00", "hash": "h1"}, decoded, "ack", db_manager
|
||||
)
|
||||
|
||||
rp = db_session.execute(select(RawPacket)).scalar_one()
|
||||
assert rp.path_hash_bytes is None
|
||||
|
||||
def test_null_decoded_returns_none(self, db_manager, db_session):
|
||||
"""A None decoded_packet leaves path_hash_bytes as None."""
|
||||
store_raw_packet(
|
||||
"a" * 64, {"raw": "00", "hash": "h1"}, None, "unknown", db_manager
|
||||
)
|
||||
|
||||
rp = db_session.execute(select(RawPacket)).scalar_one()
|
||||
assert rp.path_hash_bytes is None
|
||||
|
||||
def test_path_len_still_persisted(self, db_manager, db_session):
|
||||
"""path_len is still persisted alongside path_hash_bytes."""
|
||||
decoded = {
|
||||
"payloadType": 1,
|
||||
"path": ["aa", "bb", "cc"],
|
||||
"pathLength": 3,
|
||||
"payload": {"decoded": {"sourceHash": "AABBCCDD"}},
|
||||
}
|
||||
store_raw_packet(
|
||||
"a" * 64, {"raw": "00", "hash": "h1"}, decoded, "flood", db_manager
|
||||
)
|
||||
|
||||
rp = db_session.execute(select(RawPacket)).scalar_one()
|
||||
assert rp.path_len == 3
|
||||
assert rp.path_hash_bytes == 1
|
||||
|
||||
@@ -109,6 +109,7 @@ class TestRawPacketModel:
|
||||
assert packet.raw_hex is None
|
||||
assert packet.channel_idx is None
|
||||
assert packet.source_pubkey_prefix is None
|
||||
assert packet.path_hash_bytes is None
|
||||
assert packet.decoded is None
|
||||
|
||||
def test_raw_packet_indexes(self) -> None:
|
||||
|
||||
Reference in New Issue
Block a user