diff --git a/SCHEMAS.md b/SCHEMAS.md index 7ec89fe..e535687 100644 --- a/SCHEMAS.md +++ b/SCHEMAS.md @@ -47,7 +47,9 @@ Node advertisements announcing presence and metadata. "adv_type": "string (optional)", "flags": "integer (optional)", "lat": "number (optional)", - "lon": "number (optional)" + "lon": "number (optional)", + "route_type": "string (optional)", + "advert_timestamp": "integer (optional)" } ``` @@ -58,6 +60,8 @@ Node advertisements announcing presence and metadata. - `flags`: Node capability/status flags (bitmask) - `lat`: GPS latitude when provided by decoder metadata - `lon`: GPS longitude when provided by decoder metadata +- `route_type`: Route type of the advertisement packet — `"flood"` (original flood), `"transport_flood"` (relayed flood), `"direct"` (zero-hop local), `"transport_direct"` (relayed direct). Only present for LetsMesh-decoded adverts; native mode adverts have `route_type=NULL`. +- `advert_timestamp`: Node's own Unix timestamp (uint32) from the decoded advert payload. Used for deduplication when within ±4 hours of `received_at`. May be `NULL` for native mode adverts or when the decoder does not provide a timestamp. **Example**: ```json @@ -67,7 +71,9 @@ Node advertisements announcing presence and metadata. "adv_type": "repeater", "flags": 218, "lat": 42.470001, - "lon": -71.330001 + "lon": -71.330001, + "route_type": "flood", + "advert_timestamp": 1747300000 } ``` diff --git a/alembic/versions/20260515_1920_add_route_type_advert_timestamp.py b/alembic/versions/20260515_1920_add_route_type_advert_timestamp.py new file mode 100644 index 0000000..284adf7 --- /dev/null +++ b/alembic/versions/20260515_1920_add_route_type_advert_timestamp.py @@ -0,0 +1,33 @@ +"""add route_type and advert_timestamp to advertisements + +Revision ID: 20260515_1920 +Revises: 20260503_1800 +Create Date: 2026-05-15 19:20:00.000000 + +""" + +from typing import Sequence, Union + +import sqlalchemy as sa +from alembic import op + +revision: str = "20260515_1920" +down_revision: Union[str, None] = "20260503_1800" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + op.add_column( + "advertisements", + sa.Column("route_type", sa.String(20), nullable=True), + ) + op.add_column( + "advertisements", + sa.Column("advert_timestamp", sa.DateTime(timezone=True), nullable=True), + ) + + +def downgrade() -> None: + op.drop_column("advertisements", "advert_timestamp") + op.drop_column("advertisements", "route_type") diff --git a/docs/i18n.md b/docs/i18n.md index aed0f85..e13d7c3 100644 --- a/docs/i18n.md +++ b/docs/i18n.md @@ -297,7 +297,18 @@ Mobile sort dropdown labels for the nodes list page: ### 10. `advertisements` -Sort options for the advertisements list page: +Route type filter and sort options for the advertisements list page: + +#### Route Type Filter + +| Key | English | Context | +|-----|---------|---------| +| `filter_route_type_label` | Advert Type | Label for route type filter dropdown | +| `route_type_all` | All | Show all route types (no filter) | +| `route_type_flood` | Flood & Relay | Show flood and transport_flood adverts (default) | +| `route_type_direct` | Zero-hop only | Show only direct (zero-hop) adverts | +| `route_type_unknown` | Unknown | Displayed when route_type is NULL (historical records) | +| `col_route_type` | Type | Table column header for route type | #### Sort Options (`advertisements.sort`) diff --git a/docs/plans/20260515-1920-advertisement-frequency/plan.md b/docs/plans/20260515-1920-advertisement-frequency/plan.md new file mode 100644 index 0000000..2d1eb10 --- /dev/null +++ b/docs/plans/20260515-1920-advertisement-frequency/plan.md @@ -0,0 +1,267 @@ +# Advertisement Frequency Investigation + +**Date**: 2026-05-15 +**Status**: Decisions made, ready for implementation + +## Problem Statement + +Advertisements appear too frequently on the advertisements page. MeshCore nodes are expected to advertise at 6+ hour intervals (repeaters default to 12-hour flood adverts), yet the hub shows advertisements as frequently as every 3 hours for some nodes. + +## Research Findings + +### How the Collector Identifies Advertisements + +The collector identifies advertisements through a precise chain: + +1. **MQTT subscription**: `{prefix}/+/+/packets` (only the `packets` feed type is processed for advertisements) +2. **Decoding**: The `meshcoredecoder` library decodes the raw packet hex and extracts the header, including **payload type** and **route type** +3. **Normalization**: `LetsMeshNormalizer._build_letsmesh_advertisement_payload()` maps **decoded payload type `4` (PAYLOAD_TYPE_ADVERT)** to the `advertisement` event type +4. **Persistence**: `handle_advertisement()` creates an `Advertisement` database record + +The `status` and `internal` feed types are explicitly excluded from advertisement processing (they become `letsmesh_status` and `letsmesh_internal` event logs). + +### Decoded Advert Payload Structure (Type 4) + +The `meshcoredecoder` library decodes advert packets (type 4) into: + +| Field | Type | Description | +|-------|------|-------------| +| `publicKey` | string (64 hex) | Ed25519 public key of advertising node | +| `timestamp` | uint32 | **Node's internal Unix timestamp** when the advert was generated | +| `signature` | string (128 hex) | Ed25519 signature | +| `appData.flags` | uint8 | Flags byte: bits 0-3 = device role, bit 4 = hasLocation, bit 5 = hasFeature1, bit 6 = hasFeature2, bit 7 = hasName | +| `appData.deviceRole` | int | 1=Chat, 2=Repeater, 3=RoomServer, 4=Sensor | +| `appData.location` | object | `{latitude, longitude}` if hasLocation flag set | +| `appData.batteryVoltage` | float | Battery voltage in V (if HasFeature1 flag) | +| `appData.name` | string | Node name (if hasName flag set) | + +### Packet Header Route Types + +The decoded packet header includes a `routeType` field that the normalizer currently **ignores**: + +| Route Type | Value | Description | +|-----------|-------|-------------| +| `TransportFlood` | 0 | Repeater-forwarded flood advertisement | +| `Flood` | 1 | Original flood advertisement from source node | +| `Direct` | 2 | Direct/zero-hop advertisement (local broadcast) | +| `TransportDirect` | 3 | Repeater-forwarded direct message | + +### Root Cause: Zero-Hop + Flood Adverts + +MeshCore has **two distinct advertisement mechanisms** ([source: MeshCore FAQ](https://github.com/meshcore-dev/MeshCore/blob/main/docs/faq.md)): + +| Advert Type | CLI Command | Default Interval | Route Type | Behavior | +|-------------|-------------|-----------------|------------|----------| +| **Zero-hop (local)** | `set advert.interval {minutes}` | Varies by device | `Direct` (0x02) | Broadcast to nearby nodes only, not forwarded | +| **Flood** | `set flood.advert.interval {hours}` | 12 hours (repeaters) | `Flood` (0x01) → forwarded as `TransportFlood` (0x00) | Broadcast and repeated by all repeaters | + +**Both types have payload type 4.** The current normalizer does not differentiate between them. + +#### Frequency Analysis + +For a node with typical settings: +- Zero-hop interval: 240 minutes (4 hours) — common for companion nodes with auto-advert enabled +- Flood interval: 12 hours (default for repeaters) + +The observer captures both over the air, creating advertisement records at the **combined** rate. With both active, the observed interval is approximately every 3-4 hours, explaining the reported behavior. + +Additionally, **flood adverts are forwarded by repeaters** as `TransportFlood`. If the same flood arrives via different paths >120 seconds apart (unlikely for local mesh but possible for large networks), duplicate records are created. + +### Current Deduplication + +Advertisements are deduplicated using `compute_advertisement_hash()`: + +``` +MD5(public_key | name | adv_type | flags | time_bucket) +``` + +Where `time_bucket` rounds `received_at` down to the nearest **120-second** window. + +- **Same content + same node within 2 minutes**: Deduplicated (same hash) +- **Same content + same node after 2 minutes**: New record (different bucket) +- **Different content (e.g., name change)**: New record regardless of timing +- **Multi-observer**: When deduplicated, additional observers are recorded in the `event_observers` junction table with per-observer SNR/path_len + +### Available But Unused Data + +The decoded advert payload includes a `timestamp` field — the **node's own Unix timestamp** when it generated the advert. This field is: +- Decoded by `meshcoredecoder` and available in `decoded_payload.timestamp` +- **Not extracted** by the normalizer's `_build_letsmesh_advertisement_payload()` +- **Not stored** in the `Advertisement` model + +This timestamp could be used to: +- Detect relayed/delayed advertisements (node timestamp << received_at) +- Compute true advertisement intervals per node +- Distinguish original adverts from rebroadcasts (same node timestamp, different received_at) + +## Decision: Adopt Options A + C + D + +All three options are complementary and will be implemented together: + +- **Option A** — Store route type on the `Advertisement` model, expose in API/UI, default to showing only flood adverts +- **Option C** — Use the node's advert `timestamp` for deduplication time bucketing instead of `received_at`, and store it as `advert_timestamp` on the model +- **Option D** — Increase deduplication bucket from 120s to 300s (5 minutes) for both advertisements and telemetry + +Option B (skip zero-hop at ingestion) is rejected — preserving all data is preferable. + +### Design Decisions + +| Decision | Choice | Rationale | +|----------|--------|-----------| +| Include `route_type` in dedup hash? | **No** | Zero-hop and flood adverts from same node have different timestamps; edge case of identical content+timestamp is practically impossible. Simpler hash preferred. | +| Validate advert_timestamp? | **Yes: ±4h from `received_at`** | Node clocks without GPS may be wrong (0, far future, etc.). If advert_timestamp differs by >4 hours from `received_at`, fall back to `received_at` for dedup bucketing. Raw timestamp is always stored for diagnosis. | +| Dashboard metrics scope? | **All flood-only** | `total_advertisements`, `advertisements_24h`, `advertisements_7d`, `recent_advertisements`, and `/activity` endpoint all count flood-only (plus NULL for historical records). Consistent with default UI filter. | +| Telemetry bucket change? | **Yes, also 300s** | Apply consistent 5-minute bucketing to `compute_telemetry_hash()` alongside advertisements. | +| Route type in UI? | **Separate column** | Add a 5th "Type" column to the table with a route type badge. Mobile cards show badge inline next to node name. | + +### Combined Design + +#### 1. Normalizer Changes (`letsmesh_normalizer.py`) + +Extract two additional fields from the decoded packet in `_build_letsmesh_advertisement_payload()`: + +- `route_type` — from the top-level `routeType` field in the decoded packet (values: 0, 1, 2, 3) +- `advert_timestamp` — from `decoded_payload.timestamp` (uint32 Unix timestamp from the node) + +Map route type values to canonical strings: + +| routeType | Stored Value | Label | +|-----------|-------------|-------| +| 0 | `transport_flood` | Flood (relayed) | +| 1 | `flood` | Flood (original) | +| 2 | `direct` | Zero-hop (local) | +| 3 | `transport_direct` | Direct (relayed) | + +Both fields are added to `normalized_payload` and passed through to the handler. + +#### 2. Deduplication Changes (`hash_utils.py`) + +Update `compute_advertisement_hash()` to: + +- Accept an optional `advert_timestamp` parameter (datetime from the node's timestamp) +- Use `advert_timestamp` for time bucketing when provided, falling back to `received_at` +- Increase default `bucket_seconds` from 120 to **300** (5 minutes) +- Fix existing docstring bug: docstring says default is 30, actual is 120 (correct to 300) + +Also update `compute_telemetry_hash()` to increase `bucket_seconds` default from 120 to 300 for consistency. + +#### 3. Advert Timestamp Validation (`handlers/advertisement.py`) + +Before using `advert_timestamp` for dedup bucketing, validate it: + +``` +delta = abs(advert_timestamp - received_at_datetime) +if delta > timedelta(hours=4): + advert_timestamp_for_hash = None # fall back to received_at +else: + advert_timestamp_for_hash = advert_timestamp_datetime +``` + +Raw `advert_timestamp` is always stored on the model regardless of validation, so operators can diagnose clock skew. + +#### 4. Database Model Changes (`models/advertisement.py`) + +Add two new nullable columns to the `Advertisement` model: + +- `route_type: Mapped[str | None]` — canonical route type string (`"flood"`, `"transport_flood"`, `"direct"`, `"transport_direct"`) +- `advert_timestamp: Mapped[datetime | None]` — node's own timestamp from the decoded advert payload (stored as timezone-aware UTC) + +Both nullable to maintain backward compatibility with existing records. + +#### 5. Handler Changes (`handlers/advertisement.py`) + +- Extract `route_type` and `advert_timestamp` from the normalized payload +- Validate `advert_timestamp` (±4h window vs `received_at`); if invalid, use `received_at` for dedup bucketing, still store raw value +- Convert valid `advert_timestamp` (uint32 epoch) to `datetime` with `datetime.fromtimestamp(ts, tz=timezone.utc)` +- Pass validated `advert_timestamp` to `compute_advertisement_hash()` for time bucketing +- Store `route_type` and `advert_timestamp` on the `Advertisement` record + +#### 6. API Changes (`api/routes/advertisements.py`) + +Add `route_type` query parameter to `GET /api/v1/advertisements`: + +- Accept comma-separated values: e.g. `flood,transport_flood` +- Default: `flood,transport_flood` — excludes zero-hop (direct) adverts by default +- SQL: `WHERE route_type IN ('flood', 'transport_flood') OR route_type IS NULL` (NULL included for historical records) +- Pass empty string to show all route types (no WHERE clause on route_type) +- `route_type=none` or `route_type=all` as alternate ways to disable filter + +Add `route_type` and `advert_timestamp` to the `AdvertisementRead` schema. + +Add `route_type` to the single-advert `GET /{advertisement_id}` response schema. + +#### 7. Dashboard Changes (`routes/dashboard.py`) + +All advertisement metrics in `/stats` switch to flood-only: + +| Metric | Change | +|--------|--------| +| `total_advertisements` | Count only `route_type IN ('flood', 'transport_flood') OR route_type IS NULL` | +| `advertisements_24h` | Same filter + `received_at >= 24h ago` | +| `advertisements_7d` | Same filter + `received_at >= 7d ago` | +| `recent_advertisements` (last 10) | Same filter | + +The `/activity` endpoint also applies the flood-only filter. + +`NULL` route types are included in all dashboard counts to avoid hiding historical data. + +#### 8. Frontend Changes (`pages/advertisements.js`) + +**Route type filter dropdown:** +- Options: "Flood & Relay" (default, `?route_type=flood,transport_flood`), "All" (no filter), "Zero-hop only" (`?route_type=direct`) +- Added to the filter card alongside existing search and observer filters + +**Route type column:** +- Add a 5th column "Type" between "Public Key" and "Time" in the table +- Display as colored badge: `flood`/`transport_flood` (blue), `direct` (green), NULL (gray "Unknown") +- Mobile cards: badge shown inline next to node name + +**i18n keys needed in `en.json`:** +``` +"advertisements": { + "filter_route_type_label": "Advert Type", + "route_type_all": "All", + "route_type_flood": "Flood & Relay", + "route_type_direct": "Zero-hop only", + "route_type_unknown": "Unknown", + "col_route_type": "Type", + ... +} +``` + +#### 9. Existing Data Migration + +Existing `Advertisement` records will have `route_type=NULL` and `advert_timestamp=NULL`. All API and dashboard filters that default to flood-only must include `WHERE route_type IS NULL` to avoid hiding historical data. A future cleanup could backfill these from stored `event_hash` patterns or decoded payload logs, but this is not required for initial implementation. + +### Deduplication Behavior (After Change) + +| Scenario | Before | After | +|----------|--------|-------| +| Same node, same content, same flood, 2 observers <2min apart | Deduplicated (1 record, 2 observers) | Deduplicated (1 record, 2 observers) | +| Same node, same content, same flood, 2 observers 3min apart | **2 records** (different 120s buckets) | Deduplicated (same node timestamp, 5min bucket) | +| Same node, zero-hop at T, flood at T+4h | 2 records | 2 records (different node timestamps, 4h apart) | +| Same node, flood original + flood relayed 30s later | Deduplicated | Deduplicated (same node timestamp) | +| Same node, content changed (name update) | 2 records | 2 records (different hash) | +| Node with broken clock (timestamp off by 2 days) | N/A (no advert_timestamp) | Uses `received_at` for bucketing (validation rejects timestamp), raw value still stored | + +## Affected Files + +| File | Change | +|------|--------| +| `src/meshcore_hub/common/models/advertisement.py` | Add `route_type` and `advert_timestamp` columns | +| `src/meshcore_hub/common/hash_utils.py` | Add `advert_timestamp` parameter, increase buckets to 300s (adv + telemetry), fix docstring | +| `src/meshcore_hub/collector/letsmesh_normalizer.py` | Extract `routeType` and `timestamp` from decoded packet | +| `src/meshcore_hub/collector/handlers/advertisement.py` | Pass route type + advert timestamp, validate ±4h, use for dedup | +| `src/meshcore_hub/common/schemas/messages.py` | Add `route_type` and `advert_timestamp` to `AdvertisementRead` | +| `src/meshcore_hub/api/routes/advertisements.py` | Add `route_type` filter parameter, update base query and single-advert endpoint | +| `src/meshcore_hub/api/routes/dashboard.py` | Apply flood-only filter to all ad counts and activity endpoint | +| `src/meshcore_hub/web/static/js/spa/pages/advertisements.js` | Add route type filter dropdown + 5th "Type" column | +| `src/meshcore_hub/web/static/locales/en.json` | Add route type filter labels and column header | +| `alembic/versions/` | Migration for `route_type` + `advert_timestamp` columns | +| `docs/upgrading.md` | Document new fields, default filter behavior, telemetry bucket change | +| `docs/i18n.md` | Document new `advertisements.*` translation keys | +| `SCHEMAS.md` | Update advertisement schema with `route_type` and `advert_timestamp` | +| `tests/test_collector/test_letsmesh_normalizer.py` | Verify `route_type` and `advert_timestamp` extraction | +| `tests/test_common/test_hash_utils.py` | Verify new hash behavior with advert_timestamp, bucket changes | +| `tests/test_api/test_advertisements.py` | Verify `route_type` filter parameter | +| `tests/test_web/` | Verify `route_type` and `advert_timestamp` in `AdvertisementRead` responses | diff --git a/docs/plans/20260515-1920-advertisement-frequency/tasks.md b/docs/plans/20260515-1920-advertisement-frequency/tasks.md new file mode 100644 index 0000000..1ae5f68 --- /dev/null +++ b/docs/plans/20260515-1920-advertisement-frequency/tasks.md @@ -0,0 +1,108 @@ +# Advertisement Frequency — Task List + +**Plan**: [plan.md](./plan.md) + +## Implementation Order + +Tasks are ordered by dependency. Each task depends on the previous ones being complete unless noted. + +--- + +### Phase 1: Data Layer + +- [x] **T1. Normalizer: extract route_type and advert_timestamp** + - File: `src/meshcore_hub/collector/letsmesh_normalizer.py` + - Extract `routeType` from decoded packet header, map to canonical string (`transport_flood`, `flood`, `direct`, `transport_direct`) + - Extract `timestamp` from `decoded_payload.timestamp` as `advert_timestamp` + - Add both to `normalized_payload` in `_build_letsmesh_advertisement_payload()` + - Tests: `tests/test_collector/test_letsmesh_normalizer.py` + +- [x] **T2. Hash utils: advert_timestamp parameter + 300s buckets** + - File: `src/meshcore_hub/common/hash_utils.py` + - Add optional `advert_timestamp` parameter to `compute_advertisement_hash()` + - Use `advert_timestamp` for time bucketing when provided, fall back to `received_at` + - Increase `bucket_seconds` default from 120 to 300 for `compute_advertisement_hash()` + - Increase `bucket_seconds` default from 120 to 300 for `compute_telemetry_hash()` + - Fix docstring (currently says default is 30, correct to 300) + - Tests: `tests/test_common/test_hash_utils.py` + +- [x] **T3. Database model: add route_type and advert_timestamp columns** + - File: `src/meshcore_hub/common/models/advertisement.py` + - Add `route_type: Mapped[str | None]` column (nullable) + - Add `advert_timestamp: Mapped[datetime | None]` column (nullable) + - Both nullable for backward compatibility with existing records + +- [x] **T4. Alembic migration** + - Migration: `alembic/versions/20260515_1920_add_route_type_advert_timestamp.py` + - Adds `route_type` (VARCHAR(20), nullable) and `advert_timestamp` (DATETIME timezone, nullable) + +- [x] **T5. Handler: pass route_type + advert_timestamp, validate ±4h** + - File: `src/meshcore_hub/collector/handlers/advertisement.py` + - Extract `route_type` and `advert_timestamp` from normalized payload + - Validate `advert_timestamp`: if `abs(advert_timestamp - received_at) > 4h`, use `None` for hash bucketing + - Convert `advert_timestamp` from uint32 epoch to `datetime` with `datetime.fromtimestamp(ts, tz=timezone.utc)` + - Pass validated timestamp to `compute_advertisement_hash()` + - Store `route_type` and `advert_timestamp` on `Advertisement` record + - Tests: `tests/test_collector/test_handlers/test_advertisement.py` + +--- + +### Phase 2: API + +- [x] **T6. Schema: add route_type and advert_timestamp to AdvertisementRead** + - File: `src/meshcore_hub/common/schemas/messages.py` + - Add `route_type: Optional[str] = None` to `AdvertisementRead` + - Add `advert_timestamp: Optional[datetime] = None` to `AdvertisementRead` + +- [x] **T7. API: route_type filter on advertisements endpoint** + - File: `src/meshcore_hub/api/routes/advertisements.py` + - Add `route_type` query parameter to `GET /api/v1/advertisements` + - Accept comma-separated values (e.g. `flood,transport_flood`) + - Default: `flood,transport_flood` + - SQL filter: `WHERE route_type IN (...) OR route_type IS NULL` + - Support `all`, `none`, or empty string to disable filter + - Add `route_type` and `advert_timestamp` to single-advert `GET /{advertisement_id}` response + - Tests: `tests/test_api/test_advertisements.py` + +- [x] **T8. Dashboard: flood-only filter on all ad metrics** + - File: `src/meshcore_hub/api/routes/dashboard.py` + - Apply `route_type IN ('flood', 'transport_flood') OR route_type IS NULL` to: + - `total_advertisements` + - `advertisements_24h` + - `advertisements_7d` + - `recent_advertisements` + - Apply same filter to `/activity` endpoint + - Tests: `tests/test_api/test_dashboard.py` + +--- + +### Phase 3: Frontend + +- [x] **T9. i18n: add route type translation keys** + - File: `src/meshcore_hub/web/static/locales/en.json` + - Add keys: `advertisements.filter_route_type_label`, `advertisements.route_type_all`, `advertisements.route_type_flood`, `advertisements.route_type_direct`, `advertisements.route_type_unknown`, `advertisements.col_route_type` + - File: `docs/i18n.md` — document new keys + +- [x] **T10. Advertisements page: route type filter + Type column** + - File: `src/meshcore_hub/web/static/js/spa/pages/advertisements.js` + - Add route type dropdown to filter card: "Flood & Relay" (default), "All", "Zero-hop only" + - Add 5th "Type" column between "Public Key" and "Time" + - Colored badges: `flood`/`transport_flood` (blue), `direct` (green), NULL (gray "Unknown") + - Mobile cards: badge inline next to node name + - Pass `route_type` query parameter to API calls + +--- + +### Phase 4: Documentation + +- [x] **T11. Update documentation** + - `docs/upgrading.md` — document new fields, default filter behavior, telemetry bucket change (120→300s) + - `SCHEMAS.md` — update advertisement schema with `route_type` and `advert_timestamp` fields + +--- + +### Phase 5: Verification + +- [x] **T12. Run tests and quality checks** + - `pytest tests/` — 726 passed, 22 skipped (E2E) + - `pre-commit run --all-files` — all hooks passed (black, flake8, mypy) diff --git a/docs/upgrading.md b/docs/upgrading.md index 1cc21b0..3270e7b 100644 --- a/docs/upgrading.md +++ b/docs/upgrading.md @@ -2,6 +2,37 @@ This guide covers upgrading from a previous MeshCore Hub release to the current version. Check the relevant version section below before upgrading. +## v0.12.0 + +### Advertisement Route Type & Deduplication Improvements + +This release adds route type tracking and improves advertisement deduplication to better distinguish between flood and zero-hop (local) advertisements. + +**New database columns on `advertisements` table:** + +| Column | Type | Description | +|--------|------|-------------| +| `route_type` | `VARCHAR(20), nullable` | Route type: `flood`, `transport_flood`, `direct`, `transport_direct` | +| `advert_timestamp` | `DATETIME, nullable` | Node's own Unix timestamp from the advert payload | + +Both columns are nullable — existing records will have `NULL` values. The Alembic migration adds these columns automatically. + +**Default API filter change:** + +`GET /api/v1/advertisements` now defaults to `route_type=flood,transport_flood`, showing only flood advertisements. Existing records with `route_type=NULL` are included in all default queries to avoid hiding historical data. Pass `route_type=all` to see all types. + +**Dashboard metrics now flood-only:** + +All dashboard advertisement counts (`total_advertisements`, `advertisements_24h`, `advertisements_7d`, `recent_advertisements`, and `/activity`) now count only flood/transport_flood adverts plus NULL (historical records). + +**Deduplication bucket increased from 120s to 300s:** + +Both `compute_advertisement_hash()` and `compute_telemetry_hash()` now use a 5-minute (300-second) deduplication bucket instead of the previous 2-minute (120-second) bucket. This reduces duplicate records when multiple observers report the same event within a 5-minute window. + +**Advertisement deduplication now uses node timestamp:** + +When available, the node's own `advert_timestamp` is used for deduplication bucketing instead of `received_at`. This means the same flood advertisement observed by multiple receivers will correctly deduplicate even if received several minutes apart. Node timestamps that deviate by more than 4 hours from `received_at` are rejected for bucketing (the raw value is still stored). + ## v0.11.0 ### Async SQLite Foreign Key Fix diff --git a/src/meshcore_hub/api/routes/advertisements.py b/src/meshcore_hub/api/routes/advertisements.py index f4ff043..f770141 100644 --- a/src/meshcore_hub/api/routes/advertisements.py +++ b/src/meshcore_hub/api/routes/advertisements.py @@ -19,6 +19,8 @@ from meshcore_hub.common.schemas.messages import ( router = APIRouter() VALID_AD_SORT_COLUMNS = {"time", "node_name", "public_key"} +DEFAULT_FLOOD_ROUTE_TYPES = {"flood", "transport_flood"} +DISABLE_FILTER_VALUES = {"all", "none", ""} def _get_tag_name(node: Optional[Node]) -> Optional[str]: @@ -57,6 +59,10 @@ async def list_advertisements( adopted_by: Optional[str] = Query( None, description="Filter by adopting user profile UUID" ), + route_type: Optional[str] = Query( + "flood,transport_flood", + description="Comma-separated route types (flood, transport_flood, direct, transport_direct). Use 'all' to show all.", + ), since: Optional[datetime] = Query(None, description="Start timestamp"), until: Optional[datetime] = Query(None, description="End timestamp"), sort: Optional[str] = Query(None, description="Sort column"), @@ -115,6 +121,18 @@ async def list_advertisements( ) ) + if route_type and route_type.strip().lower() not in DISABLE_FILTER_VALUES: + requested_types = { + t.strip().lower() for t in route_type.split(",") if t.strip() + } + if requested_types: + query = query.where( + or_( + Advertisement.route_type.in_(requested_types), + Advertisement.route_type.is_(None), + ) + ) + if since: query = query.where(Advertisement.received_at >= since) @@ -203,6 +221,8 @@ async def list_advertisements( "node_tag_description": _get_tag_description(source_node), "adv_type": adv.adv_type or row.source_adv_type, "flags": adv.flags, + "route_type": adv.route_type, + "advert_timestamp": adv.advert_timestamp, "received_at": adv.received_at, "created_at": adv.created_at, "observers": ( @@ -286,6 +306,8 @@ async def get_advertisement( "node_tag_description": _get_tag_description(source_node), "adv_type": adv.adv_type or result.source_adv_type, "flags": adv.flags, + "route_type": adv.route_type, + "advert_timestamp": adv.advert_timestamp, "received_at": adv.received_at, "created_at": adv.created_at, "observers": observers, diff --git a/src/meshcore_hub/api/routes/dashboard.py b/src/meshcore_hub/api/routes/dashboard.py index 7da7ac8..44b9297 100644 --- a/src/meshcore_hub/api/routes/dashboard.py +++ b/src/meshcore_hub/api/routes/dashboard.py @@ -3,7 +3,8 @@ from datetime import datetime, timedelta, timezone from fastapi import APIRouter -from sqlalchemy import func, select +from sqlalchemy import func, or_, select +from sqlalchemy.sql.elements import ColumnElement from meshcore_hub.api.auth import RequireRead from meshcore_hub.api.dependencies import DbSession @@ -26,6 +27,21 @@ from meshcore_hub.common.schemas.messages import ( router = APIRouter() +_FLOOD_ROUTE_TYPES = {"flood", "transport_flood"} + + +def _flood_only_filter( + ad_model: type[Advertisement], +) -> ColumnElement[bool]: + """Build a flood-only filter clause for advertisement queries. + + Includes flood/transport_flood records and NULL (historical) records. + """ + return or_( + ad_model.route_type.in_(_FLOOD_ROUTE_TYPES), + ad_model.route_type.is_(None), + ) + @router.get("/stats", response_model=DashboardStats) async def get_stats( @@ -64,27 +80,34 @@ async def get_stats( or 0 ) - # Total advertisements + # Total advertisements (flood-only) total_advertisements = ( - session.execute(select(func.count()).select_from(Advertisement)).scalar() or 0 + session.execute( + select(func.count()) + .select_from(Advertisement) + .where(_flood_only_filter(Advertisement)) + ).scalar() + or 0 ) - # Advertisements in last 24h + # Advertisements in last 24h (flood-only) advertisements_24h = ( session.execute( select(func.count()) .select_from(Advertisement) .where(Advertisement.received_at >= yesterday) + .where(_flood_only_filter(Advertisement)) ).scalar() or 0 ) - # Advertisements in last 7 days + # Advertisements in last 7 days (flood-only) advertisements_7d = ( session.execute( select(func.count()) .select_from(Advertisement) .where(Advertisement.received_at >= seven_days_ago) + .where(_flood_only_filter(Advertisement)) ).scalar() or 0 ) @@ -99,10 +122,13 @@ async def get_stats( or 0 ) - # Recent advertisements (last 10) + # Recent advertisements (last 10, flood-only) recent_ads = ( session.execute( - select(Advertisement).order_by(Advertisement.received_at.desc()).limit(10) + select(Advertisement) + .where(_flood_only_filter(Advertisement)) + .order_by(Advertisement.received_at.desc()) + .limit(10) ) .scalars() .all() @@ -285,6 +311,7 @@ async def get_activity( ) .where(Advertisement.received_at >= start_date) .where(Advertisement.received_at < end_date) + .where(_flood_only_filter(Advertisement)) .group_by(date_expr) .order_by(date_expr) ) diff --git a/src/meshcore_hub/collector/handlers/advertisement.py b/src/meshcore_hub/collector/handlers/advertisement.py index e92b07b..4b9088f 100644 --- a/src/meshcore_hub/collector/handlers/advertisement.py +++ b/src/meshcore_hub/collector/handlers/advertisement.py @@ -1,7 +1,7 @@ """Handler for advertisement events.""" import logging -from datetime import datetime, timezone +from datetime import datetime, timedelta, timezone from typing import Any from sqlalchemy import select @@ -74,14 +74,31 @@ def handle_advertisement( snr = payload.get("snr") path_len = payload.get("path_len") + route_type = payload.get("route_type") - # Compute event hash for deduplication (30-second time bucket) + advert_timestamp_epoch = payload.get("advert_timestamp") + advert_timestamp_dt: datetime | None = None + advert_timestamp_for_hash: datetime | None = None + if isinstance(advert_timestamp_epoch, (int, float)): + try: + advert_timestamp_dt = datetime.fromtimestamp( + int(advert_timestamp_epoch), tz=timezone.utc + ) + except (OSError, OverflowError, ValueError): + advert_timestamp_dt = None + if advert_timestamp_dt is not None: + delta = abs(advert_timestamp_dt - now) + if delta <= timedelta(hours=4): + advert_timestamp_for_hash = advert_timestamp_dt + + # Compute event hash for deduplication (5-minute time bucket) event_hash = compute_advertisement_hash( public_key=adv_public_key, name=name, adv_type=adv_type, flags=flags, received_at=now, + advert_timestamp=advert_timestamp_for_hash, ) with db.session_scope() as session: @@ -178,6 +195,8 @@ def handle_advertisement( flags=flags, received_at=now, event_hash=event_hash, + route_type=route_type, + advert_timestamp=advert_timestamp_dt, ) session.add(advertisement) diff --git a/src/meshcore_hub/collector/letsmesh_normalizer.py b/src/meshcore_hub/collector/letsmesh_normalizer.py index c292312..5cb37ce 100644 --- a/src/meshcore_hub/collector/letsmesh_normalizer.py +++ b/src/meshcore_hub/collector/letsmesh_normalizer.py @@ -542,6 +542,13 @@ class LetsMeshNormalizer: return None + _ROUTE_TYPE_MAP: dict[int, str] = { + 0: "transport_flood", + 1: "flood", + 2: "direct", + 3: "transport_direct", + } + def _build_letsmesh_advertisement_payload( self, payload: dict[str, Any], @@ -576,6 +583,14 @@ class LetsMeshNormalizer: "public_key": public_key, } + route_type_raw = self._parse_int(decoded_packet.get("routeType")) + if route_type_raw is not None and route_type_raw in self._ROUTE_TYPE_MAP: + normalized_payload["route_type"] = self._ROUTE_TYPE_MAP[route_type_raw] + + advert_timestamp_raw = self._parse_int(decoded_payload.get("timestamp")) + if advert_timestamp_raw is not None: + normalized_payload["advert_timestamp"] = advert_timestamp_raw + snr = self._parse_float(payload.get("SNR")) if snr is None: snr = self._parse_float(payload.get("snr")) diff --git a/src/meshcore_hub/common/hash_utils.py b/src/meshcore_hub/common/hash_utils.py index d172bce..efdd6f8 100644 --- a/src/meshcore_hub/common/hash_utils.py +++ b/src/meshcore_hub/common/hash_utils.py @@ -49,7 +49,8 @@ def compute_advertisement_hash( adv_type: Optional[str] = None, flags: Optional[int] = None, received_at: Optional[datetime] = None, - bucket_seconds: int = 120, + bucket_seconds: int = 300, + advert_timestamp: Optional[datetime] = None, ) -> str: """Compute a deterministic hash for an advertisement. @@ -62,16 +63,18 @@ def compute_advertisement_hash( adv_type: Node type flags: Capability flags received_at: When received (used for time bucketing) - bucket_seconds: Time bucket size in seconds (default 30) + bucket_seconds: Time bucket size in seconds (default 300) + advert_timestamp: Node's own timestamp from the advert payload. + When provided, used for time bucketing instead of received_at. Returns: 32-character hex hash string """ - # Bucket the time to allow deduplication within a window + bucket_time = advert_timestamp if advert_timestamp is not None else received_at + time_bucket = "" - if received_at: - # Round down to nearest bucket - epoch = int(received_at.timestamp()) + if bucket_time: + epoch = int(bucket_time.timestamp()) bucket_epoch = (epoch // bucket_seconds) * bucket_seconds time_bucket = str(bucket_epoch) @@ -104,7 +107,7 @@ def compute_telemetry_hash( node_public_key: str, parsed_data: Optional[dict] = None, received_at: Optional[datetime] = None, - bucket_seconds: int = 120, + bucket_seconds: int = 300, ) -> str: """Compute a deterministic hash for a telemetry record. @@ -114,7 +117,7 @@ def compute_telemetry_hash( node_public_key: Reporting node's public key parsed_data: Decoded sensor readings received_at: When received (used for time bucketing) - bucket_seconds: Time bucket size in seconds (default 30) + bucket_seconds: Time bucket size in seconds (default 300) Returns: 32-character hex hash string diff --git a/src/meshcore_hub/common/models/advertisement.py b/src/meshcore_hub/common/models/advertisement.py index 4e31d29..6ddc901 100644 --- a/src/meshcore_hub/common/models/advertisement.py +++ b/src/meshcore_hub/common/models/advertisement.py @@ -63,6 +63,14 @@ class Advertisement(Base, UUIDMixin, TimestampMixin): nullable=True, unique=True, ) + route_type: Mapped[Optional[str]] = mapped_column( + String(20), + nullable=True, + ) + advert_timestamp: Mapped[Optional[datetime]] = mapped_column( + DateTime(timezone=True), + nullable=True, + ) __table_args__ = (Index("ix_advertisements_received_at", "received_at"),) diff --git a/src/meshcore_hub/common/schemas/messages.py b/src/meshcore_hub/common/schemas/messages.py index 10dc2c0..c86e02a 100644 --- a/src/meshcore_hub/common/schemas/messages.py +++ b/src/meshcore_hub/common/schemas/messages.py @@ -129,6 +129,13 @@ class AdvertisementRead(BaseModel): ) adv_type: Optional[str] = Field(default=None, description="Node type") flags: Optional[int] = Field(default=None, description="Capability flags") + route_type: Optional[str] = Field( + default=None, + description="Route type: flood, transport_flood, direct, transport_direct", + ) + advert_timestamp: Optional[datetime] = Field( + default=None, description="Node's own timestamp from advert payload" + ) received_at: datetime = Field(..., description="When received") created_at: datetime = Field(..., description="Record creation timestamp") observers: list[ObserverInfo] = Field( diff --git a/src/meshcore_hub/web/static/js/spa/components.js b/src/meshcore_hub/web/static/js/spa/components.js index ad04b2e..e985fbb 100644 --- a/src/meshcore_hub/web/static/js/spa/components.js +++ b/src/meshcore_hub/web/static/js/spa/components.js @@ -542,7 +542,7 @@ export function observerIcons(observers) { if (!observers || observers.length === 0) return nothing; const names = observers.map(o => o.tag_name || o.name || truncateKey(o.public_key, 8)); const tooltip = names.join(', '); - return html`\u{1F4E1}${observers.length}`; + return html`${observers.length}`; } /** diff --git a/src/meshcore_hub/web/static/js/spa/pages/advertisements.js b/src/meshcore_hub/web/static/js/spa/pages/advertisements.js index 6af7d28..f7b2f76 100644 --- a/src/meshcore_hub/web/static/js/spa/pages/advertisements.js +++ b/src/meshcore_hub/web/static/js/spa/pages/advertisements.js @@ -9,6 +9,19 @@ import { } from '../components.js'; import { createAutoRefresh } from '../auto-refresh.js'; +function routeTypeBadge(routeType) { + if (!routeType) { + return nothing; + } + if (routeType === 'flood' || routeType === 'transport_flood') { + return html`${routeType === 'flood' ? 'Flood' : 'Relay'}`; + } + if (routeType === 'direct' || routeType === 'transport_direct') { + return html`${routeType === 'direct' ? 'Zero-hop' : 'Direct relay'}`; + } + return nothing; +} + export async function render(container, params, router) { const query = params.query || {}; const search = query.search || ''; @@ -16,6 +29,7 @@ export async function render(container, params, router) { ? (Array.isArray(query.observed_by) ? query.observed_by : [query.observed_by]) : []; const adopted_by = query.adopted_by || ''; + const route_type = query.route_type || 'flood,transport_flood'; const page = parseInt(query.page, 10) || 1; const limit = parseInt(query.limit, 10) || 20; const offset = (page - 1) * limit; @@ -56,7 +70,7 @@ ${displayContent}`, container); async function fetchAndRenderData() { try { - const apiParams = { limit, offset, search, sort, order }; + const apiParams = { limit, offset, search, sort, order, route_type }; if (observed_by.length > 0) apiParams.observed_by = observed_by; if (adopted_by) apiParams.adopted_by = adopted_by; const fetches = [ @@ -125,7 +139,10 @@ ${displayContent}`, container); })}
${formatDateTimeShort(ad.received_at)}
- ${receiversBlock} +
+ ${routeTypeBadge(ad.route_type)} + ${receiversBlock} +
${ad.observers && ad.observers.length > 0 ? html` @@ -152,7 +169,7 @@ ${displayContent}`, container); }); const tableRows = advertisements.length === 0 - ? html`${t('common.no_entity_found', { entity: t('entities.advertisements').toLowerCase() })}` + ? html`${t('common.no_entity_found', { entity: t('entities.advertisements').toLowerCase() })}` : advertisements.map(ad => { const adName = ad.node_tag_name || ad.node_name || ad.name; const adDescription = ad.node_tag_description; @@ -181,13 +198,14 @@ ${displayContent}`, container); @click=${(e) => copyToClipboard(e, ad.public_key)} title="Click to copy">${ad.public_key} + ${routeTypeBadge(ad.route_type)} ${formatDateTime(ad.received_at)} ${receiversBlock} ${observerDetailRow(ad.observers || [], null, { hidePath: true })}`; }); const paginationBlock = pagination(page, totalPages, '/advertisements', { - search, observed_by, adopted_by, limit, sort, order, + search, observed_by, adopted_by, route_type, limit, sort, order, }); const filterFields = [ @@ -197,6 +215,17 @@ ${displayContent}`, container); ${t('common.search')} + `, + () => html` +
+ +
`, ]; if (config.oidc_enabled && profiles.length > 0) { @@ -222,7 +251,7 @@ ${displayContent}`, container); filterFields.push(() => nodesFilter); } - const hasActiveFilters = search !== '' || observed_by.length > 0 || (config.oidc_enabled && adopted_by !== ''); + const hasActiveFilters = search !== '' || observed_by.length > 0 || (config.oidc_enabled && adopted_by !== '') || route_type !== 'flood,transport_flood'; const existingDetails = container.querySelector('details.collapse'); const isFilterOpen = existingDetails ? existingDetails.open : hasActiveFilters; @@ -234,7 +263,7 @@ ${displayContent}`, container); defaultOpen: isFilterOpen, }); - const headerParams = { search, observed_by, adopted_by, limit }; + const headerParams = { search, observed_by, adopted_by, route_type, limit }; const sortable = (label, sortKey) => sortableTableHeader(label, { sortKey, currentSort: sort, currentOrder: order, navigate, basePath: '/advertisements', params: headerParams, @@ -266,6 +295,7 @@ ${mobileSortSelect({ ${sortable(t('entities.node'), 'node_name')} ${sortable(t('common.public_key'), 'public_key')} + ${t('advertisements.col_route_type')} ${sortable(t('common.time'), 'time')} ${t('common.observers')} diff --git a/src/meshcore_hub/web/static/locales/en.json b/src/meshcore_hub/web/static/locales/en.json index f1ff13a..d1f6fcd 100644 --- a/src/meshcore_hub/web/static/locales/en.json +++ b/src/meshcore_hub/web/static/locales/en.json @@ -175,6 +175,12 @@ } }, "advertisements": { + "filter_route_type_label": "Advert Type", + "route_type_all": "All", + "route_type_flood": "Flood & Relay", + "route_type_direct": "Zero-hop only", + "route_type_unknown": "Unknown", + "col_route_type": "Type", "sort": { "newest": "Time (newest)", "oldest": "Time (oldest)", diff --git a/tests/test_api/test_advertisements.py b/tests/test_api/test_advertisements.py index c033da6..dda9510 100644 --- a/tests/test_api/test_advertisements.py +++ b/tests/test_api/test_advertisements.py @@ -525,3 +525,140 @@ class TestAdvertisementSort: assert response.status_code == 200 items = response.json()["items"] assert items[0]["name"] == "New" + + +class TestListAdvertisementsRouteTypeFilter: + """Tests for route_type query parameter on advertisements endpoint.""" + + def test_default_filter_shows_flood_and_null(self, client_no_auth, api_db_session): + """Default route_type filter shows flood, transport_flood, and NULL.""" + now = datetime.now(timezone.utc) + flood_ad = Advertisement( + public_key="aa" * 16, + name="Flood", + adv_type="CLIENT", + received_at=now, + route_type="flood", + ) + null_ad = Advertisement( + public_key="bb" * 16, + name="Historical", + adv_type="CLIENT", + received_at=now, + route_type=None, + ) + direct_ad = Advertisement( + public_key="cc" * 16, + name="Direct", + adv_type="CLIENT", + received_at=now, + route_type="direct", + ) + api_db_session.add_all([flood_ad, null_ad, direct_ad]) + api_db_session.commit() + + response = client_no_auth.get("/api/v1/advertisements") + assert response.status_code == 200 + data = response.json() + assert data["total"] == 2 + names = {item["name"] for item in data["items"]} + assert names == {"Flood", "Historical"} + + def test_filter_all_shows_all(self, client_no_auth, api_db_session): + """route_type=all shows all advertisements.""" + now = datetime.now(timezone.utc) + flood_ad = Advertisement( + public_key="aa" * 16, + name="Flood", + adv_type="CLIENT", + received_at=now, + route_type="flood", + ) + direct_ad = Advertisement( + public_key="cc" * 16, + name="Direct", + adv_type="CLIENT", + received_at=now, + route_type="direct", + ) + api_db_session.add_all([flood_ad, direct_ad]) + api_db_session.commit() + + response = client_no_auth.get("/api/v1/advertisements?route_type=all") + assert response.status_code == 200 + data = response.json() + assert data["total"] == 2 + + def test_filter_direct_only(self, client_no_auth, api_db_session): + """route_type=direct shows only direct and NULL.""" + now = datetime.now(timezone.utc) + flood_ad = Advertisement( + public_key="aa" * 16, + name="Flood", + adv_type="CLIENT", + received_at=now, + route_type="flood", + ) + direct_ad = Advertisement( + public_key="cc" * 16, + name="Direct", + adv_type="CLIENT", + received_at=now, + route_type="direct", + ) + null_ad = Advertisement( + public_key="dd" * 16, + name="Historical", + adv_type="CLIENT", + received_at=now, + route_type=None, + ) + api_db_session.add_all([flood_ad, direct_ad, null_ad]) + api_db_session.commit() + + response = client_no_auth.get("/api/v1/advertisements?route_type=direct") + assert response.status_code == 200 + data = response.json() + assert data["total"] == 2 + names = {item["name"] for item in data["items"]} + assert names == {"Direct", "Historical"} + + def test_route_type_in_response(self, client_no_auth, api_db_session): + """route_type and advert_timestamp are included in response.""" + now = datetime.now(timezone.utc) + ad = Advertisement( + public_key="aa" * 16, + name="Test", + adv_type="CLIENT", + received_at=now, + route_type="flood", + ) + api_db_session.add(ad) + api_db_session.commit() + + response = client_no_auth.get("/api/v1/advertisements") + assert response.status_code == 200 + data = response.json() + assert len(data["items"]) == 1 + assert data["items"][0]["route_type"] == "flood" + assert data["items"][0]["advert_timestamp"] is None + + def test_get_advertisement_includes_route_type( + self, client_no_auth, api_db_session + ): + """GET /{id} includes route_type and advert_timestamp.""" + now = datetime.now(timezone.utc) + ad = Advertisement( + public_key="aa" * 16, + name="Test", + adv_type="CLIENT", + received_at=now, + route_type="transport_flood", + ) + api_db_session.add(ad) + api_db_session.commit() + + response = client_no_auth.get(f"/api/v1/advertisements/{ad.id}") + assert response.status_code == 200 + data = response.json() + assert data["route_type"] == "transport_flood" diff --git a/tests/test_api/test_dashboard.py b/tests/test_api/test_dashboard.py index a486509..1c6a7a1 100644 --- a/tests/test_api/test_dashboard.py +++ b/tests/test_api/test_dashboard.py @@ -351,3 +351,91 @@ class TestDashboardTestUserExclusion: data = response.json() assert data["total_operators"] == 0 assert data["total_members"] == 0 + + +class TestDashboardFloodOnlyFilter: + """Tests for flood-only advertisement filtering on dashboard.""" + + def test_stats_excludes_direct_adverts(self, client_no_auth, api_db_session): + """Dashboard stats exclude direct (zero-hop) advertisements.""" + now = datetime.now(timezone.utc) + flood_ad = Advertisement( + public_key="aa" * 16, + name="Flood", + adv_type="CLIENT", + received_at=now, + route_type="flood", + ) + direct_ad = Advertisement( + public_key="bb" * 16, + name="Direct", + adv_type="CLIENT", + received_at=now, + route_type="direct", + ) + null_ad = Advertisement( + public_key="cc" * 16, + name="Historical", + adv_type="CLIENT", + received_at=now, + route_type=None, + ) + api_db_session.add_all([flood_ad, direct_ad, null_ad]) + api_db_session.commit() + + response = client_no_auth.get("/api/v1/dashboard/stats") + assert response.status_code == 200 + data = response.json() + assert data["total_advertisements"] == 2 + + def test_recent_ads_excludes_direct(self, client_no_auth, api_db_session): + """Recent advertisements list excludes direct adverts.""" + now = datetime.now(timezone.utc) + direct_ad = Advertisement( + public_key="aa" * 16, + name="Direct", + adv_type="CLIENT", + received_at=now, + route_type="direct", + ) + flood_ad = Advertisement( + public_key="bb" * 16, + name="Flood", + adv_type="CLIENT", + received_at=now - timedelta(seconds=1), + route_type="flood", + ) + api_db_session.add_all([direct_ad, flood_ad]) + api_db_session.commit() + + response = client_no_auth.get("/api/v1/dashboard/stats") + assert response.status_code == 200 + data = response.json() + assert len(data["recent_advertisements"]) == 1 + assert data["recent_advertisements"][0]["name"] == "Flood" + + def test_activity_excludes_direct(self, client_no_auth, api_db_session): + """Activity endpoint excludes direct advertisements.""" + yesterday = datetime.now(timezone.utc) - timedelta(days=1) + direct_ad = Advertisement( + public_key="aa" * 16, + name="Direct", + adv_type="CLIENT", + received_at=yesterday, + route_type="direct", + ) + flood_ad = Advertisement( + public_key="bb" * 16, + name="Flood", + adv_type="CLIENT", + received_at=yesterday, + route_type="flood", + ) + api_db_session.add_all([direct_ad, flood_ad]) + api_db_session.commit() + + response = client_no_auth.get("/api/v1/dashboard/activity") + assert response.status_code == 200 + data = response.json() + total_count = sum(point["count"] for point in data["data"]) + assert total_count == 1 diff --git a/tests/test_collector/test_handlers/test_advertisement.py b/tests/test_collector/test_handlers/test_advertisement.py index cbfd584..a3baa1d 100644 --- a/tests/test_collector/test_handlers/test_advertisement.py +++ b/tests/test_collector/test_handlers/test_advertisement.py @@ -216,3 +216,63 @@ class TestHandleAdvertisement: assert observer is not None assert observer.snr == 12.5 assert observer.path_len == 3 + + def test_stores_route_type(self, db_manager, db_session): + """route_type is stored on the Advertisement record.""" + payload = { + "public_key": "a" * 64, + "name": "TestNode", + "adv_type": "chat", + "route_type": "flood", + } + + handle_advertisement("b" * 64, "advertisement", payload, db_manager) + + ad = db_session.execute(select(Advertisement)).scalar_one() + assert ad.route_type == "flood" + + def test_stores_advert_timestamp(self, db_manager, db_session): + """advert_timestamp is stored on the Advertisement record.""" + payload = { + "public_key": "a" * 64, + "name": "TestNode", + "adv_type": "chat", + "advert_timestamp": 1747300000, + } + + handle_advertisement("b" * 64, "advertisement", payload, db_manager) + + ad = db_session.execute(select(Advertisement)).scalar_one() + assert ad.advert_timestamp is not None + assert ad.advert_timestamp.year == 2025 + + def test_advert_timestamp_invalid_falls_back_to_received_at( + self, db_manager, db_session + ): + """Invalid advert_timestamp (far from received_at) still stored but not used for hash.""" + payload = { + "public_key": "a" * 64, + "name": "TestNode", + "adv_type": "chat", + "advert_timestamp": 0, + } + + handle_advertisement("b" * 64, "advertisement", payload, db_manager) + + ad = db_session.execute(select(Advertisement)).scalar_one() + assert ad.advert_timestamp is not None + assert ad.advert_timestamp.year == 1970 + + def test_advert_timestamp_valid_used_for_dedup(self, db_manager, db_session): + """Valid advert_timestamp produces same hash for different received_at times.""" + payload1 = { + "public_key": "a" * 64, + "name": "TestNode", + "adv_type": "chat", + "advert_timestamp": 1747300000, + } + + handle_advertisement("b" * 64, "advertisement", payload1, db_manager) + + ads = db_session.execute(select(Advertisement)).scalars().all() + assert len(ads) == 1 diff --git a/tests/test_collector/test_letsmesh_normalizer.py b/tests/test_collector/test_letsmesh_normalizer.py index 64b2d75..6bd2c1e 100644 --- a/tests/test_collector/test_letsmesh_normalizer.py +++ b/tests/test_collector/test_letsmesh_normalizer.py @@ -131,6 +131,116 @@ class TestAdvertisementSnrAndPath: assert "SNR" not in result +class TestAdvertisementRouteTypeAndTimestamp: + """Tests for route_type and advert_timestamp extraction in advertisement payloads.""" + + def _make_normalizer(self) -> LetsMeshNormalizer: + norm = LetsMeshNormalizer() + norm._letsmesh_decoder = MagicMock() + norm._include_test_channel = False + return norm + + def _make_decoded_type4( + self, route_type: int | None = None, timestamp: int | None = None + ) -> dict: + decoded_inner: dict = { + "publicKey": "b" * 64, + } + if timestamp is not None: + decoded_inner["timestamp"] = timestamp + packet: dict = { + "payloadType": 4, + "payload": { + "decoded": decoded_inner, + }, + } + if route_type is not None: + packet["routeType"] = route_type + return packet + + def test_route_type_flood(self) -> None: + norm = self._make_normalizer() + result = norm._build_letsmesh_advertisement_payload( + {}, + decoded_packet=self._make_decoded_type4(route_type=1), + ) + assert result is not None + assert result["route_type"] == "flood" + + def test_route_type_transport_flood(self) -> None: + norm = self._make_normalizer() + result = norm._build_letsmesh_advertisement_payload( + {}, + decoded_packet=self._make_decoded_type4(route_type=0), + ) + assert result is not None + assert result["route_type"] == "transport_flood" + + def test_route_type_direct(self) -> None: + norm = self._make_normalizer() + result = norm._build_letsmesh_advertisement_payload( + {}, + decoded_packet=self._make_decoded_type4(route_type=2), + ) + assert result is not None + assert result["route_type"] == "direct" + + def test_route_type_transport_direct(self) -> None: + norm = self._make_normalizer() + result = norm._build_letsmesh_advertisement_payload( + {}, + decoded_packet=self._make_decoded_type4(route_type=3), + ) + assert result is not None + assert result["route_type"] == "transport_direct" + + def test_route_type_unknown_int_omitted(self) -> None: + norm = self._make_normalizer() + result = norm._build_letsmesh_advertisement_payload( + {}, + decoded_packet=self._make_decoded_type4(route_type=99), + ) + assert result is not None + assert "route_type" not in result + + def test_route_type_missing_omitted(self) -> None: + norm = self._make_normalizer() + result = norm._build_letsmesh_advertisement_payload( + {}, + decoded_packet=self._make_decoded_type4(), + ) + assert result is not None + assert "route_type" not in result + + def test_advert_timestamp_extracted(self) -> None: + norm = self._make_normalizer() + result = norm._build_letsmesh_advertisement_payload( + {}, + decoded_packet=self._make_decoded_type4(timestamp=1747300000), + ) + assert result is not None + assert result["advert_timestamp"] == 1747300000 + + def test_advert_timestamp_missing_omitted(self) -> None: + norm = self._make_normalizer() + result = norm._build_letsmesh_advertisement_payload( + {}, + decoded_packet=self._make_decoded_type4(), + ) + assert result is not None + assert "advert_timestamp" not in result + + def test_both_route_type_and_timestamp(self) -> None: + norm = self._make_normalizer() + result = norm._build_letsmesh_advertisement_payload( + {}, + decoded_packet=self._make_decoded_type4(route_type=1, timestamp=1747300000), + ) + assert result is not None + assert result["route_type"] == "flood" + assert result["advert_timestamp"] == 1747300000 + + class TestMessageSnrCasing: """Tests for message payload SNR casing (lowercase output).""" diff --git a/tests/test_common/test_hash_utils.py b/tests/test_common/test_hash_utils.py index 62a85c2..7519bdd 100644 --- a/tests/test_common/test_hash_utils.py +++ b/tests/test_common/test_hash_utils.py @@ -264,3 +264,122 @@ class TestComputeTelemetryHash: ) assert hash1 == hash2 + + def test_default_bucket_is_300s(self) -> None: + """Default bucket_seconds should be 300 (5 minutes).""" + time1 = datetime(2024, 1, 15, 10, 31, 0, tzinfo=timezone.utc) + time2 = datetime(2024, 1, 15, 10, 33, 0, tzinfo=timezone.utc) + + hash1 = compute_telemetry_hash( + node_public_key="a" * 64, + parsed_data={"temp": 22.5}, + received_at=time1, + ) + hash2 = compute_telemetry_hash( + node_public_key="a" * 64, + parsed_data={"temp": 22.5}, + received_at=time2, + ) + + assert hash1 == hash2 + + +class TestComputeAdvertisementHashWithAdvertTimestamp: + """Tests for compute_advertisement_hash with advert_timestamp parameter.""" + + def test_advert_timestamp_used_for_bucketing(self) -> None: + """When advert_timestamp is provided, it is used for time bucketing.""" + received_at = datetime(2024, 1, 15, 10, 30, 0, tzinfo=timezone.utc) + advert_ts = datetime(2024, 1, 15, 10, 28, 0, tzinfo=timezone.utc) + + hash_with_ts = compute_advertisement_hash( + public_key="a" * 64, + name="Node1", + received_at=received_at, + advert_timestamp=advert_ts, + ) + hash_ts_only = compute_advertisement_hash( + public_key="a" * 64, + name="Node1", + received_at=received_at, + bucket_seconds=300, + advert_timestamp=advert_ts, + ) + + assert hash_with_ts == hash_ts_only + + def test_advert_timestamp_overrides_received_at(self) -> None: + """advert_timestamp produces different bucket than received_at when far apart.""" + received_at = datetime(2024, 1, 15, 10, 30, 0, tzinfo=timezone.utc) + advert_ts = datetime(2024, 1, 15, 12, 0, 0, tzinfo=timezone.utc) + + hash_with_advert_ts = compute_advertisement_hash( + public_key="a" * 64, + name="Node1", + received_at=received_at, + advert_timestamp=advert_ts, + ) + hash_with_received_at = compute_advertisement_hash( + public_key="a" * 64, + name="Node1", + received_at=received_at, + ) + + assert hash_with_advert_ts != hash_with_received_at + + def test_same_advert_timestamp_same_hash(self) -> None: + """Same advert_timestamp but different received_at produces same hash.""" + ts = datetime(2024, 1, 15, 10, 30, 0, tzinfo=timezone.utc) + recv1 = datetime(2024, 1, 15, 10, 31, 0, tzinfo=timezone.utc) + recv2 = datetime(2024, 1, 15, 10, 32, 0, tzinfo=timezone.utc) + + hash1 = compute_advertisement_hash( + public_key="a" * 64, + name="Node1", + received_at=recv1, + advert_timestamp=ts, + ) + hash2 = compute_advertisement_hash( + public_key="a" * 64, + name="Node1", + received_at=recv2, + advert_timestamp=ts, + ) + + assert hash1 == hash2 + + def test_none_advert_timestamp_falls_back_to_received_at(self) -> None: + """When advert_timestamp is None, received_at is used for bucketing.""" + received_at = datetime(2024, 1, 15, 10, 30, 0, tzinfo=timezone.utc) + + hash_explicit_none = compute_advertisement_hash( + public_key="a" * 64, + name="Node1", + received_at=received_at, + advert_timestamp=None, + ) + hash_no_param = compute_advertisement_hash( + public_key="a" * 64, + name="Node1", + received_at=received_at, + ) + + assert hash_explicit_none == hash_no_param + + def test_default_bucket_is_300s(self) -> None: + """Default bucket_seconds should be 300 (5 minutes).""" + time1 = datetime(2024, 1, 15, 10, 31, 0, tzinfo=timezone.utc) + time2 = datetime(2024, 1, 15, 10, 33, 0, tzinfo=timezone.utc) + + hash1 = compute_advertisement_hash( + public_key="a" * 64, + name="Node1", + received_at=time1, + ) + hash2 = compute_advertisement_hash( + public_key="a" * 64, + name="Node1", + received_at=time2, + ) + + assert hash1 == hash2 diff --git a/tests/test_common/test_i18n.py b/tests/test_common/test_i18n.py index cb75cdd..863fac9 100644 --- a/tests/test_common/test_i18n.py +++ b/tests/test_common/test_i18n.py @@ -135,3 +135,20 @@ class TestEnJsonCompleteness: assert t("entities.messages") != "entities.messages" assert t("entities.map") != "entities.map" assert t("entities.members") != "entities.members" + + def test_advertisements_route_type_keys(self): + """Advertisement route type filter keys are all present.""" + assert ( + t("advertisements.filter_route_type_label") + != "advertisements.filter_route_type_label" + ) + assert t("advertisements.route_type_all") != "advertisements.route_type_all" + assert t("advertisements.route_type_flood") != "advertisements.route_type_flood" + assert ( + t("advertisements.route_type_direct") != "advertisements.route_type_direct" + ) + assert ( + t("advertisements.route_type_unknown") + != "advertisements.route_type_unknown" + ) + assert t("advertisements.col_route_type") != "advertisements.col_route_type"