Merge pull request #310 from ipnet-mesh/feat/api-caching-and-event-dedup

feat: HTTP Cache-Control, dashboard TTL bump, and route event-hash dedup
This commit is contained in:
JingleManSweep
2026-07-18 12:45:32 +01:00
committed by GitHub
26 changed files with 1587 additions and 129 deletions
+9 -2
View File
@@ -411,8 +411,15 @@ PROMETHEUS_PORT=9090
# Default cache TTL in seconds (matches web auto-refresh interval)
# REDIS_CACHE_TTL=30
# Cache TTL for dashboard endpoints (seconds)
# REDIS_CACHE_TTL_DASHBOARD=30
# Cache TTL for dashboard endpoints (seconds; covers /dashboard/* and /routes/{id}/history)
# REDIS_CACHE_TTL_DASHBOARD=300
# Cache TTL for /routes/{id} detail endpoint (seconds)
# REDIS_CACHE_TTL_ROUTE_DETAIL=300
# Emit HTTP Cache-Control on /api/v1/* responses + ETag/If-None-Match on
# cached endpoints. Disable to suppress all client-side caching directives.
# API_CACHE_CONTROL_ENABLED=true
# External Alertmanager port (when using --profile metrics)
ALERTMANAGER_PORT=9093
@@ -0,0 +1,63 @@
"""add event_hash to raw packets and path hops
Revision ID: 6b3430fd84f4
Revises: ec40c67c8c83
Create Date: 2026-07-18 12:00:00.000000+00:00
Adds a nullable ``event_hash`` column to ``raw_packets`` and
``packet_path_hops``. The column denormalizes the underlying structured
event's identity (advertisement / message / telemetry / trace
``event_hash``) so the route evaluator can deduplicate matches by their
underlying event rather than by per-transmission wire hash.
Background: a single advert or message retransmitted (or flooded) through
the mesh produces one ``RawPacket`` row per on-air copy, each with a
fresh wire ``packet_hash``. Counting those as distinct matches let a
single underlying event satisfy ``packet_count_threshold`` within seconds
and bias the route's health. Joining through ``event_hash`` collapses
all receptions of the same underlying event into one match.
Rows captured before this migration (and any unclassified wire packets)
keep ``event_hash IS NULL``; the evaluator falls back to the wire
``packet_hash`` for those, preserving today's behaviour until they age
out of the configured ``window_hours``. No data backfill is performed.
"""
from typing import Sequence, Union
import sqlalchemy as sa
from alembic import op
revision: str = "6b3430fd84f4"
down_revision: Union[str, None] = "ec40c67c8c83"
branch_labels: Union[str, Sequence[str], None] = None
depends_on: Union[str, Sequence[str], None] = None
def upgrade() -> None:
with op.batch_alter_table("raw_packets", schema=None) as batch_op:
batch_op.add_column(
sa.Column("event_hash", sa.String(length=32), nullable=True)
)
batch_op.create_index("ix_raw_packets_event_hash", ["event_hash"], unique=False)
with op.batch_alter_table("packet_path_hops", schema=None) as batch_op:
batch_op.add_column(
sa.Column("event_hash", sa.String(length=32), nullable=True)
)
batch_op.create_index(
"ix_packet_path_hops_event_hash_received_at",
["event_hash", "received_at"],
unique=False,
)
def downgrade() -> None:
with op.batch_alter_table("packet_path_hops", schema=None) as batch_op:
batch_op.drop_index("ix_packet_path_hops_event_hash_received_at")
batch_op.drop_column("event_hash")
with op.batch_alter_table("raw_packets", schema=None) as batch_op:
batch_op.drop_index("ix_raw_packets_event_hash")
batch_op.drop_column("event_hash")
+3 -1
View File
@@ -320,7 +320,9 @@ services:
- REDIS_PASSWORD=${REDIS_PASSWORD:-}
- REDIS_KEY_PREFIX=${REDIS_KEY_PREFIX:-hub}
- REDIS_CACHE_TTL=${REDIS_CACHE_TTL:-30}
- REDIS_CACHE_TTL_DASHBOARD=${REDIS_CACHE_TTL_DASHBOARD:-30}
- REDIS_CACHE_TTL_DASHBOARD=${REDIS_CACHE_TTL_DASHBOARD:-300}
- REDIS_CACHE_TTL_ROUTE_DETAIL=${REDIS_CACHE_TTL_ROUTE_DETAIL:-300}
- API_CACHE_CONTROL_ENABLED=${API_CACHE_CONTROL_ENABLED:-true}
# Spam detection: hide-filter switch + threshold (bridged from
# FEATURE_SPAM_DETECTION so it tracks the collector and web toggle). On by
# default; opt out with FEATURE_SPAM_DETECTION=false.
+22 -1
View File
@@ -62,7 +62,28 @@ Optional Redis-backed caching for API responses. When disabled or unavailable, t
| `REDIS_PASSWORD` | _(none)_ | Redis password (optional) |
| `REDIS_KEY_PREFIX` | `hub` | Cache key prefix for multi-instance isolation |
| `REDIS_CACHE_TTL` | `30` | Default cache TTL in seconds |
| `REDIS_CACHE_TTL_DASHBOARD` | `30` | Cache TTL for dashboard endpoints in seconds |
| `REDIS_CACHE_TTL_DASHBOARD` | `300` | Cache TTL for `/dashboard/*` endpoints and `/routes/{id}/history` (seconds). Trend/aggregation data tolerates longer staleness than the default TTL. |
| `REDIS_CACHE_TTL_ROUTE_DETAIL` | `300` | Cache TTL for `/routes/{id}` detail endpoint in seconds |
### HTTP Cache-Control
On top of the Redis server cache, the API emits HTTP-layer caching directives so browsers and HTTP clients can reuse recent responses without re-fetching. Disabled with `API_CACHE_CONTROL_ENABLED=false` (default `true`).
| Variable | Default | Description |
| --- | --- | --- |
| `API_CACHE_CONTROL_ENABLED` | `true` | Emit HTTP `Cache-Control` on `/api/v1/*` responses and `ETag` / `If-None-Match` handling on `@cached` endpoints |
Responses fall into three buckets:
| Bucket | Endpoints | `Cache-Control` | `ETag` |
| --- | --- | --- | --- |
| `@cached` GETs | All Redis-cached endpoints (`/nodes`, `/routes`, `/routes/{id}`, `/dashboard/*`, `/packets`, `/packet-groups`, `/messages`, `/advertisements`, `/channels`, `/user/profiles`) | `private, max-age=<redis_ttl>` | Strong SHA-256 hash of the body; `If-None-Match` returns `304 Not Modified` |
| Other GETs | Per-id detail endpoints (`/nodes/{key}`, `/packets/{id}`, `/messages/{id}`, `/user/profile/{id}`, `/trace-paths`, `/telemetry`, etc.) | `private, max-age=0, must-revalidate` | _(none)_ |
| Mutating + health | `POST`/`PUT`/`DELETE` + `/health*` | `no-store` | _(none)_ |
All API responses use `private` because several `@cached` endpoints are role-aware — their response shape/redaction varies by trusted-proxy `X-User-Id` / `X-User-Roles` headers, so shared/CDN caches must never store them. Browser caches key by URL + request headers and so remain correct.
Client `max-age` matches the configured Redis TTL for the endpoint (e.g. 300 s on `/routes/{id}`, 30 s on `/dashboard/*`). The `X-Cache: HIT|MISS` observability header continues to be emitted regardless of this setting.
## Collector
+3 -1
View File
@@ -10,13 +10,15 @@ For the environment-variable reference, see [configuration.md → Feature Flags]
A route is an ordered list of two or more nodes (the configured path). For each captured packet reception, the evaluator walks the reception's path-hash sequence and checks whether the route's nodes appear **in order, as a subsequence** (intermediate hops are allowed). When `reversible` is set (the default), the reverse-ordered path is also accepted, so a packet travelling `B → ... → A` counts toward an `A → B` route.
Matches are deduplicated by their **underlying event identity**, not per on-air transmission. The collector denormalizes the structured event's `event_hash` (the same key used to dedup advertisements, messages, telemetry, and traces at the structured-event layer) onto each captured raw packet at ingest time. The evaluator prefers `event_hash` when set, so retransmissions or floods of the same underlying advert/message count once toward `packet_count_threshold` instead of once per on-air copy. Packets captured before this column existed (and any unclassified wire packets) have `event_hash IS NULL` and fall back to the wire `packet_hash`, preserving the previous behaviour until they age out of the configured `window_hours`.
Each route carries these knobs:
| Field | Default | Description |
| --- | --- | --- |
| `match_width` | `1` | Path-hash prefix width in bytes (1/2/3). Higher widths disambiguate nodes that share a short public-key prefix. |
| `window_hours` | `24` | Rolling lookback window for the live status card. |
| `packet_count_threshold` | `3` | Distinct matching packets at/above which the route is `healthy`. |
| `packet_count_threshold` | `3` | Distinct matching packets at/above which the route is `healthy`. "Distinct" is per underlying event, not per transmission — see [How health is evaluated](#how-health-is-evaluated) above. |
| `clear_threshold` | _(2× threshold)_ | Comfort bar for the `clear`/`marginal` split. Omit/null to use twice the threshold. |
| `max_hop_span` | _(unlimited)_ | Caps the position gap between the first and last matched node, to reject matches that wander too far. |
| `reversible` | `true` | Also match the path in reverse direction. |
+34
View File
@@ -40,6 +40,39 @@ Remote observers contribute to the Hub by publishing decoded packets to your MQT
In Docker Compose, set `OBSERVER_ALLOWLIST` / `OBSERVER_DENYLIST` in your `.env`; they are wired into the collector service. See [configuration.md → Observer Ingestion Filters](configuration.md#observer-ingestion-filters) and [observer.md](observer.md).
### HTTP Cache-Control on API responses
The API now emits HTTP `Cache-Control` headers on every `/api/v1/*` response and supports `ETag` / `If-None-Match``304 Not Modified` on Redis-cached endpoints. **On by default**, non-breaking, no migration required.
- `@cached` GET endpoints get `Cache-Control: private, max-age=<redis_ttl>` (matching their existing Redis TTL) plus a strong `ETag`. On a matching `If-None-Match`, the server returns `304 Not Modified` with an empty body — saving bandwidth on cache hits.
- Uncached GETs under `/api/v1/*` get `Cache-Control: private, max-age=0, must-revalidate` so browsers may store but always revalidate.
- `POST`/`PUT`/`DELETE`/`PATCH` and `/health*` get `Cache-Control: no-store`.
- All API responses are `private` (browser cache only) because several `@cached` endpoints are role-aware — their response shape varies by `X-User-Id`/`X-User-Roles`. Shared/CDN caches must not store them.
Existing Redis entries written in the legacy bare-JSON format are still read; they're transparently upgraded to the new `{"body": ..., "etag": ...}` envelope on the next cache miss. No manual Redis flush is required; old entries expire within one TTL period (≤ 300 s by default).
To suppress all client-side caching directives (e.g. for debugging):
| Variable | Default | Description |
| --- | --- | --- |
| `API_CACHE_CONTROL_ENABLED` | `true` | Emit `Cache-Control` on `/api/v1/*` + `ETag`/`If-None-Match` on `@cached` endpoints. Set `false` to disable. |
The `X-Cache: HIT|MISS` observability header is unaffected — it is still emitted whenever a request flows through the `@cached` decorator, regardless of this setting.
### Dashboard Cache TTL default raised (30s → 300s)
The default for `REDIS_CACHE_TTL_DASHBOARD` has been raised from **30 s** to **300 s** (5 minutes). This setting covers all `/dashboard/*` endpoints (`stats`, `activity`, `packet-activity`, `packet-breakdown`, `message-activity`, `node-count`) **and** the per-route health strip `GET /api/v1/routes/{id}/history`. These all return trend/aggregation data where minute-level staleness is invisible to operators but every cache miss costs seconds of SQL/aggregation work — the old 30 s default was leaving most of that work unpaid on typical dashboards.
| Variable | Old default | New default |
| --- | --- | --- |
| `REDIS_CACHE_TTL_DASHBOARD` | `30` | `300` |
- **Operators who already set the env var explicitly:** no change — your value still wins.
- **Operators on the default:** silently bumped to 5 min staleness on the dashboard and route health strips. If you relied on the old 30 s behaviour (e.g. for development or tight freshness SLOs), set `REDIS_CACHE_TTL_DASHBOARD=30` to restore it.
- **Existing Redis entries at deploy time:** keep their original TTL and expire naturally within ≤ 30 s. No flush is required.
- **Browser `max-age`:** the HTTP `Cache-Control: max-age=<ttl>` emitted by the middleware follows the Redis TTL, so browser-side caching on these endpoints also extends to 5 min. Clicking around the dashboard will be instant for that window. A hard refresh (`Ctrl+Shift+R`) bypasses this if you need the latest data.
- **Side-effect fix:** the reported short-TTL behaviour on the per-route healthchart (which shares this setting) is resolved.
## v0.15.0
### Spam Detection (score, hide, and toggle likely-spam messages)
@@ -260,6 +293,7 @@ A new optional Redis-backed caching layer reduces database load for read-heavy A
| `REDIS_KEY_PREFIX` | `hub` | Cache key prefix (change per instance for multi-instance setups) |
| `REDIS_CACHE_TTL` | `30` | Default cache TTL in seconds |
| `REDIS_CACHE_TTL_DASHBOARD` | `30` | Cache TTL for dashboard endpoints |
| `REDIS_CACHE_TTL_ROUTE_DETAIL` | `300` | Cache TTL for `/routes/{id}` detail endpoint |
**Docker Compose:** Redis is available via the `cache` profile:
+61 -2
View File
@@ -90,7 +90,9 @@ def create_app(
redis_password: str | None = None,
redis_key_prefix: str = "hub",
redis_cache_ttl: int = 30,
redis_cache_ttl_dashboard: int = 30,
redis_cache_ttl_dashboard: int = 300,
redis_cache_ttl_route_detail: int = 300,
api_cache_control_enabled: bool = True,
spam_detection_enabled: bool = False,
spam_score_threshold: float = 0.65,
) -> FastAPI:
@@ -119,6 +121,9 @@ def create_app(
redis_key_prefix: Prefix for all cache keys
redis_cache_ttl: Default cache TTL in seconds
redis_cache_ttl_dashboard: Cache TTL for dashboard endpoints
redis_cache_ttl_route_detail: Cache TTL for /routes/{id} detail endpoint
api_cache_control_enabled: Emit HTTP Cache-Control on /api/v1/* and
ETag/If-None-Match handling on @cached endpoints.
Returns:
Configured FastAPI application
@@ -154,6 +159,8 @@ def create_app(
app.state.redis_key_prefix = redis_key_prefix
app.state.redis_cache_ttl = redis_cache_ttl
app.state.redis_cache_ttl_dashboard = redis_cache_ttl_dashboard
app.state.redis_cache_ttl_route_detail = redis_cache_ttl_route_detail
app.state.api_cache_control_enabled = api_cache_control_enabled
app.state.spam_detection_enabled = spam_detection_enabled
app.state.spam_score_threshold = spam_score_threshold
@@ -170,11 +177,61 @@ def create_app(
)
@app.middleware("http")
async def cache_header_middleware(request: Request, call_next: Any) -> Any:
async def api_cache_middleware(request: Request, call_next: Any) -> Any:
"""Apply X-Cache observability + HTTP Cache-Control / ETag headers.
Buckets (only when ``app.state.api_cache_control_enabled`` is True):
* ``@cached`` endpoints (``request.state.cache_control_ttl`` set by
the decorator): ``private, max-age=<ttl>`` + ``ETag`` echoed back.
* Uncached GETs under ``/api/v1``: ``private, max-age=0,
must-revalidate`` so browsers revalidate but may store.
* Mutating methods (POST/PUT/DELETE/PATCH): ``no-store``.
* ``/health*`` endpoints: ``no-store``.
``private`` is used everywhere because several cached endpoints are
role-aware — their response shape/redaction varies by trusted-proxy
``X-User-Id`` / ``X-User-Roles`` headers, so shared/CDN caches must
never store them.
"""
response = await call_next(request)
# X-Cache observability header (always emitted, even when the kill
# switch is on, so monitoring can still see hit/miss ratios).
cache_status = getattr(request.state, "cache_status", None)
if cache_status is not None:
response.headers["X-Cache"] = cache_status
# ETag from the @cached decorator (304 responses already carry it;
# this branch covers the 200 path where the decorator returned a
# plain model and FastAPI serialized it).
etag = getattr(request.state, "api_etag", None)
if etag is not None and "etag" not in response.headers:
response.headers["ETag"] = etag
if not getattr(app.state, "api_cache_control_enabled", True):
return response
# Don't overwrite a Cache-Control header set explicitly by a handler
# or by the @cached decorator's 304 Response.
if "cache-control" in response.headers:
return response
method = request.method.upper()
path = request.url.path
if method in ("POST", "PUT", "DELETE", "PATCH"):
response.headers["Cache-Control"] = "no-store"
elif path.startswith("/health"):
response.headers["Cache-Control"] = "no-store"
elif path.startswith("/api/"):
ttl = getattr(request.state, "cache_control_ttl", 0)
if isinstance(ttl, int) and ttl > 0:
response.headers["Cache-Control"] = f"private, max-age={ttl}"
else:
response.headers["Cache-Control"] = (
"private, max-age=0, must-revalidate"
)
return response
# Include routers
@@ -272,6 +329,8 @@ def create_app_from_env() -> FastAPI:
redis_key_prefix=settings.redis_key_prefix,
redis_cache_ttl=settings.redis_cache_ttl,
redis_cache_ttl_dashboard=settings.redis_cache_ttl_dashboard,
redis_cache_ttl_route_detail=settings.redis_cache_ttl_route_detail,
api_cache_control_enabled=settings.api_cache_control_enabled,
spam_detection_enabled=settings.spam_detection_enabled,
spam_score_threshold=settings.spam_score_threshold,
)
+141 -41
View File
@@ -1,6 +1,7 @@
"""Cache decorator for API endpoints."""
import functools
import hashlib
import inspect
import json
import logging
@@ -8,6 +9,7 @@ from typing import Any, Callable, Optional
from urllib.parse import urlencode
from fastapi import Request
from fastapi.responses import Response
logger = logging.getLogger(__name__)
@@ -50,36 +52,105 @@ def _build_cache_key(
return f"{endpoint_name}:{sorted_query_string(request)}"
def _lookup(cache: Any, cache_key: str, request: Request) -> Any:
"""Return the cached value, or _MISS, and record the cache status."""
def _compute_etag(serialized_body: str) -> str:
"""Compute a strong ETag header value (quoted hex hash) for a body.
Uses the first 32 hex chars of SHA-256 collision-safe for cache keys
and short enough to fit comfortably in a request header.
"""
digest = hashlib.sha256(serialized_body.encode("utf-8")).hexdigest()[:32]
return f'"{digest}"'
def _etag_matches(if_none_match: str, etag: str) -> bool:
"""Return True if the client's If-None-Match header matches the ETag.
Per RFC 7232 the header may carry a comma-separated list or the wildcard
``*``. Weak indicators (``W/``) are accepted on the client side. We do not
emit weak ETags, but clients are allowed to send them.
"""
if if_none_match.strip() == "*":
return True
for token in if_none_match.split(","):
candidate = token.strip()
if candidate.startswith("W/"):
candidate = candidate[2:]
if candidate == etag:
return True
return False
def _serialize_for_cache(result: Any) -> tuple[Any, str]:
"""Convert a handler result into (body_json, serialized_json) for storage.
Returns the JSON-compatible body (suitable for JSONResponse) and the
canonical serialized string used to compute the ETag. Pydantic models are
dumped via ``model_dump(mode="json")`` so datetimes become ISO strings
deterministically.
"""
if hasattr(result, "model_dump"):
body = result.model_dump(mode="json")
elif isinstance(result, dict):
body = result
else:
body = result
serialized = json.dumps(body, default=str)
return body, serialized
def _store(cache: Any, cache_key: str, result: Any, ttl: int) -> tuple[Any, str]:
"""Serialize, ETag, and store a handler result in the cache.
Returns ``(body, etag)`` so the caller can return the body to FastAPI
without re-serializing. Stores the new envelope format
``{"body": ..., "etag": "..."}`` so future hits can return the ETag
without re-hashing.
"""
body, serialized = _serialize_for_cache(result)
etag = _compute_etag(serialized)
envelope = json.dumps({"body": body, "etag": etag})
try:
cached_value = cache.get(cache_key)
except Exception as e:
logger.warning("Redis GET error for %s: %s", cache_key, e)
cached_value = None
if cached_value is not None:
logger.debug("Cache HIT: %s", cache_key)
request.state.cache_status = "HIT"
return json.loads(cached_value)
logger.debug("Cache MISS: %s", cache_key)
request.state.cache_status = "MISS"
return _MISS
def _store(cache: Any, cache_key: str, result: Any, ttl: int) -> None:
"""Serialize and store a handler result in the cache."""
try:
if hasattr(result, "model_dump"):
serialized = json.dumps(result.model_dump(mode="json"))
elif isinstance(result, dict):
serialized = json.dumps(result)
else:
serialized = json.dumps(result, default=str)
cache.set(cache_key, serialized, ttl)
cache.set(cache_key, envelope, ttl)
except Exception as e:
logger.warning("Cache store error for %s: %s", cache_key, e)
return body, etag
def _lookup(cache: Any, cache_key: str, request: Request) -> tuple[Any, str]:
"""Return ``(body, etag)`` on hit, or ``(_MISS, "")`` on miss.
Reads both the new envelope format ``{"body": ..., "etag": "..."}`` and
the legacy bare-body format (which carried no ETag). Legacy entries are
hashed on read so they still serve correctly; natural expiry migrates
them to the envelope format on the next write.
"""
try:
raw = cache.get(cache_key)
except Exception as e:
logger.warning("Redis GET error for %s: %s", cache_key, e)
return _MISS, ""
if raw is None:
logger.debug("Cache MISS: %s", cache_key)
request.state.cache_status = "MISS"
return _MISS, ""
logger.debug("Cache HIT: %s", cache_key)
request.state.cache_status = "HIT"
try:
parsed = json.loads(raw)
except (json.JSONDecodeError, TypeError):
# Garbage in cache — treat as miss and let the next write repair it.
request.state.cache_status = "MISS"
return _MISS, ""
if isinstance(parsed, dict) and "body" in parsed and "etag" in parsed:
return parsed["body"], parsed["etag"]
# Legacy entry: body stored bare, no ETag envelope. Hash on the fly so
# the client still gets a usable ETag; the next MISS overwrites it with
# the envelope format.
serialized = json.dumps(parsed, default=str)
return parsed, _compute_etag(serialized)
def cached(
@@ -106,19 +177,37 @@ def cached(
async def async_wrapper(*args: Any, **kwargs: Any) -> Any:
request = _find_request(kwargs)
cache = getattr(request.app.state, "redis_cache", None)
ttl = getattr(request.app.state, ttl_setting, 30)
# Always advertise the client-cache TTL so the middleware can
# emit Cache-Control regardless of whether Redis is configured.
request.state.cache_control_ttl = ttl
if cache is None:
return await func(*args, **kwargs)
ttl = getattr(request.app.state, ttl_setting, 30)
cache_key = _build_cache_key(request, endpoint_name, key_builder)
body, etag = _lookup(cache, cache_key, request)
cached = _lookup(cache, cache_key, request)
if cached is not _MISS:
return cached
if body is _MISS:
# MISS: call the handler, store, and return the original
# result so FastAPI's response_model handling still sees
# the Pydantic model (not the JSON-dumped dict).
result = await func(*args, **kwargs)
_, etag = _store(cache, cache_key, result, ttl)
return_value = result
else:
# HIT: only have the JSON-deserialized body.
return_value = body
result = await func(*args, **kwargs)
_store(cache, cache_key, result, ttl)
return result
# ETag + If-None-Match handling.
request.state.api_etag = etag
if_none_match = request.headers.get("if-none-match")
if if_none_match and _etag_matches(if_none_match, etag):
return Response(
status_code=304,
headers={"ETag": etag},
)
return return_value
return async_wrapper
@@ -126,19 +215,30 @@ def cached(
def sync_wrapper(*args: Any, **kwargs: Any) -> Any:
request = _find_request(kwargs)
cache = getattr(request.app.state, "redis_cache", None)
ttl = getattr(request.app.state, ttl_setting, 30)
request.state.cache_control_ttl = ttl
if cache is None:
return func(*args, **kwargs)
ttl = getattr(request.app.state, ttl_setting, 30)
cache_key = _build_cache_key(request, endpoint_name, key_builder)
body, etag = _lookup(cache, cache_key, request)
cached = _lookup(cache, cache_key, request)
if cached is not _MISS:
return cached
if body is _MISS:
result = func(*args, **kwargs)
_, etag = _store(cache, cache_key, result, ttl)
return_value = result
else:
return_value = body
result = func(*args, **kwargs)
_store(cache, cache_key, result, ttl)
return result
request.state.api_etag = etag
if_none_match = request.headers.get("if-none-match")
if if_none_match and _etag_matches(if_none_match, etag):
return Response(
status_code=304,
headers={"ETag": etag},
)
return return_value
return sync_wrapper
+25 -2
View File
@@ -173,10 +173,26 @@ import click
@click.option(
"--redis-cache-ttl-dashboard",
type=int,
default=30,
default=300,
envvar="REDIS_CACHE_TTL_DASHBOARD",
help="Cache TTL for dashboard endpoints (seconds)",
)
@click.option(
"--redis-cache-ttl-route-detail",
type=int,
default=300,
envvar="REDIS_CACHE_TTL_ROUTE_DETAIL",
help="Cache TTL for /routes/{id} detail endpoint (seconds)",
)
@click.option(
"--api-cache-control-enabled/--no-api-cache-control",
default=True,
envvar="API_CACHE_CONTROL_ENABLED",
help=(
"Emit HTTP Cache-Control headers on /api/v1/* responses and ETag / "
"If-None-Match handling on @cached endpoints"
),
)
@click.option(
"--reload",
is_flag=True,
@@ -223,6 +239,8 @@ def api(
redis_key_prefix: str,
redis_cache_ttl: int,
redis_cache_ttl_dashboard: int,
redis_cache_ttl_route_detail: int,
api_cache_control_enabled: bool,
reload: bool,
workers: int,
) -> None:
@@ -280,8 +298,11 @@ def api(
click.echo(f"Redis: {redis_host}:{redis_port}/{redis_db}")
click.echo(f"Redis key prefix: {redis_key_prefix}")
click.echo(
f"Redis cache TTL: {redis_cache_ttl}s (dashboard: {redis_cache_ttl_dashboard}s)"
f"Redis cache TTL: {redis_cache_ttl}s "
f"(dashboard: {redis_cache_ttl_dashboard}s, "
f"route detail: {redis_cache_ttl_route_detail}s)"
)
click.echo(f"API Cache-Control enabled: {api_cache_control_enabled}")
click.echo(f"Reload mode: {reload}")
click.echo(f"Workers: {workers}")
click.echo("=" * 50)
@@ -343,6 +364,8 @@ def api(
redis_key_prefix=redis_key_prefix,
redis_cache_ttl=redis_cache_ttl,
redis_cache_ttl_dashboard=redis_cache_ttl_dashboard,
redis_cache_ttl_route_detail=redis_cache_ttl_route_detail,
api_cache_control_enabled=api_cache_control_enabled,
)
click.echo("\nStarting API server...")
+5
View File
@@ -214,6 +214,11 @@ def create_route(
@router.get("/{route_id}", response_model=RouteDetail)
@cached(
"routes/{id}",
ttl_setting="redis_cache_ttl_route_detail",
key_builder=_routes_key_builder,
)
def get_route(
_: RequireRead,
session: DbSession,
@@ -2,7 +2,7 @@
import logging
from datetime import datetime, timedelta, timezone
from typing import Any
from typing import Any, Optional
from sqlalchemy import select
from sqlalchemy.exc import IntegrityError
@@ -33,7 +33,7 @@ def handle_advertisement(
event_type: str,
payload: dict[str, Any],
db: DatabaseManager,
) -> None:
) -> Optional[str]:
"""Handle an advertisement event.
1. Upserts the node in the nodes table
@@ -45,11 +45,16 @@ def handle_advertisement(
event_type: Event type name
payload: Advertisement payload
db: Database manager
Returns:
The ``event_hash`` of the underlying advertisement (used by the
subscriber to denormalize the identity onto the captured raw
packet), or ``None`` on early exit.
"""
adv_public_key = payload.get("public_key")
if not adv_public_key:
logger.warning("Advertisement missing public_key")
return
return None
name = payload.get("name")
adv_type = payload.get("adv_type")
@@ -154,7 +159,7 @@ def handle_advertisement(
f"Added receiver {public_key[:12]}... to advertisement "
f"(hash={event_hash[:8]}...)"
)
return
return event_hash
# Find or create advertised node
node_query = select(Node).where(Node.public_key == adv_public_key)
@@ -237,8 +242,9 @@ def handle_advertisement(
path_len=path_len,
observed_at=now,
)
return
return event_hash
logger.info(
f"Stored advertisement from {name or adv_public_key[:12]!r} (type={adv_type})"
)
return event_hash
+23 -9
View File
@@ -2,7 +2,7 @@
import logging
from datetime import datetime, timezone
from typing import Any
from typing import Any, Optional
from sqlalchemy import select
from sqlalchemy.exc import IntegrityError
@@ -25,7 +25,7 @@ def handle_contact_message(
event_type: str,
payload: dict[str, Any],
db: DatabaseManager,
) -> None:
) -> Optional[str]:
"""Handle a contact message event.
Args:
@@ -33,8 +33,12 @@ def handle_contact_message(
event_type: Event type name
payload: Message payload
db: Database manager
Returns:
The ``event_hash`` of the underlying message, or ``None`` on early
exit.
"""
_handle_message(public_key, "contact", payload, db)
return _handle_message(public_key, "contact", payload, db)
def handle_channel_message(
@@ -42,7 +46,7 @@ def handle_channel_message(
event_type: str,
payload: dict[str, Any],
db: DatabaseManager,
) -> None:
) -> Optional[str]:
"""Handle a channel message event.
Args:
@@ -50,8 +54,12 @@ def handle_channel_message(
event_type: Event type name
payload: Message payload
db: Database manager
Returns:
The ``event_hash`` of the underlying message, or ``None`` on early
exit.
"""
_handle_message(public_key, "channel", payload, db)
return _handle_message(public_key, "channel", payload, db)
def _handle_message(
@@ -59,7 +67,7 @@ def _handle_message(
message_type: str,
payload: dict[str, Any],
db: DatabaseManager,
) -> None:
) -> Optional[str]:
"""Handle a message event (contact or channel).
Args:
@@ -67,11 +75,15 @@ def _handle_message(
message_type: Message type ('contact' or 'channel')
payload: Message payload
db: Database manager
Returns:
The ``event_hash`` of the underlying message, or ``None`` on early
exit.
"""
text = payload.get("text")
if not text:
logger.warning("Message missing text content")
return
return None
now = datetime.now(timezone.utc)
@@ -144,7 +156,7 @@ def _handle_message(
f"Added receiver {public_key[:12]}... to message "
f"(hash={event_hash[:8]}...)"
)
return
return event_hash
# Spam scoring (online): only on the first insert of an event_hash, and
# only when the feature is switched on. When off, the columns stay null
@@ -227,7 +239,7 @@ def _handle_message(
path_len=path_len,
observed_at=now,
)
return
return event_hash
# Surface the spam score (and the signals that drove it) in the log so it is
# observable without querying the DB. Likely-spam (>= threshold) is logged at
@@ -252,3 +264,5 @@ def _handle_message(
logger.warning(line)
else:
logger.info(line)
return event_hash
@@ -11,7 +11,7 @@ import logging
from datetime import datetime, timezone
from typing import Any, Optional
from sqlalchemy import select
from sqlalchemy import select, update
from meshcore_hub.collector.letsmesh_normalizer import LetsMeshNormalizer
from meshcore_hub.common.database import DatabaseManager
@@ -56,7 +56,7 @@ def store_raw_packet(
decoded_packet: Optional[dict[str, Any]],
event_type: str,
db: DatabaseManager,
) -> None:
) -> Optional[str]:
"""Capture a single raw packet from the LetsMesh ``packets`` feed.
Args:
@@ -65,6 +65,11 @@ def store_raw_packet(
decoded_packet: Decoder output already produced during normalization
event_type: How the collector classified the packet
db: Database manager
Returns:
The ``raw_packet.id`` of the inserted row, or ``None`` on failure.
The caller uses the id to backfill ``event_hash`` after the
structured handler resolves the underlying event identity.
"""
now = datetime.now(timezone.utc)
@@ -168,3 +173,34 @@ def store_raw_packet(
)
logger.debug("Captured raw packet: %s (%s)", packet_hash or "unknown", event_type)
return raw_packet.id
def update_raw_packet_event_hash(
raw_packet_id: str,
event_hash: str,
db: DatabaseManager,
) -> None:
"""Backfill ``event_hash`` onto a captured raw packet and its hops.
Called by the subscriber after the structured handler resolves the
underlying event identity. Two single-row UPDATEs scoped by
``raw_packet_id``; no-op if the row no longer exists (e.g. retention
cleanup ran between capture and dispatch).
Args:
raw_packet_id: The id returned by :func:`store_raw_packet`.
event_hash: The ``event_hash`` returned by the structured handler.
db: Database manager.
"""
with db.session_scope() as session:
session.execute(
update(RawPacket)
.where(RawPacket.id == raw_packet_id)
.values(event_hash=event_hash)
)
session.execute(
update(PacketPathHop)
.where(PacketPathHop.raw_packet_id == raw_packet_id)
.values(event_hash=event_hash)
)
@@ -2,7 +2,7 @@
import logging
from datetime import datetime, timezone
from typing import Any
from typing import Any, Optional
from sqlalchemy import select
from sqlalchemy.exc import IntegrityError
@@ -19,7 +19,7 @@ def handle_telemetry(
event_type: str,
payload: dict[str, Any],
db: DatabaseManager,
) -> None:
) -> Optional[str]:
"""Handle a telemetry response event.
Args:
@@ -27,11 +27,15 @@ def handle_telemetry(
event_type: Event type name
payload: Telemetry payload
db: Database manager
Returns:
The ``event_hash`` of the underlying telemetry record, or ``None``
on early exit.
"""
node_public_key = payload.get("node_public_key")
if not node_public_key:
logger.warning("Telemetry missing node_public_key")
return
return None
now = datetime.now(timezone.utc)
@@ -100,7 +104,7 @@ def handle_telemetry(
f"Added receiver {public_key[:12]}... to telemetry "
f"(node={node_public_key[:12]}...)"
)
return
return event_hash
# Find or create reporting node
reporting_node = None
@@ -164,7 +168,7 @@ def handle_telemetry(
path_len=path_len,
observed_at=now,
)
return
return event_hash
# Log telemetry values
if parsed_data:
@@ -172,3 +176,5 @@ def handle_telemetry(
logger.info(f"Stored telemetry from {node_public_key[:12]!r}: {values}")
else:
logger.info(f"Stored telemetry from {node_public_key[:12]!r}")
return event_hash
+10 -5
View File
@@ -2,7 +2,7 @@
import logging
from datetime import datetime, timezone
from typing import Any
from typing import Any, Optional
from sqlalchemy import select
from sqlalchemy.exc import IntegrityError
@@ -19,7 +19,7 @@ def handle_trace_data(
event_type: str,
payload: dict[str, Any],
db: DatabaseManager,
) -> None:
) -> Optional[str]:
"""Handle a trace data event.
Args:
@@ -27,11 +27,15 @@ def handle_trace_data(
event_type: Event type name
payload: Trace data payload
db: Database manager
Returns:
The ``event_hash`` of the underlying trace record, or ``None`` on
early exit.
"""
initiator_tag = payload.get("initiator_tag")
if initiator_tag is None:
logger.warning("Trace data missing initiator_tag")
return
return None
now = datetime.now(timezone.utc)
@@ -86,7 +90,7 @@ def handle_trace_data(
f"Added receiver {public_key[:12]}... to trace "
f"(tag={initiator_tag})"
)
return
return event_hash
# Create trace path record
trace_path = TracePath(
@@ -135,6 +139,7 @@ def handle_trace_data(
path_len=path_len,
observed_at=now,
)
return
return event_hash
logger.info(f"Stored trace data: tag={initiator_tag}, hops={hop_count}")
return event_hash
+64 -22
View File
@@ -74,6 +74,23 @@ def derive_quality(
return RouteQuality.UNKNOWN.value
def _match_identity(hops: list[dict[str, Any]]) -> Optional[str]:
"""Identity used to dedup matched receptions into one match per event.
Prefers the denormalized ``event_hash`` (the underlying structured
event's identity, populated at ingest) so that retransmissions of the
same advert/message/telemetry/trace count once instead of once per
on-air copy. Falls back to the per-transmission wire ``packet_hash``
when ``event_hash`` is NULL (legacy rows captured before the column
existed, or unclassified packets that didn't trigger a structured
handler).
"""
if not hops:
return None
first = hops[0]
return first.get("event_hash") or first.get("packet_hash")
def _subsequence_indices(
path: list[dict[str, Any]],
expected: list[str],
@@ -281,6 +298,7 @@ def fetch_candidate_paths(
PacketPathHop.position,
PacketPathHop.node_hash,
PacketPathHop.packet_hash,
PacketPathHop.event_hash,
PacketPathHop.received_at,
PacketPathHop.observer_node_id,
)
@@ -298,6 +316,7 @@ def fetch_candidate_paths(
"position": row.position,
"node_hash": row.node_hash,
"packet_hash": row.packet_hash,
"event_hash": row.event_hash,
"received_at": row.received_at,
"observer_node_id": row.observer_node_id,
}
@@ -337,6 +356,7 @@ def _fetch_matching_hops(
PacketPathHop.position,
PacketPathHop.node_hash,
PacketPathHop.packet_hash,
PacketPathHop.event_hash,
PacketPathHop.received_at,
)
.where(
@@ -359,6 +379,7 @@ def _fetch_matching_hops(
"position": row.position,
"node_hash": row.node_hash,
"packet_hash": row.packet_hash,
"event_hash": row.event_hash,
"received_at": row.received_at,
}
)
@@ -475,9 +496,9 @@ def evaluate_route(
matched_packets: set[str] = set()
for hops in paths.values():
if _match_hops(hops, expected, route.max_hop_span, reversible):
ph = hops[0]["packet_hash"]
if ph:
matched_packets.add(ph)
identity = _match_identity(hops)
if identity:
matched_packets.add(identity)
if len(matched_packets) >= eff_clear:
return (
RouteState.HEALTHY.value,
@@ -528,9 +549,9 @@ def evaluate_route_day(
matched_packets: set[str] = set()
for hops in paths.values():
if _match_hops(hops, expected, route.max_hop_span, reversible):
ph = hops[0]["packet_hash"]
if ph:
matched_packets.add(ph)
identity = _match_identity(hops)
if identity:
matched_packets.add(identity)
if len(matched_packets) >= eff_clear:
return (
RouteQuality.CLEAR.value,
@@ -673,9 +694,9 @@ def evaluate_route_history(
matched_packets: set[str] = set()
for hops in day_paths[i].values():
if _match_hops(hops, expected, route.max_hop_span, reversible):
ph = hops[0]["packet_hash"]
if ph:
matched_packets.add(ph)
identity = _match_identity(hops)
if identity:
matched_packets.add(identity)
if len(matched_packets) >= eff_clear:
break
@@ -765,7 +786,13 @@ def recent_matches(
limit: int = 3,
now: Optional[datetime] = None,
) -> list[dict[str, Any]]:
"""Return the latest *limit* matching paths for a route."""
"""Return the latest *limit* matching paths for a route.
Deduplicates by event identity (preferring ``event_hash``, falling back
to wire ``packet_hash``) so the UI shows one row per underlying event
rather than one row per retransmission. When multiple receptions share
an identity, the newest is returned.
"""
expected = _route_expected_hashes(route)
if len(expected) < 2:
return []
@@ -781,22 +808,37 @@ def recent_matches(
session, expected, since, observer_ids, reversible
)
matches: list[dict[str, Any]] = []
# Keep the newest match per identity so the UI lists distinct underlying
# events rather than every retransmission of the same event.
matches_by_identity: dict[str, dict[str, Any]] = {}
for hops in paths.values():
subpath = _matched_subpath(hops, expected, route.max_hop_span, reversible)
if not subpath:
continue
identity = _match_identity(subpath)
if identity is None:
# Fall back to a synthetic unique key so unmatched-identity
# receptions still surface (one row each).
identity = f"__rawid_{id(subpath)}"
first = subpath[0] if subpath else {}
matches.append(
{
"packet_hash": first.get("packet_hash"),
"hops": subpath,
"received_at": first.get("received_at"),
"observer_node_id": first.get("observer_node_id"),
}
received_at = first.get("received_at") or datetime.min.replace(
tzinfo=timezone.utc
)
candidate = {
"packet_hash": first.get("packet_hash"),
"event_hash": first.get("event_hash"),
"hops": subpath,
"received_at": first.get("received_at"),
"observer_node_id": first.get("observer_node_id"),
}
existing = matches_by_identity.get(identity)
if existing is None or received_at > (
existing.get("received_at") or datetime.min.replace(tzinfo=timezone.utc)
):
matches_by_identity[identity] = candidate
matches.sort(
matches = sorted(
matches_by_identity.values(),
key=lambda m: m["received_at"] or datetime.min.replace(tzinfo=timezone.utc),
reverse=True,
)
@@ -878,9 +920,9 @@ def preview_route(
for hops in paths.values():
if _match_hops(hops, expected, max_hop_span, reversible):
ph = hops[0]["packet_hash"]
if ph:
matched_packets.add(ph)
identity = _match_identity(hops)
if identity:
matched_packets.add(identity)
obs = hops[0]["observer_node_id"]
if obs:
contributing[obs] = contributing.get(obs, 0) + 1
+62 -14
View File
@@ -31,8 +31,10 @@ if TYPE_CHECKING:
logger = logging.getLogger(__name__)
# Handler type: receives (public_key, event_type, payload, db_manager)
EventHandler = Callable[[str, str, dict[str, Any], DatabaseManager], None]
# Handler type: receives (public_key, event_type, payload, db_manager),
# returns the underlying event's event_hash (or None) so the subscriber can
# denormalize the identity onto the captured raw packet.
EventHandler = Callable[[str, str, dict[str, Any], DatabaseManager], Optional[str]]
class Subscriber(LetsMeshNormalizer):
@@ -253,13 +255,22 @@ class Subscriber(LetsMeshNormalizer):
public_key, event_type, normalized_payload = parsed
logger.debug("Received event: %s from %s...", event_type, public_key[:12])
# Capture the raw packet (packets feed only) independent of, and before,
# structured dispatch so the raw_packets table is complete. The boolean
# short-circuit avoids the insert entirely when capture is disabled.
# Capture the raw packet (packets feed only) independent of, and
# before, structured dispatch so the raw_packets table is complete.
# The boolean short-circuit avoids the insert entirely when capture
# is disabled. The returned id lets us backfill ``event_hash`` onto
# the row (and its hops) once the structured handler resolves the
# underlying event identity.
raw_packet_id: str | None = None
if self._raw_packet_capture_enabled:
self._maybe_capture_raw_packet(topic, public_key, event_type, payload)
raw_packet_id = self._maybe_capture_raw_packet(
topic, public_key, event_type, payload
)
self._dispatch_event(public_key, event_type, normalized_payload)
event_hash = self._dispatch_event(public_key, event_type, normalized_payload)
if raw_packet_id and event_hash:
self._backfill_raw_packet_event_hash(raw_packet_id, event_hash)
def _maybe_capture_raw_packet(
self,
@@ -267,41 +278,76 @@ class Subscriber(LetsMeshNormalizer):
public_key: str,
event_type: str,
payload: dict[str, Any],
) -> None:
) -> str | None:
"""Persist a raw_packets row for a packets-feed reception.
Reuses the decode the normalizer already performed (the decoder caches
per raw hex, so this ``decode_payload`` call is a cache hit). Capture
failures are logged and never block event dispatch.
Returns:
The inserted ``raw_packet.id``, or ``None`` when the topic is
not a packets-feed message or capture fails. The caller uses the
id to backfill ``event_hash`` after dispatch.
"""
try:
parsed_topic = self.mqtt.topic_builder.parse_letsmesh_upload_topic(topic)
if not parsed_topic:
return
return None
_, feed_type = parsed_topic
if feed_type != "packets":
return
return None
from meshcore_hub.collector.handlers.raw_packet import store_raw_packet
decoded_packet = self._letsmesh_decoder.decode_payload(payload)
store_raw_packet(public_key, payload, decoded_packet, event_type, self.db)
return store_raw_packet(
public_key, payload, decoded_packet, event_type, self.db
)
except Exception as e:
logger.error("Error capturing raw packet: %s", e)
return None
def _backfill_raw_packet_event_hash(
self,
raw_packet_id: str,
event_hash: str,
) -> None:
"""Propagate ``event_hash`` onto a captured raw packet and its hops.
Runs after structured dispatch resolves the underlying event. Two
single-row UPDATEs scoped by ``raw_packet_id``; failures are logged
and swallowed so they never affect subsequent ingest.
"""
try:
from meshcore_hub.collector.handlers.raw_packet import (
update_raw_packet_event_hash,
)
update_raw_packet_event_hash(raw_packet_id, event_hash, self.db)
except Exception as e:
logger.error("Error backfilling raw packet event_hash: %s", e)
def _dispatch_event(
self,
public_key: str,
event_type: str,
payload: dict[str, Any],
) -> None:
"""Route a normalized event to the appropriate handler."""
) -> str | None:
"""Route a normalized event to the appropriate handler.
Returns:
The ``event_hash`` returned by the structured handler (used by
the caller to denormalize the identity onto the captured raw
packet), or ``None`` when no handler ran or it returned None.
"""
event_hash: str | None = None
# Find and call handler
handler = self._handlers.get(event_type)
if handler:
try:
handler(public_key, event_type, payload, self.db)
event_hash = handler(public_key, event_type, payload, self.db)
except Exception as e:
logger.error(f"Error handling {event_type}: {e}")
else:
@@ -317,6 +363,8 @@ class Subscriber(LetsMeshNormalizer):
if self._webhook_dispatcher and self._webhook_dispatcher.webhooks:
self._queue_webhook_event(event_type, payload, public_key)
return event_hash
def _queue_webhook_event(
self, event_type: str, payload: dict[str, Any], public_key: str
) -> None:
+19 -2
View File
@@ -418,8 +418,25 @@ class APISettings(CommonSettings):
description="Default cache TTL in seconds",
)
redis_cache_ttl_dashboard: int = Field(
default=30,
description="Cache TTL for dashboard endpoints (seconds)",
default=300,
description=(
"Cache TTL in seconds for dashboard endpoints and per-route "
"health history (trend/aggregation data tolerates longer staleness)"
),
)
redis_cache_ttl_route_detail: int = Field(
default=300,
description="Cache TTL for /routes/{id} detail endpoint (seconds)",
)
# HTTP Cache-Control headers
api_cache_control_enabled: bool = Field(
default=True,
description=(
"Emit HTTP Cache-Control headers on /api/v1/* responses and ETag / "
"If-None-Match handling on @cached endpoints. Disable to suppress "
"all client-side caching directives."
),
)
@@ -27,6 +27,12 @@ class PacketPathHop(Base, UUIDMixin, TimestampMixin):
position: Zero-based hop position in the ordered path
node_hash: Normalized (uppercase) hex prefix for this hop
packet_hash: Denormalized packet hash from raw_packets
event_hash: Denormalized identity of the underlying structured event
from raw_packets. The route evaluator prefers ``event_hash`` over
``packet_hash`` when deduplicating matches, so retransmissions
of the same underlying advert/message/telemetry/trace count once
instead of once per on-air copy. ``NULL`` for legacy rows and
unclassified packets; the evaluator falls back to ``packet_hash``.
received_at: Denormalized reception timestamp from raw_packets
observer_node_id: Denormalized observer node FK
"""
@@ -49,6 +55,10 @@ class PacketPathHop(Base, UUIDMixin, TimestampMixin):
String(32),
nullable=True,
)
event_hash: Mapped[Optional[str]] = mapped_column(
String(32),
nullable=True,
)
received_at: Mapped[datetime] = mapped_column(
DateTime(timezone=True),
nullable=False,
@@ -73,6 +83,11 @@ class PacketPathHop(Base, UUIDMixin, TimestampMixin):
"ix_packet_path_hops_received_at",
"received_at",
),
Index(
"ix_packet_path_hops_event_hash_received_at",
"event_hash",
"received_at",
),
)
def __repr__(self) -> str:
@@ -22,6 +22,14 @@ class RawPacket(Base, UUIDMixin, TimestampMixin):
observer_node_id: FK to nodes (the receiving interface)
packet_hash: LetsMesh packet hash, links rows to structured records and
groups multi-observer receptions
event_hash: Denormalized identity of the underlying structured event
(advertisement / message / telemetry / trace) computed by the
structured handler. Populated after dispatch by the subscriber.
``NULL`` for packets that did not trigger a structured handler and
for rows captured before this column was added. The route
evaluator prefers ``event_hash`` over ``packet_hash`` when
deduplicating matches, so retransmissions of the same underlying
event count once instead of once per on-air copy.
raw_hex: The on-air bytes from ``payload["raw"]``
packet_type: Wire packet type
payload_type: Decoder payload type
@@ -51,6 +59,11 @@ class RawPacket(Base, UUIDMixin, TimestampMixin):
nullable=True,
index=True,
)
event_hash: Mapped[Optional[str]] = mapped_column(
String(32),
nullable=True,
index=True,
)
raw_hex: Mapped[Optional[str]] = mapped_column(
Text,
nullable=True,
@@ -200,15 +200,41 @@ function renderDetailContent(route, detail, { navigate, packetsEnabled, history
const detailUrl = packetDetailUrl(m.packet_hash);
return html`<div class="flex flex-wrap items-center gap-0.5 text-xs pb-1 border-b border-base-300 last:border-0 ${detailUrl ? 'hover:bg-base-200 cursor-pointer -mx-1 px-1 rounded transition-colors' : ''}"
@click=${detailUrl ? (e) => { e.stopPropagation(); navigate(detailUrl); } : undefined}>
${(m.hops || []).map((h, i) => {
const rn = pathLookup.get((h.node_hash || '').toLowerCase().slice(0, prefixLen));
return html`
${i > 0 ? html`<span class="opacity-30 mx-0.5">\u2192</span>` : nothing}
${rn
? html`<span class="badge badge-primary badge-sm">${(h.node_hash || '').toLowerCase()}</span>`
: html`<span class="badge badge-ghost badge-sm opacity-50">${(h.node_hash || '').toLowerCase()}</span>`}
`;
})}
${(() => {
const hops = m.hops || [];
const PATH_MAX = 5;
const PATH_HEAD = 2;
const PATH_TAIL = 2;
let entries;
if (hops.length > PATH_MAX) {
const hidden = hops.length - PATH_HEAD - PATH_TAIL;
const head = hops.slice(0, PATH_HEAD);
const tail = hops.slice(-PATH_TAIL);
const ellipsis = html`<span class="badge badge-ghost badge-sm cursor-help" title=${t('packets.hops_hidden', { count: hidden })}>\u2026</span>`;
entries = [
...head.map(h => [h, true]),
[ellipsis, false],
...tail.map(h => [h, true]),
];
} else {
entries = hops.map(h => [h, true]);
}
return entries.map(([h, isHop], i) => {
if (!isHop) {
return html`
${i > 0 ? html`<span class="opacity-30 mx-0.5">\u2192</span>` : nothing}
${h}
`;
}
const rn = pathLookup.get((h.node_hash || '').toLowerCase().slice(0, prefixLen));
return html`
${i > 0 ? html`<span class="opacity-30 mx-0.5">\u2192</span>` : nothing}
${rn
? html`<span class="badge badge-primary badge-sm">${(h.node_hash || '').toLowerCase()}</span>`
: html`<span class="badge badge-ghost badge-sm opacity-50">${(h.node_hash || '').toLowerCase()}</span>`}
`;
});
})()}
${m.received_at ? html`<span class="ml-auto opacity-40 whitespace-nowrap">${new Date(m.received_at).toLocaleString()}</span>` : nothing}
</div>`;
})}
+471 -4
View File
@@ -430,6 +430,33 @@ class TestCachedDecorator:
call_args = mock_cache.set.call_args
assert call_args[0][2] == 60 # TTL should be 60, not 30
async def test_route_detail_ttl_override(self):
app = FastAPI()
mock_cache = MagicMock()
mock_cache.get.return_value = None
app.state.redis_cache = mock_cache
app.state.redis_cache_ttl = 30
app.state.redis_cache_ttl_route_detail = 90
@cached("routes/{id}", ttl_setting="redis_cache_ttl_route_detail")
async def handler(request: Request):
return {"id": "abc", "matches": []}
scope = {
"type": "http",
"query_string": b"",
"headers": [],
"app": app,
}
from starlette.datastructures import State
request = Request(scope)
request._state = State()
await handler(request=request)
call_args = mock_cache.set.call_args
assert call_args[0][2] == 90 # TTL should be 90, not 30
async def test_serializes_pydantic_model_result(self):
app = FastAPI()
mock_cache = MagicMock()
@@ -455,8 +482,10 @@ class TestCachedDecorator:
result = await handler(request=request)
assert result.items == [1, 2]
set_call = mock_cache.set.call_args
stored = json.loads(set_call[0][1])
assert stored == {"items": [1, 2], "total": 2}
envelope = json.loads(set_call[0][1])
assert envelope["body"] == {"items": [1, 2], "total": 2}
assert isinstance(envelope["etag"], str)
assert envelope["etag"].startswith('"')
async def test_serializes_dict_result(self):
app = FastAPI()
@@ -483,7 +512,9 @@ class TestCachedDecorator:
result = await handler(request=request)
assert result == {"key": "value"}
set_call = mock_cache.set.call_args
assert set_call[0][1] == '{"key": "value"}'
envelope = json.loads(set_call[0][1])
assert envelope["body"] == {"key": "value"}
assert isinstance(envelope["etag"], str)
async def test_serializes_other_result_with_default_str(self):
app = FastAPI()
@@ -510,7 +541,8 @@ class TestCachedDecorator:
result = await handler(request=request)
assert result == ["a", "b"]
set_call = mock_cache.set.call_args
assert json.loads(set_call[0][1]) == ["a", "b"]
envelope = json.loads(set_call[0][1])
assert envelope["body"] == ["a", "b"]
async def test_cache_set_error_falls_through(self):
app = FastAPI()
@@ -560,6 +592,394 @@ class TestCachedDecorator:
assert result == {"items": ["direct"]}
class TestCachedEtag:
"""ETag / If-None-Match behavior of @cached."""
@staticmethod
def _make_request(app, headers=None):
scope = {
"type": "http",
"query_string": b"",
"headers": [(k.lower().encode(), v.encode()) for k, v in (headers or [])],
"app": app,
}
from starlette.datastructures import State
request = Request(scope)
request._state = State()
return request
async def test_miss_sets_request_state_etag(self):
app = FastAPI()
mock_cache = MagicMock()
mock_cache.get.return_value = None
app.state.redis_cache = mock_cache
app.state.redis_cache_ttl = 30
@cached("nodes")
async def handler(request: Request):
return {"items": ["a"], "total": 1}
request = self._make_request(app)
result = await handler(request=request)
assert result == {"items": ["a"], "total": 1}
etag = request.state.api_etag
assert isinstance(etag, str)
assert etag.startswith('"')
async def test_hit_reads_etag_from_envelope(self):
app = FastAPI()
mock_cache = MagicMock()
body = {"items": [], "total": 0}
etag = '"deadbeefdeadbeefdeadbeefdeadbeef"'
mock_cache.get.return_value = json.dumps({"body": body, "etag": etag})
app.state.redis_cache = mock_cache
app.state.redis_cache_ttl = 30
@cached("nodes")
async def handler(request: Request):
return {"items": ["should not appear"], "total": 1}
request = self._make_request(app)
result = await handler(request=request)
assert result == body
assert request.state.api_etag == etag
assert request.state.cache_status == "HIT"
async def test_hit_computes_etag_for_legacy_entry(self):
"""Legacy bare-JSON cache entries (no etag envelope) still serve and
get an ETag computed on the fly."""
app = FastAPI()
mock_cache = MagicMock()
body = {"items": [], "total": 0}
mock_cache.get.return_value = json.dumps(body)
app.state.redis_cache = mock_cache
app.state.redis_cache_ttl = 30
@cached("nodes")
async def handler(request: Request):
return {"items": ["should not appear"], "total": 1}
request = self._make_request(app)
result = await handler(request=request)
assert result == body
assert request.state.cache_status == "HIT"
# ETag is computed deterministically from the legacy body bytes.
assert request.state.api_etag.startswith('"')
async def test_garbage_cache_entry_treated_as_miss(self):
app = FastAPI()
mock_cache = MagicMock()
mock_cache.get.return_value = "not valid json {"
app.state.redis_cache = mock_cache
app.state.redis_cache_ttl = 30
@cached("nodes")
async def handler(request: Request):
return {"items": ["real"], "total": 1}
request = self._make_request(app)
result = await handler(request=request)
assert result == {"items": ["real"], "total": 1}
assert request.state.cache_status == "MISS"
async def test_if_none_match_returns_304_on_miss(self):
app = FastAPI()
mock_cache = MagicMock()
mock_cache.get.return_value = None
app.state.redis_cache = mock_cache
app.state.redis_cache_ttl = 30
@cached("nodes")
async def handler(request: Request):
return {"items": ["a"], "total": 1}
# First call to learn the ETag.
request = self._make_request(app)
await handler(request=request)
etag = request.state.api_etag
# Second call with matching If-None-Match.
request2 = self._make_request(app, headers=[("if-none-match", etag)])
response = await handler(request=request2)
from fastapi.responses import Response
assert isinstance(response, Response)
assert response.status_code == 304
assert response.headers["ETag"] == etag
async def test_if_none_match_returns_304_on_hit(self):
app = FastAPI()
mock_cache = MagicMock()
body = {"items": [], "total": 0}
etag = '"abc123abc123abc123abc123abc123ab"'
mock_cache.get.return_value = json.dumps({"body": body, "etag": etag})
app.state.redis_cache = mock_cache
app.state.redis_cache_ttl = 30
@cached("nodes")
async def handler(request: Request):
return {"items": ["should not appear"], "total": 1}
request = self._make_request(app, headers=[("if-none-match", etag)])
response = await handler(request=request)
from fastapi.responses import Response
assert isinstance(response, Response)
assert response.status_code == 304
assert response.headers["ETag"] == etag
async def test_if_none_match_non_matching_returns_body(self):
app = FastAPI()
mock_cache = MagicMock()
body = {"items": [], "total": 0}
etag = '"abc123abc123abc123abc123abc123ab"'
mock_cache.get.return_value = json.dumps({"body": body, "etag": etag})
app.state.redis_cache = mock_cache
app.state.redis_cache_ttl = 30
@cached("nodes")
async def handler(request: Request):
return {"items": ["should not appear"], "total": 1}
request = self._make_request(
app, headers=[("if-none-match", '"different-etag-value"')]
)
result = await handler(request=request)
assert result == body
async def test_if_none_match_wildcard_matches_any_etag(self):
app = FastAPI()
mock_cache = MagicMock()
body = {"items": [], "total": 0}
etag = '"abc123abc123abc123abc123abc123ab"'
mock_cache.get.return_value = json.dumps({"body": body, "etag": etag})
app.state.redis_cache = mock_cache
app.state.redis_cache_ttl = 30
@cached("nodes")
async def handler(request: Request):
return {"items": ["nope"], "total": 1}
request = self._make_request(app, headers=[("if-none-match", "*")])
from fastapi.responses import Response
response = await handler(request=request)
assert isinstance(response, Response)
assert response.status_code == 304
async def test_if_none_match_accepts_weak_indicator(self):
app = FastAPI()
mock_cache = MagicMock()
body = {"items": [], "total": 0}
etag = '"abc123abc123abc123abc123abc123ab"'
mock_cache.get.return_value = json.dumps({"body": body, "etag": etag})
app.state.redis_cache = mock_cache
app.state.redis_cache_ttl = 30
@cached("nodes")
async def handler(request: Request):
return {"items": ["nope"], "total": 1}
request = self._make_request(app, headers=[("if-none-match", f"W/{etag}")])
from fastapi.responses import Response
response = await handler(request=request)
assert isinstance(response, Response)
assert response.status_code == 304
async def test_cache_control_ttl_always_set(self):
"""Even when Redis is NullCache, request.state.cache_control_ttl
is set so the middleware can emit Cache-Control."""
app = FastAPI()
app.state.redis_cache = NullCache()
app.state.redis_cache_ttl = 30
@cached("nodes")
async def handler(request: Request):
return {"items": [], "total": 0}
request = self._make_request(app)
await handler(request=request)
assert request.state.cache_control_ttl == 30
async def test_cache_control_ttl_uses_overridden_setting(self):
app = FastAPI()
app.state.redis_cache = NullCache()
app.state.redis_cache_ttl = 30
app.state.redis_cache_ttl_route_detail = 90
@cached("routes/{id}", ttl_setting="redis_cache_ttl_route_detail")
async def handler(request: Request):
return {"id": "abc"}
request = self._make_request(app)
await handler(request=request)
assert request.state.cache_control_ttl == 90
class TestCachedEtagHelpers:
"""Direct unit tests for the etag helpers."""
def test_compute_etag_is_quoted_hex(self):
from meshcore_hub.api.cache import _compute_etag
etag = _compute_etag('{"a": 1}')
assert etag.startswith('"')
assert etag.endswith('"')
assert len(etag) == 34 # 32 hex chars + 2 quotes
def test_compute_etag_is_deterministic(self):
from meshcore_hub.api.cache import _compute_etag
assert _compute_etag("body") == _compute_etag("body")
def test_compute_etag_changes_with_input(self):
from meshcore_hub.api.cache import _compute_etag
assert _compute_etag("body1") != _compute_etag("body2")
def test_etag_matches_strong_equality(self):
from meshcore_hub.api.cache import _etag_matches
etag = '"abc123"'
assert _etag_matches('"abc123"', etag)
def test_etag_matches_wildcard(self):
from meshcore_hub.api.cache import _etag_matches
assert _etag_matches("*", '"any"')
def test_etag_matches_weak_indicator(self):
from meshcore_hub.api.cache import _etag_matches
etag = '"abc123"'
assert _etag_matches('W/"abc123"', etag)
def test_etag_matches_one_of_list(self):
from meshcore_hub.api.cache import _etag_matches
etag = '"abc123"'
assert _etag_matches('"xyz", "abc123", "def"', etag)
def test_etag_no_match(self):
from meshcore_hub.api.cache import _etag_matches
assert not _etag_matches('"other"', '"abc123"')
class TestCacheControlMiddleware:
"""End-to-end Cache-Control / ETag tests via the FastAPI test client."""
def test_cached_get_emits_cache_control_and_etag(self, client_no_auth):
mock_cache = MagicMock()
mock_cache.get.return_value = None
client_no_auth.app.state.redis_cache = mock_cache
client_no_auth.app.state.redis_cache_ttl = 30
response = client_no_auth.get("/api/v1/nodes")
assert response.status_code == 200
assert response.headers["cache-control"] == "private, max-age=30"
assert "etag" in response.headers
def test_cached_get_cache_control_uses_overridden_ttl(self, client_no_auth):
"""Route detail endpoint should use the route-detail TTL (300s)."""
mock_cache = MagicMock()
mock_cache.get.return_value = None
client_no_auth.app.state.redis_cache = mock_cache
client_no_auth.app.state.redis_cache_ttl = 30
client_no_auth.app.state.redis_cache_ttl_route_detail = 300
# Need a route to exist; this is a happy-path check of the header value
# so we stub the cache and just hit the endpoint with a fake id. The
# endpoint will return 404 but still flow through the @cached decorator
# and the middleware.
response = client_no_auth.get("/api/v1/routes/nonexistent-id")
# 404 is fine; we only care about the header applied by the decorator
# via request.state.cache_control_ttl.
assert response.headers.get("cache-control") == "private, max-age=300"
def test_cached_get_304_on_matching_if_none_match(self, client_no_auth):
mock_cache = MagicMock()
body = {
"items": [],
"total": 0,
"limit": 50,
"offset": 0,
}
etag = '"abc123abc123abc123abc123abc123ab"'
mock_cache.get.return_value = json.dumps({"body": body, "etag": etag})
client_no_auth.app.state.redis_cache = mock_cache
client_no_auth.app.state.redis_cache_ttl = 30
response = client_no_auth.get("/api/v1/nodes", headers={"If-None-Match": etag})
assert response.status_code == 304
assert response.headers["ETag"] == etag
assert response.headers["cache-control"] == "private, max-age=30"
assert response.headers["x-cache"] == "HIT"
# 304 must not carry a body.
assert response.content in (b"", b"null")
def test_uncached_get_emits_must_revalidate(self, client_no_auth, sample_node):
"""Uncached GET detail endpoints get max-age=0, must-revalidate."""
# Force the @cached list endpoint to NOT be the target by hitting the
# per-id endpoint, which is not cached.
if hasattr(client_no_auth.app.state, "redis_cache"):
del client_no_auth.app.state.redis_cache
response = client_no_auth.get(f"/api/v1/nodes/{sample_node.public_key}")
assert response.status_code == 200
assert (
response.headers["cache-control"] == "private, max-age=0, must-revalidate"
)
def test_post_emits_no_store(self, client_no_auth, api_db_session):
"""POST endpoints always get Cache-Control: no-store.
Depends on ``api_db_session`` so its teardown truncates the channel
row even if the create succeeds (channel creates mutate the DB and
would otherwise leak into later tests in the same module).
"""
response = client_no_auth.post(
"/api/v1/channels",
json={
"name": "CacheControlTestChan",
"key_hex": "AABBCCDDEEFF00112233445566778899",
"visibility": "community",
},
)
# May succeed (201) or fail (400/500) depending on validation; we
# only care that the middleware set no-store on the response.
assert response.headers["cache-control"] == "no-store"
def test_health_emits_no_store(self, client_no_auth):
response = client_no_auth.get("/health")
assert response.status_code == 200
assert response.headers["cache-control"] == "no-store"
def test_health_ready_emits_no_store(self, client_no_auth):
response = client_no_auth.get("/health/ready")
assert response.status_code == 200
assert response.headers["cache-control"] == "no-store"
def test_kill_switch_suppresses_cache_control(self, client_no_auth):
"""When api_cache_control_enabled is False, no Cache-Control is added."""
client_no_auth.app.state.api_cache_control_enabled = False
if hasattr(client_no_auth.app.state, "redis_cache"):
del client_no_auth.app.state.redis_cache
response = client_no_auth.get("/api/v1/nodes")
assert "cache-control" not in response.headers
def test_kill_switch_preserves_x_cache_header(self, client_no_auth):
"""X-Cache is observability, not a client-caching directive, so the
kill switch should not suppress it."""
mock_cache = MagicMock()
mock_cache.get.return_value = None
client_no_auth.app.state.redis_cache = mock_cache
client_no_auth.app.state.redis_cache_ttl = 30
client_no_auth.app.state.api_cache_control_enabled = False
response = client_no_auth.get("/api/v1/nodes")
assert response.headers.get("x-cache") == "MISS"
assert "cache-control" not in response.headers
class TestLifespanRedis:
async def test_lifespan_creates_null_cache_when_disabled(self):
import meshcore_hub.api.app as app_module
@@ -750,6 +1170,9 @@ class TestCliRedis:
"60",
"--redis-cache-ttl-dashboard",
"120",
"--redis-cache-ttl-route-detail",
"90",
"--no-api-cache-control",
],
catch_exceptions=False,
)
@@ -762,6 +1185,50 @@ class TestCliRedis:
assert call_kwargs["redis_key_prefix"] == "pre"
assert call_kwargs["redis_cache_ttl"] == 60
assert call_kwargs["redis_cache_ttl_dashboard"] == 120
assert call_kwargs["redis_cache_ttl_route_detail"] == 90
assert call_kwargs["api_cache_control_enabled"] is False
def test_api_cache_control_enabled_default(self):
"""Without --no-api-cache-control, the flag defaults to True."""
from click.testing import CliRunner
from meshcore_hub.api.cli import api
runner = CliRunner()
with patch("uvicorn.run"):
with patch("meshcore_hub.common.config.get_api_settings") as mock_settings:
mock_settings.return_value = MagicMock(
data_home="/tmp/test",
effective_database_url="sqlite:///test.db",
)
with patch("meshcore_hub.api.app.create_app") as mock_create_app:
mock_create_app.return_value = MagicMock()
runner.invoke(api, catch_exceptions=False)
call_kwargs = mock_create_app.call_args[1]
assert call_kwargs["api_cache_control_enabled"] is True
def test_redis_cache_ttl_dashboard_default(self):
"""--redis-cache-ttl-dashboard defaults to 300 (raised from 30).
Covers /dashboard/* endpoints and /routes/{id}/history. Trend data
tolerates longer staleness than the original 30s default.
"""
from click.testing import CliRunner
from meshcore_hub.api.cli import api
runner = CliRunner()
with patch("uvicorn.run"):
with patch("meshcore_hub.common.config.get_api_settings") as mock_settings:
mock_settings.return_value = MagicMock(
data_home="/tmp/test",
effective_database_url="sqlite:///test.db",
)
with patch("meshcore_hub.api.app.create_app") as mock_create_app:
mock_create_app.return_value = MagicMock()
runner.invoke(api, catch_exceptions=False)
call_kwargs = mock_create_app.call_args[1]
assert call_kwargs["redis_cache_ttl_dashboard"] == 300
class TestKeyBuilders:
+80
View File
@@ -247,6 +247,86 @@ class TestGetRouteDetail:
resp = client_no_auth.get("/api/v1/routes/nonexistent")
assert resp.status_code == 404
def test_detail_response_is_cached(self, client_no_auth, api_db_session):
"""Detail endpoint writes its response to the cache after a miss."""
import json
from unittest.mock import MagicMock
nodes = _sample_nodes(api_db_session, 2)
route = Route(from_label="Alpha", to_label="Beta")
api_db_session.add(route)
api_db_session.flush()
for pos, n in enumerate(nodes):
api_db_session.add(
RouteNode(
route_id=route.id,
node_id=n.id,
position=pos,
expected_hash=n.public_key[:2].upper(),
)
)
api_db_session.commit()
mock_cache = MagicMock()
mock_cache.get.return_value = None
client_no_auth.app.state.redis_cache = mock_cache
client_no_auth.app.state.redis_cache_ttl_route_detail = 90
resp = client_no_auth.get(f"/api/v1/routes/{route.id}")
assert resp.status_code == 200
assert resp.headers.get("x-cache") == "MISS"
mock_cache.set.assert_called_once()
cache_key, serialized, ttl = mock_cache.set.call_args[0]
assert cache_key.startswith(f"/api/v1/routes/{route.id}:")
assert "role=anonymous" in cache_key
assert ttl == 90
envelope = json.loads(serialized)
assert envelope["body"]["from_label"] == "Alpha"
assert isinstance(envelope["etag"], str)
def test_detail_serves_from_cache_on_hit(self, client_no_auth, api_db_session):
"""A second call within the TTL window is served from the cache."""
nodes = _sample_nodes(api_db_session, 2)
route = Route(from_label="Alpha", to_label="Beta")
api_db_session.add(route)
api_db_session.flush()
for pos, n in enumerate(nodes):
api_db_session.add(
RouteNode(
route_id=route.id,
node_id=n.id,
position=pos,
expected_hash=n.public_key[:2].upper(),
)
)
api_db_session.commit()
store: dict[str, str] = {}
class _FakeCache:
def get(self, key):
return store.get(key)
def set(self, key, value, ttl):
store[key] = value
def ping(self):
return True
client_no_auth.app.state.redis_cache = _FakeCache()
client_no_auth.app.state.redis_cache_ttl_route_detail = 60
first = client_no_auth.get(f"/api/v1/routes/{route.id}")
assert first.status_code == 200
assert first.headers.get("x-cache") == "MISS"
first_body = first.json()
second = client_no_auth.get(f"/api/v1/routes/{route.id}")
assert second.status_code == 200
assert second.headers.get("x-cache") == "HIT"
assert second.json() == first_body
class TestUpdateRoute:
def test_update_from_to(self, client_no_auth, api_db_session):
@@ -2,7 +2,10 @@
from sqlalchemy import select
from meshcore_hub.collector.handlers.raw_packet import store_raw_packet
from meshcore_hub.collector.handlers.raw_packet import (
store_raw_packet,
update_raw_packet_event_hash,
)
from meshcore_hub.common.models import Node, PacketPathHop, RawPacket
@@ -28,7 +31,7 @@ class TestStoreRawPacket:
"""A single packet writes exactly one raw_packets row."""
payload = {"raw": "0011223344", "hash": "deadbeef", "SNR": 9.5, "path": "0102"}
store_raw_packet(
rp_id = store_raw_packet(
"a" * 64, payload, _channel_decoded(), "channel_msg_recv", db_manager
)
@@ -39,6 +42,7 @@ class TestStoreRawPacket:
assert rp.packet_hash == "deadbeef"
assert rp.event_type == "channel_msg_recv"
assert rp.snr == 9.5
assert rp_id == rp.id
def test_derives_channel_idx_and_source_prefix(self, db_manager, db_session):
"""channel_idx from channelHash; source prefix from sourceHash."""
@@ -309,3 +313,69 @@ class TestStoreRawPacketPathHops:
)
assert len(hops) == 2
assert [h.node_hash for h in hops] == ["AABB", "CCDD"]
class TestUpdateRawPacketEventHash:
"""Tests for the post-dispatch event_hash backfill."""
def test_backfills_raw_packet_and_hops(self, db_manager, db_session):
"""update_raw_packet_event_hash writes event_hash onto both rows."""
decoded = {
"payloadType": 1,
"path": ["aa", "bb"],
"payload": {"decoded": {}},
}
rp_id = store_raw_packet(
"a" * 64, {"raw": "00", "hash": "wire1"}, decoded, "flood", db_manager
)
assert rp_id is not None
update_raw_packet_event_hash(rp_id, "evt-aaa", db_manager)
rp = db_session.execute(select(RawPacket)).scalar_one()
db_session.refresh(rp)
assert rp.event_hash == "evt-aaa"
hops = db_session.execute(select(PacketPathHop)).scalars().all()
assert len(hops) == 2
for hop in hops:
db_session.refresh(hop)
assert hop.event_hash == "evt-aaa"
def test_backfill_idempotent_on_replay(self, db_manager, db_session):
"""Calling backfill twice with the same hash is a no-op."""
decoded = {"payloadType": 1, "path": ["aa"], "payload": {"decoded": {}}}
rp_id = store_raw_packet(
"a" * 64, {"raw": "00", "hash": "wire1"}, decoded, "flood", db_manager
)
assert rp_id is not None
update_raw_packet_event_hash(rp_id, "evt-aaa", db_manager)
update_raw_packet_event_hash(rp_id, "evt-aaa", db_manager)
rp = db_session.execute(select(RawPacket)).scalar_one()
db_session.refresh(rp)
assert rp.event_hash == "evt-aaa"
def test_backfill_overwrites_prior_value(self, db_manager, db_session):
"""A second backfill with a different hash replaces the prior one."""
decoded = {"payloadType": 1, "path": ["aa"], "payload": {"decoded": {}}}
rp_id = store_raw_packet(
"a" * 64, {"raw": "00", "hash": "wire1"}, decoded, "flood", db_manager
)
assert rp_id is not None
update_raw_packet_event_hash(rp_id, "evt-first", db_manager)
update_raw_packet_event_hash(rp_id, "evt-second", db_manager)
rp = db_session.execute(select(RawPacket)).scalar_one()
db_session.refresh(rp)
assert rp.event_hash == "evt-second"
def test_backfill_no_op_when_row_missing(self, db_manager, db_session):
"""Backfilling a non-existent id silently does nothing (retention
cleanup may have removed the row between capture and dispatch)."""
# No row created; just verify it doesn't raise.
update_raw_packet_event_hash("nonexistent-id", "evt-aaa", db_manager)
assert db_session.execute(select(RawPacket)).scalars().all() == []
+167
View File
@@ -51,6 +51,7 @@ def _make_reception(
packet_hash: str,
path_hashes: list[str],
received_at: datetime | None = None,
event_hash: str | None = None,
) -> str:
"""Insert a RawPacket + PacketPathHop rows for a test reception."""
ts = received_at or _NOW
@@ -59,6 +60,7 @@ def _make_reception(
id=rp_id,
observer_node_id=observer_node_id,
packet_hash=packet_hash,
event_hash=event_hash,
received_at=ts,
)
db_session.add(rp)
@@ -70,6 +72,7 @@ def _make_reception(
position=pos,
node_hash=nh,
packet_hash=packet_hash,
event_hash=event_hash,
received_at=ts,
observer_node_id=observer_node_id,
)
@@ -326,6 +329,92 @@ class TestEvaluateRoute:
assert count == 1
assert state == RouteState.HEALTHY.value
def test_retransmissions_dedup_by_event_hash(self, db_session):
"""Three retransmissions of the same underlying event count once.
Each reception has a distinct wire ``packet_hash`` (the LetsMesh
``payload["hash"]`` is unique per transmission) but shares the same
underlying ``event_hash``. Without the event_hash dedup a single
advert broadcast three times would satisfy ``threshold=3`` within
seconds and bias the route's health.
"""
node_a = _make_node(db_session, "aa" + "0" * 62)
node_b = _make_node(db_session, "bb" + "0" * 62)
route = _make_route(db_session, "R1", [node_a, node_b], threshold=3)
_make_reception(
db_session, None, "wire1", ["AA", "BB"], event_hash="evt-shared"
)
_make_reception(
db_session, None, "wire2", ["AA", "BB"], event_hash="evt-shared"
)
_make_reception(
db_session, None, "wire3", ["AA", "BB"], event_hash="evt-shared"
)
db_session.commit()
since = _NOW - timedelta(hours=24)
state, quality, count = evaluate_route(db_session, route, since)
assert count == 1
assert state == RouteState.UNHEALTHY.value
def test_distinct_event_hashes_count_separately(self, db_session):
"""Two underlying events (distinct event_hash) count as two matches."""
node_a = _make_node(db_session, "aa" + "0" * 62)
node_b = _make_node(db_session, "bb" + "0" * 62)
route = _make_route(db_session, "R1", [node_a, node_b], threshold=2)
_make_reception(db_session, None, "wire1", ["AA", "BB"], event_hash="evt-aaa")
_make_reception(db_session, None, "wire2", ["AA", "BB"], event_hash="evt-bbb")
db_session.commit()
since = _NOW - timedelta(hours=24)
state, _, count = evaluate_route(db_session, route, since)
assert count == 2
assert state == RouteState.HEALTHY.value
def test_null_event_hash_falls_back_to_packet_hash(self, db_session):
"""Legacy rows with NULL event_hash dedup by wire packet_hash only."""
node_a = _make_node(db_session, "aa" + "0" * 62)
node_b = _make_node(db_session, "bb" + "0" * 62)
route = _make_route(db_session, "R1", [node_a, node_b], threshold=3)
# Three distinct wire hashes, no event_hash (legacy/NULL).
_make_reception(db_session, None, "wire1", ["AA", "BB"])
_make_reception(db_session, None, "wire2", ["AA", "BB"])
_make_reception(db_session, None, "wire3", ["AA", "BB"])
db_session.commit()
since = _NOW - timedelta(hours=24)
state, _, count = evaluate_route(db_session, route, since)
assert count == 3
assert state == RouteState.HEALTHY.value
def test_mixed_event_and_wire_identities(self, db_session):
"""Mix of NULL and shared event_hash: each NULL is unique, shared
event_hash collapses to one."""
node_a = _make_node(db_session, "aa" + "0" * 62)
node_b = _make_node(db_session, "bb" + "0" * 62)
route = _make_route(db_session, "R1", [node_a, node_b], threshold=3)
# Two legacy rows (distinct wire hashes, NULL event_hash) → 2 matches.
_make_reception(db_session, None, "wire1", ["AA", "BB"])
_make_reception(db_session, None, "wire2", ["AA", "BB"])
# Three retransmissions of one event → 1 match.
for i in range(3):
_make_reception(
db_session,
None,
f"wire-shared-{i}",
["AA", "BB"],
event_hash="evt-shared",
)
db_session.commit()
since = _NOW - timedelta(hours=24)
_, _, count = evaluate_route(db_session, route, since)
assert count == 3 # 2 legacy + 1 collapsed event
def test_observer_scope_filter(self, db_session):
"""Only in-scope observers are considered."""
node_a = _make_node(db_session, "aa" + "0" * 62)
@@ -497,6 +586,84 @@ class TestRecentMatches:
hops = matches[0]["hops"]
assert [h["node_hash"] for h in hops] == ["BB", "YY", "AA"]
def test_dedup_by_event_hash_keeps_newest(self, db_session):
"""Multiple retransmissions of one event return one row, newest first.
Without this dedup, the recent-matches list showed three rows with
near-identical timestamps whenever a single advert traversed the
route multiple times the symptom that surfaced this bug.
"""
node_a = _make_node(db_session, "aa" + "0" * 62)
node_b = _make_node(db_session, "bb" + "0" * 62)
route = _make_route(db_session, "R1", [node_a, node_b])
# Three retransmissions of one event with monotonic timestamps.
base = _NOW - timedelta(hours=5)
for i in range(3):
_make_reception(
db_session,
None,
f"wire{i}",
["AA", "BB"],
received_at=base + timedelta(minutes=i),
event_hash="evt-shared",
)
db_session.commit()
matches = recent_matches(db_session, route, limit=3, now=_NOW)
assert len(matches) == 1
# The newest reception of the shared event is the one returned.
assert matches[0]["packet_hash"] == "wire2"
assert matches[0]["event_hash"] == "evt-shared"
def test_distinct_events_returned_in_newest_first_order(self, db_session):
"""Two distinct events surface as two rows, newest first."""
node_a = _make_node(db_session, "aa" + "0" * 62)
node_b = _make_node(db_session, "bb" + "0" * 62)
route = _make_route(db_session, "R1", [node_a, node_b])
_make_reception(
db_session,
None,
"wire-older",
["AA", "BB"],
received_at=_NOW - timedelta(hours=2),
event_hash="evt-older",
)
_make_reception(
db_session,
None,
"wire-newer",
["AA", "BB"],
received_at=_NOW - timedelta(hours=1),
event_hash="evt-newer",
)
db_session.commit()
matches = recent_matches(db_session, route, limit=3, now=_NOW)
assert len(matches) == 2
assert matches[0]["event_hash"] == "evt-newer"
assert matches[1]["event_hash"] == "evt-older"
def test_legacy_null_event_hash_one_row_per_wire_hash(self, db_session):
"""NULL event_hash rows preserve today's per-wire-hash behaviour."""
node_a = _make_node(db_session, "aa" + "0" * 62)
node_b = _make_node(db_session, "bb" + "0" * 62)
route = _make_route(db_session, "R1", [node_a, node_b])
for i in range(3):
_make_reception(
db_session,
None,
f"wire{i}",
["AA", "BB"],
received_at=_NOW - timedelta(hours=i),
)
db_session.commit()
matches = recent_matches(db_session, route, limit=3, now=_NOW)
assert len(matches) == 3
class TestPreviewRoute:
def test_normal_preview(self, db_session):
+130
View File
@@ -154,6 +154,136 @@ class TestSubscriber:
finally:
session.close()
def test_event_hash_backfilled_after_dispatch(self, mock_mqtt_client, db_manager):
"""When the structured handler returns an event_hash, the subscriber
propagates it onto the captured raw_packet (and its hops)."""
from sqlalchemy import select
from meshcore_hub.common.models import PacketPathHop, RawPacket
mock_mqtt_client.topic_builder.parse_letsmesh_upload_topic.return_value = (
"a" * 64,
"packets",
)
subscriber = Subscriber(
mock_mqtt_client, db_manager, raw_packet_capture_enabled=True
)
subscriber._normalize_letsmesh_event = MagicMock( # type: ignore[method-assign]
return_value=("a" * 64, "channel_msg_recv", {"text": "hi"})
)
subscriber._letsmesh_decoder.decode_payload = MagicMock( # type: ignore[method-assign]
return_value={
"payloadType": 4,
"path": ["aa", "bb"],
"payload": {"decoded": {}},
}
)
# Structured handler resolves the underlying event and returns its hash.
subscriber.register_handler(
"channel_msg_recv", lambda pk, et, payload, db: "evt-resolved"
)
subscriber._handle_mqtt_message(
topic="meshcore/STN/abc/packets",
pattern="meshcore/+/+/packets",
payload={"raw": "0011", "hash": "wire1"},
)
session = db_manager.get_session()
try:
rp = session.execute(select(RawPacket)).scalar_one()
session.refresh(rp)
assert rp.event_hash == "evt-resolved"
hops = session.execute(select(PacketPathHop)).scalars().all()
assert len(hops) == 2
for hop in hops:
session.refresh(hop)
assert hop.event_hash == "evt-resolved"
finally:
session.close()
def test_event_hash_skipped_when_handler_returns_none(
self, mock_mqtt_client, db_manager
):
"""A handler that returns None (or no handler) leaves event_hash NULL."""
from sqlalchemy import select
from meshcore_hub.common.models import RawPacket
mock_mqtt_client.topic_builder.parse_letsmesh_upload_topic.return_value = (
"a" * 64,
"packets",
)
subscriber = Subscriber(
mock_mqtt_client, db_manager, raw_packet_capture_enabled=True
)
subscriber._normalize_letsmesh_event = MagicMock( # type: ignore[method-assign]
return_value=("a" * 64, "channel_msg_recv", {"text": "hi"})
)
subscriber._letsmesh_decoder.decode_payload = MagicMock( # type: ignore[method-assign]
return_value={"payloadType": 4, "payload": {"decoded": {}}}
)
# Handler returns None (event unresolvable / unclassified).
subscriber.register_handler(
"channel_msg_recv", lambda pk, et, payload, db: None
)
subscriber._handle_mqtt_message(
topic="meshcore/STN/abc/packets",
pattern="meshcore/+/+/packets",
payload={"raw": "0011", "hash": "wire1"},
)
session = db_manager.get_session()
try:
rp = session.execute(select(RawPacket)).scalar_one()
session.refresh(rp)
assert rp.event_hash is None
finally:
session.close()
def test_event_hash_skipped_when_handler_raises(self, mock_mqtt_client, db_manager):
"""A throwing handler leaves event_hash NULL but the raw_packet is
still captured (capture happens before dispatch)."""
from sqlalchemy import select
from meshcore_hub.common.models import RawPacket
mock_mqtt_client.topic_builder.parse_letsmesh_upload_topic.return_value = (
"a" * 64,
"packets",
)
subscriber = Subscriber(
mock_mqtt_client, db_manager, raw_packet_capture_enabled=True
)
subscriber._normalize_letsmesh_event = MagicMock( # type: ignore[method-assign]
return_value=("a" * 64, "channel_msg_recv", {"text": "hi"})
)
subscriber._letsmesh_decoder.decode_payload = MagicMock( # type: ignore[method-assign]
return_value={"payloadType": 4, "payload": {"decoded": {}}}
)
def _boom(pk, et, payload, db):
raise RuntimeError("handler failed")
subscriber.register_handler("channel_msg_recv", _boom)
subscriber._handle_mqtt_message(
topic="meshcore/STN/abc/packets",
pattern="meshcore/+/+/packets",
payload={"raw": "0011", "hash": "wire1"},
)
session = db_manager.get_session()
try:
rp = session.execute(select(RawPacket)).scalar_one()
session.refresh(rp)
assert rp.event_hash is None
assert rp.raw_hex == "0011" # capture survived the handler failure
finally:
session.close()
def test_raw_capture_skips_status_feed(self, mock_mqtt_client, db_manager):
"""Capture is packets-feed only; status feed writes no raw rows."""
from sqlalchemy import select