diff --git a/src/meshcore_hub/api/routes/packet_groups.py b/src/meshcore_hub/api/routes/packet_groups.py index 086327a..5d90ce2 100644 --- a/src/meshcore_hub/api/routes/packet_groups.py +++ b/src/meshcore_hub/api/routes/packet_groups.py @@ -4,7 +4,7 @@ from datetime import datetime, timedelta, timezone from typing import Any, Optional from fastapi import APIRouter, HTTPException, Query, Request -from sqlalchemy import func, or_, select +from sqlalchemy import asc, desc, func, or_, select from sqlalchemy.orm import aliased, selectinload from meshcore_hub.api.auth import RequireRead @@ -58,12 +58,16 @@ def list_packet_groups( _: RequireRead, session: DbSession, request: Request, - search: Optional[str] = Query(None, description="Search in packet hash or observer name"), + search: Optional[str] = Query( + None, description="Search in packet hash or observer name" + ), event_type: Optional[str] = Query(None, description="Filter by event type"), channel_idx: Optional[int] = Query(None, description="Filter by channel index"), since: Optional[datetime] = Query(None, description="Start timestamp"), until: Optional[datetime] = Query(None, description="End timestamp"), - sort: Optional[str] = Query(None, description="Sort column: time, event_type, reception_count"), + sort: Optional[str] = Query( + None, description="Sort column: time, event_type, reception_count" + ), order: Optional[str] = Query(None, description="asc or desc"), limit: int = Query(50, ge=1, le=100), offset: int = Query(0, ge=0), @@ -83,9 +87,13 @@ def list_packet_groups( # Default time window — keeps GROUP BY bounded when no explicit since/until. if since is None: if search: - since = datetime.now(timezone.utc) - timedelta(days=SEARCH_DEFAULT_WINDOW_DAYS) + since = datetime.now(timezone.utc) - timedelta( + days=SEARCH_DEFAULT_WINDOW_DAYS + ) else: - since = datetime.now(timezone.utc) - timedelta(days=SEARCH_DEFAULT_WINDOW_DAYS) + since = datetime.now(timezone.utc) - timedelta( + days=SEARCH_DEFAULT_WINDOW_DAYS + ) ObserverNode = aliased(Node) @@ -125,13 +133,14 @@ def list_packet_groups( count_query = select(func.count()).select_from(group_query.subquery()) total = session.execute(count_query).scalar() or 0 - sort_col = { + sort_exprs: dict[str, Any] = { "time": func.min(RawPacket.received_at), "event_type": func.min(RawPacket.event_type), "reception_count": func.count(RawPacket.id), - }[sort] + } + sort_col = sort_exprs[sort] group_query = group_query.order_by( - sort_col.desc() if order == "desc" else sort_col.asc() + desc(sort_col) if order == "desc" else asc(sort_col) ) group_query = group_query.offset(offset).limit(limit) @@ -142,17 +151,21 @@ def list_packet_groups( return GroupedPacketList(items=[], total=total, limit=limit, offset=offset) # ── Phase 2: Lightweight metadata fetch — no raw_hex, no decoded ────────── - meta_query = select( - RawPacket.id, - RawPacket.packet_hash, - RawPacket.event_type, - RawPacket.channel_idx, - RawPacket.packet_type, - RawPacket.payload_type, - RawPacket.route_type, - RawPacket.source_pubkey_prefix, - RawPacket.received_at, - ).where(RawPacket.packet_hash.in_(hashes)).order_by(RawPacket.received_at.asc()) + meta_query = ( + select( + RawPacket.id, + RawPacket.packet_hash, + RawPacket.event_type, + RawPacket.channel_idx, + RawPacket.packet_type, + RawPacket.payload_type, + RawPacket.route_type, + RawPacket.source_pubkey_prefix, + RawPacket.received_at, + ) + .where(RawPacket.packet_hash.in_(hashes)) + .order_by(RawPacket.received_at.asc()) + ) meta_rows = session.execute(meta_query).all() @@ -244,8 +257,7 @@ def get_packet_group( for row in rows: packet = row[0] is_redacted = ( - packet.channel_idx is not None - and packet.channel_idx not in visible_indices + packet.channel_idx is not None and packet.channel_idx not in visible_indices ) observer_node = nodes_by_id.get(row.observer_id) if row.observer_id else None receptions.append( @@ -256,7 +268,9 @@ def get_packet_group( observer_tag_name=_get_tag_name(observer_node), snr=packet.snr, path_len=packet.path_len, - path_hashes=None if is_redacted else _extract_path_hashes(packet.decoded), + path_hashes=( + None if is_redacted else _extract_path_hashes(packet.decoded) + ), received_at=packet.received_at, redacted=is_redacted, ) @@ -281,7 +295,9 @@ def get_packet_group( packet_type=first_packet.packet_type, payload_type=first_packet.payload_type, route_type=first_packet.route_type, - source_pubkey_prefix=None if all_redacted else first_packet.source_pubkey_prefix, + source_pubkey_prefix=( + None if all_redacted else first_packet.source_pubkey_prefix + ), reception_count=len(receptions), observer_count=unique_observers, receptions=receptions, diff --git a/src/meshcore_hub/common/schemas/raw_packets.py b/src/meshcore_hub/common/schemas/raw_packets.py index e682fd1..2ddc312 100644 --- a/src/meshcore_hub/common/schemas/raw_packets.py +++ b/src/meshcore_hub/common/schemas/raw_packets.py @@ -73,11 +73,14 @@ class PacketReceptionInfo(BaseModel): packet_id: str = Field(..., description="Raw packet UUID") observed_by: Optional[str] = Field(default=None, description="Observer public key") observer_name: Optional[str] = Field(default=None, description="Observer node name") - observer_tag_name: Optional[str] = Field(default=None, description="Observer name from tags") + observer_tag_name: Optional[str] = Field( + default=None, description="Observer name from tags" + ) snr: Optional[float] = Field(default=None, description="SNR at this observer") path_len: Optional[int] = Field(default=None, description="Hop count") path_hashes: Optional[list[str]] = Field( - default=None, description="Hop node hash sequence from decoded.payload.decoded.pathHashes" + default=None, + description="Hop node hash sequence from decoded.payload.decoded.pathHashes", ) received_at: datetime = Field(..., description="When received") redacted: bool = Field(default=False) @@ -99,10 +102,18 @@ class GroupedPacketRead(BaseModel): default_factory=list, description="Individual receptions (populated for detail, empty for list)", ) - first_seen: datetime = Field(..., description="Earliest received_at across all receptions") - redacted: bool = Field(default=False, description="True when all receptions are redacted") - raw_hex: Optional[str] = Field(default=None, description="Representative raw hex (null for list view)") - decoded: Optional[dict[str, Any]] = Field(default=None, description="Representative decoded JSON (null for list view)") + first_seen: datetime = Field( + ..., description="Earliest received_at across all receptions" + ) + redacted: bool = Field( + default=False, description="True when all receptions are redacted" + ) + raw_hex: Optional[str] = Field( + default=None, description="Representative raw hex (null for list view)" + ) + decoded: Optional[dict[str, Any]] = Field( + default=None, description="Representative decoded JSON (null for list view)" + ) class GroupedPacketList(BaseModel): diff --git a/tests/test_api/test_packet_groups.py b/tests/test_api/test_packet_groups.py new file mode 100644 index 0000000..85cbcba --- /dev/null +++ b/tests/test_api/test_packet_groups.py @@ -0,0 +1,546 @@ +"""Tests for GET /api/v1/packet-groups endpoints.""" + +from datetime import datetime, timezone + +import pytest + +from meshcore_hub.common.models import Channel, Node, NodeTag, RawPacket + + +def _now() -> datetime: + return datetime.now(timezone.utc) + + +class TestListPacketGroups: + """Tests for GET /packet-groups (grouped list).""" + + def test_empty_db(self, client_no_auth): + response = client_no_auth.get("/api/v1/packet-groups") + assert response.status_code == 200 + data = response.json() + assert data["items"] == [] + assert data["total"] == 0 + + def test_null_hash_rows_excluded(self, client_no_auth, api_db_session): + api_db_session.add( + RawPacket(raw_hex="AA", packet_hash=None, received_at=_now()) + ) + api_db_session.commit() + + data = client_no_auth.get("/api/v1/packet-groups").json() + assert data["total"] == 0 + + def test_single_group(self, client_no_auth, api_db_session): + now = _now() + api_db_session.add_all( + [ + RawPacket( + raw_hex="AA", + packet_hash="H1", + event_type="advertisement", + received_at=now, + ), + RawPacket( + raw_hex="BB", + packet_hash="H1", + event_type="advertisement", + received_at=now, + ), + ] + ) + api_db_session.commit() + + data = client_no_auth.get("/api/v1/packet-groups").json() + assert data["total"] == 1 + item = data["items"][0] + assert item["packet_hash"] == "H1" + assert item["reception_count"] == 2 + assert item["event_type"] == "advertisement" + + def test_multiple_groups(self, client_no_auth, api_db_session): + now = _now() + api_db_session.add_all( + [ + RawPacket(raw_hex="AA", packet_hash="H1", received_at=now), + RawPacket(raw_hex="BB", packet_hash="H1", received_at=now), + RawPacket(raw_hex="CC", packet_hash="H2", received_at=now), + ] + ) + api_db_session.commit() + + data = client_no_auth.get("/api/v1/packet-groups").json() + assert data["total"] == 2 + hashes = {i["packet_hash"] for i in data["items"]} + assert hashes == {"H1", "H2"} + + def test_observer_count(self, client_no_auth, api_db_session): + now = _now() + obs1 = Node(public_key="a" * 64) + obs2 = Node(public_key="b" * 64) + api_db_session.add_all([obs1, obs2]) + api_db_session.flush() + api_db_session.add_all( + [ + RawPacket( + raw_hex="AA", + packet_hash="H1", + observer_node_id=obs1.id, + received_at=now, + ), + RawPacket( + raw_hex="BB", + packet_hash="H1", + observer_node_id=obs2.id, + received_at=now, + ), + RawPacket( + raw_hex="CC", + packet_hash="H1", + observer_node_id=obs1.id, + received_at=now, + ), + ] + ) + api_db_session.commit() + + data = client_no_auth.get("/api/v1/packet-groups").json() + item = data["items"][0] + assert item["reception_count"] == 3 + assert item["observer_count"] == 2 + + def test_filter_event_type(self, client_no_auth, api_db_session): + now = _now() + api_db_session.add_all( + [ + RawPacket( + raw_hex="AA", packet_hash="H1", event_type="advert", received_at=now + ), + RawPacket( + raw_hex="BB", packet_hash="H2", event_type="path", received_at=now + ), + ] + ) + api_db_session.commit() + + data = client_no_auth.get("/api/v1/packet-groups?event_type=advert").json() + assert data["total"] == 1 + assert data["items"][0]["event_type"] == "advert" + + def test_filter_channel_idx(self, client_no_auth, api_db_session): + now = _now() + api_db_session.add_all( + [ + RawPacket( + raw_hex="AA", packet_hash="H1", channel_idx=7, received_at=now + ), + RawPacket( + raw_hex="BB", packet_hash="H2", channel_idx=9, received_at=now + ), + ] + ) + api_db_session.commit() + + data = client_no_auth.get("/api/v1/packet-groups?channel_idx=7").json() + assert data["total"] == 1 + assert data["items"][0]["channel_idx"] == 7 + + def test_filter_search_by_hash(self, client_no_auth, api_db_session): + now = _now() + api_db_session.add_all( + [ + RawPacket(raw_hex="AA", packet_hash="FINDME", received_at=now), + RawPacket(raw_hex="BB", packet_hash="OTHER", received_at=now), + ] + ) + api_db_session.commit() + + data = client_no_auth.get("/api/v1/packet-groups?search=FINDME").json() + assert data["total"] == 1 + assert data["items"][0]["packet_hash"] == "FINDME" + + def test_since_filter(self, client_no_auth, api_db_session): + old = datetime(2020, 1, 1, tzinfo=timezone.utc) + recent = _now() + api_db_session.add_all( + [ + RawPacket(raw_hex="AA", packet_hash="OLD", received_at=old), + RawPacket(raw_hex="BB", packet_hash="NEW", received_at=recent), + ] + ) + api_db_session.commit() + + data = client_no_auth.get( + "/api/v1/packet-groups", params={"since": "2021-01-01T00:00:00+00:00"} + ).json() + assert data["total"] == 1 + assert data["items"][0]["packet_hash"] == "NEW" + + def test_until_filter(self, client_no_auth, api_db_session): + old = datetime(2020, 1, 1, tzinfo=timezone.utc) + recent = _now() + api_db_session.add_all( + [ + RawPacket(raw_hex="AA", packet_hash="OLD", received_at=old), + RawPacket(raw_hex="BB", packet_hash="NEW", received_at=recent), + ] + ) + api_db_session.commit() + + # Pass explicit since to bypass the 7-day default window + data = client_no_auth.get( + "/api/v1/packet-groups", + params={ + "since": "2019-01-01T00:00:00+00:00", + "until": "2021-01-01T00:00:00+00:00", + }, + ).json() + assert data["total"] == 1 + assert data["items"][0]["packet_hash"] == "OLD" + + def test_sort_by_reception_count(self, client_no_auth, api_db_session): + now = _now() + api_db_session.add_all( + [ + RawPacket(raw_hex="A1", packet_hash="FEW", received_at=now), + RawPacket(raw_hex="B1", packet_hash="MANY", received_at=now), + RawPacket(raw_hex="B2", packet_hash="MANY", received_at=now), + RawPacket(raw_hex="B3", packet_hash="MANY", received_at=now), + ] + ) + api_db_session.commit() + + data = client_no_auth.get( + "/api/v1/packet-groups?sort=reception_count&order=desc" + ).json() + assert data["items"][0]["packet_hash"] == "MANY" + assert data["items"][1]["packet_hash"] == "FEW" + + def test_sort_by_event_type(self, client_no_auth, api_db_session): + now = _now() + api_db_session.add_all( + [ + RawPacket( + raw_hex="AA", + packet_hash="Z_HASH", + event_type="zzz", + received_at=now, + ), + RawPacket( + raw_hex="BB", + packet_hash="A_HASH", + event_type="aaa", + received_at=now, + ), + ] + ) + api_db_session.commit() + + data = client_no_auth.get( + "/api/v1/packet-groups?sort=event_type&order=asc" + ).json() + assert data["items"][0]["event_type"] == "aaa" + + def test_pagination_params_echoed(self, client_no_auth): + data = client_no_auth.get("/api/v1/packet-groups?limit=10&offset=5").json() + assert data["limit"] == 10 + assert data["offset"] == 5 + + def test_receptions_not_populated_in_list(self, client_no_auth, api_db_session): + api_db_session.add( + RawPacket(raw_hex="AA", packet_hash="H1", received_at=_now()) + ) + api_db_session.commit() + + data = client_no_auth.get("/api/v1/packet-groups").json() + assert data["items"][0]["receptions"] == [] + + def test_raw_hex_and_decoded_not_in_list(self, client_no_auth, api_db_session): + api_db_session.add( + RawPacket( + raw_hex="DEADBEEF", + packet_hash="H1", + decoded={"foo": "bar"}, + received_at=_now(), + ) + ) + api_db_session.commit() + + data = client_no_auth.get("/api/v1/packet-groups").json() + item = data["items"][0] + assert item["raw_hex"] is None + assert item["decoded"] is None + + def test_invalid_sort_defaults_to_time(self, client_no_auth, api_db_session): + api_db_session.add( + RawPacket(raw_hex="AA", packet_hash="H1", received_at=_now()) + ) + api_db_session.commit() + + data = client_no_auth.get( + "/api/v1/packet-groups?sort=bogus&order=invalid" + ).json() + assert data["total"] == 1 + + def test_key_builder_role_aware(self): + from starlette.requests import Request + + from meshcore_hub.api.routes.packet_groups import _group_key_builder + + req = Request( + { + "type": "http", + "method": "GET", + "headers": [], + "query_string": b"limit=10", + } + ) + key = _group_key_builder(req) + assert key.startswith("packet_groups:role=anonymous:") + assert "limit=10" in key + + +class TestGetPacketGroup: + """Tests for GET /packet-groups/{hash} (detail).""" + + def test_404_for_unknown_hash(self, client_no_auth): + response = client_no_auth.get("/api/v1/packet-groups/NOSUCHPACKET") + assert response.status_code == 404 + + def test_returns_all_receptions(self, client_no_auth, api_db_session): + now = _now() + api_db_session.add_all( + [ + RawPacket(raw_hex="AA", packet_hash="H1", received_at=now), + RawPacket(raw_hex="BB", packet_hash="H1", received_at=now), + RawPacket(raw_hex="CC", packet_hash="H1", received_at=now), + ] + ) + api_db_session.commit() + + data = client_no_auth.get("/api/v1/packet-groups/H1").json() + assert data["packet_hash"] == "H1" + assert data["reception_count"] == 3 + assert len(data["receptions"]) == 3 + + def test_observer_hydration(self, client_no_auth, api_db_session): + obs = Node(public_key="o" * 64, name="ObsName") + api_db_session.add(obs) + api_db_session.flush() + api_db_session.add(NodeTag(node_id=obs.id, key="name", value="TaggedObs")) + api_db_session.add( + RawPacket( + raw_hex="AA", + packet_hash="H1", + observer_node_id=obs.id, + received_at=_now(), + ) + ) + api_db_session.commit() + + data = client_no_auth.get("/api/v1/packet-groups/H1").json() + r = data["receptions"][0] + assert r["observed_by"] == "o" * 64 + assert r["observer_name"] == "ObsName" + assert r["observer_tag_name"] == "TaggedObs" + + def test_path_hashes_extracted(self, client_no_auth, api_db_session): + decoded = { + "payload": { + "decoded": { + "pathHashes": ["AA", "BB", "CC"], + } + } + } + api_db_session.add( + RawPacket( + raw_hex="AA", + packet_hash="H1", + decoded=decoded, + path_len=3, + received_at=_now(), + ) + ) + api_db_session.commit() + + data = client_no_auth.get("/api/v1/packet-groups/H1").json() + r = data["receptions"][0] + assert r["path_hashes"] == ["AA", "BB", "CC"] + assert r["path_len"] == 3 + + def test_path_hashes_missing_returns_none(self, client_no_auth, api_db_session): + api_db_session.add( + RawPacket( + raw_hex="AA", + packet_hash="H1", + decoded={"payload": {"decoded": {}}}, + received_at=_now(), + ) + ) + api_db_session.commit() + + data = client_no_auth.get("/api/v1/packet-groups/H1").json() + assert data["receptions"][0]["path_hashes"] is None + + def test_representative_raw_hex_and_decoded(self, client_no_auth, api_db_session): + now = _now() + api_db_session.add_all( + [ + RawPacket( + raw_hex="AABBCC", + packet_hash="H1", + decoded={"info": "first"}, + received_at=now, + ), + RawPacket( + raw_hex="DDEEFF", + packet_hash="H1", + decoded={"info": "second"}, + received_at=now, + ), + ] + ) + api_db_session.commit() + + data = client_no_auth.get("/api/v1/packet-groups/H1").json() + assert data["raw_hex"] is not None + assert data["decoded"] is not None + + def test_observer_count_distinct(self, client_no_auth, api_db_session): + obs = Node(public_key="x" * 64) + api_db_session.add(obs) + api_db_session.flush() + now = _now() + # Same observer, two different paths + api_db_session.add_all( + [ + RawPacket( + raw_hex="AA", + packet_hash="H1", + observer_node_id=obs.id, + path_len=2, + received_at=now, + ), + RawPacket( + raw_hex="BB", + packet_hash="H1", + observer_node_id=obs.id, + path_len=3, + received_at=now, + ), + ] + ) + api_db_session.commit() + + data = client_no_auth.get("/api/v1/packet-groups/H1").json() + assert data["reception_count"] == 2 + assert data["observer_count"] == 1 + + +class TestPacketGroupRedaction: + """Tests for channel-visibility redaction in packet groups.""" + + @pytest.fixture + def channel_packets(self, api_db_session): + pub_key = "AABBCCDDEEFF00112233445566778899" + adm_key = "FFEEDDCCBBAA99887766554433221100" + pub_idx = int(Channel.compute_channel_hash(pub_key), 16) + adm_idx = int(Channel.compute_channel_hash(adm_key), 16) + + api_db_session.add_all( + [ + Channel( + name="Pub", + key_hex=pub_key, + channel_hash=Channel.compute_channel_hash(pub_key), + visibility="community", + enabled=True, + ), + Channel( + name="Adm", + key_hex=adm_key, + channel_hash=Channel.compute_channel_hash(adm_key), + visibility="admin", + enabled=True, + ), + ] + ) + now = _now() + api_db_session.add_all( + [ + RawPacket( + raw_hex="PUBLIC", + packet_hash="PUB_HASH", + channel_idx=pub_idx, + source_pubkey_prefix="AABBCC", + received_at=now, + ), + RawPacket( + raw_hex="SECRET", + packet_hash="ADM_HASH", + channel_idx=adm_idx, + source_pubkey_prefix="FFEEDD", + received_at=now, + ), + ] + ) + api_db_session.commit() + return pub_idx, adm_idx + + def test_list_redacts_admin_channel(self, client_no_auth, channel_packets): + data = client_no_auth.get("/api/v1/packet-groups").json() + assert data["total"] == 2 + adm = next(i for i in data["items"] if i["packet_hash"] == "ADM_HASH") + assert adm["redacted"] is True + assert adm["source_pubkey_prefix"] is None + + def test_list_admin_role_sees_all(self, client_no_auth, channel_packets): + data = client_no_auth.get( + "/api/v1/packet-groups", headers={"X-User-Roles": "admin"} + ).json() + assert all(not i["redacted"] for i in data["items"]) + + def test_detail_redacted_reception(self, client_no_auth, channel_packets): + data = client_no_auth.get("/api/v1/packet-groups/ADM_HASH").json() + assert data["redacted"] is True + r = data["receptions"][0] + assert r["redacted"] is True + assert r["path_hashes"] is None + assert data["raw_hex"] is None + + def test_detail_admin_sees_payload(self, client_no_auth, channel_packets): + data = client_no_auth.get( + "/api/v1/packet-groups/ADM_HASH", headers={"X-User-Roles": "admin"} + ).json() + assert data["redacted"] is False + assert data["raw_hex"] == "SECRET" + + +class TestExtractPathHashes: + """Unit tests for the _extract_path_hashes helper.""" + + def test_extracts_valid_hashes(self): + from meshcore_hub.api.routes.packet_groups import _extract_path_hashes + + decoded = {"payload": {"decoded": {"pathHashes": ["AA", "BB"]}}} + assert _extract_path_hashes(decoded) == ["AA", "BB"] + + def test_none_input(self): + from meshcore_hub.api.routes.packet_groups import _extract_path_hashes + + assert _extract_path_hashes(None) is None + + def test_missing_path_hashes(self): + from meshcore_hub.api.routes.packet_groups import _extract_path_hashes + + assert _extract_path_hashes({"payload": {"decoded": {}}}) is None + + def test_non_list_path_hashes(self): + from meshcore_hub.api.routes.packet_groups import _extract_path_hashes + + decoded = {"payload": {"decoded": {"pathHashes": "not-a-list"}}} + assert _extract_path_hashes(decoded) is None + + def test_empty_decoded(self): + from meshcore_hub.api.routes.packet_groups import _extract_path_hashes + + assert _extract_path_hashes({}) is None