Merge pull request #350 from ipnet-mesh/fix/dashboard-packet-retention-window

fix(dashboard): make packet displays retention-aware

docker-compose.yml

@@ -236,9 +236,9 @@ services:
       - NODE_CLEANUP_ENABLED=${NODE_CLEANUP_ENABLED:-true}
       - NODE_CLEANUP_DAYS=${NODE_CLEANUP_DAYS:-30}
       # Raw packet capture (derived from FEATURE_PACKETS so one var drives both
-      # capture and the web Packets page). Retention defaults to 7 days.
+      # capture and the web Packets page). Retention defaults to 2 days.
       - RAW_PACKET_CAPTURE_ENABLED=${FEATURE_PACKETS:-true}
-      - RAW_PACKET_RETENTION_DAYS=${RAW_PACKET_RETENTION_DAYS:-7}
+      - RAW_PACKET_RETENTION_DAYS=${RAW_PACKET_RETENTION_DAYS:-2}
       # Observer ingestion filter (allow/deny remote observers by public key/prefix).
       # Allowlist overrides denylist; both empty accepts all observers (default).
       - OBSERVER_ALLOWLIST=${OBSERVER_ALLOWLIST:-}

docs/configuration.md

@@ -92,7 +92,7 @@ The collector subscribes to MQTT events and persists them to the database. For p
 | --- | --- | --- |
 | `CHANNEL_REFRESH_INTERVAL_SECONDS` | `300` | Seconds between channel-key refresh from the database (minimum `10`) |
 | `ROUTE_EVALUATOR_INTERVAL_SECONDS` | `300` | Seconds between route health evaluations. `0` disables the background evaluator (route cards then stay `unknown`). See [routes.md](routes.md) |
-| `ROUTE_HISTORY_BACKFILL_INTERVAL_SECONDS` | `3600` | Seconds between route-health history backfill sweeps (recomputes completed-day buckets for the retention window). `0` disables the backfill; the dashboard strip and `/routes/{id}/history` then only reflect the live 60 s evaluator sweeps. See [routes.md](routes.md) |
+| `ROUTE_HISTORY_BACKFILL_INTERVAL_SECONDS` | `3600` | Seconds between route-health history backfill sweeps (recomputes completed-day buckets for the retention window). `0` disables the backfill; the dashboard strip and `/routes/{id}/history` then only reflect the live 300 s evaluator sweeps. See [routes.md](routes.md) |
 
 ### Observer Ingestion Filters
 

docs/seeding.md

@@ -126,7 +126,7 @@ routes:
     description: A140 corridor route
     visibility: community
     match_width: 1
-    window_hours: 48
+    window_hours: 6
     packet_count_threshold: 5
     # clear_threshold: 15      # optional; omit/null = 3x threshold
     # max_hop_span: 8           # optional; omit/null = unlimited

example/seed/routes.yaml

@@ -17,7 +17,7 @@ routes:
     description: A140 corridor route
     visibility: community
     match_width: 1
-    window_hours: 24
+    window_hours: 6
     packet_count_threshold: 3
     # clear_threshold: 10        # optional; omit/null = 2x threshold
     # max_hop_span: 8             # optional; omit/null = unlimited

src/meshcore_hub/api/routes/dashboard.py

@@ -185,12 +185,19 @@ def get_stats(
     advertisements_24h = adv_row[1] or 0
     advertisements_7d = adv_row[2] or 0
 
-    # Raw-packet counts (total + last 7 days), observer-level volume metric
+    # Raw-packet counts (total + recent window), observer-level volume metric
     # with no role/channel filter (payload redaction lives on list/detail only).
+    # The "recent" window is capped at min(7, retention) because raw packets
+    # (and their cascaded packet_path_hops) are purged at RAW_PACKET_RETENTION_DAYS;
+    # a wider window would silently undercount against the "Last 7 days" label.
+    packets_window_days = min(
+        7, get_collector_settings().effective_raw_packet_retention_days
+    )
+    packets_since = now - timedelta(days=packets_window_days)
     packet_row = session.execute(
         select(
             func.count(RawPacket.id).label("total_packets"),
-            func.sum(case((RawPacket.received_at >= seven_days_ago, 1), else_=0)).label(
+            func.sum(case((RawPacket.received_at >= packets_since, 1), else_=0)).label(
                 "packets_7d"
             ),
         ).select_from(RawPacket)
@@ -252,6 +259,7 @@ def get_stats(
         total_members=total_members,
         total_packets=total_packets,
         packets_7d=packets_7d,
+        packets_window_days=packets_window_days,
     )
 
 
@@ -470,7 +478,14 @@ def get_packet_activity(
     Returns:
         Daily raw-packet counts for each day in the period (excluding today)
     """
-    days = min(days, 90)
+    # Clamp to the raw-packet retention window so the response never advertises
+    # days whose data has been purged (the route evaluator / dashboard stat uses
+    # the same cap). See get_dashboard_stats for the rationale.
+    days = min(
+        days,
+        90,
+        get_collector_settings().effective_raw_packet_retention_days,
+    )
 
     now = datetime.now(timezone.utc)
     end_date = now.replace(hour=0, minute=0, second=0, microsecond=0)
@@ -524,7 +539,12 @@ def get_packet_breakdown(
         Counts bucketed by event type (top 6 + "other") and by path-hash
         byte width (1b/2b/3b, NULL excluded) for the period (excluding today).
     """
-    days = min(days, 90)
+    # Clamp to the raw-packet retention window (see get_dashboard_stats).
+    days = min(
+        days,
+        90,
+        get_collector_settings().effective_raw_packet_retention_days,
+    )
 
     now = datetime.now(timezone.utc)
     end_date = now.replace(hour=0, minute=0, second=0, microsecond=0)

src/meshcore_hub/api/routes/routes.py

@@ -426,7 +426,7 @@ def _load_recent_matches(
     """Return the route's top-3 recent matches in the ``RecentMatchPath`` shape.
 
     Reads the normalized ``route_recent_matches`` table (populated by the
-    background evaluator on every 60s tick), JOINs through ``raw_packets``
+    background evaluator on every 300s tick), JOINs through ``raw_packets``
     for the packet-level metadata, then fetches the matched hop slice from
     ``packet_path_hops`` in a second indexed query and slices
     ``[first_position .. last_position]`` per match in Python. Falls back

src/meshcore_hub/collector/subscriber.py

@@ -52,7 +52,7 @@ class Subscriber(LetsMeshNormalizer):
         node_cleanup_days: int = 90,
         channel_refresh_interval_seconds: int = 300,
         raw_packet_capture_enabled: bool = False,
-        raw_packet_retention_days: int = 7,
+        raw_packet_retention_days: int = 2,
         observer_filter: Optional[ObserverFilter] = None,
     ):
         """Initialize subscriber.
@@ -903,7 +903,7 @@ def create_subscriber(
     node_cleanup_days: int = 90,
     channel_refresh_interval_seconds: int = 300,
     raw_packet_capture_enabled: bool = False,
-    raw_packet_retention_days: int = 7,
+    raw_packet_retention_days: int = 2,
     observer_filter: Optional[ObserverFilter] = None,
 ) -> Subscriber:
     """Create a configured subscriber instance.
@@ -989,7 +989,7 @@ def run_collector(
     node_cleanup_days: int = 90,
     channel_refresh_interval_seconds: int = 300,
     raw_packet_capture_enabled: bool = False,
-    raw_packet_retention_days: int = 7,
+    raw_packet_retention_days: int = 2,
     observer_filter: Optional[ObserverFilter] = None,
 ) -> None:
     """Run the collector (blocking).

src/meshcore_hub/common/schemas/messages.py

@@ -296,6 +296,14 @@ class DashboardStats(BaseModel):
     )
     total_packets: int = Field(default=0, description="Total raw packets captured")
     packets_7d: int = Field(default=0, description="Packets captured in last 7 days")
+    packets_window_days: int = Field(
+        default=7,
+        description=(
+            "Actual window in days for packets_7d = min(7, "
+            "raw_packet_retention_days). The UI labels the stat with this "
+            "value so it never claims a 7-day window when retention is shorter."
+        ),
+    )
 
 
 class RecentActivity(BaseModel):

src/meshcore_hub/web/static/js/spa-react/pages/Dashboard.tsx

@@ -41,9 +41,11 @@ interface DashboardStats {
   advertisements_7d: number;
   messages_7d: number;
   packets_7d: number;
+  packets_window_days: number;
 }
 
 interface PacketBreakdown {
+  days: number;
   by_event_type: BreakdownBucket[];
   by_path_width: BreakdownBucket[];
 }
@@ -491,7 +493,9 @@ export function DashboardPage() {
                 colorVar="--color-packets"
                 icon={<IconPackets className="h-5 w-5" />}
                 title={t("entities.packets")}
-                subtitle={t("time.per_day_last_7_days")}
+                subtitle={t("time.per_day_last_n_days", {
+                  n: data.packetActivity?.days ?? 7,
+                })}
                 value={stats.packets_7d}
               >
                 <TrendLineChart
@@ -511,7 +515,9 @@ export function DashboardPage() {
                   colorVar="--color-packets"
                   icon={<IconPackets className="h-5 w-5" />}
                   title={t("entities.packet_event_types")}
-                  subtitle={t("time.last_7_days")}
+                  subtitle={t("time.last_n_days", {
+                    n: packetBreakdown?.days ?? 7,
+                  })}
                   value={eventTypeTotal}
                 >
                   <StackedBarChart
@@ -525,7 +531,9 @@ export function DashboardPage() {
                   colorVar="--color-packets"
                   icon={<IconPackets className="h-5 w-5" />}
                   title={t("entities.path_hash_width")}
-                  subtitle={t("time.last_7_days")}
+                  subtitle={t("time.last_n_days", {
+                    n: packetBreakdown?.days ?? 7,
+                  })}
                   value={pathWidthTotal}
                 >
                   <StackedBarChart

src/meshcore_hub/web/static/js/spa-react/pages/Home.tsx

@@ -41,6 +41,7 @@ interface DashboardStats {
   advertisements_7d: number;
   messages_7d: number;
   packets_7d: number;
+  packets_window_days: number;
   total_operators: number;
   total_members: number;
 }
@@ -371,7 +372,9 @@ export function HomePage() {
                 color={getPageColor("packets")}
                 title={t("entities.packets")}
                 value={stats.packets_7d}
-                description={t("time.last_7_days")}
+                description={t("time.last_n_days", {
+                  n: stats.packets_window_days,
+                })}
               />
             )}
           </div>

src/meshcore_hub/web/static/js/spa-react/utils/charts.ts

@@ -16,6 +16,9 @@ export interface ActivityPoint {
 }
 
 export interface ActivitySeries {
+  // `days` is present on all DailyActivity API responses but optional here so
+  // the type can be reused for activity charts that don't surface the window.
+  days?: number;
   data: ActivityPoint[];
 }
 

src/meshcore_hub/web/static/locales/en.json

@@ -137,7 +137,9 @@
     "minutes_ago": "{{count}}m ago",
     "less_than_minute": "<1m ago",
     "last_7_days": "Last 7 days",
+    "last_n_days": "Last {{n}} days",
     "per_day_last_7_days": "Per day (last 7 days)",
+    "per_day_last_n_days": "Per day (last {{n}} days)",
     "over_time_last_7_days": "Over time (last 7 days)",
     "activity_per_day_last_7_days": "Activity per day (last 7 days)",
     "routes_over_last_n_days": "Matched packets per route (last {{n}} days)"

src/meshcore_hub/web/static/locales/nl.json

@@ -128,7 +128,9 @@
     "minutes_ago": "{{count}}m geleden",
     "less_than_minute": "<1m geleden",
     "last_7_days": "Laatste 7 dagen",
+    "last_n_days": "Laatste {{n}} dagen",
     "per_day_last_7_days": "Per dag (laatste 7 dagen)",
+    "per_day_last_n_days": "Per dag (laatste {{n}} dagen)",
     "over_time_last_7_days": "In de tijd (laatste 7 dagen)",
     "activity_per_day_last_7_days": "Activiteit per dag (laatste 7 dagen)",
     "routes_over_last_n_days": "Overeenkomende pakketten per route (laatste {{n}} dagen)"

tests/test_api/test_dashboard.py

@@ -51,6 +51,21 @@ class TestDateBucketKey:
 class TestDashboardStats:
     """Tests for GET /dashboard/stats endpoint."""
 
+    @pytest.fixture(autouse=True)
+    def _high_raw_packet_retention(self, monkeypatch):
+        """Bump raw-packet retention to 90d for this class.
+
+        The packet stat window is min(7, retention); at the default retention
+        of 2d it would clamp to 2d and break the days-handling assertions
+        below. The clamp itself is covered by
+        ``test_packets_window_days_tracks_retention``.
+        """
+        from meshcore_hub.common.config import CollectorSettings
+
+        monkeypatch.setattr(
+            CollectorSettings, "effective_raw_packet_retention_days", 90
+        )
+
     def test_get_stats_empty(self, client_no_auth):
         """Test getting stats with empty database."""
         response = client_no_auth.get("/api/v1/dashboard/stats")
@@ -158,6 +173,43 @@ class TestDashboardStats:
         assert data["total_packets"] == 3
         assert data["packets_7d"] == 2  # now + 3d (10d excluded)
 
+    def test_packets_window_days_tracks_retention(
+        self, client_no_auth, api_db_session, monkeypatch
+    ):
+        """The packet stat window is min(7, retention) and the count respects it.
+
+        With the default retention of 2d, a packet 3 days ago falls outside
+        the window even though it would be inside a 7d window — so packets_7d
+        undercounts relative to the old fixed-7d behaviour, and the response
+        exposes the actual window via packets_window_days so the UI can label
+        it honestly ("Last 2 days") instead of lying ("Last 7 days").
+        """
+        from meshcore_hub.common.config import CollectorSettings
+
+        now = datetime.now(timezone.utc)
+        api_db_session.add_all(
+            [
+                RawPacket(event_type="message", received_at=now),
+                RawPacket(event_type="message", received_at=now - timedelta(days=3)),
+            ]
+        )
+        api_db_session.commit()
+
+        # Default retention (2d): window clamps to 2, so the 3-day-old packet
+        # is excluded and the window is exposed for honest labelling.
+        monkeypatch.setattr(CollectorSettings, "effective_raw_packet_retention_days", 2)
+        data = client_no_auth.get("/api/v1/dashboard/stats").json()
+        assert data["packets_window_days"] == 2
+        assert data["packets_7d"] == 1  # only "now"; 3d is outside the 2d window
+
+        # Retention >= 7: window is the full 7 days, both packets counted.
+        monkeypatch.setattr(
+            CollectorSettings, "effective_raw_packet_retention_days", 30
+        )
+        data = client_no_auth.get("/api/v1/dashboard/stats").json()
+        assert data["packets_window_days"] == 7
+        assert data["packets_7d"] == 2
+
 
 class TestDashboardHtmlRemoved:
     """Tests that legacy HTML dashboard endpoint has been removed."""
@@ -296,6 +348,17 @@ class TestDashboardActivity:
 class TestPacketActivity:
     """Tests for GET /dashboard/packet-activity endpoint."""
 
+    @pytest.fixture(autouse=True)
+    def _high_raw_packet_retention(self, monkeypatch):
+        """Bump raw-packet retention to 90d so the days-handling tests below
+        exercise the request parameter (the retention clamp is covered by
+        ``test_days_clamped_to_retention``)."""
+        from meshcore_hub.common.config import CollectorSettings
+
+        monkeypatch.setattr(
+            CollectorSettings, "effective_raw_packet_retention_days", 90
+        )
+
     def test_get_packet_activity_empty(self, client_no_auth):
         """Test getting packet activity with empty database."""
         response = client_no_auth.get("/api/v1/dashboard/packet-activity")
@@ -323,6 +386,18 @@ class TestPacketActivity:
         assert data["days"] == 90
         assert len(data["data"]) == 90
 
+    def test_days_clamped_to_retention(self, client_no_auth, monkeypatch):
+        """Days are clamped to the raw-packet retention window so the chart
+        never advertises days whose data has been purged."""
+        from meshcore_hub.common.config import CollectorSettings
+
+        monkeypatch.setattr(CollectorSettings, "effective_raw_packet_retention_days", 2)
+        response = client_no_auth.get("/api/v1/dashboard/packet-activity?days=30")
+        assert response.status_code == 200
+        data = response.json()
+        assert data["days"] == 2
+        assert len(data["data"]) == 2
+
     def test_get_packet_activity_with_data(self, client_no_auth, api_db_session):
         """Test getting packet activity with packets across two days.
 
@@ -391,6 +466,17 @@ class TestPacketActivity:
 class TestPacketBreakdown:
     """Tests for GET /dashboard/packet-breakdown endpoint."""
 
+    @pytest.fixture(autouse=True)
+    def _high_raw_packet_retention(self, monkeypatch):
+        """Bump raw-packet retention to 90d so the days-handling tests below
+        exercise the request parameter (the retention clamp is covered by
+        ``test_days_clamped_to_retention``)."""
+        from meshcore_hub.common.config import CollectorSettings
+
+        monkeypatch.setattr(
+            CollectorSettings, "effective_raw_packet_retention_days", 90
+        )
+
     def test_get_packet_breakdown_empty(self, client_no_auth):
         """Empty database returns empty bucket lists."""
         response = client_no_auth.get("/api/v1/dashboard/packet-breakdown")
@@ -418,6 +504,17 @@ class TestPacketBreakdown:
         data = response.json()
         assert data["days"] == 90
 
+    def test_days_clamped_to_retention(self, client_no_auth, monkeypatch):
+        """Days are clamped to the raw-packet retention window so the breakdown
+        never advertises days whose data has been purged."""
+        from meshcore_hub.common.config import CollectorSettings
+
+        monkeypatch.setattr(CollectorSettings, "effective_raw_packet_retention_days", 2)
+        response = client_no_auth.get("/api/v1/dashboard/packet-breakdown?days=14")
+        assert response.status_code == 200
+        data = response.json()
+        assert data["days"] == 2
+
     def test_breakdown_excludes_today(self, client_no_auth, api_db_session):
         """Today's packets are excluded from the breakdown window."""
         now = datetime.now(timezone.utc)