fix(dashboard): make packet displays retention-aware

PR #348 lowered the RAW_PACKET_RETENTION_DAYS default 7->2, but the
dashboard packet count, activity chart, and breakdown chart still
advertised "Last 7 days" — a label that lied once only 2 days of data
remained. Make every raw-packet-derived window track the effective
retention so labels stay honest:

- get_dashboard_stats: packets_7d window = min(7, retention); expose
  packets_window_days on DashboardStats so the UI renders the real window
- packet-activity / packet-breakdown: clamp days = min(days, 90, retention)
  so responses never advertise days whose data has been purged
- Home StatCard + Dashboard chart subtitles: dynamic "Last N days" /
  "Per day (last N days)" via new time.last_n_days / time.per_day_last_n_days
  i18n keys ({{n}} interpolation, matching routes_over_last_n_days)
- ActivitySeries.days made optional (type shared across activity charts);
  PacketBreakdown.days added (backend always serializes it)

Messages/ads 7-day displays are unchanged — those draw on event data
which retains the full 30-day DATA_RETENTION_DAYS window.

Tests: autouse retention fixture (->90) on the three packet test classes
so existing days-handling tests stay valid; added retention-clamp tests
proving the window tracks retention=2 and relaxes to 7 at retention>=7.
This commit is contained in:
Louis King
2026-07-25 14:39:33 +01:00
parent 153ba14ea0
commit 7802a00db3
8 changed files with 151 additions and 8 deletions
+24 -4
View File
@@ -185,12 +185,19 @@ def get_stats(
advertisements_24h = adv_row[1] or 0
advertisements_7d = adv_row[2] or 0
# Raw-packet counts (total + last 7 days), observer-level volume metric
# Raw-packet counts (total + recent window), observer-level volume metric
# with no role/channel filter (payload redaction lives on list/detail only).
# The "recent" window is capped at min(7, retention) because raw packets
# (and their cascaded packet_path_hops) are purged at RAW_PACKET_RETENTION_DAYS;
# a wider window would silently undercount against the "Last 7 days" label.
packets_window_days = min(
7, get_collector_settings().effective_raw_packet_retention_days
)
packets_since = now - timedelta(days=packets_window_days)
packet_row = session.execute(
select(
func.count(RawPacket.id).label("total_packets"),
func.sum(case((RawPacket.received_at >= seven_days_ago, 1), else_=0)).label(
func.sum(case((RawPacket.received_at >= packets_since, 1), else_=0)).label(
"packets_7d"
),
).select_from(RawPacket)
@@ -252,6 +259,7 @@ def get_stats(
total_members=total_members,
total_packets=total_packets,
packets_7d=packets_7d,
packets_window_days=packets_window_days,
)
@@ -470,7 +478,14 @@ def get_packet_activity(
Returns:
Daily raw-packet counts for each day in the period (excluding today)
"""
days = min(days, 90)
# Clamp to the raw-packet retention window so the response never advertises
# days whose data has been purged (the route evaluator / dashboard stat uses
# the same cap). See get_dashboard_stats for the rationale.
days = min(
days,
90,
get_collector_settings().effective_raw_packet_retention_days,
)
now = datetime.now(timezone.utc)
end_date = now.replace(hour=0, minute=0, second=0, microsecond=0)
@@ -524,7 +539,12 @@ def get_packet_breakdown(
Counts bucketed by event type (top 6 + "other") and by path-hash
byte width (1b/2b/3b, NULL excluded) for the period (excluding today).
"""
days = min(days, 90)
# Clamp to the raw-packet retention window (see get_dashboard_stats).
days = min(
days,
90,
get_collector_settings().effective_raw_packet_retention_days,
)
now = datetime.now(timezone.utc)
end_date = now.replace(hour=0, minute=0, second=0, microsecond=0)
@@ -296,6 +296,14 @@ class DashboardStats(BaseModel):
)
total_packets: int = Field(default=0, description="Total raw packets captured")
packets_7d: int = Field(default=0, description="Packets captured in last 7 days")
packets_window_days: int = Field(
default=7,
description=(
"Actual window in days for packets_7d = min(7, "
"raw_packet_retention_days). The UI labels the stat with this "
"value so it never claims a 7-day window when retention is shorter."
),
)
class RecentActivity(BaseModel):
@@ -41,9 +41,11 @@ interface DashboardStats {
advertisements_7d: number;
messages_7d: number;
packets_7d: number;
packets_window_days: number;
}
interface PacketBreakdown {
days: number;
by_event_type: BreakdownBucket[];
by_path_width: BreakdownBucket[];
}
@@ -491,7 +493,9 @@ export function DashboardPage() {
colorVar="--color-packets"
icon={<IconPackets className="h-5 w-5" />}
title={t("entities.packets")}
subtitle={t("time.per_day_last_7_days")}
subtitle={t("time.per_day_last_n_days", {
n: data.packetActivity?.days ?? 7,
})}
value={stats.packets_7d}
>
<TrendLineChart
@@ -511,7 +515,9 @@ export function DashboardPage() {
colorVar="--color-packets"
icon={<IconPackets className="h-5 w-5" />}
title={t("entities.packet_event_types")}
subtitle={t("time.last_7_days")}
subtitle={t("time.last_n_days", {
n: packetBreakdown?.days ?? 7,
})}
value={eventTypeTotal}
>
<StackedBarChart
@@ -525,7 +531,9 @@ export function DashboardPage() {
colorVar="--color-packets"
icon={<IconPackets className="h-5 w-5" />}
title={t("entities.path_hash_width")}
subtitle={t("time.last_7_days")}
subtitle={t("time.last_n_days", {
n: packetBreakdown?.days ?? 7,
})}
value={pathWidthTotal}
>
<StackedBarChart
@@ -41,6 +41,7 @@ interface DashboardStats {
advertisements_7d: number;
messages_7d: number;
packets_7d: number;
packets_window_days: number;
total_operators: number;
total_members: number;
}
@@ -371,7 +372,9 @@ export function HomePage() {
color={getPageColor("packets")}
title={t("entities.packets")}
value={stats.packets_7d}
description={t("time.last_7_days")}
description={t("time.last_n_days", {
n: stats.packets_window_days,
})}
/>
)}
</div>
@@ -16,6 +16,9 @@ export interface ActivityPoint {
}
export interface ActivitySeries {
// `days` is present on all DailyActivity API responses but optional here so
// the type can be reused for activity charts that don't surface the window.
days?: number;
data: ActivityPoint[];
}
@@ -137,7 +137,9 @@
"minutes_ago": "{{count}}m ago",
"less_than_minute": "<1m ago",
"last_7_days": "Last 7 days",
"last_n_days": "Last {{n}} days",
"per_day_last_7_days": "Per day (last 7 days)",
"per_day_last_n_days": "Per day (last {{n}} days)",
"over_time_last_7_days": "Over time (last 7 days)",
"activity_per_day_last_7_days": "Activity per day (last 7 days)",
"routes_over_last_n_days": "Matched packets per route (last {{n}} days)"
@@ -128,7 +128,9 @@
"minutes_ago": "{{count}}m geleden",
"less_than_minute": "<1m geleden",
"last_7_days": "Laatste 7 dagen",
"last_n_days": "Laatste {{n}} dagen",
"per_day_last_7_days": "Per dag (laatste 7 dagen)",
"per_day_last_n_days": "Per dag (laatste {{n}} dagen)",
"over_time_last_7_days": "In de tijd (laatste 7 dagen)",
"activity_per_day_last_7_days": "Activiteit per dag (laatste 7 dagen)",
"routes_over_last_n_days": "Overeenkomende pakketten per route (laatste {{n}} dagen)"
+97
View File
@@ -51,6 +51,21 @@ class TestDateBucketKey:
class TestDashboardStats:
"""Tests for GET /dashboard/stats endpoint."""
@pytest.fixture(autouse=True)
def _high_raw_packet_retention(self, monkeypatch):
"""Bump raw-packet retention to 90d for this class.
The packet stat window is min(7, retention); at the default retention
of 2d it would clamp to 2d and break the days-handling assertions
below. The clamp itself is covered by
``test_packets_window_days_tracks_retention``.
"""
from meshcore_hub.common.config import CollectorSettings
monkeypatch.setattr(
CollectorSettings, "effective_raw_packet_retention_days", 90
)
def test_get_stats_empty(self, client_no_auth):
"""Test getting stats with empty database."""
response = client_no_auth.get("/api/v1/dashboard/stats")
@@ -158,6 +173,43 @@ class TestDashboardStats:
assert data["total_packets"] == 3
assert data["packets_7d"] == 2 # now + 3d (10d excluded)
def test_packets_window_days_tracks_retention(
self, client_no_auth, api_db_session, monkeypatch
):
"""The packet stat window is min(7, retention) and the count respects it.
With the default retention of 2d, a packet 3 days ago falls outside
the window even though it would be inside a 7d window — so packets_7d
undercounts relative to the old fixed-7d behaviour, and the response
exposes the actual window via packets_window_days so the UI can label
it honestly ("Last 2 days") instead of lying ("Last 7 days").
"""
from meshcore_hub.common.config import CollectorSettings
now = datetime.now(timezone.utc)
api_db_session.add_all(
[
RawPacket(event_type="message", received_at=now),
RawPacket(event_type="message", received_at=now - timedelta(days=3)),
]
)
api_db_session.commit()
# Default retention (2d): window clamps to 2, so the 3-day-old packet
# is excluded and the window is exposed for honest labelling.
monkeypatch.setattr(CollectorSettings, "effective_raw_packet_retention_days", 2)
data = client_no_auth.get("/api/v1/dashboard/stats").json()
assert data["packets_window_days"] == 2
assert data["packets_7d"] == 1 # only "now"; 3d is outside the 2d window
# Retention >= 7: window is the full 7 days, both packets counted.
monkeypatch.setattr(
CollectorSettings, "effective_raw_packet_retention_days", 30
)
data = client_no_auth.get("/api/v1/dashboard/stats").json()
assert data["packets_window_days"] == 7
assert data["packets_7d"] == 2
class TestDashboardHtmlRemoved:
"""Tests that legacy HTML dashboard endpoint has been removed."""
@@ -296,6 +348,17 @@ class TestDashboardActivity:
class TestPacketActivity:
"""Tests for GET /dashboard/packet-activity endpoint."""
@pytest.fixture(autouse=True)
def _high_raw_packet_retention(self, monkeypatch):
"""Bump raw-packet retention to 90d so the days-handling tests below
exercise the request parameter (the retention clamp is covered by
``test_days_clamped_to_retention``)."""
from meshcore_hub.common.config import CollectorSettings
monkeypatch.setattr(
CollectorSettings, "effective_raw_packet_retention_days", 90
)
def test_get_packet_activity_empty(self, client_no_auth):
"""Test getting packet activity with empty database."""
response = client_no_auth.get("/api/v1/dashboard/packet-activity")
@@ -323,6 +386,18 @@ class TestPacketActivity:
assert data["days"] == 90
assert len(data["data"]) == 90
def test_days_clamped_to_retention(self, client_no_auth, monkeypatch):
"""Days are clamped to the raw-packet retention window so the chart
never advertises days whose data has been purged."""
from meshcore_hub.common.config import CollectorSettings
monkeypatch.setattr(CollectorSettings, "effective_raw_packet_retention_days", 2)
response = client_no_auth.get("/api/v1/dashboard/packet-activity?days=30")
assert response.status_code == 200
data = response.json()
assert data["days"] == 2
assert len(data["data"]) == 2
def test_get_packet_activity_with_data(self, client_no_auth, api_db_session):
"""Test getting packet activity with packets across two days.
@@ -391,6 +466,17 @@ class TestPacketActivity:
class TestPacketBreakdown:
"""Tests for GET /dashboard/packet-breakdown endpoint."""
@pytest.fixture(autouse=True)
def _high_raw_packet_retention(self, monkeypatch):
"""Bump raw-packet retention to 90d so the days-handling tests below
exercise the request parameter (the retention clamp is covered by
``test_days_clamped_to_retention``)."""
from meshcore_hub.common.config import CollectorSettings
monkeypatch.setattr(
CollectorSettings, "effective_raw_packet_retention_days", 90
)
def test_get_packet_breakdown_empty(self, client_no_auth):
"""Empty database returns empty bucket lists."""
response = client_no_auth.get("/api/v1/dashboard/packet-breakdown")
@@ -418,6 +504,17 @@ class TestPacketBreakdown:
data = response.json()
assert data["days"] == 90
def test_days_clamped_to_retention(self, client_no_auth, monkeypatch):
"""Days are clamped to the raw-packet retention window so the breakdown
never advertises days whose data has been purged."""
from meshcore_hub.common.config import CollectorSettings
monkeypatch.setattr(CollectorSettings, "effective_raw_packet_retention_days", 2)
response = client_no_auth.get("/api/v1/dashboard/packet-breakdown?days=14")
assert response.status_code == 200
data = response.json()
assert data["days"] == 2
def test_breakdown_excludes_today(self, client_no_auth, api_db_session):
"""Today's packets are excluded from the breakdown window."""
now = datetime.now(timezone.utc)