mirror of
https://github.com/ipnet-mesh/meshcore-hub.git
synced 2026-08-09 18:32:44 +02:00
fix(dashboard): make packet displays retention-aware
PR #348 lowered the RAW_PACKET_RETENTION_DAYS default 7->2, but the dashboard packet count, activity chart, and breakdown chart still advertised "Last 7 days" — a label that lied once only 2 days of data remained. Make every raw-packet-derived window track the effective retention so labels stay honest: - get_dashboard_stats: packets_7d window = min(7, retention); expose packets_window_days on DashboardStats so the UI renders the real window - packet-activity / packet-breakdown: clamp days = min(days, 90, retention) so responses never advertise days whose data has been purged - Home StatCard + Dashboard chart subtitles: dynamic "Last N days" / "Per day (last N days)" via new time.last_n_days / time.per_day_last_n_days i18n keys ({{n}} interpolation, matching routes_over_last_n_days) - ActivitySeries.days made optional (type shared across activity charts); PacketBreakdown.days added (backend always serializes it) Messages/ads 7-day displays are unchanged — those draw on event data which retains the full 30-day DATA_RETENTION_DAYS window. Tests: autouse retention fixture (->90) on the three packet test classes so existing days-handling tests stay valid; added retention-clamp tests proving the window tracks retention=2 and relaxes to 7 at retention>=7.
This commit is contained in:
@@ -185,12 +185,19 @@ def get_stats(
|
||||
advertisements_24h = adv_row[1] or 0
|
||||
advertisements_7d = adv_row[2] or 0
|
||||
|
||||
# Raw-packet counts (total + last 7 days), observer-level volume metric
|
||||
# Raw-packet counts (total + recent window), observer-level volume metric
|
||||
# with no role/channel filter (payload redaction lives on list/detail only).
|
||||
# The "recent" window is capped at min(7, retention) because raw packets
|
||||
# (and their cascaded packet_path_hops) are purged at RAW_PACKET_RETENTION_DAYS;
|
||||
# a wider window would silently undercount against the "Last 7 days" label.
|
||||
packets_window_days = min(
|
||||
7, get_collector_settings().effective_raw_packet_retention_days
|
||||
)
|
||||
packets_since = now - timedelta(days=packets_window_days)
|
||||
packet_row = session.execute(
|
||||
select(
|
||||
func.count(RawPacket.id).label("total_packets"),
|
||||
func.sum(case((RawPacket.received_at >= seven_days_ago, 1), else_=0)).label(
|
||||
func.sum(case((RawPacket.received_at >= packets_since, 1), else_=0)).label(
|
||||
"packets_7d"
|
||||
),
|
||||
).select_from(RawPacket)
|
||||
@@ -252,6 +259,7 @@ def get_stats(
|
||||
total_members=total_members,
|
||||
total_packets=total_packets,
|
||||
packets_7d=packets_7d,
|
||||
packets_window_days=packets_window_days,
|
||||
)
|
||||
|
||||
|
||||
@@ -470,7 +478,14 @@ def get_packet_activity(
|
||||
Returns:
|
||||
Daily raw-packet counts for each day in the period (excluding today)
|
||||
"""
|
||||
days = min(days, 90)
|
||||
# Clamp to the raw-packet retention window so the response never advertises
|
||||
# days whose data has been purged (the route evaluator / dashboard stat uses
|
||||
# the same cap). See get_dashboard_stats for the rationale.
|
||||
days = min(
|
||||
days,
|
||||
90,
|
||||
get_collector_settings().effective_raw_packet_retention_days,
|
||||
)
|
||||
|
||||
now = datetime.now(timezone.utc)
|
||||
end_date = now.replace(hour=0, minute=0, second=0, microsecond=0)
|
||||
@@ -524,7 +539,12 @@ def get_packet_breakdown(
|
||||
Counts bucketed by event type (top 6 + "other") and by path-hash
|
||||
byte width (1b/2b/3b, NULL excluded) for the period (excluding today).
|
||||
"""
|
||||
days = min(days, 90)
|
||||
# Clamp to the raw-packet retention window (see get_dashboard_stats).
|
||||
days = min(
|
||||
days,
|
||||
90,
|
||||
get_collector_settings().effective_raw_packet_retention_days,
|
||||
)
|
||||
|
||||
now = datetime.now(timezone.utc)
|
||||
end_date = now.replace(hour=0, minute=0, second=0, microsecond=0)
|
||||
|
||||
@@ -296,6 +296,14 @@ class DashboardStats(BaseModel):
|
||||
)
|
||||
total_packets: int = Field(default=0, description="Total raw packets captured")
|
||||
packets_7d: int = Field(default=0, description="Packets captured in last 7 days")
|
||||
packets_window_days: int = Field(
|
||||
default=7,
|
||||
description=(
|
||||
"Actual window in days for packets_7d = min(7, "
|
||||
"raw_packet_retention_days). The UI labels the stat with this "
|
||||
"value so it never claims a 7-day window when retention is shorter."
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
class RecentActivity(BaseModel):
|
||||
|
||||
@@ -41,9 +41,11 @@ interface DashboardStats {
|
||||
advertisements_7d: number;
|
||||
messages_7d: number;
|
||||
packets_7d: number;
|
||||
packets_window_days: number;
|
||||
}
|
||||
|
||||
interface PacketBreakdown {
|
||||
days: number;
|
||||
by_event_type: BreakdownBucket[];
|
||||
by_path_width: BreakdownBucket[];
|
||||
}
|
||||
@@ -491,7 +493,9 @@ export function DashboardPage() {
|
||||
colorVar="--color-packets"
|
||||
icon={<IconPackets className="h-5 w-5" />}
|
||||
title={t("entities.packets")}
|
||||
subtitle={t("time.per_day_last_7_days")}
|
||||
subtitle={t("time.per_day_last_n_days", {
|
||||
n: data.packetActivity?.days ?? 7,
|
||||
})}
|
||||
value={stats.packets_7d}
|
||||
>
|
||||
<TrendLineChart
|
||||
@@ -511,7 +515,9 @@ export function DashboardPage() {
|
||||
colorVar="--color-packets"
|
||||
icon={<IconPackets className="h-5 w-5" />}
|
||||
title={t("entities.packet_event_types")}
|
||||
subtitle={t("time.last_7_days")}
|
||||
subtitle={t("time.last_n_days", {
|
||||
n: packetBreakdown?.days ?? 7,
|
||||
})}
|
||||
value={eventTypeTotal}
|
||||
>
|
||||
<StackedBarChart
|
||||
@@ -525,7 +531,9 @@ export function DashboardPage() {
|
||||
colorVar="--color-packets"
|
||||
icon={<IconPackets className="h-5 w-5" />}
|
||||
title={t("entities.path_hash_width")}
|
||||
subtitle={t("time.last_7_days")}
|
||||
subtitle={t("time.last_n_days", {
|
||||
n: packetBreakdown?.days ?? 7,
|
||||
})}
|
||||
value={pathWidthTotal}
|
||||
>
|
||||
<StackedBarChart
|
||||
|
||||
@@ -41,6 +41,7 @@ interface DashboardStats {
|
||||
advertisements_7d: number;
|
||||
messages_7d: number;
|
||||
packets_7d: number;
|
||||
packets_window_days: number;
|
||||
total_operators: number;
|
||||
total_members: number;
|
||||
}
|
||||
@@ -371,7 +372,9 @@ export function HomePage() {
|
||||
color={getPageColor("packets")}
|
||||
title={t("entities.packets")}
|
||||
value={stats.packets_7d}
|
||||
description={t("time.last_7_days")}
|
||||
description={t("time.last_n_days", {
|
||||
n: stats.packets_window_days,
|
||||
})}
|
||||
/>
|
||||
)}
|
||||
</div>
|
||||
|
||||
@@ -16,6 +16,9 @@ export interface ActivityPoint {
|
||||
}
|
||||
|
||||
export interface ActivitySeries {
|
||||
// `days` is present on all DailyActivity API responses but optional here so
|
||||
// the type can be reused for activity charts that don't surface the window.
|
||||
days?: number;
|
||||
data: ActivityPoint[];
|
||||
}
|
||||
|
||||
|
||||
@@ -137,7 +137,9 @@
|
||||
"minutes_ago": "{{count}}m ago",
|
||||
"less_than_minute": "<1m ago",
|
||||
"last_7_days": "Last 7 days",
|
||||
"last_n_days": "Last {{n}} days",
|
||||
"per_day_last_7_days": "Per day (last 7 days)",
|
||||
"per_day_last_n_days": "Per day (last {{n}} days)",
|
||||
"over_time_last_7_days": "Over time (last 7 days)",
|
||||
"activity_per_day_last_7_days": "Activity per day (last 7 days)",
|
||||
"routes_over_last_n_days": "Matched packets per route (last {{n}} days)"
|
||||
|
||||
@@ -128,7 +128,9 @@
|
||||
"minutes_ago": "{{count}}m geleden",
|
||||
"less_than_minute": "<1m geleden",
|
||||
"last_7_days": "Laatste 7 dagen",
|
||||
"last_n_days": "Laatste {{n}} dagen",
|
||||
"per_day_last_7_days": "Per dag (laatste 7 dagen)",
|
||||
"per_day_last_n_days": "Per dag (laatste {{n}} dagen)",
|
||||
"over_time_last_7_days": "In de tijd (laatste 7 dagen)",
|
||||
"activity_per_day_last_7_days": "Activiteit per dag (laatste 7 dagen)",
|
||||
"routes_over_last_n_days": "Overeenkomende pakketten per route (laatste {{n}} dagen)"
|
||||
|
||||
@@ -51,6 +51,21 @@ class TestDateBucketKey:
|
||||
class TestDashboardStats:
|
||||
"""Tests for GET /dashboard/stats endpoint."""
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _high_raw_packet_retention(self, monkeypatch):
|
||||
"""Bump raw-packet retention to 90d for this class.
|
||||
|
||||
The packet stat window is min(7, retention); at the default retention
|
||||
of 2d it would clamp to 2d and break the days-handling assertions
|
||||
below. The clamp itself is covered by
|
||||
``test_packets_window_days_tracks_retention``.
|
||||
"""
|
||||
from meshcore_hub.common.config import CollectorSettings
|
||||
|
||||
monkeypatch.setattr(
|
||||
CollectorSettings, "effective_raw_packet_retention_days", 90
|
||||
)
|
||||
|
||||
def test_get_stats_empty(self, client_no_auth):
|
||||
"""Test getting stats with empty database."""
|
||||
response = client_no_auth.get("/api/v1/dashboard/stats")
|
||||
@@ -158,6 +173,43 @@ class TestDashboardStats:
|
||||
assert data["total_packets"] == 3
|
||||
assert data["packets_7d"] == 2 # now + 3d (10d excluded)
|
||||
|
||||
def test_packets_window_days_tracks_retention(
|
||||
self, client_no_auth, api_db_session, monkeypatch
|
||||
):
|
||||
"""The packet stat window is min(7, retention) and the count respects it.
|
||||
|
||||
With the default retention of 2d, a packet 3 days ago falls outside
|
||||
the window even though it would be inside a 7d window — so packets_7d
|
||||
undercounts relative to the old fixed-7d behaviour, and the response
|
||||
exposes the actual window via packets_window_days so the UI can label
|
||||
it honestly ("Last 2 days") instead of lying ("Last 7 days").
|
||||
"""
|
||||
from meshcore_hub.common.config import CollectorSettings
|
||||
|
||||
now = datetime.now(timezone.utc)
|
||||
api_db_session.add_all(
|
||||
[
|
||||
RawPacket(event_type="message", received_at=now),
|
||||
RawPacket(event_type="message", received_at=now - timedelta(days=3)),
|
||||
]
|
||||
)
|
||||
api_db_session.commit()
|
||||
|
||||
# Default retention (2d): window clamps to 2, so the 3-day-old packet
|
||||
# is excluded and the window is exposed for honest labelling.
|
||||
monkeypatch.setattr(CollectorSettings, "effective_raw_packet_retention_days", 2)
|
||||
data = client_no_auth.get("/api/v1/dashboard/stats").json()
|
||||
assert data["packets_window_days"] == 2
|
||||
assert data["packets_7d"] == 1 # only "now"; 3d is outside the 2d window
|
||||
|
||||
# Retention >= 7: window is the full 7 days, both packets counted.
|
||||
monkeypatch.setattr(
|
||||
CollectorSettings, "effective_raw_packet_retention_days", 30
|
||||
)
|
||||
data = client_no_auth.get("/api/v1/dashboard/stats").json()
|
||||
assert data["packets_window_days"] == 7
|
||||
assert data["packets_7d"] == 2
|
||||
|
||||
|
||||
class TestDashboardHtmlRemoved:
|
||||
"""Tests that legacy HTML dashboard endpoint has been removed."""
|
||||
@@ -296,6 +348,17 @@ class TestDashboardActivity:
|
||||
class TestPacketActivity:
|
||||
"""Tests for GET /dashboard/packet-activity endpoint."""
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _high_raw_packet_retention(self, monkeypatch):
|
||||
"""Bump raw-packet retention to 90d so the days-handling tests below
|
||||
exercise the request parameter (the retention clamp is covered by
|
||||
``test_days_clamped_to_retention``)."""
|
||||
from meshcore_hub.common.config import CollectorSettings
|
||||
|
||||
monkeypatch.setattr(
|
||||
CollectorSettings, "effective_raw_packet_retention_days", 90
|
||||
)
|
||||
|
||||
def test_get_packet_activity_empty(self, client_no_auth):
|
||||
"""Test getting packet activity with empty database."""
|
||||
response = client_no_auth.get("/api/v1/dashboard/packet-activity")
|
||||
@@ -323,6 +386,18 @@ class TestPacketActivity:
|
||||
assert data["days"] == 90
|
||||
assert len(data["data"]) == 90
|
||||
|
||||
def test_days_clamped_to_retention(self, client_no_auth, monkeypatch):
|
||||
"""Days are clamped to the raw-packet retention window so the chart
|
||||
never advertises days whose data has been purged."""
|
||||
from meshcore_hub.common.config import CollectorSettings
|
||||
|
||||
monkeypatch.setattr(CollectorSettings, "effective_raw_packet_retention_days", 2)
|
||||
response = client_no_auth.get("/api/v1/dashboard/packet-activity?days=30")
|
||||
assert response.status_code == 200
|
||||
data = response.json()
|
||||
assert data["days"] == 2
|
||||
assert len(data["data"]) == 2
|
||||
|
||||
def test_get_packet_activity_with_data(self, client_no_auth, api_db_session):
|
||||
"""Test getting packet activity with packets across two days.
|
||||
|
||||
@@ -391,6 +466,17 @@ class TestPacketActivity:
|
||||
class TestPacketBreakdown:
|
||||
"""Tests for GET /dashboard/packet-breakdown endpoint."""
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _high_raw_packet_retention(self, monkeypatch):
|
||||
"""Bump raw-packet retention to 90d so the days-handling tests below
|
||||
exercise the request parameter (the retention clamp is covered by
|
||||
``test_days_clamped_to_retention``)."""
|
||||
from meshcore_hub.common.config import CollectorSettings
|
||||
|
||||
monkeypatch.setattr(
|
||||
CollectorSettings, "effective_raw_packet_retention_days", 90
|
||||
)
|
||||
|
||||
def test_get_packet_breakdown_empty(self, client_no_auth):
|
||||
"""Empty database returns empty bucket lists."""
|
||||
response = client_no_auth.get("/api/v1/dashboard/packet-breakdown")
|
||||
@@ -418,6 +504,17 @@ class TestPacketBreakdown:
|
||||
data = response.json()
|
||||
assert data["days"] == 90
|
||||
|
||||
def test_days_clamped_to_retention(self, client_no_auth, monkeypatch):
|
||||
"""Days are clamped to the raw-packet retention window so the breakdown
|
||||
never advertises days whose data has been purged."""
|
||||
from meshcore_hub.common.config import CollectorSettings
|
||||
|
||||
monkeypatch.setattr(CollectorSettings, "effective_raw_packet_retention_days", 2)
|
||||
response = client_no_auth.get("/api/v1/dashboard/packet-breakdown?days=14")
|
||||
assert response.status_code == 200
|
||||
data = response.json()
|
||||
assert data["days"] == 2
|
||||
|
||||
def test_breakdown_excludes_today(self, client_no_auth, api_db_session):
|
||||
"""Today's packets are excluded from the breakdown window."""
|
||||
now = datetime.now(timezone.utc)
|
||||
|
||||
Reference in New Issue
Block a user