Show statistics for regional messages. Closes #338.

This commit is contained in:
Jack Kingsman
2026-07-25 12:46:38 -07:00
parent 58cd61dc61
commit 50d063b195
10 changed files with 626 additions and 11 deletions
+1 -1
View File
@@ -393,7 +393,7 @@ All endpoints are prefixed with `/api` (e.g., `/api/health`).
| PATCH | `/api/fanout/{id}` | Update fanout config (triggers module reload) |
| DELETE | `/api/fanout/{id}` | Delete fanout config (stops module) |
| POST | `/api/fanout/bots/disable-until-restart` | Stop bot fanout modules and keep bots disabled until the process restarts |
| GET | `/api/statistics` | Aggregated mesh network statistics |
| GET | `/api/statistics` | Aggregated mesh network statistics, including `region_scope_24h` regional flood-scope adoption |
| GET | `/api/push/vapid-public-key` | VAPID public key for browser push subscription |
| POST | `/api/push/subscribe` | Register/upsert a push subscription |
| GET | `/api/push/subscriptions` | List all push subscriptions |
+12 -1
View File
@@ -155,6 +155,17 @@ app/
- Candidate region names come from `app_settings.known_regions` (user-editable, seeded by migration 063 from `flood_scope` + channel `flood_scope_override`).
- Channel messages persist `messages.transport_code` (uint16, NULL = unscoped plain flood) and `messages.region` (resolved name, NULL = scoped but no list match) at ingest, so the chat region badge survives raw-packet purge. The packet inspector (`GET /packets/{id}` and the `raw_packet` WS broadcast) resolves region on the fly against the current list since it still holds the raw payload.
### Region-scope adoption stats (`region_scope_24h`)
`GET /statistics` reports regional flood-scope uptake as two views with different denominators that intentionally will not agree:
- **Traffic** (`bucket_region_scope` in `path_utils.py`) counts flood-routed (`route_type` 0/1) GroupText packets across all channels, including undecryptable ones. Zero-hop/direct sends are excluded because firmware reaches them through the non-transport `sendZeroHop`/`sendDirect` overloads and they can never carry transport codes.
- **Senders** (`StatisticsRepository._region_scope_senders_24h`) counts distinct senders with at least one scoped message. Attribution requires decryption, so it only covers channels we hold keys for — narrower, but self-validating (a decrypted packet is provably not a corrupt capture) and immune to one chatty node skewing the result. Identity is `sender_key` falling back to `sender_name`; scoping reads `messages.transport_code`, falling back to the linked raw packet for rows stored before region tagging existed.
`false_positive_floor` exists because corrupt RF captures land in `raw_packets` with effectively random headers and a share of them claim `TRANSPORT_FLOOD`. That garbage spreads near-uniformly across payload-type buckets, so it is measured directly from payload types the protocol does not define (`0x0C`/`0x0D`/`0x0E`) and averaged per bucket. **A `scoped_messages` count at or below the floor is not evidence of adoption**; surface the two together and never show the percentage alone. Do not "fix" the floor by removing it — without it the metric reads several times higher than reality.
Both traffic buckets come from one 24h raw-packet scan (`_packet_shape_24h`) shared with `path_hash_width_24h`, so adding region stats costs no extra query or parse pass.
### Raw packet dedup policy
- Raw packet storage deduplicates by payload hash (`RawPacketRepository.create`), excluding routing/path bytes.
@@ -297,7 +308,7 @@ Web Push is a standalone subsystem in `app/push/`, separate from the fanout modu
- `POST /fanout/bots/disable-until-restart` — stop bot modules and keep bots disabled until restart
### Statistics
- `GET /statistics` — aggregated mesh network stats (entity counts, message/packet splits, activity windows, busiest channels)
- `GET /statistics` — aggregated mesh network stats (entity counts, message/packet splits, activity windows, busiest channels, `region_scope_24h` regional adoption)
### Push
- `GET /push/vapid-public-key` — VAPID public key for browser `PushManager.subscribe()`
+36
View File
@@ -1083,6 +1083,41 @@ class PacketsPerHourBucket(BaseModel):
count: int = Field(description="Number of packets received in that hour")
class RegionScopeStats(BaseModel):
"""Regional flood-scope adoption over the last 24 hours.
Two independent views, deliberately not merged they have different
denominators and will not agree:
- Traffic (``total_messages``/``scoped_messages``) counts flood-routed
channel-message packets across all channels, including ones we cannot
decrypt. Broad coverage, but corrupt RF captures contribute false
positives, hence ``false_positive_floor``.
- Senders (``total_senders``/``scoped_senders``) counts distinct message
senders, which requires decryption and so only covers channels we hold
keys for. Narrower, but noise-free and immune to one chatty node skewing
the result.
"""
total_messages: int = Field(
description="Flood-routed channel-message packets heard in the last 24h (unique payloads)"
)
scoped_messages: int = Field(description="Of those, how many carried a regional transport code")
scoped_pct: float
false_positive_floor: float = Field(
description=(
"Estimated false positives in scoped_messages, measured from transport-routed "
"packets claiming a payload type the protocol does not define. A scoped_messages "
"value at or below this is not evidence of regional adoption."
)
)
total_senders: int = Field(
description="Distinct channel-message senders in the last 24h (decryptable channels only)"
)
scoped_senders: int = Field(description="Of those, how many sent at least one scoped message")
scoped_senders_pct: float
class StatisticsResponse(BaseModel):
busiest_channels_24h: list[BusyChannel]
contact_count: int
@@ -1098,6 +1133,7 @@ class StatisticsResponse(BaseModel):
repeaters_heard: ContactActivityCounts
known_channels_active: ContactActivityCounts
path_hash_width_24h: PathHashWidthStats
region_scope_24h: RegionScopeStats
packets_per_hour_72h: list[PacketsPerHourBucket]
noise_floor_24h: NoiseFloorHistoryStats
+63
View File
@@ -304,6 +304,69 @@ def bucket_path_hash_widths(rows: Iterable) -> dict[str, int | float]:
}
# Payload types with no meaning in the MeshCore protocol. Any packet claiming one
# is corrupt by definition, which makes them a usable gauge for how much RF garbage
# is sitting in a given route-type bucket. See bucket_region_scope().
UNDEFINED_PAYLOAD_TYPES = (0x0C, 0x0D, 0x0E)
_PAYLOAD_TYPE_GROUP_TEXT = 0x05
_FLOOD_ROUTE_TYPES = (0x00, 0x01) # TRANSPORT_FLOOD, FLOOD
def bucket_region_scope(rows: Iterable) -> dict[str, int | float]:
"""Count flood-routed channel messages carrying a regional transport code.
*rows* must be an already-fetched list whose elements have a ``data`` column
containing raw packet bytes.
Only flood-routed packets are counted. Zero-hop and direct sends can never
carry transport codes (firmware reaches them through the non-transport
``sendZeroHop``/``sendDirect`` overloads), so including them would silently
dilute the percentage.
``false_positive_floor`` exists because corrupt RF captures still land in
``raw_packets`` with effectively random header bytes, and a share of them
claim TRANSPORT_FLOOD. That garbage spreads near-uniformly across payload-type
buckets, so we measure it directly: count transport-routed packets claiming a
payload type the protocol does not define, and average per bucket. A
``scoped_messages`` count at or below ``false_positive_floor`` is not evidence
of regional adoption, and callers should present the two together.
"""
total = 0
scoped = 0
undefined_scoped = 0
for row in rows:
envelope = parse_packet_envelope(bytes(row["data"]))
if envelope is None:
continue
if (
envelope.route_type == 0x00
and envelope.payload_type in UNDEFINED_PAYLOAD_TYPES
and envelope.transport_codes is not None
):
undefined_scoped += 1
continue
if envelope.payload_type != _PAYLOAD_TYPE_GROUP_TEXT:
continue
if envelope.route_type not in _FLOOD_ROUTE_TYPES:
continue
total += 1
if envelope.transport_codes is not None:
scoped += 1
noise_floor = undefined_scoped / len(UNDEFINED_PAYLOAD_TYPES)
return {
"total_messages": total,
"scoped_messages": scoped,
"scoped_pct": (scoped / total) * 100 if total else 0.0,
"false_positive_floor": noise_floor,
}
def calculate_packet_hash(raw_bytes: bytes) -> str:
"""Calculate packet hash matching MeshCore's Packet::calculatePacketHash().
+65 -5
View File
@@ -7,7 +7,7 @@ import aiosqlite
from app.database import db
from app.models import AppSettings
from app.path_utils import bucket_path_hash_widths
from app.path_utils import bucket_path_hash_widths, bucket_region_scope, parse_packet_envelope
from app.telemetry_interval import DEFAULT_TELEMETRY_INTERVAL_HOURS
logger = logging.getLogger(__name__)
@@ -513,8 +513,12 @@ class StatisticsRepository:
return [{"timestamp": row["hour_ts"], "count": row["count"]} for row in rows]
@staticmethod
async def _path_hash_width_24h() -> dict[str, int | float]:
"""Count parsed raw packets from the last 24h by hop hash width."""
async def _packet_shape_24h() -> tuple[dict[str, int | float], dict[str, int | float]]:
"""Bucket the last 24h of raw packets by hop hash width and region scope.
Both buckets come from one fetch so the snapshot only scans the packet
table once. Returns ``(path_hash_width, region_scope)``.
"""
now = int(time.time())
async with db.readonly() as conn:
async with conn.execute(
@@ -522,7 +526,61 @@ class StatisticsRepository:
(now - SECONDS_24H,),
) as cursor:
rows = await cursor.fetchall()
return bucket_path_hash_widths(rows)
return bucket_path_hash_widths(rows), bucket_region_scope(rows)
@staticmethod
async def _region_scope_senders_24h() -> dict[str, int | float]:
"""Count distinct channel-message senders who scoped at least one send.
Sender attribution requires having decrypted the message, so this only
covers channels we hold keys for a narrower population than the
packet-level count, but a self-validating one: a message we decrypted is
provably not a corrupt capture, so this number needs no noise floor.
Scoping is read from ``messages.transport_code`` where present, falling
back to the linked raw packet for rows stored before region tagging
existed. Senders are keyed by ``sender_key`` where resolved, falling back
to ``sender_name`` one physical operator may run several nodes, hence
"senders" rather than "users".
"""
now = int(time.time())
async with db.readonly() as conn:
async with conn.execute(
"""
SELECT m.id, m.sender_key, m.sender_name, m.transport_code, p.data
FROM messages m
LEFT JOIN raw_packets p ON p.message_id = m.id
WHERE m.type = 'CHAN' AND m.outgoing = 0 AND m.received_at >= ?
""",
(now - SECONDS_24H,),
) as cursor:
rows = await cursor.fetchall()
senders: set[str] = set()
scoped_senders: set[str] = set()
for row in rows:
identity = row["sender_key"] or row["sender_name"]
if not identity:
continue
senders.add(identity)
if row["transport_code"] is not None:
scoped_senders.add(identity)
continue
raw = row["data"]
if raw is None:
continue
envelope = parse_packet_envelope(bytes(raw))
if envelope is not None and envelope.transport_codes is not None:
scoped_senders.add(identity)
total = len(senders)
scoped = len(scoped_senders)
return {
"total_senders": total,
"scoped_senders": scoped,
"scoped_senders_pct": (scoped / total) * 100 if total else 0.0,
}
@staticmethod
async def get_all() -> dict:
@@ -600,7 +658,8 @@ class StatisticsRepository:
contacts_heard = await StatisticsRepository._activity_counts(contact_type=2, exclude=True)
repeaters_heard = await StatisticsRepository._activity_counts(contact_type=2)
known_channels_active = await StatisticsRepository._known_channels_active()
path_hash_width_24h = await StatisticsRepository._path_hash_width_24h()
path_hash_width_24h, region_scope = await StatisticsRepository._packet_shape_24h()
region_scope.update(await StatisticsRepository._region_scope_senders_24h())
packets_per_hour_72h = await StatisticsRepository._packets_per_hour_72h()
return {
@@ -618,5 +677,6 @@ class StatisticsRepository:
"repeaters_heard": repeaters_heard,
"known_channels_active": known_channels_active,
"path_hash_width_24h": path_hash_width_24h,
"region_scope_24h": region_scope,
"packets_per_hour_72h": packets_per_hour_72h,
}
+10 -1
View File
@@ -152,7 +152,7 @@ frontend/src/
│ │ ├── SettingsFanoutSection.tsx # Fanout integrations: MQTT, bots, config CRUD
│ │ ├── SettingsRadioAppSection.tsx # Radio-App Management: tracked telemetry, contact management, blocked lists
│ │ ├── SettingsDatabaseSection.tsx # Database: DB size, storage cleanup, auto-decrypt
│ │ ├── SettingsStatisticsSection.tsx # Read-only mesh network stats
│ │ ├── SettingsStatisticsSection.tsx # Read-only mesh network stats (incl. region-scope adoption)
│ │ ├── SettingsAboutSection.tsx # Version, author, license, links
│ │ ├── ThemeSelector.tsx # Color theme picker
│ │ └── BulkDeleteContactsModal.tsx # Bulk contact deletion dialog
@@ -479,6 +479,15 @@ Key conventions documented in the reference:
- **Badges/tags** use `text-[0.625rem] uppercase tracking-wider px-1.5 py-0.5 rounded` with `bg-muted` (neutral) or `bg-primary/10` (active).
- **Clickable text** (copy-to-clipboard, navigational links) uses `role="button" tabIndex={0}` with `cursor-pointer hover:text-primary transition-colors`.
### Region-scope adoption panel
`SettingsStatisticsSection.tsx` renders `stats.region_scope_24h` via `RegionScopeStatsPanel`. Two presentation rules exist because regional adoption is currently very sparse, and both are deliberate:
- **Fractions, not bare percentages.** "3 of 117" carries the sample size that "2.6%" hides.
- **The traffic percentage is withheld** when the scoped count is at or below `false_positive_floor` (corrupt-capture noise) or when the share would round to `0.0%`. The floor caveat is always shown alongside a non-zero scoped count. The sender figure is never suppressed — it requires successful decryption and so carries no noise.
Traffic and sender figures use different denominators (all channels vs. decryptable-only) and are not expected to match.
## Security Posture (intentional)
- No authentication UI.
@@ -13,12 +13,76 @@ import {
} from 'recharts';
import { Separator } from '../ui/separator';
import { api } from '../../api';
import type { StatisticsResponse } from '../../types';
import type { RegionScopeStats, StatisticsResponse } from '../../types';
function formatPercent(value: number): string {
return `${value.toFixed(1)}%`;
}
/**
* Regional flood-scope adoption. Deliberately shows fractions rather than bare
* percentages: with adoption this sparse, "3 of 117" communicates the sample
* size that "2.6%" hides.
*/
function RegionScopeStatsPanel({ stats }: { stats: RegionScopeStats }) {
// Corrupt RF captures land in the packet table with random headers, some of
// which claim to be region-scoped. At or below the measured floor there is
// nothing to report but noise, so withhold the percentage and say so.
const floor = stats.false_positive_floor;
const withinNoise = stats.scoped_messages <= floor;
// Real-world scoping is currently rare enough that the share rounds to "0.0%",
// which reads as a broken widget. Below that resolution the fraction alone is
// the honest presentation.
const showTrafficPct = stats.total_messages > 0 && !withinNoise && stats.scoped_pct >= 0.05;
return (
<div>
<h3 className="text-base font-semibold tracking-tight mb-2">Region Scope (24h)</h3>
<p className="text-[0.8125rem] text-muted-foreground mb-3">
How much local traffic uses regional flood scoping. Traffic covers all channel messages
heard, including channels you have no key for; senders only counts channels you can decrypt,
so the two use different denominators and will not match.
</p>
<div className="space-y-2">
<div className="flex justify-between items-center gap-4">
<span className="text-sm text-muted-foreground">Scoped messages</span>
<span className="font-medium text-right">
{stats.scoped_messages.toLocaleString()} of {stats.total_messages.toLocaleString()}
{showTrafficPct && (
<span className="text-muted-foreground"> ({formatPercent(stats.scoped_pct)})</span>
)}
</span>
</div>
<div className="flex justify-between items-center gap-4">
<span className="text-sm text-muted-foreground">Senders using regions</span>
<span className="font-medium text-right">
{stats.scoped_senders.toLocaleString()} of {stats.total_senders.toLocaleString()}
{stats.total_senders > 0 && (
<span className="text-muted-foreground">
{' '}
({formatPercent(stats.scoped_senders_pct)})
</span>
)}
</span>
</div>
</div>
{floor > 0 && stats.scoped_messages > 0 && (
<p className="text-[0.8125rem] text-muted-foreground mt-2">
{withinNoise
? `Scoped message count is at or below the estimated false-positive floor (${floor.toFixed(0)}) from corrupt packet captures, so it is not evidence of regional adoption.`
: `Includes an estimated ${floor.toFixed(0)} false positives from corrupt packet captures.`}{' '}
The sender count is unaffected it requires successful decryption.
</p>
)}
{stats.total_messages === 0 && (
<p className="text-sm text-muted-foreground mt-2">
No channel messages heard in the last 24 hours.
</p>
)}
</div>
);
}
const CHANNEL_BAR_COLORS = ['#0ea5e9', '#10b981', '#f59e0b', '#f43f5e', '#8b5cf6'];
const TOOLTIP_STYLE = {
@@ -404,6 +468,11 @@ export function SettingsStatisticsSection({ className }: { className?: string })
)}
</div>
<Separator />
{/* Region Scope */}
<RegionScopeStatsPanel stats={stats.region_scope_24h} />
{/* Busiest Channels */}
{stats.busiest_channels_24h.length > 0 && (
<>
+158
View File
@@ -777,6 +777,15 @@ describe('SettingsModal', () => {
double_byte_pct: 30,
triple_byte_pct: 20,
},
region_scope_24h: {
total_messages: 120,
scoped_messages: 40,
scoped_pct: 33.3,
false_positive_floor: 2,
total_senders: 12,
scoped_senders: 3,
scoped_senders_pct: 25,
},
packets_per_hour_72h: [
{ timestamp: 1711792800, count: 12 },
{ timestamp: 1711796400, count: 8 },
@@ -824,6 +833,146 @@ describe('SettingsModal', () => {
expect(screen.getByText('Known-channels active')).toBeInTheDocument();
expect(screen.getByText('Busiest Channels (24h)')).toBeInTheDocument();
expect(screen.getByText('Noise Floor (24h)')).toBeInTheDocument();
expect(screen.getByText('Region Scope (24h)')).toBeInTheDocument();
// Fractions, not bare percentages — the sample size matters at this sparsity
expect(screen.getByText(/40 of 120/)).toBeInTheDocument();
expect(screen.getByText(/3 of 12/)).toBeInTheDocument();
// 40 scoped is well above the floor of 2, so the percentage is shown
expect(screen.getByText(/33\.3%/)).toBeInTheDocument();
expect(
screen.queryByText(/at or below the estimated false-positive floor/)
).not.toBeInTheDocument();
});
it('discloses the false-positive floor and withholds a sub-0.1% scoped share', async () => {
const mockStats: StatisticsResponse = {
busiest_channels_24h: [],
contact_count: 0,
repeater_count: 0,
channel_count: 0,
total_packets: 0,
decrypted_packets: 0,
undecrypted_packets: 0,
total_dms: 0,
total_channel_messages: 0,
total_outgoing: 0,
contacts_heard: { last_hour: 0, last_24_hours: 0, last_week: 0 },
repeaters_heard: { last_hour: 0, last_24_hours: 0, last_week: 0 },
known_channels_active: { last_hour: 0, last_24_hours: 0, last_week: 0 },
path_hash_width_24h: {
total_packets: 0,
single_byte: 0,
double_byte: 0,
triple_byte: 0,
single_byte_pct: 0,
double_byte_pct: 0,
triple_byte_pct: 0,
},
// Mirrors real-world data: 70 "scoped" packets against a measured floor of
// 60 is corrupt-capture noise, not adoption.
region_scope_24h: {
total_messages: 391757,
scoped_messages: 70,
scoped_pct: 0.0179,
false_positive_floor: 60.3,
total_senders: 117,
scoped_senders: 3,
scoped_senders_pct: 2.56,
},
packets_per_hour_72h: [],
noise_floor_24h: {
sample_interval_seconds: 60,
coverage_seconds: 0,
latest_noise_floor_dbm: null,
latest_timestamp: null,
samples: [],
},
};
vi.spyOn(globalThis, 'fetch').mockResolvedValue(
new Response(JSON.stringify(mockStats), {
status: 200,
headers: { 'Content-Type': 'application/json' },
})
);
renderModal({ externalSidebarNav: true, desktopSection: 'statistics' });
await waitFor(() => {
expect(screen.getByText('Region Scope (24h)')).toBeInTheDocument();
});
// 70 scoped sits just above the 60.3 floor, so most of it is corrupt captures
expect(screen.getByText(/Includes an estimated 60 false positives/)).toBeInTheDocument();
// 0.0179% would render as a meaningless "0.0%", so the share is withheld
expect(screen.queryByText(/0\.0%/)).not.toBeInTheDocument();
expect(screen.getByText(/70 of 391,757/)).toBeInTheDocument();
// ...but the decryption-backed sender figure still stands
expect(screen.getByText(/3 of 117/)).toBeInTheDocument();
expect(screen.getByText(/2\.6%/)).toBeInTheDocument();
});
it('reports scoped traffic as noise when it is at or below the floor', async () => {
const mockStats: StatisticsResponse = {
busiest_channels_24h: [],
contact_count: 0,
repeater_count: 0,
channel_count: 0,
total_packets: 0,
decrypted_packets: 0,
undecrypted_packets: 0,
total_dms: 0,
total_channel_messages: 0,
total_outgoing: 0,
contacts_heard: { last_hour: 0, last_24_hours: 0, last_week: 0 },
repeaters_heard: { last_hour: 0, last_24_hours: 0, last_week: 0 },
known_channels_active: { last_hour: 0, last_24_hours: 0, last_week: 0 },
path_hash_width_24h: {
total_packets: 0,
single_byte: 0,
double_byte: 0,
triple_byte: 0,
single_byte_pct: 0,
double_byte_pct: 0,
triple_byte_pct: 0,
},
region_scope_24h: {
total_messages: 5000,
scoped_messages: 12,
scoped_pct: 0.24,
false_positive_floor: 20,
total_senders: 40,
scoped_senders: 0,
scoped_senders_pct: 0,
},
packets_per_hour_72h: [],
noise_floor_24h: {
sample_interval_seconds: 60,
coverage_seconds: 0,
latest_noise_floor_dbm: null,
latest_timestamp: null,
samples: [],
},
};
vi.spyOn(globalThis, 'fetch').mockResolvedValue(
new Response(JSON.stringify(mockStats), {
status: 200,
headers: { 'Content-Type': 'application/json' },
})
);
renderModal({ externalSidebarNav: true, desktopSection: 'statistics' });
await waitFor(() => {
expect(screen.getByText('Region Scope (24h)')).toBeInTheDocument();
});
expect(
screen.getByText(/at or below the estimated false-positive floor \(20\)/)
).toBeInTheDocument();
// Percentage withheld even though 0.24% would round visibly — it is noise
expect(screen.queryByText(/0\.2%/)).not.toBeInTheDocument();
});
it('fetches statistics when expanded in mobile external-nav mode', async () => {
@@ -850,6 +999,15 @@ describe('SettingsModal', () => {
double_byte_pct: 30,
triple_byte_pct: 20,
},
region_scope_24h: {
total_messages: 0,
scoped_messages: 0,
scoped_pct: 0,
false_positive_floor: 0,
total_senders: 0,
scoped_senders: 0,
scoped_senders_pct: 0,
},
packets_per_hour_72h: [],
noise_floor_24h: {
sample_interval_seconds: 60,
+19
View File
@@ -674,6 +674,24 @@ interface PacketsPerHourBucket {
count: number;
}
/**
* Regional flood-scope adoption over the last 24h. Two views with different
* denominators that will not agree traffic spans all channels including
* undecryptable ones (so it carries a false-positive floor from corrupt RF
* captures), while senders requires decryption and is therefore noise-free but
* limited to channels we hold keys for.
*/
export interface RegionScopeStats {
total_messages: number;
scoped_messages: number;
scoped_pct: number;
/** Estimated false positives in scoped_messages. At or below this = not adoption. */
false_positive_floor: number;
total_senders: number;
scoped_senders: number;
scoped_senders_pct: number;
}
export interface StatisticsResponse {
busiest_channels_24h: BusyChannel[];
contact_count: number;
@@ -697,6 +715,7 @@ export interface StatisticsResponse {
double_byte_pct: number;
triple_byte_pct: number;
};
region_scope_24h: RegionScopeStats;
packets_per_hour_72h: PacketsPerHourBucket[];
noise_floor_24h: NoiseFloorHistoryStats;
}
+192 -2
View File
@@ -43,6 +43,15 @@ class TestStatisticsEmpty:
"double_byte_pct": 0.0,
"triple_byte_pct": 0.0,
}
assert result["region_scope_24h"] == {
"total_messages": 0,
"scoped_messages": 0,
"scoped_pct": 0.0,
"false_positive_floor": 0.0,
"total_senders": 0,
"scoped_senders": 0,
"scoped_senders_pct": 0.0,
}
assert result["packets_per_hour_72h"] == []
@@ -378,10 +387,18 @@ class TestPathHashWidthStats:
hash_size = hash_sizes.get(raw_packet)
if hash_size is None:
return None
return SimpleNamespace(hash_size=hash_size)
# The scan is shared with the region-scope bucketer, so the stub has
# to carry the envelope fields that one reads too. Plain flood
# GroupText keeps it out of the region counters' way.
return SimpleNamespace(
hash_size=hash_size,
route_type=0x01,
payload_type=0x05,
transport_codes=None,
)
with patch("app.path_utils.parse_packet_envelope", side_effect=fake_parse):
breakdown = await StatisticsRepository._path_hash_width_24h()
breakdown, _region_scope = await StatisticsRepository._packet_shape_24h()
assert breakdown["total_packets"] == 3
assert breakdown["single_byte"] == 1
@@ -389,6 +406,179 @@ class TestPathHashWidthStats:
assert breakdown["triple_byte"] == 1
class TestRegionScopeStats:
"""Regional flood-scope adoption counters.
Packet fixtures are built by hand so the header bits are explicit:
header = (payload_type << 2) | route_type, then a 4-byte transport-code
block for TRANSPORT_* routes, then the packed path byte, then payload.
"""
# GROUP_TEXT (0x05) flood (0x01) -> header 0x15; path byte 0x00 = 0 hops, 1-byte hashes
UNSCOPED_GROUP_TEXT = bytes.fromhex("1500AA")
# GROUP_TEXT (0x05) transport-flood (0x00) -> header 0x14, + codes AABB/0000
SCOPED_GROUP_TEXT = bytes.fromhex("14AABB000000AA")
# Undefined payload type 0x0C, transport-flood -> header 0x30. Corrupt by
# definition, so it feeds the false-positive floor rather than the counts.
SCOPED_UNDEFINED_TYPE = bytes.fromhex("30AABB000000AA")
# GROUP_TEXT direct (0x02) -> header 0x16. Direct sends can never be scoped.
DIRECT_GROUP_TEXT = bytes.fromhex("1600AA")
async def _insert_packet(self, conn, data: bytes, tag: bytes, timestamp: int):
await conn.execute(
"INSERT INTO raw_packets (timestamp, data, payload_hash) VALUES (?, ?, ?)",
(timestamp, data, tag * 32),
)
@pytest.mark.asyncio
async def test_counts_scoped_flood_group_text_only(self, test_db):
"""Only flood-routed GroupText packets count; direct sends are excluded."""
now = int(time.time())
conn = test_db.conn
await self._insert_packet(conn, self.SCOPED_GROUP_TEXT, b"\x11", now)
await self._insert_packet(conn, self.UNSCOPED_GROUP_TEXT, b"\x22", now)
await self._insert_packet(conn, self.DIRECT_GROUP_TEXT, b"\x33", now)
# Outside the 24h window
await self._insert_packet(conn, self.SCOPED_GROUP_TEXT, b"\x44", now - 90000)
await conn.commit()
stats = (await StatisticsRepository.get_all())["region_scope_24h"]
# Direct + stale packets excluded, so 2 in the denominator
assert stats["total_messages"] == 2
assert stats["scoped_messages"] == 1
assert stats["scoped_pct"] == pytest.approx(50.0)
@pytest.mark.asyncio
async def test_undefined_payload_types_feed_false_positive_floor(self, test_db):
"""Corrupt packets claiming an undefined type estimate the noise floor."""
now = int(time.time())
conn = test_db.conn
await self._insert_packet(conn, self.UNSCOPED_GROUP_TEXT, b"\x11", now)
# Three corrupt transport-routed packets across the undefined-type buckets
for i, header in enumerate(("30", "34", "38")): # types 0x0C, 0x0D, 0x0E
await self._insert_packet(
conn, bytes.fromhex(f"{header}AABB000000AA"), bytes([0x20 + i]), now
)
await conn.commit()
stats = (await StatisticsRepository.get_all())["region_scope_24h"]
# Garbage must not inflate the real counts...
assert stats["total_messages"] == 1
assert stats["scoped_messages"] == 0
# ...but should surface as the floor: 3 packets over 3 undefined buckets
assert stats["false_positive_floor"] == pytest.approx(1.0)
@pytest.mark.asyncio
async def test_counts_distinct_senders_who_scoped(self, test_db):
"""Sender adoption is per distinct sender, not per message."""
now = int(time.time())
conn = test_db.conn
# One chatty scoping sender, one quiet scoping sender, one unscoped sender.
# Traffic share would read 4/5; sender share should read 2/3.
rows = [
("alice_key", "Alice", 0xAABB),
("alice_key", "Alice", 0xAABB),
("alice_key", "Alice", 0xAABB),
("bob_key", "Bob", 0xCCDD),
("carol_key", "Carol", None),
]
for i, (sender_key, sender_name, code) in enumerate(rows):
await conn.execute(
"""INSERT INTO messages
(type, conversation_key, text, received_at, outgoing,
sender_key, sender_name, transport_code)
VALUES ('CHAN', ?, ?, ?, 0, ?, ?, ?)""",
("EE" * 16, f"msg{i}", now, sender_key, sender_name, code),
)
await conn.commit()
stats = (await StatisticsRepository.get_all())["region_scope_24h"]
assert stats["total_senders"] == 3
assert stats["scoped_senders"] == 2
assert stats["scoped_senders_pct"] == pytest.approx(200 / 3, rel=1e-3)
@pytest.mark.asyncio
async def test_sender_scoping_falls_back_to_linked_raw_packet(self, test_db):
"""Rows predating region tagging are resolved via their retained packet."""
now = int(time.time())
conn = test_db.conn
async with conn.execute(
"""INSERT INTO messages
(type, conversation_key, text, received_at, outgoing, sender_key, sender_name)
VALUES ('CHAN', ?, 'legacy', ?, 0, 'dave_key', 'Dave')""",
("EE" * 16, now),
) as cursor:
message_id = cursor.lastrowid
# transport_code is NULL, but the linked packet still shows the scoping
await conn.execute(
"""INSERT INTO raw_packets (timestamp, data, payload_hash, message_id)
VALUES (?, ?, ?, ?)""",
(now, self.SCOPED_GROUP_TEXT, b"\x99" * 32, message_id),
)
await conn.commit()
stats = (await StatisticsRepository.get_all())["region_scope_24h"]
assert stats["total_senders"] == 1
assert stats["scoped_senders"] == 1
@pytest.mark.asyncio
async def test_senders_fall_back_to_name_without_resolved_key(self, test_db):
"""sender_key is only ~69% resolved, so name is the fallback identity."""
now = int(time.time())
conn = test_db.conn
for i, code in enumerate((0xAABB, None)):
await conn.execute(
"""INSERT INTO messages
(type, conversation_key, text, received_at, outgoing,
sender_key, sender_name, transport_code)
VALUES ('CHAN', ?, ?, ?, 0, NULL, ?, ?)""",
("EE" * 16, f"msg{i}", now, f"NamedOnly{i}", code),
)
# No identity at all -> not counted in either side of the fraction
await conn.execute(
"""INSERT INTO messages
(type, conversation_key, text, received_at, outgoing, sender_key, sender_name)
VALUES ('CHAN', ?, 'anon', ?, 0, NULL, NULL)""",
("EE" * 16, now),
)
await conn.commit()
stats = (await StatisticsRepository.get_all())["region_scope_24h"]
assert stats["total_senders"] == 2
assert stats["scoped_senders"] == 1
@pytest.mark.asyncio
async def test_outgoing_messages_excluded_from_sender_counts(self, test_db):
"""Our own sends are not evidence of anyone else's adoption."""
now = int(time.time())
conn = test_db.conn
await conn.execute(
"""INSERT INTO messages
(type, conversation_key, text, received_at, outgoing,
sender_key, sender_name, transport_code)
VALUES ('CHAN', ?, 'ours', ?, 1, 'me_key', 'Me', ?)""",
("EE" * 16, now, 0xAABB),
)
await conn.commit()
stats = (await StatisticsRepository.get_all())["region_scope_24h"]
assert stats["total_senders"] == 0
assert stats["scoped_senders"] == 0
assert stats["scoped_senders_pct"] == 0.0
class TestPacketsPerHour:
@pytest.mark.asyncio
async def test_buckets_packets_by_hour(self, test_db):