diff --git a/AGENTS.md b/AGENTS.md index 64ef5d3..652e217 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -393,7 +393,7 @@ All endpoints are prefixed with `/api` (e.g., `/api/health`). | PATCH | `/api/fanout/{id}` | Update fanout config (triggers module reload) | | DELETE | `/api/fanout/{id}` | Delete fanout config (stops module) | | POST | `/api/fanout/bots/disable-until-restart` | Stop bot fanout modules and keep bots disabled until the process restarts | -| GET | `/api/statistics` | Aggregated mesh network statistics | +| GET | `/api/statistics` | Aggregated mesh network statistics, including `region_scope_24h` regional flood-scope adoption | | GET | `/api/push/vapid-public-key` | VAPID public key for browser push subscription | | POST | `/api/push/subscribe` | Register/upsert a push subscription | | GET | `/api/push/subscriptions` | List all push subscriptions | diff --git a/app/AGENTS.md b/app/AGENTS.md index 46e8747..80d15c7 100644 --- a/app/AGENTS.md +++ b/app/AGENTS.md @@ -155,6 +155,17 @@ app/ - Candidate region names come from `app_settings.known_regions` (user-editable, seeded by migration 063 from `flood_scope` + channel `flood_scope_override`). - Channel messages persist `messages.transport_code` (uint16, NULL = unscoped plain flood) and `messages.region` (resolved name, NULL = scoped but no list match) at ingest, so the chat region badge survives raw-packet purge. The packet inspector (`GET /packets/{id}` and the `raw_packet` WS broadcast) resolves region on the fly against the current list since it still holds the raw payload. +### Region-scope adoption stats (`region_scope_24h`) + +`GET /statistics` reports regional flood-scope uptake as two views with different denominators that intentionally will not agree: + +- **Traffic** (`bucket_region_scope` in `path_utils.py`) counts flood-routed (`route_type` 0/1) GroupText packets across all channels, including undecryptable ones. Zero-hop/direct sends are excluded because firmware reaches them through the non-transport `sendZeroHop`/`sendDirect` overloads and they can never carry transport codes. +- **Senders** (`StatisticsRepository._region_scope_senders_24h`) counts distinct senders with at least one scoped message. Attribution requires decryption, so it only covers channels we hold keys for — narrower, but self-validating (a decrypted packet is provably not a corrupt capture) and immune to one chatty node skewing the result. Identity is `sender_key` falling back to `sender_name`; scoping reads `messages.transport_code`, falling back to the linked raw packet for rows stored before region tagging existed. + +`false_positive_floor` exists because corrupt RF captures land in `raw_packets` with effectively random headers and a share of them claim `TRANSPORT_FLOOD`. That garbage spreads near-uniformly across payload-type buckets, so it is measured directly from payload types the protocol does not define (`0x0C`/`0x0D`/`0x0E`) and averaged per bucket. **A `scoped_messages` count at or below the floor is not evidence of adoption**; surface the two together and never show the percentage alone. Do not "fix" the floor by removing it — without it the metric reads several times higher than reality. + +Both traffic buckets come from one 24h raw-packet scan (`_packet_shape_24h`) shared with `path_hash_width_24h`, so adding region stats costs no extra query or parse pass. + ### Raw packet dedup policy - Raw packet storage deduplicates by payload hash (`RawPacketRepository.create`), excluding routing/path bytes. @@ -297,7 +308,7 @@ Web Push is a standalone subsystem in `app/push/`, separate from the fanout modu - `POST /fanout/bots/disable-until-restart` — stop bot modules and keep bots disabled until restart ### Statistics -- `GET /statistics` — aggregated mesh network stats (entity counts, message/packet splits, activity windows, busiest channels) +- `GET /statistics` — aggregated mesh network stats (entity counts, message/packet splits, activity windows, busiest channels, `region_scope_24h` regional adoption) ### Push - `GET /push/vapid-public-key` — VAPID public key for browser `PushManager.subscribe()` diff --git a/app/models.py b/app/models.py index 263077e..6507682 100644 --- a/app/models.py +++ b/app/models.py @@ -1083,6 +1083,41 @@ class PacketsPerHourBucket(BaseModel): count: int = Field(description="Number of packets received in that hour") +class RegionScopeStats(BaseModel): + """Regional flood-scope adoption over the last 24 hours. + + Two independent views, deliberately not merged — they have different + denominators and will not agree: + + - Traffic (``total_messages``/``scoped_messages``) counts flood-routed + channel-message packets across all channels, including ones we cannot + decrypt. Broad coverage, but corrupt RF captures contribute false + positives, hence ``false_positive_floor``. + - Senders (``total_senders``/``scoped_senders``) counts distinct message + senders, which requires decryption and so only covers channels we hold + keys for. Narrower, but noise-free and immune to one chatty node skewing + the result. + """ + + total_messages: int = Field( + description="Flood-routed channel-message packets heard in the last 24h (unique payloads)" + ) + scoped_messages: int = Field(description="Of those, how many carried a regional transport code") + scoped_pct: float + false_positive_floor: float = Field( + description=( + "Estimated false positives in scoped_messages, measured from transport-routed " + "packets claiming a payload type the protocol does not define. A scoped_messages " + "value at or below this is not evidence of regional adoption." + ) + ) + total_senders: int = Field( + description="Distinct channel-message senders in the last 24h (decryptable channels only)" + ) + scoped_senders: int = Field(description="Of those, how many sent at least one scoped message") + scoped_senders_pct: float + + class StatisticsResponse(BaseModel): busiest_channels_24h: list[BusyChannel] contact_count: int @@ -1098,6 +1133,7 @@ class StatisticsResponse(BaseModel): repeaters_heard: ContactActivityCounts known_channels_active: ContactActivityCounts path_hash_width_24h: PathHashWidthStats + region_scope_24h: RegionScopeStats packets_per_hour_72h: list[PacketsPerHourBucket] noise_floor_24h: NoiseFloorHistoryStats diff --git a/app/path_utils.py b/app/path_utils.py index e20eb39..be643ca 100644 --- a/app/path_utils.py +++ b/app/path_utils.py @@ -304,6 +304,69 @@ def bucket_path_hash_widths(rows: Iterable) -> dict[str, int | float]: } +# Payload types with no meaning in the MeshCore protocol. Any packet claiming one +# is corrupt by definition, which makes them a usable gauge for how much RF garbage +# is sitting in a given route-type bucket. See bucket_region_scope(). +UNDEFINED_PAYLOAD_TYPES = (0x0C, 0x0D, 0x0E) + +_PAYLOAD_TYPE_GROUP_TEXT = 0x05 +_FLOOD_ROUTE_TYPES = (0x00, 0x01) # TRANSPORT_FLOOD, FLOOD + + +def bucket_region_scope(rows: Iterable) -> dict[str, int | float]: + """Count flood-routed channel messages carrying a regional transport code. + + *rows* must be an already-fetched list whose elements have a ``data`` column + containing raw packet bytes. + + Only flood-routed packets are counted. Zero-hop and direct sends can never + carry transport codes (firmware reaches them through the non-transport + ``sendZeroHop``/``sendDirect`` overloads), so including them would silently + dilute the percentage. + + ``false_positive_floor`` exists because corrupt RF captures still land in + ``raw_packets`` with effectively random header bytes, and a share of them + claim TRANSPORT_FLOOD. That garbage spreads near-uniformly across payload-type + buckets, so we measure it directly: count transport-routed packets claiming a + payload type the protocol does not define, and average per bucket. A + ``scoped_messages`` count at or below ``false_positive_floor`` is not evidence + of regional adoption, and callers should present the two together. + """ + total = 0 + scoped = 0 + undefined_scoped = 0 + + for row in rows: + envelope = parse_packet_envelope(bytes(row["data"])) + if envelope is None: + continue + + if ( + envelope.route_type == 0x00 + and envelope.payload_type in UNDEFINED_PAYLOAD_TYPES + and envelope.transport_codes is not None + ): + undefined_scoped += 1 + continue + + if envelope.payload_type != _PAYLOAD_TYPE_GROUP_TEXT: + continue + if envelope.route_type not in _FLOOD_ROUTE_TYPES: + continue + + total += 1 + if envelope.transport_codes is not None: + scoped += 1 + + noise_floor = undefined_scoped / len(UNDEFINED_PAYLOAD_TYPES) + return { + "total_messages": total, + "scoped_messages": scoped, + "scoped_pct": (scoped / total) * 100 if total else 0.0, + "false_positive_floor": noise_floor, + } + + def calculate_packet_hash(raw_bytes: bytes) -> str: """Calculate packet hash matching MeshCore's Packet::calculatePacketHash(). diff --git a/app/repository/settings.py b/app/repository/settings.py index 3cd6681..884faa8 100644 --- a/app/repository/settings.py +++ b/app/repository/settings.py @@ -7,7 +7,7 @@ import aiosqlite from app.database import db from app.models import AppSettings -from app.path_utils import bucket_path_hash_widths +from app.path_utils import bucket_path_hash_widths, bucket_region_scope, parse_packet_envelope from app.telemetry_interval import DEFAULT_TELEMETRY_INTERVAL_HOURS logger = logging.getLogger(__name__) @@ -513,8 +513,12 @@ class StatisticsRepository: return [{"timestamp": row["hour_ts"], "count": row["count"]} for row in rows] @staticmethod - async def _path_hash_width_24h() -> dict[str, int | float]: - """Count parsed raw packets from the last 24h by hop hash width.""" + async def _packet_shape_24h() -> tuple[dict[str, int | float], dict[str, int | float]]: + """Bucket the last 24h of raw packets by hop hash width and region scope. + + Both buckets come from one fetch so the snapshot only scans the packet + table once. Returns ``(path_hash_width, region_scope)``. + """ now = int(time.time()) async with db.readonly() as conn: async with conn.execute( @@ -522,7 +526,61 @@ class StatisticsRepository: (now - SECONDS_24H,), ) as cursor: rows = await cursor.fetchall() - return bucket_path_hash_widths(rows) + return bucket_path_hash_widths(rows), bucket_region_scope(rows) + + @staticmethod + async def _region_scope_senders_24h() -> dict[str, int | float]: + """Count distinct channel-message senders who scoped at least one send. + + Sender attribution requires having decrypted the message, so this only + covers channels we hold keys for — a narrower population than the + packet-level count, but a self-validating one: a message we decrypted is + provably not a corrupt capture, so this number needs no noise floor. + + Scoping is read from ``messages.transport_code`` where present, falling + back to the linked raw packet for rows stored before region tagging + existed. Senders are keyed by ``sender_key`` where resolved, falling back + to ``sender_name`` — one physical operator may run several nodes, hence + "senders" rather than "users". + """ + now = int(time.time()) + async with db.readonly() as conn: + async with conn.execute( + """ + SELECT m.id, m.sender_key, m.sender_name, m.transport_code, p.data + FROM messages m + LEFT JOIN raw_packets p ON p.message_id = m.id + WHERE m.type = 'CHAN' AND m.outgoing = 0 AND m.received_at >= ? + """, + (now - SECONDS_24H,), + ) as cursor: + rows = await cursor.fetchall() + + senders: set[str] = set() + scoped_senders: set[str] = set() + for row in rows: + identity = row["sender_key"] or row["sender_name"] + if not identity: + continue + senders.add(identity) + + if row["transport_code"] is not None: + scoped_senders.add(identity) + continue + raw = row["data"] + if raw is None: + continue + envelope = parse_packet_envelope(bytes(raw)) + if envelope is not None and envelope.transport_codes is not None: + scoped_senders.add(identity) + + total = len(senders) + scoped = len(scoped_senders) + return { + "total_senders": total, + "scoped_senders": scoped, + "scoped_senders_pct": (scoped / total) * 100 if total else 0.0, + } @staticmethod async def get_all() -> dict: @@ -600,7 +658,8 @@ class StatisticsRepository: contacts_heard = await StatisticsRepository._activity_counts(contact_type=2, exclude=True) repeaters_heard = await StatisticsRepository._activity_counts(contact_type=2) known_channels_active = await StatisticsRepository._known_channels_active() - path_hash_width_24h = await StatisticsRepository._path_hash_width_24h() + path_hash_width_24h, region_scope = await StatisticsRepository._packet_shape_24h() + region_scope.update(await StatisticsRepository._region_scope_senders_24h()) packets_per_hour_72h = await StatisticsRepository._packets_per_hour_72h() return { @@ -618,5 +677,6 @@ class StatisticsRepository: "repeaters_heard": repeaters_heard, "known_channels_active": known_channels_active, "path_hash_width_24h": path_hash_width_24h, + "region_scope_24h": region_scope, "packets_per_hour_72h": packets_per_hour_72h, } diff --git a/frontend/AGENTS.md b/frontend/AGENTS.md index 9524839..1f136a8 100644 --- a/frontend/AGENTS.md +++ b/frontend/AGENTS.md @@ -152,7 +152,7 @@ frontend/src/ │ │ ├── SettingsFanoutSection.tsx # Fanout integrations: MQTT, bots, config CRUD │ │ ├── SettingsRadioAppSection.tsx # Radio-App Management: tracked telemetry, contact management, blocked lists │ │ ├── SettingsDatabaseSection.tsx # Database: DB size, storage cleanup, auto-decrypt -│ │ ├── SettingsStatisticsSection.tsx # Read-only mesh network stats +│ │ ├── SettingsStatisticsSection.tsx # Read-only mesh network stats (incl. region-scope adoption) │ │ ├── SettingsAboutSection.tsx # Version, author, license, links │ │ ├── ThemeSelector.tsx # Color theme picker │ │ └── BulkDeleteContactsModal.tsx # Bulk contact deletion dialog @@ -479,6 +479,15 @@ Key conventions documented in the reference: - **Badges/tags** use `text-[0.625rem] uppercase tracking-wider px-1.5 py-0.5 rounded` with `bg-muted` (neutral) or `bg-primary/10` (active). - **Clickable text** (copy-to-clipboard, navigational links) uses `role="button" tabIndex={0}` with `cursor-pointer hover:text-primary transition-colors`. +### Region-scope adoption panel + +`SettingsStatisticsSection.tsx` renders `stats.region_scope_24h` via `RegionScopeStatsPanel`. Two presentation rules exist because regional adoption is currently very sparse, and both are deliberate: + +- **Fractions, not bare percentages.** "3 of 117" carries the sample size that "2.6%" hides. +- **The traffic percentage is withheld** when the scoped count is at or below `false_positive_floor` (corrupt-capture noise) or when the share would round to `0.0%`. The floor caveat is always shown alongside a non-zero scoped count. The sender figure is never suppressed — it requires successful decryption and so carries no noise. + +Traffic and sender figures use different denominators (all channels vs. decryptable-only) and are not expected to match. + ## Security Posture (intentional) - No authentication UI. diff --git a/frontend/src/components/settings/SettingsStatisticsSection.tsx b/frontend/src/components/settings/SettingsStatisticsSection.tsx index 76b35ae..69a2e13 100644 --- a/frontend/src/components/settings/SettingsStatisticsSection.tsx +++ b/frontend/src/components/settings/SettingsStatisticsSection.tsx @@ -13,12 +13,76 @@ import { } from 'recharts'; import { Separator } from '../ui/separator'; import { api } from '../../api'; -import type { StatisticsResponse } from '../../types'; +import type { RegionScopeStats, StatisticsResponse } from '../../types'; function formatPercent(value: number): string { return `${value.toFixed(1)}%`; } +/** + * Regional flood-scope adoption. Deliberately shows fractions rather than bare + * percentages: with adoption this sparse, "3 of 117" communicates the sample + * size that "2.6%" hides. + */ +function RegionScopeStatsPanel({ stats }: { stats: RegionScopeStats }) { + // Corrupt RF captures land in the packet table with random headers, some of + // which claim to be region-scoped. At or below the measured floor there is + // nothing to report but noise, so withhold the percentage and say so. + const floor = stats.false_positive_floor; + const withinNoise = stats.scoped_messages <= floor; + // Real-world scoping is currently rare enough that the share rounds to "0.0%", + // which reads as a broken widget. Below that resolution the fraction alone is + // the honest presentation. + const showTrafficPct = stats.total_messages > 0 && !withinNoise && stats.scoped_pct >= 0.05; + + return ( +
+

Region Scope (24h)

+

+ How much local traffic uses regional flood scoping. Traffic covers all channel messages + heard, including channels you have no key for; senders only counts channels you can decrypt, + so the two use different denominators and will not match. +

+
+
+ Scoped messages + + {stats.scoped_messages.toLocaleString()} of {stats.total_messages.toLocaleString()} + {showTrafficPct && ( + ({formatPercent(stats.scoped_pct)}) + )} + +
+
+ Senders using regions + + {stats.scoped_senders.toLocaleString()} of {stats.total_senders.toLocaleString()} + {stats.total_senders > 0 && ( + + {' '} + ({formatPercent(stats.scoped_senders_pct)}) + + )} + +
+
+ {floor > 0 && stats.scoped_messages > 0 && ( +

+ {withinNoise + ? `Scoped message count is at or below the estimated false-positive floor (${floor.toFixed(0)}) from corrupt packet captures, so it is not evidence of regional adoption.` + : `Includes an estimated ${floor.toFixed(0)} false positives from corrupt packet captures.`}{' '} + The sender count is unaffected — it requires successful decryption. +

+ )} + {stats.total_messages === 0 && ( +

+ No channel messages heard in the last 24 hours. +

+ )} +
+ ); +} + const CHANNEL_BAR_COLORS = ['#0ea5e9', '#10b981', '#f59e0b', '#f43f5e', '#8b5cf6']; const TOOLTIP_STYLE = { @@ -404,6 +468,11 @@ export function SettingsStatisticsSection({ className }: { className?: string }) )} + + + {/* Region Scope */} + + {/* Busiest Channels */} {stats.busiest_channels_24h.length > 0 && ( <> diff --git a/frontend/src/test/settingsModal.test.tsx b/frontend/src/test/settingsModal.test.tsx index 9867099..2d8e5f1 100644 --- a/frontend/src/test/settingsModal.test.tsx +++ b/frontend/src/test/settingsModal.test.tsx @@ -777,6 +777,15 @@ describe('SettingsModal', () => { double_byte_pct: 30, triple_byte_pct: 20, }, + region_scope_24h: { + total_messages: 120, + scoped_messages: 40, + scoped_pct: 33.3, + false_positive_floor: 2, + total_senders: 12, + scoped_senders: 3, + scoped_senders_pct: 25, + }, packets_per_hour_72h: [ { timestamp: 1711792800, count: 12 }, { timestamp: 1711796400, count: 8 }, @@ -824,6 +833,146 @@ describe('SettingsModal', () => { expect(screen.getByText('Known-channels active')).toBeInTheDocument(); expect(screen.getByText('Busiest Channels (24h)')).toBeInTheDocument(); expect(screen.getByText('Noise Floor (24h)')).toBeInTheDocument(); + expect(screen.getByText('Region Scope (24h)')).toBeInTheDocument(); + // Fractions, not bare percentages — the sample size matters at this sparsity + expect(screen.getByText(/40 of 120/)).toBeInTheDocument(); + expect(screen.getByText(/3 of 12/)).toBeInTheDocument(); + // 40 scoped is well above the floor of 2, so the percentage is shown + expect(screen.getByText(/33\.3%/)).toBeInTheDocument(); + expect( + screen.queryByText(/at or below the estimated false-positive floor/) + ).not.toBeInTheDocument(); + }); + + it('discloses the false-positive floor and withholds a sub-0.1% scoped share', async () => { + const mockStats: StatisticsResponse = { + busiest_channels_24h: [], + contact_count: 0, + repeater_count: 0, + channel_count: 0, + total_packets: 0, + decrypted_packets: 0, + undecrypted_packets: 0, + total_dms: 0, + total_channel_messages: 0, + total_outgoing: 0, + contacts_heard: { last_hour: 0, last_24_hours: 0, last_week: 0 }, + repeaters_heard: { last_hour: 0, last_24_hours: 0, last_week: 0 }, + known_channels_active: { last_hour: 0, last_24_hours: 0, last_week: 0 }, + path_hash_width_24h: { + total_packets: 0, + single_byte: 0, + double_byte: 0, + triple_byte: 0, + single_byte_pct: 0, + double_byte_pct: 0, + triple_byte_pct: 0, + }, + // Mirrors real-world data: 70 "scoped" packets against a measured floor of + // 60 is corrupt-capture noise, not adoption. + region_scope_24h: { + total_messages: 391757, + scoped_messages: 70, + scoped_pct: 0.0179, + false_positive_floor: 60.3, + total_senders: 117, + scoped_senders: 3, + scoped_senders_pct: 2.56, + }, + packets_per_hour_72h: [], + noise_floor_24h: { + sample_interval_seconds: 60, + coverage_seconds: 0, + latest_noise_floor_dbm: null, + latest_timestamp: null, + samples: [], + }, + }; + + vi.spyOn(globalThis, 'fetch').mockResolvedValue( + new Response(JSON.stringify(mockStats), { + status: 200, + headers: { 'Content-Type': 'application/json' }, + }) + ); + + renderModal({ externalSidebarNav: true, desktopSection: 'statistics' }); + + await waitFor(() => { + expect(screen.getByText('Region Scope (24h)')).toBeInTheDocument(); + }); + + // 70 scoped sits just above the 60.3 floor, so most of it is corrupt captures + expect(screen.getByText(/Includes an estimated 60 false positives/)).toBeInTheDocument(); + // 0.0179% would render as a meaningless "0.0%", so the share is withheld + expect(screen.queryByText(/0\.0%/)).not.toBeInTheDocument(); + expect(screen.getByText(/70 of 391,757/)).toBeInTheDocument(); + // ...but the decryption-backed sender figure still stands + expect(screen.getByText(/3 of 117/)).toBeInTheDocument(); + expect(screen.getByText(/2\.6%/)).toBeInTheDocument(); + }); + + it('reports scoped traffic as noise when it is at or below the floor', async () => { + const mockStats: StatisticsResponse = { + busiest_channels_24h: [], + contact_count: 0, + repeater_count: 0, + channel_count: 0, + total_packets: 0, + decrypted_packets: 0, + undecrypted_packets: 0, + total_dms: 0, + total_channel_messages: 0, + total_outgoing: 0, + contacts_heard: { last_hour: 0, last_24_hours: 0, last_week: 0 }, + repeaters_heard: { last_hour: 0, last_24_hours: 0, last_week: 0 }, + known_channels_active: { last_hour: 0, last_24_hours: 0, last_week: 0 }, + path_hash_width_24h: { + total_packets: 0, + single_byte: 0, + double_byte: 0, + triple_byte: 0, + single_byte_pct: 0, + double_byte_pct: 0, + triple_byte_pct: 0, + }, + region_scope_24h: { + total_messages: 5000, + scoped_messages: 12, + scoped_pct: 0.24, + false_positive_floor: 20, + total_senders: 40, + scoped_senders: 0, + scoped_senders_pct: 0, + }, + packets_per_hour_72h: [], + noise_floor_24h: { + sample_interval_seconds: 60, + coverage_seconds: 0, + latest_noise_floor_dbm: null, + latest_timestamp: null, + samples: [], + }, + }; + + vi.spyOn(globalThis, 'fetch').mockResolvedValue( + new Response(JSON.stringify(mockStats), { + status: 200, + headers: { 'Content-Type': 'application/json' }, + }) + ); + + renderModal({ externalSidebarNav: true, desktopSection: 'statistics' }); + + await waitFor(() => { + expect(screen.getByText('Region Scope (24h)')).toBeInTheDocument(); + }); + + expect( + screen.getByText(/at or below the estimated false-positive floor \(20\)/) + ).toBeInTheDocument(); + // Percentage withheld even though 0.24% would round visibly — it is noise + expect(screen.queryByText(/0\.2%/)).not.toBeInTheDocument(); }); it('fetches statistics when expanded in mobile external-nav mode', async () => { @@ -850,6 +999,15 @@ describe('SettingsModal', () => { double_byte_pct: 30, triple_byte_pct: 20, }, + region_scope_24h: { + total_messages: 0, + scoped_messages: 0, + scoped_pct: 0, + false_positive_floor: 0, + total_senders: 0, + scoped_senders: 0, + scoped_senders_pct: 0, + }, packets_per_hour_72h: [], noise_floor_24h: { sample_interval_seconds: 60, diff --git a/frontend/src/types.ts b/frontend/src/types.ts index ea258b5..fe71d00 100644 --- a/frontend/src/types.ts +++ b/frontend/src/types.ts @@ -674,6 +674,24 @@ interface PacketsPerHourBucket { count: number; } +/** + * Regional flood-scope adoption over the last 24h. Two views with different + * denominators that will not agree — traffic spans all channels including + * undecryptable ones (so it carries a false-positive floor from corrupt RF + * captures), while senders requires decryption and is therefore noise-free but + * limited to channels we hold keys for. + */ +export interface RegionScopeStats { + total_messages: number; + scoped_messages: number; + scoped_pct: number; + /** Estimated false positives in scoped_messages. At or below this = not adoption. */ + false_positive_floor: number; + total_senders: number; + scoped_senders: number; + scoped_senders_pct: number; +} + export interface StatisticsResponse { busiest_channels_24h: BusyChannel[]; contact_count: number; @@ -697,6 +715,7 @@ export interface StatisticsResponse { double_byte_pct: number; triple_byte_pct: number; }; + region_scope_24h: RegionScopeStats; packets_per_hour_72h: PacketsPerHourBucket[]; noise_floor_24h: NoiseFloorHistoryStats; } diff --git a/tests/test_statistics.py b/tests/test_statistics.py index 6db352e..6f88116 100644 --- a/tests/test_statistics.py +++ b/tests/test_statistics.py @@ -43,6 +43,15 @@ class TestStatisticsEmpty: "double_byte_pct": 0.0, "triple_byte_pct": 0.0, } + assert result["region_scope_24h"] == { + "total_messages": 0, + "scoped_messages": 0, + "scoped_pct": 0.0, + "false_positive_floor": 0.0, + "total_senders": 0, + "scoped_senders": 0, + "scoped_senders_pct": 0.0, + } assert result["packets_per_hour_72h"] == [] @@ -378,10 +387,18 @@ class TestPathHashWidthStats: hash_size = hash_sizes.get(raw_packet) if hash_size is None: return None - return SimpleNamespace(hash_size=hash_size) + # The scan is shared with the region-scope bucketer, so the stub has + # to carry the envelope fields that one reads too. Plain flood + # GroupText keeps it out of the region counters' way. + return SimpleNamespace( + hash_size=hash_size, + route_type=0x01, + payload_type=0x05, + transport_codes=None, + ) with patch("app.path_utils.parse_packet_envelope", side_effect=fake_parse): - breakdown = await StatisticsRepository._path_hash_width_24h() + breakdown, _region_scope = await StatisticsRepository._packet_shape_24h() assert breakdown["total_packets"] == 3 assert breakdown["single_byte"] == 1 @@ -389,6 +406,179 @@ class TestPathHashWidthStats: assert breakdown["triple_byte"] == 1 +class TestRegionScopeStats: + """Regional flood-scope adoption counters. + + Packet fixtures are built by hand so the header bits are explicit: + header = (payload_type << 2) | route_type, then a 4-byte transport-code + block for TRANSPORT_* routes, then the packed path byte, then payload. + """ + + # GROUP_TEXT (0x05) flood (0x01) -> header 0x15; path byte 0x00 = 0 hops, 1-byte hashes + UNSCOPED_GROUP_TEXT = bytes.fromhex("1500AA") + # GROUP_TEXT (0x05) transport-flood (0x00) -> header 0x14, + codes AABB/0000 + SCOPED_GROUP_TEXT = bytes.fromhex("14AABB000000AA") + # Undefined payload type 0x0C, transport-flood -> header 0x30. Corrupt by + # definition, so it feeds the false-positive floor rather than the counts. + SCOPED_UNDEFINED_TYPE = bytes.fromhex("30AABB000000AA") + # GROUP_TEXT direct (0x02) -> header 0x16. Direct sends can never be scoped. + DIRECT_GROUP_TEXT = bytes.fromhex("1600AA") + + async def _insert_packet(self, conn, data: bytes, tag: bytes, timestamp: int): + await conn.execute( + "INSERT INTO raw_packets (timestamp, data, payload_hash) VALUES (?, ?, ?)", + (timestamp, data, tag * 32), + ) + + @pytest.mark.asyncio + async def test_counts_scoped_flood_group_text_only(self, test_db): + """Only flood-routed GroupText packets count; direct sends are excluded.""" + now = int(time.time()) + conn = test_db.conn + + await self._insert_packet(conn, self.SCOPED_GROUP_TEXT, b"\x11", now) + await self._insert_packet(conn, self.UNSCOPED_GROUP_TEXT, b"\x22", now) + await self._insert_packet(conn, self.DIRECT_GROUP_TEXT, b"\x33", now) + # Outside the 24h window + await self._insert_packet(conn, self.SCOPED_GROUP_TEXT, b"\x44", now - 90000) + await conn.commit() + + stats = (await StatisticsRepository.get_all())["region_scope_24h"] + + # Direct + stale packets excluded, so 2 in the denominator + assert stats["total_messages"] == 2 + assert stats["scoped_messages"] == 1 + assert stats["scoped_pct"] == pytest.approx(50.0) + + @pytest.mark.asyncio + async def test_undefined_payload_types_feed_false_positive_floor(self, test_db): + """Corrupt packets claiming an undefined type estimate the noise floor.""" + now = int(time.time()) + conn = test_db.conn + + await self._insert_packet(conn, self.UNSCOPED_GROUP_TEXT, b"\x11", now) + # Three corrupt transport-routed packets across the undefined-type buckets + for i, header in enumerate(("30", "34", "38")): # types 0x0C, 0x0D, 0x0E + await self._insert_packet( + conn, bytes.fromhex(f"{header}AABB000000AA"), bytes([0x20 + i]), now + ) + await conn.commit() + + stats = (await StatisticsRepository.get_all())["region_scope_24h"] + + # Garbage must not inflate the real counts... + assert stats["total_messages"] == 1 + assert stats["scoped_messages"] == 0 + # ...but should surface as the floor: 3 packets over 3 undefined buckets + assert stats["false_positive_floor"] == pytest.approx(1.0) + + @pytest.mark.asyncio + async def test_counts_distinct_senders_who_scoped(self, test_db): + """Sender adoption is per distinct sender, not per message.""" + now = int(time.time()) + conn = test_db.conn + + # One chatty scoping sender, one quiet scoping sender, one unscoped sender. + # Traffic share would read 4/5; sender share should read 2/3. + rows = [ + ("alice_key", "Alice", 0xAABB), + ("alice_key", "Alice", 0xAABB), + ("alice_key", "Alice", 0xAABB), + ("bob_key", "Bob", 0xCCDD), + ("carol_key", "Carol", None), + ] + for i, (sender_key, sender_name, code) in enumerate(rows): + await conn.execute( + """INSERT INTO messages + (type, conversation_key, text, received_at, outgoing, + sender_key, sender_name, transport_code) + VALUES ('CHAN', ?, ?, ?, 0, ?, ?, ?)""", + ("EE" * 16, f"msg{i}", now, sender_key, sender_name, code), + ) + await conn.commit() + + stats = (await StatisticsRepository.get_all())["region_scope_24h"] + + assert stats["total_senders"] == 3 + assert stats["scoped_senders"] == 2 + assert stats["scoped_senders_pct"] == pytest.approx(200 / 3, rel=1e-3) + + @pytest.mark.asyncio + async def test_sender_scoping_falls_back_to_linked_raw_packet(self, test_db): + """Rows predating region tagging are resolved via their retained packet.""" + now = int(time.time()) + conn = test_db.conn + + async with conn.execute( + """INSERT INTO messages + (type, conversation_key, text, received_at, outgoing, sender_key, sender_name) + VALUES ('CHAN', ?, 'legacy', ?, 0, 'dave_key', 'Dave')""", + ("EE" * 16, now), + ) as cursor: + message_id = cursor.lastrowid + # transport_code is NULL, but the linked packet still shows the scoping + await conn.execute( + """INSERT INTO raw_packets (timestamp, data, payload_hash, message_id) + VALUES (?, ?, ?, ?)""", + (now, self.SCOPED_GROUP_TEXT, b"\x99" * 32, message_id), + ) + await conn.commit() + + stats = (await StatisticsRepository.get_all())["region_scope_24h"] + + assert stats["total_senders"] == 1 + assert stats["scoped_senders"] == 1 + + @pytest.mark.asyncio + async def test_senders_fall_back_to_name_without_resolved_key(self, test_db): + """sender_key is only ~69% resolved, so name is the fallback identity.""" + now = int(time.time()) + conn = test_db.conn + + for i, code in enumerate((0xAABB, None)): + await conn.execute( + """INSERT INTO messages + (type, conversation_key, text, received_at, outgoing, + sender_key, sender_name, transport_code) + VALUES ('CHAN', ?, ?, ?, 0, NULL, ?, ?)""", + ("EE" * 16, f"msg{i}", now, f"NamedOnly{i}", code), + ) + # No identity at all -> not counted in either side of the fraction + await conn.execute( + """INSERT INTO messages + (type, conversation_key, text, received_at, outgoing, sender_key, sender_name) + VALUES ('CHAN', ?, 'anon', ?, 0, NULL, NULL)""", + ("EE" * 16, now), + ) + await conn.commit() + + stats = (await StatisticsRepository.get_all())["region_scope_24h"] + + assert stats["total_senders"] == 2 + assert stats["scoped_senders"] == 1 + + @pytest.mark.asyncio + async def test_outgoing_messages_excluded_from_sender_counts(self, test_db): + """Our own sends are not evidence of anyone else's adoption.""" + now = int(time.time()) + conn = test_db.conn + + await conn.execute( + """INSERT INTO messages + (type, conversation_key, text, received_at, outgoing, + sender_key, sender_name, transport_code) + VALUES ('CHAN', ?, 'ours', ?, 1, 'me_key', 'Me', ?)""", + ("EE" * 16, now, 0xAABB), + ) + await conn.commit() + + stats = (await StatisticsRepository.get_all())["region_scope_24h"] + + assert stats["total_senders"] == 0 + assert stats["scoped_senders"] == 0 + assert stats["scoped_senders_pct"] == 0.0 + + class TestPacketsPerHour: @pytest.mark.asyncio async def test_buckets_packets_by_hour(self, test_db):