Offer multiple timing windows for repeater telemetry pickup. Closes #192.

This commit is contained in:
Jack Kingsman
2026-04-16 13:55:01 -07:00
parent 8efbbd97bd
commit 4b69ec4519
15 changed files with 1046 additions and 135 deletions
+2 -1
View File
@@ -108,7 +108,8 @@ CREATE TABLE IF NOT EXISTS app_settings (
blocked_names TEXT DEFAULT '[]',
discovery_blocked_types TEXT DEFAULT '[]',
tracked_telemetry_repeaters TEXT DEFAULT '[]',
auto_resend_channel INTEGER DEFAULT 0
auto_resend_channel INTEGER DEFAULT 0,
telemetry_interval_hours INTEGER DEFAULT 8
);
INSERT OR IGNORE INTO app_settings (id) VALUES (1);
@@ -0,0 +1,22 @@
import logging
import aiosqlite
logger = logging.getLogger(__name__)
async def migrate(conn: aiosqlite.Connection) -> None:
"""Add telemetry_interval_hours integer column to app_settings."""
tables_cursor = await conn.execute("SELECT name FROM sqlite_master WHERE type='table'")
if "app_settings" not in {row[0] for row in await tables_cursor.fetchall()}:
await conn.commit()
return
col_cursor = await conn.execute("PRAGMA table_info(app_settings)")
columns = {row[1] for row in await col_cursor.fetchall()}
if "telemetry_interval_hours" not in columns:
# Default to 8 hours, matching the previous hard-coded interval
# so existing users see no behavior change until they opt in.
await conn.execute(
"ALTER TABLE app_settings ADD COLUMN telemetry_interval_hours INTEGER DEFAULT 8"
)
await conn.commit()
+8
View File
@@ -842,6 +842,14 @@ class AppSettings(BaseModel):
default_factory=list,
description="Public keys of repeaters opted into periodic telemetry collection (max 8)",
)
telemetry_interval_hours: int = Field(
default=8,
description=(
"User-preferred telemetry collection interval in hours. The backend "
"clamps this up to the shortest legal interval given the number of "
"tracked repeaters so daily checks stay under a 24/day ceiling."
),
)
auto_resend_channel: bool = Field(
default=False,
description=(
+116 -57
View File
@@ -14,6 +14,7 @@ import logging
import math
import time
from contextlib import asynccontextmanager
from datetime import UTC, datetime, timedelta
from typing import Literal
from meshcore import EventType, MeshCore
@@ -36,6 +37,7 @@ from app.services.contact_reconciliation import (
)
from app.services.messages import create_fallback_channel_message
from app.services.radio_runtime import radio_runtime as radio_manager
from app.telemetry_interval import clamp_telemetry_interval
from app.websocket import broadcast_error, broadcast_event
logger = logging.getLogger(__name__)
@@ -159,10 +161,10 @@ MIN_ADVERT_INTERVAL = 3600
# Periodic telemetry collection task handle
_telemetry_collect_task: asyncio.Task | None = None
# Telemetry collection interval (8 hours)
TELEMETRY_COLLECT_INTERVAL = 8 * 3600
# Initial delay before the first telemetry collection cycle (let radio settle)
# Initial delay before the scheduler starts (let radio settle). After this,
# the loop wakes at each UTC top-of-hour and decides whether to run a cycle
# based on the user's telemetry_interval_hours preference, clamped up to
# the shortest-legal interval for the current tracked-repeater count.
TELEMETRY_COLLECT_INITIAL_DELAY = 60
# Counter to pause polling during repeater operations (supports nested pauses)
@@ -1656,62 +1658,122 @@ async def _collect_repeater_telemetry(mc: MeshCore, contact: Contact) -> bool:
return False
async def _run_telemetry_cycle() -> None:
"""Collect one telemetry sample from every tracked repeater."""
if not radio_manager.is_connected:
logger.debug("Telemetry collect: radio not connected, skipping cycle")
return
app_settings = await AppSettingsRepository.get()
tracked = app_settings.tracked_telemetry_repeaters
if not tracked:
return
logger.info("Telemetry collect: starting cycle for %d repeater(s)", len(tracked))
collected = 0
for pub_key in tracked:
contact = await ContactRepository.get_by_key(pub_key)
if not contact or contact.type != 2:
logger.debug(
"Telemetry collect: skipping %s (not found or not repeater)",
pub_key[:12],
)
continue
try:
async with radio_manager.radio_operation(
"telemetry_collect",
blocking=False,
suspend_auto_fetch=True,
) as mc:
if await _collect_repeater_telemetry(mc, contact):
collected += 1
except RadioOperationBusyError:
logger.debug(
"Telemetry collect: radio busy, skipping %s",
pub_key[:12],
)
logger.info(
"Telemetry collect: cycle complete, %d/%d successful",
collected,
len(tracked),
)
async def _sleep_until_next_utc_top_of_hour() -> None:
"""Sleep until the next UTC top-of-hour (or a minimum of 1 second)."""
now = datetime.now(UTC)
next_top = now.replace(minute=0, second=0, microsecond=0) + timedelta(hours=1)
delay = (next_top - now).total_seconds()
if delay < 1:
delay = 1
await asyncio.sleep(delay)
async def _maybe_run_scheduled_cycle(now: datetime) -> None:
"""Evaluate the modulo gate for the given UTC time and run a cycle if due.
Factored out of the loop so we can also invoke it immediately after the
post-boot initial delay — otherwise a restart within the initial-delay
window before a scheduled boundary would carry the task past that boundary
and skip a due cycle (for 24h cadence users, that's a full day of missed
telemetry).
"""
app_settings = await AppSettingsRepository.get()
tracked_count = len(app_settings.tracked_telemetry_repeaters)
if tracked_count == 0:
return
effective_hours = clamp_telemetry_interval(app_settings.telemetry_interval_hours, tracked_count)
if effective_hours <= 0:
return
if now.hour % effective_hours != 0:
return
await _run_telemetry_cycle()
async def _telemetry_collect_loop() -> None:
"""Background task that collects telemetry from tracked repeaters every 8 hours.
"""Background task that runs tracked-repeater telemetry collection.
Runs a first cycle after a short initial delay (so newly tracked repeaters
get a sample promptly), then sleeps the full interval between subsequent cycles.
After an initial post-boot delay we evaluate the modulo gate once
(covers the edge case where the initial delay crossed a scheduled
boundary on restart). Then we wake at every UTC top-of-hour and
evaluate the gate again. A cycle runs only when
``current_utc_hour % effective_interval_hours == 0``, where the
effective interval is the user preference clamped up to the shortest
legal interval for the current tracked-repeater count. This keeps the
total daily check count bounded at ``DAILY_CHECK_CEILING`` (24).
Acquires the radio lock per-repeater (non-blocking) so manual operations can
The loop never updates the stored user preference. If the user picks a
short interval and then adds repeaters that make it illegal, they keep
their pick stored and we silently use the clamped value until they drop
repeaters.
Radio lock is acquired per-repeater (non-blocking) so manual ops can
interleave. Failures are logged and skipped.
"""
first_run = True
try:
await asyncio.sleep(TELEMETRY_COLLECT_INITIAL_DELAY)
except asyncio.CancelledError:
logger.info("Telemetry collect task cancelled before initial delay")
return
# Post-boot boundary check: if the delay carried us into a matching hour
# (or we booted exactly at a matching hour), run now rather than waiting
# another full cycle.
try:
await _maybe_run_scheduled_cycle(datetime.now(UTC))
except asyncio.CancelledError:
logger.info("Telemetry collect task cancelled after initial delay")
return
except Exception as e:
logger.error("Error in post-boot telemetry check: %s", e, exc_info=True)
while True:
try:
delay = TELEMETRY_COLLECT_INITIAL_DELAY if first_run else TELEMETRY_COLLECT_INTERVAL
await asyncio.sleep(delay)
first_run = False
if not radio_manager.is_connected:
logger.debug("Telemetry collect: radio not connected, skipping cycle")
continue
app_settings = await AppSettingsRepository.get()
tracked = app_settings.tracked_telemetry_repeaters
if not tracked:
continue
logger.info("Telemetry collect: starting cycle for %d repeater(s)", len(tracked))
collected = 0
for pub_key in tracked:
contact = await ContactRepository.get_by_key(pub_key)
if not contact or contact.type != 2:
logger.debug(
"Telemetry collect: skipping %s (not found or not repeater)",
pub_key[:12],
)
continue
try:
async with radio_manager.radio_operation(
"telemetry_collect",
blocking=False,
suspend_auto_fetch=True,
) as mc:
if await _collect_repeater_telemetry(mc, contact):
collected += 1
except RadioOperationBusyError:
logger.debug(
"Telemetry collect: radio busy, skipping %s",
pub_key[:12],
)
logger.info(
"Telemetry collect: cycle complete, %d/%d successful",
collected,
len(tracked),
)
await _sleep_until_next_utc_top_of_hour()
await _maybe_run_scheduled_cycle(datetime.now(UTC))
except asyncio.CancelledError:
logger.info("Telemetry collect task cancelled")
@@ -1725,10 +1787,7 @@ def start_telemetry_collect() -> None:
global _telemetry_collect_task
if _telemetry_collect_task is None or _telemetry_collect_task.done():
_telemetry_collect_task = asyncio.create_task(_telemetry_collect_loop())
logger.info(
"Started periodic telemetry collection (interval: %ds)",
TELEMETRY_COLLECT_INTERVAL,
)
logger.info("Started periodic telemetry collection (UTC-hourly scheduler)")
async def stop_telemetry_collect() -> None:
+19 -1
View File
@@ -6,6 +6,7 @@ from typing import Any
from app.database import db
from app.models import AppSettings
from app.path_utils import bucket_path_hash_widths
from app.telemetry_interval import DEFAULT_TELEMETRY_INTERVAL_HOURS
logger = logging.getLogger(__name__)
@@ -30,7 +31,8 @@ class AppSettingsRepository:
last_message_times,
advert_interval, last_advert_time, flood_scope,
blocked_keys, blocked_names, discovery_blocked_types,
tracked_telemetry_repeaters, auto_resend_channel
tracked_telemetry_repeaters, auto_resend_channel,
telemetry_interval_hours
FROM app_settings WHERE id = 1
"""
)
@@ -91,6 +93,16 @@ class AppSettingsRepository:
except (KeyError, TypeError):
auto_resend_channel = False
# Parse telemetry_interval_hours (migration adds the column with
# default=8, but guard against older rows / partial migrations).
try:
raw_interval = row["telemetry_interval_hours"]
telemetry_interval_hours = (
int(raw_interval) if raw_interval is not None else DEFAULT_TELEMETRY_INTERVAL_HOURS
)
except (KeyError, TypeError, ValueError):
telemetry_interval_hours = DEFAULT_TELEMETRY_INTERVAL_HOURS
return AppSettings(
max_radio_contacts=row["max_radio_contacts"],
auto_decrypt_dm_on_advert=bool(row["auto_decrypt_dm_on_advert"]),
@@ -103,6 +115,7 @@ class AppSettingsRepository:
discovery_blocked_types=discovery_blocked_types,
tracked_telemetry_repeaters=tracked_telemetry_repeaters,
auto_resend_channel=auto_resend_channel,
telemetry_interval_hours=telemetry_interval_hours,
)
@staticmethod
@@ -118,6 +131,7 @@ class AppSettingsRepository:
discovery_blocked_types: list[int] | None = None,
tracked_telemetry_repeaters: list[str] | None = None,
auto_resend_channel: bool | None = None,
telemetry_interval_hours: int | None = None,
) -> AppSettings:
"""Update app settings. Only provided fields are updated."""
updates = []
@@ -167,6 +181,10 @@ class AppSettingsRepository:
updates.append("auto_resend_channel = ?")
params.append(1 if auto_resend_channel else 0)
if telemetry_interval_hours is not None:
updates.append("telemetry_interval_hours = ?")
params.append(telemetry_interval_hours)
if updates:
query = f"UPDATE app_settings SET {', '.join(updates)} WHERE id = 1"
await db.conn.execute(query, params)
+88
View File
@@ -8,6 +8,13 @@ from pydantic import BaseModel, Field
from app.models import CONTACT_TYPE_REPEATER, AppSettings
from app.region_scope import normalize_region_scope
from app.repository import AppSettingsRepository, ChannelRepository, ContactRepository
from app.telemetry_interval import (
DEFAULT_TELEMETRY_INTERVAL_HOURS,
TELEMETRY_INTERVAL_OPTIONS_HOURS,
clamp_telemetry_interval,
legal_interval_options,
next_run_timestamp_utc,
)
logger = logging.getLogger(__name__)
router = APIRouter(prefix="/settings", tags=["settings"])
@@ -57,6 +64,15 @@ class AppSettingsUpdate(BaseModel):
default=None,
description="Auto-resend channel messages once if no echo heard within 2 seconds",
)
telemetry_interval_hours: int | None = Field(
default=None,
description=(
"Preferred tracked-repeater telemetry interval in hours. "
f"Must be one of {list(TELEMETRY_INTERVAL_OPTIONS_HOURS)}. "
"Effective interval is clamped up to the shortest legal value "
"based on the current tracked-repeater count."
),
)
class BlockKeyRequest(BaseModel):
@@ -82,6 +98,29 @@ class TrackedTelemetryRequest(BaseModel):
public_key: str = Field(description="Public key of the repeater to toggle tracking")
class TelemetrySchedule(BaseModel):
"""Surface of telemetry scheduling derivations for the UI.
``preferred_hours`` is the stored user choice. ``effective_hours`` is the
value the scheduler actually uses (preferred, clamped up to the shortest
legal interval given the current tracked-repeater count). ``options``
lists the subset of the menu that is legal at the current count; the UI
should hide anything not in this list. ``next_run_at`` is the Unix
timestamp (seconds, UTC) of the next scheduled cycle, or ``None`` when
no repeaters are tracked (nothing to schedule).
"""
preferred_hours: int = Field(description="User's saved telemetry interval preference")
effective_hours: int = Field(description="Scheduler's clamped interval")
options: list[int] = Field(description="Legal interval choices at the current count")
tracked_count: int = Field(description="Number of repeaters currently tracked")
max_tracked: int = Field(description="Maximum number of repeaters that can be tracked")
next_run_at: int | None = Field(
default=None,
description="Unix timestamp (UTC seconds) of the next scheduled cycle",
)
class TrackedTelemetryResponse(BaseModel):
tracked_telemetry_repeaters: list[str] = Field(
description="Current list of tracked repeater public keys"
@@ -89,6 +128,24 @@ class TrackedTelemetryResponse(BaseModel):
names: dict[str, str] = Field(
description="Map of public key to display name for tracked repeaters"
)
schedule: TelemetrySchedule = Field(description="Current scheduling state")
def _build_schedule(tracked_count: int, preferred_hours: int | None) -> TelemetrySchedule:
pref = (
preferred_hours
if preferred_hours in TELEMETRY_INTERVAL_OPTIONS_HOURS
else DEFAULT_TELEMETRY_INTERVAL_HOURS
)
effective = clamp_telemetry_interval(pref, tracked_count)
return TelemetrySchedule(
preferred_hours=pref,
effective_hours=effective,
options=legal_interval_options(tracked_count),
tracked_count=tracked_count,
max_tracked=MAX_TRACKED_TELEMETRY_REPEATERS,
next_run_at=next_run_timestamp_utc(effective) if tracked_count > 0 else None,
)
@router.get("", response_model=AppSettings)
@@ -136,6 +193,20 @@ async def update_settings(update: AppSettingsUpdate) -> AppSettings:
if update.auto_resend_channel is not None:
kwargs["auto_resend_channel"] = update.auto_resend_channel
# Telemetry interval preference. Invalid values fall back to default
# rather than 400-ing so a stale client can't brick settings saves.
if update.telemetry_interval_hours is not None:
raw_interval = update.telemetry_interval_hours
if raw_interval not in TELEMETRY_INTERVAL_OPTIONS_HOURS:
logger.warning(
"telemetry_interval_hours=%r is not in the menu; defaulting to %d",
raw_interval,
DEFAULT_TELEMETRY_INTERVAL_HOURS,
)
raw_interval = DEFAULT_TELEMETRY_INTERVAL_HOURS
logger.info("Updating telemetry_interval_hours to %d", raw_interval)
kwargs["telemetry_interval_hours"] = raw_interval
# Flood scope
flood_scope_changed = False
if update.flood_scope is not None:
@@ -229,6 +300,7 @@ async def toggle_tracked_telemetry(request: TrackedTelemetryRequest) -> TrackedT
return TrackedTelemetryResponse(
tracked_telemetry_repeaters=new_list,
names=await _resolve_names(new_list),
schedule=_build_schedule(len(new_list), settings.telemetry_interval_hours),
)
# Validate it's a repeater
@@ -255,4 +327,20 @@ async def toggle_tracked_telemetry(request: TrackedTelemetryRequest) -> TrackedT
return TrackedTelemetryResponse(
tracked_telemetry_repeaters=new_list,
names=await _resolve_names(new_list),
schedule=_build_schedule(len(new_list), settings.telemetry_interval_hours),
)
@router.get("/tracked-telemetry/schedule", response_model=TelemetrySchedule)
async def get_telemetry_schedule() -> TelemetrySchedule:
"""Return the current telemetry scheduling derivation.
The UI uses this to render the interval dropdown (legal options),
surface saved-vs-effective when they differ, and show the next-run-at
timestamp so users know when the next cycle will fire.
"""
app_settings = await AppSettingsRepository.get()
return _build_schedule(
len(app_settings.tracked_telemetry_repeaters),
app_settings.telemetry_interval_hours,
)
+88
View File
@@ -0,0 +1,88 @@
"""Shared math for the tracked-repeater telemetry scheduler.
The app enforces a ceiling of 24 repeater status checks per 24 hours across
all tracked repeaters. With N repeaters tracked, the shortest legal interval
is ``24 // floor(24 / N)`` hours. Longer intervals (``12`` or ``24``) are
always legal at any N and are offered as user choices on top of the derived
shortest-legal value.
The user picks an interval via settings. The scheduler uses
``clamp_telemetry_interval`` to push that pick up to the shortest legal
interval if the user has added repeaters that invalidated their choice.
The stored preference is *not* mutated on clamp users get their pick back
if they later drop repeaters.
"""
from datetime import UTC, datetime
# Daily check budget: total number of repeater status checks we allow
# across all tracked repeaters per 24-hour window.
DAILY_CHECK_CEILING = 24
# Menu of interval values shown to users. The derivation-based options
# (1..8) are filtered per current repeater count via
# ``legal_interval_options``; 12 and 24 are always legal.
TELEMETRY_INTERVAL_OPTIONS_HOURS: tuple[int, ...] = (1, 2, 3, 4, 6, 8, 12, 24)
DEFAULT_TELEMETRY_INTERVAL_HOURS = 8
def shortest_legal_interval_hours(n_tracked: int) -> int:
"""Return the shortest interval (hours) that keeps under the daily ceiling.
With ``N`` repeaters, each full cycle costs ``N`` checks. We're capped at
``DAILY_CHECK_CEILING`` checks/day, so the maximum cycles/day is
``floor(24 / N)`` and the resulting interval is ``24 // cycles_per_day``.
For ``N == 0`` we return the default so the math still terminates, though
the scheduler skips empty-tracked cycles regardless.
"""
if n_tracked <= 0:
return DEFAULT_TELEMETRY_INTERVAL_HOURS
cycles_per_day = DAILY_CHECK_CEILING // n_tracked
if cycles_per_day <= 0:
# Would exceed ceiling even at 24h cadence; fall back to 24h.
return 24
return 24 // cycles_per_day
def clamp_telemetry_interval(preferred_hours: int, n_tracked: int) -> int:
"""Return the effective interval: max of user preference and shortest legal.
Unrecognized values fall back to the default.
"""
if preferred_hours not in TELEMETRY_INTERVAL_OPTIONS_HOURS:
preferred_hours = DEFAULT_TELEMETRY_INTERVAL_HOURS
shortest = shortest_legal_interval_hours(n_tracked)
return max(preferred_hours, shortest)
def legal_interval_options(n_tracked: int) -> list[int]:
"""Return the subset of the interval menu that is legal for a given N."""
shortest = shortest_legal_interval_hours(n_tracked)
return [h for h in TELEMETRY_INTERVAL_OPTIONS_HOURS if h >= shortest]
def next_run_timestamp_utc(effective_hours: int, now: datetime | None = None) -> int:
"""Return Unix timestamp for the next UTC top-of-hour where
``hour % effective_hours == 0``.
Returns the next matching hour strictly in the future (never ``now``
itself, even if ``now`` lies exactly on a matching boundary).
"""
if effective_hours <= 0:
effective_hours = DEFAULT_TELEMETRY_INTERVAL_HOURS
if now is None:
now = datetime.now(UTC)
else:
now = now.astimezone(UTC)
# Round up to the next top-of-hour, then skip forward until the modulo matches.
candidate = now.replace(minute=0, second=0, microsecond=0)
# Always move at least one hour forward so "now" never matches.
candidate = candidate.replace(hour=candidate.hour)
from datetime import timedelta
candidate = candidate + timedelta(hours=1)
while candidate.hour % effective_hours != 0:
candidate = candidate + timedelta(hours=1)
return int(candidate.timestamp())