diff --git a/app/device_manager.py b/app/device_manager.py index 3881b06..922a3da 100644 --- a/app/device_manager.py +++ b/app/device_manager.py @@ -170,6 +170,10 @@ class DeviceManager: self._ble_keepalive_task = None # asyncio.Task for BLE keepalive self._ble_permanently_failed = False # True when all reconnect attempts exhausted + # Liveness telemetry for /health/strict and the watchdog + self._last_rx_at: float = 0.0 # unix ts of last RX_LOG_DATA / event from device + self._consecutive_stats_failures: int = 0 # incremented on get_stats_* / get_bat failures + @property def is_connected(self) -> bool: return self._connected and self.mc is not None @@ -1116,6 +1120,7 @@ class DeviceManager: """ try: import io + self._last_rx_at = time.time() data = getattr(event, 'payload', {}) payload_hex = data.get('payload', '') logger.debug(f"RX_LOG_DATA received: {len(payload_hex)//2} bytes, snr={data.get('snr')}") @@ -2500,8 +2505,10 @@ class DeviceManager: try: event = self.execute(self.mc.commands.get_bat(), timeout=5) if event and hasattr(event, 'data'): + self._consecutive_stats_failures = 0 return getattr(event, 'payload', {}) except Exception as e: + self._consecutive_stats_failures += 1 logger.error(f"Failed to get battery: {e}") return None @@ -2511,10 +2518,12 @@ class DeviceManager: return {} stats = {} + any_success = False try: event = self.execute(self.mc.commands.get_stats_core(), timeout=5) if event and hasattr(event, 'payload'): stats['core'] = event.payload + any_success = True except Exception as e: logger.debug(f"get_stats_core failed: {e}") @@ -2522,6 +2531,7 @@ class DeviceManager: event = self.execute(self.mc.commands.get_stats_radio(), timeout=5) if event and hasattr(event, 'payload'): stats['radio'] = event.payload + any_success = True except Exception as e: logger.debug(f"get_stats_radio failed: {e}") @@ -2529,9 +2539,15 @@ class DeviceManager: event = self.execute(self.mc.commands.get_stats_packets(), timeout=5) if event and hasattr(event, 'payload'): stats['packets'] = event.payload + any_success = True except Exception as e: logger.debug(f"get_stats_packets failed: {e}") + if any_success: + self._consecutive_stats_failures = 0 + else: + self._consecutive_stats_failures += 1 + return stats def request_telemetry(self, contact_name: str) -> Optional[Dict]: diff --git a/app/routes/views.py b/app/routes/views.py index 18db095..343217c 100644 --- a/app/routes/views.py +++ b/app/routes/views.py @@ -2,14 +2,21 @@ HTML views for mc-webui """ +import os +import time import logging -from flask import Blueprint, render_template, request +from flask import Blueprint, render_template, request, jsonify from app.config import config, runtime_config logger = logging.getLogger(__name__) views_bp = Blueprint('views', __name__) +# Thresholds for the strict health check (used by the external watchdog). +# Kept as module constants so they can be tuned without code review. +HEALTH_STRICT_MAX_RX_STALE_SEC = 300 # >5 min since last RX event → unhealthy +HEALTH_STRICT_MAX_STATS_FAILURES = 5 # ≥5 consecutive get_stats/battery failures → unhealthy + @views_bp.route('/') def index(): @@ -114,3 +121,56 @@ def health(): if dm and getattr(dm, '_ble_permanently_failed', False): return 'BLE connection permanently failed', 503 return 'OK', 200 + + +@views_bp.route('/health/strict') +def health_strict(): + """Stricter device-health check for the external watchdog. + + Returns 503 when: + - BLE reconnection has permanently failed (same as /health), or + - The device is connected but has produced N consecutive stats/battery + failures (firmware/USB stalled), or + - The device is connected via USB and we haven't received any RX event + in HEALTH_STRICT_MAX_RX_STALE_SEC seconds. + + The watchdog uses this to catch "sluggish" failures the regular /health + endpoint can't see. Returns 200 otherwise. Always returns JSON so the + caller can log the specific reason. + """ + from flask import current_app + dm = getattr(current_app, 'device_manager', None) + if dm is None: + return jsonify({'status': 'ok', 'reason': 'no_device_manager'}), 200 + + if getattr(dm, '_ble_permanently_failed', False): + return jsonify({'status': 'fail', 'reason': 'ble_permanent_failure'}), 503 + + if not getattr(dm, 'is_connected', False): + # Don't fail strict on "not yet connected" — let DM keep retrying. + return jsonify({'status': 'ok', 'reason': 'not_connected'}), 200 + + failures = getattr(dm, '_consecutive_stats_failures', 0) + if failures >= HEALTH_STRICT_MAX_STATS_FAILURES: + return jsonify({ + 'status': 'fail', + 'reason': 'consecutive_stats_failures', + 'count': failures, + }), 503 + + transport = getattr(config, 'transport_type', 'serial') + last_rx = getattr(dm, '_last_rx_at', 0.0) or 0.0 + if transport in ('serial', 'usb') and last_rx > 0: + stale = time.time() - last_rx + if stale > HEALTH_STRICT_MAX_RX_STALE_SEC: + return jsonify({ + 'status': 'fail', + 'reason': 'rx_stale', + 'seconds_since_last_rx': int(stale), + }), 503 + + return jsonify({ + 'status': 'ok', + 'consecutive_stats_failures': failures, + 'seconds_since_last_rx': int(time.time() - last_rx) if last_rx else None, + }), 200 diff --git a/scripts/watchdog/watchdog.py b/scripts/watchdog/watchdog.py index 9f1575c..e233e0a 100755 --- a/scripts/watchdog/watchdog.py +++ b/scripts/watchdog/watchdog.py @@ -469,23 +469,46 @@ def handle_unhealthy_container(container_name: str, status: dict): def check_device_unresponsive(container_name: str) -> bool: - """Check if the container logs indicate the USB device is unresponsive.""" + """Check if the container logs indicate the USB device is unresponsive. + + Two classes of patterns: + - HARD: any single occurrence triggers a restart. These are the + long-standing "device clearly dead" messages. + - SOFT: any of these failing >=5 times in the last 2 minutes triggers + a restart. Catches the "sluggish but not dead" mode (firmware + stalls on get_stats / get_bat commands while still answering + passive RX events). + """ success, stdout, stderr = run_compose_command([ - 'logs', '--since', '1m', container_name + 'logs', '--since', '2m', container_name ]) if not success: return False - - error_patterns = [ + + hard_patterns = [ "No response from meshcore node, disconnecting", "Device connected but self_info is empty", - "Failed to connect after 10 attempts" + "Failed to connect after 10 attempts", ] - - for pattern in error_patterns: + + soft_patterns = [ + "get_stats_core failed:", + "get_stats_radio failed:", + "get_stats_packets failed:", + "Failed to get battery:", + "Failed to get channel", + ] + SOFT_THRESHOLD = 5 + + for pattern in hard_patterns: if pattern in stdout: return True - + + for pattern in soft_patterns: + if stdout.count(pattern) >= SOFT_THRESHOLD: + log(f"Soft-failure threshold tripped: '{pattern}' x{stdout.count(pattern)} in 2m", "WARN") + return True + return False