feat(watchdog): catch sluggish-device failures via soft-pattern counting

The container watchdog only restarted on three legacy "device clearly dead"
log lines, so today's failure mode (firmware briefly stalls and get_stats_*
/ get_battery commands time out with an empty error while passive RX
keeps working) never tripped it — leaving the user with 10-15 s freezes
several times a day and no automatic recovery.

DeviceManager now tracks two liveness signals:
- _last_rx_at, bumped on every RX_LOG_DATA event
- _consecutive_stats_failures, incremented on get_stats_* / get_bat
  exceptions and cleared on success

New /health/strict endpoint exposes these to the watchdog. It returns 503
when the device is connected but has 5+ consecutive stats failures, or
when no RX event has been seen for over 5 minutes on a serial transport.
The cheap /health endpoint keeps its lenient behavior so Docker's
healthcheck doesn't suddenly start tripping.

The watchdog's check_device_unresponsive() gains a "soft" pattern class
with a count threshold of 5 in the last 2 minutes — matching against
get_stats_core/radio/packets failed:, Failed to get battery:, and
Failed to get channel. Hard patterns still trigger on a single hit.

Deploy note: the watchdog runs as a host-level systemd service and is
NOT restarted by mcupdate, so after deploy run:
  sudo systemctl restart mc-webui-watchdog.service

Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>

app/device_manager.py

@@ -170,6 +170,10 @@ class DeviceManager:
         self._ble_keepalive_task = None   # asyncio.Task for BLE keepalive
         self._ble_permanently_failed = False  # True when all reconnect attempts exhausted
 
+        # Liveness telemetry for /health/strict and the watchdog
+        self._last_rx_at: float = 0.0           # unix ts of last RX_LOG_DATA / event from device
+        self._consecutive_stats_failures: int = 0  # incremented on get_stats_* / get_bat failures
+
     @property
     def is_connected(self) -> bool:
         return self._connected and self.mc is not None
@@ -1116,6 +1120,7 @@ class DeviceManager:
         """
         try:
             import io
+            self._last_rx_at = time.time()
             data = getattr(event, 'payload', {})
             payload_hex = data.get('payload', '')
             logger.debug(f"RX_LOG_DATA received: {len(payload_hex)//2} bytes, snr={data.get('snr')}")
@@ -2500,8 +2505,10 @@ class DeviceManager:
         try:
             event = self.execute(self.mc.commands.get_bat(), timeout=5)
             if event and hasattr(event, 'data'):
+                self._consecutive_stats_failures = 0
                 return getattr(event, 'payload', {})
         except Exception as e:
+            self._consecutive_stats_failures += 1
             logger.error(f"Failed to get battery: {e}")
         return None
 
@@ -2511,10 +2518,12 @@ class DeviceManager:
             return {}
 
         stats = {}
+        any_success = False
         try:
             event = self.execute(self.mc.commands.get_stats_core(), timeout=5)
             if event and hasattr(event, 'payload'):
                 stats['core'] = event.payload
+                any_success = True
         except Exception as e:
             logger.debug(f"get_stats_core failed: {e}")
 
@@ -2522,6 +2531,7 @@ class DeviceManager:
             event = self.execute(self.mc.commands.get_stats_radio(), timeout=5)
             if event and hasattr(event, 'payload'):
                 stats['radio'] = event.payload
+                any_success = True
         except Exception as e:
             logger.debug(f"get_stats_radio failed: {e}")
 
@@ -2529,9 +2539,15 @@ class DeviceManager:
             event = self.execute(self.mc.commands.get_stats_packets(), timeout=5)
             if event and hasattr(event, 'payload'):
                 stats['packets'] = event.payload
+                any_success = True
         except Exception as e:
             logger.debug(f"get_stats_packets failed: {e}")
 
+        if any_success:
+            self._consecutive_stats_failures = 0
+        else:
+            self._consecutive_stats_failures += 1
+
         return stats
 
     def request_telemetry(self, contact_name: str) -> Optional[Dict]:

app/routes/views.py

@@ -2,14 +2,21 @@
 HTML views for mc-webui
 """
 
+import os
+import time
 import logging
-from flask import Blueprint, render_template, request
+from flask import Blueprint, render_template, request, jsonify
 from app.config import config, runtime_config
 
 logger = logging.getLogger(__name__)
 
 views_bp = Blueprint('views', __name__)
 
+# Thresholds for the strict health check (used by the external watchdog).
+# Kept as module constants so they can be tuned without code review.
+HEALTH_STRICT_MAX_RX_STALE_SEC = 300        # >5 min since last RX event → unhealthy
+HEALTH_STRICT_MAX_STATS_FAILURES = 5        # ≥5 consecutive get_stats/battery failures → unhealthy
+
 
 @views_bp.route('/')
 def index():
@@ -114,3 +121,56 @@ def health():
     if dm and getattr(dm, '_ble_permanently_failed', False):
         return 'BLE connection permanently failed', 503
     return 'OK', 200
+
+
+@views_bp.route('/health/strict')
+def health_strict():
+    """Stricter device-health check for the external watchdog.
+
+    Returns 503 when:
+      - BLE reconnection has permanently failed (same as /health), or
+      - The device is connected but has produced N consecutive stats/battery
+        failures (firmware/USB stalled), or
+      - The device is connected via USB and we haven't received any RX event
+        in HEALTH_STRICT_MAX_RX_STALE_SEC seconds.
+
+    The watchdog uses this to catch "sluggish" failures the regular /health
+    endpoint can't see. Returns 200 otherwise. Always returns JSON so the
+    caller can log the specific reason.
+    """
+    from flask import current_app
+    dm = getattr(current_app, 'device_manager', None)
+    if dm is None:
+        return jsonify({'status': 'ok', 'reason': 'no_device_manager'}), 200
+
+    if getattr(dm, '_ble_permanently_failed', False):
+        return jsonify({'status': 'fail', 'reason': 'ble_permanent_failure'}), 503
+
+    if not getattr(dm, 'is_connected', False):
+        # Don't fail strict on "not yet connected" — let DM keep retrying.
+        return jsonify({'status': 'ok', 'reason': 'not_connected'}), 200
+
+    failures = getattr(dm, '_consecutive_stats_failures', 0)
+    if failures >= HEALTH_STRICT_MAX_STATS_FAILURES:
+        return jsonify({
+            'status': 'fail',
+            'reason': 'consecutive_stats_failures',
+            'count': failures,
+        }), 503
+
+    transport = getattr(config, 'transport_type', 'serial')
+    last_rx = getattr(dm, '_last_rx_at', 0.0) or 0.0
+    if transport in ('serial', 'usb') and last_rx > 0:
+        stale = time.time() - last_rx
+        if stale > HEALTH_STRICT_MAX_RX_STALE_SEC:
+            return jsonify({
+                'status': 'fail',
+                'reason': 'rx_stale',
+                'seconds_since_last_rx': int(stale),
+            }), 503
+
+    return jsonify({
+        'status': 'ok',
+        'consecutive_stats_failures': failures,
+        'seconds_since_last_rx': int(time.time() - last_rx) if last_rx else None,
+    }), 200

scripts/watchdog/watchdog.py

@@ -469,23 +469,46 @@ def handle_unhealthy_container(container_name: str, status: dict):
 
 
 def check_device_unresponsive(container_name: str) -> bool:
-    """Check if the container logs indicate the USB device is unresponsive."""
+    """Check if the container logs indicate the USB device is unresponsive.
+
+    Two classes of patterns:
+      - HARD: any single occurrence triggers a restart. These are the
+        long-standing "device clearly dead" messages.
+      - SOFT: any of these failing >=5 times in the last 2 minutes triggers
+        a restart. Catches the "sluggish but not dead" mode (firmware
+        stalls on get_stats / get_bat commands while still answering
+        passive RX events).
+    """
     success, stdout, stderr = run_compose_command([
-        'logs', '--since', '1m', container_name
+        'logs', '--since', '2m', container_name
     ])
     if not success:
         return False
-        
-    error_patterns = [
+
+    hard_patterns = [
         "No response from meshcore node, disconnecting",
         "Device connected but self_info is empty",
-        "Failed to connect after 10 attempts"
+        "Failed to connect after 10 attempts",
     ]
-    
-    for pattern in error_patterns:
+
+    soft_patterns = [
+        "get_stats_core failed:",
+        "get_stats_radio failed:",
+        "get_stats_packets failed:",
+        "Failed to get battery:",
+        "Failed to get channel",
+    ]
+    SOFT_THRESHOLD = 5
+
+    for pattern in hard_patterns:
         if pattern in stdout:
             return True
-            
+
+    for pattern in soft_patterns:
+        if stdout.count(pattern) >= SOFT_THRESHOLD:
+            log(f"Soft-failure threshold tripped: '{pattern}' x{stdout.count(pattern)} in 2m", "WARN")
+            return True
+
     return False