Rework more coverage in e2e tests and don't force radio restart + better startup error handling

app/config.py

@@ -48,6 +48,40 @@ class Settings(BaseSettings):
 settings = Settings()
 
 
+class _RepeatSquelch(logging.Filter):
+    """Suppress rapid-fire identical messages and emit a summary instead.
+
+    Attached to the ``meshcore`` library logger to catch its repeated
+    "Serial Connection started" lines that flood the log when another
+    process holds the serial port.
+    """
+
+    def __init__(self, threshold: int = 3) -> None:
+        super().__init__()
+        self._last_msg: str | None = None
+        self._repeat_count: int = 0
+        self._threshold = threshold
+
+    def filter(self, record: logging.LogRecord) -> bool:
+        msg = record.getMessage()
+        if msg == self._last_msg:
+            self._repeat_count += 1
+            if self._repeat_count == self._threshold:
+                record.msg = (
+                    "%s (repeated %d times — possible serial port contention from another process)"
+                )
+                record.args = (msg, self._repeat_count)
+                record.levelno = logging.WARNING
+                record.levelname = "WARNING"
+                return True
+            # Suppress further repeats beyond the threshold
+            return self._repeat_count < self._threshold
+        else:
+            self._last_msg = msg
+            self._repeat_count = 1
+            return True
+
+
 def setup_logging() -> None:
     """Configure logging for the application."""
     logging.basicConfig(
@@ -55,3 +89,6 @@ def setup_logging() -> None:
         format="%(asctime)s - %(name)s - %(levelname)s - %(message)s",
         datefmt="%Y-%m-%d %H:%M:%S",
     )
+    # Squelch repeated messages from the meshcore library (e.g. rapid-fire
+    # "Serial Connection started" when the port is contended).
+    logging.getLogger("meshcore").addFilter(_RepeatSquelch())

app/radio.py

@@ -470,6 +470,8 @@ class RadioManager:
             from app.websocket import broadcast_health
 
             CHECK_INTERVAL_SECONDS = 5
+            UNRESPONSIVE_THRESHOLD = 3
+            consecutive_setup_failures = 0
 
             while True:
                 try:
@@ -483,6 +485,7 @@ class RadioManager:
                         logger.warning("Radio connection lost, broadcasting status change")
                         broadcast_health(False, self._connection_info)
                         self._last_connected = False
+                        consecutive_setup_failures = 0
 
                     if not current_connected:
                         # Attempt reconnection on every loop while disconnected
@@ -492,6 +495,7 @@ class RadioManager:
                             await self.post_connect_setup()
                             broadcast_health(True, self._connection_info)
                             self._last_connected = True
+                            consecutive_setup_failures = 0
 
                     elif not self._last_connected and current_connected:
                         # Connection restored (might have reconnected automatically).
@@ -500,19 +504,34 @@ class RadioManager:
                         await self.post_connect_setup()
                         broadcast_health(True, self._connection_info)
                         self._last_connected = True
+                        consecutive_setup_failures = 0
 
                     elif current_connected and not self._setup_complete:
                         # Transport connected but setup incomplete — retry
                         logger.info("Retrying post-connect setup...")
                         await self.post_connect_setup()
                         broadcast_health(True, self._connection_info)
+                        consecutive_setup_failures = 0
 
                 except asyncio.CancelledError:
                     # Task is being cancelled, exit cleanly
                     break
                 except Exception as e:
-                    # Log error but continue monitoring - don't let the monitor die
-                    logger.exception("Error in connection monitor, continuing: %s", e)
+                    consecutive_setup_failures += 1
+                    if consecutive_setup_failures == UNRESPONSIVE_THRESHOLD:
+                        logger.error(
+                            "Post-connect setup has failed %d times in a row. "
+                            "The radio port appears open but the radio is not "
+                            "responding to commands. Common causes: another "
+                            "process has the serial port open (check for other "
+                            "RemoteTerm instances, serial monitors, etc.), the "
+                            "firmware is in repeater mode (not client), or the "
+                            "radio needs a power cycle. Will keep retrying.",
+                            consecutive_setup_failures,
+                        )
+                    elif consecutive_setup_failures < UNRESPONSIVE_THRESHOLD:
+                        logger.exception("Error in connection monitor, continuing: %s", e)
+                    # After the threshold, silently retry (avoid log spam)
 
         self._reconnect_task = asyncio.create_task(monitor_loop())
         logger.info("Radio connection monitor started")

app/radio_sync.py

@@ -117,7 +117,16 @@ async def sync_and_offload_contacts(mc: MeshCore) -> dict:
         result = await mc.commands.get_contacts()
 
         if result is None or result.type == EventType.ERROR:
-            logger.error("Failed to get contacts from radio: %s", result)
+            logger.error(
+                "Failed to get contacts from radio: %s. "
+                "If you see this repeatedly, the radio may be visible on the "
+                "serial/TCP/BLE port but not responding to commands. Check for "
+                "another process with the serial port open (other RemoteTerm "
+                "instances, serial monitors, etc.), verify the firmware is "
+                "up-to-date and in client mode (not repeater), or try a "
+                "power cycle.",
+                result,
+            )
             return {"synced": 0, "removed": 0, "error": str(result)}
 
         contacts = result.payload or {}
@@ -662,8 +671,19 @@ async def _sync_contacts_to_radio_inner(mc: MeshCore) -> dict:
                 logger.debug("Loaded contact %s to radio", contact.public_key[:12])
             else:
                 failed += 1
+                reason = result.payload
+                hint = ""
+                if reason is None:
+                    hint = (
+                        " (no response from radio — if this repeats, check for "
+                        "serial port contention from another process or try a "
+                        "power cycle)"
+                    )
                 logger.warning(
-                    "Failed to load contact %s: %s", contact.public_key[:12], result.payload
+                    "Failed to load contact %s: %s%s",
+                    contact.public_key[:12],
+                    reason,
+                    hint,
                 )
         except Exception as e:
             failed += 1