From 61d6b6287ed109b5efee21e5783bfe6a14e43908 Mon Sep 17 00:00:00 2001 From: Louis King Date: Thu, 8 Jan 2026 10:22:27 +0000 Subject: [PATCH] Add contact cleanup to interface RECEIVER mode - Add CONTACT_CLEANUP_ENABLED and CONTACT_CLEANUP_DAYS settings - Implement remove_contact and schedule_remove_contact on device classes - During contact sync, remove stale contacts from companion node - Stale contacts (not advertised for > N days) not published to MQTT - Update Python version to 3.13 across project config - Remove brittle config tests that assumed default env values --- .pre-commit-config.yaml | 2 +- .python-version | 2 +- AGENTS.md | 24 +++++++- Dockerfile | 4 +- pyproject.toml | 9 ++- src/meshcore_hub/common/config.py | 11 ++++ src/meshcore_hub/interface/cli.py | 35 +++++++++++ src/meshcore_hub/interface/device.py | 74 +++++++++++++++++++++++ src/meshcore_hub/interface/mock_device.py | 24 ++++++++ src/meshcore_hub/interface/receiver.py | 67 ++++++++++++++++++-- tests/test_common/test_config.py | 74 +++-------------------- 11 files changed, 244 insertions(+), 82 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 43d24ac..62c0f40 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -14,7 +14,7 @@ repos: rev: 24.3.0 hooks: - id: black - language_version: python3.11 + language_version: python3.13 args: ["--line-length=88"] - repo: https://github.com/pycqa/flake8 diff --git a/.python-version b/.python-version index 2c07333..24ee5b1 100644 --- a/.python-version +++ b/.python-version @@ -1 +1 @@ -3.11 +3.13 diff --git a/AGENTS.md b/AGENTS.md index 3740ea8..e707eb6 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -18,7 +18,7 @@ This document provides context and guidelines for AI coding assistants working o ## Project Overview -MeshCore Hub is a Python 3.11+ monorepo for managing and orchestrating MeshCore mesh networks. It consists of five main components: +MeshCore Hub is a Python 3.13+ monorepo for managing and orchestrating MeshCore mesh networks. It consists of five main components: - **meshcore_interface**: Serial/USB interface to MeshCore companion nodes, publishes/subscribes to MQTT - **meshcore_collector**: Collects MeshCore events from MQTT and stores them in a database @@ -37,7 +37,7 @@ MeshCore Hub is a Python 3.11+ monorepo for managing and orchestrating MeshCore | Category | Technology | |----------|------------| -| Language | Python 3.11+ | +| Language | Python 3.13+ | | Package Management | pip with pyproject.toml | | CLI Framework | Click | | Configuration | Pydantic Settings | @@ -545,6 +545,22 @@ When enabled, the collector automatically removes nodes where: **Note:** Both event data and node cleanup run on the same schedule (DATA_RETENTION_INTERVAL_HOURS). +**Contact Cleanup (Interface RECEIVER):** + +The interface RECEIVER mode can automatically remove stale contacts from the MeshCore companion node's contact database. This prevents the companion node from resyncing old/dead contacts back to the collector, freeing up memory on the device (typically limited to ~100 contacts). + +| Variable | Description | +|----------|-------------| +| `CONTACT_CLEANUP_ENABLED` | Enable automatic removal of stale contacts (default: true) | +| `CONTACT_CLEANUP_DAYS` | Remove contacts not advertised for this many days (default: 7) | + +When enabled, during each contact sync the receiver checks each contact's `last_advert` timestamp: +- Contacts with `last_advert` older than `CONTACT_CLEANUP_DAYS` are removed from the device +- Stale contacts are not published to MQTT (preventing collector database pollution) +- Contacts without a `last_advert` timestamp are preserved (no removal without data) + +This cleanup runs automatically whenever the receiver syncs contacts (on startup and after each advertisement event). + Manual cleanup can be triggered at any time with: ```bash # Dry run to see what would be deleted @@ -571,6 +587,10 @@ Webhook payload structure: 2. **Database Migration Errors**: Ensure `DATA_HOME` is writable, run `meshcore-hub db upgrade` 3. **Import Errors**: Ensure package is installed with `pip install -e .` 4. **Type Errors**: Run `pre-commit run --all-files` to check type annotations and other issues +5. **NixOS greenlet errors**: On NixOS, the pre-built greenlet wheel may fail with `libstdc++.so.6` errors. Rebuild from source: + ```bash + pip install --no-binary greenlet greenlet + ``` ### Debugging diff --git a/Dockerfile b/Dockerfile index 30b3c08..091bdaf 100644 --- a/Dockerfile +++ b/Dockerfile @@ -4,7 +4,7 @@ # ============================================================================= # Stage 1: Builder - Install dependencies and build package # ============================================================================= -FROM python:3.11-slim AS builder +FROM python:3.13-slim AS builder # Set environment variables ENV PYTHONDONTWRITEBYTECODE=1 \ @@ -39,7 +39,7 @@ RUN sed -i "s|__version__ = \"dev\"|__version__ = \"${BUILD_VERSION}\"|" src/mes # ============================================================================= # Stage 2: Runtime - Final production image # ============================================================================= -FROM python:3.11-slim AS runtime +FROM python:3.13-slim AS runtime # Labels LABEL org.opencontainers.image.title="MeshCore Hub" \ diff --git a/pyproject.toml b/pyproject.toml index c333366..937f706 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -8,7 +8,7 @@ version = "0.0.0" description = "Python monorepo for managing and orchestrating MeshCore mesh networks" readme = "README.md" license = {text = "GPL-3.0-or-later"} -requires-python = ">=3.11" +requires-python = ">=3.13" authors = [ {name = "MeshCore Hub Contributors"} ] @@ -18,8 +18,7 @@ classifiers = [ "License :: OSI Approved :: GNU General Public License v3 or later (GPLv3+)", "Operating System :: OS Independent", "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.11", - "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", "Topic :: Communications", "Topic :: System :: Networking", ] @@ -78,7 +77,7 @@ meshcore_hub = ["py.typed"] [tool.black] line-length = 88 -target-version = ["py311"] +target-version = ["py312"] include = '\.pyi?$' extend-exclude = ''' /( @@ -97,7 +96,7 @@ extend-exclude = ''' ''' [tool.mypy] -python_version = "3.11" +python_version = "3.13" warn_return_any = true warn_unused_ignores = true disallow_untyped_defs = true diff --git a/src/meshcore_hub/common/config.py b/src/meshcore_hub/common/config.py index 1adc61b..0739149 100644 --- a/src/meshcore_hub/common/config.py +++ b/src/meshcore_hub/common/config.py @@ -78,6 +78,17 @@ class InterfaceSettings(CommonSettings): default=None, description="Device/node name (optional)" ) + # Contact cleanup settings + contact_cleanup_enabled: bool = Field( + default=True, + description="Enable automatic removal of stale contacts from companion node", + ) + contact_cleanup_days: int = Field( + default=7, + description="Remove contacts not advertised for this many days", + ge=1, + ) + class CollectorSettings(CommonSettings): """Settings for the Collector component.""" diff --git a/src/meshcore_hub/interface/cli.py b/src/meshcore_hub/interface/cli.py index 46a47ed..09f8db2 100644 --- a/src/meshcore_hub/interface/cli.py +++ b/src/meshcore_hub/interface/cli.py @@ -100,6 +100,19 @@ def interface() -> None: envvar="MQTT_TLS", help="Enable TLS/SSL for MQTT connection", ) +@click.option( + "--contact-cleanup/--no-contact-cleanup", + default=True, + envvar="CONTACT_CLEANUP_ENABLED", + help="Enable/disable automatic removal of stale contacts (RECEIVER mode only)", +) +@click.option( + "--contact-cleanup-days", + type=int, + default=7, + envvar="CONTACT_CLEANUP_DAYS", + help="Remove contacts not advertised for this many days (RECEIVER mode only)", +) @click.option( "--log-level", type=click.Choice(["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"]), @@ -120,6 +133,8 @@ def run( mqtt_password: str | None, prefix: str, mqtt_tls: bool, + contact_cleanup: bool, + contact_cleanup_days: int, log_level: str, ) -> None: """Run the interface component. @@ -162,6 +177,8 @@ def run( mqtt_password=mqtt_password, mqtt_prefix=prefix, mqtt_tls=mqtt_tls, + contact_cleanup_enabled=contact_cleanup, + contact_cleanup_days=contact_cleanup_days, ) elif mode_upper == "SENDER": from meshcore_hub.interface.sender import run_sender @@ -262,6 +279,19 @@ def run( envvar="MQTT_TLS", help="Enable TLS/SSL for MQTT connection", ) +@click.option( + "--contact-cleanup/--no-contact-cleanup", + default=True, + envvar="CONTACT_CLEANUP_ENABLED", + help="Enable/disable automatic removal of stale contacts", +) +@click.option( + "--contact-cleanup-days", + type=int, + default=7, + envvar="CONTACT_CLEANUP_DAYS", + help="Remove contacts not advertised for this many days", +) def receiver( port: str, baud: int, @@ -274,6 +304,8 @@ def receiver( mqtt_password: str | None, prefix: str, mqtt_tls: bool, + contact_cleanup: bool, + contact_cleanup_days: int, ) -> None: """Run interface in RECEIVER mode. @@ -293,12 +325,15 @@ def receiver( baud=baud, mock=mock, node_address=node_address, + device_name=device_name, mqtt_host=mqtt_host, mqtt_port=mqtt_port, mqtt_username=mqtt_username, mqtt_password=mqtt_password, mqtt_prefix=prefix, mqtt_tls=mqtt_tls, + contact_cleanup_enabled=contact_cleanup, + contact_cleanup_days=contact_cleanup_days, ) diff --git a/src/meshcore_hub/interface/device.py b/src/meshcore_hub/interface/device.py index 1ebe46e..814457d 100644 --- a/src/meshcore_hub/interface/device.py +++ b/src/meshcore_hub/interface/device.py @@ -211,6 +211,32 @@ class BaseMeshCoreDevice(ABC): """ pass + @abstractmethod + def remove_contact(self, public_key: str) -> bool: + """Remove a contact from the device's contact database. + + Args: + public_key: The 64-character hex public key of the contact to remove + + Returns: + True if contact was removed successfully + """ + pass + + @abstractmethod + def schedule_remove_contact(self, public_key: str) -> bool: + """Schedule a remove_contact request on the event loop. + + This is safe to call from event handlers while the event loop is running. + + Args: + public_key: The 64-character hex public key of the contact to remove + + Returns: + True if request was scheduled successfully + """ + pass + @abstractmethod def run(self) -> None: """Run the device event loop (blocking).""" @@ -627,6 +653,54 @@ class MeshCoreDevice(BaseMeshCoreDevice): logger.error(f"Failed to schedule get contacts: {e}") return False + def remove_contact(self, public_key: str) -> bool: + """Remove a contact from the device's contact database. + + Note: This method should only be called before the event loop is running + (e.g., during initialization). For calling during event processing, + use schedule_remove_contact() instead. + """ + if not self._connected or not self._mc: + logger.error("Cannot remove contact: not connected") + return False + + try: + + async def _remove_contact() -> None: + await self._mc.commands.remove_contact(public_key) + + self._loop.run_until_complete(_remove_contact()) + logger.info(f"Removed contact {public_key[:12]}...") + return True + except Exception as e: + logger.error(f"Failed to remove contact: {e}") + return False + + def schedule_remove_contact(self, public_key: str) -> bool: + """Schedule a remove_contact request on the event loop. + + This is safe to call from event handlers while the event loop is running. + The request is scheduled as a task on the event loop. + + Returns: + True if request was scheduled, False if device not connected + """ + if not self._connected or not self._mc: + logger.error("Cannot remove contact: not connected") + return False + + try: + + async def _remove_contact() -> None: + await self._mc.commands.remove_contact(public_key) + + asyncio.run_coroutine_threadsafe(_remove_contact(), self._loop) + logger.debug(f"Scheduled removal of contact {public_key[:12]}...") + return True + except Exception as e: + logger.error(f"Failed to schedule remove contact: {e}") + return False + def run(self) -> None: """Run the device event loop.""" self._running = True diff --git a/src/meshcore_hub/interface/mock_device.py b/src/meshcore_hub/interface/mock_device.py index 2c11f55..b3038ab 100644 --- a/src/meshcore_hub/interface/mock_device.py +++ b/src/meshcore_hub/interface/mock_device.py @@ -329,6 +329,30 @@ class MockMeshCoreDevice(BaseMeshCoreDevice): """ return self.get_contacts() + def remove_contact(self, public_key: str) -> bool: + """Remove a contact from the mock device's contact database.""" + if not self._connected: + logger.error("Cannot remove contact: not connected") + return False + + # Find and remove the contact from mock_config.nodes + for i, node in enumerate(self.mock_config.nodes): + if node.public_key == public_key: + del self.mock_config.nodes[i] + logger.info(f"Mock: Removed contact {public_key[:12]}...") + return True + + logger.warning(f"Mock: Contact {public_key[:12]}... not found") + return True # Return True even if not found (idempotent) + + def schedule_remove_contact(self, public_key: str) -> bool: + """Schedule a remove_contact request. + + For the mock device, this is the same as remove_contact() since we + don't have a real async event loop. + """ + return self.remove_contact(public_key) + def run(self) -> None: """Run the mock device event loop.""" self._running = True diff --git a/src/meshcore_hub/interface/receiver.py b/src/meshcore_hub/interface/receiver.py index 55778c1..e9f71c9 100644 --- a/src/meshcore_hub/interface/receiver.py +++ b/src/meshcore_hub/interface/receiver.py @@ -20,6 +20,9 @@ from meshcore_hub.interface.device import ( create_device, ) +# Default contact cleanup settings +DEFAULT_CONTACT_CLEANUP_DAYS = 7 + logger = logging.getLogger(__name__) @@ -34,6 +37,8 @@ class Receiver: device: BaseMeshCoreDevice, mqtt_client: MQTTClient, device_name: Optional[str] = None, + contact_cleanup_enabled: bool = True, + contact_cleanup_days: int = DEFAULT_CONTACT_CLEANUP_DAYS, ): """Initialize receiver. @@ -41,10 +46,14 @@ class Receiver: device: MeshCore device instance mqtt_client: MQTT client instance device_name: Optional device/node name to set on startup + contact_cleanup_enabled: Whether to remove stale contacts from device + contact_cleanup_days: Remove contacts not advertised for this many days """ self.device = device self.mqtt = mqtt_client self.device_name = device_name + self.contact_cleanup_enabled = contact_cleanup_enabled + self.contact_cleanup_days = contact_cleanup_days self._running = False self._shutdown_event = threading.Event() self._device_connected = False @@ -167,6 +176,8 @@ class Receiver: The device returns contacts as a dict keyed by public_key. We split this into individual 'contact' events for cleaner processing. + Stale contacts (not advertised for > contact_cleanup_days) are removed + from the device and not published. Args: payload: Dict of contacts keyed by public_key @@ -188,22 +199,54 @@ class Receiver: return device_key = self.device.public_key # Capture for type narrowing - count = 0 + current_time = int(time.time()) + stale_threshold = current_time - (self.contact_cleanup_days * 24 * 60 * 60) + + published_count = 0 + removed_count = 0 + for contact in contacts: if not isinstance(contact, dict): continue + public_key = contact.get("public_key") + if not public_key: + continue + + # Check if contact is stale based on last_advert timestamp + # Only check if cleanup is enabled and last_advert exists + if self.contact_cleanup_enabled: + last_advert = contact.get("last_advert") + if last_advert is not None and last_advert > 0: + if last_advert < stale_threshold: + # Contact is stale - remove from device + adv_name = contact.get("adv_name", contact.get("name", "")) + logger.info( + f"Removing stale contact {public_key[:12]}... " + f"({adv_name}) - last advertised " + f"{(current_time - last_advert) // 86400} days ago" + ) + self.device.schedule_remove_contact(public_key) + removed_count += 1 + continue # Don't publish stale contacts + try: self.mqtt.publish_event( device_key, "contact", # Use singular 'contact' for individual events contact, ) - count += 1 + published_count += 1 except Exception as e: logger.error(f"Failed to publish contact event: {e}") - logger.info(f"Published {count} contact events to MQTT") + if removed_count > 0: + logger.info( + f"Contact sync: published {published_count}, " + f"removed {removed_count} stale contacts" + ) + else: + logger.info(f"Published {published_count} contact events to MQTT") def start(self) -> None: """Start the receiver.""" @@ -306,6 +349,8 @@ def create_receiver( mqtt_password: Optional[str] = None, mqtt_prefix: str = "meshcore", mqtt_tls: bool = False, + contact_cleanup_enabled: bool = True, + contact_cleanup_days: int = DEFAULT_CONTACT_CLEANUP_DAYS, ) -> Receiver: """Create a configured receiver instance. @@ -321,6 +366,8 @@ def create_receiver( mqtt_password: MQTT password mqtt_prefix: MQTT topic prefix mqtt_tls: Enable TLS/SSL for MQTT connection + contact_cleanup_enabled: Whether to remove stale contacts from device + contact_cleanup_days: Remove contacts not advertised for this many days Returns: Configured Receiver instance @@ -345,7 +392,13 @@ def create_receiver( ) mqtt_client = MQTTClient(mqtt_config) - return Receiver(device, mqtt_client, device_name=device_name) + return Receiver( + device, + mqtt_client, + device_name=device_name, + contact_cleanup_enabled=contact_cleanup_enabled, + contact_cleanup_days=contact_cleanup_days, + ) def run_receiver( @@ -360,6 +413,8 @@ def run_receiver( mqtt_password: Optional[str] = None, mqtt_prefix: str = "meshcore", mqtt_tls: bool = False, + contact_cleanup_enabled: bool = True, + contact_cleanup_days: int = DEFAULT_CONTACT_CLEANUP_DAYS, ) -> None: """Run the receiver (blocking). @@ -377,6 +432,8 @@ def run_receiver( mqtt_password: MQTT password mqtt_prefix: MQTT topic prefix mqtt_tls: Enable TLS/SSL for MQTT connection + contact_cleanup_enabled: Whether to remove stale contacts from device + contact_cleanup_days: Remove contacts not advertised for this many days """ receiver = create_receiver( port=port, @@ -390,6 +447,8 @@ def run_receiver( mqtt_password=mqtt_password, mqtt_prefix=mqtt_prefix, mqtt_tls=mqtt_tls, + contact_cleanup_enabled=contact_cleanup_enabled, + contact_cleanup_days=contact_cleanup_days, ) # Set up signal handlers diff --git a/tests/test_common/test_config.py b/tests/test_common/test_config.py index ea6c1b3..a823a2d 100644 --- a/tests/test_common/test_config.py +++ b/tests/test_common/test_config.py @@ -6,26 +6,12 @@ from meshcore_hub.common.config import ( CollectorSettings, APISettings, WebSettings, - LogLevel, - InterfaceMode, ) class TestCommonSettings: """Tests for CommonSettings.""" - def test_default_values(self) -> None: - """Test default setting values without .env file influence.""" - settings = CommonSettings(_env_file=None) - - assert settings.data_home == "./data" - assert settings.log_level == LogLevel.INFO - assert settings.mqtt_host == "localhost" - assert settings.mqtt_port == 1883 - assert settings.mqtt_username is None - assert settings.mqtt_password is None - assert settings.mqtt_prefix == "meshcore" - def test_custom_data_home(self) -> None: """Test custom DATA_HOME setting.""" settings = CommonSettings(_env_file=None, data_home="/custom/data") @@ -36,37 +22,19 @@ class TestCommonSettings: class TestInterfaceSettings: """Tests for InterfaceSettings.""" - def test_default_values(self) -> None: - """Test default setting values without .env file influence.""" - settings = InterfaceSettings(_env_file=None) + def test_custom_values(self) -> None: + """Test custom setting values.""" + settings = InterfaceSettings( + _env_file=None, serial_port="/dev/ttyACM0", serial_baud=9600 + ) - assert settings.interface_mode == InterfaceMode.RECEIVER - assert settings.serial_port == "/dev/ttyUSB0" - assert settings.serial_baud == 115200 - assert settings.mock_device is False + assert settings.serial_port == "/dev/ttyACM0" + assert settings.serial_baud == 9600 class TestCollectorSettings: """Tests for CollectorSettings.""" - def test_default_values(self) -> None: - """Test default setting values without .env file influence.""" - settings = CollectorSettings(_env_file=None) - - # database_url is None by default, effective_database_url computes it - assert settings.database_url is None - # Path normalizes ./data to data - assert settings.effective_database_url == "sqlite:///data/collector/meshcore.db" - assert settings.data_home == "./data" - assert settings.collector_data_dir == "data/collector" - - # seed_home defaults to ./seed (normalized to "seed") - assert settings.seed_home == "./seed" - assert settings.effective_seed_home == "seed" - # node_tags_file and members_file are derived from effective_seed_home - assert settings.node_tags_file == "seed/node_tags.yaml" - assert settings.members_file == "seed/members.yaml" - def test_custom_data_home(self) -> None: """Test that custom data_home affects effective paths.""" settings = CollectorSettings(_env_file=None, data_home="/custom/data") @@ -76,10 +44,6 @@ class TestCollectorSettings: == "sqlite:////custom/data/collector/meshcore.db" ) assert settings.collector_data_dir == "/custom/data/collector" - # seed_home is independent of data_home - assert settings.effective_seed_home == "seed" - assert settings.node_tags_file == "seed/node_tags.yaml" - assert settings.members_file == "seed/members.yaml" def test_explicit_database_url_overrides(self) -> None: """Test that explicit database_url overrides the default.""" @@ -103,19 +67,6 @@ class TestCollectorSettings: class TestAPISettings: """Tests for APISettings.""" - def test_default_values(self) -> None: - """Test default setting values without .env file influence.""" - settings = APISettings(_env_file=None) - - assert settings.api_host == "0.0.0.0" - assert settings.api_port == 8000 - # database_url is None by default, effective_database_url computes it - assert settings.database_url is None - # Path normalizes ./data to data - assert settings.effective_database_url == "sqlite:///data/collector/meshcore.db" - assert settings.api_read_key is None - assert settings.api_admin_key is None - def test_custom_data_home(self) -> None: """Test that custom data_home affects effective database path.""" settings = APISettings(_env_file=None, data_home="/custom/data") @@ -136,17 +87,6 @@ class TestAPISettings: class TestWebSettings: """Tests for WebSettings.""" - def test_default_values(self) -> None: - """Test default setting values without .env file influence.""" - settings = WebSettings(_env_file=None) - - assert settings.web_host == "0.0.0.0" - assert settings.web_port == 8080 - assert settings.api_base_url == "http://localhost:8000" - assert settings.network_name == "MeshCore Network" - # Path normalizes ./data to data - assert settings.web_data_dir == "data/web" - def test_custom_data_home(self) -> None: """Test that custom data_home affects effective paths.""" settings = WebSettings(_env_file=None, data_home="/custom/data")