data: provide frequency and modem preset for meshcore (#694)

* data: provide frequency and modem preset for meshcore

* data: provide frequency and modem preset for meshcore

* ingestor: address review comments

* fix: remove duplicate _mark_packet_seen entry from handlers __all__

* ci: install meshcore in Python workflow

protocols/meshcore.py now imports meshcore at module level (required to
fix a self-referential import failure after the providers/ → protocols/
rename).  test_provider_unit.py imports that module unconditionally, so
meshcore must be present in the test environment.

* data: run black

data/mesh.sh

@@ -15,8 +15,14 @@
 
 set -euo pipefail
 
-python -m venv .venv
-source .venv/bin/activate
-pip install -U pip
-pip install -r "$(dirname "$0")/requirements.txt"
-exec python mesh.py
+# Recreate the venv only when its embedded Python is missing or points to the
+# wrong prefix (e.g. a stale shebang from a sibling project's venv).  Avoid
+# --clear on every run: it wipes installed packages before each start, so any
+# restart during a PyPI outage turns a transient network failure into hard
+# ingestor downtime.
+if ! .venv/bin/python -c "import sys; exit(0 if '.venv' in sys.prefix else 1)" 2>/dev/null; then
+    python -m venv --clear .venv
+fi
+.venv/bin/pip install -U pip
+.venv/bin/pip install -r "$(dirname "$0")/requirements.txt"
+exec .venv/bin/python mesh.py

data/mesh_ingestor/CONTRACTS.md

@@ -5,7 +5,7 @@ This repo’s ingestion pipeline is split into:
 - **Python collector** (`data/mesh_ingestor/*`) which normalizes packets/events and POSTs JSON to the web app.
 - **Sinatra web app** (`web/`) which accepts those payloads on `POST /api/*` ingest routes and persists them into SQLite tables defined under `data/*.sql`.
 
-This document records the **contracts that future providers must preserve**. The intent is to enable adding new providers (MeshCore, Reticulum, …) without changing the Ruby/DB/UI read-side.
+This document records the **contracts that future protocols must preserve**. The intent is to enable adding new protocols (MeshCore, Reticulum, …) without changing the Ruby/DB/UI read-side.
 
 ### Canonical node identity
 
@@ -16,7 +16,7 @@ This document records the **contracts that future providers must preserve**. The
   - Ruby normalizes via `web/lib/potato_mesh/application/data_processing.rb:canonical_node_parts`.
 - **Dual addressing**: Ruby routes and queries accept either a canonical `!xxxxxxxx` string or a numeric node id; they normalize to `node_id`.
 
-Note: non-Meshtastic providers will need a strategy to map their native node identifiers into this `!%08x` space. That mapping is intentionally not standardized in code yet.
+Note: non-Meshtastic protocols will need a strategy to map their native node identifiers into this `!%08x` space. That mapping is intentionally not standardized in code yet.
 
 ### Ingest HTTP routes and payload shapes
 

data/mesh_ingestor/config.py

@@ -65,21 +65,31 @@ CHANNEL_INDEX = int(os.environ.get("CHANNEL_INDEX", str(DEFAULT_CHANNEL_INDEX)))
 
 DEBUG = os.environ.get("DEBUG") == "1"
 
-_KNOWN_PROVIDERS = ("meshtastic", "meshcore")
+_KNOWN_PROTOCOLS = ("meshtastic", "meshcore")
 
-_raw_provider = os.environ.get("PROVIDER", "meshtastic").strip().lower()
-if _raw_provider not in _KNOWN_PROVIDERS:
+# Prefer the canonical PROTOCOL env var; fall back to legacy PROVIDER for
+# backwards compatibility with existing deployments.
+_raw_protocol = (
+    (os.environ.get("PROTOCOL") or os.environ.get("PROVIDER", "meshtastic"))
+    .strip()
+    .lower()
+)
+if _raw_protocol not in _KNOWN_PROTOCOLS:
     raise ValueError(
-        f"Unknown PROVIDER={_raw_provider!r}. "
-        f"Valid options: {', '.join(_KNOWN_PROVIDERS)}"
+        f"Unknown PROTOCOL={_raw_protocol!r}. "
+        f"Valid options: {', '.join(_KNOWN_PROTOCOLS)}"
     )
 
-PROVIDER = _raw_provider
-"""Active ingestion provider, selected via the :envvar:`PROVIDER` environment variable.
+PROTOCOL = _raw_protocol
+"""Active ingestion protocol, selected via the :envvar:`PROTOCOL` environment variable.
 
 Accepted values are ``meshtastic`` (default) and ``meshcore``.
+The legacy :envvar:`PROVIDER` environment variable is still accepted as a fallback.
 """
 
+PROVIDER = PROTOCOL
+"""Deprecated alias for :data:`PROTOCOL`; kept for backwards compatibility."""
+
 
 def _parse_channel_names(raw_value: str | None) -> tuple[str, ...]:
     """Normalise a comma-separated list of channel names.
@@ -228,12 +238,16 @@ class _ConfigModule(ModuleType):
     """Module proxy that keeps connection aliases synchronised."""
 
     def __setattr__(self, name: str, value: Any) -> None:  # type: ignore[override]
-        """Propagate CONNECTION/PORT assignments to both attributes."""
+        """Propagate CONNECTION/PORT and PROTOCOL/PROVIDER assignments to both attributes."""
 
         if name in {"CONNECTION", "PORT"}:
             super().__setattr__("CONNECTION", value)
             super().__setattr__("PORT", value)
             return
+        if name in {"PROTOCOL", "PROVIDER"}:
+            super().__setattr__("PROTOCOL", value)
+            super().__setattr__("PROVIDER", value)
+            return
         super().__setattr__(name, value)
 
 

data/mesh_ingestor/daemon.py

@@ -25,7 +25,7 @@ import time
 from pubsub import pub
 
 from . import config, handlers, ingestors, interfaces
-from .provider import Provider
+from .mesh_protocol import MeshProtocol
 from .utils import _retry_dict_snapshot
 
 _RECEIVE_TOPICS = (
@@ -245,7 +245,7 @@ def _connected_state(candidate) -> bool | None:
 class _DaemonState:
     """All mutable state for the :func:`main` daemon loop."""
 
-    provider: Provider
+    provider: MeshProtocol
     stop: threading.Event
     configured_port: str | None
     inactivity_reconnect_secs: float
@@ -549,16 +549,16 @@ def _loop_iteration(state: _DaemonState) -> bool:
 # ---------------------------------------------------------------------------
 
 
-def main(*, provider: Provider | None = None) -> None:
+def main(*, provider: MeshProtocol | None = None) -> None:
     """Run the mesh ingestion daemon until interrupted."""
 
     if provider is None:
-        if config.PROVIDER == "meshcore":
-            from .providers.meshcore import MeshcoreProvider
+        if config.PROTOCOL == "meshcore":
+            from .protocols.meshcore import MeshcoreProvider
 
             provider = MeshcoreProvider()
         else:
-            from .providers.meshtastic import MeshtasticProvider
+            from .protocols.meshtastic import MeshtasticProvider
 
             provider = MeshtasticProvider()
 

data/mesh_ingestor/handlers/__init__.py

@@ -26,7 +26,7 @@ This package is organised into focused submodules:
 - :mod:`.generic` — packet dispatcher, node upsert, and the main receive callback
 
 All public names from the original flat ``handlers`` module are re-exported
-here so existing callers (e.g. ``daemon.py``, ``providers/``) require no
+here so existing callers (e.g. ``daemon.py``, ``protocols/``) require no
 changes.
 """
 
@@ -80,13 +80,13 @@ __all__ = [
     "_apply_radio_metadata",
     "_apply_radio_metadata_to_nodes",
     "_is_encrypted_flag",
+    "_mark_packet_seen",
     "_normalize_trace_hops",
     "_portnum_candidates",
     "_queue_post_json",
     "_radio_metadata_fields",
     "_record_ignored_packet",
     "base64_payload",
-    "_mark_packet_seen",
     "host_node_id",
     "last_packet_monotonic",
     "on_receive",

data/mesh_ingestor/handlers/telemetry.py

@@ -123,7 +123,7 @@ def store_telemetry_packet(packet: Mapping, decoded: Mapping) -> None:
     # Priority order matters: deviceMetrics is checked first because the device
     # sub-object also carries a voltage field that overlaps with powerMetrics.
     # Meshtastic uses a protobuf oneof so only one sub-object can be populated per
-    # packet; the elif chain handles any hypothetical overlap from future providers.
+    # packet; the elif chain handles any hypothetical overlap from future protocols.
     if isinstance(_dm, Mapping):
         telemetry_type: str | None = "device"
     elif isinstance(_em, Mapping):

data/mesh_ingestor/ingestors.py

@@ -113,7 +113,7 @@ def queue_ingestor_heartbeat(
         "start_time": STATE.start_time,
         "last_seen_time": now,
         "version": INGESTOR_VERSION,
-        "protocol": getattr(config, "PROVIDER", "meshtastic") or "meshtastic",
+        "protocol": getattr(config, "PROTOCOL", "meshtastic") or "meshtastic",
     }
     if getattr(config, "LORA_FREQ", None) is not None:
         payload["lora_freq"] = config.LORA_FREQ

data/mesh_ingestor/mesh_protocol.py

@@ -12,11 +12,10 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-"""Provider interface for ingestion sources.
+"""MeshProtocol interface for ingestion sources.
 
-Today the repo ships a Meshtastic provider only. This module defines the seam so
-future providers (MeshCore, Reticulum, ...) can be added without changing the
-web app ingest contract.
+This module defines the seam so future protocols (MeshCore, Reticulum, ...) can
+be added without changing the web app ingest contract.
 """
 
 from __future__ import annotations
@@ -26,8 +25,8 @@ from typing import Protocol, runtime_checkable
 
 
 @runtime_checkable
-class Provider(Protocol):
-    """Abstract source of mesh observations."""
+class MeshProtocol(Protocol):
+    """Abstract mesh protocol source."""
 
     name: str
 
@@ -51,5 +50,8 @@ class Provider(Protocol):
 
 
 __all__ = [
-    "Provider",
+    "MeshProtocol",
 ]
+
+# Backwards-compatibility alias — import Provider from here during transition.
+Provider = MeshProtocol

data/mesh_ingestor/protocols/__init__.py

@@ -12,9 +12,9 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-"""Provider implementations.
+"""Protocol implementations.
 
-This package contains protocol-specific provider implementations (Meshtastic,
+This package contains protocol-specific implementations (Meshtastic,
 MeshCore, and others in the future).
 """
 
@@ -24,11 +24,11 @@ from .meshtastic import MeshtasticProvider
 
 
 def __getattr__(name: str) -> object:
-    """Lazy-load provider classes and exceptions that carry optional heavy dependencies.
+    """Lazy-load protocol classes and exceptions that carry optional heavy dependencies.
 
     ``MeshcoreProvider`` and ``ClosedBeforeConnectedError`` are imported on
     demand so that the MeshCore library (once wired in) is not loaded at
-    startup when ``PROVIDER=meshtastic``.
+    startup when ``PROTOCOL=meshtastic``.
     """
     if name == "MeshcoreProvider":
         from .meshcore import MeshcoreProvider

data/mesh_ingestor/protocols/meshcore.py

@@ -12,17 +12,17 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-"""MeshCore provider implementation.
+"""MeshCore protocol implementation.
 
 This module defines :class:`MeshcoreProvider`, which satisfies the
-:class:`~data.mesh_ingestor.provider.Provider` protocol for MeshCore nodes
-connected via serial port, BLE, or TCP/IP.
+:class:`~data.mesh_ingestor.mesh_protocol.MeshProtocol` interface for MeshCore
+nodes connected via serial port, BLE, or TCP/IP.
 
-The provider runs MeshCore's ``asyncio`` event loop in a background daemon
-thread so that incoming events are dispatched without blocking the
+The protocol backend runs MeshCore's ``asyncio`` event loop in a background
+daemon thread so that incoming events are dispatched without blocking the
 synchronous daemon loop.  Received contacts, channel messages, and direct
 messages are forwarded to the shared HTTP ingest queue via the same
-:mod:`~data.mesh_ingestor.handlers` helpers used by the Meshtastic provider.
+:mod:`~data.mesh_ingestor.handlers` helpers used by the Meshtastic protocol.
 
 Connection type is detected automatically from the target string:
 
@@ -50,6 +50,25 @@ import time
 from datetime import datetime, timezone
 from pathlib import Path
 
+# Import meshcore symbols at module level rather than lazily inside functions.
+# The original deferred-import pattern was introduced so that loading
+# ``protocols/__init__.py`` under ``PROTOCOL=meshtastic`` would not pull in the
+# meshcore library.  That protection is preserved: ``protocols/__init__.py``
+# only imports THIS module on demand (via its ``__getattr__`` lazy loader), so
+# this top-level import still never executes for meshtastic-only deployments.
+# The import was hoisted because, after the rename from ``providers/meshcore``
+# to ``protocols/meshcore``, Python's absolute import resolver matched the
+# module's own short name (``meshcore``) against the installed package, causing
+# a ``ModuleNotFoundError`` when the deferred ``from meshcore import …`` ran
+# inside a background thread at connect time.
+from meshcore import (
+    BLEConnection,
+    EventType,
+    MeshCore,
+    SerialConnection,
+    TCPConnection,
+)
+
 from .. import config
 from ..connection import default_serial_targets, parse_ble_target, parse_tcp_target
 
@@ -234,6 +253,23 @@ def _contact_to_node_dict(contact: dict) -> dict:
     return node
 
 
+def _derive_modem_preset(sf: object, bw: object, cr: object) -> str | None:
+    """Return a compact radio-parameter string from spreading factor, bandwidth, and coding rate.
+
+    Parameters:
+        sf: Spreading factor (int, e.g. ``12``).
+        bw: Bandwidth in kHz (int or float, e.g. ``125.0``).
+        cr: Coding rate denominator (int, e.g. ``5`` meaning 4/5).
+
+    Returns:
+        A string such as ``"SF12/BW125/CR5"``, or ``None`` when any parameter
+        is absent or zero (meaning the radio config was not reported).
+    """
+    if not sf or not bw or not cr:
+        return None
+    return f"SF{int(sf)}/BW{int(bw)}/CR{int(cr)}"
+
+
 def _self_info_to_node_dict(self_info: dict) -> dict:
     """Convert a MeshCore ``SELF_INFO`` payload to a Meshtastic-ish node dict.
 
@@ -414,11 +450,13 @@ class _MeshcoreInterface:
 def _process_self_info(
     payload: dict, iface: _MeshcoreInterface, handlers: object
 ) -> None:
-    """Apply a ``SELF_INFO`` payload: set host_node_id and upsert the host node.
+    """Apply a ``SELF_INFO`` payload: set host_node_id, upsert the host node,
+    and capture LoRa radio metadata into the shared config cache.
 
     Parameters:
         payload: Event payload dict containing at minimum ``public_key`` and
-            optionally ``name``, ``adv_lat``, ``adv_lon``.
+            optionally ``name``, ``adv_lat``, ``adv_lon``, ``radio_freq``,
+            ``radio_bw``, ``radio_sf``, ``radio_cr``.
         iface: Active interface whose :attr:`host_node_id` will be updated.
         handlers: Module reference for :func:`~data.mesh_ingestor.handlers`
             functions (passed to avoid circular-import issues).
@@ -429,6 +467,25 @@ def _process_self_info(
         iface.host_node_id = node_id
         handlers.register_host_node_id(node_id)
         handlers.upsert_node(node_id, _self_info_to_node_dict(payload))
+
+    # Capture radio metadata once — never overwrite a previously cached value.
+    # Mirrors the guard used by interfaces._ensure_radio_metadata for Meshtastic.
+    radio_freq = payload.get("radio_freq")
+    if radio_freq is not None and getattr(config, "LORA_FREQ", None) is None:
+        config.LORA_FREQ = radio_freq
+    modem_preset = _derive_modem_preset(
+        payload.get("radio_sf"), payload.get("radio_bw"), payload.get("radio_cr")
+    )
+    if modem_preset is not None and getattr(config, "MODEM_PRESET", None) is None:
+        config.MODEM_PRESET = modem_preset
+    config._debug_log(
+        "MeshCore radio metadata captured",
+        context="meshcore.self_info.radio",
+        severity="info",
+        lora_freq=radio_freq,
+        modem_preset=modem_preset,
+    )
+
     handlers._mark_packet_seen()
     config._debug_log(
         "MeshCore self-info received",
@@ -620,8 +677,6 @@ def _make_connection(target: str, baudrate: int) -> object:
     Returns:
         An unconnected ``meshcore`` connection object.
     """
-    from meshcore import BLEConnection, SerialConnection, TCPConnection
-
     ble_addr = parse_ble_target(target)
     if ble_addr:
         return BLEConnection(address=ble_addr)
@@ -655,8 +710,6 @@ async def _run_meshcore(
         error_holder: Single-element list; set to the raised exception when
             the connection attempt fails so the caller can re-raise it.
     """
-    from meshcore import EventType, MeshCore
-
     # Install early so :meth:`_MeshcoreInterface.close` can signal shutdown with
     # ``stop_event.set()`` instead of ``loop.stop()`` while ``connect()`` or the
     # ``finally`` disconnect is still running (avoids RuntimeError from

data/mesh_ingestor/protocols/meshtastic.py

@@ -12,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-"""Meshtastic provider implementation."""
+"""Meshtastic protocol implementation."""
 
 from __future__ import annotations
 
@@ -23,7 +23,7 @@ from ..utils import _retry_dict_snapshot
 
 
 class MeshtasticProvider:
-    """Meshtastic ingestion provider (current default)."""
+    """Meshtastic ingestion protocol (current default)."""
 
     name = "meshtastic"