Document mesh ingestor modules with PDoc-style docstrings (#255)

This commit is contained in:
l5y
2025-10-07 08:59:38 +02:00
committed by GitHub
parent 363b4c5525
commit a5a2ae5edc
10 changed files with 418 additions and 21 deletions

View File

@@ -60,6 +60,8 @@ class _MeshIngestorModule(types.ModuleType):
"""Module proxy that forwards config and interface state."""
def __getattr__(self, name: str): # type: ignore[override]
"""Resolve attributes by delegating to the underlying submodules."""
if name in _CONFIG_ATTRS:
return getattr(config, name)
if name in _INTERFACE_ATTRS:
@@ -69,6 +71,8 @@ class _MeshIngestorModule(types.ModuleType):
raise AttributeError(name)
def __setattr__(self, name: str, value): # type: ignore[override]
"""Propagate assignments to the appropriate submodule."""
if name in _CONFIG_ATTRS:
setattr(config, name, value)
super().__setattr__(name, value)

View File

@@ -18,7 +18,11 @@ _CLOSE_TIMEOUT_SECS = float(os.environ.get("MESH_CLOSE_TIMEOUT", "5"))
def _debug_log(message: str) -> None:
"""Print ``message`` with a UTC timestamp when ``DEBUG`` is enabled."""
"""Print ``message`` with a UTC timestamp when ``DEBUG`` is enabled.
Parameters:
message: Text to display when debug logging is active.
"""
if not DEBUG:
return

View File

@@ -25,6 +25,12 @@ _RECEIVE_TOPICS = (
def _event_wait_allows_default_timeout() -> bool:
"""Return ``True`` when :meth:`threading.Event.wait` accepts ``timeout``.
The behaviour changed between Python versions; this helper shields the
daemon from ``TypeError`` when the default timeout parameter is absent.
"""
try:
wait_signature = inspect.signature(threading.Event.wait)
except (TypeError, ValueError): # pragma: no cover
@@ -45,6 +51,8 @@ def _event_wait_allows_default_timeout() -> bool:
def _subscribe_receive_topics() -> list[str]:
"""Subscribe the packet handler to all receive-related pubsub topics."""
subscribed = []
for topic in _RECEIVE_TOPICS:
try:
@@ -58,6 +66,18 @@ def _subscribe_receive_topics() -> list[str]:
def _node_items_snapshot(
nodes_obj, retries: int = 3
) -> list[tuple[str, object]] | None:
"""Snapshot ``nodes_obj`` to avoid iteration errors during updates.
Parameters:
nodes_obj: Meshtastic nodes mapping or iterable.
retries: Number of attempts when encountering "dictionary changed"
runtime errors.
Returns:
A list of ``(node_id, node)`` tuples, ``None`` when retries are
exhausted, or an empty list when no nodes exist.
"""
if not nodes_obj:
return []
@@ -87,6 +107,8 @@ def _node_items_snapshot(
def _close_interface(iface_obj) -> None:
"""Close ``iface_obj`` while respecting configured timeouts."""
if iface_obj is None:
return
@@ -112,6 +134,8 @@ def _close_interface(iface_obj) -> None:
def main() -> None:
"""Run the mesh ingestion daemon until interrupted."""
subscribed = _subscribe_receive_topics()
if config.DEBUG and subscribed:
config._debug_log(f"subscribed to receive topics: {', '.join(subscribed)}")

View File

@@ -29,6 +29,16 @@ from .serialization import (
def upsert_node(node_id, node) -> None:
"""Schedule an upsert for a single node.
Parameters:
node_id: Canonical identifier for the node in the ``!xxxxxxxx`` format.
node: Node object or mapping to serialise for the API payload.
Returns:
``None``. The payload is forwarded to the shared HTTP queue.
"""
payload = upsert_payload(node_id, node)
_queue_post_json("/api/nodes", payload, priority=queue._NODE_POST_PRIORITY)
@@ -42,6 +52,16 @@ def upsert_node(node_id, node) -> None:
def store_position_packet(packet: Mapping, decoded: Mapping) -> None:
"""Persist a decoded position packet.
Parameters:
packet: Raw packet metadata emitted by Meshtastic.
decoded: Decoded payload extracted from ``packet['decoded']``.
Returns:
``None``. The formatted position data is queued for HTTP submission.
"""
node_ref = _first(packet, "fromId", "from_id", "from", default=None)
if node_ref is None:
node_ref = _first(decoded, "num", default=None)
@@ -215,12 +235,32 @@ def store_position_packet(packet: Mapping, decoded: Mapping) -> None:
def base64_payload(payload_bytes: bytes | None) -> str | None:
"""Encode raw payload bytes for JSON transport.
Parameters:
payload_bytes: Optional payload to encode. ``None`` is returned when
the payload is empty or missing.
Returns:
The Base64 encoded payload string or ``None`` when no payload exists.
"""
if not payload_bytes:
return None
return base64.b64encode(payload_bytes).decode("ascii")
def store_telemetry_packet(packet: Mapping, decoded: Mapping) -> None:
"""Persist telemetry metrics extracted from a packet.
Parameters:
packet: Packet metadata received from the radio interface.
decoded: Meshtastic-decoded view containing telemetry structures.
Returns:
``None``. The telemetry payload is added to the HTTP queue.
"""
telemetry_section = (
decoded.get("telemetry") if isinstance(decoded, Mapping) else None
)
@@ -392,6 +432,17 @@ def store_telemetry_packet(packet: Mapping, decoded: Mapping) -> None:
def store_nodeinfo_packet(packet: Mapping, decoded: Mapping) -> None:
"""Persist node information updates.
Parameters:
packet: Raw packet metadata describing the update.
decoded: Decoded payload that may include ``user`` and ``position``
sections.
Returns:
``None``. The node payload is merged into the API queue.
"""
payload_bytes = _extract_payload_bytes(decoded)
node_info = _decode_nodeinfo_payload(payload_bytes)
decoded_user = decoded.get("user")
@@ -544,6 +595,16 @@ def store_nodeinfo_packet(packet: Mapping, decoded: Mapping) -> None:
def store_neighborinfo_packet(packet: Mapping, decoded: Mapping) -> None:
"""Persist neighbour information gathered from a packet.
Parameters:
packet: Raw Meshtastic packet metadata.
decoded: Decoded view containing the neighbour information section.
Returns:
``None``. The neighbour snapshot is queued for submission.
"""
neighbor_section = (
decoded.get("neighborinfo") if isinstance(decoded, Mapping) else None
)
@@ -637,6 +698,15 @@ def store_neighborinfo_packet(packet: Mapping, decoded: Mapping) -> None:
def store_packet_dict(packet: Mapping) -> None:
"""Route a decoded packet to the appropriate storage handler.
Parameters:
packet: Packet dictionary emitted by the mesh interface.
Returns:
``None``. Side-effects depend on the specific handler invoked.
"""
decoded = packet.get("decoded") or {}
portnum_raw = _first(decoded, "portnum", default=None)
@@ -733,6 +803,17 @@ def store_packet_dict(packet: Mapping) -> None:
def on_receive(packet, interface) -> None:
"""Callback registered with Meshtastic to capture incoming packets.
Parameters:
packet: Packet payload supplied by the Meshtastic pubsub topic.
interface: Interface instance that produced the packet. Only used for
compatibility with Meshtastic's callback signature.
Returns:
``None``. Packets are serialised and enqueued asynchronously.
"""
if isinstance(packet, dict):
if packet.get("_potatomesh_seen"):
return

View File

@@ -1,4 +1,4 @@
"""Mesh interface discovery helpers."""
"""Mesh interface discovery helpers for interacting with Meshtastic hardware."""
from __future__ import annotations
@@ -181,7 +181,14 @@ class _DummySerialInterface:
def _parse_ble_target(value: str) -> str | None:
"""Return an uppercase BLE MAC address when ``value`` matches the format."""
"""Return an uppercase BLE MAC address when ``value`` matches the format.
Parameters:
value: User-provided target string.
Returns:
The normalised MAC address or ``None`` when validation fails.
"""
if not value:
return None
@@ -194,7 +201,14 @@ def _parse_ble_target(value: str) -> str | None:
def _parse_network_target(value: str) -> tuple[str, int] | None:
"""Return ``(host, port)`` when ``value`` is an IP address string."""
"""Return ``(host, port)`` when ``value`` is an IP address string.
Parameters:
value: Hostname or URL describing the TCP interface.
Returns:
A ``(host, port)`` tuple or ``None`` when parsing fails.
"""
if not value:
return None
@@ -240,7 +254,14 @@ def _parse_network_target(value: str) -> tuple[str, int] | None:
def _load_ble_interface():
"""Return :class:`meshtastic.ble_interface.BLEInterface` when available."""
"""Return :class:`meshtastic.ble_interface.BLEInterface` when available.
Returns:
The resolved BLE interface class.
Raises:
RuntimeError: If the BLE dependencies are not installed.
"""
global BLEInterface
if BLEInterface is not None:
@@ -267,7 +288,14 @@ def _load_ble_interface():
def _create_serial_interface(port: str) -> tuple[object, str]:
"""Return an appropriate mesh interface for ``port``."""
"""Return an appropriate mesh interface for ``port``.
Parameters:
port: User-supplied port string which may represent serial, BLE or TCP.
Returns:
``(interface, resolved_target)`` describing the created interface.
"""
port_value = (port or "").strip()
if port_value.lower() in {"", "mock", "none", "null", "disabled"}:
@@ -294,7 +322,7 @@ class NoAvailableMeshInterface(RuntimeError):
def _default_serial_targets() -> list[str]:
"""Return a list of candidate serial device paths for auto-discovery."""
"""Return candidate serial device paths for auto-discovery."""
candidates: list[str] = []
seen: set[str] = set()
@@ -309,7 +337,14 @@ def _default_serial_targets() -> list[str]:
def _create_default_interface() -> tuple[object, str]:
"""Attempt to create the default mesh interface, raising on failure."""
"""Attempt to create the default mesh interface, raising on failure.
Returns:
``(interface, resolved_target)`` for the discovered connection.
Raises:
NoAvailableMeshInterface: When no usable connection can be created.
"""
errors: list[tuple[str, Exception]] = []
for candidate in _default_serial_targets():

View File

@@ -40,7 +40,14 @@ def _post_json(
instance: str | None = None,
api_token: str | None = None,
) -> None:
"""Send a JSON payload to the configured web API."""
"""Send a JSON payload to the configured web API.
Parameters:
path: API path relative to the configured instance root.
payload: JSON-serialisable body to transmit.
instance: Optional override for :data:`config.INSTANCE`.
api_token: Optional override for :data:`config.API_TOKEN`.
"""
if instance is None:
instance = config.INSTANCE
@@ -70,7 +77,14 @@ def _enqueue_post_json(
*,
state: QueueState = STATE,
) -> None:
"""Store a POST request in the priority queue."""
"""Store a POST request in the priority queue.
Parameters:
path: API path for the queued request.
payload: JSON-serialisable body.
priority: Lower values execute first.
state: Shared queue state, injectable for testing.
"""
with state.lock:
counter = next(state.counter)
@@ -80,7 +94,12 @@ def _enqueue_post_json(
def _drain_post_queue(
state: QueueState = STATE, send: Callable[[str, dict], None] | None = None
) -> None:
"""Process queued POST requests in priority order."""
"""Process queued POST requests in priority order.
Parameters:
state: Queue container holding pending items.
send: Optional callable used to transmit requests.
"""
if send is None:
send = _post_json
@@ -105,7 +124,15 @@ def _queue_post_json(
state: QueueState = STATE,
send: Callable[[str, dict], None] | None = None,
) -> None:
"""Queue a POST request and start processing if idle."""
"""Queue a POST request and start processing if idle.
Parameters:
path: API path for the request.
payload: JSON payload to send.
priority: Scheduling priority where lower values run first.
state: Queue container used to store pending requests.
send: Optional transport override, primarily for tests.
"""
if send is None:
send = _post_json
@@ -119,7 +146,11 @@ def _queue_post_json(
def _clear_post_queue(state: QueueState = STATE) -> None:
"""Clear the pending POST queue (used by tests)."""
"""Clear the pending POST queue.
Parameters:
state: Queue state to reset. Defaults to the global queue.
"""
with state.lock:
state.queue.clear()

View File

@@ -1,4 +1,8 @@
"""Utilities for converting Meshtastic structures into JSON-friendly forms."""
"""Utilities for converting Meshtastic structures into JSON-friendly forms.
The helpers normalise loosely structured Meshtastic packets so they can be
forwarded to the web application using predictable field names and types.
"""
from __future__ import annotations
@@ -15,12 +19,33 @@ from google.protobuf.message import Message as ProtoMessage
def _get(obj, key, default=None):
"""Return ``obj[key]`` or ``getattr(obj, key)`` when available.
Parameters:
obj: Mapping or object supplying attributes.
key: Name of the attribute or mapping key to retrieve.
default: Fallback value when ``key`` is not present.
Returns:
The resolved value or ``default`` if the lookup fails.
"""
if isinstance(obj, dict):
return obj.get(key, default)
return getattr(obj, key, default)
def _node_to_dict(n) -> dict:
"""Convert ``n`` into a JSON-serialisable mapping.
Parameters:
n: Arbitrary data structure, commonly a protobuf message, dataclass or
nested containers produced by Meshtastic.
Returns:
A plain dictionary containing recursively converted values.
"""
def _convert(value):
if isinstance(value, dict):
return {k: _convert(v) for k, v in value.items()}
@@ -61,11 +86,23 @@ def _node_to_dict(n) -> dict:
def upsert_payload(node_id, node) -> dict:
"""Return the payload expected by ``/api/nodes`` upsert requests.
Parameters:
node_id: Canonical node identifier.
node: Node representation to convert with :func:`_node_to_dict`.
Returns:
A mapping keyed by ``node_id`` describing the node.
"""
ndict = _node_to_dict(node)
return {node_id: ndict}
def _iso(ts: int | float) -> str:
"""Convert ``ts`` into an ISO-8601 timestamp in UTC."""
import datetime
return (
@@ -76,6 +113,18 @@ def _iso(ts: int | float) -> str:
def _first(d, *names, default=None):
"""Return the first matching attribute or key from ``d``.
Parameters:
d: Mapping or object providing nested attributes.
*names: Candidate names, optionally using ``dot.separated`` notation
for nested lookups.
default: Value returned when no candidates succeed.
Returns:
The first non-empty value encountered or ``default``.
"""
def _mapping_get(obj, key):
if isinstance(obj, Mapping) and key in obj:
return True, obj[key]
@@ -105,6 +154,15 @@ def _first(d, *names, default=None):
def _coerce_int(value):
"""Best-effort conversion of ``value`` to an integer.
Parameters:
value: Any type supported by Meshtastic payloads.
Returns:
An integer or ``None`` when conversion is not possible.
"""
if value is None:
return None
if isinstance(value, bool):
@@ -134,6 +192,15 @@ def _coerce_int(value):
def _coerce_float(value):
"""Best-effort conversion of ``value`` to a float.
Parameters:
value: Any type supported by Meshtastic payloads.
Returns:
A float or ``None`` when conversion fails or results in ``NaN``.
"""
if value is None:
return None
if isinstance(value, bool):
@@ -159,6 +226,15 @@ def _coerce_float(value):
def _pkt_to_dict(packet) -> dict:
"""Normalise a packet into a plain dictionary.
Parameters:
packet: Packet object or mapping emitted by Meshtastic.
Returns:
A dictionary representation suitable for downstream processing.
"""
if isinstance(packet, dict):
return packet
if isinstance(packet, ProtoMessage):
@@ -179,6 +255,15 @@ def _pkt_to_dict(packet) -> dict:
def _canonical_node_id(value) -> str | None:
"""Convert node identifiers into the canonical ``!xxxxxxxx`` format.
Parameters:
value: Input identifier which may be an int, float or string.
Returns:
The canonical identifier or ``None`` if conversion fails.
"""
if value is None:
return None
if isinstance(value, (int, float)):
@@ -218,6 +303,15 @@ def _canonical_node_id(value) -> str | None:
def _node_num_from_id(node_id) -> int | None:
"""Extract the numeric node ID from a canonical identifier.
Parameters:
node_id: Identifier value accepted by :func:`_canonical_node_id`.
Returns:
The numeric node ID or ``None`` when parsing fails.
"""
if node_id is None:
return None
if isinstance(node_id, (int, float)):
@@ -246,6 +340,17 @@ def _node_num_from_id(node_id) -> int | None:
def _merge_mappings(base, extra):
"""Merge two mapping-like objects recursively.
Parameters:
base: Existing mapping or mapping-like structure.
extra: Mapping or compatible object whose entries should overlay
``base``.
Returns:
A new dictionary containing the merged values.
"""
base_dict: dict
if isinstance(base, Mapping):
base_dict = dict(base)
@@ -271,6 +376,15 @@ def _merge_mappings(base, extra):
def _extract_payload_bytes(decoded_section: Mapping) -> bytes | None:
"""Return raw payload bytes from ``decoded_section`` when available.
Parameters:
decoded_section: Mapping that may include a ``payload`` entry.
Returns:
Raw payload bytes or ``None`` when the payload is missing or invalid.
"""
if not isinstance(decoded_section, Mapping):
return None
payload = decoded_section.get("payload")
@@ -292,6 +406,15 @@ def _extract_payload_bytes(decoded_section: Mapping) -> bytes | None:
def _decode_nodeinfo_payload(payload_bytes):
"""Decode ``NodeInfo`` protobuf payloads from raw bytes.
Parameters:
payload_bytes: Serialized protobuf data from a NODEINFO packet.
Returns:
A :class:`meshtastic.protobuf.mesh_pb2.NodeInfo` instance or ``None``.
"""
if not payload_bytes:
return None
try:
@@ -315,6 +438,16 @@ def _decode_nodeinfo_payload(payload_bytes):
def _nodeinfo_metrics_dict(node_info) -> dict | None:
"""Extract device metric fields from a NodeInfo message.
Parameters:
node_info: Parsed NodeInfo protobuf message.
Returns:
A dictionary containing selected metric fields, or ``None`` when no
metrics are present.
"""
if not node_info:
return None
metrics_field_names = {f[0].name for f in node_info.ListFields()}
@@ -343,6 +476,15 @@ def _nodeinfo_metrics_dict(node_info) -> dict | None:
def _nodeinfo_position_dict(node_info) -> dict | None:
"""Return a dictionary view of positional data from NodeInfo.
Parameters:
node_info: Parsed NodeInfo protobuf message.
Returns:
A dictionary of positional fields or ``None`` if no data exists.
"""
if not node_info:
return None
fields = {f[0].name for f in node_info.ListFields()}
@@ -388,6 +530,18 @@ def _nodeinfo_position_dict(node_info) -> dict | None:
def _nodeinfo_user_dict(node_info, decoded_user):
"""Combine protobuf and decoded user information into a mapping.
Parameters:
node_info: Parsed NodeInfo protobuf message that may contain a ``user``
field.
decoded_user: Mapping or protobuf message representing decoded user
data from the packet payload.
Returns:
A merged mapping of user information or ``None`` when no data exists.
"""
user_dict = None
if node_info:
field_names = {f[0].name for f in node_info.ListFields()}

View File

@@ -14,6 +14,8 @@
# See the License for the specific language governing permissions and
# limitations under the License.
"""Interactive debugging helpers for live Meshtastic sessions."""
import time, json, base64, threading
from pubsub import pub # comes with meshtastic
from meshtastic.serial_interface import SerialInterface
@@ -28,7 +30,14 @@ stop = threading.Event()
def to_jsonable(obj):
"""Recursively convert protobuf/bytes/etc. into JSON-serializable structures."""
"""Recursively convert complex objects into JSON-serialisable structures.
Parameters:
obj: Any Meshtastic-related payload or protobuf message.
Returns:
A structure composed of standard Python types.
"""
if obj is None:
return None
if isinstance(obj, ProtoMessage):
@@ -49,7 +58,14 @@ def to_jsonable(obj):
def extract_text(d):
"""Best-effort pull of decoded text from a dict produced by to_jsonable()."""
"""Best-effort pull of decoded text from :func:`to_jsonable` output.
Parameters:
d: Mapping derived from :func:`to_jsonable`.
Returns:
The decoded text when available, otherwise ``None``.
"""
dec = d.get("decoded") or {}
# Text packets usually at decoded.payload.text
payload = dec.get("payload") or {}
@@ -62,6 +78,12 @@ def extract_text(d):
def on_receive(packet, interface):
"""Display human-readable output for each received packet.
Parameters:
packet: Packet instance supplied by Meshtastic.
interface: Interface that produced the packet.
"""
global packet_count, last_rx_ts
packet_count += 1
last_rx_ts = time.time()
@@ -86,14 +108,20 @@ def on_receive(packet, interface):
def on_connected(interface, *args, **kwargs):
"""Log when a connection is established."""
print("[info] connection established")
def on_disconnected(interface, *args, **kwargs):
"""Log when the interface disconnects."""
print("[info] disconnected")
def main():
"""Run the interactive debugging loop."""
print(f"Opening Meshtastic on {PORT}")
# Use PubSub topics (reliable in current meshtastic)

View File

@@ -1,9 +1,17 @@
#!/usr/bin/env python3
import json, os, signal, sys, time, threading
"""Utility script to dump Meshtastic traffic for offline analysis."""
from __future__ import annotations
import json
import os
import signal
import sys
import time
from datetime import datetime, timezone
from meshtastic.serial_interface import SerialInterface
from meshtastic.mesh_interface import MeshInterface
from meshtastic.serial_interface import SerialInterface
from pubsub import pub
PORT = os.environ.get("MESH_SERIAL", "/dev/ttyACM0")
@@ -13,11 +21,20 @@ OUT = os.environ.get("MESH_DUMP_FILE", "meshtastic-dump.ndjson")
f = open(OUT, "a", buffering=1, encoding="utf-8")
def now():
def now() -> str:
"""Return the current UTC timestamp in ISO 8601 format."""
return datetime.now(timezone.utc).isoformat()
def write(kind, payload):
def write(kind: str, payload: dict) -> None:
"""Append a JSON record to the dump file.
Parameters:
kind: Logical record type such as ``"packet"`` or ``"node"``.
payload: Serializable payload containing the record body.
"""
rec = {"ts": now(), "kind": kind, **payload}
f.write(json.dumps(rec, ensure_ascii=False, default=str) + "\n")
@@ -28,12 +45,26 @@ iface: MeshInterface = SerialInterface(PORT)
# Packet callback: every RF/Mesh packet the node receives/decodes lands here
def on_packet(packet, iface):
"""Write packet metadata whenever the radio receives a frame.
Parameters:
packet: Meshtastic packet object or dictionary.
iface: Interface instance delivering the packet.
"""
# 'packet' already includes decoded fields when available (portnum, payload, position, telemetry, etc.)
write("packet", {"packet": packet})
# Node callback: topology/metadata updates (nodeinfo, hops, lastHeard, etc.)
def on_node(node, iface):
"""Write node metadata updates produced by Meshtastic.
Parameters:
node: Meshtastic node object or mapping.
iface: Interface instance emitting the update.
"""
write("node", {"node": node})
@@ -57,6 +88,8 @@ except Exception as e:
# Keep the process alive until Ctrl-C
def _stop(signum, frame):
"""Handle termination signals by flushing buffers and exiting."""
write("meta", {"event": "stopping"})
try:
try:

View File

@@ -2,6 +2,9 @@ import base64
import importlib
import sys
import types
"""End-to-end tests covering the mesh ingestion package."""
from dataclasses import dataclass
from pathlib import Path
from types import SimpleNamespace
@@ -11,7 +14,7 @@ import pytest
@pytest.fixture
def mesh_module(monkeypatch):
"""Import data.mesh with stubbed dependencies."""
"""Import :mod:`data.mesh` with stubbed dependencies."""
repo_root = Path(__file__).resolve().parents[1]
monkeypatch.syspath_prepend(str(repo_root))