Floor override frequencies to MHz integers (#476)

* Floor override frequencies to MHz integers

* Handle stub protobuf nodeinfo dictionaries

* Handle zero override frequency defaults

* Add defensive serialization coverage

* Stabilize nodeinfo user role normalization test

data/mesh_ingestor/config.py

@@ -66,8 +66,8 @@ API_TOKEN = os.environ.get("API_TOKEN", "")
 ENERGY_SAVING = os.environ.get("ENERGY_SAVING") == "1"
 """When ``True``, enables the ingestor's energy saving mode."""
 
-LORA_FREQ: int | None = None
-"""Frequency of the local node's configured LoRa region in MHz."""
+LORA_FREQ: float | int | str | None = None
+"""Frequency of the local node's configured LoRa region in MHz or raw region label."""
 
 MODEM_PRESET: str | None = None
 """CamelCase modem preset name reported by the local node."""

data/mesh_ingestor/interfaces.py

@@ -20,6 +20,7 @@ import contextlib
 import glob
 import importlib
 import ipaddress
+import math
 import re
 import sys
 import urllib.parse
@@ -471,11 +472,24 @@ def _resolve_lora_message(local_config: Any) -> Any | None:
     return None
 
 
-def _region_frequency(lora_message: Any) -> int | None:
-    """Derive the LoRa region frequency in MHz from ``lora_message``."""
+def _region_frequency(lora_message: Any) -> int | float | str | None:
+    """Derive the LoRa region frequency in MHz or the region label from ``lora_message``.
+
+    Numeric override values are floored to the nearest MHz to align with the
+    integer frequencies expected elsewhere in the ingestion pipeline.
+    """
 
     if lora_message is None:
         return None
+
+    override_frequency = getattr(lora_message, "override_frequency", None)
+    if override_frequency is not None:
+        if isinstance(override_frequency, (int, float)):
+            if override_frequency > 0:
+                return math.floor(override_frequency)
+        elif override_frequency:
+            return override_frequency
+
     region_value = getattr(lora_message, "region", None)
     if region_value is None:
         return None
@@ -494,8 +508,11 @@ def _region_frequency(lora_message: Any) -> int | None:
                 return int(token)
             except ValueError:  # pragma: no cover - defensive only
                 continue
+        return enum_name
     if isinstance(region_value, int) and region_value >= 100:
         return region_value
+    if isinstance(region_value, str) and region_value:
+        return region_value
     return None
 
 

data/mesh_ingestor/serialization.py

@@ -172,6 +172,12 @@ def _node_to_dict(n) -> dict:
         if dataclasses.is_dataclass(value):
             return {k: _convert(getattr(value, k)) for k in value.__dataclass_fields__}
         if isinstance(value, ProtoMessage):
+            manual_to_dict = getattr(value, "to_dict", None)
+            if callable(manual_to_dict):
+                try:
+                    return manual_to_dict()
+                except Exception:
+                    pass
             try:
                 return MessageToDict(
                     value,
@@ -715,6 +721,12 @@ def _nodeinfo_user_dict(node_info, decoded_user):
     if node_info:
         field_names = {f[0].name for f in node_info.ListFields()}
         if "user" in field_names:
+            manual_to_dict = getattr(node_info.user, "to_dict", None)
+            if callable(manual_to_dict):
+                try:
+                    user_dict = manual_to_dict()
+                except Exception:
+                    user_dict = None
             try:
                 user_dict = MessageToDict(
                     node_info.user,
@@ -723,16 +735,28 @@ def _nodeinfo_user_dict(node_info, decoded_user):
                 )
             except Exception:
                 user_dict = _node_to_dict(node_info.user)
+            if user_dict is None and callable(manual_to_dict):
+                try:
+                    user_dict = manual_to_dict()
+                except Exception:
+                    user_dict = None
 
     if isinstance(decoded_user, ProtoMessage):
-        try:
-            decoded_user = MessageToDict(
-                decoded_user,
-                preserving_proto_field_name=False,
-                use_integers_for_enums=False,
-            )
-        except Exception:
-            decoded_user = _node_to_dict(decoded_user)
+        manual_to_dict = getattr(decoded_user, "to_dict", None)
+        if callable(manual_to_dict):
+            try:
+                decoded_user = manual_to_dict()
+            except Exception:
+                decoded_user = decoded_user
+        if isinstance(decoded_user, ProtoMessage):
+            try:
+                decoded_user = MessageToDict(
+                    decoded_user,
+                    preserving_proto_field_name=False,
+                    use_integers_for_enums=False,
+                )
+            except Exception:
+                decoded_user = _node_to_dict(decoded_user)
 
     if isinstance(decoded_user, Mapping):
         user_dict = _merge_mappings(user_dict, decoded_user)

tests/test_helpers.py

@@ -205,7 +205,10 @@ def test_region_frequency_and_resolution_helpers():
 
     class EnumType:
         def __init__(self):
-            self.values_by_number = {1: EnumValue("REGION_915")}
+            self.values_by_number = {
+                1: EnumValue("REGION_915"),
+                2: EnumValue("US"),
+            }
 
     class FieldDesc:
         def __init__(self):
@@ -216,13 +219,33 @@ def test_region_frequency_and_resolution_helpers():
             self.fields_by_name = {"region": FieldDesc()}
 
     class LoraMessage:
-        def __init__(self, region):
+        def __init__(self, region, override_frequency=None):
             self.region = region
+            self.override_frequency = override_frequency
             self.DESCRIPTOR = Descriptor()
 
     freq = interfaces._region_frequency(LoraMessage(1))
     assert freq == 915
 
+    freq = interfaces._region_frequency(LoraMessage(1, override_frequency=0))
+    assert freq == 915
+
+    freq = interfaces._region_frequency(LoraMessage(1, override_frequency=921.5))
+    assert freq == 921
+
+    freq = interfaces._region_frequency(LoraMessage(1, override_frequency="915MHz"))
+    assert freq == "915MHz"
+
+    freq = interfaces._region_frequency(LoraMessage(2))
+    assert freq == "US"
+
+    class StringRegionMessage:
+        def __init__(self, region):
+            self.region = region
+
+    freq = interfaces._region_frequency(StringRegionMessage("EU"))
+    assert freq == "EU"
+
     class LocalConfig:
         def __init__(self, lora):
             self.lora = lora

tests/test_serialization_unit.py

@@ -0,0 +1,392 @@
+# Copyright © 2025-26 l5yth & contributors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Focused serialization helper coverage for defensive branches."""
+
+from __future__ import annotations
+
+import builtins
+import importlib
+import sys
+import types
+from typing import Any
+
+import pytest
+
+from data.mesh_ingestor import serialization
+
+
+class _StubFieldDesc:
+    def __init__(self, name: str) -> None:
+        self.name = name
+
+
+class _StubContainer:
+    def __init__(self, fields: dict[str, Any]) -> None:
+        self._fields = fields
+
+    def ListFields(self):  # noqa: D401 - protobuf-compatible stub
+        return [(_StubFieldDesc(name), value) for name, value in self._fields.items()]
+
+
+class _StubProto(serialization.ProtoMessage):
+    """Simple ProtoMessage subclass usable for monkeypatched checks."""
+
+    def __init__(self) -> None:
+        self._copied_from = None
+
+    def ParseFromString(
+        self, payload: bytes
+    ) -> None:  # noqa: D401 - protobuf-compatible stub
+        raise serialization.DecodeError("boom")
+
+    def CopyFrom(self, other):  # noqa: D401 - protobuf-compatible stub
+        self._copied_from = other
+
+
+@pytest.fixture(autouse=True)
+def reset_cli_cache(monkeypatch):
+    """Ensure the CLI lookup cache is cleared between tests."""
+
+    serialization._reset_cli_role_cache()
+    monkeypatch.setattr(serialization, "_CLI_ROLE_LOOKUP", None, raising=False)
+    yield
+    serialization._reset_cli_role_cache()
+
+
+def test_load_cli_role_lookup_prefers_cache(monkeypatch):
+    """Return cached CLI role mappings without re-importing modules."""
+
+    sentinel = {1: "ADMIN"}
+    monkeypatch.setattr(serialization, "_CLI_ROLE_LOOKUP", sentinel, raising=False)
+    assert serialization._load_cli_role_lookup() is sentinel
+
+
+def test_load_cli_role_lookup_members_mapping(monkeypatch):
+    """Resolve role names from a stub module exposing a __members__ mapping."""
+
+    stub_module = types.SimpleNamespace()
+
+    class MembersRole:
+        __members__ = {
+            "one": types.SimpleNamespace(value=1),
+            "TWO": types.SimpleNamespace(value=2),
+        }
+
+    stub_module.Roles = MembersRole
+
+    def fake_import(name):
+        if name == serialization._CLI_ROLE_MODULE_NAMES[0]:
+            return stub_module
+        raise ImportError("skip")
+
+    monkeypatch.setattr(importlib, "import_module", fake_import)
+    lookup = serialization._load_cli_role_lookup()
+    assert lookup == {1: "ONE", 2: "TWO"}
+
+
+def test_load_cli_role_lookup_skips_empty_candidates(monkeypatch):
+    """Skip candidates that do not expose usable members."""
+
+    stub_module = types.SimpleNamespace(Role=types.SimpleNamespace(__members__={}))
+
+    def fake_import(name):
+        if name == serialization._CLI_ROLE_MODULE_NAMES[0]:
+            return stub_module
+        raise ImportError("skip")
+
+    monkeypatch.setattr(importlib, "import_module", fake_import)
+    assert serialization._load_cli_role_lookup() == {}
+
+
+class _ExplodingStr:
+    def __str__(self) -> str:  # noqa: D401 - custom str to trigger json.dumps fallback
+        return "repr"
+
+
+def test_node_to_dict_json_error(monkeypatch):
+    """Fallback to ``str`` conversion when ``json.dumps`` raises."""
+
+    def boom(*_args, **_kwargs):
+        raise ValueError("explode")
+
+    monkeypatch.setattr(serialization.json, "dumps", boom)
+    result = serialization._node_to_dict(_ExplodingStr())
+    assert result == "repr"
+
+
+@pytest.mark.parametrize(
+    "value,expected",
+    [
+        ("   ", None),
+        (object(), None),
+    ],
+)
+def test_normalize_user_role_edge_cases(value, expected):
+    """Normalize blank strings and uncoercible values to ``None``."""
+
+    assert serialization._normalize_user_role(value) is expected
+
+
+def test_coerce_float_value_error():
+    """Invalid strings should return ``None`` when conversion fails."""
+
+    assert serialization._coerce_float("not-a-number") is None
+
+
+class _BadProto(serialization.ProtoMessage):
+    """Proto-like object that raises from MessageToDict and to_dict."""
+
+    def __str__(self) -> str:  # noqa: D401 - string representation for fallbacks
+        return "bad-proto"
+
+    def to_dict(self):  # noqa: D401 - protobuf-compatible stub
+        raise ValueError("nope")
+
+
+def test_pkt_to_dict_fallbacks(monkeypatch):
+    """Exercise MessageToDict and to_dict failure handling."""
+
+    def fail_message_to_dict(*_args, **_kwargs):
+        raise Exception("fail")
+
+    monkeypatch.setattr(serialization, "MessageToDict", fail_message_to_dict)
+    result = serialization._pkt_to_dict(_BadProto())
+    assert "_unparsed" not in result
+    assert isinstance(result, str)
+
+
+class _Digitish(str):
+    def isdigit(self) -> bool:  # noqa: D401 - force digit handling despite letters
+        return True
+
+    def strip(self) -> "_Digitish":  # noqa: D401 - preserve subclass through stripping
+        return self
+
+
+@pytest.mark.parametrize(
+    "value,expected",
+    [
+        (float("nan"), None),
+        (-5, None),
+        (object(), None),
+        ("^alias", "^alias"),
+        (_Digitish("xyz"), None),
+        ("!", None),
+    ],
+)
+def test_canonical_node_id_defensive_paths(value, expected):
+    """Cover defensive branches in node id normalisation."""
+
+    assert serialization._canonical_node_id(value) == expected
+
+
+@pytest.mark.parametrize(
+    "value,expected",
+    [
+        (float("nan"), None),
+        (object(), None),
+        ("not-a-number", None),
+    ],
+)
+def test_node_num_from_id_defensive_paths(value, expected):
+    """Cover numeric and string parsing failures in id extraction."""
+
+    assert serialization._node_num_from_id(value) == expected
+
+
+def test_merge_mappings_with_non_mapping_extra():
+    """Ignore extras that cannot be converted into mappings."""
+
+    assert serialization._merge_mappings({"a": 1}, 5) == {"a": 1}
+
+
+def test_extract_payload_bytes_invalid_base64():
+    """Return ``None`` for strings that are not valid base64 payloads."""
+
+    original_b64decode = serialization.base64.b64decode
+
+    def boom(_value):
+        raise ValueError("bad")
+
+    serialization.base64.b64decode = boom
+    try:
+        decoded = serialization._extract_payload_bytes({"payload": "$$$"})
+        assert decoded is None
+    finally:
+        serialization.base64.b64decode = original_b64decode
+
+
+def test_normalize_user_role_uppercases():
+    """Convert role strings to canonical uppercase names."""
+
+    assert serialization._normalize_user_role(" member ") == "MEMBER"
+
+
+def test_decode_nodeinfo_payload_import_and_parse_failures(monkeypatch):
+    """Handle import errors and parse failures when decoding NodeInfo."""
+
+    assert serialization._decode_nodeinfo_payload(b"") is None
+
+    original_import = builtins.__import__
+
+    def raising_import(name, *args, **kwargs):
+        if name.startswith("meshtastic.protobuf"):
+            raise ModuleNotFoundError(name)
+        return original_import(name, *args, **kwargs)
+
+    monkeypatch.setattr(builtins, "__import__", raising_import)
+    assert serialization._decode_nodeinfo_payload(b"payload") is None
+    monkeypatch.setattr(builtins, "__import__", original_import)
+
+    mesh_pb2 = types.SimpleNamespace(NodeInfo=_StubProto, User=_StubProto)
+    protobuf_pkg = types.SimpleNamespace(mesh_pb2=mesh_pb2)
+    monkeypatch.setitem(sys.modules, "meshtastic", types.ModuleType("meshtastic"))
+    monkeypatch.setitem(sys.modules, "meshtastic.protobuf", protobuf_pkg)
+    monkeypatch.setitem(sys.modules, "meshtastic.protobuf.mesh_pb2", mesh_pb2)
+    assert serialization._decode_nodeinfo_payload(b"payload") is None
+
+
+def test_nodeinfo_metrics_dict_handles_optional_fields():
+    """Extract only present metrics fields or return ``None`` when absent."""
+
+    assert serialization._nodeinfo_metrics_dict(None) is None
+
+    device_metrics = _StubContainer(
+        {
+            "battery_level": 5.0,
+            "humidity": 42.0,
+            "temperature": 21.5,
+            "barometric_pressure": 1000.5,
+        }
+    )
+    node_info = types.SimpleNamespace(device_metrics=device_metrics)
+    node_info.ListFields = lambda: [(_StubFieldDesc("device_metrics"), device_metrics)]
+    metrics = serialization._nodeinfo_metrics_dict(node_info)
+    assert metrics == {
+        "batteryLevel": 5.0,
+        "humidity": 42.0,
+        "temperature": 21.5,
+        "barometricPressure": 1000.5,
+    }
+
+
+def test_nodeinfo_position_dict_variant_fields():
+    """Cover integer conversions and derived latitude/longitude values."""
+
+    assert serialization._nodeinfo_position_dict(None) is None
+
+    position = _StubContainer(
+        {
+            "latitude_i": 100000000,
+            "longitude_i": 200000000,
+            "altitude": 7,
+            "ground_speed": 1.5,
+            "ground_track": 2.5,
+            "precision_bits": 3,
+            "location_source": 4,
+        }
+    )
+    node_info = types.SimpleNamespace(position=position)
+    node_info.ListFields = lambda: [(_StubFieldDesc("position"), position)]
+    result = serialization._nodeinfo_position_dict(node_info)
+    assert result == {
+        "latitudeI": 100000000,
+        "longitudeI": 200000000,
+        "latitude": 10.0,
+        "longitude": 20.0,
+        "altitude": 7,
+        "groundSpeed": 1.5,
+        "groundTrack": 2.5,
+        "precisionBits": 3,
+        "locationSource": 4,
+    }
+
+    position_with_floats = _StubContainer(
+        {
+            "latitude": 1.5,
+            "longitude": 2.5,
+        }
+    )
+    node_info_float = types.SimpleNamespace(position=position_with_floats)
+    node_info_float.ListFields = lambda: [
+        (_StubFieldDesc("position"), position_with_floats)
+    ]
+    direct_result = serialization._nodeinfo_position_dict(node_info_float)
+    assert direct_result == {"latitude": 1.5, "longitude": 2.5}
+
+
+def test_nodeinfo_user_dict_monkeypatched_paths(monkeypatch):
+    """Cover manual_to_dict failures and decoded user ProtoMessage conversion."""
+
+    monkeypatch.setattr(serialization, "_load_cli_role_lookup", lambda: {})
+
+    proto_pkg = types.SimpleNamespace()
+    failing_role = types.SimpleNamespace(
+        Name=lambda _value: (_ for _ in ()).throw(ValueError("nope"))
+    )
+    failing_user = types.SimpleNamespace(Role=failing_role)
+    monkeypatch.setitem(
+        sys.modules, "meshtastic", types.SimpleNamespace(protobuf=proto_pkg)
+    )
+    monkeypatch.setitem(sys.modules, "meshtastic.protobuf", proto_pkg)
+    monkeypatch.setitem(
+        sys.modules,
+        "meshtastic.protobuf.mesh_pb2",
+        types.SimpleNamespace(User=failing_user),
+    )
+    monkeypatch.setitem(
+        sys.modules,
+        "meshtastic.protobuf.config_pb2",
+        types.SimpleNamespace(Config=types.SimpleNamespace(DeviceConfig=failing_user)),
+    )
+
+    def raising_to_dict():
+        raise ValueError("nope")
+
+    user_proto = types.SimpleNamespace(to_dict=raising_to_dict)
+    user_proto.ListFields = lambda: []
+    node_info = types.SimpleNamespace(user=user_proto)
+    node_info.ListFields = lambda: [(_StubFieldDesc("user"), user_proto)]
+
+    def failing_message_to_dict(*_args, **_kwargs):
+        raise Exception("fail")
+
+    monkeypatch.setattr(serialization, "MessageToDict", failing_message_to_dict)
+
+    decoded_user = _BadProto()
+    decoded_user.to_dict = lambda: {"role": 0}
+
+    result = serialization._nodeinfo_user_dict(node_info, decoded_user)
+    assert result == {"role": "0"}
+
+
+def test_nodeinfo_user_dict_proto_fallback(monkeypatch):
+    """Exercise decoded user ProtoMessage fallbacks when conversions fail."""
+
+    def failing_to_dict():
+        raise ValueError("nope")
+
+    class DecodedProto(serialization.ProtoMessage):
+        def __str__(self):  # noqa: D401
+            return "decoded-proto"
+
+        to_dict = staticmethod(failing_to_dict)
+
+    def failing_message_to_dict(*_args, **_kwargs):
+        raise Exception("fail")
+
+    monkeypatch.setattr(serialization, "MessageToDict", failing_message_to_dict)
+
+    decoded_user = DecodedProto()
+    assert serialization._nodeinfo_user_dict(None, decoded_user) is None