Merge pull request #116 from kizniche/feat-int-mc-map-auto-uploader

Add automatic mesh map upload (integration/fanout module). Closes #108. Thank you!!
This commit is contained in:
Jack Kingsman
2026-03-26 18:08:34 -07:00
committed by GitHub
12 changed files with 1864 additions and 43 deletions

View File

@@ -89,6 +89,19 @@ Amazon SQS delivery. Config blob:
- Publishes a JSON envelope of the form `{"event_type":"message"|"raw_packet","data":...}` - Publishes a JSON envelope of the form `{"event_type":"message"|"raw_packet","data":...}`
- Supports both decoded messages and raw packets via normal scope selection - Supports both decoded messages and raw packets via normal scope selection
### map_upload (map_upload.py)
Uploads heard repeater and room-server advertisements to map.meshcore.dev. Config blob:
- `api_url` (optional, default `""`) — upload endpoint; empty falls back to the public map.meshcore.dev API
- `dry_run` (bool, default `true`) — when true, logs the payload at INFO level without sending
- `geofence_enabled` (bool, default `false`) — when true, only uploads nodes within `geofence_radius_km` of the radio's own configured lat/lon
- `geofence_radius_km` (float, default `0`) — filter radius in kilometres
Geofence notes:
- The reference center is always the radio's own `adv_lat`/`adv_lon` from `radio_runtime.meshcore.self_info`, read **live at upload time** — no lat/lon is stored in the fanout config itself.
- If the radio's lat/lon is `(0, 0)` or the radio is not connected, the geofence check is silently skipped so uploads continue normally until coordinates are configured.
- Requires the radio to have `ENABLE_PRIVATE_KEY_EXPORT=1` firmware to sign uploads.
- Scope is always `{"messages": "none", "raw_packets": "all"}` — only raw RF packets are processed.
## Adding a New Integration Type ## Adding a New Integration Type
### Step-by-step checklist ### Step-by-step checklist
@@ -291,6 +304,7 @@ Migrations:
- `app/fanout/webhook.py` — Webhook fanout module - `app/fanout/webhook.py` — Webhook fanout module
- `app/fanout/apprise_mod.py` — Apprise fanout module - `app/fanout/apprise_mod.py` — Apprise fanout module
- `app/fanout/sqs.py` — Amazon SQS fanout module - `app/fanout/sqs.py` — Amazon SQS fanout module
- `app/fanout/map_upload.py` — Map Upload fanout module
- `app/repository/fanout.py` — Database CRUD - `app/repository/fanout.py` — Database CRUD
- `app/routers/fanout.py` — REST API - `app/routers/fanout.py` — REST API
- `app/websocket.py``broadcast_event()` dispatches to fanout - `app/websocket.py``broadcast_event()` dispatches to fanout

View File

@@ -20,9 +20,9 @@ from datetime import datetime
from typing import Any, Protocol from typing import Any, Protocol
import aiomqtt import aiomqtt
import nacl.bindings
from app.fanout.mqtt_base import BaseMqttPublisher from app.fanout.mqtt_base import BaseMqttPublisher
from app.keystore import ed25519_sign_expanded
from app.path_utils import parse_packet_envelope, split_path_hex from app.path_utils import parse_packet_envelope, split_path_hex
from app.version_info import get_app_build_info from app.version_info import get_app_build_info
@@ -40,9 +40,6 @@ _TOKEN_RENEWAL_THRESHOLD = _TOKEN_LIFETIME - 3600 # 23 hours
_STATS_REFRESH_INTERVAL = 300 # 5 minutes _STATS_REFRESH_INTERVAL = 300 # 5 minutes
_STATS_MIN_CACHE_SECS = 60 # Don't re-fetch stats within 60s _STATS_MIN_CACHE_SECS = 60 # Don't re-fetch stats within 60s
# Ed25519 group order
_L = 2**252 + 27742317777372353535851937790883648493
# Route type mapping: bottom 2 bits of first byte # Route type mapping: bottom 2 bits of first byte
_ROUTE_MAP = {0: "F", 1: "F", 2: "D", 3: "T"} _ROUTE_MAP = {0: "F", 1: "F", 2: "D", 3: "T"}
@@ -69,28 +66,6 @@ def _base64url_encode(data: bytes) -> str:
return base64.urlsafe_b64encode(data).rstrip(b"=").decode("ascii") return base64.urlsafe_b64encode(data).rstrip(b"=").decode("ascii")
def _ed25519_sign_expanded(
message: bytes, scalar: bytes, prefix: bytes, public_key: bytes
) -> bytes:
"""Sign a message using MeshCore's expanded Ed25519 key format.
MeshCore stores 64-byte "orlp" format keys: scalar(32) || prefix(32).
Standard Ed25519 libraries expect seed format and would re-SHA-512 the key.
This performs the signing manually using the already-expanded key material.
Port of meshcore-packet-capture's ed25519_sign_with_expanded_key().
"""
# r = SHA-512(prefix || message) mod L
r = int.from_bytes(hashlib.sha512(prefix + message).digest(), "little") % _L
# R = r * B (base point multiplication)
R = nacl.bindings.crypto_scalarmult_ed25519_base_noclamp(r.to_bytes(32, "little"))
# k = SHA-512(R || public_key || message) mod L
k = int.from_bytes(hashlib.sha512(R + public_key + message).digest(), "little") % _L
# s = (r + k * scalar) mod L
s = (r + k * int.from_bytes(scalar, "little")) % _L
return R + s.to_bytes(32, "little")
def _generate_jwt_token( def _generate_jwt_token(
private_key: bytes, private_key: bytes,
public_key: bytes, public_key: bytes,
@@ -127,7 +102,7 @@ def _generate_jwt_token(
scalar = private_key[:32] scalar = private_key[:32]
prefix = private_key[32:] prefix = private_key[32:]
signature = _ed25519_sign_expanded(signing_input, scalar, prefix, public_key) signature = ed25519_sign_expanded(signing_input, scalar, prefix, public_key)
return f"{header_b64}.{payload_b64}.{signature.hex()}" return f"{header_b64}.{payload_b64}.{signature.hex()}"

View File

@@ -21,6 +21,7 @@ def _register_module_types() -> None:
return return
from app.fanout.apprise_mod import AppriseModule from app.fanout.apprise_mod import AppriseModule
from app.fanout.bot import BotModule from app.fanout.bot import BotModule
from app.fanout.map_upload import MapUploadModule
from app.fanout.mqtt_community import MqttCommunityModule from app.fanout.mqtt_community import MqttCommunityModule
from app.fanout.mqtt_private import MqttPrivateModule from app.fanout.mqtt_private import MqttPrivateModule
from app.fanout.sqs import SqsModule from app.fanout.sqs import SqsModule
@@ -32,6 +33,7 @@ def _register_module_types() -> None:
_MODULE_TYPES["webhook"] = WebhookModule _MODULE_TYPES["webhook"] = WebhookModule
_MODULE_TYPES["apprise"] = AppriseModule _MODULE_TYPES["apprise"] = AppriseModule
_MODULE_TYPES["sqs"] = SqsModule _MODULE_TYPES["sqs"] = SqsModule
_MODULE_TYPES["map_upload"] = MapUploadModule
def _matches_filter(filter_value: Any, key: str) -> bool: def _matches_filter(filter_value: Any, key: str) -> bool:

320
app/fanout/map_upload.py Normal file
View File

@@ -0,0 +1,320 @@
"""Fanout module for uploading heard advert packets to map.meshcore.dev.
Mirrors the logic of the standalone map.meshcore.dev-uploader project:
- Listens on raw RF packets via on_raw
- Filters for ADVERT packets, only processes repeaters (role 2) and rooms (role 3)
- Skips nodes with no valid location (lat/lon None)
- Applies per-pubkey rate-limiting (1-hour window, matching the uploader)
- Signs the upload request with the radio's own Ed25519 private key
- POSTs to the map API (or logs in dry-run mode)
Dry-run mode (default: True) logs the full would-be payload at INFO level
without making any HTTP requests. Disable it only after verifying the log
output looks correct — in particular the radio params (freq/bw/sf/cr) and
the raw hex link.
Config keys
-----------
api_url : str, default ""
Upload endpoint. Empty string falls back to the public map.meshcore.dev API.
dry_run : bool, default True
When True, log the payload at INFO level instead of sending it.
geofence_enabled : bool, default False
When True, only upload nodes whose location falls within geofence_radius_km of
the radio's own configured latitude/longitude (read live from the radio at upload
time — no lat/lon is stored in this config). When the radio's lat/lon is not set
(0, 0) or unavailable, the geofence check is silently skipped so uploads continue
normally until coordinates are configured.
geofence_radius_km : float, default 0.0
Radius of the geofence in kilometres. Nodes further than this distance
from the radio's own position are skipped.
"""
from __future__ import annotations
import hashlib
import json
import logging
import math
import httpx
from app.decoder import parse_advertisement, parse_packet
from app.fanout.base import FanoutModule
from app.keystore import ed25519_sign_expanded, get_private_key, get_public_key
from app.services.radio_runtime import radio_runtime
logger = logging.getLogger(__name__)
_DEFAULT_API_URL = "https://map.meshcore.dev/api/v1/uploader/node"
# Re-upload guard: skip re-uploading a pubkey seen within this window (AU parity)
_REUPLOAD_SECONDS = 3600
# Only upload repeaters (2) and rooms (3). Any other role — including future
# roles not yet defined — is rejected. An allowlist is used rather than a
# blocklist so that new roles cannot accidentally start populating the map.
_ALLOWED_DEVICE_ROLES = {2, 3}
def _get_radio_params() -> dict:
"""Read radio frequency parameters from the connected radio's self_info.
The Python meshcore library returns radio_freq in MHz (e.g. 910.525) and
radio_bw in kHz (e.g. 62.5). These are exactly the units the map API
expects, matching what the JS reference uploader produces after its own
/1000 division on raw integer values. No further scaling is applied here.
"""
try:
mc = radio_runtime.meshcore
if not mc:
return {"freq": 0, "cr": 0, "sf": 0, "bw": 0}
info = mc.self_info
if not isinstance(info, dict):
return {"freq": 0, "cr": 0, "sf": 0, "bw": 0}
freq = info.get("radio_freq", 0) or 0
bw = info.get("radio_bw", 0) or 0
sf = info.get("radio_sf", 0) or 0
cr = info.get("radio_cr", 0) or 0
return {
"freq": freq,
"cr": cr,
"sf": sf,
"bw": bw,
}
except Exception as exc:
logger.debug("MapUpload: could not read radio params: %s", exc)
return {"freq": 0, "cr": 0, "sf": 0, "bw": 0}
_ROLE_NAMES: dict[int, str] = {2: "repeater", 3: "room"}
def _haversine_km(lat1: float, lon1: float, lat2: float, lon2: float) -> float:
"""Return the great-circle distance in kilometres between two lat/lon points."""
r = 6371.0
phi1, phi2 = math.radians(lat1), math.radians(lat2)
dphi = math.radians(lat2 - lat1)
dlam = math.radians(lon2 - lon1)
a = math.sin(dphi / 2) ** 2 + math.cos(phi1) * math.cos(phi2) * math.sin(dlam / 2) ** 2
return 2 * r * math.asin(math.sqrt(a))
class MapUploadModule(FanoutModule):
"""Uploads heard ADVERT packets to the MeshCore community map."""
def __init__(self, config_id: str, config: dict, *, name: str = "") -> None:
super().__init__(config_id, config, name=name)
self._client: httpx.AsyncClient | None = None
self._last_error: str | None = None
# Per-pubkey rate limiting: pubkey_hex -> last_uploaded_advert_timestamp
self._seen: dict[str, int] = {}
async def start(self) -> None:
self._client = httpx.AsyncClient(timeout=httpx.Timeout(15.0))
self._last_error = None
self._seen.clear()
async def stop(self) -> None:
if self._client:
await self._client.aclose()
self._client = None
self._last_error = None
async def on_raw(self, data: dict) -> None:
if data.get("payload_type") != "ADVERT":
return
raw_hex = data.get("data", "")
if not raw_hex:
return
try:
raw_bytes = bytes.fromhex(raw_hex)
except ValueError:
return
packet_info = parse_packet(raw_bytes)
if packet_info is None:
return
advert = parse_advertisement(packet_info.payload, raw_packet=raw_bytes)
if advert is None:
return
# TODO: advert Ed25519 signature verification is skipped here.
# The radio has already validated the packet before passing it to RT,
# so re-verification is redundant in practice. If added, verify that
# nacl.bindings.crypto_sign_open(sig + (pubkey_bytes || timestamp_bytes),
# advert.public_key_bytes) succeeds before proceeding.
# Only process repeaters (2) and rooms (3) — any other role is rejected
if advert.device_role not in _ALLOWED_DEVICE_ROLES:
return
# Skip nodes with no valid location — the decoder already nulls out
# impossible values, so None means either no location flag or bad coords.
if advert.lat is None or advert.lon is None:
logger.debug(
"MapUpload: skipping %s — no valid location",
advert.public_key[:12],
)
return
pubkey = advert.public_key.lower()
# Rate-limit: skip if this pubkey's timestamp hasn't advanced enough
last_seen = self._seen.get(pubkey)
if last_seen is not None:
if last_seen >= advert.timestamp:
logger.debug(
"MapUpload: skipping %s — possible replay (last=%d, advert=%d)",
pubkey[:12],
last_seen,
advert.timestamp,
)
return
if advert.timestamp < last_seen + _REUPLOAD_SECONDS:
logger.debug(
"MapUpload: skipping %s — within 1-hr rate-limit window (delta=%ds)",
pubkey[:12],
advert.timestamp - last_seen,
)
return
await self._upload(
pubkey, advert.timestamp, advert.device_role, raw_hex, advert.lat, advert.lon
)
async def _upload(
self,
pubkey: str,
advert_timestamp: int,
device_role: int,
raw_hex: str,
lat: float,
lon: float,
) -> None:
# Geofence check: if enabled, skip nodes outside the configured radius.
# The reference center is the radio's own lat/lon read live from self_info —
# no coordinates are stored in the fanout config. If the radio lat/lon is
# (0, 0) or unavailable the check is skipped transparently so uploads
# continue normally until the operator sets coordinates in radio settings.
geofence_dist_km: float | None = None
if self.config.get("geofence_enabled"):
try:
mc = radio_runtime.meshcore
sinfo = mc.self_info if mc else None
fence_lat = float((sinfo or {}).get("adv_lat", 0) or 0)
fence_lon = float((sinfo or {}).get("adv_lon", 0) or 0)
except Exception as exc:
logger.debug("MapUpload: could not read radio lat/lon for geofence: %s", exc)
fence_lat = 0.0
fence_lon = 0.0
if fence_lat == 0.0 and fence_lon == 0.0:
logger.debug(
"MapUpload: geofence skipped for %s — radio lat/lon not configured",
pubkey[:12],
)
else:
fence_radius_km = float(self.config.get("geofence_radius_km", 0) or 0)
geofence_dist_km = _haversine_km(fence_lat, fence_lon, lat, lon)
if geofence_dist_km > fence_radius_km:
logger.debug(
"MapUpload: skipping %s — outside geofence (%.2f km > %.2f km)",
pubkey[:12],
geofence_dist_km,
fence_radius_km,
)
return
private_key = get_private_key()
public_key = get_public_key()
if private_key is None or public_key is None:
logger.warning(
"MapUpload: private key not available — cannot sign upload for %s. "
"Ensure radio firmware has ENABLE_PRIVATE_KEY_EXPORT=1.",
pubkey[:12],
)
return
api_url = str(self.config.get("api_url", "") or _DEFAULT_API_URL).strip()
dry_run = bool(self.config.get("dry_run", True))
role_name = _ROLE_NAMES.get(device_role, f"role={device_role}")
params = _get_radio_params()
upload_data = {
"params": params,
"links": [f"meshcore://{raw_hex}"],
}
# Sign: SHA-256 the compact JSON, then Ed25519-sign the hash
json_str = json.dumps(upload_data, separators=(",", ":"))
data_hash = hashlib.sha256(json_str.encode()).digest()
scalar = private_key[:32]
prefix_bytes = private_key[32:]
signature = ed25519_sign_expanded(data_hash, scalar, prefix_bytes, public_key)
request_payload = {
"data": json_str,
"signature": signature.hex(),
"publicKey": public_key.hex(),
}
if dry_run:
geofence_note = (
f" | geofence: {geofence_dist_km:.2f} km from observer"
if geofence_dist_km is not None
else ""
)
logger.info(
"MapUpload [DRY RUN] %s (%s)%s → would POST to %s\n payload: %s",
pubkey[:12],
role_name,
geofence_note,
api_url,
json.dumps(request_payload, separators=(",", ":")),
)
# Still update _seen so rate-limiting works during dry-run testing
self._seen[pubkey] = advert_timestamp
return
if not self._client:
return
try:
resp = await self._client.post(
api_url,
content=json.dumps(request_payload, separators=(",", ":")),
headers={"Content-Type": "application/json"},
)
resp.raise_for_status()
self._seen[pubkey] = advert_timestamp
self._last_error = None
logger.info(
"MapUpload: uploaded %s (%s) → HTTP %d",
pubkey[:12],
role_name,
resp.status_code,
)
except httpx.HTTPStatusError as exc:
self._last_error = f"HTTP {exc.response.status_code}"
logger.warning(
"MapUpload: server returned %d for %s: %s",
exc.response.status_code,
pubkey[:12],
exc.response.text[:200],
)
except httpx.RequestError as exc:
self._last_error = str(exc)
logger.warning("MapUpload: request error for %s: %s", pubkey[:12], exc)
@property
def status(self) -> str:
if self._client is None:
return "disconnected"
if self._last_error:
return "error"
return "connected"

View File

@@ -1,14 +1,18 @@
""" """
Ephemeral keystore for storing sensitive keys in memory. Ephemeral keystore for storing sensitive keys in memory, plus the Ed25519
signing primitive used by fanout modules that need to sign requests with the
radio's own key.
The private key is stored in memory only and is never persisted to disk. The private key is stored in memory only and is never persisted to disk.
It's exported from the radio on startup and reconnect, then used for It's exported from the radio on startup and reconnect, then used for
server-side decryption of direct messages. server-side decryption of direct messages.
""" """
import hashlib
import logging import logging
from typing import TYPE_CHECKING from typing import TYPE_CHECKING
import nacl.bindings
from meshcore import EventType from meshcore import EventType
from app.decoder import derive_public_key from app.decoder import derive_public_key
@@ -25,11 +29,30 @@ NO_EVENT_RECEIVED_GUIDANCE = (
"issue commands to the radio." "issue commands to the radio."
) )
# Ed25519 group order (L) — used in the expanded signing primitive below
_L = 2**252 + 27742317777372353535851937790883648493
# In-memory storage for the private key and derived public key # In-memory storage for the private key and derived public key
_private_key: bytes | None = None _private_key: bytes | None = None
_public_key: bytes | None = None _public_key: bytes | None = None
def ed25519_sign_expanded(message: bytes, scalar: bytes, prefix: bytes, public_key: bytes) -> bytes:
"""Sign a message using MeshCore's expanded Ed25519 key format.
MeshCore stores 64-byte keys as scalar(32) || prefix(32). Standard
Ed25519 libraries expect seed format and would re-SHA-512 the key, so we
perform the signing manually using the already-expanded key material.
Port of meshcore-packet-capture's ed25519_sign_with_expanded_key().
"""
r = int.from_bytes(hashlib.sha512(prefix + message).digest(), "little") % _L
R = nacl.bindings.crypto_scalarmult_ed25519_base_noclamp(r.to_bytes(32, "little"))
k = int.from_bytes(hashlib.sha512(R + public_key + message).digest(), "little") % _L
s = (r + k * int.from_bytes(scalar, "little")) % _L
return R + s.to_bytes(32, "little")
def clear_keys() -> None: def clear_keys() -> None:
"""Clear any stored private/public key material from memory.""" """Clear any stored private/public key material from memory."""
global _private_key, _public_key global _private_key, _public_key

View File

@@ -16,7 +16,7 @@ from app.repository.fanout import FanoutConfigRepository
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
router = APIRouter(prefix="/fanout", tags=["fanout"]) router = APIRouter(prefix="/fanout", tags=["fanout"])
_VALID_TYPES = {"mqtt_private", "mqtt_community", "bot", "webhook", "apprise", "sqs"} _VALID_TYPES = {"mqtt_private", "mqtt_community", "bot", "webhook", "apprise", "sqs", "map_upload"}
_IATA_RE = re.compile(r"^[A-Z]{3}$") _IATA_RE = re.compile(r"^[A-Z]{3}$")
_DEFAULT_COMMUNITY_MQTT_TOPIC_TEMPLATE = "meshcore/{IATA}/{PUBLIC_KEY}/packets" _DEFAULT_COMMUNITY_MQTT_TOPIC_TEMPLATE = "meshcore/{IATA}/{PUBLIC_KEY}/packets"
@@ -94,6 +94,8 @@ def _validate_and_normalize_config(config_type: str, config: dict) -> dict:
_validate_apprise_config(normalized) _validate_apprise_config(normalized)
elif config_type == "sqs": elif config_type == "sqs":
_validate_sqs_config(normalized) _validate_sqs_config(normalized)
elif config_type == "map_upload":
_validate_map_upload_config(normalized)
return normalized return normalized
@@ -295,10 +297,33 @@ def _validate_sqs_config(config: dict) -> None:
) )
def _validate_map_upload_config(config: dict) -> None:
"""Validate and normalize map_upload config blob."""
api_url = str(config.get("api_url", "")).strip()
if api_url and not api_url.startswith(("http://", "https://")):
raise HTTPException(
status_code=400,
detail="api_url must start with http:// or https://",
)
# Persist the cleaned value (empty string means use the module default)
config["api_url"] = api_url
config["dry_run"] = bool(config.get("dry_run", True))
config["geofence_enabled"] = bool(config.get("geofence_enabled", False))
try:
radius = float(config.get("geofence_radius_km", 0) or 0)
except (TypeError, ValueError):
raise HTTPException(status_code=400, detail="geofence_radius_km must be a number") from None
if radius < 0:
raise HTTPException(status_code=400, detail="geofence_radius_km must be >= 0")
config["geofence_radius_km"] = radius
def _enforce_scope(config_type: str, scope: dict) -> dict: def _enforce_scope(config_type: str, scope: dict) -> dict:
"""Enforce type-specific scope constraints. Returns normalized scope.""" """Enforce type-specific scope constraints. Returns normalized scope."""
if config_type == "mqtt_community": if config_type == "mqtt_community":
return {"messages": "none", "raw_packets": "all"} return {"messages": "none", "raw_packets": "all"}
if config_type == "map_upload":
return {"messages": "none", "raw_packets": "all"}
if config_type == "bot": if config_type == "bot":
return {"messages": "all", "raw_packets": "none"} return {"messages": "all", "raw_packets": "none"}
if config_type in ("webhook", "apprise"): if config_type in ("webhook", "apprise"):

View File

@@ -16,11 +16,12 @@ const BotCodeEditor = lazy(() =>
const TYPE_LABELS: Record<string, string> = { const TYPE_LABELS: Record<string, string> = {
mqtt_private: 'Private MQTT', mqtt_private: 'Private MQTT',
mqtt_community: 'Community MQTT', mqtt_community: 'Community Sharing',
bot: 'Python Bot', bot: 'Python Bot',
webhook: 'Webhook', webhook: 'Webhook',
apprise: 'Apprise', apprise: 'Apprise',
sqs: 'Amazon SQS', sqs: 'Amazon SQS',
map_upload: 'Map Upload',
}; };
const DEFAULT_COMMUNITY_PACKET_TOPIC_TEMPLATE = 'meshcore/{IATA}/{PUBLIC_KEY}/packets'; const DEFAULT_COMMUNITY_PACKET_TOPIC_TEMPLATE = 'meshcore/{IATA}/{PUBLIC_KEY}/packets';
@@ -100,7 +101,8 @@ type DraftType =
| 'webhook' | 'webhook'
| 'apprise' | 'apprise'
| 'sqs' | 'sqs'
| 'bot'; | 'bot'
| 'map_upload';
type CreateIntegrationDefinition = { type CreateIntegrationDefinition = {
value: DraftType; value: DraftType;
@@ -143,7 +145,7 @@ const CREATE_INTEGRATION_DEFINITIONS: readonly CreateIntegrationDefinition[] = [
value: 'mqtt_community', value: 'mqtt_community',
savedType: 'mqtt_community', savedType: 'mqtt_community',
label: 'Community MQTT/meshcoretomqtt', label: 'Community MQTT/meshcoretomqtt',
section: 'Community MQTT', section: 'Community Sharing',
description: description:
'MeshcoreToMQTT-compatible raw-packet feed publishing, compatible with community aggregators (in other words, make your companion radio also serve as an observer node). Superset of other Community MQTT presets.', 'MeshcoreToMQTT-compatible raw-packet feed publishing, compatible with community aggregators (in other words, make your companion radio also serve as an observer node). Superset of other Community MQTT presets.',
defaultName: 'Community MQTT', defaultName: 'Community MQTT',
@@ -157,7 +159,7 @@ const CREATE_INTEGRATION_DEFINITIONS: readonly CreateIntegrationDefinition[] = [
value: 'mqtt_community_meshrank', value: 'mqtt_community_meshrank',
savedType: 'mqtt_community', savedType: 'mqtt_community',
label: 'MeshRank', label: 'MeshRank',
section: 'Community MQTT', section: 'Community Sharing',
description: description:
'A community MQTT config preconfigured for MeshRank, requiring only the provided topic from your MeshRank configuration. A subset of the primary Community MQTT/meshcoretomqtt configuration; you are free to edit all configuration after creation.', 'A community MQTT config preconfigured for MeshRank, requiring only the provided topic from your MeshRank configuration. A subset of the primary Community MQTT/meshcoretomqtt configuration; you are free to edit all configuration after creation.',
defaultName: 'MeshRank', defaultName: 'MeshRank',
@@ -180,7 +182,7 @@ const CREATE_INTEGRATION_DEFINITIONS: readonly CreateIntegrationDefinition[] = [
value: 'mqtt_community_letsmesh_us', value: 'mqtt_community_letsmesh_us',
savedType: 'mqtt_community', savedType: 'mqtt_community',
label: 'LetsMesh (US)', label: 'LetsMesh (US)',
section: 'Community MQTT', section: 'Community Sharing',
description: description:
'A community MQTT config preconfigured for the LetsMesh US-ingest endpoint, requiring only your email and IATA region code. Good to use with an additional EU configuration for redundancy. A subset of the primary Community MQTT/meshcoretomqtt configuration; you are free to edit all configuration after creation.', 'A community MQTT config preconfigured for the LetsMesh US-ingest endpoint, requiring only your email and IATA region code. Good to use with an additional EU configuration for redundancy. A subset of the primary Community MQTT/meshcoretomqtt configuration; you are free to edit all configuration after creation.',
defaultName: 'LetsMesh (US)', defaultName: 'LetsMesh (US)',
@@ -197,7 +199,7 @@ const CREATE_INTEGRATION_DEFINITIONS: readonly CreateIntegrationDefinition[] = [
value: 'mqtt_community_letsmesh_eu', value: 'mqtt_community_letsmesh_eu',
savedType: 'mqtt_community', savedType: 'mqtt_community',
label: 'LetsMesh (EU)', label: 'LetsMesh (EU)',
section: 'Community MQTT', section: 'Community Sharing',
description: description:
'A community MQTT config preconfigured for the LetsMesh EU-ingest endpoint, requiring only your email and IATA region code. Good to use with an additional US configuration for redundancy. A subset of the primary Community MQTT/meshcoretomqtt configuration; you are free to edit all configuration after creation.', 'A community MQTT config preconfigured for the LetsMesh EU-ingest endpoint, requiring only your email and IATA region code. Good to use with an additional US configuration for redundancy. A subset of the primary Community MQTT/meshcoretomqtt configuration; you are free to edit all configuration after creation.',
defaultName: 'LetsMesh (EU)', defaultName: 'LetsMesh (EU)',
@@ -284,6 +286,23 @@ const CREATE_INTEGRATION_DEFINITIONS: readonly CreateIntegrationDefinition[] = [
scope: { messages: 'all', raw_packets: 'none' }, scope: { messages: 'all', raw_packets: 'none' },
}, },
}, },
{
value: 'map_upload',
savedType: 'map_upload',
label: 'Map Upload',
section: 'Community Sharing',
description:
'Upload repeaters and room servers to map.meshcore.dev or a compatible map API endpoint.',
defaultName: 'Map Upload',
nameMode: 'counted',
defaults: {
config: {
api_url: '',
dry_run: true,
},
scope: { messages: 'none', raw_packets: 'all' },
},
},
]; ];
const CREATE_INTEGRATION_DEFINITIONS_BY_VALUE = Object.fromEntries( const CREATE_INTEGRATION_DEFINITIONS_BY_VALUE = Object.fromEntries(
@@ -566,7 +585,9 @@ function getDefaultIntegrationName(type: string, configs: FanoutConfig[]) {
function getStatusLabel(status: string | undefined, type?: string) { function getStatusLabel(status: string | undefined, type?: string) {
if (status === 'connected') if (status === 'connected')
return type === 'bot' || type === 'webhook' || type === 'apprise' ? 'Active' : 'Connected'; return type === 'bot' || type === 'webhook' || type === 'apprise' || type === 'map_upload'
? 'Active'
: 'Connected';
if (status === 'error') return 'Error'; if (status === 'error') return 'Error';
if (status === 'disconnected') return 'Disconnected'; if (status === 'disconnected') return 'Disconnected';
return 'Inactive'; return 'Inactive';
@@ -1059,6 +1080,152 @@ function BotConfigEditor({
); );
} }
function MapUploadConfigEditor({
config,
onChange,
}: {
config: Record<string, unknown>;
onChange: (config: Record<string, unknown>) => void;
}) {
const isDryRun = config.dry_run !== false;
const [radioLat, setRadioLat] = useState<number | null>(null);
const [radioLon, setRadioLon] = useState<number | null>(null);
useEffect(() => {
api
.getRadioConfig()
.then((rc) => {
setRadioLat(rc.lat ?? 0);
setRadioLon(rc.lon ?? 0);
})
.catch(() => {
setRadioLat(0);
setRadioLon(0);
});
}, []);
const radioLatLonConfigured =
radioLat !== null && radioLon !== null && !(radioLat === 0 && radioLon === 0);
return (
<div className="space-y-3">
<p className="text-xs text-muted-foreground">
Automatically upload heard repeater and room server advertisements to{' '}
<a
href="https://map.meshcore.dev"
target="_blank"
rel="noopener noreferrer"
className="underline hover:text-foreground"
>
map.meshcore.dev
</a>
. Requires the radio&apos;s private key to be available (firmware must have{' '}
<code>ENABLE_PRIVATE_KEY_EXPORT=1</code>). Only raw RF packets are shared &mdash; never
decrypted messages.
</p>
<div className="rounded-md border border-warning/50 bg-warning/10 px-3 py-2 text-xs text-warning">
<strong>Dry Run is {isDryRun ? 'ON' : 'OFF'}.</strong>{' '}
{isDryRun
? 'No uploads will be sent. Check the backend logs to verify the payload looks correct before enabling live sends.'
: 'Live uploads are enabled. Each advert is rate-limited to once per hour per node.'}
</div>
<label className="flex items-center gap-3 cursor-pointer">
<input
type="checkbox"
checked={isDryRun}
onChange={(e) => onChange({ ...config, dry_run: e.target.checked })}
className="h-4 w-4 rounded border-border"
/>
<div>
<span className="text-sm font-medium">Dry Run (log only, no uploads)</span>
<p className="text-xs text-muted-foreground">
When enabled, upload payloads are logged at INFO level but not sent. Disable once you
have confirmed the logged output looks correct.
</p>
</div>
</label>
<Separator />
<div className="space-y-2">
<Label htmlFor="fanout-map-api-url">API URL (optional)</Label>
<Input
id="fanout-map-api-url"
type="url"
placeholder="https://map.meshcore.dev/api/v1/uploader/node"
value={(config.api_url as string) || ''}
onChange={(e) => onChange({ ...config, api_url: e.target.value })}
/>
<p className="text-xs text-muted-foreground">
Leave blank to use the default <code>map.meshcore.dev</code> endpoint.
</p>
</div>
<Separator />
<label className="flex items-center gap-3 cursor-pointer">
<input
type="checkbox"
checked={!!config.geofence_enabled}
onChange={(e) => onChange({ ...config, geofence_enabled: e.target.checked })}
className="h-4 w-4 rounded border-border"
/>
<div>
<span className="text-sm font-medium">Enable Geofence</span>
<p className="text-xs text-muted-foreground">
Only upload nodes whose location falls within the configured radius of your radio&apos;s
own position. Helps exclude nodes with false or spoofed coordinates. Uses the
latitude/longitude set in Radio Settings.
</p>
</div>
</label>
{!!config.geofence_enabled && (
<div className="space-y-3 pl-7">
{!radioLatLonConfigured && (
<div className="rounded-md border border-warning/50 bg-warning/10 px-3 py-2 text-xs text-warning">
Your radio does not currently have a latitude/longitude configured. Geofencing will be
silently skipped until coordinates are set in{' '}
<strong>Settings &rarr; Radio &rarr; Location</strong>.
</div>
)}
{radioLatLonConfigured && (
<p className="text-xs text-muted-foreground">
Using radio position{' '}
<code>
{radioLat?.toFixed(5)}, {radioLon?.toFixed(5)}
</code>{' '}
as the geofence center. Update coordinates in Radio Settings to move the center.
</p>
)}
<div className="space-y-2">
<Label htmlFor="fanout-map-geofence-radius">Radius (km)</Label>
<Input
id="fanout-map-geofence-radius"
type="number"
min="0"
step="any"
placeholder="e.g. 100"
value={(config.geofence_radius_km as number | undefined) ?? ''}
onChange={(e) =>
onChange({
...config,
geofence_radius_km: e.target.value === '' ? 0 : parseFloat(e.target.value),
})
}
/>
<p className="text-xs text-muted-foreground">
Nodes further than this distance from your radio&apos;s position will not be uploaded.
</p>
</div>
</div>
)}
</div>
);
}
type ScopeMode = 'all' | 'none' | 'only' | 'except'; type ScopeMode = 'all' | 'none' | 'only' | 'except';
function getScopeMode(value: unknown): ScopeMode { function getScopeMode(value: unknown): ScopeMode {
@@ -1975,6 +2142,10 @@ export function SettingsFanoutSection({
/> />
)} )}
{detailType === 'map_upload' && (
<MapUploadConfigEditor config={editConfig} onChange={setEditConfig} />
)}
<Separator /> <Separator />
<div className="flex gap-2"> <div className="flex gap-2">

View File

@@ -12,6 +12,7 @@ vi.mock('../api', () => ({
deleteFanoutConfig: vi.fn(), deleteFanoutConfig: vi.fn(),
getChannels: vi.fn(), getChannels: vi.fn(),
getContacts: vi.fn(), getContacts: vi.fn(),
getRadioConfig: vi.fn(),
}, },
})); }));
@@ -96,6 +97,17 @@ beforeEach(() => {
mockedApi.getFanoutConfigs.mockResolvedValue([]); mockedApi.getFanoutConfigs.mockResolvedValue([]);
mockedApi.getChannels.mockResolvedValue([]); mockedApi.getChannels.mockResolvedValue([]);
mockedApi.getContacts.mockResolvedValue([]); mockedApi.getContacts.mockResolvedValue([]);
mockedApi.getRadioConfig.mockResolvedValue({
public_key: 'aa'.repeat(32),
name: 'TestNode',
lat: 0,
lon: 0,
tx_power: 17,
max_tx_power: 22,
radio: { freq: 910.525, bw: 62.5, sf: 7, cr: 5 },
path_hash_mode: 0,
path_hash_mode_supported: false,
});
}); });
describe('SettingsFanoutSection', () => { describe('SettingsFanoutSection', () => {
@@ -106,7 +118,7 @@ describe('SettingsFanoutSection', () => {
const optionButtons = within(dialog) const optionButtons = within(dialog)
.getAllByRole('button') .getAllByRole('button')
.filter((button) => button.hasAttribute('aria-pressed')); .filter((button) => button.hasAttribute('aria-pressed'));
expect(optionButtons).toHaveLength(9); expect(optionButtons).toHaveLength(10);
expect(within(dialog).getByRole('button', { name: 'Close' })).toBeInTheDocument(); expect(within(dialog).getByRole('button', { name: 'Close' })).toBeInTheDocument();
expect(within(dialog).getByRole('button', { name: 'Create' })).toBeInTheDocument(); expect(within(dialog).getByRole('button', { name: 'Create' })).toBeInTheDocument();
expect( expect(
@@ -138,6 +150,9 @@ describe('SettingsFanoutSection', () => {
expect( expect(
within(dialog).getByRole('button', { name: startsWithAccessibleName('Python Bot') }) within(dialog).getByRole('button', { name: startsWithAccessibleName('Python Bot') })
).toBeInTheDocument(); ).toBeInTheDocument();
expect(
within(dialog).getByRole('button', { name: startsWithAccessibleName('Map Upload') })
).toBeInTheDocument();
expect(within(dialog).getByRole('heading', { level: 3 })).toBeInTheDocument(); expect(within(dialog).getByRole('heading', { level: 3 })).toBeInTheDocument();
const genericCommunityIndex = optionButtons.findIndex((button) => const genericCommunityIndex = optionButtons.findIndex((button) =>
@@ -916,7 +931,7 @@ describe('SettingsFanoutSection', () => {
await waitFor(() => expect(screen.getByText('← Back to list')).toBeInTheDocument()); await waitFor(() => expect(screen.getByText('← Back to list')).toBeInTheDocument());
expect(screen.getByLabelText('Name')).toHaveValue('Community MQTT #1'); expect(screen.getByLabelText('Name')).toHaveValue('Community Sharing #1');
expect(screen.getByLabelText('Broker Host')).toBeInTheDocument(); expect(screen.getByLabelText('Broker Host')).toBeInTheDocument();
expect(screen.getByLabelText('Authentication')).toBeInTheDocument(); expect(screen.getByLabelText('Authentication')).toBeInTheDocument();
expect(screen.getByLabelText('Packet Topic Template')).toBeInTheDocument(); expect(screen.getByLabelText('Packet Topic Template')).toBeInTheDocument();

View File

@@ -19,7 +19,6 @@ from app.fanout.community_mqtt import (
_build_status_topic, _build_status_topic,
_calculate_packet_hash, _calculate_packet_hash,
_decode_packet_fields, _decode_packet_fields,
_ed25519_sign_expanded,
_format_raw_packet, _format_raw_packet,
_generate_jwt_token, _generate_jwt_token,
_get_client_version, _get_client_version,
@@ -29,6 +28,7 @@ from app.fanout.mqtt_community import (
_publish_community_packet, _publish_community_packet,
_render_packet_topic, _render_packet_topic,
) )
from app.keystore import ed25519_sign_expanded
def _make_test_keys() -> tuple[bytes, bytes]: def _make_test_keys() -> tuple[bytes, bytes]:
@@ -173,13 +173,13 @@ class TestEddsaSignExpanded:
def test_produces_64_byte_signature(self): def test_produces_64_byte_signature(self):
private_key, public_key = _make_test_keys() private_key, public_key = _make_test_keys()
message = b"test message" message = b"test message"
sig = _ed25519_sign_expanded(message, private_key[:32], private_key[32:], public_key) sig = ed25519_sign_expanded(message, private_key[:32], private_key[32:], public_key)
assert len(sig) == 64 assert len(sig) == 64
def test_signature_verifies_with_nacl(self): def test_signature_verifies_with_nacl(self):
private_key, public_key = _make_test_keys() private_key, public_key = _make_test_keys()
message = b"hello world" message = b"hello world"
sig = _ed25519_sign_expanded(message, private_key[:32], private_key[32:], public_key) sig = ed25519_sign_expanded(message, private_key[:32], private_key[32:], public_key)
signed_message = sig + message signed_message = sig + message
verified = nacl.bindings.crypto_sign_open(signed_message, public_key) verified = nacl.bindings.crypto_sign_open(signed_message, public_key)
@@ -187,8 +187,8 @@ class TestEddsaSignExpanded:
def test_different_messages_produce_different_signatures(self): def test_different_messages_produce_different_signatures(self):
private_key, public_key = _make_test_keys() private_key, public_key = _make_test_keys()
sig1 = _ed25519_sign_expanded(b"msg1", private_key[:32], private_key[32:], public_key) sig1 = ed25519_sign_expanded(b"msg1", private_key[:32], private_key[32:], public_key)
sig2 = _ed25519_sign_expanded(b"msg2", private_key[:32], private_key[32:], public_key) sig2 = ed25519_sign_expanded(b"msg2", private_key[:32], private_key[32:], public_key)
assert sig1 != sig2 assert sig1 != sig2

View File

@@ -707,6 +707,98 @@ class TestSqsValidation:
{"queue_url": "https://sqs.us-east-1.amazonaws.com/123456789012/mesh-events"} {"queue_url": "https://sqs.us-east-1.amazonaws.com/123456789012/mesh-events"}
) )
class TestMapUploadValidation:
def test_rejects_bad_api_url_scheme(self):
from fastapi import HTTPException
from app.routers.fanout import _validate_map_upload_config
with pytest.raises(HTTPException) as exc_info:
_validate_map_upload_config({"api_url": "ftp://example.com"})
assert exc_info.value.status_code == 400
assert "api_url" in exc_info.value.detail
def test_accepts_empty_api_url(self):
from app.routers.fanout import _validate_map_upload_config
config = {"api_url": ""}
_validate_map_upload_config(config)
assert config["api_url"] == ""
def test_accepts_valid_api_url(self):
from app.routers.fanout import _validate_map_upload_config
config = {"api_url": "https://custom.example.com/upload"}
_validate_map_upload_config(config)
assert config["api_url"] == "https://custom.example.com/upload"
def test_normalizes_dry_run_to_bool(self):
from app.routers.fanout import _validate_map_upload_config
config = {"dry_run": 1}
_validate_map_upload_config(config)
assert config["dry_run"] is True
def test_normalizes_geofence_enabled_to_bool(self):
from app.routers.fanout import _validate_map_upload_config
config = {"geofence_enabled": 1}
_validate_map_upload_config(config)
assert config["geofence_enabled"] is True
def test_normalizes_geofence_radius_to_float(self):
from app.routers.fanout import _validate_map_upload_config
config = {"geofence_radius_km": 100}
_validate_map_upload_config(config)
assert config["geofence_radius_km"] == 100.0
assert isinstance(config["geofence_radius_km"], float)
def test_rejects_negative_geofence_radius(self):
from fastapi import HTTPException
from app.routers.fanout import _validate_map_upload_config
with pytest.raises(HTTPException) as exc_info:
_validate_map_upload_config({"geofence_radius_km": -1})
assert exc_info.value.status_code == 400
assert "geofence_radius_km" in exc_info.value.detail
def test_rejects_non_numeric_geofence_radius(self):
from fastapi import HTTPException
from app.routers.fanout import _validate_map_upload_config
with pytest.raises(HTTPException) as exc_info:
_validate_map_upload_config({"geofence_radius_km": "bad"})
assert exc_info.value.status_code == 400
assert "geofence_radius_km" in exc_info.value.detail
def test_accepts_zero_geofence_radius(self):
from app.routers.fanout import _validate_map_upload_config
config = {"geofence_radius_km": 0}
_validate_map_upload_config(config)
assert config["geofence_radius_km"] == 0.0
def test_defaults_applied_when_keys_absent(self):
from app.routers.fanout import _validate_map_upload_config
config = {}
_validate_map_upload_config(config)
assert config["api_url"] == ""
assert config["dry_run"] is True
assert config["geofence_enabled"] is False
assert config["geofence_radius_km"] == 0.0
def test_enforce_scope_map_upload_forces_raw_only(self):
"""map_upload scope is always fixed regardless of what the caller passes."""
from app.routers.fanout import _enforce_scope
scope = _enforce_scope("map_upload", {"messages": "all", "raw_packets": "none"})
assert scope == {"messages": "none", "raw_packets": "all"}
def test_enforce_scope_sqs_preserves_raw_packets_setting(self): def test_enforce_scope_sqs_preserves_raw_packets_setting(self):
from app.routers.fanout import _enforce_scope from app.routers.fanout import _enforce_scope

View File

@@ -1790,3 +1790,100 @@ class TestManagerRestartFailure:
assert len(healthy.messages_received) == 1 assert len(healthy.messages_received) == 1
assert len(dead.messages_received) == 0 assert len(dead.messages_received) == 0
# ---------------------------------------------------------------------------
# MapUploadModule integration tests
# ---------------------------------------------------------------------------
class TestMapUploadIntegration:
"""Integration tests: FanoutManager loads and dispatches to MapUploadModule."""
@pytest.mark.asyncio
async def test_map_upload_module_loaded_and_receives_raw(self, integration_db):
"""Enabled map_upload config is loaded by the manager and its on_raw is called."""
from unittest.mock import AsyncMock, patch
cfg = await FanoutConfigRepository.create(
config_type="map_upload",
name="Map",
config={"dry_run": True, "api_url": ""},
scope={"messages": "none", "raw_packets": "all"},
enabled=True,
)
manager = FanoutManager()
await manager.load_from_db()
assert cfg["id"] in manager._modules
module, scope = manager._modules[cfg["id"]]
assert scope == {"messages": "none", "raw_packets": "all"}
# Raw ADVERT event should be dispatched to on_raw
advert_data = {
"payload_type": "ADVERT",
"data": "aabbccdd",
"timestamp": 1000,
"id": 1,
"observation_id": 1,
}
with patch.object(module, "_upload", new_callable=AsyncMock):
# Provide a parseable but minimal packet so on_raw gets past hex decode;
# parse_packet/parse_advertisement returning None is fine — on_raw silently exits
await manager.broadcast_raw(advert_data)
# Give the asyncio task a chance to run
import asyncio
await asyncio.sleep(0.05)
# _upload may or may not be called depending on parse result, but no exception
await manager.stop_all()
@pytest.mark.asyncio
async def test_map_upload_disabled_not_loaded(self, integration_db):
"""Disabled map_upload config is not loaded by the manager."""
await FanoutConfigRepository.create(
config_type="map_upload",
name="Map Disabled",
config={"dry_run": True, "api_url": ""},
scope={"messages": "none", "raw_packets": "all"},
enabled=False,
)
manager = FanoutManager()
await manager.load_from_db()
assert len(manager._modules) == 0
await manager.stop_all()
@pytest.mark.asyncio
async def test_map_upload_does_not_receive_messages(self, integration_db):
"""map_upload scope forces raw_packets only — message events must not reach it."""
from unittest.mock import AsyncMock, patch
cfg = await FanoutConfigRepository.create(
config_type="map_upload",
name="Map",
config={"dry_run": True, "api_url": ""},
scope={"messages": "none", "raw_packets": "all"},
enabled=True,
)
manager = FanoutManager()
await manager.load_from_db()
assert cfg["id"] in manager._modules
module, _ = manager._modules[cfg["id"]]
with patch.object(module, "on_message", new_callable=AsyncMock) as mock_msg:
await manager.broadcast_message(
{"type": "CHAN", "conversation_key": "k1", "text": "hi"}
)
import asyncio
await asyncio.sleep(0.05)
mock_msg.assert_not_called()
await manager.stop_all()

1087
tests/test_map_upload.py Normal file

File diff suppressed because it is too large Load Diff