diff --git a/.gitignore b/.gitignore index 7a94de8..d4f6cdd 100644 --- a/.gitignore +++ b/.gitignore @@ -22,6 +22,7 @@ meshview-web.pid /table_details.py config.ini *.log +*.status.json # Screenshots screenshots/* diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 528a8c7..9dafea5 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -52,7 +52,7 @@ Unacceptable behavior includes harassment, insults, hate speech, personal attack --- -## I Want to Contribute +## I Want to Contribute! ### Legal Notice By contributing to Meshview, you agree that: diff --git a/README.md b/README.md index 0726141..0d21e36 100644 --- a/README.md +++ b/README.md @@ -9,7 +9,7 @@ The project serves as a real-time monitoring and diagnostic tool for the Meshtas - Node list/map UX: added node list filtering for wider active-node ranges, included all database nodes where appropriate, and made map node colors persistent across reboots. - QR/API fixes: corrected QR code generation when a node role is missing or unavailable. - Database cleanup: removed stored node public key data and added the Alembic migration to drop `node.public_key`. -- Protobufs: refreshed vendored Meshtastic protobuf Python modules to upstream commit `9ab4a1d08cb833d897aee2367013318718391f2f`, including new hardware models such as `RAK6421`. +- Protobufs: refreshed Meshtastic protobuf Python modules to upstream commit - Tooling: improved the protobuf updater so it supports raw commit SHAs, the newer upstream `meshtastic/*.proto` layout, generated `.pyi` stubs, and the vendored `nanopb`/`serial_hal` outputs. ### Version 3.0.6 — March 2026 diff --git a/docs/API_Documentation.md b/docs/API_Documentation.md index bc8e10a..5531c26 100644 --- a/docs/API_Documentation.md +++ b/docs/API_Documentation.md @@ -203,6 +203,12 @@ Returns network edges (connections between nodes) based on traceroutes and neigh Traceroute edges are collected over the last 12 hours. Neighbor edges are based on port 71 packets. +A completed traceroute is observed as a response packet, whose `from`/`to` are reversed +with respect to the traced path, so edges are built as `to -> route[] -> from`. `route[]` +may contain `0xFFFFFFFF` placeholders for hops the mesh could not identify; the edges on +either side of such a placeholder are omitted, because the nodes around an unknown hop +are not neighbours. + Query Parameters - `type` (optional, string): `traceroute` or `neighbor`. If omitted, returns both. - `node_id` (optional, int): Filter edges to only those touching a node. @@ -324,13 +330,30 @@ Returns traceroute details and derived paths for a packet. Path Parameters - `packet_id` (required, int): Packet ID. +A completed traceroute is observed as a response packet, whose `from`/`to` are reversed +with respect to the traced path. `initiator` and `target` resolve this: the forward path +runs `initiator -> forward_hops -> target` and the return path runs +`target -> reverse_hops -> initiator`, in both directions of travel. `forward_hops` and +`reverse_hops` contain only the intermediate hops, never the endpoints. + +A gateway usually reports a response that is still travelling home, so `reverse_hops` is +often incomplete. `reverse_complete` says whether that observation had reached the +initiator; only then does the corresponding `winning_paths.reverse` entry end at the +initiator. + +`winning_paths` is built from response observations, since a response proves the request +reached the target. `complete` on a `unique_forward_paths` entry says whether some +response reported that hop list, and so whether it is known to end at the target. + Response Example ```json { "packet": { "id": 123, - "from": 111, - "to": 222, + "from": 222, + "to": 111, + "initiator": 111, + "target": 222, "channel": "main" }, "traceroute_packets": [ @@ -338,19 +361,21 @@ Response Example "index": 0, "gateway_node_id": 333, "done": true, - "forward_hops": [111, 444, 222], - "reverse_hops": [222, 444, 111] + "forward_hops": [444, 555], + "reverse_hops": [555, 444], + "reverse_complete": true } ], "unique_forward_paths": [ - { "path": [111, 444, 222], "count": 2 } + { "path": [444, 555], "count": 2, "complete": true } ], "unique_reverse_paths": [ - [222, 444, 111] + [555, 444] ], - "winning_paths": [ - [111, 444, 222] - ] + "winning_paths": { + "forward": [[111, 444, 555, 222]], + "reverse": [[222, 555, 444, 111]] + } } ``` @@ -368,6 +393,49 @@ Response Example "timestamp": "2025-07-22T12:45:00+00:00", "version": "3.0.3", "git_revision": "abc1234", + "cleanup": { + "enabled": true, + "days_to_keep": 14, + "scheduled_time": "02:00", + "vacuum": false, + "status": "ok", + "status_file": "dbcleanup.status.json", + "last_run": { + "status": "ok", + "started_at": "2026-06-04T09:00:00+00:00", + "completed_at": "2026-06-04T09:00:03+00:00", + "cutoff_at": "2026-05-21T09:00:00+00:00", + "days_to_keep": 14, + "vacuum_requested": false, + "vacuum_completed": false, + "rows_deleted": { + "packet": 1200, + "packet_seen": 3400, + "traceroute": 42, + "node": 3 + }, + "error": null + } + }, + "backup": { + "enabled": true, + "backup_dir": "./backups", + "scheduled_time": "02:00", + "status": "ok", + "status_file": "dbbackup.status.json", + "last_run": { + "status": "ok", + "started_at": "2026-06-04T09:00:00+00:00", + "completed_at": "2026-06-04T09:00:02+00:00", + "backup_dir": "./backups", + "database_path": "packets.db", + "backup_file": "backups/packets_backup_20260604_090000.db.gz", + "original_size_bytes": 12939444, + "compressed_size_bytes": 4211560, + "compression_percent": 67.5, + "error": null + } + }, "database": "connected", "database_size": "12.34 MB", "database_size_bytes": 12939444 diff --git a/meshview/__version__.py b/meshview/__version__.py index 1abafc8..50d30b1 100644 --- a/meshview/__version__.py +++ b/meshview/__version__.py @@ -3,7 +3,7 @@ import subprocess from pathlib import Path -__version__ = "3.0.7" +__version__ = "3.0.8" __release_date__ = "2026-5-12" diff --git a/meshview/lang/en.json b/meshview/lang/en.json index 3b6b9d9..6eec50e 100644 --- a/meshview/lang/en.json +++ b/meshview/lang/en.json @@ -159,6 +159,7 @@ "times_seen": "Seen (24h)", "avg_gateways": "Avg Gateways", "showing_nodes": "Showing", + "all_channels": "All Channels", "nodes_suffix": "nodes" }, diff --git a/meshview/lang/es.json b/meshview/lang/es.json index 7d21ec1..e7561d9 100644 --- a/meshview/lang/es.json +++ b/meshview/lang/es.json @@ -155,6 +155,7 @@ "times_seen": "Visto (24h)", "avg_gateways": "Promedio de Gateways", "showing_nodes": "Mostrando", + "all_channels": "Todos los Canales", "nodes_suffix": "nodos" }, diff --git a/meshview/lang/ru.json b/meshview/lang/ru.json index f1727ba..902aa64 100644 --- a/meshview/lang/ru.json +++ b/meshview/lang/ru.json @@ -145,6 +145,7 @@ "times_seen": "Принято (24ч)", "avg_gateways": "Среднее количество шлюзов", "showing_nodes": "Показать", + "all_channels": "Все каналы", "nodes_suffix": "ноды" }, "nodegraph": { diff --git a/meshview/mqtt_reader.py b/meshview/mqtt_reader.py index 642034a..6345d58 100644 --- a/meshview/mqtt_reader.py +++ b/meshview/mqtt_reader.py @@ -12,7 +12,10 @@ from meshtastic.protobuf.mesh_pb2 import Data from meshtastic.protobuf.mqtt_pb2 import ServiceEnvelope from meshview.config import CONFIG -PRIMARY_KEY = base64.b64decode("1PG7OiApB1nwvP+rz05pAQ==") +# Meshtastic's well-known default PSK, i.e. the expansion of the single byte PSK index 1. +DEFAULT_PSK = base64.b64decode("1PG7OiApB1nwvP+rz05pAQ==") +PRIMARY_KEY = DEFAULT_PSK +VALID_KEY_LENGTHS = (16, 32) logging.basicConfig( level=logging.INFO, @@ -52,6 +55,10 @@ def _strip_quotes(value): return value +def _expand_short_psk(index): + return DEFAULT_PSK[:-1] + bytes([(DEFAULT_PSK[-1] + index - 1) % 256]) + + def _parse_secondary_keys(): mqtt_config = CONFIG.get("mqtt", {}) raw_value = mqtt_config.get("secondary_keys", "") @@ -68,12 +75,33 @@ def _parse_secondary_keys(): keys = [] for value in values: + cleaned = _strip_quotes(str(value).strip()) + if not cleaned: + continue + try: - cleaned = _strip_quotes(str(value).strip()) - if cleaned: - keys.append(base64.b64decode(cleaned)) + key = base64.b64decode(cleaned) except (TypeError, ValueError): logger.warning("Invalid base64 key in mqtt.secondary_keys: %s", value) + continue + + if len(key) == 1: + if key[0] == 0: + logger.warning( + "Skipping PSK %s in mqtt.secondary_keys: index 0 means no encryption", value + ) + continue + key = _expand_short_psk(key[0]) + + if len(key) not in VALID_KEY_LENGTHS: + logger.error( + "Skipping key %s in mqtt.secondary_keys: decoded to %d bytes, expected 16 or 32", + value, + len(key), + ) + continue + + keys.append(key) return keys diff --git a/meshview/mqtt_store.py b/meshview/mqtt_store.py index 825de1f..68acd8e 100644 --- a/meshview/mqtt_store.py +++ b/meshview/mqtt_store.py @@ -17,6 +17,34 @@ from meshview.models import DailySnapshot, Node, Packet, PacketSeen, Traceroute logger = logging.getLogger(__name__) MQTT_GATEWAY_CACHE: set[int] = set() +UNKNOWN_NODE_NAME = "unknown name" +NODE_NAME_CONTROL_CHARS = re.compile(r"[\x00-\x1f\x7f]") +MQTT_DIRECT_NODE_ID = 1 + + +def normalize_node_name(name: str) -> str: + name = name.strip() + if not name or NODE_NAME_CONTROL_CHARS.search(name): + return UNKNOWN_NODE_NAME + return name + + +def storage_packet_id(packet) -> int | None: + if packet.id: + return packet.id + + if packet.decoded.portnum != PortNum.MAP_REPORT_APP: + return None + + from_node_id = getattr(packet, "from", 0) or 0 + now_us = int(time.time() * 1_000_000) + return -(now_us * 2048 + (from_node_id & 0x7FF)) + + +def storage_to_node_id(packet) -> int: + if packet.decoded.portnum == PortNum.MAP_REPORT_APP: + return MQTT_DIRECT_NODE_ID + return packet.to async def capture_daily_snapshot() -> None: @@ -138,20 +166,21 @@ async def process_envelope(topic, env): await session.commit() - if not env.packet.id: + packet_id = storage_packet_id(env.packet) + if packet_id is None: return async with mqtt_database.async_session() as session: # --- Packet insert with ON CONFLICT DO NOTHING - result = await session.execute(select(Packet).where(Packet.id == env.packet.id)) + result = await session.execute(select(Packet).where(Packet.id == packet_id)) packet = result.scalar_one_or_none() if not packet: now_us = int(time.time() * 1_000_000) packet_values = { - "id": env.packet.id, + "id": packet_id, "portnum": env.packet.decoded.portnum, "from_node_id": getattr(env.packet, "from"), - "to_node_id": env.packet.to, + "to_node_id": storage_to_node_id(env.packet), "payload": env.packet.SerializeToString(), "import_time_us": now_us, "channel": env.channel_id, @@ -199,7 +228,7 @@ async def process_envelope(topic, env): now_us = int(time.time() * 1_000_000) seen_values = { - "packet_id": env.packet.id, + "packet_id": packet_id, "node_id": int(env.gateway_id[1:], 16), "channel": env.channel_id, "rx_time": env.packet.rx_time, @@ -268,11 +297,13 @@ async def process_envelope(topic, env): ).scalar_one_or_none() now_us = int(time.time() * 1_000_000) + long_name = normalize_node_name(user.long_name) + short_name = normalize_node_name(user.short_name) if node: node.node_id = node_id - node.long_name = user.long_name - node.short_name = user.short_name + node.long_name = long_name + node.short_name = short_name node.hw_model = hw_model node.role = role node.channel = env.channel_id @@ -283,8 +314,8 @@ async def process_envelope(topic, env): node = Node( id=user.id, node_id=node_id, - long_name=user.long_name, - short_name=user.short_name, + long_name=long_name, + short_name=short_name, hw_model=hw_model, role=role, channel=env.channel_id, diff --git a/meshview/templates/firehose.html b/meshview/templates/firehose.html index bc5266a..e109cc1 100644 --- a/meshview/templates/firehose.html +++ b/meshview/templates/firehose.html @@ -241,7 +241,7 @@ function logPacketTimes(packet) { const times = formatTimes(packet.import_time_us); console.log( "[firehose] packet time", - "id=" + packet.id, + "id=" + (packet.packet_id ?? packet.id), "epoch_us=" + times.epoch, "local=" + times.local, "utc=" + times.utc @@ -365,13 +365,21 @@ async function fetchUpdates() { if (match) traceId = match[1]; inlineLinks += ` ⮕`; + href="/traceroute/${traceId}" + >⮕`; } const safePayload = (pkt.payload || "") .replace(//g, ">"); + const packetIdCell = pkt.packet_id + ? ` + ${pkt.packet_id} + ` + : pkt.portnum === 73 + ? `Not at Packet` + : `—`; const html = ` @@ -382,10 +390,7 @@ async function fetchUpdates() { ▶ - - ${pkt.id} - + ${packetIdCell} ${from} diff --git a/meshview/templates/map.html b/meshview/templates/map.html index 0fae47a..4844f4c 100644 --- a/meshview/templates/map.html +++ b/meshview/templates/map.html @@ -148,6 +148,7 @@ +