From aaf77988e92d9e9d2b141ba883fd666d9fd2672b Mon Sep 17 00:00:00 2001 From: l5y <220195275+l5yth@users.noreply.github.com> Date: Sat, 13 Sep 2025 08:39:00 +0200 Subject: [PATCH] data: add mesh node data ingestor python daemon --- README.md | 6 ++++ data/.gitignore | 3 ++ data/nodes.py | 80 +++++++++++++++++++++++++++++++++++++++++++++++++ data/nodes.sh | 5 ++++ data/nodes.sql | 33 ++++++++++++++++++++ 5 files changed, 127 insertions(+) create mode 100644 data/.gitignore create mode 100644 data/nodes.py create mode 100755 data/nodes.sh create mode 100644 data/nodes.sql diff --git a/README.md b/README.md index 1e0f6d8..31872d6 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,8 @@ # potato-mesh a simple node dashboard for berlin mediumfast + + +### data +uses python meshtastic library to ingest mesh data into an sqlite3 database locally + +run `nodes.sh` in `data/` to keep updating node records. diff --git a/data/.gitignore b/data/.gitignore new file mode 100644 index 0000000..c743496 --- /dev/null +++ b/data/.gitignore @@ -0,0 +1,3 @@ +*.db +*.db-wal +*.db-shm diff --git a/data/nodes.py b/data/nodes.py new file mode 100644 index 0000000..a4304ea --- /dev/null +++ b/data/nodes.py @@ -0,0 +1,80 @@ +import json, sqlite3, time, threading +from pathlib import Path +from meshtastic.serial_interface import SerialInterface +from meshtastic.mesh_interface import MeshInterface + +DB = "nodes.db" + +schema = Path("nodes.sql").read_text() +conn = sqlite3.connect(DB, check_same_thread=False) +conn.executescript(schema) +conn.commit() + +def upsert_node(node_id, n): + user = (n.get("user") or {}) + met = (n.get("deviceMetrics") or {}) + pos = (n.get("position") or {}) + row = ( + node_id, + n.get("num"), + user.get("shortName"), + user.get("longName"), + user.get("macaddr"), + user.get("hwModel") or n.get("hwModel"), + user.get("role"), + user.get("publicKey"), + user.get("isUnmessagable"), + n.get("isFavorite"), + n.get("hopsAway"), + n.get("snr"), + n.get("lastHeard"), + met.get("batteryLevel"), + met.get("voltage"), + met.get("channelUtilization"), + met.get("airUtilTx"), + met.get("uptimeSeconds"), + pos.get("time"), + pos.get("locationSource"), + pos.get("latitude"), + pos.get("longitude"), + pos.get("altitude"), + json.dumps(n, ensure_ascii=False) + ) + conn.execute(""" + INSERT INTO nodes(node_id,num,short_name,long_name,macaddr,hw_model,role,public_key,is_unmessagable,is_favorite, + hops_away,snr,last_heard,battery_level,voltage,channel_utilization,air_util_tx,uptime_seconds, + position_time,location_source,latitude,longitude,altitude,node_json) + VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?) + ON CONFLICT(node_id) DO UPDATE SET + num=excluded.num, short_name=excluded.short_name, long_name=excluded.long_name, macaddr=excluded.macaddr, + hw_model=excluded.hw_model, role=excluded.role, public_key=excluded.public_key, is_unmessagable=excluded.is_unmessagable, + is_favorite=excluded.is_favorite, hops_away=excluded.hops_away, snr=excluded.snr, last_heard=excluded.last_heard, + battery_level=excluded.battery_level, voltage=excluded.voltage, channel_utilization=excluded.channel_utilization, + air_util_tx=excluded.air_util_tx, uptime_seconds=excluded.uptime_seconds, position_time=excluded.position_time, + location_source=excluded.location_source, latitude=excluded.latitude, longitude=excluded.longitude, + altitude=excluded.altitude, node_json=excluded.node_json + """, row) + +def snapshot_nodes_periodically(iface: MeshInterface, every_sec=30): + time.sleep(5) # let the library sync initial node DB + while True: + try: + for node_id, n in (getattr(iface, "nodes", {}) or {}).items(): + upsert_node(node_id, n) + conn.commit() + except Exception as e: + print("node snapshot error:", e) + time.sleep(every_sec) + +def main(): + iface = SerialInterface(devPath="/dev/ttyACM0") + # optional: also update on any incoming nodeinfo packets via onReceive if you want + threading.Thread(target=snapshot_nodes_periodically, args=(iface, 30), daemon=True).start() + print("Nodes ingestor running. Ctrl+C to stop.") + try: + while True: time.sleep(60) + except KeyboardInterrupt: + pass + +if __name__ == "__main__": + main() diff --git a/data/nodes.sh b/data/nodes.sh new file mode 100755 index 0000000..621afda --- /dev/null +++ b/data/nodes.sh @@ -0,0 +1,5 @@ +#!/usr/bin/env bash + +python -m venv .venv && source .venv/bin/activate +pip install meshtastic +python nodes.py diff --git a/data/nodes.sql b/data/nodes.sql new file mode 100644 index 0000000..242f78a --- /dev/null +++ b/data/nodes.sql @@ -0,0 +1,33 @@ +-- nodes.sql +PRAGMA journal_mode=WAL; + +CREATE TABLE IF NOT EXISTS nodes ( + node_id TEXT PRIMARY KEY, -- e.g. "!0c63e027" + num INTEGER, -- numeric node number + short_name TEXT, + long_name TEXT, + macaddr TEXT, + hw_model TEXT,nodes + role TEXT, + public_key TEXT, + is_unmessagable BOOLEAN, + is_favorite BOOLEAN, + hops_away INTEGER, + snr REAL, + last_heard INTEGER, -- unix seconds + battery_level REAL, + voltage REAL, + channel_utilization REAL, + air_util_tx REAL, + uptime_seconds INTEGER, + position_time INTEGER, + location_source TEXT, + latitude REAL, + longitude REAL, + altitude REAL, + node_json TEXT NOT NULL -- full original node object for debugging +); + +CREATE INDEX IF NOT EXISTS idx_nodes_last_heard ON nodes(last_heard); +CREATE INDEX IF NOT EXISTS idx_nodes_hw_model ON nodes(hw_model); +CREATE INDEX IF NOT EXISTS idx_nodes_latlon ON nodes(latitude, longitude);