From 94171b779c347429fae1a8434b06a5be85ac95ca Mon Sep 17 00:00:00 2001 From: l5y <220195275+l5yth@users.noreply.github.com> Date: Sat, 13 Sep 2025 14:21:31 +0200 Subject: [PATCH] Remove raw node json column (#12) --- data/nodes.py | 21 +++------------------ data/nodes.sql | 3 +-- test/test_nodes_serialization.py | 6 ++---- web/app.rb | 9 ++++----- 4 files changed, 10 insertions(+), 29 deletions(-) diff --git a/data/nodes.py b/data/nodes.py index ab36779..980a5ff 100644 --- a/data/nodes.py +++ b/data/nodes.py @@ -1,5 +1,4 @@ import json, os, sqlite3, time, threading -from dataclasses import asdict, is_dataclass from pathlib import Path try: # meshtastic is optional for tests @@ -24,19 +23,6 @@ def _get(obj, key, default=None): return getattr(obj, key, default) -def _jsonable(obj): - """Recursively convert dataclasses and objects into JSON-serialisable types.""" - if is_dataclass(obj): - return _jsonable(asdict(obj)) - if isinstance(obj, dict): - return {k: _jsonable(v) for k, v in obj.items()} - if isinstance(obj, (list, tuple)): - return [_jsonable(v) for v in obj] - if hasattr(obj, "__dict__"): - return _jsonable(vars(obj)) - return obj - - def upsert_node(node_id, n): user = _get(n, "user") or {} met = _get(n, "deviceMetrics") or {} @@ -65,14 +51,13 @@ def upsert_node(node_id, n): _get(pos, "latitude"), _get(pos, "longitude"), _get(pos, "altitude"), - json.dumps(_jsonable(n), ensure_ascii=False), ) conn.execute( """ INSERT INTO nodes(node_id,num,short_name,long_name,macaddr,hw_model,role,public_key,is_unmessagable,is_favorite, hops_away,snr,last_heard,battery_level,voltage,channel_utilization,air_util_tx,uptime_seconds, - position_time,location_source,latitude,longitude,altitude,node_json) - VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?) + position_time,location_source,latitude,longitude,altitude) + VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?) ON CONFLICT(node_id) DO UPDATE SET num=excluded.num, short_name=excluded.short_name, long_name=excluded.long_name, macaddr=excluded.macaddr, hw_model=excluded.hw_model, role=excluded.role, public_key=excluded.public_key, is_unmessagable=excluded.is_unmessagable, @@ -80,7 +65,7 @@ def upsert_node(node_id, n): battery_level=excluded.battery_level, voltage=excluded.voltage, channel_utilization=excluded.channel_utilization, air_util_tx=excluded.air_util_tx, uptime_seconds=excluded.uptime_seconds, position_time=excluded.position_time, location_source=excluded.location_source, latitude=excluded.latitude, longitude=excluded.longitude, - altitude=excluded.altitude, node_json=excluded.node_json + altitude=excluded.altitude """, row, ) diff --git a/data/nodes.sql b/data/nodes.sql index 6755f80..a163230 100644 --- a/data/nodes.sql +++ b/data/nodes.sql @@ -24,8 +24,7 @@ CREATE TABLE IF NOT EXISTS nodes ( location_source TEXT, latitude REAL, longitude REAL, - altitude REAL, - node_json TEXT NOT NULL -- full original node object for debugging + altitude REAL ); CREATE INDEX IF NOT EXISTS idx_nodes_last_heard ON nodes(last_heard); diff --git a/test/test_nodes_serialization.py b/test/test_nodes_serialization.py index 46a993b..d02f087 100644 --- a/test/test_nodes_serialization.py +++ b/test/test_nodes_serialization.py @@ -1,4 +1,3 @@ -import json import os import sqlite3 import sys @@ -47,10 +46,9 @@ def test_upsert_node_handles_position(tmp_path): nodes.upsert_node("node1", n) nodes.conn.commit() row = nodes.conn.execute( - "SELECT node_json FROM nodes WHERE node_id=?", ("node1",) + "SELECT latitude FROM nodes WHERE node_id=?", ("node1",) ).fetchone() assert row is not None - data = json.loads(row[0]) - assert data["position"]["latitude"] == 52.5 + assert row[0] == 52.5 finally: os.chdir(cwd) diff --git a/web/app.rb b/web/app.rb index cb7c6f5..5ee52a1 100644 --- a/web/app.rb +++ b/web/app.rb @@ -61,14 +61,13 @@ def upsert_node(db, node_id, n) pos["locationSource"], pos["latitude"], pos["longitude"], - pos["altitude"], - JSON.dump(n) + pos["altitude"] ] db.execute <<~SQL, row INSERT INTO nodes(node_id,num,short_name,long_name,macaddr,hw_model,role,public_key,is_unmessagable,is_favorite, hops_away,snr,last_heard,battery_level,voltage,channel_utilization,air_util_tx,uptime_seconds, - position_time,location_source,latitude,longitude,altitude,node_json) - VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?) + position_time,location_source,latitude,longitude,altitude) + VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?) ON CONFLICT(node_id) DO UPDATE SET num=excluded.num, short_name=excluded.short_name, long_name=excluded.long_name, macaddr=excluded.macaddr, hw_model=excluded.hw_model, role=excluded.role, public_key=excluded.public_key, is_unmessagable=excluded.is_unmessagable, @@ -76,7 +75,7 @@ def upsert_node(db, node_id, n) battery_level=excluded.battery_level, voltage=excluded.voltage, channel_utilization=excluded.channel_utilization, air_util_tx=excluded.air_util_tx, uptime_seconds=excluded.uptime_seconds, position_time=excluded.position_time, location_source=excluded.location_source, latitude=excluded.latitude, longitude=excluded.longitude, - altitude=excluded.altitude, node_json=excluded.node_json + altitude=excluded.altitude WHERE COALESCE(excluded.last_heard,0) >= COALESCE(nodes.last_heard,0) SQL end