From 03b5a10fe434c8861428874d884ca2ff2601fa46 Mon Sep 17 00:00:00 2001 From: l5y <220195275+l5yth@users.noreply.github.com> Date: Tue, 14 Oct 2025 14:51:28 +0200 Subject: [PATCH] Add LoRa metadata fields to nodes and messages (#331) * Add LoRa metadata fields to nodes and messages * Filter numeric SQLite keys from message rows --- data/messages.sql | 5 +- data/migrations/20250301_add_lora_columns.sql | 22 +++++++++ data/nodes.sql | 4 +- tests/messages.json | 6 +++ tests/nodes.json | 8 +++- .../application/data_processing.rb | 48 ++++++++++++++++--- web/lib/potato_mesh/application/database.rb | 23 +++++++++ web/lib/potato_mesh/application/queries.rb | 40 ++++++++++++---- web/spec/app_spec.rb | 23 ++++++++- 9 files changed, 159 insertions(+), 20 deletions(-) create mode 100644 data/migrations/20250301_add_lora_columns.sql diff --git a/data/messages.sql b/data/messages.sql index 026851d..6aaecc9 100644 --- a/data/messages.sql +++ b/data/messages.sql @@ -24,7 +24,10 @@ CREATE TABLE IF NOT EXISTS messages ( encrypted TEXT, snr REAL, rssi INTEGER, - hop_limit INTEGER + hop_limit INTEGER, + lora_freq INTEGER, + modem_preset TEXT, + channel_name TEXT ); CREATE INDEX IF NOT EXISTS idx_messages_rx_time ON messages(rx_time); diff --git a/data/migrations/20250301_add_lora_columns.sql b/data/migrations/20250301_add_lora_columns.sql new file mode 100644 index 0000000..cf43b3b --- /dev/null +++ b/data/migrations/20250301_add_lora_columns.sql @@ -0,0 +1,22 @@ +-- Copyright (C) 2025 l5yth +-- +-- Licensed under the Apache License, Version 2.0 (the "License"); +-- you may not use this file except in compliance with the License. +-- You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- +-- Extend the nodes and messages tables with LoRa metadata columns. +BEGIN; +ALTER TABLE nodes ADD COLUMN lora_freq INTEGER; +ALTER TABLE nodes ADD COLUMN modem_preset TEXT; +ALTER TABLE messages ADD COLUMN lora_freq INTEGER; +ALTER TABLE messages ADD COLUMN modem_preset TEXT; +ALTER TABLE messages ADD COLUMN channel_name TEXT; +COMMIT; diff --git a/data/nodes.sql b/data/nodes.sql index fad51a7..c1bba28 100644 --- a/data/nodes.sql +++ b/data/nodes.sql @@ -39,7 +39,9 @@ CREATE TABLE IF NOT EXISTS nodes ( precision_bits INTEGER, latitude REAL, longitude REAL, - altitude REAL + altitude REAL, + lora_freq INTEGER, + modem_preset TEXT ); CREATE INDEX IF NOT EXISTS idx_nodes_last_heard ON nodes(last_heard); diff --git a/tests/messages.json b/tests/messages.json index a601252..80c4705 100644 --- a/tests/messages.json +++ b/tests/messages.json @@ -11,6 +11,9 @@ "rssi": -121, "hop_limit": 1, "snr": -13.25, + "lora_freq": 915, + "modem_preset": "LONG_FAST", + "channel_name": "SpecChannel", "node": { "snr": -13.25, "node_id": "!bba83318", @@ -50,6 +53,9 @@ "rssi": -117, "hop_limit": 3, "snr": -12.0, + "lora_freq": 868, + "modem_preset": "MEDIUM_SLOW", + "channel_name": "SpecChannel", "node": { "snr": -12.0, "node_id": "!43b6e530", diff --git a/tests/nodes.json b/tests/nodes.json index 81ff54c..01524ca 100644 --- a/tests/nodes.json +++ b/tests/nodes.json @@ -20,7 +20,9 @@ "last_seen_iso": "2025-09-16T12:05:30Z", "pos_time_iso": "2025-09-16T12:05:30Z", "location_source": "LOC_FIXTURE_0", - "precision_bits": 10 + "precision_bits": 10, + "lora_freq": 915, + "modem_preset": "LONG_FAST" }, { "node_id": "!d1edc388", @@ -65,7 +67,9 @@ "last_seen_iso": "2025-09-16T12:05:05Z", "pos_time_iso": "2025-09-16T12:05:05Z", "location_source": "LOC_FIXTURE_2", - "precision_bits": 12 + "precision_bits": 12, + "lora_freq": 868, + "modem_preset": "MEDIUM_SLOW" }, { "node_id": "!33602324", diff --git a/web/lib/potato_mesh/application/data_processing.rb b/web/lib/potato_mesh/application/data_processing.rb index b4a4a54..6c6b82c 100644 --- a/web/lib/potato_mesh/application/data_processing.rb +++ b/web/lib/potato_mesh/application/data_processing.rb @@ -220,6 +220,9 @@ module PotatoMesh update_prometheus_metrics(node_id, user, role, met, pos) + lora_freq = coerce_integer(n["lora_freq"] || n["loraFrequency"]) + modem_preset = string_or_nil(n["modem_preset"] || n["modemPreset"]) + row = [ node_id, node_num, @@ -250,13 +253,15 @@ module PotatoMesh pos["latitude"], pos["longitude"], pos["altitude"], + lora_freq, + modem_preset, ] with_busy_retry do db.execute <<~SQL, row INSERT INTO nodes(node_id,num,short_name,long_name,macaddr,hw_model,role,public_key,is_unmessagable,is_favorite, hops_away,snr,last_heard,first_heard,battery_level,voltage,channel_utilization,air_util_tx,uptime_seconds, - position_time,location_source,precision_bits,latitude,longitude,altitude) - VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?) + position_time,location_source,precision_bits,latitude,longitude,altitude,lora_freq,modem_preset) + VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?) ON CONFLICT(node_id) DO UPDATE SET num=excluded.num, short_name=excluded.short_name, long_name=excluded.long_name, macaddr=excluded.macaddr, hw_model=excluded.hw_model, role=excluded.role, public_key=excluded.public_key, is_unmessagable=excluded.is_unmessagable, @@ -265,7 +270,7 @@ module PotatoMesh battery_level=excluded.battery_level, voltage=excluded.voltage, channel_utilization=excluded.channel_utilization, air_util_tx=excluded.air_util_tx, uptime_seconds=excluded.uptime_seconds, position_time=excluded.position_time, location_source=excluded.location_source, precision_bits=excluded.precision_bits, latitude=excluded.latitude, longitude=excluded.longitude, - altitude=excluded.altitude + altitude=excluded.altitude, lora_freq=excluded.lora_freq, modem_preset=excluded.modem_preset WHERE COALESCE(excluded.last_heard,0) >= COALESCE(nodes.last_heard,0) SQL end @@ -941,6 +946,10 @@ module PotatoMesh source: :message, ) + lora_freq = coerce_integer(message["lora_freq"] || message["loraFrequency"]) + modem_preset = string_or_nil(message["modem_preset"] || message["modemPreset"]) + channel_name = string_or_nil(message["channel_name"] || message["channelName"]) + row = [ msg_id, rx_time, @@ -954,11 +963,14 @@ module PotatoMesh message["snr"], message["rssi"], message["hop_limit"], + lora_freq, + modem_preset, + channel_name, ] with_busy_retry do existing = db.get_first_row( - "SELECT from_id, to_id, encrypted FROM messages WHERE id = ?", + "SELECT from_id, to_id, encrypted, lora_freq, modem_preset, channel_name FROM messages WHERE id = ?", [msg_id], ) if existing @@ -988,6 +1000,27 @@ module PotatoMesh updates["encrypted"] = encrypted if should_update end + unless lora_freq.nil? + existing_lora = existing.is_a?(Hash) ? existing["lora_freq"] : existing[3] + updates["lora_freq"] = lora_freq if existing_lora != lora_freq + end + + if modem_preset + existing_preset = existing.is_a?(Hash) ? existing["modem_preset"] : existing[4] + existing_preset_str = existing_preset&.to_s + should_update = existing_preset_str.nil? || existing_preset_str.strip.empty? + should_update ||= existing_preset != modem_preset + updates["modem_preset"] = modem_preset if should_update + end + + if channel_name + existing_channel = existing.is_a?(Hash) ? existing["channel_name"] : existing[5] + existing_channel_str = existing_channel&.to_s + should_update = existing_channel_str.nil? || existing_channel_str.strip.empty? + should_update ||= existing_channel != channel_name + updates["channel_name"] = channel_name if should_update + end + unless updates.empty? assignments = updates.keys.map { |column| "#{column} = ?" }.join(", ") db.execute("UPDATE messages SET #{assignments} WHERE id = ?", updates.values + [msg_id]) @@ -997,14 +1030,17 @@ module PotatoMesh begin db.execute <<~SQL, row - INSERT INTO messages(id,rx_time,rx_iso,from_id,to_id,channel,portnum,text,encrypted,snr,rssi,hop_limit) - VALUES (?,?,?,?,?,?,?,?,?,?,?,?) + INSERT INTO messages(id,rx_time,rx_iso,from_id,to_id,channel,portnum,text,encrypted,snr,rssi,hop_limit,lora_freq,modem_preset,channel_name) + VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?) SQL rescue SQLite3::ConstraintException fallback_updates = {} fallback_updates["from_id"] = from_id if from_id fallback_updates["to_id"] = to_id if to_id fallback_updates["encrypted"] = encrypted if encrypted + fallback_updates["lora_freq"] = lora_freq unless lora_freq.nil? + fallback_updates["modem_preset"] = modem_preset if modem_preset + fallback_updates["channel_name"] = channel_name if channel_name unless fallback_updates.empty? assignments = fallback_updates.keys.map { |column| "#{column} = ?" }.join(", ") db.execute("UPDATE messages SET #{assignments} WHERE id = ?", fallback_updates.values + [msg_id]) diff --git a/web/lib/potato_mesh/application/database.rb b/web/lib/potato_mesh/application/database.rb index 05530d0..b82b8c3 100644 --- a/web/lib/potato_mesh/application/database.rb +++ b/web/lib/potato_mesh/application/database.rb @@ -89,6 +89,29 @@ module PotatoMesh node_columns = db.execute("PRAGMA table_info(nodes)").map { |row| row[1] } unless node_columns.include?("precision_bits") db.execute("ALTER TABLE nodes ADD COLUMN precision_bits INTEGER") + node_columns << "precision_bits" + end + + unless node_columns.include?("lora_freq") + db.execute("ALTER TABLE nodes ADD COLUMN lora_freq INTEGER") + end + + unless node_columns.include?("modem_preset") + db.execute("ALTER TABLE nodes ADD COLUMN modem_preset TEXT") + end + + message_columns = db.execute("PRAGMA table_info(messages)").map { |row| row[1] } + + unless message_columns.include?("lora_freq") + db.execute("ALTER TABLE messages ADD COLUMN lora_freq INTEGER") + end + + unless message_columns.include?("modem_preset") + db.execute("ALTER TABLE messages ADD COLUMN modem_preset TEXT") + end + + unless message_columns.include?("channel_name") + db.execute("ALTER TABLE messages ADD COLUMN channel_name TEXT") end tables = db.execute("SELECT name FROM sqlite_master WHERE type='table' AND name='instances'").flatten diff --git a/web/lib/potato_mesh/application/queries.rb b/web/lib/potato_mesh/application/queries.rb index 2343e4b..cf11e05 100644 --- a/web/lib/potato_mesh/application/queries.rb +++ b/web/lib/potato_mesh/application/queries.rb @@ -148,7 +148,7 @@ module PotatoMesh battery_level, voltage, last_heard, first_heard, uptime_seconds, channel_utilization, air_util_tx, position_time, location_source, precision_bits, - latitude, longitude, altitude + latitude, longitude, altitude, lora_freq, modem_preset FROM nodes SQL sql += " WHERE #{where_clauses.join(" AND ")}\n" if where_clauses.any? @@ -203,7 +203,28 @@ module PotatoMesh end sql = <<~SQL - SELECT m.*, n.*, m.snr AS msg_snr + SELECT m.id, m.rx_time, m.rx_iso, m.from_id, m.to_id, m.channel, + m.portnum, m.text, m.encrypted, m.rssi, m.hop_limit, + m.lora_freq AS msg_lora_freq, m.modem_preset AS msg_modem_preset, + m.channel_name AS msg_channel_name, m.snr AS msg_snr, + n.node_id AS node_node_id, n.num AS node_num, + n.short_name AS node_short_name, n.long_name AS node_long_name, + n.macaddr AS node_macaddr, n.hw_model AS node_hw_model, + n.role AS node_role, n.public_key AS node_public_key, + n.is_unmessagable AS node_is_unmessagable, + n.is_favorite AS node_is_favorite, + n.hops_away AS node_hops_away, n.snr AS node_snr, + n.last_heard AS node_last_heard, n.first_heard AS node_first_heard, + n.battery_level AS node_battery_level, n.voltage AS node_voltage, + n.channel_utilization AS node_channel_utilization, + n.air_util_tx AS node_air_util_tx, + n.uptime_seconds AS node_uptime_seconds, + n.position_time AS node_position_time, + n.location_source AS node_location_source, + n.precision_bits AS node_precision_bits, + n.latitude AS node_latitude, n.longitude AS node_longitude, + n.altitude AS node_altitude, + n.lora_freq AS node_lora_freq, n.modem_preset AS node_modem_preset FROM messages m LEFT JOIN nodes n ON ( m.from_id IS NOT NULL AND TRIM(m.from_id) <> '' AND ( @@ -220,8 +241,12 @@ module PotatoMesh SQL params << limit rows = db.execute(sql, params) - msg_fields = %w[id rx_time rx_iso from_id to_id channel portnum text encrypted msg_snr rssi hop_limit] rows.each do |r| + r.delete_if { |key, _| key.is_a?(Integer) } + r["lora_freq"] = r.delete("msg_lora_freq") + r["modem_preset"] = r.delete("msg_modem_preset") + r["channel_name"] = r.delete("msg_channel_name") + snr_value = r.delete("msg_snr") if PotatoMesh::Config.debug? && (r["from_id"].nil? || r["from_id"].to_s.empty?) raw = db.execute("SELECT * FROM messages WHERE id = ?", [r["id"]]).first debug_log( @@ -238,11 +263,11 @@ module PotatoMesh ) end node = {} - r.keys.each do |k| - next if msg_fields.include?(k) - node[k] = r.delete(k) + r.keys.grep(/^node_/).each do |k| + attribute = k.delete_prefix("node_") + node[attribute] = r.delete(k) end - r["snr"] = r.delete("msg_snr") + r["snr"] = snr_value references = [r["from_id"]].compact if references.any? && (node["node_id"].nil? || node["node_id"].to_s.empty?) lookup_keys = [] @@ -260,7 +285,6 @@ module PotatoMesh if fallback fallback.each do |key, value| next unless key.is_a?(String) - next if msg_fields.include?(key) node[key] = value if node[key].nil? end end diff --git a/web/spec/app_spec.rb b/web/spec/app_spec.rb index bbcaa41..3db34af 100644 --- a/web/spec/app_spec.rb +++ b/web/spec/app_spec.rb @@ -122,6 +122,9 @@ RSpec.describe "Potato Mesh Sinatra app" do ) payload["position"] = position unless position.empty? + payload["lora_freq"] = node["lora_freq"] if node.key?("lora_freq") + payload["modem_preset"] = node["modem_preset"] if node.key?("modem_preset") + payload end @@ -159,6 +162,8 @@ RSpec.describe "Potato Mesh Sinatra app" do "latitude" => node["latitude"], "longitude" => node["longitude"], "altitude" => node["altitude"], + "lora_freq" => node["lora_freq"], + "modem_preset" => node["modem_preset"], } end @@ -1151,7 +1156,8 @@ RSpec.describe "Potato Mesh Sinatra app" do SELECT node_id, short_name, long_name, hw_model, role, snr, battery_level, voltage, last_heard, first_heard, uptime_seconds, channel_utilization, air_util_tx, - position_time, latitude, longitude, altitude + position_time, location_source, precision_bits, + latitude, longitude, altitude, lora_freq, modem_preset FROM nodes ORDER BY node_id SQL @@ -1173,9 +1179,13 @@ RSpec.describe "Potato Mesh Sinatra app" do expect_same_value(row["channel_utilization"], expected["channel_utilization"]) expect_same_value(row["air_util_tx"], expected["air_util_tx"]) expect_same_value(row["position_time"], expected["position_time"]) + expect(row["location_source"]).to eq(expected["location_source"]) + expect_same_value(row["precision_bits"], expected["precision_bits"]) expect_same_value(row["latitude"], expected["latitude"]) expect_same_value(row["longitude"], expected["longitude"]) expect_same_value(row["altitude"], expected["altitude"]) + expect_same_value(row["lora_freq"], expected["lora_freq"]) + expect(row["modem_preset"]).to eq(expected["modem_preset"]) end end end @@ -1408,7 +1418,8 @@ RSpec.describe "Potato Mesh Sinatra app" do db.results_as_hash = true rows = db.execute(<<~SQL) SELECT id, rx_time, rx_iso, from_id, to_id, channel, - portnum, text, snr, rssi, hop_limit + portnum, text, snr, rssi, hop_limit, + lora_freq, modem_preset, channel_name FROM messages ORDER BY id SQL @@ -1427,6 +1438,9 @@ RSpec.describe "Potato Mesh Sinatra app" do expect_same_value(row["snr"], expected["snr"]) expect(row["rssi"]).to eq(expected["rssi"]) expect(row["hop_limit"]).to eq(expected["hop_limit"]) + expect(row["lora_freq"]).to eq(expected["lora_freq"]) + expect(row["modem_preset"]).to eq(expected["modem_preset"]) + expect(row["channel_name"]).to eq(expected["channel_name"]) end end end @@ -2419,6 +2433,9 @@ RSpec.describe "Potato Mesh Sinatra app" do expect_same_value(actual_row["snr"], expected["snr"]) expect(actual_row["rssi"]).to eq(expected["rssi"]) expect(actual_row["hop_limit"]).to eq(expected["hop_limit"]) + expect(actual_row["lora_freq"]).to eq(expected["lora_freq"]) + expect(actual_row["modem_preset"]).to eq(expected["modem_preset"]) + expect(actual_row["channel_name"]).to eq(expected["channel_name"]) if expected["from_id"] lookup_id = expected["from_id"] @@ -2438,6 +2455,8 @@ RSpec.describe "Potato Mesh Sinatra app" do expect(node_actual["long_name"]).to eq(node_expected["long_name"]) expect(node_actual["role"]).to eq(node_expected["role"]) expect_same_value(node_actual["snr"], node_expected["snr"]) + expect(node_actual["lora_freq"]).to eq(node_expected["lora_freq"]) + expect(node_actual["modem_preset"]).to eq(node_expected["modem_preset"]) expect_same_value(node_actual["battery_level"], node_expected["battery_level"]) expect_same_value(node_actual["voltage"], node_expected["voltage"]) expected_last_heard = node_expected["last_heard"]