From 6775de3ccad76246ed4e64f479f825cf5d72d33d Mon Sep 17 00:00:00 2001 From: l5y <220195275+l5yth@users.noreply.github.com> Date: Sat, 18 Oct 2025 20:16:14 +0200 Subject: [PATCH] Prune blank values from API responses (#386) --- web/lib/potato_mesh/application/queries.rb | 35 +++- web/spec/app_spec.rb | 228 +++++++++++++++++++-- 2 files changed, 240 insertions(+), 23 deletions(-) diff --git a/web/lib/potato_mesh/application/queries.rb b/web/lib/potato_mesh/application/queries.rb index 12458fd..420c65a 100644 --- a/web/lib/potato_mesh/application/queries.rb +++ b/web/lib/potato_mesh/application/queries.rb @@ -17,6 +17,33 @@ module PotatoMesh module Queries MAX_QUERY_LIMIT = 1000 + # Remove nil or empty values from an API response hash to reduce payload size. + # Integer keys emitted by SQLite are ignored because the JSON representation + # only exposes symbolic keys. Strings containing only whitespace are treated + # as empty to mirror sanitisation elsewhere in the application. + # + # @param row [Hash] raw database row to compact. + # @return [Hash] cleaned hash without blank values. + def compact_api_row(row) + return {} unless row.is_a?(Hash) + + row.each_with_object({}) do |(key, value), acc| + next if key.is_a?(Integer) + next if value.nil? + + if value.is_a?(String) + trimmed = value.strip + next if trimmed.empty? + acc[key] = value + next + end + + next if value.respond_to?(:empty?) && value.empty? + + acc[key] = value + end + end + # Normalise a caller-provided limit to a sane, positive integer. # # @param limit [Object] value coerced to an integer. @@ -179,7 +206,7 @@ module PotatoMesh pb = r["precision_bits"] r["precision_bits"] = pb.to_i if pb end - rows + rows.map { |row| compact_api_row(row) } ensure db&.close end @@ -301,7 +328,7 @@ module PotatoMesh r["pdop"] = coerce_float(r["pdop"]) r["snr"] = coerce_float(r["snr"]) end - rows + rows.map { |row| compact_api_row(row) } ensure db&.close end @@ -341,7 +368,7 @@ module PotatoMesh r["rx_iso"] = Time.at(rx_time).utc.iso8601 if rx_time r["snr"] = coerce_float(r["snr"]) end - rows + rows.map { |row| compact_api_row(row) } ensure db&.close end @@ -419,7 +446,7 @@ module PotatoMesh r["soil_moisture"] = coerce_integer(r["soil_moisture"]) r["soil_temperature"] = coerce_float(r["soil_temperature"]) end - rows + rows.map { |row| compact_api_row(row) } ensure db&.close end diff --git a/web/spec/app_spec.rb b/web/spec/app_spec.rb index 7f50440..14da9a7 100644 --- a/web/spec/app_spec.rb +++ b/web/spec/app_spec.rb @@ -3323,30 +3323,19 @@ RSpec.describe "Potato Mesh Sinatra app" do expected = expected_node_row(node) actual_row = actual_by_id.fetch(node["node_id"]) - expect(actual_row["short_name"]).to eq(expected["short_name"]) - expect(actual_row["long_name"]).to eq(expected["long_name"]) - expect(actual_row["hw_model"]).to eq(expected["hw_model"]) - expect(actual_row["role"]).to eq(expected["role"]) - expect_same_value(actual_row["snr"], expected["snr"]) - expect_same_value(actual_row["battery_level"], expected["battery_level"]) - expect_same_value(actual_row["voltage"], expected["voltage"]) - expect(actual_row["last_heard"]).to eq(expected["last_heard"]) - expect(actual_row["first_heard"]).to eq(expected["first_heard"]) - expect_same_value(actual_row["uptime_seconds"], expected["uptime_seconds"]) - expect_same_value(actual_row["channel_utilization"], expected["channel_utilization"]) - expect_same_value(actual_row["air_util_tx"], expected["air_util_tx"]) - expect_same_value(actual_row["position_time"], expected["position_time"]) - expect(actual_row["location_source"]).to eq(expected["location_source"]) - expect_same_value(actual_row["precision_bits"], expected["precision_bits"]) - expect_same_value(actual_row["latitude"], expected["latitude"]) - expect_same_value(actual_row["longitude"], expected["longitude"]) - expect_same_value(actual_row["altitude"], expected["altitude"]) + expected.each do |key, value| + if value.nil? + expect(actual_row).not_to have_key(key), "expected #{key} to be omitted" + else + expect_same_value(actual_row[key], value) + end + end if expected["last_heard"] expected_last_seen_iso = Time.at(expected["last_heard"]).utc.iso8601 expect(actual_row["last_seen_iso"]).to eq(expected_last_seen_iso) else - expect(actual_row["last_seen_iso"]).to be_nil + expect(actual_row).not_to have_key("last_seen_iso") end if node["position_time"] @@ -3391,6 +3380,64 @@ RSpec.describe "Potato Mesh Sinatra app" do payload = JSON.parse(last_response.body) expect(payload["node_id"]).to eq("!fresh-node") end + + it "omits blank values from node responses" do + clear_database + allow(Time).to receive(:now).and_return(reference_time) + now = reference_time.to_i + + with_db do |db| + db.execute( + "INSERT INTO nodes(node_id, short_name, long_name, hw_model, role, snr, battery_level, voltage, last_heard, first_heard, uptime_seconds, channel_utilization, air_util_tx, position_time, location_source, precision_bits, latitude, longitude, altitude) VALUES(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)", + [ + "!blank", + " ", + nil, + "", + nil, + nil, + nil, + nil, + now, + now, + nil, + nil, + nil, + nil, + " ", + nil, + nil, + nil, + nil, + ], + ) + end + + get "/api/nodes" + + expect(last_response).to be_ok + nodes = JSON.parse(last_response.body) + expect(nodes.length).to eq(1) + entry = nodes.first + expect(entry["node_id"]).to eq("!blank") + %w[short_name long_name hw_model snr battery_level voltage uptime_seconds channel_utilization air_util_tx position_time location_source precision_bits latitude longitude altitude].each do |attribute| + expect(entry).not_to have_key(attribute), "expected #{attribute} to be omitted" + end + + expect(entry["role"]).to eq("CLIENT") + expect(entry["last_heard"]).to eq(now) + expect(entry["first_heard"]).to eq(now) + expect(entry["last_seen_iso"]).to eq(Time.at(now).utc.iso8601) + expect(entry).not_to have_key("pos_time_iso") + + get "/api/nodes/!blank" + + expect(last_response).to be_ok + payload = JSON.parse(last_response.body) + expect(payload["node_id"]).to eq("!blank") + expect(payload).not_to have_key("short_name") + expect(payload).not_to have_key("hw_model") + end end describe "GET /api/messages" do @@ -3680,6 +3727,55 @@ RSpec.describe "Potato Mesh Sinatra app" do filtered = JSON.parse(last_response.body) expect(filtered.map { |row| row["id"] }).to eq([2]) end + + it "omits blank values from position responses" do + clear_database + allow(Time).to receive(:now).and_return(reference_time) + now = reference_time.to_i + + with_db do |db| + db.execute( + "INSERT INTO positions(id, node_id, node_num, rx_time, rx_iso, position_time, latitude, longitude, altitude, location_source, precision_bits, sats_in_view, pdop, payload_b64) VALUES(?,?,?,?,?,?,?,?,?,?,?,?,?,?)", + [ + 7, + "!pos-blank", + nil, + now, + " ", + nil, + nil, + nil, + nil, + " ", + nil, + nil, + nil, + "", + ], + ) + end + + get "/api/positions" + + expect(last_response).to be_ok + rows = JSON.parse(last_response.body) + expect(rows.length).to eq(1) + entry = rows.first + expect(entry["node_id"]).to eq("!pos-blank") + expect(entry["rx_time"]).to eq(now) + expect(entry["rx_iso"]).to eq(Time.at(now).utc.iso8601) + %w[position_time latitude longitude altitude location_source precision_bits sats_in_view pdop payload_b64].each do |attribute| + expect(entry).not_to have_key(attribute), "expected #{attribute} to be omitted" + end + + get "/api/positions/!pos-blank" + + expect(last_response).to be_ok + filtered = JSON.parse(last_response.body) + expect(filtered.length).to eq(1) + expect(filtered.first).not_to have_key("payload_b64") + expect(filtered.first).not_to have_key("location_source") + end end describe "GET /api/neighbors" do @@ -3729,6 +3825,45 @@ RSpec.describe "Potato Mesh Sinatra app" do expect(filtered.first["neighbor_id"]).to eq("!neighbor-new") expect(filtered.first["rx_time"]).to eq(fresh_rx) end + + it "omits blank values from neighbor responses" do + clear_database + allow(Time).to receive(:now).and_return(reference_time) + now = reference_time.to_i + + with_db do |db| + db.execute( + "INSERT INTO nodes(node_id, short_name, long_name, hw_model, role, snr, last_heard, first_heard) VALUES(?,?,?,?,?,?,?,?)", + ["!origin", "orig", "Origin", "TBEAM", "CLIENT", 0.0, now, now], + ) + db.execute( + "INSERT INTO nodes(node_id, short_name, long_name, hw_model, role, snr, last_heard, first_heard) VALUES(?,?,?,?,?,?,?,?)", + ["!neighbor", "neig", "Neighbor", "TBEAM", "CLIENT", 0.0, now, now], + ) + db.execute( + "INSERT INTO neighbors(node_id, neighbor_id, snr, rx_time) VALUES(?,?,?,?)", + ["!origin", "!neighbor", nil, now], + ) + end + + get "/api/neighbors" + + expect(last_response).to be_ok + payload = JSON.parse(last_response.body) + expect(payload.length).to eq(1) + entry = payload.first + expect(entry["node_id"]).to eq("!origin") + expect(entry["neighbor_id"]).to eq("!neighbor") + expect(entry["rx_time"]).to eq(now) + expect(entry).not_to have_key("snr") + + get "/api/neighbors/!origin" + + expect(last_response).to be_ok + filtered = JSON.parse(last_response.body) + expect(filtered.length).to eq(1) + expect(filtered.first).not_to have_key("snr") + end end describe "GET /api/telemetry" do @@ -3819,5 +3954,60 @@ RSpec.describe "Potato Mesh Sinatra app" do filtered = JSON.parse(last_response.body) expect(filtered.map { |row| row["id"] }).to eq([2]) end + + it "omits blank values from telemetry responses" do + clear_database + allow(Time).to receive(:now).and_return(reference_time) + now = reference_time.to_i + + with_db do |db| + db.execute( + "INSERT INTO telemetry(id, node_id, node_num, rx_time, rx_iso, telemetry_time, channel, portnum, hop_limit, snr, rssi, bitfield, payload_b64, battery_level, voltage, channel_utilization, air_util_tx, uptime_seconds, temperature, relative_humidity) VALUES(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)", + [ + 77, + "!tele-blank", + nil, + now, + " ", + nil, + nil, + "", + nil, + nil, + nil, + nil, + "", + nil, + nil, + nil, + nil, + nil, + nil, + nil, + ], + ) + end + + get "/api/telemetry" + + expect(last_response).to be_ok + rows = JSON.parse(last_response.body) + expect(rows.length).to eq(1) + entry = rows.first + expect(entry["node_id"]).to eq("!tele-blank") + expect(entry["rx_time"]).to eq(now) + expect(entry["rx_iso"]).to eq(Time.at(now).utc.iso8601) + %w[telemetry_time channel portnum hop_limit snr rssi bitfield payload_b64 battery_level voltage channel_utilization air_util_tx uptime_seconds temperature relative_humidity].each do |attribute| + expect(entry).not_to have_key(attribute), "expected #{attribute} to be omitted" + end + + get "/api/telemetry/!tele-blank" + + expect(last_response).to be_ok + filtered = JSON.parse(last_response.body) + expect(filtered.length).to eq(1) + expect(filtered.first).not_to have_key("battery_level") + expect(filtered.first).not_to have_key("portnum") + end end end