diff --git a/data/mesh_ingestor/handlers.py b/data/mesh_ingestor/handlers.py index 10fab90..8405eb1 100644 --- a/data/mesh_ingestor/handlers.py +++ b/data/mesh_ingestor/handlers.py @@ -372,6 +372,132 @@ def base64_payload(payload_bytes: bytes | None) -> str | None: return base64.b64encode(payload_bytes).decode("ascii") +def _normalize_trace_hops(hops_value) -> list[int]: + """Coerce hop entries to integers while preserving order.""" + + if hops_value is None: + return [] + hop_entries = hops_value if isinstance(hops_value, list) else [hops_value] + normalized: list[int] = [] + for hop in hop_entries: + hop_value = hop + if isinstance(hop, Mapping): + hop_value = _first(hop, "node_id", "nodeId", "id", "num", default=None) + + canonical = _canonical_node_id(hop_value) + hop_id = _node_num_from_id(canonical or hop_value) + if hop_id is None: + hop_id = _coerce_int(hop_value) + if hop_id is not None: + normalized.append(hop_id) + return normalized + + +def store_traceroute_packet(packet: Mapping, decoded: Mapping) -> None: + """Persist traceroute details and hop path to the API.""" + + traceroute_section = ( + decoded.get("traceroute") if isinstance(decoded, Mapping) else None + ) + request_id = _coerce_int( + _first( + traceroute_section, + "requestId", + "request_id", + default=_first(decoded, "req", "requestId", "request_id", default=None), + ) + ) + pkt_id = _coerce_int(_first(packet, "id", "packet_id", "packetId", default=None)) + if pkt_id is None: + pkt_id = request_id + + rx_time = _coerce_int(_first(packet, "rxTime", "rx_time", default=time.time())) + if rx_time is None: + rx_time = int(time.time()) + + src = _coerce_int( + _first( + decoded, + "src", + "source", + default=_first(packet, "fromId", "from_id", "from", default=None), + ) + ) + dest = _coerce_int( + _first( + decoded, + "dest", + "destination", + default=_first(packet, "toId", "to_id", "to", default=None), + ) + ) + + metrics = traceroute_section if isinstance(traceroute_section, Mapping) else {} + rssi = _coerce_int( + _first(metrics, "rssi", default=_first(packet, "rssi", "rx_rssi", "rxRssi")) + ) + snr = _coerce_float( + _first(metrics, "snr", default=_first(packet, "snr", "rx_snr", "rxSnr")) + ) + elapsed_ms = _coerce_int( + _first(metrics, "elapsed_ms", "latency_ms", "latencyMs", default=None) + ) + + hop_candidates = ( + _first(metrics, "hops", default=None), + _first(metrics, "path", default=None), + _first(metrics, "route", default=None), + _first(decoded, "hops", default=None), + _first(decoded, "path", default=None), + ( + _first(traceroute_section, "route", default=None) + if isinstance(traceroute_section, Mapping) + else None + ), + ) + hops: list[int] = [] + seen_hops: set[int] = set() + for candidate in hop_candidates: + for hop in _normalize_trace_hops(candidate): + if hop in seen_hops: + continue + seen_hops.add(hop) + hops.append(hop) + + if pkt_id is None and request_id is None and not hops: + _record_ignored_packet(packet, reason="traceroute-missing-identifiers") + return + + payload = { + "id": pkt_id, + "request_id": request_id, + "src": src, + "dest": dest, + "rx_time": rx_time, + "rx_iso": _iso(rx_time), + "hops": hops, + "rssi": rssi, + "snr": snr, + "elapsed_ms": elapsed_ms, + } + + _queue_post_json( + "/api/traces", + _apply_radio_metadata(payload), + priority=queue._TRACE_POST_PRIORITY, + ) + + if config.DEBUG: + config._debug_log( + "Queued traceroute payload", + context="handlers.store_traceroute_packet", + request_id=request_id, + src=src, + dest=dest, + hop_count=len(hops), + ) + + def store_telemetry_packet(packet: Mapping, decoded: Mapping) -> None: """Persist telemetry metrics extracted from a packet. @@ -1084,6 +1210,40 @@ def store_packet_dict(packet: Mapping) -> None: store_telemetry_packet(packet, decoded) return + traceroute_section = ( + decoded.get("traceroute") if isinstance(decoded, Mapping) else None + ) + traceroute_port_ints: set[int] = set() + for module_name in ( + "meshtastic.portnums_pb2", + "meshtastic.protobuf.portnums_pb2", + ): + module = sys.modules.get(module_name) + if module is None: + with contextlib.suppress(ModuleNotFoundError): + module = importlib.import_module(module_name) + if module is None: + continue + portnum_enum = getattr(module, "PortNum", None) + value_lookup = getattr(portnum_enum, "Value", None) if portnum_enum else None + if callable(value_lookup): + with contextlib.suppress(Exception): + candidate = _coerce_int(value_lookup("TRACEROUTE_APP")) + if candidate is not None: + traceroute_port_ints.add(candidate) + constant_value = getattr(module, "TRACEROUTE_APP", None) + candidate = _coerce_int(constant_value) + if candidate is not None: + traceroute_port_ints.add(candidate) + + if ( + portnum == "TRACEROUTE_APP" + or (portnum_int is not None and portnum_int in traceroute_port_ints) + or isinstance(traceroute_section, Mapping) + ): + store_traceroute_packet(packet, decoded) + return + if portnum in {"5", "NODEINFO_APP"}: store_nodeinfo_packet(packet, decoded) return diff --git a/data/mesh_ingestor/queue.py b/data/mesh_ingestor/queue.py index c5c5cfa..d601088 100644 --- a/data/mesh_ingestor/queue.py +++ b/data/mesh_ingestor/queue.py @@ -75,6 +75,7 @@ def _payload_key_value_pairs(payload: Mapping[str, object]) -> str: _MESSAGE_POST_PRIORITY = 10 _NEIGHBOR_POST_PRIORITY = 20 +_TRACE_POST_PRIORITY = 25 _POSITION_POST_PRIORITY = 30 _TELEMETRY_POST_PRIORITY = 40 _NODE_POST_PRIORITY = 50 @@ -261,6 +262,7 @@ __all__ = [ "_NEIGHBOR_POST_PRIORITY", "_NODE_POST_PRIORITY", "_POSITION_POST_PRIORITY", + "_TRACE_POST_PRIORITY", "_TELEMETRY_POST_PRIORITY", "_clear_post_queue", "_drain_post_queue", diff --git a/data/traces.sql b/data/traces.sql new file mode 100644 index 0000000..f003aa7 --- /dev/null +++ b/data/traces.sql @@ -0,0 +1,38 @@ +-- Copyright © 2025-26 l5yth & contributors +-- +-- Licensed under the Apache License, Version 2.0 (the "License"); +-- you may not use this file except in compliance with the License. +-- You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. + +CREATE TABLE IF NOT EXISTS traces ( + id INTEGER PRIMARY KEY, + request_id INTEGER, + src INTEGER, + dest INTEGER, + rx_time INTEGER NOT NULL, + rx_iso TEXT NOT NULL, + rssi INTEGER, + snr REAL, + elapsed_ms INTEGER +); + +CREATE TABLE IF NOT EXISTS trace_hops ( + id INTEGER PRIMARY KEY, + trace_id INTEGER NOT NULL, + hop_index INTEGER NOT NULL, + node_id INTEGER NOT NULL, + FOREIGN KEY(trace_id) REFERENCES traces(id) ON DELETE CASCADE +); + +CREATE INDEX IF NOT EXISTS idx_traces_rx_time ON traces(rx_time); +CREATE INDEX IF NOT EXISTS idx_traces_request ON traces(request_id); +CREATE INDEX IF NOT EXISTS idx_trace_hops_trace ON trace_hops(trace_id); +CREATE INDEX IF NOT EXISTS idx_trace_hops_node ON trace_hops(node_id); diff --git a/tests/test_mesh.py b/tests/test_mesh.py index 8714276..5b7839d 100644 --- a/tests/test_mesh.py +++ b/tests/test_mesh.py @@ -1978,6 +1978,109 @@ def test_store_packet_dict_handles_environment_telemetry(mesh_module, monkeypatc assert payload["modem_preset"] == "MediumFast" +def test_store_packet_dict_handles_traceroute_packet(mesh_module, monkeypatch): + mesh = mesh_module + captured = [] + monkeypatch.setattr( + mesh, + "_queue_post_json", + lambda path, payload, *, priority: captured.append((path, payload, priority)), + ) + + mesh.config.LORA_FREQ = 915 + mesh.config.MODEM_PRESET = "LongFast" + + packet = { + "id": 2_934_054_466, + "rxTime": 1_763_183_133, + "rssi": -70, + "snr": 10.25, + "fromId": "3664074452", + "decoded": { + "portnum": "PAXCOUNTER_APP", + "dest": "2660618080", + "traceroute": { + "requestId": 17, + "route": [3_663_643_096, "!beadf00d", "c0ffee99", 1_150_717_793], + "snrTowards": [42, -14, 41], + }, + }, + } + + mesh.store_packet_dict(packet) + + assert captured + path, payload, priority = captured[0] + assert path == "/api/traces" + assert priority == mesh._TRACE_POST_PRIORITY + assert payload["id"] == packet["id"] + assert payload["request_id"] == 17 + assert payload["src"] == 3_664_074_452 + assert payload["dest"] == 2_660_618_080 + assert payload["rx_time"] == 1_763_183_133 + assert payload["rx_iso"] == "2025-11-15T05:05:33Z" + assert payload["hops"] == [ + 3_663_643_096, + 3_199_070_221, + 3_237_998_233, + 1_150_717_793, + ] + assert payload["rssi"] == -70 + assert payload["snr"] == pytest.approx(10.25) + assert "elapsed_ms" in payload + assert payload["lora_freq"] == 915 + assert payload["modem_preset"] == "LongFast" + + +def test_traceroute_hop_normalization_supports_mappings(mesh_module, monkeypatch): + mesh = mesh_module + captured = [] + monkeypatch.setattr( + mesh, + "_queue_post_json", + lambda path, payload, *, priority: captured.append((path, payload, priority)), + ) + + packet = { + "id": 1_111, + "decoded": { + "portnum": "TRACEROUTE_APP", + "traceroute": { + "requestId": 42, + "route": [{"node_id": "!beadf00d"}, {"num": "0xc0ffee99"}, {"id": 123}], + }, + }, + } + + mesh.store_packet_dict(packet) + + assert captured + _, payload, _ = captured[0] + assert payload["hops"] == [0xBEADF00D, 0xC0FFEE99, 123] + + +def test_traceroute_packet_without_identifiers_is_ignored(mesh_module, monkeypatch): + mesh = mesh_module + captured = [] + monkeypatch.setattr( + mesh, + "_queue_post_json", + lambda path, payload, *, priority: captured.append((path, payload, priority)), + ) + + packet = { + "decoded": { + "portnum": "TRACEROUTE_APP", + "traceroute": {}, + }, + "rxTime": 123, + } + + mesh.store_packet_dict(packet) + + assert captured == [] + + def test_post_queue_prioritises_messages(mesh_module, monkeypatch): mesh = mesh_module mesh._clear_post_queue() diff --git a/web/lib/potato_mesh/application/data_processing.rb b/web/lib/potato_mesh/application/data_processing.rb index c83cc38..5d796d1 100644 --- a/web/lib/potato_mesh/application/data_processing.rb +++ b/web/lib/potato_mesh/application/data_processing.rb @@ -777,6 +777,44 @@ module PotatoMesh private :resolve_numeric_metric + # Normalise a traceroute hop entry to a numeric node identifier. + # + # @param hop [Object] raw hop entry from the payload. + # @return [Integer, nil] coerced node ID or nil when the value is unusable. + def coerce_trace_node_id(hop) + case hop + when Integer + return hop + when Numeric + return hop.to_i + when String + trimmed = hop.strip + return nil if trimmed.empty? + return Integer(trimmed, 10) if trimmed.match?(/\A-?\d+\z/) + + parts = canonical_node_parts(trimmed) + return parts[1] if parts + when Hash + candidate = hop["node_id"] || hop[:node_id] || hop["id"] || hop[:id] || hop["num"] || hop[:num] + return coerce_trace_node_id(candidate) + end + + nil + rescue ArgumentError + nil + end + + # Extract hop identifiers from a traceroute payload preserving order. + # + # @param hops_value [Object] raw hops array or path collection. + # @return [Array] ordered list of coerced hop identifiers. + def normalize_trace_hops(hops_value) + return [] if hops_value.nil? + + hop_entries = hops_value.is_a?(Array) ? hops_value : [hops_value] + hop_entries.filter_map { |entry| coerce_trace_node_id(entry) } + end + def insert_telemetry(db, payload) return unless payload.is_a?(Hash) @@ -1206,6 +1244,74 @@ module PotatoMesh }) end + # Persist a traceroute observation and its hop path. + # + # @param db [SQLite3::Database] open database handle. + # @param payload [Hash] traceroute payload as produced by the ingestor. + # @return [void] + def insert_trace(db, payload) + return unless payload.is_a?(Hash) + + trace_identifier = coerce_integer(payload["id"] || payload["packet_id"] || payload["packetId"]) + trace_identifier ||= coerce_integer(payload["trace_id"]) + request_id = coerce_integer(payload["request_id"] || payload["req"]) + trace_identifier ||= request_id + + now = Time.now.to_i + rx_time = coerce_integer(payload["rx_time"]) + rx_time = now if rx_time.nil? || rx_time > now + rx_iso = string_or_nil(payload["rx_iso"]) || Time.at(rx_time).utc.iso8601 + + metrics = normalize_json_object(payload["metrics"]) + src = coerce_integer(payload["src"] || payload["source"] || payload["from"]) + dest = coerce_integer(payload["dest"] || payload["destination"] || payload["to"]) + rssi = coerce_integer(payload["rssi"]) || coerce_integer(metrics["rssi"]) + snr = coerce_float(payload["snr"]) || coerce_float(metrics["snr"]) + elapsed_ms = coerce_integer( + payload["elapsed_ms"] || + payload["latency_ms"] || + metrics&.[]("elapsed_ms") || + metrics&.[]("latency_ms") || + metrics&.[]("latencyMs"), + ) + + hops_value = payload.key?("hops") ? payload["hops"] : payload["path"] + hops = normalize_trace_hops(hops_value) + + all_nodes = [src, dest, *hops].compact.uniq + all_nodes.each do |node| + ensure_unknown_node(db, node, node, heard_time: rx_time) + touch_node_last_seen(db, node, node, rx_time: rx_time, source: :trace) + end + + with_busy_retry do + db.execute <<~SQL, [trace_identifier, request_id, src, dest, rx_time, rx_iso, rssi, snr, elapsed_ms] + INSERT INTO traces(id, request_id, src, dest, rx_time, rx_iso, rssi, snr, elapsed_ms) + VALUES(?,?,?,?,?,?,?,?,?) + ON CONFLICT(id) DO UPDATE SET + request_id=COALESCE(excluded.request_id,traces.request_id), + src=COALESCE(excluded.src,traces.src), + dest=COALESCE(excluded.dest,traces.dest), + rx_time=excluded.rx_time, + rx_iso=excluded.rx_iso, + rssi=COALESCE(excluded.rssi,traces.rssi), + snr=COALESCE(excluded.snr,traces.snr), + elapsed_ms=COALESCE(excluded.elapsed_ms,traces.elapsed_ms) + SQL + + trace_id = trace_identifier || db.last_insert_row_id + return unless trace_id + + db.execute("DELETE FROM trace_hops WHERE trace_id = ?", [trace_id]) + hops.each_with_index do |hop_id, index| + db.execute( + "INSERT INTO trace_hops(trace_id, hop_index, node_id) VALUES(?,?,?)", + [trace_id, index, hop_id], + ) + end + end + end + def insert_message(db, message) return unless message.is_a?(Hash) diff --git a/web/lib/potato_mesh/application/database.rb b/web/lib/potato_mesh/application/database.rb index d5120d6..fac2d14 100644 --- a/web/lib/potato_mesh/application/database.rb +++ b/web/lib/potato_mesh/application/database.rb @@ -81,10 +81,10 @@ module PotatoMesh return false unless File.exist?(PotatoMesh::Config.db_path) db = open_database(readonly: true) - required = %w[nodes messages positions telemetry neighbors instances] + required = %w[nodes messages positions telemetry neighbors instances traces trace_hops] tables = db.execute( - "SELECT name FROM sqlite_master WHERE type='table' AND name IN ('nodes','messages','positions','telemetry','neighbors','instances')", + "SELECT name FROM sqlite_master WHERE type='table' AND name IN ('nodes','messages','positions','telemetry','neighbors','instances','traces','trace_hops')", ).flatten (required - tables).empty? rescue SQLite3::Exception @@ -99,7 +99,7 @@ module PotatoMesh def init_db FileUtils.mkdir_p(File.dirname(PotatoMesh::Config.db_path)) db = open_database - %w[nodes messages positions telemetry neighbors instances].each do |schema| + %w[nodes messages positions telemetry neighbors instances traces].each do |schema| sql_file = File.expand_path("../../../../data/#{schema}.sql", __dir__) db.execute_batch(File.read(sql_file)) end @@ -178,6 +178,15 @@ module PotatoMesh db.execute("ALTER TABLE telemetry ADD COLUMN #{name} #{type}") telemetry_columns << name end + + trace_tables = + db.execute( + "SELECT name FROM sqlite_master WHERE type='table' AND name IN ('traces','trace_hops')", + ).flatten + unless trace_tables.include?("traces") && trace_tables.include?("trace_hops") + traces_schema = File.expand_path("../../../../data/traces.sql", __dir__) + db.execute_batch(File.read(traces_schema)) + end rescue SQLite3::SQLException, Errno::ENOENT => e warn_log( "Failed to apply schema upgrade", diff --git a/web/lib/potato_mesh/application/queries.rb b/web/lib/potato_mesh/application/queries.rb index 4beca86..35ae042 100644 --- a/web/lib/potato_mesh/application/queries.rb +++ b/web/lib/potato_mesh/application/queries.rb @@ -110,7 +110,7 @@ module PotatoMesh cleaned_strings = string_values.compact.map(&:to_s).map(&:strip).reject(&:empty?).uniq cleaned_numbers = numeric_values.compact.map do |value| begin - Integer(value, 10) + value.is_a?(String) ? Integer(value, 10) : Integer(value) rescue ArgumentError, TypeError nil end @@ -464,6 +464,79 @@ module PotatoMesh ensure db&.close end + + def query_traces(limit, node_ref: nil) + limit = coerce_query_limit(limit) + db = open_database(readonly: true) + db.results_as_hash = true + params = [] + where_clauses = [] + + if node_ref + tokens = node_reference_tokens(node_ref) + numeric_values = tokens[:numeric_values] + if numeric_values.empty? + return [] + end + placeholders = Array.new(numeric_values.length, "?").join(", ") + candidate_clauses = [] + candidate_clauses << "src IN (#{placeholders})" + candidate_clauses << "dest IN (#{placeholders})" + candidate_clauses << "id IN (SELECT trace_id FROM trace_hops WHERE node_id IN (#{placeholders}))" + where_clauses << "(#{candidate_clauses.join(" OR ")})" + 3.times { params.concat(numeric_values) } + end + + sql = <<~SQL + SELECT id, request_id, src, dest, rx_time, rx_iso, rssi, snr, elapsed_ms + FROM traces + SQL + sql += " WHERE #{where_clauses.join(" AND ")}\n" if where_clauses.any? + sql += <<~SQL + ORDER BY rx_time DESC + LIMIT ? + SQL + params << limit + rows = db.execute(sql, params) + + trace_ids = rows.map { |row| coerce_integer(row["id"]) }.compact + hops_by_trace = Hash.new { |hash, key| hash[key] = [] } + unless trace_ids.empty? + placeholders = Array.new(trace_ids.length, "?").join(", ") + hop_rows = + db.execute( + "SELECT trace_id, hop_index, node_id FROM trace_hops WHERE trace_id IN (#{placeholders}) ORDER BY trace_id, hop_index", + trace_ids, + ) + hop_rows.each do |hop| + trace_id = coerce_integer(hop["trace_id"]) + node_id = coerce_integer(hop["node_id"]) + next unless trace_id && node_id + + hops_by_trace[trace_id] << node_id + end + end + + rows.each do |r| + rx_time = coerce_integer(r["rx_time"]) + r["rx_time"] = rx_time if rx_time + r["rx_iso"] = Time.at(rx_time).utc.iso8601 if rx_time && string_or_nil(r["rx_iso"]).nil? + r["request_id"] = coerce_integer(r["request_id"]) + r["src"] = coerce_integer(r["src"]) + r["dest"] = coerce_integer(r["dest"]) + r["rssi"] = coerce_integer(r["rssi"]) + r["snr"] = coerce_float(r["snr"]) + r["elapsed_ms"] = coerce_integer(r["elapsed_ms"]) + + trace_id = coerce_integer(r["id"]) + if trace_id && hops_by_trace.key?(trace_id) + r["hops"] = hops_by_trace[trace_id] + end + end + rows.map { |row| compact_api_row(row) } + ensure + db&.close + end end end end diff --git a/web/lib/potato_mesh/application/routes/api.rb b/web/lib/potato_mesh/application/routes/api.rb index 3cf92ed..0e83db2 100644 --- a/web/lib/potato_mesh/application/routes/api.rb +++ b/web/lib/potato_mesh/application/routes/api.rb @@ -135,6 +135,20 @@ module PotatoMesh query_telemetry(limit, node_ref: node_ref).to_json end + app.get "/api/traces" do + content_type :json + limit = [params["limit"]&.to_i || 200, 1000].min + query_traces(limit).to_json + end + + app.get "/api/traces/:id" do + content_type :json + node_ref = string_or_nil(params["id"]) + halt 400, { error: "missing node id" }.to_json unless node_ref + limit = [params["limit"]&.to_i || 200, 1000].min + query_traces(limit, node_ref: node_ref).to_json + end + app.get "/api/instances" do # Prevent the federation catalog from being exposed when federation is disabled. halt 404 unless federation_enabled? diff --git a/web/lib/potato_mesh/application/routes/ingest.rb b/web/lib/potato_mesh/application/routes/ingest.rb index 98fdff5..c625450 100644 --- a/web/lib/potato_mesh/application/routes/ingest.rb +++ b/web/lib/potato_mesh/application/routes/ingest.rb @@ -321,6 +321,25 @@ module PotatoMesh ensure db&.close end + + app.post "/api/traces" do + require_token! + content_type :json + begin + data = JSON.parse(read_json_body) + rescue JSON::ParserError + halt 400, { error: "invalid JSON" }.to_json + end + trace_packets = data.is_a?(Array) ? data : [data] + halt 400, { error: "too many traces" }.to_json if trace_packets.size > 1000 + db = open_database + trace_packets.each do |packet| + insert_trace(db, packet) + end + { status: "ok" }.to_json + ensure + db&.close + end end end end diff --git a/web/spec/app_spec.rb b/web/spec/app_spec.rb index d7e5c42..d10413d 100644 --- a/web/spec/app_spec.rb +++ b/web/spec/app_spec.rb @@ -96,6 +96,8 @@ RSpec.describe "Potato Mesh Sinatra app" do def clear_database with_db do |db| db.execute("DELETE FROM instances") + db.execute("DELETE FROM trace_hops") + db.execute("DELETE FROM traces") db.execute("DELETE FROM neighbors") db.execute("DELETE FROM messages") db.execute("DELETE FROM nodes") @@ -270,6 +272,30 @@ RSpec.describe "Potato Mesh Sinatra app" do let(:nodes_fixture) { JSON.parse(File.read(fixture_path("nodes.json"))) } let(:messages_fixture) { JSON.parse(File.read(fixture_path("messages.json"))) } let(:telemetry_fixture) { JSON.parse(File.read(fixture_path("telemetry.json"))) } + let(:trace_fixture) do + [ + { + "id" => 9_001, + "request_id" => 17, + "src" => 2_658_361_180, + "dest" => 4_242_424_242, + "rx_time" => reference_time.to_i - 2, + "hops" => [2_658_361_180, 19_088_743, 4_242_424_242], + "rssi" => -83, + "snr" => 5.0, + "elapsed_ms" => 842, + }, + { + "packet_id" => 9_002, + "req" => 21, + "from" => 19_088_743, + "destination" => 2_658_361_180, + "rx_time" => reference_time.to_i - 5, + "path" => [{ "node_id" => "0xbeadf00d" }, { "node_id" => 19_088_743 }], + "metrics" => { "snr" => 3.5, "latency_ms" => 1_020 }, + }, + ] + end let(:reference_time) do latest = nodes_fixture.map { |node| node["last_heard"] }.compact.max Time.at((latest || Time.now.to_i) + 1000) @@ -3186,6 +3212,88 @@ RSpec.describe "Potato Mesh Sinatra app" do end end + describe "POST /api/traces" do + it "stores traces with hop paths and updates last heard timestamps" do + payload = trace_fixture + + post "/api/traces", payload.to_json, auth_headers + + expect(last_response).to be_ok + expect(JSON.parse(last_response.body)).to eq("status" => "ok") + + with_db(readonly: true) do |db| + db.results_as_hash = true + + traces = db.execute("SELECT * FROM traces ORDER BY rx_time DESC") + expect(traces.size).to eq(payload.size) + + primary = traces.find { |row| row["id"] == payload.first["id"] } + expect(primary["request_id"]).to eq(payload.first["request_id"]) + expect(primary["src"]).to eq(payload.first["src"]) + expect(primary["dest"]).to eq(payload.first["dest"]) + expect(primary["rx_time"]).to eq(payload.first["rx_time"]) + expect(primary["rx_iso"]).to eq(Time.at(payload.first["rx_time"]).utc.iso8601) + expect(primary["rssi"]).to eq(payload.first["rssi"]) + expect(primary["snr"]).to eq(payload.first["snr"]) + expect(primary["elapsed_ms"]).to eq(payload.first["elapsed_ms"]) + + primary_hops = db.execute( + "SELECT hop_index, node_id FROM trace_hops WHERE trace_id = ? ORDER BY hop_index", + [primary["id"]], + ) + expect(primary_hops.map { |row| row["node_id"] }).to eq(payload.first["hops"]) + + secondary = traces.find { |row| row["id"] == payload.last["packet_id"] } + expect(secondary["request_id"]).to eq(payload.last["req"]) + expect(secondary["src"]).to eq(payload.last["from"]) + expect(secondary["dest"]).to eq(payload.last["destination"]) + expect(secondary["rssi"]).to be_nil + expect(secondary["snr"]).to eq(payload.last.dig("metrics", "snr")) + expect(secondary["elapsed_ms"]).to eq(payload.last.dig("metrics", "latency_ms")) + + secondary_hops = db.execute( + "SELECT hop_index, node_id FROM trace_hops WHERE trace_id = ? ORDER BY hop_index", + [secondary["id"]], + ) + expect(secondary_hops.map { |row| row["node_id"] }).to eq([0xBEADF00D, 19_088_743]) + + node_ids = [ + payload.first["src"], + payload.first["dest"], + payload.first["hops"][1], + 0xBEADF00D, + ].map { |num| format("!%08x", num & 0xFFFFFFFF) } + + placeholders = node_ids.map { "?" }.join(",") + rows = db.execute("SELECT node_id, last_heard FROM nodes WHERE node_id IN (#{placeholders})", node_ids) + expect(rows.size).to eq(node_ids.size) + latest_last_heard = rows.map { |row| row["last_heard"] }.max + expect(latest_last_heard).to eq(payload.first["rx_time"]) + end + end + + it "returns 400 when the payload is not valid JSON" do + post "/api/traces", "{", auth_headers + + expect(last_response.status).to eq(400) + expect(JSON.parse(last_response.body)).to eq("error" => "invalid JSON") + end + + it "returns 400 when more than 1000 traces are provided" do + payload = Array.new(1001) { |i| { "id" => i + 1, "rx_time" => reference_time.to_i - i } } + + post "/api/traces", payload.to_json, auth_headers + + expect(last_response.status).to eq(400) + expect(JSON.parse(last_response.body)).to eq("error" => "too many traces") + + with_db(readonly: true) do |db| + count = db.get_first_value("SELECT COUNT(*) FROM traces") + expect(count).to eq(0) + end + end + end + it "returns 400 when more than 1000 messages are provided" do payload = Array.new(1001) { |i| { "packet_id" => i + 1 } } @@ -4242,6 +4350,60 @@ RSpec.describe "Potato Mesh Sinatra app" do end end + describe "GET /api/traces" do + it "returns stored traces ordered by receive time" do + clear_database + post "/api/traces", trace_fixture.to_json, auth_headers + expect(last_response).to be_ok + + get "/api/traces" + + expect(last_response).to be_ok + payload = JSON.parse(last_response.body) + expect(payload.length).to eq(trace_fixture.length) + expect(payload.map { |row| row["id"] }).to eq([trace_fixture.first["id"], trace_fixture.last["packet_id"]]) + + latest = payload.first + expect(latest["request_id"]).to eq(trace_fixture.first["request_id"]) + expect(latest["src"]).to eq(trace_fixture.first["src"]) + expect(latest["dest"]).to eq(trace_fixture.first["dest"]) + expect(latest["hops"]).to eq(trace_fixture.first["hops"]) + expect(latest["rx_iso"]).to eq(Time.at(trace_fixture.first["rx_time"]).utc.iso8601) + + earlier = payload.last + expect(earlier["request_id"]).to eq(trace_fixture.last["req"]) + expect(earlier["hops"]).to eq([0xBEADF00D, 19_088_743]) + expect(earlier["elapsed_ms"]).to eq(trace_fixture.last.dig("metrics", "latency_ms")) + end + + it "filters traces by node reference across sources" do + clear_database + post "/api/traces", trace_fixture.to_json, auth_headers + expect(last_response).to be_ok + + get "/api/traces/#{trace_fixture.first["src"]}" + + expect(last_response).to be_ok + filtered = JSON.parse(last_response.body) + expect(filtered.map { |row| row["id"] }).to include(trace_fixture.first["id"], trace_fixture.last["packet_id"]) + + get "/api/traces/!beadf00d" + + expect(last_response).to be_ok + bead_filtered = JSON.parse(last_response.body) + expect(bead_filtered.map { |row| row["id"] }).to eq([trace_fixture.last["packet_id"]]) + expect(bead_filtered.first["hops"]).to eq([0xBEADF00D, 19_088_743]) + end + + it "returns an empty list when no traces are stored" do + clear_database + get "/api/traces" + + expect(last_response).to be_ok + expect(JSON.parse(last_response.body)).to eq([]) + end + end + describe "GET /nodes/:id" do before do import_nodes_fixture diff --git a/web/spec/database_spec.rb b/web/spec/database_spec.rb index de2cf6e..6be4f77 100644 --- a/web/spec/database_spec.rb +++ b/web/spec/database_spec.rb @@ -165,4 +165,22 @@ RSpec.describe PotatoMesh::App::Database do expect(telemetry_columns).to include("soil_temperature", "lux", "iaq") expect(telemetry_columns).to include("rx_time", "battery_level") end + + it "creates trace tables when absent" do + SQLite3::Database.new(PotatoMesh::Config.db_path) do |db| + db.execute("CREATE TABLE nodes(node_id TEXT)") + db.execute("CREATE TABLE messages(id INTEGER PRIMARY KEY)") + end + + expect(column_names_for("traces")).to be_empty + expect(column_names_for("trace_hops")).to be_empty + + harness_class.ensure_schema_upgrades + + traces_columns = column_names_for("traces") + expect(traces_columns).to include("request_id", "src", "dest", "rx_time", "rx_iso", "elapsed_ms") + + hop_columns = column_names_for("trace_hops") + expect(hop_columns).to include("trace_id", "hop_index", "node_id") + end end