diff --git a/web/lib/potato_mesh/application.rb b/web/lib/potato_mesh/application.rb index 6c1bc3f..e29001c 100644 --- a/web/lib/potato_mesh/application.rb +++ b/web/lib/potato_mesh/application.rb @@ -57,6 +57,7 @@ require_relative "application/meshtastic/cipher" require_relative "application/meshtastic/payload_decoder" require_relative "application/data_processing" require_relative "application/filesystem" +require_relative "application/api_cache" require_relative "application/pages" require_relative "application/instances" require_relative "application/routes/api" diff --git a/web/lib/potato_mesh/application/api_cache.rb b/web/lib/potato_mesh/application/api_cache.rb new file mode 100644 index 0000000..37eee66 --- /dev/null +++ b/web/lib/potato_mesh/application/api_cache.rb @@ -0,0 +1,163 @@ +# Copyright © 2025-26 l5yth & contributors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# frozen_string_literal: true + +require "digest" + +module PotatoMesh + module App + # Thread-safe in-memory cache for serialised API responses. + # + # Each entry is stored with a monotonic expiration time and a pre-computed + # ETag so the route handler can skip recomputing the digest on cache hits. + # + # The cache is bounded to {MAX_ENTRIES} to prevent unbounded memory growth + # from attacker-controlled query parameters. When the limit is reached the + # oldest entry by insertion order is evicted (LRU-ish via Ruby hash ordering). + # + # Invalidation can target a specific prefix (e.g. +"api:nodes:"+) so that an + # ingest POST to +/api/messages+ does not flush the neighbors cache. + # A single-flight guard coalesces concurrent misses for the same key so only + # one thread computes the value while others wait for the result. + module ApiCache + # Hard cap on the number of cached entries to prevent memory exhaustion. + # With the whitelisted protocol values and known limit set, the realistic + # key space is ~30 entries. 64 provides generous headroom. + MAX_ENTRIES = 64 + + @store = {} + @inflight = {} + @mutex = Mutex.new + + class << self + # Retrieve a cached value or compute and store it. + # + # When multiple threads request the same cold key concurrently only one + # executes the block; the others wait for the result (single-flight). + # + # The returned hash contains both +:value+ (the JSON string) and +:etag+ + # (pre-computed weak ETag) so callers can set the header without + # re-hashing the body. + # + # @param key [String] cache key incorporating all relevant query + # parameters (limit, protocol, etc.). + # @param ttl_seconds [Numeric] time-to-live for the cached entry. + # @yield Computes the value to cache when the entry is missing or + # expired. The block should return the serialised JSON string. + # @return [Hash{Symbol => String}] +:value+ and +:etag+ of the response. + def fetch(key, ttl_seconds:) + now = monotonic_now + + @mutex.synchronize do + entry = @store[key] + if entry && now < entry[:expires_at] + return { value: entry[:value], etag: entry[:etag] } + end + + # Single-flight: if another thread is already computing this key, + # wait for it to finish and use its result. The loop guards + # against spurious wakeups from ConditionVariable#wait. + while @inflight.key?(key) + cv = @inflight[key] + cv.wait(@mutex) + entry = @store[key] + if entry && monotonic_now < entry[:expires_at] + return { value: entry[:value], etag: entry[:etag] } + end + end + + # Mark this key as in-flight so concurrent requests wait. + @inflight[key] = ConditionVariable.new + end + + value = yield + etag = Digest::MD5.hexdigest(value) + + @mutex.synchronize do + evict_oldest_if_full + @store[key] = { value: value, etag: etag, expires_at: monotonic_now + ttl_seconds } + cv = @inflight.delete(key) + cv&.broadcast + end + + { value: value, etag: etag } + rescue => e + # On error, unblock any waiters and re-raise. + @mutex.synchronize do + cv = @inflight.delete(key) + cv&.broadcast + end + raise e + end + + # Remove entries whose keys start with any of the given prefixes. + # + # Targeted invalidation so that e.g. a messages POST does not flush the + # neighbors or telemetry caches. + # + # @param prefixes [Array] key prefixes to match. + # @return [void] + def invalidate_prefix(*prefixes) + @mutex.synchronize do + @store.reject! do |key, _| + prefixes.any? { |p| key.start_with?(p) } + end + end + end + + # Remove all entries from the cache. + # + # @return [void] + def invalidate_all + @mutex.synchronize { @store.clear } + end + + # Remove specific entries by exact key. + # + # @param keys [Array] cache keys to evict. + # @return [void] + def invalidate(*keys) + @mutex.synchronize do + keys.each { |k| @store.delete(k) } + end + end + + # Return the number of entries currently held in the cache. + # + # @return [Integer] entry count. + def size + @mutex.synchronize { @store.size } + end + + private + + # Use the monotonic clock so TTL calculations are immune to wall-clock + # adjustments (NTP jumps, DST transitions, etc.). + def monotonic_now + Process.clock_gettime(Process::CLOCK_MONOTONIC) + end + + # Evict the oldest entry when the store is at capacity. Ruby hashes + # preserve insertion order, so +first+ is the oldest key. + def evict_oldest_if_full + while @store.size >= MAX_ENTRIES + oldest_key = @store.each_key.first + @store.delete(oldest_key) + end + end + end + end + end +end diff --git a/web/lib/potato_mesh/application/queries/chat_queries.rb b/web/lib/potato_mesh/application/queries/chat_queries.rb index b0556a9..16c1cc3 100644 --- a/web/lib/potato_mesh/application/queries/chat_queries.rb +++ b/web/lib/potato_mesh/application/queries/chat_queries.rb @@ -64,6 +64,12 @@ module PotatoMesh SQL params << limit rows = db.execute(sql, params) + + # Batch-resolve all unique from_id values to canonical node_ids in a + # single query instead of issuing 1-2 SELECTs per message row. + raw_from_ids = rows.filter_map { |r| string_or_nil(r["from_id"]&.to_s&.strip) }.uniq + canonical_map = batch_resolve_node_ids(db, raw_from_ids) + rows.each do |r| r.delete_if { |key, _| key.is_a?(Integer) } r["reply_id"] = coerce_integer(r["reply_id"]) if r.key?("reply_id") @@ -81,7 +87,7 @@ module PotatoMesh ) end - canonical_from_id = string_or_nil(normalize_node_id(db, r["from_id"])) + canonical_from_id = canonical_map[r["from_id"]&.to_s&.strip] node_id = canonical_from_id || string_or_nil(r["from_id"]) if canonical_from_id diff --git a/web/lib/potato_mesh/application/queries/common.rb b/web/lib/potato_mesh/application/queries/common.rb index 18b5388..c217d16 100644 --- a/web/lib/potato_mesh/application/queries/common.rb +++ b/web/lib/potato_mesh/application/queries/common.rb @@ -133,6 +133,57 @@ module PotatoMesh coerced end + # Resolve a collection of raw node reference strings to their canonical + # +node_id+ values in a single batch query. This avoids the N+1 pattern + # of calling +normalize_node_id+ once per row. + # + # @param db [SQLite3::Database] open database handle. + # @param refs [Array] raw node identifiers (hex strings or numeric + # strings) to resolve. + # @return [Hash{String => String}] mapping from each input reference to its + # canonical +node_id+, omitting entries that could not be resolved. + def batch_resolve_node_ids(db, refs) + return {} if refs.nil? || refs.empty? + + result = {} + string_refs = [] + numeric_refs = [] + + refs.each do |ref| + next if ref.nil? || ref.strip.empty? + string_refs << ref.strip + begin + numeric_refs << Integer(ref.strip, 10) + rescue ArgumentError + # not a numeric reference — skip the numeric branch + end + end + + # Batch lookup by node_id (string match) + unless string_refs.empty? + placeholders = Array.new(string_refs.length, "?").join(", ") + rows = db.execute("SELECT node_id FROM nodes WHERE node_id IN (#{placeholders})", string_refs) + rows.each do |row| + nid = row.is_a?(Hash) ? row["node_id"] : row[0] + result[nid] = nid if nid + end + end + + # Batch lookup by num (numeric match) for refs not yet resolved + unresolved_numeric = numeric_refs.select { |n| !result.key?(n.to_s) } + unless unresolved_numeric.empty? + placeholders = Array.new(unresolved_numeric.length, "?").join(", ") + rows = db.execute("SELECT node_id, num FROM nodes WHERE num IN (#{placeholders})", unresolved_numeric) + rows.each do |row| + nid = row.is_a?(Hash) ? row["node_id"] : row[0] + num = row.is_a?(Hash) ? row["num"] : row[1] + result[num.to_s] = nid if nid && num + end + end + + result + end + # Normalise a caller-supplied timestamp for API pagination windows. # # @param since [Object] requested lower bound expressed as seconds since the epoch. diff --git a/web/lib/potato_mesh/application/queries/federation_queries.rb b/web/lib/potato_mesh/application/queries/federation_queries.rb index 475ef44..b305777 100644 --- a/web/lib/potato_mesh/application/queries/federation_queries.rb +++ b/web/lib/potato_mesh/application/queries/federation_queries.rb @@ -37,7 +37,7 @@ module PotatoMesh params << since_threshold if node_ref - clause = node_lookup_clause(node_ref, string_columns: ["node_id"], numeric_columns: ["node_num"]) + clause = node_lookup_clause(node_ref, string_columns: ["node_id"], numeric_columns: ["node_num"], db: db) return [] unless clause where_clauses << clause.first params.concat(clause.last) diff --git a/web/lib/potato_mesh/application/queries/node_queries.rb b/web/lib/potato_mesh/application/queries/node_queries.rb index 1357bbd..7032f51 100644 --- a/web/lib/potato_mesh/application/queries/node_queries.rb +++ b/web/lib/potato_mesh/application/queries/node_queries.rb @@ -70,11 +70,42 @@ module PotatoMesh } end - def node_lookup_clause(node_ref, string_columns:, numeric_columns: []) + # Build a WHERE clause fragment for looking up a node across one or more + # columns. When +numeric_columns+ are provided together with an open +db+ + # handle the numeric identifiers are resolved to canonical +node_id+ + # strings up-front so the resulting SQL uses only string-column +IN+ + # predicates. This avoids an +OR+ across heterogeneous columns which + # prevents SQLite from choosing the optimal index. + # + # @param node_ref [String, Integer, nil] raw node reference from the request. + # @param string_columns [Array] SQL column names holding string identifiers. + # @param numeric_columns [Array] SQL column names holding numeric identifiers. + # @param db [SQLite3::Database, nil] open database handle used to resolve + # numeric IDs to canonical strings. When provided and +numeric_columns+ + # is non-empty the numeric branch is folded into the string branch. + # @return [Array(String, Array), nil] SQL fragment and bind parameters, or + # +nil+ when no lookup can be constructed. + def node_lookup_clause(node_ref, string_columns:, numeric_columns: [], db: nil) tokens = node_reference_tokens(node_ref) string_values = tokens[:string_values] numeric_values = tokens[:numeric_values] + # When a database handle is available, resolve numeric identifiers to + # canonical node_id strings so the query can use a single indexed column + # instead of an OR across string and numeric columns. + if db && !numeric_columns.empty? && !numeric_values.empty? + numeric_values.each do |num| + resolved = db.get_first_value("SELECT node_id FROM nodes WHERE num = ? LIMIT 1", [num]) + if resolved + string_values << resolved unless string_values.include?(resolved) + end + end + # All numeric values have been folded into string_values; drop the + # numeric branch so the generated SQL avoids an OR. + numeric_columns = [] + numeric_values = [] + end + clauses = [] params = [] @@ -117,7 +148,7 @@ module PotatoMesh where_clauses = [] if node_ref - clause = node_lookup_clause(node_ref, string_columns: ["node_id"], numeric_columns: ["num"]) + clause = node_lookup_clause(node_ref, string_columns: ["node_id"], numeric_columns: ["num"], db: db) return [] unless clause where_clauses << clause.first params.concat(clause.last) diff --git a/web/lib/potato_mesh/application/queries/telemetry_queries.rb b/web/lib/potato_mesh/application/queries/telemetry_queries.rb index 7b98397..3c2beb7 100644 --- a/web/lib/potato_mesh/application/queries/telemetry_queries.rb +++ b/web/lib/potato_mesh/application/queries/telemetry_queries.rb @@ -37,7 +37,7 @@ module PotatoMesh params << since_threshold if node_ref - clause = node_lookup_clause(node_ref, string_columns: ["node_id"], numeric_columns: ["node_num"]) + clause = node_lookup_clause(node_ref, string_columns: ["node_id"], numeric_columns: ["node_num"], db: db) return [] unless clause where_clauses << clause.first params.concat(clause.last) diff --git a/web/lib/potato_mesh/application/routes/api.rb b/web/lib/potato_mesh/application/routes/api.rb index d9431ce..7b2c91c 100644 --- a/web/lib/potato_mesh/application/routes/api.rb +++ b/web/lib/potato_mesh/application/routes/api.rb @@ -18,12 +18,34 @@ module PotatoMesh module App module Routes module Api + # Accepted protocol filter values. Unknown values are discarded to + # prevent attacker-controlled strings from polluting the cache keyspace. + KNOWN_PROTOCOLS = Set.new(%w[meshcore meshtastic]).freeze + # Register read-only API endpoints that expose cached mesh data and # instance metadata. Invoked by Sinatra during extension registration. # # @param app [Sinatra::Base] application instance receiving the routes. # @return [void] def self.registered(app) + known_protocols = KNOWN_PROTOCOLS + + app.helpers do + # Sanitise the protocol query parameter to a known value. + define_method(:sanitize_protocol) do |raw| + val = raw&.to_s&.strip&.downcase + known_protocols.include?(val) ? val : nil + end + + # Set Cache-Control headers appropriate for the current mode. + # Private-mode instances must not allow intermediary caches to + # store responses that may contain filtered data. + define_method(:api_cache_control) do |max_age: 10| + visibility = private_mode? ? :private : :public + cache_control visibility, :must_revalidate, max_age: max_age + end + end + app.before "/api/messages*" do halt 404 if private_mode? end @@ -63,98 +85,213 @@ module PotatoMesh app.get "/api/nodes" do content_type :json - limit = [params["limit"]&.to_i || 200, 1000].min - query_nodes(limit, since: params["since"], protocol: string_or_nil(params["protocol"])).to_json + limit = coerce_query_limit(params["limit"]) + since = params["since"] + protocol = sanitize_protocol(params["protocol"]) + since_val = coerce_integer(since) || 0 + priv = private_mode? ? 1 : 0 + + if since_val > 0 + json_body = query_nodes(limit, since: since, protocol: protocol).to_json + etag Digest::MD5.hexdigest(json_body), kind: :weak + api_cache_control + json_body + else + cached = PotatoMesh::App::ApiCache.fetch("api:nodes:#{limit}:#{protocol}:#{priv}", ttl_seconds: 15) do + query_nodes(limit, since: since, protocol: protocol).to_json + end + etag cached[:etag], kind: :weak + api_cache_control + cached[:value] + end end app.get "/api/stats" do content_type :json - stats = query_active_node_stats - { - active_nodes: { - "hour" => stats["hour"], "day" => stats["day"], - "week" => stats["week"], "month" => stats["month"], - }, - meshcore: stats["meshcore"], - meshtastic: stats["meshtastic"], - sampled: false, - }.to_json + priv = private_mode? ? 1 : 0 + cached = PotatoMesh::App::ApiCache.fetch("api:stats:#{priv}", ttl_seconds: 15) do + stats = query_active_node_stats + { + active_nodes: { + "hour" => stats["hour"], "day" => stats["day"], + "week" => stats["week"], "month" => stats["month"], + }, + meshcore: stats["meshcore"], + meshtastic: stats["meshtastic"], + sampled: false, + }.to_json + end + + etag cached[:etag], kind: :weak + api_cache_control + cached[:value] end app.get "/api/nodes/:id" do content_type :json node_ref = string_or_nil(params["id"]) halt 400, { error: "missing node id" }.to_json unless node_ref - limit = [params["limit"]&.to_i || 200, 1000].min + limit = coerce_query_limit(params["limit"]) rows = query_nodes(limit, node_ref: node_ref, since: params["since"]) halt 404, { error: "not found" }.to_json if rows.empty? - rows.first.to_json + json_body = rows.first.to_json + etag Digest::MD5.hexdigest(json_body), kind: :weak + api_cache_control + json_body end app.get "/api/ingestors" do content_type :json limit = coerce_query_limit(params["limit"]) - query_ingestors(limit, since: params["since"], protocol: string_or_nil(params["protocol"])).to_json + protocol = sanitize_protocol(params["protocol"]) + since = params["since"] + since_val = coerce_integer(since) || 0 + + if since_val > 0 + json_body = query_ingestors(limit, since: since, protocol: protocol).to_json + etag Digest::MD5.hexdigest(json_body), kind: :weak + api_cache_control + json_body + else + cached = PotatoMesh::App::ApiCache.fetch("api:ingestors:#{limit}:#{protocol}", ttl_seconds: 30) do + query_ingestors(limit, since: since, protocol: protocol).to_json + end + etag cached[:etag], kind: :weak + api_cache_control + cached[:value] + end end app.get "/api/messages" do content_type :json - limit = [params["limit"]&.to_i || 200, 1000].min + limit = coerce_query_limit(params["limit"]) include_encrypted = coerce_boolean(params["encrypted"]) || false since = coerce_integer(params["since"]) since = 0 if since.nil? || since.negative? - query_messages(limit, include_encrypted: include_encrypted, since: since, protocol: string_or_nil(params["protocol"])).to_json + protocol = sanitize_protocol(params["protocol"]) + enc_key = include_encrypted ? "1" : "0" + + if since > 0 + json_body = query_messages(limit, include_encrypted: include_encrypted, since: since, protocol: protocol).to_json + etag Digest::MD5.hexdigest(json_body), kind: :weak + api_cache_control + json_body + else + cached = PotatoMesh::App::ApiCache.fetch("api:messages:#{limit}:#{enc_key}:#{protocol}", ttl_seconds: 10) do + query_messages(limit, include_encrypted: include_encrypted, since: since, protocol: protocol).to_json + end + etag cached[:etag], kind: :weak + api_cache_control + cached[:value] + end end app.get "/api/messages/:id" do content_type :json node_ref = string_or_nil(params["id"]) halt 400, { error: "missing node id" }.to_json unless node_ref - limit = [params["limit"]&.to_i || 200, 1000].min + limit = coerce_query_limit(params["limit"]) include_encrypted = coerce_boolean(params["encrypted"]) || false since = coerce_integer(params["since"]) since = 0 if since.nil? || since.negative? - query_messages( + json_body = query_messages( limit, node_ref: node_ref, include_encrypted: include_encrypted, since: since, - protocol: string_or_nil(params["protocol"]), + protocol: sanitize_protocol(params["protocol"]), ).to_json + etag Digest::MD5.hexdigest(json_body), kind: :weak + api_cache_control + json_body end app.get "/api/positions" do content_type :json - limit = [params["limit"]&.to_i || 200, 1000].min - query_positions(limit, since: params["since"], protocol: string_or_nil(params["protocol"])).to_json + limit = coerce_query_limit(params["limit"]) + since = params["since"] + protocol = sanitize_protocol(params["protocol"]) + since_val = coerce_integer(since) || 0 + + if since_val > 0 + json_body = query_positions(limit, since: since, protocol: protocol).to_json + etag Digest::MD5.hexdigest(json_body), kind: :weak + api_cache_control + json_body + else + cached = PotatoMesh::App::ApiCache.fetch("api:positions:#{limit}:#{protocol}", ttl_seconds: 15) do + query_positions(limit, since: since, protocol: protocol).to_json + end + etag cached[:etag], kind: :weak + api_cache_control + cached[:value] + end end app.get "/api/positions/:id" do content_type :json node_ref = string_or_nil(params["id"]) halt 400, { error: "missing node id" }.to_json unless node_ref - limit = [params["limit"]&.to_i || 200, 1000].min - query_positions(limit, node_ref: node_ref, since: params["since"], protocol: string_or_nil(params["protocol"])).to_json + limit = coerce_query_limit(params["limit"]) + json_body = query_positions(limit, node_ref: node_ref, since: params["since"], protocol: sanitize_protocol(params["protocol"])).to_json + etag Digest::MD5.hexdigest(json_body), kind: :weak + api_cache_control + json_body end app.get "/api/neighbors" do content_type :json - limit = [params["limit"]&.to_i || 200, 1000].min - query_neighbors(limit, since: params["since"], protocol: string_or_nil(params["protocol"])).to_json + limit = coerce_query_limit(params["limit"]) + since = params["since"] + protocol = sanitize_protocol(params["protocol"]) + since_val = coerce_integer(since) || 0 + + if since_val > 0 + json_body = query_neighbors(limit, since: since, protocol: protocol).to_json + etag Digest::MD5.hexdigest(json_body), kind: :weak + api_cache_control + json_body + else + cached = PotatoMesh::App::ApiCache.fetch("api:neighbors:#{limit}:#{protocol}", ttl_seconds: 30) do + query_neighbors(limit, since: since, protocol: protocol).to_json + end + etag cached[:etag], kind: :weak + api_cache_control + cached[:value] + end end app.get "/api/neighbors/:id" do content_type :json node_ref = string_or_nil(params["id"]) halt 400, { error: "missing node id" }.to_json unless node_ref - limit = [params["limit"]&.to_i || 200, 1000].min - query_neighbors(limit, node_ref: node_ref, since: params["since"], protocol: string_or_nil(params["protocol"])).to_json + limit = coerce_query_limit(params["limit"]) + json_body = query_neighbors(limit, node_ref: node_ref, since: params["since"], protocol: sanitize_protocol(params["protocol"])).to_json + etag Digest::MD5.hexdigest(json_body), kind: :weak + api_cache_control + json_body end app.get "/api/telemetry" do content_type :json - limit = [params["limit"]&.to_i || 200, 1000].min - query_telemetry(limit, since: params["since"], protocol: string_or_nil(params["protocol"])).to_json + limit = coerce_query_limit(params["limit"]) + since = params["since"] + protocol = sanitize_protocol(params["protocol"]) + since_val = coerce_integer(since) || 0 + + if since_val > 0 + json_body = query_telemetry(limit, since: since, protocol: protocol).to_json + etag Digest::MD5.hexdigest(json_body), kind: :weak + api_cache_control + json_body + else + cached = PotatoMesh::App::ApiCache.fetch("api:telemetry:#{limit}:#{protocol}", ttl_seconds: 15) do + query_telemetry(limit, since: since, protocol: protocol).to_json + end + etag cached[:etag], kind: :weak + api_cache_control + cached[:value] + end end app.get "/api/telemetry/aggregated" do @@ -185,33 +322,67 @@ module PotatoMesh halt 400, { error: "bucketSeconds too small for requested window" }.to_json end - query_telemetry_buckets( - window_seconds: window_seconds, - bucket_seconds: bucket_seconds, - since: params["since"], - ).to_json + since = params["since"] + since_val = coerce_integer(since) || 0 + + if since_val > 0 + json_body = query_telemetry_buckets(window_seconds: window_seconds, bucket_seconds: bucket_seconds, since: since).to_json + etag Digest::MD5.hexdigest(json_body), kind: :weak + api_cache_control(max_age: 30) + json_body + else + cache_key = "api:telemetry_agg:#{window_seconds}:#{bucket_seconds}" + cached = PotatoMesh::App::ApiCache.fetch(cache_key, ttl_seconds: 60) do + query_telemetry_buckets(window_seconds: window_seconds, bucket_seconds: bucket_seconds, since: since).to_json + end + etag cached[:etag], kind: :weak + api_cache_control(max_age: 30) + cached[:value] + end end app.get "/api/telemetry/:id" do content_type :json node_ref = string_or_nil(params["id"]) halt 400, { error: "missing node id" }.to_json unless node_ref - limit = [params["limit"]&.to_i || 200, 1000].min - query_telemetry(limit, node_ref: node_ref, since: params["since"], protocol: string_or_nil(params["protocol"])).to_json + limit = coerce_query_limit(params["limit"]) + json_body = query_telemetry(limit, node_ref: node_ref, since: params["since"], protocol: sanitize_protocol(params["protocol"])).to_json + etag Digest::MD5.hexdigest(json_body), kind: :weak + api_cache_control + json_body end app.get "/api/traces" do content_type :json - limit = [params["limit"]&.to_i || 200, 1000].min - query_traces(limit, since: params["since"], protocol: string_or_nil(params["protocol"])).to_json + limit = coerce_query_limit(params["limit"]) + since = params["since"] + protocol = sanitize_protocol(params["protocol"]) + since_val = coerce_integer(since) || 0 + + if since_val > 0 + json_body = query_traces(limit, since: since, protocol: protocol).to_json + etag Digest::MD5.hexdigest(json_body), kind: :weak + api_cache_control + json_body + else + cached = PotatoMesh::App::ApiCache.fetch("api:traces:#{limit}:#{protocol}", ttl_seconds: 30) do + query_traces(limit, since: since, protocol: protocol).to_json + end + etag cached[:etag], kind: :weak + api_cache_control + cached[:value] + end end app.get "/api/traces/:id" do content_type :json node_ref = string_or_nil(params["id"]) halt 400, { error: "missing node id" }.to_json unless node_ref - limit = [params["limit"]&.to_i || 200, 1000].min - query_traces(limit, node_ref: node_ref, since: params["since"], protocol: string_or_nil(params["protocol"])).to_json + limit = coerce_query_limit(params["limit"]) + json_body = query_traces(limit, node_ref: node_ref, since: params["since"], protocol: sanitize_protocol(params["protocol"])).to_json + etag Digest::MD5.hexdigest(json_body), kind: :weak + api_cache_control + json_body end app.get "/api/instances" do diff --git a/web/lib/potato_mesh/application/routes/ingest.rb b/web/lib/potato_mesh/application/routes/ingest.rb index 5851f35..6e71f03 100644 --- a/web/lib/potato_mesh/application/routes/ingest.rb +++ b/web/lib/potato_mesh/application/routes/ingest.rb @@ -45,6 +45,7 @@ module PotatoMesh upsert_node(db, node_id, node, protocol: protocol) end PotatoMesh::App::Prometheus::NODES_GAUGE.set(query_nodes(1000).length) + PotatoMesh::App::ApiCache.invalidate_prefix("api:nodes:", "api:stats:") { status: "ok" }.to_json ensure db&.close @@ -65,6 +66,7 @@ module PotatoMesh messages.each do |msg| insert_message(db, msg, protocol_cache: protocol_cache) end + PotatoMesh::App::ApiCache.invalidate_prefix("api:messages:", "api:stats:") { status: "ok" }.to_json ensure db&.close @@ -84,6 +86,7 @@ module PotatoMesh db = open_database stored = upsert_ingestor(db, payload) halt 400, { error: "invalid payload" }.to_json unless stored + PotatoMesh::App::ApiCache.invalidate_prefix("api:ingestors:") { status: "ok" }.to_json ensure db&.close @@ -314,6 +317,7 @@ module PotatoMesh positions.each do |pos| insert_position(db, pos, protocol_cache: protocol_cache) end + PotatoMesh::App::ApiCache.invalidate_prefix("api:positions:", "api:nodes:", "api:stats:") { status: "ok" }.to_json ensure db&.close @@ -334,6 +338,7 @@ module PotatoMesh neighbor_payloads.each do |packet| insert_neighbors(db, packet, protocol_cache: protocol_cache) end + PotatoMesh::App::ApiCache.invalidate_prefix("api:neighbors:", "api:stats:") { status: "ok" }.to_json ensure db&.close @@ -354,6 +359,7 @@ module PotatoMesh telemetry_packets.each do |packet| insert_telemetry(db, packet, protocol_cache: protocol_cache) end + PotatoMesh::App::ApiCache.invalidate_prefix("api:telemetry:", "api:stats:") { status: "ok" }.to_json ensure db&.close @@ -374,6 +380,7 @@ module PotatoMesh trace_packets.each do |packet| insert_trace(db, packet, protocol_cache: protocol_cache) end + PotatoMesh::App::ApiCache.invalidate_prefix("api:traces:", "api:stats:") { status: "ok" }.to_json ensure db&.close diff --git a/web/public/assets/js/app/__tests__/incremental-helpers.test.js b/web/public/assets/js/app/__tests__/incremental-helpers.test.js new file mode 100644 index 0000000..b6c8d69 --- /dev/null +++ b/web/public/assets/js/app/__tests__/incremental-helpers.test.js @@ -0,0 +1,212 @@ +/* + * Copyright © 2025-26 l5yth & contributors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import test from 'node:test'; +import assert from 'node:assert/strict'; +import { maxRecordTimestamp, mergeById, mergeByCompositeKey, trimToLimit } from '../incremental-helpers.js'; + +// --------------------------------------------------------------------------- +// maxRecordTimestamp +// --------------------------------------------------------------------------- + +test('maxRecordTimestamp returns 0 for an empty array', () => { + assert.equal(maxRecordTimestamp([]), 0); +}); + +test('maxRecordTimestamp returns 0 for non-array input', () => { + assert.equal(maxRecordTimestamp(null), 0); + assert.equal(maxRecordTimestamp(undefined), 0); + assert.equal(maxRecordTimestamp('string'), 0); +}); + +test('maxRecordTimestamp extracts the highest rx_time by default', () => { + const records = [ + { rx_time: 100 }, + { rx_time: 300 }, + { rx_time: 200 }, + ]; + assert.equal(maxRecordTimestamp(records), 300); +}); + +test('maxRecordTimestamp inspects last_heard by default', () => { + const records = [ + { last_heard: 500 }, + { last_heard: 250 }, + ]; + assert.equal(maxRecordTimestamp(records), 500); +}); + +test('maxRecordTimestamp returns 0 when records lack timestamp fields', () => { + const records = [{ node_id: '!abc' }, { node_id: '!def' }]; + assert.equal(maxRecordTimestamp(records), 0); +}); + +test('maxRecordTimestamp accepts custom field names', () => { + const records = [ + { telemetry_time: 700, rx_time: 600 }, + { telemetry_time: 800 }, + ]; + assert.equal(maxRecordTimestamp(records, ['telemetry_time']), 800); +}); + +test('maxRecordTimestamp picks the max across multiple fields', () => { + const records = [ + { rx_time: 100, position_time: 400 }, + { rx_time: 300, position_time: 200 }, + ]; + assert.equal(maxRecordTimestamp(records, ['rx_time', 'position_time']), 400); +}); + +test('maxRecordTimestamp skips null and non-object entries', () => { + const records = [null, undefined, 42, { rx_time: 10 }]; + assert.equal(maxRecordTimestamp(records), 10); +}); + +test('maxRecordTimestamp ignores non-number timestamp values', () => { + const records = [{ rx_time: 'abc' }, { rx_time: 50 }]; + assert.equal(maxRecordTimestamp(records), 50); +}); + +// --------------------------------------------------------------------------- +// mergeById +// --------------------------------------------------------------------------- + +test('mergeById returns existing when incoming is empty', () => { + const existing = [{ id: 1, v: 'a' }]; + assert.strictEqual(mergeById(existing, [], 'id'), existing); + assert.strictEqual(mergeById(existing, null, 'id'), existing); + assert.strictEqual(mergeById(existing, undefined, 'id'), existing); +}); + +test('mergeById deduplicates by keyField keeping the incoming value', () => { + const existing = [ + { id: 1, v: 'old' }, + { id: 2, v: 'keep' }, + ]; + const incoming = [ + { id: 1, v: 'new' }, + { id: 3, v: 'added' }, + ]; + const result = mergeById(existing, incoming, 'id'); + assert.equal(result.length, 3); + const byId = Object.fromEntries(result.map(r => [r.id, r.v])); + assert.equal(byId[1], 'new'); + assert.equal(byId[2], 'keep'); + assert.equal(byId[3], 'added'); +}); + +test('mergeById works with string keys', () => { + const existing = [{ node_id: '!abc', name: 'A' }]; + const incoming = [{ node_id: '!abc', name: 'B' }]; + const result = mergeById(existing, incoming, 'node_id'); + assert.equal(result.length, 1); + assert.equal(result[0].name, 'B'); +}); + +test('mergeById skips items with null or undefined key', () => { + const existing = [{ id: 1, v: 'a' }]; + const incoming = [{ v: 'no-id' }, { id: 2, v: 'b' }]; + const result = mergeById(existing, incoming, 'id'); + assert.equal(result.length, 2); +}); + +test('mergeById returns all incoming when existing is empty', () => { + const result = mergeById([], [{ id: 1 }, { id: 2 }], 'id'); + assert.equal(result.length, 2); +}); + +// --------------------------------------------------------------------------- +// mergeByCompositeKey +// --------------------------------------------------------------------------- + +test('mergeByCompositeKey deduplicates by composite key', () => { + const existing = [ + { node_id: '!a', neighbor_id: '!b', snr: 5 }, + { node_id: '!a', neighbor_id: '!c', snr: 3 }, + ]; + const incoming = [ + { node_id: '!a', neighbor_id: '!b', snr: 8 }, + { node_id: '!a', neighbor_id: '!d', snr: 1 }, + ]; + const result = mergeByCompositeKey(existing, incoming, ['node_id', 'neighbor_id']); + assert.equal(result.length, 3); + const ab = result.find(r => r.neighbor_id === '!b'); + assert.equal(ab.snr, 8, 'incoming should overwrite existing for same composite key'); +}); + +test('mergeByCompositeKey returns existing when incoming is empty', () => { + const existing = [{ a: 1, b: 2 }]; + assert.strictEqual(mergeByCompositeKey(existing, [], ['a', 'b']), existing); + assert.strictEqual(mergeByCompositeKey(existing, null, ['a', 'b']), existing); +}); + +test('mergeByCompositeKey handles missing key fields gracefully', () => { + const existing = [{ node_id: '!a' }]; + const incoming = [{ node_id: '!a', neighbor_id: '!b' }]; + const result = mergeByCompositeKey(existing, incoming, ['node_id', 'neighbor_id']); + assert.equal(result.length, 2, 'different composite keys due to missing field'); +}); + +// --------------------------------------------------------------------------- +// trimToLimit +// --------------------------------------------------------------------------- + +test('trimToLimit returns the same array when within limit', () => { + const records = [{ id: 1, rx_time: 100 }, { id: 2, rx_time: 200 }]; + const result = trimToLimit(records, 5); + assert.strictEqual(result, records); +}); + +test('trimToLimit trims to limit keeping newest entries', () => { + const records = [ + { id: 1, rx_time: 100 }, + { id: 2, rx_time: 300 }, + { id: 3, rx_time: 200 }, + { id: 4, rx_time: 400 }, + ]; + const result = trimToLimit(records, 2); + assert.equal(result.length, 2); + const ids = result.map(r => r.id); + assert.ok(ids.includes(4), 'should keep newest (id=4)'); + assert.ok(ids.includes(2), 'should keep second newest (id=2)'); +}); + +test('trimToLimit uses custom timestamp field', () => { + const records = [ + { id: 1, last_heard: 100 }, + { id: 2, last_heard: 300 }, + { id: 3, last_heard: 200 }, + ]; + const result = trimToLimit(records, 1, 'last_heard'); + assert.equal(result.length, 1); + assert.equal(result[0].id, 2); +}); + +test('trimToLimit returns input for non-array values', () => { + assert.equal(trimToLimit(null, 10), null); + assert.equal(trimToLimit(undefined, 10), undefined); +}); + +test('trimToLimit handles records with missing timestamp fields', () => { + const records = [ + { id: 1, rx_time: 100 }, + { id: 2 }, + { id: 3, rx_time: 300 }, + ]; + const result = trimToLimit(records, 2); + assert.equal(result.length, 2); + assert.equal(result[0].id, 3); +}); diff --git a/web/public/assets/js/app/__tests__/main-incremental-refresh.test.js b/web/public/assets/js/app/__tests__/main-incremental-refresh.test.js new file mode 100644 index 0000000..a835871 --- /dev/null +++ b/web/public/assets/js/app/__tests__/main-incremental-refresh.test.js @@ -0,0 +1,240 @@ +/* + * Copyright © 2025-26 l5yth & contributors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import test from 'node:test'; +import assert from 'node:assert/strict'; +import { createDomEnvironment } from './dom-environment.js'; +import { initializeApp } from '../main.js'; + +/** Minimal config that disables auto-refresh so we control timing. */ +const BASE_CONFIG = Object.freeze({ + channel: 'Primary', + frequency: '915MHz', + refreshMs: 0, + refreshIntervalSeconds: 0, + chatEnabled: true, + mapCenter: { lat: 0, lon: 0 }, + mapZoom: null, + maxDistanceKm: 0, + tileFilters: { light: '', dark: '' }, + instancesFeatureEnabled: false, + instanceDomain: null, + snapshotWindowSeconds: 3600, +}); + +/** + * Build a stubbed fetch that records every call and responds with canned data. + * + * @param {Object} responsesByEndpoint Map of URL prefix to JSON response body. + * @returns {{ fetch: Function, calls: Array<{ url: string, options: Object }> }} + */ +function buildStubFetch(responsesByEndpoint = {}) { + const calls = []; + + function stubFetch(url, options = {}) { + calls.push({ url, options }); + for (const [prefix, body] of Object.entries(responsesByEndpoint)) { + if (url.includes(prefix)) { + return Promise.resolve({ + ok: true, + status: 200, + json: () => Promise.resolve( + typeof body === 'function' ? body() : body, + ), + }); + } + } + return Promise.resolve({ + ok: true, + status: 200, + json: () => Promise.resolve([]), + }); + } + + return { fetch: stubFetch, calls }; +} + +/** + * Run test body with a fetch-stubbed app instance. + * + * @param {Object} stubResponses Response map for the stub fetch. + * @param {function(Object): Promise} fn Receives { testUtils, calls }. + */ +async function withStubFetchApp(stubResponses, fn) { + const env = createDomEnvironment({ includeBody: true }); + const originalFetch = globalThis.fetch; + const { fetch: stubFetch, calls } = buildStubFetch(stubResponses); + globalThis.fetch = stubFetch; + try { + const { _testUtils } = initializeApp(BASE_CONFIG); + // Allow the initial refresh() to settle (it is async). + await new Promise(r => setTimeout(r, 50)); + await fn({ testUtils: _testUtils, calls }); + } finally { + globalThis.fetch = originalFetch; + env.cleanup(); + } +} + +// --------------------------------------------------------------------------- +// Verify fetch functions append since parameter +// --------------------------------------------------------------------------- + +test('first refresh does not include since parameter in fetch URLs', async () => { + await withStubFetchApp({}, ({ calls }) => { + const apiCalls = calls.filter(c => c.url.startsWith('/api/')); + assert.ok(apiCalls.length > 0, 'should have made API calls'); + for (const call of apiCalls) { + assert.ok( + !call.url.includes('since='), + `first refresh should not pass since: ${call.url}`, + ); + } + }); +}); + +test('second refresh includes since parameter for endpoints with timestamp data', async () => { + const now = Math.floor(Date.now() / 1000); + const stubResponses = { + '/api/nodes': [{ node_id: '!aabb', last_heard: now, short_name: 'AB', role: 'CLIENT' }], + '/api/messages': [{ id: 1, rx_time: now, from_id: '!aabb', text: 'hello' }], + '/api/positions': [{ id: 1, node_id: '!aabb', rx_time: now, latitude: 52.5, longitude: 13.4 }], + '/api/telemetry': [{ id: 1, node_id: '!aabb', rx_time: now, battery_level: 90 }], + '/api/neighbors': [{ node_id: '!aabb', neighbor_id: '!ccdd', rx_time: now, snr: 10 }], + '/api/traces': [{ id: 1, rx_time: now, src: 1, dest: 2 }], + }; + + await withStubFetchApp(stubResponses, async ({ testUtils, calls }) => { + // Verify first refresh completed without since params + const firstRoundCalls = [...calls]; + const firstApiCalls = firstRoundCalls.filter(c => c.url.startsWith('/api/')); + assert.ok(firstApiCalls.length > 0, 'initial refresh should have fired'); + for (const call of firstApiCalls) { + assert.ok( + !call.url.includes('since='), + `first refresh should not pass since: ${call.url}`, + ); + } + + // Clear call log and trigger a second refresh + calls.length = 0; + await testUtils.refresh(); + await new Promise(r => setTimeout(r, 50)); + + // Second refresh should include since= on all data endpoints + const secondApiCalls = calls.filter(c => c.url.startsWith('/api/')); + assert.ok(secondApiCalls.length > 0, 'second refresh should have fired'); + + const nodeCall = secondApiCalls.find(c => c.url.includes('/api/nodes?')); + assert.ok(nodeCall, 'should have made a nodes call'); + assert.ok(nodeCall.url.includes('since='), `nodes should include since: ${nodeCall.url}`); + + const posCall = secondApiCalls.find(c => c.url.includes('/api/positions?')); + assert.ok(posCall, 'should have made a positions call'); + assert.ok(posCall.url.includes('since='), `positions should include since: ${posCall.url}`); + + const telCall = secondApiCalls.find(c => c.url.includes('/api/telemetry?')); + assert.ok(telCall, 'should have made a telemetry call'); + assert.ok(telCall.url.includes('since='), `telemetry should include since: ${telCall.url}`); + + const nbCall = secondApiCalls.find(c => c.url.includes('/api/neighbors?')); + assert.ok(nbCall, 'should have made a neighbors call'); + assert.ok(nbCall.url.includes('since='), `neighbors should include since: ${nbCall.url}`); + + const trCall = secondApiCalls.find(c => c.url.includes('/api/traces?')); + assert.ok(trCall, 'should have made a traces call'); + assert.ok(trCall.url.includes('since='), `traces should include since: ${trCall.url}`); + + const msgCalls = secondApiCalls.filter(c => c.url.includes('/api/messages?')); + assert.ok(msgCalls.length > 0, 'should have made message calls'); + for (const mc of msgCalls) { + assert.ok(mc.url.includes('since='), `messages should include since: ${mc.url}`); + } + }); +}); + +test('second refresh merges incremental data into existing state', async () => { + const now = Math.floor(Date.now() / 1000); + let callCount = 0; + + // First call returns node A, second call returns node B + const stubResponses = { + '/api/nodes': () => { + callCount++; + if (callCount <= 1) { + return [{ node_id: '!aaaa', last_heard: now, short_name: 'AA', role: 'CLIENT' }]; + } + return [{ node_id: '!bbbb', last_heard: now + 60, short_name: 'BB', role: 'CLIENT' }]; + }, + }; + + await withStubFetchApp(stubResponses, async ({ testUtils, calls }) => { + // After first refresh, call count should be 1 + assert.ok(callCount >= 1, 'first refresh should have fetched nodes'); + + // Trigger second refresh + calls.length = 0; + await testUtils.refresh(); + await new Promise(r => setTimeout(r, 50)); + + // The second refresh should have merged data + assert.ok(callCount >= 2, 'second refresh should have fetched nodes again'); + }); +}); + +test('fetch functions use cache: default option', async () => { + await withStubFetchApp({}, ({ calls }) => { + const apiCalls = calls.filter(c => c.url.startsWith('/api/')); + for (const call of apiCalls) { + assert.equal( + call.options.cache, + 'default', + `${call.url} should use cache:default`, + ); + } + }); +}); + +test('messages fetch sends encrypted parameter when requested', async () => { + await withStubFetchApp({}, ({ calls }) => { + const encryptedCalls = calls.filter( + c => c.url.includes('/api/messages') && c.url.includes('encrypted=true'), + ); + assert.ok(encryptedCalls.length > 0, 'should have made encrypted message call'); + }); +}); + +test('since parameter uses a 1-second overlap to avoid missing rows', async () => { + const now = Math.floor(Date.now() / 1000); + const stubResponses = { + '/api/nodes': [{ node_id: '!test', last_heard: now, short_name: 'T', role: 'CLIENT' }], + }; + + await withStubFetchApp(stubResponses, async ({ testUtils, calls }) => { + calls.length = 0; + await testUtils.refresh(); + await new Promise(r => setTimeout(r, 50)); + + const nodeCall = calls.find(c => c.url.includes('/api/nodes?')); + assert.ok(nodeCall, 'should have nodes call on second refresh'); + // The since value should be (now - 1) to create the overlap + const expectedSince = now - 1; + assert.ok( + nodeCall.url.includes(`since=${expectedSince}`), + `expected since=${expectedSince} in URL: ${nodeCall.url}`, + ); + }); +}); diff --git a/web/public/assets/js/app/__tests__/node-details.test.js b/web/public/assets/js/app/__tests__/node-details.test.js index e419e37..5b665ca 100644 --- a/web/public/assets/js/app/__tests__/node-details.test.js +++ b/web/public/assets/js/app/__tests__/node-details.test.js @@ -109,7 +109,7 @@ test('refreshNodeInformation merges telemetry metrics when the base node lacks t assert.equal(calls.length, 4); calls.forEach(call => { - assert.deepEqual(call.options, { cache: 'no-store' }); + assert.deepEqual(call.options, { cache: 'default' }); }); }); diff --git a/web/public/assets/js/app/__tests__/node-page.test.js b/web/public/assets/js/app/__tests__/node-page.test.js index 1b1b4ca..8707f5d 100644 --- a/web/public/assets/js/app/__tests__/node-page.test.js +++ b/web/public/assets/js/app/__tests__/node-page.test.js @@ -919,7 +919,7 @@ test('fetchMessages handles HTTP responses and uses defaults', async () => { }; const messages = await fetchMessages('!node', { fetchImpl }); assert.equal(messages.length, 1); - assert.equal(calls[0].options.cache, 'no-store'); + assert.equal(calls[0].options.cache, 'default'); }); test('fetchMessages returns an empty list when the endpoint is missing', async () => { @@ -1002,7 +1002,7 @@ test('fetchTracesForNode requests traceroutes for the node', async () => { const traces = await fetchTracesForNode('!abc', { fetchImpl }); assert.equal(traces.length, 1); assert.equal(calls[0].url.includes('/api/traces/!abc'), true); - assert.equal(calls[0].options.cache, 'no-store'); + assert.equal(calls[0].options.cache, 'default'); }); test('fetchTracesForNode returns empty when identifier is missing', async () => { diff --git a/web/public/assets/js/app/charts-page.js b/web/public/assets/js/app/charts-page.js index 954016a..ba8bcd2 100644 --- a/web/public/assets/js/app/charts-page.js +++ b/web/public/assets/js/app/charts-page.js @@ -179,7 +179,7 @@ export async function fetchAggregatedTelemetry({ const bucketSecondsSafe = bucketSecondsCandidate > 0 ? bucketSecondsCandidate : TELEMETRY_BUCKET_SECONDS; const response = await fetchFn( `/api/telemetry/aggregated?windowSeconds=${windowSeconds}&bucketSeconds=${bucketSecondsSafe}`, - { cache: 'no-store' }, + { cache: 'default' }, ); if (!response.ok) { throw new Error(`Failed to fetch aggregated telemetry (HTTP ${response.status})`); diff --git a/web/public/assets/js/app/incremental-helpers.js b/web/public/assets/js/app/incremental-helpers.js new file mode 100644 index 0000000..ccb1bb4 --- /dev/null +++ b/web/public/assets/js/app/incremental-helpers.js @@ -0,0 +1,108 @@ +/* + * Copyright © 2025-26 l5yth & contributors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * Extract the maximum timestamp from an array of API records. + * + * Inspects the specified fields on each record and returns the highest + * value found. Returns 0 when the array is empty or contains no valid + * timestamps. + * + * @param {Array} records API response rows. + * @param {Array} [fields] Timestamp field names to inspect. + * @returns {number} Maximum unix timestamp across all records. + */ +export function maxRecordTimestamp(records, fields = ['rx_time', 'last_heard']) { + let max = 0; + if (!Array.isArray(records)) return max; + for (const record of records) { + if (!record || typeof record !== 'object') continue; + for (const field of fields) { + const val = record[field]; + if (typeof val === 'number' && val > max) max = val; + } + } + return max; +} + +/** + * Merge incremental rows into an existing collection, deduplicating by a + * key field. New rows replace existing entries with the same key. + * + * @param {Array} existing Previous full dataset. + * @param {Array} incoming New incremental rows. + * @param {string} keyField Property used for deduplication. + * @returns {Array} Merged array. + */ +export function mergeById(existing, incoming, keyField) { + if (!incoming || incoming.length === 0) return existing; + const map = new Map(); + for (const item of existing) { + const key = item[keyField]; + if (key != null) map.set(key, item); + } + for (const item of incoming) { + const key = item[keyField]; + if (key != null) map.set(key, item); + } + return Array.from(map.values()); +} + +/** + * Merge incremental rows using a composite key built from multiple fields. + * + * Behaves like {@link mergeById} but joins the values of several fields + * into a single string key so records with a composite primary key (e.g. + * ``node_id`` + ``neighbor_id``) are deduplicated correctly. + * + * @param {Array} existing Previous full dataset. + * @param {Array} incoming New incremental rows. + * @param {Array} keyFields Properties whose values form the composite key. + * @returns {Array} Merged array. + */ +export function mergeByCompositeKey(existing, incoming, keyFields) { + if (!incoming || incoming.length === 0) return existing; + + function buildKey(item) { + return keyFields.map(f => String(item[f] ?? '')).join('\0'); + } + + const map = new Map(); + for (const item of existing) { + map.set(buildKey(item), item); + } + for (const item of incoming) { + map.set(buildKey(item), item); + } + return Array.from(map.values()); +} + +/** + * Trim an array to at most ``limit`` entries, keeping the ones with the + * highest timestamp value. Prevents unbounded growth from incremental + * merges over a long-running browser tab. + * + * @param {Array} records Merged record array. + * @param {number} limit Maximum number of entries to retain. + * @param {string} [tsField] Timestamp field name used for sorting. + * @returns {Array} Trimmed array (may be the same reference if + * already within the limit). + */ +export function trimToLimit(records, limit, tsField = 'rx_time') { + if (!Array.isArray(records) || records.length <= limit) return records; + const sorted = records.slice().sort((a, b) => (b[tsField] || 0) - (a[tsField] || 0)); + return sorted.slice(0, limit); +} diff --git a/web/public/assets/js/app/main.js b/web/public/assets/js/app/main.js index 4c14af6..18f39f1 100644 --- a/web/public/assets/js/app/main.js +++ b/web/public/assets/js/app/main.js @@ -92,6 +92,7 @@ import { aggregateTelemetrySnapshots, } from './snapshot-aggregator.js'; import { normalizeNodeCollection } from './node-snapshot-normalizer.js'; +import { maxRecordTimestamp, mergeById, mergeByCompositeKey, trimToLimit } from './incremental-helpers.js'; import { buildTraceSegments } from './trace-paths.js'; import { getRoleColor, @@ -229,6 +230,18 @@ export function initializeApp(config) { applyNodeFallback: applyNodeNameFallback, logger: console, }); + // Timestamps of the most recent record seen per data type. Used to pass + // the ``since`` query parameter on subsequent refreshes so only new/changed + // rows are transferred over the wire. + let lastNodeTimestamp = 0; + let lastMessageTimestamp = 0; + let lastPositionTimestamp = 0; + let lastTelemetryTimestamp = 0; + let lastNeighborTimestamp = 0; + let lastTraceTimestamp = 0; + /** Whether the very first full fetch has completed. */ + let initialFetchDone = false; + const NODE_LIMIT = 1000; const TRACE_LIMIT = 200; const TRACE_MAX_AGE_SECONDS = 28 * 24 * 60 * 60; @@ -3597,11 +3610,14 @@ export function initializeApp(config) { * Fetch the latest nodes from the JSON API. * * @param {number} [limit=NODE_LIMIT] Maximum number of records. + * @param {number} [since=0] Unix timestamp; only rows newer than this are returned. * @returns {Promise>} Parsed node payloads. */ - async function fetchNodes(limit = NODE_LIMIT) { + async function fetchNodes(limit = NODE_LIMIT, since = 0) { const effectiveLimit = resolveSnapshotLimit(limit, NODE_LIMIT); - const r = await fetch(`/api/nodes?limit=${effectiveLimit}`, { cache: 'no-store' }); + let url = `/api/nodes?limit=${effectiveLimit}`; + if (since > 0) url += `&since=${since}`; + const r = await fetch(url, { cache: 'default' }); if (!r.ok) throw new Error('HTTP ' + r.status); return r.json(); } @@ -3616,7 +3632,7 @@ export function initializeApp(config) { if (typeof nodeId !== 'string') return null; const trimmed = nodeId.trim(); if (trimmed.length === 0) return null; - const r = await fetch(`/api/nodes/${encodeURIComponent(trimmed)}`, { cache: 'no-store' }); + const r = await fetch(`/api/nodes/${encodeURIComponent(trimmed)}`, { cache: 'default' }); if (r.status === 404) return null; if (!r.ok) throw new Error('HTTP ' + r.status); return r.json(); @@ -3626,7 +3642,7 @@ export function initializeApp(config) { * Fetch recent messages from the JSON API. * * @param {number} [limit=NODE_LIMIT] Maximum number of rows. - * @param {{ encrypted?: boolean }} [options] Optional retrieval flags. + * @param {{ encrypted?: boolean, since?: number }} [options] Optional retrieval flags. * @returns {Promise>} Parsed message payloads. */ async function fetchMessages(limit = MESSAGE_LIMIT, options = {}) { @@ -3636,8 +3652,11 @@ export function initializeApp(config) { if (options && options.encrypted) { params.set('encrypted', 'true'); } + if (options && options.since > 0) { + params.set('since', String(options.since)); + } const query = params.toString(); - const r = await fetch(`/api/messages?${query}`, { cache: 'no-store' }); + const r = await fetch(`/api/messages?${query}`, { cache: 'default' }); if (!r.ok) throw new Error('HTTP ' + r.status); return r.json(); } @@ -3646,11 +3665,14 @@ export function initializeApp(config) { * Fetch neighbour information from the JSON API. * * @param {number} [limit=NODE_LIMIT] Maximum number of rows. + * @param {number} [since=0] Unix timestamp; only rows newer than this are returned. * @returns {Promise>} Parsed neighbour payloads. */ - async function fetchNeighbors(limit = NODE_LIMIT) { + async function fetchNeighbors(limit = NODE_LIMIT, since = 0) { const effectiveLimit = resolveSnapshotLimit(limit, NODE_LIMIT); - const r = await fetch(`/api/neighbors?limit=${effectiveLimit}`, { cache: 'no-store' }); + let url = `/api/neighbors?limit=${effectiveLimit}`; + if (since > 0) url += `&since=${since}`; + const r = await fetch(url, { cache: 'default' }); if (!r.ok) throw new Error('HTTP ' + r.status); return r.json(); } @@ -3659,12 +3681,15 @@ export function initializeApp(config) { * Fetch traceroute observations from the JSON API. * * @param {number} [limit=TRACE_LIMIT] Maximum number of records. + * @param {number} [since=0] Unix timestamp; only rows newer than this are returned. * @returns {Promise>} Parsed trace payloads. */ - async function fetchTraces(limit = TRACE_LIMIT) { + async function fetchTraces(limit = TRACE_LIMIT, since = 0) { const safeLimit = Number.isFinite(limit) && limit > 0 ? Math.floor(limit) : TRACE_LIMIT; const effectiveLimit = Math.min(safeLimit, NODE_LIMIT); - const r = await fetch(`/api/traces?limit=${effectiveLimit}`, { cache: 'no-store' }); + let url = `/api/traces?limit=${effectiveLimit}`; + if (since > 0) url += `&since=${since}`; + const r = await fetch(url, { cache: 'default' }); if (!r.ok) throw new Error('HTTP ' + r.status); const traces = await r.json(); return filterRecentTraces(traces, TRACE_MAX_AGE_SECONDS); @@ -3674,11 +3699,14 @@ export function initializeApp(config) { * Fetch telemetry entries from the JSON API. * * @param {number} [limit=NODE_LIMIT] Maximum number of rows. + * @param {number} [since=0] Unix timestamp; only rows newer than this are returned. * @returns {Promise>} Parsed telemetry payloads. */ - async function fetchTelemetry(limit = NODE_LIMIT) { + async function fetchTelemetry(limit = NODE_LIMIT, since = 0) { const effectiveLimit = resolveSnapshotLimit(limit, NODE_LIMIT); - const r = await fetch(`/api/telemetry?limit=${effectiveLimit}`, { cache: 'no-store' }); + let url = `/api/telemetry?limit=${effectiveLimit}`; + if (since > 0) url += `&since=${since}`; + const r = await fetch(url, { cache: 'default' }); if (!r.ok) throw new Error('HTTP ' + r.status); return r.json(); } @@ -3687,11 +3715,14 @@ export function initializeApp(config) { * Fetch position packets from the JSON API. * * @param {number} [limit=NODE_LIMIT] Maximum number of rows. + * @param {number} [since=0] Unix timestamp; only rows newer than this are returned. * @returns {Promise>} Parsed position payloads. */ - async function fetchPositions(limit = NODE_LIMIT) { + async function fetchPositions(limit = NODE_LIMIT, since = 0) { const effectiveLimit = resolveSnapshotLimit(limit, NODE_LIMIT); - const r = await fetch(`/api/positions?limit=${effectiveLimit}`, { cache: 'no-store' }); + let url = `/api/positions?limit=${effectiveLimit}`; + if (since > 0) url += `&since=${since}`; + const r = await fetch(url, { cache: 'default' }); if (!r.ok) throw new Error('HTTP ' + r.status); return r.json(); } @@ -3708,6 +3739,7 @@ export function initializeApp(config) { return Number.isFinite(num) ? num : null; } + /** * Determine the best-effort timestamp in seconds from numeric or ISO values. * @@ -4486,49 +4518,102 @@ export function initializeApp(config) { if (statusEl) { statusEl.textContent = 'refreshing…'; } + // On the first load fetch the full dataset; subsequent refreshes pass + // the ``since`` timestamp so only new/changed rows are transferred. + // A 1-second overlap avoids missing rows that arrive at the boundary. + const useSince = initialFetchDone; + const nodeSince = useSince ? Math.max(0, lastNodeTimestamp - 1) : 0; + const msgSince = useSince ? Math.max(0, lastMessageTimestamp - 1) : 0; + const posSince = useSince ? Math.max(0, lastPositionTimestamp - 1) : 0; + const telSince = useSince ? Math.max(0, lastTelemetryTimestamp - 1) : 0; + const nbSince = useSince ? Math.max(0, lastNeighborTimestamp - 1) : 0; + const trSince = useSince ? Math.max(0, lastTraceTimestamp - 1) : 0; + // Secondary fetches are fire-and-forget with individual error handlers so // that a failure in one stream (e.g. telemetry) does not abort the whole // refresh cycle. Each promise resolves to an empty array on error, which // preserves the previous data until the next successful fetch. - const neighborPromise = fetchNeighbors().catch(err => { + const neighborPromise = fetchNeighbors(NODE_LIMIT, nbSince).catch(err => { console.warn('neighbor refresh failed; continuing without connections', err); return []; }); - const telemetryPromise = fetchTelemetry().catch(err => { + const telemetryPromise = fetchTelemetry(NODE_LIMIT, telSince).catch(err => { console.warn('telemetry refresh failed; continuing without telemetry', err); return []; }); - const positionsPromise = fetchPositions().catch(err => { + const positionsPromise = fetchPositions(NODE_LIMIT, posSince).catch(err => { console.warn('position refresh failed; continuing without updates', err); return []; }); - const tracesPromise = fetchTraces().catch(err => { + const tracesPromise = fetchTraces(TRACE_LIMIT, trSince).catch(err => { console.warn('trace refresh failed; continuing without traceroutes', err); return []; }); - const encryptedMessagesPromise = fetchMessages(MESSAGE_LIMIT, { encrypted: true }).catch(err => { + const encryptedMessagesPromise = fetchMessages(MESSAGE_LIMIT, { encrypted: true, since: msgSince }).catch(err => { console.warn('encrypted message refresh failed; continuing without encrypted entries', err); return []; }); // Fan-out all requests simultaneously; nodes are the primary resource and // must succeed for rendering to proceed. const [ - nodes, - positions, - neighborTuples, - traceEntries, - messages, - telemetryEntries, - encryptedMessages + incomingNodes, + incomingPositions, + incomingNeighbors, + incomingTraces, + incomingMessages, + incomingTelemetry, + incomingEncryptedMessages ] = await Promise.all([ - fetchNodes(), + fetchNodes(NODE_LIMIT, nodeSince), positionsPromise, neighborPromise, tracesPromise, - fetchMessages(MESSAGE_LIMIT), + fetchMessages(MESSAGE_LIMIT, { since: msgSince }), telemetryPromise, encryptedMessagesPromise ]); + + // Update high-water marks for incremental fetching. + const incomingNodeTs = maxRecordTimestamp(incomingNodes, ['last_heard']); + const incomingMsgTs = maxRecordTimestamp(incomingMessages, ['rx_time']); + const incomingEncMsgTs = maxRecordTimestamp(incomingEncryptedMessages, ['rx_time']); + const incomingPosTs = maxRecordTimestamp(incomingPositions, ['rx_time', 'position_time']); + const incomingTelTs = maxRecordTimestamp(incomingTelemetry, ['rx_time', 'telemetry_time']); + const incomingNbTs = maxRecordTimestamp(incomingNeighbors, ['rx_time']); + const incomingTrTs = maxRecordTimestamp(incomingTraces, ['rx_time']); + if (incomingNodeTs > lastNodeTimestamp) lastNodeTimestamp = incomingNodeTs; + const latestMsgTs = Math.max(incomingMsgTs, incomingEncMsgTs); + if (latestMsgTs > lastMessageTimestamp) lastMessageTimestamp = latestMsgTs; + if (incomingPosTs > lastPositionTimestamp) lastPositionTimestamp = incomingPosTs; + if (incomingTelTs > lastTelemetryTimestamp) lastTelemetryTimestamp = incomingTelTs; + if (incomingNbTs > lastNeighborTimestamp) lastNeighborTimestamp = incomingNbTs; + if (incomingTrTs > lastTraceTimestamp) lastTraceTimestamp = incomingTrTs; + + // Merge incremental results with existing data. On first load the + // existing arrays are empty so the merge is effectively a no-op. + // Merge incremental results with existing data then trim to the + // configured limits so long-running tabs do not accumulate stale + // entries beyond what the server would return on a fresh fetch. + const nodes = useSince ? mergeById(allNodes, incomingNodes, 'node_id') : incomingNodes; + const positions = useSince + ? trimToLimit(mergeById(allPositionEntries, incomingPositions, 'id'), NODE_LIMIT) + : incomingPositions; + const neighborTuples = useSince + ? mergeByCompositeKey(allNeighbors, incomingNeighbors, ['node_id', 'neighbor_id']) + : incomingNeighbors; + const telemetryEntries = useSince + ? trimToLimit(mergeById(allTelemetryEntries, incomingTelemetry, 'id'), NODE_LIMIT) + : incomingTelemetry; + const traceEntries = useSince + ? trimToLimit(mergeById(allTraces, incomingTraces, 'id'), TRACE_LIMIT) + : incomingTraces; + const messages = useSince + ? trimToLimit(mergeById(allMessages, incomingMessages, 'id'), MESSAGE_LIMIT) + : incomingMessages; + const encryptedMessages = useSince + ? trimToLimit(mergeById(allEncryptedMessages, incomingEncryptedMessages, 'id'), MESSAGE_LIMIT) + : incomingEncryptedMessages; + // Collapse per-source snapshot arrays into single merged records; the // snapshot window de-duplicates entries from multiple ingestors. const aggregatedNodes = aggregateNodeSnapshots(nodes); @@ -4558,6 +4643,7 @@ export function initializeApp(config) { allPositionEntries = aggregatedPositions; allNeighbors = aggregatedNeighbors; allTraces = Array.isArray(traceEntries) ? traceEntries : []; + initialFetchDone = true; applyFilter(); if (statusEl) { statusEl.textContent = 'updated ' + new Date().toLocaleTimeString(); @@ -4744,6 +4830,8 @@ export function initializeApp(config) { meshcoreColEl = mc; meshtasticColEl = mt; }, + /** Trigger a manual refresh cycle (test use only). */ + refresh, }, }; } diff --git a/web/public/assets/js/app/node-details.js b/web/public/assets/js/app/node-details.js index 894e909..fb76c53 100644 --- a/web/public/assets/js/app/node-details.js +++ b/web/public/assets/js/app/node-details.js @@ -24,7 +24,7 @@ import { aggregateTelemetrySnapshots, } from './snapshot-aggregator.js'; -const DEFAULT_FETCH_OPTIONS = Object.freeze({ cache: 'no-store' }); +const DEFAULT_FETCH_OPTIONS = Object.freeze({ cache: 'default' }); const TELEMETRY_LIMIT = 1000; const POSITION_LIMIT = SNAPSHOT_WINDOW; const NEIGHBOR_LIMIT = 1000; diff --git a/web/public/assets/js/app/node-page-data.js b/web/public/assets/js/app/node-page-data.js index 573bf8d..5efe93f 100644 --- a/web/public/assets/js/app/node-page-data.js +++ b/web/public/assets/js/app/node-page-data.js @@ -24,8 +24,8 @@ * @module node-page-data */ -/** Shared fetch options that disable the browser HTTP cache for all API calls. */ -const DEFAULT_FETCH_OPTIONS = Object.freeze({ cache: 'no-store' }); +/** Shared fetch options for API calls, allowing conditional ETag revalidation. */ +const DEFAULT_FETCH_OPTIONS = Object.freeze({ cache: 'default' }); /** Maximum number of messages to request from the messages API. */ const MESSAGE_LIMIT = 50; diff --git a/web/public/assets/js/app/node-page.js b/web/public/assets/js/app/node-page.js index 4128445..f485eaa 100644 --- a/web/public/assets/js/app/node-page.js +++ b/web/public/assets/js/app/node-page.js @@ -78,7 +78,7 @@ import { } from './node-page-charts.js'; import { fetchMessages, fetchTracesForNode } from './node-page-data.js'; -const DEFAULT_FETCH_OPTIONS = Object.freeze({ cache: 'no-store' }); +const DEFAULT_FETCH_OPTIONS = Object.freeze({ cache: 'default' }); const MESSAGE_LIMIT = 50; const RENDER_WAIT_INTERVAL_MS = 20; const RENDER_WAIT_TIMEOUT_MS = 500; diff --git a/web/public/assets/js/app/stats.js b/web/public/assets/js/app/stats.js index 3b06ece..cb76e82 100644 --- a/web/public/assets/js/app/stats.js +++ b/web/public/assets/js/app/stats.js @@ -138,7 +138,7 @@ async function fetchRemoteActiveNodeStats(fetchImpl) { activeNodeStatsFetchImpl = fetchImpl; activeNodeStatsFetchPromise = (async () => { - const response = await fetchImpl('/api/stats', { cache: 'no-store' }); + const response = await fetchImpl('/api/stats', { cache: 'default' }); if (!response?.ok) { throw new Error(`stats HTTP ${response?.status ?? 'unknown'}`); } diff --git a/web/spec/api_cache_spec.rb b/web/spec/api_cache_spec.rb new file mode 100644 index 0000000..e664776 --- /dev/null +++ b/web/spec/api_cache_spec.rb @@ -0,0 +1,165 @@ +# Copyright © 2025-26 l5yth & contributors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# frozen_string_literal: true + +require "spec_helper" + +RSpec.describe PotatoMesh::App::ApiCache do + after { described_class.invalidate_all } + + describe ".fetch" do + it "returns a hash with :value and :etag on cache miss" do + result = described_class.fetch("test:key", ttl_seconds: 60) { "value_a" } + expect(result).to be_a(Hash) + expect(result[:value]).to eq("value_a") + expect(result[:etag]).to match(/\A[0-9a-f]+\z/) + end + + it "returns the cached value within the TTL" do + described_class.fetch("test:ttl", ttl_seconds: 60) { "first" } + result = described_class.fetch("test:ttl", ttl_seconds: 60) { "second" } + expect(result[:value]).to eq("first") + end + + it "recomputes the value after the TTL expires" do + described_class.fetch("test:expired", ttl_seconds: 0) { "first" } + sleep 0.01 + result = described_class.fetch("test:expired", ttl_seconds: 0) { "second" } + expect(result[:value]).to eq("second") + end + + it "caches different keys independently" do + described_class.fetch("key:a", ttl_seconds: 60) { "alpha" } + described_class.fetch("key:b", ttl_seconds: 60) { "beta" } + + a = described_class.fetch("key:a", ttl_seconds: 60) { "stale" } + b = described_class.fetch("key:b", ttl_seconds: 60) { "stale" } + expect(a[:value]).to eq("alpha") + expect(b[:value]).to eq("beta") + end + + it "stores a pre-computed weak ETag matching the value digest" do + result = described_class.fetch("test:etag", ttl_seconds: 60) { '{"ok":true}' } + expected_digest = Digest::MD5.hexdigest('{"ok":true}') + expect(result[:etag]).to eq(expected_digest) + end + + it "returns the same ETag on cache hit without recomputing" do + first = described_class.fetch("test:etag-hit", ttl_seconds: 60) { "body" } + second = described_class.fetch("test:etag-hit", ttl_seconds: 60) { "other" } + expect(second[:etag]).to eq(first[:etag]) + end + end + + describe ".invalidate_all" do + it "clears all cached entries" do + described_class.fetch("inv:x", ttl_seconds: 60) { "x" } + described_class.fetch("inv:y", ttl_seconds: 60) { "y" } + expect(described_class.size).to eq(2) + + described_class.invalidate_all + expect(described_class.size).to eq(0) + + result = described_class.fetch("inv:x", ttl_seconds: 60) { "fresh" } + expect(result[:value]).to eq("fresh") + end + end + + describe ".invalidate" do + it "removes only the specified keys" do + described_class.fetch("sel:a", ttl_seconds: 60) { "a" } + described_class.fetch("sel:b", ttl_seconds: 60) { "b" } + described_class.fetch("sel:c", ttl_seconds: 60) { "c" } + + described_class.invalidate("sel:a", "sel:c") + expect(described_class.size).to eq(1) + + result = described_class.fetch("sel:b", ttl_seconds: 60) { "stale" } + expect(result[:value]).to eq("b") + + result = described_class.fetch("sel:a", ttl_seconds: 60) { "new_a" } + expect(result[:value]).to eq("new_a") + end + end + + describe ".invalidate_prefix" do + it "removes entries whose keys start with any of the given prefixes" do + described_class.fetch("api:nodes:200:", ttl_seconds: 60) { "n" } + described_class.fetch("api:nodes:1000:", ttl_seconds: 60) { "n2" } + described_class.fetch("api:messages:200:", ttl_seconds: 60) { "m" } + described_class.fetch("api:stats:0", ttl_seconds: 60) { "s" } + + described_class.invalidate_prefix("api:nodes:", "api:stats:") + expect(described_class.size).to eq(1) + + result = described_class.fetch("api:messages:200:", ttl_seconds: 60) { "stale" } + expect(result[:value]).to eq("m") + end + + it "is a no-op when no keys match" do + described_class.fetch("api:nodes:x", ttl_seconds: 60) { "n" } + described_class.invalidate_prefix("api:telemetry:") + expect(described_class.size).to eq(1) + end + end + + describe "MAX_ENTRIES eviction" do + it "evicts the oldest entry when the store exceeds MAX_ENTRIES" do + max = described_class::MAX_ENTRIES + # Fill the cache to capacity + max.times do |i| + described_class.fetch("fill:#{i}", ttl_seconds: 60) { "v#{i}" } + end + expect(described_class.size).to eq(max) + + # Adding one more should evict the oldest + described_class.fetch("fill:overflow", ttl_seconds: 60) { "new" } + expect(described_class.size).to eq(max) + + # The first entry should have been evicted + result = described_class.fetch("fill:0", ttl_seconds: 60) { "recomputed" } + expect(result[:value]).to eq("recomputed") + end + end + + describe ".size" do + it "reports the number of cached entries" do + expect(described_class.size).to eq(0) + described_class.fetch("sz:1", ttl_seconds: 60) { "v" } + expect(described_class.size).to eq(1) + end + end + + describe "error handling" do + it "does not cache the value when the block raises" do + expect { + described_class.fetch("err:raise", ttl_seconds: 60) { raise "boom" } + }.to raise_error(RuntimeError, "boom") + + expect(described_class.size).to eq(0) + end + + it "allows a subsequent fetch after a block error" do + begin + described_class.fetch("err:retry", ttl_seconds: 60) { raise "first" } + rescue RuntimeError + # expected + end + + result = described_class.fetch("err:retry", ttl_seconds: 60) { "recovered" } + expect(result[:value]).to eq("recovered") + end + end +end diff --git a/web/spec/app_spec.rb b/web/spec/app_spec.rb index 1b945a1..7aab983 100644 --- a/web/spec/app_spec.rb +++ b/web/spec/app_spec.rb @@ -496,6 +496,7 @@ RSpec.describe "Potato Mesh Sinatra app" do ENV.delete("PRIVATE") allow(Time).to receive(:now).and_return(reference_time) clear_database + PotatoMesh::App::ApiCache.invalidate_all end after do diff --git a/web/spec/queries_spec.rb b/web/spec/queries_spec.rb index 8f9ed87..63b9ef0 100644 --- a/web/spec/queries_spec.rb +++ b/web/spec/queries_spec.rb @@ -657,4 +657,94 @@ RSpec.describe PotatoMesh::App::Queries do expect(rows.length).to be >= 1 end end + + # --------------------------------------------------------------------------- + # batch_resolve_node_ids + # --------------------------------------------------------------------------- + describe "#batch_resolve_node_ids" do + before do + with_db do |db| + db.execute( + "INSERT INTO nodes(node_id, num, short_name, last_heard, first_heard, role) VALUES (?,?,?,?,?,?)", + ["!aabb0001", 0xaabb0001, "N1", now, now, "CLIENT"], + ) + db.execute( + "INSERT INTO nodes(node_id, num, short_name, last_heard, first_heard, role) VALUES (?,?,?,?,?,?)", + ["!aabb0002", 0xaabb0002, "N2", now, now, "CLIENT"], + ) + end + end + + it "resolves string node_id references" do + with_db do |db| + result = queries.batch_resolve_node_ids(db, ["!aabb0001", "!aabb0002"]) + expect(result["!aabb0001"]).to eq("!aabb0001") + expect(result["!aabb0002"]).to eq("!aabb0002") + end + end + + it "resolves numeric references to canonical node_id" do + with_db do |db| + num_str = 0xaabb0001.to_s + result = queries.batch_resolve_node_ids(db, [num_str]) + expect(result[num_str]).to eq("!aabb0001") + end + end + + it "returns an empty hash for empty input" do + with_db do |db| + expect(queries.batch_resolve_node_ids(db, [])).to eq({}) + expect(queries.batch_resolve_node_ids(db, nil)).to eq({}) + end + end + + it "omits references that cannot be resolved" do + with_db do |db| + result = queries.batch_resolve_node_ids(db, ["!nonexistent"]) + expect(result).not_to have_key("!nonexistent") + end + end + end + + # --------------------------------------------------------------------------- + # node_lookup_clause with db parameter + # --------------------------------------------------------------------------- + describe "#node_lookup_clause with db" do + before do + with_db do |db| + db.execute( + "INSERT INTO nodes(node_id, num, short_name, last_heard, first_heard, role) VALUES (?,?,?,?,?,?)", + ["!deadbeef", 0xdeadbeef, "DB", now, now, "CLIENT"], + ) + end + end + + it "folds numeric columns into string columns when db is provided" do + with_db do |db| + clause = queries.node_lookup_clause( + "!deadbeef", + string_columns: ["node_id"], + numeric_columns: ["node_num"], + db: db, + ) + expect(clause).not_to be_nil + sql_fragment, _params = clause + # When db is provided and numeric values are resolved, the OR with + # node_num should not appear in the SQL. + expect(sql_fragment).not_to include("node_num") + expect(sql_fragment).to include("node_id") + end + end + + it "falls back to OR when db is not provided" do + clause = queries.node_lookup_clause( + 0xdeadbeef, + string_columns: ["node_id"], + numeric_columns: ["node_num"], + ) + expect(clause).not_to be_nil + sql_fragment, _params = clause + expect(sql_fragment).to include("OR") + end + end end