web: schema upgrades fail loudly, reach parity, index nodes.num (#1036)

CHANGELOG.md

@@ -189,6 +189,11 @@
 * Web: a federation instance row is replaced in one transaction: a failed write keeps the stored row, and two writes for one domain at once no longer fail on the unique domain index (SPEC FK1)
 * Web: a database error while a crawl stores one peer is logged and the crawl goes on with the next peer (SPEC FK2)
 * Web: a keyfile that cannot be parsed stops the boot with an error naming the file and is left as it is, instead of being replaced by a new key and instance id; a legacy keyfile that cannot be parsed is not copied, and the error names it; restore the named file from a backup or delete it to start with a new key (SPEC FK3)
+* Web: looking up a node by its number no longer scans the nodes table (SPEC SU1)
+* Web: a database upgrade step that fails stops the boot with an error naming the step and leaves the database as it was, instead of logging a warning and serving a half-upgraded database (SPEC SU2)
+* Web: the boot recreates every index from `data/*.sql` that the database lacks, and stops, naming what is missing, when a table or column is still missing after the upgrade (SPEC SU3)
+* Web: a database created by v0.2.0 gains `messages.encrypted`; a database created before v0.6.0 gets the MeshCore text index and the duplicate purge on its first boot, not its second; the upgrade matches column names in any case (SPEC SU4)
+* Web: a process that starts while another one upgrades the same database no longer fails on a duplicate column or a locked database (SPEC SU5)
 * Data: the Reticulum ingestor posts interface names without peer addresses: a TCP, Backbone, UDP or AutoInterface peer name keeps its class and the operator-given name and drops the address and port, an outbound I2P peer drops its I2P destination, a Weave peer posts its class, and RNode names are unchanged (SPEC RI1)
 * Web: interface names are stored without peer addresses whatever the ingestor posts, and stored names that carry one lose it at the next boot, so `GET /api/destinations` serves none, with or without `PRIVATE=1` (SPEC RI2/RI3)
 * Web: `/metrics` request metrics label `path` with the route that answered, such as `GET /api/nodes/:id`, or `static`, `metrics` or `unmatched`, instead of the request path; 8000 requests to distinct missing paths no longer add 8000 series (SPEC PG1)

data/migrations/20261009_add_nodes_num_index.sql

@@ -0,0 +1,25 @@
+-- Copyright © 2025-26 l5yth & contributors
+--
+-- Licensed under the Apache License, Version 2.0 (the "License");
+-- you may not use this file except in compliance with the License.
+-- You may obtain a copy of the License at
+--
+--     http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+
+-- Index the numeric node id (SPEC SU1): additive index only.
+--
+-- nodes.num: the Meshtastic node number. A per-id read resolves a numeric
+--   reference through it, ingest resolves a decimal sender or recipient
+--   through it, and the batch resolver looks up many at once; without the
+--   index each lookup scans every node row.
+--
+-- The web app applies this conditionally at boot (database.rb); this file is
+-- the standalone mirror for CLI/manual migration of older installations.
+
+CREATE INDEX IF NOT EXISTS idx_nodes_num ON nodes(num);

data/nodes.sql

@@ -62,3 +62,6 @@ CREATE INDEX IF NOT EXISTS idx_nodes_last_heard ON nodes(last_heard);
 CREATE INDEX IF NOT EXISTS idx_nodes_hw_model  ON nodes(hw_model);
 CREATE INDEX IF NOT EXISTS idx_nodes_latlon    ON nodes(latitude, longitude);
 CREATE INDEX IF NOT EXISTS idx_nodes_long_name ON nodes(long_name);
+-- Numeric node references (per-id reads, decimal ids on ingest) search this
+-- index (SPEC SU1). ensure_schema_upgrades creates it on upgraded databases.
+CREATE INDEX IF NOT EXISTS idx_nodes_num       ON nodes(num);

web/lib/potato_mesh/application.rb

@@ -260,6 +260,7 @@ module PotatoMesh
       ensure_schema_upgrades
       scrub_destination_interfaces # SPEC RI3: needs the destinations table, so after the upgrades
       init_db unless db_schema_present?
+      ensure_schema_parity!
 
       log_instance_domain_resolution
       log_instance_public_key

web/lib/potato_mesh/application/database.rb

@@ -14,6 +14,8 @@
 
 # frozen_string_literal: true
 
+require_relative "database/schema_guard"
+
 module PotatoMesh
   module App
     module Database
@@ -125,16 +127,36 @@ module PotatoMesh
 
       # Apply any schema migrations required for older installations.
       #
+      # Every step runs in one IMMEDIATE transaction (SPEC SU2, SU5). A boot
+      # that races another boot on the same database waits for the write
+      # lock, then reads the upgraded schema and changes nothing. A failing
+      # step rolls the whole upgrade back and fails the boot with an error
+      # naming the step; the +dest_hash+ DROP is the one step allowed to
+      # fail. A new database gets its full schema here too, in the same
+      # transaction. Column guards match names in any case, as SQLite does
+      # (SPEC SU4), and every index +data/*.sql+ defines that the database
+      # lacks is created last (SPEC SU3).
+      #
       # @return [void]
+      # @raise [SchemaUpgradeError] when a step fails.
       def ensure_schema_upgrades
         FileUtils.mkdir_p(File.dirname(PotatoMesh::Config.db_path))
         db = open_database
+        db.extend(UpgradeConnection)
+        # Some data/*.sql files set WAL, and journal_mode cannot change
+        # inside a transaction, so the switch comes before the one they run
+        # in. A new file is still in rollback-journal mode: there the switch
+        # takes the write lock, and SQLite reports BUSY at once, without the
+        # busy timeout, while a racing boot holds it. So it retries like the
+        # BEGIN (SPEC SU5).
+        with_busy_retry { db.execute("PRAGMA journal_mode=WAL") }
+        with_busy_retry { db.execute("BEGIN IMMEDIATE") }
 
         node_table_exists = db.get_first_value(
           "SELECT COUNT(*) FROM sqlite_master WHERE type='table' AND name='nodes'",
         ).to_i > 0
         if node_table_exists
-          node_columns = db.execute("PRAGMA table_info(nodes)").map { |row| row[1] }
+          node_columns = schema_column_names(db, "nodes")
           unless node_columns.include?("precision_bits")
             db.execute("ALTER TABLE nodes ADD COLUMN precision_bits INTEGER")
             node_columns << "precision_bits"
@@ -306,9 +328,15 @@ module PotatoMesh
         message_table_exists = db.get_first_value(
           "SELECT COUNT(*) FROM sqlite_master WHERE type='table' AND name='messages'",
         ).to_i > 0
-        message_columns = message_table_exists ? db.execute("PRAGMA table_info(messages)").map { |row| row[1] } : []
+        message_columns = message_table_exists ? schema_column_names(db, "messages") : []
 
         if message_table_exists
+          # v0.2.0 created messages without this column, and no later boot
+          # added it, so every message insert there failed (SPEC SU4).
+          unless message_columns.include?("encrypted")
+            db.execute("ALTER TABLE messages ADD COLUMN encrypted TEXT")
+          end
+
           unless message_columns.include?("lora_freq")
             db.execute("ALTER TABLE messages ADD COLUMN lora_freq INTEGER")
           end
@@ -342,6 +370,10 @@ module PotatoMesh
           unless message_columns.include?("protocol")
             db.execute("ALTER TABLE messages ADD COLUMN protocol TEXT NOT NULL DEFAULT 'meshtastic'")
             db.execute("UPDATE messages SET protocol = 'meshtastic' WHERE protocol IS NULL OR TRIM(protocol) = ''")
+            # The MX6 index and the #756 purge below key on protocol; keep
+            # the list current so they run on this boot, not the next one
+            # (SPEC SU4).
+            message_columns << "protocol"
           end
 
           # RF metrics (SPEC RF1/RF2/RF6): hops actually travelled (distinct
@@ -500,7 +532,7 @@ module PotatoMesh
           db.execute_batch(File.read(sql_file))
         end
 
-        instance_columns = db.execute("PRAGMA table_info(instances)").map { |row| row[1] }
+        instance_columns = schema_column_names(db, "instances")
         unless instance_columns.include?("contact_link")
           db.execute("ALTER TABLE instances ADD COLUMN contact_link TEXT")
           instance_columns << "contact_link"
@@ -533,7 +565,7 @@ module PotatoMesh
           db.execute_batch(File.read(telemetry_schema))
         end
 
-        telemetry_columns = db.execute("PRAGMA table_info(telemetry)").map { |row| row[1] }
+        telemetry_columns = schema_column_names(db, "telemetry")
         # The environment expansion plus the extended metric families (TI-A2:
         # power / air-quality / health / local / host / traffic stats and the
         # one-wire probe list) share one idempotent backfill loop; the extended
@@ -562,7 +594,7 @@ module PotatoMesh
           positions_schema = File.expand_path("../../../../data/positions.sql", __dir__)
           db.execute_batch(File.read(positions_schema))
         end
-        position_columns = db.execute("PRAGMA table_info(positions)").map { |row| row[1] }
+        position_columns = schema_column_names(db, "positions")
         unless position_columns.include?("ingestor")
           db.execute("ALTER TABLE positions ADD COLUMN ingestor TEXT")
         end
@@ -578,7 +610,7 @@ module PotatoMesh
           neighbors_schema = File.expand_path("../../../../data/neighbors.sql", __dir__)
           db.execute_batch(File.read(neighbors_schema))
         end
-        neighbor_columns = db.execute("PRAGMA table_info(neighbors)").map { |row| row[1] }
+        neighbor_columns = schema_column_names(db, "neighbors")
         unless neighbor_columns.include?("ingestor")
           db.execute("ALTER TABLE neighbors ADD COLUMN ingestor TEXT")
         end
@@ -596,7 +628,7 @@ module PotatoMesh
           traces_schema = File.expand_path("../../../../data/traces.sql", __dir__)
           db.execute_batch(File.read(traces_schema))
         end
-        trace_columns = db.execute("PRAGMA table_info(traces)").map { |row| row[1] }
+        trace_columns = schema_column_names(db, "traces")
         unless trace_columns.include?("ingestor")
           db.execute("ALTER TABLE traces ADD COLUMN ingestor TEXT")
         end
@@ -612,7 +644,7 @@ module PotatoMesh
           ingestors_schema = File.expand_path("../../../../data/ingestors.sql", __dir__)
           db.execute_batch(File.read(ingestors_schema))
         else
-          ingestor_columns = db.execute("PRAGMA table_info(ingestors)").map { |row| row[1] }
+          ingestor_columns = schema_column_names(db, "ingestors")
           unless ingestor_columns.include?("version")
             db.execute("ALTER TABLE ingestors ADD COLUMN version TEXT")
           end
@@ -656,14 +688,30 @@ module PotatoMesh
           destinations_schema = File.expand_path("../../../../data/destinations.sql", __dir__)
           db.execute_batch(File.read(destinations_schema))
         end
-      rescue SQLite3::SQLException, Errno::ENOENT => e
-        warn_log(
-          "Failed to apply schema upgrade",
-          context: "database.schema",
-          error_class: e.class.name,
-          error_message: e.message,
-        )
+
+        # A new database has no nodes or messages table. They come last, after
+        # the blocks above skipped them as before, so a fresh schema commits
+        # whole in this transaction and a racing boot never reads it half
+        # built (SPEC SU5); init_db then finds the schema present.
+        %w[nodes messages].each do |schema|
+          table_exists = db.get_first_value(
+            "SELECT COUNT(*) FROM sqlite_master WHERE type='table' AND name=?", [schema],
+          ).to_i > 0
+          db.execute_batch(File.read(File.join(SCHEMA_DIRECTORY, "#{schema}.sql"))) unless table_exists
+        end
+
+        # An index an operator dropped comes back here, after every other
+        # step (SPEC SU3).
+        create_missing_schema_indexes(db)
+
+        db.execute("COMMIT")
+      rescue SQLite3::Exception => e
+        step = db.respond_to?(:upgrade_step_label) ? db.upgrade_step_label : "opening the database"
+        raise SchemaUpgradeError,
+              "Schema upgrade step failed: #{step} (#{e.class}: #{e.message.lines.first.to_s.chomp.chomp(":")})"
       ensure
+        # Closing rolls back the transaction a failed step left open, so the
+        # database keeps its pre-boot schema.
         db&.close
       end
     end

web/lib/potato_mesh/application/database/schema_guard.rb

@@ -0,0 +1,210 @@
+# Copyright © 2025-26 l5yth & contributors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# frozen_string_literal: true
+
+module PotatoMesh
+  module App
+    # Raised at boot when a schema upgrade step fails, or when the upgraded
+    # database still lacks a table, column or index that +data/*.sql+
+    # defines (SPEC SU2, SU3). The message names the failed step or every
+    # missing object, and the boot stops.
+    class SchemaUpgradeError < StandardError; end
+
+    module Database
+      # Directory holding the fresh-install schema files, +data/*.sql+.
+      SCHEMA_DIRECTORY = File.expand_path("../../../../../data", __dir__)
+
+      # Longest statement text a failed-step message quotes.
+      UPGRADE_STEP_LABEL_LIMIT = 200
+
+      # Behaviour +ensure_schema_upgrades+ gives the one connection it runs
+      # its single IMMEDIATE transaction on (SPEC SU2, SU5). Mixed into that
+      # connection with +extend+, so every other handle keeps the stock
+      # +SQLite3::Database+ behaviour.
+      module UpgradeConnection
+        # @return [String, nil] the SQL prepared last: the step running when
+        #   a statement fails.
+        attr_reader :upgrade_step
+
+        # Record +sql+ as the current step, then prepare it.
+        #
+        # +execute+, +get_first_value+ and every statement of +execute_batch+
+        # prepare through here, so the record is the statement SQLite
+        # rejected, not a call site.
+        #
+        # @param sql [String] statement text; inside +execute_batch+ the
+        #   remaining batch, whose first statement is the one prepared.
+        # @return [SQLite3::Statement, Object] the statement, or the block's
+        #   result when a block is given.
+        def prepare(sql, &block)
+          @upgrade_step = sql
+          super
+        end
+
+        # Run a nested +transaction+ block inside the upgrade's transaction.
+        #
+        # The upgrade already holds an IMMEDIATE transaction, which a BEGIN
+        # cannot nest in. A nested block's statements commit or roll back
+        # with the rest of the upgrade, so the block keeps the all-or-nothing
+        # guarantee it asked for. Outside a transaction this is the stock
+        # +SQLite3::Database#transaction+.
+        #
+        # @param mode [Symbol, nil] transaction mode, used outside a transaction.
+        # @yieldparam db [SQLite3::Database] this connection.
+        # @return [Object] the block's result, or the stock method's.
+        def transaction(mode = nil, &block)
+          return super unless block && transaction_active?
+
+          yield self
+        end
+
+        # The current step as one line: its first statement without SQL
+        # comments, cut at {UPGRADE_STEP_LABEL_LIMIT} characters.
+        #
+        # @return [String] the statement text, or an empty string before the
+        #   first statement.
+        def upgrade_step_label
+          statement = upgrade_step.to_s.gsub(/--[^\n]*/, "")
+          statement = statement.split(";").first.to_s.split.join(" ")
+          return statement if statement.length <= UPGRADE_STEP_LABEL_LIMIT
+
+          "#{statement[0, UPGRADE_STEP_LABEL_LIMIT - 3]}..."
+        end
+      end
+
+      # Fail the boot unless the database holds every table, column and
+      # index the fresh schema defines (SPEC SU3).
+      #
+      # Runs after +ensure_schema_upgrades+ and +init_db+, so an upgraded
+      # database reaches here with every step applied and every index
+      # created. Objects the fresh schema does not define (a legacy column,
+      # a hand-made index) are allowed; only absences fail.
+      #
+      # @return [void]
+      # @raise [SchemaUpgradeError] naming every missing object.
+      def ensure_schema_parity!
+        db = open_database(readonly: true)
+        missing = missing_schema_objects(db)
+        return if missing.empty?
+
+        raise SchemaUpgradeError,
+              "Database schema lacks what data/*.sql defines after the upgrade: #{missing.join(", ")}"
+      ensure
+        db&.close
+      end
+
+      # List what the fresh schema defines and +db+ lacks.
+      #
+      # SQLite compares identifiers case-insensitively, and so does this.
+      #
+      # @param db [SQLite3::Database] open database handle.
+      # @param expected [Hash] schema shape to hold +db+ against; defaults to
+      #   {#fresh_schema_shape}.
+      # @return [Array<String>] +table <name>+, +<table>.<column>+ and
+      #   +index <name>+ entries, in schema order.
+      def missing_schema_objects(db, expected = fresh_schema_shape)
+        missing = []
+        expected.fetch(:tables).each do |table, columns|
+          present = schema_column_names(db, table)
+          if present.empty?
+            missing << "table #{table}"
+          else
+            missing.concat(columns.reject { |column| present.include?(column.downcase) }.map { |column| "#{table}.#{column}" })
+          end
+        end
+        indexes = schema_index_names(db).map(&:downcase)
+        missing.concat(expected.fetch(:indexes).keys.reject { |name| indexes.include?(name.downcase) }.map { |name| "index #{name}" })
+      end
+
+      # Create each index +data/*.sql+ defines that +db+ lacks, with the
+      # statement the fresh schema holds for it (SPEC SU3). An index is safe
+      # to build again, so one an operator dropped comes back on the next
+      # boot instead of failing it.
+      #
+      # The upgrade runs this last, in its transaction. An index whose table
+      # is missing or lacks a column of the fresh schema is skipped: the
+      # parity check then names that table or column, and the index.
+      #
+      # @param db [SQLite3::Database] the upgrade's connection.
+      # @return [void]
+      def create_missing_schema_indexes(db)
+        shape = fresh_schema_shape
+        present = schema_index_names(db).map(&:downcase)
+        shape.fetch(:indexes).each do |name, index|
+          next if present.include?(name.downcase)
+
+          columns = schema_column_names(db, index[:table])
+          next if columns.empty?
+          next unless shape.fetch(:tables).fetch(index[:table]).all? { |column| columns.include?(column.downcase) }
+
+          db.execute(index[:sql])
+        end
+      end
+
+      # Tables, columns and named indexes +data/*.sql+ defines, read from an
+      # in-memory database built from every file in {SCHEMA_DIRECTORY}.
+      #
+      # @return [Hash] +:tables+ maps each table to its column names in
+      #   definition order; +:indexes+ maps each index name to its +:table+
+      #   and the +:sql+ that creates it, as SQLite stores it (without
+      #   +IF NOT EXISTS+).
+      def fresh_schema_shape
+        SQLite3::Database.open(":memory:") do |db|
+          Dir.glob(File.join(SCHEMA_DIRECTORY, "*.sql")).sort.each { |file| db.execute_batch(File.read(file)) }
+          tables = db.execute("SELECT name FROM sqlite_master WHERE type = 'table' AND name NOT LIKE 'sqlite_%' ORDER BY rowid").flatten
+          # SQLite's own autoindexes carry no statement.
+          indexes = db.execute("SELECT name, tbl_name, sql FROM sqlite_master WHERE type = 'index' AND sql IS NOT NULL ORDER BY rowid")
+          {
+            tables: tables.to_h { |table| [table, schema_table_columns(db, table)] },
+            indexes: indexes.to_h { |name, table, sql| [name, { table: table, sql: sql }] },
+          }
+        end
+      end
+
+      private
+
+      # Column names of +table+ in +db+, in lower case. SQLite matches
+      # identifiers in any case, so the upgrade's column guards and the
+      # parity check do too (SPEC SU3, SU4).
+      #
+      # @param db [SQLite3::Database] open database handle.
+      # @param table [String] table name.
+      # @return [Array<String>] lower-case column names; empty when the table
+      #   is absent.
+      def schema_column_names(db, table)
+        schema_table_columns(db, table).map(&:downcase)
+      end
+
+      # Column names of +table+ in +db+.
+      #
+      # @param db [SQLite3::Database] open database handle.
+      # @param table [String] table name, quoted for the pragma.
+      # @return [Array<String>] column names; empty when the table is absent.
+      def schema_table_columns(db, table)
+        db.execute(%(PRAGMA table_info("#{table.gsub('"', '""')}"))).map { |row| row[1] }
+      end
+
+      # Named indexes in +db+. SQLite's own +sqlite_autoindex_*+ entries
+      # belong to a table's key constraints, not to the schema files, and
+      # are left out.
+      #
+      # @param db [SQLite3::Database] open database handle.
+      # @return [Array<String>] index names in creation order.
+      def schema_index_names(db)
+        db.execute("SELECT name FROM sqlite_master WHERE type = 'index' AND name NOT LIKE 'sqlite_%' ORDER BY rowid").flatten
+      end
+    end
+  end
+end

web/spec/database_spec.rb

@@ -93,8 +93,11 @@ RSpec.describe PotatoMesh::App::Database do
 
   it "adds missing telemetry columns when upgrading an existing schema" do
     SQLite3::Database.new(PotatoMesh::Config.db_path) do |db|
-      db.execute("CREATE TABLE nodes(node_id TEXT)")
-      db.execute("CREATE TABLE messages(id INTEGER PRIMARY KEY)")
+      # The second boot's #747 and #755 backfills read long_name, role and
+      # from_id, and delete through the neighbors foreign key, which needs a
+      # keyed node_id. A failing step stops the boot (SPEC SU2).
+      db.execute("CREATE TABLE nodes(node_id TEXT PRIMARY KEY, long_name TEXT, role TEXT)")
+      db.execute("CREATE TABLE messages(id INTEGER PRIMARY KEY, from_id TEXT)")
       db.execute <<~SQL
                    CREATE TABLE telemetry (
                      id INTEGER PRIMARY KEY,
@@ -155,8 +158,9 @@ RSpec.describe PotatoMesh::App::Database do
     # #893 shipped dest_hash as a JSON column; the destinations table replaced
     # it, and a column and a table modelling one relationship would drift.
     SQLite3::Database.new(PotatoMesh::Config.db_path) do |db|
-      db.execute("CREATE TABLE nodes(node_id TEXT, dest_hash TEXT)")
-      db.execute("CREATE TABLE messages(id INTEGER PRIMARY KEY)")
+      # Keyed node_id, long_name, role and from_id: the second boot's backfills.
+      db.execute("CREATE TABLE nodes(node_id TEXT PRIMARY KEY, long_name TEXT, role TEXT, dest_hash TEXT)")
+      db.execute("CREATE TABLE messages(id INTEGER PRIMARY KEY, from_id TEXT)")
     end
 
     harness_class.ensure_schema_upgrades
@@ -188,8 +192,9 @@ RSpec.describe PotatoMesh::App::Database do
 
   it "backfills every extended telemetry metric column on an existing schema (TI-A2)" do
     SQLite3::Database.new(PotatoMesh::Config.db_path) do |db|
-      db.execute("CREATE TABLE nodes(node_id TEXT)")
-      db.execute("CREATE TABLE messages(id INTEGER PRIMARY KEY)")
+      # Keyed node_id, long_name, role and from_id: the second boot's backfills.
+      db.execute("CREATE TABLE nodes(node_id TEXT PRIMARY KEY, long_name TEXT, role TEXT)")
+      db.execute("CREATE TABLE messages(id INTEGER PRIMARY KEY, from_id TEXT)")
       db.execute("CREATE TABLE telemetry(id INTEGER PRIMARY KEY, rx_time INTEGER NOT NULL, rx_iso TEXT NOT NULL)")
     end
 
@@ -332,7 +337,8 @@ RSpec.describe PotatoMesh::App::Database do
           protocol TEXT NOT NULL DEFAULT 'meshtastic', synthetic BOOLEAN NOT NULL DEFAULT 0
         )
       SQL
-      db.execute("CREATE TABLE messages(id INTEGER PRIMARY KEY)")
+      # from_id: read by the #755 backfill that follows #747 on this boot.
+      db.execute("CREATE TABLE messages(id INTEGER PRIMARY KEY, from_id TEXT)")
 
       # Misclassified meshcore placeholder (bug #747)
       db.execute(

web/spec/fixtures/schema/v0.2.0.sql

@@ -0,0 +1,89 @@
+-- Copyright © 2025-26 l5yth & contributors
+--
+-- Licensed under the Apache License, Version 2.0 (the "License");
+-- you may not use this file except in compliance with the License.
+-- You may obtain a copy of the License at
+--
+--     http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+
+-- Schema fixture (SPEC SU6): the database a fresh v0.2.0 install created.
+-- The DDL is that release's init_db file list, each file verbatim from git
+-- without its license header:
+-- nodes, messages.
+-- The sample rows at the end must survive the boot-time schema upgrade
+-- unchanged.
+
+-- data/nodes.sql at v0.2.0
+
+PRAGMA journal_mode=WAL;
+
+CREATE TABLE IF NOT EXISTS nodes (
+  node_id            TEXT PRIMARY KEY,
+  num                INTEGER,
+  short_name         TEXT,
+  long_name          TEXT,
+  macaddr            TEXT,
+  hw_model           TEXT,
+  role               TEXT,
+  public_key         TEXT,
+  is_unmessagable    BOOLEAN,
+  is_favorite        BOOLEAN,
+  hops_away          INTEGER,
+  snr                REAL,
+  last_heard         INTEGER,
+  first_heard        INTEGER,
+  battery_level      REAL,
+  voltage            REAL,
+  channel_utilization REAL,
+  air_util_tx        REAL,
+  uptime_seconds     INTEGER,
+  position_time      INTEGER,
+  location_source    TEXT,
+  latitude           REAL,
+  longitude          REAL,
+  altitude           REAL
+);
+
+CREATE INDEX IF NOT EXISTS idx_nodes_last_heard ON nodes(last_heard);
+CREATE INDEX IF NOT EXISTS idx_nodes_hw_model  ON nodes(hw_model);
+CREATE INDEX IF NOT EXISTS idx_nodes_latlon    ON nodes(latitude, longitude);
+
+-- data/messages.sql at v0.2.0
+
+CREATE TABLE IF NOT EXISTS messages (
+    id        INTEGER PRIMARY KEY,
+    rx_time   INTEGER NOT NULL,
+    rx_iso    TEXT NOT NULL,
+    from_id   TEXT,
+    to_id     TEXT,
+    channel   INTEGER,
+    portnum   TEXT,
+    text      TEXT,
+    snr       REAL,
+    rssi      INTEGER,
+    hop_limit INTEGER,
+    raw_json  TEXT
+);
+
+CREATE INDEX IF NOT EXISTS idx_messages_rx_time   ON messages(rx_time);
+CREATE INDEX IF NOT EXISTS idx_messages_from_id   ON messages(from_id);
+CREATE INDEX IF NOT EXISTS idx_messages_to_id     ON messages(to_id);
+CREATE INDEX IF NOT EXISTS idx_messages_channel   ON messages(channel);
+CREATE INDEX IF NOT EXISTS idx_messages_portnum   ON messages(portnum);
+
+-- Sample rows.
+
+INSERT INTO nodes(node_id, num, short_name, long_name, hw_model, role, snr, last_heard, first_heard, latitude, longitude, altitude)
+  VALUES ('!a1b2c3d4', 2712847316, 'A1B2', 'Legacy Alpha', 'TBEAM', 'CLIENT', 6.5, 1735689600, 1735600000, 52.52, 13.405, 34.0);
+INSERT INTO nodes(node_id, num, short_name, long_name, hw_model, role, snr, last_heard, first_heard)
+  VALUES ('!0badc0de', 195936478, 'BADC', 'Legacy Bravo', 'HELTEC_V3', 'ROUTER', -3.25, 1735689000, 1735500000);
+INSERT INTO messages(id, rx_time, rx_iso, from_id, to_id, channel, portnum, text, snr, rssi, hop_limit, raw_json)
+  VALUES (1001, 1735689600, '2025-01-01T00:00:00Z', '!a1b2c3d4', '^all', 0, 'TEXT_MESSAGE_APP', 'hello from v0.2.0', -7.5, -92, 3, '{"id":1001}');
+INSERT INTO messages(id, rx_time, rx_iso, from_id, to_id, channel, portnum, text, snr, rssi, hop_limit)
+  VALUES (1002, 1735689660, '2025-01-01T00:01:00Z', '!0badc0de', '!a1b2c3d4', 0, 'TEXT_MESSAGE_APP', 'direct reply', 5.25, -80, 2);

web/spec/fixtures/schema/v0.4.0.sql

@@ -0,0 +1,172 @@
+-- Copyright © 2025-26 l5yth & contributors
+--
+-- Licensed under the Apache License, Version 2.0 (the "License");
+-- you may not use this file except in compliance with the License.
+-- You may obtain a copy of the License at
+--
+--     http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+
+-- Schema fixture (SPEC SU6): the database a fresh v0.4.0 install created.
+-- The DDL is that release's init_db file list, each file verbatim from git
+-- without its license header:
+-- nodes, messages, positions, telemetry, neighbors.
+-- The sample rows at the end must survive the boot-time schema upgrade
+-- unchanged.
+
+-- data/nodes.sql at v0.4.0
+
+PRAGMA journal_mode=WAL;
+
+CREATE TABLE IF NOT EXISTS nodes (
+  node_id            TEXT PRIMARY KEY,
+  num                INTEGER,
+  short_name         TEXT,
+  long_name          TEXT,
+  macaddr            TEXT,
+  hw_model           TEXT,
+  role               TEXT,
+  public_key         TEXT,
+  is_unmessagable    BOOLEAN,
+  is_favorite        BOOLEAN,
+  hops_away          INTEGER,
+  snr                REAL,
+  last_heard         INTEGER,
+  first_heard        INTEGER,
+  battery_level      REAL,
+  voltage            REAL,
+  channel_utilization REAL,
+  air_util_tx        REAL,
+  uptime_seconds     INTEGER,
+  position_time      INTEGER,
+  location_source    TEXT,
+  precision_bits     INTEGER,
+  latitude           REAL,
+  longitude          REAL,
+  altitude           REAL
+);
+
+CREATE INDEX IF NOT EXISTS idx_nodes_last_heard ON nodes(last_heard);
+CREATE INDEX IF NOT EXISTS idx_nodes_hw_model  ON nodes(hw_model);
+CREATE INDEX IF NOT EXISTS idx_nodes_latlon    ON nodes(latitude, longitude);
+
+-- data/messages.sql at v0.4.0
+
+CREATE TABLE IF NOT EXISTS messages (
+    id        INTEGER PRIMARY KEY,
+    rx_time   INTEGER NOT NULL,
+    rx_iso    TEXT NOT NULL,
+    from_id   TEXT,
+    to_id     TEXT,
+    channel   INTEGER,
+    portnum   TEXT,
+    text      TEXT,
+    encrypted TEXT,
+    snr       REAL,
+    rssi      INTEGER,
+    hop_limit INTEGER
+);
+
+CREATE INDEX IF NOT EXISTS idx_messages_rx_time   ON messages(rx_time);
+CREATE INDEX IF NOT EXISTS idx_messages_from_id   ON messages(from_id);
+CREATE INDEX IF NOT EXISTS idx_messages_to_id     ON messages(to_id);
+CREATE INDEX IF NOT EXISTS idx_messages_channel   ON messages(channel);
+CREATE INDEX IF NOT EXISTS idx_messages_portnum   ON messages(portnum);
+
+-- data/positions.sql at v0.4.0
+
+CREATE TABLE IF NOT EXISTS positions (
+    id             INTEGER PRIMARY KEY,
+    node_id        TEXT,
+    node_num       INTEGER,
+    rx_time        INTEGER NOT NULL,
+    rx_iso         TEXT NOT NULL,
+    position_time  INTEGER,
+    to_id          TEXT,
+    latitude       REAL,
+    longitude      REAL,
+    altitude       REAL,
+    location_source TEXT,
+    precision_bits INTEGER,
+    sats_in_view   INTEGER,
+    pdop           REAL,
+    ground_speed   REAL,
+    ground_track   REAL,
+    snr            REAL,
+    rssi           INTEGER,
+    hop_limit      INTEGER,
+    bitfield       INTEGER,
+    payload_b64    TEXT
+);
+
+CREATE INDEX IF NOT EXISTS idx_positions_rx_time ON positions(rx_time);
+CREATE INDEX IF NOT EXISTS idx_positions_node_id ON positions(node_id);
+
+-- data/telemetry.sql at v0.4.0
+
+CREATE TABLE IF NOT EXISTS telemetry (
+    id                      INTEGER PRIMARY KEY,
+    node_id                 TEXT,
+    node_num                INTEGER,
+    from_id                 TEXT,
+    to_id                   TEXT,
+    rx_time                 INTEGER NOT NULL,
+    rx_iso                  TEXT NOT NULL,
+    telemetry_time          INTEGER,
+    channel                 INTEGER,
+    portnum                 TEXT,
+    hop_limit               INTEGER,
+    snr                     REAL,
+    rssi                    INTEGER,
+    bitfield                INTEGER,
+    payload_b64             TEXT,
+    battery_level           REAL,
+    voltage                 REAL,
+    channel_utilization     REAL,
+    air_util_tx             REAL,
+    uptime_seconds          INTEGER,
+    temperature             REAL,
+    relative_humidity       REAL,
+    barometric_pressure     REAL
+);
+
+CREATE INDEX IF NOT EXISTS idx_telemetry_rx_time ON telemetry(rx_time);
+CREATE INDEX IF NOT EXISTS idx_telemetry_node_id ON telemetry(node_id);
+CREATE INDEX IF NOT EXISTS idx_telemetry_time ON telemetry(telemetry_time);
+
+-- data/neighbors.sql at v0.4.0
+
+CREATE TABLE IF NOT EXISTS neighbors (
+    node_id     TEXT NOT NULL,
+    neighbor_id TEXT NOT NULL,
+    snr         REAL,
+    rx_time     INTEGER NOT NULL,
+    PRIMARY KEY (node_id, neighbor_id),
+    FOREIGN KEY (node_id) REFERENCES nodes(node_id) ON DELETE CASCADE,
+    FOREIGN KEY (neighbor_id) REFERENCES nodes(node_id) ON DELETE CASCADE
+);
+
+CREATE INDEX IF NOT EXISTS idx_neighbors_rx_time ON neighbors(rx_time);
+CREATE INDEX IF NOT EXISTS idx_neighbors_neighbor_id ON neighbors(neighbor_id);
+
+-- Sample rows.
+
+INSERT INTO nodes(node_id, num, short_name, long_name, hw_model, role, snr, last_heard, first_heard, precision_bits, latitude, longitude, altitude)
+  VALUES ('!a1b2c3d4', 2712847316, 'A1B2', 'Legacy Alpha', 'TBEAM', 'CLIENT', 6.5, 1735689600, 1735600000, 13, 52.52, 13.405, 34.0);
+INSERT INTO nodes(node_id, num, short_name, long_name, hw_model, role, snr, last_heard, first_heard)
+  VALUES ('!0badc0de', 195936478, 'BADC', 'Legacy Bravo', 'HELTEC_V3', 'ROUTER', -3.25, 1735689000, 1735500000);
+INSERT INTO messages(id, rx_time, rx_iso, from_id, to_id, channel, portnum, text, encrypted, snr, rssi, hop_limit)
+  VALUES (1001, 1735689600, '2025-01-01T00:00:00Z', '!a1b2c3d4', '^all', 0, 'TEXT_MESSAGE_APP', 'hello from v0.4.0', NULL, -7.5, -92, 3);
+INSERT INTO messages(id, rx_time, rx_iso, from_id, to_id, channel, portnum, text, encrypted, snr, rssi, hop_limit)
+  VALUES (1002, 1735689660, '2025-01-01T00:01:00Z', '!0badc0de', '!a1b2c3d4', 1, NULL, NULL, 'q83vEjRWeJA=', 5.25, -80, 2);
+INSERT INTO positions(id, node_id, node_num, rx_time, rx_iso, position_time, to_id, latitude, longitude, altitude, location_source, precision_bits, sats_in_view, snr, rssi, hop_limit)
+  VALUES (2001, '!a1b2c3d4', 2712847316, 1735689700, '2025-01-01T00:01:40Z', 1735689690, '^all', 52.52, 13.405, 34.0, 'LOC_INTERNAL', 13, 7, 6.5, -91, 3);
+INSERT INTO telemetry(id, node_id, node_num, from_id, to_id, rx_time, rx_iso, telemetry_time, channel, portnum, hop_limit, snr, rssi, battery_level, voltage, channel_utilization, air_util_tx, uptime_seconds)
+  VALUES (3001, '!a1b2c3d4', 2712847316, '!a1b2c3d4', '^all', 1735689800, '2025-01-01T00:03:20Z', 1735689795, 0, 'TELEMETRY_APP', 3, 6.0, -90, 87.0, 4.05, 12.5, 1.25, 86400);
+INSERT INTO neighbors(node_id, neighbor_id, snr, rx_time)
+  VALUES ('!a1b2c3d4', '!0badc0de', 4.5, 1735689900);

web/spec/fixtures/schema/v0.7.0.sql

@@ -0,0 +1,292 @@
+-- Copyright © 2025-26 l5yth & contributors
+--
+-- Licensed under the Apache License, Version 2.0 (the "License");
+-- you may not use this file except in compliance with the License.
+-- You may obtain a copy of the License at
+--
+--     http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+
+-- Schema fixture (SPEC SU6): the database a fresh v0.7.0 install created.
+-- The DDL is that release's init_db file list, each file verbatim from git
+-- without its license header:
+-- nodes, messages, positions, telemetry, neighbors, instances, traces, ingestors.
+-- The sample rows at the end must survive the boot-time schema upgrade
+-- unchanged.
+
+-- data/nodes.sql at v0.7.0
+
+PRAGMA journal_mode=WAL;
+
+CREATE TABLE IF NOT EXISTS nodes (
+  node_id            TEXT PRIMARY KEY,
+  num                INTEGER,
+  short_name         TEXT,
+  long_name          TEXT,
+  macaddr            TEXT,
+  hw_model           TEXT,
+  role               TEXT,
+  public_key         TEXT,
+  is_unmessagable    BOOLEAN,
+  is_favorite        BOOLEAN,
+  hops_away          INTEGER,
+  snr                REAL,
+  last_heard         INTEGER,
+  first_heard        INTEGER,
+  battery_level      REAL,
+  voltage            REAL,
+  channel_utilization REAL,
+  air_util_tx        REAL,
+  uptime_seconds     INTEGER,
+  position_time      INTEGER,
+  location_source    TEXT,
+  precision_bits     INTEGER,
+  latitude           REAL,
+  longitude          REAL,
+  altitude           REAL,
+  lora_freq          INTEGER,
+  modem_preset       TEXT,
+  protocol           TEXT NOT NULL DEFAULT 'meshtastic',
+  synthetic          BOOLEAN NOT NULL DEFAULT 0
+);
+
+CREATE INDEX IF NOT EXISTS idx_nodes_last_heard ON nodes(last_heard);
+CREATE INDEX IF NOT EXISTS idx_nodes_hw_model  ON nodes(hw_model);
+CREATE INDEX IF NOT EXISTS idx_nodes_latlon    ON nodes(latitude, longitude);
+CREATE INDEX IF NOT EXISTS idx_nodes_long_name ON nodes(long_name);
+
+-- data/messages.sql at v0.7.0
+
+CREATE TABLE IF NOT EXISTS messages (
+    id        INTEGER PRIMARY KEY,
+    rx_time   INTEGER NOT NULL,
+    rx_iso    TEXT NOT NULL,
+    from_id   TEXT,
+    to_id     TEXT,
+    channel   INTEGER,
+    portnum   TEXT,
+    text      TEXT,
+    encrypted TEXT,
+    snr       REAL,
+    rssi      INTEGER,
+    hop_limit INTEGER,
+    lora_freq INTEGER,
+    modem_preset TEXT,
+    channel_name TEXT,
+    reply_id INTEGER,
+    emoji TEXT,
+    ingestor TEXT,
+    protocol TEXT NOT NULL DEFAULT 'meshtastic'
+);
+
+CREATE INDEX IF NOT EXISTS idx_messages_rx_time   ON messages(rx_time);
+CREATE INDEX IF NOT EXISTS idx_messages_from_id   ON messages(from_id);
+CREATE INDEX IF NOT EXISTS idx_messages_to_id     ON messages(to_id);
+CREATE INDEX IF NOT EXISTS idx_messages_channel   ON messages(channel);
+CREATE INDEX IF NOT EXISTS idx_messages_portnum   ON messages(portnum);
+CREATE INDEX IF NOT EXISTS idx_messages_reply_id  ON messages(reply_id);
+
+-- data/positions.sql at v0.7.0
+
+CREATE TABLE IF NOT EXISTS positions (
+    id             INTEGER PRIMARY KEY,
+    node_id        TEXT,
+    node_num       INTEGER,
+    rx_time        INTEGER NOT NULL,
+    rx_iso         TEXT NOT NULL,
+    position_time  INTEGER,
+    to_id          TEXT,
+    latitude       REAL,
+    longitude      REAL,
+    altitude       REAL,
+    location_source TEXT,
+    precision_bits INTEGER,
+    sats_in_view   INTEGER,
+    pdop           REAL,
+    ground_speed   REAL,
+    ground_track   REAL,
+    snr            REAL,
+    rssi           INTEGER,
+    hop_limit      INTEGER,
+    bitfield       INTEGER,
+    payload_b64    TEXT,
+    ingestor       TEXT,
+    protocol       TEXT NOT NULL DEFAULT 'meshtastic'
+);
+
+CREATE INDEX IF NOT EXISTS idx_positions_rx_time ON positions(rx_time);
+CREATE INDEX IF NOT EXISTS idx_positions_node_id ON positions(node_id);
+
+-- data/telemetry.sql at v0.7.0
+
+CREATE TABLE IF NOT EXISTS telemetry (
+    id                      INTEGER PRIMARY KEY,
+    node_id                 TEXT,
+    node_num                INTEGER,
+    from_id                 TEXT,
+    to_id                   TEXT,
+    rx_time                 INTEGER NOT NULL,
+    rx_iso                  TEXT NOT NULL,
+    telemetry_time          INTEGER,
+    channel                 INTEGER,
+    portnum                 TEXT,
+    hop_limit               INTEGER,
+    snr                     REAL,
+    rssi                    INTEGER,
+    bitfield                INTEGER,
+    payload_b64             TEXT,
+    battery_level           REAL,
+    voltage                 REAL,
+    channel_utilization     REAL,
+    air_util_tx             REAL,
+    uptime_seconds          INTEGER,
+    temperature             REAL,
+    relative_humidity       REAL,
+    barometric_pressure     REAL,
+    gas_resistance          REAL,
+    current                 REAL,
+    iaq                     INTEGER,
+    distance                REAL,
+    lux                     REAL,
+    white_lux               REAL,
+    ir_lux                  REAL,
+    uv_lux                  REAL,
+    wind_direction          INTEGER,
+    wind_speed              REAL,
+    weight                  REAL,
+    wind_gust               REAL,
+    wind_lull               REAL,
+    radiation               REAL,
+    rainfall_1h             REAL,
+    rainfall_24h            REAL,
+    soil_moisture           INTEGER,
+    soil_temperature        REAL,
+    ingestor                TEXT,
+    protocol                TEXT NOT NULL DEFAULT 'meshtastic',
+    telemetry_type          TEXT
+);
+
+CREATE INDEX IF NOT EXISTS idx_telemetry_rx_time ON telemetry(rx_time);
+CREATE INDEX IF NOT EXISTS idx_telemetry_node_id ON telemetry(node_id);
+CREATE INDEX IF NOT EXISTS idx_telemetry_time ON telemetry(telemetry_time);
+
+-- data/neighbors.sql at v0.7.0
+
+CREATE TABLE IF NOT EXISTS neighbors (
+    node_id     TEXT NOT NULL,
+    neighbor_id TEXT NOT NULL,
+    snr         REAL,
+    rx_time     INTEGER NOT NULL,
+    ingestor    TEXT,
+    protocol    TEXT NOT NULL DEFAULT 'meshtastic',
+    PRIMARY KEY (node_id, neighbor_id),
+    FOREIGN KEY (node_id) REFERENCES nodes(node_id) ON DELETE CASCADE,
+    FOREIGN KEY (neighbor_id) REFERENCES nodes(node_id) ON DELETE CASCADE
+);
+
+CREATE INDEX IF NOT EXISTS idx_neighbors_rx_time ON neighbors(rx_time);
+CREATE INDEX IF NOT EXISTS idx_neighbors_neighbor_id ON neighbors(neighbor_id);
+
+-- data/instances.sql at v0.7.0
+
+PRAGMA journal_mode=WAL;
+
+CREATE TABLE IF NOT EXISTS instances (
+  id                TEXT PRIMARY KEY,
+  domain            TEXT NOT NULL,
+  pubkey            TEXT NOT NULL,
+  name              TEXT,
+  version           TEXT,
+  channel           TEXT,
+  frequency         TEXT,
+  latitude          REAL,
+  longitude         REAL,
+  last_update_time  INTEGER,
+  is_private        BOOLEAN NOT NULL DEFAULT 0,
+  nodes_count       INTEGER,
+  meshcore_nodes_count   INTEGER,
+  meshtastic_nodes_count INTEGER,
+  contact_link      TEXT,
+  signature         TEXT
+);
+
+CREATE UNIQUE INDEX IF NOT EXISTS idx_instances_domain ON instances(domain);
+
+-- data/traces.sql at v0.7.0
+
+CREATE TABLE IF NOT EXISTS traces (
+    id          INTEGER PRIMARY KEY,
+    request_id  INTEGER,
+    src         INTEGER,
+    dest        INTEGER,
+    rx_time     INTEGER NOT NULL,
+    rx_iso      TEXT NOT NULL,
+    rssi        INTEGER,
+    snr         REAL,
+    elapsed_ms  INTEGER,
+    ingestor    TEXT,
+    protocol    TEXT NOT NULL DEFAULT 'meshtastic'
+);
+
+CREATE TABLE IF NOT EXISTS trace_hops (
+    id         INTEGER PRIMARY KEY,
+    trace_id   INTEGER NOT NULL,
+    hop_index  INTEGER NOT NULL,
+    node_id    INTEGER NOT NULL,
+    FOREIGN KEY(trace_id) REFERENCES traces(id) ON DELETE CASCADE
+);
+
+CREATE INDEX IF NOT EXISTS idx_traces_rx_time ON traces(rx_time);
+CREATE INDEX IF NOT EXISTS idx_traces_request ON traces(request_id);
+CREATE INDEX IF NOT EXISTS idx_trace_hops_trace ON trace_hops(trace_id);
+CREATE INDEX IF NOT EXISTS idx_trace_hops_node ON trace_hops(node_id);
+
+-- data/ingestors.sql at v0.7.0
+
+PRAGMA journal_mode=WAL;
+
+CREATE TABLE IF NOT EXISTS ingestors (
+  node_id        TEXT PRIMARY KEY,
+  start_time     INTEGER NOT NULL,
+  last_seen_time INTEGER NOT NULL,
+  version        TEXT,
+  lora_freq      INTEGER,
+  modem_preset   TEXT,
+  protocol       TEXT NOT NULL DEFAULT 'meshtastic'
+);
+
+CREATE INDEX IF NOT EXISTS idx_ingestors_last_seen ON ingestors(last_seen_time);
+
+-- Sample rows.
+
+INSERT INTO nodes(node_id, num, short_name, long_name, hw_model, role, snr, last_heard, first_heard, precision_bits, latitude, longitude, altitude, lora_freq, modem_preset, protocol, synthetic)
+  VALUES ('!a1b2c3d4', 2712847316, 'A1B2', 'Legacy Alpha', 'TBEAM', 'CLIENT', 6.5, 1735689600, 1735600000, 13, 52.52, 13.405, 34.0, 868, 'LongFast', 'meshtastic', 0);
+INSERT INTO nodes(node_id, num, short_name, long_name, hw_model, role, snr, last_heard, first_heard, protocol, synthetic)
+  VALUES ('!0badc0de', 195936478, 'BADC', 'Legacy Bravo', 'HELTEC_V3', 'ROUTER', -3.25, 1735689000, 1735500000, 'meshtastic', 0);
+INSERT INTO nodes(node_id, num, short_name, long_name, role, last_heard, first_heard, protocol, synthetic)
+  VALUES ('!5e7c0a11', 1585187345, '5E7C', 'Harbour Repeater', 'REPEATER', 1735689500, 1735400000, 'meshcore', 0);
+INSERT INTO messages(id, rx_time, rx_iso, from_id, to_id, channel, portnum, text, snr, rssi, hop_limit, lora_freq, modem_preset, channel_name, reply_id, emoji, ingestor, protocol)
+  VALUES (1001, 1735689600, '2025-01-01T00:00:00Z', '!a1b2c3d4', '^all', 0, 'TEXT_MESSAGE_APP', 'hello from v0.7.0', -7.5, -92, 3, 868, 'LongFast', 'LongFast', NULL, NULL, '!0000beef', 'meshtastic');
+INSERT INTO messages(id, rx_time, rx_iso, from_id, to_id, channel, portnum, text, snr, rssi, hop_limit, reply_id, emoji, ingestor, protocol)
+  VALUES (1002, 1735689660, '2025-01-01T00:01:00Z', '!0badc0de', '^all', 0, 'TEXT_MESSAGE_APP', NULL, 5.25, -80, 2, 1001, '👍', '!0000beef', 'meshtastic');
+INSERT INTO messages(id, rx_time, rx_iso, from_id, to_id, channel, text, channel_name, ingestor, protocol)
+  VALUES (1003, 1735689700, '2025-01-01T00:01:40Z', '!5e7c0a11', '^all', 0, 'Harbour Repeater: tide is high', '#public', '!0000cafe', 'meshcore');
+INSERT INTO positions(id, node_id, node_num, rx_time, rx_iso, position_time, to_id, latitude, longitude, altitude, location_source, precision_bits, sats_in_view, snr, rssi, hop_limit, ingestor, protocol)
+  VALUES (2001, '!a1b2c3d4', 2712847316, 1735689700, '2025-01-01T00:01:40Z', 1735689690, '^all', 52.52, 13.405, 34.0, 'LOC_INTERNAL', 13, 7, 6.5, -91, 3, '!0000beef', 'meshtastic');
+INSERT INTO telemetry(id, node_id, node_num, from_id, to_id, rx_time, rx_iso, telemetry_time, channel, portnum, hop_limit, snr, rssi, battery_level, voltage, channel_utilization, air_util_tx, uptime_seconds, temperature, lux, ingestor, protocol, telemetry_type)
+  VALUES (3001, '!a1b2c3d4', 2712847316, '!a1b2c3d4', '^all', 1735689800, '2025-01-01T00:03:20Z', 1735689795, 0, 'TELEMETRY_APP', 3, 6.0, -90, 87.0, 4.05, 12.5, 1.25, 86400, NULL, NULL, '!0000beef', 'meshtastic', 'device');
+INSERT INTO neighbors(node_id, neighbor_id, snr, rx_time, ingestor, protocol)
+  VALUES ('!a1b2c3d4', '!0badc0de', 4.5, 1735689900, '!0000beef', 'meshtastic');
+INSERT INTO instances(id, domain, pubkey, name, version, channel, frequency, latitude, longitude, last_update_time, is_private, nodes_count, meshcore_nodes_count, meshtastic_nodes_count, contact_link, signature)
+  VALUES ('3f6c2a', 'peer.example.test', 'PUBKEY-PEM', 'Peer Mesh', 'v0.7.0', 'LongFast', '868', 50.0, 8.0, 1735689600, 0, 12, 2, 10, NULL, 'c2lnbmF0dXJl');
+INSERT INTO traces(id, request_id, src, dest, rx_time, rx_iso, rssi, snr, elapsed_ms, ingestor, protocol)
+  VALUES (4001, 77, 2712847316, 195936478, 1735690000, '2025-01-01T00:06:40Z', -95, 3.5, 1200, '!0000beef', 'meshtastic');
+INSERT INTO trace_hops(id, trace_id, hop_index, node_id) VALUES (1, 4001, 0, 2712847316);
+INSERT INTO trace_hops(id, trace_id, hop_index, node_id) VALUES (2, 4001, 1, 195936478);
+INSERT INTO ingestors(node_id, start_time, last_seen_time, version, lora_freq, modem_preset, protocol)
+  VALUES ('!0000beef', 1735600000, 1735690000, 'v0.7.0', 868, 'LongFast', 'meshtastic');

web/spec/schema_upgrades_spec.rb

@@ -0,0 +1,624 @@
+# Copyright © 2025-26 l5yth & contributors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# frozen_string_literal: true
+
+require "spec_helper"
+require "sqlite3"
+require "json"
+require "fileutils"
+
+# Boot-time schema upgrades on release databases: the nodes.num index, a
+# failing step that stops the boot, the parity check and the indexes the
+# upgrade recreates, the steps upgraded databases lacked, and boots racing
+# on one database (SPEC SU1-SU6).
+RSpec.describe "schema upgrades" do
+  # The database methods the configure block calls, outside any app.
+  let(:boot) do
+    Class.new do
+      extend PotatoMesh::App::Database
+
+      # No upgrade step logs since SU2; an older tree logged a swallowed
+      # failure here, which these examples judge by the schema it left.
+      #
+      # @return [void]
+      def self.warn_log(*, **); end
+    end
+  end
+
+  let(:fixture_dir) { File.expand_path("fixtures/schema", __dir__) }
+  let(:plan_on_num) { "SEARCH nodes USING INDEX idx_nodes_num (num=?)" }
+
+  around do |example|
+    Dir.mktmpdir("schema-upgrades-spec-") do |dir|
+      RSpec::Mocks.with_temporary_scope do
+        allow(PotatoMesh::Config).to receive(:db_path).and_return(File.join(dir, "mesh.db"))
+        example.run
+      end
+    end
+  end
+
+  # Build the example's database from a release fixture.
+  #
+  # @param tag [String] release tag naming +spec/fixtures/schema/<tag>.sql+.
+  # @return [void]
+  def load_fixture(tag)
+    db = SQLite3::Database.new(PotatoMesh::Config.db_path)
+    db.execute_batch(File.read(File.join(fixture_dir, "#{tag}.sql")))
+  ensure
+    db&.close
+  end
+
+  # Run +block+ on a fresh handle on the example's database.
+  #
+  # @yieldparam db [SQLite3::Database] open handle, closed afterwards.
+  # @return [Object] the block's result.
+  def with_db
+    db = SQLite3::Database.new(PotatoMesh::Config.db_path)
+    db.busy_timeout = 5000
+    yield db
+  ensure
+    db&.close
+  end
+
+  # @param table [String] table name.
+  # @return [Array<String>] the table's column names.
+  def columns(table)
+    with_db { |db| db.execute("PRAGMA table_info(#{table})").map { |row| row[1] } }
+  end
+
+  # @return [Array<String>] every index name in the example's database.
+  def indexes
+    with_db { |db| db.execute("SELECT name FROM sqlite_master WHERE type='index'").flatten }
+  end
+
+  # @return [Integer] the database's +PRAGMA user_version+.
+  def user_version
+    with_db { |db| db.get_first_value("PRAGMA user_version").to_i }
+  end
+
+  # The configure block's first two steps: upgrade, then create the fresh
+  # schema when a required table is still absent.
+  #
+  # @return [void]
+  def upgrade!
+    boot.ensure_schema_upgrades
+    boot.init_db unless boot.db_schema_present?
+  end
+
+  # What +data/*.sql+ defines and the example's database lacks, worked out
+  # here without the app's own check: +<table>.<column>+, +table <name>+ and
+  # +index <name>+ entries.
+  #
+  # @return [Array<String>] the gaps, sorted.
+  def schema_gaps
+    fresh = SQLite3::Database.new(":memory:")
+    Dir[File.expand_path("../../data/*.sql", __dir__)].each { |file| fresh.execute_batch(File.read(file)) }
+    tables = fresh.execute("SELECT name FROM sqlite_master WHERE type='table' AND name NOT LIKE 'sqlite_%'").flatten
+    gaps = tables.flat_map do |table|
+      have = columns(table)
+      next ["table #{table}"] if have.empty?
+
+      (fresh.execute("PRAGMA table_info(#{table})").map { |row| row[1] } - have).map { |column| "#{table}.#{column}" }
+    end
+    wanted = fresh.execute("SELECT name FROM sqlite_master WHERE type='index' AND name NOT LIKE 'sqlite_%'").flatten
+    (gaps + (wanted - indexes).map { |name| "index #{name}" }).sort
+  ensure
+    fresh&.close
+  end
+
+  # Every row of each table, holding only the given columns.
+  #
+  # @param columns_by_table [Hash{String => Array<String>}] columns to read.
+  # @return [Hash{String => Array<Hash>}] rows in rowid order, per table.
+  def snapshot(columns_by_table)
+    with_db do |db|
+      db.results_as_hash = true
+      columns_by_table.to_h do |table, cols|
+        [table, db.execute("SELECT #{cols.map { |col| %("#{col}") }.join(", ")} FROM #{table} ORDER BY rowid")]
+      end
+    end
+  end
+
+  # Hold the write lock in a second process, as a boot racing this one does:
+  # BEGIN IMMEDIATE, run +sql+, report the lock held, wait +hold+ seconds,
+  # then COMMIT. Returns once the lock is held.
+  #
+  # @param sql [String] statement the racing boot applies.
+  # @param hold [Float] seconds the lock is held after +sql+.
+  # @return [Integer] the child's pid, for +expect_clean_exit+.
+  def racing_boot(sql, hold:)
+    reader, writer = IO.pipe
+    pid = fork do
+      reader.close
+      db = SQLite3::Database.new(PotatoMesh::Config.db_path)
+      db.execute("BEGIN IMMEDIATE")
+      db.execute(sql)
+      writer.puts("locked")
+      writer.flush
+      sleep hold
+      db.execute("COMMIT")
+      db.close
+      exit!(0)
+    rescue StandardError
+      exit!(1)
+    end
+    writer.close
+    expect(reader.gets).to eq("locked\n")
+    reader.close
+    pid
+  end
+
+  # @param pid [Integer] a +racing_boot+ child.
+  # @return [void]
+  def expect_clean_exit(pid)
+    _, status = Process.wait2(pid)
+    expect(status.exitstatus).to eq(0)
+  end
+
+  # Make the upgrade's connection raise +error+ when it prepares the first
+  # statement that starts with +prefix+, where SQLite reports a failure.
+  # The module goes on before +ensure_schema_upgrades+ adds its own, so the
+  # step is recorded first.
+  #
+  # @param prefix [String] start of the statement that fails.
+  # @param error [SQLite3::Exception] exception to raise.
+  # @return [void]
+  def fail_step(prefix, error)
+    failing = Module.new do
+      define_method(:prepare) do |sql, &block|
+        raise error if sql.lstrip.start_with?(prefix)
+
+        super(sql, &block)
+      end
+    end
+    allow(boot).to receive(:open_database).and_wrap_original do |original, *args, **kwargs|
+      original.call(*args, **kwargs).extend(failing)
+    end
+  end
+
+  # Point the fresh schema at a copy of +data/*.sql+ in which +file+ ends
+  # with +sql+, as a schema change made without its upgrade step would.
+  #
+  # @param file [String] schema file name, such as +nodes.sql+.
+  # @param sql [String] statements appended to the copy.
+  # @return [void]
+  def schema_files_with(file, sql)
+    dir = File.join(File.dirname(PotatoMesh::Config.db_path), "schema-files")
+    FileUtils.mkdir_p(dir)
+    FileUtils.cp(Dir[File.expand_path("../../data/*.sql", __dir__)], dir)
+    File.write(File.join(dir, file), "\n#{sql}\n", mode: "a")
+    stub_const("PotatoMesh::App::Database::SCHEMA_DIRECTORY", dir)
+  end
+
+  # Make +db+ record each statement that reads +nodes+ through +execute+
+  # or +get_first_value+, with its bind values, into +log+.
+  #
+  # @param db [SQLite3::Database] handle to record on.
+  # @param log [Array<Array(String, Array)>] receives +[sql, binds]+ pairs.
+  # @return [SQLite3::Database] +db+.
+  def record_node_reads(db, log)
+    %i[execute get_first_value].each do |name|
+      original = db.method(name)
+      db.define_singleton_method(name) do |sql, *binds, &blk|
+        log << [sql, binds.flatten] if sql.include?("FROM nodes")
+        original.call(sql, *binds, &blk)
+      end
+    end
+    db
+  end
+
+  # @param db [SQLite3::Database] open handle.
+  # @param sql [String] statement text.
+  # @param binds [Array] bind values.
+  # @return [Array<String>] the EXPLAIN QUERY PLAN detail lines.
+  def plan(db, sql, binds)
+    db.execute("EXPLAIN QUERY PLAN #{sql}", binds).map { |row| row[3] }
+  end
+
+  describe "the nodes.num index (SPEC SU1)" do
+    # The three numeric lookups, as their call sites issue them, run on the
+    # example's database, which holds the fixture's two Meshtastic nodes.
+    #
+    # @return [Array<Array(String, Array)>] the num lookups, with binds.
+    def num_lookups
+      reads = []
+      handle = record_node_reads(SQLite3::Database.new(PotatoMesh::Config.db_path), reads)
+      app = PotatoMesh::Application
+      expect(app.node_lookup_clause("!a1b2c3d4", string_columns: ["node_id"], numeric_columns: ["num"], db: handle)).not_to be_nil
+      expect(app.normalize_node_id(handle, "2712847316")).to eq("!a1b2c3d4")
+      expect(app.batch_resolve_node_ids(handle, ["2712847316", "195936478"])).to include("195936478" => "!0badc0de")
+      reads.select { |sql, _binds| sql.match?(/WHERE num (=|IN)/) }
+    ensure
+      handle&.close
+    end
+
+    it "searches idx_nodes_num for every numeric lookup on a fresh database" do
+      boot.init_db
+      with_db { |db| db.execute("INSERT INTO nodes(node_id, num) VALUES ('!a1b2c3d4', 2712847316), ('!0badc0de', 195936478)") }
+
+      lookups = num_lookups
+
+      expect(lookups.length).to eq(3)
+      with_db { |db| lookups.each { |sql, binds| expect(plan(db, sql, binds)).to eq([plan_on_num]) } }
+    end
+
+    it "searches idx_nodes_num for every numeric lookup on an upgraded v0.7.0 database" do
+      load_fixture("v0.7.0")
+      upgrade!
+
+      lookups = num_lookups
+
+      expect(lookups.length).to eq(3)
+      with_db { |db| lookups.each { |sql, binds| expect(plan(db, sql, binds)).to eq([plan_on_num]) } }
+    end
+
+    it "leaves the plans of the numeric opt-out subqueries unchanged" do
+      boot.init_db
+      now = Time.now.to_i
+      with_db do |db|
+        db.execute("INSERT INTO nodes(node_id, num) VALUES ('!a1b2c3d4', 2712847316), ('!0badc0de', 195936478)")
+        db.execute("INSERT INTO traces(id, src, dest, rx_time, rx_iso) VALUES (1, 2712847316, 195936478, ?, 'now')", [now])
+        db.execute("INSERT INTO trace_hops(trace_id, hop_index, node_id) VALUES (1, 0, 195936478)")
+      end
+      app = PotatoMesh::Application
+      statements = []
+      allow(app).to receive(:open_database).and_wrap_original do |original, *args, **kwargs|
+        record_node_reads(original.call(*args, **kwargs), statements)
+      end
+      cutoffs = %w[hour day week month].to_h { |window| [window, now - 86_400] }
+
+      # Public and private reads: the opt-out filters, then the
+      # CLIENT_HIDDEN filters beside them.
+      [false, true].each do |private_mode|
+        allow(app).to receive(:private_mode?).and_return(private_mode)
+        expect(app.query_traces(10, node_ref: "!a1b2c3d4").length).to eq(1)
+        with_db do |db|
+          db.results_as_hash = true
+          expect(app.telemetry_activity_counts(record_node_reads(db, statements), cutoffs)).to include("total")
+        end
+      end
+
+      subqueries = statements.select { |sql, _binds| sql.include?("SELECT num FROM nodes") }
+      expect(subqueries.length).to eq(6)
+      expect(subqueries.map(&:first).join).to include("role = 'CLIENT_HIDDEN'").and include("LIKE")
+      with_db do |db|
+        indexed = subqueries.map { |sql, binds| plan(db, sql, binds) }
+        db.execute("DROP INDEX idx_nodes_num")
+        expect(subqueries.map { |sql, binds| plan(db, sql, binds) }).to eq(indexed)
+      end
+    end
+
+    it "ships a reference migration that creates the index" do
+      load_fixture("v0.7.0")
+      migration = File.expand_path("../../data/migrations/20261009_add_nodes_num_index.sql", __dir__)
+
+      with_db { |db| db.execute_batch(File.read(migration)) }
+
+      expect(indexes).to include("idx_nodes_num")
+    end
+  end
+
+  describe "a failing upgrade step (SPEC SU2)" do
+    it "fails the boot naming the step and leaves the schema as it was" do
+      load_fixture("v0.7.0")
+      fail_step("ALTER TABLE telemetry ADD COLUMN ch1_voltage", SQLite3::IOException.new("disk I/O error"))
+
+      expect { boot.ensure_schema_upgrades }.to raise_error(
+        StandardError,
+        "Schema upgrade step failed: ALTER TABLE telemetry ADD COLUMN ch1_voltage REAL " \
+        "(SQLite3::IOException: disk I/O error)",
+      ) { |error| expect(error.class.name).to eq("PotatoMesh::App::SchemaUpgradeError") }
+      # The steps before it rolled back with it.
+      expect(columns("nodes")).not_to include("rssi", "identity_hash")
+      expect(columns("messages")).not_to include("scope")
+      expect(user_version).to eq(0)
+    end
+
+    it "names the database open when the connection cannot be made" do
+      allow(boot).to receive(:open_database).and_raise(SQLite3::CantOpenException, "unable to open database file")
+
+      expect { boot.ensure_schema_upgrades }.to raise_error(
+        PotatoMesh::App::SchemaUpgradeError,
+        "Schema upgrade step failed: opening the database (SQLite3::CantOpenException: unable to open database file)",
+      )
+    end
+
+    it "names the BEGIN when a racing boot keeps the write lock past every retry" do
+      load_fixture("v0.7.0")
+      allow(PotatoMesh::Config).to receive(:db_busy_timeout_ms).and_return(20)
+      allow(PotatoMesh::Config).to receive(:db_busy_max_retries).and_return(1)
+      pid = racing_boot("SELECT 1", hold: 1.0)
+
+      expect { boot.ensure_schema_upgrades }.to raise_error(
+        PotatoMesh::App::SchemaUpgradeError,
+        "Schema upgrade step failed: BEGIN IMMEDIATE (SQLite3::BusyException: database is locked)",
+      )
+      expect_clean_exit(pid)
+    end
+  end
+
+  describe "the schema parity check (SPEC SU3)" do
+    it "passes on a fresh database" do
+      upgrade!
+
+      expect { boot.ensure_schema_parity! }.not_to raise_error
+    end
+
+    it "fails naming every table, column and index the database lacks" do
+      upgrade!
+      with_db do |db|
+        db.execute("DROP TABLE waypoints")
+        db.execute("ALTER TABLE messages DROP COLUMN scope")
+        db.execute("DROP INDEX idx_nodes_num")
+      end
+
+      expect { boot.ensure_schema_parity! }.to raise_error(PotatoMesh::App::SchemaUpgradeError) do |error|
+        expect(error.message).to eq(
+          "Database schema lacks what data/*.sql defines after the upgrade: messages.scope, table waypoints, " \
+          "index idx_nodes_num, index idx_waypoints_rx_time, index idx_waypoints_node_id",
+        )
+      end
+    end
+
+    it "allows objects the fresh schema does not define and matches names in any case" do
+      upgrade!
+      with_db do |db|
+        db.execute("ALTER TABLE messages ADD COLUMN raw_json TEXT")
+        db.execute("CREATE INDEX idx_messages_raw_json ON messages(raw_json)")
+        db.execute("ALTER TABLE messages RENAME COLUMN encrypted TO ENCRYPTED")
+      end
+
+      expect { boot.ensure_schema_parity! }.not_to raise_error
+    end
+
+    it "recreates on the next boot every index the operator dropped" do
+      load_fixture("v0.7.0")
+      upgrade!
+      dropped = %w[idx_messages_rx_time idx_nodes_num idx_instances_domain]
+      with_db { |db| dropped.each { |name| db.execute("DROP INDEX #{name}") } }
+
+      upgrade!
+
+      expect { boot.ensure_schema_parity! }.not_to raise_error
+      expect(indexes).to include(*dropped)
+    end
+
+    it "leaves the indexes of a table lacking a column the upgrade cannot add to the check, which names them" do
+      load_fixture("v0.7.0")
+      # A schema change made without its upgrade step: a column and an index.
+      schema_files_with(
+        "nodes.sql",
+        "ALTER TABLE nodes ADD COLUMN shoe_size INTEGER;\nCREATE INDEX IF NOT EXISTS idx_nodes_shoe_size ON nodes(shoe_size);",
+      )
+
+      expect { upgrade! }.not_to raise_error
+      expect { boot.ensure_schema_parity! }.to raise_error(
+        PotatoMesh::App::SchemaUpgradeError,
+        "Database schema lacks what data/*.sql defines after the upgrade: nodes.shoe_size, " \
+        "index idx_nodes_identity_hash, index idx_nodes_num, index idx_nodes_shoe_size",
+      )
+    end
+
+    it "skips the indexes of a table the upgrade did not create" do
+      upgrade!
+      with_db do |db|
+        db.execute("DROP TABLE waypoints")
+        boot.create_missing_schema_indexes(db)
+      end
+
+      expect(indexes).not_to include("idx_waypoints_rx_time", "idx_waypoints_node_id")
+    end
+
+    it "lets a failed open stop the boot as it is" do
+      allow(boot).to receive(:open_database).and_raise(SQLite3::CantOpenException, "unable to open database file")
+
+      expect { boot.ensure_schema_parity! }.to raise_error(SQLite3::CantOpenException, "unable to open database file")
+    end
+
+    it "reads the fresh schema from every data/*.sql file" do
+      shape = boot.fresh_schema_shape
+
+      expect(shape[:tables].keys).to contain_exactly(
+        "destinations", "ingestor_activity", "ingestors", "instances", "messages", "neighbors",
+        "nodes", "positions", "telemetry", "traces", "trace_hops", "waypoints",
+      )
+      expect(shape[:tables]["nodes"]).to include("num", "identity_hash")
+      expect(shape[:indexes].keys).to include("idx_nodes_num", "idx_nodes_identity_hash", "idx_messages_meshcore_text")
+      expect(shape[:indexes].keys.grep(/\Asqlite_/)).to be_empty
+      expect(shape[:indexes]["idx_nodes_num"]).to eq(table: "nodes", sql: "CREATE INDEX idx_nodes_num       ON nodes(num)")
+    end
+  end
+
+  describe "steps upgraded databases lacked (SPEC SU4)" do
+    let(:app) { Sinatra::Application }
+
+    around do |example|
+      token = ENV["API_TOKEN"]
+      ENV["API_TOKEN"] = "schema-spec-token"
+      example.run
+    ensure
+      ENV["API_TOKEN"] = token
+    end
+
+    it "adds messages.encrypted to a v0.2.0 database, so a message POST succeeds" do
+      load_fixture("v0.2.0")
+
+      upgrade!
+
+      expect(columns("messages")).to include("encrypted")
+      post "/api/messages",
+           { "id" => 1003, "rx_time" => Time.now.to_i, "from_id" => "!a1b2c3d4", "to_id" => "^all", "channel" => 0,
+             "portnum" => "TEXT_MESSAGE_APP", "text" => "after the upgrade", "ingestor" => "!0000beef",
+             "protocol" => "meshtastic" }.to_json,
+           { "CONTENT_TYPE" => "application/json", "HTTP_AUTHORIZATION" => "Bearer schema-spec-token" }
+      expect(last_response.status).to eq(201)
+      expect(with_db { |db| db.get_first_value("SELECT text FROM messages WHERE id = 1003") }).to eq("after the upgrade")
+    end
+
+    it "creates idx_nodes_identity_hash on an upgraded v0.7.0 database" do
+      load_fixture("v0.7.0")
+
+      upgrade!
+
+      expect(indexes).to include("idx_nodes_identity_hash")
+    end
+
+    it "matches column names in any case, so a renamed column is not added again" do
+      load_fixture("v0.7.0")
+      upgrade!
+      with_db do |db|
+        db.execute("ALTER TABLE nodes RENAME COLUMN rssi TO RSSI")
+        db.execute("ALTER TABLE messages RENAME COLUMN encrypted TO ENCRYPTED")
+        db.execute("ALTER TABLE telemetry RENAME COLUMN ch1_voltage TO CH1_VOLTAGE")
+      end
+
+      expect { upgrade! }.not_to raise_error
+      expect { boot.ensure_schema_parity! }.not_to raise_error
+      expect(columns("nodes")).to include("RSSI")
+      expect(columns("messages")).to include("ENCRYPTED")
+      expect(columns("telemetry")).to include("CH1_VOLTAGE")
+    end
+
+    it "builds the MX6 index and runs the #756 purge on the boot that adds messages.protocol" do
+      load_fixture("v0.4.0")
+      expect(columns("messages")).not_to include("protocol")
+
+      boot.ensure_schema_upgrades
+
+      expect(indexes).to include("idx_messages_meshcore_text")
+      expect(user_version).to eq(PotatoMesh::App::Database::MESHCORE_CONTENT_DEDUP_BACKFILL_VERSION)
+    end
+  end
+
+  describe "boots racing on one database (SPEC SU5)" do
+    it "waits for a racing boot's upgrade, then adds nothing twice" do
+      load_fixture("v0.7.0")
+      # The racing boot holds the lock with the first step this boot needs.
+      pid = racing_boot("ALTER TABLE nodes ADD COLUMN rssi INTEGER", hold: 0.4)
+
+      expect { boot.ensure_schema_upgrades }.not_to raise_error
+      expect_clean_exit(pid)
+      expect(columns("nodes").count("rssi")).to eq(1)
+      expect(columns("telemetry")).to include("ch1_voltage", "one_wire_temperature")
+      expect(indexes).to include("idx_nodes_num", "idx_nodes_identity_hash")
+    end
+
+    it "retries the write lock while a racing boot holds it past the busy timeout" do
+      load_fixture("v0.7.0")
+      allow(PotatoMesh::Config).to receive(:db_busy_timeout_ms).and_return(50)
+      pid = racing_boot("ALTER TABLE nodes ADD COLUMN rssi INTEGER", hold: 0.3)
+
+      expect { boot.ensure_schema_upgrades }.not_to raise_error
+      expect_clean_exit(pid)
+      expect(columns("telemetry")).to include("ch1_voltage")
+    end
+
+    it "retries the WAL switch while a racing boot holds a new database's write lock" do
+      # A new file is in rollback-journal mode. Switching it to WAL takes the
+      # write lock, and SQLite reports BUSY at once, without the busy timeout.
+      pid = racing_boot("SELECT 1", hold: 0.2)
+
+      expect { boot.ensure_schema_upgrades }.not_to raise_error
+      expect_clean_exit(pid)
+      expect(with_db { |db| db.get_first_value("PRAGMA journal_mode") }).to eq("wal")
+      expect(with_db { |db| boot.missing_schema_objects(db) }).to be_empty
+    end
+
+    it "creates a new database's whole schema in its one transaction" do
+      boot.ensure_schema_upgrades
+
+      expect(boot.db_schema_present?).to be(true)
+      expect(with_db { |db| boot.missing_schema_objects(db) }).to be_empty
+      # The upgrade blocks still skip a new database's tables, as before.
+      expect(user_version).to eq(0)
+    end
+  end
+
+  describe "the upgrade connection (SPEC SU2, SU5)" do
+    # @return [SQLite3::Database] a handle with the upgrade behaviour.
+    def upgrade_connection
+      SQLite3::Database.new(PotatoMesh::Config.db_path).extend(PotatoMesh::App::Database::UpgradeConnection)
+    end
+
+    it "labels the step by its first statement, without comments, on one line" do
+      db = upgrade_connection
+      db.execute_batch("-- a comment\nCREATE TABLE t(\n  a INTEGER  -- inline\n);\nCREATE TABLE u(b);")
+      expect(db.upgrade_step_label).to eq("CREATE TABLE u(b)")
+
+      db.prepare("-- a comment\nCREATE TABLE IF NOT EXISTS t(\n  a INTEGER  -- inline\n);").close
+      expect(db.upgrade_step_label).to eq("CREATE TABLE IF NOT EXISTS t( a INTEGER )")
+      db.prepare("-- header\n-- more\nSELECT a,\n       1\n  FROM t;\nSELECT 2").close
+      expect(db.upgrade_step_label).to eq("SELECT a, 1 FROM t")
+      db.prepare("SELECT '#{"x" * 300}'").close
+      expect(db.upgrade_step_label).to eq("SELECT '#{"x" * 189}...")
+      expect(db.upgrade_step_label.length).to eq(PotatoMesh::App::Database::UPGRADE_STEP_LABEL_LIMIT)
+    ensure
+      db&.close
+    end
+
+    it "has an empty label before the first statement" do
+      db = SQLite3::Database.new(":memory:").extend(PotatoMesh::App::Database::UpgradeConnection)
+
+      expect(db.upgrade_step_label).to eq("")
+    ensure
+      db&.close
+    end
+
+    it "runs a nested transaction block inside the upgrade's transaction" do
+      db = upgrade_connection
+      db.execute("CREATE TABLE t(a)")
+      db.execute("BEGIN IMMEDIATE")
+
+      expect(db.transaction { |handle| handle.execute("INSERT INTO t VALUES (1)") && :joined }).to eq(:joined)
+      expect(db.transaction_active?).to be(true)
+      db.execute("ROLLBACK")
+      expect(db.get_first_value("SELECT COUNT(*) FROM t")).to eq(0)
+    ensure
+      db&.close
+    end
+
+    it "keeps the stock transaction outside a transaction" do
+      db = upgrade_connection
+      db.execute("CREATE TABLE t(a)")
+
+      db.transaction { |handle| handle.execute("INSERT INTO t VALUES (1)") }
+      expect(db.transaction_active?).to be(false)
+      expect(db.transaction).to be(true)
+      expect(db.transaction_active?).to be(true)
+      db.rollback
+      expect(db.get_first_value("SELECT COUNT(*) FROM t")).to eq(1)
+    ensure
+      db&.close
+    end
+  end
+
+  describe "release databases (SPEC SU6)" do
+    %w[v0.2.0 v0.4.0 v0.7.0].each do |tag|
+      it "upgrades a #{tag} database to parity and keeps its rows" do
+        load_fixture(tag)
+        tables = with_db { |db| db.execute("SELECT name FROM sqlite_master WHERE type='table'").flatten }
+        legacy_columns = tables.to_h { |table| [table, columns(table)] }
+        before = snapshot(legacy_columns)
+        expect(before.values.sum(&:length)).to be >= 4
+
+        upgrade!
+
+        expect(schema_gaps).to eq([])
+        expect { boot.ensure_schema_parity! }.not_to raise_error
+        expect(snapshot(legacy_columns)).to eq(before)
+      end
+    end
+  end
+end