From 03472b151ffb8b626a4bc3dbb8513560077f22f6 Mon Sep 17 00:00:00 2001 From: Afri <220195275+l5yth@users.noreply.github.com> Date: Sun, 11 Oct 2026 13:36:24 +0200 Subject: [PATCH] web: schema upgrades fail loudly, reach parity, index nodes.num (#1036) --- CHANGELOG.md | 5 + .../20261009_add_nodes_num_index.sql | 25 + data/nodes.sql | 3 + web/lib/potato_mesh/application.rb | 1 + web/lib/potato_mesh/application/database.rb | 78 ++- .../application/database/schema_guard.rb | 210 ++++++ web/spec/database_spec.rb | 20 +- web/spec/fixtures/schema/v0.2.0.sql | 89 +++ web/spec/fixtures/schema/v0.4.0.sql | 172 +++++ web/spec/fixtures/schema/v0.7.0.sql | 292 ++++++++ web/spec/schema_upgrades_spec.rb | 624 ++++++++++++++++++ 11 files changed, 1497 insertions(+), 22 deletions(-) create mode 100644 data/migrations/20261009_add_nodes_num_index.sql create mode 100644 web/lib/potato_mesh/application/database/schema_guard.rb create mode 100644 web/spec/fixtures/schema/v0.2.0.sql create mode 100644 web/spec/fixtures/schema/v0.4.0.sql create mode 100644 web/spec/fixtures/schema/v0.7.0.sql create mode 100644 web/spec/schema_upgrades_spec.rb diff --git a/CHANGELOG.md b/CHANGELOG.md index 507854d1..5d7e81d7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -189,6 +189,11 @@ * Web: a federation instance row is replaced in one transaction: a failed write keeps the stored row, and two writes for one domain at once no longer fail on the unique domain index (SPEC FK1) * Web: a database error while a crawl stores one peer is logged and the crawl goes on with the next peer (SPEC FK2) * Web: a keyfile that cannot be parsed stops the boot with an error naming the file and is left as it is, instead of being replaced by a new key and instance id; a legacy keyfile that cannot be parsed is not copied, and the error names it; restore the named file from a backup or delete it to start with a new key (SPEC FK3) +* Web: looking up a node by its number no longer scans the nodes table (SPEC SU1) +* Web: a database upgrade step that fails stops the boot with an error naming the step and leaves the database as it was, instead of logging a warning and serving a half-upgraded database (SPEC SU2) +* Web: the boot recreates every index from `data/*.sql` that the database lacks, and stops, naming what is missing, when a table or column is still missing after the upgrade (SPEC SU3) +* Web: a database created by v0.2.0 gains `messages.encrypted`; a database created before v0.6.0 gets the MeshCore text index and the duplicate purge on its first boot, not its second; the upgrade matches column names in any case (SPEC SU4) +* Web: a process that starts while another one upgrades the same database no longer fails on a duplicate column or a locked database (SPEC SU5) * Data: the Reticulum ingestor posts interface names without peer addresses: a TCP, Backbone, UDP or AutoInterface peer name keeps its class and the operator-given name and drops the address and port, an outbound I2P peer drops its I2P destination, a Weave peer posts its class, and RNode names are unchanged (SPEC RI1) * Web: interface names are stored without peer addresses whatever the ingestor posts, and stored names that carry one lose it at the next boot, so `GET /api/destinations` serves none, with or without `PRIVATE=1` (SPEC RI2/RI3) * Web: `/metrics` request metrics label `path` with the route that answered, such as `GET /api/nodes/:id`, or `static`, `metrics` or `unmatched`, instead of the request path; 8000 requests to distinct missing paths no longer add 8000 series (SPEC PG1) diff --git a/data/migrations/20261009_add_nodes_num_index.sql b/data/migrations/20261009_add_nodes_num_index.sql new file mode 100644 index 00000000..50f83aed --- /dev/null +++ b/data/migrations/20261009_add_nodes_num_index.sql @@ -0,0 +1,25 @@ +-- Copyright © 2025-26 l5yth & contributors +-- +-- Licensed under the Apache License, Version 2.0 (the "License"); +-- you may not use this file except in compliance with the License. +-- You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. + +-- Index the numeric node id (SPEC SU1): additive index only. +-- +-- nodes.num: the Meshtastic node number. A per-id read resolves a numeric +-- reference through it, ingest resolves a decimal sender or recipient +-- through it, and the batch resolver looks up many at once; without the +-- index each lookup scans every node row. +-- +-- The web app applies this conditionally at boot (database.rb); this file is +-- the standalone mirror for CLI/manual migration of older installations. + +CREATE INDEX IF NOT EXISTS idx_nodes_num ON nodes(num); diff --git a/data/nodes.sql b/data/nodes.sql index 7b196224..79fb6e1e 100644 --- a/data/nodes.sql +++ b/data/nodes.sql @@ -62,3 +62,6 @@ CREATE INDEX IF NOT EXISTS idx_nodes_last_heard ON nodes(last_heard); CREATE INDEX IF NOT EXISTS idx_nodes_hw_model ON nodes(hw_model); CREATE INDEX IF NOT EXISTS idx_nodes_latlon ON nodes(latitude, longitude); CREATE INDEX IF NOT EXISTS idx_nodes_long_name ON nodes(long_name); +-- Numeric node references (per-id reads, decimal ids on ingest) search this +-- index (SPEC SU1). ensure_schema_upgrades creates it on upgraded databases. +CREATE INDEX IF NOT EXISTS idx_nodes_num ON nodes(num); diff --git a/web/lib/potato_mesh/application.rb b/web/lib/potato_mesh/application.rb index 8f9d137b..096d4a12 100644 --- a/web/lib/potato_mesh/application.rb +++ b/web/lib/potato_mesh/application.rb @@ -260,6 +260,7 @@ module PotatoMesh ensure_schema_upgrades scrub_destination_interfaces # SPEC RI3: needs the destinations table, so after the upgrades init_db unless db_schema_present? + ensure_schema_parity! log_instance_domain_resolution log_instance_public_key diff --git a/web/lib/potato_mesh/application/database.rb b/web/lib/potato_mesh/application/database.rb index 904f51ef..7b3ede82 100644 --- a/web/lib/potato_mesh/application/database.rb +++ b/web/lib/potato_mesh/application/database.rb @@ -14,6 +14,8 @@ # frozen_string_literal: true +require_relative "database/schema_guard" + module PotatoMesh module App module Database @@ -125,16 +127,36 @@ module PotatoMesh # Apply any schema migrations required for older installations. # + # Every step runs in one IMMEDIATE transaction (SPEC SU2, SU5). A boot + # that races another boot on the same database waits for the write + # lock, then reads the upgraded schema and changes nothing. A failing + # step rolls the whole upgrade back and fails the boot with an error + # naming the step; the +dest_hash+ DROP is the one step allowed to + # fail. A new database gets its full schema here too, in the same + # transaction. Column guards match names in any case, as SQLite does + # (SPEC SU4), and every index +data/*.sql+ defines that the database + # lacks is created last (SPEC SU3). + # # @return [void] + # @raise [SchemaUpgradeError] when a step fails. def ensure_schema_upgrades FileUtils.mkdir_p(File.dirname(PotatoMesh::Config.db_path)) db = open_database + db.extend(UpgradeConnection) + # Some data/*.sql files set WAL, and journal_mode cannot change + # inside a transaction, so the switch comes before the one they run + # in. A new file is still in rollback-journal mode: there the switch + # takes the write lock, and SQLite reports BUSY at once, without the + # busy timeout, while a racing boot holds it. So it retries like the + # BEGIN (SPEC SU5). + with_busy_retry { db.execute("PRAGMA journal_mode=WAL") } + with_busy_retry { db.execute("BEGIN IMMEDIATE") } node_table_exists = db.get_first_value( "SELECT COUNT(*) FROM sqlite_master WHERE type='table' AND name='nodes'", ).to_i > 0 if node_table_exists - node_columns = db.execute("PRAGMA table_info(nodes)").map { |row| row[1] } + node_columns = schema_column_names(db, "nodes") unless node_columns.include?("precision_bits") db.execute("ALTER TABLE nodes ADD COLUMN precision_bits INTEGER") node_columns << "precision_bits" @@ -306,9 +328,15 @@ module PotatoMesh message_table_exists = db.get_first_value( "SELECT COUNT(*) FROM sqlite_master WHERE type='table' AND name='messages'", ).to_i > 0 - message_columns = message_table_exists ? db.execute("PRAGMA table_info(messages)").map { |row| row[1] } : [] + message_columns = message_table_exists ? schema_column_names(db, "messages") : [] if message_table_exists + # v0.2.0 created messages without this column, and no later boot + # added it, so every message insert there failed (SPEC SU4). + unless message_columns.include?("encrypted") + db.execute("ALTER TABLE messages ADD COLUMN encrypted TEXT") + end + unless message_columns.include?("lora_freq") db.execute("ALTER TABLE messages ADD COLUMN lora_freq INTEGER") end @@ -342,6 +370,10 @@ module PotatoMesh unless message_columns.include?("protocol") db.execute("ALTER TABLE messages ADD COLUMN protocol TEXT NOT NULL DEFAULT 'meshtastic'") db.execute("UPDATE messages SET protocol = 'meshtastic' WHERE protocol IS NULL OR TRIM(protocol) = ''") + # The MX6 index and the #756 purge below key on protocol; keep + # the list current so they run on this boot, not the next one + # (SPEC SU4). + message_columns << "protocol" end # RF metrics (SPEC RF1/RF2/RF6): hops actually travelled (distinct @@ -500,7 +532,7 @@ module PotatoMesh db.execute_batch(File.read(sql_file)) end - instance_columns = db.execute("PRAGMA table_info(instances)").map { |row| row[1] } + instance_columns = schema_column_names(db, "instances") unless instance_columns.include?("contact_link") db.execute("ALTER TABLE instances ADD COLUMN contact_link TEXT") instance_columns << "contact_link" @@ -533,7 +565,7 @@ module PotatoMesh db.execute_batch(File.read(telemetry_schema)) end - telemetry_columns = db.execute("PRAGMA table_info(telemetry)").map { |row| row[1] } + telemetry_columns = schema_column_names(db, "telemetry") # The environment expansion plus the extended metric families (TI-A2: # power / air-quality / health / local / host / traffic stats and the # one-wire probe list) share one idempotent backfill loop; the extended @@ -562,7 +594,7 @@ module PotatoMesh positions_schema = File.expand_path("../../../../data/positions.sql", __dir__) db.execute_batch(File.read(positions_schema)) end - position_columns = db.execute("PRAGMA table_info(positions)").map { |row| row[1] } + position_columns = schema_column_names(db, "positions") unless position_columns.include?("ingestor") db.execute("ALTER TABLE positions ADD COLUMN ingestor TEXT") end @@ -578,7 +610,7 @@ module PotatoMesh neighbors_schema = File.expand_path("../../../../data/neighbors.sql", __dir__) db.execute_batch(File.read(neighbors_schema)) end - neighbor_columns = db.execute("PRAGMA table_info(neighbors)").map { |row| row[1] } + neighbor_columns = schema_column_names(db, "neighbors") unless neighbor_columns.include?("ingestor") db.execute("ALTER TABLE neighbors ADD COLUMN ingestor TEXT") end @@ -596,7 +628,7 @@ module PotatoMesh traces_schema = File.expand_path("../../../../data/traces.sql", __dir__) db.execute_batch(File.read(traces_schema)) end - trace_columns = db.execute("PRAGMA table_info(traces)").map { |row| row[1] } + trace_columns = schema_column_names(db, "traces") unless trace_columns.include?("ingestor") db.execute("ALTER TABLE traces ADD COLUMN ingestor TEXT") end @@ -612,7 +644,7 @@ module PotatoMesh ingestors_schema = File.expand_path("../../../../data/ingestors.sql", __dir__) db.execute_batch(File.read(ingestors_schema)) else - ingestor_columns = db.execute("PRAGMA table_info(ingestors)").map { |row| row[1] } + ingestor_columns = schema_column_names(db, "ingestors") unless ingestor_columns.include?("version") db.execute("ALTER TABLE ingestors ADD COLUMN version TEXT") end @@ -656,14 +688,30 @@ module PotatoMesh destinations_schema = File.expand_path("../../../../data/destinations.sql", __dir__) db.execute_batch(File.read(destinations_schema)) end - rescue SQLite3::SQLException, Errno::ENOENT => e - warn_log( - "Failed to apply schema upgrade", - context: "database.schema", - error_class: e.class.name, - error_message: e.message, - ) + + # A new database has no nodes or messages table. They come last, after + # the blocks above skipped them as before, so a fresh schema commits + # whole in this transaction and a racing boot never reads it half + # built (SPEC SU5); init_db then finds the schema present. + %w[nodes messages].each do |schema| + table_exists = db.get_first_value( + "SELECT COUNT(*) FROM sqlite_master WHERE type='table' AND name=?", [schema], + ).to_i > 0 + db.execute_batch(File.read(File.join(SCHEMA_DIRECTORY, "#{schema}.sql"))) unless table_exists + end + + # An index an operator dropped comes back here, after every other + # step (SPEC SU3). + create_missing_schema_indexes(db) + + db.execute("COMMIT") + rescue SQLite3::Exception => e + step = db.respond_to?(:upgrade_step_label) ? db.upgrade_step_label : "opening the database" + raise SchemaUpgradeError, + "Schema upgrade step failed: #{step} (#{e.class}: #{e.message.lines.first.to_s.chomp.chomp(":")})" ensure + # Closing rolls back the transaction a failed step left open, so the + # database keeps its pre-boot schema. db&.close end end diff --git a/web/lib/potato_mesh/application/database/schema_guard.rb b/web/lib/potato_mesh/application/database/schema_guard.rb new file mode 100644 index 00000000..2b7c7299 --- /dev/null +++ b/web/lib/potato_mesh/application/database/schema_guard.rb @@ -0,0 +1,210 @@ +# Copyright © 2025-26 l5yth & contributors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# frozen_string_literal: true + +module PotatoMesh + module App + # Raised at boot when a schema upgrade step fails, or when the upgraded + # database still lacks a table, column or index that +data/*.sql+ + # defines (SPEC SU2, SU3). The message names the failed step or every + # missing object, and the boot stops. + class SchemaUpgradeError < StandardError; end + + module Database + # Directory holding the fresh-install schema files, +data/*.sql+. + SCHEMA_DIRECTORY = File.expand_path("../../../../../data", __dir__) + + # Longest statement text a failed-step message quotes. + UPGRADE_STEP_LABEL_LIMIT = 200 + + # Behaviour +ensure_schema_upgrades+ gives the one connection it runs + # its single IMMEDIATE transaction on (SPEC SU2, SU5). Mixed into that + # connection with +extend+, so every other handle keeps the stock + # +SQLite3::Database+ behaviour. + module UpgradeConnection + # @return [String, nil] the SQL prepared last: the step running when + # a statement fails. + attr_reader :upgrade_step + + # Record +sql+ as the current step, then prepare it. + # + # +execute+, +get_first_value+ and every statement of +execute_batch+ + # prepare through here, so the record is the statement SQLite + # rejected, not a call site. + # + # @param sql [String] statement text; inside +execute_batch+ the + # remaining batch, whose first statement is the one prepared. + # @return [SQLite3::Statement, Object] the statement, or the block's + # result when a block is given. + def prepare(sql, &block) + @upgrade_step = sql + super + end + + # Run a nested +transaction+ block inside the upgrade's transaction. + # + # The upgrade already holds an IMMEDIATE transaction, which a BEGIN + # cannot nest in. A nested block's statements commit or roll back + # with the rest of the upgrade, so the block keeps the all-or-nothing + # guarantee it asked for. Outside a transaction this is the stock + # +SQLite3::Database#transaction+. + # + # @param mode [Symbol, nil] transaction mode, used outside a transaction. + # @yieldparam db [SQLite3::Database] this connection. + # @return [Object] the block's result, or the stock method's. + def transaction(mode = nil, &block) + return super unless block && transaction_active? + + yield self + end + + # The current step as one line: its first statement without SQL + # comments, cut at {UPGRADE_STEP_LABEL_LIMIT} characters. + # + # @return [String] the statement text, or an empty string before the + # first statement. + def upgrade_step_label + statement = upgrade_step.to_s.gsub(/--[^\n]*/, "") + statement = statement.split(";").first.to_s.split.join(" ") + return statement if statement.length <= UPGRADE_STEP_LABEL_LIMIT + + "#{statement[0, UPGRADE_STEP_LABEL_LIMIT - 3]}..." + end + end + + # Fail the boot unless the database holds every table, column and + # index the fresh schema defines (SPEC SU3). + # + # Runs after +ensure_schema_upgrades+ and +init_db+, so an upgraded + # database reaches here with every step applied and every index + # created. Objects the fresh schema does not define (a legacy column, + # a hand-made index) are allowed; only absences fail. + # + # @return [void] + # @raise [SchemaUpgradeError] naming every missing object. + def ensure_schema_parity! + db = open_database(readonly: true) + missing = missing_schema_objects(db) + return if missing.empty? + + raise SchemaUpgradeError, + "Database schema lacks what data/*.sql defines after the upgrade: #{missing.join(", ")}" + ensure + db&.close + end + + # List what the fresh schema defines and +db+ lacks. + # + # SQLite compares identifiers case-insensitively, and so does this. + # + # @param db [SQLite3::Database] open database handle. + # @param expected [Hash] schema shape to hold +db+ against; defaults to + # {#fresh_schema_shape}. + # @return [Array] +table +, +.+ and + # +index + entries, in schema order. + def missing_schema_objects(db, expected = fresh_schema_shape) + missing = [] + expected.fetch(:tables).each do |table, columns| + present = schema_column_names(db, table) + if present.empty? + missing << "table #{table}" + else + missing.concat(columns.reject { |column| present.include?(column.downcase) }.map { |column| "#{table}.#{column}" }) + end + end + indexes = schema_index_names(db).map(&:downcase) + missing.concat(expected.fetch(:indexes).keys.reject { |name| indexes.include?(name.downcase) }.map { |name| "index #{name}" }) + end + + # Create each index +data/*.sql+ defines that +db+ lacks, with the + # statement the fresh schema holds for it (SPEC SU3). An index is safe + # to build again, so one an operator dropped comes back on the next + # boot instead of failing it. + # + # The upgrade runs this last, in its transaction. An index whose table + # is missing or lacks a column of the fresh schema is skipped: the + # parity check then names that table or column, and the index. + # + # @param db [SQLite3::Database] the upgrade's connection. + # @return [void] + def create_missing_schema_indexes(db) + shape = fresh_schema_shape + present = schema_index_names(db).map(&:downcase) + shape.fetch(:indexes).each do |name, index| + next if present.include?(name.downcase) + + columns = schema_column_names(db, index[:table]) + next if columns.empty? + next unless shape.fetch(:tables).fetch(index[:table]).all? { |column| columns.include?(column.downcase) } + + db.execute(index[:sql]) + end + end + + # Tables, columns and named indexes +data/*.sql+ defines, read from an + # in-memory database built from every file in {SCHEMA_DIRECTORY}. + # + # @return [Hash] +:tables+ maps each table to its column names in + # definition order; +:indexes+ maps each index name to its +:table+ + # and the +:sql+ that creates it, as SQLite stores it (without + # +IF NOT EXISTS+). + def fresh_schema_shape + SQLite3::Database.open(":memory:") do |db| + Dir.glob(File.join(SCHEMA_DIRECTORY, "*.sql")).sort.each { |file| db.execute_batch(File.read(file)) } + tables = db.execute("SELECT name FROM sqlite_master WHERE type = 'table' AND name NOT LIKE 'sqlite_%' ORDER BY rowid").flatten + # SQLite's own autoindexes carry no statement. + indexes = db.execute("SELECT name, tbl_name, sql FROM sqlite_master WHERE type = 'index' AND sql IS NOT NULL ORDER BY rowid") + { + tables: tables.to_h { |table| [table, schema_table_columns(db, table)] }, + indexes: indexes.to_h { |name, table, sql| [name, { table: table, sql: sql }] }, + } + end + end + + private + + # Column names of +table+ in +db+, in lower case. SQLite matches + # identifiers in any case, so the upgrade's column guards and the + # parity check do too (SPEC SU3, SU4). + # + # @param db [SQLite3::Database] open database handle. + # @param table [String] table name. + # @return [Array] lower-case column names; empty when the table + # is absent. + def schema_column_names(db, table) + schema_table_columns(db, table).map(&:downcase) + end + + # Column names of +table+ in +db+. + # + # @param db [SQLite3::Database] open database handle. + # @param table [String] table name, quoted for the pragma. + # @return [Array] column names; empty when the table is absent. + def schema_table_columns(db, table) + db.execute(%(PRAGMA table_info("#{table.gsub('"', '""')}"))).map { |row| row[1] } + end + + # Named indexes in +db+. SQLite's own +sqlite_autoindex_*+ entries + # belong to a table's key constraints, not to the schema files, and + # are left out. + # + # @param db [SQLite3::Database] open database handle. + # @return [Array] index names in creation order. + def schema_index_names(db) + db.execute("SELECT name FROM sqlite_master WHERE type = 'index' AND name NOT LIKE 'sqlite_%' ORDER BY rowid").flatten + end + end + end +end diff --git a/web/spec/database_spec.rb b/web/spec/database_spec.rb index 7f46dffc..6942f0b9 100644 --- a/web/spec/database_spec.rb +++ b/web/spec/database_spec.rb @@ -93,8 +93,11 @@ RSpec.describe PotatoMesh::App::Database do it "adds missing telemetry columns when upgrading an existing schema" do SQLite3::Database.new(PotatoMesh::Config.db_path) do |db| - db.execute("CREATE TABLE nodes(node_id TEXT)") - db.execute("CREATE TABLE messages(id INTEGER PRIMARY KEY)") + # The second boot's #747 and #755 backfills read long_name, role and + # from_id, and delete through the neighbors foreign key, which needs a + # keyed node_id. A failing step stops the boot (SPEC SU2). + db.execute("CREATE TABLE nodes(node_id TEXT PRIMARY KEY, long_name TEXT, role TEXT)") + db.execute("CREATE TABLE messages(id INTEGER PRIMARY KEY, from_id TEXT)") db.execute <<~SQL CREATE TABLE telemetry ( id INTEGER PRIMARY KEY, @@ -155,8 +158,9 @@ RSpec.describe PotatoMesh::App::Database do # #893 shipped dest_hash as a JSON column; the destinations table replaced # it, and a column and a table modelling one relationship would drift. SQLite3::Database.new(PotatoMesh::Config.db_path) do |db| - db.execute("CREATE TABLE nodes(node_id TEXT, dest_hash TEXT)") - db.execute("CREATE TABLE messages(id INTEGER PRIMARY KEY)") + # Keyed node_id, long_name, role and from_id: the second boot's backfills. + db.execute("CREATE TABLE nodes(node_id TEXT PRIMARY KEY, long_name TEXT, role TEXT, dest_hash TEXT)") + db.execute("CREATE TABLE messages(id INTEGER PRIMARY KEY, from_id TEXT)") end harness_class.ensure_schema_upgrades @@ -188,8 +192,9 @@ RSpec.describe PotatoMesh::App::Database do it "backfills every extended telemetry metric column on an existing schema (TI-A2)" do SQLite3::Database.new(PotatoMesh::Config.db_path) do |db| - db.execute("CREATE TABLE nodes(node_id TEXT)") - db.execute("CREATE TABLE messages(id INTEGER PRIMARY KEY)") + # Keyed node_id, long_name, role and from_id: the second boot's backfills. + db.execute("CREATE TABLE nodes(node_id TEXT PRIMARY KEY, long_name TEXT, role TEXT)") + db.execute("CREATE TABLE messages(id INTEGER PRIMARY KEY, from_id TEXT)") db.execute("CREATE TABLE telemetry(id INTEGER PRIMARY KEY, rx_time INTEGER NOT NULL, rx_iso TEXT NOT NULL)") end @@ -332,7 +337,8 @@ RSpec.describe PotatoMesh::App::Database do protocol TEXT NOT NULL DEFAULT 'meshtastic', synthetic BOOLEAN NOT NULL DEFAULT 0 ) SQL - db.execute("CREATE TABLE messages(id INTEGER PRIMARY KEY)") + # from_id: read by the #755 backfill that follows #747 on this boot. + db.execute("CREATE TABLE messages(id INTEGER PRIMARY KEY, from_id TEXT)") # Misclassified meshcore placeholder (bug #747) db.execute( diff --git a/web/spec/fixtures/schema/v0.2.0.sql b/web/spec/fixtures/schema/v0.2.0.sql new file mode 100644 index 00000000..481bc4ba --- /dev/null +++ b/web/spec/fixtures/schema/v0.2.0.sql @@ -0,0 +1,89 @@ +-- Copyright © 2025-26 l5yth & contributors +-- +-- Licensed under the Apache License, Version 2.0 (the "License"); +-- you may not use this file except in compliance with the License. +-- You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. + +-- Schema fixture (SPEC SU6): the database a fresh v0.2.0 install created. +-- The DDL is that release's init_db file list, each file verbatim from git +-- without its license header: +-- nodes, messages. +-- The sample rows at the end must survive the boot-time schema upgrade +-- unchanged. + +-- data/nodes.sql at v0.2.0 + +PRAGMA journal_mode=WAL; + +CREATE TABLE IF NOT EXISTS nodes ( + node_id TEXT PRIMARY KEY, + num INTEGER, + short_name TEXT, + long_name TEXT, + macaddr TEXT, + hw_model TEXT, + role TEXT, + public_key TEXT, + is_unmessagable BOOLEAN, + is_favorite BOOLEAN, + hops_away INTEGER, + snr REAL, + last_heard INTEGER, + first_heard INTEGER, + battery_level REAL, + voltage REAL, + channel_utilization REAL, + air_util_tx REAL, + uptime_seconds INTEGER, + position_time INTEGER, + location_source TEXT, + latitude REAL, + longitude REAL, + altitude REAL +); + +CREATE INDEX IF NOT EXISTS idx_nodes_last_heard ON nodes(last_heard); +CREATE INDEX IF NOT EXISTS idx_nodes_hw_model ON nodes(hw_model); +CREATE INDEX IF NOT EXISTS idx_nodes_latlon ON nodes(latitude, longitude); + +-- data/messages.sql at v0.2.0 + +CREATE TABLE IF NOT EXISTS messages ( + id INTEGER PRIMARY KEY, + rx_time INTEGER NOT NULL, + rx_iso TEXT NOT NULL, + from_id TEXT, + to_id TEXT, + channel INTEGER, + portnum TEXT, + text TEXT, + snr REAL, + rssi INTEGER, + hop_limit INTEGER, + raw_json TEXT +); + +CREATE INDEX IF NOT EXISTS idx_messages_rx_time ON messages(rx_time); +CREATE INDEX IF NOT EXISTS idx_messages_from_id ON messages(from_id); +CREATE INDEX IF NOT EXISTS idx_messages_to_id ON messages(to_id); +CREATE INDEX IF NOT EXISTS idx_messages_channel ON messages(channel); +CREATE INDEX IF NOT EXISTS idx_messages_portnum ON messages(portnum); + +-- Sample rows. + +INSERT INTO nodes(node_id, num, short_name, long_name, hw_model, role, snr, last_heard, first_heard, latitude, longitude, altitude) + VALUES ('!a1b2c3d4', 2712847316, 'A1B2', 'Legacy Alpha', 'TBEAM', 'CLIENT', 6.5, 1735689600, 1735600000, 52.52, 13.405, 34.0); +INSERT INTO nodes(node_id, num, short_name, long_name, hw_model, role, snr, last_heard, first_heard) + VALUES ('!0badc0de', 195936478, 'BADC', 'Legacy Bravo', 'HELTEC_V3', 'ROUTER', -3.25, 1735689000, 1735500000); +INSERT INTO messages(id, rx_time, rx_iso, from_id, to_id, channel, portnum, text, snr, rssi, hop_limit, raw_json) + VALUES (1001, 1735689600, '2025-01-01T00:00:00Z', '!a1b2c3d4', '^all', 0, 'TEXT_MESSAGE_APP', 'hello from v0.2.0', -7.5, -92, 3, '{"id":1001}'); +INSERT INTO messages(id, rx_time, rx_iso, from_id, to_id, channel, portnum, text, snr, rssi, hop_limit) + VALUES (1002, 1735689660, '2025-01-01T00:01:00Z', '!0badc0de', '!a1b2c3d4', 0, 'TEXT_MESSAGE_APP', 'direct reply', 5.25, -80, 2); diff --git a/web/spec/fixtures/schema/v0.4.0.sql b/web/spec/fixtures/schema/v0.4.0.sql new file mode 100644 index 00000000..3206bfe4 --- /dev/null +++ b/web/spec/fixtures/schema/v0.4.0.sql @@ -0,0 +1,172 @@ +-- Copyright © 2025-26 l5yth & contributors +-- +-- Licensed under the Apache License, Version 2.0 (the "License"); +-- you may not use this file except in compliance with the License. +-- You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. + +-- Schema fixture (SPEC SU6): the database a fresh v0.4.0 install created. +-- The DDL is that release's init_db file list, each file verbatim from git +-- without its license header: +-- nodes, messages, positions, telemetry, neighbors. +-- The sample rows at the end must survive the boot-time schema upgrade +-- unchanged. + +-- data/nodes.sql at v0.4.0 + +PRAGMA journal_mode=WAL; + +CREATE TABLE IF NOT EXISTS nodes ( + node_id TEXT PRIMARY KEY, + num INTEGER, + short_name TEXT, + long_name TEXT, + macaddr TEXT, + hw_model TEXT, + role TEXT, + public_key TEXT, + is_unmessagable BOOLEAN, + is_favorite BOOLEAN, + hops_away INTEGER, + snr REAL, + last_heard INTEGER, + first_heard INTEGER, + battery_level REAL, + voltage REAL, + channel_utilization REAL, + air_util_tx REAL, + uptime_seconds INTEGER, + position_time INTEGER, + location_source TEXT, + precision_bits INTEGER, + latitude REAL, + longitude REAL, + altitude REAL +); + +CREATE INDEX IF NOT EXISTS idx_nodes_last_heard ON nodes(last_heard); +CREATE INDEX IF NOT EXISTS idx_nodes_hw_model ON nodes(hw_model); +CREATE INDEX IF NOT EXISTS idx_nodes_latlon ON nodes(latitude, longitude); + +-- data/messages.sql at v0.4.0 + +CREATE TABLE IF NOT EXISTS messages ( + id INTEGER PRIMARY KEY, + rx_time INTEGER NOT NULL, + rx_iso TEXT NOT NULL, + from_id TEXT, + to_id TEXT, + channel INTEGER, + portnum TEXT, + text TEXT, + encrypted TEXT, + snr REAL, + rssi INTEGER, + hop_limit INTEGER +); + +CREATE INDEX IF NOT EXISTS idx_messages_rx_time ON messages(rx_time); +CREATE INDEX IF NOT EXISTS idx_messages_from_id ON messages(from_id); +CREATE INDEX IF NOT EXISTS idx_messages_to_id ON messages(to_id); +CREATE INDEX IF NOT EXISTS idx_messages_channel ON messages(channel); +CREATE INDEX IF NOT EXISTS idx_messages_portnum ON messages(portnum); + +-- data/positions.sql at v0.4.0 + +CREATE TABLE IF NOT EXISTS positions ( + id INTEGER PRIMARY KEY, + node_id TEXT, + node_num INTEGER, + rx_time INTEGER NOT NULL, + rx_iso TEXT NOT NULL, + position_time INTEGER, + to_id TEXT, + latitude REAL, + longitude REAL, + altitude REAL, + location_source TEXT, + precision_bits INTEGER, + sats_in_view INTEGER, + pdop REAL, + ground_speed REAL, + ground_track REAL, + snr REAL, + rssi INTEGER, + hop_limit INTEGER, + bitfield INTEGER, + payload_b64 TEXT +); + +CREATE INDEX IF NOT EXISTS idx_positions_rx_time ON positions(rx_time); +CREATE INDEX IF NOT EXISTS idx_positions_node_id ON positions(node_id); + +-- data/telemetry.sql at v0.4.0 + +CREATE TABLE IF NOT EXISTS telemetry ( + id INTEGER PRIMARY KEY, + node_id TEXT, + node_num INTEGER, + from_id TEXT, + to_id TEXT, + rx_time INTEGER NOT NULL, + rx_iso TEXT NOT NULL, + telemetry_time INTEGER, + channel INTEGER, + portnum TEXT, + hop_limit INTEGER, + snr REAL, + rssi INTEGER, + bitfield INTEGER, + payload_b64 TEXT, + battery_level REAL, + voltage REAL, + channel_utilization REAL, + air_util_tx REAL, + uptime_seconds INTEGER, + temperature REAL, + relative_humidity REAL, + barometric_pressure REAL +); + +CREATE INDEX IF NOT EXISTS idx_telemetry_rx_time ON telemetry(rx_time); +CREATE INDEX IF NOT EXISTS idx_telemetry_node_id ON telemetry(node_id); +CREATE INDEX IF NOT EXISTS idx_telemetry_time ON telemetry(telemetry_time); + +-- data/neighbors.sql at v0.4.0 + +CREATE TABLE IF NOT EXISTS neighbors ( + node_id TEXT NOT NULL, + neighbor_id TEXT NOT NULL, + snr REAL, + rx_time INTEGER NOT NULL, + PRIMARY KEY (node_id, neighbor_id), + FOREIGN KEY (node_id) REFERENCES nodes(node_id) ON DELETE CASCADE, + FOREIGN KEY (neighbor_id) REFERENCES nodes(node_id) ON DELETE CASCADE +); + +CREATE INDEX IF NOT EXISTS idx_neighbors_rx_time ON neighbors(rx_time); +CREATE INDEX IF NOT EXISTS idx_neighbors_neighbor_id ON neighbors(neighbor_id); + +-- Sample rows. + +INSERT INTO nodes(node_id, num, short_name, long_name, hw_model, role, snr, last_heard, first_heard, precision_bits, latitude, longitude, altitude) + VALUES ('!a1b2c3d4', 2712847316, 'A1B2', 'Legacy Alpha', 'TBEAM', 'CLIENT', 6.5, 1735689600, 1735600000, 13, 52.52, 13.405, 34.0); +INSERT INTO nodes(node_id, num, short_name, long_name, hw_model, role, snr, last_heard, first_heard) + VALUES ('!0badc0de', 195936478, 'BADC', 'Legacy Bravo', 'HELTEC_V3', 'ROUTER', -3.25, 1735689000, 1735500000); +INSERT INTO messages(id, rx_time, rx_iso, from_id, to_id, channel, portnum, text, encrypted, snr, rssi, hop_limit) + VALUES (1001, 1735689600, '2025-01-01T00:00:00Z', '!a1b2c3d4', '^all', 0, 'TEXT_MESSAGE_APP', 'hello from v0.4.0', NULL, -7.5, -92, 3); +INSERT INTO messages(id, rx_time, rx_iso, from_id, to_id, channel, portnum, text, encrypted, snr, rssi, hop_limit) + VALUES (1002, 1735689660, '2025-01-01T00:01:00Z', '!0badc0de', '!a1b2c3d4', 1, NULL, NULL, 'q83vEjRWeJA=', 5.25, -80, 2); +INSERT INTO positions(id, node_id, node_num, rx_time, rx_iso, position_time, to_id, latitude, longitude, altitude, location_source, precision_bits, sats_in_view, snr, rssi, hop_limit) + VALUES (2001, '!a1b2c3d4', 2712847316, 1735689700, '2025-01-01T00:01:40Z', 1735689690, '^all', 52.52, 13.405, 34.0, 'LOC_INTERNAL', 13, 7, 6.5, -91, 3); +INSERT INTO telemetry(id, node_id, node_num, from_id, to_id, rx_time, rx_iso, telemetry_time, channel, portnum, hop_limit, snr, rssi, battery_level, voltage, channel_utilization, air_util_tx, uptime_seconds) + VALUES (3001, '!a1b2c3d4', 2712847316, '!a1b2c3d4', '^all', 1735689800, '2025-01-01T00:03:20Z', 1735689795, 0, 'TELEMETRY_APP', 3, 6.0, -90, 87.0, 4.05, 12.5, 1.25, 86400); +INSERT INTO neighbors(node_id, neighbor_id, snr, rx_time) + VALUES ('!a1b2c3d4', '!0badc0de', 4.5, 1735689900); diff --git a/web/spec/fixtures/schema/v0.7.0.sql b/web/spec/fixtures/schema/v0.7.0.sql new file mode 100644 index 00000000..0a09f90b --- /dev/null +++ b/web/spec/fixtures/schema/v0.7.0.sql @@ -0,0 +1,292 @@ +-- Copyright © 2025-26 l5yth & contributors +-- +-- Licensed under the Apache License, Version 2.0 (the "License"); +-- you may not use this file except in compliance with the License. +-- You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. + +-- Schema fixture (SPEC SU6): the database a fresh v0.7.0 install created. +-- The DDL is that release's init_db file list, each file verbatim from git +-- without its license header: +-- nodes, messages, positions, telemetry, neighbors, instances, traces, ingestors. +-- The sample rows at the end must survive the boot-time schema upgrade +-- unchanged. + +-- data/nodes.sql at v0.7.0 + +PRAGMA journal_mode=WAL; + +CREATE TABLE IF NOT EXISTS nodes ( + node_id TEXT PRIMARY KEY, + num INTEGER, + short_name TEXT, + long_name TEXT, + macaddr TEXT, + hw_model TEXT, + role TEXT, + public_key TEXT, + is_unmessagable BOOLEAN, + is_favorite BOOLEAN, + hops_away INTEGER, + snr REAL, + last_heard INTEGER, + first_heard INTEGER, + battery_level REAL, + voltage REAL, + channel_utilization REAL, + air_util_tx REAL, + uptime_seconds INTEGER, + position_time INTEGER, + location_source TEXT, + precision_bits INTEGER, + latitude REAL, + longitude REAL, + altitude REAL, + lora_freq INTEGER, + modem_preset TEXT, + protocol TEXT NOT NULL DEFAULT 'meshtastic', + synthetic BOOLEAN NOT NULL DEFAULT 0 +); + +CREATE INDEX IF NOT EXISTS idx_nodes_last_heard ON nodes(last_heard); +CREATE INDEX IF NOT EXISTS idx_nodes_hw_model ON nodes(hw_model); +CREATE INDEX IF NOT EXISTS idx_nodes_latlon ON nodes(latitude, longitude); +CREATE INDEX IF NOT EXISTS idx_nodes_long_name ON nodes(long_name); + +-- data/messages.sql at v0.7.0 + +CREATE TABLE IF NOT EXISTS messages ( + id INTEGER PRIMARY KEY, + rx_time INTEGER NOT NULL, + rx_iso TEXT NOT NULL, + from_id TEXT, + to_id TEXT, + channel INTEGER, + portnum TEXT, + text TEXT, + encrypted TEXT, + snr REAL, + rssi INTEGER, + hop_limit INTEGER, + lora_freq INTEGER, + modem_preset TEXT, + channel_name TEXT, + reply_id INTEGER, + emoji TEXT, + ingestor TEXT, + protocol TEXT NOT NULL DEFAULT 'meshtastic' +); + +CREATE INDEX IF NOT EXISTS idx_messages_rx_time ON messages(rx_time); +CREATE INDEX IF NOT EXISTS idx_messages_from_id ON messages(from_id); +CREATE INDEX IF NOT EXISTS idx_messages_to_id ON messages(to_id); +CREATE INDEX IF NOT EXISTS idx_messages_channel ON messages(channel); +CREATE INDEX IF NOT EXISTS idx_messages_portnum ON messages(portnum); +CREATE INDEX IF NOT EXISTS idx_messages_reply_id ON messages(reply_id); + +-- data/positions.sql at v0.7.0 + +CREATE TABLE IF NOT EXISTS positions ( + id INTEGER PRIMARY KEY, + node_id TEXT, + node_num INTEGER, + rx_time INTEGER NOT NULL, + rx_iso TEXT NOT NULL, + position_time INTEGER, + to_id TEXT, + latitude REAL, + longitude REAL, + altitude REAL, + location_source TEXT, + precision_bits INTEGER, + sats_in_view INTEGER, + pdop REAL, + ground_speed REAL, + ground_track REAL, + snr REAL, + rssi INTEGER, + hop_limit INTEGER, + bitfield INTEGER, + payload_b64 TEXT, + ingestor TEXT, + protocol TEXT NOT NULL DEFAULT 'meshtastic' +); + +CREATE INDEX IF NOT EXISTS idx_positions_rx_time ON positions(rx_time); +CREATE INDEX IF NOT EXISTS idx_positions_node_id ON positions(node_id); + +-- data/telemetry.sql at v0.7.0 + +CREATE TABLE IF NOT EXISTS telemetry ( + id INTEGER PRIMARY KEY, + node_id TEXT, + node_num INTEGER, + from_id TEXT, + to_id TEXT, + rx_time INTEGER NOT NULL, + rx_iso TEXT NOT NULL, + telemetry_time INTEGER, + channel INTEGER, + portnum TEXT, + hop_limit INTEGER, + snr REAL, + rssi INTEGER, + bitfield INTEGER, + payload_b64 TEXT, + battery_level REAL, + voltage REAL, + channel_utilization REAL, + air_util_tx REAL, + uptime_seconds INTEGER, + temperature REAL, + relative_humidity REAL, + barometric_pressure REAL, + gas_resistance REAL, + current REAL, + iaq INTEGER, + distance REAL, + lux REAL, + white_lux REAL, + ir_lux REAL, + uv_lux REAL, + wind_direction INTEGER, + wind_speed REAL, + weight REAL, + wind_gust REAL, + wind_lull REAL, + radiation REAL, + rainfall_1h REAL, + rainfall_24h REAL, + soil_moisture INTEGER, + soil_temperature REAL, + ingestor TEXT, + protocol TEXT NOT NULL DEFAULT 'meshtastic', + telemetry_type TEXT +); + +CREATE INDEX IF NOT EXISTS idx_telemetry_rx_time ON telemetry(rx_time); +CREATE INDEX IF NOT EXISTS idx_telemetry_node_id ON telemetry(node_id); +CREATE INDEX IF NOT EXISTS idx_telemetry_time ON telemetry(telemetry_time); + +-- data/neighbors.sql at v0.7.0 + +CREATE TABLE IF NOT EXISTS neighbors ( + node_id TEXT NOT NULL, + neighbor_id TEXT NOT NULL, + snr REAL, + rx_time INTEGER NOT NULL, + ingestor TEXT, + protocol TEXT NOT NULL DEFAULT 'meshtastic', + PRIMARY KEY (node_id, neighbor_id), + FOREIGN KEY (node_id) REFERENCES nodes(node_id) ON DELETE CASCADE, + FOREIGN KEY (neighbor_id) REFERENCES nodes(node_id) ON DELETE CASCADE +); + +CREATE INDEX IF NOT EXISTS idx_neighbors_rx_time ON neighbors(rx_time); +CREATE INDEX IF NOT EXISTS idx_neighbors_neighbor_id ON neighbors(neighbor_id); + +-- data/instances.sql at v0.7.0 + +PRAGMA journal_mode=WAL; + +CREATE TABLE IF NOT EXISTS instances ( + id TEXT PRIMARY KEY, + domain TEXT NOT NULL, + pubkey TEXT NOT NULL, + name TEXT, + version TEXT, + channel TEXT, + frequency TEXT, + latitude REAL, + longitude REAL, + last_update_time INTEGER, + is_private BOOLEAN NOT NULL DEFAULT 0, + nodes_count INTEGER, + meshcore_nodes_count INTEGER, + meshtastic_nodes_count INTEGER, + contact_link TEXT, + signature TEXT +); + +CREATE UNIQUE INDEX IF NOT EXISTS idx_instances_domain ON instances(domain); + +-- data/traces.sql at v0.7.0 + +CREATE TABLE IF NOT EXISTS traces ( + id INTEGER PRIMARY KEY, + request_id INTEGER, + src INTEGER, + dest INTEGER, + rx_time INTEGER NOT NULL, + rx_iso TEXT NOT NULL, + rssi INTEGER, + snr REAL, + elapsed_ms INTEGER, + ingestor TEXT, + protocol TEXT NOT NULL DEFAULT 'meshtastic' +); + +CREATE TABLE IF NOT EXISTS trace_hops ( + id INTEGER PRIMARY KEY, + trace_id INTEGER NOT NULL, + hop_index INTEGER NOT NULL, + node_id INTEGER NOT NULL, + FOREIGN KEY(trace_id) REFERENCES traces(id) ON DELETE CASCADE +); + +CREATE INDEX IF NOT EXISTS idx_traces_rx_time ON traces(rx_time); +CREATE INDEX IF NOT EXISTS idx_traces_request ON traces(request_id); +CREATE INDEX IF NOT EXISTS idx_trace_hops_trace ON trace_hops(trace_id); +CREATE INDEX IF NOT EXISTS idx_trace_hops_node ON trace_hops(node_id); + +-- data/ingestors.sql at v0.7.0 + +PRAGMA journal_mode=WAL; + +CREATE TABLE IF NOT EXISTS ingestors ( + node_id TEXT PRIMARY KEY, + start_time INTEGER NOT NULL, + last_seen_time INTEGER NOT NULL, + version TEXT, + lora_freq INTEGER, + modem_preset TEXT, + protocol TEXT NOT NULL DEFAULT 'meshtastic' +); + +CREATE INDEX IF NOT EXISTS idx_ingestors_last_seen ON ingestors(last_seen_time); + +-- Sample rows. + +INSERT INTO nodes(node_id, num, short_name, long_name, hw_model, role, snr, last_heard, first_heard, precision_bits, latitude, longitude, altitude, lora_freq, modem_preset, protocol, synthetic) + VALUES ('!a1b2c3d4', 2712847316, 'A1B2', 'Legacy Alpha', 'TBEAM', 'CLIENT', 6.5, 1735689600, 1735600000, 13, 52.52, 13.405, 34.0, 868, 'LongFast', 'meshtastic', 0); +INSERT INTO nodes(node_id, num, short_name, long_name, hw_model, role, snr, last_heard, first_heard, protocol, synthetic) + VALUES ('!0badc0de', 195936478, 'BADC', 'Legacy Bravo', 'HELTEC_V3', 'ROUTER', -3.25, 1735689000, 1735500000, 'meshtastic', 0); +INSERT INTO nodes(node_id, num, short_name, long_name, role, last_heard, first_heard, protocol, synthetic) + VALUES ('!5e7c0a11', 1585187345, '5E7C', 'Harbour Repeater', 'REPEATER', 1735689500, 1735400000, 'meshcore', 0); +INSERT INTO messages(id, rx_time, rx_iso, from_id, to_id, channel, portnum, text, snr, rssi, hop_limit, lora_freq, modem_preset, channel_name, reply_id, emoji, ingestor, protocol) + VALUES (1001, 1735689600, '2025-01-01T00:00:00Z', '!a1b2c3d4', '^all', 0, 'TEXT_MESSAGE_APP', 'hello from v0.7.0', -7.5, -92, 3, 868, 'LongFast', 'LongFast', NULL, NULL, '!0000beef', 'meshtastic'); +INSERT INTO messages(id, rx_time, rx_iso, from_id, to_id, channel, portnum, text, snr, rssi, hop_limit, reply_id, emoji, ingestor, protocol) + VALUES (1002, 1735689660, '2025-01-01T00:01:00Z', '!0badc0de', '^all', 0, 'TEXT_MESSAGE_APP', NULL, 5.25, -80, 2, 1001, '👍', '!0000beef', 'meshtastic'); +INSERT INTO messages(id, rx_time, rx_iso, from_id, to_id, channel, text, channel_name, ingestor, protocol) + VALUES (1003, 1735689700, '2025-01-01T00:01:40Z', '!5e7c0a11', '^all', 0, 'Harbour Repeater: tide is high', '#public', '!0000cafe', 'meshcore'); +INSERT INTO positions(id, node_id, node_num, rx_time, rx_iso, position_time, to_id, latitude, longitude, altitude, location_source, precision_bits, sats_in_view, snr, rssi, hop_limit, ingestor, protocol) + VALUES (2001, '!a1b2c3d4', 2712847316, 1735689700, '2025-01-01T00:01:40Z', 1735689690, '^all', 52.52, 13.405, 34.0, 'LOC_INTERNAL', 13, 7, 6.5, -91, 3, '!0000beef', 'meshtastic'); +INSERT INTO telemetry(id, node_id, node_num, from_id, to_id, rx_time, rx_iso, telemetry_time, channel, portnum, hop_limit, snr, rssi, battery_level, voltage, channel_utilization, air_util_tx, uptime_seconds, temperature, lux, ingestor, protocol, telemetry_type) + VALUES (3001, '!a1b2c3d4', 2712847316, '!a1b2c3d4', '^all', 1735689800, '2025-01-01T00:03:20Z', 1735689795, 0, 'TELEMETRY_APP', 3, 6.0, -90, 87.0, 4.05, 12.5, 1.25, 86400, NULL, NULL, '!0000beef', 'meshtastic', 'device'); +INSERT INTO neighbors(node_id, neighbor_id, snr, rx_time, ingestor, protocol) + VALUES ('!a1b2c3d4', '!0badc0de', 4.5, 1735689900, '!0000beef', 'meshtastic'); +INSERT INTO instances(id, domain, pubkey, name, version, channel, frequency, latitude, longitude, last_update_time, is_private, nodes_count, meshcore_nodes_count, meshtastic_nodes_count, contact_link, signature) + VALUES ('3f6c2a', 'peer.example.test', 'PUBKEY-PEM', 'Peer Mesh', 'v0.7.0', 'LongFast', '868', 50.0, 8.0, 1735689600, 0, 12, 2, 10, NULL, 'c2lnbmF0dXJl'); +INSERT INTO traces(id, request_id, src, dest, rx_time, rx_iso, rssi, snr, elapsed_ms, ingestor, protocol) + VALUES (4001, 77, 2712847316, 195936478, 1735690000, '2025-01-01T00:06:40Z', -95, 3.5, 1200, '!0000beef', 'meshtastic'); +INSERT INTO trace_hops(id, trace_id, hop_index, node_id) VALUES (1, 4001, 0, 2712847316); +INSERT INTO trace_hops(id, trace_id, hop_index, node_id) VALUES (2, 4001, 1, 195936478); +INSERT INTO ingestors(node_id, start_time, last_seen_time, version, lora_freq, modem_preset, protocol) + VALUES ('!0000beef', 1735600000, 1735690000, 'v0.7.0', 868, 'LongFast', 'meshtastic'); diff --git a/web/spec/schema_upgrades_spec.rb b/web/spec/schema_upgrades_spec.rb new file mode 100644 index 00000000..06216288 --- /dev/null +++ b/web/spec/schema_upgrades_spec.rb @@ -0,0 +1,624 @@ +# Copyright © 2025-26 l5yth & contributors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# frozen_string_literal: true + +require "spec_helper" +require "sqlite3" +require "json" +require "fileutils" + +# Boot-time schema upgrades on release databases: the nodes.num index, a +# failing step that stops the boot, the parity check and the indexes the +# upgrade recreates, the steps upgraded databases lacked, and boots racing +# on one database (SPEC SU1-SU6). +RSpec.describe "schema upgrades" do + # The database methods the configure block calls, outside any app. + let(:boot) do + Class.new do + extend PotatoMesh::App::Database + + # No upgrade step logs since SU2; an older tree logged a swallowed + # failure here, which these examples judge by the schema it left. + # + # @return [void] + def self.warn_log(*, **); end + end + end + + let(:fixture_dir) { File.expand_path("fixtures/schema", __dir__) } + let(:plan_on_num) { "SEARCH nodes USING INDEX idx_nodes_num (num=?)" } + + around do |example| + Dir.mktmpdir("schema-upgrades-spec-") do |dir| + RSpec::Mocks.with_temporary_scope do + allow(PotatoMesh::Config).to receive(:db_path).and_return(File.join(dir, "mesh.db")) + example.run + end + end + end + + # Build the example's database from a release fixture. + # + # @param tag [String] release tag naming +spec/fixtures/schema/.sql+. + # @return [void] + def load_fixture(tag) + db = SQLite3::Database.new(PotatoMesh::Config.db_path) + db.execute_batch(File.read(File.join(fixture_dir, "#{tag}.sql"))) + ensure + db&.close + end + + # Run +block+ on a fresh handle on the example's database. + # + # @yieldparam db [SQLite3::Database] open handle, closed afterwards. + # @return [Object] the block's result. + def with_db + db = SQLite3::Database.new(PotatoMesh::Config.db_path) + db.busy_timeout = 5000 + yield db + ensure + db&.close + end + + # @param table [String] table name. + # @return [Array] the table's column names. + def columns(table) + with_db { |db| db.execute("PRAGMA table_info(#{table})").map { |row| row[1] } } + end + + # @return [Array] every index name in the example's database. + def indexes + with_db { |db| db.execute("SELECT name FROM sqlite_master WHERE type='index'").flatten } + end + + # @return [Integer] the database's +PRAGMA user_version+. + def user_version + with_db { |db| db.get_first_value("PRAGMA user_version").to_i } + end + + # The configure block's first two steps: upgrade, then create the fresh + # schema when a required table is still absent. + # + # @return [void] + def upgrade! + boot.ensure_schema_upgrades + boot.init_db unless boot.db_schema_present? + end + + # What +data/*.sql+ defines and the example's database lacks, worked out + # here without the app's own check: +
.+, +table + and + # +index + entries. + # + # @return [Array] the gaps, sorted. + def schema_gaps + fresh = SQLite3::Database.new(":memory:") + Dir[File.expand_path("../../data/*.sql", __dir__)].each { |file| fresh.execute_batch(File.read(file)) } + tables = fresh.execute("SELECT name FROM sqlite_master WHERE type='table' AND name NOT LIKE 'sqlite_%'").flatten + gaps = tables.flat_map do |table| + have = columns(table) + next ["table #{table}"] if have.empty? + + (fresh.execute("PRAGMA table_info(#{table})").map { |row| row[1] } - have).map { |column| "#{table}.#{column}" } + end + wanted = fresh.execute("SELECT name FROM sqlite_master WHERE type='index' AND name NOT LIKE 'sqlite_%'").flatten + (gaps + (wanted - indexes).map { |name| "index #{name}" }).sort + ensure + fresh&.close + end + + # Every row of each table, holding only the given columns. + # + # @param columns_by_table [Hash{String => Array}] columns to read. + # @return [Hash{String => Array}] rows in rowid order, per table. + def snapshot(columns_by_table) + with_db do |db| + db.results_as_hash = true + columns_by_table.to_h do |table, cols| + [table, db.execute("SELECT #{cols.map { |col| %("#{col}") }.join(", ")} FROM #{table} ORDER BY rowid")] + end + end + end + + # Hold the write lock in a second process, as a boot racing this one does: + # BEGIN IMMEDIATE, run +sql+, report the lock held, wait +hold+ seconds, + # then COMMIT. Returns once the lock is held. + # + # @param sql [String] statement the racing boot applies. + # @param hold [Float] seconds the lock is held after +sql+. + # @return [Integer] the child's pid, for +expect_clean_exit+. + def racing_boot(sql, hold:) + reader, writer = IO.pipe + pid = fork do + reader.close + db = SQLite3::Database.new(PotatoMesh::Config.db_path) + db.execute("BEGIN IMMEDIATE") + db.execute(sql) + writer.puts("locked") + writer.flush + sleep hold + db.execute("COMMIT") + db.close + exit!(0) + rescue StandardError + exit!(1) + end + writer.close + expect(reader.gets).to eq("locked\n") + reader.close + pid + end + + # @param pid [Integer] a +racing_boot+ child. + # @return [void] + def expect_clean_exit(pid) + _, status = Process.wait2(pid) + expect(status.exitstatus).to eq(0) + end + + # Make the upgrade's connection raise +error+ when it prepares the first + # statement that starts with +prefix+, where SQLite reports a failure. + # The module goes on before +ensure_schema_upgrades+ adds its own, so the + # step is recorded first. + # + # @param prefix [String] start of the statement that fails. + # @param error [SQLite3::Exception] exception to raise. + # @return [void] + def fail_step(prefix, error) + failing = Module.new do + define_method(:prepare) do |sql, &block| + raise error if sql.lstrip.start_with?(prefix) + + super(sql, &block) + end + end + allow(boot).to receive(:open_database).and_wrap_original do |original, *args, **kwargs| + original.call(*args, **kwargs).extend(failing) + end + end + + # Point the fresh schema at a copy of +data/*.sql+ in which +file+ ends + # with +sql+, as a schema change made without its upgrade step would. + # + # @param file [String] schema file name, such as +nodes.sql+. + # @param sql [String] statements appended to the copy. + # @return [void] + def schema_files_with(file, sql) + dir = File.join(File.dirname(PotatoMesh::Config.db_path), "schema-files") + FileUtils.mkdir_p(dir) + FileUtils.cp(Dir[File.expand_path("../../data/*.sql", __dir__)], dir) + File.write(File.join(dir, file), "\n#{sql}\n", mode: "a") + stub_const("PotatoMesh::App::Database::SCHEMA_DIRECTORY", dir) + end + + # Make +db+ record each statement that reads +nodes+ through +execute+ + # or +get_first_value+, with its bind values, into +log+. + # + # @param db [SQLite3::Database] handle to record on. + # @param log [Array] receives +[sql, binds]+ pairs. + # @return [SQLite3::Database] +db+. + def record_node_reads(db, log) + %i[execute get_first_value].each do |name| + original = db.method(name) + db.define_singleton_method(name) do |sql, *binds, &blk| + log << [sql, binds.flatten] if sql.include?("FROM nodes") + original.call(sql, *binds, &blk) + end + end + db + end + + # @param db [SQLite3::Database] open handle. + # @param sql [String] statement text. + # @param binds [Array] bind values. + # @return [Array] the EXPLAIN QUERY PLAN detail lines. + def plan(db, sql, binds) + db.execute("EXPLAIN QUERY PLAN #{sql}", binds).map { |row| row[3] } + end + + describe "the nodes.num index (SPEC SU1)" do + # The three numeric lookups, as their call sites issue them, run on the + # example's database, which holds the fixture's two Meshtastic nodes. + # + # @return [Array] the num lookups, with binds. + def num_lookups + reads = [] + handle = record_node_reads(SQLite3::Database.new(PotatoMesh::Config.db_path), reads) + app = PotatoMesh::Application + expect(app.node_lookup_clause("!a1b2c3d4", string_columns: ["node_id"], numeric_columns: ["num"], db: handle)).not_to be_nil + expect(app.normalize_node_id(handle, "2712847316")).to eq("!a1b2c3d4") + expect(app.batch_resolve_node_ids(handle, ["2712847316", "195936478"])).to include("195936478" => "!0badc0de") + reads.select { |sql, _binds| sql.match?(/WHERE num (=|IN)/) } + ensure + handle&.close + end + + it "searches idx_nodes_num for every numeric lookup on a fresh database" do + boot.init_db + with_db { |db| db.execute("INSERT INTO nodes(node_id, num) VALUES ('!a1b2c3d4', 2712847316), ('!0badc0de', 195936478)") } + + lookups = num_lookups + + expect(lookups.length).to eq(3) + with_db { |db| lookups.each { |sql, binds| expect(plan(db, sql, binds)).to eq([plan_on_num]) } } + end + + it "searches idx_nodes_num for every numeric lookup on an upgraded v0.7.0 database" do + load_fixture("v0.7.0") + upgrade! + + lookups = num_lookups + + expect(lookups.length).to eq(3) + with_db { |db| lookups.each { |sql, binds| expect(plan(db, sql, binds)).to eq([plan_on_num]) } } + end + + it "leaves the plans of the numeric opt-out subqueries unchanged" do + boot.init_db + now = Time.now.to_i + with_db do |db| + db.execute("INSERT INTO nodes(node_id, num) VALUES ('!a1b2c3d4', 2712847316), ('!0badc0de', 195936478)") + db.execute("INSERT INTO traces(id, src, dest, rx_time, rx_iso) VALUES (1, 2712847316, 195936478, ?, 'now')", [now]) + db.execute("INSERT INTO trace_hops(trace_id, hop_index, node_id) VALUES (1, 0, 195936478)") + end + app = PotatoMesh::Application + statements = [] + allow(app).to receive(:open_database).and_wrap_original do |original, *args, **kwargs| + record_node_reads(original.call(*args, **kwargs), statements) + end + cutoffs = %w[hour day week month].to_h { |window| [window, now - 86_400] } + + # Public and private reads: the opt-out filters, then the + # CLIENT_HIDDEN filters beside them. + [false, true].each do |private_mode| + allow(app).to receive(:private_mode?).and_return(private_mode) + expect(app.query_traces(10, node_ref: "!a1b2c3d4").length).to eq(1) + with_db do |db| + db.results_as_hash = true + expect(app.telemetry_activity_counts(record_node_reads(db, statements), cutoffs)).to include("total") + end + end + + subqueries = statements.select { |sql, _binds| sql.include?("SELECT num FROM nodes") } + expect(subqueries.length).to eq(6) + expect(subqueries.map(&:first).join).to include("role = 'CLIENT_HIDDEN'").and include("LIKE") + with_db do |db| + indexed = subqueries.map { |sql, binds| plan(db, sql, binds) } + db.execute("DROP INDEX idx_nodes_num") + expect(subqueries.map { |sql, binds| plan(db, sql, binds) }).to eq(indexed) + end + end + + it "ships a reference migration that creates the index" do + load_fixture("v0.7.0") + migration = File.expand_path("../../data/migrations/20261009_add_nodes_num_index.sql", __dir__) + + with_db { |db| db.execute_batch(File.read(migration)) } + + expect(indexes).to include("idx_nodes_num") + end + end + + describe "a failing upgrade step (SPEC SU2)" do + it "fails the boot naming the step and leaves the schema as it was" do + load_fixture("v0.7.0") + fail_step("ALTER TABLE telemetry ADD COLUMN ch1_voltage", SQLite3::IOException.new("disk I/O error")) + + expect { boot.ensure_schema_upgrades }.to raise_error( + StandardError, + "Schema upgrade step failed: ALTER TABLE telemetry ADD COLUMN ch1_voltage REAL " \ + "(SQLite3::IOException: disk I/O error)", + ) { |error| expect(error.class.name).to eq("PotatoMesh::App::SchemaUpgradeError") } + # The steps before it rolled back with it. + expect(columns("nodes")).not_to include("rssi", "identity_hash") + expect(columns("messages")).not_to include("scope") + expect(user_version).to eq(0) + end + + it "names the database open when the connection cannot be made" do + allow(boot).to receive(:open_database).and_raise(SQLite3::CantOpenException, "unable to open database file") + + expect { boot.ensure_schema_upgrades }.to raise_error( + PotatoMesh::App::SchemaUpgradeError, + "Schema upgrade step failed: opening the database (SQLite3::CantOpenException: unable to open database file)", + ) + end + + it "names the BEGIN when a racing boot keeps the write lock past every retry" do + load_fixture("v0.7.0") + allow(PotatoMesh::Config).to receive(:db_busy_timeout_ms).and_return(20) + allow(PotatoMesh::Config).to receive(:db_busy_max_retries).and_return(1) + pid = racing_boot("SELECT 1", hold: 1.0) + + expect { boot.ensure_schema_upgrades }.to raise_error( + PotatoMesh::App::SchemaUpgradeError, + "Schema upgrade step failed: BEGIN IMMEDIATE (SQLite3::BusyException: database is locked)", + ) + expect_clean_exit(pid) + end + end + + describe "the schema parity check (SPEC SU3)" do + it "passes on a fresh database" do + upgrade! + + expect { boot.ensure_schema_parity! }.not_to raise_error + end + + it "fails naming every table, column and index the database lacks" do + upgrade! + with_db do |db| + db.execute("DROP TABLE waypoints") + db.execute("ALTER TABLE messages DROP COLUMN scope") + db.execute("DROP INDEX idx_nodes_num") + end + + expect { boot.ensure_schema_parity! }.to raise_error(PotatoMesh::App::SchemaUpgradeError) do |error| + expect(error.message).to eq( + "Database schema lacks what data/*.sql defines after the upgrade: messages.scope, table waypoints, " \ + "index idx_nodes_num, index idx_waypoints_rx_time, index idx_waypoints_node_id", + ) + end + end + + it "allows objects the fresh schema does not define and matches names in any case" do + upgrade! + with_db do |db| + db.execute("ALTER TABLE messages ADD COLUMN raw_json TEXT") + db.execute("CREATE INDEX idx_messages_raw_json ON messages(raw_json)") + db.execute("ALTER TABLE messages RENAME COLUMN encrypted TO ENCRYPTED") + end + + expect { boot.ensure_schema_parity! }.not_to raise_error + end + + it "recreates on the next boot every index the operator dropped" do + load_fixture("v0.7.0") + upgrade! + dropped = %w[idx_messages_rx_time idx_nodes_num idx_instances_domain] + with_db { |db| dropped.each { |name| db.execute("DROP INDEX #{name}") } } + + upgrade! + + expect { boot.ensure_schema_parity! }.not_to raise_error + expect(indexes).to include(*dropped) + end + + it "leaves the indexes of a table lacking a column the upgrade cannot add to the check, which names them" do + load_fixture("v0.7.0") + # A schema change made without its upgrade step: a column and an index. + schema_files_with( + "nodes.sql", + "ALTER TABLE nodes ADD COLUMN shoe_size INTEGER;\nCREATE INDEX IF NOT EXISTS idx_nodes_shoe_size ON nodes(shoe_size);", + ) + + expect { upgrade! }.not_to raise_error + expect { boot.ensure_schema_parity! }.to raise_error( + PotatoMesh::App::SchemaUpgradeError, + "Database schema lacks what data/*.sql defines after the upgrade: nodes.shoe_size, " \ + "index idx_nodes_identity_hash, index idx_nodes_num, index idx_nodes_shoe_size", + ) + end + + it "skips the indexes of a table the upgrade did not create" do + upgrade! + with_db do |db| + db.execute("DROP TABLE waypoints") + boot.create_missing_schema_indexes(db) + end + + expect(indexes).not_to include("idx_waypoints_rx_time", "idx_waypoints_node_id") + end + + it "lets a failed open stop the boot as it is" do + allow(boot).to receive(:open_database).and_raise(SQLite3::CantOpenException, "unable to open database file") + + expect { boot.ensure_schema_parity! }.to raise_error(SQLite3::CantOpenException, "unable to open database file") + end + + it "reads the fresh schema from every data/*.sql file" do + shape = boot.fresh_schema_shape + + expect(shape[:tables].keys).to contain_exactly( + "destinations", "ingestor_activity", "ingestors", "instances", "messages", "neighbors", + "nodes", "positions", "telemetry", "traces", "trace_hops", "waypoints", + ) + expect(shape[:tables]["nodes"]).to include("num", "identity_hash") + expect(shape[:indexes].keys).to include("idx_nodes_num", "idx_nodes_identity_hash", "idx_messages_meshcore_text") + expect(shape[:indexes].keys.grep(/\Asqlite_/)).to be_empty + expect(shape[:indexes]["idx_nodes_num"]).to eq(table: "nodes", sql: "CREATE INDEX idx_nodes_num ON nodes(num)") + end + end + + describe "steps upgraded databases lacked (SPEC SU4)" do + let(:app) { Sinatra::Application } + + around do |example| + token = ENV["API_TOKEN"] + ENV["API_TOKEN"] = "schema-spec-token" + example.run + ensure + ENV["API_TOKEN"] = token + end + + it "adds messages.encrypted to a v0.2.0 database, so a message POST succeeds" do + load_fixture("v0.2.0") + + upgrade! + + expect(columns("messages")).to include("encrypted") + post "/api/messages", + { "id" => 1003, "rx_time" => Time.now.to_i, "from_id" => "!a1b2c3d4", "to_id" => "^all", "channel" => 0, + "portnum" => "TEXT_MESSAGE_APP", "text" => "after the upgrade", "ingestor" => "!0000beef", + "protocol" => "meshtastic" }.to_json, + { "CONTENT_TYPE" => "application/json", "HTTP_AUTHORIZATION" => "Bearer schema-spec-token" } + expect(last_response.status).to eq(201) + expect(with_db { |db| db.get_first_value("SELECT text FROM messages WHERE id = 1003") }).to eq("after the upgrade") + end + + it "creates idx_nodes_identity_hash on an upgraded v0.7.0 database" do + load_fixture("v0.7.0") + + upgrade! + + expect(indexes).to include("idx_nodes_identity_hash") + end + + it "matches column names in any case, so a renamed column is not added again" do + load_fixture("v0.7.0") + upgrade! + with_db do |db| + db.execute("ALTER TABLE nodes RENAME COLUMN rssi TO RSSI") + db.execute("ALTER TABLE messages RENAME COLUMN encrypted TO ENCRYPTED") + db.execute("ALTER TABLE telemetry RENAME COLUMN ch1_voltage TO CH1_VOLTAGE") + end + + expect { upgrade! }.not_to raise_error + expect { boot.ensure_schema_parity! }.not_to raise_error + expect(columns("nodes")).to include("RSSI") + expect(columns("messages")).to include("ENCRYPTED") + expect(columns("telemetry")).to include("CH1_VOLTAGE") + end + + it "builds the MX6 index and runs the #756 purge on the boot that adds messages.protocol" do + load_fixture("v0.4.0") + expect(columns("messages")).not_to include("protocol") + + boot.ensure_schema_upgrades + + expect(indexes).to include("idx_messages_meshcore_text") + expect(user_version).to eq(PotatoMesh::App::Database::MESHCORE_CONTENT_DEDUP_BACKFILL_VERSION) + end + end + + describe "boots racing on one database (SPEC SU5)" do + it "waits for a racing boot's upgrade, then adds nothing twice" do + load_fixture("v0.7.0") + # The racing boot holds the lock with the first step this boot needs. + pid = racing_boot("ALTER TABLE nodes ADD COLUMN rssi INTEGER", hold: 0.4) + + expect { boot.ensure_schema_upgrades }.not_to raise_error + expect_clean_exit(pid) + expect(columns("nodes").count("rssi")).to eq(1) + expect(columns("telemetry")).to include("ch1_voltage", "one_wire_temperature") + expect(indexes).to include("idx_nodes_num", "idx_nodes_identity_hash") + end + + it "retries the write lock while a racing boot holds it past the busy timeout" do + load_fixture("v0.7.0") + allow(PotatoMesh::Config).to receive(:db_busy_timeout_ms).and_return(50) + pid = racing_boot("ALTER TABLE nodes ADD COLUMN rssi INTEGER", hold: 0.3) + + expect { boot.ensure_schema_upgrades }.not_to raise_error + expect_clean_exit(pid) + expect(columns("telemetry")).to include("ch1_voltage") + end + + it "retries the WAL switch while a racing boot holds a new database's write lock" do + # A new file is in rollback-journal mode. Switching it to WAL takes the + # write lock, and SQLite reports BUSY at once, without the busy timeout. + pid = racing_boot("SELECT 1", hold: 0.2) + + expect { boot.ensure_schema_upgrades }.not_to raise_error + expect_clean_exit(pid) + expect(with_db { |db| db.get_first_value("PRAGMA journal_mode") }).to eq("wal") + expect(with_db { |db| boot.missing_schema_objects(db) }).to be_empty + end + + it "creates a new database's whole schema in its one transaction" do + boot.ensure_schema_upgrades + + expect(boot.db_schema_present?).to be(true) + expect(with_db { |db| boot.missing_schema_objects(db) }).to be_empty + # The upgrade blocks still skip a new database's tables, as before. + expect(user_version).to eq(0) + end + end + + describe "the upgrade connection (SPEC SU2, SU5)" do + # @return [SQLite3::Database] a handle with the upgrade behaviour. + def upgrade_connection + SQLite3::Database.new(PotatoMesh::Config.db_path).extend(PotatoMesh::App::Database::UpgradeConnection) + end + + it "labels the step by its first statement, without comments, on one line" do + db = upgrade_connection + db.execute_batch("-- a comment\nCREATE TABLE t(\n a INTEGER -- inline\n);\nCREATE TABLE u(b);") + expect(db.upgrade_step_label).to eq("CREATE TABLE u(b)") + + db.prepare("-- a comment\nCREATE TABLE IF NOT EXISTS t(\n a INTEGER -- inline\n);").close + expect(db.upgrade_step_label).to eq("CREATE TABLE IF NOT EXISTS t( a INTEGER )") + db.prepare("-- header\n-- more\nSELECT a,\n 1\n FROM t;\nSELECT 2").close + expect(db.upgrade_step_label).to eq("SELECT a, 1 FROM t") + db.prepare("SELECT '#{"x" * 300}'").close + expect(db.upgrade_step_label).to eq("SELECT '#{"x" * 189}...") + expect(db.upgrade_step_label.length).to eq(PotatoMesh::App::Database::UPGRADE_STEP_LABEL_LIMIT) + ensure + db&.close + end + + it "has an empty label before the first statement" do + db = SQLite3::Database.new(":memory:").extend(PotatoMesh::App::Database::UpgradeConnection) + + expect(db.upgrade_step_label).to eq("") + ensure + db&.close + end + + it "runs a nested transaction block inside the upgrade's transaction" do + db = upgrade_connection + db.execute("CREATE TABLE t(a)") + db.execute("BEGIN IMMEDIATE") + + expect(db.transaction { |handle| handle.execute("INSERT INTO t VALUES (1)") && :joined }).to eq(:joined) + expect(db.transaction_active?).to be(true) + db.execute("ROLLBACK") + expect(db.get_first_value("SELECT COUNT(*) FROM t")).to eq(0) + ensure + db&.close + end + + it "keeps the stock transaction outside a transaction" do + db = upgrade_connection + db.execute("CREATE TABLE t(a)") + + db.transaction { |handle| handle.execute("INSERT INTO t VALUES (1)") } + expect(db.transaction_active?).to be(false) + expect(db.transaction).to be(true) + expect(db.transaction_active?).to be(true) + db.rollback + expect(db.get_first_value("SELECT COUNT(*) FROM t")).to eq(1) + ensure + db&.close + end + end + + describe "release databases (SPEC SU6)" do + %w[v0.2.0 v0.4.0 v0.7.0].each do |tag| + it "upgrades a #{tag} database to parity and keeps its rows" do + load_fixture(tag) + tables = with_db { |db| db.execute("SELECT name FROM sqlite_master WHERE type='table'").flatten } + legacy_columns = tables.to_h { |table| [table, columns(table)] } + before = snapshot(legacy_columns) + expect(before.values.sum(&:length)).to be >= 4 + + upgrade! + + expect(schema_gaps).to eq([]) + expect { boot.ensure_schema_parity! }.not_to raise_error + expect(snapshot(legacy_columns)).to eq(before) + end + end + end +end