diff --git a/app/database.py b/app/database.py index 7e267be..6f886e1 100644 --- a/app/database.py +++ b/app/database.py @@ -79,6 +79,17 @@ class Database: Path(self.db_path).parent.mkdir(parents=True, exist_ok=True) self._connection = await aiosqlite.connect(self.db_path) self._connection.row_factory = aiosqlite.Row + + # WAL mode: faster writes, concurrent readers during writes, no journal file churn. + # Persists in the DB file but we set it explicitly on every connection. + await self._connection.execute("PRAGMA journal_mode = WAL") + + # Incremental auto-vacuum: freed pages are reclaimable via + # PRAGMA incremental_vacuum without a full VACUUM. Must be set before + # the first table is created (for new databases); for existing databases + # migration 20 handles the one-time VACUUM to restructure the file. + await self._connection.execute("PRAGMA auto_vacuum = INCREMENTAL") + await self._connection.executescript(SCHEMA) await self._connection.commit() logger.debug("Database schema initialized") diff --git a/app/migrations.py b/app/migrations.py index 146f567..4588003 100644 --- a/app/migrations.py +++ b/app/migrations.py @@ -170,16 +170,17 @@ async def run_migrations(conn: aiosqlite.Connection) -> int: await set_version(conn, 19) applied += 1 + # Migration 20: Enable WAL journal mode and incremental auto-vacuum + if version < 20: + logger.info("Applying migration 20: enable WAL mode and incremental auto-vacuum") + await _migrate_020_enable_wal_and_auto_vacuum(conn) + await set_version(conn, 20) + applied += 1 + if applied > 0: logger.info( "Applied %d migration(s), schema now at version %d", applied, await get_version(conn) ) - - # Reclaim disk space after table-rebuild migrations - if version < 19: - logger.info("Running VACUUM to reclaim disk space (this may take a moment)...") - await conn.execute("VACUUM") - logger.info("VACUUM complete") else: logger.debug("Schema up to date at version %d", version) @@ -1211,3 +1212,43 @@ async def _migrate_019_drop_messages_unique_constraint(conn: aiosqlite.Connectio await conn.commit() logger.info("messages table rebuilt without UNIQUE constraint") + + +async def _migrate_020_enable_wal_and_auto_vacuum(conn: aiosqlite.Connection) -> None: + """ + Enable WAL journal mode and incremental auto-vacuum. + + WAL (Write-Ahead Logging): + - Faster writes: appends to a WAL file instead of rewriting the main DB + - Concurrent reads during writes (readers don't block writers) + - No journal file create/delete churn on every commit + + Incremental auto-vacuum: + - Pages freed by DELETE become reclaimable without a full VACUUM + - Call PRAGMA incremental_vacuum to reclaim on demand + - Less overhead than FULL auto-vacuum (which reorganizes on every commit) + + auto_vacuum mode change requires a VACUUM to restructure the file. + The VACUUM is performed before switching to WAL so it runs under the + current journal mode; WAL is then set as the final step. + """ + # Check current auto_vacuum mode + cursor = await conn.execute("PRAGMA auto_vacuum") + row = await cursor.fetchone() + current_auto_vacuum = row[0] if row else 0 + + if current_auto_vacuum != 2: # 2 = INCREMENTAL + logger.info("Switching auto_vacuum to INCREMENTAL (requires VACUUM)...") + await conn.execute("PRAGMA auto_vacuum = INCREMENTAL") + await conn.execute("VACUUM") + logger.info("VACUUM complete, auto_vacuum set to INCREMENTAL") + else: + logger.debug("auto_vacuum already INCREMENTAL, skipping VACUUM") + + # Enable WAL mode (idempotent — returns current mode) + cursor = await conn.execute("PRAGMA journal_mode = WAL") + row = await cursor.fetchone() + mode = row[0] if row else "unknown" + logger.info("Journal mode set to %s", mode) + + await conn.commit() diff --git a/tests/test_migrations.py b/tests/test_migrations.py index d4f1fbb..5093d55 100644 --- a/tests/test_migrations.py +++ b/tests/test_migrations.py @@ -100,8 +100,8 @@ class TestMigration001: # Run migrations applied = await run_migrations(conn) - assert applied == 19 # All 17 migrations run - assert await get_version(conn) == 19 + assert applied == 20 # All 17 migrations run + assert await get_version(conn) == 20 # Verify columns exist by inserting and selecting await conn.execute( @@ -183,9 +183,9 @@ class TestMigration001: applied1 = await run_migrations(conn) applied2 = await run_migrations(conn) - assert applied1 == 19 # All 19 migrations run + assert applied1 == 20 # All 20 migrations run assert applied2 == 0 # No migrations on second run - assert await get_version(conn) == 19 + assert await get_version(conn) == 20 finally: await conn.close() @@ -246,8 +246,8 @@ class TestMigration001: applied = await run_migrations(conn) # All 17 migrations applied (version incremented) but no error - assert applied == 19 - assert await get_version(conn) == 19 + assert applied == 20 + assert await get_version(conn) == 20 finally: await conn.close() @@ -374,10 +374,10 @@ class TestMigration013: ) await conn.commit() - # Run migration 13 (plus 14-19 which also run) + # Run migration 13 (plus 14-20 which also run) applied = await run_migrations(conn) - assert applied == 7 - assert await get_version(conn) == 19 + assert applied == 8 + assert await get_version(conn) == 20 # Verify bots array was created with migrated data cursor = await conn.execute("SELECT bots FROM app_settings WHERE id = 1") @@ -497,7 +497,7 @@ class TestMigration018: assert await cursor.fetchone() is not None await run_migrations(conn) - assert await get_version(conn) == 19 + assert await get_version(conn) == 20 # Verify autoindex is gone cursor = await conn.execute( @@ -571,8 +571,8 @@ class TestMigration018: await conn.commit() applied = await run_migrations(conn) - assert applied == 2 # Migrations 18+19 run (but both skip internally) - assert await get_version(conn) == 19 + assert applied == 3 # Migrations 18+19+20 run (18+19 skip internally) + assert await get_version(conn) == 20 finally: await conn.close() @@ -644,7 +644,7 @@ class TestMigration019: assert await cursor.fetchone() is not None await run_migrations(conn) - assert await get_version(conn) == 19 + assert await get_version(conn) == 20 # Verify autoindex is gone cursor = await conn.execute( @@ -681,3 +681,76 @@ class TestMigration019: assert await cursor.fetchone() is not None finally: await conn.close() + + +class TestMigration020: + """Test migration 020: enable WAL mode and incremental auto-vacuum.""" + + @pytest.mark.asyncio + async def test_migration_enables_wal_and_incremental_auto_vacuum(self, tmp_path): + """Migration switches journal mode to WAL and auto_vacuum to INCREMENTAL.""" + db_path = str(tmp_path / "test.db") + conn = await aiosqlite.connect(db_path) + conn.row_factory = aiosqlite.Row + try: + await set_version(conn, 19) + + # Create minimal tables so migration 20 can run + await conn.execute( + "CREATE TABLE raw_packets (id INTEGER PRIMARY KEY, data BLOB NOT NULL)" + ) + await conn.execute( + "CREATE TABLE messages (id INTEGER PRIMARY KEY, text TEXT NOT NULL)" + ) + await conn.commit() + + # Verify defaults before migration + cursor = await conn.execute("PRAGMA auto_vacuum") + assert (await cursor.fetchone())[0] == 0 # NONE + + cursor = await conn.execute("PRAGMA journal_mode") + assert (await cursor.fetchone())[0] == "delete" + + applied = await run_migrations(conn) + assert applied == 1 + assert await get_version(conn) == 20 + + # Verify WAL mode + cursor = await conn.execute("PRAGMA journal_mode") + assert (await cursor.fetchone())[0] == "wal" + + # Verify incremental auto-vacuum + cursor = await conn.execute("PRAGMA auto_vacuum") + assert (await cursor.fetchone())[0] == 2 # INCREMENTAL + finally: + await conn.close() + + @pytest.mark.asyncio + async def test_migration_is_idempotent(self, tmp_path): + """Running migration 20 twice doesn't error or re-VACUUM.""" + db_path = str(tmp_path / "test.db") + conn = await aiosqlite.connect(db_path) + conn.row_factory = aiosqlite.Row + try: + # Set up as if already at version 20 with WAL + incremental + await conn.execute("PRAGMA auto_vacuum = INCREMENTAL") + await conn.execute("PRAGMA journal_mode = WAL") + await conn.execute( + "CREATE TABLE raw_packets (id INTEGER PRIMARY KEY, data BLOB NOT NULL)" + ) + await conn.execute( + "CREATE TABLE messages (id INTEGER PRIMARY KEY, text TEXT NOT NULL)" + ) + await conn.commit() + await set_version(conn, 20) + + applied = await run_migrations(conn) + assert applied == 0 # Already at version 20 + + # Still WAL + INCREMENTAL + cursor = await conn.execute("PRAGMA journal_mode") + assert (await cursor.fetchone())[0] == "wal" + cursor = await conn.execute("PRAGMA auto_vacuum") + assert (await cursor.fetchone())[0] == 2 + finally: + await conn.close()