WAL + incremental vacuum for space happiness

app/database.py

@@ -79,6 +79,17 @@ class Database:
         Path(self.db_path).parent.mkdir(parents=True, exist_ok=True)
         self._connection = await aiosqlite.connect(self.db_path)
         self._connection.row_factory = aiosqlite.Row
+
+        # WAL mode: faster writes, concurrent readers during writes, no journal file churn.
+        # Persists in the DB file but we set it explicitly on every connection.
+        await self._connection.execute("PRAGMA journal_mode = WAL")
+
+        # Incremental auto-vacuum: freed pages are reclaimable via
+        # PRAGMA incremental_vacuum without a full VACUUM. Must be set before
+        # the first table is created (for new databases); for existing databases
+        # migration 20 handles the one-time VACUUM to restructure the file.
+        await self._connection.execute("PRAGMA auto_vacuum = INCREMENTAL")
+
         await self._connection.executescript(SCHEMA)
         await self._connection.commit()
         logger.debug("Database schema initialized")

app/migrations.py

@@ -170,16 +170,17 @@ async def run_migrations(conn: aiosqlite.Connection) -> int:
         await set_version(conn, 19)
         applied += 1
 
+    # Migration 20: Enable WAL journal mode and incremental auto-vacuum
+    if version < 20:
+        logger.info("Applying migration 20: enable WAL mode and incremental auto-vacuum")
+        await _migrate_020_enable_wal_and_auto_vacuum(conn)
+        await set_version(conn, 20)
+        applied += 1
+
     if applied > 0:
         logger.info(
             "Applied %d migration(s), schema now at version %d", applied, await get_version(conn)
         )
-
-        # Reclaim disk space after table-rebuild migrations
-        if version < 19:
-            logger.info("Running VACUUM to reclaim disk space (this may take a moment)...")
-            await conn.execute("VACUUM")
-            logger.info("VACUUM complete")
     else:
         logger.debug("Schema up to date at version %d", version)
 
@@ -1211,3 +1212,43 @@ async def _migrate_019_drop_messages_unique_constraint(conn: aiosqlite.Connectio
 
     await conn.commit()
     logger.info("messages table rebuilt without UNIQUE constraint")
+
+
+async def _migrate_020_enable_wal_and_auto_vacuum(conn: aiosqlite.Connection) -> None:
+    """
+    Enable WAL journal mode and incremental auto-vacuum.
+
+    WAL (Write-Ahead Logging):
+    - Faster writes: appends to a WAL file instead of rewriting the main DB
+    - Concurrent reads during writes (readers don't block writers)
+    - No journal file create/delete churn on every commit
+
+    Incremental auto-vacuum:
+    - Pages freed by DELETE become reclaimable without a full VACUUM
+    - Call PRAGMA incremental_vacuum to reclaim on demand
+    - Less overhead than FULL auto-vacuum (which reorganizes on every commit)
+
+    auto_vacuum mode change requires a VACUUM to restructure the file.
+    The VACUUM is performed before switching to WAL so it runs under the
+    current journal mode; WAL is then set as the final step.
+    """
+    # Check current auto_vacuum mode
+    cursor = await conn.execute("PRAGMA auto_vacuum")
+    row = await cursor.fetchone()
+    current_auto_vacuum = row[0] if row else 0
+
+    if current_auto_vacuum != 2:  # 2 = INCREMENTAL
+        logger.info("Switching auto_vacuum to INCREMENTAL (requires VACUUM)...")
+        await conn.execute("PRAGMA auto_vacuum = INCREMENTAL")
+        await conn.execute("VACUUM")
+        logger.info("VACUUM complete, auto_vacuum set to INCREMENTAL")
+    else:
+        logger.debug("auto_vacuum already INCREMENTAL, skipping VACUUM")
+
+    # Enable WAL mode (idempotent — returns current mode)
+    cursor = await conn.execute("PRAGMA journal_mode = WAL")
+    row = await cursor.fetchone()
+    mode = row[0] if row else "unknown"
+    logger.info("Journal mode set to %s", mode)
+
+    await conn.commit()

tests/test_migrations.py

@@ -100,8 +100,8 @@ class TestMigration001:
             # Run migrations
             applied = await run_migrations(conn)
 
-            assert applied == 19  # All 17 migrations run
-            assert await get_version(conn) == 19
+            assert applied == 20  # All 17 migrations run
+            assert await get_version(conn) == 20
 
             # Verify columns exist by inserting and selecting
             await conn.execute(
@@ -183,9 +183,9 @@ class TestMigration001:
             applied1 = await run_migrations(conn)
             applied2 = await run_migrations(conn)
 
-            assert applied1 == 19  # All 19 migrations run
+            assert applied1 == 20  # All 20 migrations run
             assert applied2 == 0  # No migrations on second run
-            assert await get_version(conn) == 19
+            assert await get_version(conn) == 20
         finally:
             await conn.close()
 
@@ -246,8 +246,8 @@ class TestMigration001:
             applied = await run_migrations(conn)
 
             # All 17 migrations applied (version incremented) but no error
-            assert applied == 19
-            assert await get_version(conn) == 19
+            assert applied == 20
+            assert await get_version(conn) == 20
         finally:
             await conn.close()
 
@@ -374,10 +374,10 @@ class TestMigration013:
             )
             await conn.commit()
 
-            # Run migration 13 (plus 14-19 which also run)
+            # Run migration 13 (plus 14-20 which also run)
             applied = await run_migrations(conn)
-            assert applied == 7
-            assert await get_version(conn) == 19
+            assert applied == 8
+            assert await get_version(conn) == 20
 
             # Verify bots array was created with migrated data
             cursor = await conn.execute("SELECT bots FROM app_settings WHERE id = 1")
@@ -497,7 +497,7 @@ class TestMigration018:
             assert await cursor.fetchone() is not None
 
             await run_migrations(conn)
-            assert await get_version(conn) == 19
+            assert await get_version(conn) == 20
 
             # Verify autoindex is gone
             cursor = await conn.execute(
@@ -571,8 +571,8 @@ class TestMigration018:
             await conn.commit()
 
             applied = await run_migrations(conn)
-            assert applied == 2  # Migrations 18+19 run (but both skip internally)
-            assert await get_version(conn) == 19
+            assert applied == 3  # Migrations 18+19+20 run (18+19 skip internally)
+            assert await get_version(conn) == 20
         finally:
             await conn.close()
 
@@ -644,7 +644,7 @@ class TestMigration019:
             assert await cursor.fetchone() is not None
 
             await run_migrations(conn)
-            assert await get_version(conn) == 19
+            assert await get_version(conn) == 20
 
             # Verify autoindex is gone
             cursor = await conn.execute(
@@ -681,3 +681,76 @@ class TestMigration019:
             assert await cursor.fetchone() is not None
         finally:
             await conn.close()
+
+
+class TestMigration020:
+    """Test migration 020: enable WAL mode and incremental auto-vacuum."""
+
+    @pytest.mark.asyncio
+    async def test_migration_enables_wal_and_incremental_auto_vacuum(self, tmp_path):
+        """Migration switches journal mode to WAL and auto_vacuum to INCREMENTAL."""
+        db_path = str(tmp_path / "test.db")
+        conn = await aiosqlite.connect(db_path)
+        conn.row_factory = aiosqlite.Row
+        try:
+            await set_version(conn, 19)
+
+            # Create minimal tables so migration 20 can run
+            await conn.execute(
+                "CREATE TABLE raw_packets (id INTEGER PRIMARY KEY, data BLOB NOT NULL)"
+            )
+            await conn.execute(
+                "CREATE TABLE messages (id INTEGER PRIMARY KEY, text TEXT NOT NULL)"
+            )
+            await conn.commit()
+
+            # Verify defaults before migration
+            cursor = await conn.execute("PRAGMA auto_vacuum")
+            assert (await cursor.fetchone())[0] == 0  # NONE
+
+            cursor = await conn.execute("PRAGMA journal_mode")
+            assert (await cursor.fetchone())[0] == "delete"
+
+            applied = await run_migrations(conn)
+            assert applied == 1
+            assert await get_version(conn) == 20
+
+            # Verify WAL mode
+            cursor = await conn.execute("PRAGMA journal_mode")
+            assert (await cursor.fetchone())[0] == "wal"
+
+            # Verify incremental auto-vacuum
+            cursor = await conn.execute("PRAGMA auto_vacuum")
+            assert (await cursor.fetchone())[0] == 2  # INCREMENTAL
+        finally:
+            await conn.close()
+
+    @pytest.mark.asyncio
+    async def test_migration_is_idempotent(self, tmp_path):
+        """Running migration 20 twice doesn't error or re-VACUUM."""
+        db_path = str(tmp_path / "test.db")
+        conn = await aiosqlite.connect(db_path)
+        conn.row_factory = aiosqlite.Row
+        try:
+            # Set up as if already at version 20 with WAL + incremental
+            await conn.execute("PRAGMA auto_vacuum = INCREMENTAL")
+            await conn.execute("PRAGMA journal_mode = WAL")
+            await conn.execute(
+                "CREATE TABLE raw_packets (id INTEGER PRIMARY KEY, data BLOB NOT NULL)"
+            )
+            await conn.execute(
+                "CREATE TABLE messages (id INTEGER PRIMARY KEY, text TEXT NOT NULL)"
+            )
+            await conn.commit()
+            await set_version(conn, 20)
+
+            applied = await run_migrations(conn)
+            assert applied == 0  # Already at version 20
+
+            # Still WAL + INCREMENTAL
+            cursor = await conn.execute("PRAGMA journal_mode")
+            assert (await cursor.fetchone())[0] == "wal"
+            cursor = await conn.execute("PRAGMA auto_vacuum")
+            assert (await cursor.fetchone())[0] == 2
+        finally:
+            await conn.close()