Add more efficient message pagination index to eliminate temporary b-tree indexing

app/database.py

@@ -84,7 +84,6 @@ CREATE TABLE IF NOT EXISTS contact_name_history (
     FOREIGN KEY (public_key) REFERENCES contacts(public_key)
 );
 
-CREATE INDEX IF NOT EXISTS idx_messages_conversation ON messages(type, conversation_key);
 CREATE INDEX IF NOT EXISTS idx_messages_received ON messages(received_at);
 CREATE UNIQUE INDEX IF NOT EXISTS idx_messages_dedup_null_safe
     ON messages(type, conversation_key, text, COALESCE(sender_timestamp, 0));

app/migrations.py

@@ -240,6 +240,13 @@ async def run_migrations(conn: aiosqlite.Connection) -> int:
         await set_version(conn, 29)
         applied += 1
 
+    # Migration 30: Add pagination index, drop redundant idx_messages_conversation
+    if version < 30:
+        logger.info("Applying migration 30: add pagination index for message queries")
+        await _migrate_030_add_pagination_index(conn)
+        await set_version(conn, 30)
+        applied += 1
+
     if applied > 0:
         logger.info(
             "Applied %d migration(s), schema now at version %d", applied, await get_version(conn)
@@ -1819,3 +1826,29 @@ async def _migrate_029_add_unread_covering_index(conn: aiosqlite.Connection) ->
             "ON messages(type, conversation_key, outgoing, received_at)"
         )
     await conn.commit()
+
+
+async def _migrate_030_add_pagination_index(conn: aiosqlite.Connection) -> None:
+    """
+    Add a composite index for message pagination and drop the now-redundant
+    idx_messages_conversation.
+
+    The pagination query (ORDER BY received_at DESC, id DESC LIMIT N) hits a
+    temp B-tree sort without this index. With it, SQLite walks the index in
+    order and stops after N rows — critical for channels with 30K+ messages.
+
+    idx_messages_conversation(type, conversation_key) is a strict prefix of
+    both this index and idx_messages_unread_covering, so SQLite never picks it.
+    Dropping it saves ~6 MB and one index to maintain per INSERT.
+    """
+    # Guard: table or columns may not exist in partial-schema test setups
+    cursor = await conn.execute("PRAGMA table_info(messages)")
+    columns = {row[1] for row in await cursor.fetchall()}
+    required = {"type", "conversation_key", "received_at", "id"}
+    if required <= columns:
+        await conn.execute(
+            "CREATE INDEX IF NOT EXISTS idx_messages_pagination "
+            "ON messages(type, conversation_key, received_at DESC, id DESC)"
+        )
+        await conn.execute("DROP INDEX IF EXISTS idx_messages_conversation")
+    await conn.commit()

tests/test_migrations.py

@@ -100,8 +100,8 @@ class TestMigration001:
             # Run migrations
             applied = await run_migrations(conn)
 
-            assert applied == 29  # All migrations run
-            assert await get_version(conn) == 29
+            assert applied == 30  # All migrations run
+            assert await get_version(conn) == 30
 
             # Verify columns exist by inserting and selecting
             await conn.execute(
@@ -183,9 +183,9 @@ class TestMigration001:
             applied1 = await run_migrations(conn)
             applied2 = await run_migrations(conn)
 
-            assert applied1 == 29  # All migrations run
+            assert applied1 == 30  # All migrations run
             assert applied2 == 0  # No migrations on second run
-            assert await get_version(conn) == 29
+            assert await get_version(conn) == 30
         finally:
             await conn.close()
 
@@ -246,8 +246,8 @@ class TestMigration001:
             applied = await run_migrations(conn)
 
             # All migrations applied (version incremented) but no error
-            assert applied == 29
-            assert await get_version(conn) == 29
+            assert applied == 30
+            assert await get_version(conn) == 30
         finally:
             await conn.close()
 
@@ -376,8 +376,8 @@ class TestMigration013:
 
             # Run migration 13 (plus 14-27 which also run)
             applied = await run_migrations(conn)
-            assert applied == 17
-            assert await get_version(conn) == 29
+            assert applied == 18
+            assert await get_version(conn) == 30
 
             # Verify bots array was created with migrated data
             cursor = await conn.execute("SELECT bots FROM app_settings WHERE id = 1")
@@ -497,7 +497,7 @@ class TestMigration018:
             assert await cursor.fetchone() is not None
 
             await run_migrations(conn)
-            assert await get_version(conn) == 29
+            assert await get_version(conn) == 30
 
             # Verify autoindex is gone
             cursor = await conn.execute(
@@ -575,8 +575,8 @@ class TestMigration018:
             await conn.commit()
 
             applied = await run_migrations(conn)
-            assert applied == 12  # Migrations 18-29 run (18+19 skip internally)
-            assert await get_version(conn) == 29
+            assert applied == 13  # Migrations 18-30 run (18+19 skip internally)
+            assert await get_version(conn) == 30
         finally:
             await conn.close()
 
@@ -648,7 +648,7 @@ class TestMigration019:
             assert await cursor.fetchone() is not None
 
             await run_migrations(conn)
-            assert await get_version(conn) == 29
+            assert await get_version(conn) == 30
 
             # Verify autoindex is gone
             cursor = await conn.execute(
@@ -714,8 +714,8 @@ class TestMigration020:
             assert (await cursor.fetchone())[0] == "delete"
 
             applied = await run_migrations(conn)
-            assert applied == 10  # Migrations 20-29
-            assert await get_version(conn) == 29
+            assert applied == 11  # Migrations 20-30
+            assert await get_version(conn) == 30
 
             # Verify WAL mode
             cursor = await conn.execute("PRAGMA journal_mode")
@@ -745,7 +745,7 @@ class TestMigration020:
             await set_version(conn, 20)
 
             applied = await run_migrations(conn)
-            assert applied == 9  # Migrations 21-29 still run
+            assert applied == 10  # Migrations 21-30 still run
 
             # Still WAL + INCREMENTAL
             cursor = await conn.execute("PRAGMA journal_mode")
@@ -803,8 +803,8 @@ class TestMigration028:
             await conn.commit()
 
             applied = await run_migrations(conn)
-            assert applied == 2
-            assert await get_version(conn) == 29
+            assert applied == 3
+            assert await get_version(conn) == 30
 
             # Verify payload_hash column is now BLOB
             cursor = await conn.execute("PRAGMA table_info(raw_packets)")
@@ -873,8 +873,8 @@ class TestMigration028:
             await conn.commit()
 
             applied = await run_migrations(conn)
-            assert applied == 2  # Version still bumped
-            assert await get_version(conn) == 29
+            assert applied == 3  # Version still bumped
+            assert await get_version(conn) == 30
 
             # Verify data unchanged
             cursor = await conn.execute("SELECT payload_hash FROM raw_packets")