From 727ac913de1bdad545feb1966599e63e4a32c5da Mon Sep 17 00:00:00 2001 From: Jack Kingsman Date: Sat, 28 Feb 2026 21:00:16 -0800 Subject: [PATCH] Add more efficient message pagination index to eliminate temporary b-tree indexing --- app/database.py | 1 - app/migrations.py | 33 +++++++++++++++++++++++++++++++++ tests/test_migrations.py | 38 +++++++++++++++++++------------------- 3 files changed, 52 insertions(+), 20 deletions(-) diff --git a/app/database.py b/app/database.py index 0c25ce4..637fd6f 100644 --- a/app/database.py +++ b/app/database.py @@ -84,7 +84,6 @@ CREATE TABLE IF NOT EXISTS contact_name_history ( FOREIGN KEY (public_key) REFERENCES contacts(public_key) ); -CREATE INDEX IF NOT EXISTS idx_messages_conversation ON messages(type, conversation_key); CREATE INDEX IF NOT EXISTS idx_messages_received ON messages(received_at); CREATE UNIQUE INDEX IF NOT EXISTS idx_messages_dedup_null_safe ON messages(type, conversation_key, text, COALESCE(sender_timestamp, 0)); diff --git a/app/migrations.py b/app/migrations.py index 2d689f0..5194a9c 100644 --- a/app/migrations.py +++ b/app/migrations.py @@ -240,6 +240,13 @@ async def run_migrations(conn: aiosqlite.Connection) -> int: await set_version(conn, 29) applied += 1 + # Migration 30: Add pagination index, drop redundant idx_messages_conversation + if version < 30: + logger.info("Applying migration 30: add pagination index for message queries") + await _migrate_030_add_pagination_index(conn) + await set_version(conn, 30) + applied += 1 + if applied > 0: logger.info( "Applied %d migration(s), schema now at version %d", applied, await get_version(conn) @@ -1819,3 +1826,29 @@ async def _migrate_029_add_unread_covering_index(conn: aiosqlite.Connection) -> "ON messages(type, conversation_key, outgoing, received_at)" ) await conn.commit() + + +async def _migrate_030_add_pagination_index(conn: aiosqlite.Connection) -> None: + """ + Add a composite index for message pagination and drop the now-redundant + idx_messages_conversation. + + The pagination query (ORDER BY received_at DESC, id DESC LIMIT N) hits a + temp B-tree sort without this index. With it, SQLite walks the index in + order and stops after N rows — critical for channels with 30K+ messages. + + idx_messages_conversation(type, conversation_key) is a strict prefix of + both this index and idx_messages_unread_covering, so SQLite never picks it. + Dropping it saves ~6 MB and one index to maintain per INSERT. + """ + # Guard: table or columns may not exist in partial-schema test setups + cursor = await conn.execute("PRAGMA table_info(messages)") + columns = {row[1] for row in await cursor.fetchall()} + required = {"type", "conversation_key", "received_at", "id"} + if required <= columns: + await conn.execute( + "CREATE INDEX IF NOT EXISTS idx_messages_pagination " + "ON messages(type, conversation_key, received_at DESC, id DESC)" + ) + await conn.execute("DROP INDEX IF EXISTS idx_messages_conversation") + await conn.commit() diff --git a/tests/test_migrations.py b/tests/test_migrations.py index 987e16d..b8a7f11 100644 --- a/tests/test_migrations.py +++ b/tests/test_migrations.py @@ -100,8 +100,8 @@ class TestMigration001: # Run migrations applied = await run_migrations(conn) - assert applied == 29 # All migrations run - assert await get_version(conn) == 29 + assert applied == 30 # All migrations run + assert await get_version(conn) == 30 # Verify columns exist by inserting and selecting await conn.execute( @@ -183,9 +183,9 @@ class TestMigration001: applied1 = await run_migrations(conn) applied2 = await run_migrations(conn) - assert applied1 == 29 # All migrations run + assert applied1 == 30 # All migrations run assert applied2 == 0 # No migrations on second run - assert await get_version(conn) == 29 + assert await get_version(conn) == 30 finally: await conn.close() @@ -246,8 +246,8 @@ class TestMigration001: applied = await run_migrations(conn) # All migrations applied (version incremented) but no error - assert applied == 29 - assert await get_version(conn) == 29 + assert applied == 30 + assert await get_version(conn) == 30 finally: await conn.close() @@ -376,8 +376,8 @@ class TestMigration013: # Run migration 13 (plus 14-27 which also run) applied = await run_migrations(conn) - assert applied == 17 - assert await get_version(conn) == 29 + assert applied == 18 + assert await get_version(conn) == 30 # Verify bots array was created with migrated data cursor = await conn.execute("SELECT bots FROM app_settings WHERE id = 1") @@ -497,7 +497,7 @@ class TestMigration018: assert await cursor.fetchone() is not None await run_migrations(conn) - assert await get_version(conn) == 29 + assert await get_version(conn) == 30 # Verify autoindex is gone cursor = await conn.execute( @@ -575,8 +575,8 @@ class TestMigration018: await conn.commit() applied = await run_migrations(conn) - assert applied == 12 # Migrations 18-29 run (18+19 skip internally) - assert await get_version(conn) == 29 + assert applied == 13 # Migrations 18-30 run (18+19 skip internally) + assert await get_version(conn) == 30 finally: await conn.close() @@ -648,7 +648,7 @@ class TestMigration019: assert await cursor.fetchone() is not None await run_migrations(conn) - assert await get_version(conn) == 29 + assert await get_version(conn) == 30 # Verify autoindex is gone cursor = await conn.execute( @@ -714,8 +714,8 @@ class TestMigration020: assert (await cursor.fetchone())[0] == "delete" applied = await run_migrations(conn) - assert applied == 10 # Migrations 20-29 - assert await get_version(conn) == 29 + assert applied == 11 # Migrations 20-30 + assert await get_version(conn) == 30 # Verify WAL mode cursor = await conn.execute("PRAGMA journal_mode") @@ -745,7 +745,7 @@ class TestMigration020: await set_version(conn, 20) applied = await run_migrations(conn) - assert applied == 9 # Migrations 21-29 still run + assert applied == 10 # Migrations 21-30 still run # Still WAL + INCREMENTAL cursor = await conn.execute("PRAGMA journal_mode") @@ -803,8 +803,8 @@ class TestMigration028: await conn.commit() applied = await run_migrations(conn) - assert applied == 2 - assert await get_version(conn) == 29 + assert applied == 3 + assert await get_version(conn) == 30 # Verify payload_hash column is now BLOB cursor = await conn.execute("PRAGMA table_info(raw_packets)") @@ -873,8 +873,8 @@ class TestMigration028: await conn.commit() applied = await run_migrations(conn) - assert applied == 2 # Version still bumped - assert await get_version(conn) == 29 + assert applied == 3 # Version still bumped + assert await get_version(conn) == 30 # Verify data unchanged cursor = await conn.execute("SELECT payload_hash FROM raw_packets")