From a55166989ea12fcbd9b415797a1b273170a49bb5 Mon Sep 17 00:00:00 2001 From: Jack Kingsman Date: Sat, 28 Feb 2026 20:10:38 -0800 Subject: [PATCH] Improve performance on unread endpoint --- app/migrations.py | 29 +++++++++++++++++++++++++++++ tests/test_migrations.py | 38 +++++++++++++++++++------------------- 2 files changed, 48 insertions(+), 19 deletions(-) diff --git a/app/migrations.py b/app/migrations.py index 27b08e8..2d689f0 100644 --- a/app/migrations.py +++ b/app/migrations.py @@ -233,6 +233,13 @@ async def run_migrations(conn: aiosqlite.Connection) -> int: await set_version(conn, 28) applied += 1 + # Migration 29: Add covering index for unread counts query + if version < 29: + logger.info("Applying migration 29: add covering index for unread counts query") + await _migrate_029_add_unread_covering_index(conn) + await set_version(conn, 29) + applied += 1 + if applied > 0: logger.info( "Applied %d migration(s), schema now at version %d", applied, await get_version(conn) @@ -1790,3 +1797,25 @@ async def _migrate_028_payload_hash_text_to_blob(conn: aiosqlite.Connection) -> await conn.commit() logger.info("Converted %d payload_hash values from TEXT to BLOB", total) + + +async def _migrate_029_add_unread_covering_index(conn: aiosqlite.Connection) -> None: + """ + Add a covering index for the unread counts query. + + The /api/read-state/unreads endpoint runs three queries against messages. + The last-message-times query (GROUP BY type, conversation_key + MAX(received_at)) + was doing a full table scan. This covering index lets SQLite resolve the + grouping and MAX entirely from the index without touching the table. + It also improves the unread count queries which filter on outgoing and received_at. + """ + # Guard: table or columns may not exist in partial-schema test setups + cursor = await conn.execute("PRAGMA table_info(messages)") + columns = {row[1] for row in await cursor.fetchall()} + required = {"type", "conversation_key", "outgoing", "received_at"} + if required <= columns: + await conn.execute( + "CREATE INDEX IF NOT EXISTS idx_messages_unread_covering " + "ON messages(type, conversation_key, outgoing, received_at)" + ) + await conn.commit() diff --git a/tests/test_migrations.py b/tests/test_migrations.py index 95c9fa1..987e16d 100644 --- a/tests/test_migrations.py +++ b/tests/test_migrations.py @@ -100,8 +100,8 @@ class TestMigration001: # Run migrations applied = await run_migrations(conn) - assert applied == 28 # All migrations run - assert await get_version(conn) == 28 + assert applied == 29 # All migrations run + assert await get_version(conn) == 29 # Verify columns exist by inserting and selecting await conn.execute( @@ -183,9 +183,9 @@ class TestMigration001: applied1 = await run_migrations(conn) applied2 = await run_migrations(conn) - assert applied1 == 28 # All migrations run + assert applied1 == 29 # All migrations run assert applied2 == 0 # No migrations on second run - assert await get_version(conn) == 28 + assert await get_version(conn) == 29 finally: await conn.close() @@ -246,8 +246,8 @@ class TestMigration001: applied = await run_migrations(conn) # All migrations applied (version incremented) but no error - assert applied == 28 - assert await get_version(conn) == 28 + assert applied == 29 + assert await get_version(conn) == 29 finally: await conn.close() @@ -376,8 +376,8 @@ class TestMigration013: # Run migration 13 (plus 14-27 which also run) applied = await run_migrations(conn) - assert applied == 16 - assert await get_version(conn) == 28 + assert applied == 17 + assert await get_version(conn) == 29 # Verify bots array was created with migrated data cursor = await conn.execute("SELECT bots FROM app_settings WHERE id = 1") @@ -497,7 +497,7 @@ class TestMigration018: assert await cursor.fetchone() is not None await run_migrations(conn) - assert await get_version(conn) == 28 + assert await get_version(conn) == 29 # Verify autoindex is gone cursor = await conn.execute( @@ -575,8 +575,8 @@ class TestMigration018: await conn.commit() applied = await run_migrations(conn) - assert applied == 11 # Migrations 18-28 run (18+19 skip internally) - assert await get_version(conn) == 28 + assert applied == 12 # Migrations 18-29 run (18+19 skip internally) + assert await get_version(conn) == 29 finally: await conn.close() @@ -648,7 +648,7 @@ class TestMigration019: assert await cursor.fetchone() is not None await run_migrations(conn) - assert await get_version(conn) == 28 + assert await get_version(conn) == 29 # Verify autoindex is gone cursor = await conn.execute( @@ -714,8 +714,8 @@ class TestMigration020: assert (await cursor.fetchone())[0] == "delete" applied = await run_migrations(conn) - assert applied == 9 # Migrations 20-28 - assert await get_version(conn) == 28 + assert applied == 10 # Migrations 20-29 + assert await get_version(conn) == 29 # Verify WAL mode cursor = await conn.execute("PRAGMA journal_mode") @@ -745,7 +745,7 @@ class TestMigration020: await set_version(conn, 20) applied = await run_migrations(conn) - assert applied == 8 # Migrations 21-28 still run + assert applied == 9 # Migrations 21-29 still run # Still WAL + INCREMENTAL cursor = await conn.execute("PRAGMA journal_mode") @@ -803,8 +803,8 @@ class TestMigration028: await conn.commit() applied = await run_migrations(conn) - assert applied == 1 - assert await get_version(conn) == 28 + assert applied == 2 + assert await get_version(conn) == 29 # Verify payload_hash column is now BLOB cursor = await conn.execute("PRAGMA table_info(raw_packets)") @@ -873,8 +873,8 @@ class TestMigration028: await conn.commit() applied = await run_migrations(conn) - assert applied == 1 # Version still bumped - assert await get_version(conn) == 28 + assert applied == 2 # Version still bumped + assert await get_version(conn) == 29 # Verify data unchanged cursor = await conn.execute("SELECT payload_hash FROM raw_packets")