From e22514882f03ffdafb81d89a3fc39181d7a9224f Mon Sep 17 00:00:00 2001 From: agessaman Date: Wed, 8 Jul 2026 00:02:49 -0700 Subject: [PATCH] perf: force timestamp range scan for windowed packet-stats GROUP BYs The GROUP BY type and GROUP BY drop_reason sub-queries in get_packet_stats (and get_packet_type_stats) filter by a time window but let the planner pick idx_packets_type / idx_packets_transmitted to get grouping for free. It then heap-checks the timestamp filter across the entire table, turning a bounded window into a full scan: on a 1.5M-row packets table the 24h stats load spent ~4.8s (type) + ~2.3s (drop_reason) instead of ~0.1s each. Pin these to idx_packets_timestamp with INDEXED BY so they range-scan the window (~50k rows) and group via a small temp b-tree. Verified on the live DB: 4.80s -> 0.10s and 2.32s -> 0.10s. Unlike a covering index this adds no write-path cost on the packet-insert hot path. --- repeater/data_acquisition/sqlite_handler.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/repeater/data_acquisition/sqlite_handler.py b/repeater/data_acquisition/sqlite_handler.py index 9a7183c..b4ecc61 100644 --- a/repeater/data_acquisition/sqlite_handler.py +++ b/repeater/data_acquisition/sqlite_handler.py @@ -973,10 +973,16 @@ class SQLiteHandler: (cutoff,), ).fetchone() + # INDEXED BY forces the timestamp range scan. Without it the + # planner picks idx_packets_type / idx_packets_transmitted to get + # grouping for free, then heap-checks the timestamp filter across + # the entire table — turning a bounded window into a full scan + # (~5s vs ~0.1s at 1.5M rows). A small temp b-tree over the + # windowed rows is far cheaper. types = conn.execute( """ SELECT type, COUNT(*) as count - FROM packets + FROM packets INDEXED BY idx_packets_timestamp WHERE timestamp > ? GROUP BY type ORDER BY count DESC @@ -987,7 +993,7 @@ class SQLiteHandler: drop_reasons = conn.execute( """ SELECT drop_reason, COUNT(*) as count - FROM packets + FROM packets INDEXED BY idx_packets_timestamp WHERE timestamp > ? AND transmitted = 0 AND drop_reason IS NOT NULL GROUP BY drop_reason ORDER BY count DESC @@ -1314,10 +1320,12 @@ class SQLiteHandler: with self._connect() as conn: conn.row_factory = sqlite3.Row + # See get_packet_stats: force the timestamp range scan so the + # windowed GROUP BY doesn't degrade into a full-table scan. type_rows = conn.execute( """ SELECT type, COUNT(*) as count - FROM packets + FROM packets INDEXED BY idx_packets_timestamp WHERE timestamp > ? GROUP BY type """,