Fix async db handling. Closes #179.

app/path_utils.py

@@ -9,6 +9,7 @@ The path_len wire byte is packed as [hash_mode:2][hop_count:6]:
 Mode 3 (hash_size=4) is reserved and rejected.
 """
 
+from collections.abc import Iterable
 from dataclasses import dataclass
 
 MAX_PATH_SIZE = 64
@@ -246,30 +247,26 @@ def parse_explicit_hop_route(route_text: str) -> tuple[str, int, int]:
     return "".join(hops), len(hops), hash_size - 1
 
 
-async def bucket_path_hash_widths(cursor, *, batch_size: int = 500) -> dict[str, int | float]:
+def bucket_path_hash_widths(rows: Iterable) -> dict[str, int | float]:
     """Bucket raw packet rows by hop hash width and return counts + percentages.
 
-    *cursor* must be an already-executed async cursor whose rows have a ``data``
+    *rows* must be an already-fetched list whose elements have a ``data``
     column containing raw packet bytes.
     """
     single_byte = 0
     double_byte = 0
     triple_byte = 0
 
-    while True:
-        rows = await cursor.fetchmany(batch_size)
-        if not rows:
-            break
-        for row in rows:
-            envelope = parse_packet_envelope(bytes(row["data"]))
-            if envelope is None:
-                continue
-            if envelope.hash_size == 1:
-                single_byte += 1
-            elif envelope.hash_size == 2:
-                double_byte += 1
-            elif envelope.hash_size == 3:
-                triple_byte += 1
+    for row in rows:
+        envelope = parse_packet_envelope(bytes(row["data"]))
+        if envelope is None:
+            continue
+        if envelope.hash_size == 1:
+            single_byte += 1
+        elif envelope.hash_size == 2:
+            double_byte += 1
+        elif envelope.hash_size == 3:
+            triple_byte += 1
 
     total = single_byte + double_byte + triple_byte
     if total == 0:

app/repository/messages.py

@@ -868,7 +868,8 @@ class MessageRepository:
             """,
             (conversation_key, t_24h),
         )
-        path_hash_width_24h = await bucket_path_hash_widths(cursor3)
+        rows3 = await cursor3.fetchall()
+        path_hash_width_24h = bucket_path_hash_widths(rows3)
 
         return {
             "message_counts": message_counts,

app/repository/raw_packets.py

@@ -74,41 +74,52 @@ class RawPacketRepository:
     async def stream_all_undecrypted(
         batch_size: int = UNDECRYPTED_PACKET_BATCH_SIZE,
     ) -> AsyncIterator[tuple[int, bytes, int]]:
-        """Yield all undecrypted packets as (id, data, timestamp) in bounded batches."""
-        cursor = await db.conn.execute(
-            "SELECT id, data, timestamp FROM raw_packets WHERE message_id IS NULL ORDER BY timestamp ASC"
-        )
-        try:
-            while True:
-                rows = await cursor.fetchmany(batch_size)
-                if not rows:
-                    break
-                for row in rows:
-                    yield (row["id"], bytes(row["data"]), row["timestamp"])
-        finally:
+        """Yield all undecrypted packets as (id, data, timestamp) in bounded batches.
+
+        Uses keyset pagination so each batch is a fresh query with a fully
+        consumed cursor — no open statement held across yield boundaries.
+        """
+        last_id = -1
+        while True:
+            cursor = await db.conn.execute(
+                "SELECT id, data, timestamp FROM raw_packets "
+                "WHERE message_id IS NULL AND id > ? ORDER BY id ASC LIMIT ?",
+                (last_id, batch_size),
+            )
+            rows = await cursor.fetchall()
             await cursor.close()
+            if not rows:
+                break
+            for row in rows:
+                last_id = row["id"]
+                yield (row["id"], bytes(row["data"]), row["timestamp"])
 
     @staticmethod
     async def stream_undecrypted_text_messages(
         batch_size: int = UNDECRYPTED_PACKET_BATCH_SIZE,
     ) -> AsyncIterator[tuple[int, bytes, int]]:
-        """Yield undecrypted TEXT_MESSAGE packets in bounded-size batches."""
-        cursor = await db.conn.execute(
-            "SELECT id, data, timestamp FROM raw_packets WHERE message_id IS NULL ORDER BY timestamp ASC"
-        )
-        try:
-            while True:
-                rows = await cursor.fetchmany(batch_size)
-                if not rows:
-                    break
+        """Yield undecrypted TEXT_MESSAGE packets in bounded-size batches.
 
-                for row in rows:
-                    data = bytes(row["data"])
-                    payload_type = get_packet_payload_type(data)
-                    if payload_type == PayloadType.TEXT_MESSAGE:
-                        yield (row["id"], data, row["timestamp"])
-        finally:
+        Uses keyset pagination so each batch is a fresh query with a fully
+        consumed cursor — no open statement held across yield boundaries.
+        """
+        last_id = -1
+        while True:
+            cursor = await db.conn.execute(
+                "SELECT id, data, timestamp FROM raw_packets "
+                "WHERE message_id IS NULL AND id > ? ORDER BY id ASC LIMIT ?",
+                (last_id, batch_size),
+            )
+            rows = await cursor.fetchall()
             await cursor.close()
+            if not rows:
+                break
+            for row in rows:
+                last_id = row["id"]
+                data = bytes(row["data"])
+                payload_type = get_packet_payload_type(data)
+                if payload_type == PayloadType.TEXT_MESSAGE:
+                    yield (row["id"], data, row["timestamp"])
 
     @staticmethod
     async def count_undecrypted_text_messages(

app/repository/settings.py

@@ -13,7 +13,6 @@ SECONDS_1H = 3600
 SECONDS_24H = 86400
 SECONDS_72H = 259200
 SECONDS_7D = 604800
-RAW_PACKET_STATS_BATCH_SIZE = 500
 
 
 class AppSettingsRepository:
@@ -302,7 +301,8 @@ class StatisticsRepository:
             "SELECT data FROM raw_packets WHERE timestamp >= ?",
             (now - SECONDS_24H,),
         )
-        return await bucket_path_hash_widths(cursor, batch_size=RAW_PACKET_STATS_BATCH_SIZE)
+        rows = await cursor.fetchall()
+        return bucket_path_hash_widths(rows)
 
     @staticmethod
     async def get_all() -> dict: