From b0e19b13af643a497dbe15dbae715d83f1b4c195 Mon Sep 17 00:00:00 2001
From: Lloyd <lloyd@rightup.co.uk>
Date: Tue, 3 Feb 2026 10:21:59 +0000
Subject: [PATCH] Add  bulk packet retrieval API with gzip compression and
 pagination support.

---
 repeater/data_acquisition/sqlite_handler.py   |  6 ++-
 .../data_acquisition/storage_collector.py     |  3 +-
 repeater/web/api_endpoints.py                 | 39 +++++++++++++++++++
 3 files changed, 45 insertions(+), 3 deletions(-)

diff --git a/repeater/data_acquisition/sqlite_handler.py b/repeater/data_acquisition/sqlite_handler.py
index 205b43d..0f198c9 100644
--- a/repeater/data_acquisition/sqlite_handler.py
+++ b/repeater/data_acquisition/sqlite_handler.py
@@ -560,7 +560,8 @@ class SQLiteHandler:
                            route: Optional[int] = None,
                            start_timestamp: Optional[float] = None,
                            end_timestamp: Optional[float] = None,
-                           limit: int = 1000) -> list:
+                           limit: int = 1000,
+                           offset: int = 0) -> list:
         try:
             with sqlite3.connect(self.sqlite_path) as conn:
                 conn.row_factory = sqlite3.Row
@@ -599,8 +600,9 @@ class SQLiteHandler:
                 else:
                     query = base_query
                 
-                query += " ORDER BY timestamp DESC LIMIT ?"
+                query += " ORDER BY timestamp DESC LIMIT ? OFFSET ?"
                 params.append(limit)
+                params.append(offset)
                 
                 packets = conn.execute(query, params).fetchall()
                 
diff --git a/repeater/data_acquisition/storage_collector.py b/repeater/data_acquisition/storage_collector.py
index 241e53b..1019415 100644
--- a/repeater/data_acquisition/storage_collector.py
+++ b/repeater/data_acquisition/storage_collector.py
@@ -216,9 +216,10 @@ class StorageCollector:
         start_timestamp: Optional[float] = None,
         end_timestamp: Optional[float] = None,
         limit: int = 1000,
+        offset: int = 0,
     ) -> list:
         return self.sqlite_handler.get_filtered_packets(
-            packet_type, route, start_timestamp, end_timestamp, limit
+            packet_type, route, start_timestamp, end_timestamp, limit, offset
         )
 
     def get_packet_by_hash(self, packet_hash: str) -> Optional[dict]:
diff --git a/repeater/web/api_endpoints.py b/repeater/web/api_endpoints.py
index c45f91a..0051a97 100644
--- a/repeater/web/api_endpoints.py
+++ b/repeater/web/api_endpoints.py
@@ -841,6 +841,45 @@ class APIEndpoints:
             logger.error(f"Error getting recent packets: {e}")
             return self._error(e)
 
+    @cherrypy.expose
+    @cherrypy.tools.gzip(compress_level=6)
+    @cherrypy.tools.json_out()
+    def bulk_packets(self, limit=1000, offset=0):
+        """
+        Optimized bulk packet retrieval with gzip compression and DB-level pagination.
+        """
+        try:
+            # Enforce reasonable limits
+            limit = min(int(limit), 10000)
+            offset = max(int(offset), 0)
+            
+            # Get packets from storage with TRUE DB-level pagination
+            # Uses SQL "LIMIT ? OFFSET ?" - no Python slicing needed!
+            storage = self._get_storage()
+            packets = storage.get_filtered_packets(
+                packet_type=None,
+                route=None,
+                start_timestamp=None,
+                end_timestamp=None,
+                limit=limit,
+                offset=offset
+            )
+            
+            response = {
+                "success": True,
+                "data": packets,
+                "count": len(packets),
+                "offset": offset,
+                "limit": limit,
+                "compressed": True
+            }
+            
+            return response
+            
+        except Exception as e:
+            logger.error(f"Error getting bulk packets: {e}")
+            return self._error(e)
+
     @cherrypy.expose
     @cherrypy.tools.json_out()
     def filtered_packets(self, start_timestamp=None, end_timestamp=None, limit=1000, type=None, route=None):