adam/Remote-Terminal-for-MeshCore
1
0
Fork 0
mirror of https://github.com/jkingsman/Remote-Terminal-for-MeshCore.git synced 2026-08-08 09:43:03 +02:00
Code Issues Packages Projects Releases Wiki Activity

Add database-level payload dedupe

Browse Source
This commit is contained in:
Jack Kingsman
2026-01-17 17:06:25 -08:00
parent 4219f96894
commit 850ce2d6c7
17 changed files with 412 additions and 243 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/test_api.py
+112 -9
View File
@@ -477,15 +477,19 @@ class TestRawPacketRepository:
conn = await aiosqlite.connect(":memory:")
conn.row_factory = aiosqlite.Row
# Create the raw_packets table
# Create the raw_packets table with payload_hash for deduplication
await conn.execute("""
CREATE TABLE raw_packets (
id INTEGER PRIMARY KEY,
timestamp INTEGER NOT NULL,
data BLOB NOT NULL UNIQUE,
message_id INTEGER
data BLOB NOT NULL,
message_id INTEGER,
payload_hash TEXT
)
""")
await conn.execute(
"CREATE UNIQUE INDEX IF NOT EXISTS idx_raw_packets_payload_hash ON raw_packets(payload_hash)"
)
await conn.commit()
# Patch the db._connection to use our test connection
@@ -494,10 +498,11 @@ class TestRawPacketRepository:
try:
packet_data = b"\x01\x02\x03\x04\x05"
packet_id = await RawPacketRepository.create(packet_data, 1234567890)
packet_id, is_new = await RawPacketRepository.create(packet_data, 1234567890)
assert packet_id is not None
assert packet_id > 0
assert is_new is True
finally:
db._connection = original_conn
await conn.close()
@@ -514,15 +519,19 @@ class TestRawPacketRepository:
conn = await aiosqlite.connect(":memory:")
conn.row_factory = aiosqlite.Row
# Create the raw_packets table
# Create the raw_packets table with payload_hash for deduplication
await conn.execute("""
CREATE TABLE raw_packets (
id INTEGER PRIMARY KEY,
timestamp INTEGER NOT NULL,
data BLOB NOT NULL UNIQUE,
message_id INTEGER
data BLOB NOT NULL,
message_id INTEGER,
payload_hash TEXT
)
""")
await conn.execute(
"CREATE UNIQUE INDEX IF NOT EXISTS idx_raw_packets_payload_hash ON raw_packets(payload_hash)"
)
await conn.commit()
# Patch the db._connection to use our test connection
@@ -533,12 +542,106 @@ class TestRawPacketRepository:
packet1 = b"\x01\x02\x03"
packet2 = b"\x04\x05\x06"
id1 = await RawPacketRepository.create(packet1, 1234567890)
id2 = await RawPacketRepository.create(packet2, 1234567891)
id1, is_new1 = await RawPacketRepository.create(packet1, 1234567890)
id2, is_new2 = await RawPacketRepository.create(packet2, 1234567891)
assert id1 is not None
assert id2 is not None
assert id1 != id2
assert is_new1 is True
assert is_new2 is True
finally:
db._connection = original_conn
await conn.close()
@pytest.mark.asyncio
async def test_duplicate_packet_returns_existing_id(self):
"""Inserting same payload twice returns existing ID and is_new=False."""
import aiosqlite
from app.database import db
from app.repository import RawPacketRepository
conn = await aiosqlite.connect(":memory:")
conn.row_factory = aiosqlite.Row
# Create the raw_packets table with payload_hash for deduplication
await conn.execute("""
CREATE TABLE raw_packets (
id INTEGER PRIMARY KEY,
timestamp INTEGER NOT NULL,
data BLOB NOT NULL,
message_id INTEGER,
payload_hash TEXT
)
""")
await conn.execute(
"CREATE UNIQUE INDEX IF NOT EXISTS idx_raw_packets_payload_hash ON raw_packets(payload_hash)"
)
await conn.commit()
original_conn = db._connection
db._connection = conn
try:
# Same packet data inserted twice
packet_data = b"\x01\x02\x03\x04\x05"
id1, is_new1 = await RawPacketRepository.create(packet_data, 1234567890)
id2, is_new2 = await RawPacketRepository.create(packet_data, 1234567891)
# Both should return the same ID
assert id1 == id2
# First is new, second is not
assert is_new1 is True
assert is_new2 is False
finally:
db._connection = original_conn
await conn.close()
@pytest.mark.asyncio
async def test_malformed_packet_uses_full_data_hash(self):
"""Malformed packets (can't extract payload) hash full data for dedup."""
import aiosqlite
from app.database import db
from app.repository import RawPacketRepository
conn = await aiosqlite.connect(":memory:")
conn.row_factory = aiosqlite.Row
await conn.execute("""
CREATE TABLE raw_packets (
id INTEGER PRIMARY KEY,
timestamp INTEGER NOT NULL,
data BLOB NOT NULL,
message_id INTEGER,
payload_hash TEXT
)
""")
await conn.execute(
"CREATE UNIQUE INDEX IF NOT EXISTS idx_raw_packets_payload_hash ON raw_packets(payload_hash)"
)
await conn.commit()
original_conn = db._connection
db._connection = conn
try:
# Single byte is too short to be valid packet (extract_payload returns None)
malformed = b"\x01"
id1, is_new1 = await RawPacketRepository.create(malformed, 1234567890)
id2, is_new2 = await RawPacketRepository.create(malformed, 1234567891)
# Should still deduplicate using full data hash
assert id1 == id2
assert is_new1 is True
assert is_new2 is False
# Different malformed packet should get different ID
different_malformed = b"\x02"
id3, is_new3 = await RawPacketRepository.create(different_malformed, 1234567892)
assert id3 != id1
assert is_new3 is True
finally:
db._connection = original_conn
await conn.close()
tests/test_migrations.py
+7 -7
View File
@@ -83,8 +83,8 @@ class TestMigration001:
# Run migrations
applied = await run_migrations(conn)
assert applied == 3 # All 3 migrations run
assert await get_version(conn) == 3
assert applied == 5 # All 5 migrations run
assert await get_version(conn) == 5
# Verify columns exist by inserting and selecting
await conn.execute(
@@ -149,9 +149,9 @@ class TestMigration001:
applied1 = await run_migrations(conn)
applied2 = await run_migrations(conn)
assert applied1 == 3 # All 3 migrations run
assert applied1 == 5 # All 5 migrations run
assert applied2 == 0 # No migrations on second run
assert await get_version(conn) == 3
assert await get_version(conn) == 5
finally:
await conn.close()
@@ -194,9 +194,9 @@ class TestMigration001:
# Run migrations - should not fail
applied = await run_migrations(conn)
# All 3 migrations applied (version incremented) but no error
assert applied == 3
assert await get_version(conn) == 3
# All 5 migrations applied (version incremented) but no error
assert applied == 5
assert await get_version(conn) == 5
finally:
await conn.close()
tests/test_packet_pipeline.py
+5 -5
View File
@@ -393,7 +393,7 @@ class TestCreateMessageFromDecrypted:
from app.packet_processor import create_message_from_decrypted
# Create a raw packet first (required for the function)
packet_id = await RawPacketRepository.create(b"test_packet_data", 1700000000)
packet_id, _ = await RawPacketRepository.create(b"test_packet_data", 1700000000)
broadcasts, mock_broadcast = captured_broadcasts
@@ -439,7 +439,7 @@ class TestCreateMessageFromDecrypted:
"""create_message_from_decrypted handles messages without sender prefix."""
from app.packet_processor import create_message_from_decrypted
packet_id = await RawPacketRepository.create(b"test_packet_data_2", 1700000000)
packet_id, _ = await RawPacketRepository.create(b"test_packet_data_2", 1700000000)
broadcasts, mock_broadcast = captured_broadcasts
@@ -467,8 +467,8 @@ class TestCreateMessageFromDecrypted:
"""create_message_from_decrypted returns None for duplicate message."""
from app.packet_processor import create_message_from_decrypted
packet_id_1 = await RawPacketRepository.create(b"packet_1", 1700000000)
packet_id_2 = await RawPacketRepository.create(b"packet_2", 1700000001)
packet_id_1, _ = await RawPacketRepository.create(b"packet_1", 1700000000)
packet_id_2, _ = await RawPacketRepository.create(b"packet_2", 1700000001)
broadcasts, mock_broadcast = captured_broadcasts
@@ -505,7 +505,7 @@ class TestCreateMessageFromDecrypted:
"""create_message_from_decrypted links raw packet to created message."""
from app.packet_processor import create_message_from_decrypted
packet_id = await RawPacketRepository.create(b"test_packet", 1700000000)
packet_id, _ = await RawPacketRepository.create(b"test_packet", 1700000000)
broadcasts, mock_broadcast = captured_broadcasts