From 1fe3fb1779e7d4fadca2ea4914c439983f6afbfb Mon Sep 17 00:00:00 2001 From: agessaman Date: Tue, 14 Jul 2026 15:51:40 -0700 Subject: [PATCH] fix(storage): retain signed advert packets --- repeater/companion/frame_server.py | 11 ++ repeater/data_acquisition/sqlite_handler.py | 38 ++++- tests/test_companion_advert_persistence.py | 164 ++++++++++++++++++++ 3 files changed, 208 insertions(+), 5 deletions(-) create mode 100644 tests/test_companion_advert_persistence.py diff --git a/repeater/companion/frame_server.py b/repeater/companion/frame_server.py index 770f90d..b92e5e5 100644 --- a/repeater/companion/frame_server.py +++ b/repeater/companion/frame_server.py @@ -123,6 +123,16 @@ class CompanionFrameServer(_BaseFrameServer): def _contact_to_dict(c) -> dict: """Convert a Contact object to a persistence dict.""" pk = c.public_key if isinstance(c.public_key, bytes) else bytes.fromhex(c.public_key) + raw_advert = getattr(c, "last_advert_packet", None) + if isinstance(raw_advert, bytearray): + raw_advert = bytes(raw_advert) + elif isinstance(raw_advert, str): + try: + raw_advert = bytes.fromhex(raw_advert) + except ValueError: + raw_advert = None + elif not isinstance(raw_advert, bytes): + raw_advert = None return { "pubkey": pk, "name": c.name, @@ -135,6 +145,7 @@ class CompanionFrameServer(_BaseFrameServer): else (bytes.fromhex(c.out_path) if c.out_path else b"") ), "last_advert_timestamp": c.last_advert_timestamp, + "last_advert_packet": raw_advert, "lastmod": c.lastmod, "gps_lat": c.gps_lat, "gps_lon": c.gps_lon, diff --git a/repeater/data_acquisition/sqlite_handler.py b/repeater/data_acquisition/sqlite_handler.py index 73becd8..7ac67fa 100644 --- a/repeater/data_acquisition/sqlite_handler.py +++ b/repeater/data_acquisition/sqlite_handler.py @@ -403,6 +403,7 @@ class SQLiteHandler: out_path_len INTEGER NOT NULL DEFAULT -1, out_path BLOB, last_advert_timestamp INTEGER NOT NULL DEFAULT 0, + last_advert_packet BLOB, lastmod INTEGER NOT NULL DEFAULT 0, gps_lat REAL NOT NULL DEFAULT 0, gps_lon REAL NOT NULL DEFAULT 0, @@ -615,6 +616,27 @@ class SQLiteHandler: ) logger.info(f"Migration '{migration_name}' applied successfully") + # Migration 11: Preserve the exact verified ADVERT wire packet + # for MeshCore-compatible CMD_EXPORT_CONTACT after restart. + migration_name = "add_last_advert_packet_to_companion_contacts" + existing = conn.execute( + "SELECT migration_name FROM migrations WHERE migration_name = ?", + (migration_name,), + ).fetchone() + if not existing: + cursor = conn.execute("PRAGMA table_info(companion_contacts)") + columns = [column[1] for column in cursor.fetchall()] + if "last_advert_packet" not in columns: + conn.execute( + "ALTER TABLE companion_contacts ADD COLUMN last_advert_packet BLOB" + ) + logger.info("Added last_advert_packet column to companion_contacts") + conn.execute( + "INSERT INTO migrations (migration_name, applied_at) VALUES (?, ?)", + (migration_name, time.time()), + ) + logger.info(f"Migration '{migration_name}' applied successfully") + conn.commit() except Exception as e: @@ -2990,7 +3012,8 @@ class SQLiteHandler: cursor = conn.execute( """ SELECT pubkey, name, adv_type, flags, out_path_len, out_path, - last_advert_timestamp, lastmod, gps_lat, gps_lon, sync_since + last_advert_timestamp, last_advert_packet, + lastmod, gps_lat, gps_lon, sync_since FROM companion_contacts WHERE companion_hash = ? """, (companion_hash,), @@ -3019,6 +3042,7 @@ class SQLiteHandler: c.get("out_path_len", -1), c.get("out_path", b""), c.get("last_advert_timestamp", 0), + c.get("last_advert_packet"), c.get("lastmod", 0), c.get("gps_lat", 0.0), c.get("gps_lon", 0.0), @@ -3032,8 +3056,9 @@ class SQLiteHandler: """ INSERT INTO companion_contacts (companion_hash, pubkey, name, adv_type, flags, out_path_len, out_path, - last_advert_timestamp, lastmod, gps_lat, gps_lon, sync_since, updated_at) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + last_advert_timestamp, last_advert_packet, + lastmod, gps_lat, gps_lon, sync_since, updated_at) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) """, rows, ) @@ -3052,14 +3077,16 @@ class SQLiteHandler: """ INSERT INTO companion_contacts (companion_hash, pubkey, name, adv_type, flags, out_path_len, out_path, - last_advert_timestamp, lastmod, gps_lat, gps_lon, sync_since, updated_at) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + last_advert_timestamp, last_advert_packet, + lastmod, gps_lat, gps_lon, sync_since, updated_at) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) ON CONFLICT(companion_hash, pubkey) DO UPDATE SET name=excluded.name, adv_type=excluded.adv_type, flags=excluded.flags, out_path_len=excluded.out_path_len, out_path=excluded.out_path, last_advert_timestamp=excluded.last_advert_timestamp, + last_advert_packet=excluded.last_advert_packet, lastmod=excluded.lastmod, gps_lat=excluded.gps_lat, gps_lon=excluded.gps_lon, sync_since=excluded.sync_since, updated_at=excluded.updated_at @@ -3073,6 +3100,7 @@ class SQLiteHandler: contact.get("out_path_len", -1), contact.get("out_path", b""), contact.get("last_advert_timestamp", 0), + contact.get("last_advert_packet"), contact.get("lastmod", 0), contact.get("gps_lat", 0.0), contact.get("gps_lon", 0.0), diff --git a/tests/test_companion_advert_persistence.py b/tests/test_companion_advert_persistence.py new file mode 100644 index 0000000..6de4bb0 --- /dev/null +++ b/tests/test_companion_advert_persistence.py @@ -0,0 +1,164 @@ +"""SQLite regression tests for raw MeshCore ADVERT contact exchange.""" + +import asyncio +import sqlite3 +from unittest.mock import AsyncMock + +import pytest + +from openhop_core.companion.models import Contact +from openhop_core.protocol import LocalIdentity +from repeater.companion.bridge import RepeaterCompanionBridge +from repeater.companion.frame_server import CompanionFrameServer +from repeater.data_acquisition.sqlite_handler import SQLiteHandler + + +# The same literal MeshCore Packet::writeTo ADVERT fixture exercised by Core. +# It deliberately includes header/path bytes and a signature, so BLOB handling +# is tested with the real protocol shape instead of a synthetic text payload. +MESHCORE_ADVERT_WIRE = bytes.fromhex( + "1100d75a980182b10ab7d54bfed3c964073a0ee172f3daa62325af021a68f707511a" + "7856341200844b4710c3083898f85b990be03edd18c9c4d05b82590915f72d3f04" + "ac2eefc5446a5379bf25fbbbc7fcab87570e7552ed7d30def3291ea495be646cb1" + "ef078146776d566563" +) +MESHCORE_ADVERT_PUBKEY = bytes.fromhex( + "d75a980182b10ab7d54bfed3c964073a0ee172f3daa62325af021a68f707511a" +) +COMPANION_HASH = "0x42" + + +async def _drain_loopback() -> None: + for _ in range(3): + await asyncio.sleep(0) + + +def _contact(raw_advert=MESHCORE_ADVERT_WIRE) -> dict: + return { + "pubkey": MESHCORE_ADVERT_PUBKEY, + "name": "FwmVec", + "adv_type": 1, + "flags": 0, + "out_path_len": -1, + "out_path": b"", + "last_advert_timestamp": 0x12345678, + "last_advert_packet": raw_advert, + "lastmod": 123, + "gps_lat": 0.0, + "gps_lon": 0.0, + "sync_since": 0, + } + + +def _bridge(sqlite_handler) -> RepeaterCompanionBridge: + return RepeaterCompanionBridge( + LocalIdentity(), + AsyncMock(return_value=True), + sqlite_handler=sqlite_handler, + companion_hash=COMPANION_HASH, + ) + + +def test_bulk_save_load_preserves_raw_advert_blob_and_upsert_replaces_it(tmp_path): + handler = SQLiteHandler(tmp_path) + assert handler.companion_save_contacts(COMPANION_HASH, [_contact()]) + assert ( + handler.companion_load_contacts(COMPANION_HASH)[0]["last_advert_packet"] + == MESHCORE_ADVERT_WIRE + ) + + updated = MESHCORE_ADVERT_WIRE[:-1] + b"!" + assert handler.companion_upsert_contact(COMPANION_HASH, _contact(updated)) + assert handler.companion_load_contacts(COMPANION_HASH)[0]["last_advert_packet"] == updated + + +@pytest.mark.asyncio +async def test_import_persist_restart_and_export_are_byte_exact(tmp_path): + handler = SQLiteHandler(tmp_path) + first = _bridge(handler) + + assert first.import_contact(MESHCORE_ADVERT_WIRE) is True + await _drain_loopback() + contact = first.get_contact_by_key(MESHCORE_ADVERT_PUBKEY) + assert contact is not None + + # RepeaterFrameServer is the production persistence adapter used by the + # contact-updated callback. Persist through it rather than writing a row + # directly, then rebuild state exactly as the daemon restart path does. + server = CompanionFrameServer(first, COMPANION_HASH, port=0, sqlite_handler=handler) + await server._persist_contact(contact) + + restarted_handler = SQLiteHandler(tmp_path) + rows = restarted_handler.companion_load_contacts(COMPANION_HASH) + assert rows is not None and len(rows) == 1 + assert rows[0]["last_advert_packet"] == MESHCORE_ADVERT_WIRE + + restarted = _bridge(restarted_handler) + records = [] + for row in rows: + record = dict(row) + record["public_key"] = record.pop("pubkey") + records.append(record) + restarted.contacts.load_from_dicts(records) + + assert restarted.export_contact(MESHCORE_ADVERT_PUBKEY) == MESHCORE_ADVERT_WIRE + + +def test_migration_adds_nullable_advert_blob_without_losing_existing_contacts(tmp_path): + handler = SQLiteHandler(tmp_path) + conn = sqlite3.connect(str(handler.sqlite_path)) + conn.execute( + "DELETE FROM migrations " + "WHERE migration_name = 'add_last_advert_packet_to_companion_contacts'" + ) + conn.execute("ALTER TABLE companion_contacts RENAME TO companion_contacts_old") + conn.execute( + """ + CREATE TABLE companion_contacts ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + companion_hash TEXT NOT NULL, + pubkey BLOB NOT NULL, + name TEXT NOT NULL, + adv_type INTEGER NOT NULL DEFAULT 0, + flags INTEGER NOT NULL DEFAULT 0, + out_path_len INTEGER NOT NULL DEFAULT -1, + out_path BLOB, + last_advert_timestamp INTEGER NOT NULL DEFAULT 0, + lastmod INTEGER NOT NULL DEFAULT 0, + gps_lat REAL NOT NULL DEFAULT 0, + gps_lon REAL NOT NULL DEFAULT 0, + sync_since INTEGER NOT NULL DEFAULT 0, + updated_at REAL NOT NULL + ) + """ + ) + conn.execute( + "INSERT INTO companion_contacts " + "(companion_hash, pubkey, name, updated_at) VALUES (?, ?, ?, ?)", + (COMPANION_HASH, MESHCORE_ADVERT_PUBKEY, "old-contact", 1.0), + ) + conn.execute("DROP TABLE companion_contacts_old") + conn.commit() + conn.close() + + migrated = SQLiteHandler(tmp_path) + columns = { + row[1] + for row in sqlite3.connect(str(migrated.sqlite_path)) + .execute("PRAGMA table_info(companion_contacts)") + .fetchall() + } + assert "last_advert_packet" in columns + rows = migrated.companion_load_contacts(COMPANION_HASH) + assert rows is not None and rows[0]["name"] == "old-contact" + assert rows[0]["last_advert_packet"] is None + + +def test_persistence_adapter_keeps_raw_advert_bytes(): + contact = Contact( + public_key=MESHCORE_ADVERT_PUBKEY, + name="FwmVec", + last_advert_packet=MESHCORE_ADVERT_WIRE, + ) + record = CompanionFrameServer._contact_to_dict(contact) + assert record["last_advert_packet"] == MESHCORE_ADVERT_WIRE