Actually persist out_path_hash_mode instead of lossily deriving it

This commit is contained in:
Jack Kingsman
2026-03-07 22:14:22 -08:00
parent 69c812cfd4
commit 76d11b01a7
13 changed files with 459 additions and 88 deletions

View File

@@ -15,6 +15,7 @@ CREATE TABLE IF NOT EXISTS contacts (
flags INTEGER DEFAULT 0,
last_path TEXT,
last_path_len INTEGER DEFAULT -1,
out_path_hash_mode INTEGER DEFAULT 0,
last_advert INTEGER,
lat REAL,
lon REAL,

View File

@@ -211,6 +211,7 @@ async def on_path_update(event: "Event") -> None:
# so if path fields are absent here we treat this as informational only.
path = payload.get("path")
path_len = payload.get("path_len")
path_hash_mode = payload.get("path_hash_mode")
if path is None or path_len is None:
logger.debug(
"PATH_UPDATE for %s has no path payload, skipping DB update", contact.public_key[:12]
@@ -225,7 +226,28 @@ async def on_path_update(event: "Event") -> None:
)
return
await ContactRepository.update_path(contact.public_key, str(path), normalized_path_len)
normalized_path_hash_mode: int | None
if path_hash_mode is None:
# Legacy firmware/library payloads only support 1-byte hop hashes.
normalized_path_hash_mode = -1 if normalized_path_len == -1 else 0
else:
normalized_path_hash_mode = None
try:
normalized_path_hash_mode = int(path_hash_mode)
except (TypeError, ValueError):
logger.warning(
"Invalid path_hash_mode in PATH_UPDATE for %s: %r",
contact.public_key[:12],
path_hash_mode,
)
normalized_path_hash_mode = None
await ContactRepository.update_path(
contact.public_key,
str(path),
normalized_path_len,
normalized_path_hash_mode,
)
async def on_new_contact(event: "Event") -> None:

View File

@@ -303,6 +303,13 @@ async def run_migrations(conn: aiosqlite.Connection) -> int:
await set_version(conn, 38)
applied += 1
# Migration 39: Persist contacts.out_path_hash_mode for multibyte path round-tripping
if version < 39:
logger.info("Applying migration 39: add contacts.out_path_hash_mode")
await _migrate_039_add_contact_out_path_hash_mode(conn)
await set_version(conn, 39)
applied += 1
if applied > 0:
logger.info(
"Applied %d migration(s), schema now at version %d", applied, await get_version(conn)
@@ -2288,3 +2295,63 @@ async def _migrate_038_drop_legacy_columns(conn: aiosqlite.Connection) -> None:
raise
await conn.commit()
async def _migrate_039_add_contact_out_path_hash_mode(conn: aiosqlite.Connection) -> None:
"""Add contacts.out_path_hash_mode and backfill legacy rows.
Historical databases predate multibyte routing support. Backfill rules:
- contacts with last_path_len = -1 are flood routes -> out_path_hash_mode = -1
- all other existing contacts default to 0 (1-byte legacy hop identifiers)
"""
cursor = await conn.execute(
"SELECT name FROM sqlite_master WHERE type='table' AND name='contacts'"
)
if await cursor.fetchone() is None:
await conn.commit()
return
column_cursor = await conn.execute("PRAGMA table_info(contacts)")
columns = {row[1] for row in await column_cursor.fetchall()}
added_column = False
try:
await conn.execute(
"ALTER TABLE contacts ADD COLUMN out_path_hash_mode INTEGER NOT NULL DEFAULT 0"
)
added_column = True
logger.debug("Added out_path_hash_mode to contacts table")
except aiosqlite.OperationalError as e:
if "duplicate column name" in str(e).lower():
logger.debug("contacts.out_path_hash_mode already exists, skipping add")
else:
raise
if "last_path_len" not in columns:
await conn.commit()
return
if added_column:
await conn.execute(
"""
UPDATE contacts
SET out_path_hash_mode = CASE
WHEN last_path_len = -1 THEN -1
ELSE 0
END
"""
)
else:
await conn.execute(
"""
UPDATE contacts
SET out_path_hash_mode = CASE
WHEN last_path_len = -1 THEN -1
ELSE 0
END
WHERE out_path_hash_mode NOT IN (-1, 0, 1, 2)
OR (last_path_len = -1 AND out_path_hash_mode != -1)
"""
)
await conn.commit()

View File

@@ -2,8 +2,6 @@ from typing import Literal
from pydantic import BaseModel, Field
from app.path_utils import infer_hash_size
class Contact(BaseModel):
public_key: str = Field(description="Public key (64-char hex)")
@@ -12,6 +10,7 @@ class Contact(BaseModel):
flags: int = 0
last_path: str | None = None
last_path_len: int = -1
out_path_hash_mode: int = 0
last_advert: int | None = None
lat: float | None = None
lon: float | None = None
@@ -34,7 +33,7 @@ class Contact(BaseModel):
"flags": self.flags,
"out_path": self.last_path or "",
"out_path_len": self.last_path_len,
"out_path_hash_mode": infer_hash_size(self.last_path or "", self.last_path_len) - 1,
"out_path_hash_mode": self.out_path_hash_mode,
"adv_lat": self.lat if self.lat is not None else 0.0,
"adv_lon": self.lon if self.lon is not None else 0.0,
"last_advert": self.last_advert if self.last_advert is not None else 0,
@@ -54,6 +53,10 @@ class Contact(BaseModel):
"flags": radio_data.get("flags", 0),
"last_path": radio_data.get("out_path"),
"last_path_len": radio_data.get("out_path_len", -1),
"out_path_hash_mode": radio_data.get(
"out_path_hash_mode",
-1 if radio_data.get("out_path_len", -1) == -1 else 0,
),
"lat": radio_data.get("adv_lat"),
"lon": radio_data.get("adv_lon"),
"last_advert": radio_data.get("last_advert"),

View File

@@ -691,9 +691,11 @@ async def _process_advertisement(
assert existing is not None # Guaranteed by the conditions that set use_existing_path
path_len = existing.last_path_len if existing.last_path_len is not None else -1
path_hex = existing.last_path or ""
out_path_hash_mode = existing.out_path_hash_mode
else:
path_len = new_path_len
path_hex = new_path_hex
out_path_hash_mode = packet_info.path_hash_size - 1
logger.debug(
"Parsed advertisement from %s: %s (role=%d, lat=%s, lon=%s, path_len=%d)",
@@ -738,6 +740,7 @@ async def _process_advertisement(
"last_seen": timestamp,
"last_path": path_hex,
"last_path_len": path_len,
"out_path_hash_mode": out_path_hash_mode,
"first_seen": timestamp, # COALESCE in upsert preserves existing value
}

View File

@@ -54,17 +54,3 @@ def first_hop_hex(path_hex: str, hop_count: int) -> str | None:
"""
hops = split_path_hex(path_hex, hop_count)
return hops[0] if hops else None
def infer_hash_size(path_hex: str, hop_count: int) -> int:
"""Derive bytes-per-hop from path hex length and hop count.
Returns 1 as default for ambiguous or legacy cases.
"""
if hop_count <= 0 or not path_hex:
return 1
hex_per_hop = len(path_hex) // hop_count
byte_per_hop = hex_per_hop // 2
if byte_per_hop in (1, 2, 3) and hex_per_hop * hop_count == len(path_hex):
return byte_per_hop
return 1

View File

@@ -23,18 +23,24 @@ class AmbiguousPublicKeyPrefixError(ValueError):
class ContactRepository:
@staticmethod
async def upsert(contact: dict[str, Any]) -> None:
out_path_hash_mode = contact.get("out_path_hash_mode")
if out_path_hash_mode is None:
out_path_hash_mode = -1 if contact.get("last_path_len", -1) == -1 else 0
await db.conn.execute(
"""
INSERT INTO contacts (public_key, name, type, flags, last_path, last_path_len,
out_path_hash_mode,
last_advert, lat, lon, last_seen,
on_radio, last_contacted, first_seen)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
ON CONFLICT(public_key) DO UPDATE SET
name = COALESCE(excluded.name, contacts.name),
type = CASE WHEN excluded.type = 0 THEN contacts.type ELSE excluded.type END,
flags = excluded.flags,
last_path = COALESCE(excluded.last_path, contacts.last_path),
last_path_len = excluded.last_path_len,
out_path_hash_mode = excluded.out_path_hash_mode,
last_advert = COALESCE(excluded.last_advert, contacts.last_advert),
lat = COALESCE(excluded.lat, contacts.lat),
lon = COALESCE(excluded.lon, contacts.lon),
@@ -50,6 +56,7 @@ class ContactRepository:
contact.get("flags", 0),
contact.get("last_path"),
contact.get("last_path_len", -1),
out_path_hash_mode,
contact.get("last_advert"),
contact.get("lat"),
contact.get("lon"),
@@ -71,6 +78,7 @@ class ContactRepository:
flags=row["flags"],
last_path=row["last_path"],
last_path_len=row["last_path_len"],
out_path_hash_mode=row["out_path_hash_mode"],
last_advert=row["last_advert"],
lat=row["lat"],
lon=row["lon"],
@@ -201,11 +209,17 @@ class ContactRepository:
return [ContactRepository._row_to_contact(row) for row in rows]
@staticmethod
async def update_path(public_key: str, path: str, path_len: int) -> None:
async def update_path(
public_key: str,
path: str,
path_len: int,
out_path_hash_mode: int | None = None,
) -> None:
await db.conn.execute(
"""UPDATE contacts SET last_path = ?, last_path_len = ?,
out_path_hash_mode = COALESCE(?, out_path_hash_mode),
last_seen = ? WHERE public_key = ?""",
(path, path_len, int(time.time()), public_key.lower()),
(path, path_len, out_path_hash_mode, int(time.time()), public_key.lower()),
)
await db.conn.commit()

View File

@@ -110,6 +110,7 @@ async def create_contact(
"flags": existing.flags,
"last_path": existing.last_path,
"last_path_len": existing.last_path_len,
"out_path_hash_mode": existing.out_path_hash_mode,
"last_advert": existing.last_advert,
"lat": existing.lat,
"lon": existing.lon,
@@ -139,6 +140,7 @@ async def create_contact(
"flags": 0,
"last_path": None,
"last_path_len": -1,
"out_path_hash_mode": -1,
"last_advert": None,
"lat": None,
"lon": None,
@@ -462,7 +464,7 @@ async def reset_contact_path(public_key: str) -> dict:
"""Reset a contact's routing path to flood."""
contact = await _resolve_contact_or_404(public_key)
await ContactRepository.update_path(contact.public_key, "", -1)
await ContactRepository.update_path(contact.public_key, "", -1, -1)
logger.info("Reset path to flood for %s", contact.public_key[:12])
# Push the updated path to radio if connected and contact is on radio

View File

@@ -651,6 +651,7 @@ class TestResetPath:
contact = await ContactRepository.get_by_key(KEY_A)
assert contact.last_path == ""
assert contact.last_path_len == -1
assert contact.out_path_hash_mode == -1
@pytest.mark.asyncio
async def test_reset_path_not_found(self, test_db, client):
@@ -661,7 +662,13 @@ class TestResetPath:
@pytest.mark.asyncio
async def test_reset_path_pushes_to_radio(self, test_db, client):
"""When radio connected and contact on_radio, pushes updated path."""
await _insert_contact(KEY_A, on_radio=True, last_path="1122", last_path_len=1)
await _insert_contact(
KEY_A,
on_radio=True,
last_path="1122",
last_path_len=1,
out_path_hash_mode=0,
)
mock_mc = MagicMock()
mock_result = MagicMock()
@@ -678,6 +685,10 @@ class TestResetPath:
assert response.status_code == 200
mock_mc.commands.add_contact.assert_called_once()
contact_payload = mock_mc.commands.add_contact.call_args.args[0]
assert contact_payload["out_path"] == ""
assert contact_payload["out_path_len"] == -1
assert contact_payload["out_path_hash_mode"] == -1
@pytest.mark.asyncio
async def test_reset_path_broadcasts_websocket_event(self, test_db, client):
@@ -726,6 +737,34 @@ class TestAddRemoveRadio:
contact = await ContactRepository.get_by_key(KEY_A)
assert contact.on_radio is True
@pytest.mark.asyncio
async def test_add_to_radio_preserves_stored_out_path_hash_mode(self, test_db, client):
await _insert_contact(
KEY_A,
last_path="aa00bb00",
last_path_len=2,
out_path_hash_mode=1,
)
mock_mc = MagicMock()
mock_mc.get_contact_by_key_prefix = MagicMock(return_value=None)
mock_result = MagicMock()
mock_result.type = EventType.OK
mock_mc.commands.add_contact = AsyncMock(return_value=mock_result)
radio_manager._meshcore = mock_mc
with patch("app.dependencies.radio_manager") as mock_dep_rm:
mock_dep_rm.is_connected = True
mock_dep_rm.meshcore = mock_mc
response = await client.post(f"/api/contacts/{KEY_A}/add-to-radio")
assert response.status_code == 200
payload = mock_mc.commands.add_contact.call_args.args[0]
assert payload["out_path"] == "aa00bb00"
assert payload["out_path_len"] == 2
assert payload["out_path_hash_mode"] == 1
@pytest.mark.asyncio
async def test_add_already_on_radio(self, test_db, client):
"""Adding a contact already on radio returns ok without calling add_contact."""

View File

@@ -608,6 +608,37 @@ class TestOnPathUpdate:
assert contact is not None
assert contact.last_path == "0102"
assert contact.last_path_len == 2
assert contact.out_path_hash_mode == 0
@pytest.mark.asyncio
async def test_updates_path_hash_mode_when_present(self, test_db):
"""PATH_UPDATE persists explicit multibyte path hash mode."""
from app.event_handlers import on_path_update
await ContactRepository.upsert(
{
"public_key": "ab" * 32,
"name": "Alice",
"type": 1,
"flags": 0,
}
)
class MockEvent:
payload = {
"public_key": "ab" * 32,
"path": "aa00bb00",
"path_len": 2,
"path_hash_mode": 1,
}
await on_path_update(MockEvent())
contact = await ContactRepository.get_by_key("ab" * 32)
assert contact is not None
assert contact.last_path == "aa00bb00"
assert contact.last_path_len == 2
assert contact.out_path_hash_mode == 1
@pytest.mark.asyncio
async def test_does_nothing_when_contact_not_found(self, test_db):

View File

@@ -1064,3 +1064,144 @@ class TestMigration033:
assert row["on_radio"] == 1 # Not overwritten
finally:
await conn.close()
class TestMigration039:
"""Test migration 039: persist contacts.out_path_hash_mode."""
@pytest.mark.asyncio
async def test_adds_column_and_backfills_legacy_rows(self):
"""Pre-039 contacts get flood=-1 and legacy routed paths=0."""
conn = await aiosqlite.connect(":memory:")
conn.row_factory = aiosqlite.Row
try:
await set_version(conn, 38)
await conn.execute("""
CREATE TABLE contacts (
public_key TEXT PRIMARY KEY,
name TEXT,
type INTEGER DEFAULT 0,
flags INTEGER DEFAULT 0,
last_path TEXT,
last_path_len INTEGER DEFAULT -1,
last_advert INTEGER,
lat REAL,
lon REAL,
last_seen INTEGER,
on_radio INTEGER DEFAULT 0,
last_contacted INTEGER,
first_seen INTEGER
)
""")
await conn.execute(
"""
INSERT INTO contacts (
public_key, name, last_path, last_path_len, first_seen
) VALUES (?, ?, ?, ?, ?), (?, ?, ?, ?, ?)
""",
(
"aa" * 32,
"Flood",
"",
-1,
1000,
"bb" * 32,
"LegacyPath",
"1122",
1,
1001,
),
)
await conn.commit()
applied = await run_migrations(conn)
assert applied == 1
assert await get_version(conn) == 39
cursor = await conn.execute(
"""
SELECT public_key, last_path_len, out_path_hash_mode
FROM contacts
ORDER BY public_key
"""
)
rows = await cursor.fetchall()
assert rows[0]["public_key"] == "aa" * 32
assert rows[0]["last_path_len"] == -1
assert rows[0]["out_path_hash_mode"] == -1
assert rows[1]["public_key"] == "bb" * 32
assert rows[1]["last_path_len"] == 1
assert rows[1]["out_path_hash_mode"] == 0
finally:
await conn.close()
@pytest.mark.asyncio
async def test_existing_valid_modes_are_preserved_when_column_already_exists(self):
"""Migration does not clobber post-upgrade multibyte rows."""
conn = await aiosqlite.connect(":memory:")
conn.row_factory = aiosqlite.Row
try:
await set_version(conn, 38)
await conn.execute("""
CREATE TABLE contacts (
public_key TEXT PRIMARY KEY,
name TEXT,
type INTEGER DEFAULT 0,
flags INTEGER DEFAULT 0,
last_path TEXT,
last_path_len INTEGER DEFAULT -1,
out_path_hash_mode INTEGER NOT NULL DEFAULT 0,
last_advert INTEGER,
lat REAL,
lon REAL,
last_seen INTEGER,
on_radio INTEGER DEFAULT 0,
last_contacted INTEGER,
first_seen INTEGER
)
""")
await conn.execute(
"""
INSERT INTO contacts (
public_key, name, last_path, last_path_len, out_path_hash_mode, first_seen
) VALUES (?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?)
""",
(
"cc" * 32,
"Multi",
"aa00bb00",
2,
1,
1000,
"dd" * 32,
"Flood",
"",
-1,
0,
1001,
),
)
await conn.commit()
applied = await run_migrations(conn)
assert applied == 1
assert await get_version(conn) == 39
cursor = await conn.execute(
"""
SELECT public_key, out_path_hash_mode
FROM contacts
WHERE public_key IN (?, ?)
ORDER BY public_key
""",
("cc" * 32, "dd" * 32),
)
rows = await cursor.fetchall()
assert rows[0]["public_key"] == "cc" * 32
assert rows[0]["out_path_hash_mode"] == 1
assert rows[1]["public_key"] == "dd" * 32
assert rows[1]["out_path_hash_mode"] == -1
finally:
await conn.close()

View File

@@ -5,7 +5,6 @@ import pytest
from app.path_utils import (
decode_path_byte,
first_hop_hex,
infer_hash_size,
path_wire_len,
split_path_hex,
)
@@ -125,60 +124,96 @@ class TestFirstHopHex:
assert first_hop_hex("", 0) is None
class TestInferHashSize:
def test_one_byte(self):
assert infer_hash_size("1a2b3c", 3) == 1
def test_two_byte(self):
assert infer_hash_size("1a2b3c4d", 2) == 2
def test_three_byte(self):
assert infer_hash_size("1a2b3c4d5e6f", 2) == 3
def test_empty_defaults_to_1(self):
assert infer_hash_size("", 0) == 1
def test_inconsistent_defaults_to_1(self):
assert infer_hash_size("1a2b3", 2) == 1
def test_zero_hop_count_defaults_to_1(self):
assert infer_hash_size("1a2b", 0) == 1
class TestContactToRadioDictHashMode:
"""Test that Contact.to_radio_dict() correctly derives out_path_hash_mode."""
"""Test that Contact.to_radio_dict() preserves the stored out_path_hash_mode."""
def test_1byte_hops(self):
def test_preserves_1byte_mode(self):
from app.models import Contact
c = Contact(public_key="aa" * 32, last_path="1a2b3c", last_path_len=3)
d = c.to_radio_dict()
assert d["out_path_hash_mode"] == 0 # infer_hash_size=1, mode=0
def test_2byte_hops(self):
from app.models import Contact
c = Contact(public_key="bb" * 32, last_path="1a2b3c4d", last_path_len=2)
d = c.to_radio_dict()
assert d["out_path_hash_mode"] == 1 # infer_hash_size=2, mode=1
def test_3byte_hops(self):
from app.models import Contact
c = Contact(public_key="cc" * 32, last_path="1a2b3c4d5e6f", last_path_len=2)
d = c.to_radio_dict()
assert d["out_path_hash_mode"] == 2 # infer_hash_size=3, mode=2
def test_no_path_defaults_to_mode0(self):
from app.models import Contact
c = Contact(public_key="dd" * 32, last_path=None, last_path_len=-1)
c = Contact(
public_key="aa" * 32,
last_path="1a2b3c",
last_path_len=3,
out_path_hash_mode=0,
)
d = c.to_radio_dict()
assert d["out_path_hash_mode"] == 0
def test_empty_path_defaults_to_mode0(self):
def test_preserves_2byte_mode(self):
from app.models import Contact
c = Contact(public_key="ee" * 32, last_path="", last_path_len=0)
c = Contact(
public_key="bb" * 32,
last_path="1a2b3c4d",
last_path_len=2,
out_path_hash_mode=1,
)
d = c.to_radio_dict()
assert d["out_path_hash_mode"] == 0
assert d["out_path_hash_mode"] == 1
def test_preserves_3byte_mode(self):
from app.models import Contact
c = Contact(
public_key="cc" * 32,
last_path="1a2b3c4d5e6f",
last_path_len=2,
out_path_hash_mode=2,
)
d = c.to_radio_dict()
assert d["out_path_hash_mode"] == 2
def test_preserves_flood_mode(self):
from app.models import Contact
c = Contact(
public_key="dd" * 32,
last_path=None,
last_path_len=-1,
out_path_hash_mode=-1,
)
d = c.to_radio_dict()
assert d["out_path_hash_mode"] == -1
def test_preserves_mode_with_zero_bytes_in_path(self):
from app.models import Contact
c = Contact(
public_key="ee" * 32,
last_path="aa00bb00",
last_path_len=2,
out_path_hash_mode=1,
)
d = c.to_radio_dict()
assert d["out_path_hash_mode"] == 1
class TestContactFromRadioDictHashMode:
"""Test that Contact.from_radio_dict() preserves explicit path hash mode."""
def test_preserves_mode_from_radio_payload(self):
from app.models import Contact
d = Contact.from_radio_dict(
"aa" * 32,
{
"adv_name": "Alice",
"out_path": "aa00bb00",
"out_path_len": 2,
"out_path_hash_mode": 1,
},
)
assert d["out_path_hash_mode"] == 1
def test_flood_falls_back_to_minus_one(self):
from app.models import Contact
d = Contact.from_radio_dict(
"bb" * 32,
{
"adv_name": "Bob",
"out_path": "",
"out_path_len": -1,
},
)
assert d["out_path_hash_mode"] == -1

View File

@@ -56,24 +56,24 @@ def _make_mc(name="TestNode"):
return mc
async def _insert_contact(public_key, name="Alice"):
async def _insert_contact(public_key, name="Alice", **overrides):
"""Insert a contact into the test database."""
await ContactRepository.upsert(
{
"public_key": public_key,
"name": name,
"type": 0,
"flags": 0,
"last_path": None,
"last_path_len": -1,
"last_advert": None,
"lat": None,
"lon": None,
"last_seen": None,
"on_radio": False,
"last_contacted": None,
}
)
data = {
"public_key": public_key,
"name": name,
"type": 0,
"flags": 0,
"last_path": None,
"last_path_len": -1,
"last_advert": None,
"lat": None,
"lon": None,
"last_seen": None,
"on_radio": False,
"last_contacted": None,
}
data.update(overrides)
await ContactRepository.upsert(data)
class TestOutgoingDMBroadcast:
@@ -125,6 +125,33 @@ class TestOutgoingDMBroadcast:
assert exc_info.value.status_code == 409
assert "ambiguous" in exc_info.value.detail.lower()
@pytest.mark.asyncio
async def test_send_dm_preserves_stored_out_path_hash_mode(self, test_db):
"""Direct-message send pushes the persisted path hash mode back to the radio."""
mc = _make_mc()
pub_key = "cd" * 32
await _insert_contact(
pub_key,
"Alice",
last_path="aa00bb00",
last_path_len=2,
out_path_hash_mode=1,
)
with (
patch("app.routers.messages.require_connected", return_value=mc),
patch.object(radio_manager, "_meshcore", mc),
patch("app.routers.messages.broadcast_event"),
):
request = SendDirectMessageRequest(destination=pub_key, text="Hello")
await send_direct_message(request)
contact_payload = mc.commands.add_contact.call_args.args[0]
assert contact_payload["public_key"] == pub_key
assert contact_payload["out_path"] == "aa00bb00"
assert contact_payload["out_path_len"] == 2
assert contact_payload["out_path_hash_mode"] == 1
class TestOutgoingChannelBroadcast:
"""Test that outgoing channel messages are broadcast via broadcast_event for fanout dispatch."""