First wave of work

This commit is contained in:
Jack Kingsman
2026-03-06 17:41:37 -08:00
parent b5e2a4c269
commit 9c54ea623e
18 changed files with 256 additions and 42 deletions

View File

@@ -79,9 +79,11 @@ class PacketInfo:
route_type: RouteType
payload_type: PayloadType
payload_version: int
path_length: int
path: bytes # The routing path (empty if path_length is 0)
path_length: int # Hop count encoded in the lower 6 bits of the path byte
path: bytes # The routing path bytes (empty if path_length is 0)
payload: bytes
path_hash_size: int = 1 # Bytes per hop encoded in the upper 2 bits of the path byte
path_byte_length: int = 0
def calculate_channel_hash(channel_key: bytes) -> str:
@@ -93,6 +95,14 @@ def calculate_channel_hash(channel_key: bytes) -> str:
return format(hash_bytes[0], "02x")
def _decode_path_metadata(path_byte: int) -> tuple[int, int, int]:
"""Decode the packed path byte into hop count and byte length."""
path_hash_size = (path_byte >> 6) + 1
path_length = path_byte & 0x3F
path_byte_length = path_length * path_hash_size
return path_length, path_hash_size, path_byte_length
def extract_payload(raw_packet: bytes) -> bytes | None:
"""
Extract just the payload from a raw packet, skipping header and path.
@@ -100,8 +110,10 @@ def extract_payload(raw_packet: bytes) -> bytes | None:
Packet structure:
- Byte 0: header (route_type, payload_type, version)
- For TRANSPORT routes: bytes 1-4 are transport codes
- Next byte: path_length
- Next path_length bytes: path data
- Next byte: packed path metadata
- upper 2 bits: bytes per hop minus 1
- lower 6 bits: hop count
- Next hop_count * path_hash_size bytes: path data
- Remaining: payload
Returns the payload bytes, or None if packet is malformed.
@@ -120,16 +132,16 @@ def extract_payload(raw_packet: bytes) -> bytes | None:
return None
offset += 4
# Get path length
# Decode packed path metadata
if len(raw_packet) < offset + 1:
return None
path_length = raw_packet[offset]
path_length, _path_hash_size, path_byte_length = _decode_path_metadata(raw_packet[offset])
offset += 1
# Skip path data
if len(raw_packet) < offset + path_length:
if len(raw_packet) < offset + path_byte_length:
return None
offset += path_length
offset += path_byte_length
# Rest is payload
return raw_packet[offset:]
@@ -156,17 +168,17 @@ def parse_packet(raw_packet: bytes) -> PacketInfo | None:
return None
offset += 4
# Get path length
# Decode packed path metadata
if len(raw_packet) < offset + 1:
return None
path_length = raw_packet[offset]
path_length, path_hash_size, path_byte_length = _decode_path_metadata(raw_packet[offset])
offset += 1
# Extract path data
if len(raw_packet) < offset + path_length:
if len(raw_packet) < offset + path_byte_length:
return None
path = raw_packet[offset : offset + path_length]
offset += path_length
path = raw_packet[offset : offset + path_byte_length]
offset += path_byte_length
# Rest is payload
payload = raw_packet[offset:]
@@ -178,6 +190,8 @@ def parse_packet(raw_packet: bytes) -> PacketInfo | None:
path_length=path_length,
path=path,
payload=payload,
path_hash_size=path_hash_size,
path_byte_length=path_byte_length,
)
except (ValueError, IndexError):
return None

View File

@@ -106,13 +106,21 @@ async def on_contact_message(event: "Event") -> None:
ts = payload.get("sender_timestamp")
sender_timestamp = ts if ts is not None else received_at
sender_name = contact.name if contact else None
path = payload.get("path")
payload_path_len = payload.get("path_len")
normalized_path_len = (
payload_path_len
if isinstance(payload_path_len, int)
else (len(path) // 2 if path is not None else None)
)
msg_id = await MessageRepository.create(
msg_type="PRIV",
text=payload.get("text", ""),
conversation_key=sender_pubkey,
sender_timestamp=sender_timestamp,
received_at=received_at,
path=payload.get("path"),
path=path,
path_len=normalized_path_len,
txt_type=txt_type,
signature=payload.get("signature"),
sender_key=sender_pubkey,
@@ -129,8 +137,11 @@ async def on_contact_message(event: "Event") -> None:
logger.debug("DM from %s handled by event handler (fallback path)", sender_pubkey[:12])
# Build paths array for broadcast
path = payload.get("path")
paths = [MessagePath(path=path or "", received_at=received_at)] if path is not None else None
paths = (
[MessagePath(path=path or "", received_at=received_at, path_len=normalized_path_len)]
if path is not None
else None
)
# Broadcast the new message
broadcast_event(

View File

@@ -56,7 +56,13 @@ def _format_body(data: dict, *, include_path: bool) -> str:
if path_str == "":
via = " **via:** [`direct`]"
else:
hops = [path_str[i : i + 2] for i in range(0, len(path_str), 2)]
path_len = paths[0].get("path_len") if isinstance(paths[0], dict) else None
hop_chars = (
len(path_str) // path_len
if isinstance(path_len, int) and path_len > 0 and len(path_str) % path_len == 0
else 2
)
hops = [path_str[i : i + hop_chars] for i in range(0, len(path_str), hop_chars)]
if hops:
hop_list = ", ".join(f"`{h}`" for h in hops)
via = f" **via:** [{hop_list}]"

View File

@@ -19,6 +19,26 @@ class Contact(BaseModel):
last_read_at: int | None = None # Server-side read state tracking
first_seen: int | None = None
@staticmethod
def _derive_out_path_hash_mode(path_hex: str | None, path_len: int) -> int:
"""Infer the contact path hash mode from stored path bytes and hop count."""
if path_len < 0:
return -1
if path_len == 0 or not path_hex:
return 0
if len(path_hex) % 2 != 0:
return 0
path_bytes = len(path_hex) // 2
if path_bytes == 0 or path_bytes % path_len != 0:
return 0
bytes_per_hop = path_bytes // path_len
if bytes_per_hop < 1:
return 0
return bytes_per_hop - 1
def to_radio_dict(self) -> dict:
"""Convert to the dict format expected by meshcore radio commands.
@@ -32,6 +52,9 @@ class Contact(BaseModel):
"flags": self.flags,
"out_path": self.last_path or "",
"out_path_len": self.last_path_len,
"out_path_hash_mode": self._derive_out_path_hash_mode(
self.last_path, self.last_path_len
),
"adv_lat": self.lat if self.lat is not None else 0.0,
"adv_lon": self.lon if self.lon is not None else 0.0,
"last_advert": self.last_advert if self.last_advert is not None else 0,
@@ -176,8 +199,11 @@ class ChannelDetail(BaseModel):
class MessagePath(BaseModel):
"""A single path that a message took to reach us."""
path: str = Field(description="Hex-encoded routing path (2 chars per hop)")
path: str = Field(description="Hex-encoded routing path")
received_at: int = Field(description="Unix timestamp when this path was received")
path_len: int | None = Field(
default=None, description="Number of hops in the path, when known"
)
class Message(BaseModel):

View File

@@ -57,6 +57,7 @@ async def _handle_duplicate_message(
text: str,
sender_timestamp: int,
path: str | None,
path_len: int | None,
received: int,
) -> None:
"""Handle a duplicate message by updating paths/acks on the existing record.
@@ -90,7 +91,9 @@ async def _handle_duplicate_message(
# Add path if provided
if path is not None:
paths = await MessageRepository.add_path(existing_msg.id, path, received)
paths = await MessageRepository.add_path(
existing_msg.id, path, received, path_len=path_len
)
else:
# Get current paths for broadcast
paths = existing_msg.paths or []
@@ -128,6 +131,7 @@ async def create_message_from_decrypted(
timestamp: int,
received_at: int | None = None,
path: str | None = None,
path_len: int | None = None,
channel_name: str | None = None,
realtime: bool = True,
) -> int | None:
@@ -150,6 +154,9 @@ async def create_message_from_decrypted(
Returns the message ID if created, None if duplicate.
"""
received = received_at or int(time.time())
normalized_path_len = (
path_len if isinstance(path_len, int) else (len(path) // 2 if path is not None else None)
)
# Format the message text with sender prefix if present
text = f"{sender}: {message_text}" if sender else message_text
@@ -172,6 +179,7 @@ async def create_message_from_decrypted(
sender_timestamp=timestamp,
received_at=received,
path=path,
path_len=normalized_path_len,
sender_name=sender,
sender_key=resolved_sender_key,
)
@@ -182,7 +190,14 @@ async def create_message_from_decrypted(
# 2. Same message arrives via multiple paths before first is committed
# In either case, add the path to the existing message.
await _handle_duplicate_message(
packet_id, "CHAN", channel_key_normalized, text, timestamp, path, received
packet_id,
"CHAN",
channel_key_normalized,
text,
timestamp,
path,
normalized_path_len,
received,
)
return None
@@ -193,7 +208,11 @@ async def create_message_from_decrypted(
# Build paths array for broadcast
# Use "is not None" to include empty string (direct/0-hop messages)
paths = [MessagePath(path=path or "", received_at=received)] if path is not None else None
paths = (
[MessagePath(path=path or "", received_at=received, path_len=normalized_path_len)]
if path is not None
else None
)
# Broadcast new message to connected clients (and fanout modules when realtime)
broadcast_event(
@@ -223,6 +242,7 @@ async def create_dm_message_from_decrypted(
our_public_key: str | None,
received_at: int | None = None,
path: str | None = None,
path_len: int | None = None,
outgoing: bool = False,
realtime: bool = True,
) -> int | None:
@@ -255,6 +275,9 @@ async def create_dm_message_from_decrypted(
return None
received = received_at or int(time.time())
normalized_path_len = (
path_len if isinstance(path_len, int) else (len(path) // 2 if path is not None else None)
)
# conversation_key is always the other party's public key
conversation_key = their_public_key.lower()
@@ -270,6 +293,7 @@ async def create_dm_message_from_decrypted(
sender_timestamp=decrypted.timestamp,
received_at=received,
path=path,
path_len=normalized_path_len,
outgoing=outgoing,
sender_key=conversation_key if not outgoing else None,
sender_name=sender_name,
@@ -284,6 +308,7 @@ async def create_dm_message_from_decrypted(
decrypted.message,
decrypted.timestamp,
path,
normalized_path_len,
received,
)
return None
@@ -299,7 +324,11 @@ async def create_dm_message_from_decrypted(
await RawPacketRepository.mark_decrypted(packet_id, msg_id)
# Build paths array for broadcast
paths = [MessagePath(path=path or "", received_at=received)] if path is not None else None
paths = (
[MessagePath(path=path or "", received_at=received, path_len=normalized_path_len)]
if path is not None
else None
)
# Broadcast new message to connected clients (and fanout modules when realtime)
sender_name = contact.name if contact and not outgoing else None
@@ -391,6 +420,7 @@ async def run_historical_dm_decryption(
our_public_key=our_public_key_bytes.hex(),
received_at=packet_timestamp,
path=path_hex,
path_len=packet_info.path_length if packet_info else None,
outgoing=outgoing,
realtime=False, # Historical decryption should not trigger fanout
)
@@ -606,6 +636,7 @@ async def _process_group_text(
timestamp=decrypted.timestamp,
received_at=timestamp,
path=packet_info.path.hex() if packet_info else None,
path_len=packet_info.path_length if packet_info else None,
)
return {
@@ -872,6 +903,7 @@ async def _process_direct_message(
our_public_key=our_public_key.hex(),
received_at=timestamp,
path=packet_info.path.hex() if packet_info else None,
path_len=packet_info.path_length if packet_info else None,
outgoing=is_outgoing,
)

View File

@@ -26,6 +26,7 @@ class MessageRepository:
conversation_key: str,
sender_timestamp: int | None = None,
path: str | None = None,
path_len: int | None = None,
txt_type: int = 0,
signature: str | None = None,
outgoing: bool = False,
@@ -43,7 +44,10 @@ class MessageRepository:
# Convert single path to paths array format
paths_json = None
if path is not None:
paths_json = json.dumps([{"path": path, "received_at": received_at}])
normalized_path_len = path_len if isinstance(path_len, int) else len(path) // 2
path_entry: dict[str, Any] = {"path": path, "received_at": received_at}
path_entry["path_len"] = normalized_path_len
paths_json = json.dumps([path_entry])
cursor = await db.conn.execute(
"""
@@ -74,7 +78,7 @@ class MessageRepository:
@staticmethod
async def add_path(
message_id: int, path: str, received_at: int | None = None
message_id: int, path: str, received_at: int | None = None, path_len: int | None = None
) -> list[MessagePath]:
"""Add a new path to an existing message.
@@ -85,7 +89,10 @@ class MessageRepository:
# Atomic append: use json_insert to avoid read-modify-write race when
# multiple duplicate packets arrive concurrently for the same message.
new_entry = json.dumps({"path": path, "received_at": ts})
normalized_path_len = path_len if isinstance(path_len, int) else len(path) // 2
new_entry_dict: dict[str, Any] = {"path": path, "received_at": ts}
new_entry_dict["path_len"] = normalized_path_len
new_entry = json.dumps(new_entry_dict)
await db.conn.execute(
"""UPDATE messages SET paths = json_insert(
COALESCE(paths, '[]'), '$[#]', json(?)

View File

@@ -71,6 +71,7 @@ async def _run_historical_channel_decryption(
timestamp=result.timestamp,
received_at=packet_timestamp,
path=path_hex,
path_len=packet_info.path_length if packet_info else None,
realtime=False, # Historical decryption should not trigger fanout
)

View File

@@ -52,7 +52,7 @@ export function PathModal({
const resolvedPaths = hasPaths
? paths.map((p) => ({
...p,
resolved: resolvePath(p.path, senderInfo, contacts, config),
resolved: resolvePath(p.path, senderInfo, contacts, config, p.path_len),
}))
: [];
@@ -90,7 +90,7 @@ export function PathModal({
{/* Raw path summary */}
<div className="text-sm">
{paths.map((p, index) => {
const hops = parsePathHops(p.path);
const hops = parsePathHops(p.path, p.path_len);
const rawPath = hops.length > 0 ? hops.join('->') : 'direct';
return (
<div key={index}>

View File

@@ -60,6 +60,10 @@ describe('parsePathHops', () => {
expect(parsePathHops('1A2B3C')).toEqual(['1A', '2B', '3C']);
});
it('parses multi-byte hops when path length is provided', () => {
expect(parsePathHops('1A2B3C4D', 2)).toEqual(['1A2B', '3C4D']);
});
it('converts to uppercase', () => {
expect(parsePathHops('1a2b')).toEqual(['1A', '2B']);
});
@@ -197,6 +201,29 @@ describe('resolvePath', () => {
expect(result.receiver.prefix).toBe('FF');
});
it('resolves multi-byte hop prefixes when path length is provided', () => {
const wideContacts = [
createContact({
public_key: '1A2B' + 'A'.repeat(60),
name: 'WideRepeater1',
type: CONTACT_TYPE_REPEATER,
}),
createContact({
public_key: '3C4D' + 'B'.repeat(60),
name: 'WideRepeater2',
type: CONTACT_TYPE_REPEATER,
}),
];
const result = resolvePath('1A2B3C4D', sender, wideContacts, config, 2);
expect(result.hops).toHaveLength(2);
expect(result.hops[0].prefix).toBe('1A2B');
expect(result.hops[0].matches[0].name).toBe('WideRepeater1');
expect(result.hops[1].prefix).toBe('3C4D');
expect(result.hops[1].matches[0].name).toBe('WideRepeater2');
});
it('handles unknown repeaters (no matches)', () => {
const result = resolvePath('XX', sender, contacts, config);
@@ -545,6 +572,15 @@ describe('formatHopCounts', () => {
expect(result.hasMultiple).toBe(false);
});
it('uses explicit path_len for multi-byte hop counts', () => {
const result = formatHopCounts([
{ path: '1A2B3C4D', path_len: 2, received_at: 1700000000 },
]);
expect(result.display).toBe('2');
expect(result.allDirect).toBe(false);
expect(result.hasMultiple).toBe(false);
});
it('formats multiple paths sorted by hop count', () => {
const result = formatHopCounts([
{ path: '1A2B3C', received_at: 1700000000 }, // 3 hops

View File

@@ -149,10 +149,12 @@ export interface ChannelDetail {
/** A single path that a message took to reach us */
export interface MessagePath {
/** Hex-encoded routing path (2 chars per hop) */
/** Hex-encoded routing path */
path: string;
/** Unix timestamp when this path was received */
received_at: number;
/** Number of hops in the path, when known */
path_len?: number;
}
export interface Message {

View File

@@ -2,7 +2,7 @@ import type { Contact, RadioConfig, MessagePath } from '../types';
import { CONTACT_TYPE_REPEATER } from '../types';
export interface PathHop {
prefix: string; // 2-char hex prefix (e.g., "1A")
prefix: string; // Hex prefix for a single hop
matches: Contact[]; // Matched repeaters (empty=unknown, multiple=ambiguous)
distanceFromPrev: number | null; // km from previous hop
}
@@ -30,20 +30,21 @@ export interface SenderInfo {
}
/**
* Split hex string into 2-char hops
* Split hex string into hop-sized chunks.
*/
export function parsePathHops(path: string | null | undefined): string[] {
export function parsePathHops(path: string | null | undefined, pathLen?: number): string[] {
if (!path || path.length === 0) {
return [];
}
const normalized = path.toUpperCase();
const hopCount = pathLen ?? Math.floor(normalized.length / 2);
const charsPerHop =
hopCount > 0 && normalized.length % hopCount === 0 ? normalized.length / hopCount : 2;
const hops: string[] = [];
for (let i = 0; i < normalized.length; i += 2) {
if (i + 1 < normalized.length) {
hops.push(normalized.slice(i, i + 2));
}
for (let i = 0; i + charsPerHop <= normalized.length; i += charsPerHop) {
hops.push(normalized.slice(i, i + charsPerHop));
}
return hops;
@@ -148,11 +149,11 @@ function sortContactsByDistance(
/**
* Get simple hop count from path string
*/
function getHopCount(path: string | null | undefined): number {
function getHopCount(path: string | null | undefined, pathLen?: number): number {
if (!path || path.length === 0) {
return 0;
}
return Math.floor(path.length / 2);
return pathLen ?? Math.floor(path.length / 2);
}
/**
@@ -170,7 +171,7 @@ export function formatHopCounts(paths: MessagePath[] | null | undefined): {
}
// Get hop counts for all paths and sort ascending
const hopCounts = paths.map((p) => getHopCount(p.path)).sort((a, b) => a - b);
const hopCounts = paths.map((p) => getHopCount(p.path, p.path_len)).sort((a, b) => a - b);
const allDirect = hopCounts.every((h) => h === 0);
const hasMultiple = paths.length > 1;
@@ -189,9 +190,10 @@ export function resolvePath(
path: string | null | undefined,
sender: SenderInfo,
contacts: Contact[],
config: RadioConfig | null
config: RadioConfig | null,
pathLen?: number
): ResolvedPath {
const hopPrefixes = parsePathHops(path);
const hopPrefixes = parsePathHops(path, pathLen);
// Build sender info
const senderPrefix = sender.publicKeyOrPrefix.toUpperCase().slice(0, 2);

View File

@@ -127,6 +127,7 @@ export function deleteContact(publicKey: string): Promise<{ status: string }> {
export interface MessagePath {
path: string;
received_at: number;
path_len?: number;
}
export interface Message {

View File

@@ -19,6 +19,7 @@ from app.decoder import (
decrypt_group_text,
derive_public_key,
derive_shared_secret,
extract_payload,
parse_packet,
try_decrypt_dm,
try_decrypt_packet_with_channel_key,
@@ -81,8 +82,31 @@ class TestPacketParsing:
assert result.route_type == RouteType.DIRECT
assert result.payload_type == PayloadType.TEXT_MESSAGE
assert result.path_length == 3
assert result.path_hash_size == 1
assert result.path_byte_length == 3
assert result.payload == b"msg"
def test_parse_packet_with_two_byte_hops(self):
"""Packets with multi-byte hop identifiers decode hop count separately from byte length."""
packet = bytes([0x0A, 0x42, 0x01, 0x02, 0x03, 0x04]) + b"msg"
result = parse_packet(packet)
assert result is not None
assert result.route_type == RouteType.DIRECT
assert result.payload_type == PayloadType.TEXT_MESSAGE
assert result.path_length == 2
assert result.path_hash_size == 2
assert result.path_byte_length == 4
assert result.path == bytes([0x01, 0x02, 0x03, 0x04])
assert result.payload == b"msg"
def test_extract_payload_with_two_byte_hops(self):
"""Payload extraction skips the full path byte length for multi-byte hops."""
packet = bytes([0x15, 0x42, 0xAA, 0xBB, 0xCC, 0xDD]) + b"payload_data"
assert extract_payload(packet) == b"payload_data"
def test_parse_transport_flood_skips_transport_code(self):
"""TRANSPORT_FLOOD packets have 4-byte transport code to skip."""
# Header: route_type=TRANSPORT_FLOOD(0), payload_type=GROUP_TEXT(5)

View File

@@ -682,7 +682,7 @@ class TestDirectMessageDirectionDetection:
message_broadcasts = [b for b in broadcasts if b["type"] == "message"]
assert len(message_broadcasts) == 1
assert message_broadcasts[0]["data"]["paths"] == [
{"path": "", "received_at": SENDER_TIMESTAMP}
{"path": "", "received_at": SENDER_TIMESTAMP, "path_len": 0}
]
@pytest.mark.asyncio

View File

@@ -652,6 +652,21 @@ class TestAppriseFormatBody:
assert "`20`" in body
assert "`27`" in body
def test_dm_with_multi_byte_path(self):
from app.fanout.apprise_mod import _format_body
body = _format_body(
{
"type": "PRIV",
"text": "hi",
"sender_name": "Alice",
"paths": [{"path": "20273031", "path_len": 2}],
},
include_path=True,
)
assert "`2027`" in body
assert "`3031`" in body
def test_dm_no_path_shows_direct(self):
from app.fanout.apprise_mod import _format_body

View File

@@ -678,6 +678,7 @@ class TestMessageBroadcastStructure:
assert broadcast["paths"] is not None
assert len(broadcast["paths"]) == 1
assert broadcast["paths"][0]["path"] == "" # Empty string = direct/flood
assert broadcast["paths"][0]["path_len"] == 0
class TestRawPacketStorage:
@@ -927,6 +928,7 @@ class TestCreateDMMessageFromDecrypted:
our_public_key=self.FACE12_PUB,
received_at=1700000001,
path="aabbcc", # Path through 3 repeaters
path_len=3,
outgoing=False,
)
@@ -937,6 +939,7 @@ class TestCreateDMMessageFromDecrypted:
assert broadcast["paths"] is not None
assert len(broadcast["paths"]) == 1
assert broadcast["paths"][0]["path"] == "aabbcc"
assert broadcast["paths"][0]["path_len"] == 3
assert broadcast["paths"][0]["received_at"] == 1700000001

View File

@@ -36,12 +36,13 @@ class TestMessageRepositoryAddPath:
msg_id = await _create_message(test_db)
result = await MessageRepository.add_path(
message_id=msg_id, path="1A2B", received_at=1700000000
message_id=msg_id, path="1A2B", received_at=1700000000, path_len=1
)
assert len(result) == 1
assert result[0].path == "1A2B"
assert result[0].received_at == 1700000000
assert result[0].path_len == 1
@pytest.mark.asyncio
async def test_add_path_to_message_with_existing_paths(self, test_db):

View File

@@ -125,6 +125,39 @@ class TestOutgoingDMBroadcast:
assert exc_info.value.status_code == 409
assert "ambiguous" in exc_info.value.detail.lower()
@pytest.mark.asyncio
async def test_send_dm_add_contact_preserves_out_path_hash_mode(self, test_db):
"""Direct-send contact export includes the inferred path hash mode for multi-byte routes."""
mc = _make_mc()
pub_key = "cd" * 32
await ContactRepository.upsert(
{
"public_key": pub_key,
"name": "Bob",
"type": 0,
"flags": 0,
"last_path": "11223344",
"last_path_len": 2,
"last_advert": None,
"lat": None,
"lon": None,
"last_seen": None,
"on_radio": False,
"last_contacted": None,
}
)
with (
patch("app.routers.messages.require_connected", return_value=mc),
patch.object(radio_manager, "_meshcore", mc),
):
await send_direct_message(SendDirectMessageRequest(destination=pub_key, text="hi"))
add_contact_arg = mc.commands.add_contact.await_args.args[0]
assert add_contact_arg["out_path"] == "11223344"
assert add_contact_arg["out_path_len"] == 2
assert add_contact_arg["out_path_hash_mode"] == 1
class TestOutgoingChannelBroadcast:
"""Test that outgoing channel messages are broadcast via broadcast_event for fanout dispatch."""