Add draft reactions + gifs; region resolution

This commit is contained in:
Jack Kingsman
2026-06-20 21:12:47 -07:00
parent cb4d4ca584
commit 06556a853d
35 changed files with 1111 additions and 44 deletions
+8
View File
@@ -145,6 +145,13 @@ app/
- Incoming PRIV message uniqueness (`idx_messages_incoming_priv_dedup`): `(type, conversation_key, text, COALESCE(sender_timestamp, 0), COALESCE(sender_key, ''))` where `type = 'PRIV' AND outgoing = 0``sender_key` was added in migration 056 to distinguish room-server posts from different senders in the same second.
- Duplicate insert is treated as an echo/repeat: the new path (if any) is appended, and the ACK count is incremented only for outgoing channel messages. Incoming direct messages with the same dedup identity also collapse onto one stored row, with later observations merging path data instead of creating a second DM.
### Region scope decoding (transport codes)
- `ROUTE_TYPE_TRANSPORT_FLOOD`/`ROUTE_TYPE_TRANSPORT_DIRECT` packets carry a 4-byte transport-code block; `parse_packet_envelope` exposes it as `transport_codes = (code_1, code_2)` (little-endian uint16s; `code_2` is reserved/0).
- `code_1` is a keyed MAC over the payload, not a stable per-region id: `code = HMAC-SHA256(SHA256("#" + region_name)[:16], payload_type || payload)[:2]` (firmware `TransportKeyStore.cpp`; reserved values `0x0000`/`0xFFFF` are nudged to `0x0001`/`0xFFFE`). There is **no** reverse lookup table — to name a packet's region you recompute the code per candidate region and check for a match (`app/region_resolver.py`).
- Candidate region names come from `app_settings.known_regions` (user-editable, seeded by migration 064 from `flood_scope` + channel `flood_scope_override`).
- Channel messages persist `messages.transport_code` (uint16, NULL = unscoped plain flood) and `messages.region` (resolved name, NULL = scoped but no list match) at ingest, so the chat region badge survives raw-packet purge. The packet inspector (`GET /packets/{id}` and the `raw_packet` WS broadcast) resolves region on the fly against the current list since it still holds the raw payload.
### Raw packet dedup policy
- Raw packet storage deduplicates by payload hash (`RawPacketRepository.create`), excluding routing/path bytes.
@@ -350,6 +357,7 @@ Repository writes should prefer typed models such as `ContactUpsert` over ad hoc
- `advert_interval`
- `last_advert_time`
- `flood_scope`
- `known_regions`
- `blocked_keys`, `blocked_names`, `discovery_blocked_types`
- `tracked_telemetry_repeaters`, `tracked_telemetry_contacts`
- `auto_resend_channel`
+2
View File
@@ -105,6 +105,7 @@ class PacketInfo:
path: bytes # The routing path bytes (empty if path_length is 0)
payload: bytes
path_hash_size: int = 1 # Bytes per hop: 1, 2, or 3
transport_codes: tuple[int, int] | None = None # (code_1, code_2) for TRANSPORT_* routes
def _is_valid_advert_location(lat: float, lon: float) -> bool:
@@ -146,6 +147,7 @@ def parse_packet(raw_packet: bytes) -> PacketInfo | None:
path_hash_size=envelope.hash_size,
path=envelope.path,
payload=envelope.payload,
transport_codes=envelope.transport_codes,
)
except ValueError:
return None
@@ -0,0 +1,86 @@
import json
import logging
import aiosqlite
logger = logging.getLogger(__name__)
async def migrate(conn: aiosqlite.Connection) -> None:
"""Add region-scope decoding support.
1. Add ``transport_code`` (uint16) and ``region`` columns to ``messages`` so
transport-routed channel messages persist their resolved region even after
the source raw packet is purged by maintenance.
2. Add ``known_regions`` (JSON list) to ``app_settings`` and seed it from any
region names we already know about: the global ``flood_scope`` and per-channel
``flood_scope_override`` values. Names are stored without the ``#`` prefix.
"""
tables_cursor = await conn.execute("SELECT name FROM sqlite_master WHERE type='table'")
existing_tables = {row[0] for row in await tables_cursor.fetchall()}
# --- messages columns ---
if "messages" in existing_tables:
col_cursor = await conn.execute("PRAGMA table_info(messages)")
message_columns = {row[1] for row in await col_cursor.fetchall()}
if "transport_code" not in message_columns:
await conn.execute("ALTER TABLE messages ADD COLUMN transport_code INTEGER")
if "region" not in message_columns:
await conn.execute("ALTER TABLE messages ADD COLUMN region TEXT")
await conn.commit()
# --- app_settings.known_regions column + seed ---
if "app_settings" not in existing_tables:
await conn.commit()
return
col_cursor = await conn.execute("PRAGMA table_info(app_settings)")
settings_columns = {row[1] for row in await col_cursor.fetchall()}
if "known_regions" not in settings_columns:
await conn.execute("ALTER TABLE app_settings ADD COLUMN known_regions TEXT")
await conn.commit()
def _clean(name: str | None) -> str | None:
stripped = (name or "").strip()
if stripped.startswith("#"):
stripped = stripped[1:].strip()
return stripped or None
seeded: list[str] = []
seen: set[str] = set()
def _add(name: str | None) -> None:
cleaned = _clean(name)
if cleaned and cleaned.lower() not in seen:
seen.add(cleaned.lower())
seeded.append(cleaned)
# Global flood scope
try:
cursor = await conn.execute("SELECT flood_scope FROM app_settings WHERE id = 1")
row = await cursor.fetchone()
if row:
_add(row[0])
except aiosqlite.Error:
pass
# Per-channel flood scope overrides
if "channels" in existing_tables:
chan_cols_cursor = await conn.execute("PRAGMA table_info(channels)")
chan_cols = {row[1] for row in await chan_cols_cursor.fetchall()}
if "flood_scope_override" in chan_cols:
cursor = await conn.execute(
"SELECT DISTINCT flood_scope_override FROM channels "
"WHERE flood_scope_override IS NOT NULL AND flood_scope_override != ''"
)
for row in await cursor.fetchall():
_add(row[0])
await conn.execute(
"UPDATE app_settings SET known_regions = ? WHERE id = 1",
(json.dumps(seeded),),
)
await conn.commit()
if seeded:
logger.info("Seeded %d known region(s) for scope decoding: %s", len(seeded), seeded)
+34
View File
@@ -429,6 +429,17 @@ class Message(BaseModel):
default=None,
description="Representative raw packet row ID when archival raw bytes exist",
)
transport_code: int | None = Field(
default=None,
description=(
"Region scope transport code (uint16) when the message arrived via a "
"TransportFlood/TransportDirect packet; None for unscoped (plain flood) messages"
),
)
region: str | None = Field(
default=None,
description="Resolved region name for the transport code, if it matched a known region",
)
class MessagesAroundResponse(BaseModel):
@@ -474,6 +485,14 @@ class RawPacketBroadcast(BaseModel):
rssi: int | None = Field(default=None, description="Received signal strength in dBm")
decrypted: bool = False
decrypted_info: RawPacketDecryptedInfo | None = None
transport_code: int | None = Field(
default=None,
description="Region scope transport code (uint16) for TransportFlood/TransportDirect packets",
)
region: str | None = Field(
default=None,
description="Resolved region name for the transport code, if it matched a known region",
)
class RawPacketDetail(BaseModel):
@@ -489,6 +508,14 @@ class RawPacketDetail(BaseModel):
)
decrypted: bool = False
decrypted_info: RawPacketDecryptedInfo | None = None
transport_code: int | None = Field(
default=None,
description="Region scope transport code (uint16) for TransportFlood/TransportDirect packets",
)
region: str | None = Field(
default=None,
description="Resolved region name for the transport code, if it matched a known region",
)
class SendMessageRequest(BaseModel):
@@ -842,6 +869,13 @@ class AppSettings(BaseModel):
default="",
description="Outbound flood scope / region name (empty = disabled, no tagging)",
)
known_regions: list[str] = Field(
default_factory=list,
description=(
"Region scope names used to resolve incoming TransportFlood/TransportDirect "
"packets back to a readable region label (packet inspector + channel decoration)"
),
)
blocked_keys: list[str] = Field(
default_factory=list,
description="Public keys whose messages are hidden from the UI",
+40 -1
View File
@@ -36,7 +36,9 @@ from app.models import (
RawPacketDecryptedInfo,
)
from app.path_utils import calculate_packet_hash
from app.region_resolver import resolve_region
from app.repository import (
AppSettingsRepository,
ChannelRepository,
ContactAdvertPathRepository,
ContactRepository,
@@ -75,6 +77,8 @@ async def create_message_from_decrypted(
channel_name: str | None = None,
realtime: bool = True,
packet_hash: str | None = None,
transport_code: int | None = None,
region: str | None = None,
) -> int | None:
"""Store a decrypted channel message via the shared message service."""
return await _create_message_from_decrypted(
@@ -92,6 +96,8 @@ async def create_message_from_decrypted(
realtime=realtime,
broadcast_fn=broadcast_event,
packet_hash=packet_hash,
transport_code=transport_code,
region=region,
)
@@ -338,13 +344,40 @@ async def process_raw_packet(
# Compute packet hash once for threading into message broadcasts (used by bot fanout).
pkt_hash = calculate_packet_hash(raw_bytes)
# Resolve regional flood-scope for transport-routed packets. The transport code
# is a keyed MAC over the payload, so we recompute it for each known region name
# and keep the first match. Only transport-routed packets carry codes, so this is
# skipped for the common (unscoped) flood/direct case.
transport_code: int | None = None
region: str | None = None
if packet_info is not None and packet_info.transport_codes is not None:
transport_code = packet_info.transport_codes[0]
try:
settings = await AppSettingsRepository.get()
region = resolve_region(
int(packet_info.payload_type),
packet_info.payload,
transport_code,
settings.known_regions,
)
except Exception:
logger.debug("Region resolution failed for packet %d", packet_id, exc_info=True)
# Process packets based on payload type
# For GROUP_TEXT, we always try to decrypt even for duplicate packets - the message
# deduplication in create_message_from_decrypted handles adding paths to existing messages.
# This is more reliable than trying to look up the message via raw packet linking.
if payload_type == PayloadType.GROUP_TEXT:
decrypt_result = await _process_group_text(
raw_bytes, packet_id, ts, packet_info, rssi=rssi, snr=snr, packet_hash=pkt_hash
raw_bytes,
packet_id,
ts,
packet_info,
rssi=rssi,
snr=snr,
packet_hash=pkt_hash,
transport_code=transport_code,
region=region,
)
if decrypt_result:
result.update(decrypt_result)
@@ -403,6 +436,8 @@ async def process_raw_packet(
)
if result["decrypted"]
else None,
transport_code=transport_code,
region=region,
)
broadcast_event("raw_packet", broadcast_payload.model_dump())
@@ -417,6 +452,8 @@ async def _process_group_text(
rssi: int | None = None,
snr: float | None = None,
packet_hash: str | None = None,
transport_code: int | None = None,
region: str | None = None,
) -> dict | None:
"""
Process a GroupText (channel message) packet.
@@ -456,6 +493,8 @@ async def _process_group_text(
rssi=rssi,
snr=snr,
packet_hash=packet_hash,
transport_code=transport_code,
region=region,
)
return {
+12
View File
@@ -31,6 +31,12 @@ class ParsedPacketEnvelope:
path: bytes
payload: bytes
payload_offset: int
transport_codes: tuple[int, int] | None = None
"""Region transport codes (code_1, code_2) for TRANSPORT_* routes, else None.
Each is a little-endian uint16 read from the 4-byte transport-code block.
``code_1`` is the region-scope code; ``code_2`` is currently reserved (0).
"""
def decode_path_byte(path_byte: int) -> tuple[int, int]:
@@ -91,9 +97,14 @@ def parse_packet_envelope(raw_packet: bytes) -> ParsedPacketEnvelope | None:
payload_version = (header >> 6) & 0x03
offset = 1
transport_codes: tuple[int, int] | None = None
if route_type in (0x00, 0x03):
if len(raw_packet) < offset + 4:
return None
transport_codes = (
int.from_bytes(raw_packet[offset : offset + 2], "little"),
int.from_bytes(raw_packet[offset + 2 : offset + 4], "little"),
)
offset += 4
if len(raw_packet) < offset + 1:
@@ -123,6 +134,7 @@ def parse_packet_envelope(raw_packet: bytes) -> ParsedPacketEnvelope | None:
path=path,
payload=raw_packet[offset:],
payload_offset=offset,
transport_codes=transport_codes,
)
except (IndexError, ValueError):
return None
+79
View File
@@ -0,0 +1,79 @@
"""Resolve a packet's regional flood-scope (transport code) back to a region name.
MeshCore's TransportFlood/TransportDirect packets carry a 16-bit "transport code"
derived from the region name *and the packet payload* it is a keyed MAC, not a
stable per-region identifier:
key = SHA256("#" + region_name)[:16] # firmware TransportKey
code = HMAC-SHA256(key, payload_type || payload)[:2] # little-endian uint16
(see ``references/MeshCore/src/helpers/TransportKeyStore.cpp``). Codes ``0x0000``
and ``0xFFFF`` are reserved, so the firmware nudges them to ``0x0001`` / ``0xFFFE``.
Because the code depends on the payload, there is no reverse lookup table: to
name a packet's region we recompute the code for each known region name and check
for a match. The candidate region names come from the server-side
``app_settings.known_regions`` list.
"""
import hashlib
import hmac
from app.region_scope import normalize_region_scope
# SHA256("#name")[:16] is deterministic and cheap, but region lists are tiny and
# packets are frequent, so cache the derived 16-byte keys by normalized name.
_key_cache: dict[str, bytes] = {}
def _region_key(region_name: str) -> bytes | None:
"""Return the 16-byte TransportKey for a region name, or None if blank.
Region names are hashed in their hashtag form (``#name``), matching the
firmware (``getAutoKeyFor(id, "#" + name, ...)``).
"""
normalized = normalize_region_scope(region_name)
if not normalized:
return None
key = _key_cache.get(normalized)
if key is None:
key = hashlib.sha256(normalized.encode("utf-8")).digest()[:16]
_key_cache[normalized] = key
return key
def compute_transport_code(region_name: str, payload_type: int, payload: bytes) -> int | None:
"""Compute the transport code a region would produce for this payload.
Returns the little-endian uint16 code (with the firmware's reserved-value
adjustment applied), or None if the region name is blank.
"""
key = _region_key(region_name)
if key is None:
return None
digest = hmac.new(key, bytes([payload_type & 0xFF]) + payload, hashlib.sha256).digest()
code = int.from_bytes(digest[:2], "little")
if code == 0:
code = 1
elif code == 0xFFFF:
code = 0xFFFE
return code
def resolve_region(
payload_type: int,
payload: bytes,
transport_code: int,
region_names: list[str],
) -> str | None:
"""Return the first region name whose code matches ``transport_code``, else None.
The returned name is the user-facing form as stored in the candidate list
(no ``#`` prefix is added or stripped here).
"""
for name in region_names:
if not name:
continue
if compute_transport_code(name, payload_type, payload) == transport_code:
return name
return None
+14 -2
View File
@@ -64,6 +64,8 @@ class MessageRepository:
outgoing: bool = False,
sender_name: str | None = None,
sender_key: str | None = None,
transport_code: int | None = None,
region: str | None = None,
) -> int | None:
"""Create a message, returning the ID or None if duplicate.
@@ -94,8 +96,8 @@ class MessageRepository:
"""
INSERT OR IGNORE INTO messages (type, conversation_key, text, sender_timestamp,
received_at, paths, txt_type, signature, outgoing,
sender_name, sender_key)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
sender_name, sender_key, transport_code, region)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
""",
(
msg_type,
@@ -109,6 +111,8 @@ class MessageRepository:
outgoing,
sender_name,
normalized_sender_key,
transport_code,
region,
),
) as cursor:
rowcount = cursor.rowcount
@@ -357,10 +361,16 @@ class MessageRepository:
def _row_to_message(row: Any) -> Message:
"""Convert a database row to a Message model."""
packet_id = None
transport_code = None
region = None
if hasattr(row, "keys"):
row_keys = row.keys()
if "packet_id" in row_keys:
packet_id = row["packet_id"]
if "transport_code" in row_keys:
transport_code = row["transport_code"]
if "region" in row_keys:
region = row["region"]
return Message(
id=row["id"],
@@ -377,6 +387,8 @@ class MessageRepository:
acked=row["acked"],
sender_name=row["sender_name"],
packet_id=packet_id,
transport_code=transport_code,
region=region,
)
@staticmethod
+18 -1
View File
@@ -39,7 +39,7 @@ class AppSettingsRepository:
"""
SELECT max_radio_contacts, auto_decrypt_dm_on_advert,
last_message_times,
advert_interval, last_advert_time, flood_scope,
advert_interval, last_advert_time, flood_scope, known_regions,
blocked_keys, blocked_names, discovery_blocked_types,
tracked_telemetry_repeaters, tracked_telemetry_contacts,
auto_resend_channel,
@@ -81,6 +81,15 @@ class AppSettingsRepository:
except (json.JSONDecodeError, TypeError):
blocked_names = []
# Parse known_regions JSON
known_regions: list[str] = []
try:
raw_regions = row["known_regions"]
if raw_regions:
known_regions = json.loads(raw_regions)
except (json.JSONDecodeError, TypeError, KeyError):
known_regions = []
# Parse discovery_blocked_types JSON
discovery_blocked_types: list[int] = []
if row["discovery_blocked_types"]:
@@ -136,6 +145,7 @@ class AppSettingsRepository:
advert_interval=row["advert_interval"] or 0,
last_advert_time=row["last_advert_time"] or 0,
flood_scope=row["flood_scope"] or "",
known_regions=known_regions,
blocked_keys=blocked_keys,
blocked_names=blocked_names,
discovery_blocked_types=discovery_blocked_types,
@@ -156,6 +166,7 @@ class AppSettingsRepository:
advert_interval: int | None = None,
last_advert_time: int | None = None,
flood_scope: str | None = None,
known_regions: list[str] | None = None,
blocked_keys: list[str] | None = None,
blocked_names: list[str] | None = None,
discovery_blocked_types: list[int] | None = None,
@@ -197,6 +208,10 @@ class AppSettingsRepository:
updates.append("flood_scope = ?")
params.append(flood_scope)
if known_regions is not None:
updates.append("known_regions = ?")
params.append(json.dumps(known_regions))
if blocked_keys is not None:
updates.append("blocked_keys = ?")
params.append(json.dumps(blocked_keys))
@@ -251,6 +266,7 @@ class AppSettingsRepository:
advert_interval: int | None = None,
last_advert_time: int | None = None,
flood_scope: str | None = None,
known_regions: list[str] | None = None,
blocked_keys: list[str] | None = None,
blocked_names: list[str] | None = None,
discovery_blocked_types: list[int] | None = None,
@@ -270,6 +286,7 @@ class AppSettingsRepository:
advert_interval=advert_interval,
last_advert_time=last_advert_time,
flood_scope=flood_scope,
known_regions=known_regions,
blocked_keys=blocked_keys,
blocked_names=blocked_names,
discovery_blocked_types=discovery_blocked_types,
+40 -1
View File
@@ -10,7 +10,13 @@ from app.database import db
from app.decoder import parse_packet, try_decrypt_packet_with_channel_key
from app.models import RawPacketDecryptedInfo, RawPacketDetail
from app.packet_processor import create_message_from_decrypted, run_historical_dm_decryption
from app.repository import ChannelRepository, MessageRepository, RawPacketRepository
from app.region_resolver import resolve_region
from app.repository import (
AppSettingsRepository,
ChannelRepository,
MessageRepository,
RawPacketRepository,
)
from app.websocket import broadcast_success
logger = logging.getLogger(__name__)
@@ -58,6 +64,8 @@ async def _run_historical_channel_decryption(
logger.info("Starting historical channel decryption of %d packets", total)
known_regions = (await AppSettingsRepository.get()).known_regions
async for (
packet_id,
packet_data,
@@ -70,6 +78,18 @@ async def _run_historical_channel_decryption(
packet_info = parse_packet(packet_data)
path_hex = packet_info.path.hex() if packet_info else None
# Resolve regional flood-scope if this is a transport-routed packet.
transport_code: int | None = None
region: str | None = None
if packet_info is not None and packet_info.transport_codes is not None:
transport_code = packet_info.transport_codes[0]
region = resolve_region(
int(packet_info.payload_type),
packet_info.payload,
transport_code,
known_regions,
)
msg_id = await create_message_from_decrypted(
packet_id=packet_id,
channel_key=channel_key_hex,
@@ -81,6 +101,8 @@ async def _run_historical_channel_decryption(
path=path_hex,
path_len=packet_info.path_length if packet_info else None,
realtime=False, # Historical decryption should not trigger fanout
transport_code=transport_code,
region=region,
)
if msg_id is not None:
@@ -117,6 +139,21 @@ async def get_raw_packet(packet_id: int) -> RawPacketDetail:
packet_info = parse_packet(packet_data)
payload_type_name = packet_info.payload_type.name if packet_info else "Unknown"
# Resolve regional flood-scope for transport-routed packets against the
# current known-region list (we have the raw payload here, so this stays
# accurate even if the stored message predates a region-list change).
transport_code: int | None = None
region: str | None = None
if packet_info is not None and packet_info.transport_codes is not None:
transport_code = packet_info.transport_codes[0]
settings = await AppSettingsRepository.get()
region = resolve_region(
int(packet_info.payload_type),
packet_info.payload,
transport_code,
settings.known_regions,
)
decrypted_info: RawPacketDecryptedInfo | None = None
if message_id is not None:
message = await MessageRepository.get_by_id(message_id)
@@ -146,6 +183,8 @@ async def get_raw_packet(packet_id: int) -> RawPacketDetail:
payload_type=payload_type_name,
decrypted=message_id is not None,
decrypted_info=decrypted_info,
transport_code=transport_code,
region=region,
)
+21
View File
@@ -46,6 +46,13 @@ class AppSettingsUpdate(BaseModel):
default=None,
description="Outbound flood scope / region name (empty = disabled)",
)
known_regions: list[str] | None = Field(
default=None,
description=(
"Region scope names used to decode incoming transport-scoped packets "
"(stored without a leading '#')"
),
)
blocked_keys: list[str] | None = Field(
default=None,
description="Public keys whose messages are hidden from the UI",
@@ -212,6 +219,20 @@ async def update_settings(update: AppSettingsUpdate) -> AppSettings:
logger.info("Updating advert_interval to %d", interval)
kwargs["advert_interval"] = interval
# Known regions for scope decoding. Normalize to user-facing form (no leading
# '#'), trim blanks, and dedupe case-insensitively while preserving order.
if update.known_regions is not None:
cleaned_regions: list[str] = []
seen_regions: set[str] = set()
for raw_name in update.known_regions:
name = (raw_name or "").strip()
if name.startswith("#"):
name = name[1:].strip()
if name and name.lower() not in seen_regions:
seen_regions.add(name.lower())
cleaned_regions.append(name)
kwargs["known_regions"] = cleaned_regions
# Block lists
if update.blocked_keys is not None:
kwargs["blocked_keys"] = [k.lower() for k in update.blocked_keys]
+10
View File
@@ -69,6 +69,8 @@ def build_message_model(
sender_name: str | None = None,
channel_name: str | None = None,
packet_id: int | None = None,
transport_code: int | None = None,
region: str | None = None,
) -> Message:
"""Build a Message model with the canonical backend payload shape."""
return Message(
@@ -87,6 +89,8 @@ def build_message_model(
sender_name=sender_name,
channel_name=channel_name,
packet_id=packet_id,
transport_code=transport_code,
region=region,
)
@@ -276,6 +280,8 @@ async def create_message_from_decrypted(
realtime: bool = True,
broadcast_fn: BroadcastFn,
packet_hash: str | None = None,
transport_code: int | None = None,
region: str | None = None,
) -> int | None:
"""Store and broadcast a decrypted channel message."""
received = received_at or int(time.time())
@@ -300,6 +306,8 @@ async def create_message_from_decrypted(
snr=snr,
sender_name=sender,
sender_key=resolved_sender_key,
transport_code=transport_code,
region=region,
)
if msg_id is None:
@@ -341,6 +349,8 @@ async def create_message_from_decrypted(
sender_key=resolved_sender_key,
channel_name=channel_name,
packet_id=packet_id,
transport_code=transport_code,
region=region,
),
broadcast_fn=broadcast_fn,
realtime=realtime,