Support 2- and 3-byte path hashes and transport-routed packets

The MeshCore path_length byte encodes hop_count in its low 6 bits and a
hash_size_code in its high 2 bits (0/1/2 -> 1/2/3 bytes per hop), and
transport route types (TRANSPORT_FLOOD/TRANSPORT_DIRECT) insert a 4-byte
transport_codes field between the header and path_length. The packet
decoder previously assumed every hop was a single byte and that
path_length always sat at byte 2, so it only handled 1-byte-hash,
non-transport packets; anything else decoded to an over-long path and an
empty payload.

Migration 007 reworks the meshcore_packets read-time aliases to honor the
transport_codes offset and compute the path as hop_count * hash_size
bytes, and exposes hop_count / hash_size_code / hash_size (bytes per hop)
as columns. payload, path and packet_hash now decode correctly for every
route type and hash size; the adverts and public-channel derived tables
are rebuilt from the corrected decode (invalid hash_size_code 3 packets
are skipped per spec).

hash_size is carried through the chat and advert APIs so the path
visualization splits a path into hops of the correct width
(pathUtils/PathVisualization), instead of always slicing one byte per hop.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

ingest/migrations/007_fix_packet_parsing.sql

@@ -0,0 +1,415 @@
+-- +goose Up
+-- Fix MeshCore packet parsing in the meshcore_packets read-time ALIAS columns.
+--
+-- Two bugs vs. the Core Protocol wire format
+-- (header(1) | transport_codes(4, only if route_type in {0,3}) | path_length(1) | path(V) | payload):
+--   1. The path_length byte / path / payload aliases always read path_length at byte 2 and never
+--      skip the 4-byte transport_codes present on transport route types (0=TRANSPORT_FLOOD,
+--      3=TRANSPORT_DIRECT). For transport packets byte 2 is a transport-code byte (garbage).
+--   2. The path_length byte is NOT a raw byte count. Its low 6 bits are hop_count (0-63) and its
+--      high 2 bits are a hash_size_code (0/1/2 -> 1/2/3 bytes per hop; 3 is invalid). The on-wire
+--      path is hop_count * hash_size bytes. The old aliases treated the whole byte as the path
+--      byte length, so any packet with hash_size_code != 0 (byte >= 64) overran the path and
+--      decoded an empty payload.
+--
+-- Net effect: ~22% of packets decoded an empty payload, collapsing to a degenerate packet_hash
+-- (used as the chat message_id) and a degenerate '' advert public_key, and silently dropping their
+-- adverts/messages. The raw `packet` bytes are stored correctly, so fixing these ALIASes corrects
+-- all history on read. New helper columns are exposed (notably `hash_size` = bytes per hop) so
+-- downstream hop-splitters can chunk the path correctly.
+
+-- New helper alias columns (added in dependency order; all metadata-only, retroactive on read).
+ALTER TABLE meshcore_packets ADD COLUMN IF NOT EXISTS transport_off UInt8
+    ALIAS if(route_type IN (0, 3), 4, 0)
+    COMMENT 'Byte offset of transport_codes: 4 when route_type is TRANSPORT_FLOOD(0)/TRANSPORT_DIRECT(3), else 0';
+ALTER TABLE meshcore_packets ADD COLUMN IF NOT EXISTS path_len_byte UInt8
+    ALIAS reinterpretAsUInt8(substring(packet, 2 + transport_off, 1))
+    COMMENT 'Raw path_length byte (after the optional transport_codes): low 6 bits hop_count, high 2 bits hash_size_code';
+ALTER TABLE meshcore_packets ADD COLUMN IF NOT EXISTS hop_count UInt8
+    ALIAS bitAnd(path_len_byte, 0x3F)
+    COMMENT 'Number of hops in the path (low 6 bits of the path_length byte)';
+ALTER TABLE meshcore_packets ADD COLUMN IF NOT EXISTS hash_size_code UInt8
+    ALIAS bitShiftRight(path_len_byte, 6)
+    COMMENT 'Hash size code (high 2 bits of the path_length byte): 0->1B, 1->2B, 2->3B, 3->invalid';
+ALTER TABLE meshcore_packets ADD COLUMN IF NOT EXISTS hash_size UInt8
+    ALIAS [1, 2, 3, 0][hash_size_code + 1]
+    COMMENT 'Bytes per hop in the path (1/2/3); 0 means invalid hash_size_code (3) and the packet should be ignored';
+
+-- Correct the path/payload aliases. path_len now means the true path byte-length (hop_count*hash_size),
+-- matching its column comment. packet_hash references payload/path_len by name, so it self-corrects.
+ALTER TABLE meshcore_packets MODIFY COLUMN path_len UInt8
+    ALIAS hop_count * hash_size
+    COMMENT 'Length of the path field in bytes (hop_count * hash_size)';
+ALTER TABLE meshcore_packets MODIFY COLUMN path String
+    ALIAS hex(substring(packet, 3 + transport_off, hop_count * hash_size))
+    COMMENT 'Routing path as hex string (starts after header+transport_codes+path_length byte, length hop_count*hash_size)';
+ALTER TABLE meshcore_packets MODIFY COLUMN payload String
+    ALIAS substring(packet, 3 + transport_off + hop_count * hash_size)
+    COMMENT 'Payload (starts after the path)';
+
+-- Drop the incremental MVs BEFORE the chat-table schema change (a live MV whose SELECT column count
+-- no longer matches its target table would fail on insert), then recreate them: skip invalid
+-- hash_size_code=3 packets, and carry hash_size into the chat rows for correct path hop-splitting.
+DROP VIEW IF EXISTS meshcore_adverts_latest_mv;
+DROP VIEW IF EXISTS meshcore_public_channel_messages_mv;
+
+-- hash_size travels with each chat row so the UI can split the path into hops (hop = hash_size bytes).
+ALTER TABLE meshcore_public_channel_messages_raw ADD COLUMN IF NOT EXISTS hash_size UInt8;
+
+-- +goose StatementBegin
+CREATE MATERIALIZED VIEW IF NOT EXISTS meshcore_adverts_latest_mv
+TO meshcore_adverts_latest_state
+AS
+SELECT
+    hex(substring(payload, 1, 32)) AS public_key,
+    minState(ingest_timestamp) AS first_heard,
+    maxState(ingest_timestamp) AS last_seen,
+    argMaxState(toString(broker), ingest_timestamp) AS broker,
+    argMaxState(toString(topic), ingest_timestamp) AS topic,
+    argMaxState(multiIf(lower(meshcore_packets.topic) IN ('meshcore','meshcore/salish'), 'SEA', match(splitByChar('/', lower(meshcore_packets.topic))[2], '^[a-z]{3}$'), upper(splitByChar('/', lower(meshcore_packets.topic))[2]), ''), ingest_timestamp) AS region,
+    argMaxState(origin, ingest_timestamp) AS origin,
+    argMaxState(mesh_timestamp, ingest_timestamp) AS mesh_timestamp,
+    argMaxState(packet, ingest_timestamp) AS packet,
+    argMaxState(path_len, ingest_timestamp) AS path_len,
+    argMaxState(path, ingest_timestamp) AS path,
+    argMaxState(reinterpretAsUInt32(substring(payload, 33, 4)), ingest_timestamp) AS adv_timestamp,
+    argMaxState(hex(substring(payload, 37, 64)), ingest_timestamp) AS signature,
+    argMaxState(reinterpretAsUInt8(substring(payload, 101, 1)), ingest_timestamp) AS appdata_flags,
+    argMaxState(toUInt8(bitAnd(reinterpretAsUInt8(substring(payload, 101, 1)), 0x01) = 0x01), ingest_timestamp) AS is_chat_node,
+    argMaxState(toUInt8(bitAnd(reinterpretAsUInt8(substring(payload, 101, 1)), 0x02) = 0x02), ingest_timestamp) AS is_repeater,
+    argMaxState(toUInt8(bitAnd(reinterpretAsUInt8(substring(payload, 101, 1)), 0x03) = 0x03), ingest_timestamp) AS is_room_server,
+    argMaxState(toUInt8(bitAnd(reinterpretAsUInt8(substring(payload, 101, 1)), 0x10) = 0x10), ingest_timestamp) AS has_location,
+    argMaxState(toUInt8(bitAnd(reinterpretAsUInt8(substring(payload, 101, 1)), 0x20) = 0x20), ingest_timestamp) AS has_feature1,
+    argMaxState(toUInt8(bitAnd(reinterpretAsUInt8(substring(payload, 101, 1)), 0x40) = 0x40), ingest_timestamp) AS has_feature2,
+    argMaxState(toUInt8(bitAnd(reinterpretAsUInt8(substring(payload, 101, 1)), 0x80) = 0x80), ingest_timestamp) AS has_name,
+    argMaxState(CASE WHEN bitAnd(reinterpretAsUInt8(substring(payload, 101, 1)), 0x10) = 0x10 THEN reinterpretAsInt32(substring(payload, 102, 4)) ELSE NULL END, ingest_timestamp) AS latitude_i,
+    argMaxState(CASE WHEN bitAnd(reinterpretAsUInt8(substring(payload, 101, 1)), 0x10) = 0x10 THEN reinterpretAsInt32(substring(payload, 106, 4)) ELSE NULL END, ingest_timestamp) AS longitude_i,
+    argMaxState((CASE WHEN bitAnd(reinterpretAsUInt8(substring(payload, 101, 1)), 0x10) = 0x10 THEN reinterpretAsInt32(substring(payload, 102, 4)) ELSE NULL END) * 1e-6, ingest_timestamp) AS latitude,
+    argMaxState((CASE WHEN bitAnd(reinterpretAsUInt8(substring(payload, 101, 1)), 0x10) = 0x10 THEN reinterpretAsInt32(substring(payload, 106, 4)) ELSE NULL END) * 1e-6, ingest_timestamp) AS longitude,
+    argMaxState(substring(payload, 102 + multiIf(bitAnd(reinterpretAsUInt8(substring(payload, 101, 1)), 0x10) = 0x10, 8, 0)), ingest_timestamp) AS node_name,
+    argMaxState(hex(substring(payload, 1, 1)), ingest_timestamp) AS node_hash,
+    argMaxState(packet_hash, ingest_timestamp) AS packet_hash
+FROM meshcore_packets
+WHERE payload_type = 4 AND hash_size_code != 3
+GROUP BY public_key;
+-- +goose StatementEnd
+
+-- +goose StatementBegin
+CREATE MATERIALIZED VIEW IF NOT EXISTS meshcore_public_channel_messages_mv
+TO meshcore_public_channel_messages_raw
+AS
+SELECT
+    ingest_timestamp,
+    mesh_timestamp,
+    hex(substring(payload, 1, 1)) AS channel_hash,
+    hex(substring(payload, 2, 2)) AS mac,
+    substring(payload, 4) AS encrypted_message,
+    packet_hash AS message_id,
+    origin,
+    hex(origin_pubkey) AS origin_pubkey,
+    path,
+    toString(broker) AS broker,
+    toString(topic) AS topic,
+    multiIf(lower(meshcore_packets.topic) IN ('meshcore','meshcore/salish'), 'SEA', match(splitByChar('/', lower(meshcore_packets.topic))[2], '^[a-z]{3}$'), upper(splitByChar('/', lower(meshcore_packets.topic))[2]), '') AS region,
+    hash_size
+FROM meshcore_packets
+WHERE payload_type = 5 AND hash_size_code != 3;
+-- +goose StatementEnd
+
+-- Rebuild the derived tables from the now-correctly-decoded base table. Duplicate rows from the
+-- short MV-recreate -> backfill overlap are harmless: argMax/min/max states collapse on merge, and
+-- the chat table dedups by message_id at read.
+TRUNCATE TABLE meshcore_adverts_latest_state;
+-- +goose StatementBegin
+INSERT INTO meshcore_adverts_latest_state
+SELECT
+    hex(substring(payload, 1, 32)) AS public_key,
+    minState(ingest_timestamp),
+    maxState(ingest_timestamp),
+    argMaxState(toString(broker), ingest_timestamp),
+    argMaxState(toString(topic), ingest_timestamp),
+    argMaxState(multiIf(lower(meshcore_packets.topic) IN ('meshcore','meshcore/salish'), 'SEA', match(splitByChar('/', lower(meshcore_packets.topic))[2], '^[a-z]{3}$'), upper(splitByChar('/', lower(meshcore_packets.topic))[2]), ''), ingest_timestamp),
+    argMaxState(origin, ingest_timestamp),
+    argMaxState(mesh_timestamp, ingest_timestamp),
+    argMaxState(packet, ingest_timestamp),
+    argMaxState(path_len, ingest_timestamp),
+    argMaxState(path, ingest_timestamp),
+    argMaxState(reinterpretAsUInt32(substring(payload, 33, 4)), ingest_timestamp),
+    argMaxState(hex(substring(payload, 37, 64)), ingest_timestamp),
+    argMaxState(reinterpretAsUInt8(substring(payload, 101, 1)), ingest_timestamp),
+    argMaxState(toUInt8(bitAnd(reinterpretAsUInt8(substring(payload, 101, 1)), 0x01) = 0x01), ingest_timestamp),
+    argMaxState(toUInt8(bitAnd(reinterpretAsUInt8(substring(payload, 101, 1)), 0x02) = 0x02), ingest_timestamp),
+    argMaxState(toUInt8(bitAnd(reinterpretAsUInt8(substring(payload, 101, 1)), 0x03) = 0x03), ingest_timestamp),
+    argMaxState(toUInt8(bitAnd(reinterpretAsUInt8(substring(payload, 101, 1)), 0x10) = 0x10), ingest_timestamp),
+    argMaxState(toUInt8(bitAnd(reinterpretAsUInt8(substring(payload, 101, 1)), 0x20) = 0x20), ingest_timestamp),
+    argMaxState(toUInt8(bitAnd(reinterpretAsUInt8(substring(payload, 101, 1)), 0x40) = 0x40), ingest_timestamp),
+    argMaxState(toUInt8(bitAnd(reinterpretAsUInt8(substring(payload, 101, 1)), 0x80) = 0x80), ingest_timestamp),
+    argMaxState(CASE WHEN bitAnd(reinterpretAsUInt8(substring(payload, 101, 1)), 0x10) = 0x10 THEN reinterpretAsInt32(substring(payload, 102, 4)) ELSE NULL END, ingest_timestamp),
+    argMaxState(CASE WHEN bitAnd(reinterpretAsUInt8(substring(payload, 101, 1)), 0x10) = 0x10 THEN reinterpretAsInt32(substring(payload, 106, 4)) ELSE NULL END, ingest_timestamp),
+    argMaxState((CASE WHEN bitAnd(reinterpretAsUInt8(substring(payload, 101, 1)), 0x10) = 0x10 THEN reinterpretAsInt32(substring(payload, 102, 4)) ELSE NULL END) * 1e-6, ingest_timestamp),
+    argMaxState((CASE WHEN bitAnd(reinterpretAsUInt8(substring(payload, 101, 1)), 0x10) = 0x10 THEN reinterpretAsInt32(substring(payload, 106, 4)) ELSE NULL END) * 1e-6, ingest_timestamp),
+    argMaxState(substring(payload, 102 + multiIf(bitAnd(reinterpretAsUInt8(substring(payload, 101, 1)), 0x10) = 0x10, 8, 0)), ingest_timestamp),
+    argMaxState(hex(substring(payload, 1, 1)), ingest_timestamp),
+    argMaxState(packet_hash, ingest_timestamp)
+FROM meshcore_packets
+WHERE payload_type = 4 AND hash_size_code != 3
+GROUP BY public_key;
+-- +goose StatementEnd
+
+TRUNCATE TABLE meshcore_public_channel_messages_raw;
+-- +goose StatementBegin
+INSERT INTO meshcore_public_channel_messages_raw
+SELECT
+    ingest_timestamp,
+    mesh_timestamp,
+    hex(substring(payload, 1, 1)) AS channel_hash,
+    hex(substring(payload, 2, 2)) AS mac,
+    substring(payload, 4) AS encrypted_message,
+    packet_hash AS message_id,
+    origin,
+    hex(origin_pubkey) AS origin_pubkey,
+    path,
+    toString(broker) AS broker,
+    toString(topic) AS topic,
+    multiIf(lower(meshcore_packets.topic) IN ('meshcore','meshcore/salish'), 'SEA', match(splitByChar('/', lower(meshcore_packets.topic))[2], '^[a-z]{3}$'), upper(splitByChar('/', lower(meshcore_packets.topic))[2]), '') AS region,
+    hash_size
+FROM meshcore_packets
+WHERE payload_type = 5 AND hash_size_code != 3;
+-- +goose StatementEnd
+
+-- Expose hash_size on the adverts view so the node page can split advert paths into hops too.
+-- +goose StatementBegin
+CREATE OR REPLACE VIEW meshcore_adverts AS
+SELECT
+    ingest_timestamp,
+    origin,
+    origin_pubkey,
+    mesh_timestamp,
+    packet,
+    path_len,
+    path,
+    hash_size,
+    broker,
+    topic,
+    multiIf(lower(topic) IN ('meshcore','meshcore/salish'), 'SEA', match(splitByChar('/', lower(topic))[2], '^[a-z]{3}$'), upper(splitByChar('/', lower(topic))[2]), '') AS region,
+    hex(substring(payload, 1, 32)) AS public_key,
+    reinterpretAsUInt32(substring(payload, 33, 4)) AS adv_timestamp,
+    hex(substring(payload, 37, 64)) AS signature,
+    reinterpretAsUInt8(substring(payload, 101, 1)) AS appdata_flags,
+    bitAnd(reinterpretAsUInt8(substring(payload, 101, 1)), 0x01) = 0x01 AS is_chat_node,
+    bitAnd(reinterpretAsUInt8(substring(payload, 101, 1)), 0x02) = 0x02 AS is_repeater,
+    bitAnd(reinterpretAsUInt8(substring(payload, 101, 1)), 0x03) = 0x03 AS is_room_server,
+    bitAnd(reinterpretAsUInt8(substring(payload, 101, 1)), 0x10) = 0x10 AS has_location,
+    bitAnd(reinterpretAsUInt8(substring(payload, 101, 1)), 0x20) = 0x20 AS has_feature1,
+    bitAnd(reinterpretAsUInt8(substring(payload, 101, 1)), 0x40) = 0x40 AS has_feature2,
+    bitAnd(reinterpretAsUInt8(substring(payload, 101, 1)), 0x80) = 0x80 AS has_name,
+    CASE WHEN bitAnd(reinterpretAsUInt8(substring(payload, 101, 1)), 0x10) = 0x10
+        THEN reinterpretAsInt32(substring(payload, 102, 4))
+        ELSE NULL
+    END AS latitude_i,
+    CASE WHEN bitAnd(reinterpretAsUInt8(substring(payload, 101, 1)), 0x10) = 0x10
+        THEN reinterpretAsInt32(substring(payload, 106, 4))
+        ELSE NULL
+    END AS longitude_i,
+    latitude_i * 1e-6 AS latitude,
+    longitude_i * 1e-6 AS longitude,
+    substring(
+        payload,
+        102
+        + multiIf(bitAnd(reinterpretAsUInt8(substring(payload, 101, 1)), 0x10) = 0x10, 8, 0)
+    ) AS     node_name,
+    hex(substring(payload, 1, 1)) AS node_hash,
+    packet_hash
+FROM meshcore_packets
+WHERE payload_type = 4;
+-- +goose StatementEnd
+
+-- The neighbor/region REFRESH MVs (003/004) recompute hourly from the corrected aliases; trigger
+-- an immediate refresh so they pick up the fix now.
+SYSTEM REFRESH VIEW meshcore_all_neighbor_edges;
+SYSTEM REFRESH VIEW meshcore_node_direct_neighbors;
+SYSTEM REFRESH VIEW meshcore_regions;
+
+
+-- +goose Down
+-- Restore the migration-004 adverts view (no hash_size).
+-- +goose StatementBegin
+CREATE OR REPLACE VIEW meshcore_adverts AS
+SELECT
+    ingest_timestamp,
+    origin,
+    origin_pubkey,
+    mesh_timestamp,
+    packet,
+    path_len,
+    path,
+    broker,
+    topic,
+    multiIf(lower(topic) IN ('meshcore','meshcore/salish'), 'SEA', match(splitByChar('/', lower(topic))[2], '^[a-z]{3}$'), upper(splitByChar('/', lower(topic))[2]), '') AS region,
+    hex(substring(payload, 1, 32)) AS public_key,
+    reinterpretAsUInt32(substring(payload, 33, 4)) AS adv_timestamp,
+    hex(substring(payload, 37, 64)) AS signature,
+    reinterpretAsUInt8(substring(payload, 101, 1)) AS appdata_flags,
+    bitAnd(reinterpretAsUInt8(substring(payload, 101, 1)), 0x01) = 0x01 AS is_chat_node,
+    bitAnd(reinterpretAsUInt8(substring(payload, 101, 1)), 0x02) = 0x02 AS is_repeater,
+    bitAnd(reinterpretAsUInt8(substring(payload, 101, 1)), 0x03) = 0x03 AS is_room_server,
+    bitAnd(reinterpretAsUInt8(substring(payload, 101, 1)), 0x10) = 0x10 AS has_location,
+    bitAnd(reinterpretAsUInt8(substring(payload, 101, 1)), 0x20) = 0x20 AS has_feature1,
+    bitAnd(reinterpretAsUInt8(substring(payload, 101, 1)), 0x40) = 0x40 AS has_feature2,
+    bitAnd(reinterpretAsUInt8(substring(payload, 101, 1)), 0x80) = 0x80 AS has_name,
+    CASE WHEN bitAnd(reinterpretAsUInt8(substring(payload, 101, 1)), 0x10) = 0x10
+        THEN reinterpretAsInt32(substring(payload, 102, 4))
+        ELSE NULL
+    END AS latitude_i,
+    CASE WHEN bitAnd(reinterpretAsUInt8(substring(payload, 101, 1)), 0x10) = 0x10
+        THEN reinterpretAsInt32(substring(payload, 106, 4))
+        ELSE NULL
+    END AS longitude_i,
+    latitude_i * 1e-6 AS latitude,
+    longitude_i * 1e-6 AS longitude,
+    substring(
+        payload,
+        102
+        + multiIf(bitAnd(reinterpretAsUInt8(substring(payload, 101, 1)), 0x10) = 0x10, 8, 0)
+    ) AS     node_name,
+    hex(substring(payload, 1, 1)) AS node_hash,
+    packet_hash
+FROM meshcore_packets
+WHERE payload_type = 4;
+-- +goose StatementEnd
+
+DROP VIEW IF EXISTS meshcore_adverts_latest_mv;
+DROP VIEW IF EXISTS meshcore_public_channel_messages_mv;
+ALTER TABLE meshcore_public_channel_messages_raw DROP COLUMN IF EXISTS hash_size;
+
+-- Restore the original (buggy) aliases from migration 001 and drop the helper columns.
+ALTER TABLE meshcore_packets MODIFY COLUMN path_len UInt8
+    ALIAS reinterpretAsUInt8(substring(packet, 2, 1))
+    COMMENT 'Length of the path field in bytes (byte 2 of packet)';
+ALTER TABLE meshcore_packets MODIFY COLUMN path String
+    ALIAS hex(substring(packet, 3, path_len))
+    COMMENT 'Routing path as hex string (variable length, starts at byte 3, length path_len)';
+ALTER TABLE meshcore_packets MODIFY COLUMN payload String
+    ALIAS substring(packet, 3 + path_len, length(packet) - 2 - path_len)
+    COMMENT 'Payload (starts after path, up to 184 bytes)';
+ALTER TABLE meshcore_packets DROP COLUMN IF EXISTS hash_size;
+ALTER TABLE meshcore_packets DROP COLUMN IF EXISTS hash_size_code;
+ALTER TABLE meshcore_packets DROP COLUMN IF EXISTS hop_count;
+ALTER TABLE meshcore_packets DROP COLUMN IF EXISTS path_len_byte;
+ALTER TABLE meshcore_packets DROP COLUMN IF EXISTS transport_off;
+
+-- +goose StatementBegin
+CREATE MATERIALIZED VIEW IF NOT EXISTS meshcore_adverts_latest_mv
+TO meshcore_adverts_latest_state
+AS
+SELECT
+    hex(substring(payload, 1, 32)) AS public_key,
+    minState(ingest_timestamp) AS first_heard,
+    maxState(ingest_timestamp) AS last_seen,
+    argMaxState(toString(broker), ingest_timestamp) AS broker,
+    argMaxState(toString(topic), ingest_timestamp) AS topic,
+    argMaxState(multiIf(lower(meshcore_packets.topic) IN ('meshcore','meshcore/salish'), 'SEA', match(splitByChar('/', lower(meshcore_packets.topic))[2], '^[a-z]{3}$'), upper(splitByChar('/', lower(meshcore_packets.topic))[2]), ''), ingest_timestamp) AS region,
+    argMaxState(origin, ingest_timestamp) AS origin,
+    argMaxState(mesh_timestamp, ingest_timestamp) AS mesh_timestamp,
+    argMaxState(packet, ingest_timestamp) AS packet,
+    argMaxState(path_len, ingest_timestamp) AS path_len,
+    argMaxState(path, ingest_timestamp) AS path,
+    argMaxState(reinterpretAsUInt32(substring(payload, 33, 4)), ingest_timestamp) AS adv_timestamp,
+    argMaxState(hex(substring(payload, 37, 64)), ingest_timestamp) AS signature,
+    argMaxState(reinterpretAsUInt8(substring(payload, 101, 1)), ingest_timestamp) AS appdata_flags,
+    argMaxState(toUInt8(bitAnd(reinterpretAsUInt8(substring(payload, 101, 1)), 0x01) = 0x01), ingest_timestamp) AS is_chat_node,
+    argMaxState(toUInt8(bitAnd(reinterpretAsUInt8(substring(payload, 101, 1)), 0x02) = 0x02), ingest_timestamp) AS is_repeater,
+    argMaxState(toUInt8(bitAnd(reinterpretAsUInt8(substring(payload, 101, 1)), 0x03) = 0x03), ingest_timestamp) AS is_room_server,
+    argMaxState(toUInt8(bitAnd(reinterpretAsUInt8(substring(payload, 101, 1)), 0x10) = 0x10), ingest_timestamp) AS has_location,
+    argMaxState(toUInt8(bitAnd(reinterpretAsUInt8(substring(payload, 101, 1)), 0x20) = 0x20), ingest_timestamp) AS has_feature1,
+    argMaxState(toUInt8(bitAnd(reinterpretAsUInt8(substring(payload, 101, 1)), 0x40) = 0x40), ingest_timestamp) AS has_feature2,
+    argMaxState(toUInt8(bitAnd(reinterpretAsUInt8(substring(payload, 101, 1)), 0x80) = 0x80), ingest_timestamp) AS has_name,
+    argMaxState(CASE WHEN bitAnd(reinterpretAsUInt8(substring(payload, 101, 1)), 0x10) = 0x10 THEN reinterpretAsInt32(substring(payload, 102, 4)) ELSE NULL END, ingest_timestamp) AS latitude_i,
+    argMaxState(CASE WHEN bitAnd(reinterpretAsUInt8(substring(payload, 101, 1)), 0x10) = 0x10 THEN reinterpretAsInt32(substring(payload, 106, 4)) ELSE NULL END, ingest_timestamp) AS longitude_i,
+    argMaxState((CASE WHEN bitAnd(reinterpretAsUInt8(substring(payload, 101, 1)), 0x10) = 0x10 THEN reinterpretAsInt32(substring(payload, 102, 4)) ELSE NULL END) * 1e-6, ingest_timestamp) AS latitude,
+    argMaxState((CASE WHEN bitAnd(reinterpretAsUInt8(substring(payload, 101, 1)), 0x10) = 0x10 THEN reinterpretAsInt32(substring(payload, 106, 4)) ELSE NULL END) * 1e-6, ingest_timestamp) AS longitude,
+    argMaxState(substring(payload, 102 + multiIf(bitAnd(reinterpretAsUInt8(substring(payload, 101, 1)), 0x10) = 0x10, 8, 0)), ingest_timestamp) AS node_name,
+    argMaxState(hex(substring(payload, 1, 1)), ingest_timestamp) AS node_hash,
+    argMaxState(packet_hash, ingest_timestamp) AS packet_hash
+FROM meshcore_packets
+WHERE payload_type = 4
+GROUP BY public_key;
+-- +goose StatementEnd
+
+-- +goose StatementBegin
+CREATE MATERIALIZED VIEW IF NOT EXISTS meshcore_public_channel_messages_mv
+TO meshcore_public_channel_messages_raw
+AS
+SELECT
+    ingest_timestamp,
+    mesh_timestamp,
+    hex(substring(payload, 1, 1)) AS channel_hash,
+    hex(substring(payload, 2, 2)) AS mac,
+    substring(payload, 4) AS encrypted_message,
+    packet_hash AS message_id,
+    origin,
+    hex(origin_pubkey) AS origin_pubkey,
+    path,
+    toString(broker) AS broker,
+    toString(topic) AS topic,
+    multiIf(lower(meshcore_packets.topic) IN ('meshcore','meshcore/salish'), 'SEA', match(splitByChar('/', lower(meshcore_packets.topic))[2], '^[a-z]{3}$'), upper(splitByChar('/', lower(meshcore_packets.topic))[2]), '') AS region
+FROM meshcore_packets
+WHERE payload_type = 5;
+-- +goose StatementEnd
+
+TRUNCATE TABLE meshcore_adverts_latest_state;
+-- +goose StatementBegin
+INSERT INTO meshcore_adverts_latest_state
+SELECT
+    hex(substring(payload, 1, 32)) AS public_key,
+    minState(ingest_timestamp),
+    maxState(ingest_timestamp),
+    argMaxState(toString(broker), ingest_timestamp),
+    argMaxState(toString(topic), ingest_timestamp),
+    argMaxState(multiIf(lower(meshcore_packets.topic) IN ('meshcore','meshcore/salish'), 'SEA', match(splitByChar('/', lower(meshcore_packets.topic))[2], '^[a-z]{3}$'), upper(splitByChar('/', lower(meshcore_packets.topic))[2]), ''), ingest_timestamp),
+    argMaxState(origin, ingest_timestamp),
+    argMaxState(mesh_timestamp, ingest_timestamp),
+    argMaxState(packet, ingest_timestamp),
+    argMaxState(path_len, ingest_timestamp),
+    argMaxState(path, ingest_timestamp),
+    argMaxState(reinterpretAsUInt32(substring(payload, 33, 4)), ingest_timestamp),
+    argMaxState(hex(substring(payload, 37, 64)), ingest_timestamp),
+    argMaxState(reinterpretAsUInt8(substring(payload, 101, 1)), ingest_timestamp),
+    argMaxState(toUInt8(bitAnd(reinterpretAsUInt8(substring(payload, 101, 1)), 0x01) = 0x01), ingest_timestamp),
+    argMaxState(toUInt8(bitAnd(reinterpretAsUInt8(substring(payload, 101, 1)), 0x02) = 0x02), ingest_timestamp),
+    argMaxState(toUInt8(bitAnd(reinterpretAsUInt8(substring(payload, 101, 1)), 0x03) = 0x03), ingest_timestamp),
+    argMaxState(toUInt8(bitAnd(reinterpretAsUInt8(substring(payload, 101, 1)), 0x10) = 0x10), ingest_timestamp),
+    argMaxState(toUInt8(bitAnd(reinterpretAsUInt8(substring(payload, 101, 1)), 0x20) = 0x20), ingest_timestamp),
+    argMaxState(toUInt8(bitAnd(reinterpretAsUInt8(substring(payload, 101, 1)), 0x40) = 0x40), ingest_timestamp),
+    argMaxState(toUInt8(bitAnd(reinterpretAsUInt8(substring(payload, 101, 1)), 0x80) = 0x80), ingest_timestamp),
+    argMaxState(CASE WHEN bitAnd(reinterpretAsUInt8(substring(payload, 101, 1)), 0x10) = 0x10 THEN reinterpretAsInt32(substring(payload, 102, 4)) ELSE NULL END, ingest_timestamp),
+    argMaxState(CASE WHEN bitAnd(reinterpretAsUInt8(substring(payload, 101, 1)), 0x10) = 0x10 THEN reinterpretAsInt32(substring(payload, 106, 4)) ELSE NULL END, ingest_timestamp),
+    argMaxState((CASE WHEN bitAnd(reinterpretAsUInt8(substring(payload, 101, 1)), 0x10) = 0x10 THEN reinterpretAsInt32(substring(payload, 102, 4)) ELSE NULL END) * 1e-6, ingest_timestamp),
+    argMaxState((CASE WHEN bitAnd(reinterpretAsUInt8(substring(payload, 101, 1)), 0x10) = 0x10 THEN reinterpretAsInt32(substring(payload, 106, 4)) ELSE NULL END) * 1e-6, ingest_timestamp),
+    argMaxState(substring(payload, 102 + multiIf(bitAnd(reinterpretAsUInt8(substring(payload, 101, 1)), 0x10) = 0x10, 8, 0)), ingest_timestamp),
+    argMaxState(hex(substring(payload, 1, 1)), ingest_timestamp),
+    argMaxState(packet_hash, ingest_timestamp)
+FROM meshcore_packets
+WHERE payload_type = 4
+GROUP BY public_key;
+-- +goose StatementEnd
+
+TRUNCATE TABLE meshcore_public_channel_messages_raw;
+-- +goose StatementBegin
+INSERT INTO meshcore_public_channel_messages_raw
+SELECT
+    ingest_timestamp,
+    mesh_timestamp,
+    hex(substring(payload, 1, 1)) AS channel_hash,
+    hex(substring(payload, 2, 2)) AS mac,
+    substring(payload, 4) AS encrypted_message,
+    packet_hash AS message_id,
+    origin,
+    hex(origin_pubkey) AS origin_pubkey,
+    path,
+    toString(broker) AS broker,
+    toString(topic) AS topic,
+    multiIf(lower(meshcore_packets.topic) IN ('meshcore','meshcore/salish'), 'SEA', match(splitByChar('/', lower(meshcore_packets.topic))[2], '^[a-z]{3}$'), upper(splitByChar('/', lower(meshcore_packets.topic))[2]), '') AS region
+FROM meshcore_packets
+WHERE payload_type = 5;
+-- +goose StatementEnd

meshexplorer/src/components/AdvertDetails.tsx

@@ -19,6 +19,7 @@ interface AdvertDetailsProps {
     is_room_server: number;
     has_location: number;
     packet_hash: string;
+    hash_size?: number; // bytes per path hop (1/2/3); used to split path into hops
   };
   initiatingNodeKey?: string;
 }
@@ -95,7 +96,8 @@ export default function AdvertDetails({ advert, initiatingNodeKey }: AdvertDetai
                 paths={advert.origin_path_pubkey_tuples.map(([origin, path, origin_pubkey], index) => ({
                   origin: origin || origin_pubkey.substring(0, 8), // Use origin name if available, fallback to pubkey
                   pubkey: origin_pubkey,
-                  path: path
+                  path: path,
+                  hashSize: advert.hash_size
                 }))}
                 className="text-sm"
                 initiatingNodeKey={initiatingNodeKey}

meshexplorer/src/components/ChatMessageItem.tsx

@@ -19,6 +19,7 @@ export interface ChatMessage {
   encrypted_message: string;
   message_count: number;
   origin_path_info: Array<[string, string, string, string, string]>; // Array of [origin, origin_pubkey, path, broker, topic] tuples
+  hash_size?: number; // bytes per path hop (1/2/3); used to split path into hops
 }
 
 
@@ -134,9 +135,10 @@ function ChatMessageItem({ msg, showErrorRow }: { msg: ChatMessage, showErrorRow
     originPathInfo.map(([origin, origin_pubkey, path, broker, topic]) => ({
       origin,
       pubkey: origin_pubkey,
-      path
+      path,
+      hashSize: msg.hash_size
     })),
-    [originPathInfo]
+    [originPathInfo, msg.hash_size]
   );
 
 

meshexplorer/src/components/PathVisualization.tsx

@@ -50,11 +50,14 @@ export default function PathVisualization({
     [paths]
   );
 
+  // All paths in one render share the same hash size (same message/advert).
+  const hashSize = paths[0]?.hashSize;
+
   // Process data for tree visualization
   const treeData = useMemo(() => {
     if (!showGraph || pathsCount === 0) return null;
-    return buildTreeFromPathGroups(pathGroups, initiatingNodeKey);
-  }, [showGraph, pathsCount, pathGroups, initiatingNodeKey]);
+    return buildTreeFromPathGroups(pathGroups, initiatingNodeKey, hashSize);
+  }, [showGraph, pathsCount, pathGroups, initiatingNodeKey, hashSize]);
 
   // Extract unique prefixes from tree data for name lookups
   const uniquePrefixes = useMemo(() => 

meshexplorer/src/hooks/useNodeData.ts

@@ -30,6 +30,7 @@ export interface Advert {
   is_room_server: number;
   has_location: number;
   packet_hash: string;
+  hash_size?: number; // bytes per path hop (1/2/3); used to split path into hops
 }
 
 export interface LocationHistory {

meshexplorer/src/lib/clickhouse/actions.ts

@@ -94,7 +94,7 @@ export async function getLatestChatMessages({ limit = 20, before, after, channel
     }
 
     const whereClause = outerWhere.length > 0 ? `WHERE ${outerWhere.join(' AND ')}` : '';
-    const query = `SELECT ingest_timestamp, mesh_timestamp, channel_hash, mac, hex(encrypted_message) AS encrypted_message, message_count, origin_path_info, message_id FROM ${publicChannelMessagesSubquery(innerWhere)} ${whereClause} ORDER BY ingest_timestamp DESC LIMIT {limit:UInt32}`;
+    const query = `SELECT ingest_timestamp, mesh_timestamp, channel_hash, mac, hex(encrypted_message) AS encrypted_message, message_count, origin_path_info, hash_size, message_id FROM ${publicChannelMessagesSubquery(innerWhere)} ${whereClause} ORDER BY ingest_timestamp DESC LIMIT {limit:UInt32}`;
     const resultSet = await clickhouse.query({ query, query_params: params, format: 'JSONEachRow' });
     const rows = await resultSet.json();
     return rows as Array<{
@@ -105,6 +105,7 @@ export async function getLatestChatMessages({ limit = 20, before, after, channel
       encrypted_message: string;
       message_count: number;
       origin_path_info: Array<[string, string, string, string, string]>; // Array of [origin, origin_pubkey, path, broker, topic] tuples
+      hash_size: number; // bytes per path hop (1/2/3); used to split path into hops
       message_id: string;
     }>;
   } catch (error) {
@@ -189,14 +190,16 @@ export async function getMeshcoreNodeInfo(publicKey: string, limit: number = 50)
         argMax(is_chat_node, ingest_timestamp) as is_chat_node,
         argMax(is_room_server, ingest_timestamp) as is_room_server,
         argMax(has_location, ingest_timestamp) as has_location,
+        any(hash_size) as hash_size,
         packet_hash
       FROM (
-        SELECT 
+        SELECT
           ingest_timestamp,
           mesh_timestamp,
           adv_timestamp,
           hex(path) as path,
           path_len,
+          hash_size,
           latitude,
           longitude,
           is_repeater,

meshexplorer/src/lib/clickhouse/chatMessages.ts

@@ -31,6 +31,7 @@ export function publicChannelMessagesSubquery(innerConditions: string[] = []): s
       any(encrypted_message) AS encrypted_message,
       count() AS message_count,
       groupArray((origin, origin_pubkey, path, broker, topic)) AS origin_path_info,
+      any(hash_size) AS hash_size,
       message_id
     FROM meshcore_public_channel_messages_raw
     ${where}

meshexplorer/src/lib/clickhouse/streaming.ts

@@ -274,6 +274,7 @@ export function createChatMessagesStreamerConfig(
         hex(encrypted_message) AS encrypted_message,
         message_count,
         origin_path_info,
+        hash_size,
         message_id
       FROM ${publicChannelMessagesSubquery(innerConditions)}
       WHERE ingest_timestamp > {lastTimestamp:DateTime64}

meshexplorer/src/lib/pathUtils.ts

@@ -2,6 +2,10 @@ export interface PathData {
   origin: string;
   pubkey: string;
   path: string;
+  // Bytes per path hop (MeshCore path hash size: 1/2/3). The hex `path` is a
+  // sequence of hop identifiers, each hashSize bytes (2*hashSize hex chars).
+  // Defaults to 1 for backward compatibility / unknown.
+  hashSize?: number;
 }
 
 export interface PathGroup {
@@ -22,10 +26,13 @@ export interface TreeNode {
 export function groupPathsByStructure(paths: PathData[]): PathGroup[] {
   const pathGroups: PathGroup[] = [];
   
-  paths.forEach(({ origin, pubkey, path }, index) => {
-    // Parse path into 2-character slices and include pubkey as final hop
-    const pathSlices = path.match(/.{1,2}/g) || [];
-    const pubkeyPrefix = pubkey.substring(0, 2);
+  paths.forEach(({ origin, pubkey, path, hashSize }, index) => {
+    // Each hop identifier is hashSize bytes = 2*hashSize hex chars. Split the path
+    // accordingly, and take the matching-length prefix of the origin pubkey as the
+    // final hop so it's comparable to the in-path hop identifiers.
+    const hopChars = 2 * (hashSize && hashSize > 0 ? hashSize : 1);
+    const pathSlices = path.match(new RegExp(`.{1,${hopChars}}`, "g")) || [];
+    const pubkeyPrefix = pubkey.substring(0, hopChars);
     const fullPathSlices = [...pathSlices, pubkeyPrefix];
     
     // Find existing group with same path structure
@@ -53,8 +60,9 @@ export function groupPathsByStructure(paths: PathData[]): PathGroup[] {
 /**
  * Builds a tree structure from path groups for visualization
  */
-export function buildTreeFromPathGroups(pathGroups: PathGroup[], initiatingNodeKey?: string): TreeNode {
-  const rootName = initiatingNodeKey ? initiatingNodeKey.substring(0, 2) : "??";
+export function buildTreeFromPathGroups(pathGroups: PathGroup[], initiatingNodeKey?: string, hashSize?: number): TreeNode {
+  const hopChars = 2 * (hashSize && hashSize > 0 ? hashSize : 1);
+  const rootName = initiatingNodeKey ? initiatingNodeKey.substring(0, hopChars) : "??";
   const root: TreeNode = { name: rootName, children: [] };
   
   pathGroups.forEach(group => {