map changes and other fixes

2026-06-27 05:21:19 +02:00 · 2026-06-25 17:19:35 -07:00
parent 5386f39611
commit e2adb73fea
6 changed files with 190 additions and 28 deletions
@@ -19,6 +19,7 @@ logger = logging.getLogger(__name__)
 MQTT_GATEWAY_CACHE: set[int] = set()
 UNKNOWN_NODE_NAME = "unknown name"
 NODE_NAME_CONTROL_CHARS = re.compile(r"[\x00-\x1f\x7f]")
+MQTT_DIRECT_NODE_ID = 1


 def normalize_node_name(name: str) -> str:
@@ -28,6 +29,24 @@ def normalize_node_name(name: str) -> str:
    return name


+def storage_packet_id(packet) -> int | None:
+    if packet.id:
+        return packet.id
+
+    if packet.decoded.portnum != PortNum.MAP_REPORT_APP:
+        return None
+
+    from_node_id = getattr(packet, "from", 0) or 0
+    now_us = int(time.time() * 1_000_000)
+    return -(now_us * 2048 + (from_node_id & 0x7FF))
+
+
+def storage_to_node_id(packet) -> int:
+    if packet.decoded.portnum == PortNum.MAP_REPORT_APP:
+        return MQTT_DIRECT_NODE_ID
+    return packet.to
+
+
 async def capture_daily_snapshot() -> None:
    today = datetime.now(UTC).date()

@@ -147,20 +166,21 @@ async def process_envelope(topic, env):

            await session.commit()

-    if not env.packet.id:
+    packet_id = storage_packet_id(env.packet)
+    if packet_id is None:
        return

    async with mqtt_database.async_session() as session:
        # --- Packet insert with ON CONFLICT DO NOTHING
-        result = await session.execute(select(Packet).where(Packet.id == env.packet.id))
+        result = await session.execute(select(Packet).where(Packet.id == packet_id))
        packet = result.scalar_one_or_none()
        if not packet:
            now_us = int(time.time() * 1_000_000)
            packet_values = {
-                "id": env.packet.id,
+                "id": packet_id,
                "portnum": env.packet.decoded.portnum,
                "from_node_id": getattr(env.packet, "from"),
-                "to_node_id": env.packet.to,
+                "to_node_id": storage_to_node_id(env.packet),
                "payload": env.packet.SerializeToString(),
                "import_time_us": now_us,
                "channel": env.channel_id,
@@ -208,7 +228,7 @@ async def process_envelope(topic, env):

        now_us = int(time.time() * 1_000_000)
        seen_values = {
-            "packet_id": env.packet.id,
+            "packet_id": packet_id,
            "node_id": int(env.gateway_id[1:], 16),
            "channel": env.channel_id,
            "rx_time": env.packet.rx_time,
@@ -241,7 +241,7 @@ function logPacketTimes(packet) {
    const times = formatTimes(packet.import_time_us);
    console.log(
        "[firehose] packet time",
-        "id=" + packet.id,
+        "id=" + (packet.packet_id ?? packet.id),
        "epoch_us=" + times.epoch,
        "local=" + times.local,
        "utc=" + times.utc
@@ -372,6 +372,14 @@ async function fetchUpdates() {
            const safePayload = (pkt.payload || "")
                .replace(/</g, "&lt;")
                .replace(/>/g, "&gt;");
+            const packetIdCell = pkt.packet_id
+                ? `<a href="/packet/${pkt.storage_id || pkt.id}"
+                      style="text-decoration:underline; color:inherit;">
+                       ${pkt.packet_id}
+                   </a>`
+                : pkt.portnum === 73
+                ? `<span class="text-secondary" title="MQTT map report">Not at Packet</span>`
+                : `<span class="text-secondary" title="No Meshtastic packet ID">—</span>`;

            const html = `
 <tr class="packet-row">
@@ -382,10 +390,7 @@ async function fetchUpdates() {

    <td>
        <span class="toggle-btn">▶</span>
-        <a href="/packet/${pkt.id}"
-           style="text-decoration:underline; color:inherit;">
-            ${pkt.id}
-        </a>
+        ${packetIdCell}
    </td>

    <td>${from}</td>
@@ -148,6 +148,7 @@
 <script src="https://unpkg.com/leaflet-polylinedecorator@1.6.0/dist/leaflet.polylinedecorator.js"
        integrity="sha384-FhPn/2P/fJGhQLeNWDn9B/2Gml2bPOrKJwFqJXgR3xOPYxWg5mYQ5XZdhUSugZT0"
        crossorigin></script>
+<script src="https://unpkg.com/leaflet.heat/dist/leaflet-heat.js"></script>
 <script src="/static/portmaps.js"></script>

 <script>
@@ -188,20 +189,50 @@ function applyTranslationsMap(root = document) {
   ====================================================== */

 var map = L.map('map');
-L.tileLayer('https://tile.openstreetmap.org/{z}/{x}/{y}.png',
-    { maxZoom:19, attribution:'&copy; OpenStreetMap' }).addTo(map);
+var baseLayers = {
+    "OpenStreetMap": L.tileLayer('https://tile.openstreetmap.org/{z}/{x}/{y}.png', {
+        maxZoom: 19,
+        attribution: '&copy; OpenStreetMap'
+    }),
+    "OpenTopoMap": L.tileLayer('https://{s}.tile.opentopomap.org/{z}/{x}/{y}.png', {
+        maxZoom: 17,
+        attribution: 'Map data: &copy; OpenStreetMap, SRTM | Map style: &copy; OpenTopoMap'
+    })
+};
+baseLayers["OpenStreetMap"].addTo(map);
 L.control.scale({
    position: 'bottomleft',
    metric: true,
    imperial: true
 }).addTo(map);
+var nodeLayer = L.layerGroup().addTo(map);
+var edgeLayer = L.layerGroup().addTo(map);
+var trafficHeatLayer = L.heatLayer([], {
+    radius: 34,
+    blur: 18,
+    maxZoom: 12,
+    minOpacity: 0.3,
+    gradient: {
+        0.2: "#0b1f4d",
+        0.45: "#12436b",
+        0.7: "#7f2704",
+        1.0: "#4d0000"
+    }
+});
+var overlayLayers = {
+    "Nodes": nodeLayer,
+    "Traffic HeatMap": trafficHeatLayer
+};
+L.control.layers(baseLayers, overlayLayers, { collapsed: false }).addTo(map);

 // Data structures
 var nodes = [], markers = {}, markerById = {}, nodeMap = new Map();
-var edgeLayer = L.layerGroup().addTo(map), selectedNodeId = null;
+var selectedNodeId = null;
 var activeBlinks = new Map(), lastImportTime = null;
 var mapInterval = 0;
 var unmappedPackets = [];
+var trafficHeatLoaded = false;
+var trafficHeatLoading = false;
 const UNMAPPED_LIMIT = 50;
 const UNMAPPED_TTL_MS = 5000;

@@ -389,6 +420,15 @@ document.addEventListener("visibilitychange",()=>{
    document.hidden?stopPacketFetcher():startPacketFetcher();
 });

+map.on("overlayadd", e => {
+    if(e.layer === nodeLayer) updateNodeVisibility();
+    if(e.layer === trafficHeatLayer) loadTrafficHeat();
+});
+
+map.on("overlayremove", e => {
+    if(e.layer === nodeLayer) clearActiveBlinks();
+});
+
 async function waitForConfig() {
    while (typeof window._siteConfigPromise === "undefined") {
        await new Promise(r => setTimeout(r, 100));
@@ -470,6 +510,9 @@ fetch('/api/nodes?days_active=3')

        renderNodesOnMap();
        createChannelFilters();
+        if(map.hasLayer(trafficHeatLayer)){
+            loadTrafficHeat();
+        }
    })
    .catch(console.error);

@@ -490,7 +533,7 @@ function renderNodesOnMap(){
            fillColor: color,
            fillOpacity: 1,
            weight: 0.7
-        }).addTo(map);
+        }).addTo(nodeLayer);

        marker.nodeId = node.key;
        marker.originalColor = color;
@@ -513,6 +556,56 @@ function renderNodesOnMap(){
    setTimeout(() => applyTranslationsMap(), 50);
 }

+/* ======================================================
+   TRAFFIC HEAT LAYER
+   ====================================================== */
+
+async function loadTrafficHeat(){
+    if(trafficHeatLoaded || trafficHeatLoading) return;
+    if(nodeMap.size === 0) return;
+
+    trafficHeatLoading = true;
+
+    try {
+        const url = new URL("/api/packets", window.location.origin);
+        url.searchParams.set("limit", 1000);
+
+        const res = await fetch(url);
+        if(!res.ok) return;
+
+        const data = await res.json();
+        const counts = new Map();
+
+        (data.packets || []).forEach(pkt => {
+            const nodeId = pkt.from_node_id;
+            if(nodeId == null) return;
+            counts.set(nodeId, (counts.get(nodeId) || 0) + 1);
+        });
+
+        const maxCount = Math.max(...counts.values(), 0);
+        const heatPoints = [];
+
+        counts.forEach((count, nodeId) => {
+            const node = nodeMap.get(nodeId);
+            if(!node || isInvalidCoord(node)) return;
+
+            const markerLatLng = markerById[nodeId]?.getLatLng();
+            const lat = markerLatLng?.lat ?? node.lat;
+            const lng = markerLatLng?.lng ?? node.long;
+            const intensity = maxCount > 0 ? count / maxCount : 0;
+
+            heatPoints.push([lat, lng, Math.max(0.15, intensity)]);
+        });
+
+        trafficHeatLayer.setLatLngs(heatPoints);
+        trafficHeatLoaded = true;
+    } catch(err) {
+        console.error("Failed to load traffic heat layer:", err);
+    } finally {
+        trafficHeatLoading = false;
+    }
+}
+
 /* ======================================================
   UNMAPPED PACKETS LIST
   ====================================================== */
@@ -651,13 +744,33 @@ function isTextPort(portnum){
    return portnum === 1 || portnum === 7;
 }

+function isNodeVisibleForPackets(marker){
+    return map.hasLayer(nodeLayer) && nodeLayer.hasLayer(marker);
+}
+
+function clearBlinkForMarker(marker){
+    if(activeBlinks.has(marker)){
+        const interval = activeBlinks.get(marker);
+        clearInterval(interval);
+        activeBlinks.delete(marker);
+    }
+    marker.setStyle({ fillColor: marker.originalColor });
+    if(marker.tooltip){
+        map.removeLayer(marker.tooltip);
+        marker.tooltip = null;
+    }
+}
+
+function clearActiveBlinks(){
+    activeBlinks.forEach((interval, marker) => clearBlinkForMarker(marker));
+    activeBlinks.clear();
+}
+
 function blinkNode(marker,longName,portnum,payload){
-    if(!map.hasLayer(marker)) return;
+    if(!isNodeVisibleForPackets(marker)) return;

    if(activeBlinks.has(marker)){
-        clearInterval(activeBlinks.get(marker));
-        marker.setStyle({ fillColor: marker.originalColor });
-        if(marker.tooltip) map.removeLayer(marker.tooltip);
+        clearBlinkForMarker(marker);
    }

    let blinkCount = 0;
@@ -685,19 +798,19 @@ function blinkNode(marker,longName,portnum,payload){
    marker.tooltip = tooltip;

    const interval = setInterval(()=>{
-        if(map.hasLayer(marker)){
+        if(isNodeVisibleForPackets(marker)){
            marker.setStyle({
                fillColor: blinkCount%2===0 ? 'yellow' : marker.originalColor
            });
            marker.bringToFront();
+        } else {
+            clearBlinkForMarker(marker);
+            return;
        }
        blinkCount++;

        if(Date.now() - blinkStart > blinkDurationMs){
-            clearInterval(interval);
-            marker.setStyle({ fillColor: marker.originalColor });
-            map.removeLayer(tooltip);
-            activeBlinks.delete(marker);
+            clearBlinkForMarker(marker);
        }

    },500);
@@ -757,6 +870,8 @@ function saveFiltersToLocalStorage(){
 }

 function updateNodeVisibility(){
+    if(!map.hasLayer(nodeLayer)) return;
+
    const routerOnly = document.getElementById("filter-routers-only").checked;
    const mqttOnly = document.getElementById("filter-mqtt-only").checked;
    const activeChannels = [...channelSet].filter(ch =>
@@ -771,7 +886,12 @@ function updateNodeVisibility(){
                (!mqttOnly || n.is_mqtt_gateway) &&
                activeChannels.includes(n.channel);

-            visible ? map.addLayer(marker) : map.removeLayer(marker);
+            if(visible){
+                nodeLayer.addLayer(marker);
+            } else {
+                clearBlinkForMarker(marker);
+                nodeLayer.removeLayer(marker);
+            }
        }
    });
 }
@@ -1075,14 +1075,20 @@ async function loadPackets(filters = {}) {
        }

        const sizeBytes = packetSizeBytes(pkt);
+        const packetIdCell = pkt.packet_id
+            ? `<a href="/packet/${pkt.storage_id || pkt.id}"
+                  style="text-decoration:underline; color:inherit;">
+                   ${pkt.packet_id}
+               </a>`
+            : pkt.portnum === 73
+            ? `<span class="text-secondary" title="MQTT map report">Not a Packet</span>`
+            : `<span class="text-secondary" title="No Meshtastic packet ID">—</span>`;

        list.insertAdjacentHTML("afterbegin", `
        <tr class="packet-row">
            <td>${localTime}</td>
            <td><span class="toggle-btn">▶</span>
-                <a href="/packet/${pkt.id}" style="text-decoration:underline; color:inherit;">
-                    ${pkt.id}
-                </a>
+                ${packetIdCell}
            </td>
            <td>${fromCell}</td>
            <td>${toCell}</td>
@@ -1706,6 +1712,7 @@ async function loadNodeStats(nodeId) {
            : "";

        const portName = PORT_LABEL_MAP[pkt.portnum] || `Port ${pkt.portnum}`;
+        const displayPacketId = pkt.packet_id || (pkt.portnum === 73 ? "Not a Packet" : "");

        // Escape quotes + line breaks for CSV safety
        const payload = (pkt.payload || "")
@@ -1714,7 +1721,7 @@ async function loadNodeStats(nodeId) {

        rows.push([
            time,
-            pkt.id,
+            displayPacketId,
            pkt.from_node_id,
            pkt.to_node_id,
            pkt.portnum,
@@ -53,6 +53,7 @@ class Packet:
    """UI-friendly packet wrapper for templates and API payloads."""

    id: int
+    packet_id: int | None
    from_node_id: int
    from_node: models.Node
    to_node_id: int
@@ -99,8 +100,13 @@ class Packet:
                    f'<a href="https://www.google.com/maps/search/?api=1&query={payload.latitude_i * 1e-7},{payload.longitude_i * 1e-7}" target="_blank">map</a>'
                )

+        packet_id = None
+        if mesh_packet and mesh_packet.id:
+            packet_id = mesh_packet.id
+
        return cls(
            id=packet.id,
+            packet_id=packet_id,
            from_node=packet.from_node,
            from_node_id=packet.from_node_id,
            to_node=packet.to_node,
@@ -250,6 +250,8 @@ async def api_packets(request):
            p = Packet.from_model(packet)
            data = {
                "id": p.id,
+                "storage_id": p.id,
+                "packet_id": p.packet_id,
                "from_node_id": p.from_node_id,
                "to_node_id": p.to_node_id,
                "portnum": int(p.portnum) if p.portnum is not None else None,
@@ -339,6 +341,8 @@ async def api_packets(request):
        for p in ui_packets:
            packet_dict = {
                "id": p.id,
+                "storage_id": p.id,
+                "packet_id": p.packet_id,
                "import_time_us": p.import_time_us,
                "channel": p.channel,
                "from_node_id": p.from_node_id,