From e2adb73feaa871eee6a8bcc71dcc4e1355b7becf Mon Sep 17 00:00:00 2001
From: pablorevilla-meshtastic <pablorevilla@gmail.com>
Date: Thu, 25 Jun 2026 17:19:35 -0700
Subject: [PATCH] map changes and other fixes

---
 meshview/mqtt_store.py           |  30 +++++--
 meshview/templates/firehose.html |  15 ++--
 meshview/templates/map.html      | 148 ++++++++++++++++++++++++++++---
 meshview/templates/node.html     |  15 +++-
 meshview/web.py                  |   6 ++
 meshview/web_api/api.py          |   4 +
 6 files changed, 190 insertions(+), 28 deletions(-)

diff --git a/meshview/mqtt_store.py b/meshview/mqtt_store.py
index 6bd9bab..68acd8e 100644
--- a/meshview/mqtt_store.py
+++ b/meshview/mqtt_store.py
@@ -19,6 +19,7 @@ logger = logging.getLogger(__name__)
 MQTT_GATEWAY_CACHE: set[int] = set()
 UNKNOWN_NODE_NAME = "unknown name"
 NODE_NAME_CONTROL_CHARS = re.compile(r"[\x00-\x1f\x7f]")
+MQTT_DIRECT_NODE_ID = 1
 
 
 def normalize_node_name(name: str) -> str:
@@ -28,6 +29,24 @@ def normalize_node_name(name: str) -> str:
     return name
 
 
+def storage_packet_id(packet) -> int | None:
+    if packet.id:
+        return packet.id
+
+    if packet.decoded.portnum != PortNum.MAP_REPORT_APP:
+        return None
+
+    from_node_id = getattr(packet, "from", 0) or 0
+    now_us = int(time.time() * 1_000_000)
+    return -(now_us * 2048 + (from_node_id & 0x7FF))
+
+
+def storage_to_node_id(packet) -> int:
+    if packet.decoded.portnum == PortNum.MAP_REPORT_APP:
+        return MQTT_DIRECT_NODE_ID
+    return packet.to
+
+
 async def capture_daily_snapshot() -> None:
     today = datetime.now(UTC).date()
 
@@ -147,20 +166,21 @@ async def process_envelope(topic, env):
 
             await session.commit()
 
-    if not env.packet.id:
+    packet_id = storage_packet_id(env.packet)
+    if packet_id is None:
         return
 
     async with mqtt_database.async_session() as session:
         # --- Packet insert with ON CONFLICT DO NOTHING
-        result = await session.execute(select(Packet).where(Packet.id == env.packet.id))
+        result = await session.execute(select(Packet).where(Packet.id == packet_id))
         packet = result.scalar_one_or_none()
         if not packet:
             now_us = int(time.time() * 1_000_000)
             packet_values = {
-                "id": env.packet.id,
+                "id": packet_id,
                 "portnum": env.packet.decoded.portnum,
                 "from_node_id": getattr(env.packet, "from"),
-                "to_node_id": env.packet.to,
+                "to_node_id": storage_to_node_id(env.packet),
                 "payload": env.packet.SerializeToString(),
                 "import_time_us": now_us,
                 "channel": env.channel_id,
@@ -208,7 +228,7 @@ async def process_envelope(topic, env):
 
         now_us = int(time.time() * 1_000_000)
         seen_values = {
-            "packet_id": env.packet.id,
+            "packet_id": packet_id,
             "node_id": int(env.gateway_id[1:], 16),
             "channel": env.channel_id,
             "rx_time": env.packet.rx_time,
diff --git a/meshview/templates/firehose.html b/meshview/templates/firehose.html
index bc5266a..cc42be0 100644
--- a/meshview/templates/firehose.html
+++ b/meshview/templates/firehose.html
@@ -241,7 +241,7 @@ function logPacketTimes(packet) {
     const times = formatTimes(packet.import_time_us);
     console.log(
         "[firehose] packet time",
-        "id=" + packet.id,
+        "id=" + (packet.packet_id ?? packet.id),
         "epoch_us=" + times.epoch,
         "local=" + times.local,
         "utc=" + times.utc
@@ -372,6 +372,14 @@ async function fetchUpdates() {
             const safePayload = (pkt.payload || "")
                 .replace(/</g, "&lt;")
                 .replace(/>/g, "&gt;");
+            const packetIdCell = pkt.packet_id
+                ? `<a href="/packet/${pkt.storage_id || pkt.id}"
+                      style="text-decoration:underline; color:inherit;">
+                       ${pkt.packet_id}
+                   </a>`
+                : pkt.portnum === 73
+                ? `<span class="text-secondary" title="MQTT map report">Not at Packet</span>`
+                : `<span class="text-secondary" title="No Meshtastic packet ID">—</span>`;
 
             const html = `
 <tr class="packet-row">
@@ -382,10 +390,7 @@ async function fetchUpdates() {
 
     <td>
         <span class="toggle-btn">▶</span>
-        <a href="/packet/${pkt.id}"
-           style="text-decoration:underline; color:inherit;">
-            ${pkt.id}
-        </a>
+        ${packetIdCell}
     </td>
 
     <td>${from}</td>
diff --git a/meshview/templates/map.html b/meshview/templates/map.html
index 7602aa6..babe229 100644
--- a/meshview/templates/map.html
+++ b/meshview/templates/map.html
@@ -148,6 +148,7 @@
 <script src="https://unpkg.com/leaflet-polylinedecorator@1.6.0/dist/leaflet.polylinedecorator.js"
         integrity="sha384-FhPn/2P/fJGhQLeNWDn9B/2Gml2bPOrKJwFqJXgR3xOPYxWg5mYQ5XZdhUSugZT0"
         crossorigin></script>
+<script src="https://unpkg.com/leaflet.heat/dist/leaflet-heat.js"></script>
 <script src="/static/portmaps.js"></script>
 
 <script>
@@ -188,20 +189,50 @@ function applyTranslationsMap(root = document) {
    ====================================================== */
 
 var map = L.map('map');
-L.tileLayer('https://tile.openstreetmap.org/{z}/{x}/{y}.png',
-    { maxZoom:19, attribution:'&copy; OpenStreetMap' }).addTo(map);
+var baseLayers = {
+    "OpenStreetMap": L.tileLayer('https://tile.openstreetmap.org/{z}/{x}/{y}.png', {
+        maxZoom: 19,
+        attribution: '&copy; OpenStreetMap'
+    }),
+    "OpenTopoMap": L.tileLayer('https://{s}.tile.opentopomap.org/{z}/{x}/{y}.png', {
+        maxZoom: 17,
+        attribution: 'Map data: &copy; OpenStreetMap, SRTM | Map style: &copy; OpenTopoMap'
+    })
+};
+baseLayers["OpenStreetMap"].addTo(map);
 L.control.scale({
     position: 'bottomleft',
     metric: true,
     imperial: true
 }).addTo(map);
+var nodeLayer = L.layerGroup().addTo(map);
+var edgeLayer = L.layerGroup().addTo(map);
+var trafficHeatLayer = L.heatLayer([], {
+    radius: 34,
+    blur: 18,
+    maxZoom: 12,
+    minOpacity: 0.3,
+    gradient: {
+        0.2: "#0b1f4d",
+        0.45: "#12436b",
+        0.7: "#7f2704",
+        1.0: "#4d0000"
+    }
+});
+var overlayLayers = {
+    "Nodes": nodeLayer,
+    "Traffic HeatMap": trafficHeatLayer
+};
+L.control.layers(baseLayers, overlayLayers, { collapsed: false }).addTo(map);
 
 // Data structures
 var nodes = [], markers = {}, markerById = {}, nodeMap = new Map();
-var edgeLayer = L.layerGroup().addTo(map), selectedNodeId = null;
+var selectedNodeId = null;
 var activeBlinks = new Map(), lastImportTime = null;
 var mapInterval = 0;
 var unmappedPackets = [];
+var trafficHeatLoaded = false;
+var trafficHeatLoading = false;
 const UNMAPPED_LIMIT = 50;
 const UNMAPPED_TTL_MS = 5000;
 
@@ -389,6 +420,15 @@ document.addEventListener("visibilitychange",()=>{
     document.hidden?stopPacketFetcher():startPacketFetcher();
 });
 
+map.on("overlayadd", e => {
+    if(e.layer === nodeLayer) updateNodeVisibility();
+    if(e.layer === trafficHeatLayer) loadTrafficHeat();
+});
+
+map.on("overlayremove", e => {
+    if(e.layer === nodeLayer) clearActiveBlinks();
+});
+
 async function waitForConfig() {
     while (typeof window._siteConfigPromise === "undefined") {
         await new Promise(r => setTimeout(r, 100));
@@ -470,6 +510,9 @@ fetch('/api/nodes?days_active=3')
 
         renderNodesOnMap();
         createChannelFilters();
+        if(map.hasLayer(trafficHeatLayer)){
+            loadTrafficHeat();
+        }
     })
     .catch(console.error);
 
@@ -490,7 +533,7 @@ function renderNodesOnMap(){
             fillColor: color,
             fillOpacity: 1,
             weight: 0.7
-        }).addTo(map);
+        }).addTo(nodeLayer);
 
         marker.nodeId = node.key;
         marker.originalColor = color;
@@ -513,6 +556,56 @@ function renderNodesOnMap(){
     setTimeout(() => applyTranslationsMap(), 50);
 }
 
+/* ======================================================
+   TRAFFIC HEAT LAYER
+   ====================================================== */
+
+async function loadTrafficHeat(){
+    if(trafficHeatLoaded || trafficHeatLoading) return;
+    if(nodeMap.size === 0) return;
+
+    trafficHeatLoading = true;
+
+    try {
+        const url = new URL("/api/packets", window.location.origin);
+        url.searchParams.set("limit", 1000);
+
+        const res = await fetch(url);
+        if(!res.ok) return;
+
+        const data = await res.json();
+        const counts = new Map();
+
+        (data.packets || []).forEach(pkt => {
+            const nodeId = pkt.from_node_id;
+            if(nodeId == null) return;
+            counts.set(nodeId, (counts.get(nodeId) || 0) + 1);
+        });
+
+        const maxCount = Math.max(...counts.values(), 0);
+        const heatPoints = [];
+
+        counts.forEach((count, nodeId) => {
+            const node = nodeMap.get(nodeId);
+            if(!node || isInvalidCoord(node)) return;
+
+            const markerLatLng = markerById[nodeId]?.getLatLng();
+            const lat = markerLatLng?.lat ?? node.lat;
+            const lng = markerLatLng?.lng ?? node.long;
+            const intensity = maxCount > 0 ? count / maxCount : 0;
+
+            heatPoints.push([lat, lng, Math.max(0.15, intensity)]);
+        });
+
+        trafficHeatLayer.setLatLngs(heatPoints);
+        trafficHeatLoaded = true;
+    } catch(err) {
+        console.error("Failed to load traffic heat layer:", err);
+    } finally {
+        trafficHeatLoading = false;
+    }
+}
+
 /* ======================================================
    UNMAPPED PACKETS LIST
    ====================================================== */
@@ -651,13 +744,33 @@ function isTextPort(portnum){
     return portnum === 1 || portnum === 7;
 }
 
+function isNodeVisibleForPackets(marker){
+    return map.hasLayer(nodeLayer) && nodeLayer.hasLayer(marker);
+}
+
+function clearBlinkForMarker(marker){
+    if(activeBlinks.has(marker)){
+        const interval = activeBlinks.get(marker);
+        clearInterval(interval);
+        activeBlinks.delete(marker);
+    }
+    marker.setStyle({ fillColor: marker.originalColor });
+    if(marker.tooltip){
+        map.removeLayer(marker.tooltip);
+        marker.tooltip = null;
+    }
+}
+
+function clearActiveBlinks(){
+    activeBlinks.forEach((interval, marker) => clearBlinkForMarker(marker));
+    activeBlinks.clear();
+}
+
 function blinkNode(marker,longName,portnum,payload){
-    if(!map.hasLayer(marker)) return;
+    if(!isNodeVisibleForPackets(marker)) return;
 
     if(activeBlinks.has(marker)){
-        clearInterval(activeBlinks.get(marker));
-        marker.setStyle({ fillColor: marker.originalColor });
-        if(marker.tooltip) map.removeLayer(marker.tooltip);
+        clearBlinkForMarker(marker);
     }
 
     let blinkCount = 0;
@@ -685,19 +798,19 @@ function blinkNode(marker,longName,portnum,payload){
     marker.tooltip = tooltip;
 
     const interval = setInterval(()=>{
-        if(map.hasLayer(marker)){
+        if(isNodeVisibleForPackets(marker)){
             marker.setStyle({
                 fillColor: blinkCount%2===0 ? 'yellow' : marker.originalColor
             });
             marker.bringToFront();
+        } else {
+            clearBlinkForMarker(marker);
+            return;
         }
         blinkCount++;
 
         if(Date.now() - blinkStart > blinkDurationMs){
-            clearInterval(interval);
-            marker.setStyle({ fillColor: marker.originalColor });
-            map.removeLayer(tooltip);
-            activeBlinks.delete(marker);
+            clearBlinkForMarker(marker);
         }
 
     },500);
@@ -757,6 +870,8 @@ function saveFiltersToLocalStorage(){
 }
 
 function updateNodeVisibility(){
+    if(!map.hasLayer(nodeLayer)) return;
+
     const routerOnly = document.getElementById("filter-routers-only").checked;
     const mqttOnly = document.getElementById("filter-mqtt-only").checked;
     const activeChannels = [...channelSet].filter(ch =>
@@ -771,7 +886,12 @@ function updateNodeVisibility(){
                 (!mqttOnly || n.is_mqtt_gateway) &&
                 activeChannels.includes(n.channel);
 
-            visible ? map.addLayer(marker) : map.removeLayer(marker);
+            if(visible){
+                nodeLayer.addLayer(marker);
+            } else {
+                clearBlinkForMarker(marker);
+                nodeLayer.removeLayer(marker);
+            }
         }
     });
 }
diff --git a/meshview/templates/node.html b/meshview/templates/node.html
index 1287179..e8aa5f2 100644
--- a/meshview/templates/node.html
+++ b/meshview/templates/node.html
@@ -1075,14 +1075,20 @@ async function loadPackets(filters = {}) {
         }
 
         const sizeBytes = packetSizeBytes(pkt);
+        const packetIdCell = pkt.packet_id
+            ? `<a href="/packet/${pkt.storage_id || pkt.id}"
+                  style="text-decoration:underline; color:inherit;">
+                   ${pkt.packet_id}
+               </a>`
+            : pkt.portnum === 73
+            ? `<span class="text-secondary" title="MQTT map report">Not a Packet</span>`
+            : `<span class="text-secondary" title="No Meshtastic packet ID">—</span>`;
 
         list.insertAdjacentHTML("afterbegin", `
         <tr class="packet-row">
             <td>${localTime}</td>
             <td><span class="toggle-btn">▶</span>
-                <a href="/packet/${pkt.id}" style="text-decoration:underline; color:inherit;">
-                    ${pkt.id}
-                </a>
+                ${packetIdCell}
             </td>
             <td>${fromCell}</td>
             <td>${toCell}</td>
@@ -1706,6 +1712,7 @@ async function loadNodeStats(nodeId) {
             : "";
 
         const portName = PORT_LABEL_MAP[pkt.portnum] || `Port ${pkt.portnum}`;
+        const displayPacketId = pkt.packet_id || (pkt.portnum === 73 ? "Not a Packet" : "");
 
         // Escape quotes + line breaks for CSV safety
         const payload = (pkt.payload || "")
@@ -1714,7 +1721,7 @@ async function loadNodeStats(nodeId) {
 
         rows.push([
             time,
-            pkt.id,
+            displayPacketId,
             pkt.from_node_id,
             pkt.to_node_id,
             pkt.portnum,
diff --git a/meshview/web.py b/meshview/web.py
index 82de041..03f61fd 100644
--- a/meshview/web.py
+++ b/meshview/web.py
@@ -53,6 +53,7 @@ class Packet:
     """UI-friendly packet wrapper for templates and API payloads."""
 
     id: int
+    packet_id: int | None
     from_node_id: int
     from_node: models.Node
     to_node_id: int
@@ -99,8 +100,13 @@ class Packet:
                     f'<a href="https://www.google.com/maps/search/?api=1&query={payload.latitude_i * 1e-7},{payload.longitude_i * 1e-7}" target="_blank">map</a>'
                 )
 
+        packet_id = None
+        if mesh_packet and mesh_packet.id:
+            packet_id = mesh_packet.id
+
         return cls(
             id=packet.id,
+            packet_id=packet_id,
             from_node=packet.from_node,
             from_node_id=packet.from_node_id,
             to_node=packet.to_node,
diff --git a/meshview/web_api/api.py b/meshview/web_api/api.py
index 71e5c99..d9ca3ac 100644
--- a/meshview/web_api/api.py
+++ b/meshview/web_api/api.py
@@ -250,6 +250,8 @@ async def api_packets(request):
             p = Packet.from_model(packet)
             data = {
                 "id": p.id,
+                "storage_id": p.id,
+                "packet_id": p.packet_id,
                 "from_node_id": p.from_node_id,
                 "to_node_id": p.to_node_id,
                 "portnum": int(p.portnum) if p.portnum is not None else None,
@@ -339,6 +341,8 @@ async def api_packets(request):
         for p in ui_packets:
             packet_dict = {
                 "id": p.id,
+                "storage_id": p.id,
+                "packet_id": p.packet_id,
                 "import_time_us": p.import_time_us,
                 "channel": p.channel,
                 "from_node_id": p.from_node_id,