6 Commits

Author SHA1 Message Date
pablorevilla-meshtastic 992296b0c1 Work on new Traceroute page 2026-07-02 15:01:24 -07:00
pablorevilla-meshtastic e2adb73fea map changes and other fixes 2026-06-25 17:19:35 -07:00
pablorevilla-meshtastic 5386f39611 report change 2026-06-11 16:16:08 -07:00
pablorevilla-meshtastic 2175fcb44d remove some of the changes 2026-06-11 14:49:26 -07:00
pablorevilla-meshtastic c16e19336a more fixes 2026-06-08 12:02:20 -07:00
pablorevilla-meshtastic 7f5b50b6f3 Fix timezone or health report 2026-06-08 11:59:20 -07:00
8 changed files with 333 additions and 52 deletions
+25 -5
View File
@@ -19,6 +19,7 @@ logger = logging.getLogger(__name__)
MQTT_GATEWAY_CACHE: set[int] = set()
UNKNOWN_NODE_NAME = "unknown name"
NODE_NAME_CONTROL_CHARS = re.compile(r"[\x00-\x1f\x7f]")
MQTT_DIRECT_NODE_ID = 1
def normalize_node_name(name: str) -> str:
@@ -28,6 +29,24 @@ def normalize_node_name(name: str) -> str:
return name
def storage_packet_id(packet) -> int | None:
if packet.id:
return packet.id
if packet.decoded.portnum != PortNum.MAP_REPORT_APP:
return None
from_node_id = getattr(packet, "from", 0) or 0
now_us = int(time.time() * 1_000_000)
return -(now_us * 2048 + (from_node_id & 0x7FF))
def storage_to_node_id(packet) -> int:
if packet.decoded.portnum == PortNum.MAP_REPORT_APP:
return MQTT_DIRECT_NODE_ID
return packet.to
async def capture_daily_snapshot() -> None:
today = datetime.now(UTC).date()
@@ -147,20 +166,21 @@ async def process_envelope(topic, env):
await session.commit()
if not env.packet.id:
packet_id = storage_packet_id(env.packet)
if packet_id is None:
return
async with mqtt_database.async_session() as session:
# --- Packet insert with ON CONFLICT DO NOTHING
result = await session.execute(select(Packet).where(Packet.id == env.packet.id))
result = await session.execute(select(Packet).where(Packet.id == packet_id))
packet = result.scalar_one_or_none()
if not packet:
now_us = int(time.time() * 1_000_000)
packet_values = {
"id": env.packet.id,
"id": packet_id,
"portnum": env.packet.decoded.portnum,
"from_node_id": getattr(env.packet, "from"),
"to_node_id": env.packet.to,
"to_node_id": storage_to_node_id(env.packet),
"payload": env.packet.SerializeToString(),
"import_time_us": now_us,
"channel": env.channel_id,
@@ -208,7 +228,7 @@ async def process_envelope(topic, env):
now_us = int(time.time() * 1_000_000)
seen_values = {
"packet_id": env.packet.id,
"packet_id": packet_id,
"node_id": int(env.gateway_id[1:], 16),
"channel": env.channel_id,
"rx_time": env.packet.rx_time,
+12 -7
View File
@@ -241,7 +241,7 @@ function logPacketTimes(packet) {
const times = formatTimes(packet.import_time_us);
console.log(
"[firehose] packet time",
"id=" + packet.id,
"id=" + (packet.packet_id ?? packet.id),
"epoch_us=" + times.epoch,
"local=" + times.local,
"utc=" + times.utc
@@ -365,13 +365,21 @@ async function fetchUpdates() {
if (match) traceId = match[1];
inlineLinks += ` <a class="inline-link"
href="/graph/traceroute/${traceId}"
target="_blank">⮕</a>`;
href="/traceroute/${traceId}"
>⮕</a>`;
}
const safePayload = (pkt.payload || "")
.replace(/</g, "&lt;")
.replace(/>/g, "&gt;");
const packetIdCell = pkt.packet_id
? `<a href="/packet/${pkt.storage_id || pkt.id}"
style="text-decoration:underline; color:inherit;">
${pkt.packet_id}
</a>`
: pkt.portnum === 73
? `<span class="text-secondary" title="MQTT map report">Not at Packet</span>`
: `<span class="text-secondary" title="No Meshtastic packet ID">—</span>`;
const html = `
<tr class="packet-row">
@@ -382,10 +390,7 @@ async function fetchUpdates() {
<td>
<span class="toggle-btn">▶</span>
<a href="/packet/${pkt.id}"
style="text-decoration:underline; color:inherit;">
${pkt.id}
</a>
${packetIdCell}
</td>
<td>${from}</td>
+134 -14
View File
@@ -148,6 +148,7 @@
<script src="https://unpkg.com/leaflet-polylinedecorator@1.6.0/dist/leaflet.polylinedecorator.js"
integrity="sha384-FhPn/2P/fJGhQLeNWDn9B/2Gml2bPOrKJwFqJXgR3xOPYxWg5mYQ5XZdhUSugZT0"
crossorigin></script>
<script src="https://unpkg.com/leaflet.heat/dist/leaflet-heat.js"></script>
<script src="/static/portmaps.js"></script>
<script>
@@ -188,20 +189,50 @@ function applyTranslationsMap(root = document) {
====================================================== */
var map = L.map('map');
L.tileLayer('https://tile.openstreetmap.org/{z}/{x}/{y}.png',
{ maxZoom:19, attribution:'&copy; OpenStreetMap' }).addTo(map);
var baseLayers = {
"OpenStreetMap": L.tileLayer('https://tile.openstreetmap.org/{z}/{x}/{y}.png', {
maxZoom: 19,
attribution: '&copy; OpenStreetMap'
}),
"OpenTopoMap": L.tileLayer('https://{s}.tile.opentopomap.org/{z}/{x}/{y}.png', {
maxZoom: 17,
attribution: 'Map data: &copy; OpenStreetMap, SRTM | Map style: &copy; OpenTopoMap'
})
};
baseLayers["OpenStreetMap"].addTo(map);
L.control.scale({
position: 'bottomleft',
metric: true,
imperial: true
}).addTo(map);
var nodeLayer = L.layerGroup().addTo(map);
var edgeLayer = L.layerGroup().addTo(map);
var trafficHeatLayer = L.heatLayer([], {
radius: 34,
blur: 18,
maxZoom: 12,
minOpacity: 0.3,
gradient: {
0.2: "#0b1f4d",
0.45: "#12436b",
0.7: "#7f2704",
1.0: "#4d0000"
}
});
var overlayLayers = {
"Nodes": nodeLayer,
"Traffic HeatMap": trafficHeatLayer
};
L.control.layers(baseLayers, overlayLayers, { collapsed: false }).addTo(map);
// Data structures
var nodes = [], markers = {}, markerById = {}, nodeMap = new Map();
var edgeLayer = L.layerGroup().addTo(map), selectedNodeId = null;
var selectedNodeId = null;
var activeBlinks = new Map(), lastImportTime = null;
var mapInterval = 0;
var unmappedPackets = [];
var trafficHeatLoaded = false;
var trafficHeatLoading = false;
const UNMAPPED_LIMIT = 50;
const UNMAPPED_TTL_MS = 5000;
@@ -389,6 +420,15 @@ document.addEventListener("visibilitychange",()=>{
document.hidden?stopPacketFetcher():startPacketFetcher();
});
map.on("overlayadd", e => {
if(e.layer === nodeLayer) updateNodeVisibility();
if(e.layer === trafficHeatLayer) loadTrafficHeat();
});
map.on("overlayremove", e => {
if(e.layer === nodeLayer) clearActiveBlinks();
});
async function waitForConfig() {
while (typeof window._siteConfigPromise === "undefined") {
await new Promise(r => setTimeout(r, 100));
@@ -470,6 +510,9 @@ fetch('/api/nodes?days_active=3')
renderNodesOnMap();
createChannelFilters();
if(map.hasLayer(trafficHeatLayer)){
loadTrafficHeat();
}
})
.catch(console.error);
@@ -490,7 +533,7 @@ function renderNodesOnMap(){
fillColor: color,
fillOpacity: 1,
weight: 0.7
}).addTo(map);
}).addTo(nodeLayer);
marker.nodeId = node.key;
marker.originalColor = color;
@@ -513,6 +556,56 @@ function renderNodesOnMap(){
setTimeout(() => applyTranslationsMap(), 50);
}
/* ======================================================
TRAFFIC HEAT LAYER
====================================================== */
async function loadTrafficHeat(){
if(trafficHeatLoaded || trafficHeatLoading) return;
if(nodeMap.size === 0) return;
trafficHeatLoading = true;
try {
const url = new URL("/api/packets", window.location.origin);
url.searchParams.set("limit", 1000);
const res = await fetch(url);
if(!res.ok) return;
const data = await res.json();
const counts = new Map();
(data.packets || []).forEach(pkt => {
const nodeId = pkt.from_node_id;
if(nodeId == null) return;
counts.set(nodeId, (counts.get(nodeId) || 0) + 1);
});
const maxCount = Math.max(...counts.values(), 0);
const heatPoints = [];
counts.forEach((count, nodeId) => {
const node = nodeMap.get(nodeId);
if(!node || isInvalidCoord(node)) return;
const markerLatLng = markerById[nodeId]?.getLatLng();
const lat = markerLatLng?.lat ?? node.lat;
const lng = markerLatLng?.lng ?? node.long;
const intensity = maxCount > 0 ? count / maxCount : 0;
heatPoints.push([lat, lng, Math.max(0.15, intensity)]);
});
trafficHeatLayer.setLatLngs(heatPoints);
trafficHeatLoaded = true;
} catch(err) {
console.error("Failed to load traffic heat layer:", err);
} finally {
trafficHeatLoading = false;
}
}
/* ======================================================
UNMAPPED PACKETS LIST
====================================================== */
@@ -651,13 +744,33 @@ function isTextPort(portnum){
return portnum === 1 || portnum === 7;
}
function isNodeVisibleForPackets(marker){
return map.hasLayer(nodeLayer) && nodeLayer.hasLayer(marker);
}
function clearBlinkForMarker(marker){
if(activeBlinks.has(marker)){
const interval = activeBlinks.get(marker);
clearInterval(interval);
activeBlinks.delete(marker);
}
marker.setStyle({ fillColor: marker.originalColor });
if(marker.tooltip){
map.removeLayer(marker.tooltip);
marker.tooltip = null;
}
}
function clearActiveBlinks(){
activeBlinks.forEach((interval, marker) => clearBlinkForMarker(marker));
activeBlinks.clear();
}
function blinkNode(marker,longName,portnum,payload){
if(!map.hasLayer(marker)) return;
if(!isNodeVisibleForPackets(marker)) return;
if(activeBlinks.has(marker)){
clearInterval(activeBlinks.get(marker));
marker.setStyle({ fillColor: marker.originalColor });
if(marker.tooltip) map.removeLayer(marker.tooltip);
clearBlinkForMarker(marker);
}
let blinkCount = 0;
@@ -685,19 +798,19 @@ function blinkNode(marker,longName,portnum,payload){
marker.tooltip = tooltip;
const interval = setInterval(()=>{
if(map.hasLayer(marker)){
if(isNodeVisibleForPackets(marker)){
marker.setStyle({
fillColor: blinkCount%2===0 ? 'yellow' : marker.originalColor
});
marker.bringToFront();
} else {
clearBlinkForMarker(marker);
return;
}
blinkCount++;
if(Date.now() - blinkStart > blinkDurationMs){
clearInterval(interval);
marker.setStyle({ fillColor: marker.originalColor });
map.removeLayer(tooltip);
activeBlinks.delete(marker);
clearBlinkForMarker(marker);
}
},500);
@@ -757,6 +870,8 @@ function saveFiltersToLocalStorage(){
}
function updateNodeVisibility(){
if(!map.hasLayer(nodeLayer)) return;
const routerOnly = document.getElementById("filter-routers-only").checked;
const mqttOnly = document.getElementById("filter-mqtt-only").checked;
const activeChannels = [...channelSet].filter(ch =>
@@ -771,7 +886,12 @@ function updateNodeVisibility(){
(!mqttOnly || n.is_mqtt_gateway) &&
activeChannels.includes(n.channel);
visible ? map.addLayer(marker) : map.removeLayer(marker);
if(visible){
nodeLayer.addLayer(marker);
} else {
clearBlinkForMarker(marker);
nodeLayer.removeLayer(marker);
}
}
});
}
+12 -5
View File
@@ -1071,18 +1071,24 @@ async function loadPackets(filters = {}) {
const match = pkt.payload?.match(/ID:\s*(\d+)/i);
if (match) traceId = match[1];
inlineLinks +=
` <a class="inline-link" href="/graph/traceroute/${traceId}" target="_blank"></a>`;
` <a class="inline-link" href="/traceroute/${traceId}" target="_blank"></a>`;
}
const sizeBytes = packetSizeBytes(pkt);
const packetIdCell = pkt.packet_id
? `<a href="/packet/${pkt.storage_id || pkt.id}"
style="text-decoration:underline; color:inherit;">
${pkt.packet_id}
</a>`
: pkt.portnum === 73
? `<span class="text-secondary" title="MQTT map report">Not a Packet</span>`
: `<span class="text-secondary" title="No Meshtastic packet ID"></span>`;
list.insertAdjacentHTML("afterbegin", `
<tr class="packet-row">
<td>${localTime}</td>
<td><span class="toggle-btn"></span>
<a href="/packet/${pkt.id}" style="text-decoration:underline; color:inherit;">
${pkt.id}
</a>
${packetIdCell}
</td>
<td>${fromCell}</td>
<td>${toCell}</td>
@@ -1706,6 +1712,7 @@ async function loadNodeStats(nodeId) {
: "";
const portName = PORT_LABEL_MAP[pkt.portnum] || `Port ${pkt.portnum}`;
const displayPacketId = pkt.packet_id || (pkt.portnum === 73 ? "Not a Packet" : "");
// Escape quotes + line breaks for CSV safety
const payload = (pkt.payload || "")
@@ -1714,7 +1721,7 @@ async function loadNodeStats(nodeId) {
rows.push([
time,
pkt.id,
displayPacketId,
pkt.from_node_id,
pkt.to_node_id,
pkt.portnum,
+133 -11
View File
@@ -7,12 +7,53 @@
{% block css %}
#traceroute-graph {
width: 100%;
height: 85vh;
height: calc(100vh - 270px);
border: 1px solid #2a2f36;
background: linear-gradient(135deg, #0f1216 0%, #171b22 100%);
border-radius: 10px;
}
#traceroute-graph-wrap {
position: relative;
}
#traceroute-legend {
position: absolute;
right: 16px;
bottom: 16px;
z-index: 2;
display: grid;
gap: 8px;
padding: 10px 12px;
color: #d6dde6;
background: rgba(15, 18, 22, 0.88);
border: 1px solid #303743;
border-radius: 8px;
font-size: 13px;
}
.traceroute-legend-row {
display: grid;
grid-template-columns: 34px auto;
align-items: center;
gap: 8px;
white-space: nowrap;
}
.traceroute-legend-line {
height: 0;
border-top: 3px solid var(--line-color);
}
.traceroute-legend-line.muted {
border-top-width: 2px;
opacity: 0.55;
}
.traceroute-legend-line.dashed {
border-top-style: dashed;
}
#traceroute-meta {
padding: 12px 16px;
color: #c8d0da;
@@ -28,7 +69,23 @@
<div><b>Traceroute</b> <span id="traceroute-title"></span></div>
<div id="traceroute-error"></div>
</div>
<div id="traceroute-graph"></div>
<div id="traceroute-graph-wrap">
<div id="traceroute-graph"></div>
<div id="traceroute-legend" aria-label="Traceroute edge legend">
<div class="traceroute-legend-row">
<span class="traceroute-legend-line" style="--line-color: #34c759"></span>
<span>Winning forward</span>
</div>
<div class="traceroute-legend-row">
<span class="traceroute-legend-line dashed" style="--line-color: #ff3b30"></span>
<span>Reported reverse</span>
</div>
<div class="traceroute-legend-row">
<span class="traceroute-legend-line muted" style="--line-color: #7c858f"></span>
<span>Reported route</span>
</div>
</div>
</div>
<script>
const el = document.getElementById("traceroute-graph");
@@ -49,6 +106,33 @@ function addPathEdges(path, edges, style) {
}
}
function pathKey(path) {
return path.map(id => String(id)).join(">");
}
function addMissingEndpoints(path, startId, endId) {
const fullPath = path.map(id => String(id));
if (startId && (!fullPath.length || fullPath[0] !== startId)) {
fullPath.unshift(startId);
}
if (endId && (!fullPath.length || fullPath[fullPath.length - 1] !== endId)) {
fullPath.push(endId);
}
return fullPath;
}
function uniqueForwardPaths(data, originId, targetId) {
return (data?.unique_forward_paths || [])
.map(item => addMissingEndpoints(item.path || [], originId, targetId))
.filter(path => path.length > 1);
}
function uniqueReversePaths(data, originId, targetId) {
return (data?.unique_reverse_paths || [])
.map(path => addMissingEndpoints(path || [], targetId, null))
.filter(path => path.length > 1);
}
async function loadTraceroute() {
const packetId = packetIdFromPath();
document.getElementById("traceroute-title").textContent = `#${packetId}`;
@@ -73,26 +157,61 @@ async function loadTraceroute() {
const nodes = new Map();
const edges = [];
const forwardPaths = data?.winning_paths?.forward || [];
const reversePaths = data?.winning_paths?.reverse || [];
const originId = data?.packet?.from != null ? String(data.packet.from) : null;
const targetId = data?.packet?.to != null ? String(data.packet.to) : null;
const forwardPaths = (data?.winning_paths?.forward || []).map(path => path.map(id => String(id)));
const reversePaths = (data?.winning_paths?.reverse || []).map(path => path.map(id => String(id)));
const winningPathKeys = new Set([
...forwardPaths.map(pathKey),
...reversePaths.map(pathKey)
]);
const winningForwardNodeIds = new Set(forwardPaths.flat());
const hasWinningReversePath = reversePaths.some(path => path.length > 1);
const winningForwardPath = forwardPaths.find(path => path.length > 1);
const colorStartId = winningForwardPath ? winningForwardPath[0] : originId;
const colorEndId = winningForwardPath ? winningForwardPath[winningForwardPath.length - 1] : targetId;
const startNodeColor = hasWinningReversePath ? "#ff3b30" : "#34c759";
const endNodeColor = hasWinningReversePath ? "#34c759" : "#ff3b30";
uniqueForwardPaths(data, originId, targetId).forEach(path => {
if (winningPathKeys.has(pathKey(path))) {
return;
}
path.forEach(id => nodes.set(String(id), { name: String(id) }));
addPathEdges(path, edges, { color: "#7c858f", width: 1.5, opacity: 0.45 });
});
uniqueReversePaths(data, originId, targetId).forEach(path => {
if (winningPathKeys.has(pathKey(path))) {
return;
}
path.forEach(id => nodes.set(String(id), { name: String(id) }));
addPathEdges(path, edges, { color: "#7c858f", width: 1.5, opacity: 0.45, type: "dashed" });
});
forwardPaths.forEach(path => {
path.forEach(id => nodes.set(String(id), { name: String(id) }));
addPathEdges(path, edges, { color: "#ff5733", width: 3 });
addPathEdges(path, edges, { color: "#34c759", width: 3 });
});
reversePaths.forEach(path => {
path.forEach(id => nodes.set(String(id), { name: String(id) }));
addPathEdges(path, edges, { color: "#00c3ff", width: 2, type: "dashed" });
addPathEdges(path, edges, { color: "#ff3b30", width: 2, type: "dashed" });
});
const graphNodes = Array.from(nodes.values()).map(n => {
const isOrigin = originId && n.name === originId;
const isTarget = targetId && n.name === targetId;
const color = isOrigin ? "#ff3b30" : isTarget ? "#34c759" : "#8aa4c8";
const size = isOrigin || isTarget ? 44 : 36;
const isColorStart = colorStartId && n.name === colorStartId;
const isColorEnd = colorEndId && n.name === colorEndId;
const isWinningForwardNode = winningForwardNodeIds.has(n.name);
let color = "#3f454d";
if (isColorStart) {
color = startNodeColor;
} else if (isColorEnd) {
color = endNodeColor;
} else if (isWinningForwardNode) {
color = "#b88a2a";
}
const size = isColorStart || isColorEnd ? 44 : 36;
return {
id: n.name,
name: nodeShortNameById.get(n.name) || n.name,
@@ -104,7 +223,10 @@ async function loadTraceroute() {
fontWeight: "bold"
},
tooltip: {
formatter: () => nodeLongNameById.get(n.name) || n.name
formatter: () => {
const longName = nodeLongNameById.get(n.name) || n.name;
return longName === n.name ? n.name : `${longName} (${n.name})`;
}
}
};
});
+6
View File
@@ -53,6 +53,7 @@ class Packet:
"""UI-friendly packet wrapper for templates and API payloads."""
id: int
packet_id: int | None
from_node_id: int
from_node: models.Node
to_node_id: int
@@ -99,8 +100,13 @@ class Packet:
f'<a href="https://www.google.com/maps/search/?api=1&query={payload.latitude_i * 1e-7},{payload.longitude_i * 1e-7}" target="_blank">map</a>'
)
packet_id = None
if mesh_packet and mesh_packet.id:
packet_id = mesh_packet.id
return cls(
id=packet.id,
packet_id=packet_id,
from_node=packet.from_node,
from_node_id=packet.from_node_id,
to_node=packet.to_node,
+9 -8
View File
@@ -250,6 +250,8 @@ async def api_packets(request):
p = Packet.from_model(packet)
data = {
"id": p.id,
"storage_id": p.id,
"packet_id": p.packet_id,
"from_node_id": p.from_node_id,
"to_node_id": p.to_node_id,
"portnum": int(p.portnum) if p.portnum is not None else None,
@@ -339,6 +341,8 @@ async def api_packets(request):
for p in ui_packets:
packet_dict = {
"id": p.id,
"storage_id": p.id,
"packet_id": p.packet_id,
"import_time_us": p.import_time_us,
"channel": p.channel,
"from_node_id": p.from_node_id,
@@ -964,11 +968,10 @@ async def api_traceroute(request):
if tr["reverse_hops"]:
reverse_paths.append(r)
if tr["done"]:
if tr["reverse_hops"]:
if tr["forward_hops"]:
winning_forward_paths.append(f)
if tr["reverse_hops"]:
winning_reverse_paths.append(r)
winning_reverse_paths.append(r)
# Deduplicate
unique_forward_paths = sorted(set(forward_paths))
@@ -989,10 +992,10 @@ async def api_traceroute(request):
winning_forward_with_endpoints = []
for path in set(winning_forward_paths):
full_path = list(path)
if from_node_id is not None and (not full_path or full_path[0] != from_node_id):
full_path = [from_node_id, *full_path]
if to_node_id is not None and (not full_path or full_path[-1] != to_node_id):
full_path = [*full_path, to_node_id]
full_path = [to_node_id, *full_path]
if from_node_id is not None and (not full_path or full_path[-1] != from_node_id):
full_path = [*full_path, from_node_id]
winning_forward_with_endpoints.append(full_path)
winning_reverse_with_endpoints = []
@@ -1000,8 +1003,6 @@ async def api_traceroute(request):
full_path = list(path)
if to_node_id is not None and (not full_path or full_path[0] != to_node_id):
full_path = [to_node_id, *full_path]
if from_node_id is not None and (not full_path or full_path[-1] != from_node_id):
full_path = [*full_path, from_node_id]
winning_reverse_with_endpoints.append(full_path)
winning_paths_json = {
+2 -2
View File
@@ -60,7 +60,7 @@ def write_cleanup_status(status: dict) -> None:
tmp_path = status_path.with_suffix(f"{status_path.suffix}.tmp")
try:
with open(tmp_path, "w", encoding="utf-8") as f:
json.dump(status, f, indent=2, sort_keys=True)
json.dump(status, f, indent=2)
tmp_path.replace(status_path)
except Exception as e:
cleanup_logger.warning(f"Failed to write cleanup status file: {e}")
@@ -71,7 +71,7 @@ def write_backup_status(status: dict) -> None:
tmp_path = status_path.with_suffix(f"{status_path.suffix}.tmp")
try:
with open(tmp_path, "w", encoding="utf-8") as f:
json.dump(status, f, indent=2, sort_keys=True)
json.dump(status, f, indent=2)
tmp_path.replace(status_path)
except Exception as e:
cleanup_logger.warning(f"Failed to write backup status file: {e}")