From 99cb5654e49482578c4a8a0f82f41065f5956f25 Mon Sep 17 00:00:00 2001 From: Pablo Revilla Date: Mon, 24 Nov 2025 16:11:11 -0800 Subject: [PATCH] More changes... almost ready for release. Ranamed 2 pages for easy or reading. --- meshview/store.py | 82 ++++++++---- meshview/templates/base.html | 4 - meshview/templates/chat.html | 4 +- meshview/templates/firehose.html | 2 +- meshview/templates/node.html | 2 +- .../{new_packet.html => packet.html} | 119 ++++++++++-------- meshview/web.py | 10 +- meshview/web_api/api.py | 82 ++++++------ 8 files changed, 177 insertions(+), 128 deletions(-) rename meshview/templates/{new_packet.html => packet.html} (81%) diff --git a/meshview/store.py b/meshview/store.py index 2d6a2e0..4845156 100644 --- a/meshview/store.py +++ b/meshview/store.py @@ -1,9 +1,9 @@ from datetime import datetime, timedelta -from sqlalchemy import func, nullslast, select, text +from sqlalchemy import and_, func, or_, select, text from sqlalchemy.orm import lazyload -from meshview import database +from meshview import database, models from meshview.models import Node, Packet, PacketSeen, Traceroute @@ -24,38 +24,68 @@ async def get_fuzzy_nodes(query): return result.scalars() + async def get_packets( - node_id=None, portnum=None, after=None, before=None, limit=None, packet_id=None + from_node_id=None, + to_node_id=None, + node_id=None, # legacy: match either from/to + portnum=None, + after=None, + limit=50, ): + """ + SQLAlchemy 2.0 async ORM version. + Supports strict from/to/node filtering, portnum, since, limit. + """ + async with database.async_session() as session: - # --- Fast path: fetch by packet_id (uses primary key lookup) --- - if packet_id is not None: - packet = await session.get(Packet, packet_id) - return [packet] if packet else [] - # --- Normal query path --- - q = select(Packet) + # Start select + stmt = select(models.Packet) - if node_id: - q = q.where((Packet.from_node_id == node_id) | (Packet.to_node_id == node_id)) - if portnum: - q = q.where(Packet.portnum == portnum) - if after: - q = q.where(Packet.import_time_us > after) - if before: - q = q.where(Packet.import_time_us < before) + conditions = [] - # Order by import_time_us when available, fallback to import_time for old data - # This handles databases where import_time_us may be NULL (old backups) - # First sort by import_time_us (NULLs last), then by import_time for those rows - q = q.order_by(nullslast(Packet.import_time_us.desc()), Packet.import_time.desc()) + # Strict FROM filter + if from_node_id is not None: + conditions.append(models.Packet.from_node_id == from_node_id) - if limit is not None: - q = q.limit(limit) + # Strict TO filter + if to_node_id is not None: + conditions.append(models.Packet.to_node_id == to_node_id) + + # Legacy "node_id = match either direction" + if node_id is not None: + conditions.append( + or_( + models.Packet.from_node_id == node_id, + models.Packet.to_node_id == node_id + ) + ) + + # Port filter + if portnum is not None: + conditions.append(models.Packet.portnum == portnum) + + # Time filter + if after is not None: + conditions.append(models.Packet.import_time_us > after) + + # Apply WHERE clause if needed + if conditions: + stmt = stmt.where(and_(*conditions)) + + # Sort newest → oldest + stmt = stmt.order_by(models.Packet.import_time_us.desc()) + + # Limit + stmt = stmt.limit(limit) + + # Execute + result = await session.execute(stmt) + + # Convert ORM rows to models + return result.scalars().all() - result = await session.execute(q) - packets = list(result.scalars()) - return packets async def get_packets_from(node_id=None, portnum=None, since=None, limit=500): diff --git a/meshview/templates/base.html b/meshview/templates/base.html index a16566d..44f3d0d 100644 --- a/meshview/templates/base.html +++ b/meshview/templates/base.html @@ -6,11 +6,7 @@ - - - - diff --git a/meshview/templates/chat.html b/meshview/templates/chat.html index 198f2db..bfba95b 100644 --- a/meshview/templates/chat.html +++ b/meshview/templates/chat.html @@ -131,7 +131,7 @@ document.addEventListener("DOMContentLoaded", async () => { replyHtml = `
${replyPrefix} - ${packet.reply_id} + ${packet.reply_id}
`; } } @@ -142,7 +142,7 @@ document.addEventListener("DOMContentLoaded", async () => { div.innerHTML = ` ${formattedTimestamp} - 🔎 + 🔎 ${escapeHtml(packet.channel || "")} diff --git a/meshview/templates/firehose.html b/meshview/templates/firehose.html index 1b0667e..d40a1e7 100644 --- a/meshview/templates/firehose.html +++ b/meshview/templates/firehose.html @@ -265,7 +265,7 @@ async function fetchUpdates() { const html = ` ${localTime} - ▶ ${pkt.id} + ▶ ${pkt.id} ${from} ${to} ${portLabel(pkt.portnum, pkt.payload, inlineLinks)} diff --git a/meshview/templates/node.html b/meshview/templates/node.html index 95f3311..0d9a917 100644 --- a/meshview/templates/node.html +++ b/meshview/templates/node.html @@ -572,7 +572,7 @@ async function loadPackets(){ list.insertAdjacentHTML("afterbegin", ` ${localTime} - ▶ ${pkt.id} + ▶ ${pkt.id} ${fromCell} ${toCell} ${portLabel(pkt.portnum)}${inlineLinks} diff --git a/meshview/templates/new_packet.html b/meshview/templates/packet.html similarity index 81% rename from meshview/templates/new_packet.html rename to meshview/templates/packet.html index 3954428..b9362d1 100644 --- a/meshview/templates/new_packet.html +++ b/meshview/templates/packet.html @@ -48,10 +48,10 @@ display: none; } -/* --- Source Marker: 2px bigger than gateway (26px vs 24px) --- */ +/* --- SOURCE MARKER (slightly bigger) --- */ .source-marker { - width: 26px; - height: 26px; + width: 24px; + height: 24px; background: rgba(255,0,0,0.55); border: 3px solid #ff0000; border-radius: 50%; @@ -135,13 +135,17 @@ document.addEventListener("DOMContentLoaded", async () => { const seenTableBody = document.getElementById("seen-table-body"); const seenCountSpan = document.getElementById("seen-count"); - const match = window.location.pathname.match(/\/new_packet\/(\d+)/); + /* --------------------------------------------- + Identify packet ID + ----------------------------------------------*/ + const match = window.location.pathname.match(/\/packet\/(\d+)/); if (!match) { loading.textContent = "Invalid packet URL"; return; } const packetId = match[1]; + /* PORT NAME MAP */ const PORT_NAMES = { 0:"UNKNOWN APP", 1:"Text", @@ -154,9 +158,9 @@ document.addEventListener("DOMContentLoaded", async () => { 71:"Neighbor" }; - /* ------------------------- + /* --------------------------------------------- Fetch packet - -------------------------- */ + ----------------------------------------------*/ const packetRes = await fetch(`/api/packets?packet_id=${packetId}`); const packetData = await packetRes.json(); if (!packetData.packets.length) { @@ -165,24 +169,24 @@ document.addEventListener("DOMContentLoaded", async () => { } const p = packetData.packets[0]; - /* ------------------------- - Fetch nodes - -------------------------- */ + /* --------------------------------------------- + Fetch all nodes + ----------------------------------------------*/ const nodesRes = await fetch("/api/nodes"); const nodesData = await nodesRes.json(); const nodeLookup = {}; (nodesData.nodes || []).forEach(n => nodeLookup[n.node_id] = n); const fromNodeObj = nodeLookup[p.from_node_id]; - const fromNodeLabel = fromNodeObj?.long_name || p.from_node_id; + const toNodeObj = nodeLookup[p.to_node_id]; - const toNodeObj = nodeLookup[p.to_node_id]; + const fromNodeLabel = fromNodeObj?.long_name || p.from_node_id; const toNodeLabel = p.to_node_id == 4294967295 ? "All" : (toNodeObj?.long_name || p.to_node_id); - /* ------------------------- - Parse payload - -------------------------- */ + /* --------------------------------------------- + Parse payload for lat/lon if this *packet* is a position packet + ----------------------------------------------*/ let lat = null, lon = null; const parsed = {}; @@ -197,19 +201,19 @@ document.addEventListener("DOMContentLoaded", async () => { }); } + /* --------------------------------------------- + Render packet header & details + ----------------------------------------------*/ + const time = p.import_time_us + ? new Date(p.import_time_us / 1000).toLocaleString() + : "—"; + const telemetryExtras = []; if (parsed.PDOP) telemetryExtras.push(`PDOP: ${parsed.PDOP}`); if (parsed.sats_in_view) telemetryExtras.push(`Sats: ${parsed.sats_in_view}`); if (parsed.ground_speed) telemetryExtras.push(`Speed: ${parsed.ground_speed}`); if (parsed.altitude) telemetryExtras.push(`Altitude: ${parsed.altitude}`); - const time = p.import_time_us - ? new Date(p.import_time_us / 1000).toLocaleString() - : "—"; - - /* ------------------------- - Render packet card - -------------------------- */ packetCard.innerHTML = `
Packet ID: ${p.id} @@ -256,9 +260,9 @@ document.addEventListener("DOMContentLoaded", async () => { loading.classList.add("d-none"); packetCard.classList.remove("d-none"); - /* ------------------------- - Map init - -------------------------- */ + /* --------------------------------------------- + Map initialization + ----------------------------------------------*/ const map = L.map("map"); mapDiv.style.display = "block"; @@ -268,11 +272,22 @@ document.addEventListener("DOMContentLoaded", async () => { const allBounds = []; - /* -------------------------------------------------------------------- - SOURCE MARKER (26px vs 24px) - --------------------------------------------------------------------- */ - if (lat && lon) { - allBounds.push([lat, lon]); + /* --------------------------------------------- + ALWAYS SHOW SOURCE POSITION + Priority: + 1) position from packet payload + 2) fallback: last_lat/last_long from /api/nodes + ----------------------------------------------*/ + let srcLat = lat; + let srcLon = lon; + + if ((!srcLat || !srcLon) && fromNodeObj?.last_lat && fromNodeObj?.last_long) { + srcLat = fromNodeObj.last_lat / 1e7; + srcLon = fromNodeObj.last_long / 1e7; + } + + if (srcLat && srcLon) { + allBounds.push([srcLat, srcLon]); const sourceIcon = L.divIcon({ html: `
`, @@ -281,7 +296,7 @@ document.addEventListener("DOMContentLoaded", async () => { iconAnchor: [13, 13] }); - const sourceMarker = L.marker([lat, lon], { + const sourceMarker = L.marker([srcLat, srcLon], { icon: sourceIcon, zIndexOffset: 9999 }).addTo(map); @@ -289,34 +304,32 @@ document.addEventListener("DOMContentLoaded", async () => { sourceMarker.bindPopup(`
Packet Source
- Lat: ${lat.toFixed(6)}
- Lon: ${lon.toFixed(6)}
+ Lat: ${srcLat.toFixed(6)}
+ Lon: ${srcLon.toFixed(6)}
From Node: ${fromNodeLabel}
Channel: ${p.channel ?? "—"}
Port: ${PORT_NAMES[p.portnum] || "UNKNOWN"} (${p.portnum})
`); + } else { - map.setView([0, 0], 2); + map.setView([0,0], 2); } - /* ------------------------- - Helpers - -------------------------- */ - - /* 0 = warmest → 7 = coldest */ + /* --------------------------------------------- + Color for hop indicator markers (warm → cold) + ----------------------------------------------*/ function hopColor(hopValue){ const colors = [ - "#ff3b30", // 0 red + "#ff3b30", "#ff6b22", "#ff9f0c", "#ffd60a", "#87d957", "#57d9c4", "#3db2ff", - "#1e63ff" // 7 deep cold blue + "#1e63ff" ]; - let h = Number(hopValue); if (isNaN(h)) return "#aaa"; if (h < 0) h = 0; @@ -324,6 +337,7 @@ document.addEventListener("DOMContentLoaded", async () => { return colors[h]; } + /* Distance helper */ function haversine(lat1,lon1,lat2,lon2){ const R=6371; const dLat=(lat2-lat1)*Math.PI/180; @@ -335,13 +349,14 @@ document.addEventListener("DOMContentLoaded", async () => { return R*(2*Math.atan2(Math.sqrt(a),Math.sqrt(1-a))); } - /* ------------------------- - Fetch seen list - -------------------------- */ + /* --------------------------------------------- + Fetch packets_seen + ----------------------------------------------*/ const seenRes = await fetch(`/api/packets_seen/${packetId}`); const seenData = await seenRes.json(); const seenList = seenData.seen ?? []; + /* sort by hop_start descending (warm → cold) */ const seenSorted = seenList.slice().sort((a,b)=>{ const A=a.hop_start??-999; const B=b.hop_start??-999; @@ -353,9 +368,9 @@ document.addEventListener("DOMContentLoaded", async () => { seenCountSpan.textContent=`(${seenSorted.length} gateways)`; } - /* ------------------------- - Gateway markers + table - -------------------------- */ + /* --------------------------------------------- + Gateway markers and seen table + ----------------------------------------------*/ seenTableBody.innerHTML = seenSorted.map(s=>{ const node=nodeLookup[s.node_id]; const label=node?(node.long_name||node.node_id):s.node_id; @@ -401,8 +416,8 @@ document.addEventListener("DOMContentLoaded", async () => { }).addTo(map); let distKm=null,distMi=null; - if(lat&&lon){ - distKm=haversine(lat,lon,rlat,rlon); + if(srcLat&&srcLon){ + distKm=haversine(srcLat,srcLon,rlat,rlon); distMi=distKm*0.621371; } @@ -437,10 +452,16 @@ document.addEventListener("DOMContentLoaded", async () => { `; }).join(""); + /* --------------------------------------------- + Fit map to all markers + ----------------------------------------------*/ if(allBounds.length>0){ map.fitBounds(allBounds,{padding:[40,40]}); } + /* --------------------------------------------- + Escape HTML + ----------------------------------------------*/ function escapeHtml(unsafe) { return (unsafe??"").replace(/[&<"'>]/g,m=>({ "&":"&", diff --git a/meshview/web.py b/meshview/web.py index c4fd497..1f64cee 100644 --- a/meshview/web.py +++ b/meshview/web.py @@ -200,12 +200,6 @@ async def redirect_packet_list(request): raise web.HTTPFound(location=f"/node/{packet_id}") -# redirect for backwards compatibility -@routes.get("/packet/{packet_id}") -async def redirect_packet(request): - packet_id = request.match_info["packet_id"] - raise web.HTTPFound(location=f"/packet/{packet_id}") - @routes.get("/net") async def net(request): return web.Response( @@ -246,9 +240,9 @@ async def chat(request): ) -@routes.get("/new_packet/{packet_id}") +@routes.get("/packet/{packet_id}") async def new_packet(request): - template = env.get_template("new_packet.html") + template = env.get_template("packet.html") return web.Response( text=template.render(), content_type="text/html", diff --git a/meshview/web_api/api.py b/meshview/web_api/api.py index a732c3d..2d38979 100644 --- a/meshview/web_api/api.py +++ b/meshview/web_api/api.py @@ -100,9 +100,13 @@ async def api_packets(request): since_str = request.query.get("since") portnum_str = request.query.get("portnum") contains = request.query.get("contains") - from_node_id_str = request.query.get("from_node_id") - # --- If a packet_id is provided, fetch just that one --- + # NEW — explicit filters + from_node_id_str = request.query.get("from_node_id") + to_node_id_str = request.query.get("to_node_id") + node_id_str = request.query.get("node_id") # legacy: match either from/to + + # --- If a packet_id is provided, return only that packet --- if packet_id_str: try: packet_id = int(packet_id_str) @@ -111,7 +115,7 @@ async def api_packets(request): packet = await store.get_packet(packet_id) if not packet: - return web.json_response({"packets": []}) # consistent shape + return web.json_response({"packets": []}) p = Packet.from_model(packet) data = { @@ -121,24 +125,19 @@ async def api_packets(request): "portnum": int(p.portnum) if p.portnum is not None else None, "payload": (p.payload or "").strip(), "import_time_us": p.import_time_us, - # NOTE: Temporary stopgap - include import_time as fallback until old data - # with import_time_us=0 is migrated/cleaned up. Can be removed once all - # legacy records have been updated. "import_time": p.import_time.isoformat() if p.import_time else None, "channel": getattr(p.from_node, "channel", ""), "long_name": getattr(p.from_node, "long_name", ""), } - return web.json_response({"packets": [data]}) # unified key + return web.json_response({"packets": [data]}) - # --- Otherwise: multi-packet listing mode --- - - # Limit validation + # --- Parse limit --- try: limit = min(max(int(limit_str), 1), 100) except ValueError: limit = 50 - # Parse 'since' timestamp + # --- Parse since timestamp --- since = None if since_str: try: @@ -146,15 +145,7 @@ async def api_packets(request): except ValueError: logger.warning(f"Invalid 'since' value (expected microseconds): {since_str}") - # Parse from_node_id (decimal or hex) - node_id = None - if from_node_id_str: - try: - node_id = int(from_node_id_str, 0) - except ValueError: - logger.warning(f"Invalid from_node_id: {from_node_id_str}") - - # Parse portnum safely + # --- Parse portnum --- portnum = None if portnum_str: try: @@ -162,9 +153,34 @@ async def api_packets(request): except ValueError: logger.warning(f"Invalid portnum: {portnum_str}") - # --- Fetch packets --- + # --- Parse node filters --- + from_node_id = None + to_node_id = None + node_id = None # legacy: match either from/to + + if from_node_id_str: + try: + from_node_id = int(from_node_id_str, 0) + except ValueError: + logger.warning(f"Invalid from_node_id: {from_node_id_str}") + + if to_node_id_str: + try: + to_node_id = int(to_node_id_str, 0) + except ValueError: + logger.warning(f"Invalid to_node_id: {to_node_id_str}") + + if node_id_str: + try: + node_id = int(node_id_str, 0) + except ValueError: + logger.warning(f"Invalid node_id: {node_id_str}") + + # --- Fetch packets using explicit filters --- packets = await store.get_packets( - node_id=node_id, + from_node_id=from_node_id, + to_node_id=to_node_id, + node_id=node_id, # old behavior: match from OR to portnum=portnum, after=since, limit=limit, @@ -179,21 +195,18 @@ async def api_packets(request): ui_packets = [p for p in ui_packets if contains.lower() in p.payload.lower()] # --- Sort descending by import_time_us --- - # Handle None values by treating them as smallest (will be sorted last) ui_packets.sort( - key=lambda p: (p.import_time_us is not None, p.import_time_us or 0), reverse=True + key=lambda p: (p.import_time_us is not None, p.import_time_us or 0), + reverse=True ) ui_packets = ui_packets[:limit] - # --- Prepare output --- + # --- Build JSON output --- packets_data = [] for p in ui_packets: packet_dict = { "id": p.id, "import_time_us": p.import_time_us, - # NOTE: Temporary stopgap - include import_time as fallback until old data - # with import_time_us=0 is migrated/cleaned up. Can be removed once all - # legacy records have been updated. "import_time": p.import_time.isoformat() if p.import_time else None, "channel": getattr(p.from_node, "channel", ""), "from_node_id": p.from_node_id, @@ -213,25 +226,19 @@ async def api_packets(request): packets_data.append(packet_dict) - # Calculate latest_import_time for incremental updates - # NOTE: Temporary stopgap - fallback to import_time until old data with - # import_time_us=0 is migrated/cleaned up. Can be simplified once all - # legacy records have been updated. - # Use the highest import_time_us, with fallback to import_time + # --- Latest import_time for incremental fetch --- latest_import_time = None if packets_data: for p in packets_data: if p.get("import_time_us") and p["import_time_us"] > 0: - if latest_import_time is None or p["import_time_us"] > latest_import_time: - latest_import_time = p["import_time_us"] + latest_import_time = max(latest_import_time or 0, p["import_time_us"]) elif p.get("import_time") and latest_import_time is None: - # Fallback: convert ISO string to microseconds if import_time_us is missing try: dt = datetime.datetime.fromisoformat( p["import_time"].replace("Z", "+00:00") ) latest_import_time = int(dt.timestamp() * 1_000_000) - except (ValueError, AttributeError): + except Exception: pass response = {"packets": packets_data} @@ -245,6 +252,7 @@ async def api_packets(request): return web.json_response({"error": "Failed to fetch packets"}, status=500) + @routes.get("/api/stats") async def api_stats(request): """