From 257bf7ffac0f8bdc53903925f8fc8be63977d315 Mon Sep 17 00:00:00 2001 From: Nathan Date: Sat, 11 Oct 2025 16:28:34 -0700 Subject: [PATCH] Add channel filters to stats, chat, and firehose views --- meshview/store.py | 12 +- meshview/templates/chat.html | 144 +++++++++++++- meshview/templates/firehose.html | 206 ++++++++++++++++---- meshview/templates/stats.html | 317 +++++++++++++++++++++++++------ meshview/web.py | 53 +++++- 5 files changed, 617 insertions(+), 115 deletions(-) diff --git a/meshview/store.py b/meshview/store.py index ef3c7ae..6bfa225 100644 --- a/meshview/store.py +++ b/meshview/store.py @@ -24,7 +24,7 @@ async def get_fuzzy_nodes(query): return result.scalars() -async def get_packets(node_id=None, portnum=None, after=None, before=None, limit=None): +async def get_packets(node_id=None, portnum=None, after=None, before=None, limit=None, channel: str | None = None): async with database.async_session() as session: q = select(Packet) @@ -36,6 +36,8 @@ async def get_packets(node_id=None, portnum=None, after=None, before=None, limit q = q.where(Packet.import_time > after) if before: q = q.where(Packet.import_time < before) + if channel: + q = q.where(func.lower(Packet.channel) == channel.lower()) q = q.order_by(Packet.import_time.desc()) @@ -147,17 +149,21 @@ async def get_mqtt_neighbors(since): # We count the total amount of packages # This is to be used by /stats in web.py -async def get_total_packet_count(): +async def get_total_packet_count(channel: str | None = None) -> int: async with database.async_session() as session: q = select(func.count(Packet.id)) # Use SQLAlchemy's func to count packets + if channel: + q = q.where(func.lower(Packet.channel) == channel.lower()) result = await session.execute(q) return result.scalar() # Return the total count of packets # We count the total amount of seen packets -async def get_total_packet_seen_count(): +async def get_total_packet_seen_count(channel: str | None = None) -> int: async with database.async_session() as session: q = select(func.count(PacketSeen.node_id)) # Use SQLAlchemy's func to count nodes + if channel: + q = q.where(func.lower(PacketSeen.channel) == channel.lower()) result = await session.execute(q) return result.scalar() # Return the` total count of seen packets diff --git a/meshview/templates/chat.html b/meshview/templates/chat.html index 3df51a9..f297b6a 100644 --- a/meshview/templates/chat.html +++ b/meshview/templates/chat.html @@ -17,19 +17,61 @@ /* Nested reply style */ .replying-to { font-size: 0.85em; color: #aaa; margin-top: 4px; padding-left: 20px; } .replying-to .reply-preview { color: #aaa; } + +.filter-bar { + display: flex; + justify-content: center; + align-items: center; + gap: 10px; + margin-bottom: 16px; +} + +.filter-label { + color: #ccc; + font-size: 14px; +} + +#chatChannelSelect { + padding: 4px 6px; + background: #444; + color: #fff; + border: none; + border-radius: 4px; +} + +.status-message { + color: #bbb; + margin-bottom: 12px; + display: none; +} {% endblock %} {% block body %}
+
+ + +
+
diff --git a/meshview/templates/stats.html b/meshview/templates/stats.html index e045371..38bd3fe 100644 --- a/meshview/templates/stats.html +++ b/meshview/templates/stats.html @@ -77,14 +77,27 @@ font-weight: bold; } +.filter-bar { + display: flex; + justify-content: flex-end; + align-items: center; + gap: 10px; + margin-bottom: 20px; +} + +.filter-label { + color: #ccc; + font-size: 14px; +} + #channelSelect { - margin-bottom: 8px; padding: 4px 6px; background:#444; color:#fff; border:none; border-radius:4px; } + {% endblock %} {% block head %} @@ -95,18 +108,25 @@

Mesh Statistics - Summary (all available in Database)

+
+ + +
+

Total Nodes

-
{{ "{:,}".format(total_nodes) }}
+
{{ "{:,}".format(total_nodes) }}

Total Packets

-
{{ "{:,}".format(total_packets) }}
+
{{ "{:,}".format(total_packets) }}

Total Packets Seen

-
{{ "{:,}".format(total_packets_seen) }}
+
{{ "{:,}".format(total_packets_seen) }}
@@ -122,9 +142,6 @@

Packet Types - Last 24 Hours

-
@@ -201,12 +218,25 @@ const PORTNUM_LABELS = { 71: "Neighbor Info" }; +const PORT_CONFIG = [ + { port: 1, color: '#ff5722', domId: 'chart_portnum_1', totalId: 'total_portnum_1' }, + { port: 3, color: '#2196f3', domId: 'chart_portnum_3', totalId: 'total_portnum_3' }, + { port: 4, color: '#9c27b0', domId: 'chart_portnum_4', totalId: 'total_portnum_4' }, + { port: 67, color: '#ffeb3b', domId: 'chart_portnum_67', totalId: 'total_portnum_67' }, + { port: 70, color: '#795548', domId: 'chart_portnum_70', totalId: 'total_portnum_70' }, + { port: 71, color: '#4caf50', domId: 'chart_portnum_71', totalId: 'total_portnum_71' }, +]; + +const CHANNEL_PRESETS = ["LongFast", "MediumSlow"]; + +let currentChannel = ""; + // --- Fetch & Processing --- async function fetchStats(period_type,length,portnum=null,channel=null){ try{ let url=`/api/stats?period_type=${period_type}&length=${length}`; if(portnum!==null) url+=`&portnum=${portnum}`; - if(channel) url+=`&channel=${channel}`; + if(channel) url+=`&channel=${encodeURIComponent(channel)}`; const res=await fetch(url); if(!res.ok) return []; const json=await res.json(); @@ -214,28 +244,119 @@ async function fetchStats(period_type,length,portnum=null,channel=null){ }catch{return [];} } -async function fetchNodes(){ try{ const res=await fetch("/api/nodes"); const json=await res.json(); return json.nodes||[];}catch{return [];} } -async function fetchChannels(){ try{ const res = await fetch("/api/channels"); const json = await res.json(); return json.channels || [];}catch{return [];} } +async function fetchNodes(channel=null){ + try{ + const base="/api/nodes"; + const url=channel?`${base}?channel=${encodeURIComponent(channel)}`:base; + const res=await fetch(url); + const json=await res.json(); + return json.nodes||[]; + }catch{return [];} +} -function processCountField(nodes,field){ const counts={}; nodes.forEach(n=>{ const key=n[field]||"Unknown"; counts[key]=(counts[key]||0)+1; }); return Object.entries(counts).map(([name,value])=>({name,value})); } -function updateTotalCount(domId,data){ const el=document.getElementById(domId); if(!el||!data.length) return; const total=data.reduce((acc,d)=>acc+(d.count??d.packet_count??0),0); el.textContent=`Total: ${total.toLocaleString()}`; } -function prepareTopN(data,n=20){ data.sort((a,b)=>b.value-a.value); let top=data.slice(0,n); if(data.length>n){ const otherValue=data.slice(n).reduce((sum,item)=>sum+item.value,0); top.push({name:"Other", value:otherValue}); } return top; } +async function fetchChannels(){ + try{ + const res = await fetch("/api/channels"); + const json = await res.json(); + return json.channels || []; + }catch{return [];} +} + +async function fetchSummary(channel=null){ + try{ + const base="/api/stats/summary"; + const url=channel?`${base}?channel=${encodeURIComponent(channel)}`:base; + const res=await fetch(url); + if(!res.ok) return null; + return await res.json(); + }catch{return null;} +} + +function processCountField(nodes,field){ + const counts={}; + (nodes||[]).forEach(n=>{ + const key=n?.[field]||"Unknown"; + counts[key]=(counts[key]||0)+1; + }); + return Object.entries(counts).map(([name,value])=>({name,value})); +} + +function updateTotalCount(domId,data){ + const el=document.getElementById(domId); + if(!el) return; + const dataset=Array.isArray(data)?data:[]; + const total=dataset.reduce((acc,d)=>acc+(d?.count??d?.packet_count??0),0); + el.textContent=`Total: ${total.toLocaleString()}`; +} + +function prepareTopN(data=[],n=20){ + const sorted=[...(data||[])].sort((a,b)=>b.value-a.value); + const top=sorted.slice(0,n); + if(sorted.length>n){ + const otherValue=sorted.slice(n).reduce((sum,item)=>sum+item.value,0); + top.push({name:"Other", value:otherValue}); + } + return top; +} // --- Chart Rendering --- -function renderChart(domId,data,type,color){ const el=document.getElementById(domId); if(!el) return; const chart=echarts.init(el); const periods=data.map(d=>(d.period??d.period===0)?d.period.toString():''); const counts=data.map(d=>d.count??d.packet_count??0); chart.setOption({backgroundColor:'#272b2f', tooltip:{trigger:'axis'}, grid:{left:'6%', right:'6%', bottom:'18%'}, xAxis:{type:'category', data:periods, axisLine:{lineStyle:{color:'#aaa'}}, axisLabel:{rotate:45,color:'#ccc'}}, yAxis:{type:'value', axisLine:{lineStyle:{color:'#aaa'}}, axisLabel:{color:'#ccc'}}, series:[{data:counts,type:type,smooth:type==='line',itemStyle:{color:color}, areaStyle:type==='line'?{}:undefined}]}); return chart; } +function renderChart(domId,data,type,color){ + const el=document.getElementById(domId); + if(!el) return null; + const existing=echarts.getInstanceByDom(el); + if(existing) existing.dispose(); + const chart=echarts.init(el); + const source=Array.isArray(data)?data:[]; + const periods=source.map(d=>{ + const periodValue=d?.period; + return (periodValue||periodValue===0) ? String(periodValue) : ''; + }); + const counts=source.map(d=>d?.count??d?.packet_count??0); + chart.setOption({ + backgroundColor:'#272b2f', + tooltip:{trigger:'axis'}, + grid:{left:'6%', right:'6%', bottom:'18%'}, + xAxis:{type:'category', data:periods, axisLine:{lineStyle:{color:'#aaa'}}, axisLabel:{rotate:45,color:'#ccc'}}, + yAxis:{type:'value', axisLine:{lineStyle:{color:'#aaa'}}, axisLabel:{color:'#ccc'}}, + series:[{data:counts,type:type,smooth:type==='line',itemStyle:{color:color}, areaStyle:type==='line'?{}:undefined}] + }); + return chart; +} -function renderPieChart(elId,data,name){ const el=document.getElementById(elId); if(!el) return; const chart=echarts.init(el); const top20=prepareTopN(data,20); chart.setOption({backgroundColor:"#272b2f", tooltip:{trigger:"item", formatter: params=>`${params.name}: ${Math.round(params.percent)}% (${params.value})`}, series:[{name:name, type:"pie", radius:["30%","70%"], center:["50%","50%"], avoidLabelOverlap:true, itemStyle:{borderRadius:6,borderColor:"#272b2f",borderWidth:2}, label:{show:true,formatter:"{b}\n{d}%", color:"#ccc", fontSize:10}, labelLine:{show:true,length:10,length2:6}, data:top20}]}); return chart; } +function renderPieChart(elId,data,name){ + const el=document.getElementById(elId); + if(!el) return null; + const existing=echarts.getInstanceByDom(el); + if(existing) existing.dispose(); + const chart=echarts.init(el); + const top20=prepareTopN(data,20); + chart.setOption({ + backgroundColor:"#272b2f", + tooltip:{trigger:"item", formatter: params=>`${params.name}: ${Math.round(params.percent)}% (${params.value})`}, + series:[{ + name:name, + type:"pie", + radius:["30%","70%"], + center:["50%","50%"], + avoidLabelOverlap:true, + itemStyle:{borderRadius:6,borderColor:"#272b2f",borderWidth:2}, + label:{show:true,formatter:"{b}\n{d}%", color:"#ccc", fontSize:10}, + labelLine:{show:true,length:10,length2:6}, + data:top20 + }] + }); + return chart; +} // --- Packet Type Pie Chart --- async function fetchPacketTypeBreakdown(channel=null) { - const portnums = [1,3,4,67,70,71]; - const requests = portnums.map(async pn => { - const data = await fetchStats('hour',24,pn,channel); - const total = (data || []).reduce((sum,d)=>sum+(d.count??d.packet_count??0),0); - return {portnum: pn, count: total}; + const requests = PORT_CONFIG.map(async ({port}) => { + const data = await fetchStats('hour',24,port,channel); + const total = (data || []).reduce((sum,d)=>sum+(d?.count??d?.packet_count??0),0); + return {portnum: port, count: total}; }); const allData = await fetchStats('hour',24,null,channel); - const totalAll = allData.reduce((sum,d)=>sum+(d.count??d.packet_count??0),0); + const totalAll = (allData||[]).reduce((sum,d)=>sum+(d?.count??d?.packet_count??0),0); const results = await Promise.all(requests); const trackedTotal = results.reduce((sum,d)=>sum+d.count,0); const other = Math.max(totalAll - trackedTotal,0); @@ -243,44 +364,130 @@ async function fetchPacketTypeBreakdown(channel=null) { return results; } +function updateSummaryCards(summary){ + if(!summary) return; + const nodesEl=document.getElementById("summaryTotalNodes"); + const packetsEl=document.getElementById("summaryTotalPackets"); + const packetsSeenEl=document.getElementById("summaryTotalPacketsSeen"); + if(nodesEl) nodesEl.textContent=Number(summary.total_nodes||0).toLocaleString(); + if(packetsEl) packetsEl.textContent=Number(summary.total_packets||0).toLocaleString(); + if(packetsSeenEl) packetsSeenEl.textContent=Number(summary.total_packets_seen||0).toLocaleString(); +} + +function populateChannelSelect(channels){ + const select=document.getElementById("channelSelect"); + if(!select) return; + const unique=Array.from(new Set((channels||[]).filter(ch=>typeof ch==="string" && ch.trim().length>0))).sort((a,b)=>a.localeCompare(b)); + const prioritized=[]; + CHANNEL_PRESETS.forEach(preset=>{ + const idx=unique.indexOf(preset); + if(idx>=0){ + prioritized.push(unique[idx]); + unique.splice(idx,1); + } + }); + const ordered=[...new Set([...prioritized, ...unique])]; + select.innerHTML=""; + const allOption=document.createElement("option"); + allOption.value=""; + allOption.textContent="All Channels"; + allOption.setAttribute("data-translate-lang","all_channels"); + select.appendChild(allOption); + ordered.forEach(ch=>{ + const opt=document.createElement("option"); + opt.value=ch; + opt.textContent=ch; + select.appendChild(opt); + }); + let preferred=currentChannel; + if(!preferred){ + preferred=CHANNEL_PRESETS.find(preset=>ordered.includes(preset))||""; + } + if(preferred && !ordered.includes(preferred)){ + preferred=""; + } + select.value=preferred; + currentChannel=select.value; +} + +function setPortChart(port, chart){ + switch(port){ + case 1: chartPortnum1 = chart; break; + case 3: chartPortnum3 = chart; break; + case 4: chartPortnum4 = chart; break; + case 67: chartPortnum67 = chart; break; + case 70: chartPortnum70 = chart; break; + case 71: chartPortnum71 = chart; break; + default: break; + } +} + // --- Init --- let chartHourlyAll, chartPortnum1, chartPortnum3, chartPortnum4, chartPortnum67, chartPortnum70, chartPortnum71; let chartDailyAll, chartDailyPortnum1; let chartHwModel, chartRole, chartChannel; let chartPacketTypes; +let isRefreshing=false; + +async function refreshDashboard(){ + if(isRefreshing) return; + isRefreshing=true; + const channel=currentChannel||null; + try{ + const [summary, dailyAllData, dailyPort1Data, hourlyAllData, portDataSets, nodes, packetTypesData] = await Promise.all([ + fetchSummary(channel), + fetchStats('day',14,null,channel), + fetchStats('day',14,1,channel), + fetchStats('hour',24,null,channel), + Promise.all(PORT_CONFIG.map(cfg=>fetchStats('hour',24,cfg.port,channel))), + fetchNodes(channel), + fetchPacketTypeBreakdown(channel) + ]); + + updateSummaryCards(summary); + + updateTotalCount('total_daily_all',dailyAllData); + chartDailyAll=renderChart('chart_daily_all',dailyAllData,'line','#66bb6a'); + + updateTotalCount('total_daily_portnum_1',dailyPort1Data); + chartDailyPortnum1=renderChart('chart_daily_portnum_1',dailyPort1Data,'bar','#ff5722'); + + updateTotalCount('total_hourly_all',hourlyAllData); + chartHourlyAll=renderChart('chart_hourly_all',hourlyAllData,'bar','#03dac6'); + + PORT_CONFIG.forEach((cfg,idx)=>{ + const data=portDataSets?.[idx]||[]; + updateTotalCount(cfg.totalId,data); + const chart=renderChart(cfg.domId,data,'bar',cfg.color); + setPortChart(cfg.port,chart); + }); + + chartHwModel=renderPieChart("chart_hw_model",processCountField(nodes,"hw_model"),"Hardware"); + chartRole=renderPieChart("chart_role",processCountField(nodes,"role"),"Role"); + chartChannel=renderPieChart("chart_channel",processCountField(nodes,"channel"),"Channel"); + + const formatted=(packetTypesData||[]).filter(d=>d.count>0).map(d=>({ + name: d.portnum==="other" ? "Other" : (PORTNUM_LABELS[d.portnum]||`Port ${d.portnum}`), + value: d.count + })); + chartPacketTypes=renderPieChart("chart_packet_types",formatted,"Packet Types (Last 24h)"); + } finally { + isRefreshing=false; + } +} async function init(){ const channels = await fetchChannels(); - const select = document.getElementById("channelSelect"); - channels.forEach(ch=>{ const opt = document.createElement("option"); opt.value = ch; opt.textContent = ch; select.appendChild(opt); }); - - const dailyAllData=await fetchStats('day',14); - updateTotalCount('total_daily_all',dailyAllData); - chartDailyAll=renderChart('chart_daily_all',dailyAllData,'line','#66bb6a'); - - const dailyPort1Data=await fetchStats('day',14,1); - updateTotalCount('total_daily_portnum_1',dailyPort1Data); - chartDailyPortnum1=renderChart('chart_daily_portnum_1',dailyPort1Data,'bar','#ff5722'); - - const hourlyAllData=await fetchStats('hour',24); - updateTotalCount('total_hourly_all',hourlyAllData); - chartHourlyAll=renderChart('chart_hourly_all',hourlyAllData,'bar','#03dac6'); - - const portnums=[1,3,4,67,70,71]; - const colors=['#ff5722','#2196f3','#9c27b0','#ffeb3b','#795548','#4caf50']; - const domIds=['chart_portnum_1','chart_portnum_3','chart_portnum_4','chart_portnum_67','chart_portnum_70','chart_portnum_71']; - const totalIds=['total_portnum_1','total_portnum_3','total_portnum_4','total_portnum_67','total_portnum_70','total_portnum_71']; - const allData=await Promise.all(portnums.map(pn=>fetchStats('hour',24,pn))); - for(let i=0;id.count>0).map(d=>({ name: d.portnum==="other" ? "Other" : (PORTNUM_LABELS[d.portnum]||`Port ${d.portnum}`), value: d.count })); - chartPacketTypes = renderPieChart("chart_packet_types",formatted,"Packet Types (Last 24h)"); + populateChannelSelect(channels); + const select=document.getElementById("channelSelect"); + if(select && !select.dataset.listenerAttached){ + select.addEventListener("change",async e=>{ + currentChannel=e.target.value; + await refreshDashboard(); + }); + select.dataset.listenerAttached="true"; + } + await refreshDashboard(); } window.addEventListener('resize',()=>{ [chartHourlyAll,chartPortnum1,chartPortnum3,chartPortnum4,chartPortnum67,chartPortnum70,chartPortnum71, chartDailyAll,chartDailyPortnum1,chartHwModel,chartRole,chartChannel,chartPacketTypes].forEach(c=>c?.resize()); }); @@ -342,14 +549,6 @@ document.querySelectorAll(".export-btn").forEach(btn=>{ }); }); -document.getElementById("channelSelect").addEventListener("change", async (e)=>{ - const channel = e.target.value; - const packetTypesData = await fetchPacketTypeBreakdown(channel); - const formatted = packetTypesData.filter(d=>d.count>0).map(d=>({ name: d.portnum==="other" ? "Other" : (PORTNUM_LABELS[d.portnum]||`Port ${d.portnum}`), value: d.count })); - chartPacketTypes?.dispose(); - chartPacketTypes = renderPieChart("chart_packet_types",formatted,"Packet Types (Last 24h)"); -}); - init(); // --- Translation Loader --- diff --git a/meshview/web.py b/meshview/web.py index 54f015b..d56db07 100644 --- a/meshview/web.py +++ b/meshview/web.py @@ -425,15 +425,27 @@ async def packet_details(request): @routes.get("/firehose") async def packet_details_firehose(request): - portnum = request.query.get("portnum") - if portnum: - portnum = int(portnum) - packets = await store.get_packets(portnum=portnum, limit=10) + portnum_value = request.query.get("portnum") + channel = request.query.get("channel") + + portnum = None + if portnum_value: + try: + portnum = int(portnum_value) + except ValueError: + logger.warning("Invalid portnum '%s' provided to /firehose", portnum_value) + + packets = await store.get_packets(portnum=portnum, limit=10, channel=channel) + ui_packets = [Packet.from_model(p) for p in packets] + latest_time = ui_packets[0].import_time.isoformat() if ui_packets else None + template = env.get_template("firehose.html") return web.Response( text=template.render( - packets=(Packet.from_model(p) for p in packets), + packets=ui_packets, portnum=portnum, + channel=channel, + last_time=latest_time, site_config=CONFIG, SOFTWARE_RELEASE=SOFTWARE_RELEASE, ), @@ -454,8 +466,18 @@ async def firehose_updates(request): logger.error(f"Failed to parse last_time '{last_time_str}': {e}") last_time = None + portnum_value = request.query.get("portnum") + channel = request.query.get("channel") + + portnum = None + if portnum_value: + try: + portnum = int(portnum_value) + except ValueError: + logger.warning("Invalid portnum '%s' provided to /firehose/updates", portnum_value) + # Query packets after last_time (microsecond precision) - packets = await store.get_packets(after=last_time, limit=10) + packets = await store.get_packets(after=last_time, limit=10, portnum=portnum, channel=channel) # Convert to UI model ui_packets = [Packet.from_model(p) for p in packets] @@ -1458,6 +1480,7 @@ async def api_chat(request): # Parse query params limit_str = request.query.get("limit", "20") since_str = request.query.get("since") + channel_filter = request.query.get("channel") # Clamp limit between 1 and 200 try: @@ -1477,7 +1500,9 @@ async def api_chat(request): packets = await store.get_packets( node_id=0xFFFFFFFF, portnum=PortNum.TEXT_MESSAGE_APP, + after=since, limit=limit, + channel=channel_filter, ) ui_packets = [Packet.from_model(p) for p in packets] @@ -1678,6 +1703,22 @@ async def api_stats(request): return web.json_response(stats) +@routes.get("/api/stats/summary") +async def api_stats_summary(request): + channel = request.query.get("channel") or None + total_packets = await store.get_total_packet_count(channel=channel) + total_nodes = await store.get_total_node_count(channel=channel) + total_packets_seen = await store.get_total_packet_seen_count(channel=channel) + + return web.json_response( + { + "total_packets": total_packets, + "total_nodes": total_nodes, + "total_packets_seen": total_packets_seen, + } + ) + + @routes.get("/api/config") async def api_config(request): try: