diff --git a/alembic/versions/4f1d2a9c8b71_add_daily_snapshot_table.py b/alembic/versions/4f1d2a9c8b71_add_daily_snapshot_table.py new file mode 100644 index 0000000..7ff18c4 --- /dev/null +++ b/alembic/versions/4f1d2a9c8b71_add_daily_snapshot_table.py @@ -0,0 +1,35 @@ +"""Add daily_snapshot table + +Revision ID: 4f1d2a9c8b71 +Revises: 23dad03d2e42 +Create Date: 2026-03-05 00:00:00.000000 + +""" + +from collections.abc import Sequence + +import sqlalchemy as sa + +from alembic import op + +# revision identifiers, used by Alembic. +revision: str = "4f1d2a9c8b71" +down_revision: str | None = "23dad03d2e42" +branch_labels: str | Sequence[str] | None = None +depends_on: str | Sequence[str] | None = None + + +def upgrade() -> None: + op.create_table( + "daily_snapshot", + sa.Column("snapshot_date", sa.Date(), nullable=False), + sa.Column("node_count", sa.BigInteger(), nullable=False), + sa.Column("packet_count", sa.BigInteger(), nullable=False), + sa.Column("gateway_count", sa.BigInteger(), nullable=False), + sa.Column("captured_at_us", sa.BigInteger(), nullable=False), + sa.PrimaryKeyConstraint("snapshot_date"), + ) + + +def downgrade() -> None: + op.drop_table("daily_snapshot") diff --git a/meshview/lang/en.json b/meshview/lang/en.json index 302fe7d..71d95e6 100644 --- a/meshview/lang/en.json +++ b/meshview/lang/en.json @@ -127,6 +127,7 @@ "total_gateways": "Total Gateways", "total_packets": "Total Packets", "total_packets_seen": "Total Packets Seen", + "daily_snapshot_histogram": "Daily Network Snapshot Histogram (Last 30 Days)", "packets_per_day_all": "Packets per Day - All Ports (Last 14 Days)", "packets_per_day_text": "Packets per Day - Text Messages (Port 1, Last 14 Days)", "packets_per_hour_all": "Packets per Hour - All Ports", diff --git a/meshview/lang/es.json b/meshview/lang/es.json index 573d2a4..0c33e9a 100644 --- a/meshview/lang/es.json +++ b/meshview/lang/es.json @@ -123,6 +123,7 @@ "total_gateways": "Gateways Totales", "total_packets": "Paquetes Totales", "total_packets_seen": "Paquetes Totales Vistos", + "daily_snapshot_histogram": "Histograma Diario de Instantáneas de Red (Últimos 30 Días)", "packets_per_day_all": "Paquetes por Día - Todos los Puertos (Últimos 14 Días)", "packets_per_day_text": "Paquetes por Día - Mensajes de Texto (Puerto 1, Últimos 14 Días)", "packets_per_hour_all": "Paquetes por Hora - Todos los Puertos", diff --git a/meshview/models.py b/meshview/models.py index b6a2ffb..21c1ab3 100644 --- a/meshview/models.py +++ b/meshview/models.py @@ -1,4 +1,6 @@ -from sqlalchemy import BigInteger, ForeignKey, Index, desc +from datetime import date + +from sqlalchemy import BigInteger, Date, ForeignKey, Index, desc from sqlalchemy.ext.asyncio import AsyncAttrs from sqlalchemy.orm import DeclarativeBase, Mapped, mapped_column, relationship @@ -120,3 +122,13 @@ class NodePublicKey(Base): Index("idx_node_public_key_node_id", "node_id"), Index("idx_node_public_key_public_key", "public_key"), ) + + +class DailySnapshot(Base): + __tablename__ = "daily_snapshot" + + snapshot_date: Mapped[date] = mapped_column(Date, primary_key=True) + node_count: Mapped[int] = mapped_column(BigInteger, nullable=False) + packet_count: Mapped[int] = mapped_column(BigInteger, nullable=False) + gateway_count: Mapped[int] = mapped_column(BigInteger, nullable=False) + captured_at_us: Mapped[int] = mapped_column(BigInteger, nullable=False) diff --git a/meshview/mqtt_store.py b/meshview/mqtt_store.py index 341e0f6..03f1ad2 100644 --- a/meshview/mqtt_store.py +++ b/meshview/mqtt_store.py @@ -1,8 +1,9 @@ import logging import re import time +from datetime import UTC, datetime -from sqlalchemy import select, update +from sqlalchemy import func, select, update from sqlalchemy.dialects.postgresql import insert as pg_insert from sqlalchemy.dialects.sqlite import insert as sqlite_insert from sqlalchemy.exc import IntegrityError @@ -11,13 +12,69 @@ from meshtastic.protobuf.config_pb2 import Config from meshtastic.protobuf.mesh_pb2 import HardwareModel from meshtastic.protobuf.portnums_pb2 import PortNum from meshview import decode_payload, mqtt_database -from meshview.models import Node, NodePublicKey, Packet, PacketSeen, Traceroute +from meshview.models import DailySnapshot, Node, NodePublicKey, Packet, PacketSeen, Traceroute logger = logging.getLogger(__name__) MQTT_GATEWAY_CACHE: set[int] = set() +async def capture_daily_snapshot() -> None: + today = datetime.now(UTC).date() + + async with mqtt_database.async_session() as session: + node_count = (await session.execute(select(func.count()).select_from(Node))).scalar_one() + packet_count = ( + await session.execute(select(func.count()).select_from(Packet)) + ).scalar_one() + gateway_count = ( + await session.execute( + select(func.count()).select_from(Node).where(Node.is_mqtt_gateway.is_(True)) + ) + ).scalar_one() + captured_at_us = int(time.time() * 1_000_000) + values = { + "snapshot_date": today, + "node_count": node_count, + "packet_count": packet_count, + "gateway_count": gateway_count, + "captured_at_us": captured_at_us, + } + + dialect = session.get_bind().dialect.name + stmt = None + if dialect == "sqlite": + stmt = ( + sqlite_insert(DailySnapshot) + .values(**values) + .on_conflict_do_update(index_elements=["snapshot_date"], set_=values) + ) + elif dialect == "postgresql": + stmt = ( + pg_insert(DailySnapshot) + .values(**values) + .on_conflict_do_update(index_elements=["snapshot_date"], set_=values) + ) + + if stmt is not None: + await session.execute(stmt) + else: + snapshot = ( + await session.execute( + select(DailySnapshot).where(DailySnapshot.snapshot_date == today) + ) + ).scalar_one_or_none() + if snapshot is None: + session.add(DailySnapshot(**values)) + else: + snapshot.node_count = node_count + snapshot.packet_count = packet_count + snapshot.gateway_count = gateway_count + snapshot.captured_at_us = captured_at_us + + await session.commit() + + async def process_envelope(topic, env): # MAP_REPORT_APP if env.packet.decoded.portnum == PortNum.MAP_REPORT_APP: diff --git a/meshview/templates/stats.html b/meshview/templates/stats.html index ff072cb..74c7c0f 100644 --- a/meshview/templates/stats.html +++ b/meshview/templates/stats.html @@ -118,6 +118,15 @@ +
+

+ Daily Network Snapshot Histogram (Last 30 Days) +

+ + +
+
+

@@ -247,6 +256,15 @@ async function fetchStats(period_type,length,portnum=null,channel=null){ }catch{return [];} } +async function fetchDailySnapshots(length=30){ + try{ + const res=await fetch(`/api/snapshots/daily?length=${length}`); + if(!res.ok) return []; + const json=await res.json(); + return json.data||[]; + }catch{return [];} +} + async function fetchNodes(){ try{ const res=await fetch("/api/nodes"); @@ -365,6 +383,42 @@ function renderPieChart(elId,data,name){ return chart; } +function renderDailySnapshotChart(domId,data){ + const el=document.getElementById(domId); + if(!el) return; + const chart=echarts.init(el); + const labels=data.map(d=>d.date||""); + const nodeCounts=data.map(d=>d.node_count??0); + const packetCounts=data.map(d=>d.packet_count??0); + const gatewayCounts=data.map(d=>d.gateway_count??0); + chart.setOption({ + backgroundColor:'#272b2f', + tooltip:{trigger:'axis'}, + legend:{ + data:['Nodes','Packets','Gateways'], + textStyle:{color:'#ccc'} + }, + grid:{left:'6%', right:'6%', bottom:'18%'}, + xAxis:{ + type:'category', + data:labels, + axisLine:{lineStyle:{color:'#aaa'}}, + axisLabel:{rotate:45,color:'#ccc'} + }, + yAxis:{ + type:'value', + axisLine:{lineStyle:{color:'#aaa'}}, + axisLabel:{color:'#ccc'} + }, + series:[ + {name:'Nodes', type:'bar', data:nodeCounts, itemStyle:{color:'#5e81ac'}}, + {name:'Packets', type:'bar', data:packetCounts, itemStyle:{color:'#88c0d0'}}, + {name:'Gateways', type:'bar', data:gatewayCounts, itemStyle:{color:'#a3be8c'}} + ] + }); + return chart; +} + // --- Packet Type Pie Chart --- async function fetchPacketTypeBreakdown(channel=null) { @@ -388,6 +442,7 @@ async function fetchPacketTypeBreakdown(channel=null) { // --- Init --- let chartHourlyAll, chartPortnum1, chartPortnum3, chartPortnum4, chartPortnum67, chartPortnum70, chartPortnum71; let chartDailyAll, chartDailyPortnum1; +let chartDailySnapshot; let chartHwModel, chartRole, chartChannel; let chartGatewayChannel, chartGatewayRole, chartGatewayFirmware; let chartPacketTypes; @@ -403,6 +458,10 @@ async function init(){ select.appendChild(opt); }); + // Daily snapshot histogram + const snapshots = await fetchDailySnapshots(30); + chartDailySnapshot = renderDailySnapshotChart("chart_daily_snapshot", snapshots); + // Daily all ports const dailyAllData=await fetchStats('day',14); updateTotalCount('total_daily_all',dailyAllData); @@ -502,6 +561,7 @@ window.addEventListener('resize',()=>{ chartPortnum67, chartPortnum70, chartPortnum71, + chartDailySnapshot, chartDailyAll, chartDailyPortnum1, chartHwModel, @@ -554,10 +614,23 @@ document.querySelectorAll(".export-btn").forEach(btn=>{ const option=chart.getOption(); let rows=[]; if(option.series[0].type==="bar"||option.series[0].type==="line"){ - rows.push(["Period","Count"]); const xData=option.xAxis[0].data; - const yData=option.series[0].data; - for(let i=0;i 1){ + const headers = ["Period", ...series.map(s => s.name || "Series")]; + rows.push(headers); + for(let i=0;i{ + row.push((s.data && s.data[i]!==undefined) ? s.data[i] : ""); + }); + rows.push(row); + } + }else{ + rows.push(["Period","Count"]); + const yData=series[0].data; + for(let i=0;i