Merge branch 'pablorevilla-meshtastic:master' into master

This commit is contained in:
madeofstown
2025-02-16 17:09:07 -08:00
committed by GitHub
20 changed files with 655 additions and 213 deletions

2
README
View File

@@ -1,6 +1,8 @@
Meshview
========
Now running at https://meshview.bayme.sh
This project watches a MQTT topic for meshtastic messages, imports them to a
database and has a web UI to view them.
Requires Python 3.12

BIN
images/main.png Normal file

Binary file not shown.

After

Width:  |  Height:  |  Size: 32 KiB

View File

@@ -1,20 +1,16 @@
from sqlalchemy.ext.asyncio import async_sessionmaker
from sqlalchemy.ext.asyncio import AsyncSession
from sqlalchemy.ext.asyncio import create_async_engine
from meshview import models
from sqlalchemy.ext.asyncio import create_async_engine, async_sessionmaker
def init_database(database_connetion_string):
def init_database(database_connection_string):
global engine, async_session
kwargs = {}
if not database_connetion_string.startswith('sqlite'):
if not database_connection_string.startswith('sqlite'):
kwargs['pool_size'] = 20
kwargs['max_overflow'] = 50
engine = create_async_engine(database_connetion_string, echo=False, **kwargs)
print (**kwargs)
engine = create_async_engine(database_connection_string, echo=False, connect_args={"timeout": 15})
async_session = async_sessionmaker(engine, expire_on_commit=False)
async def create_tables():
async with engine.begin() as conn:
await conn.run_sync(models.Base.metadata.create_all)

View File

@@ -1,3 +1,4 @@
from meshtastic.protobuf.mqtt_pb2 import MapReport
from meshtastic.protobuf.portnums_pb2 import PortNum
from meshtastic.protobuf.mesh_pb2 import (
Position,
@@ -24,6 +25,7 @@ DECODE_MAP = {
PortNum.TRACEROUTE_APP: RouteDiscovery.FromString,
PortNum.ROUTING_APP: Routing.FromString,
PortNum.TEXT_MESSAGE_APP: text_message,
PortNum.MAP_REPORT_APP: MapReport.FromString
}

View File

@@ -14,30 +14,31 @@ class Node(Base):
__tablename__ = "node"
id: Mapped[str] = mapped_column(primary_key=True)
node_id: Mapped[int] = mapped_column(BigInteger, nullable=True, unique=True)
long_name: Mapped[str]
short_name: Mapped[str]
hw_model: Mapped[str]
long_name: Mapped[str] = mapped_column(nullable=True)
short_name: Mapped[str] = mapped_column(nullable=True)
hw_model: Mapped[str] = mapped_column(nullable=True)
firmware: Mapped[str] = mapped_column(nullable=True)
role: Mapped[str] = mapped_column(nullable=True)
last_lat: Mapped[int] = mapped_column(BigInteger, nullable=True)
last_long: Mapped[int] = mapped_column(BigInteger, nullable=True)
channel: Mapped[str]
channel: Mapped[str] = mapped_column(nullable=True)
last_update: Mapped[datetime] = mapped_column(nullable=True)
class Packet(Base):
__tablename__ = "packet"
id: Mapped[int] = mapped_column(BigInteger, primary_key=True)
portnum: Mapped[int]
from_node_id: Mapped[int] = mapped_column(BigInteger)
portnum: Mapped[int] = mapped_column(nullable=True)
from_node_id: Mapped[int] = mapped_column(BigInteger, nullable=True)
from_node: Mapped["Node"] = relationship(
primaryjoin="Packet.from_node_id == foreign(Node.node_id)", lazy="joined"
)
to_node_id: Mapped[int] = mapped_column(BigInteger)
to_node_id: Mapped[int] = mapped_column(BigInteger,nullable=True)
to_node: Mapped["Node"] = relationship(
primaryjoin="Packet.to_node_id == foreign(Node.node_id)", lazy="joined"
)
payload: Mapped[bytes]
import_time: Mapped[datetime]
channel: Mapped[str]
payload: Mapped[bytes] = mapped_column(nullable=True)
import_time: Mapped[datetime] = mapped_column(nullable=True)
channel: Mapped[str] = mapped_column(nullable=True)
class PacketSeen(Base):
@@ -48,13 +49,13 @@ class PacketSeen(Base):
lazy="joined", primaryjoin="PacketSeen.node_id == foreign(Node.node_id)"
)
rx_time: Mapped[int] = mapped_column(BigInteger, primary_key=True)
hop_limit: Mapped[int]
hop_limit: Mapped[int] = mapped_column(nullable=True)
hop_start: Mapped[int] = mapped_column(nullable=True)
channel: Mapped[str]
channel: Mapped[str] = mapped_column(nullable=True)
rx_snr: Mapped[float] = mapped_column(nullable=True)
rx_rssi: Mapped[int] = mapped_column(nullable=True)
topic: Mapped[str]
import_time: Mapped[datetime]
topic: Mapped[str] = mapped_column(nullable=True)
import_time: Mapped[datetime] = mapped_column(nullable=True)
class Traceroute(Base):
@@ -64,8 +65,8 @@ class Traceroute(Base):
packet: Mapped["Packet"] = relationship(
primaryjoin="Traceroute.packet_id == foreign(Packet.id)", lazy="joined"
)
gateway_node_id: Mapped[int] = mapped_column(BigInteger)
done: Mapped[bool]
route: Mapped[bytes]
import_time: Mapped[datetime]
gateway_node_id: Mapped[int] = mapped_column(BigInteger, nullable=True)
done: Mapped[bool] = mapped_column(nullable=True)
route: Mapped[bytes] = mapped_column(nullable=True)
import_time: Mapped[datetime] = mapped_column(nullable=True)

View File

@@ -1,11 +1,9 @@
import base64
import asyncio
import random
import aiomqtt
from google.protobuf.message import DecodeError
from cryptography.hazmat.primitives.ciphers import Cipher, algorithms, modes
from meshtastic.protobuf.mqtt_pb2 import ServiceEnvelope
KEY = base64.b64decode("1PG7OiApB1nwvP+rz05pAQ==")

View File

@@ -34,16 +34,13 @@ def create_event(node_id):
def remove_event(node_event):
print("removing event")
waiting_node_ids_events[node_event.node_id].remove(node_event)
def notify_packet(node_id, packet):
for event in waiting_node_ids_events[node_id]:
event.packets.append(packet)
event.set()
def notify_uplinked(node_id, packet):
for event in waiting_node_ids_events[node_id]:
event.uplinked.append(packet)
@@ -52,8 +49,15 @@ def notify_uplinked(node_id, packet):
@contextlib.contextmanager
def subscribe(node_id):
"""
Context manager for subscribing to events for a node_id.
Automatically manages event creation and cleanup.
"""
event = create_event(node_id)
try:
yield event
except Exception as e:
print(f"Error during subscription for node_id={node_id}: {e}")
raise
finally:
remove_event(event)

View File

@@ -2,7 +2,7 @@ import datetime
from sqlalchemy import select, func
from sqlalchemy.orm import lazyload
from sqlalchemy import update
from meshtastic.protobuf.config_pb2 import Config
from meshtastic.protobuf.portnums_pb2 import PortNum
from meshtastic.protobuf.mesh_pb2 import User, HardwareModel
@@ -12,7 +12,34 @@ from meshview.models import Packet, PacketSeen, Node, Traceroute
from meshview import notify
async def process_envelope(topic, env):
# Checking if the received packet is a MAP_REPORT
# Update the node table with the firmware version
if env.packet.decoded.portnum == PortNum.MAP_REPORT_APP:
# Extract the node ID from the packet (renamed from 'id' to 'node_id' to avoid conflicts with Python's built-in id function)
node_id = getattr(env.packet, "from")
# Decode the MAP report payload to extract the firmware version
map_report = decode_payload.decode_payload(PortNum.MAP_REPORT_APP, env.packet.decoded.payload)
# Establish an asynchronous database session
async with database.async_session() as session:
# Construct an SQLAlchemy update statement
stmt = (
update(Node)
.where(Node.node_id == node_id) # Ensure correct column reference
.values(firmware=map_report.firmware_version) # Assign new firmware value
)
# Execute the update statement asynchronously
await session.execute(stmt)
# Commit the changes to the database
await session.commit()
# This ignores any packet that does not have a ID
if not env.packet.id:
return
@@ -58,6 +85,8 @@ async def process_envelope(topic, env):
)
session.add(seen)
if env.packet.decoded.portnum == PortNum.NODEINFO_APP:
user = decode_payload.decode_payload(
PortNum.NODEINFO_APP, env.packet.decoded.payload
@@ -88,7 +117,7 @@ async def process_envelope(topic, env):
node.short_name = user.short_name
node.hw_model = hw_model
node.role = role
# if need to update time of last update it may be here
node.last_update =datetime.datetime.now()
else:
node = Node(
@@ -415,6 +444,7 @@ async def get_total_packet_count():
async def get_total_node_count():
async with database.async_session() as session:
q = select(func.count(Node.id)) # Use SQLAlchemy's func to count nodes
q = q.where(Node.last_update > datetime.datetime.now() - datetime.timedelta(days=1)) # Look for nodes with nodeinfo updates in the last 24 hours
result = await session.execute(q)
return result.scalar() # Return the total count of nodes
@@ -427,24 +457,13 @@ async def get_total_packet_seen_count():
async def get_total_node_count_longfast() -> int:
"""
Retrieves the total count of nodes where the channel is equal to 'LongFast'.
This function queries the database asynchronously to count the number of nodes
in the `Node` table that meet the condition `channel == 'LongFast'`. It uses
SQLAlchemy's asynchronous session management and query construction.
Returns:
int: The total count of nodes with `channel == 'LongFast'`.
Raises:
Exception: If an error occurs during the database query execution.
"""
try:
# Open an asynchronous session with the database
async with database.async_session() as session:
# Build the query to count nodes where channel == 'LongFast'
q = select(func.count(Node.id)).filter(Node.channel == 'LongFast')
q = select(func.count(Node.id))
q = q.where(Node.last_update > datetime.datetime.now() - datetime.timedelta( days=1)) # Look for nodes with nodeinfo updates in the last 24 hours
q = q.where(Node.channel == 'LongFast') #
# Execute the query asynchronously and fetch the result
result = await session.execute(q)
@@ -458,25 +477,14 @@ async def get_total_node_count_longfast() -> int:
async def get_total_node_count_mediumslow() -> int:
"""
Retrieves the total count of nodes where the channel is equal to 'MediumSlow'.
This function queries the database asynchronously to count the number of nodes
in the `Node` table that meet the condition `channel == 'MediumSlow'`. It uses
SQLAlchemy's asynchronous session management and query construction.
Returns:
int: The total count of nodes with `channel == 'MediumSlow'`.
Raises:
Exception: If an error occurs during the database query execution.
"""
try:
# Open an asynchronous session with the database
async with database.async_session() as session:
# Build the query to count nodes where channel == 'LongFast'
q = select(func.count(Node.id)).filter(Node.channel == 'MediumSlow')
q = select(func.count(Node.id))
q = q.where(Node.last_update > datetime.datetime.now() - datetime.timedelta(
days=1)) # Look for nodes with nodeinfo updates in the last 24 hours
q = q.where(Node.channel == 'MediumSlow') #
# Execute the query asynchronously and fetch the result
result = await session.execute(q)
@@ -498,6 +506,51 @@ async def get_nodes_mediumslow():
(Node.channel == "MediumSlow")
)
)
return result.scalars()
async def get_nodes(role=None, channel=None, hw_model=None):
"""
Fetches nodes from the database based on optional filtering criteria.
Parameters:
role (str, optional): The role of the node (converted to uppercase for consistency).
channel (str, optional): The communication channel associated with the node.
hw_model (str, optional): The hardware model of the node.
Returns:
list: A list of Node objects that match the given criteria.
"""
try:
async with database.async_session() as session:
#print(channel) # Debugging output (consider replacing with logging)
# Start with a base query selecting all nodes
query = select(Node)
# Apply filters based on provided parameters
if role is not None:
query = query.where(Node.role == role.upper()) # Ensure role is uppercase
if channel is not None:
query = query.where(Node.channel == channel)
if hw_model is not None:
query = query.where(Node.hw_model == hw_model)
# Exclude nodes where last_update is an empty string
query = query.where(Node.last_update != "")
# Order results by long_name in ascending order
query = query.order_by(Node.long_name.asc())
# Execute the query and retrieve results
result = await session.execute(query)
nodes = result.scalars().all()
return nodes # Return the list of nodes
except Exception as e:
print("error reading DB") # Consider using logging instead of print
return [] # Return an empty list in case of failure

View File

@@ -34,7 +34,9 @@
</head>
<body hx-indicator="#spinner">
<br><div style="text-align:center"><strong>Bay Area Mesh - http://bayme.sh</strong></div>
<div style="text-align:center">Quick Links:&nbsp;&nbsp;<a href="/">Search for a node </a>&nbsp;-&nbsp;<a href="/chat">Conversations</a>&nbsp;-&nbsp;<a href="/firehose">See <strong>everything</strong> </a>&nbsp;-&nbsp;Mesh Graph <a href="/graph/longfast">LG</a>&nbsp;-&nbsp;<a href="/graph/mediumslow">MS</a>&nbsp;-&nbsp;<a href="/stats">Stats </a></div><br>
<div style="text-align:center">Quick Links:&nbsp;&nbsp;<a href="/nodelist">Nodes</a>&nbsp;-&nbsp;<a href="/chat">Conversations</a>&nbsp;-&nbsp;<a href="/firehose">See <strong>everything</strong> </a>
&nbsp;-&nbsp;Mesh Graph <a href="/graph/longfast">LF</a>&nbsp;-&nbsp;<a href="/graph/mediumslow">MS </a>&nbsp;-&nbsp;<a href="/stats">Stats </a>
&nbsp;-&nbsp;<a href="/net">Weekly Net</a></div><br>
<div id="spinner" class="spinner-border secondary-primary htmx-indicator position-absolute top-50 start-50" role="status">
<span class="visually-hidden">Loading...</span>
</div>

View File

@@ -5,11 +5,17 @@
min-width:10em;
}
.chat-packet:nth-of-type(odd){
background-color:#1f1f1f;
background-color: #3a3a3a; /* Lighter than #2a2a2a */
}
.chat-packet {
border-bottom: 1px solid #555;
padding: 8px;
border-radius: 8px; /* Adjust the value to make the corners more or less rounded */
}
.chat-packet:nth-of-type(even){
background-color:#181818;
background-color: #333333; /* Slightly lighter than the previous #181818 */
}
{% endblock %}

View File

@@ -1,5 +1,6 @@
<div class="row chat-packet">
<span class="col-3 timestamp">{{packet.import_time | format_timestamp}} <a href="/packet/{{packet.id}}">✉️</a></span>
<span class="col-3 username"><a href="/packet_list/{{packet.from_node_id}}">{{packet.from_node.long_name or (packet.from_node_id | node_id_to_hex) }}</a></span>
<span class="col message">{{packet.payload}}</span>
</div>
<span class="col-2 timestamp">{{packet.import_time.strftime('%-I:%M:%S %p - %d-%m-%Y')}} </span>
<span class="col-1 timestamp"><a href="/packet/{{packet.id}}">✉️</a> {{packet.from_node.channel}}</span>
<span class="col-2 username"><a href="/packet_list/{{packet.from_node_id}}">{{packet.from_node.long_name or (packet.from_node_id | node_id_to_hex) }}</a></span>
<span class="col-6 message">{{packet.payload}}</span>
</div>

View File

@@ -23,7 +23,7 @@
>{{ name }}</option>
{% endfor %}
</select>
<input type="submit" class="col-2 m-2"/>
<input type="Submit" value="Refresh" class="col-2 m-2"/>
</form>
<div class="row">
<div class="col-xs" id="packet_list" sse-swap="packet" hx-swap="afterbegin">

View File

@@ -1,27 +1,34 @@
{% extends "base.html" %}
{% block css %}
#packet_details{
height: 95vh;
overflow: scroll;
}
.timestamp {
min-width:10em;
}
.chat-packet:nth-of-type(odd){
background-color: #3a3a3a; /* Lighter than #2a2a2a */
}
.chat-packet {
border-bottom: 1px solid #555;
padding: 8px;
border-radius: 8px; /* Adjust the value to make the corners more or less rounded */
}
.chat-packet:nth-of-type(even){
background-color: #333333; /* Slightly lighter than the previous #181818 */
}
{% endblock %}
{% block body %}
<div
id="net"
class="container text-center"
hx-ext="sse"
sse-connect="/events?portnum=1"
>
<div class="row">
<div class="col-6" id="packet_list" sse-swap="net_packet" hx-swap="afterbegin">
{% for packet in net_packets %}
{% include 'net_packet.html' %}
{% endfor %}
</div>
<div class="col-6 sticky-top" id="packet_details">
</div>
<div class="container" > Weekly Mesh check-in. We will keep it open on every Wednesday from 5:00pm for checkins so you do not have to rush.<br>
The message format should be (LONG NAME) - (CITY YOU ARE IN) #BayMeshNet.<br><br>
</div>
<div class="container" hx-ext="sse" sse-connect="/net_events" sse-swap="net_packet" hx-swap="afterbegin">
{% for packet in packets %}
{% include 'chat_packet.html' %}
{% else %}
No packets found.
{% endfor %}
</div>
</div>
{% endblock body %}
{% endblock %}

View File

@@ -1,12 +1,6 @@
<div class="card mt-2">
<div class="card-header">
<span class="fw-bold">{{packet.from_node.long_name}}</span>
<a href="/packet_details/{{packet.id}}" hx-trigger="click" hx-target="#packet_details" hx-get="/packet_details/{{packet.id}}" hx-swap="innerHTML scroll:top">🔎</a>
</div>
<div class="card-text text-start">
<dl>
<div>{{packet.import_time | format_timestamp}}</div>
<div>{{packet.payload}}</div>
</dl>
</div>
</div>
<div class="row chat-packet">
<span class="col-2 timestamp">{{packet.import_time.strftime('%-I:%M:%S %p - %d-%m-%Y')}} </span>
<span class="col-1 timestamp"><a href="/packet/{{packet.id}}">✉️</a> {{packet.from_node.channel}}</span>
<span class="col-2 username"><a href="/packet_list/{{packet.from_node_id}}">{{packet.from_node.long_name or (packet.from_node_id | node_id_to_hex) }}</a></span>
<span class="col-6 message">{{packet.payload}}</span>
</div>

View File

@@ -61,7 +61,7 @@
<div class="row">
<div class="col">
{% include "buttons.html" %}
<!-- {% include "buttons.html" %}-->
</div>
</div>

View File

@@ -0,0 +1,206 @@
{% extends "base.html" %}
{% block css %}
table {
width: 100%;
border-collapse: collapse;
margin-top: 1em;
}
th, td {
padding: 10px;
border: 1px solid #333;
text-align: left;
}
th {
background-color: #1f1f1f;
color: white;
cursor: pointer;
}
tr:nth-child(even) {
background-color: #181818;
}
tr:nth-child(odd) {
background-color: #222;
}
.search-container {
display: flex;
gap: 10px;
margin-bottom: 10px;
}
.search, .filter-role, .filter-channel, .filter-hw_model, .export-btn {
padding: 8px;
border: 1px solid #333;
border-radius: 4px;
}
.filter-role, .filter-channel, .filter-hw_model {
cursor: pointer;
}
.export-btn {
background: #28a745;
color: white;
border: none;
cursor: pointer;
}
.export-btn:hover {
background: #218838;
}
.count-container {
margin-bottom: 10px;
font-weight: bold;
color: white;
}
{% endblock %}
{% block body %}
<div id="node-list">
<div class="search-container">
<input class="search" placeholder="Search nodes..." />
<!-- Filter by Role -->
<select class="filter-role" onchange="applyFilters()">
<option value="">Filter by Role</option>
{% for node in nodes|groupby('role') %}
<option value="{{ node.grouper }}">{{ node.grouper }}</option>
{% endfor %}
</select>
<!-- Filter by Channel -->
<select class="filter-channel" onchange="applyFilters()">
<option value="">Filter by Channel</option>
{% for node in nodes|groupby('channel') %}
<option value="{{ node.grouper }}">{{ node.grouper }}</option>
{% endfor %}
</select>
<!-- Filter by HW Model -->
<select class="filter-hw_model" onchange="applyFilters()">
<option value="">Filter by HW Model</option>
{% for node in nodes|groupby('hw_model') %}
<option value="{{ node.grouper }}">{{ node.grouper }}</option>
{% endfor %}
</select>
<button class="export-btn" onclick="exportToCSV()">Export to CSV</button>
</div>
<!-- Count Display -->
<div class="count-container">Showing <span id="node-count-value">0</span> nodes</div>
{% if nodes %}
<table id="node-table">
<thead>
<tr>
<th class="sort" data-sort="long_name">Long Name</th>
<th class="sort" data-sort="short_name">Short Name</th>
<th class="sort" data-sort="hw_model">HW Model</th>
<th class="sort" data-sort="firmware">Firmware</th>
<th class="sort" data-sort="role">Role</th>
<th class="sort" data-sort="last_lat">Last Latitude</th>
<th class="sort" data-sort="last_long">Last Longitude</th>
<th class="sort" data-sort="channel">Channel</th>
<th class="sort" data-sort="last_update" data-order="desc">Last Update</th>
</tr>
</thead>
<tbody class="list">
{% for node in nodes %}
<tr>
<td class="long_name"> <a href="/packet_list/{{ node.node_id }}">{{ node.long_name }}</a></td>
<td class="short_name">{{ node.short_name }}</td>
<td class="hw_model">{{ node.hw_model if node.hw_model else "N/A" }}</td>
<td class="firmware">{{ node.firmware }}</td>
<td class="role">{{ node.role if node.role else "N/A" }}</td>
<td class="last_lat">{{ "{:.7f}".format(node.last_lat / 10**7) if node.last_lat else "N/A" }}</td>
<td class="last_long">{{ "{:.7f}".format(node.last_long / 10**7) if node.last_long else "N/A" }}</td>
<td class="channel">{{ node.channel if node.channel else "N/A" }}</td>
<td class="last_update" data-timestamp="{{ node.last_update.timestamp() if node.last_update else 0 }}">
{{ node.last_update.strftime('%-I:%M:%S %p - %m-%d-%Y') if node.last_update else "N/A" }}
</td>
</tr>
{% endfor %}
</tbody>
</table>
{% else %}
<p>No nodes found.</p>
{% endif %}
</div>
<script src="https://cdnjs.cloudflare.com/ajax/libs/list.js/2.3.1/list.min.js"></script>
<script>
var nodeList;
document.addEventListener("DOMContentLoaded", function () {
var options = {
valueNames: [
"long_name", "short_name", "hw_model", "firmware", "role",
"last_lat", "last_long", "channel", { name: "last_update", attr: "data-timestamp" }
]
};
nodeList = new List("node-list", options);
updateCount(); // Update count on load
nodeList.on("updated", function () {
updateCount(); // Update count when search or sort changes
});
});
function applyFilters() {
var selectedRole = document.querySelector(".filter-role").value;
var selectedChannel = document.querySelector(".filter-channel").value;
var selectedHWModel = document.querySelector(".filter-hw_model").value;
nodeList.filter(function (item) {
var matchesRole = selectedRole === "" || item.values().role === selectedRole;
var matchesChannel = selectedChannel === "" || item.values().channel === selectedChannel;
var matchesHWModel = selectedHWModel === "" || item.values().hw_model === selectedHWModel;
return matchesRole && matchesChannel && matchesHWModel;
});
updateCount(); // Update the count after filtering
}
function updateCount() {
var visibleRows = document.querySelectorAll("#node-table tbody tr:not([style*='display: none'])").length;
document.getElementById("node-count-value").innerText = visibleRows;
}
function exportToCSV() {
var table = document.getElementById("node-table");
var rows = table.querySelectorAll("tr");
var csvContent = [];
// Extract header row
var headers = [];
table.querySelectorAll("th").forEach(th => headers.push(th.innerText));
csvContent.push(headers.join(","));
// Extract table rows
rows.forEach(row => {
var cells = row.querySelectorAll("td");
if (cells.length > 0) {
var rowData = [];
cells.forEach(cell => rowData.push(cell.innerText));
csvContent.push(rowData.join(","));
}
});
// Create CSV file and trigger download
var csvString = csvContent.join("\n");
var blob = new Blob([csvString], { type: "text/csv" });
var a = document.createElement("a");
a.href = URL.createObjectURL(blob);
a.download = "nodes_list.csv";
a.click();
}
</script>
{% endblock %}

View File

@@ -34,8 +34,8 @@
</div>
<div class="card-text text-start">
<dl>
<dt>import_time</dt>
<dd>{{packet.import_time | format_timestamp}}</dd>
<dt>Import Time</dt>
<dd>{{packet.import_time.strftime('%-I:%M:%S %p - %d-%m-%Y')}}</dd>
<dt>packet</dt>
<dd><pre>{{packet.data}}</pre></dd>
<dt>payload</dt>

View File

@@ -12,8 +12,8 @@
<div class="card-body">
<div class="card-text text-start">
<dl>
<dt>import_time</dt>
<dd>{{seen.import_time|format_timestamp}}</dd>
<dt>Import Time</dt>
<dd>{{seen.import_time.strftime('%-I:%M:%S %p - %d-%m-%Y')}}</dd>
<dt>rx_time</dt>
<dd>{{seen.rx_time|format_timestamp}}</dd>
<dt>hop_limit</dt>

View File

@@ -16,14 +16,14 @@
<!-- Section for Total Nodes -->
<div style="background: rgba(255, 255, 255, 0.1); padding: 15px; border-radius: 10px; margin-bottom: 15px;">
<p style="font-size: 16px; margin: 0; font-weight: 500;">
Total Nodes:
Total Active Nodes (Last 24 hours):
<span style="font-weight: 700; color: #ffeb3b;">{{ total_nodes }}</span>
</p>
</div>
<!-- Section for Total Nodes LongFast -->
<div style="background: rgba(255, 255, 255, 0.1); padding: 15px; border-radius: 10px; margin-bottom: 10px;">
<p style="font-size: 14px; margin: 0; font-weight: 500;">
Total Nodes LongFast:
Total Active Nodes LongFast: <br>
<span style="font-weight: 700; color: #03dac6;">{{ total_nodes_longfast }}</span>
<span style="font-size: 12px; color: #ffeb3b; font-weight: 400;">
({{ (total_nodes_longfast / total_nodes * 100) | round(2) }}%)
@@ -33,7 +33,7 @@
<!-- Section for Total Nodes MediumSlow -->
<div style="background: rgba(255, 255, 255, 0.1); padding: 15px; border-radius: 10px; margin-bottom: 10px;">
<p style="font-size: 14px; margin: 0; font-weight: 500;">
Total Nodes MediumSlow:
Total Active Nodes MediumSlow: <br>
<span style="font-weight: 700; color: #03dac6;">{{ total_nodes_mediumslow }}</span>
<span style="font-size: 12px; color: #ffeb3b; font-weight: 400;">
({{ (total_nodes_mediumslow / total_nodes * 100) | round(2) }}%)

View File

@@ -307,71 +307,121 @@ async def _packet_list(request, raw_packets, packet_event):
@routes.get("/chat_events")
async def chat_events(request):
"""
Server-Sent Events (SSE) endpoint for real-time chat packet updates.
This endpoint listens for new chat packets related to `PortNum.TEXT_MESSAGE_APP`
and streams them to connected clients. Messages matching the pattern `"seq \d+$"`
are filtered out before sending.
Args:
request (aiohttp.web.Request): The incoming HTTP request.
Returns:
aiohttp.web.StreamResponse: SSE response streaming chat events.
"""
chat_packet = env.get_template("chat_packet.html")
# Precompile regex for filtering out unwanted messages (case insensitive)
seq_pattern = re.compile(r"seq \d+$", re.IGNORECASE)
# Subscribe to notifications for packets from all nodes (0xFFFFFFFF = broadcast)
with notify.subscribe(node_id=0xFFFFFFFF) as event:
async with sse_response(request) as resp:
while resp.is_connected():
while resp.is_connected(): # Keep the connection open while the client is connected
try:
async with asyncio.timeout(10):
await event.wait()
except TimeoutError:
# Wait for an event with a timeout of 10 seconds
await asyncio.wait_for(event.wait(), timeout=10)
except asyncio.TimeoutError:
# Timeout reached, continue looping to keep connection alive
continue
if event.is_set():
# Extract relevant packets, ensuring event.packets is not None
packets = [
p
for p in event.packets
if PortNum.TEXT_MESSAGE_APP == p.portnum
p for p in (event.packets or [])
if p.portnum == PortNum.TEXT_MESSAGE_APP
]
event.clear()
event.clear() # Reset event flag
try:
for packet in packets:
ui_packet = Packet.from_model(packet)
if not re.match(r"seq \d+$", ui_packet.payload):
# Filter out packets that match "seq <number>"
if not seq_pattern.match(ui_packet.payload):
await resp.send(
chat_packet.render(
packet=ui_packet,
),
event="chat_packet",
chat_packet.render(packet=ui_packet),
event="chat_packet", # SSE event type
)
except ConnectionResetError:
return
# Log when a client disconnects unexpectedly
logging.warning("Client disconnected from SSE stream.")
return # Exit the loop and close the connection
@routes.get("/events")
async def events(request):
"""
Server-Sent Events (SSE) endpoint for real-time packet updates.
This endpoint listens for new network packets and streams them to connected clients.
Clients can optionally filter packets based on `node_id` and `portnum` query parameters.
Query Parameters:
- node_id (int, optional): Filter packets for a specific node (default: all nodes).
- portnum (int, optional): Filter packets for a specific port number (default: all ports).
Args:
request (aiohttp.web.Request): The incoming HTTP request.
Returns:
aiohttp.web.StreamResponse: SSE response streaming network events.
"""
# Extract and convert query parameters (if provided)
node_id = request.query.get("node_id")
if node_id:
node_id = int(node_id)
node_id = int(node_id) # Convert node_id to an integer
portnum = request.query.get("portnum")
if portnum:
portnum = int(portnum)
portnum = int(portnum) # Convert portnum to an integer
# Load Jinja2 templates for rendering packets
packet_template = env.get_template("packet.html")
net_packet_template = env.get_template("net_packet.html")
# Subscribe to packet notifications for the given node_id (or all nodes if None)
with notify.subscribe(node_id) as event:
async with sse_response(request) as resp:
while resp.is_connected():
while resp.is_connected(): # Keep connection open while client is connected
try:
async with asyncio.timeout(10):
await event.wait()
except TimeoutError:
continue
# Wait for an event with a timeout of 10 seconds
await asyncio.wait_for(event.wait(), timeout=10)
except asyncio.TimeoutError:
continue # No new packets, continue waiting
if event.is_set():
# Extract relevant packets based on `portnum` filter (if provided)
packets = [
p
for p in event.packets
p for p in (event.packets or [])
if portnum is None or portnum == p.portnum
]
# Extract uplinked packets (if port filter applies)
uplinked = [
u
for u in event.uplinked
u for u in (event.uplinked or [])
if portnum is None or portnum == u.portnum
]
event.clear()
event.clear() # Reset event flag
try:
# Process and send incoming packets
for packet in packets:
ui_packet = Packet.from_model(packet)
# Send standard packet event
await resp.send(
packet_template.render(
is_hx_request="HX-Request" in request.headers,
@@ -380,12 +430,16 @@ async def events(request):
),
event="packet",
)
# If the packet belongs to `PortNum.TEXT_MESSAGE_APP` and contains "#baymeshnet",
# send it as a network event
if ui_packet.portnum == PortNum.TEXT_MESSAGE_APP and '#baymeshnet' in ui_packet.payload.lower():
await resp.send(
net_packet_template.render(packet=ui_packet),
event="net_packet",
)
# Process and send uplinked packets separately
for packet in uplinked:
await resp.send(
packet_template.render(
@@ -395,8 +449,10 @@ async def events(request):
),
event="uplinked",
)
except ConnectionResetError:
return
logging.warning("Client disconnected from SSE stream.")
return # Gracefully exit on disconnection
@dataclass
class UplinkedNode:
@@ -408,7 +464,7 @@ class UplinkedNode:
snr: float
rssi: float
# Updated code p.r.
@routes.get("/packet_details/{packet_id}")
async def packet_details(request):
packet_id = int(request.match_info["packet_id"])
@@ -416,8 +472,11 @@ async def packet_details(request):
packet = await store.get_packet(packet_id)
from_node_cord = None
if packet.from_node and packet.from_node.last_lat:
from_node_cord = [packet.from_node.last_lat * 1e-7 , packet.from_node.last_long * 1e-7]
if packet and packet.from_node and packet.from_node.last_lat:
from_node_cord = [
packet.from_node.last_lat * 1e-7,
packet.from_node.last_long * 1e-7,
]
uplinked_nodes = []
for p in packets_seen:
@@ -444,6 +503,7 @@ async def packet_details(request):
elif uplinked_nodes:
map_center = [uplinked_nodes[0].lat, uplinked_nodes[0].long]
# Render the template and return the response
template = env.get_template("packet_details.html")
return web.Response(
text=template.render(
@@ -461,7 +521,7 @@ async def packet_details(request):
portnum = request.query.get("portnum")
if portnum:
portnum = int(portnum)
packets = await store.get_packets(portnum=portnum)
packets = await store.get_packets(portnum=portnum, limit=50)
template = env.get_template("firehose.html")
return web.Response(
text=template.render(
@@ -471,20 +531,43 @@ async def packet_details(request):
content_type="text/html",
)
@routes.get("/chat")
async def chat(request):
packets = await store.get_packets(
node_id=0xFFFFFFFF, portnum=PortNum.TEXT_MESSAGE_APP
)
template = env.get_template("chat.html")
ui_packets = (Packet.from_model(p) for p in packets)
return web.Response(
text=template.render(
packets=(p for p in ui_packets if not re.match(r"seq \d+$", p.payload)),
),
content_type="text/html",
)
try:
# Fetch packets for the given node ID and port number
#print("Fetching packets...")
packets = await store.get_packets(
node_id=0xFFFFFFFF, portnum=PortNum.TEXT_MESSAGE_APP, limit=100
)
#print(f"Fetched {len(packets)} packets.")
# Convert packets to UI packets
#print("Processing packets...")
ui_packets = [Packet.from_model(p) for p in packets]
# Filter packets
#print("Filtering packets...")
filtered_packets = [
p for p in ui_packets if not re.match(r"seq \d+$", p.payload)
]
# Render template
#print("Rendering template...")
template = env.get_template("chat.html")
return web.Response(
text=template.render(packets=filtered_packets),
content_type="text/html",
)
except Exception as e:
# Log the error and return an appropriate response
#print(f"Error in chat handler: {e}")
return web.Response(
text="An error occurred while processing your request.",
status=500,
content_type="text/plain",
)
@routes.get("/packet/{packet_id}")
@@ -598,9 +681,6 @@ async def graph_chutil(request):
],
)
@routes.get("/graph/wind_speed/{node_id}")
async def graph_wind_speed(request):
return await graph_telemetry(
@@ -702,7 +782,7 @@ async def graph_power_metrics(request):
@routes.get("/graph/neighbors/{node_id}")
async def graph_neighbors(request):
oldest = datetime.datetime.utcnow() - datetime.timedelta(days=4)
oldest = datetime.datetime.now() - datetime.timedelta(days=4)
data = {}
dates =[]
@@ -750,7 +830,7 @@ async def graph_neighbors(request):
@routes.get("/graph/neighbors2/{node_id}")
async def graph_neighbors2(request):
oldest = datetime.datetime.utcnow() - datetime.timedelta(days=30)
oldest = datetime.datetime.now() - datetime.timedelta(days=30)
data = []
node_ids = set()
@@ -992,10 +1072,10 @@ async def graph_network(request):
used_nodes = new_used_nodes
edges = new_edges
# Create the graph
graph = pydot.Dot('network', graph_type="digraph", layout="sfdp", overlap="prism", esep="+10", nodesep="0.5",
ranksep="1")
#graph = pydot.Dot('network', graph_type="digraph", layout="sfdp", overlap="prism", quadtree="2", repulsiveforce="1.5", k="1", overlap_scaling="1.5", concentrate=True)
#graph = pydot.Dot('network', graph_type="digraph", layout="sfdp", overlap="prism1000", overlap_scaling="-4", sep="1000", pack="true")
graph = pydot.Dot('network', graph_type="digraph", layout="neato", overlap="false", model='subset', esep="+5")
for node_id in used_nodes:
node = await nodes[node_id]
color = '#000000'
@@ -1053,50 +1133,37 @@ async def graph_network(request):
)
@routes.get("/net")
async def net(request):
if "date" in request.query:
start_date = datetime.date.fromisoformat(request.query["date"])
else:
start_date = datetime.date.today()
while start_date.weekday() != 2:
start_date = start_date - datetime.timedelta(days=5)
start_time = datetime.datetime.combine(start_date, datetime.time(0,0))
text_packets = [
Packet.from_model(p)
for p in await store.get_packets(
portnum=PortNum.TEXT_MESSAGE_APP,
after=start_time,
before=start_time + datetime.timedelta(hours=74),
)
]
net_packets = [p for p in text_packets if '#baymeshnet' in p.payload.lower()]
template = env.get_template("net.html")
return web.Response(
text=template.render(net_packets=text_packets),
content_type="text/html",
)
@routes.get("/stats")
async def stats(request):
# Fetch total packet count from the store
total_packets = await store.get_total_packet_count()
total_nodes = await store.get_total_node_count()
total_packets_seen = await store.get_total_packet_seen_count()
total_nodes_longfast = await store.get_total_node_count_longfast()
total_nodes_mediumslow = await store.get_total_node_count_mediumslow()
# Render the stats template with the total packet count
template = env.get_template("stats.html")
return web.Response(
text=template.render(total_packets=total_packets, total_nodes=total_nodes,total_packets_seen=total_packets_seen,total_nodes_longfast=total_nodes_longfast, total_nodes_mediumslow=total_nodes_mediumslow ),
content_type="text/html",
)
try:
# Add logging to track execution
total_packets = await store.get_total_packet_count()
total_nodes = await store.get_total_node_count()
total_packets_seen = await store.get_total_packet_seen_count()
total_nodes_longfast = await store.get_total_node_count_longfast()
total_nodes_mediumslow = await store.get_total_node_count_mediumslow()
# Render template
#print("Rendering template...")
template = env.get_template("stats.html")
return web.Response(
text=template.render(
total_packets=total_packets,
total_nodes=total_nodes,
total_packets_seen=total_packets_seen,
total_nodes_longfast=total_nodes_longfast,
total_nodes_mediumslow=total_nodes_mediumslow,
),
content_type="text/html",
)
except Exception as e:
# Log and return error response
#print(f"Error in stats handler: {e}")
return web.Response(
text="An error occurred while processing your request.",
status=500,
content_type="text/plain",
)
@routes.get("/graph/longfast")
@@ -1215,20 +1282,21 @@ async def graph_network_longfast(request):
edges = new_edges
# Create graph
graph = pydot.Dot('network', graph_type="digraph", layout="neato", overlap="false", model='subset', esep="+5")
graph = pydot.Dot('network', graph_type="digraph", layout="sfdp", overlap="scale", model='subset', splines="true")
for node_id in used_nodes:
node = await nodes[node_id]
color = '#000000'
node_name = await get_node_name(node_id)
if node and node.role in ('ROUTER', 'ROUTER_CLIENT', 'REPEATER'):
color = '#0000FF'
elif node and node.role == 'CLIENT_MUTE':
color = '#00FF00'
#elif node and node.role == 'CLIENT_MUTE':
# color = '#00FF00'
graph.add_node(pydot.Node(
str(node_id),
label=node_name,
shape='box',
color=color,
fontsize="10", width="0", height="0",
href=f"/graph/network?root={node_id}&amp;depth={depth-1}",
))
@@ -1247,7 +1315,7 @@ async def graph_network_longfast(request):
if edge_type[(src, dest)] in ('ni'):
color = '#FF0000'
elif edge_type[(src, dest)] in ('sni'):
color = '#00FF00'
color = '#040fb3'
else:
color = '#000000'
edge_dir = "forward"
@@ -1262,8 +1330,9 @@ async def graph_network_longfast(request):
str(dest),
color=color,
tooltip=f'{await get_node_name(src)} -> {await get_node_name(dest)}',
penwidth=1.85,
penwidth=.5,
dir=edge_dir,
arrowsize=".5",
))
return web.Response(
@@ -1281,7 +1350,7 @@ async def graph_network_longfast(request):
async def graph_network_mediumslow(request):
try:
root = request.query.get("root")
depth = int(request.query.get("depth", 5))
depth = int(request.query.get("depth", 3))
hours = int(request.query.get("hours", 24))
minutes = int(request.query.get("minutes", 0))
@@ -1393,7 +1462,8 @@ async def graph_network_mediumslow(request):
edges = new_edges
# Create graph
graph = pydot.Dot('network', graph_type="digraph", layout="neato", overlap="false", model='subset', esep="+5")
graph = pydot.Dot('network', graph_type="digraph", layout="sfdp", overlap="scale", model='subset', esep="+5", splines="true", nodesep="2", ranksep="2")
for node_id in used_nodes:
node = await nodes[node_id]
color = '#000000'
@@ -1407,6 +1477,7 @@ async def graph_network_mediumslow(request):
label=node_name,
shape='box',
color=color,
fontsize="10", width="0", height="0",
href=f"/graph/mediumslow?root={node_id}&amp;depth={depth-1}",
))
@@ -1425,7 +1496,7 @@ async def graph_network_mediumslow(request):
if edge_type[(src, dest)] in ('ni'):
color = '#FF0000'
elif edge_type[(src, dest)] in ('sni'):
color = '#00FF00'
color = '#040fb3'
else:
color = '#000000'
edge_dir = "forward"
@@ -1440,8 +1511,9 @@ async def graph_network_mediumslow(request):
str(dest),
color=color,
tooltip=f'{await get_node_name(src)} -> {await get_node_name(dest)}',
penwidth=1.85,
penwidth=.5,
dir=edge_dir,
arrowsize=".5",
))
return web.Response(
@@ -1453,6 +1525,104 @@ async def graph_network_mediumslow(request):
print(f"Error in graph_network_longfast: {e}")
return web.Response(status=500, text="Internal Server Error")
@routes.get("/nodelist")
async def nodelist(request):
try:
role = request.query.get("role")
#print(role)
channel = request.query.get("channel")
#print(channel)
hw_model = request.query.get("hw_model")
#print(hw_model)
nodes= await store.get_nodes(role,channel, hw_model)
template = env.get_template("nodelist.html")
return web.Response(
text=template.render(nodes=nodes),
content_type="text/html",
)
except Exception as e:
return web.Response(
text="An error occurred while processing your request.",
status=500,
content_type="text/plain",
)
@routes.get("/net")
async def net(request):
try:
# Fetch packets for the given node ID and port number
packets = await store.get_packets(
node_id=0xFFFFFFFF, portnum=PortNum.TEXT_MESSAGE_APP, limit=200
)
# Convert packets to UI packets
ui_packets = [Packet.from_model(p) for p in packets]
# Precompile regex for performance
seq_pattern = re.compile(r"seq \d+$")
# Filter packets: exclude "seq \d+$" but include those containing "pablo-test"
filtered_packets = [
p for p in ui_packets
if not seq_pattern.match(p.payload) and "baymeshnet" in p.payload.lower()
]
# Render template
template = env.get_template("net.html")
return web.Response(
text=template.render(packets=filtered_packets),
content_type="text/html",
)
except web.HTTPException as e:
raise # Let aiohttp handle HTTP exceptions properly
except Exception as e:
logging.exception("Error processing chat request")
return web.Response(
text="An internal server error occurred.",
status=500,
content_type="text/plain",
)
@routes.get("/net_events")
async def net_events(request):
chat_packet = env.get_template("net_packet.html")
# Precompile regex for performance (case insensitive)
seq_pattern = re.compile(r"seq \d+$")
with notify.subscribe(node_id=0xFFFFFFFF) as event:
async with sse_response(request) as resp:
while resp.is_connected():
try:
await asyncio.wait_for(event.wait(), timeout=10)
except asyncio.TimeoutError:
continue # Timeout occurred, loop again
if event.is_set():
# Ensure event.packets is valid before accessing it
packets = [
p for p in (event.packets or [])
if p.portnum == PortNum.TEXT_MESSAGE_APP
]
event.clear()
try:
for packet in packets:
ui_packet = Packet.from_model(packet)
if not seq_pattern.match(ui_packet.payload) and "baymeshnet" in ui_packet.payload.lower():
await resp.send(
chat_packet.render(packet=ui_packet),
event="net_packet",
)
except ConnectionResetError:
print("Client disconnected from SSE stream.")
return # Gracefully exit on disconnection
async def run_server(bind, port, tls_cert):