diff --git a/README b/README index d4c2f7f..1273ae6 100644 --- a/README +++ b/README @@ -1,6 +1,8 @@ Meshview ======== +Now running at https://meshview.bayme.sh + This project watches a MQTT topic for meshtastic messages, imports them to a database and has a web UI to view them. Requires Python 3.12 diff --git a/images/main.png b/images/main.png new file mode 100644 index 0000000..d2aab24 Binary files /dev/null and b/images/main.png differ diff --git a/meshview/database.py b/meshview/database.py index 1d027ee..a20e37b 100644 --- a/meshview/database.py +++ b/meshview/database.py @@ -1,20 +1,16 @@ -from sqlalchemy.ext.asyncio import async_sessionmaker -from sqlalchemy.ext.asyncio import AsyncSession -from sqlalchemy.ext.asyncio import create_async_engine - from meshview import models +from sqlalchemy.ext.asyncio import create_async_engine, async_sessionmaker - -def init_database(database_connetion_string): +def init_database(database_connection_string): global engine, async_session kwargs = {} - if not database_connetion_string.startswith('sqlite'): + if not database_connection_string.startswith('sqlite'): kwargs['pool_size'] = 20 kwargs['max_overflow'] = 50 - engine = create_async_engine(database_connetion_string, echo=False, **kwargs) + print (**kwargs) + engine = create_async_engine(database_connection_string, echo=False, connect_args={"timeout": 15}) async_session = async_sessionmaker(engine, expire_on_commit=False) - async def create_tables(): async with engine.begin() as conn: await conn.run_sync(models.Base.metadata.create_all) diff --git a/meshview/decode_payload.py b/meshview/decode_payload.py index 99fe696..f8f9a20 100644 --- a/meshview/decode_payload.py +++ b/meshview/decode_payload.py @@ -1,3 +1,4 @@ +from meshtastic.protobuf.mqtt_pb2 import MapReport from meshtastic.protobuf.portnums_pb2 import PortNum from meshtastic.protobuf.mesh_pb2 import ( Position, @@ -24,6 +25,7 @@ DECODE_MAP = { PortNum.TRACEROUTE_APP: RouteDiscovery.FromString, PortNum.ROUTING_APP: Routing.FromString, PortNum.TEXT_MESSAGE_APP: text_message, + PortNum.MAP_REPORT_APP: MapReport.FromString } diff --git a/meshview/models.py b/meshview/models.py index 8932032..569943c 100644 --- a/meshview/models.py +++ b/meshview/models.py @@ -14,30 +14,31 @@ class Node(Base): __tablename__ = "node" id: Mapped[str] = mapped_column(primary_key=True) node_id: Mapped[int] = mapped_column(BigInteger, nullable=True, unique=True) - long_name: Mapped[str] - short_name: Mapped[str] - hw_model: Mapped[str] + long_name: Mapped[str] = mapped_column(nullable=True) + short_name: Mapped[str] = mapped_column(nullable=True) + hw_model: Mapped[str] = mapped_column(nullable=True) + firmware: Mapped[str] = mapped_column(nullable=True) role: Mapped[str] = mapped_column(nullable=True) last_lat: Mapped[int] = mapped_column(BigInteger, nullable=True) last_long: Mapped[int] = mapped_column(BigInteger, nullable=True) - channel: Mapped[str] - + channel: Mapped[str] = mapped_column(nullable=True) + last_update: Mapped[datetime] = mapped_column(nullable=True) class Packet(Base): __tablename__ = "packet" id: Mapped[int] = mapped_column(BigInteger, primary_key=True) - portnum: Mapped[int] - from_node_id: Mapped[int] = mapped_column(BigInteger) + portnum: Mapped[int] = mapped_column(nullable=True) + from_node_id: Mapped[int] = mapped_column(BigInteger, nullable=True) from_node: Mapped["Node"] = relationship( primaryjoin="Packet.from_node_id == foreign(Node.node_id)", lazy="joined" ) - to_node_id: Mapped[int] = mapped_column(BigInteger) + to_node_id: Mapped[int] = mapped_column(BigInteger,nullable=True) to_node: Mapped["Node"] = relationship( primaryjoin="Packet.to_node_id == foreign(Node.node_id)", lazy="joined" ) - payload: Mapped[bytes] - import_time: Mapped[datetime] - channel: Mapped[str] + payload: Mapped[bytes] = mapped_column(nullable=True) + import_time: Mapped[datetime] = mapped_column(nullable=True) + channel: Mapped[str] = mapped_column(nullable=True) class PacketSeen(Base): @@ -48,13 +49,13 @@ class PacketSeen(Base): lazy="joined", primaryjoin="PacketSeen.node_id == foreign(Node.node_id)" ) rx_time: Mapped[int] = mapped_column(BigInteger, primary_key=True) - hop_limit: Mapped[int] + hop_limit: Mapped[int] = mapped_column(nullable=True) hop_start: Mapped[int] = mapped_column(nullable=True) - channel: Mapped[str] + channel: Mapped[str] = mapped_column(nullable=True) rx_snr: Mapped[float] = mapped_column(nullable=True) rx_rssi: Mapped[int] = mapped_column(nullable=True) - topic: Mapped[str] - import_time: Mapped[datetime] + topic: Mapped[str] = mapped_column(nullable=True) + import_time: Mapped[datetime] = mapped_column(nullable=True) class Traceroute(Base): @@ -64,8 +65,8 @@ class Traceroute(Base): packet: Mapped["Packet"] = relationship( primaryjoin="Traceroute.packet_id == foreign(Packet.id)", lazy="joined" ) - gateway_node_id: Mapped[int] = mapped_column(BigInteger) - done: Mapped[bool] - route: Mapped[bytes] - import_time: Mapped[datetime] + gateway_node_id: Mapped[int] = mapped_column(BigInteger, nullable=True) + done: Mapped[bool] = mapped_column(nullable=True) + route: Mapped[bytes] = mapped_column(nullable=True) + import_time: Mapped[datetime] = mapped_column(nullable=True) diff --git a/meshview/mqtt_reader.py b/meshview/mqtt_reader.py index 1bac492..5453f5c 100644 --- a/meshview/mqtt_reader.py +++ b/meshview/mqtt_reader.py @@ -1,11 +1,9 @@ import base64 import asyncio import random - import aiomqtt from google.protobuf.message import DecodeError from cryptography.hazmat.primitives.ciphers import Cipher, algorithms, modes - from meshtastic.protobuf.mqtt_pb2 import ServiceEnvelope KEY = base64.b64decode("1PG7OiApB1nwvP+rz05pAQ==") diff --git a/meshview/notify.py b/meshview/notify.py index d74b383..57268e9 100644 --- a/meshview/notify.py +++ b/meshview/notify.py @@ -34,16 +34,13 @@ def create_event(node_id): def remove_event(node_event): - print("removing event") waiting_node_ids_events[node_event.node_id].remove(node_event) - def notify_packet(node_id, packet): for event in waiting_node_ids_events[node_id]: event.packets.append(packet) event.set() - def notify_uplinked(node_id, packet): for event in waiting_node_ids_events[node_id]: event.uplinked.append(packet) @@ -52,8 +49,15 @@ def notify_uplinked(node_id, packet): @contextlib.contextmanager def subscribe(node_id): + """ + Context manager for subscribing to events for a node_id. + Automatically manages event creation and cleanup. + """ event = create_event(node_id) try: yield event + except Exception as e: + print(f"Error during subscription for node_id={node_id}: {e}") + raise finally: remove_event(event) diff --git a/meshview/store.py b/meshview/store.py index 97c13d2..3ff6118 100644 --- a/meshview/store.py +++ b/meshview/store.py @@ -2,7 +2,7 @@ import datetime from sqlalchemy import select, func from sqlalchemy.orm import lazyload - +from sqlalchemy import update from meshtastic.protobuf.config_pb2 import Config from meshtastic.protobuf.portnums_pb2 import PortNum from meshtastic.protobuf.mesh_pb2 import User, HardwareModel @@ -12,7 +12,34 @@ from meshview.models import Packet, PacketSeen, Node, Traceroute from meshview import notify + async def process_envelope(topic, env): + + # Checking if the received packet is a MAP_REPORT + # Update the node table with the firmware version + if env.packet.decoded.portnum == PortNum.MAP_REPORT_APP: + # Extract the node ID from the packet (renamed from 'id' to 'node_id' to avoid conflicts with Python's built-in id function) + node_id = getattr(env.packet, "from") + + # Decode the MAP report payload to extract the firmware version + map_report = decode_payload.decode_payload(PortNum.MAP_REPORT_APP, env.packet.decoded.payload) + + # Establish an asynchronous database session + async with database.async_session() as session: + # Construct an SQLAlchemy update statement + stmt = ( + update(Node) + .where(Node.node_id == node_id) # Ensure correct column reference + .values(firmware=map_report.firmware_version) # Assign new firmware value + ) + + # Execute the update statement asynchronously + await session.execute(stmt) + + # Commit the changes to the database + await session.commit() + + # This ignores any packet that does not have a ID if not env.packet.id: return @@ -58,6 +85,8 @@ async def process_envelope(topic, env): ) session.add(seen) + + if env.packet.decoded.portnum == PortNum.NODEINFO_APP: user = decode_payload.decode_payload( PortNum.NODEINFO_APP, env.packet.decoded.payload @@ -88,7 +117,7 @@ async def process_envelope(topic, env): node.short_name = user.short_name node.hw_model = hw_model node.role = role - # if need to update time of last update it may be here + node.last_update =datetime.datetime.now() else: node = Node( @@ -415,6 +444,7 @@ async def get_total_packet_count(): async def get_total_node_count(): async with database.async_session() as session: q = select(func.count(Node.id)) # Use SQLAlchemy's func to count nodes + q = q.where(Node.last_update > datetime.datetime.now() - datetime.timedelta(days=1)) # Look for nodes with nodeinfo updates in the last 24 hours result = await session.execute(q) return result.scalar() # Return the total count of nodes @@ -427,24 +457,13 @@ async def get_total_packet_seen_count(): async def get_total_node_count_longfast() -> int: - """ - Retrieves the total count of nodes where the channel is equal to 'LongFast'. - - This function queries the database asynchronously to count the number of nodes - in the `Node` table that meet the condition `channel == 'LongFast'`. It uses - SQLAlchemy's asynchronous session management and query construction. - - Returns: - int: The total count of nodes with `channel == 'LongFast'`. - - Raises: - Exception: If an error occurs during the database query execution. - """ try: # Open an asynchronous session with the database async with database.async_session() as session: # Build the query to count nodes where channel == 'LongFast' - q = select(func.count(Node.id)).filter(Node.channel == 'LongFast') + q = select(func.count(Node.id)) + q = q.where(Node.last_update > datetime.datetime.now() - datetime.timedelta( days=1)) # Look for nodes with nodeinfo updates in the last 24 hours + q = q.where(Node.channel == 'LongFast') # # Execute the query asynchronously and fetch the result result = await session.execute(q) @@ -458,25 +477,14 @@ async def get_total_node_count_longfast() -> int: async def get_total_node_count_mediumslow() -> int: - """ - Retrieves the total count of nodes where the channel is equal to 'MediumSlow'. - - This function queries the database asynchronously to count the number of nodes - in the `Node` table that meet the condition `channel == 'MediumSlow'`. It uses - SQLAlchemy's asynchronous session management and query construction. - - Returns: - int: The total count of nodes with `channel == 'MediumSlow'`. - - Raises: - Exception: If an error occurs during the database query execution. - """ try: # Open an asynchronous session with the database async with database.async_session() as session: # Build the query to count nodes where channel == 'LongFast' - q = select(func.count(Node.id)).filter(Node.channel == 'MediumSlow') - + q = select(func.count(Node.id)) + q = q.where(Node.last_update > datetime.datetime.now() - datetime.timedelta( + days=1)) # Look for nodes with nodeinfo updates in the last 24 hours + q = q.where(Node.channel == 'MediumSlow') # # Execute the query asynchronously and fetch the result result = await session.execute(q) @@ -498,6 +506,51 @@ async def get_nodes_mediumslow(): (Node.channel == "MediumSlow") ) ) + return result.scalars() +async def get_nodes(role=None, channel=None, hw_model=None): + """ + Fetches nodes from the database based on optional filtering criteria. + + Parameters: + role (str, optional): The role of the node (converted to uppercase for consistency). + channel (str, optional): The communication channel associated with the node. + hw_model (str, optional): The hardware model of the node. + + Returns: + list: A list of Node objects that match the given criteria. + """ + try: + async with database.async_session() as session: + #print(channel) # Debugging output (consider replacing with logging) + + # Start with a base query selecting all nodes + query = select(Node) + + # Apply filters based on provided parameters + if role is not None: + query = query.where(Node.role == role.upper()) # Ensure role is uppercase + if channel is not None: + query = query.where(Node.channel == channel) + if hw_model is not None: + query = query.where(Node.hw_model == hw_model) + + # Exclude nodes where last_update is an empty string + query = query.where(Node.last_update != "") + + # Order results by long_name in ascending order + query = query.order_by(Node.long_name.asc()) + + # Execute the query and retrieve results + result = await session.execute(query) + nodes = result.scalars().all() + return nodes # Return the list of nodes + + except Exception as e: + print("error reading DB") # Consider using logging instead of print + return [] # Return an empty list in case of failure + + + diff --git a/meshview/templates/base.html b/meshview/templates/base.html index c82bbc8..c089e4d 100644 --- a/meshview/templates/base.html +++ b/meshview/templates/base.html @@ -34,7 +34,9 @@
Bay Area Mesh - http://bayme.sh
-
Quick Links:  Search for a node  - Conversations - See everything  - Mesh Graph LG - MS - Stats

+
Quick Links:  Nodes - Conversations - See everything +  - Mesh Graph LF - MS  - Stats +  - Weekly Net

Loading...
diff --git a/meshview/templates/chat.html b/meshview/templates/chat.html index 48eb783..821f601 100644 --- a/meshview/templates/chat.html +++ b/meshview/templates/chat.html @@ -5,11 +5,17 @@ min-width:10em; } .chat-packet:nth-of-type(odd){ - background-color:#1f1f1f; + background-color: #3a3a3a; /* Lighter than #2a2a2a */ +} +.chat-packet { + border-bottom: 1px solid #555; + padding: 8px; + border-radius: 8px; /* Adjust the value to make the corners more or less rounded */ } .chat-packet:nth-of-type(even){ - background-color:#181818; + background-color: #333333; /* Slightly lighter than the previous #181818 */ } + {% endblock %} diff --git a/meshview/templates/chat_packet.html b/meshview/templates/chat_packet.html index 1c95df4..9a6d833 100644 --- a/meshview/templates/chat_packet.html +++ b/meshview/templates/chat_packet.html @@ -1,5 +1,6 @@
- {{packet.import_time | format_timestamp}} ✉️ - {{packet.from_node.long_name or (packet.from_node_id | node_id_to_hex) }} - {{packet.payload}} -
+ {{packet.import_time.strftime('%-I:%M:%S %p - %d-%m-%Y')}} + ✉️ {{packet.from_node.channel}} + {{packet.from_node.long_name or (packet.from_node_id | node_id_to_hex) }} + {{packet.payload}} + \ No newline at end of file diff --git a/meshview/templates/firehose.html b/meshview/templates/firehose.html index c5e36b8..c8da75b 100644 --- a/meshview/templates/firehose.html +++ b/meshview/templates/firehose.html @@ -23,7 +23,7 @@ >{{ name }} {% endfor %} - +
diff --git a/meshview/templates/net.html b/meshview/templates/net.html index e0288f0..b16e0db 100644 --- a/meshview/templates/net.html +++ b/meshview/templates/net.html @@ -1,27 +1,34 @@ {% extends "base.html" %} {% block css %} - #packet_details{ - height: 95vh; - overflow: scroll; - } +.timestamp { + min-width:10em; +} +.chat-packet:nth-of-type(odd){ + background-color: #3a3a3a; /* Lighter than #2a2a2a */ +} +.chat-packet { + border-bottom: 1px solid #555; + padding: 8px; + border-radius: 8px; /* Adjust the value to make the corners more or less rounded */ +} +.chat-packet:nth-of-type(even){ + background-color: #333333; /* Slightly lighter than the previous #181818 */ +} + {% endblock %} + {% block body %} -
-
-
- {% for packet in net_packets %} - {% include 'net_packet.html' %} - {% endfor %} -
-
-
+ +
Weekly Mesh check-in. We will keep it open on every Wednesday from 5:00pm for checkins so you do not have to rush.
+ The message format should be (LONG NAME) - (CITY YOU ARE IN) #BayMeshNet.

+
+
+ {% for packet in packets %} + {% include 'chat_packet.html' %} + {% else %} + No packets found. + {% endfor %}
-
-{% endblock body %} +{% endblock %} diff --git a/meshview/templates/net_packet.html b/meshview/templates/net_packet.html index 7582a1f..9a6d833 100644 --- a/meshview/templates/net_packet.html +++ b/meshview/templates/net_packet.html @@ -1,12 +1,6 @@ -
-
- {{packet.from_node.long_name}} - 🔎 -
-
-
-
{{packet.import_time | format_timestamp}}
-
{{packet.payload}}
-
-
-
+
+ {{packet.import_time.strftime('%-I:%M:%S %p - %d-%m-%Y')}} + ✉️ {{packet.from_node.channel}} + {{packet.from_node.long_name or (packet.from_node_id | node_id_to_hex) }} + {{packet.payload}} +
\ No newline at end of file diff --git a/meshview/templates/node.html b/meshview/templates/node.html index 88f7da6..2bf8c57 100644 --- a/meshview/templates/node.html +++ b/meshview/templates/node.html @@ -61,7 +61,7 @@
- {% include "buttons.html" %} +
diff --git a/meshview/templates/nodelist.html b/meshview/templates/nodelist.html new file mode 100644 index 0000000..00dc104 --- /dev/null +++ b/meshview/templates/nodelist.html @@ -0,0 +1,206 @@ +{% extends "base.html" %} + +{% block css %} +table { + width: 100%; + border-collapse: collapse; + margin-top: 1em; +} + +th, td { + padding: 10px; + border: 1px solid #333; + text-align: left; +} + +th { + background-color: #1f1f1f; + color: white; + cursor: pointer; +} + +tr:nth-child(even) { + background-color: #181818; +} + +tr:nth-child(odd) { + background-color: #222; +} + +.search-container { + display: flex; + gap: 10px; + margin-bottom: 10px; +} + +.search, .filter-role, .filter-channel, .filter-hw_model, .export-btn { + padding: 8px; + border: 1px solid #333; + border-radius: 4px; +} + +.filter-role, .filter-channel, .filter-hw_model { + cursor: pointer; +} + +.export-btn { + background: #28a745; + color: white; + border: none; + cursor: pointer; +} + +.export-btn:hover { + background: #218838; +} + +.count-container { + margin-bottom: 10px; + font-weight: bold; + color: white; +} +{% endblock %} + +{% block body %} +
+
+ + + + + + + + + + + + +
+ + +
Showing 0 nodes
+ + {% if nodes %} + + + + + + + + + + + + + + + + {% for node in nodes %} + + + + + + + + + + + + {% endfor %} + +
Long NameShort NameHW ModelFirmwareRoleLast LatitudeLast LongitudeChannelLast Update
{{ node.long_name }}{{ node.short_name }}{{ node.hw_model if node.hw_model else "N/A" }}{{ node.firmware }}{{ node.role if node.role else "N/A" }}{{ "{:.7f}".format(node.last_lat / 10**7) if node.last_lat else "N/A" }}{{ "{:.7f}".format(node.last_long / 10**7) if node.last_long else "N/A" }}{{ node.channel if node.channel else "N/A" }} + {{ node.last_update.strftime('%-I:%M:%S %p - %m-%d-%Y') if node.last_update else "N/A" }} +
+ {% else %} +

No nodes found.

+ {% endif %} +
+ + + +{% endblock %} diff --git a/meshview/templates/packet.html b/meshview/templates/packet.html index e430d32..a6f7ceb 100644 --- a/meshview/templates/packet.html +++ b/meshview/templates/packet.html @@ -34,8 +34,8 @@
-
import_time
-
{{packet.import_time | format_timestamp}}
+
Import Time
+
{{packet.import_time.strftime('%-I:%M:%S %p - %d-%m-%Y')}}
packet
{{packet.data}}
payload
diff --git a/meshview/templates/packet_details.html b/meshview/templates/packet_details.html index 3de8ebc..d273f0a 100644 --- a/meshview/templates/packet_details.html +++ b/meshview/templates/packet_details.html @@ -12,8 +12,8 @@
-
import_time
-
{{seen.import_time|format_timestamp}}
+
Import Time
+
{{seen.import_time.strftime('%-I:%M:%S %p - %d-%m-%Y')}}
rx_time
{{seen.rx_time|format_timestamp}}
hop_limit
diff --git a/meshview/templates/stats.html b/meshview/templates/stats.html index 3fc73d1..aca49cf 100644 --- a/meshview/templates/stats.html +++ b/meshview/templates/stats.html @@ -16,14 +16,14 @@

- Total Nodes: + Total Active Nodes (Last 24 hours): {{ total_nodes }}

- Total Nodes LongFast: + Total Active Nodes LongFast:
{{ total_nodes_longfast }} ({{ (total_nodes_longfast / total_nodes * 100) | round(2) }}%) @@ -33,7 +33,7 @@

- Total Nodes MediumSlow: + Total Active Nodes MediumSlow:
{{ total_nodes_mediumslow }} ({{ (total_nodes_mediumslow / total_nodes * 100) | round(2) }}%) diff --git a/meshview/web.py b/meshview/web.py index 52080b3..b917882 100644 --- a/meshview/web.py +++ b/meshview/web.py @@ -307,71 +307,121 @@ async def _packet_list(request, raw_packets, packet_event): @routes.get("/chat_events") async def chat_events(request): + """ + Server-Sent Events (SSE) endpoint for real-time chat packet updates. + + This endpoint listens for new chat packets related to `PortNum.TEXT_MESSAGE_APP` + and streams them to connected clients. Messages matching the pattern `"seq \d+$"` + are filtered out before sending. + + Args: + request (aiohttp.web.Request): The incoming HTTP request. + + Returns: + aiohttp.web.StreamResponse: SSE response streaming chat events. + """ chat_packet = env.get_template("chat_packet.html") + # Precompile regex for filtering out unwanted messages (case insensitive) + seq_pattern = re.compile(r"seq \d+$", re.IGNORECASE) + + # Subscribe to notifications for packets from all nodes (0xFFFFFFFF = broadcast) with notify.subscribe(node_id=0xFFFFFFFF) as event: async with sse_response(request) as resp: - while resp.is_connected(): + while resp.is_connected(): # Keep the connection open while the client is connected try: - async with asyncio.timeout(10): - await event.wait() - except TimeoutError: + # Wait for an event with a timeout of 10 seconds + await asyncio.wait_for(event.wait(), timeout=10) + except asyncio.TimeoutError: + # Timeout reached, continue looping to keep connection alive continue + if event.is_set(): + # Extract relevant packets, ensuring event.packets is not None packets = [ - p - for p in event.packets - if PortNum.TEXT_MESSAGE_APP == p.portnum + p for p in (event.packets or []) + if p.portnum == PortNum.TEXT_MESSAGE_APP ] - event.clear() + event.clear() # Reset event flag + try: for packet in packets: ui_packet = Packet.from_model(packet) - if not re.match(r"seq \d+$", ui_packet.payload): + + # Filter out packets that match "seq " + if not seq_pattern.match(ui_packet.payload): await resp.send( - chat_packet.render( - packet=ui_packet, - ), - event="chat_packet", + chat_packet.render(packet=ui_packet), + event="chat_packet", # SSE event type ) except ConnectionResetError: - return + # Log when a client disconnects unexpectedly + logging.warning("Client disconnected from SSE stream.") + return # Exit the loop and close the connection @routes.get("/events") async def events(request): + """ + Server-Sent Events (SSE) endpoint for real-time packet updates. + + This endpoint listens for new network packets and streams them to connected clients. + Clients can optionally filter packets based on `node_id` and `portnum` query parameters. + + Query Parameters: + - node_id (int, optional): Filter packets for a specific node (default: all nodes). + - portnum (int, optional): Filter packets for a specific port number (default: all ports). + + Args: + request (aiohttp.web.Request): The incoming HTTP request. + + Returns: + aiohttp.web.StreamResponse: SSE response streaming network events. + """ + # Extract and convert query parameters (if provided) node_id = request.query.get("node_id") if node_id: - node_id = int(node_id) + node_id = int(node_id) # Convert node_id to an integer + portnum = request.query.get("portnum") if portnum: - portnum = int(portnum) + portnum = int(portnum) # Convert portnum to an integer + # Load Jinja2 templates for rendering packets packet_template = env.get_template("packet.html") net_packet_template = env.get_template("net_packet.html") + + # Subscribe to packet notifications for the given node_id (or all nodes if None) with notify.subscribe(node_id) as event: async with sse_response(request) as resp: - while resp.is_connected(): + while resp.is_connected(): # Keep connection open while client is connected try: - async with asyncio.timeout(10): - await event.wait() - except TimeoutError: - continue + # Wait for an event with a timeout of 10 seconds + await asyncio.wait_for(event.wait(), timeout=10) + except asyncio.TimeoutError: + continue # No new packets, continue waiting + if event.is_set(): + # Extract relevant packets based on `portnum` filter (if provided) packets = [ - p - for p in event.packets + p for p in (event.packets or []) if portnum is None or portnum == p.portnum ] + + # Extract uplinked packets (if port filter applies) uplinked = [ - u - for u in event.uplinked + u for u in (event.uplinked or []) if portnum is None or portnum == u.portnum ] - event.clear() + + event.clear() # Reset event flag + try: + # Process and send incoming packets for packet in packets: ui_packet = Packet.from_model(packet) + + # Send standard packet event await resp.send( packet_template.render( is_hx_request="HX-Request" in request.headers, @@ -380,12 +430,16 @@ async def events(request): ), event="packet", ) + + # If the packet belongs to `PortNum.TEXT_MESSAGE_APP` and contains "#baymeshnet", + # send it as a network event if ui_packet.portnum == PortNum.TEXT_MESSAGE_APP and '#baymeshnet' in ui_packet.payload.lower(): await resp.send( net_packet_template.render(packet=ui_packet), event="net_packet", ) + # Process and send uplinked packets separately for packet in uplinked: await resp.send( packet_template.render( @@ -395,8 +449,10 @@ async def events(request): ), event="uplinked", ) + except ConnectionResetError: - return + logging.warning("Client disconnected from SSE stream.") + return # Gracefully exit on disconnection @dataclass class UplinkedNode: @@ -408,7 +464,7 @@ class UplinkedNode: snr: float rssi: float - +# Updated code p.r. @routes.get("/packet_details/{packet_id}") async def packet_details(request): packet_id = int(request.match_info["packet_id"]) @@ -416,8 +472,11 @@ async def packet_details(request): packet = await store.get_packet(packet_id) from_node_cord = None - if packet.from_node and packet.from_node.last_lat: - from_node_cord = [packet.from_node.last_lat * 1e-7 , packet.from_node.last_long * 1e-7] + if packet and packet.from_node and packet.from_node.last_lat: + from_node_cord = [ + packet.from_node.last_lat * 1e-7, + packet.from_node.last_long * 1e-7, + ] uplinked_nodes = [] for p in packets_seen: @@ -444,6 +503,7 @@ async def packet_details(request): elif uplinked_nodes: map_center = [uplinked_nodes[0].lat, uplinked_nodes[0].long] + # Render the template and return the response template = env.get_template("packet_details.html") return web.Response( text=template.render( @@ -461,7 +521,7 @@ async def packet_details(request): portnum = request.query.get("portnum") if portnum: portnum = int(portnum) - packets = await store.get_packets(portnum=portnum) + packets = await store.get_packets(portnum=portnum, limit=50) template = env.get_template("firehose.html") return web.Response( text=template.render( @@ -471,20 +531,43 @@ async def packet_details(request): content_type="text/html", ) - @routes.get("/chat") async def chat(request): - packets = await store.get_packets( - node_id=0xFFFFFFFF, portnum=PortNum.TEXT_MESSAGE_APP - ) - template = env.get_template("chat.html") - ui_packets = (Packet.from_model(p) for p in packets) - return web.Response( - text=template.render( - packets=(p for p in ui_packets if not re.match(r"seq \d+$", p.payload)), - ), - content_type="text/html", - ) + try: + # Fetch packets for the given node ID and port number + #print("Fetching packets...") + packets = await store.get_packets( + node_id=0xFFFFFFFF, portnum=PortNum.TEXT_MESSAGE_APP, limit=100 + ) + #print(f"Fetched {len(packets)} packets.") + + # Convert packets to UI packets + #print("Processing packets...") + ui_packets = [Packet.from_model(p) for p in packets] + + # Filter packets + #print("Filtering packets...") + filtered_packets = [ + p for p in ui_packets if not re.match(r"seq \d+$", p.payload) + ] + + # Render template + #print("Rendering template...") + template = env.get_template("chat.html") + return web.Response( + text=template.render(packets=filtered_packets), + content_type="text/html", + ) + + except Exception as e: + # Log the error and return an appropriate response + #print(f"Error in chat handler: {e}") + return web.Response( + text="An error occurred while processing your request.", + status=500, + content_type="text/plain", + ) + @routes.get("/packet/{packet_id}") @@ -598,9 +681,6 @@ async def graph_chutil(request): ], ) - - - @routes.get("/graph/wind_speed/{node_id}") async def graph_wind_speed(request): return await graph_telemetry( @@ -702,7 +782,7 @@ async def graph_power_metrics(request): @routes.get("/graph/neighbors/{node_id}") async def graph_neighbors(request): - oldest = datetime.datetime.utcnow() - datetime.timedelta(days=4) + oldest = datetime.datetime.now() - datetime.timedelta(days=4) data = {} dates =[] @@ -750,7 +830,7 @@ async def graph_neighbors(request): @routes.get("/graph/neighbors2/{node_id}") async def graph_neighbors2(request): - oldest = datetime.datetime.utcnow() - datetime.timedelta(days=30) + oldest = datetime.datetime.now() - datetime.timedelta(days=30) data = [] node_ids = set() @@ -992,10 +1072,10 @@ async def graph_network(request): used_nodes = new_used_nodes edges = new_edges + # Create the graph + graph = pydot.Dot('network', graph_type="digraph", layout="sfdp", overlap="prism", esep="+10", nodesep="0.5", + ranksep="1") - #graph = pydot.Dot('network', graph_type="digraph", layout="sfdp", overlap="prism", quadtree="2", repulsiveforce="1.5", k="1", overlap_scaling="1.5", concentrate=True) - #graph = pydot.Dot('network', graph_type="digraph", layout="sfdp", overlap="prism1000", overlap_scaling="-4", sep="1000", pack="true") - graph = pydot.Dot('network', graph_type="digraph", layout="neato", overlap="false", model='subset', esep="+5") for node_id in used_nodes: node = await nodes[node_id] color = '#000000' @@ -1053,50 +1133,37 @@ async def graph_network(request): ) -@routes.get("/net") -async def net(request): - if "date" in request.query: - start_date = datetime.date.fromisoformat(request.query["date"]) - else: - start_date = datetime.date.today() - while start_date.weekday() != 2: - start_date = start_date - datetime.timedelta(days=5) - - start_time = datetime.datetime.combine(start_date, datetime.time(0,0)) - - text_packets = [ - Packet.from_model(p) - for p in await store.get_packets( - portnum=PortNum.TEXT_MESSAGE_APP, - after=start_time, - before=start_time + datetime.timedelta(hours=74), - ) - ] - net_packets = [p for p in text_packets if '#baymeshnet' in p.payload.lower()] - - template = env.get_template("net.html") - return web.Response( - text=template.render(net_packets=text_packets), - content_type="text/html", - ) - @routes.get("/stats") async def stats(request): - # Fetch total packet count from the store - total_packets = await store.get_total_packet_count() - total_nodes = await store.get_total_node_count() - total_packets_seen = await store.get_total_packet_seen_count() - total_nodes_longfast = await store.get_total_node_count_longfast() - total_nodes_mediumslow = await store.get_total_node_count_mediumslow() - - - # Render the stats template with the total packet count - template = env.get_template("stats.html") - return web.Response( - text=template.render(total_packets=total_packets, total_nodes=total_nodes,total_packets_seen=total_packets_seen,total_nodes_longfast=total_nodes_longfast, total_nodes_mediumslow=total_nodes_mediumslow ), - content_type="text/html", - ) + try: + # Add logging to track execution + total_packets = await store.get_total_packet_count() + total_nodes = await store.get_total_node_count() + total_packets_seen = await store.get_total_packet_seen_count() + total_nodes_longfast = await store.get_total_node_count_longfast() + total_nodes_mediumslow = await store.get_total_node_count_mediumslow() + # Render template + #print("Rendering template...") + template = env.get_template("stats.html") + return web.Response( + text=template.render( + total_packets=total_packets, + total_nodes=total_nodes, + total_packets_seen=total_packets_seen, + total_nodes_longfast=total_nodes_longfast, + total_nodes_mediumslow=total_nodes_mediumslow, + ), + content_type="text/html", + ) + except Exception as e: + # Log and return error response + #print(f"Error in stats handler: {e}") + return web.Response( + text="An error occurred while processing your request.", + status=500, + content_type="text/plain", + ) @routes.get("/graph/longfast") @@ -1215,20 +1282,21 @@ async def graph_network_longfast(request): edges = new_edges # Create graph - graph = pydot.Dot('network', graph_type="digraph", layout="neato", overlap="false", model='subset', esep="+5") + graph = pydot.Dot('network', graph_type="digraph", layout="sfdp", overlap="scale", model='subset', splines="true") for node_id in used_nodes: node = await nodes[node_id] color = '#000000' node_name = await get_node_name(node_id) if node and node.role in ('ROUTER', 'ROUTER_CLIENT', 'REPEATER'): color = '#0000FF' - elif node and node.role == 'CLIENT_MUTE': - color = '#00FF00' + #elif node and node.role == 'CLIENT_MUTE': + # color = '#00FF00' graph.add_node(pydot.Node( str(node_id), label=node_name, shape='box', color=color, + fontsize="10", width="0", height="0", href=f"/graph/network?root={node_id}&depth={depth-1}", )) @@ -1247,7 +1315,7 @@ async def graph_network_longfast(request): if edge_type[(src, dest)] in ('ni'): color = '#FF0000' elif edge_type[(src, dest)] in ('sni'): - color = '#00FF00' + color = '#040fb3' else: color = '#000000' edge_dir = "forward" @@ -1262,8 +1330,9 @@ async def graph_network_longfast(request): str(dest), color=color, tooltip=f'{await get_node_name(src)} -> {await get_node_name(dest)}', - penwidth=1.85, + penwidth=.5, dir=edge_dir, + arrowsize=".5", )) return web.Response( @@ -1281,7 +1350,7 @@ async def graph_network_longfast(request): async def graph_network_mediumslow(request): try: root = request.query.get("root") - depth = int(request.query.get("depth", 5)) + depth = int(request.query.get("depth", 3)) hours = int(request.query.get("hours", 24)) minutes = int(request.query.get("minutes", 0)) @@ -1393,7 +1462,8 @@ async def graph_network_mediumslow(request): edges = new_edges # Create graph - graph = pydot.Dot('network', graph_type="digraph", layout="neato", overlap="false", model='subset', esep="+5") + graph = pydot.Dot('network', graph_type="digraph", layout="sfdp", overlap="scale", model='subset', esep="+5", splines="true", nodesep="2", ranksep="2") + for node_id in used_nodes: node = await nodes[node_id] color = '#000000' @@ -1407,6 +1477,7 @@ async def graph_network_mediumslow(request): label=node_name, shape='box', color=color, + fontsize="10", width="0", height="0", href=f"/graph/mediumslow?root={node_id}&depth={depth-1}", )) @@ -1425,7 +1496,7 @@ async def graph_network_mediumslow(request): if edge_type[(src, dest)] in ('ni'): color = '#FF0000' elif edge_type[(src, dest)] in ('sni'): - color = '#00FF00' + color = '#040fb3' else: color = '#000000' edge_dir = "forward" @@ -1440,8 +1511,9 @@ async def graph_network_mediumslow(request): str(dest), color=color, tooltip=f'{await get_node_name(src)} -> {await get_node_name(dest)}', - penwidth=1.85, + penwidth=.5, dir=edge_dir, + arrowsize=".5", )) return web.Response( @@ -1453,6 +1525,104 @@ async def graph_network_mediumslow(request): print(f"Error in graph_network_longfast: {e}") return web.Response(status=500, text="Internal Server Error") +@routes.get("/nodelist") +async def nodelist(request): + try: + role = request.query.get("role") + #print(role) + channel = request.query.get("channel") + #print(channel) + hw_model = request.query.get("hw_model") + #print(hw_model) + nodes= await store.get_nodes(role,channel, hw_model) + template = env.get_template("nodelist.html") + + return web.Response( + text=template.render(nodes=nodes), + content_type="text/html", + ) + except Exception as e: + + return web.Response( + text="An error occurred while processing your request.", + status=500, + content_type="text/plain", + ) + + +@routes.get("/net") +async def net(request): + try: + # Fetch packets for the given node ID and port number + packets = await store.get_packets( + node_id=0xFFFFFFFF, portnum=PortNum.TEXT_MESSAGE_APP, limit=200 + ) + + # Convert packets to UI packets + ui_packets = [Packet.from_model(p) for p in packets] + + # Precompile regex for performance + seq_pattern = re.compile(r"seq \d+$") + + # Filter packets: exclude "seq \d+$" but include those containing "pablo-test" + filtered_packets = [ + p for p in ui_packets + if not seq_pattern.match(p.payload) and "baymeshnet" in p.payload.lower() + ] + + # Render template + template = env.get_template("net.html") + return web.Response( + text=template.render(packets=filtered_packets), + content_type="text/html", + ) + + except web.HTTPException as e: + raise # Let aiohttp handle HTTP exceptions properly + + except Exception as e: + logging.exception("Error processing chat request") + return web.Response( + text="An internal server error occurred.", + status=500, + content_type="text/plain", + ) + + +@routes.get("/net_events") +async def net_events(request): + chat_packet = env.get_template("net_packet.html") + + # Precompile regex for performance (case insensitive) + seq_pattern = re.compile(r"seq \d+$") + + with notify.subscribe(node_id=0xFFFFFFFF) as event: + async with sse_response(request) as resp: + while resp.is_connected(): + try: + await asyncio.wait_for(event.wait(), timeout=10) + except asyncio.TimeoutError: + continue # Timeout occurred, loop again + + if event.is_set(): + # Ensure event.packets is valid before accessing it + packets = [ + p for p in (event.packets or []) + if p.portnum == PortNum.TEXT_MESSAGE_APP + ] + event.clear() + + try: + for packet in packets: + ui_packet = Packet.from_model(packet) + if not seq_pattern.match(ui_packet.payload) and "baymeshnet" in ui_packet.payload.lower(): + await resp.send( + chat_packet.render(packet=ui_packet), + event="net_packet", + ) + except ConnectionResetError: + print("Client disconnected from SSE stream.") + return # Gracefully exit on disconnection async def run_server(bind, port, tls_cert):