Merge branch 'pablo-merge' into site_config

.idea/.gitignore

@@ -1,3 +0,0 @@
-# Default ignored files
-/shelf/
-/workspace.xml

.idea/inspectionProfiles/profiles_settings.xml

@@ -0,0 +1,6 @@
+<component name="InspectionProjectProfileManager">
+  <settings>
+    <option name="USE_PROJECT_PROFILE" value="false" />
+    <version value="1.0" />
+  </settings>
+</component>

.idea/meshview-2.iml

@@ -0,0 +1,10 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<module type="PYTHON_MODULE" version="4">
+  <component name="NewModuleRootManager">
+    <content url="file://$MODULE_DIR$">
+      <excludeFolder url="file://$MODULE_DIR$/.venv" />
+    </content>
+    <orderEntry type="jdk" jdkName="Python 3.12 (meshview-2)" jdkType="Python SDK" />
+    <orderEntry type="sourceFolder" forTests="false" />
+  </component>
+</module>

.idea/modules.xml

@@ -0,0 +1,8 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="ProjectModuleManager">
+    <modules>
+      <module fileurl="file://$PROJECT_DIR$/.idea/meshview-2.iml" filepath="$PROJECT_DIR$/.idea/meshview-2.iml" />
+    </modules>
+  </component>
+</project>

README.md

@@ -58,10 +58,16 @@ connection_string = sqlite+aiosqlite:///packets.db
 ```
 
 ## Running Meshview
-
+Start the database connection.
+``` bash
+./env/bin/python startdb.py
+```
+Start the web server.
 ``` bash
 ./env/bin/python main.py
 ```
+
+
 Now you can hit http://localhost:8081/ ***(if you did not change the web server port )***
 
 You can specify the path to your `config.ini` file with the run command argument `--config`

main.py

@@ -1,18 +1,19 @@
 import asyncio
 import argparse
 import configparser
-import json
 from meshview import mqtt_reader
 from meshview import database
-from meshview import store
+from meshview import mqtt_store
 from meshview import web
 from meshview import http
 from meshview import models
+import json
+
 
 
 async def load_database_from_mqtt(mqtt_server: str , mqtt_port: int, topic: list, mqtt_user: str | None = None, mqtt_passwd: str | None = None):
     async for topic, env in mqtt_reader.get_topic_envelopes(mqtt_server, mqtt_port, topic, mqtt_user, mqtt_passwd):
-        await store.process_envelope(topic, env)
+        await mqtt_store.process_envelope(topic, env)
 
 
 async def main(config):
@@ -39,9 +40,6 @@ async def main(config):
         # print("Site configuration loaded to database")
 
     async with asyncio.TaskGroup() as tg:
-        tg.create_task(
-            load_database_from_mqtt(config["mqtt"]["server"], int(config["mqtt"]["port"]), mqtt_topics, mqtt_user, mqtt_passwd)
-        )
         tg.create_task(
             web.run_server(
                 config["server"]["bind"],
@@ -49,13 +47,6 @@ async def main(config):
                 config["server"].get("tls_cert"),
             )
         )
-        if config["server"].get("acme_challenge"):
-            tg.create_task(
-                http.run_server(
-                    config["server"]["bind"], config["server"]["acme_challenge"]
-                )
-            )
-
 
 def load_config(file_path):
     """Load configuration from an INI-style text file."""

meshview/database.py

@@ -1,15 +1,36 @@
 from meshview import models
-from sqlalchemy.ext.asyncio import create_async_engine, async_sessionmaker
+from sqlalchemy.ext.asyncio import async_sessionmaker
+from sqlalchemy.ext.asyncio import create_async_engine, AsyncSession
 
-def init_database(database_connection_string):
+
+
+engine = None
+async_session = None
+
+
+def init_database(database_connection_string, read_only=False):
     global engine, async_session
-    kwargs = {}
-    if not database_connection_string.startswith('sqlite'):
-        kwargs['pool_size'] = 20
-        kwargs['max_overflow'] = 50
-    print (**kwargs)
-    engine = create_async_engine(database_connection_string, echo=False, connect_args={"timeout": 15})
-    async_session = async_sessionmaker(engine, expire_on_commit=False)
+
+    kwargs = {"echo": False}
+
+    if database_connection_string.startswith("sqlite"):
+        if read_only:
+            # Ensure SQLite is opened in read-only mode
+            database_connection_string += "?mode=ro"
+            kwargs["connect_args"] = {"uri": True}
+        else:
+            kwargs["connect_args"] = {"timeout": 15}
+    else:
+        kwargs["pool_size"] = 20
+        kwargs["max_overflow"] = 50
+
+    print("Database connection settings:", kwargs)  # Debugging output
+
+    engine = create_async_engine(database_connection_string, **kwargs)
+    async_session = async_sessionmaker( bind=engine,
+    class_=AsyncSession,
+    expire_on_commit=False,
+)
 
 async def create_tables():
     async with engine.begin() as conn:

meshview/models.py

@@ -1,5 +1,4 @@
 from datetime import datetime
-
 from sqlalchemy.orm import DeclarativeBase, foreign
 from sqlalchemy.ext.asyncio import AsyncAttrs
 from sqlalchemy.orm import mapped_column, relationship, Mapped

meshview/mqtt_database.py

@@ -0,0 +1,16 @@
+from meshview import models
+from sqlalchemy.ext.asyncio import create_async_engine, async_sessionmaker
+
+def init_database(database_connection_string):
+    global engine, async_session
+    kwargs = {}
+    if not database_connection_string.startswith('sqlite'):
+        kwargs['pool_size'] = 20
+        kwargs['max_overflow'] = 50
+    print (**kwargs)
+    engine = create_async_engine(database_connection_string, echo=False, connect_args={"timeout": 15})
+    async_session = async_sessionmaker(engine, expire_on_commit=False)
+
+async def create_tables():
+    async with engine.begin() as conn:
+        await conn.run_sync(models.Base.metadata.create_all)

meshview/mqtt_reader.py

@@ -33,6 +33,7 @@ async def get_topic_envelopes(mqtt_server, mqtt_port, topics, mqtt_user, mqtt_pa
                 mqtt_server, port=mqtt_port , username=mqtt_user, password=mqtt_passwd , identifier=identifier,
             ) as client:
                 for topic in topics:
+                    print(topic)
                     await client.subscribe(topic)
                 async for msg in client.messages:
                     try:

meshview/mqtt_store.py

@@ -0,0 +1,165 @@
+import datetime
+from sqlalchemy import select
+from sqlalchemy import update
+from meshtastic.protobuf.config_pb2 import Config
+from meshtastic.protobuf.portnums_pb2 import PortNum
+from meshtastic.protobuf.mesh_pb2 import User, HardwareModel
+from meshview import mqtt_database
+from meshview import decode_payload
+from meshview.models import Packet, PacketSeen, Node, Traceroute
+
+
+
+
+async def process_envelope(topic, env):
+
+    # Checking if the received packet is a MAP_REPORT
+    # Update the node table with the firmware version
+    if env.packet.decoded.portnum == PortNum.MAP_REPORT_APP:
+        # Extract the node ID from the packet (renamed from 'id' to 'node_id' to avoid conflicts with Python's built-in id function)
+        node_id = getattr(env.packet, "from")
+
+        # Decode the MAP report payload to extract the firmware version
+        map_report = decode_payload.decode_payload(PortNum.MAP_REPORT_APP, env.packet.decoded.payload)
+
+        # Establish an asynchronous database session
+        async with mqtt_database.async_session() as session:
+            # Construct an SQLAlchemy update statement
+            stmt = (
+                update(Node)
+                .where(Node.node_id == node_id)  # Ensure correct column reference
+                .values(firmware=map_report.firmware_version)  # Assign new firmware value
+            )
+
+            # Execute the update statement asynchronously
+            await session.execute(stmt)
+
+            # Commit the changes to the database
+            await session.commit()
+
+    # This ignores any packet that does not have a ID
+    if not env.packet.id:
+        return
+
+    async with mqtt_database.async_session() as session:
+        result = await session.execute(select(Packet).where(Packet.id == env.packet.id))
+        new_packet = False
+        packet = result.scalar_one_or_none()
+        if not packet:
+            new_packet = True
+            packet = Packet(
+                id=env.packet.id,
+                portnum=env.packet.decoded.portnum,
+                from_node_id=getattr(env.packet, "from"),
+                to_node_id=env.packet.to,
+                payload=env.packet.SerializeToString(),
+                # p.r. Here seems to be where the packet is imported on the Database and import time is set.
+                import_time=datetime.datetime.now(),
+                channel=env.channel_id,
+            )
+            session.add(packet)
+
+        result = await session.execute(
+            select(PacketSeen).where(
+                PacketSeen.packet_id == env.packet.id,
+                PacketSeen.node_id == int(env.gateway_id[1:], 16),
+                PacketSeen.rx_time == env.packet.rx_time,
+            )
+        )
+        seen = None
+        if not result.scalar_one_or_none():
+            seen = PacketSeen(
+                packet_id=env.packet.id,
+                node_id=int(env.gateway_id[1:], 16),
+                channel=env.channel_id,
+                rx_time=env.packet.rx_time,
+                rx_snr=env.packet.rx_snr,
+                rx_rssi=env.packet.rx_rssi,
+                hop_limit=env.packet.hop_limit,
+                hop_start=env.packet.hop_start,
+                topic=topic,
+                # p.r. Here seems to be where the packet is imported on the Database and import time is set.
+                import_time=datetime.datetime.now(),
+            )
+            session.add(seen)
+
+
+
+        if env.packet.decoded.portnum == PortNum.NODEINFO_APP:
+            user = decode_payload.decode_payload(
+                PortNum.NODEINFO_APP, env.packet.decoded.payload
+            )
+            if user:
+                result = await session.execute(select(Node).where(Node.id == user.id))
+                if user.id and user.id[0] == "!":
+                    try:
+                        node_id = int(user.id[1:], 16)
+                    except ValueError:
+                        node_id = None
+                        pass
+                else:
+                    node_id = None
+
+                try:
+                    hw_model = HardwareModel.Name(user.hw_model)
+                except ValueError:
+                    hw_model = "unknown"
+                try:
+                   role = Config.DeviceConfig.Role.Name(user.role)
+                except ValueError:
+                    role = "unknown"
+
+                if node := result.scalar_one_or_none():
+                    node.node_id = node_id
+                    node.long_name = user.long_name
+                    node.short_name = user.short_name
+                    node.hw_model = hw_model
+                    node.role = role
+                    node.last_update =datetime.datetime.now()
+
+                else:
+                    node = Node(
+                        id=user.id,
+                        node_id=node_id,
+                        long_name=user.long_name,
+                        short_name=user.short_name,
+                        hw_model=hw_model,
+                        role=role,
+                        channel=env.channel_id,
+                        # if need to update time of last update it may be here
+                    )
+                    session.add(node)
+
+        if env.packet.decoded.portnum == PortNum.POSITION_APP:
+            position = decode_payload.decode_payload(
+                PortNum.POSITION_APP, env.packet.decoded.payload
+            )
+            if position and position.latitude_i and position.longitude_i:
+                from_node_id = getattr(env.packet, 'from')
+                node = (await session.execute(select(Node).where(Node.node_id == from_node_id))).scalar_one_or_none()
+                if node:
+                    node.last_lat = position.latitude_i
+                    node.last_long = position.longitude_i
+                    session.add(node)
+
+        if env.packet.decoded.portnum == PortNum.TRACEROUTE_APP:
+            packet_id = None
+            if env.packet.decoded.want_response:
+                packet_id = env.packet.id
+            else:
+                result = await session.execute(select(Packet).where(Packet.id == env.packet.decoded.request_id))
+                if result.scalar_one_or_none():
+                    packet_id = env.packet.decoded.request_id
+            if packet_id is not None:
+                session.add(Traceroute(
+                    packet_id=packet_id,
+                    route=env.packet.decoded.payload,
+                    done=not env.packet.decoded.want_response,
+                    gateway_node_id=int(env.gateway_id[1:], 16),
+                    import_time=datetime.datetime.now(),
+                ))
+
+        await session.commit()
+        if new_packet:
+            await packet.awaitable_attrs.to_node
+            await packet.awaitable_attrs.from_node

meshview/store.py

@@ -1,175 +1,9 @@
 import datetime
-
 from sqlalchemy import select, func
 from sqlalchemy.orm import lazyload
-from sqlalchemy import update
-from meshtastic.protobuf.config_pb2 import Config
-from meshtastic.protobuf.portnums_pb2 import PortNum
-from meshtastic.protobuf.mesh_pb2 import User, HardwareModel
 from meshview import database
-from meshview import decode_payload
 from meshview.models import Packet, PacketSeen, Node, Traceroute, SiteConfig
-from meshview import notify
-
-
-
-async def process_envelope(topic, env):
-
-    # Checking if the received packet is a MAP_REPORT
-    # Update the node table with the firmware version
-    if env.packet.decoded.portnum == PortNum.MAP_REPORT_APP:
-        # Extract the node ID from the packet (renamed from 'id' to 'node_id' to avoid conflicts with Python's built-in id function)
-        node_id = getattr(env.packet, "from")
-
-        # Decode the MAP report payload to extract the firmware version
-        map_report = decode_payload.decode_payload(PortNum.MAP_REPORT_APP, env.packet.decoded.payload)
-
-        # Establish an asynchronous database session
-        async with database.async_session() as session:
-            # Construct an SQLAlchemy update statement
-            stmt = (
-                update(Node)
-                .where(Node.node_id == node_id)  # Ensure correct column reference
-                .values(firmware=map_report.firmware_version)  # Assign new firmware value
-            )
-
-            # Execute the update statement asynchronously
-            await session.execute(stmt)
-
-            # Commit the changes to the database
-            await session.commit()
-
-    # This ignores any packet that does not have a ID
-    if not env.packet.id:
-        return
-
-    async with database.async_session() as session:
-        result = await session.execute(select(Packet).where(Packet.id == env.packet.id))
-        new_packet = False
-        packet = result.scalar_one_or_none()
-        if not packet:
-            new_packet = True
-            packet = Packet(
-                id=env.packet.id,
-                portnum=env.packet.decoded.portnum,
-                from_node_id=getattr(env.packet, "from"),
-                to_node_id=env.packet.to,
-                payload=env.packet.SerializeToString(),
-                # p.r. Here seems to be where the packet is imported on the Database and import time is set.
-                import_time=datetime.datetime.now(),
-                channel=env.channel_id,
-            )
-            session.add(packet)
-
-        result = await session.execute(
-            select(PacketSeen).where(
-                PacketSeen.packet_id == env.packet.id,
-                PacketSeen.node_id == int(env.gateway_id[1:], 16),
-                PacketSeen.rx_time == env.packet.rx_time,
-            )
-        )
-        seen = None
-        if not result.scalar_one_or_none():
-            seen = PacketSeen(
-                packet_id=env.packet.id,
-                node_id=int(env.gateway_id[1:], 16),
-                channel=env.channel_id,
-                rx_time=env.packet.rx_time,
-                rx_snr=env.packet.rx_snr,
-                rx_rssi=env.packet.rx_rssi,
-                hop_limit=env.packet.hop_limit,
-                hop_start=env.packet.hop_start,
-                topic=topic,
-                # p.r. Here seems to be where the packet is imported on the Database and import time is set.
-                import_time=datetime.datetime.now(),
-            )
-            session.add(seen)
-
-
-
-        if env.packet.decoded.portnum == PortNum.NODEINFO_APP:
-            user = decode_payload.decode_payload(
-                PortNum.NODEINFO_APP, env.packet.decoded.payload
-            )
-            if user:
-                result = await session.execute(select(Node).where(Node.id == user.id))
-                if user.id and user.id[0] == "!":
-                    try:
-                        node_id = int(user.id[1:], 16)
-                    except ValueError:
-                        node_id = None
-                        pass
-                else:
-                    node_id = None
-
-                try:
-                    hw_model = HardwareModel.Name(user.hw_model)
-                except ValueError:
-                    hw_model = "unknown"
-                try:
-                   role = Config.DeviceConfig.Role.Name(user.role)
-                except ValueError:
-                    role = "unknown"
-
-                if node := result.scalar_one_or_none():
-                    node.node_id = node_id
-                    node.long_name = user.long_name
-                    node.short_name = user.short_name
-                    node.hw_model = hw_model
-                    node.role = role
-                    node.last_update =datetime.datetime.now()
-
-                else:
-                    node = Node(
-                        id=user.id,
-                        node_id=node_id,
-                        long_name=user.long_name,
-                        short_name=user.short_name,
-                        hw_model=hw_model,
-                        role=role,
-                        channel=env.channel_id,
-                        # if need to update time of last update it may be here
-                    )
-                    session.add(node)
-
-        if env.packet.decoded.portnum == PortNum.POSITION_APP:
-            position = decode_payload.decode_payload(
-                PortNum.POSITION_APP, env.packet.decoded.payload
-            )
-            if position and position.latitude_i and position.longitude_i:
-                from_node_id = getattr(env.packet, 'from')
-                node = (await session.execute(select(Node).where(Node.node_id == from_node_id))).scalar_one_or_none()
-                if node:
-                    node.last_lat = position.latitude_i
-                    node.last_long = position.longitude_i
-                    session.add(node)
-
-        if env.packet.decoded.portnum == PortNum.TRACEROUTE_APP:
-            packet_id = None
-            if env.packet.decoded.want_response:
-                packet_id = env.packet.id
-            else:
-                result = await session.execute(select(Packet).where(Packet.id == env.packet.decoded.request_id))
-                if result.scalar_one_or_none():
-                    packet_id = env.packet.decoded.request_id
-            if packet_id is not None:
-                session.add(Traceroute(
-                    packet_id=packet_id,
-                    route=env.packet.decoded.payload,
-                    done=not env.packet.decoded.want_response,
-                    gateway_node_id=int(env.gateway_id[1:], 16),
-                    import_time=datetime.datetime.now(),
-                ))
-
-        await session.commit()
-        if new_packet:
-            await packet.awaitable_attrs.to_node
-            await packet.awaitable_attrs.from_node
-            notify.notify_packet(packet.to_node_id, packet)
-            notify.notify_packet(packet.from_node_id, packet)
-            notify.notify_packet(None, packet)
-        if seen:
-            notify.notify_uplinked(seen.node_id, packet)
+from sqlalchemy import text
 
 
 async def get_node(node_id):
@@ -189,7 +23,7 @@ async def get_fuzzy_nodes(query):
         return result.scalars()
 
 
-async def get_packets(node_id=None, portnum=None, since=None, limit=500, before=None, after=None):
+async def get_packets(node_id=None, portnum=None, since=None, limit=1000, before=None, after=None):
     async with database.async_session() as session:
         q = select(Packet)
 
@@ -209,7 +43,8 @@ async def get_packets(node_id=None, portnum=None, since=None, limit=500, before=
             q = q.limit(limit)
 
         result = await session.execute(q.order_by(Packet.import_time.desc()))
-        return result.scalars()
+        packets = list(result.scalars())  # Convert to list
+        return packets  # Return the list
 
 
 async def get_packets_from(node_id=None, portnum=None, since=None, limit=500):
@@ -300,140 +135,6 @@ async def get_mqtt_neighbors(since):
         )
         return result
 
-# In order to provide separate network graphs for LongFast and MediumSlow, I am duplicating the procedures.
-# 3 procedures are needed. These would have to be replicated for any other network that we may need to use graphs.
-#
-# get_traceroutes_longfast
-# get_packets_longfast
-# get_mqtt_neighbors_longfast
-#
-# p.r.
-# TODO # combine the duplicated funtions back to the original 3 by letting them take a second variable to specify channel name. 
-# The default value for channel (none) should cause these functioins to operate the same as they did before they were channel specific.
-# This change will make adding new channel specific graphs much easier in the future.
-#
-# Get Traceroute for LongFast only
-async def get_traceroutes_longfast(since):
-    async with database.async_session() as session:
-        result = await session.execute(
-                select(Traceroute)
-                .join(Packet)
-                .where(
-                (Traceroute.import_time > (datetime.datetime.now() - since))
-                & (Packet.channel == "LongFast")
-                )
-                .order_by(Traceroute.import_time)
-        )
-        return result.scalars()
-
-# Get MQTT Neighbors for LongFast only
-# p.r.
-async def get_mqtt_neighbors_longfast(since):
-    async with database.async_session() as session:
-        result = await session.execute(select(PacketSeen, Packet)
-            .join(Packet)
-            .where(
-                (PacketSeen.hop_limit == PacketSeen.hop_start)
-                & (PacketSeen.hop_start != 0)
-                & (Packet.channel == "LongFast")
-            )
-
-            .options(
-                lazyload(Packet.from_node),
-                lazyload(Packet.to_node),
-            )
-        )
-        return result
-
-# Get Packets for LongFast only
-# p.r.
-async def get_packets_longfast(node_id=None, portnum=None, since=None, limit=500, before=None, after=None):
-    async with database.async_session() as session:
-        q = select(Packet)
-
-        # Add condition for channel being "LongFast"
-        q = q.where(Packet.channel == "LongFast")
-
-        if node_id:
-            q = q.where(
-                (Packet.from_node_id == node_id) | (Packet.to_node_id == node_id)
-            )
-        if portnum:
-            q = q.where(Packet.portnum == portnum)
-        if since:
-            q = q.where(Packet.import_time > (datetime.datetime.now() - since))
-        if before:
-            q = q.where(Packet.import_time < before)
-        if after:
-            q = q.where(Packet.import_time > after)
-        if limit is not None:
-            q = q.limit(limit)
-
-        result = await session.execute(q.order_by(Packet.import_time.desc()))
-        return result.scalars()
-
-# Get Traceroute for mediumslow only
-# p.r.
-async def get_traceroutes_mediumslow(since):
-    async with database.async_session() as session:
-        result = await session.execute(
-                select(Traceroute)
-                .join(Packet)
-                .where(
-                (Traceroute.import_time > (datetime.datetime.now() - since))
-                & (Packet.channel == "MediumSlow")
-                )
-                .order_by(Traceroute.import_time)
-        )
-        return result.scalars()
-
-# Get MQTT Neighbors for mediumslow only
-# p.r.
-async def get_mqtt_neighbors_mediumslow(since):
-    async with database.async_session() as session:
-        result = await session.execute(select(PacketSeen, Packet)
-            .join(Packet)
-            .where(
-                (PacketSeen.hop_limit == PacketSeen.hop_start)
-                & (PacketSeen.hop_start != 0)
-                & (Packet.channel == "MediumSlow")
-            )
-
-            .options(
-                lazyload(Packet.from_node),
-                lazyload(Packet.to_node),
-            )
-        )
-        return result
-
-# Get Packets for MediumSlow only
-# p.r.
-async def get_packets_mediumslow(node_id=None, portnum=None, since=None, limit=500, before=None, after=None):
-    async with database.async_session() as session:
-        q = select(Packet)
-
-        # Add condition for channel being "MediumSlow"
-        q = q.where(Packet.channel == "MediumSlow")
-
-        if node_id:
-            q = q.where(
-                (Packet.from_node_id == node_id) | (Packet.to_node_id == node_id)
-            )
-        if portnum:
-            q = q.where(Packet.portnum == portnum)
-        if since:
-            q = q.where(Packet.import_time > (datetime.datetime.now() - since))
-        if before:
-            q = q.where(Packet.import_time < before)
-        if after:
-            q = q.where(Packet.import_time > after)
-        if limit is not None:
-            q = q.limit(limit)
-
-        result = await session.execute(q.order_by(Packet.import_time.desc()))
-        return result.scalars()
-
-
 
 # We count the total amount of packages
 # This is to be used by /stats in web.py
@@ -512,6 +213,63 @@ async def get_nodes_mediumslow():
 
         return result.scalars()
 
+async def get_top_traffic_nodes():
+    async with database.async_session() as session:
+        result = await session.execute(text("""
+            SELECT 
+                n.node_id,
+                n.long_name,
+                n.role,
+                COUNT(p.id) AS packet_count
+            FROM 
+                packet p
+            JOIN 
+                node n
+            ON 
+                p.from_node_id = n.node_id
+            WHERE 
+                p.import_time >= DATETIME('now', '-1 day')
+            GROUP BY 
+                n.long_name, n.role
+            ORDER BY 
+                packet_count DESC
+            LIMIT 100;
+        """))
+
+        return result.fetchall()  # Returns a list of tuples
+
+async def get_node_traffic(node_id: int):
+    try:
+        async with database.async_session() as session:
+            result = await session.execute(
+                text("""
+                    SELECT 
+                        node.long_name, packet.portnum, 
+                        COUNT(*) AS packet_count
+                    FROM packet
+                    JOIN node ON packet.from_node_id = node.node_id OR packet.to_node_id = node.node_id
+                    WHERE node.node_id = :node_id 
+                    AND packet.import_time >= DATETIME('now', '-1 day') 
+                    GROUP BY packet.portnum
+                    ORDER BY packet_count DESC;
+                """), {"node_id": node_id}
+            )
+
+            # Map the result to include node.long_name and packet data
+            traffic_data = [{
+                "long_name": row[0],  # node.long_name
+                "portnum": row[1],    # packet.portnum
+                "packet_count": row[2]  # COUNT(*) as packet_count
+            } for row in result.all()]
+
+            return traffic_data
+
+    except Exception as e:
+        # Log the error or handle it as needed
+        print(f"Error fetching node traffic: {str(e)}")
+        return []
+
+
 
 async def get_nodes(role=None, channel=None, hw_model=None):
     """

meshview/templates/base.html

@@ -10,6 +10,8 @@
         <link href="https://cdn.jsdelivr.net/npm/bootstrap@5.3.3/dist/css/bootstrap.min.css" rel="stylesheet" integrity="sha384-QWTKZyjpPEjISv5WaRU9OFeRpok6YctnYmDr5pNlyT2bRjXh0JMhjY6hW+ALEwIH" crossorigin="anonymous">
         <link rel="stylesheet" href="https://unpkg.com/leaflet@1.9.4/dist/leaflet.css" integrity="sha256-p4NxAoJBhIIN+hmNHrzRCf9tD/miZyoHS5obTRR9BMY=" crossorigin=""/>
         <script src="https://unpkg.com/leaflet@1.9.4/dist/leaflet.js" integrity="sha256-20nQCchB9co0qIjJZRGuk2/Z9VM+kNiyxNV1lvTlZBo=" crossorigin=""></script>
+
+
         {% block head %}
         {% endblock %}
 
@@ -36,8 +38,8 @@
         <br><div style="text-align:center"><strong>{{ site_config.site_title }} - {{ site_config.site_domain }}</strong></div>
         <div style="text-align: center;">{{ site_config.site_message }}</div>
         <div style="text-align:center">Quick Links:&nbsp;&nbsp;<a href="/nodelist">Nodes</a>&nbsp;-&nbsp;<a href="/chat">Conversations</a>&nbsp;-&nbsp;<a href="/firehose">See <strong>everything</strong> </a>
-            &nbsp;-&nbsp;Mesh Graph <a href="/graph/longfast">LF</a>&nbsp;-&nbsp;<a href="/graph/mediumslow">MS </a>&nbsp;-&nbsp;<a href="/stats">Stats </a>
-        &nbsp;-&nbsp;<a href="/net">Weekly Net</a>&nbsp;-&nbsp;<a href="/map">Map</a></div><br>
+            &nbsp;-&nbsp;Mesh Graph <a href="/nodegraph/LongFast">LF</a>&nbsp;-&nbsp;<a href="/nodegraph/MediumSlow">MS </a>&nbsp;-&nbsp;<a href="/stats">Stats </a>
+        &nbsp;-&nbsp;<a href="/net">Weekly Net</a>&nbsp;-&nbsp;<a href="/map">Map</a>&nbsp;-&nbsp;<a href="/top">Top Traffic</a></div><br>
         <div id="spinner" class="spinner-border secondary-primary htmx-indicator position-absolute top-50 start-50" role="status">
           <span class="visually-hidden">Loading...</span>
         </div>

meshview/templates/chat.html

@@ -21,7 +21,13 @@
 
 
 {% block body %}
-    <div class="container" hx-ext="sse" sse-connect="/chat_events" sse-swap="chat_packet" hx-swap="afterbegin">
+<script>
+    setTimeout(function() {
+        location.reload();
+    }, 30000); // 10 seconds
+</script>
+
+    <div class="container" >
         {% for packet in packets %}
         {% include 'chat_packet.html' %}
         {% else %}

meshview/templates/chat_packet.html

@@ -1,6 +1,6 @@
 <div class="row chat-packet">
-    <span class="col-1 timestamp" style="font-size: smaller;"><a href="/packet/{{packet.id}}">✉️</a> {{packet.from_node.channel}}<div>{{packet.import_time.strftime('%-I:%M:%S %p - %d-%m-%Y')}}</div></span>
-    <!-- <span class="col-1 timestamp"></span> -->
-    <span class="col-4 username" style="text-align: center;"><a href="/packet_list/{{packet.from_node_id}}">{{packet.from_node.long_name or (packet.from_node_id | node_id_to_hex) }}</a></span>
-    <span class="col message" style="text-align: right;">{{packet.payload}}</span>
+    <span class="col-2 timestamp">{{packet.import_time.strftime('%-I:%M:%S %p - %d-%m-%Y')}} </span>
+    <span class="col-1 timestamp"><a href="/packet/{{packet.id}}">✉️</a> {{packet.from_node.channel}}</span>
+    <span class="col-2 username"><a href="/packet_list/{{packet.from_node_id}}">{{packet.from_node.long_name or (packet.from_node_id | node_id_to_hex) }}</a></span>
+    <span class="col-6 message">{{packet.payload}}</span>
 </div>

meshview/templates/firehose.html

@@ -1,7 +1,71 @@
 {% extends "base.html" %}
 
 {% block body %}
-<div class="container" hx-ext="sse" sse-connect="/events{% if portnum %}?portnum={{portnum}}{% endif%}">
+
+<script>
+    let refreshInterval;
+
+    function updateURLWithPort() {
+        let selectedPort = document.querySelector('select[name="portnum"]').value;
+        let url = new URL(window.location.href);
+        url.searchParams.set('portnum', selectedPort);
+
+        // Save scroll position before refreshing
+        localStorage.setItem("scrollPosition", window.scrollY);
+
+        window.location.href = url.toString();
+    }
+
+    function startAutoRefresh() {
+        refreshInterval = setInterval(updateURLWithPort, 5000);
+        localStorage.setItem("autoRefresh", "true");
+        updateButtonState(true);
+    }
+
+    function stopAutoRefresh() {
+        clearInterval(refreshInterval);
+        localStorage.setItem("autoRefresh", "false");
+        updateButtonState(false);
+    }
+
+    function toggleAutoRefresh() {
+        let isEnabled = localStorage.getItem("autoRefresh") === "true";
+        if (isEnabled) {
+            stopAutoRefresh();
+        } else {
+            startAutoRefresh();
+        }
+    }
+
+    function updateButtonState(isEnabled) {
+        let button = document.getElementById("auto-refresh-button");
+        button.innerText = isEnabled ? "Disable Auto-Refresh" : "Enable Auto-Refresh";
+    }
+
+    function restoreScrollPosition() {
+        let scrollPosition = localStorage.getItem("scrollPosition");
+        if (scrollPosition !== null) {
+            window.scrollTo(0, parseInt(scrollPosition, 10));
+        }
+    }
+
+    document.addEventListener("DOMContentLoaded", function () {
+        document.querySelector('select[name="portnum"]').addEventListener('change', updateURLWithPort);
+        document.getElementById("auto-refresh-button").addEventListener('click', toggleAutoRefresh);
+
+        // Restore auto-refresh state
+        let isEnabled = localStorage.getItem("autoRefresh") === "true";
+        updateButtonState(isEnabled);
+        if (isEnabled) {
+            startAutoRefresh();
+        }
+
+        // Restore scroll position
+        restoreScrollPosition();
+    });
+</script>
+
+<div class="container">
     <form class="row">
         {% set options = {
                 1: "Text Message",
@@ -9,31 +73,26 @@
                 4: "Node Info",
                 67: "Telemetry",
                 71: "Neighbor Info",
+                70: "Trace Route",
             }
         %}
         <select name="portnum" class="col-2 m-2">
-            <option
-                value = ""
-                {% if portnum not in options %}selected{% endif %}
-            >All</option>
+            <option value="" {% if portnum not in options %}selected{% endif %}>All</option>
             {% for value, name in options.items() %}
-            <option
-                value="{{value}}"
-                {% if value == portnum %}selected{% endif %}
-            >{{ name }}</option>
+            <option value="{{ value }}" {% if value == portnum %}selected{% endif %}>{{ name }}</option>
             {% endfor %}
         </select>
-        <input type="Submit" value="Refresh" class="col-2 m-2"/>
+        <button type="button" id="auto-refresh-button" class="col-2 m-2 btn btn-primary">Enable Auto-Refresh</button>
     </form>
     <div class="row">
-        <div class="col-xs" id="packet_list" sse-swap="packet" hx-swap="afterbegin">
+        <div class="col-xs" id="packet_list">
             {% for packet in packets %}
-            {% include 'packet.html' %}
+                {% include 'packet.html' %}
             {% else %}
-            No packets found.
+                No packets found.
             {% endfor %}
         </div>
-        <!-- <div class="col-6 sticky-top" id="packet_details" style="height: 95vh; overflow: scroll"> -->
     </div>
 </div>
+
 {% endblock %}

meshview/templates/map.html

@@ -93,11 +93,11 @@
             let isRouter = node.role.toLowerCase().includes("router");
 
             let markerOptions = {
-                radius: isRouter ? 8 : 7,
-                color: isRouter ? "black" : color,
+                radius: isRouter ? 9 : 7,
+                color: "white",
                 fillColor: color,
-                fillOpacity: 0.6,
-                weight: isRouter ? 1 : 0
+                fillOpacity: 1,
+                weight: .7,
             };
 
             var popupContent = `

meshview/templates/net.html

@@ -24,7 +24,7 @@
     <div class="container" > Weekly Mesh check-in. We will keep it open on every Wednesday from 5:00pm for checkins so you do not have to rush.<br>
         The message format should be (LONG NAME) - (CITY YOU ARE IN) #BayMeshNet.<br><br>
  </div>
-    <div class="container" hx-ext="sse" sse-connect="/net_events" sse-swap="net_packet" hx-swap="afterbegin">
+    <div class="container">
         {% for packet in packets %}
         {% include 'chat_packet.html' %}
         {% else %}

meshview/templates/node.html

@@ -1,22 +1,28 @@
 {% extends "base.html" %}
 
 {% block css %}
+    /* Styles for the node info card */
     #node_info {
-        height:100%;
+        height: 100%;
     }
-    #map{
-        height:100%;
+
+    /* Styles for the map */
+    #map {
+        height: 100%;
         min-height: 400px;
     }
-    #packet_details{
+
+    /* Styles for packet details section */
+    #packet_details {
         height: 95vh;
         overflow: scroll;
         top: 3em;
     }
+
+    /* Ensure inline display for details */
     div.tab-pane > dl {
         display: inline-block;
-       }
-
+    }
 {% endblock %}
 
 {% block body %}
@@ -28,34 +34,26 @@
          {% if node %}
          hx-ext="sse"
          sse-connect="/events?node_id={{node_id}}{% if portnum %}&portnum={{portnum}}{% endif %}"
-        {% endif %}
-        >
+         {% endif %}
+    >
         <div class="row">
             <div class="col mb-3">
+                <!-- Node Information Card -->
                 <div class="card" id="node_info">
                     {% if node %}
-                    <div class="card-header" id="node_color">
-                    <strong style="margin-right: 1em ; margin-left: 1em; font-size: x-large;">{{node.short_name}}</strong>
-                    <p style="margin-bottom: 0px; font-size: large; font-weight: bold;">{{node.long_name}}</p>
+                    <div class="card-header">
+                        {{node.long_name}} ({{node.node_id|node_id_to_hex}})
                     </div>
                     <div class="card-body">
-                        <dl >
-                            {% if trace %}
-                            <dd id="map"></dd>
-                            {% endif %}
-                            <dt>NodeID</dt>
-                            <dd>{{node.node_id|node_id_to_hex}}</dd>
-                            <dt>Channel</dt>
-                            <dd>{{node.channel}}</dd>
+                        <dl>
+                            <dt>ShortName</dt>
+                            <dd>{{node.short_name}}</dd>
                             <dt>HW Model</dt>
                             <dd>{{node.hw_model}}</dd>
                             <dt>Role</dt>
                             <dd>{{node.role}}</dd>
-                            {% if node.firmware %}
-                            <dt>Firmware</dt>
-                            <dd>{{node.firmware}}</dd>
-                            {% endif %}
                         </dl>
+                        <a href="/top?node_id={{node.node_id}}" >Get node traffic totals</a>
                         {% include "node_graphs.html" %}
                     </div>
                     {% else %}
@@ -65,69 +63,146 @@
                     {% endif %}
                 </div>
             </div>
-        </div>
 
-        <div class="row">
-            <div class="col">
-           {% include "buttons.html" %}
+            <div class="col mb-3">
+                <!-- Map Container -->
+                <div id="map"></div>
             </div>
         </div>
 
         <div class="row">
             <div class="col">
-            {% include 'packet_list.html' %}
+                <!-- Additional buttons can be included here -->
             </div>
-            
+        </div>
+
+        <div class="row">
+            <div class="col">
+                {% include 'packet_list.html' %}
+            </div>
+            <!-- <div class="col sticky-top" id="packet_details"></div> -->
         </div>
     </div>
 </div>
-<script>
-    var node_color = document.getElementById('node_color');
-    var node_id = '{{node.node_id | node_id_to_hex}}';
-    var color = node_id.slice(-6);
-    var bg_color = "#"+color;
-    node_color.style.background = bg_color;
-    var hex = /^#?([a-f\d]{2})([a-f\d]{2})([a-f\d]{2})$/i.exec(bg_color);
-    var text_color = [
-        parseInt(hex[1], 16),
-        parseInt(hex[2], 16),
-        parseInt(hex[3], 16),
-    ];
-
-    const brightness = Math.round(((parseInt(text_color[0]) * 299) +
-                      (parseInt(text_color[1]) * 587) +
-                      (parseInt(text_color[2]) * 114)) / 1000);
-    
-    if (brightness > 125) {
-        var textColor = '#212529'
-        node_color.style.color = textColor
-    }
-     /*                  const textColor = (brightness > 125) ? '#212529' : 'white';
-    node_color.style.color = textColor */
-
-
-</script>
 
 {% if trace %}
 <script>
-    var trace = {{trace | tojson}};
-    var map = L.map('map').setView(trace[0], 13);
-    var markers = L.featureGroup();
+    var trace = {{ trace | tojson }}; // Load trace data into JavaScript
+    var map = L.map('map').setView(trace[0], 13); // Initialize map centered at first trace point
+    var markers = L.featureGroup(); // Create a feature group for markers
     markers.addTo(map);
+
+    // Add tile layer (OpenStreetMap)
     L.tileLayer('https://tile.openstreetmap.org/{z}/{x}/{y}.png', {
         maxZoom: 19,
         attribution: '&copy; <a href="http://www.openstreetmap.org/copyright">OpenStreetMap</a>'
     }).addTo(map);
-    L.polyline(trace).addTo(map);
-    L.marker(trace[0]).addTo(markers);
+
+    // Draw a polyline along the trace path
+    L.polyline(trace, { color: 'blue', weight: 1}).addTo(map);
+
+    // Add a red circle marker for the starting node with a tooltip
+    var startMarker = L.circleMarker(trace[0], {
+        radius: 8,
+        color: 'red',
+        weight: 1,
+        fillColor: 'red',
+        fillOpacity: 0.5
+    }).addTo(markers) // Add to feature group
+      .bindTooltip(`
+        <b>{{node.long_name}}</b><br/>
+        <b>Short:</b> {{node.short_name}}<br/>
+        <b>Channel:</b> {{node.channel}}<br/>
+        <b>Hardware:</b> {{node.hw_model}}<br/>
+        <b>Role:</b> {{node.role}}<br/>
+        <b>Firmware:</b> {{node.firmware}}<br/>
+        <b>Coordinates:</b> [{{node.last_lat}} , {{node.last_long}}]
+      `, {permanent: false, direction: 'top', opacity: 0.9});
+
+    // Function to calculate distance and convert to miles
+    function getDistanceInMiles(latlng1, latlng2) {
+        var meters = latlng1.distanceTo(latlng2); // Get distance in meters
+        return meters * 0.000621371; // Convert meters to miles
+    }
+
     {% for n in neighbors %}
-    var m = L.circleMarker({{n.location | tojson}});
-    m.bindPopup('SNR: {{n.snr}}<br/><a href="/packet_list/{{n.node_id}}">[{{n.short_name}}] {{n.long_name}} {{n.node_id | node_id_to_hex}}</a>');
-    m.addTo(markers);
-    L.polyline([trace[0], {{n.location | tojson}}], {color: 'red'}).addTo(map);
-    map.fitBounds(markers.getBounds().pad(.7));
+        var neighborLatLng = L.latLng([{{n.location[0]}}, {{n.location[1]}}]);
+        var startLatLng = L.latLng(trace[0]);
+
+        // Calculate distance in miles with 1 decimal place
+        var distanceMiles = getDistanceInMiles(startLatLng, neighborLatLng).toFixed(1);
+
+        // Create a blue circle marker for each neighbor node
+        var m = L.circleMarker(neighborLatLng, {
+            radius: 6,
+            color: 'blue',
+            weight: 1,
+            fillColor: 'blue',
+            fillOpacity: 0.5
+        }).addTo(markers) // Add to feature group
+          .bindTooltip(`
+            <b>Neighbour: [{{n.short_name}}] {{n.long_name}}</b> <br/>
+            <b>SNR:</b> {{n.snr}} <br/>
+            <b>Distance:</b> ${distanceMiles} miles <br/>
+        `, {permanent: false, direction: 'top', opacity: 0.9});
+
+        // Draw a polyline from the first trace point to each neighbor node
+        L.polyline([startLatLng, neighborLatLng], {
+            color: 'grey',
+            weight: 1
+        }).addTo(map);
+
     {% endfor %}
+
+    // Add a legend to the map
+    var legend = L.control({ position: 'bottomleft' });
+
+    legend.onAdd = function(map) {
+        var div = L.DomUtil.create('div', 'info legend');
+        div.style.background = 'white';
+        div.style.padding = '8px';
+        div.style.border = '1px solid black';
+        div.style.borderRadius = '5px';
+        div.style.boxShadow = '0 0 5px rgba(0,0,0,0.3)';
+        div.style.color = 'black'; // Ensure text is black
+        div.style.textAlign = 'left'; /* Ensure left alignment */
+        div.innerHTML = `
+        <b>Legend</b><br>
+        <svg width="16" height="16">
+            <circle cx="8" cy="8" r="6" fill="blue" stroke="blue" stroke-width="1" fill-opacity="0.4"/>
+        </svg> Neighbor Node<br>
+        <svg width="20" height="20">
+            <circle cx="10" cy="10" r="8" fill="red" stroke="red" stroke-width="1" fill-opacity="0.4"/>
+        </svg> Home Node<br>
+        <svg width="20" height="4">
+            <line x1="0" y1="2" x2="20" y2="2" stroke="grey" stroke-width="2"/>
+        </svg> Connection to Neighbors<br>
+        <svg width="20" height="4">
+            <line x1="0" y1="2" x2="20" y2="2" stroke="blue" stroke-width="2"/>
+        </svg> Path taken by node
+        `;
+        return div;
+    };
+
+    legend.addTo(map);
+
+    // Ensure the map adjusts to fit all markers and trace points
+    setTimeout(() => {
+        if (markers.getLayers().length > 0 || trace.length > 0) {
+            var bounds = markers.getBounds(); // Get bounds from markers
+
+            // Ensure trace points are included in the bounds
+            trace.forEach(point => {
+                bounds.extend(point);
+            });
+
+            map.fitBounds(bounds.pad(0.1), { maxZoom: 15 });
+        }
+    }, 200); // Slightly longer delay to ensure all elements are fully loaded
+
 </script>
+
+
 {% endif %}
 
 {% endblock %}

meshview/templates/node2.html

@@ -0,0 +1,58 @@
+{% extends "base.html" %}
+
+{% block css %}
+    #node_info {
+        height:100%;
+    }
+    #map{
+        height:100%;
+        min-height: 400px;
+    }
+    #packet_details{
+        height: 95vh;
+        overflow: scroll;
+        top: 3em;
+    }
+    div.tab-pane > dl {
+        display: inline-block;
+       }
+{% endblock %}
+
+{% block body %}
+
+{% include "search_form.html" %}
+
+<div class="row">
+            <div class="col mb-3">
+                <div class="card" id="node_info">
+                    {% if node %}
+                    <div class="card-header">
+                        {{node.long_name}}
+                    </div>
+                    <div class="card-body">
+                        <dl >
+                            <dt>ShortName</dt>
+                            <dd>{{node.short_name}}</dd>
+                            <dt>HW Model</dt>
+                            <dd>{{node.hw_model}}</dd>
+                            <dt>Role</dt>
+                            <dd>{{node.role}}</dd>
+                        </dl>
+                    </div>
+                    {% else %}
+                    <div class="card-body">
+                        A NodeInfo has not been seen.
+                    </div>
+                    {% endif %}
+                </div>
+            </div>
+            <div class="row">
+            <div class="col">
+            {% include 'packet_list.html' %}
+            </div>
+    </div>
+            <div class="col mb-3">
+                <div id="map"></div>
+            </div>
+        </div>
+{% endblock %}

meshview/templates/node_traffic.html

@@ -0,0 +1,105 @@
+{% extends "base.html" %}
+
+{% block css %}
+.table-title {
+            font-size: 2rem;
+            text-align: center;
+            margin-bottom: 20px;
+        }
+
+        .traffic-table {
+            width: 50%;
+            border-collapse: collapse;
+            margin: 0 auto;
+            font-family: Arial, sans-serif;
+        }
+
+        .traffic-table th,
+        .traffic-table td {
+            padding: 10px 15px;
+            text-align: left;
+            border: 1px solid #474b4e;
+        }
+
+        .traffic-table th {
+            background-color: #272b2f;
+            color: white;
+        }
+
+        .traffic:nth-of-type(odd) {
+            background-color: #272b2f; /* Lighter than #2a2a2a */
+        }
+
+        .traffic {
+            border: 1px solid #474b4e;
+            padding: 8px;
+            margin-bottom: 4px;
+            border-radius: 8px;
+        }
+
+        .traffic:nth-of-type(even) {
+            background-color: #212529; /* Slightly lighter than the previous #181818 */
+        }
+
+        .footer {
+            text-align: center;
+            margin-top: 20px;
+        }
+
+{% endblock %}
+
+{% block body %}
+    <section>
+        <h2 class="table-title">{{ traffic[0].long_name }} (last 24 hours)</h2>
+        <table class="traffic-table">
+            <thead>
+                <tr>
+                    <th>Port Number</th>
+                    <th>Packet Count</th>
+                </tr>
+            </thead>
+            <tbody>
+                {% for port in traffic %}
+                    <tr class="traffic">
+                        <td>
+                            {% if port.portnum == 1 %}
+                                TEXT_MESSAGE_APP
+                            {% elif port.portnum == 3 %}
+                                POSITION_APP
+                            {% elif port.portnum == 4 %}
+                                NODEINFO_APP
+                            {% elif port.portnum == 5 %}
+                                ROUTING_APP
+                            {% elif port.portnum == 67 %}
+                                TELEMETRY_APP
+                            {% elif port.portnum == 70 %}
+                                TRACEROUTE_APP
+                            {% elif port.portnum == 71 %}
+                                NEIGHBORINFO_APP
+                            {% elif port.portnum == 73 %}
+                                MAP_REPORT_APP
+                            {% elif port.portnum == 0 %}
+                                UNKNOWN_APP
+                            {% elif port.portnum == 0 %}
+                                UNKNOWN_APP
+                            {% elif port.portnum == 0 %}
+                                UNKNOWN_APP
+                            {% else %}
+                                {{ port.portnum }}
+                            {% endif %}
+                        </td>
+                        <td>{{ port.packet_count }}</td>
+                    </tr>
+                {% else %}
+                    <tr>
+                        <td colspan="2">No traffic data available for this node.</td>
+                    </tr>
+                {% endfor %}
+            </tbody>
+        </table>
+    </section>
+
+    <footer class="footer">
+        <a href="/top">Back to Top Nodes</a>
+    </footer>
+{% endblock %}

meshview/templates/nodegraph.html

@@ -0,0 +1,190 @@
+{% extends "base.html" %}
+
+{% block head %}
+    <script src="https://cdn.jsdelivr.net/npm/echarts/dist/echarts.min.js"></script>
+{% endblock %}
+
+{% block css %}
+#mynetwork {
+    width: 100%;
+    height: 100vh;
+    max-width: 2000px;
+    max-height: 2000px;
+    border: 1px solid lightgray;
+    background-color: white;
+}
+
+.legend {
+    position: absolute;
+    bottom: 10px;
+    left: 10px;
+    background-color: rgba(255, 255, 255, 0.8);
+    padding: 5px;
+    border-radius: 5px;
+    border: 1px solid #ccc;
+    font-size: 12px;
+    color: #333;
+}
+
+#node-info {
+    position: absolute;
+    bottom: 10px;
+    right: 10px;
+    background-color: rgba(255, 255, 255, 0.9);
+    padding: 10px;
+    border-radius: 5px;
+    border: 1px solid #ccc;
+    font-size: 14px;
+    color: #333;
+    width: 250px;
+    max-height: 200px;
+    overflow-y: auto;
+}
+{% endblock %}
+
+{% block body %}
+<div id="mynetwork"></div>
+
+<!-- Legend -->
+<div class="legend">
+    <div><span style="background-color: #ff5733; width: 20px; height: 20px; display: inline-block; border-radius: 50%; margin-right: 5px;"></span> Traceroute</div>
+    <div><span style="background-color: #3388ff; width: 20px; height: 20px; display: inline-block; border-radius: 50%; margin-right: 5px;"></span> NeighborInfo</div>
+</div>
+
+<!-- Node Information Panel -->
+<div id="node-info">
+    <b>Long Name: </b> <span id="node-long-name"></span></br>
+    <b>Short Name: </b><span id="node-short-name"></span></br>
+    <b>Role: </b><span id="node-role"></span></br>
+    <b>Hardware Model: </b><span id="node-hw-model"></span>
+</div>
+
+<<script type="text/javascript">
+    // Initialize chart
+    var chart = echarts.init(document.getElementById('mynetwork'));
+
+
+    var nodes = [
+    {% for node in nodes %}
+        {
+            name: '{{ node.node_id }}',
+            value: '{{ node.long_name | escape }}',
+            symbol: 'rect',
+            symbolSize: [null, 40],
+            label: {
+                show: true,
+                position: 'inside',
+                color: '#000',
+                padding: [5, 10],
+                formatter: function(params) { return params.data.value; },
+                backgroundColor: '#f0f0f0',
+                borderColor: '#999',
+                borderWidth: 1,
+                borderRadius: 5
+            },
+            long_name: '{{ node.long_name | escape }}',  // Add long name
+            short_name: '{{ node.short_name | escape }}',  // Add short name
+            role: '{{ node.role | escape }}',  // Add role
+            hw_model: '{{ node.hw_model | escape }}'  // Add hardware model
+        }{% if not loop.last %},{% endif %}
+    {% endfor %}
+    ];
+
+    // Sample edge data (this will be passed from Python backend)
+    var edges = [
+    {% for edge in edges %}
+        {
+            source: '{{ edge.from }}',
+            target: '{{ edge.to }}',
+            originalColor: '{{ edge.originalColor }}', // Store original color
+            lineStyle: {
+                color: '#d3d3d3', // Set all edges to light gray by default
+                width: 2, // Default width for all edges
+                opacity: 0.5 // Dim edges by default
+            }
+        }{% if not loop.last %},{% endif %}
+    {% endfor %}
+    ];
+
+    var option = {
+        backgroundColor: 'white',
+        tooltip: {
+            formatter: function(params) {
+                // Only show long_name on hover
+                return params.data.long_name + ' - ' + params.data.short_name;
+
+            }
+        },
+        animationDurationUpdate: 1500,
+        animationEasingUpdate: 'quinticInOut',
+        legend: {
+            data: ['Traceroute', 'NeighborInfo'],
+            selectedMode: false, // Disable item selection
+            left: 'center',
+            bottom: '5%',
+            orient: 'vertical', // Stack legend vertically
+            textStyle: {
+                fontSize: 12,
+                color: '#333'
+            },
+            itemWidth: 10,
+            itemHeight: 10,
+            padding: [5, 15]
+        },
+        series: [
+            {
+                type: 'graph',
+                layout: 'force',
+                data: nodes,
+                links: edges,
+                roam: true,
+                force: {
+                    repulsion: 500,
+                    edgeLength: [100, 200],
+                    gravity: 0.05
+                },
+                lineStyle: {
+                    width: 2,
+                    curveness: 0
+                }
+            }
+        ]
+    };
+
+    chart.setOption(option);
+
+    // Event listener for node clicks
+    chart.on('click', function(params) {
+        if (params.dataType === 'node') {
+            var selectedNode = params.data.name;
+
+            // Update edges for the selected node: highlight connected edges
+            var updatedEdges = edges.map(edge => {
+                if (edge.source === selectedNode || edge.target === selectedNode) {
+                    return {
+                        ...edge,
+                        lineStyle: {
+                            color: edge.originalColor, // Use original color for selected edges (blue or red)
+                            width: 2, // Thinner width for highlighted edges
+                            opacity: 1 // Full opacity for selected edges
+                        }
+                    };
+                } else {
+                    return edge; // Keep the non-selected edges in light gray
+                }
+            });
+
+            // Update the chart with highlighted edges
+            chart.setOption({
+                series: [{ links: updatedEdges }]
+            });
+
+            // Update the node information panel
+            document.getElementById('node-long-name').innerText = params.data.long_name;
+            document.getElementById('node-short-name').innerText = params.data.short_name;
+            document.getElementById('node-role').innerText = params.data.role;
+            document.getElementById('node-hw-model').innerText = params.data.hw_model;
+        }
+    });
+</script>
+{% endblock %}

meshview/templates/packet.html

@@ -13,7 +13,6 @@
             {%- endif -%}
             )
         </span>
-        -&gt;
         <span {% if to_me %} class="fw-bold" {% endif %}>
             {{packet.to_node.long_name}}(
             {%- if not to_me -%}
@@ -29,8 +28,7 @@
     <div class="card-body">
         <div class="card-title">
         {{packet.id}}
-        <a href="/packet_details/{{packet.id}}" hx-target="#packet_details" hx-get="/packet_details/{{packet.id}}" hx-swap="innerHTML scroll:top">🔎</a>
-            <a href="/packet/{{packet.id}}">🔗</a>
+            <a href="/packet/{{packet.id}}">🔎</a>
         </div>
         <div class="card-text text-start">
             <dl>
@@ -41,7 +39,7 @@
                 <dt>payload</dt>
                 <dd>
                     {% if packet.pretty_payload %}
-                    <div>{{packet.pretty_payload}}<div>
+                    <div>{{packet.pretty_payload}}</div>
                     {% endif %}
                     {% if  packet.raw_mesh_packet.decoded and packet.raw_mesh_packet.decoded.portnum == 70 %}
                         <ul>

meshview/templates/packet_details.html

@@ -36,25 +36,103 @@
 
 {% if map_center %}
 <script>
-    var details_map = L.map('details_map').setView({{map_center | tojson}}, 12);
+    var details_map = L.map('details_map').setView({{ map_center | tojson }}, 12);
     var markers = L.featureGroup();
     markers.addTo(details_map);
+
     L.tileLayer('https://tile.openstreetmap.org/{z}/{x}/{y}.png', {
         maxZoom: 19,
         attribution: '&copy; <a href="http://www.openstreetmap.org/copyright">OpenStreetMap</a>'
     }).addTo(details_map);
+
+    // Function to calculate distance in miles
+    function getDistanceInMiles(latlng1, latlng2) {
+        var meters = latlng1.distanceTo(latlng2); // Get distance in meters
+        return meters * 0.000621371; // Convert meters to miles
+    }
+
     {% if from_node_cord %}
-    L.marker({{from_node_cord | tojson}}).addTo(markers);
+    var fromNodeLatLng = L.latLng({{ from_node_cord | tojson }});
+
+    var fromNode = L.circleMarker(fromNodeLatLng, {
+        radius: 8,
+        color: 'red',
+        weight: 1,
+        fillColor: 'red',
+        fillOpacity: 0.4
+    }).addTo(markers);
+
+    // Add tooltip for the from_node_cord
+    fromNode.bindTooltip(`
+        Sent by:   <b>{{node.long_name}}</b><br/>
+        <b>Short:</b> {{node.short_name}}<br/>
+        <b>Channel:</b> {{node.channel}}<br/>
+        <b>Hardware:</b> {{node.hw_model}}<br/>
+        <b>Role:</b> {{node.role}}<br/>
+        <b>Firmware:</b> {{node.firmware}}<br/>
+        <b>Coordinates:</b> [{{node.last_lat}}, {{node.last_long}}]
+    `, { permanent: false, direction: 'top', opacity: 0.9 });
     {% endif %}
-    var radioTower = L.icon({
-        iconUrl: 'data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAGAAAABgCAYAAADimHc4AAAACXBIWXMAAAsTAAALEwEAmpwYAAAGmklEQVR4nO2ca4hVVRTH/45mRkNUItO7Gz1msLAo7An2cjJIErIiQfL2NEqaCqMPfcjoSxRlDyjs4dWihLQGDD8UvQilJyY9oZKjWUlphWM2TTPOxIIzcLncc9b/nL33OfvO7B/sT3PX3ms/zlprr733AIFAIBAIBAKBQCAQCATc0AVgA4DJng/wVQAmYowxB8BfAEYA9MBfzgEwDOB9AB0YI9wNYCgefCm7ARwKP9lYp+c2AGehxbmnrkP15VH4x9VN9NwHYDZalLsSBl/KAIAT4Q8HAPg+Qdd/AHSjxbgjtqUjKWUN/KFH0VUm4TK0CJc22PykMgzg/LKVjf3RbkJfmYQz4TnHk52RstcT+7qY1FfKVgBT4SlTAHxGduQXjyKMCQDuA7Cf1P0dAJPgIU+QHRBndxT844o46mH6sAyeMQPAIKH4DgAV+Mvs2NZr/ZAobjo8oQ3Ax4TSuwB0ojV27f8S/fkwNl+lcxuhrNjXy9E6LCZN0c1lK3oggF8JRR9C6/Ei0a+dcfBRGrcSSn5CZhcnAJhXgM7zSNNxEIDviP7J11IKE1O27/WmZybZ2bXx5kwykq6zna+RK/cSYgJ+LCssvYZQ7jkyD7OhTkYykkVkO9fHbWu8QvRzAUqgftCalX4ARxD1rG4iO9+BvvObtLOSkDsWwH9KX99CCfmTAUWpFUQ9N6Vs+cXBF5HtXETIv6T0VfZA01AgCwnb30msrL9T6ugpKNspOamjiY3msE8haa+izHtEHS8rddg6OWOynasy+o9SzZCEcHsUZZYQB/TDhHO73dLZxAjxxZ5CHK2m1bGvqGjoZEURGdhjlDqeIgbldUsdmkR8sVKWE6l2bdGchgK4VlFiCzEgozckRhLKdgAHW9S5HcBPhMnTNoxfKnUwDt2YhxUlXlDkZxGrcaEDvRcR7V6g1LFKkZcv2zlvGtr/B4mdZZsDvdviutPafkCp407isMY5HylKyApP4w1F/jFCBwlh1wHoi0svmeperrQtdaZxkSL/DQpAS1B1Gcp3E4P/RxO5P+O/abl+kwGcTpx5OGenooSWftilyJ+kyK9LkZUkm0kE95sifyQRijqnX1FCyzJqp03tinxfiqzsT9JoV9oW3dKYosjLdRznaDkg7fazduaaRkWRHSHOnE1W8GRFXhaHczQTcrgiv8NgAqrEBMhv0tD2H2lMVeR/RgFsNVyBWwzyKTViAuQ3abybIrtZkT1BaVsCDOdsVpS4UJF/NSUfc4YiGxETIL9J49wUWdEtjYuVtjehANYqStyiyC9JkFttwf6zX+H6nJvIxYZfnxW0neyTivzpTWT642SXqf1n/cCMhKuI2nXJp5V25YqjcxYoSnxB1PFtg8wjhAxj/7OsxDVNTJd2W+Irpd0rUQBdihL7iUjo3oYd7GFEu4z9Z/0A4vz/YIb7ntOIdLSWhreGFkreQHRmb/zbpUR7Wew/6wcQZ25HN1AS4ZjcgZKvujBWWjieWxY/hGPu52Sx/6wfEI6Ld79y6K7xgdKe3A4vjOsUZQYJp9oOYC7ZXhb7n8UPjGZIuwhzpb0fkOvthdGu5GSkPG6xvSjHBDB+AOTlrBVKW7+X8QD9eUWpPZae81RyDH4WP8CYqf4CFxvNecQA2LCLVYMJYPxA1nC1WZF9RSlob8IGiOseLux/Vj+QxCwi9JTcUmnMJQZhk+E/v4gMJoD1A0l+7geiDe0I1imyc/yUULLHkf2PiAnK6we0m3tS3oYHdBOKSqx9tgP7XyNMVB4/sJTo05BHT20pRyUXow7JWG+NGFxmkmzf/C4t8kmiI87paEm6rPd9IsK8MGYqK88odW4jzq4L50ZFaXnuk4VKhoG17Qc6UjaaQ/HBjJckXd2TA5CsVDOYFhd+4P6EuiST6y1yofbrJivm1Bx11TIMqm0/MPpwcHtDPb2+PM5Oo7PhQcSzOeuJMpgVF35AuL6ujo2Wb2w7ZWac7+8jH+o1kmdAXewHJGj4PA4gmEMjr5hjYC+rOUyKCz+AONb39n8EuaKWYzBd+IFxS5TDnLjyA+OOisFAusoLjSuqBqbElR8YV9QMBjH4AQtEBmYk+AFDKhYcafADBlQthJLBDxhQs+BEgx8wILIQRgY/kJOKxY1U8AM5qFpMJQQ/kIOaxU1U8AM5iCymEYIfyEjFQSIt+IGSTUYt5IXKHayqg0kds0QO0sjBD3gwUJGDiR1zVB2aiuAHSh6kavAD5ZqJikPzFggEAoFAIBAIBALIw/+mMY6LjjUT0gAAAABJRU5ErkJggg==',
-        iconSize: [32, 32],
-    });
+
     {% for u in uplinked_nodes %}
-    L.marker([{{u.lat}}, {{u.long}}], {icon: radioTower})
-        .bindPopup('[{{u.short_name}}] {{u.long_name}}<br/>Hops: {{u.hops}}<br/>SNR: {{u.snr}}<br/>RSSI: {{u.rssi}}')
-        .addTo(markers);
-    details_map.fitBounds(markers.getBounds().pad(.03), {animate: false});
+        var uplinkNodeLatLng = L.latLng([{{ u.lat }}, {{ u.long }}]);
+
+        // Calculate distance in miles
+        var distanceMiles = getDistanceInMiles(fromNodeLatLng, uplinkNodeLatLng).toFixed(1);
+
+        var node = L.circleMarker(uplinkNodeLatLng, {
+            radius: 6,
+            color: 'blue',
+            weight: 1,
+            fillColor: 'blue',
+            fillOpacity: 0.4
+        }).addTo(markers);
+
+        // Add a tooltip with node details and distance
+        node.bindTooltip(`
+            Seen by: <b>[{{ u.short_name }}] {{ u.long_name }}</b><br/>
+            <b>Hops:</b> {{ u.hops }}<br/>
+            <b>SNR:</b> {{ u.snr }}<br/>
+            <b>RSSI:</b> {{ u.rssi }}<br/>
+            <b>Distance:</b> ${distanceMiles} miles <br/>
+            <b>Coordinates:</b> [{{u.lat}}, {{u.long}}]
+        `, { permanent: false, direction: 'top', opacity: 0.9 });
     {% endfor %}
+
+    // Ensure markers are added before adjusting map bounds
+    setTimeout(() => {
+        if (markers.getLayers().length > 0) {
+            details_map.fitBounds(markers.getBounds().pad(0.1), { animate: true });
+        }
+    }, 500); // Delay to ensure markers load
+
+    // Add a legend to details_map
+    var legend = L.control({ position: 'bottomleft' });
+
+    legend.onAdd = function(map) {
+        var div = L.DomUtil.create('div', 'info legend');
+        div.style.background = 'white';
+        div.style.padding = '8px';
+        div.style.border = '1px solid black';
+        div.style.borderRadius = '5px';
+        div.style.boxShadow = '0 0 5px rgba(0,0,0,0.3)';
+        div.style.color = 'black'; // Ensure text is black
+        div.style.textAlign = 'left'; /* Ensure left alignment */
+        div.innerHTML = `
+            <b>Legend</b><br>
+            <svg width="16" height="16">
+                <circle cx="8" cy="8" r="6" fill="blue" stroke="blue" stroke-width="1" fill-opacity="0.4"/>
+            </svg> Receiving Node<br>
+            <svg width="20" height="20">
+                <circle cx="10" cy="10" r="8" fill="red" stroke="red" stroke-width="1" fill-opacity="0.4"/>
+            </svg> Sending Node<br>
+        `;
+        return div;
+    };
+
+    legend.addTo(details_map);
 </script>
+
+
+
 {% endif %}

meshview/templates/packet_list.html

@@ -1,4 +1,4 @@
-<div class="col" id="packet_list" sse-swap="{{packet_event | default('packet')}}" hx-swap="afterbegin">
+<div class="col" id="packet_list">
     {% for packet in packets %}
     {% include 'packet.html' %}
     {% else %}

meshview/templates/top.html

@@ -0,0 +1,65 @@
+{% extends "base.html" %}
+{% block css %}
+.table-title {
+            font-size: 2rem;
+            text-align: center;
+            margin-bottom: 20px;
+        }
+
+        .traffic-table {
+            width: 60%;
+            border-collapse: collapse;
+            margin: 0 auto;
+            font-family: Arial, sans-serif;
+        }
+
+        .traffic-table th,
+        .traffic-table td {
+            padding: 10px 15px;
+            text-align: left;
+            border: 1px solid #474b4e;
+        }
+
+        .traffic-table th {
+            background-color: #272b2f;
+            color: white;
+        }
+
+        .traffic:nth-of-type(odd) {
+            background-color: #272b2f; /* Lighter than #2a2a2a */
+        }
+
+        .traffic {
+            border: 1px solid #474b4e;
+            padding: 8px;
+            margin-bottom: 4px;
+            border-radius: 8px;
+        }
+
+        .traffic:nth-of-type(even) {
+            background-color: #212529; /* Slightly lighter than the previous #181818 */
+        }
+{% endblock %}
+
+{% block body %}
+    <h2 class="table-title">Top Traffic Nodes (last 24 hours)</h2>
+    <table class="traffic-table">
+        <thead>
+            <tr>
+                <th>Node Name</th>
+                <th>Role</th>
+                <th>Packet Count</th>
+            </tr>
+        </thead>
+        <tbody>
+            {% for node in nodes %}
+                <tr class="traffic">
+                    <td><a href="/packet_list/{{ node[0] }}">{{ node[1] }}</a></td>  <!-- long_name -->
+                    <td>{{ node[2] }}</td>
+                    <td><a href="/top?node_id={{ node[0] }}">{{ node[3] }}</a></td> <!-- packet_count -->
+                </tr>
+            {% endfor %}
+        </tbody>
+    </table>
+
+{% endblock %}

meshview/templates/traceroute.html

@@ -0,0 +1,67 @@
+{% extends "base.html" %}
+
+{% block head %}
+    <script src="https://cdn.jsdelivr.net/npm/echarts/dist/echarts.min.js"></script>
+{% endblock %}
+
+{% block body %}
+<div id="mynetwork" style="width: 100%; height: 600px;"></div>
+
+<script type="text/javascript">
+    var chart = echarts.init(document.getElementById('mynetwork'));
+
+    // Define the nodes and edges passed from the backend
+    var nodes = {{ chart_data['nodes'] | tojson }};
+    var edges = {{ chart_data['edges'] | tojson }};
+
+    var option = {
+        backgroundColor: '#ffffff',  // Set background color to white
+        tooltip: {},
+        series: [
+            {
+                type: 'graph',
+                layout: 'force',
+                data: nodes,
+                links: edges,
+                roam: true,
+                force: {
+                    repulsion: 500,
+                    edgeLength: [100, 200],
+                    gravity: 0.1
+                },
+                label: {
+                    show: true,
+                    position: 'inside',
+                    color: '#000',
+                    padding: [5, 10],
+                    formatter: function(params) { return params.data.value; },
+                    backgroundColor: '#f0f0f0',
+                    borderColor: '#999',
+                    borderWidth: 1,
+                    borderRadius: 5,
+                    z: 5  // Label z-index is now 5, to be below the edges
+                },
+                itemStyle: {
+                    normal: {
+                        borderColor: '#1E1E1E',
+                        borderWidth: 2,
+                    }
+                },
+                lineStyle: {
+                    width: 2,
+                    color: '#ccc', // Edge color
+                    curveness: 0.1, // Slight curve for edges
+                    type: 'solid',
+                    z: 10 // Edge lines have a higher z-index than the labels
+                },
+                edgeSymbol: ['arrow', 'arrow'],  // Both ends of the edge will have arrowheads
+                edgeSymbolSize: [8, 8], // Size of the arrows
+                z: 15  // Ensure edges (arrows) are on top of both the nodes and labels
+            }
+        ]
+    };
+
+    chart.setOption(option);
+</script>
+
+{% endblock %}

requirements.txt

@@ -1,13 +1,46 @@
-protobuf
-aiomqtt
-sqlalchemy[asyncio]
-cryptography
-aiosqlite
-aiohttp
-aiodns
-Jinja2
+protobuf~=5.29.3
+aiomqtt~=2.3.0
+sqlalchemy[asyncio]~=2.0.38
+cryptography~=44.0.1
+aiosqlite~=0.21.0
+aiohttp~=3.11.12
+aiodns~=3.2.0
+Jinja2~=3.1.5
 aiohttp-sse
-asyncpg
-seaborn
-pydot
-plotly
+asyncpg~=0.30.0
+seaborn~=0.13.2
+pydot~=3.0.4
+plotly~=6.0.0
+
+numpy~=2.2.3
+pillow~=11.1.0
+pip~=23.2.1
+attrs~=25.1.0
+cffi~=1.17.1
+paho-mqtt~=2.1.0
+pytz~=2025.1
+idna~=3.10
+multidict~=6.1.0
+propcache~=0.2.1
+typing_extensions~=4.12.2
+pyparsing~=3.2.1
+pycares~=4.5.0
+MarkupSafe~=3.0.2
+pandas~=2.2.3
+matplotlib~=3.10.0
+python-dateutil~=2.9.0.post0
+packaging~=24.2
+narwhals~=1.27.1
+yarl~=1.18.3
+aiosignal~=1.3.2
+frozenlist~=1.5.0
+aiohappyeyeballs~=2.4.6
+cycler~=0.12.1
+six~=1.17.0
+greenlet~=3.1.1
+psutil~=7.0.0
+objgraph~=3.6.2
+contourpy~=1.3.1
+fonttools~=4.56.0
+pycparser~=2.22
+kiwisolver~=1.4.8

sample.config.ini

@@ -6,10 +6,13 @@ acme_challenge =
 
 [mqtt]
 server = mqtt.bayme.sh
-topics = ["msh/US/bayarea/#", "msh/US/CA/mrymesh/#"]
+topics = ["msh/US/bayarea/#", "msh/US/CA/mrymesh/#"] 
 port = 1883
 username = meshdev
 password = large4cats
 
 [database]
-connection_string = sqlite+aiosqlite:///packets.db
+connection_string = sqlite+aiosqlite:///packets.db
+
+[website]
+title = San Francisco Bay Area Mesh

startdb.py

@@ -0,0 +1,50 @@
+import asyncio
+import argparse
+import configparser
+from meshview import mqtt_reader
+from meshview import mqtt_database
+from meshview import mqtt_store
+import json
+
+
+async def load_database_from_mqtt(mqtt_server: str , mqtt_port: int, topic: list, mqtt_user: str | None = None, mqtt_passwd: str | None = None):
+    async for topic, env in mqtt_reader.get_topic_envelopes(mqtt_server, mqtt_port, topic, mqtt_user, mqtt_passwd):
+        await mqtt_store.process_envelope(topic, env)
+
+
+async def main(config):
+    mqtt_database.init_database(config["database"]["connection_string"])
+
+    await mqtt_database.create_tables()
+    mqtt_user = None
+    mqtt_passwd = None
+    if config["mqtt"]["username"] != "":
+        mqtt_user: str = config["mqtt"]["username"]
+    if config["mqtt"]["password"] != "":
+        mqtt_passwd: str = config["mqtt"]["password"]
+    mqtt_topics = json.loads(config["mqtt"]["topics"])
+
+    async with asyncio.TaskGroup() as tg:
+        tg.create_task(
+            load_database_from_mqtt(config["mqtt"]["server"], int(config["mqtt"]["port"]), mqtt_topics, mqtt_user, mqtt_passwd)
+        )
+
+
+def load_config(file_path):
+    """Load configuration from an INI-style text file."""
+    config_parser = configparser.ConfigParser()
+    config_parser.read(file_path)
+
+    # Convert to a dictionary for easier access
+    config = {section: dict(config_parser.items(section)) for section in config_parser.sections()}
+    return config
+
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser("meshview")
+    parser.add_argument("--config", help="Path to the configuration file.", default='config.ini')
+    args = parser.parse_args()
+
+    config = load_config(args.config)
+
+    asyncio.run(main(config))