update protobuf

README.md

@@ -42,17 +42,18 @@ The project serves as a real-time monitoring and diagnostic tool for the Meshtas
 Samples of currently running instances:
 
 - https://meshview.bayme.sh (SF Bay Area)
-- https://www.svme.sh/ (Sacramento Valley)
-- https://meshview.nyme.sh/   (New York)
-- https://meshview.socalmesh.org/ (LA Area)
-- https://map.wpamesh.net/ (Western Pennsylvania)
-- https://meshview.chicagolandmesh.org/ (Chicago)
+- https://www.svme.sh (Sacramento Valley)
+- https://meshview.nyme.sh   (New York)
+- https://meshview.socalmesh.org (LA Area)
+- https://map.wpamesh.net (Western Pennsylvania)
+- https://meshview.chicagolandmesh.org (Chicago)
 - https://meshview.mt.gt (Canadaverse)
 - https://meshview.meshtastic.es (Spain)
-- https://view.mtnme.sh/ (North Georgia / East Tennessee)
+- https://view.mtnme.sh (North Georgia / East Tennessee)
 - https://meshview.lsinfra.de (Hessen - Germany)
-- https://map.nswmesh.au/ (Sydney - Australia)
-- https://meshview.pvmesh.org/ (Pioneer Valley, Massachusetts)
+- https://map.nswmesh.au (Sydney - Australia)
+- https://meshview.pvmesh.org (Pioneer Valley, Massachusetts)
+- https://meshview.louisianamesh.org (Louisiana)
 ---
 
 ## Installing
@@ -62,13 +63,11 @@ Requires **`python3.11`** or above.
 Clone the repo from GitHub:
 
 ```bash
-git clone --recurse-submodules https://github.com/pablorevilla-meshtastic/meshview.git
+git clone https://github.com/pablorevilla-meshtastic/meshview.git
 ```
 
 ```bash
 cd meshview
-git submodule update --init
-ln -s ../python/meshtastic/protobuf meshtastic/protobuf
 ```
 Create a Python virtual environment:
 

meshview/models.py

@@ -84,8 +84,10 @@ class PacketSeen(Base):
     )
 
 
+
 class Traceroute(Base):
     __tablename__ = "traceroute"
+
     id: Mapped[int] = mapped_column(primary_key=True, autoincrement=True)
     packet_id = mapped_column(ForeignKey("packet.id"))
     packet: Mapped["Packet"] = relationship(
@@ -95,3 +97,7 @@ class Traceroute(Base):
     done: Mapped[bool] = mapped_column(nullable=True)
     route: Mapped[bytes] = mapped_column(nullable=True)
     import_time: Mapped[datetime] = mapped_column(nullable=True)
+
+    __table_args__ = (
+        Index("idx_traceroute_import_time", "import_time"),
+    )

meshview/store.py

@@ -111,13 +111,16 @@ async def get_traceroute(packet_id):
 
 async def get_traceroutes(since):
     async with database.async_session() as session:
-        result = await session.execute(
-                select(Traceroute)
-                .join(Packet)
-                .where(Traceroute.import_time > (datetime.now() - since))
-                .order_by(Traceroute.import_time)
+        stmt = (
+            select(Traceroute)
+            .join(Packet)
+            .where(Traceroute.import_time > since)
+            .order_by(Traceroute.import_time)
         )
-        return result.scalars()
+        stream = await session.stream_scalars(stmt)
+        async for tr in stream:
+            yield tr
+
 
 
 async def get_mqtt_neighbors(since):

meshview/web.py

@@ -899,7 +899,7 @@ async def graph_network(request):
     node_ids = set()
 
     traceroutes = []
-    for tr in await store.get_traceroutes(since):
+    async for tr in store.get_traceroutes(since):
         node_ids.add(tr.gateway_node_id)
         node_ids.add(tr.packet.from_node_id)
         node_ids.add(tr.packet.to_node_id)
@@ -1244,7 +1244,7 @@ async def nodegraph(request):
     traceroutes = []
 
     # Fetch traceroutes
-    for tr in await store.get_traceroutes(since):
+    async for tr in store.get_traceroutes(since):
         node_ids.add(tr.gateway_node_id)
         node_ids.add(tr.packet.from_node_id)
         node_ids.add(tr.packet.to_node_id)
@@ -1608,48 +1608,36 @@ async def api_config(request):
 
 @routes.get("/api/edges")
 async def api_edges(request):
-    edges_set = set()
-    edge_type = {}
     since = datetime.datetime.now() - datetime.timedelta(hours=48)
+    filter_type = request.query.get("type")
 
-    # Get optional type filter from query string
-    filter_type = request.query.get("type")  # None if not provided
+    edges = {}
 
-    # Fetch traceroutes
-    for tr in await store.get_traceroutes(since):
+    # Traceroutes
+    async for tr in store.get_traceroutes(since):
         route = decode_payload.decode_payload(PortNum.TRACEROUTE_APP, tr.route)
         path = [tr.packet.from_node_id] + list(route.route)
-        if tr.done:
-            path.append(tr.packet.to_node_id)
-        else:
-            if path[-1] != tr.gateway_node_id:
-                path.append(tr.gateway_node_id)
+        path.append(tr.packet.to_node_id if tr.done else tr.gateway_node_id)
 
-        for i in range(len(path) - 1):
-            edge_pair = (path[i], path[i + 1])
-            edges_set.add(edge_pair)
-            edge_type[edge_pair] = "traceroute"
+        for a, b in zip(path, path[1:]):
+            edges[(a, b)] = "traceroute"
 
-    # Fetch NeighborInfo packets
+    # NeighborInfo
     for packet in await store.get_packets(portnum=PortNum.NEIGHBORINFO_APP, after=since):
         try:
             _, neighbor_info = decode_payload.decode(packet)
             for node in neighbor_info.neighbors:
-                edge_pair = (node.node_id, packet.from_node_id)
-                if edge_pair not in edges_set:
-                    edges_set.add(edge_pair)
-                    edge_type[edge_pair] = "neighbor"
+                edges.setdefault((node.node_id, packet.from_node_id), "neighbor")
         except Exception as e:
             print(f"Error decoding NeighborInfo packet: {e}")
 
-    # Prepare edges with optional filtering by type
-    edges = [
-        {"from": frm, "to": to, "type": typ}
-        for (frm, to), typ in edge_type.items()
-        if filter_type is None or typ == filter_type
-    ]
-
-    return web.json_response({"edges": edges})
+    return web.json_response({
+        "edges": [
+            {"from": a, "to": b, "type": typ}
+            for (a, b), typ in edges.items()
+            if filter_type is None or typ == filter_type
+        ]
+    })
 
 
 # Generic static HTML route