diff --git a/README.md b/README.md index b47e3c7..b8be577 100644 --- a/README.md +++ b/README.md @@ -42,17 +42,18 @@ The project serves as a real-time monitoring and diagnostic tool for the Meshtas Samples of currently running instances: - https://meshview.bayme.sh (SF Bay Area) -- https://www.svme.sh/ (Sacramento Valley) -- https://meshview.nyme.sh/ (New York) -- https://meshview.socalmesh.org/ (LA Area) -- https://map.wpamesh.net/ (Western Pennsylvania) -- https://meshview.chicagolandmesh.org/ (Chicago) +- https://www.svme.sh (Sacramento Valley) +- https://meshview.nyme.sh (New York) +- https://meshview.socalmesh.org (LA Area) +- https://map.wpamesh.net (Western Pennsylvania) +- https://meshview.chicagolandmesh.org (Chicago) - https://meshview.mt.gt (Canadaverse) - https://meshview.meshtastic.es (Spain) -- https://view.mtnme.sh/ (North Georgia / East Tennessee) +- https://view.mtnme.sh (North Georgia / East Tennessee) - https://meshview.lsinfra.de (Hessen - Germany) -- https://map.nswmesh.au/ (Sydney - Australia) -- https://meshview.pvmesh.org/ (Pioneer Valley, Massachusetts) +- https://map.nswmesh.au (Sydney - Australia) +- https://meshview.pvmesh.org (Pioneer Valley, Massachusetts) +- https://meshview.louisianamesh.org (Louisiana) --- ## Installing @@ -62,13 +63,11 @@ Requires **`python3.11`** or above. Clone the repo from GitHub: ```bash -git clone --recurse-submodules https://github.com/pablorevilla-meshtastic/meshview.git +git clone https://github.com/pablorevilla-meshtastic/meshview.git ``` ```bash cd meshview -git submodule update --init -ln -s ../python/meshtastic/protobuf meshtastic/protobuf ``` Create a Python virtual environment: diff --git a/meshview/models.py b/meshview/models.py index cabaf0f..6185d74 100644 --- a/meshview/models.py +++ b/meshview/models.py @@ -84,8 +84,10 @@ class PacketSeen(Base): ) + class Traceroute(Base): __tablename__ = "traceroute" + id: Mapped[int] = mapped_column(primary_key=True, autoincrement=True) packet_id = mapped_column(ForeignKey("packet.id")) packet: Mapped["Packet"] = relationship( @@ -95,3 +97,7 @@ class Traceroute(Base): done: Mapped[bool] = mapped_column(nullable=True) route: Mapped[bytes] = mapped_column(nullable=True) import_time: Mapped[datetime] = mapped_column(nullable=True) + + __table_args__ = ( + Index("idx_traceroute_import_time", "import_time"), + ) diff --git a/meshview/store.py b/meshview/store.py index 7c7454d..6187413 100644 --- a/meshview/store.py +++ b/meshview/store.py @@ -111,13 +111,16 @@ async def get_traceroute(packet_id): async def get_traceroutes(since): async with database.async_session() as session: - result = await session.execute( - select(Traceroute) - .join(Packet) - .where(Traceroute.import_time > (datetime.now() - since)) - .order_by(Traceroute.import_time) + stmt = ( + select(Traceroute) + .join(Packet) + .where(Traceroute.import_time > since) + .order_by(Traceroute.import_time) ) - return result.scalars() + stream = await session.stream_scalars(stmt) + async for tr in stream: + yield tr + async def get_mqtt_neighbors(since): diff --git a/meshview/web.py b/meshview/web.py index ad36c4e..03796d5 100644 --- a/meshview/web.py +++ b/meshview/web.py @@ -899,7 +899,7 @@ async def graph_network(request): node_ids = set() traceroutes = [] - for tr in await store.get_traceroutes(since): + async for tr in store.get_traceroutes(since): node_ids.add(tr.gateway_node_id) node_ids.add(tr.packet.from_node_id) node_ids.add(tr.packet.to_node_id) @@ -1244,7 +1244,7 @@ async def nodegraph(request): traceroutes = [] # Fetch traceroutes - for tr in await store.get_traceroutes(since): + async for tr in store.get_traceroutes(since): node_ids.add(tr.gateway_node_id) node_ids.add(tr.packet.from_node_id) node_ids.add(tr.packet.to_node_id) @@ -1608,48 +1608,36 @@ async def api_config(request): @routes.get("/api/edges") async def api_edges(request): - edges_set = set() - edge_type = {} since = datetime.datetime.now() - datetime.timedelta(hours=48) + filter_type = request.query.get("type") - # Get optional type filter from query string - filter_type = request.query.get("type") # None if not provided + edges = {} - # Fetch traceroutes - for tr in await store.get_traceroutes(since): + # Traceroutes + async for tr in store.get_traceroutes(since): route = decode_payload.decode_payload(PortNum.TRACEROUTE_APP, tr.route) path = [tr.packet.from_node_id] + list(route.route) - if tr.done: - path.append(tr.packet.to_node_id) - else: - if path[-1] != tr.gateway_node_id: - path.append(tr.gateway_node_id) + path.append(tr.packet.to_node_id if tr.done else tr.gateway_node_id) - for i in range(len(path) - 1): - edge_pair = (path[i], path[i + 1]) - edges_set.add(edge_pair) - edge_type[edge_pair] = "traceroute" + for a, b in zip(path, path[1:]): + edges[(a, b)] = "traceroute" - # Fetch NeighborInfo packets + # NeighborInfo for packet in await store.get_packets(portnum=PortNum.NEIGHBORINFO_APP, after=since): try: _, neighbor_info = decode_payload.decode(packet) for node in neighbor_info.neighbors: - edge_pair = (node.node_id, packet.from_node_id) - if edge_pair not in edges_set: - edges_set.add(edge_pair) - edge_type[edge_pair] = "neighbor" + edges.setdefault((node.node_id, packet.from_node_id), "neighbor") except Exception as e: print(f"Error decoding NeighborInfo packet: {e}") - # Prepare edges with optional filtering by type - edges = [ - {"from": frm, "to": to, "type": typ} - for (frm, to), typ in edge_type.items() - if filter_type is None or typ == filter_type - ] - - return web.json_response({"edges": edges}) + return web.json_response({ + "edges": [ + {"from": a, "to": b, "type": typ} + for (a, b), typ in edges.items() + if filter_type is None or typ == filter_type + ] + }) # Generic static HTML route