Database change to remove import time columns

This commit is contained in:
pablorevilla-meshtastic
2026-01-09 13:30:14 -08:00
parent da31794d8d
commit e12e3a2a41
12 changed files with 432 additions and 350 deletions
+1
View File
@@ -45,3 +45,4 @@ __pycache__/
# OS
.DS_Store
Thumbs.db
packets.db-journal
@@ -0,0 +1,64 @@
"""Drop import_time columns.
Revision ID: 9f3b1a8d2c4f
Revises: 2b5a61bb2b75
Create Date: 2026-01-09 09:55:00.000000
"""
from collections.abc import Sequence
import sqlalchemy as sa
from alembic import op
# revision identifiers, used by Alembic.
revision: str = "9f3b1a8d2c4f"
down_revision: str | None = "2b5a61bb2b75"
branch_labels: str | Sequence[str] | None = None
depends_on: str | Sequence[str] | None = None
def upgrade() -> None:
conn = op.get_bind()
inspector = sa.inspect(conn)
packet_indexes = {idx["name"] for idx in inspector.get_indexes("packet")}
packet_columns = {col["name"] for col in inspector.get_columns("packet")}
with op.batch_alter_table("packet", schema=None) as batch_op:
if "idx_packet_import_time" in packet_indexes:
batch_op.drop_index("idx_packet_import_time")
if "idx_packet_from_node_time" in packet_indexes:
batch_op.drop_index("idx_packet_from_node_time")
if "import_time" in packet_columns:
batch_op.drop_column("import_time")
packet_seen_columns = {col["name"] for col in inspector.get_columns("packet_seen")}
with op.batch_alter_table("packet_seen", schema=None) as batch_op:
if "import_time" in packet_seen_columns:
batch_op.drop_column("import_time")
traceroute_indexes = {idx["name"] for idx in inspector.get_indexes("traceroute")}
traceroute_columns = {col["name"] for col in inspector.get_columns("traceroute")}
with op.batch_alter_table("traceroute", schema=None) as batch_op:
if "idx_traceroute_import_time" in traceroute_indexes:
batch_op.drop_index("idx_traceroute_import_time")
if "import_time" in traceroute_columns:
batch_op.drop_column("import_time")
def downgrade() -> None:
with op.batch_alter_table("traceroute", schema=None) as batch_op:
batch_op.add_column(sa.Column("import_time", sa.DateTime(), nullable=True))
batch_op.create_index("idx_traceroute_import_time", ["import_time"], unique=False)
with op.batch_alter_table("packet_seen", schema=None) as batch_op:
batch_op.add_column(sa.Column("import_time", sa.DateTime(), nullable=True))
with op.batch_alter_table("packet", schema=None) as batch_op:
batch_op.add_column(sa.Column("import_time", sa.DateTime(), nullable=True))
batch_op.create_index("idx_packet_import_time", [sa.text("import_time DESC")], unique=False)
batch_op.create_index(
"idx_packet_from_node_time",
["from_node_id", sa.text("import_time DESC")],
unique=False,
)
+231 -184
View File
@@ -1,82 +1,38 @@
# API Documentation
## 1. Chat API
Base URL: `http(s)://<host>`
### GET `/api/chat`
Returns the most recent chat messages.
All endpoints return JSON. Timestamps are either ISO 8601 strings or `*_us` values in
microseconds since epoch.
**Query Parameters**
- `limit` (optional, int): Maximum number of messages to return. Default: `100`.
**Response Example**
```json
{
"packets": [
{
"id": 123,
"import_time": "2025-07-22T12:45:00",
"from_node_id": 987654,
"from_node": "Alice",
"channel": "main",
"payload": "Hello, world!"
}
]
}
```
---
### GET `/api/chat/updates`
Returns chat messages imported after a given timestamp.
**Query Parameters**
- `last_time` (optional, ISO timestamp): Only messages imported after this time are returned.
**Response Example**
```json
{
"packets": [
{
"id": 124,
"import_time": "2025-07-22T12:50:00",
"from_node_id": 987654,
"from_node": "Alice",
"channel": "main",
"payload": "New message!"
}
],
"latest_import_time": "2025-07-22T12:50:00"
}
```
---
## 2. Nodes API
## 1. Nodes API
### GET `/api/nodes`
Returns a list of all nodes, with optional filtering by last seen.
Returns a list of nodes, with optional filtering.
**Query Parameters**
- `hours` (optional, int): Return nodes seen in the last N hours.
- `days` (optional, int): Return nodes seen in the last N days.
- `last_seen_after` (optional, ISO timestamp): Return nodes seen after this time.
Query Parameters
- `node_id` (optional, int): Exact node ID.
- `role` (optional, string): Node role.
- `channel` (optional, string): Channel name.
- `hw_model` (optional, string): Hardware model.
- `days_active` (optional, int): Nodes seen within the last N days.
**Response Example**
Response Example
```json
{
"nodes": [
{
"id": 42,
"node_id": 1234,
"long_name": "Alice",
"short_name": "A",
"channel": "main",
"last_seen": "2025-07-22T12:40:00",
"hardware": "T-Beam",
"hw_model": "T-Beam",
"firmware": "1.2.3",
"role": "client",
"last_lat": 37.7749,
"last_long": -122.4194
"last_lat": 377749000,
"last_long": -1224194000,
"channel": "main",
"last_seen_us": 1736370123456789
}
]
}
@@ -84,45 +40,58 @@ Returns a list of all nodes, with optional filtering by last seen.
---
## 3. Packets API
## 2. Packets API
### GET `/api/packets`
Returns a list of packets with optional filters.
Returns packets with optional filters.
**Query Parameters**
- `limit` (optional, int): Maximum number of packets to return. Default: `200`.
- `since` (optional, ISO timestamp): Only packets imported after this timestamp are returned.
Query Parameters
- `packet_id` (optional, int): Return exactly one packet (overrides other filters).
- `limit` (optional, int): Max packets to return, clamped 1-1000. Default: `50`.
- `since` (optional, int): Only packets imported after this microsecond timestamp.
- `portnum` (optional, int): Filter by port number.
- `contains` (optional, string): Payload substring filter.
- `from_node_id` (optional, int): Filter by sender node ID.
- `to_node_id` (optional, int): Filter by recipient node ID.
- `node_id` (optional, int): Legacy filter matching either from or to node ID.
**Response Example**
Response Example
```json
{
"packets": [
{
"id": 123,
"import_time_us": 1736370123456789,
"channel": "main",
"from_node_id": 5678,
"to_node_id": 91011,
"portnum": 1,
"import_time": "2025-07-22T12:45:00",
"payload": "Hello, Bob!"
"long_name": "Alice",
"payload": "Hello, Bob!",
"to_long_name": "Bob",
"reply_id": 122
}
]
],
"latest_import_time": 1736370123456789
}
```
---
Notes
- For `portnum=1` (text messages), packets are filtered to remove sequence-only payloads.
- `latest_import_time` is returned when available for incremental polling (microseconds).
---
## 4. Channels API
## 3. Channels API
### GET `/api/channels`
Returns a list of channels seen in a given time period.
Returns channels seen in a time period.
**Query Parameters**
- `period_type` (optional, string): Time granularity (`hour` or `day`). Default: `hour`.
Query Parameters
- `period_type` (optional, string): `hour` or `day`. Default: `hour`.
- `length` (optional, int): Number of periods to look back. Default: `24`.
**Response Example**
Response Example
```json
{
"channels": ["LongFast", "MediumFast", "ShortFast"]
@@ -131,29 +100,21 @@ Returns a list of channels seen in a given time period.
---
## 5. Statistics API
## 4. Stats API
### GET `/api/stats`
Returns packet statistics aggregated by time periods, with optional filtering.
Retrieve packet statistics aggregated by time periods, with optional filtering.
---
## Query Parameters
| Parameter | Type | Required | Default | Description |
|--------------|---------|----------|----------|-------------------------------------------------------------------------------------------------|
| `period_type` | string | No | `hour` | Time granularity of the stats. Allowed values: `hour`, `day`. |
| `length` | integer | No | 24 | Number of periods to include (hours or days). |
| `channel` | string | No | — | Filter results by channel name (case-insensitive). |
| `portnum` | integer | No | — | Filter results by port number. |
| `to_node` | integer | No | — | Filter results to packets sent **to** this node ID. |
| `from_node` | integer | No | — | Filter results to packets sent **from** this node ID. |
---
## Response
Query Parameters
- `period_type` (optional, string): `hour` or `day`. Default: `hour`.
- `length` (optional, int): Number of periods to include. Default: `24`.
- `channel` (optional, string): Filter by channel (case-insensitive).
- `portnum` (optional, int): Filter by port number.
- `to_node` (optional, int): Filter by destination node ID.
- `from_node` (optional, int): Filter by source node ID.
- `node` (optional, int): If provided, return combined `sent` and `seen` totals for that node.
Response Example (series)
```json
{
"period_type": "hour",
@@ -163,65 +124,117 @@ Retrieve packet statistics aggregated by time periods, with optional filtering.
"to_node": 12345678,
"from_node": 87654321,
"data": [
{ "period": "2025-08-08 14:00", "count": 10 },
{ "period": "2025-08-08 15:00", "count": 7 }
]
}
```
Response Example (`node` totals)
```json
{
"node_id": 12345678,
"period_type": "hour",
"length": 24,
"sent": 42,
"seen": 58
}
```
---
### GET `/api/stats/count`
Returns total packet counts, optionally filtered.
Query Parameters
- `packet_id` (optional, int): Filter packet_seen by packet ID.
- `period_type` (optional, string): `hour` or `day`.
- `length` (optional, int): Number of periods to include.
- `channel` (optional, string): Filter by channel.
- `from_node` (optional, int): Filter by source node ID.
- `to_node` (optional, int): Filter by destination node ID.
Response Example
```json
{
"total_packets": 12345,
"total_seen": 67890
}
```
---
### GET `/api/stats/top`
Returns nodes sorted by packets seen, with pagination.
Query Parameters
- `period_type` (optional, string): `hour` or `day`. Default: `day`.
- `length` (optional, int): Number of periods to include. Default: `1`.
- `channel` (optional, string): Filter by channel.
- `limit` (optional, int): Max nodes to return. Default: `20`, max `100`.
- `offset` (optional, int): Pagination offset. Default: `0`.
Response Example
```json
{
"total": 250,
"limit": 20,
"offset": 0,
"nodes": [
{
"period": "2025-08-08 14:00",
"count": 10
},
{
"period": "2025-08-08 15:00",
"count": 7
"node_id": 1234,
"long_name": "Alice",
"short_name": "A",
"channel": "main",
"sent": 100,
"seen": 240,
"avg": 2.4
}
// more entries...
]
}
```
---
## 6. Edges API
## 5. Edges API
### GET `/api/edges`
Returns network edges (connections between nodes) based on traceroutes and neighbor info.
Traceroute edges are collected over the last 48 hours. Neighbor edges are based on
port 71 packets.
**Query Parameters**
- `type` (optional, string): Filter by edge type (`traceroute` or `neighbor`). If omitted, returns both types.
Query Parameters
- `type` (optional, string): `traceroute` or `neighbor`. If omitted, returns both.
- `node_id` (optional, int): Filter edges to only those touching a node.
**Response Example**
Response Example
```json
{
"edges": [
{
"from": 12345678,
"to": 87654321,
"type": "traceroute"
},
{
"from": 11111111,
"to": 22222222,
"type": "neighbor"
}
{ "from": 12345678, "to": 87654321, "type": "traceroute" },
{ "from": 11111111, "to": 22222222, "type": "neighbor" }
]
}
```
---
## 7. Configuration API
## 6. Config API
### GET `/api/config`
Returns the current site configuration (safe subset exposed to clients).
Returns a safe subset of server configuration.
**Response Example**
Response Example
```json
{
"site": {
"domain": "meshview.example.com",
"domain": "example.com",
"language": "en",
"title": "Bay Area Mesh",
"message": "Real time data from around the bay area",
"title": "Meshview",
"message": "",
"starting": "/chat",
"nodes": "true",
"conversations": "true",
"chat": "true",
"everything": "true",
"graphs": "true",
"stats": "true",
@@ -236,11 +249,11 @@ Returns the current site configuration (safe subset exposed to clients).
"firehose_interval": 3,
"weekly_net_message": "Weekly Mesh check-in message.",
"net_tag": "#BayMeshNet",
"version": "2.0.8 ~ 10-22-25"
"version": "3.0.0"
},
"mqtt": {
"server": "mqtt.bayme.sh",
"topics": ["msh/US/bayarea/#"]
"server": "mqtt.example.com",
"topics": ["msh/region/#"]
},
"cleanup": {
"enabled": "false",
@@ -254,91 +267,125 @@ Returns the current site configuration (safe subset exposed to clients).
---
## 8. Language/Translations API
## 7. Language API
### GET `/api/lang`
Returns translation strings for the UI.
Returns translation strings.
**Query Parameters**
- `lang` (optional, string): Language code (e.g., `en`, `es`). Defaults to site language setting.
- `section` (optional, string): Specific section to retrieve translations for.
Query Parameters
- `lang` (optional, string): Language code (e.g., `en`, `es`). Default from config or `en`.
- `section` (optional, string): Return only one section (e.g., `nodelist`, `firehose`).
**Response Example (full)**
Response Example
```json
{
"chat": {
"title": "Chat",
"send": "Send"
},
"map": {
"title": "Map",
"zoom_in": "Zoom In"
}
}
```
**Response Example (section-specific)**
Request: `/api/lang?section=chat`
```json
{
"title": "Chat",
"send": "Send"
"title": "Meshview",
"search_placeholder": "Search..."
}
```
---
## 9. Health Check API
## 8. Packets Seen API
### GET `/api/packets_seen/{packet_id}`
Returns packet_seen entries for a packet.
Path Parameters
- `packet_id` (required, int): Packet ID.
Response Example
```json
{
"seen": [
{
"packet_id": 123,
"node_id": 456,
"rx_time": "2025-07-22T12:45:00",
"hop_limit": 7,
"hop_start": 0,
"channel": "main",
"rx_snr": 5.0,
"rx_rssi": -90,
"topic": "msh/region/#",
"import_time_us": 1736370123456789
}
]
}
```
---
## 9. Traceroute API
### GET `/api/traceroute/{packet_id}`
Returns traceroute details and derived paths for a packet.
Path Parameters
- `packet_id` (required, int): Packet ID.
Response Example
```json
{
"packet": {
"id": 123,
"from": 111,
"to": 222,
"channel": "main"
},
"traceroute_packets": [
{
"index": 0,
"gateway_node_id": 333,
"done": true,
"forward_hops": [111, 444, 222],
"reverse_hops": [222, 444, 111]
}
],
"unique_forward_paths": [
{ "path": [111, 444, 222], "count": 2 }
],
"unique_reverse_paths": [
[222, 444, 111]
],
"winning_paths": [
[111, 444, 222]
]
}
```
---
## 10. Health API
### GET `/health`
Health check endpoint for monitoring, load balancers, and orchestration systems.
Returns service health and database status.
**Response Example (Healthy)**
Response Example
```json
{
"status": "healthy",
"timestamp": "2025-11-03T14:30:00.123456Z",
"timestamp": "2025-07-22T12:45:00+00:00",
"version": "3.0.0",
"git_revision": "6416978",
"git_revision": "abc1234",
"database": "connected",
"database_size": "853.03 MB",
"database_size_bytes": 894468096
}
```
**Response Example (Unhealthy)**
Status Code: `503 Service Unavailable`
```json
{
"status": "unhealthy",
"timestamp": "2025-11-03T14:30:00.123456Z",
"version": "2.0.8",
"git_revision": "6416978",
"database": "disconnected"
"database_size": "12.34 MB",
"database_size_bytes": 12939444
}
```
---
## 10. Version API
## 11. Version API
### GET `/version`
Returns detailed version information including semver, release date, and git revision.
Returns version metadata.
**Response Example**
Response Example
```json
{
"version": "2.0.8",
"release_date": "2025-10-22",
"git_revision": "6416978a1b2c3d4e5f6g7h8i9j0k1l2m3n4o5p6q",
"git_revision_short": "6416978"
"version": "3.0.0",
"git_revision": "abc1234",
"build_time": "2025-11-01T12:00:00+00:00"
}
```
---
## Notes
- All timestamps (`import_time`, `last_seen`) are returned in ISO 8601 format.
- `portnum` is an integer representing the packet type.
- `payload` is always a UTF-8 decoded string.
- Node IDs are integers (e.g., `12345678`).
+1 -10
View File
@@ -55,17 +55,13 @@ class Packet(Base):
overlaps="from_node",
)
payload: Mapped[bytes] = mapped_column(nullable=True)
import_time: Mapped[datetime] = mapped_column(nullable=True)
import_time_us: Mapped[int] = mapped_column(BigInteger, nullable=True)
channel: Mapped[str] = mapped_column(nullable=True)
__table_args__ = (
Index("idx_packet_from_node_id", "from_node_id"),
Index("idx_packet_to_node_id", "to_node_id"),
Index("idx_packet_import_time", desc("import_time")),
Index("idx_packet_import_time_us", desc("import_time_us")),
# Composite index for /top endpoint performance - filters by from_node_id AND import_time
Index("idx_packet_from_node_time", "from_node_id", desc("import_time")),
Index("idx_packet_from_node_time_us", "from_node_id", desc("import_time_us")),
)
@@ -86,7 +82,6 @@ class PacketSeen(Base):
rx_snr: Mapped[float] = mapped_column(nullable=True)
rx_rssi: Mapped[int] = mapped_column(nullable=True)
topic: Mapped[str] = mapped_column(nullable=True)
import_time: Mapped[datetime] = mapped_column(nullable=True)
import_time_us: Mapped[int] = mapped_column(BigInteger, nullable=True)
__table_args__ = (
@@ -108,11 +103,7 @@ class Traceroute(Base):
gateway_node_id: Mapped[int] = mapped_column(BigInteger, nullable=True)
done: Mapped[bool] = mapped_column(nullable=True)
route: Mapped[bytes] = mapped_column(nullable=True)
import_time: Mapped[datetime] = mapped_column(nullable=True)
route_return: Mapped[bytes] = mapped_column(nullable=True)
import_time_us: Mapped[int] = mapped_column(BigInteger, nullable=True)
__table_args__ = (
Index("idx_traceroute_import_time", "import_time"),
Index("idx_traceroute_import_time_us", "import_time_us"),
)
__table_args__ = (Index("idx_traceroute_import_time_us", "import_time_us"),)
-5
View File
@@ -84,7 +84,6 @@ async def process_envelope(topic, env):
result = await session.execute(select(Packet).where(Packet.id == env.packet.id))
packet = result.scalar_one_or_none()
if not packet:
now = datetime.datetime.now(datetime.UTC)
now_us = int(now.timestamp() * 1_000_000)
stmt = (
@@ -95,7 +94,6 @@ async def process_envelope(topic, env):
from_node_id=getattr(env.packet, "from"),
to_node_id=env.packet.to,
payload=env.packet.SerializeToString(),
import_time=now,
import_time_us=now_us,
channel=env.channel_id,
)
@@ -132,7 +130,6 @@ async def process_envelope(topic, env):
hop_limit=env.packet.hop_limit,
hop_start=env.packet.hop_start,
topic=topic,
import_time=now,
import_time_us=now_us,
)
session.add(seen)
@@ -228,10 +225,8 @@ async def process_envelope(topic, env):
route=env.packet.decoded.payload,
done=not env.packet.decoded.want_response,
gateway_node_id=int(env.gateway_id[1:], 16),
import_time=now,
import_time_us=now_us,
)
)
await session.commit()
+36
View File
@@ -0,0 +1,36 @@
// Shared port label/color definitions for UI pages.
window.PORT_LABEL_MAP = {
0: "UNKNOWN",
1: "Text",
3: "Position",
4: "Node Info",
5: "Routing",
6: "Admin",
8: "Waypoint",
35: "Store Forward++",
65: "Store & Forward",
67: "Telemetry",
70: "Traceroute",
71: "Neighbor",
73: "Map Report",
};
window.PORT_COLOR_MAP = {
0: "#6c757d",
1: "#007bff",
3: "#28a745",
4: "#ffc107",
5: "#dc3545",
6: "#20c997",
8: "#fd7e14",
35: "#8bc34a",
65: "#6610f2",
67: "#17a2b8",
70: "#ff4444",
71: "#ff66cc",
73: "#9999ff",
};
// Aliases for pages that expect different names.
window.PORT_MAP = window.PORT_LABEL_MAP;
window.PORT_COLORS = window.PORT_COLOR_MAP;
+24 -13
View File
@@ -1,5 +1,6 @@
from datetime import datetime, timedelta
from sqlalchemy import select, and_, or_, func, cast, Text
from sqlalchemy import Text, and_, cast, func, or_, select, text
from sqlalchemy.orm import lazyload
from meshview import database, models
@@ -91,8 +92,10 @@ async def get_packets_from(node_id=None, portnum=None, since=None, limit=500):
if portnum:
q = q.where(Packet.portnum == portnum)
if since:
q = q.where(Packet.import_time > (datetime.now() - since))
result = await session.execute(q.limit(limit).order_by(Packet.import_time.desc()))
now_us = int(datetime.now().timestamp() * 1_000_000)
start_us = now_us - int(since.total_seconds() * 1_000_000)
q = q.where(Packet.import_time_us > start_us)
result = await session.execute(q.limit(limit).order_by(Packet.import_time_us.desc()))
return result.scalars()
@@ -108,7 +111,7 @@ async def get_packets_seen(packet_id):
result = await session.execute(
select(PacketSeen)
.where(PacketSeen.packet_id == packet_id)
.order_by(PacketSeen.import_time.desc())
.order_by(PacketSeen.import_time_us.desc())
)
return result.scalars()
@@ -129,18 +132,21 @@ async def get_traceroute(packet_id):
result = await session.execute(
select(Traceroute)
.where(Traceroute.packet_id == packet_id)
.order_by(Traceroute.import_time)
.order_by(Traceroute.import_time_us)
)
return result.scalars()
async def get_traceroutes(since):
if isinstance(since, datetime):
since_us = int(since.timestamp() * 1_000_000)
else:
since_us = int(since)
async with database.async_session() as session:
stmt = (
select(Traceroute)
.join(Packet)
.where(Traceroute.import_time > since)
.order_by(Traceroute.import_time)
.where(Traceroute.import_time_us > since_us)
.order_by(Traceroute.import_time_us)
)
stream = await session.stream_scalars(stmt)
async for tr in stream:
@@ -148,6 +154,8 @@ async def get_traceroutes(since):
async def get_mqtt_neighbors(since):
now_us = int(datetime.now().timestamp() * 1_000_000)
start_us = now_us - int(since.total_seconds() * 1_000_000)
async with database.async_session() as session:
result = await session.execute(
select(PacketSeen, Packet)
@@ -155,7 +163,7 @@ async def get_mqtt_neighbors(since):
.where(
(PacketSeen.hop_limit == PacketSeen.hop_start)
& (PacketSeen.hop_start != 0)
& (PacketSeen.import_time > (datetime.now() - since))
& (PacketSeen.import_time_us > start_us)
)
.options(
lazyload(Packet.from_node),
@@ -196,7 +204,7 @@ async def get_top_traffic_nodes():
COUNT(ps.packet_id) AS total_times_seen
FROM node n
LEFT JOIN packet p ON n.node_id = p.from_node_id
AND p.import_time >= DATETIME('now', 'localtime', '-24 hours')
AND p.import_time_us >= (CAST(strftime('%s','now') AS INTEGER) - 86400) * 1000000
LEFT JOIN packet_seen ps ON p.id = ps.packet_id
GROUP BY n.node_id, n.long_name, n.short_name
HAVING total_packets_sent > 0
@@ -235,7 +243,7 @@ async def get_node_traffic(node_id: int):
FROM packet
JOIN node ON packet.from_node_id = node.node_id
WHERE node.node_id = :node_id
AND packet.import_time >= DATETIME('now', 'localtime', '-24 hours')
AND packet.import_time_us >= (CAST(strftime('%s','now') AS INTEGER) - 86400) * 1000000
GROUP BY packet.portnum
ORDER BY packet_count DESC;
"""),
@@ -330,9 +338,12 @@ async def get_packet_stats(
async with database.async_session() as session:
q = select(
func.strftime(time_format, Packet.import_time).label('period'),
func.strftime(
time_format,
func.datetime(Packet.import_time_us / 1_000_000, "unixepoch"),
).label('period'),
func.count().label('count'),
).where(Packet.import_time >= start_time)
).where(Packet.import_time_us >= int(start_time.timestamp() * 1_000_000))
# Filters
if channel:
+4 -36
View File
@@ -115,6 +115,7 @@
</div>
<script src="/static/portmaps.js"></script>
<script>
/* ======================================================
FIREHOSE TRANSLATION SYSTEM (isolated from base)
@@ -177,41 +178,8 @@ function nodeName(id) {
/* ======================================================
PORT COLORS & NAMES
====================================================== */
const PORT_MAP = {
0: "UNKNOWN APP",
1: "Text Message",
3: "Position",
4: "Node Info",
5: "Routing",
6: "Administration",
8: "Waypoint",
65: "Store Forward",
67: "Telemetry",
70: "Trace Route",
71: "Neighbor Info"
};
const PORT_COLORS = {
0: "#6c757d",
1: "#007bff",
3: "#28a745",
4: "#ffc107",
5: "#dc3545",
6: "#20c997",
65: "#6610f2",
67: "#17a2b8",
68: "#fd7e14",
69: "#6f42c1",
70: "#ff4444",
71: "#ff66cc",
72: "#00cc99",
73: "#9999ff",
74: "#cc00cc",
75: "#ffbb33",
76: "#00bcd4",
77: "#8bc34a",
78: "#795548"
};
const PORT_MAP = window.PORT_MAP || {};
const PORT_COLORS = window.PORT_COLORS || {};
function portLabel(portnum, payload, linksHtml) {
const name = PORT_MAP[portnum] || "Unknown";
@@ -261,7 +229,7 @@ async function fetchUpdates() {
if (updatesPaused) return;
const url = new URL("/api/packets", window.location.origin);
url.searchParams.set("limit", 50);
url.searchParams.set("limit", 100);
if (lastImportTimeUs)
url.searchParams.set("since", lastImportTimeUs);
+1 -1
View File
@@ -141,7 +141,7 @@ document.addEventListener("DOMContentLoaded", async () => {
</span>
<span class="col-3 nodename">
<a href="/packet_list/${packet.from_node_id}">
<a href="/node/${packet.from_node_id}">
${escapeHtml(fromName)}
</a>
</span>
+3 -25
View File
@@ -285,34 +285,12 @@
</div>
<script src="https://cdn.jsdelivr.net/npm/echarts@5.5.0/dist/echarts.min.js"></script>
<script src="/static/portmaps.js"></script>
<script>
const PORT_COLOR_MAP = {
0: "#6c757d",
1: "#007bff",
3: "#28a745",
4: "#ffc107",
5: "#dc3545",
6: "#20c997",
65: "#6610f2",
67: "#17a2b8",
70: "#ff9800",
71: "#ff66cc",
};
const PORT_LABEL_MAP = {
0: "UNKNOWN",
1: "Text",
3: "Position",
4: "Node Info",
5: "Routing",
6: "Admin",
65: "Store & Forward",
67: "Telemetry",
70: "Traceroute",
71: "Neighbor"
};
const PORT_COLOR_MAP = window.PORT_COLOR_MAP || {};
const PORT_LABEL_MAP = window.PORT_LABEL_MAP || {};
/* ======================================================
NODE PAGE TRANSLATION (isolated from base)
+20 -15
View File
@@ -1,7 +1,10 @@
"""Main web server routes and page rendering for Meshview."""
import asyncio
import datetime
import logging
import os
import pathlib
import re
import ssl
from dataclasses import dataclass
@@ -12,7 +15,7 @@ from google.protobuf import text_format
from google.protobuf.message import Message
from jinja2 import Environment, PackageLoader, Undefined, select_autoescape
from markupsafe import Markup
import pathlib
from meshtastic.protobuf.portnums_pb2 import PortNum
from meshview import config, database, decode_payload, migrations, models, store
from meshview.__version__ import (
@@ -45,6 +48,8 @@ with open(os.path.join(os.path.dirname(__file__), '1x1.png'), 'rb') as png:
@dataclass
class Packet:
"""UI-friendly packet wrapper for templates and API payloads."""
id: int
from_node_id: int
from_node: models.Node
@@ -56,11 +61,11 @@ class Packet:
raw_payload: object
payload: str
pretty_payload: Markup
import_time: datetime.datetime
import_time_us: int
@classmethod
def from_model(cls, packet):
"""Convert a Packet ORM model into a presentation-friendly Packet."""
mesh_packet, payload = decode_payload.decode(packet)
pretty_payload = None
@@ -101,7 +106,6 @@ class Packet:
data=text_mesh_packet,
payload=text_payload, # now always a string
pretty_payload=pretty_payload,
import_time=packet.import_time,
import_time_us=packet.import_time_us, # <-- include microseconds
raw_mesh_packet=mesh_packet,
raw_payload=payload,
@@ -109,6 +113,7 @@ class Packet:
async def build_trace(node_id):
"""Build a recent GPS trace list for a node using position packets."""
trace = []
for raw_p in await store.get_packets_from(
node_id, PortNum.POSITION_APP, since=datetime.timedelta(hours=24)
@@ -130,6 +135,7 @@ async def build_trace(node_id):
async def build_neighbors(node_id):
"""Return neighbor node metadata for the given node ID."""
packets = await store.get_packets_from(node_id, PortNum.NEIGHBORINFO_APP, limit=1)
packet = packets.first()
@@ -159,6 +165,7 @@ async def build_neighbors(node_id):
def node_id_to_hex(node_id):
"""Format a node_id in Meshtastic hex notation."""
if node_id is None or isinstance(node_id, Undefined):
return "Invalid node_id" # i... have no clue
if node_id == 4294967295:
@@ -168,6 +175,7 @@ def node_id_to_hex(node_id):
def format_timestamp(timestamp):
"""Normalize timestamps to ISO 8601 strings."""
if isinstance(timestamp, int):
timestamp = datetime.datetime.fromtimestamp(timestamp, datetime.UTC)
return timestamp.isoformat(timespec="milliseconds")
@@ -185,6 +193,7 @@ routes = web.RouteTableDef()
@routes.get("/")
async def index(request):
"""Redirect root URL to configured starting page."""
"""
Redirect root URL '/' to the page specified in CONFIG['site']['starting'].
Defaults to '/map' if not set.
@@ -194,15 +203,10 @@ async def index(request):
raise web.HTTPFound(location=starting_url)
# redirect for backwards compatibility
@routes.get("/packet_list/{packet_id}")
async def redirect_packet_list(request):
packet_id = request.match_info["packet_id"]
raise web.HTTPFound(location=f"/node/{packet_id}")
# Generic static HTML route
@routes.get("/{page}")
async def serve_page(request):
"""Serve static HTML pages from meshview/static."""
page = request.match_info["page"]
# default to index.html if no extension
@@ -217,7 +221,6 @@ async def serve_page(request):
return web.Response(text=content, content_type="text/html")
@routes.get("/net")
async def net(request):
return web.Response(
@@ -352,8 +355,8 @@ async def graph_traceroute(request):
# It seems some nodes add them self to the list before uplinking
path.append(tr.gateway_node_id)
if not tr.done and tr.gateway_node_id not in node_seen_time and tr.import_time:
node_seen_time[path[-1]] = tr.import_time
if not tr.done and tr.gateway_node_id not in node_seen_time and tr.import_time_us:
node_seen_time[path[-1]] = tr.import_time_us
mqtt_nodes.add(tr.gateway_node_id)
node_color[path[-1]] = '#' + hex(hash(tuple(path)))[3:9]
@@ -363,7 +366,7 @@ async def graph_traceroute(request):
for path in paths:
used_nodes.update(path)
import_times = [tr.import_time for tr in traceroutes if tr.import_time]
import_times = [tr.import_time_us for tr in traceroutes if tr.import_time_us]
if import_times:
first_time = min(import_times)
else:
@@ -378,7 +381,7 @@ async def graph_traceroute(request):
f'[{node.short_name}] {node.long_name}\n{node_id_to_hex(node_id)}\n{node.role}'
)
if node_id in node_seen_time:
ms = (node_seen_time[node_id] - first_time).total_seconds() * 1000
ms = (node_seen_time[node_id] - first_time) / 1000
node_name += f'\n {ms:.2f}ms'
style = 'dashed'
if node_id == dest:
@@ -396,7 +399,7 @@ async def graph_traceroute(request):
shape='box',
color=node_color.get(node_id, 'black'),
style=style,
href=f"/packet_list/{node_id}",
href=f"/node/{node_id}",
)
)
@@ -412,6 +415,7 @@ async def graph_traceroute(request):
async def run_server():
"""Start the aiohttp web server after migrations are complete."""
# Wait for database migrations to complete before starting web server
logger.info("Checking database schema status...")
database_url = CONFIG["database"]["connection_string"]
@@ -428,6 +432,7 @@ async def run_server():
logger.info("Database schema verified - starting web server")
app = web.Application()
app.router.add_static("/static/", pathlib.Path(__file__).parent / "static")
app.add_routes(api.routes) # Add API routes
app.add_routes(routes) # Add main web routes
+47 -61
View File
@@ -126,7 +126,6 @@ async def api_packets(request):
"portnum": int(p.portnum) if p.portnum is not None else None,
"payload": (p.payload or "").strip(),
"import_time_us": p.import_time_us,
"import_time": p.import_time.isoformat() if p.import_time else None,
"channel": getattr(p.from_node, "channel", ""),
"long_name": getattr(p.from_node, "long_name", ""),
}
@@ -208,7 +207,6 @@ async def api_packets(request):
packet_dict = {
"id": p.id,
"import_time_us": p.import_time_us,
"import_time": p.import_time.isoformat() if p.import_time else None,
"channel": getattr(p.from_node, "channel", ""),
"from_node_id": p.from_node_id,
"to_node_id": p.to_node_id,
@@ -228,20 +226,12 @@ async def api_packets(request):
packets_data.append(packet_dict)
# --- Latest import_time for incremental fetch ---
# --- Latest import_time_us for incremental fetch ---
latest_import_time = None
if packets_data:
for p in packets_data:
if p.get("import_time_us") and p["import_time_us"] > 0:
latest_import_time = max(latest_import_time or 0, p["import_time_us"])
elif p.get("import_time") and latest_import_time is None:
try:
dt = datetime.datetime.fromisoformat(
p["import_time"].replace("Z", "+00:00")
)
latest_import_time = int(dt.timestamp() * 1_000_000)
except Exception:
pass
response = {"packets": packets_data}
if latest_import_time is not None:
@@ -431,14 +421,10 @@ async def api_edges(request):
try:
node_filter = int(node_filter_str)
except ValueError:
return web.json_response(
{"error": "node_id must be integer"},
status=400
)
return web.json_response({"error": "node_id must be integer"}, status=400)
edges = {}
traceroute_count = 0
neighbor_packet_count = 0
edges_added_tr = 0
edges_added_neighbor = 0
@@ -463,8 +449,6 @@ async def api_edges(request):
# --- Neighbor edges ---
if filter_type in (None, "neighbor"):
packets = await store.get_packets(portnum=71)
neighbor_packet_count = len(packets)
for packet in packets:
try:
_, neighbor_info = decode_payload.decode(packet)
@@ -479,21 +463,16 @@ async def api_edges(request):
# Convert to list
edges_list = [
{"from": frm, "to": to, "type": edge_type}
for (frm, to), edge_type in edges.items()
{"from": frm, "to": to, "type": edge_type} for (frm, to), edge_type in edges.items()
]
# NEW → apply node_id filtering
if node_filter is not None:
edges_list = [
e for e in edges_list
if e["from"] == node_filter or e["to"] == node_filter
]
edges_list = [e for e in edges_list if e["from"] == node_filter or e["to"] == node_filter]
return web.json_response({"edges": edges_list})
@routes.get("/api/config")
async def api_config(request):
try:
@@ -711,7 +690,6 @@ async def api_packets_seen(request):
"rx_snr": row.rx_snr,
"rx_rssi": row.rx_rssi,
"topic": row.topic,
"import_time": (row.import_time.isoformat() if row.import_time else None),
"import_time_us": row.import_time_us,
}
)
@@ -725,6 +703,7 @@ async def api_packets_seen(request):
status=500,
)
@routes.get("/api/traceroute/{packet_id}")
async def api_traceroute(request):
packet_id = int(request.match_info['packet_id'])
@@ -746,14 +725,15 @@ async def api_traceroute(request):
forward_list = list(route.route)
reverse_list = list(route.route_back)
tr_groups.append({
"index": idx,
"import_time": tr.import_time.isoformat() if tr.import_time else None,
"gateway_node_id": tr.gateway_node_id,
"done": tr.done,
"forward_hops": forward_list,
"reverse_hops": reverse_list,
})
tr_groups.append(
{
"index": idx,
"gateway_node_id": tr.gateway_node_id,
"done": tr.done,
"forward_hops": forward_list,
"reverse_hops": reverse_list,
}
)
# --------------------------------------------
# Compute UNIQUE paths + counts + winning path
@@ -796,18 +776,20 @@ async def api_traceroute(request):
# --------------------------------------------
# Final API output
# --------------------------------------------
return web.json_response({
"packet": {
"id": packet.id,
"from": packet.from_node_id,
"to": packet.to_node_id,
"channel": packet.channel,
},
"traceroute_packets": tr_groups,
"unique_forward_paths": unique_forward_paths_json,
"unique_reverse_paths": unique_reverse_paths_json,
"winning_paths": winning_paths_json,
})
return web.json_response(
{
"packet": {
"id": packet.id,
"from": packet.from_node_id,
"to": packet.to_node_id,
"channel": packet.channel,
},
"traceroute_packets": tr_groups,
"unique_forward_paths": unique_forward_paths_json,
"unique_reverse_paths": unique_reverse_paths_json,
"winning_paths": winning_paths_json,
}
)
@routes.get("/api/stats/top")
@@ -894,19 +876,23 @@ async def api_stats_top(request):
nodes = []
for r in rows:
avg = r.seen / max(r.sent, 1)
nodes.append({
"node_id": r.node_id,
"long_name": r.long_name,
"short_name": r.short_name,
"channel": r.channel,
"sent": r.sent,
"seen": r.seen,
"avg": round(avg, 2),
})
nodes.append(
{
"node_id": r.node_id,
"long_name": r.long_name,
"short_name": r.short_name,
"channel": r.channel,
"sent": r.sent,
"seen": r.seen,
"avg": round(avg, 2),
}
)
return web.json_response({
"total": total,
"limit": limit,
"offset": offset,
"nodes": nodes,
})
return web.json_response(
{
"total": total,
"limit": limit,
"offset": offset,
"nodes": nodes,
}
)