From 64261d3bc47c9c2628116f3c1d3ebf964639b74e Mon Sep 17 00:00:00 2001 From: Joel Krauska Date: Mon, 3 Nov 2025 20:05:04 -0800 Subject: [PATCH] ruff and docs --- alembic/env.py | 1 + .../1717fa5c6545_add_example_table.py | 8 +-- .../versions/add_import_time_us_columns.py | 30 +++++--- .../c88468b7ab0b_initial_migration.py | 68 +++++++++++++------ docs/API_Documentation.md | 6 +- meshview/__version__.py | 1 + meshview/web.py | 7 -- meshview/web_api/api.py | 35 +++++++++- 8 files changed, 113 insertions(+), 43 deletions(-) diff --git a/alembic/env.py b/alembic/env.py index e7a505c..bcc04d8 100644 --- a/alembic/env.py +++ b/alembic/env.py @@ -89,6 +89,7 @@ def run_migrations_online() -> None: loop = asyncio.get_running_loop() # Event loop is already running, schedule and run the coroutine import concurrent.futures + with concurrent.futures.ThreadPoolExecutor() as pool: pool.submit(lambda: asyncio.run(run_async_migrations())).result() except RuntimeError: diff --git a/alembic/versions/1717fa5c6545_add_example_table.py b/alembic/versions/1717fa5c6545_add_example_table.py index 4418a3c..bc4d59a 100644 --- a/alembic/versions/1717fa5c6545_add_example_table.py +++ b/alembic/versions/1717fa5c6545_add_example_table.py @@ -6,7 +6,7 @@ Create Date: 2025-10-26 20:59:04.347066 """ -from typing import Sequence, Union +from collections.abc import Sequence import sqlalchemy as sa @@ -14,9 +14,9 @@ from alembic import op # revision identifiers, used by Alembic. revision: str = '1717fa5c6545' -down_revision: Union[str, None] = 'add_time_us_cols' -branch_labels: Union[str, Sequence[str], None] = None -depends_on: Union[str, Sequence[str], None] = None +down_revision: str | None = 'add_time_us_cols' +branch_labels: str | Sequence[str] | None = None +depends_on: str | Sequence[str] | None = None def upgrade() -> None: diff --git a/alembic/versions/add_import_time_us_columns.py b/alembic/versions/add_import_time_us_columns.py index cfeb0ef..daf588e 100644 --- a/alembic/versions/add_import_time_us_columns.py +++ b/alembic/versions/add_import_time_us_columns.py @@ -5,17 +5,18 @@ Revises: c88468b7ab0b Create Date: 2025-11-03 14:10:00.000000 """ -from typing import Sequence, Union -from alembic import op +from collections.abc import Sequence + import sqlalchemy as sa +from alembic import op # revision identifiers, used by Alembic. revision: str = 'add_time_us_cols' -down_revision: Union[str, None] = 'c88468b7ab0b' -branch_labels: Union[str, Sequence[str], None] = None -depends_on: Union[str, Sequence[str], None] = None +down_revision: str | None = 'c88468b7ab0b' +branch_labels: str | Sequence[str] | None = None +depends_on: str | Sequence[str] | None = None def upgrade() -> None: @@ -28,22 +29,33 @@ def upgrade() -> None: if 'import_time_us' not in packet_columns: with op.batch_alter_table('packet', schema=None) as batch_op: batch_op.add_column(sa.Column('import_time_us', sa.BigInteger(), nullable=True)) - op.create_index('idx_packet_import_time_us', 'packet', [sa.text('import_time_us DESC')], unique=False) - op.create_index('idx_packet_from_node_time_us', 'packet', ['from_node_id', sa.text('import_time_us DESC')], unique=False) + op.create_index( + 'idx_packet_import_time_us', 'packet', [sa.text('import_time_us DESC')], unique=False + ) + op.create_index( + 'idx_packet_from_node_time_us', + 'packet', + ['from_node_id', sa.text('import_time_us DESC')], + unique=False, + ) # Add import_time_us to packet_seen table packet_seen_columns = [col['name'] for col in inspector.get_columns('packet_seen')] if 'import_time_us' not in packet_seen_columns: with op.batch_alter_table('packet_seen', schema=None) as batch_op: batch_op.add_column(sa.Column('import_time_us', sa.BigInteger(), nullable=True)) - op.create_index('idx_packet_seen_import_time_us', 'packet_seen', ['import_time_us'], unique=False) + op.create_index( + 'idx_packet_seen_import_time_us', 'packet_seen', ['import_time_us'], unique=False + ) # Add import_time_us to traceroute table traceroute_columns = [col['name'] for col in inspector.get_columns('traceroute')] if 'import_time_us' not in traceroute_columns: with op.batch_alter_table('traceroute', schema=None) as batch_op: batch_op.add_column(sa.Column('import_time_us', sa.BigInteger(), nullable=True)) - op.create_index('idx_traceroute_import_time_us', 'traceroute', ['import_time_us'], unique=False) + op.create_index( + 'idx_traceroute_import_time_us', 'traceroute', ['import_time_us'], unique=False + ) def downgrade() -> None: diff --git a/alembic/versions/c88468b7ab0b_initial_migration.py b/alembic/versions/c88468b7ab0b_initial_migration.py index a307d6e..9114a18 100644 --- a/alembic/versions/c88468b7ab0b_initial_migration.py +++ b/alembic/versions/c88468b7ab0b_initial_migration.py @@ -6,7 +6,7 @@ Create Date: 2025-10-26 20:56:50.285200 """ -from typing import Sequence, Union +from collections.abc import Sequence import sqlalchemy as sa @@ -14,9 +14,9 @@ from alembic import op # revision identifiers, used by Alembic. revision: str = 'c88468b7ab0b' -down_revision: Union[str, None] = None -branch_labels: Union[str, Sequence[str], None] = None -depends_on: Union[str, Sequence[str], None] = None +down_revision: str | None = None +branch_labels: str | Sequence[str] | None = None +depends_on: str | Sequence[str] | None = None def upgrade() -> None: @@ -28,7 +28,8 @@ def upgrade() -> None: # Create node table if it doesn't exist if 'node' not in existing_tables: - op.create_table('node', + op.create_table( + 'node', sa.Column('id', sa.String(), nullable=False), sa.Column('node_id', sa.BigInteger(), nullable=True), sa.Column('long_name', sa.String(), nullable=True), @@ -41,13 +42,14 @@ def upgrade() -> None: sa.Column('channel', sa.String(), nullable=True), sa.Column('last_update', sa.DateTime(), nullable=True), sa.PrimaryKeyConstraint('id'), - sa.UniqueConstraint('node_id') + sa.UniqueConstraint('node_id'), ) op.create_index('idx_node_node_id', 'node', ['node_id'], unique=False) # Create packet table if it doesn't exist if 'packet' not in existing_tables: - op.create_table('packet', + op.create_table( + 'packet', sa.Column('id', sa.BigInteger(), nullable=False), sa.Column('portnum', sa.Integer(), nullable=True), sa.Column('from_node_id', sa.BigInteger(), nullable=True), @@ -56,18 +58,33 @@ def upgrade() -> None: sa.Column('import_time', sa.DateTime(), nullable=True), sa.Column('import_time_us', sa.BigInteger(), nullable=True), sa.Column('channel', sa.String(), nullable=True), - sa.PrimaryKeyConstraint('id') + sa.PrimaryKeyConstraint('id'), ) op.create_index('idx_packet_from_node_id', 'packet', ['from_node_id'], unique=False) op.create_index('idx_packet_to_node_id', 'packet', ['to_node_id'], unique=False) - op.create_index('idx_packet_import_time', 'packet', [sa.text('import_time DESC')], unique=False) - op.create_index('idx_packet_import_time_us', 'packet', [sa.text('import_time_us DESC')], unique=False) - op.create_index('idx_packet_from_node_time', 'packet', ['from_node_id', sa.text('import_time DESC')], unique=False) - op.create_index('idx_packet_from_node_time_us', 'packet', ['from_node_id', sa.text('import_time_us DESC')], unique=False) + op.create_index( + 'idx_packet_import_time', 'packet', [sa.text('import_time DESC')], unique=False + ) + op.create_index( + 'idx_packet_import_time_us', 'packet', [sa.text('import_time_us DESC')], unique=False + ) + op.create_index( + 'idx_packet_from_node_time', + 'packet', + ['from_node_id', sa.text('import_time DESC')], + unique=False, + ) + op.create_index( + 'idx_packet_from_node_time_us', + 'packet', + ['from_node_id', sa.text('import_time_us DESC')], + unique=False, + ) # Create packet_seen table if it doesn't exist if 'packet_seen' not in existing_tables: - op.create_table('packet_seen', + op.create_table( + 'packet_seen', sa.Column('packet_id', sa.BigInteger(), nullable=False), sa.Column('node_id', sa.BigInteger(), nullable=False), sa.Column('rx_time', sa.BigInteger(), nullable=False), @@ -79,16 +96,22 @@ def upgrade() -> None: sa.Column('topic', sa.String(), nullable=True), sa.Column('import_time', sa.DateTime(), nullable=True), sa.Column('import_time_us', sa.BigInteger(), nullable=True), - sa.ForeignKeyConstraint(['packet_id'], ['packet.id'], ), - sa.PrimaryKeyConstraint('packet_id', 'node_id', 'rx_time') + sa.ForeignKeyConstraint( + ['packet_id'], + ['packet.id'], + ), + sa.PrimaryKeyConstraint('packet_id', 'node_id', 'rx_time'), ) op.create_index('idx_packet_seen_node_id', 'packet_seen', ['node_id'], unique=False) op.create_index('idx_packet_seen_packet_id', 'packet_seen', ['packet_id'], unique=False) - op.create_index('idx_packet_seen_import_time_us', 'packet_seen', ['import_time_us'], unique=False) + op.create_index( + 'idx_packet_seen_import_time_us', 'packet_seen', ['import_time_us'], unique=False + ) # Create traceroute table if it doesn't exist if 'traceroute' not in existing_tables: - op.create_table('traceroute', + op.create_table( + 'traceroute', sa.Column('id', sa.Integer(), nullable=False), sa.Column('packet_id', sa.BigInteger(), nullable=True), sa.Column('gateway_node_id', sa.BigInteger(), nullable=True), @@ -96,11 +119,16 @@ def upgrade() -> None: sa.Column('route', sa.LargeBinary(), nullable=True), sa.Column('import_time', sa.DateTime(), nullable=True), sa.Column('import_time_us', sa.BigInteger(), nullable=True), - sa.ForeignKeyConstraint(['packet_id'], ['packet.id'], ), - sa.PrimaryKeyConstraint('id') + sa.ForeignKeyConstraint( + ['packet_id'], + ['packet.id'], + ), + sa.PrimaryKeyConstraint('id'), ) op.create_index('idx_traceroute_import_time', 'traceroute', ['import_time'], unique=False) - op.create_index('idx_traceroute_import_time_us', 'traceroute', ['import_time_us'], unique=False) + op.create_index( + 'idx_traceroute_import_time_us', 'traceroute', ['import_time_us'], unique=False + ) # ### end Alembic commands ### diff --git a/docs/API_Documentation.md b/docs/API_Documentation.md index a0ba194..53a3ff2 100644 --- a/docs/API_Documentation.md +++ b/docs/API_Documentation.md @@ -298,9 +298,11 @@ Health check endpoint for monitoring, load balancers, and orchestration systems. { "status": "healthy", "timestamp": "2025-11-03T14:30:00.123456Z", - "version": "2.0.8", + "version": "3.0.0", "git_revision": "6416978", - "database": "connected" + "database": "connected", + "database_size": "853.03 MB", + "database_size_bytes": 894468096 } ``` diff --git a/meshview/__version__.py b/meshview/__version__.py index ae8b3f5..696a8d7 100644 --- a/meshview/__version__.py +++ b/meshview/__version__.py @@ -1,4 +1,5 @@ """Version information for MeshView.""" + import subprocess from pathlib import Path diff --git a/meshview/web.py b/meshview/web.py index e07c56b..3167bf9 100644 --- a/meshview/web.py +++ b/meshview/web.py @@ -1,12 +1,10 @@ import asyncio import datetime -import json import logging import os import pathlib import re import ssl -import subprocess import traceback from collections import Counter, defaultdict from dataclasses import dataclass @@ -19,16 +17,11 @@ from google.protobuf.message import Message from jinja2 import Environment, PackageLoader, Undefined, select_autoescape from markupsafe import Markup from pandas import DataFrame -from sqlalchemy import text from meshtastic.protobuf.portnums_pb2 import PortNum from meshview import config, database, decode_payload, migrations, models, store from meshview.__version__ import ( - __version__, __version_string__, - _git_revision, - _git_revision_short, - get_version_info, ) from meshview.web_api import api diff --git a/meshview/web_api/api.py b/meshview/web_api/api.py index dcf5e05..99834c9 100644 --- a/meshview/web_api/api.py +++ b/meshview/web_api/api.py @@ -1,9 +1,9 @@ """API endpoints for MeshView.""" + import datetime import json import logging import os -import re from aiohttp import web from sqlalchemy import text @@ -43,6 +43,7 @@ async def api_channels(request: web.Request): except Exception as e: return web.json_response({"channels": [], "error": str(e)}) + @routes.get("/api/chat") async def api_chat(request): try: @@ -130,6 +131,7 @@ async def api_chat(request): {"error": "Failed to fetch chat data", "details": str(e)}, status=500 ) + @routes.get("/api/nodes") async def api_nodes(request): try: @@ -175,6 +177,7 @@ async def api_nodes(request): logger.error(f"Error in /api/nodes: {e}") return web.json_response({"error": "Failed to fetch nodes"}, status=500) + @routes.get("/api/packets") async def api_packets(request): try: @@ -215,6 +218,7 @@ async def api_packets(request): logger.error(f"Error in /api/packets: {e}") return web.json_response({"error": "Failed to fetch packets"}, status=500) + @routes.get("/api/stats") async def api_stats(request): """ @@ -267,6 +271,7 @@ async def api_stats(request): return web.json_response(stats) + @routes.get("/api/edges") async def api_edges(request): since = datetime.datetime.now() - datetime.timedelta(hours=48) @@ -309,6 +314,7 @@ async def api_edges(request): return web.json_response({"edges": edges_list}) + @routes.get("/api/config") async def api_config(request): try: @@ -411,6 +417,7 @@ async def api_config(request): except Exception as e: return web.json_response({"error": str(e)}, status=500) + @routes.get("/api/lang") async def api_lang(request): # Language from ?lang=xx, fallback to config, then to "en" @@ -437,6 +444,7 @@ async def api_lang(request): # if no section requested → return full translation file return web.json_response(translations) + @routes.get("/health") async def health_check(request): """Health check endpoint for monitoring and load balancers.""" @@ -458,8 +466,33 @@ async def health_check(request): health_status["status"] = "unhealthy" return web.json_response(health_status, status=503) + # Get database file size + try: + db_url = CONFIG.get("database", {}).get("connection_string", "") + # Extract file path from SQLite connection string (e.g., "sqlite+aiosqlite:///packets.db") + if "sqlite" in db_url.lower(): + db_path = db_url.split("///")[-1].split("?")[0] + if os.path.exists(db_path): + db_size_bytes = os.path.getsize(db_path) + # Convert to human-readable format + if db_size_bytes < 1024: + health_status["database_size"] = f"{db_size_bytes} B" + elif db_size_bytes < 1024 * 1024: + health_status["database_size"] = f"{db_size_bytes / 1024:.2f} KB" + elif db_size_bytes < 1024 * 1024 * 1024: + health_status["database_size"] = f"{db_size_bytes / (1024 * 1024):.2f} MB" + else: + health_status["database_size"] = ( + f"{db_size_bytes / (1024 * 1024 * 1024):.2f} GB" + ) + health_status["database_size_bytes"] = db_size_bytes + except Exception as e: + logger.warning(f"Failed to get database size: {e}") + # Don't fail health check if we can't get size + return web.json_response(health_status) + @routes.get("/version") async def version_endpoint(request): """Return version information including semver and git revision."""