From d4f6a8ba9b575337aea9992fa9dee5624ee82a18 Mon Sep 17 00:00:00 2001 From: pablorevilla-meshtastic Date: Fri, 29 May 2026 14:30:44 -0700 Subject: [PATCH 01/13] fixed bug in top.html now it shows all channels as a option --- README.md | 2 +- meshview/lang/en.json | 1 + meshview/lang/es.json | 1 + meshview/lang/ru.json | 1 + meshview/templates/top.html | 7 ++++++- 5 files changed, 10 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 0726141..0d21e36 100644 --- a/README.md +++ b/README.md @@ -9,7 +9,7 @@ The project serves as a real-time monitoring and diagnostic tool for the Meshtas - Node list/map UX: added node list filtering for wider active-node ranges, included all database nodes where appropriate, and made map node colors persistent across reboots. - QR/API fixes: corrected QR code generation when a node role is missing or unavailable. - Database cleanup: removed stored node public key data and added the Alembic migration to drop `node.public_key`. -- Protobufs: refreshed vendored Meshtastic protobuf Python modules to upstream commit `9ab4a1d08cb833d897aee2367013318718391f2f`, including new hardware models such as `RAK6421`. +- Protobufs: refreshed Meshtastic protobuf Python modules to upstream commit - Tooling: improved the protobuf updater so it supports raw commit SHAs, the newer upstream `meshtastic/*.proto` layout, generated `.pyi` stubs, and the vendored `nanopb`/`serial_hal` outputs. ### Version 3.0.6 — March 2026 diff --git a/meshview/lang/en.json b/meshview/lang/en.json index 3b6b9d9..6eec50e 100644 --- a/meshview/lang/en.json +++ b/meshview/lang/en.json @@ -159,6 +159,7 @@ "times_seen": "Seen (24h)", "avg_gateways": "Avg Gateways", "showing_nodes": "Showing", + "all_channels": "All Channels", "nodes_suffix": "nodes" }, diff --git a/meshview/lang/es.json b/meshview/lang/es.json index 7d21ec1..e7561d9 100644 --- a/meshview/lang/es.json +++ b/meshview/lang/es.json @@ -155,6 +155,7 @@ "times_seen": "Visto (24h)", "avg_gateways": "Promedio de Gateways", "showing_nodes": "Mostrando", + "all_channels": "Todos los Canales", "nodes_suffix": "nodos" }, diff --git a/meshview/lang/ru.json b/meshview/lang/ru.json index f1727ba..902aa64 100644 --- a/meshview/lang/ru.json +++ b/meshview/lang/ru.json @@ -145,6 +145,7 @@ "times_seen": "Принято (24ч)", "avg_gateways": "Среднее количество шлюзов", "showing_nodes": "Показать", + "all_channels": "Все каналы", "nodes_suffix": "ноды" }, "nodegraph": { diff --git a/meshview/templates/top.html b/meshview/templates/top.html index a033a05..cc17574 100644 --- a/meshview/templates/top.html +++ b/meshview/templates/top.html @@ -151,6 +151,11 @@ async function loadChannels() { const sel = document.getElementById("channelFilter"); sel.innerHTML = ""; + const allOpt = document.createElement("option"); + allOpt.value = ""; + allOpt.textContent = topTranslations.all_channels || "All channels"; + sel.appendChild(allOpt); + for (const ch of data.channels || []) { const opt = document.createElement("option"); opt.value = ch; @@ -158,7 +163,7 @@ async function loadChannels() { sel.appendChild(opt); } - sel.value = "MediumFast"; + sel.value = ""; } /* ====================================================== From 02c77dcca8f20b7ace4b43c41bc714bfb0d5bbf8 Mon Sep 17 00:00:00 2001 From: pablorevilla-meshtastic Date: Fri, 29 May 2026 14:39:04 -0700 Subject: [PATCH 02/13] change the versio n number and release date for the next release --- meshview/__version__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/meshview/__version__.py b/meshview/__version__.py index 1abafc8..50d30b1 100644 --- a/meshview/__version__.py +++ b/meshview/__version__.py @@ -3,7 +3,7 @@ import subprocess from pathlib import Path -__version__ = "3.0.7" +__version__ = "3.0.8" __release_date__ = "2026-5-12" From 029d22ff9c83e3532689a66aae8ac7072e74f94d Mon Sep 17 00:00:00 2001 From: pablorevilla-meshtastic Date: Thu, 4 Jun 2026 17:14:43 -0700 Subject: [PATCH 03/13] report of cleanup status on health page --- .gitignore | 1 + docs/API_Documentation.md | 43 ++++++++++++++++++ meshview/mqtt_store.py | 19 ++++++-- meshview/web_api/api.py | 92 ++++++++++++++++++++++++++++++++++++++ startdb.py | 94 ++++++++++++++++++++++++++++++++++++++- 5 files changed, 243 insertions(+), 6 deletions(-) diff --git a/.gitignore b/.gitignore index 7a94de8..d4f6cdd 100644 --- a/.gitignore +++ b/.gitignore @@ -22,6 +22,7 @@ meshview-web.pid /table_details.py config.ini *.log +*.status.json # Screenshots screenshots/* diff --git a/docs/API_Documentation.md b/docs/API_Documentation.md index bc8e10a..564c135 100644 --- a/docs/API_Documentation.md +++ b/docs/API_Documentation.md @@ -368,6 +368,49 @@ Response Example "timestamp": "2025-07-22T12:45:00+00:00", "version": "3.0.3", "git_revision": "abc1234", + "cleanup": { + "enabled": true, + "days_to_keep": 14, + "scheduled_time": "02:00", + "vacuum": false, + "status": "ok", + "status_file": "dbcleanup.status.json", + "last_run": { + "status": "ok", + "started_at": "2026-06-04T09:00:00+00:00", + "completed_at": "2026-06-04T09:00:03+00:00", + "cutoff_at": "2026-05-21T09:00:00+00:00", + "days_to_keep": 14, + "vacuum_requested": false, + "vacuum_completed": false, + "rows_deleted": { + "packet": 1200, + "packet_seen": 3400, + "traceroute": 42, + "node": 3 + }, + "error": null + } + }, + "backup": { + "enabled": true, + "backup_dir": "./backups", + "scheduled_time": "02:00", + "status": "ok", + "status_file": "dbbackup.status.json", + "last_run": { + "status": "ok", + "started_at": "2026-06-04T09:00:00+00:00", + "completed_at": "2026-06-04T09:00:02+00:00", + "backup_dir": "./backups", + "database_path": "packets.db", + "backup_file": "backups/packets_backup_20260604_090000.db.gz", + "original_size_bytes": 12939444, + "compressed_size_bytes": 4211560, + "compression_percent": 67.5, + "error": null + } + }, "database": "connected", "database_size": "12.34 MB", "database_size_bytes": 12939444 diff --git a/meshview/mqtt_store.py b/meshview/mqtt_store.py index 825de1f..6bd9bab 100644 --- a/meshview/mqtt_store.py +++ b/meshview/mqtt_store.py @@ -17,6 +17,15 @@ from meshview.models import DailySnapshot, Node, Packet, PacketSeen, Traceroute logger = logging.getLogger(__name__) MQTT_GATEWAY_CACHE: set[int] = set() +UNKNOWN_NODE_NAME = "unknown name" +NODE_NAME_CONTROL_CHARS = re.compile(r"[\x00-\x1f\x7f]") + + +def normalize_node_name(name: str) -> str: + name = name.strip() + if not name or NODE_NAME_CONTROL_CHARS.search(name): + return UNKNOWN_NODE_NAME + return name async def capture_daily_snapshot() -> None: @@ -268,11 +277,13 @@ async def process_envelope(topic, env): ).scalar_one_or_none() now_us = int(time.time() * 1_000_000) + long_name = normalize_node_name(user.long_name) + short_name = normalize_node_name(user.short_name) if node: node.node_id = node_id - node.long_name = user.long_name - node.short_name = user.short_name + node.long_name = long_name + node.short_name = short_name node.hw_model = hw_model node.role = role node.channel = env.channel_id @@ -283,8 +294,8 @@ async def process_envelope(topic, env): node = Node( id=user.id, node_id=node_id, - long_name=user.long_name, - short_name=user.short_name, + long_name=long_name, + short_name=short_name, hw_model=hw_model, role=role, channel=env.channel_id, diff --git a/meshview/web_api/api.py b/meshview/web_api/api.py index 1a8956d..71e5c99 100644 --- a/meshview/web_api/api.py +++ b/meshview/web_api/api.py @@ -38,6 +38,96 @@ _LANG_CACHE = {} routes = web.RouteTableDef() +def _config_bool(section: str, key: str, default: bool = False) -> bool: + return str(CONFIG.get(section, {}).get(key, default)).lower() in ("1", "true", "yes", "on") + + +def _config_int(section: str, key: str, default: int = 0) -> int: + try: + return int(CONFIG.get(section, {}).get(key, default)) + except (TypeError, ValueError): + return default + + +def _config_str(section: str, key: str, default: str = "") -> str: + value = CONFIG.get(section, {}).get(key, default) + return str(value) if value is not None else default + + +def _cleanup_status_file() -> str: + cleanup_logfile = CONFIG.get("logging", {}).get("db_cleanup_logfile", "dbcleanup.log") + path_without_extension, _ = os.path.splitext(cleanup_logfile) + return f"{path_without_extension}.status.json" + + +def _backup_status_file() -> str: + cleanup_logfile = CONFIG.get("logging", {}).get("db_cleanup_logfile", "dbcleanup.log") + return os.path.join(os.path.dirname(cleanup_logfile), "dbbackup.status.json") + + +def _get_cleanup_health() -> dict: + cleanup_health = { + "enabled": _config_bool("cleanup", "enabled", False), + "days_to_keep": _config_int("cleanup", "days_to_keep", 14), + "scheduled_time": ( + f"{_config_int('cleanup', 'hour', 2):02d}:{_config_int('cleanup', 'minute', 0):02d}" + ), + "vacuum": _config_bool("cleanup", "vacuum", False), + "status": "disabled", + } + + if cleanup_health["enabled"]: + cleanup_health["status"] = "unknown" + + status_file = _cleanup_status_file() + cleanup_health["status_file"] = status_file + if not os.path.exists(status_file): + return cleanup_health + + try: + with open(status_file, encoding="utf-8") as f: + last_run = json.load(f) + except Exception as e: + cleanup_health["status"] = "error" + cleanup_health["error"] = f"Unable to read cleanup status: {e}" + return cleanup_health + + cleanup_health["status"] = last_run.get("status", cleanup_health["status"]) + cleanup_health["last_run"] = last_run + return cleanup_health + + +def _get_backup_health() -> dict: + backup_hour = _config_int("cleanup", "backup_hour", _config_int("cleanup", "hour", 2)) + backup_minute = _config_int("cleanup", "backup_minute", _config_int("cleanup", "minute", 0)) + backup_health = { + "enabled": _config_bool("cleanup", "backup_enabled", False), + "backup_dir": _config_str("cleanup", "backup_dir", "./backups"), + "scheduled_time": f"{backup_hour:02d}:{backup_minute:02d}", + "status": "disabled", + } + + if backup_health["enabled"]: + backup_health["status"] = "unknown" + + status_file = _backup_status_file() + backup_health["status_file"] = status_file + if not os.path.exists(status_file): + return backup_health + + try: + with open(status_file, encoding="utf-8") as f: + last_run = json.load(f) + except Exception as e: + backup_health["status"] = "error" + backup_health["error"] = f"Unable to read backup status: {e}" + return backup_health + + backup_health["status"] = last_run.get("status", backup_health["status"]) + backup_health["last_run"] = last_run + return backup_health + + def _haversine_km(lat1, lon1, lat2, lon2): r = 6371.0 phi1 = math.radians(lat1) @@ -720,6 +810,8 @@ async def health_check(request): "timestamp": datetime.datetime.now(datetime.UTC).isoformat(), "version": __version__, "git_revision": _git_revision_short, + "cleanup": _get_cleanup_health(), + "backup": _get_backup_health(), } # Check database connectivity diff --git a/startdb.py b/startdb.py index 1ef461d..6c08c38 100644 --- a/startdb.py +++ b/startdb.py @@ -33,6 +33,8 @@ file_handler.setLevel(logging.INFO) formatter = logging.Formatter('%(asctime)s [%(levelname)s] %(message)s') file_handler.setFormatter(formatter) cleanup_logger.addHandler(file_handler) +cleanup_status_file = str(Path(cleanup_logfile).with_suffix(".status.json")) +backup_status_file = str(Path(cleanup_logfile).with_name("dbbackup.status.json")) # ------------------------- @@ -49,6 +51,32 @@ def get_int(config, section, key, default=0): return default +def _rowcount(value): + return value if value is not None and value >= 0 else None + + +def write_cleanup_status(status: dict) -> None: + status_path = Path(cleanup_status_file) + tmp_path = status_path.with_suffix(f"{status_path.suffix}.tmp") + try: + with open(tmp_path, "w", encoding="utf-8") as f: + json.dump(status, f, indent=2, sort_keys=True) + tmp_path.replace(status_path) + except Exception as e: + cleanup_logger.warning(f"Failed to write cleanup status file: {e}") + + +def write_backup_status(status: dict) -> None: + status_path = Path(backup_status_file) + tmp_path = status_path.with_suffix(f"{status_path.suffix}.tmp") + try: + with open(tmp_path, "w", encoding="utf-8") as f: + json.dump(status, f, indent=2, sort_keys=True) + tmp_path.replace(status_path) + except Exception as e: + cleanup_logger.warning(f"Failed to write backup status file: {e}") + + # ------------------------- # Shared DB lock # ------------------------- @@ -66,19 +94,40 @@ async def backup_database(database_url: str, backup_dir: str = ".") -> None: database_url: SQLAlchemy connection string backup_dir: Directory to store backups (default: current directory) """ + backup_status = { + "status": "running", + "started_at": datetime.datetime.now(datetime.UTC).isoformat(), + "completed_at": None, + "backup_dir": backup_dir, + "database_path": None, + "backup_file": None, + "original_size_bytes": None, + "compressed_size_bytes": None, + "compression_percent": None, + "error": None, + } + write_backup_status(backup_status) + try: url = make_url(database_url) if not url.drivername.startswith("sqlite"): cleanup_logger.warning("Backup only supported for SQLite databases") + backup_status["status"] = "unsupported" + backup_status["error"] = "Backup only supported for SQLite databases" return if not url.database or url.database == ":memory:": cleanup_logger.error("Could not extract database path from connection string") + backup_status["status"] = "error" + backup_status["error"] = "Could not extract database path from connection string" return db_file = Path(url.database) + backup_status["database_path"] = str(db_file) if not db_file.exists(): cleanup_logger.error(f"Database file not found: {db_file}") + backup_status["status"] = "error" + backup_status["error"] = f"Database file not found: {db_file}" return # Create backup directory if it doesn't exist @@ -89,6 +138,7 @@ async def backup_database(database_url: str, backup_dir: str = ".") -> None: timestamp = datetime.datetime.now().strftime("%Y%m%d_%H%M%S") backup_filename = f"{db_file.stem}_backup_{timestamp}.db.gz" backup_file = backup_path / backup_filename + backup_status["backup_file"] = str(backup_file) cleanup_logger.info(f"Creating backup: {backup_file}") @@ -98,9 +148,15 @@ async def backup_database(database_url: str, backup_dir: str = ".") -> None: shutil.copyfileobj(f_in, f_out) # Get file sizes for logging - original_size = db_file.stat().st_size / (1024 * 1024) # MB - compressed_size = backup_file.stat().st_size / (1024 * 1024) # MB + original_size_bytes = db_file.stat().st_size + compressed_size_bytes = backup_file.stat().st_size + original_size = original_size_bytes / (1024 * 1024) # MB + compressed_size = compressed_size_bytes / (1024 * 1024) # MB compression_ratio = (1 - compressed_size / original_size) * 100 if original_size > 0 else 0 + backup_status["original_size_bytes"] = original_size_bytes + backup_status["compressed_size_bytes"] = compressed_size_bytes + backup_status["compression_percent"] = round(compression_ratio, 1) + backup_status["status"] = "ok" cleanup_logger.info( f"Backup created successfully: {backup_file.name} " @@ -110,6 +166,11 @@ async def backup_database(database_url: str, backup_dir: str = ".") -> None: except Exception as e: cleanup_logger.error(f"Error creating database backup: {e}") + backup_status["status"] = "error" + backup_status["error"] = str(e) + finally: + backup_status["completed_at"] = datetime.datetime.now(datetime.UTC).isoformat() + write_backup_status(backup_status) # ------------------------- @@ -179,6 +240,24 @@ async def daily_cleanup_at( ).replace(tzinfo=None) cutoff_us = int(cutoff_dt.timestamp() * 1_000_000) cleanup_logger.info(f"Running cleanup for records older than {cutoff_dt.isoformat()}...") + rows_deleted = { + "packet": None, + "packet_seen": None, + "traceroute": None, + "node": None, + } + cleanup_status = { + "status": "running", + "started_at": datetime.datetime.now(datetime.UTC).isoformat(), + "completed_at": None, + "cutoff_at": cutoff_dt.replace(tzinfo=datetime.UTC).isoformat(), + "days_to_keep": days_to_keep, + "vacuum_requested": vacuum_db, + "vacuum_completed": False, + "rows_deleted": rows_deleted, + "error": None, + } + write_cleanup_status(cleanup_status) try: async with db_lock: # Pause ingestion @@ -191,6 +270,7 @@ async def daily_cleanup_at( result = await session.execute( delete(models.Packet).where(models.Packet.import_time_us < cutoff_us) ) + rows_deleted["packet"] = _rowcount(result.rowcount) cleanup_logger.info(f"Deleted {result.rowcount} rows from Packet") # ------------------------- @@ -201,6 +281,7 @@ async def daily_cleanup_at( models.PacketSeen.import_time_us < cutoff_us ) ) + rows_deleted["packet_seen"] = _rowcount(result.rowcount) cleanup_logger.info(f"Deleted {result.rowcount} rows from PacketSeen") # ------------------------- @@ -211,6 +292,7 @@ async def daily_cleanup_at( models.Traceroute.import_time_us < cutoff_us ) ) + rows_deleted["traceroute"] = _rowcount(result.rowcount) cleanup_logger.info(f"Deleted {result.rowcount} rows from Traceroute") # ------------------------- @@ -219,6 +301,7 @@ async def daily_cleanup_at( result = await session.execute( delete(models.Node).where(models.Node.last_seen_us < cutoff_us) ) + rows_deleted["node"] = _rowcount(result.rowcount) cleanup_logger.info(f"Deleted {result.rowcount} rows from Node") await session.commit() @@ -228,14 +311,21 @@ async def daily_cleanup_at( async with mqtt_database.engine.begin() as conn: await conn.exec_driver_sql("VACUUM;") cleanup_logger.info("VACUUM completed.") + cleanup_status["vacuum_completed"] = True elif vacuum_db: cleanup_logger.info("VACUUM skipped (not supported for this database).") cleanup_logger.info("Cleanup completed successfully.") cleanup_logger.info("Ingestion resumed after cleanup.") + cleanup_status["status"] = "ok" except Exception as e: cleanup_logger.error(f"Error during cleanup: {e}") + cleanup_status["status"] = "error" + cleanup_status["error"] = str(e) + finally: + cleanup_status["completed_at"] = datetime.datetime.now(datetime.UTC).isoformat() + write_cleanup_status(cleanup_status) # ------------------------- From dff2ea0249fb3f5ee9383c7d55a26d3322b3b11a Mon Sep 17 00:00:00 2001 From: pablorevilla-meshtastic Date: Thu, 4 Jun 2026 17:16:11 -0700 Subject: [PATCH 04/13] change --- CONTRIBUTING.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 528a8c7..9dafea5 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -52,7 +52,7 @@ Unacceptable behavior includes harassment, insults, hate speech, personal attack --- -## I Want to Contribute +## I Want to Contribute! ### Legal Notice By contributing to Meshview, you agree that: From 7f5b50b6f391eacbe470bebf1f2dc00ead8b31f8 Mon Sep 17 00:00:00 2001 From: pablorevilla-meshtastic Date: Mon, 8 Jun 2026 11:59:20 -0700 Subject: [PATCH 05/13] Fix timezone or health report --- docs/API_Documentation.md | 15 ++++++++++++++ meshview/web_api/api.py | 42 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 57 insertions(+) diff --git a/docs/API_Documentation.md b/docs/API_Documentation.md index 564c135..522b2b8 100644 --- a/docs/API_Documentation.md +++ b/docs/API_Documentation.md @@ -372,6 +372,9 @@ Response Example "enabled": true, "days_to_keep": 14, "scheduled_time": "02:00", + "scheduled_timezone": "local", + "scheduled_timezone_name": "PDT", + "scheduled_utc_offset": "-07:00", "vacuum": false, "status": "ok", "status_file": "dbcleanup.status.json", @@ -390,12 +393,20 @@ Response Example "node": 3 }, "error": null + }, + "last_run_local": { + "started_at": "2026-06-04T02:00:00-07:00", + "completed_at": "2026-06-04T02:00:03-07:00", + "cutoff_at": "2026-05-21T02:00:00-07:00" } }, "backup": { "enabled": true, "backup_dir": "./backups", "scheduled_time": "02:00", + "scheduled_timezone": "local", + "scheduled_timezone_name": "PDT", + "scheduled_utc_offset": "-07:00", "status": "ok", "status_file": "dbbackup.status.json", "last_run": { @@ -409,6 +420,10 @@ Response Example "compressed_size_bytes": 4211560, "compression_percent": 67.5, "error": null + }, + "last_run_local": { + "started_at": "2026-06-04T02:00:00-07:00", + "completed_at": "2026-06-04T02:00:02-07:00" } }, "database": "connected", diff --git a/meshview/web_api/api.py b/meshview/web_api/api.py index 71e5c99..71c83bc 100644 --- a/meshview/web_api/api.py +++ b/meshview/web_api/api.py @@ -65,6 +65,40 @@ def _backup_status_file() -> str: return os.path.join(os.path.dirname(cleanup_logfile), "dbbackup.status.json") +def _local_timezone_metadata() -> dict: + now = datetime.datetime.now().astimezone() + offset = now.strftime("%z") + offset = f"{offset[:3]}:{offset[3:]}" if offset else "" + return { + "scheduled_timezone": "local", + "scheduled_timezone_name": now.tzname(), + "scheduled_utc_offset": offset, + } + + +def _local_iso_timestamp(value: str | None) -> str | None: + if not value: + return None + + try: + dt = datetime.datetime.fromisoformat(value.replace("Z", "+00:00")) + except ValueError: + return None + + if dt.tzinfo is None: + dt = dt.replace(tzinfo=datetime.UTC) + + return dt.astimezone().isoformat() + + +def _local_run_timestamps(last_run: dict, keys: tuple[str, ...]) -> dict: + return { + key: local_value + for key in keys + if (local_value := _local_iso_timestamp(last_run.get(key))) is not None + } + + def _get_cleanup_health() -> dict: cleanup_health = { "enabled": _config_bool("cleanup", "enabled", False), @@ -72,6 +106,7 @@ def _get_cleanup_health() -> dict: "scheduled_time": ( f"{_config_int('cleanup', 'hour', 2):02d}:{_config_int('cleanup', 'minute', 0):02d}" ), + **_local_timezone_metadata(), "vacuum": _config_bool("cleanup", "vacuum", False), "status": "disabled", } @@ -94,6 +129,9 @@ def _get_cleanup_health() -> dict: cleanup_health["status"] = last_run.get("status", cleanup_health["status"]) cleanup_health["last_run"] = last_run + cleanup_health["last_run_local"] = _local_run_timestamps( + last_run, ("started_at", "completed_at", "cutoff_at") + ) return cleanup_health @@ -104,6 +142,7 @@ def _get_backup_health() -> dict: "enabled": _config_bool("cleanup", "backup_enabled", False), "backup_dir": _config_str("cleanup", "backup_dir", "./backups"), "scheduled_time": f"{backup_hour:02d}:{backup_minute:02d}", + **_local_timezone_metadata(), "status": "disabled", } @@ -125,6 +164,9 @@ def _get_backup_health() -> dict: backup_health["status"] = last_run.get("status", backup_health["status"]) backup_health["last_run"] = last_run + backup_health["last_run_local"] = _local_run_timestamps( + last_run, ("started_at", "completed_at") + ) return backup_health From c16e19336a2e021c0fa9152c7dbf832bd9548217 Mon Sep 17 00:00:00 2001 From: pablorevilla-meshtastic Date: Mon, 8 Jun 2026 12:02:20 -0700 Subject: [PATCH 06/13] more fixes --- docs/API_Documentation.md | 9 --------- meshview/web_api/api.py | 29 ----------------------------- 2 files changed, 38 deletions(-) diff --git a/docs/API_Documentation.md b/docs/API_Documentation.md index 522b2b8..ee3843c 100644 --- a/docs/API_Documentation.md +++ b/docs/API_Documentation.md @@ -393,11 +393,6 @@ Response Example "node": 3 }, "error": null - }, - "last_run_local": { - "started_at": "2026-06-04T02:00:00-07:00", - "completed_at": "2026-06-04T02:00:03-07:00", - "cutoff_at": "2026-05-21T02:00:00-07:00" } }, "backup": { @@ -420,10 +415,6 @@ Response Example "compressed_size_bytes": 4211560, "compression_percent": 67.5, "error": null - }, - "last_run_local": { - "started_at": "2026-06-04T02:00:00-07:00", - "completed_at": "2026-06-04T02:00:02-07:00" } }, "database": "connected", diff --git a/meshview/web_api/api.py b/meshview/web_api/api.py index 71c83bc..026acc1 100644 --- a/meshview/web_api/api.py +++ b/meshview/web_api/api.py @@ -76,29 +76,6 @@ def _local_timezone_metadata() -> dict: } -def _local_iso_timestamp(value: str | None) -> str | None: - if not value: - return None - - try: - dt = datetime.datetime.fromisoformat(value.replace("Z", "+00:00")) - except ValueError: - return None - - if dt.tzinfo is None: - dt = dt.replace(tzinfo=datetime.UTC) - - return dt.astimezone().isoformat() - - -def _local_run_timestamps(last_run: dict, keys: tuple[str, ...]) -> dict: - return { - key: local_value - for key in keys - if (local_value := _local_iso_timestamp(last_run.get(key))) is not None - } - - def _get_cleanup_health() -> dict: cleanup_health = { "enabled": _config_bool("cleanup", "enabled", False), @@ -129,9 +106,6 @@ def _get_cleanup_health() -> dict: cleanup_health["status"] = last_run.get("status", cleanup_health["status"]) cleanup_health["last_run"] = last_run - cleanup_health["last_run_local"] = _local_run_timestamps( - last_run, ("started_at", "completed_at", "cutoff_at") - ) return cleanup_health @@ -164,9 +138,6 @@ def _get_backup_health() -> dict: backup_health["status"] = last_run.get("status", backup_health["status"]) backup_health["last_run"] = last_run - backup_health["last_run_local"] = _local_run_timestamps( - last_run, ("started_at", "completed_at") - ) return backup_health From 2175fcb44dee189c5e763a188c8c36db131ed2ee Mon Sep 17 00:00:00 2001 From: pablorevilla-meshtastic Date: Thu, 11 Jun 2026 14:49:26 -0700 Subject: [PATCH 07/13] remove some of the changes --- docs/API_Documentation.md | 6 ------ meshview/web_api/api.py | 13 ------------- 2 files changed, 19 deletions(-) diff --git a/docs/API_Documentation.md b/docs/API_Documentation.md index ee3843c..564c135 100644 --- a/docs/API_Documentation.md +++ b/docs/API_Documentation.md @@ -372,9 +372,6 @@ Response Example "enabled": true, "days_to_keep": 14, "scheduled_time": "02:00", - "scheduled_timezone": "local", - "scheduled_timezone_name": "PDT", - "scheduled_utc_offset": "-07:00", "vacuum": false, "status": "ok", "status_file": "dbcleanup.status.json", @@ -399,9 +396,6 @@ Response Example "enabled": true, "backup_dir": "./backups", "scheduled_time": "02:00", - "scheduled_timezone": "local", - "scheduled_timezone_name": "PDT", - "scheduled_utc_offset": "-07:00", "status": "ok", "status_file": "dbbackup.status.json", "last_run": { diff --git a/meshview/web_api/api.py b/meshview/web_api/api.py index 026acc1..71e5c99 100644 --- a/meshview/web_api/api.py +++ b/meshview/web_api/api.py @@ -65,17 +65,6 @@ def _backup_status_file() -> str: return os.path.join(os.path.dirname(cleanup_logfile), "dbbackup.status.json") -def _local_timezone_metadata() -> dict: - now = datetime.datetime.now().astimezone() - offset = now.strftime("%z") - offset = f"{offset[:3]}:{offset[3:]}" if offset else "" - return { - "scheduled_timezone": "local", - "scheduled_timezone_name": now.tzname(), - "scheduled_utc_offset": offset, - } - - def _get_cleanup_health() -> dict: cleanup_health = { "enabled": _config_bool("cleanup", "enabled", False), @@ -83,7 +72,6 @@ def _get_cleanup_health() -> dict: "scheduled_time": ( f"{_config_int('cleanup', 'hour', 2):02d}:{_config_int('cleanup', 'minute', 0):02d}" ), - **_local_timezone_metadata(), "vacuum": _config_bool("cleanup", "vacuum", False), "status": "disabled", } @@ -116,7 +104,6 @@ def _get_backup_health() -> dict: "enabled": _config_bool("cleanup", "backup_enabled", False), "backup_dir": _config_str("cleanup", "backup_dir", "./backups"), "scheduled_time": f"{backup_hour:02d}:{backup_minute:02d}", - **_local_timezone_metadata(), "status": "disabled", } From 5386f396112217c0c890a9e0d6b44605f2d8a8c1 Mon Sep 17 00:00:00 2001 From: pablorevilla-meshtastic Date: Thu, 11 Jun 2026 16:16:08 -0700 Subject: [PATCH 08/13] report change --- startdb.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/startdb.py b/startdb.py index 6c08c38..ac334a9 100644 --- a/startdb.py +++ b/startdb.py @@ -60,7 +60,7 @@ def write_cleanup_status(status: dict) -> None: tmp_path = status_path.with_suffix(f"{status_path.suffix}.tmp") try: with open(tmp_path, "w", encoding="utf-8") as f: - json.dump(status, f, indent=2, sort_keys=True) + json.dump(status, f, indent=2) tmp_path.replace(status_path) except Exception as e: cleanup_logger.warning(f"Failed to write cleanup status file: {e}") @@ -71,7 +71,7 @@ def write_backup_status(status: dict) -> None: tmp_path = status_path.with_suffix(f"{status_path.suffix}.tmp") try: with open(tmp_path, "w", encoding="utf-8") as f: - json.dump(status, f, indent=2, sort_keys=True) + json.dump(status, f, indent=2) tmp_path.replace(status_path) except Exception as e: cleanup_logger.warning(f"Failed to write backup status file: {e}") From e2adb73feaa871eee6a8bcc71dcc4e1355b7becf Mon Sep 17 00:00:00 2001 From: pablorevilla-meshtastic Date: Thu, 25 Jun 2026 17:19:35 -0700 Subject: [PATCH 09/13] map changes and other fixes --- meshview/mqtt_store.py | 30 +++++-- meshview/templates/firehose.html | 15 ++-- meshview/templates/map.html | 148 ++++++++++++++++++++++++++++--- meshview/templates/node.html | 15 +++- meshview/web.py | 6 ++ meshview/web_api/api.py | 4 + 6 files changed, 190 insertions(+), 28 deletions(-) diff --git a/meshview/mqtt_store.py b/meshview/mqtt_store.py index 6bd9bab..68acd8e 100644 --- a/meshview/mqtt_store.py +++ b/meshview/mqtt_store.py @@ -19,6 +19,7 @@ logger = logging.getLogger(__name__) MQTT_GATEWAY_CACHE: set[int] = set() UNKNOWN_NODE_NAME = "unknown name" NODE_NAME_CONTROL_CHARS = re.compile(r"[\x00-\x1f\x7f]") +MQTT_DIRECT_NODE_ID = 1 def normalize_node_name(name: str) -> str: @@ -28,6 +29,24 @@ def normalize_node_name(name: str) -> str: return name +def storage_packet_id(packet) -> int | None: + if packet.id: + return packet.id + + if packet.decoded.portnum != PortNum.MAP_REPORT_APP: + return None + + from_node_id = getattr(packet, "from", 0) or 0 + now_us = int(time.time() * 1_000_000) + return -(now_us * 2048 + (from_node_id & 0x7FF)) + + +def storage_to_node_id(packet) -> int: + if packet.decoded.portnum == PortNum.MAP_REPORT_APP: + return MQTT_DIRECT_NODE_ID + return packet.to + + async def capture_daily_snapshot() -> None: today = datetime.now(UTC).date() @@ -147,20 +166,21 @@ async def process_envelope(topic, env): await session.commit() - if not env.packet.id: + packet_id = storage_packet_id(env.packet) + if packet_id is None: return async with mqtt_database.async_session() as session: # --- Packet insert with ON CONFLICT DO NOTHING - result = await session.execute(select(Packet).where(Packet.id == env.packet.id)) + result = await session.execute(select(Packet).where(Packet.id == packet_id)) packet = result.scalar_one_or_none() if not packet: now_us = int(time.time() * 1_000_000) packet_values = { - "id": env.packet.id, + "id": packet_id, "portnum": env.packet.decoded.portnum, "from_node_id": getattr(env.packet, "from"), - "to_node_id": env.packet.to, + "to_node_id": storage_to_node_id(env.packet), "payload": env.packet.SerializeToString(), "import_time_us": now_us, "channel": env.channel_id, @@ -208,7 +228,7 @@ async def process_envelope(topic, env): now_us = int(time.time() * 1_000_000) seen_values = { - "packet_id": env.packet.id, + "packet_id": packet_id, "node_id": int(env.gateway_id[1:], 16), "channel": env.channel_id, "rx_time": env.packet.rx_time, diff --git a/meshview/templates/firehose.html b/meshview/templates/firehose.html index bc5266a..cc42be0 100644 --- a/meshview/templates/firehose.html +++ b/meshview/templates/firehose.html @@ -241,7 +241,7 @@ function logPacketTimes(packet) { const times = formatTimes(packet.import_time_us); console.log( "[firehose] packet time", - "id=" + packet.id, + "id=" + (packet.packet_id ?? packet.id), "epoch_us=" + times.epoch, "local=" + times.local, "utc=" + times.utc @@ -372,6 +372,14 @@ async function fetchUpdates() { const safePayload = (pkt.payload || "") .replace(//g, ">"); + const packetIdCell = pkt.packet_id + ? ` + ${pkt.packet_id} + ` + : pkt.portnum === 73 + ? `Not at Packet` + : `—`; const html = ` @@ -382,10 +390,7 @@ async function fetchUpdates() { ▶ - - ${pkt.id} - + ${packetIdCell} ${from} diff --git a/meshview/templates/map.html b/meshview/templates/map.html index 7602aa6..babe229 100644 --- a/meshview/templates/map.html +++ b/meshview/templates/map.html @@ -148,6 +148,7 @@ +