From cced48c10e89907bd98bb9f3c24b1e14370cb005 Mon Sep 17 00:00:00 2001 From: pe1hvh Date: Mon, 27 Apr 2026 09:13:47 +0200 Subject: [PATCH] patch: JSONL stream output for RX log alongside existing JSON archivef(#v1.22.1) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit v1.22.1: JSONL stream output for RX log alongside existing JSON archive Every received LoRa packet is now also written immediately as a single JSON line to ~/.meshcore-gui/archive/_rxlog.jsonl. This is an append-only, unbuffered stream format that lets separate local services (such as meshcore-watchlist) consume the RX feed in real time without depending on the GUI's internal batched-JSON format. - The existing _rxlog.json is unchanged (60 s flush interval, atomic rewrite). The GUI, the public REST API and the domca.nl ingest continue to work without modification. - Writes to the JSONL file are direct (no buffer), so end-to-end latency from radio reception to JSONL line is sub-second. - A failure on the JSONL path is logged via debug_print and does not affect the buffered JSON archive — the two paths are independent. - _cleanup_rxlog() now also rewrites the JSONL file to drop entries older than RXLOG_RETENTION_DAYS. Corrupt lines (e.g. a partial last line after a crash) are skipped during cleanup. No BLE/worker changes, no public REST API changes; SharedData and the BLE command pipeline are untouched. Disk usage increases modestly (one additional file per device, same retention window). PATCH bump 1.22.0 → 1.22.1: purely additive, fully backwards-compatible. --- CHANGELOG.md | 43 ++++++++++++++++++++ meshcore_gui/config.py | 2 +- meshcore_gui/services/message_archive.py | 52 +++++++++++++++++++++++- 3 files changed, 95 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7a3e468..f9e4d11 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,49 @@ Format follows [Keep a Changelog](https://keepachangelog.com/) and [Semantic Ver --- +## [1.22.1] - 2026-04-27 + +### Added +- **JSONL stream output for the RX log** (`services/message_archive.py`): + Every received LoRa packet is now also written immediately to an + append-only JSON Lines file at + `~/.meshcore-gui/archive/_rxlog.jsonl`, one JSON object per + line. This provides a real-time data source for separate local + services (such as `meshcore-watchlist`) that want to consume the raw + RX feed without depending on the GUI's internal batched JSON file + format. + + - The new file lives alongside the existing `_rxlog.json`. + The original batched archive (60 s flush interval, atomic rewrite) + is unchanged so the GUI, the public REST API and `domca.nl` keep + working without modification. + - Writes are direct (no buffer) so end-to-end latency from radio + reception to JSONL line is sub-second, suitable for live + monitoring use cases. + - A failure on the JSONL append path is logged via `debug_print` and + does not affect the buffered JSON archive — the two paths are + independent. + +### Changed +- `_cleanup_rxlog()` now also rewrites the JSONL stream file to drop + entries older than `RXLOG_RETENTION_DAYS`. Same retention policy as + the existing JSON archive; corrupt lines (e.g. a partial last line + after a crash) are skipped during cleanup. +- `VERSION` bumped `1.22.0` → `1.22.1` (PATCH: additive feature, fully + backwards-compatible). + +### Impact +- No BLE/worker changes; SharedData and the BLE command pipeline are + untouched. +- No public REST API changes; `domca.nl` ingest is unaffected. +- Existing consumers of `_rxlog.json` see no difference. +- Disk usage increases modestly (one additional file per device, + same retention window). On a Raspberry Pi 5 with SSD the extra + per-packet I/O is negligible. + +--- + + ## [1.22.0] - 2026-04-21 ### Added diff --git a/meshcore_gui/config.py b/meshcore_gui/config.py index fe5c944..733e70f 100644 --- a/meshcore_gui/config.py +++ b/meshcore_gui/config.py @@ -25,7 +25,7 @@ from typing import Any, Dict, List # ============================================================================== -VERSION: str = "1.22.0" +VERSION: str = "1.22.1" # ============================================================================== diff --git a/meshcore_gui/services/message_archive.py b/meshcore_gui/services/message_archive.py index 36ba2f0..061f117 100644 --- a/meshcore_gui/services/message_archive.py +++ b/meshcore_gui/services/message_archive.py @@ -60,6 +60,9 @@ class MessageArchive: self._messages_path = ARCHIVE_DIR / f"{safe_name}_messages.json" self._rxlog_path = ARCHIVE_DIR / f"{safe_name}_rxlog.json" + # Append-only JSONL stream for external consumers (e.g. meshcore-watchlist). + # One JSON object per line, written immediately on every RX entry. + self._rxlog_jsonl_path = ARCHIVE_DIR / f"{safe_name}_rxlog.jsonl" # In-memory batch buffers (flushed periodically) self._message_buffer: List[Dict] = [] @@ -176,7 +179,17 @@ class MessageArchive: } self._rxlog_buffer.append(entry_dict) - + + # Append-only JSONL stream write (real-time consumer source). + # Direct write — no batch — for sub-second stream latency. + # Failure here must not affect the buffered JSON archive path. + try: + ARCHIVE_DIR.mkdir(parents=True, exist_ok=True) + with self._rxlog_jsonl_path.open("a", encoding="utf-8") as f: + f.write(json.dumps(entry_dict, ensure_ascii=False) + "\n") + except OSError as exc: + debug_print(f"Archive: JSONL append error: {exc}") + # Flush if batch size reached if len(self._rxlog_buffer) >= self._batch_size: self._flush_rxlog() @@ -410,6 +423,43 @@ class MessageArchive: except (json.JSONDecodeError, OSError) as exc: debug_print(f"Archive: error cleaning up rxlog: {exc}") + # Cleanup JSONL stream file as well (same retention policy). + # Read all lines, filter on timestamp_utc, rewrite atomically. + if not self._rxlog_jsonl_path.exists(): + return + try: + cutoff = datetime.now(timezone.utc) - timedelta(days=RXLOG_RETENTION_DAYS) + kept_lines: List[str] = [] + original_lines = 0 + with self._rxlog_jsonl_path.open("r", encoding="utf-8") as f: + for line in f: + line = line.rstrip("\n") + if not line: + continue + original_lines += 1 + try: + rec = json.loads(line) + except json.JSONDecodeError: + # Skip corrupt line, do not retain. + continue + if self._is_newer_than(rec.get("timestamp_utc"), cutoff): + kept_lines.append(line) + + if len(kept_lines) < original_lines: + tmp_path = self._rxlog_jsonl_path.with_suffix(".jsonl.tmp") + tmp_path.write_text( + "\n".join(kept_lines) + ("\n" if kept_lines else ""), + encoding="utf-8", + ) + tmp_path.replace(self._rxlog_jsonl_path) + debug_print( + f"Archive: JSONL cleanup removed " + f"{original_lines - len(kept_lines)} old entries " + f"(retained: {len(kept_lines)})" + ) + except OSError as exc: + debug_print(f"Archive: error cleaning up rxlog JSONL: {exc}") + # ------------------------------------------------------------------ # Utilities # ------------------------------------------------------------------