From 0110e65b97dd42bf5ff9bee3899a6632ae62cc6a Mon Sep 17 00:00:00 2001 From: MarekWo Date: Fri, 20 Mar 2026 20:34:29 +0100 Subject: [PATCH] feat: add System Log viewer with real-time streaming In-memory ring buffer (2000 entries) captures all Python log records. New /logs page streams entries via WebSocket in real-time with: - Level filter (DEBUG/INFO/WARNING/ERROR) - Module filter (auto-populated from seen loggers) - Text search with highlighting - Auto-scroll with pause/resume - Dark theme matching Console style Menu entry added under Configuration section. Co-Authored-By: Claude Opus 4.6 --- app/log_handler.py | 84 ++++++++++++ app/main.py | 23 ++++ app/routes/api.py | 47 +++++++ app/routes/views.py | 6 + app/static/js/logs.js | 276 ++++++++++++++++++++++++++++++++++++++++ app/templates/base.html | 7 + app/templates/logs.html | 206 ++++++++++++++++++++++++++++++ 7 files changed, 649 insertions(+) create mode 100644 app/log_handler.py create mode 100644 app/static/js/logs.js create mode 100644 app/templates/logs.html diff --git a/app/log_handler.py b/app/log_handler.py new file mode 100644 index 0000000..66d9718 --- /dev/null +++ b/app/log_handler.py @@ -0,0 +1,84 @@ +""" +In-memory ring buffer log handler with WebSocket broadcast. + +Captures Python log records into a fixed-size deque and optionally +broadcasts them to connected SocketIO clients in real-time. +""" + +import logging +from collections import deque +from datetime import datetime +from threading import Lock + + +class MemoryLogHandler(logging.Handler): + """Logging handler that stores records in a ring buffer and broadcasts via SocketIO.""" + + def __init__(self, capacity=2000, socketio=None): + super().__init__() + self.capacity = capacity + self.buffer = deque(maxlen=capacity) + self.socketio = socketio + self._lock = Lock() + self._seq = 0 # monotonic sequence for client catch-up + + def emit(self, record): + try: + entry = self._format_record(record) + with self._lock: + self._seq += 1 + entry['seq'] = self._seq + self.buffer.append(entry) + + # Broadcast to connected clients + if self.socketio: + self.socketio.emit('log_entry', entry, namespace='/logs') + except Exception: + self.handleError(record) + + def _format_record(self, record): + """Convert LogRecord to a serializable dict.""" + return { + 'timestamp': datetime.fromtimestamp(record.created).strftime('%Y-%m-%d %H:%M:%S.%f')[:-3], + 'level': record.levelname, + 'logger': record.name, + 'message': record.getMessage(), + } + + def get_entries(self, level=None, logger_filter=None, search=None, limit=None): + """Return filtered log entries from the buffer. + + Args: + level: Minimum log level name (DEBUG, INFO, WARNING, ERROR, CRITICAL) + logger_filter: Logger name prefix filter (e.g. 'app.device_manager') + search: Text search in message (case-insensitive) + limit: Max entries to return (newest first before limit, returned in chronological order) + + Returns: + List of log entry dicts + """ + level_num = getattr(logging, level.upper(), 0) if level else 0 + search_lower = search.lower() if search else None + + with self._lock: + entries = list(self.buffer) + + # Apply filters + if level_num > 0: + entries = [e for e in entries if getattr(logging, e['level'], 0) >= level_num] + if logger_filter: + entries = [e for e in entries if e['logger'].startswith(logger_filter)] + if search_lower: + entries = [e for e in entries if search_lower in e['message'].lower()] + + # Limit (return newest N, in chronological order) + if limit and limit > 0 and len(entries) > limit: + entries = entries[-limit:] + + return entries + + def get_loggers(self): + """Return sorted list of unique logger names seen in the buffer.""" + with self._lock: + loggers = sorted({e['logger'] for e in self.buffer}) + return loggers diff --git a/app/main.py b/app/main.py index 9990528..9385cc7 100644 --- a/app/main.py +++ b/app/main.py @@ -13,6 +13,7 @@ from flask_socketio import SocketIO, emit from app.config import config, runtime_config from app.database import Database from app.device_manager import DeviceManager +from app.log_handler import MemoryLogHandler from app.routes.views import views_bp from app.routes.api import api_bp from app.version import VERSION_STRING, GIT_BRANCH @@ -70,6 +71,13 @@ def create_app(): # Initialize SocketIO socketio.init_app(app, cors_allowed_origins="*", async_mode='threading') + # Initialize in-memory log handler (ring buffer + WebSocket broadcast) + log_handler = MemoryLogHandler(capacity=2000, socketio=socketio) + log_handler.setLevel(logging.DEBUG) + log_handler.setFormatter(logging.Formatter('%(message)s')) + logging.getLogger().addHandler(log_handler) + app.log_handler = log_handler + # v2: Initialize database db = Database(config.db_path) app.db = db @@ -212,6 +220,21 @@ def handle_send_command(data): socketio.start_background_task(execute_and_respond) +# ============================================================ +# WebSocket handlers for System Log viewer +# ============================================================ + +@socketio.on('connect', namespace='/logs') +def handle_logs_connect(): + """Handle log viewer WebSocket connection.""" + logger.debug("Log viewer WebSocket client connected") + + +@socketio.on('disconnect', namespace='/logs') +def handle_logs_disconnect(): + logger.debug("Log viewer WebSocket client disconnected") + + def _parse_time_arg(value: str) -> int: """Parse time argument with optional suffix: s (seconds), m (minutes, default), h (hours).""" value = value.strip().lower() diff --git a/app/routes/api.py b/app/routes/api.py index 9f729a3..8c552af 100644 --- a/app/routes/api.py +++ b/app/routes/api.py @@ -3871,3 +3871,50 @@ def download_backup(): as_attachment=True, download_name=filename ) + + +# ============================================================================= +# System Log API +# ============================================================================= + +@api_bp.route('/logs', methods=['GET']) +def get_logs_api(): + """ + Get log entries from in-memory ring buffer. + + Query parameters: + level: Minimum log level (DEBUG, INFO, WARNING, ERROR) + logger: Logger name prefix filter + search: Text search in message (case-insensitive) + limit: Max entries to return (default 500) + + Returns: + JSON with log entries and available loggers + """ + try: + log_handler = getattr(current_app, 'log_handler', None) + if not log_handler: + return jsonify({'success': False, 'error': 'Log handler not available'}), 500 + + level = request.args.get('level') + logger_filter = request.args.get('logger') + search = request.args.get('search') + limit = request.args.get('limit', 500, type=int) + + entries = log_handler.get_entries( + level=level, + logger_filter=logger_filter, + search=search, + limit=limit + ) + + return jsonify({ + 'success': True, + 'entries': entries, + 'count': len(entries), + 'loggers': log_handler.get_loggers(), + }), 200 + + except Exception as e: + logger.error(f"Error getting logs: {e}") + return jsonify({'success': False, 'error': str(e)}), 500 diff --git a/app/routes/views.py b/app/routes/views.py index 22a984a..e80716c 100644 --- a/app/routes/views.py +++ b/app/routes/views.py @@ -85,6 +85,12 @@ def console(): ) +@views_bp.route('/logs') +def logs(): + """System log viewer - real-time log streaming with filters.""" + return render_template('logs.html') + + @views_bp.route('/health') def health(): """ diff --git a/app/static/js/logs.js b/app/static/js/logs.js new file mode 100644 index 0000000..e9d1c28 --- /dev/null +++ b/app/static/js/logs.js @@ -0,0 +1,276 @@ +/** + * System Log Viewer + * + * Real-time log streaming via WebSocket with filtering and search. + */ +(function () { + 'use strict'; + + // --- DOM refs --- + const logEntries = document.getElementById('logEntries'); + const loadingMsg = document.getElementById('loadingMsg'); + const logCount = document.getElementById('logCount'); + const statusDot = document.getElementById('statusDot'); + const pauseBtn = document.getElementById('pauseBtn'); + const pauseIcon = document.getElementById('pauseIcon'); + const clearBtn = document.getElementById('clearBtn'); + const levelFilter = document.getElementById('levelFilter'); + const loggerFilter = document.getElementById('loggerFilter'); + const searchFilter = document.getElementById('searchFilter'); + const resetFilters = document.getElementById('resetFilters'); + + // --- State --- + let paused = false; + let autoScroll = true; + let entries = []; // all received entries + let displayCount = 0; + const MAX_DISPLAY = 3000; // max DOM entries before trimming + let searchDebounce = null; + let knownLoggers = new Set(); + + // --- Level ordering --- + const LEVEL_ORDER = { DEBUG: 0, INFO: 1, WARNING: 2, ERROR: 3, CRITICAL: 4 }; + + // --- WebSocket --- + const socket = io('/logs', { + transports: ['websocket', 'polling'], + reconnection: true, + reconnectionDelay: 2000, + }); + + socket.on('connect', () => { + setStatus('live'); + // Load initial entries + loadInitialLogs(); + }); + + socket.on('disconnect', () => { + setStatus('disconnected'); + }); + + socket.on('log_entry', (entry) => { + addEntry(entry); + }); + + // --- Functions --- + + function setStatus(state) { + statusDot.className = 'status-indicator ' + state; + } + + function loadInitialLogs() { + const level = levelFilter.value; + const params = new URLSearchParams(); + if (level) params.set('level', level); + params.set('limit', '1000'); + + fetch('/api/logs?' + params.toString()) + .then(r => r.json()) + .then(data => { + if (!data.success) return; + loadingMsg?.remove(); + + // Update logger filter options + if (data.loggers) { + data.loggers.forEach(l => knownLoggers.add(l)); + updateLoggerOptions(); + } + + // Render entries + entries = data.entries || []; + renderAll(); + }) + .catch(err => { + if (loadingMsg) loadingMsg.textContent = 'Failed to load logs'; + console.error('Failed to load logs:', err); + }); + } + + function addEntry(entry) { + entries.push(entry); + + // Track new loggers + if (!knownLoggers.has(entry.logger)) { + knownLoggers.add(entry.logger); + updateLoggerOptions(); + } + + // If paused or filtered out, don't add to DOM + if (paused) { + updateCount(); + return; + } + + if (matchesFilter(entry)) { + appendEntryDOM(entry); + trimDOM(); + if (autoScroll) scrollToBottom(); + } + updateCount(); + } + + function matchesFilter(entry) { + // Level filter + const minLevel = levelFilter.value; + if (minLevel && (LEVEL_ORDER[entry.level] || 0) < (LEVEL_ORDER[minLevel] || 0)) { + return false; + } + + // Logger filter + const loggerVal = loggerFilter.value; + if (loggerVal && !entry.logger.startsWith(loggerVal)) { + return false; + } + + // Search filter + const search = searchFilter.value.trim().toLowerCase(); + if (search && !entry.message.toLowerCase().includes(search) && + !entry.logger.toLowerCase().includes(search)) { + return false; + } + + return true; + } + + function renderAll() { + logEntries.innerHTML = ''; + displayCount = 0; + + const filtered = entries.filter(e => matchesFilter(e)); + // Only render last MAX_DISPLAY entries + const toRender = filtered.slice(-MAX_DISPLAY); + const fragment = document.createDocumentFragment(); + + for (const entry of toRender) { + fragment.appendChild(createEntryElement(entry)); + displayCount++; + } + + logEntries.appendChild(fragment); + updateCount(); + scrollToBottom(); + } + + function appendEntryDOM(entry) { + logEntries.appendChild(createEntryElement(entry)); + displayCount++; + } + + function createEntryElement(entry) { + const div = document.createElement('div'); + div.className = 'log-line'; + + // Shorten timestamp to HH:MM:SS.mmm + const ts = entry.timestamp.length > 11 ? entry.timestamp.substring(11) : entry.timestamp; + + // Shorten logger name (remove 'app.' prefix) + let loggerName = entry.logger; + if (loggerName.startsWith('app.')) { + loggerName = loggerName.substring(4); + } + + // Pad level to 5 chars + const levelPad = entry.level.padEnd(5); + + // Build the line with color spans + const search = searchFilter.value.trim().toLowerCase(); + let message = escapeHtml(entry.message); + if (search) { + message = highlightSearch(message, search); + } + + div.innerHTML = + `${escapeHtml(ts)} ` + + `${escapeHtml(levelPad)} ` + + `${escapeHtml(loggerName.padEnd(18).substring(0, 18))} ` + + `${message}`; + + return div; + } + + function trimDOM() { + while (logEntries.children.length > MAX_DISPLAY) { + logEntries.removeChild(logEntries.firstChild); + displayCount--; + } + } + + function scrollToBottom() { + logEntries.scrollTop = logEntries.scrollHeight; + } + + function updateCount() { + const total = entries.length; + const shown = logEntries.children.length; + logCount.textContent = shown === total + ? `${total} entries` + : `${shown} / ${total} entries`; + } + + function updateLoggerOptions() { + const current = loggerFilter.value; + // Group loggers by top-level module + const sorted = Array.from(knownLoggers).sort(); + + loggerFilter.innerHTML = ''; + for (const name of sorted) { + const opt = document.createElement('option'); + opt.value = name; + // Shorten display + opt.textContent = name.startsWith('app.') ? name.substring(4) : name; + if (name === current) opt.selected = true; + loggerFilter.appendChild(opt); + } + } + + function escapeHtml(str) { + const div = document.createElement('div'); + div.textContent = str; + return div.innerHTML; + } + + function highlightSearch(html, search) { + if (!search) return html; + // Case-insensitive highlight (on already-escaped HTML) + const regex = new RegExp('(' + search.replace(/[.*+?^${}()|[\]\\]/g, '\\$&') + ')', 'gi'); + return html.replace(regex, '$1'); + } + + // --- Auto-scroll detection --- + logEntries.addEventListener('scroll', () => { + const atBottom = logEntries.scrollHeight - logEntries.scrollTop - logEntries.clientHeight < 50; + autoScroll = atBottom; + }); + + // --- Controls --- + pauseBtn.addEventListener('click', () => { + paused = !paused; + pauseIcon.className = paused ? 'bi bi-play-fill' : 'bi bi-pause-fill'; + setStatus(paused ? 'paused' : 'live'); + if (!paused) { + // Resume: re-render to catch up + renderAll(); + } + }); + + clearBtn.addEventListener('click', () => { + entries = []; + logEntries.innerHTML = ''; + displayCount = 0; + updateCount(); + }); + + // Filter handlers + levelFilter.addEventListener('change', () => renderAll()); + loggerFilter.addEventListener('change', () => renderAll()); + searchFilter.addEventListener('input', () => { + clearTimeout(searchDebounce); + searchDebounce = setTimeout(() => renderAll(), 250); + }); + resetFilters.addEventListener('click', () => { + levelFilter.value = 'INFO'; + loggerFilter.value = ''; + searchFilter.value = ''; + renderAll(); + }); +})(); diff --git a/app/templates/base.html b/app/templates/base.html index 49a88d4..f3c9d03 100644 --- a/app/templates/base.html +++ b/app/templates/base.html @@ -133,6 +133,13 @@ Direct meshcli commands + + +
+ System Log + Real-time application logs +
+ + + + + + +
+
+ + + + +
+
+ + +
+
Loading logs...
+
+ + + + + + + + + +