From 0e524ddbc19ecd1040aa84a2ebfa1baeb765109a Mon Sep 17 00:00:00 2001 From: MarekWo Date: Fri, 31 Jul 2026 07:44:48 +0200 Subject: [PATCH] perf: resolve message metadata in one batched request MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Every incoming echo triggers a sweep of the whole rendered message list, and refreshMessagesMeta() awaited one /api/messages//meta per message inside the loop. Messages that never gain a route (nothing heard them) never stop qualifying for the sweep, so the same ~180 messages were re-fetched every few seconds: 7,500 requests in nine minutes on a single tab, each opening its own SQLite connections. The single-threaded werkzeug server — the same one production runs — had no room left for anything else, so the UI hung on "Loading messages..." / "Connecting..." while the device was in fact connected. Add GET /api/messages/meta?ids=... resolving the whole sweep with a handful of queries, batching the row and echo lookups, and have the client collect ids first and fetch them in chunks. The per-message endpoint stays for forced single refreshes; both now share _build_message_meta() and the existing _build_channel_secrets / _get_row_pkt_payload helpers, so the payload is unchanged (verified byte-identical against the old response). Measured on the local container, 500 messages rendered / 181 needing meta: one 273 ms request in place of 181 sequential ones at ~82 ms each. Co-Authored-By: Claude Opus 5 --- app/database.py | 20 ++++++ app/routes/api.py | 147 +++++++++++++++++++++++++------------------ app/static/js/app.js | 35 ++++++++--- docs/whatsnew.md | 1 + 4 files changed, 133 insertions(+), 70 deletions(-) diff --git a/app/database.py b/app/database.py index a682dbc..f543b96 100644 --- a/app/database.py +++ b/app/database.py @@ -834,6 +834,26 @@ class Database: ).fetchone() return dict(row) if row else None + def get_channel_messages_by_ids(self, msg_ids: List[int]) -> Dict[int, Dict]: + """Batch-fetch channel messages by id with chunked IN queries. + + Returns {msg_id: row dict}; ids with no row are simply absent. + Chunked at 500 to stay under SQLite's host-parameter limit.""" + result: Dict[int, Dict] = {} + if not msg_ids: + return result + with self._connect() as conn: + for i in range(0, len(msg_ids), 500): + chunk = msg_ids[i:i + 500] + placeholders = ",".join("?" * len(chunk)) + rows = conn.execute( + f"SELECT * FROM channel_messages WHERE id IN ({placeholders})", + chunk + ).fetchall() + for r in rows: + result[r['id']] = dict(r) + return result + def get_channel_messages(self, channel_idx: int = None, limit: int = 50, offset: int = 0, days: int = None) -> List[Dict]: with self._connect() as conn: diff --git a/app/routes/api.py b/app/routes/api.py index fad6b6e..78bf2be 100644 --- a/app/routes/api.py +++ b/app/routes/api.py @@ -671,6 +671,88 @@ def get_path_analyzer_messages(): return jsonify({'success': False, 'error': str(e)}), 500 +def _build_message_meta(row: dict, pkt_payload, echoes: list) -> dict: + """Assemble the meta payload (SNR, hops, route, analyzer hash) for one row. + + Pure: takes the already-resolved pkt_payload and its echoes so callers can + batch the DB work.""" + path_len_raw = row.get('path_len') + hop_count = None + path_hash_size = 1 + if path_len_raw is not None: + hop_count, path_hash_size, _ = decode_path_len(path_len_raw) + + meta = { + 'success': True, + 'snr': row.get('snr'), + 'path_len': path_len_raw, + 'hop_count': hop_count, + 'path_hash_size': path_hash_size, + 'pkt_payload': pkt_payload, + } + + if pkt_payload: + meta['packet_hash'] = compute_packet_hash(pkt_payload) + if echoes: + meta['echo_count'] = len(echoes) + meta['echo_paths'] = [e.get('path', '') for e in echoes if e.get('path')] + meta['echo_snrs'] = [e.get('snr') for e in echoes if e.get('snr') is not None] + meta['echo_hash_sizes'] = [e.get('hash_size', 1) for e in echoes if e.get('path')] + + return meta + + +@api_bp.route('/messages/meta', methods=['GET']) +def get_messages_meta_batch(): + """Return metadata for many channel messages at once. + + Query: ?ids=1,2,3 -> {'success': True, 'metas': {'1': {...}, ...}} + + The UI sweeps every rendered message whenever echoes arrive; doing that one + request per message flooded the single-threaded werkzeug server (thousands + of round-trips per minute, each opening its own SQLite connections). This + resolves the whole sweep with a handful of queries. + """ + try: + db = _get_db() + if not db: + return jsonify({'success': False, 'error': 'No database'}), 500 + + raw_ids = (request.args.get('ids') or '').split(',') + msg_ids = [] + for part in raw_ids: + part = part.strip() + if part.isdigit(): + msg_ids.append(int(part)) + if not msg_ids: + return jsonify({'success': True, 'metas': {}}) + + rows = db.get_channel_messages_by_ids(msg_ids) + channel_secrets = _build_channel_secrets(db) + + payload_by_id = { + mid: _get_row_pkt_payload(row, channel_secrets) + for mid, row in rows.items() + } + echoes_by_payload = db.get_echoes_for_payloads( + list({p for p in payload_by_id.values() if p}) + ) + + metas = { + str(mid): _build_message_meta( + row, + payload_by_id[mid], + echoes_by_payload.get(payload_by_id[mid], []), + ) + for mid, row in rows.items() + } + return jsonify({'success': True, 'metas': metas}) + + except Exception as e: + logger.error(f"Error fetching batch message meta: {e}") + return jsonify({'success': False, 'error': str(e)}), 500 + + @api_bp.route('/messages//meta', methods=['GET']) def get_message_meta(msg_id): """Return metadata (SNR, hops, route, analyzer URL) for a single channel message.""" @@ -683,68 +765,9 @@ def get_message_meta(msg_id): if not row: return jsonify({'success': False, 'error': 'Not found'}), 404 - pkt_payload = row.get('pkt_payload') - sender_ts = row.get('sender_timestamp') - ch_idx = row.get('channel_idx', 0) - txt_type = row.get('txt_type', 0) - - # Compute pkt_payload if not stored - # Use DB channels (fast) to avoid blocking on device communication - if not pkt_payload and sender_ts: - db_channels = db.get_channels() if db else [] - channel_secrets = {} - for ch_info in db_channels: - ch_key = ch_info.get('secret', ch_info.get('key', '')) - ci = ch_info.get('idx', ch_info.get('index')) - if ch_key and ci is not None: - channel_secrets[ci] = ch_key - - if ch_idx in channel_secrets: - raw_text = None - raw_json_str = row.get('raw_json') - if raw_json_str: - try: - raw_text = json.loads(raw_json_str).get('text') - except (json.JSONDecodeError, TypeError): - pass - if not raw_text: - is_own = bool(row.get('is_own', 0)) - if is_own: - device_name = runtime_config.get_device_name() or '' - raw_text = f"{device_name}: {row.get('content', '')}" if device_name else row.get('content', '') - else: - sender = row.get('sender', '') - raw_text = f"{sender}: {row.get('content', '')}" if sender else row.get('content', '') - pkt_payload = compute_pkt_payload( - channel_secrets[ch_idx], sender_ts, txt_type, raw_text - ) - - # Decode path_len - path_len_raw = row.get('path_len') - hop_count = None - path_hash_size = 1 - if path_len_raw is not None: - hop_count, path_hash_size, _ = decode_path_len(path_len_raw) - - meta = { - 'success': True, - 'snr': row.get('snr'), - 'path_len': path_len_raw, - 'hop_count': hop_count, - 'path_hash_size': path_hash_size, - 'pkt_payload': pkt_payload, - } - - if pkt_payload: - meta['packet_hash'] = compute_packet_hash(pkt_payload) - echoes = db.get_echoes_for_message(pkt_payload) - if echoes: - meta['echo_count'] = len(echoes) - meta['echo_paths'] = [e.get('path', '') for e in echoes if e.get('path')] - meta['echo_snrs'] = [e.get('snr') for e in echoes if e.get('snr') is not None] - meta['echo_hash_sizes'] = [e.get('hash_size', 1) for e in echoes if e.get('path')] - - return jsonify(meta) + pkt_payload = _get_row_pkt_payload(row, _build_channel_secrets(db)) + echoes = db.get_echoes_for_message(pkt_payload) if pkt_payload else [] + return jsonify(_build_message_meta(row, pkt_payload, echoes)) except Exception as e: logger.error(f"Error fetching message meta: {e}") diff --git a/app/static/js/app.js b/app/static/js/app.js index 0b0ab36..3b93f1a 100644 --- a/app/static/js/app.js +++ b/app/static/js/app.js @@ -1218,9 +1218,17 @@ function appendMessageFromSocket(data) { markChannelAsRead(currentChannelIdx, msg.timestamp); } +// Cap on ids per /api/messages/meta request, to keep the query string short. +const META_BATCH_SIZE = 200; + /** * Refresh metadata (SNR, hops, route, analyzer) for messages missing it. - * Fetches /api/messages//meta for each incomplete message, updates DOM in-place. + * + * Every echo triggers a sweep of the whole rendered list, and messages that + * never gain a route (no repeaters heard them) never stop qualifying — so this + * must stay cheap. Ids are collected first and resolved with batched + * /api/messages/meta calls; one request per message used to flood the + * single-threaded server and hang the UI. */ async function refreshMessagesMeta(forceIds = []) { const container = document.getElementById('messagesList'); @@ -1228,7 +1236,8 @@ async function refreshMessagesMeta(forceIds = []) { const forced = new Set((forceIds || []).map(String)); - // Find message wrappers that don't have full metadata yet + // Collect message wrappers that don't have full metadata yet + const pending = new Map(); // msgId -> wrapper const wrappers = container.querySelectorAll('.message-wrapper[data-msg-id]'); for (const wrapper of wrappers) { const msgId = wrapper.dataset.msgId; @@ -1245,14 +1254,24 @@ async function refreshMessagesMeta(forceIds = []) { if (hasRoute && hasAnalyzer) continue; } - try { - const resp = await fetch(`/api/messages/${msgId}/meta`); - const meta = await resp.json(); - if (!meta.success) continue; + pending.set(msgId, wrapper); + } + if (pending.size === 0) return; - updateMessageMetaDOM(wrapper, meta); + const ids = Array.from(pending.keys()); + for (let i = 0; i < ids.length; i += META_BATCH_SIZE) { + const chunk = ids.slice(i, i + META_BATCH_SIZE); + try { + const resp = await fetch(`/api/messages/meta?ids=${chunk.join(',')}`); + const data = await resp.json(); + if (!data.success || !data.metas) continue; + + for (const [msgId, meta] of Object.entries(data.metas)) { + const wrapper = pending.get(msgId); + if (wrapper && meta.success) updateMessageMetaDOM(wrapper, meta); + } } catch (e) { - console.error(`Error fetching meta for msg #${msgId}:`, e); + console.error('Error fetching message meta batch:', e); } } } diff --git a/docs/whatsnew.md b/docs/whatsnew.md index 8058671..2be7783 100644 --- a/docs/whatsnew.md +++ b/docs/whatsnew.md @@ -14,6 +14,7 @@ For deep technical notes, see [architecture.md](architecture.md). For the full g - **Messages no longer go missing after the app has been in the background.** Coming back to a minimised app — or to a phone that had been asleep — could show a chat that quietly stopped at whatever message arrived last before the screen went off, with everything since then missing until the app was force-stopped and reopened. New messages reach an open page over a live connection, and Android tears that connection down while the app sits in the background; nothing then went back to ask the server what had been missed. Now every way back from a gap re-reads the list: the connection coming back, the app returning to the foreground, and a heartbeat that notices when the page has been frozen. The same applies to direct messages, and to a browser tab that lost its network for a while. - **A Refresh item in the menu.** The browser's pull-to-refresh has no equivalent in the Android app, so there is now a **Refresh** entry at the top of the menu that reloads the messages from the server on demand — in the app, and everywhere else too. +- **The app no longer grinds to a halt with more than one window open.** Keeping mc-webui open in two places at once — a second browser tab, or a phone and a desktop together — could leave the message list stuck on "Loading messages…" and the status on "Connecting…", looking for all the world like the mesh device had dropped off, when it was connected the whole time. Each open window was asking the server about every message on screen separately, hundreds of individual requests at a time, repeated every few seconds as new radio traffic came in; together they left the server no room to answer anything else. Those requests are now bundled into a single one, so a window that used to need hundreds needs one, and several windows at once are no longer a problem. ---