adam/mc-webui
1
0
Fork 0
mirror of https://github.com/MarekWo/mc-webui.git synced 2026-08-05 08:22:30 +02:00
Code Issues Packages Projects Releases Wiki Activity

perf: resolve message metadata in one batched request

Browse Source 
Every incoming echo triggers a sweep of the whole rendered message list, and
refreshMessagesMeta() awaited one /api/messages/<id>/meta per message inside
the loop. Messages that never gain a route (nothing heard them) never stop
qualifying for the sweep, so the same ~180 messages were re-fetched every few
seconds: 7,500 requests in nine minutes on a single tab, each opening its own
SQLite connections. The single-threaded werkzeug server — the same one
production runs — had no room left for anything else, so the UI hung on
"Loading messages..." / "Connecting..." while the device was in fact connected.

Add GET /api/messages/meta?ids=... resolving the whole sweep with a handful of
queries, batching the row and echo lookups, and have the client collect ids
first and fetch them in chunks. The per-message endpoint stays for forced
single refreshes; both now share _build_message_meta() and the existing
_build_channel_secrets / _get_row_pkt_payload helpers, so the payload is
unchanged (verified byte-identical against the old response).

Measured on the local container, 500 messages rendered / 181 needing meta:
one 273 ms request in place of 181 sequential ones at ~82 ms each.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
MarekWo
2026-07-31 07:44:48 +02:00
parent ebd2e95fe1
commit 0e524ddbc1
4 changed files with 133 additions and 70 deletions
Download Patch File Download Diff File Expand all files Collapse all files
app/database.py
+20
View File
@@ -834,6 +834,26 @@ class Database:
).fetchone()
return dict(row) if row else None
def get_channel_messages_by_ids(self, msg_ids: List[int]) -> Dict[int, Dict]:
"""Batch-fetch channel messages by id with chunked IN queries.
Returns {msg_id: row dict}; ids with no row are simply absent.
Chunked at 500 to stay under SQLite's host-parameter limit."""
result: Dict[int, Dict] = {}
if not msg_ids:
return result
with self._connect() as conn:
for i in range(0, len(msg_ids), 500):
chunk = msg_ids[i:i + 500]
placeholders = ",".join("?" * len(chunk))
rows = conn.execute(
f"SELECT * FROM channel_messages WHERE id IN ({placeholders})",
chunk
).fetchall()
for r in rows:
result[r['id']] = dict(r)
return result
def get_channel_messages(self, channel_idx: int = None, limit: int = 50,
offset: int = 0, days: int = None) -> List[Dict]:
with self._connect() as conn:
app/routes/api.py
+85 -62
View File
@@ -671,6 +671,88 @@ def get_path_analyzer_messages():
return jsonify({'success': False, 'error': str(e)}), 500
def _build_message_meta(row: dict, pkt_payload, echoes: list) -> dict:
"""Assemble the meta payload (SNR, hops, route, analyzer hash) for one row.
Pure: takes the already-resolved pkt_payload and its echoes so callers can
batch the DB work."""
path_len_raw = row.get('path_len')
hop_count = None
path_hash_size = 1
if path_len_raw is not None:
hop_count, path_hash_size, _ = decode_path_len(path_len_raw)
meta = {
'success': True,
'snr': row.get('snr'),
'path_len': path_len_raw,
'hop_count': hop_count,
'path_hash_size': path_hash_size,
'pkt_payload': pkt_payload,
}
if pkt_payload:
meta['packet_hash'] = compute_packet_hash(pkt_payload)
if echoes:
meta['echo_count'] = len(echoes)
meta['echo_paths'] = [e.get('path', '') for e in echoes if e.get('path')]
meta['echo_snrs'] = [e.get('snr') for e in echoes if e.get('snr') is not None]
meta['echo_hash_sizes'] = [e.get('hash_size', 1) for e in echoes if e.get('path')]
return meta
@api_bp.route('/messages/meta', methods=['GET'])
def get_messages_meta_batch():
"""Return metadata for many channel messages at once.
Query: ?ids=1,2,3 -> {'success': True, 'metas': {'1': {...}, ...}}
The UI sweeps every rendered message whenever echoes arrive; doing that one
request per message flooded the single-threaded werkzeug server (thousands
of round-trips per minute, each opening its own SQLite connections). This
resolves the whole sweep with a handful of queries.
"""
try:
db = _get_db()
if not db:
return jsonify({'success': False, 'error': 'No database'}), 500
raw_ids = (request.args.get('ids') or '').split(',')
msg_ids = []
for part in raw_ids:
part = part.strip()
if part.isdigit():
msg_ids.append(int(part))
if not msg_ids:
return jsonify({'success': True, 'metas': {}})
rows = db.get_channel_messages_by_ids(msg_ids)
channel_secrets = _build_channel_secrets(db)
payload_by_id = {
mid: _get_row_pkt_payload(row, channel_secrets)
for mid, row in rows.items()
}
echoes_by_payload = db.get_echoes_for_payloads(
list({p for p in payload_by_id.values() if p})
)
metas = {
str(mid): _build_message_meta(
row,
payload_by_id[mid],
echoes_by_payload.get(payload_by_id[mid], []),
)
for mid, row in rows.items()
}
return jsonify({'success': True, 'metas': metas})
except Exception as e:
logger.error(f"Error fetching batch message meta: {e}")
return jsonify({'success': False, 'error': str(e)}), 500
@api_bp.route('/messages/<int:msg_id>/meta', methods=['GET'])
def get_message_meta(msg_id):
"""Return metadata (SNR, hops, route, analyzer URL) for a single channel message."""
@@ -683,68 +765,9 @@ def get_message_meta(msg_id):
if not row:
return jsonify({'success': False, 'error': 'Not found'}), 404
pkt_payload = row.get('pkt_payload')
sender_ts = row.get('sender_timestamp')
ch_idx = row.get('channel_idx', 0)
txt_type = row.get('txt_type', 0)
# Compute pkt_payload if not stored
# Use DB channels (fast) to avoid blocking on device communication
if not pkt_payload and sender_ts:
db_channels = db.get_channels() if db else []
channel_secrets = {}
for ch_info in db_channels:
ch_key = ch_info.get('secret', ch_info.get('key', ''))
ci = ch_info.get('idx', ch_info.get('index'))
if ch_key and ci is not None:
channel_secrets[ci] = ch_key
if ch_idx in channel_secrets:
raw_text = None
raw_json_str = row.get('raw_json')
if raw_json_str:
try:
raw_text = json.loads(raw_json_str).get('text')
except (json.JSONDecodeError, TypeError):
pass
if not raw_text:
is_own = bool(row.get('is_own', 0))
if is_own:
device_name = runtime_config.get_device_name() or ''
raw_text = f"{device_name}: {row.get('content', '')}" if device_name else row.get('content', '')
else:
sender = row.get('sender', '')
raw_text = f"{sender}: {row.get('content', '')}" if sender else row.get('content', '')
pkt_payload = compute_pkt_payload(
channel_secrets[ch_idx], sender_ts, txt_type, raw_text
)
# Decode path_len
path_len_raw = row.get('path_len')
hop_count = None
path_hash_size = 1
if path_len_raw is not None:
hop_count, path_hash_size, _ = decode_path_len(path_len_raw)
meta = {
'success': True,
'snr': row.get('snr'),
'path_len': path_len_raw,
'hop_count': hop_count,
'path_hash_size': path_hash_size,
'pkt_payload': pkt_payload,
}
if pkt_payload:
meta['packet_hash'] = compute_packet_hash(pkt_payload)
echoes = db.get_echoes_for_message(pkt_payload)
if echoes:
meta['echo_count'] = len(echoes)
meta['echo_paths'] = [e.get('path', '') for e in echoes if e.get('path')]
meta['echo_snrs'] = [e.get('snr') for e in echoes if e.get('snr') is not None]
meta['echo_hash_sizes'] = [e.get('hash_size', 1) for e in echoes if e.get('path')]
return jsonify(meta)
pkt_payload = _get_row_pkt_payload(row, _build_channel_secrets(db))
echoes = db.get_echoes_for_message(pkt_payload) if pkt_payload else []
return jsonify(_build_message_meta(row, pkt_payload, echoes))
except Exception as e:
logger.error(f"Error fetching message meta: {e}")
app/static/js/app.js
+27 -8
View File
@@ -1218,9 +1218,17 @@ function appendMessageFromSocket(data) {
markChannelAsRead(currentChannelIdx, msg.timestamp);
}
// Cap on ids per /api/messages/meta request, to keep the query string short.
const META_BATCH_SIZE = 200;
/**
* Refresh metadata (SNR, hops, route, analyzer) for messages missing it.
* Fetches /api/messages/<id>/meta for each incomplete message, updates DOM in-place.
*
* Every echo triggers a sweep of the whole rendered list, and messages that
* never gain a route (no repeaters heard them) never stop qualifying — so this
* must stay cheap. Ids are collected first and resolved with batched
* /api/messages/meta calls; one request per message used to flood the
* single-threaded server and hang the UI.
*/
async function refreshMessagesMeta(forceIds = []) {
const container = document.getElementById('messagesList');
@@ -1228,7 +1236,8 @@ async function refreshMessagesMeta(forceIds = []) {
const forced = new Set((forceIds || []).map(String));
// Find message wrappers that don't have full metadata yet
// Collect message wrappers that don't have full metadata yet
const pending = new Map(); // msgId -> wrapper
const wrappers = container.querySelectorAll('.message-wrapper[data-msg-id]');
for (const wrapper of wrappers) {
const msgId = wrapper.dataset.msgId;
@@ -1245,14 +1254,24 @@ async function refreshMessagesMeta(forceIds = []) {
if (hasRoute && hasAnalyzer) continue;
}
try {
const resp = await fetch(`/api/messages/${msgId}/meta`);
const meta = await resp.json();
if (!meta.success) continue;
pending.set(msgId, wrapper);
}
if (pending.size === 0) return;
updateMessageMetaDOM(wrapper, meta);
const ids = Array.from(pending.keys());
for (let i = 0; i < ids.length; i += META_BATCH_SIZE) {
const chunk = ids.slice(i, i + META_BATCH_SIZE);
try {
const resp = await fetch(`/api/messages/meta?ids=${chunk.join(',')}`);
const data = await resp.json();
if (!data.success || !data.metas) continue;
for (const [msgId, meta] of Object.entries(data.metas)) {
const wrapper = pending.get(msgId);
if (wrapper && meta.success) updateMessageMetaDOM(wrapper, meta);
}
} catch (e) {
console.error(`Error fetching meta for msg #${msgId}:`, e);
console.error('Error fetching message meta batch:', e);
}
}
}
docs/whatsnew.md
+1
View File
@@ -14,6 +14,7 @@ For deep technical notes, see [architecture.md](architecture.md). For the full g
- **Messages no longer go missing after the app has been in the background.** Coming back to a minimised app — or to a phone that had been asleep — could show a chat that quietly stopped at whatever message arrived last before the screen went off, with everything since then missing until the app was force-stopped and reopened. New messages reach an open page over a live connection, and Android tears that connection down while the app sits in the background; nothing then went back to ask the server what had been missed. Now every way back from a gap re-reads the list: the connection coming back, the app returning to the foreground, and a heartbeat that notices when the page has been frozen. The same applies to direct messages, and to a browser tab that lost its network for a while.
- **A Refresh item in the menu.** The browser's pull-to-refresh has no equivalent in the Android app, so there is now a **Refresh** entry at the top of the menu that reloads the messages from the server on demand — in the app, and everywhere else too.
- **The app no longer grinds to a halt with more than one window open.** Keeping mc-webui open in two places at once — a second browser tab, or a phone and a desktop together — could leave the message list stuck on "Loading messages…" and the status on "Connecting…", looking for all the world like the mesh device had dropped off, when it was connected the whole time. Each open window was asking the server about every message on screen separately, hundreds of individual requests at a time, repeated every few seconds as new radio traffic came in; together they left the server no room to answer anything else. Those requests are now bundled into a single one, so a window that used to need hundreds needs one, and several windows at once are no longer a problem.
---