adam/mc-webui
1
0
Fork 0
mirror of https://github.com/MarekWo/mc-webui.git synced 2026-08-08 17:53:00 +02:00
Code Issues Packages Projects Releases Wiki Activity

perf: resolve message metadata in one batched request

Browse Source 
Every incoming echo triggers a sweep of the whole rendered message list, and
refreshMessagesMeta() awaited one /api/messages/<id>/meta per message inside
the loop. Messages that never gain a route (nothing heard them) never stop
qualifying for the sweep, so the same ~180 messages were re-fetched every few
seconds: 7,500 requests in nine minutes on a single tab, each opening its own
SQLite connections. The single-threaded werkzeug server — the same one
production runs — had no room left for anything else, so the UI hung on
"Loading messages..." / "Connecting..." while the device was in fact connected.

Add GET /api/messages/meta?ids=... resolving the whole sweep with a handful of
queries, batching the row and echo lookups, and have the client collect ids
first and fetch them in chunks. The per-message endpoint stays for forced
single refreshes; both now share _build_message_meta() and the existing
_build_channel_secrets / _get_row_pkt_payload helpers, so the payload is
unchanged (verified byte-identical against the old response).

Measured on the local container, 500 messages rendered / 181 needing meta:
one 273 ms request in place of 181 sequential ones at ~82 ms each.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
MarekWo
2026-07-31 07:44:48 +02:00
parent ebd2e95fe1
commit 0e524ddbc1
4 changed files with 133 additions and 70 deletions
Download Patch File Download Diff File Expand all files Collapse all files
app/database.py
+20
View File
@@ -834,6 +834,26 @@ class Database:
).fetchone()
return dict(row) if row else None
def get_channel_messages_by_ids(self, msg_ids: List[int]) -> Dict[int, Dict]:
"""Batch-fetch channel messages by id with chunked IN queries.
Returns {msg_id: row dict}; ids with no row are simply absent.
Chunked at 500 to stay under SQLite's host-parameter limit."""
result: Dict[int, Dict] = {}
if not msg_ids:
return result
with self._connect() as conn:
for i in range(0, len(msg_ids), 500):
chunk = msg_ids[i:i + 500]
placeholders = ",".join("?" * len(chunk))
rows = conn.execute(
f"SELECT * FROM channel_messages WHERE id IN ({placeholders})",
chunk
).fetchall()
for r in rows:
result[r['id']] = dict(r)
return result
def get_channel_messages(self, channel_idx: int = None, limit: int = 50,
offset: int = 0, days: int = None) -> List[Dict]:
with self._connect() as conn:
app/routes/api.py
+85 -62
View File
@@ -671,6 +671,88 @@ def get_path_analyzer_messages():
return jsonify({'success': False, 'error': str(e)}), 500
def _build_message_meta(row: dict, pkt_payload, echoes: list) -> dict:
"""Assemble the meta payload (SNR, hops, route, analyzer hash) for one row.
Pure: takes the already-resolved pkt_payload and its echoes so callers can
batch the DB work."""
path_len_raw = row.get('path_len')
hop_count = None
path_hash_size = 1
if path_len_raw is not None:
hop_count, path_hash_size, _ = decode_path_len(path_len_raw)
meta = {
'success': True,
'snr': row.get('snr'),
'path_len': path_len_raw,
'hop_count': hop_count,
'path_hash_size': path_hash_size,
'pkt_payload': pkt_payload,
}
if pkt_payload:
meta['packet_hash'] = compute_packet_hash(pkt_payload)
if echoes:
meta['echo_count'] = len(echoes)
meta['echo_paths'] = [e.get('path', '') for e in echoes if e.get('path')]
meta['echo_snrs'] = [e.get('snr') for e in echoes if e.get('snr') is not None]
meta['echo_hash_sizes'] = [e.get('hash_size', 1) for e in echoes if e.get('path')]
return meta
@api_bp.route('/messages/meta', methods=['GET'])
def get_messages_meta_batch():
"""Return metadata for many channel messages at once.
Query: ?ids=1,2,3 -> {'success': True, 'metas': {'1': {...}, ...}}
The UI sweeps every rendered message whenever echoes arrive; doing that one
request per message flooded the single-threaded werkzeug server (thousands
of round-trips per minute, each opening its own SQLite connections). This
resolves the whole sweep with a handful of queries.
"""
try:
db = _get_db()
if not db:
return jsonify({'success': False, 'error': 'No database'}), 500
raw_ids = (request.args.get('ids') or '').split(',')
msg_ids = []
for part in raw_ids:
part = part.strip()
if part.isdigit():
msg_ids.append(int(part))
if not msg_ids:
return jsonify({'success': True, 'metas': {}})
rows = db.get_channel_messages_by_ids(msg_ids)
channel_secrets = _build_channel_secrets(db)
payload_by_id = {
mid: _get_row_pkt_payload(row, channel_secrets)
for mid, row in rows.items()
}
echoes_by_payload = db.get_echoes_for_payloads(
list({p for p in payload_by_id.values() if p})
)
metas = {
str(mid): _build_message_meta(
row,
payload_by_id[mid],
echoes_by_payload.get(payload_by_id[mid], []),
)
for mid, row in rows.items()
}
return jsonify({'success': True, 'metas': metas})
except Exception as e:
logger.error(f"Error fetching batch message meta: {e}")
return jsonify({'success': False, 'error': str(e)}), 500
@api_bp.route('/messages/<int:msg_id>/meta', methods=['GET'])
def get_message_meta(msg_id):
"""Return metadata (SNR, hops, route, analyzer URL) for a single channel message."""
@@ -683,68 +765,9 @@ def get_message_meta(msg_id):
if not row:
return jsonify({'success': False, 'error': 'Not found'}), 404
pkt_payload = row.get('pkt_payload')
sender_ts = row.get('sender_timestamp')
ch_idx = row.get('channel_idx', 0)
txt_type = row.get('txt_type', 0)
# Compute pkt_payload if not stored
# Use DB channels (fast) to avoid blocking on device communication
if not pkt_payload and sender_ts:
db_channels = db.get_channels() if db else []
channel_secrets = {}
for ch_info in db_channels:
ch_key = ch_info.get('secret', ch_info.get('key', ''))
ci = ch_info.get('idx', ch_info.get('index'))
if ch_key and ci is not None:
channel_secrets[ci] = ch_key
if ch_idx in channel_secrets:
raw_text = None
raw_json_str = row.get('raw_json')
if raw_json_str:
try:
raw_text = json.loads(raw_json_str).get('text')
except (json.JSONDecodeError, TypeError):
pass
if not raw_text:
is_own = bool(row.get('is_own', 0))
if is_own:
device_name = runtime_config.get_device_name() or ''
raw_text = f"{device_name}: {row.get('content', '')}" if device_name else row.get('content', '')
else:
sender = row.get('sender', '')
raw_text = f"{sender}: {row.get('content', '')}" if sender else row.get('content', '')
pkt_payload = compute_pkt_payload(
channel_secrets[ch_idx], sender_ts, txt_type, raw_text
)
# Decode path_len
path_len_raw = row.get('path_len')
hop_count = None
path_hash_size = 1
if path_len_raw is not None:
hop_count, path_hash_size, _ = decode_path_len(path_len_raw)
meta = {
'success': True,
'snr': row.get('snr'),
'path_len': path_len_raw,
'hop_count': hop_count,
'path_hash_size': path_hash_size,
'pkt_payload': pkt_payload,
}
if pkt_payload:
meta['packet_hash'] = compute_packet_hash(pkt_payload)
echoes = db.get_echoes_for_message(pkt_payload)
if echoes:
meta['echo_count'] = len(echoes)
meta['echo_paths'] = [e.get('path', '') for e in echoes if e.get('path')]
meta['echo_snrs'] = [e.get('snr') for e in echoes if e.get('snr') is not None]
meta['echo_hash_sizes'] = [e.get('hash_size', 1) for e in echoes if e.get('path')]
return jsonify(meta)
pkt_payload = _get_row_pkt_payload(row, _build_channel_secrets(db))
echoes = db.get_echoes_for_message(pkt_payload) if pkt_payload else []
return jsonify(_build_message_meta(row, pkt_payload, echoes))
except Exception as e:
logger.error(f"Error fetching message meta: {e}")
app/static/js/app.js
+27 -8
View File
@@ -1218,9 +1218,17 @@ function appendMessageFromSocket(data) {
markChannelAsRead(currentChannelIdx, msg.timestamp);
}
// Cap on ids per /api/messages/meta request, to keep the query string short.
const META_BATCH_SIZE = 200;
/**
* Refresh metadata (SNR, hops, route, analyzer) for messages missing it.
* Fetches /api/messages/<id>/meta for each incomplete message, updates DOM in-place.
*
* Every echo triggers a sweep of the whole rendered list, and messages that
* never gain a route (no repeaters heard them) never stop qualifying — so this
* must stay cheap. Ids are collected first and resolved with batched
* /api/messages/meta calls; one request per message used to flood the
* single-threaded server and hang the UI.
*/
async function refreshMessagesMeta(forceIds = []) {
const container = document.getElementById('messagesList');
@@ -1228,7 +1236,8 @@ async function refreshMessagesMeta(forceIds = []) {
const forced = new Set((forceIds || []).map(String));
// Find message wrappers that don't have full metadata yet
// Collect message wrappers that don't have full metadata yet
const pending = new Map(); // msgId -> wrapper
const wrappers = container.querySelectorAll('.message-wrapper[data-msg-id]');
for (const wrapper of wrappers) {
const msgId = wrapper.dataset.msgId;
@@ -1245,14 +1254,24 @@ async function refreshMessagesMeta(forceIds = []) {
if (hasRoute && hasAnalyzer) continue;
}
try {
const resp = await fetch(`/api/messages/${msgId}/meta`);
const meta = await resp.json();
if (!meta.success) continue;
pending.set(msgId, wrapper);
}
if (pending.size === 0) return;
updateMessageMetaDOM(wrapper, meta);
const ids = Array.from(pending.keys());
for (let i = 0; i < ids.length; i += META_BATCH_SIZE) {
const chunk = ids.slice(i, i + META_BATCH_SIZE);
try {
const resp = await fetch(`/api/messages/meta?ids=${chunk.join(',')}`);
const data = await resp.json();
if (!data.success || !data.metas) continue;
for (const [msgId, meta] of Object.entries(data.metas)) {
const wrapper = pending.get(msgId);
if (wrapper && meta.success) updateMessageMetaDOM(wrapper, meta);
}
} catch (e) {
console.error(`Error fetching meta for msg #${msgId}:`, e);
console.error('Error fetching message meta batch:', e);
}
}
}