diff --git a/VERSION b/VERSION index 197c4d5..005119b 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.4.0 +2.4.1 diff --git a/android/src/app/build.gradle.kts b/android/src/app/build.gradle.kts index 96d95e2..97c673d 100644 --- a/android/src/app/build.gradle.kts +++ b/android/src/app/build.gradle.kts @@ -11,8 +11,8 @@ android { applicationId = "it.wojtaszek.mc.wrapper" minSdk = 21 targetSdk = 34 - versionCode = 2 - versionName = "1.1" + versionCode = 3 + versionName = "1.2" } buildTypes { diff --git a/android/src/app/src/main/java/it/wojtaszek/mc/wrapper/MainActivity.kt b/android/src/app/src/main/java/it/wojtaszek/mc/wrapper/MainActivity.kt index 879fd60..13a064e 100644 --- a/android/src/app/src/main/java/it/wojtaszek/mc/wrapper/MainActivity.kt +++ b/android/src/app/src/main/java/it/wojtaszek/mc/wrapper/MainActivity.kt @@ -106,6 +106,18 @@ class MainActivity : AppCompatActivity() { if (savedUrl.isNullOrEmpty()) showConfig(null) else connect(savedUrl) } + /** + * The page keeps running while the app sits in the background - that is how + * notifications keep arriving - but the connection behind it does not + * survive doze, so the message list can be minutes behind by the time the + * user looks at it again. A WebView is not guaranteed to tell the page it + * was ever hidden, so say it here instead and let mc-webui catch up. + */ + override fun onResume() { + super.onResume() + callJs("window.__mcAppResumed") + } + /** * A notification tap on a running app lands here rather than in [onCreate]. * When the app was not running, simply being launched is the whole point of diff --git a/app/database.py b/app/database.py index a682dbc..ebeb9da 100644 --- a/app/database.py +++ b/app/database.py @@ -834,6 +834,26 @@ class Database: ).fetchone() return dict(row) if row else None + def get_channel_messages_by_ids(self, msg_ids: List[int]) -> Dict[int, Dict]: + """Batch-fetch channel messages by id with chunked IN queries. + + Returns {msg_id: row dict}; ids with no row are simply absent. + Chunked at 500 to stay under SQLite's host-parameter limit.""" + result: Dict[int, Dict] = {} + if not msg_ids: + return result + with self._connect() as conn: + for i in range(0, len(msg_ids), 500): + chunk = msg_ids[i:i + 500] + placeholders = ",".join("?" * len(chunk)) + rows = conn.execute( + f"SELECT * FROM channel_messages WHERE id IN ({placeholders})", + chunk + ).fetchall() + for r in rows: + result[r['id']] = dict(r) + return result + def get_channel_messages(self, channel_idx: int = None, limit: int = 50, offset: int = 0, days: int = None) -> List[Dict]: with self._connect() as conn: @@ -1460,6 +1480,35 @@ class Database: # Maintenance # ================================================================ + def get_status_summary(self) -> Dict[str, Any]: + """Message count + newest channel timestamp, for /api/status. + + /api/status is polled by every open tab, so it gets a dedicated query + instead of reusing get_stats() + get_channel_messages(limit=1): + + - get_stats() counts 11 tables to answer a question about 2. The + COUNT(*) over echoes alone costs ~200ms on a 20k-row table and its + result was discarded — and echoes only ever grow. + - get_channel_messages(limit=1) was a `SELECT *` whose ORDER BY + timestamp has no usable index (idx_cm_channel_ts leads with + channel_idx), so it scanned the table through two temp B-trees and + materialized every column, raw_packet included, to read one field. + MAX(timestamp) is served straight off that index instead. + + Together with folding both into a single connection this is ~46ms + where the pair cost ~371ms. + """ + with self._connect() as conn: + row = conn.execute( + """SELECT (SELECT COUNT(*) FROM channel_messages) AS channel_count, + (SELECT COUNT(*) FROM direct_messages) AS dm_count, + (SELECT MAX(timestamp) FROM channel_messages) AS latest_timestamp""" + ).fetchone() + return { + 'message_count': (row['channel_count'] or 0) + (row['dm_count'] or 0), + 'latest_message_timestamp': row['latest_timestamp'], + } + def get_stats(self) -> Dict[str, Any]: """Get row counts for all tables.""" tables = ['device', 'contacts', 'channels', 'channel_messages', diff --git a/app/routes/api.py b/app/routes/api.py index fad6b6e..a8fb8f3 100644 --- a/app/routes/api.py +++ b/app/routes/api.py @@ -671,6 +671,88 @@ def get_path_analyzer_messages(): return jsonify({'success': False, 'error': str(e)}), 500 +def _build_message_meta(row: dict, pkt_payload, echoes: list) -> dict: + """Assemble the meta payload (SNR, hops, route, analyzer hash) for one row. + + Pure: takes the already-resolved pkt_payload and its echoes so callers can + batch the DB work.""" + path_len_raw = row.get('path_len') + hop_count = None + path_hash_size = 1 + if path_len_raw is not None: + hop_count, path_hash_size, _ = decode_path_len(path_len_raw) + + meta = { + 'success': True, + 'snr': row.get('snr'), + 'path_len': path_len_raw, + 'hop_count': hop_count, + 'path_hash_size': path_hash_size, + 'pkt_payload': pkt_payload, + } + + if pkt_payload: + meta['packet_hash'] = compute_packet_hash(pkt_payload) + if echoes: + meta['echo_count'] = len(echoes) + meta['echo_paths'] = [e.get('path', '') for e in echoes if e.get('path')] + meta['echo_snrs'] = [e.get('snr') for e in echoes if e.get('snr') is not None] + meta['echo_hash_sizes'] = [e.get('hash_size', 1) for e in echoes if e.get('path')] + + return meta + + +@api_bp.route('/messages/meta', methods=['GET']) +def get_messages_meta_batch(): + """Return metadata for many channel messages at once. + + Query: ?ids=1,2,3 -> {'success': True, 'metas': {'1': {...}, ...}} + + The UI sweeps every rendered message whenever echoes arrive; doing that one + request per message flooded the single-threaded werkzeug server (thousands + of round-trips per minute, each opening its own SQLite connections). This + resolves the whole sweep with a handful of queries. + """ + try: + db = _get_db() + if not db: + return jsonify({'success': False, 'error': 'No database'}), 500 + + raw_ids = (request.args.get('ids') or '').split(',') + msg_ids = [] + for part in raw_ids: + part = part.strip() + if part.isdigit(): + msg_ids.append(int(part)) + if not msg_ids: + return jsonify({'success': True, 'metas': {}}) + + rows = db.get_channel_messages_by_ids(msg_ids) + channel_secrets = _build_channel_secrets(db) + + payload_by_id = { + mid: _get_row_pkt_payload(row, channel_secrets) + for mid, row in rows.items() + } + echoes_by_payload = db.get_echoes_for_payloads( + list({p for p in payload_by_id.values() if p}) + ) + + metas = { + str(mid): _build_message_meta( + row, + payload_by_id[mid], + echoes_by_payload.get(payload_by_id[mid], []), + ) + for mid, row in rows.items() + } + return jsonify({'success': True, 'metas': metas}) + + except Exception as e: + logger.error(f"Error fetching batch message meta: {e}") + return jsonify({'success': False, 'error': str(e)}), 500 + + @api_bp.route('/messages//meta', methods=['GET']) def get_message_meta(msg_id): """Return metadata (SNR, hops, route, analyzer URL) for a single channel message.""" @@ -683,68 +765,9 @@ def get_message_meta(msg_id): if not row: return jsonify({'success': False, 'error': 'Not found'}), 404 - pkt_payload = row.get('pkt_payload') - sender_ts = row.get('sender_timestamp') - ch_idx = row.get('channel_idx', 0) - txt_type = row.get('txt_type', 0) - - # Compute pkt_payload if not stored - # Use DB channels (fast) to avoid blocking on device communication - if not pkt_payload and sender_ts: - db_channels = db.get_channels() if db else [] - channel_secrets = {} - for ch_info in db_channels: - ch_key = ch_info.get('secret', ch_info.get('key', '')) - ci = ch_info.get('idx', ch_info.get('index')) - if ch_key and ci is not None: - channel_secrets[ci] = ch_key - - if ch_idx in channel_secrets: - raw_text = None - raw_json_str = row.get('raw_json') - if raw_json_str: - try: - raw_text = json.loads(raw_json_str).get('text') - except (json.JSONDecodeError, TypeError): - pass - if not raw_text: - is_own = bool(row.get('is_own', 0)) - if is_own: - device_name = runtime_config.get_device_name() or '' - raw_text = f"{device_name}: {row.get('content', '')}" if device_name else row.get('content', '') - else: - sender = row.get('sender', '') - raw_text = f"{sender}: {row.get('content', '')}" if sender else row.get('content', '') - pkt_payload = compute_pkt_payload( - channel_secrets[ch_idx], sender_ts, txt_type, raw_text - ) - - # Decode path_len - path_len_raw = row.get('path_len') - hop_count = None - path_hash_size = 1 - if path_len_raw is not None: - hop_count, path_hash_size, _ = decode_path_len(path_len_raw) - - meta = { - 'success': True, - 'snr': row.get('snr'), - 'path_len': path_len_raw, - 'hop_count': hop_count, - 'path_hash_size': path_hash_size, - 'pkt_payload': pkt_payload, - } - - if pkt_payload: - meta['packet_hash'] = compute_packet_hash(pkt_payload) - echoes = db.get_echoes_for_message(pkt_payload) - if echoes: - meta['echo_count'] = len(echoes) - meta['echo_paths'] = [e.get('path', '') for e in echoes if e.get('path')] - meta['echo_snrs'] = [e.get('snr') for e in echoes if e.get('snr') is not None] - meta['echo_hash_sizes'] = [e.get('hash_size', 1) for e in echoes if e.get('path')] - - return jsonify(meta) + pkt_payload = _get_row_pkt_payload(row, _build_channel_secrets(db)) + echoes = db.get_echoes_for_message(pkt_payload) if pkt_payload else [] + return jsonify(_build_message_meta(row, pkt_payload, echoes)) except Exception as e: logger.error(f"Error fetching message meta: {e}") @@ -864,12 +887,9 @@ def get_status(): message_count = 0 latest_timestamp = None if db: - stats = db.get_stats() - message_count = stats.get('channel_messages', 0) + stats.get('direct_messages', 0) - # Get latest channel message timestamp - recent = db.get_channel_messages(limit=1) - if recent: - latest_timestamp = recent[0].get('timestamp') + summary = db.get_status_summary() + message_count = summary['message_count'] + latest_timestamp = summary['latest_message_timestamp'] else: message_count = parser.count_messages() latest = parser.get_latest_message() diff --git a/app/static/js/app.js b/app/static/js/app.js index a3a9dbf..12d0d8c 100644 --- a/app/static/js/app.js +++ b/app/static/js/app.js @@ -391,6 +391,86 @@ function isContactProtectedByName(senderName) { return pubkey && protectedContactPubkeys.has(pubkey.toLowerCase()); } +// ============================================================================= +// Resync after a gap +// +// The chat view is push-driven: messages arrive over the socket and are +// appended one at a time. Whatever the socket misses - Android doze tearing the +// connection down behind a locked screen, a switch between Wi-Fi and mobile - +// is never drawn, and the list silently stops at the last message that got +// through. A browser tab hides this because it reloads the page on resume; the +// Android wrapper keeps the same page alive for days, so the gap stays until +// the app is force-stopped. +// +// So every way back from a gap ends up here: the socket reconnecting, the page +// becoming visible, the heartbeat noticing it was frozen, the Refresh menu +// item, and the wrapper's onResume hook. +// ============================================================================= + +let chatSocketEverConnected = false; +let resyncInFlight = false; + +/** Re-read the message list and badges from the server. */ +async function resyncFromServer(reason, { toast = false } = {}) { + if (resyncInFlight) return; + resyncInFlight = true; + console.log(`[resync] refreshing after ${reason}`); + try { + // The archive view is a frozen snapshot of one day - reloading it would + // fight the date the user picked. Its badges still need updating. + if (!currentArchiveDate) await loadMessages(); + await checkForUpdates(); + loadStatus(); + updatePendingContactsBadge(); + checkDmUpdates(); + if (toast) showNotification('Messages refreshed', 'success'); + } catch (error) { + console.error('[resync] failed:', error); + if (toast) showNotification('Refresh failed', 'danger'); + } finally { + resyncInFlight = false; + } +} + +/** + * Heartbeat that catches the cases no event announces. + * + * A tick arriving far later than scheduled means the page was frozen or + * throttled - precisely the window in which socket events go missing - so the + * gap is worth a resync even if the socket claims it never dropped. The same + * tick nudges a socket still stuck in its reconnect backoff. + */ +function startResyncHeartbeat() { + const TICK_MS = 20000; + let lastTickAt = Date.now(); + + setInterval(() => { + const now = Date.now(); + const drift = now - lastTickAt; + lastTickAt = now; + + if (document.hidden) return; // visibilitychange covers the way back + + if (drift > TICK_MS * 3) { + resyncFromServer('timer gap'); + return; + } + // connect() during backoff simply retries now instead of later; the + // 'connect' handler then does the resync + if (chatSocket && !chatSocket.connected) chatSocket.connect(); + }, TICK_MS); +} + +/** + * Called by the Android wrapper from onResume. The WebView is not guaranteed + * to fire visibilitychange for an Activity coming back to the foreground, so + * the wrapper says so itself. + */ +window.__mcAppResumed = function() { + if (chatSocket && !chatSocket.connected) chatSocket.connect(); + resyncFromServer('app resumed'); +}; + // Initialize on page load /** * Connect to SocketIO /chat namespace for real-time message updates @@ -402,9 +482,14 @@ function connectChatSocket() { } const wsUrl = window.location.origin; + // Default transports (polling, then upgrade to websocket). Long-polling was + // pinned in 1d47c9c because werkzeug had no websocket support; python-engineio + // 4.8.1 pulled in simple-websocket and it does now. Polling holds an HTTP + // connection open per tab, and browsers only allow six per origin, so three + // tabs starved every other request of a connection for tens of seconds. + // Upgrading moves that connection out of the HTTP pool. Where the upgrade is + // blocked (a proxy that drops Upgrade), the client stays on polling by itself. chatSocket = io(wsUrl + '/chat', { - transports: ['polling'], - upgrade: false, reconnection: true, reconnectionDelay: 2000, reconnectionDelayMax: 10000, @@ -412,6 +497,15 @@ function connectChatSocket() { chatSocket.on('connect', () => { console.log('SocketIO connected to /chat'); + // Everything pushed while the socket was down is gone for good - only a + // re-read of the list brings those messages back. Skipped on the very + // first connect, where DOMContentLoaded has just loaded them anyway. + if (chatSocketEverConnected) resyncFromServer('socket reconnect'); + chatSocketEverConnected = true; + }); + + chatSocket.on('disconnect', (reason) => { + console.warn('SocketIO /chat disconnected:', reason); }); chatSocket.on('connect_error', (err) => { @@ -582,6 +676,8 @@ document.addEventListener('DOMContentLoaded', async function() { // Connect SocketIO for real-time updates connectChatSocket(); + // Safety net for the updates the socket never delivered + startResyncHeartbeat(); console.log(`[init] UI ready in ${(performance.now() - initStart).toFixed(0)}ms`); @@ -609,23 +705,34 @@ window.addEventListener('pageshow', function(event) { }); // Handle app returning from background (PWA visibility change) +let hiddenSince = null; document.addEventListener('visibilitychange', function() { - if (!document.hidden) { - // App became visible again, force viewport recalculation - console.log('App became visible, recalculating viewport'); - setTimeout(() => { - window.scrollTo(0, 0); - window.dispatchEvent(new Event('resize')); - document.body.offsetHeight; - }, 100); - - // Clear app badge when user returns to app - if ('clearAppBadge' in navigator) { - navigator.clearAppBadge().catch((error) => { - console.error('Error clearing app badge on visibility:', error); - }); - } + if (document.hidden) { + hiddenSince = Date.now(); + return; } + + // App became visible again, force viewport recalculation + console.log('App became visible, recalculating viewport'); + setTimeout(() => { + window.scrollTo(0, 0); + window.dispatchEvent(new Event('resize')); + document.body.offsetHeight; + }, 100); + + // Clear app badge when user returns to app + if ('clearAppBadge' in navigator) { + navigator.clearAppBadge().catch((error) => { + console.error('Error clearing app badge on visibility:', error); + }); + } + + // Anything longer than a glance away is long enough for the socket to have + // dropped messages, so catch up before the user reads a stale list + const away = hiddenSince ? Date.now() - hiddenSince : 0; + hiddenSince = null; + if (chatSocket && !chatSocket.connected) chatSocket.connect(); + if (away > 10000) resyncFromServer('back from background'); }); /** @@ -943,6 +1050,15 @@ function setupEventListeners() { inst.hide(); }); + // Manual refresh from the menu + const refreshBtn = document.getElementById('refreshBtn'); + if (refreshBtn) { + refreshBtn.addEventListener('click', () => { + if (chatSocket && !chatSocket.connected) chatSocket.connect(); + resyncFromServer('manual refresh', { toast: true }); + }); + } + // Notification toggle const notificationsToggle = document.getElementById('notificationsToggle'); if (notificationsToggle) { @@ -1107,9 +1223,17 @@ function appendMessageFromSocket(data) { markChannelAsRead(currentChannelIdx, msg.timestamp); } +// Cap on ids per /api/messages/meta request, to keep the query string short. +const META_BATCH_SIZE = 200; + /** * Refresh metadata (SNR, hops, route, analyzer) for messages missing it. - * Fetches /api/messages//meta for each incomplete message, updates DOM in-place. + * + * Every echo triggers a sweep of the whole rendered list, and messages that + * never gain a route (no repeaters heard them) never stop qualifying — so this + * must stay cheap. Ids are collected first and resolved with batched + * /api/messages/meta calls; one request per message used to flood the + * single-threaded server and hang the UI. */ async function refreshMessagesMeta(forceIds = []) { const container = document.getElementById('messagesList'); @@ -1117,7 +1241,8 @@ async function refreshMessagesMeta(forceIds = []) { const forced = new Set((forceIds || []).map(String)); - // Find message wrappers that don't have full metadata yet + // Collect message wrappers that don't have full metadata yet + const pending = new Map(); // msgId -> wrapper const wrappers = container.querySelectorAll('.message-wrapper[data-msg-id]'); for (const wrapper of wrappers) { const msgId = wrapper.dataset.msgId; @@ -1134,14 +1259,24 @@ async function refreshMessagesMeta(forceIds = []) { if (hasRoute && hasAnalyzer) continue; } - try { - const resp = await fetch(`/api/messages/${msgId}/meta`); - const meta = await resp.json(); - if (!meta.success) continue; + pending.set(msgId, wrapper); + } + if (pending.size === 0) return; - updateMessageMetaDOM(wrapper, meta); + const ids = Array.from(pending.keys()); + for (let i = 0; i < ids.length; i += META_BATCH_SIZE) { + const chunk = ids.slice(i, i + META_BATCH_SIZE); + try { + const resp = await fetch(`/api/messages/meta?ids=${chunk.join(',')}`); + const data = await resp.json(); + if (!data.success || !data.metas) continue; + + for (const [msgId, meta] of Object.entries(data.metas)) { + const wrapper = pending.get(msgId); + if (wrapper && meta.success) updateMessageMetaDOM(wrapper, meta); + } } catch (e) { - console.error(`Error fetching meta for msg #${msgId}:`, e); + console.error('Error fetching message meta batch:', e); } } } diff --git a/app/static/js/console.js b/app/static/js/console.js index cc2e2ee..415b7fe 100644 --- a/app/static/js/console.js +++ b/app/static/js/console.js @@ -37,9 +37,8 @@ function connectWebSocket() { console.log('Connecting to WebSocket:', wsUrl); try { + // Default transports — see the note in app.js connectChatSocket(). socket = io(wsUrl + '/console', { - transports: ['polling'], - upgrade: false, reconnection: true, reconnectionAttempts: Infinity, reconnectionDelay: 1000, diff --git a/app/static/js/dm.js b/app/static/js/dm.js index 93a9e7b..b357f7a 100644 --- a/app/static/js/dm.js +++ b/app/static/js/dm.js @@ -114,6 +114,31 @@ function resolveConversationName(conversationId) { return 'Unknown'; } +let chatSocketEverConnected = false; +let resyncInFlight = false; + +/** + * Re-read the DM lists from the server. + * + * This view is push-driven, so a socket that drops while the phone sleeps + * leaves it showing whatever arrived last - the 60s poll eventually catches up, + * but only once its timer un-throttles. Every path back from a gap comes here. + */ +async function resyncFromServer(reason) { + if (resyncInFlight) return; + resyncInFlight = true; + console.log(`DM: [resync] refreshing after ${reason}`); + try { + await loadConversations(); + if (currentConversationId) await loadMessages(); + await loadStatus(); + } catch (error) { + console.error('DM: [resync] failed:', error); + } finally { + resyncInFlight = false; + } +} + /** * Connect to SocketIO /chat namespace for real-time DM and ACK updates */ @@ -124,9 +149,8 @@ function connectChatSocket() { } const wsUrl = window.location.origin; + // Default transports — see the note in app.js connectChatSocket(). chatSocket = io(wsUrl + '/chat', { - transports: ['polling'], - upgrade: false, reconnection: true, reconnectionDelay: 2000, reconnectionDelayMax: 10000, @@ -134,10 +158,14 @@ function connectChatSocket() { chatSocket.on('connect', () => { console.log('DM: SocketIO connected to /chat'); + // Whatever arrived while the socket was down was never pushed to us; + // only re-reading the lists brings it back (see resyncFromServer) + if (chatSocketEverConnected) resyncFromServer('socket reconnect'); + chatSocketEverConnected = true; }); - chatSocket.on('disconnect', () => { - console.log('DM: SocketIO disconnected'); + chatSocket.on('disconnect', (reason) => { + console.log('DM: SocketIO disconnected:', reason); }); // Real-time new DM message @@ -330,16 +358,27 @@ window.addEventListener('pageshow', function(event) { }); // Handle app returning from background (PWA visibility change) +let hiddenSince = null; document.addEventListener('visibilitychange', function() { - if (!document.hidden) { - // App became visible again, force viewport recalculation - console.log('App became visible, recalculating viewport'); - setTimeout(() => { - window.scrollTo(0, 0); - window.dispatchEvent(new Event('resize')); - document.body.offsetHeight; - }, 100); + if (document.hidden) { + hiddenSince = Date.now(); + return; } + + // App became visible again, force viewport recalculation + console.log('App became visible, recalculating viewport'); + setTimeout(() => { + window.scrollTo(0, 0); + window.dispatchEvent(new Event('resize')); + document.body.offsetHeight; + }, 100); + + // Long enough away for the socket to have dropped updates - catch up + // rather than wait for the next 60s poll + const away = hiddenSince ? Date.now() - hiddenSince : 0; + hiddenSince = null; + if (chatSocket && !chatSocket.connected) chatSocket.connect(); + if (away > 10000) resyncFromServer('back from background'); }); /** diff --git a/app/static/js/logs.js b/app/static/js/logs.js index ae4050c..8af5d43 100644 --- a/app/static/js/logs.js +++ b/app/static/js/logs.js @@ -32,9 +32,8 @@ const LEVEL_ORDER = { DEBUG: 0, INFO: 1, WARNING: 2, ERROR: 3, CRITICAL: 4 }; // --- WebSocket --- + // Default transports — see the note in app.js connectChatSocket(). const socket = io('/logs', { - transports: ['polling'], - upgrade: false, reconnection: true, reconnectionDelay: 2000, }); diff --git a/app/templates/base.html b/app/templates/base.html index 742d3a2..224151b 100644 --- a/app/templates/base.html +++ b/app/templates/base.html @@ -111,6 +111,16 @@
+ +