From de0108d6aa4e9b0f4676e5d2cfd0c302ca42e777 Mon Sep 17 00:00:00 2001
From: MarekWo <mwojtaszek@gmail.com>
Date: Sat, 21 Feb 2026 17:13:36 +0100
Subject: [PATCH] feat: Add persistent contacts cache for @mention autocomplete

Contacts cache accumulates all known node names from device contacts
and adverts into a JSONL file, so @mentions work even after contacts
are removed from the device. Background thread scans adverts every
45s and parses advert payloads to extract public keys and node names.

Existing Contacts page now shows merged view with "Cache" badge for
contacts not on device, plus source filter (All/On device/Cache only).

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
---
 app/contacts_cache.py                | 237 +++++++++++++++++++++++++++
 app/main.py                          |  29 ++++
 app/routes/api.py                    |  41 +++++
 app/static/js/app.js                 |   4 +-
 app/static/js/contacts.js            | 190 +++++++++++++--------
 app/templates/contacts-existing.html |   7 +
 6 files changed, 438 insertions(+), 70 deletions(-)
 create mode 100644 app/contacts_cache.py

diff --git a/app/contacts_cache.py b/app/contacts_cache.py
new file mode 100644
index 0000000..cb1d8fa
--- /dev/null
+++ b/app/contacts_cache.py
@@ -0,0 +1,237 @@
+"""
+Contacts Cache - Persistent storage of all known node names + public keys.
+
+Stores every node name ever seen (from device contacts and adverts),
+so @mention autocomplete works even for removed contacts.
+
+File format: JSONL ({device_name}.contacts_cache.jsonl)
+Each line: {"public_key": "...", "name": "...", "first_seen": ts, "last_seen": ts, "source": "advert"|"device"}
+"""
+
+import json
+import logging
+import time
+from pathlib import Path
+from threading import Lock
+
+from app.config import config, runtime_config
+
+logger = logging.getLogger(__name__)
+
+_cache_lock = Lock()
+_cache: dict = {}  # {public_key: {name, first_seen, last_seen, source}}
+_cache_loaded = False
+_adverts_offset = 0  # File offset for incremental advert scanning
+
+
+def _get_cache_path() -> Path:
+    device_name = runtime_config.get_device_name()
+    return Path(config.MC_CONFIG_DIR) / f"{device_name}.contacts_cache.jsonl"
+
+
+def _get_adverts_path() -> Path:
+    device_name = runtime_config.get_device_name()
+    return Path(config.MC_CONFIG_DIR) / f"{device_name}.adverts.jsonl"
+
+
+def load_cache() -> dict:
+    """Load cache from disk into memory. Returns copy of cache dict."""
+    global _cache, _cache_loaded
+
+    with _cache_lock:
+        if _cache_loaded:
+            return _cache.copy()
+
+        cache_path = _get_cache_path()
+        _cache = {}
+
+        if not cache_path.exists():
+            _cache_loaded = True
+            logger.info("Contacts cache file does not exist yet")
+            return _cache.copy()
+
+        try:
+            with open(cache_path, 'r', encoding='utf-8') as f:
+                for line in f:
+                    line = line.strip()
+                    if not line:
+                        continue
+                    try:
+                        entry = json.loads(line)
+                        pk = entry.get('public_key', '').lower()
+                        if pk:
+                            _cache[pk] = entry
+                    except json.JSONDecodeError:
+                        continue
+
+            _cache_loaded = True
+            logger.info(f"Loaded contacts cache: {len(_cache)} entries")
+        except Exception as e:
+            logger.error(f"Failed to load contacts cache: {e}")
+            _cache_loaded = True
+
+        return _cache.copy()
+
+
+def save_cache() -> bool:
+    """Write full cache to disk (atomic write)."""
+    with _cache_lock:
+        cache_path = _get_cache_path()
+        try:
+            cache_path.parent.mkdir(parents=True, exist_ok=True)
+            temp_file = cache_path.with_suffix('.tmp')
+            with open(temp_file, 'w', encoding='utf-8') as f:
+                for entry in _cache.values():
+                    f.write(json.dumps(entry, ensure_ascii=False) + '\n')
+            temp_file.replace(cache_path)
+            logger.debug(f"Saved contacts cache: {len(_cache)} entries")
+            return True
+        except Exception as e:
+            logger.error(f"Failed to save contacts cache: {e}")
+            return False
+
+
+def upsert_contact(public_key: str, name: str, source: str = "advert") -> bool:
+    """Add or update a contact in the cache. Returns True if cache was modified."""
+    pk = public_key.lower()
+    now = int(time.time())
+
+    with _cache_lock:
+        existing = _cache.get(pk)
+        if existing:
+            changed = False
+            if name and name != existing.get('name'):
+                existing['name'] = name
+                changed = True
+            existing['last_seen'] = now
+            return changed
+        else:
+            if not name:
+                return False
+            _cache[pk] = {
+                'public_key': pk,
+                'name': name,
+                'first_seen': now,
+                'last_seen': now,
+                'source': source,
+            }
+            return True
+
+
+def get_all_contacts() -> list:
+    """Get all cached contacts as a list of dicts (shallow copies)."""
+    with _cache_lock:
+        return [entry.copy() for entry in _cache.values()]
+
+
+def get_all_names() -> list:
+    """Get all unique non-empty contact names sorted alphabetically."""
+    with _cache_lock:
+        return sorted(set(
+            entry['name'] for entry in _cache.values()
+            if entry.get('name')
+        ))
+
+
+def parse_advert_payload(pkt_payload_hex: str):
+    """
+    Parse advert pkt_payload to extract public_key and node_name.
+
+    Layout of pkt_payload (byte offsets):
+      [0:32]   Public Key (32 bytes = 64 hex chars)
+      [32:36]  Timestamp (4 bytes)
+      [36:100] Signature (64 bytes)
+      [100]    App Flags (1 byte) - bit 4: Location, bit 7: Name
+      [101+]   If Location (bit 4): Lat (4 bytes) + Lon (4 bytes)
+               If Name (bit 7): Node name (UTF-8, variable length)
+
+    Returns:
+        (public_key_hex, node_name) or (None, None) on failure
+    """
+    try:
+        raw = bytes.fromhex(pkt_payload_hex)
+        if len(raw) < 101:
+            return None, None
+
+        public_key = pkt_payload_hex[:64].lower()
+        app_flags = raw[100]
+
+        has_location = bool(app_flags & 0x10)  # bit 4
+        has_name = bool(app_flags & 0x80)      # bit 7
+
+        if not has_name:
+            return public_key, None
+
+        name_offset = 101
+        if has_location:
+            name_offset += 8  # lat(4) + lon(4)
+
+        if name_offset >= len(raw):
+            return public_key, None
+
+        name_bytes = raw[name_offset:]
+        node_name = name_bytes.decode('utf-8', errors='replace').rstrip('\x00')
+
+        return public_key, node_name if node_name else None
+    except Exception:
+        return None, None
+
+
+def scan_new_adverts() -> int:
+    """
+    Scan .adverts.jsonl for new entries since last scan.
+    Returns number of new/updated contacts.
+    """
+    global _adverts_offset
+
+    adverts_path = _get_adverts_path()
+    if not adverts_path.exists():
+        return 0
+
+    updated = 0
+    try:
+        with open(adverts_path, 'r', encoding='utf-8') as f:
+            f.seek(_adverts_offset)
+            for line in f:
+                line = line.strip()
+                if not line:
+                    continue
+                try:
+                    advert = json.loads(line)
+                    pkt_payload = advert.get('pkt_payload', '')
+                    if not pkt_payload:
+                        continue
+                    pk, name = parse_advert_payload(pkt_payload)
+                    if pk and name:
+                        if upsert_contact(pk, name, source="advert"):
+                            updated += 1
+                except json.JSONDecodeError:
+                    continue
+            _adverts_offset = f.tell()
+    except Exception as e:
+        logger.error(f"Failed to scan adverts: {e}")
+
+    if updated > 0:
+        save_cache()
+        logger.info(f"Contacts cache updated: {updated} new/changed entries")
+
+    return updated
+
+
+def initialize_from_device(contacts_detailed: dict):
+    """
+    Seed cache from /api/contacts/detailed response dict.
+    Called once at startup if cache file doesn't exist.
+
+    Args:
+        contacts_detailed: dict of {public_key: {adv_name, type, ...}} from meshcli
+    """
+    added = 0
+    for pk, details in contacts_detailed.items():
+        name = details.get('adv_name', '')
+        if upsert_contact(pk, name, source="device"):
+            added += 1
+
+    if added > 0:
+        save_cache()
+        logger.info(f"Initialized contacts cache from device: {added} contacts")
diff --git a/app/main.py b/app/main.py
index 2b2ce3c..af3d851 100644
--- a/app/main.py
+++ b/app/main.py
@@ -16,6 +16,7 @@ from app.routes.api import api_bp
 from app.version import VERSION_STRING, GIT_BRANCH
 from app.archiver.manager import schedule_daily_archiving
 from app.meshcore.cli import fetch_device_name_from_bridge
+from app.contacts_cache import load_cache, scan_new_adverts, initialize_from_device
 
 # Commands that require longer timeout (in seconds)
 SLOW_COMMANDS = {
@@ -108,6 +109,34 @@ def create_app():
 
     threading.Thread(target=init_device_name, daemon=True).start()
 
+    # Background thread: contacts cache initialization and periodic advert scanning
+    def init_contacts_cache():
+        # Wait for device name to resolve
+        time.sleep(10)
+
+        cache = load_cache()
+
+        # Seed from device contacts if cache is empty
+        if not cache:
+            try:
+                from app.routes.api import get_contacts_detailed_cached
+                success, contacts, error = get_contacts_detailed_cached()
+                if success and contacts:
+                    initialize_from_device(contacts)
+                    logger.info("Contacts cache seeded from device")
+            except Exception as e:
+                logger.error(f"Failed to seed contacts cache: {e}")
+
+        # Periodic advert scan loop
+        while True:
+            time.sleep(45)
+            try:
+                scan_new_adverts()
+            except Exception as e:
+                logger.error(f"Contacts cache scan error: {e}")
+
+    threading.Thread(target=init_contacts_cache, daemon=True).start()
+
     logger.info(f"mc-webui started - device: {config.MC_DEVICE_NAME}")
     logger.info(f"Messages file: {config.msgs_file_path}")
     logger.info(f"Serial port: {config.MC_SERIAL_PORT}")
diff --git a/app/routes/api.py b/app/routes/api.py
index ad4dd64..54299d7 100644
--- a/app/routes/api.py
+++ b/app/routes/api.py
@@ -19,6 +19,7 @@ from flask import Blueprint, jsonify, request, send_file
 from app.meshcore import cli, parser
 from app.config import config, runtime_config
 from app.archiver import manager as archive_manager
+from app.contacts_cache import get_all_names, get_all_contacts
 
 logger = logging.getLogger(__name__)
 
@@ -569,6 +570,46 @@ def get_contacts():
         }), 500
 
 
+@api_bp.route('/contacts/cached', methods=['GET'])
+def get_cached_contacts():
+    """
+    Get all known contacts from persistent cache (superset of device contacts).
+    Includes contacts seen via adverts even after removal from device.
+
+    Query params:
+        ?format=names  - Return just name strings for @mentions (default)
+        ?format=full   - Return full cache entries with public_key, timestamps, etc.
+    """
+    try:
+        fmt = request.args.get('format', 'names')
+
+        if fmt == 'full':
+            contacts = get_all_contacts()
+            # Add public_key_prefix for display
+            for c in contacts:
+                c['public_key_prefix'] = c.get('public_key', '')[:12]
+            return jsonify({
+                'success': True,
+                'contacts': contacts,
+                'count': len(contacts)
+            }), 200
+        else:
+            names = get_all_names()
+            return jsonify({
+                'success': True,
+                'contacts': names,
+                'count': len(names)
+            }), 200
+
+    except Exception as e:
+        logger.error(f"Error getting cached contacts: {e}")
+        return jsonify({
+            'success': False,
+            'error': str(e),
+            'contacts': []
+        }), 500
+
+
 def _filter_contacts_by_criteria(contacts: list, criteria: dict) -> list:
     """
     Filter contacts based on cleanup criteria.
diff --git a/app/static/js/app.js b/app/static/js/app.js
index f28b64c..e9eed45 100644
--- a/app/static/js/app.js
+++ b/app/static/js/app.js
@@ -2795,13 +2795,13 @@ async function loadContactsForMentions() {
     }
 
     try {
-        const response = await fetch('/api/contacts');
+        const response = await fetch('/api/contacts/cached');
         const data = await response.json();
 
         if (data.success && data.contacts) {
             mentionsCache = data.contacts;
             mentionsCacheTimestamp = now;
-            console.log(`[mentions] Cached ${mentionsCache.length} contacts`);
+            console.log(`[mentions] Cached ${mentionsCache.length} contacts from cache`);
         }
     } catch (error) {
         console.error('[mentions] Error loading contacts:', error);
diff --git a/app/static/js/contacts.js b/app/static/js/contacts.js
index e9072df..734f066 100644
--- a/app/static/js/contacts.js
+++ b/app/static/js/contacts.js
@@ -839,6 +839,14 @@ function attachExistingEventListeners() {
         });
     }
 
+    // Source filter (device / cache only)
+    const sourceFilter = document.getElementById('sourceFilter');
+    if (sourceFilter) {
+        sourceFilter.addEventListener('change', () => {
+            applySortAndFilters();
+        });
+    }
+
     // Type filter
     const typeFilter = document.getElementById('typeFilter');
     if (typeFilter) {
@@ -1613,7 +1621,6 @@ async function loadExistingContacts() {
     const emptyEl = document.getElementById('existingEmpty');
     const listEl = document.getElementById('existingList');
     const errorEl = document.getElementById('existingError');
-    const counterEl = document.getElementById('contactsCounter');
 
     // Show loading state
     if (loadingEl) loadingEl.style.display = 'block';
@@ -1622,30 +1629,53 @@ async function loadExistingContacts() {
     if (errorEl) errorEl.style.display = 'none';
 
     try {
-        const response = await fetch('/api/contacts/detailed');
-        const data = await response.json();
+        // Fetch device contacts and cached contacts in parallel
+        const [deviceResponse, cacheResponse] = await Promise.all([
+            fetch('/api/contacts/detailed'),
+            fetch('/api/contacts/cached?format=full')
+        ]);
+        const deviceData = await deviceResponse.json();
+        const cacheData = await cacheResponse.json();
 
         if (loadingEl) loadingEl.style.display = 'none';
 
-        if (data.success) {
-            existingContacts = data.contacts || [];
+        if (deviceData.success) {
+            const deviceContacts = deviceData.contacts || [];
+            const cachedContacts = (cacheData.success && cacheData.contacts) ? cacheData.contacts : [];
+
+            // Mark device contacts
+            const deviceKeySet = new Set(deviceContacts.map(c => c.public_key));
+            deviceContacts.forEach(c => { c.on_device = true; });
+
+            // Add cache-only contacts (not on device)
+            const cacheOnlyContacts = cachedContacts
+                .filter(c => !deviceKeySet.has(c.public_key))
+                .map(c => ({
+                    name: c.name || 'Unknown',
+                    public_key: c.public_key,
+                    public_key_prefix: c.public_key_prefix || c.public_key.substring(0, 12),
+                    type_label: '',
+                    last_seen: c.last_seen || 0,
+                    on_device: false,
+                    source: c.source || 'cache'
+                }));
+
+            existingContacts = [...deviceContacts, ...cacheOnlyContacts];
             filteredContacts = [...existingContacts];
 
-            // Update counter badge (in navbar)
-            updateCounter(data.count, data.limit);
+            // Update counter badge
+            updateCounter(deviceData.count, deviceData.limit, cachedContacts.length);
 
             if (existingContacts.length === 0) {
-                // Show empty state
                 if (emptyEl) emptyEl.style.display = 'block';
             } else {
-                // Apply filters and sort
                 applySortAndFilters();
             }
         } else {
-            console.error('Failed to load existing contacts:', data.error);
+            console.error('Failed to load existing contacts:', deviceData.error);
             if (errorEl) {
                 const errorMsg = document.getElementById('existingErrorMessage');
-                if (errorMsg) errorMsg.textContent = data.error || 'Failed to load contacts';
+                if (errorMsg) errorMsg.textContent = deviceData.error || 'Failed to load contacts';
                 errorEl.style.display = 'block';
             }
         }
@@ -1660,11 +1690,15 @@ async function loadExistingContacts() {
     }
 }
 
-function updateCounter(count, limit) {
+function updateCounter(count, limit, totalKnown) {
     const counterEl = document.getElementById('contactsCounter');
     if (!counterEl) return;
 
-    counterEl.textContent = `${count} / ${limit}`;
+    let text = `${count} / ${limit}`;
+    if (totalKnown && totalKnown > count) {
+        text += ` (${totalKnown} known)`;
+    }
+    counterEl.textContent = text;
     counterEl.style.display = 'inline-block';
 
     // Remove all counter classes
@@ -1752,21 +1786,29 @@ function updateSortUI() {
 function applySortAndFilters() {
     const searchInput = document.getElementById('searchInput');
     const typeFilter = document.getElementById('typeFilter');
+    const sourceFilter = document.getElementById('sourceFilter');
 
     const searchTerm = searchInput ? searchInput.value.toLowerCase() : '';
     const selectedType = typeFilter ? typeFilter.value : 'ALL';
+    const selectedSource = sourceFilter ? sourceFilter.value : 'ALL';
 
     // First, filter contacts
     filteredContacts = existingContacts.filter(contact => {
-        // Type filter
-        if (selectedType !== 'ALL' && contact.type_label !== selectedType) {
-            return false;
+        // Source filter
+        if (selectedSource === 'DEVICE' && !contact.on_device) return false;
+        if (selectedSource === 'CACHE' && contact.on_device) return false;
+
+        // Type filter (cache-only contacts have no type_label)
+        if (selectedType !== 'ALL') {
+            if (!contact.type_label || contact.type_label !== selectedType) {
+                return false;
+            }
         }
 
         // Search filter (name or public_key_prefix)
         if (searchTerm) {
             const nameMatch = contact.name.toLowerCase().includes(searchTerm);
-            const keyMatch = contact.public_key_prefix.toLowerCase().includes(searchTerm);
+            const keyMatch = (contact.public_key_prefix || '').toLowerCase().includes(searchTerm);
             return nameMatch || keyMatch;
         }
 
@@ -1922,31 +1964,41 @@ function createExistingContactCard(contact, index) {
         nameDiv.appendChild(lockIndicator);
     }
 
-    const typeBadge = document.createElement('span');
-    typeBadge.className = 'badge type-badge';
-    typeBadge.textContent = contact.type_label;
+    // Type badge (or "Cache only" badge for non-device contacts)
+    if (contact.on_device === false) {
+        const cacheBadge = document.createElement('span');
+        cacheBadge.className = 'badge type-badge bg-secondary';
+        cacheBadge.textContent = 'Cache';
+        cacheBadge.title = 'Not on device - known from adverts';
+        infoRow.appendChild(nameDiv);
+        infoRow.appendChild(cacheBadge);
+    } else {
+        const typeBadge = document.createElement('span');
+        typeBadge.className = 'badge type-badge';
+        typeBadge.textContent = contact.type_label;
 
-    // Color-code by type
-    switch (contact.type_label) {
-        case 'CLI':
-            typeBadge.classList.add('bg-primary');
-            break;
-        case 'REP':
-            typeBadge.classList.add('bg-success');
-            break;
-        case 'ROOM':
-            typeBadge.classList.add('bg-info');
-            break;
-        case 'SENS':
-            typeBadge.classList.add('bg-warning');
-            break;
-        default:
-            typeBadge.classList.add('bg-secondary');
+        // Color-code by type
+        switch (contact.type_label) {
+            case 'CLI':
+                typeBadge.classList.add('bg-primary');
+                break;
+            case 'REP':
+                typeBadge.classList.add('bg-success');
+                break;
+            case 'ROOM':
+                typeBadge.classList.add('bg-info');
+                break;
+            case 'SENS':
+                typeBadge.classList.add('bg-warning');
+                break;
+            default:
+                typeBadge.classList.add('bg-secondary');
+        }
+
+        infoRow.appendChild(nameDiv);
+        infoRow.appendChild(typeBadge);
     }
 
-    infoRow.appendChild(nameDiv);
-    infoRow.appendChild(typeBadge);
-
     // Public key row (clickable to copy)
     const keyDiv = document.createElement('div');
     keyDiv.className = 'contact-key clickable-key';
@@ -1994,40 +2046,42 @@ function createExistingContactCard(contact, index) {
         pathDiv.textContent = `Path: ${contact.path_or_mode}`;
     }
 
-    // Action buttons
+    // Action buttons (only for device contacts)
     const actionsDiv = document.createElement('div');
     actionsDiv.className = 'd-flex gap-2 mt-2';
 
-    // Map button (only if GPS coordinates available)
-    if (contact.adv_lat && contact.adv_lon && (contact.adv_lat !== 0 || contact.adv_lon !== 0)) {
-        const mapBtn = document.createElement('button');
-        mapBtn.className = 'btn btn-sm btn-outline-primary';
-        mapBtn.innerHTML = '<i class="bi bi-geo-alt"></i> Map';
-        mapBtn.onclick = () => window.showContactOnMap(contact.name, contact.adv_lat, contact.adv_lon);
-        actionsDiv.appendChild(mapBtn);
+    if (contact.on_device !== false) {
+        // Map button (only if GPS coordinates available)
+        if (contact.adv_lat && contact.adv_lon && (contact.adv_lat !== 0 || contact.adv_lon !== 0)) {
+            const mapBtn = document.createElement('button');
+            mapBtn.className = 'btn btn-sm btn-outline-primary';
+            mapBtn.innerHTML = '<i class="bi bi-geo-alt"></i> Map';
+            mapBtn.onclick = () => window.showContactOnMap(contact.name, contact.adv_lat, contact.adv_lon);
+            actionsDiv.appendChild(mapBtn);
+        }
+
+        // Protect button
+        const protectBtn = document.createElement('button');
+        protectBtn.className = isProtected ? 'btn btn-sm btn-warning' : 'btn btn-sm btn-outline-warning';
+        protectBtn.innerHTML = isProtected
+            ? '<i class="bi bi-lock-fill"></i> Protected'
+            : '<i class="bi bi-shield"></i> Protect';
+        protectBtn.onclick = () => toggleContactProtection(contact.public_key, protectBtn);
+        actionsDiv.appendChild(protectBtn);
+
+        // Delete button (disabled if protected)
+        const deleteBtn = document.createElement('button');
+        deleteBtn.className = 'btn btn-sm btn-outline-danger';
+        deleteBtn.innerHTML = '<i class="bi bi-trash"></i> Delete';
+        deleteBtn.onclick = () => showDeleteModal(contact);
+        deleteBtn.disabled = isProtected;
+        if (isProtected) {
+            deleteBtn.title = 'Cannot delete protected contact';
+        }
+
+        actionsDiv.appendChild(deleteBtn);
     }
 
-    // Protect button
-    const protectBtn = document.createElement('button');
-    protectBtn.className = isProtected ? 'btn btn-sm btn-warning' : 'btn btn-sm btn-outline-warning';
-    protectBtn.innerHTML = isProtected
-        ? '<i class="bi bi-lock-fill"></i> Protected'
-        : '<i class="bi bi-shield"></i> Protect';
-    protectBtn.onclick = () => toggleContactProtection(contact.public_key, protectBtn);
-    actionsDiv.appendChild(protectBtn);
-
-    // Delete button (disabled if protected)
-    const deleteBtn = document.createElement('button');
-    deleteBtn.className = 'btn btn-sm btn-outline-danger';
-    deleteBtn.innerHTML = '<i class="bi bi-trash"></i> Delete';
-    deleteBtn.onclick = () => showDeleteModal(contact);
-    deleteBtn.disabled = isProtected;
-    if (isProtected) {
-        deleteBtn.title = 'Cannot delete protected contact';
-    }
-
-    actionsDiv.appendChild(deleteBtn);
-
     // Assemble card
     card.appendChild(infoRow);
     card.appendChild(keyDiv);
diff --git a/app/templates/contacts-existing.html b/app/templates/contacts-existing.html
index be7ad92..a5e3acb 100644
--- a/app/templates/contacts-existing.html
+++ b/app/templates/contacts-existing.html
@@ -29,6 +29,13 @@
 
     <!-- Filter and Sort Toolbar -->
     <div class="filter-sort-toolbar">
+        <!-- Source Filter -->
+        <select class="form-select" id="sourceFilter">
+            <option value="ALL">All sources</option>
+            <option value="DEVICE">On device</option>
+            <option value="CACHE">Cache only</option>
+        </select>
+
         <!-- Type Filter -->
         <select class="form-select" id="typeFilter">
             <option value="ALL">All Types</option>