diff --git a/app/contacts_cache.py b/app/contacts_cache.py new file mode 100644 index 0000000..cb1d8fa --- /dev/null +++ b/app/contacts_cache.py @@ -0,0 +1,237 @@ +""" +Contacts Cache - Persistent storage of all known node names + public keys. + +Stores every node name ever seen (from device contacts and adverts), +so @mention autocomplete works even for removed contacts. + +File format: JSONL ({device_name}.contacts_cache.jsonl) +Each line: {"public_key": "...", "name": "...", "first_seen": ts, "last_seen": ts, "source": "advert"|"device"} +""" + +import json +import logging +import time +from pathlib import Path +from threading import Lock + +from app.config import config, runtime_config + +logger = logging.getLogger(__name__) + +_cache_lock = Lock() +_cache: dict = {} # {public_key: {name, first_seen, last_seen, source}} +_cache_loaded = False +_adverts_offset = 0 # File offset for incremental advert scanning + + +def _get_cache_path() -> Path: + device_name = runtime_config.get_device_name() + return Path(config.MC_CONFIG_DIR) / f"{device_name}.contacts_cache.jsonl" + + +def _get_adverts_path() -> Path: + device_name = runtime_config.get_device_name() + return Path(config.MC_CONFIG_DIR) / f"{device_name}.adverts.jsonl" + + +def load_cache() -> dict: + """Load cache from disk into memory. Returns copy of cache dict.""" + global _cache, _cache_loaded + + with _cache_lock: + if _cache_loaded: + return _cache.copy() + + cache_path = _get_cache_path() + _cache = {} + + if not cache_path.exists(): + _cache_loaded = True + logger.info("Contacts cache file does not exist yet") + return _cache.copy() + + try: + with open(cache_path, 'r', encoding='utf-8') as f: + for line in f: + line = line.strip() + if not line: + continue + try: + entry = json.loads(line) + pk = entry.get('public_key', '').lower() + if pk: + _cache[pk] = entry + except json.JSONDecodeError: + continue + + _cache_loaded = True + logger.info(f"Loaded contacts cache: {len(_cache)} entries") + except Exception as e: + logger.error(f"Failed to load contacts cache: {e}") + _cache_loaded = True + + return _cache.copy() + + +def save_cache() -> bool: + """Write full cache to disk (atomic write).""" + with _cache_lock: + cache_path = _get_cache_path() + try: + cache_path.parent.mkdir(parents=True, exist_ok=True) + temp_file = cache_path.with_suffix('.tmp') + with open(temp_file, 'w', encoding='utf-8') as f: + for entry in _cache.values(): + f.write(json.dumps(entry, ensure_ascii=False) + '\n') + temp_file.replace(cache_path) + logger.debug(f"Saved contacts cache: {len(_cache)} entries") + return True + except Exception as e: + logger.error(f"Failed to save contacts cache: {e}") + return False + + +def upsert_contact(public_key: str, name: str, source: str = "advert") -> bool: + """Add or update a contact in the cache. Returns True if cache was modified.""" + pk = public_key.lower() + now = int(time.time()) + + with _cache_lock: + existing = _cache.get(pk) + if existing: + changed = False + if name and name != existing.get('name'): + existing['name'] = name + changed = True + existing['last_seen'] = now + return changed + else: + if not name: + return False + _cache[pk] = { + 'public_key': pk, + 'name': name, + 'first_seen': now, + 'last_seen': now, + 'source': source, + } + return True + + +def get_all_contacts() -> list: + """Get all cached contacts as a list of dicts (shallow copies).""" + with _cache_lock: + return [entry.copy() for entry in _cache.values()] + + +def get_all_names() -> list: + """Get all unique non-empty contact names sorted alphabetically.""" + with _cache_lock: + return sorted(set( + entry['name'] for entry in _cache.values() + if entry.get('name') + )) + + +def parse_advert_payload(pkt_payload_hex: str): + """ + Parse advert pkt_payload to extract public_key and node_name. + + Layout of pkt_payload (byte offsets): + [0:32] Public Key (32 bytes = 64 hex chars) + [32:36] Timestamp (4 bytes) + [36:100] Signature (64 bytes) + [100] App Flags (1 byte) - bit 4: Location, bit 7: Name + [101+] If Location (bit 4): Lat (4 bytes) + Lon (4 bytes) + If Name (bit 7): Node name (UTF-8, variable length) + + Returns: + (public_key_hex, node_name) or (None, None) on failure + """ + try: + raw = bytes.fromhex(pkt_payload_hex) + if len(raw) < 101: + return None, None + + public_key = pkt_payload_hex[:64].lower() + app_flags = raw[100] + + has_location = bool(app_flags & 0x10) # bit 4 + has_name = bool(app_flags & 0x80) # bit 7 + + if not has_name: + return public_key, None + + name_offset = 101 + if has_location: + name_offset += 8 # lat(4) + lon(4) + + if name_offset >= len(raw): + return public_key, None + + name_bytes = raw[name_offset:] + node_name = name_bytes.decode('utf-8', errors='replace').rstrip('\x00') + + return public_key, node_name if node_name else None + except Exception: + return None, None + + +def scan_new_adverts() -> int: + """ + Scan .adverts.jsonl for new entries since last scan. + Returns number of new/updated contacts. + """ + global _adverts_offset + + adverts_path = _get_adverts_path() + if not adverts_path.exists(): + return 0 + + updated = 0 + try: + with open(adverts_path, 'r', encoding='utf-8') as f: + f.seek(_adverts_offset) + for line in f: + line = line.strip() + if not line: + continue + try: + advert = json.loads(line) + pkt_payload = advert.get('pkt_payload', '') + if not pkt_payload: + continue + pk, name = parse_advert_payload(pkt_payload) + if pk and name: + if upsert_contact(pk, name, source="advert"): + updated += 1 + except json.JSONDecodeError: + continue + _adverts_offset = f.tell() + except Exception as e: + logger.error(f"Failed to scan adverts: {e}") + + if updated > 0: + save_cache() + logger.info(f"Contacts cache updated: {updated} new/changed entries") + + return updated + + +def initialize_from_device(contacts_detailed: dict): + """ + Seed cache from /api/contacts/detailed response dict. + Called once at startup if cache file doesn't exist. + + Args: + contacts_detailed: dict of {public_key: {adv_name, type, ...}} from meshcli + """ + added = 0 + for pk, details in contacts_detailed.items(): + name = details.get('adv_name', '') + if upsert_contact(pk, name, source="device"): + added += 1 + + if added > 0: + save_cache() + logger.info(f"Initialized contacts cache from device: {added} contacts") diff --git a/app/main.py b/app/main.py index 2b2ce3c..af3d851 100644 --- a/app/main.py +++ b/app/main.py @@ -16,6 +16,7 @@ from app.routes.api import api_bp from app.version import VERSION_STRING, GIT_BRANCH from app.archiver.manager import schedule_daily_archiving from app.meshcore.cli import fetch_device_name_from_bridge +from app.contacts_cache import load_cache, scan_new_adverts, initialize_from_device # Commands that require longer timeout (in seconds) SLOW_COMMANDS = { @@ -108,6 +109,34 @@ def create_app(): threading.Thread(target=init_device_name, daemon=True).start() + # Background thread: contacts cache initialization and periodic advert scanning + def init_contacts_cache(): + # Wait for device name to resolve + time.sleep(10) + + cache = load_cache() + + # Seed from device contacts if cache is empty + if not cache: + try: + from app.routes.api import get_contacts_detailed_cached + success, contacts, error = get_contacts_detailed_cached() + if success and contacts: + initialize_from_device(contacts) + logger.info("Contacts cache seeded from device") + except Exception as e: + logger.error(f"Failed to seed contacts cache: {e}") + + # Periodic advert scan loop + while True: + time.sleep(45) + try: + scan_new_adverts() + except Exception as e: + logger.error(f"Contacts cache scan error: {e}") + + threading.Thread(target=init_contacts_cache, daemon=True).start() + logger.info(f"mc-webui started - device: {config.MC_DEVICE_NAME}") logger.info(f"Messages file: {config.msgs_file_path}") logger.info(f"Serial port: {config.MC_SERIAL_PORT}") diff --git a/app/routes/api.py b/app/routes/api.py index ad4dd64..54299d7 100644 --- a/app/routes/api.py +++ b/app/routes/api.py @@ -19,6 +19,7 @@ from flask import Blueprint, jsonify, request, send_file from app.meshcore import cli, parser from app.config import config, runtime_config from app.archiver import manager as archive_manager +from app.contacts_cache import get_all_names, get_all_contacts logger = logging.getLogger(__name__) @@ -569,6 +570,46 @@ def get_contacts(): }), 500 +@api_bp.route('/contacts/cached', methods=['GET']) +def get_cached_contacts(): + """ + Get all known contacts from persistent cache (superset of device contacts). + Includes contacts seen via adverts even after removal from device. + + Query params: + ?format=names - Return just name strings for @mentions (default) + ?format=full - Return full cache entries with public_key, timestamps, etc. + """ + try: + fmt = request.args.get('format', 'names') + + if fmt == 'full': + contacts = get_all_contacts() + # Add public_key_prefix for display + for c in contacts: + c['public_key_prefix'] = c.get('public_key', '')[:12] + return jsonify({ + 'success': True, + 'contacts': contacts, + 'count': len(contacts) + }), 200 + else: + names = get_all_names() + return jsonify({ + 'success': True, + 'contacts': names, + 'count': len(names) + }), 200 + + except Exception as e: + logger.error(f"Error getting cached contacts: {e}") + return jsonify({ + 'success': False, + 'error': str(e), + 'contacts': [] + }), 500 + + def _filter_contacts_by_criteria(contacts: list, criteria: dict) -> list: """ Filter contacts based on cleanup criteria. diff --git a/app/static/js/app.js b/app/static/js/app.js index f28b64c..e9eed45 100644 --- a/app/static/js/app.js +++ b/app/static/js/app.js @@ -2795,13 +2795,13 @@ async function loadContactsForMentions() { } try { - const response = await fetch('/api/contacts'); + const response = await fetch('/api/contacts/cached'); const data = await response.json(); if (data.success && data.contacts) { mentionsCache = data.contacts; mentionsCacheTimestamp = now; - console.log(`[mentions] Cached ${mentionsCache.length} contacts`); + console.log(`[mentions] Cached ${mentionsCache.length} contacts from cache`); } } catch (error) { console.error('[mentions] Error loading contacts:', error); diff --git a/app/static/js/contacts.js b/app/static/js/contacts.js index e9072df..734f066 100644 --- a/app/static/js/contacts.js +++ b/app/static/js/contacts.js @@ -839,6 +839,14 @@ function attachExistingEventListeners() { }); } + // Source filter (device / cache only) + const sourceFilter = document.getElementById('sourceFilter'); + if (sourceFilter) { + sourceFilter.addEventListener('change', () => { + applySortAndFilters(); + }); + } + // Type filter const typeFilter = document.getElementById('typeFilter'); if (typeFilter) { @@ -1613,7 +1621,6 @@ async function loadExistingContacts() { const emptyEl = document.getElementById('existingEmpty'); const listEl = document.getElementById('existingList'); const errorEl = document.getElementById('existingError'); - const counterEl = document.getElementById('contactsCounter'); // Show loading state if (loadingEl) loadingEl.style.display = 'block'; @@ -1622,30 +1629,53 @@ async function loadExistingContacts() { if (errorEl) errorEl.style.display = 'none'; try { - const response = await fetch('/api/contacts/detailed'); - const data = await response.json(); + // Fetch device contacts and cached contacts in parallel + const [deviceResponse, cacheResponse] = await Promise.all([ + fetch('/api/contacts/detailed'), + fetch('/api/contacts/cached?format=full') + ]); + const deviceData = await deviceResponse.json(); + const cacheData = await cacheResponse.json(); if (loadingEl) loadingEl.style.display = 'none'; - if (data.success) { - existingContacts = data.contacts || []; + if (deviceData.success) { + const deviceContacts = deviceData.contacts || []; + const cachedContacts = (cacheData.success && cacheData.contacts) ? cacheData.contacts : []; + + // Mark device contacts + const deviceKeySet = new Set(deviceContacts.map(c => c.public_key)); + deviceContacts.forEach(c => { c.on_device = true; }); + + // Add cache-only contacts (not on device) + const cacheOnlyContacts = cachedContacts + .filter(c => !deviceKeySet.has(c.public_key)) + .map(c => ({ + name: c.name || 'Unknown', + public_key: c.public_key, + public_key_prefix: c.public_key_prefix || c.public_key.substring(0, 12), + type_label: '', + last_seen: c.last_seen || 0, + on_device: false, + source: c.source || 'cache' + })); + + existingContacts = [...deviceContacts, ...cacheOnlyContacts]; filteredContacts = [...existingContacts]; - // Update counter badge (in navbar) - updateCounter(data.count, data.limit); + // Update counter badge + updateCounter(deviceData.count, deviceData.limit, cachedContacts.length); if (existingContacts.length === 0) { - // Show empty state if (emptyEl) emptyEl.style.display = 'block'; } else { - // Apply filters and sort applySortAndFilters(); } } else { - console.error('Failed to load existing contacts:', data.error); + console.error('Failed to load existing contacts:', deviceData.error); if (errorEl) { const errorMsg = document.getElementById('existingErrorMessage'); - if (errorMsg) errorMsg.textContent = data.error || 'Failed to load contacts'; + if (errorMsg) errorMsg.textContent = deviceData.error || 'Failed to load contacts'; errorEl.style.display = 'block'; } } @@ -1660,11 +1690,15 @@ async function loadExistingContacts() { } } -function updateCounter(count, limit) { +function updateCounter(count, limit, totalKnown) { const counterEl = document.getElementById('contactsCounter'); if (!counterEl) return; - counterEl.textContent = `${count} / ${limit}`; + let text = `${count} / ${limit}`; + if (totalKnown && totalKnown > count) { + text += ` (${totalKnown} known)`; + } + counterEl.textContent = text; counterEl.style.display = 'inline-block'; // Remove all counter classes @@ -1752,21 +1786,29 @@ function updateSortUI() { function applySortAndFilters() { const searchInput = document.getElementById('searchInput'); const typeFilter = document.getElementById('typeFilter'); + const sourceFilter = document.getElementById('sourceFilter'); const searchTerm = searchInput ? searchInput.value.toLowerCase() : ''; const selectedType = typeFilter ? typeFilter.value : 'ALL'; + const selectedSource = sourceFilter ? sourceFilter.value : 'ALL'; // First, filter contacts filteredContacts = existingContacts.filter(contact => { - // Type filter - if (selectedType !== 'ALL' && contact.type_label !== selectedType) { - return false; + // Source filter + if (selectedSource === 'DEVICE' && !contact.on_device) return false; + if (selectedSource === 'CACHE' && contact.on_device) return false; + + // Type filter (cache-only contacts have no type_label) + if (selectedType !== 'ALL') { + if (!contact.type_label || contact.type_label !== selectedType) { + return false; + } } // Search filter (name or public_key_prefix) if (searchTerm) { const nameMatch = contact.name.toLowerCase().includes(searchTerm); - const keyMatch = contact.public_key_prefix.toLowerCase().includes(searchTerm); + const keyMatch = (contact.public_key_prefix || '').toLowerCase().includes(searchTerm); return nameMatch || keyMatch; } @@ -1922,31 +1964,41 @@ function createExistingContactCard(contact, index) { nameDiv.appendChild(lockIndicator); } - const typeBadge = document.createElement('span'); - typeBadge.className = 'badge type-badge'; - typeBadge.textContent = contact.type_label; + // Type badge (or "Cache only" badge for non-device contacts) + if (contact.on_device === false) { + const cacheBadge = document.createElement('span'); + cacheBadge.className = 'badge type-badge bg-secondary'; + cacheBadge.textContent = 'Cache'; + cacheBadge.title = 'Not on device - known from adverts'; + infoRow.appendChild(nameDiv); + infoRow.appendChild(cacheBadge); + } else { + const typeBadge = document.createElement('span'); + typeBadge.className = 'badge type-badge'; + typeBadge.textContent = contact.type_label; - // Color-code by type - switch (contact.type_label) { - case 'CLI': - typeBadge.classList.add('bg-primary'); - break; - case 'REP': - typeBadge.classList.add('bg-success'); - break; - case 'ROOM': - typeBadge.classList.add('bg-info'); - break; - case 'SENS': - typeBadge.classList.add('bg-warning'); - break; - default: - typeBadge.classList.add('bg-secondary'); + // Color-code by type + switch (contact.type_label) { + case 'CLI': + typeBadge.classList.add('bg-primary'); + break; + case 'REP': + typeBadge.classList.add('bg-success'); + break; + case 'ROOM': + typeBadge.classList.add('bg-info'); + break; + case 'SENS': + typeBadge.classList.add('bg-warning'); + break; + default: + typeBadge.classList.add('bg-secondary'); + } + + infoRow.appendChild(nameDiv); + infoRow.appendChild(typeBadge); } - infoRow.appendChild(nameDiv); - infoRow.appendChild(typeBadge); - // Public key row (clickable to copy) const keyDiv = document.createElement('div'); keyDiv.className = 'contact-key clickable-key'; @@ -1994,40 +2046,42 @@ function createExistingContactCard(contact, index) { pathDiv.textContent = `Path: ${contact.path_or_mode}`; } - // Action buttons + // Action buttons (only for device contacts) const actionsDiv = document.createElement('div'); actionsDiv.className = 'd-flex gap-2 mt-2'; - // Map button (only if GPS coordinates available) - if (contact.adv_lat && contact.adv_lon && (contact.adv_lat !== 0 || contact.adv_lon !== 0)) { - const mapBtn = document.createElement('button'); - mapBtn.className = 'btn btn-sm btn-outline-primary'; - mapBtn.innerHTML = ' Map'; - mapBtn.onclick = () => window.showContactOnMap(contact.name, contact.adv_lat, contact.adv_lon); - actionsDiv.appendChild(mapBtn); + if (contact.on_device !== false) { + // Map button (only if GPS coordinates available) + if (contact.adv_lat && contact.adv_lon && (contact.adv_lat !== 0 || contact.adv_lon !== 0)) { + const mapBtn = document.createElement('button'); + mapBtn.className = 'btn btn-sm btn-outline-primary'; + mapBtn.innerHTML = ' Map'; + mapBtn.onclick = () => window.showContactOnMap(contact.name, contact.adv_lat, contact.adv_lon); + actionsDiv.appendChild(mapBtn); + } + + // Protect button + const protectBtn = document.createElement('button'); + protectBtn.className = isProtected ? 'btn btn-sm btn-warning' : 'btn btn-sm btn-outline-warning'; + protectBtn.innerHTML = isProtected + ? ' Protected' + : ' Protect'; + protectBtn.onclick = () => toggleContactProtection(contact.public_key, protectBtn); + actionsDiv.appendChild(protectBtn); + + // Delete button (disabled if protected) + const deleteBtn = document.createElement('button'); + deleteBtn.className = 'btn btn-sm btn-outline-danger'; + deleteBtn.innerHTML = ' Delete'; + deleteBtn.onclick = () => showDeleteModal(contact); + deleteBtn.disabled = isProtected; + if (isProtected) { + deleteBtn.title = 'Cannot delete protected contact'; + } + + actionsDiv.appendChild(deleteBtn); } - // Protect button - const protectBtn = document.createElement('button'); - protectBtn.className = isProtected ? 'btn btn-sm btn-warning' : 'btn btn-sm btn-outline-warning'; - protectBtn.innerHTML = isProtected - ? ' Protected' - : ' Protect'; - protectBtn.onclick = () => toggleContactProtection(contact.public_key, protectBtn); - actionsDiv.appendChild(protectBtn); - - // Delete button (disabled if protected) - const deleteBtn = document.createElement('button'); - deleteBtn.className = 'btn btn-sm btn-outline-danger'; - deleteBtn.innerHTML = ' Delete'; - deleteBtn.onclick = () => showDeleteModal(contact); - deleteBtn.disabled = isProtected; - if (isProtected) { - deleteBtn.title = 'Cannot delete protected contact'; - } - - actionsDiv.appendChild(deleteBtn); - // Assemble card card.appendChild(infoRow); card.appendChild(keyDiv); diff --git a/app/templates/contacts-existing.html b/app/templates/contacts-existing.html index be7ad92..a5e3acb 100644 --- a/app/templates/contacts-existing.html +++ b/app/templates/contacts-existing.html @@ -29,6 +29,13 @@