refactor: migrate .webui_settings.json to database + fix NEW_CONTACT edge case

All settings (protected_contacts, cleanup_settings, retention_settings,
manual_add_contacts) moved from .webui_settings.json file to SQLite database.
Startup migration auto-imports existing file and renames it to .json.bak.

Added safeguard in _on_new_contact: if firmware fires NEW_CONTACT for a
contact already on the device, skip pending and log a warning. Also added
diagnostic logging showing previous DB state (source, protected) when
contacts reappear as pending.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

app/database.py

@@ -286,6 +286,137 @@ class Database:
             ).fetchall()
             return {r['name'] for r in rows1} | {r['name'] for r in rows2}
 
+    # ================================================================
+    # Protected Contacts (DB-backed)
+    # ================================================================
+
+    def get_protected_keys(self) -> set:
+        """Return set of public_keys that are protected."""
+        with self._connect() as conn:
+            rows = conn.execute(
+                "SELECT public_key FROM contacts WHERE is_protected = 1"
+            ).fetchall()
+            return {r['public_key'] for r in rows}
+
+    # ================================================================
+    # App Settings (key-value store)
+    # ================================================================
+
+    def get_setting(self, key: str) -> Optional[str]:
+        """Get a setting value (JSON string) by key."""
+        with self._connect() as conn:
+            row = conn.execute(
+                "SELECT value FROM app_settings WHERE key = ?", (key,)
+            ).fetchone()
+            return row['value'] if row else None
+
+    def set_setting(self, key: str, value: str) -> None:
+        """Set a setting value (JSON string)."""
+        with self._connect() as conn:
+            conn.execute(
+                """INSERT INTO app_settings (key, value)
+                   VALUES (?, ?)
+                   ON CONFLICT(key) DO UPDATE SET
+                       value = excluded.value,
+                       updated_at = datetime('now')""",
+                (key, value)
+            )
+
+    def get_setting_json(self, key: str, default=None):
+        """Get a setting, JSON-decoded. Returns default if not found."""
+        import json
+        raw = self.get_setting(key)
+        if raw is None:
+            return default
+        try:
+            return json.loads(raw)
+        except (json.JSONDecodeError, TypeError):
+            return default
+
+    def set_setting_json(self, key: str, value) -> None:
+        """Set a setting, JSON-encoding the value."""
+        import json
+        self.set_setting(key, json.dumps(value, ensure_ascii=False))
+
+    def migrate_protected_contacts_from_file(self, settings_path) -> int:
+        """One-time migration: import protected_contacts from .webui_settings.json into DB.
+
+        Returns number of contacts marked as protected.
+        """
+        import json
+        from pathlib import Path
+        settings_path = Path(settings_path)
+        if not settings_path.exists():
+            return 0
+
+        try:
+            with open(settings_path, 'r', encoding='utf-8') as f:
+                settings = json.load(f)
+            protected = settings.get('protected_contacts', [])
+            if not protected:
+                return 0
+
+            count = 0
+            with self._connect() as conn:
+                for pk in protected:
+                    pk = pk.lower()
+                    cursor = conn.execute(
+                        "UPDATE contacts SET is_protected = 1, lastmod = datetime('now') WHERE public_key = ?",
+                        (pk,)
+                    )
+                    if cursor.rowcount > 0:
+                        count += 1
+                    else:
+                        # Contact not in DB yet - insert minimal record
+                        conn.execute(
+                            """INSERT OR IGNORE INTO contacts (public_key, name, is_protected, source)
+                               VALUES (?, '', 1, 'advert')""",
+                            (pk,)
+                        )
+                        count += 1
+
+            logger.info(f"Migrated {count} protected contacts from settings file to DB")
+            return count
+        except Exception as e:
+            logger.error(f"Failed to migrate protected contacts: {e}")
+            return 0
+
+    def migrate_settings_from_file(self, settings_path) -> bool:
+        """One-time migration: import cleanup/retention/manual_add settings from .webui_settings.json.
+
+        Returns True if migration was performed.
+        """
+        import json
+        from pathlib import Path
+        settings_path = Path(settings_path)
+        if not settings_path.exists():
+            return False
+
+        try:
+            with open(settings_path, 'r', encoding='utf-8') as f:
+                settings = json.load(f)
+
+            migrated = False
+
+            if 'cleanup_settings' in settings:
+                self.set_setting_json('cleanup_settings', settings['cleanup_settings'])
+                migrated = True
+
+            if 'retention_settings' in settings:
+                self.set_setting_json('retention_settings', settings['retention_settings'])
+                migrated = True
+
+            if 'manual_add_contacts' in settings:
+                self.set_setting_json('manual_add_contacts', settings['manual_add_contacts'])
+                migrated = True
+
+            if migrated:
+                logger.info("Migrated app settings from .webui_settings.json to DB")
+            return migrated
+        except Exception as e:
+            logger.error(f"Failed to migrate settings: {e}")
+            return False
+
     # ================================================================
     # Channels
     # ================================================================

app/device_manager.py

@@ -824,14 +824,9 @@ class DeviceManager:
                 }, namespace='/chat')
 
     def _is_manual_approval_enabled(self) -> bool:
-        """Check if manual contact approval is enabled (from persisted settings)."""
+        """Check if manual contact approval is enabled (from database)."""
         try:
-            from pathlib import Path
-            settings_path = Path(self.config.MC_CONFIG_DIR) / ".webui_settings.json"
-            if settings_path.exists():
-                with open(settings_path, 'r', encoding='utf-8') as f:
-                    settings = json.load(f)
-                    return bool(settings.get('manual_add_contacts', False))
+            return bool(self.db.get_setting_json('manual_add_contacts', False))
         except Exception:
             pass
         return False
@@ -871,8 +866,44 @@ class DeviceManager:
                 return
 
             if self._is_manual_approval_enabled():
+                # Check if contact already exists on the device (firmware edge case:
+                # firmware may fire NEW_CONTACT for a contact that was previously
+                # on the device but got removed by firmware-level cleanup)
+                if pubkey in (self.mc.contacts or {}):
+                    logger.warning(
+                        f"NEW_CONTACT fired for contact already on device: {name} ({pubkey[:8]}...) "
+                        f"— skipping pending, updating DB cache only"
+                    )
+                    # Just update cache, don't add to pending
+                    last_adv = data.get('last_advert')
+                    last_advert_val = (
+                        str(int(last_adv))
+                        if last_adv and isinstance(last_adv, (int, float)) and last_adv > 0
+                        else str(int(time.time()))
+                    )
+                    self.db.upsert_contact(
+                        public_key=pubkey,
+                        name=name,
+                        type=data.get('type', data.get('adv_type', 0)),
+                        adv_lat=data.get('adv_lat'),
+                        adv_lon=data.get('adv_lon'),
+                        last_advert=last_advert_val,
+                        source='device',
+                    )
+                    return
+
+                # Check if contact was previously known (in DB cache)
+                existing = self.db.get_contact(pubkey)
+                if existing:
+                    logger.info(
+                        f"Pending contact (manual mode): {name} ({pubkey[:8]}...) "
+                        f"— previously known (source={existing['source']}, "
+                        f"protected={existing['is_protected']})"
+                    )
+                else:
+                    logger.info(f"Pending contact (manual mode): {name} ({pubkey[:8]}...) — first time seen")
+
                 # Manual mode: meshcore puts it in mc.pending_contacts for approval
-                logger.info(f"Pending contact (manual mode): {name} ({pubkey[:8]}...)")
 
                 # Also add to DB cache for @mentions and Cache filter
                 last_adv = data.get('last_advert')

app/main.py

@@ -74,6 +74,21 @@ def create_app():
     db = Database(config.db_path)
     app.db = db
 
+    # Migrate settings from .webui_settings.json to DB (one-time)
+    from pathlib import Path
+    settings_file = Path(config.MC_CONFIG_DIR) / ".webui_settings.json"
+    if settings_file.exists() and db.get_setting('manual_add_contacts') is None:
+        logger.info("Migrating settings from .webui_settings.json to database...")
+        db.migrate_protected_contacts_from_file(settings_file)
+        db.migrate_settings_from_file(settings_file)
+        # Rename old file as backup
+        backup = settings_file.with_suffix('.json.bak')
+        try:
+            settings_file.rename(backup)
+            logger.info(f"Settings file backed up to {backup.name}")
+        except Exception as e:
+            logger.warning(f"Could not rename settings file: {e}")
+
     # v2: Initialize and start device manager
     device_manager = DeviceManager(config, db, socketio)
     app.device_manager = device_manager

app/meshcore/cli.py

@@ -5,8 +5,6 @@ Function signatures preserved for backward compatibility with api.py.
 """
 
 import logging
-import json
-from pathlib import Path
 from typing import Tuple, Optional, List, Dict
 from app.config import config
 
@@ -34,6 +32,22 @@ def _get_dm():
     return device_manager
 
 
+def _get_db():
+    """Get Database instance — try Flask app context first, then DeviceManager."""
+    try:
+        from flask import current_app
+        db = getattr(current_app, 'db', None)
+        if db is not None:
+            return db
+    except RuntimeError:
+        pass
+    try:
+        dm = _get_dm()
+        return dm.db
+    except Exception:
+        return None
+
+
 # =============================================================================
 # Messages
 # =============================================================================
@@ -454,16 +468,11 @@ def set_auto_retry_config(enabled=None, max_attempts=None, max_flood=None) -> Tu
 # =============================================================================
 
 def get_device_settings() -> Tuple[bool, Dict]:
-    """Get persistent device settings."""
-    settings_path = Path(config.MC_CONFIG_DIR) / ".webui_settings.json"
+    """Get persistent device settings from database."""
     try:
-        if not settings_path.exists():
-            return True, {'manual_add_contacts': False}
-        with open(settings_path, 'r', encoding='utf-8') as f:
-            settings = json.load(f)
-            if 'manual_add_contacts' not in settings:
-                settings['manual_add_contacts'] = False
-            return True, settings
+        db = _get_db()
+        manual = db.get_setting_json('manual_add_contacts', False) if db else False
+        return True, {'manual_add_contacts': manual}
     except Exception as e:
         logger.error(f"Failed to read device settings: {e}")
         return False, {'manual_add_contacts': False}
@@ -475,19 +484,10 @@ def set_manual_add_contacts(enabled: bool) -> Tuple[bool, str]:
         dm_inst = _get_dm()
         result = dm_inst.set_manual_add_contacts(enabled)
         if result['success']:
-            # Persist to settings file
-            settings_path = Path(config.MC_CONFIG_DIR) / ".webui_settings.json"
-            try:
-                settings = {}
-                if settings_path.exists():
-                    with open(settings_path, 'r') as f:
-                        settings = json.load(f)
-                settings['manual_add_contacts'] = enabled
-                settings_path.parent.mkdir(parents=True, exist_ok=True)
-                with open(settings_path, 'w') as f:
-                    json.dump(settings, f)
-            except Exception as e:
-                logger.warning(f"Failed to persist settings: {e}")
+            # Persist to database
+            db = _get_db()
+            if db:
+                db.set_setting_json('manual_add_contacts', enabled)
         return result['success'], result.get('message', result.get('error', ''))
     except Exception as e:
         return False, str(e)

app/routes/api.py

@@ -180,136 +180,55 @@ def invalidate_contacts_cache():
 
 def get_protected_contacts() -> list:
     """
-    Get list of protected contact public keys from settings.
+    Get list of protected contact public keys from database.
 
     Returns:
         List of public_key strings (64 hex chars, lowercase)
     """
-    from pathlib import Path
-    settings_path = Path(config.MC_CONFIG_DIR) / ".webui_settings.json"
-
-    try:
-        if not settings_path.exists():
-            return []
-
-        with open(settings_path, 'r', encoding='utf-8') as f:
-            settings = json.load(f)
-            # Return lowercase keys for consistent comparison
-            return [pk.lower() for pk in settings.get('protected_contacts', [])]
-    except Exception as e:
-        logger.error(f"Failed to read protected contacts: {e}")
-        return []
-
-
-def save_protected_contacts(protected_list: list) -> bool:
-    """
-    Save protected contacts list to settings file (atomic write).
-
-    Args:
-        protected_list: List of public_key strings
-
-    Returns:
-        True if successful, False otherwise
-    """
-    from pathlib import Path
-    settings_path = Path(config.MC_CONFIG_DIR) / ".webui_settings.json"
-
-    try:
-        # Read existing settings
-        settings = {}
-        if settings_path.exists():
-            with open(settings_path, 'r', encoding='utf-8') as f:
-                settings = json.load(f)
-
-        # Update protected contacts (store lowercase)
-        settings['protected_contacts'] = [pk.lower() for pk in protected_list]
-
-        # Write back atomically
-        temp_file = settings_path.with_suffix('.tmp')
-        with open(temp_file, 'w', encoding='utf-8') as f:
-            json.dump(settings, f, indent=2, ensure_ascii=False)
-        temp_file.replace(settings_path)
-
-        return True
-    except Exception as e:
-        logger.error(f"Failed to save protected contacts: {e}")
-        return False
+    db = _get_db()
+    if db:
+        return list(db.get_protected_keys())
+    return []
 
 
 # =============================================================================
 # Cleanup Settings Management
 # =============================================================================
 
+CLEANUP_DEFAULTS = {
+    'enabled': False,
+    'types': [1, 2, 3, 4],
+    'date_field': 'last_advert',
+    'days': 30,
+    'name_filter': '',
+    'hour': 1
+}
+
+RETENTION_DEFAULTS = {
+    'enabled': False,
+    'days': 90,
+    'include_dms': False,
+    'include_adverts': False,
+    'hour': 2
+}
+
+
 def get_cleanup_settings() -> dict:
-    """
-    Get auto-cleanup settings from .webui_settings.json.
-
-    Returns:
-        Dict with cleanup settings:
-        {
-            'enabled': bool,
-            'types': list[int],
-            'date_field': str,
-            'days': int,
-            'name_filter': str,
-            'hour': int (0-23, UTC)
-        }
-    """
-    from pathlib import Path
-    defaults = {
-        'enabled': False,
-        'types': [1, 2, 3, 4],
-        'date_field': 'last_advert',
-        'days': 30,
-        'name_filter': '',
-        'hour': 1
-    }
-
-    settings_path = Path(config.MC_CONFIG_DIR) / ".webui_settings.json"
-
-    try:
-        if not settings_path.exists():
-            return defaults
-
-        with open(settings_path, 'r', encoding='utf-8') as f:
-            settings = json.load(f)
-            cleanup = settings.get('cleanup_settings', {})
-            # Merge with defaults to ensure all fields exist
-            return {**defaults, **cleanup}
-    except Exception as e:
-        logger.error(f"Failed to read cleanup settings: {e}")
-        return defaults
+    """Get auto-cleanup settings from database."""
+    db = _get_db()
+    if db:
+        saved = db.get_setting_json('cleanup_settings', {})
+        return {**CLEANUP_DEFAULTS, **saved}
+    return dict(CLEANUP_DEFAULTS)
 
 
 def save_cleanup_settings(cleanup_settings: dict) -> bool:
-    """
-    Save auto-cleanup settings to .webui_settings.json (atomic write).
-
-    Args:
-        cleanup_settings: Dict with cleanup configuration
-
-    Returns:
-        True if successful, False otherwise
-    """
-    from pathlib import Path
-    settings_path = Path(config.MC_CONFIG_DIR) / ".webui_settings.json"
-
+    """Save auto-cleanup settings to database."""
+    db = _get_db()
+    if not db:
+        return False
     try:
-        # Read existing settings
-        settings = {}
-        if settings_path.exists():
-            with open(settings_path, 'r', encoding='utf-8') as f:
-                settings = json.load(f)
-
-        # Update cleanup settings
-        settings['cleanup_settings'] = cleanup_settings
-
-        # Write back atomically
-        temp_file = settings_path.with_suffix('.tmp')
-        with open(temp_file, 'w', encoding='utf-8') as f:
-            json.dump(settings, f, indent=2, ensure_ascii=False)
-        temp_file.replace(settings_path)
-
+        db.set_setting_json('cleanup_settings', cleanup_settings)
         return True
     except Exception as e:
         logger.error(f"Failed to save cleanup settings: {e}")
@@ -317,49 +236,21 @@ def save_cleanup_settings(cleanup_settings: dict) -> bool:
 
 
 def get_retention_settings() -> dict:
-    """Get message retention settings from .webui_settings.json."""
-    from pathlib import Path
-    defaults = {
-        'enabled': False,
-        'days': 90,
-        'include_dms': False,
-        'include_adverts': False,
-        'hour': 2
-    }
-
-    settings_path = Path(config.MC_CONFIG_DIR) / ".webui_settings.json"
-
-    try:
-        if not settings_path.exists():
-            return defaults
-
-        with open(settings_path, 'r', encoding='utf-8') as f:
-            settings = json.load(f)
-            retention = settings.get('retention_settings', {})
-            return {**defaults, **retention}
-    except Exception as e:
-        logger.error(f"Failed to read retention settings: {e}")
-        return defaults
+    """Get message retention settings from database."""
+    db = _get_db()
+    if db:
+        saved = db.get_setting_json('retention_settings', {})
+        return {**RETENTION_DEFAULTS, **saved}
+    return dict(RETENTION_DEFAULTS)
 
 
 def save_retention_settings(retention_settings: dict) -> bool:
-    """Save message retention settings to .webui_settings.json (atomic write)."""
-    from pathlib import Path
-    settings_path = Path(config.MC_CONFIG_DIR) / ".webui_settings.json"
-
+    """Save message retention settings to database."""
+    db = _get_db()
+    if not db:
+        return False
     try:
-        settings = {}
-        if settings_path.exists():
-            with open(settings_path, 'r', encoding='utf-8') as f:
-                settings = json.load(f)
-
-        settings['retention_settings'] = retention_settings
-
-        temp_file = settings_path.with_suffix('.tmp')
-        with open(temp_file, 'w', encoding='utf-8') as f:
-            json.dump(settings, f, indent=2, ensure_ascii=False)
-        temp_file.replace(settings_path)
-
+        db.set_setting_json('retention_settings', retention_settings)
         return True
     except Exception as e:
         logger.error(f"Failed to save retention settings: {e}")
@@ -837,13 +728,14 @@ def _filter_contacts_by_criteria(contacts: list, criteria: dict) -> list:
     current_time = int(time.time())
     days_threshold = days * 86400  # Convert days to seconds
 
-    # Get protected contacts list (exclude from cleanup)
-    protected_contacts = get_protected_contacts()
+    # Get protected contacts (exclude from cleanup)
+    db = _get_db()
+    protected_keys = db.get_protected_keys() if db else set()
 
     filtered = []
     for contact in contacts:
         # Skip protected contacts
-        if contact.get('public_key', '').lower() in protected_contacts:
+        if contact.get('public_key', '').lower() in protected_keys:
             continue
 
         # Filter by type
@@ -2369,8 +2261,8 @@ def get_contacts_detailed_api():
         contacts = []
 
         # Get protected/ignored/blocked contacts for status fields
-        protected_contacts = get_protected_contacts()
         db = _get_db()
+        protected_keys = db.get_protected_keys() if db else set()
         ignored_keys = db.get_ignored_keys() if db else set()
         blocked_keys = db.get_blocked_keys() if db else set()
 
@@ -2411,7 +2303,7 @@ def get_contacts_detailed_api():
                 'type_label': type_labels.get(details.get('type'), 'UNKNOWN'),
                 'path_or_mode': path_or_mode,  # For UI display
                 'last_seen': details.get('last_advert'),  # Alias for compatibility
-                'is_protected': public_key.lower() in protected_contacts,  # Protection status
+                'is_protected': public_key.lower() in protected_keys,  # Protection status
                 'is_ignored': public_key.lower() in ignored_keys,
                 'is_blocked': public_key.lower() in blocked_keys,
             }
@@ -2586,57 +2478,37 @@ def toggle_contact_protection(public_key):
             }), 400
 
         public_key = public_key.lower()
-
-        # Get current protected list
-        protected_contacts = get_protected_contacts()
+        db = _get_db()
+        if not db:
+            return jsonify({'success': False, 'error': 'Database unavailable'}), 500
 
         # Find matching full public_key if prefix provided
         if len(public_key) < 64:
-            # Fetch contacts to resolve prefix to full key
-            success, contacts_dict, error = cli.get_contacts_with_last_seen()
-            if not success:
-                return jsonify({
-                    'success': False,
-                    'error': error or 'Failed to get contacts'
-                }), 500
-
-            # Find matching contact
-            full_key = None
-            for pk in contacts_dict.keys():
-                if pk.lower().startswith(public_key):
-                    full_key = pk.lower()
-                    break
-
-            if not full_key:
+            contact = db.get_contact_by_prefix(public_key)
+            if not contact:
                 return jsonify({
                     'success': False,
                     'error': f'Contact not found with public_key prefix: {public_key}'
                 }), 404
-
-            public_key = full_key
+            public_key = contact['public_key']
+        else:
+            contact = db.get_contact(public_key)
 
         # Check if explicit protected value provided
         data = request.get_json() or {}
         if 'protected' in data:
-            should_protect = data['protected']
+            should_protect = bool(data['protected'])
         else:
             # Toggle current state
-            should_protect = public_key not in protected_contacts
+            is_currently_protected = contact['is_protected'] == 1 if contact else False
+            should_protect = not is_currently_protected
 
-        # Update protected list
-        if should_protect:
-            if public_key not in protected_contacts:
-                protected_contacts.append(public_key)
+        # Update in database
+        if contact:
+            db.set_contact_protected(public_key, should_protect)
         else:
-            if public_key in protected_contacts:
-                protected_contacts.remove(public_key)
-
-        # Save updated list
-        if not save_protected_contacts(protected_contacts):
-            return jsonify({
-                'success': False,
-                'error': 'Failed to save protected contacts'
-            }), 500
+            # Contact not in DB - create minimal record
+            db.upsert_contact(public_key, name='', is_protected=1 if should_protect else 0, source='advert')
 
         return jsonify({
             'success': True,
@@ -3185,7 +3057,7 @@ def update_device_settings_api():
         }
 
     This setting is:
-    1. Saved to .webui_settings.json for persistence across container restarts
+    1. Saved to database for persistence across container restarts
     2. Applied immediately to the running meshcli session
 
     Returns:

app/schema.sql

@@ -155,6 +155,13 @@ CREATE TABLE IF NOT EXISTS blocked_names (
     created_at  TEXT NOT NULL DEFAULT (datetime('now'))
 );
 
+-- Application settings (key-value store, replaces .webui_settings.json)
+CREATE TABLE IF NOT EXISTS app_settings (
+    key         TEXT PRIMARY KEY,
+    value       TEXT NOT NULL DEFAULT '',       -- JSON-encoded value
+    updated_at  TEXT NOT NULL DEFAULT (datetime('now'))
+);
+
 -- ============================================================
 -- Indexes
 -- ============================================================

tests/test_database.py

@@ -134,6 +134,41 @@ class TestContacts:
         assert abs(contact['adv_lat'] - 52.23) < 0.001
         assert abs(contact['adv_lon'] - 21.01) < 0.001
 
+    def test_get_protected_keys(self, db):
+        db.upsert_contact('AA', name='Alice')
+        db.upsert_contact('BB', name='Bob')
+        db.set_contact_protected('AA', True)
+        keys = db.get_protected_keys()
+        assert 'aa' in keys
+        assert 'bb' not in keys
+
+
+# ================================================================
+# App Settings
+# ================================================================
+
+class TestAppSettings:
+    def test_set_and_get_setting(self, db):
+        db.set_setting('test_key', 'test_value')
+        assert db.get_setting('test_key') == 'test_value'
+
+    def test_get_nonexistent_setting(self, db):
+        assert db.get_setting('nonexistent') is None
+
+    def test_set_and_get_json(self, db):
+        db.set_setting_json('cleanup', {'enabled': True, 'days': 7})
+        result = db.get_setting_json('cleanup')
+        assert result == {'enabled': True, 'days': 7}
+
+    def test_get_json_default(self, db):
+        result = db.get_setting_json('missing', {'default': True})
+        assert result == {'default': True}
+
+    def test_setting_upsert(self, db):
+        db.set_setting_json('key', 'old')
+        db.set_setting_json('key', 'new')
+        assert db.get_setting_json('key') == 'new'
+
 
 # ================================================================
 # Channels