diff --git a/app/database.py b/app/database.py index 33e974e..5ad9f59 100644 --- a/app/database.py +++ b/app/database.py @@ -286,6 +286,137 @@ class Database: ).fetchall() return {r['name'] for r in rows1} | {r['name'] for r in rows2} + # ================================================================ + # Protected Contacts (DB-backed) + # ================================================================ + + def get_protected_keys(self) -> set: + """Return set of public_keys that are protected.""" + with self._connect() as conn: + rows = conn.execute( + "SELECT public_key FROM contacts WHERE is_protected = 1" + ).fetchall() + return {r['public_key'] for r in rows} + + # ================================================================ + # App Settings (key-value store) + # ================================================================ + + def get_setting(self, key: str) -> Optional[str]: + """Get a setting value (JSON string) by key.""" + with self._connect() as conn: + row = conn.execute( + "SELECT value FROM app_settings WHERE key = ?", (key,) + ).fetchone() + return row['value'] if row else None + + def set_setting(self, key: str, value: str) -> None: + """Set a setting value (JSON string).""" + with self._connect() as conn: + conn.execute( + """INSERT INTO app_settings (key, value) + VALUES (?, ?) + ON CONFLICT(key) DO UPDATE SET + value = excluded.value, + updated_at = datetime('now')""", + (key, value) + ) + + def get_setting_json(self, key: str, default=None): + """Get a setting, JSON-decoded. Returns default if not found.""" + import json + raw = self.get_setting(key) + if raw is None: + return default + try: + return json.loads(raw) + except (json.JSONDecodeError, TypeError): + return default + + def set_setting_json(self, key: str, value) -> None: + """Set a setting, JSON-encoding the value.""" + import json + self.set_setting(key, json.dumps(value, ensure_ascii=False)) + + def migrate_protected_contacts_from_file(self, settings_path) -> int: + """One-time migration: import protected_contacts from .webui_settings.json into DB. + + Returns number of contacts marked as protected. + """ + import json + from pathlib import Path + settings_path = Path(settings_path) + if not settings_path.exists(): + return 0 + + try: + with open(settings_path, 'r', encoding='utf-8') as f: + settings = json.load(f) + protected = settings.get('protected_contacts', []) + if not protected: + return 0 + + count = 0 + with self._connect() as conn: + for pk in protected: + pk = pk.lower() + cursor = conn.execute( + "UPDATE contacts SET is_protected = 1, lastmod = datetime('now') WHERE public_key = ?", + (pk,) + ) + if cursor.rowcount > 0: + count += 1 + else: + # Contact not in DB yet - insert minimal record + conn.execute( + """INSERT OR IGNORE INTO contacts (public_key, name, is_protected, source) + VALUES (?, '', 1, 'advert')""", + (pk,) + ) + count += 1 + + logger.info(f"Migrated {count} protected contacts from settings file to DB") + return count + except Exception as e: + logger.error(f"Failed to migrate protected contacts: {e}") + return 0 + + def migrate_settings_from_file(self, settings_path) -> bool: + """One-time migration: import cleanup/retention/manual_add settings from .webui_settings.json. + + Returns True if migration was performed. + """ + import json + from pathlib import Path + settings_path = Path(settings_path) + if not settings_path.exists(): + return False + + try: + with open(settings_path, 'r', encoding='utf-8') as f: + settings = json.load(f) + + migrated = False + + if 'cleanup_settings' in settings: + self.set_setting_json('cleanup_settings', settings['cleanup_settings']) + migrated = True + + if 'retention_settings' in settings: + self.set_setting_json('retention_settings', settings['retention_settings']) + migrated = True + + if 'manual_add_contacts' in settings: + self.set_setting_json('manual_add_contacts', settings['manual_add_contacts']) + migrated = True + + if migrated: + logger.info("Migrated app settings from .webui_settings.json to DB") + return migrated + except Exception as e: + logger.error(f"Failed to migrate settings: {e}") + return False + # ================================================================ # Channels # ================================================================ diff --git a/app/device_manager.py b/app/device_manager.py index f785757..f9cfba9 100644 --- a/app/device_manager.py +++ b/app/device_manager.py @@ -824,14 +824,9 @@ class DeviceManager: }, namespace='/chat') def _is_manual_approval_enabled(self) -> bool: - """Check if manual contact approval is enabled (from persisted settings).""" + """Check if manual contact approval is enabled (from database).""" try: - from pathlib import Path - settings_path = Path(self.config.MC_CONFIG_DIR) / ".webui_settings.json" - if settings_path.exists(): - with open(settings_path, 'r', encoding='utf-8') as f: - settings = json.load(f) - return bool(settings.get('manual_add_contacts', False)) + return bool(self.db.get_setting_json('manual_add_contacts', False)) except Exception: pass return False @@ -871,8 +866,44 @@ class DeviceManager: return if self._is_manual_approval_enabled(): + # Check if contact already exists on the device (firmware edge case: + # firmware may fire NEW_CONTACT for a contact that was previously + # on the device but got removed by firmware-level cleanup) + if pubkey in (self.mc.contacts or {}): + logger.warning( + f"NEW_CONTACT fired for contact already on device: {name} ({pubkey[:8]}...) " + f"— skipping pending, updating DB cache only" + ) + # Just update cache, don't add to pending + last_adv = data.get('last_advert') + last_advert_val = ( + str(int(last_adv)) + if last_adv and isinstance(last_adv, (int, float)) and last_adv > 0 + else str(int(time.time())) + ) + self.db.upsert_contact( + public_key=pubkey, + name=name, + type=data.get('type', data.get('adv_type', 0)), + adv_lat=data.get('adv_lat'), + adv_lon=data.get('adv_lon'), + last_advert=last_advert_val, + source='device', + ) + return + + # Check if contact was previously known (in DB cache) + existing = self.db.get_contact(pubkey) + if existing: + logger.info( + f"Pending contact (manual mode): {name} ({pubkey[:8]}...) " + f"— previously known (source={existing['source']}, " + f"protected={existing['is_protected']})" + ) + else: + logger.info(f"Pending contact (manual mode): {name} ({pubkey[:8]}...) — first time seen") + # Manual mode: meshcore puts it in mc.pending_contacts for approval - logger.info(f"Pending contact (manual mode): {name} ({pubkey[:8]}...)") # Also add to DB cache for @mentions and Cache filter last_adv = data.get('last_advert') diff --git a/app/main.py b/app/main.py index 8e2886f..9990528 100644 --- a/app/main.py +++ b/app/main.py @@ -74,6 +74,21 @@ def create_app(): db = Database(config.db_path) app.db = db + # Migrate settings from .webui_settings.json to DB (one-time) + from pathlib import Path + settings_file = Path(config.MC_CONFIG_DIR) / ".webui_settings.json" + if settings_file.exists() and db.get_setting('manual_add_contacts') is None: + logger.info("Migrating settings from .webui_settings.json to database...") + db.migrate_protected_contacts_from_file(settings_file) + db.migrate_settings_from_file(settings_file) + # Rename old file as backup + backup = settings_file.with_suffix('.json.bak') + try: + settings_file.rename(backup) + logger.info(f"Settings file backed up to {backup.name}") + except Exception as e: + logger.warning(f"Could not rename settings file: {e}") + # v2: Initialize and start device manager device_manager = DeviceManager(config, db, socketio) app.device_manager = device_manager diff --git a/app/meshcore/cli.py b/app/meshcore/cli.py index 2d87b14..d033f0d 100644 --- a/app/meshcore/cli.py +++ b/app/meshcore/cli.py @@ -5,8 +5,6 @@ Function signatures preserved for backward compatibility with api.py. """ import logging -import json -from pathlib import Path from typing import Tuple, Optional, List, Dict from app.config import config @@ -34,6 +32,22 @@ def _get_dm(): return device_manager +def _get_db(): + """Get Database instance — try Flask app context first, then DeviceManager.""" + try: + from flask import current_app + db = getattr(current_app, 'db', None) + if db is not None: + return db + except RuntimeError: + pass + try: + dm = _get_dm() + return dm.db + except Exception: + return None + + # ============================================================================= # Messages # ============================================================================= @@ -454,16 +468,11 @@ def set_auto_retry_config(enabled=None, max_attempts=None, max_flood=None) -> Tu # ============================================================================= def get_device_settings() -> Tuple[bool, Dict]: - """Get persistent device settings.""" - settings_path = Path(config.MC_CONFIG_DIR) / ".webui_settings.json" + """Get persistent device settings from database.""" try: - if not settings_path.exists(): - return True, {'manual_add_contacts': False} - with open(settings_path, 'r', encoding='utf-8') as f: - settings = json.load(f) - if 'manual_add_contacts' not in settings: - settings['manual_add_contacts'] = False - return True, settings + db = _get_db() + manual = db.get_setting_json('manual_add_contacts', False) if db else False + return True, {'manual_add_contacts': manual} except Exception as e: logger.error(f"Failed to read device settings: {e}") return False, {'manual_add_contacts': False} @@ -475,19 +484,10 @@ def set_manual_add_contacts(enabled: bool) -> Tuple[bool, str]: dm_inst = _get_dm() result = dm_inst.set_manual_add_contacts(enabled) if result['success']: - # Persist to settings file - settings_path = Path(config.MC_CONFIG_DIR) / ".webui_settings.json" - try: - settings = {} - if settings_path.exists(): - with open(settings_path, 'r') as f: - settings = json.load(f) - settings['manual_add_contacts'] = enabled - settings_path.parent.mkdir(parents=True, exist_ok=True) - with open(settings_path, 'w') as f: - json.dump(settings, f) - except Exception as e: - logger.warning(f"Failed to persist settings: {e}") + # Persist to database + db = _get_db() + if db: + db.set_setting_json('manual_add_contacts', enabled) return result['success'], result.get('message', result.get('error', '')) except Exception as e: return False, str(e) diff --git a/app/routes/api.py b/app/routes/api.py index b453e65..9f729a3 100644 --- a/app/routes/api.py +++ b/app/routes/api.py @@ -180,136 +180,55 @@ def invalidate_contacts_cache(): def get_protected_contacts() -> list: """ - Get list of protected contact public keys from settings. + Get list of protected contact public keys from database. Returns: List of public_key strings (64 hex chars, lowercase) """ - from pathlib import Path - settings_path = Path(config.MC_CONFIG_DIR) / ".webui_settings.json" - - try: - if not settings_path.exists(): - return [] - - with open(settings_path, 'r', encoding='utf-8') as f: - settings = json.load(f) - # Return lowercase keys for consistent comparison - return [pk.lower() for pk in settings.get('protected_contacts', [])] - except Exception as e: - logger.error(f"Failed to read protected contacts: {e}") - return [] - - -def save_protected_contacts(protected_list: list) -> bool: - """ - Save protected contacts list to settings file (atomic write). - - Args: - protected_list: List of public_key strings - - Returns: - True if successful, False otherwise - """ - from pathlib import Path - settings_path = Path(config.MC_CONFIG_DIR) / ".webui_settings.json" - - try: - # Read existing settings - settings = {} - if settings_path.exists(): - with open(settings_path, 'r', encoding='utf-8') as f: - settings = json.load(f) - - # Update protected contacts (store lowercase) - settings['protected_contacts'] = [pk.lower() for pk in protected_list] - - # Write back atomically - temp_file = settings_path.with_suffix('.tmp') - with open(temp_file, 'w', encoding='utf-8') as f: - json.dump(settings, f, indent=2, ensure_ascii=False) - temp_file.replace(settings_path) - - return True - except Exception as e: - logger.error(f"Failed to save protected contacts: {e}") - return False + db = _get_db() + if db: + return list(db.get_protected_keys()) + return [] # ============================================================================= # Cleanup Settings Management # ============================================================================= +CLEANUP_DEFAULTS = { + 'enabled': False, + 'types': [1, 2, 3, 4], + 'date_field': 'last_advert', + 'days': 30, + 'name_filter': '', + 'hour': 1 +} + +RETENTION_DEFAULTS = { + 'enabled': False, + 'days': 90, + 'include_dms': False, + 'include_adverts': False, + 'hour': 2 +} + + def get_cleanup_settings() -> dict: - """ - Get auto-cleanup settings from .webui_settings.json. - - Returns: - Dict with cleanup settings: - { - 'enabled': bool, - 'types': list[int], - 'date_field': str, - 'days': int, - 'name_filter': str, - 'hour': int (0-23, UTC) - } - """ - from pathlib import Path - defaults = { - 'enabled': False, - 'types': [1, 2, 3, 4], - 'date_field': 'last_advert', - 'days': 30, - 'name_filter': '', - 'hour': 1 - } - - settings_path = Path(config.MC_CONFIG_DIR) / ".webui_settings.json" - - try: - if not settings_path.exists(): - return defaults - - with open(settings_path, 'r', encoding='utf-8') as f: - settings = json.load(f) - cleanup = settings.get('cleanup_settings', {}) - # Merge with defaults to ensure all fields exist - return {**defaults, **cleanup} - except Exception as e: - logger.error(f"Failed to read cleanup settings: {e}") - return defaults + """Get auto-cleanup settings from database.""" + db = _get_db() + if db: + saved = db.get_setting_json('cleanup_settings', {}) + return {**CLEANUP_DEFAULTS, **saved} + return dict(CLEANUP_DEFAULTS) def save_cleanup_settings(cleanup_settings: dict) -> bool: - """ - Save auto-cleanup settings to .webui_settings.json (atomic write). - - Args: - cleanup_settings: Dict with cleanup configuration - - Returns: - True if successful, False otherwise - """ - from pathlib import Path - settings_path = Path(config.MC_CONFIG_DIR) / ".webui_settings.json" - + """Save auto-cleanup settings to database.""" + db = _get_db() + if not db: + return False try: - # Read existing settings - settings = {} - if settings_path.exists(): - with open(settings_path, 'r', encoding='utf-8') as f: - settings = json.load(f) - - # Update cleanup settings - settings['cleanup_settings'] = cleanup_settings - - # Write back atomically - temp_file = settings_path.with_suffix('.tmp') - with open(temp_file, 'w', encoding='utf-8') as f: - json.dump(settings, f, indent=2, ensure_ascii=False) - temp_file.replace(settings_path) - + db.set_setting_json('cleanup_settings', cleanup_settings) return True except Exception as e: logger.error(f"Failed to save cleanup settings: {e}") @@ -317,49 +236,21 @@ def save_cleanup_settings(cleanup_settings: dict) -> bool: def get_retention_settings() -> dict: - """Get message retention settings from .webui_settings.json.""" - from pathlib import Path - defaults = { - 'enabled': False, - 'days': 90, - 'include_dms': False, - 'include_adverts': False, - 'hour': 2 - } - - settings_path = Path(config.MC_CONFIG_DIR) / ".webui_settings.json" - - try: - if not settings_path.exists(): - return defaults - - with open(settings_path, 'r', encoding='utf-8') as f: - settings = json.load(f) - retention = settings.get('retention_settings', {}) - return {**defaults, **retention} - except Exception as e: - logger.error(f"Failed to read retention settings: {e}") - return defaults + """Get message retention settings from database.""" + db = _get_db() + if db: + saved = db.get_setting_json('retention_settings', {}) + return {**RETENTION_DEFAULTS, **saved} + return dict(RETENTION_DEFAULTS) def save_retention_settings(retention_settings: dict) -> bool: - """Save message retention settings to .webui_settings.json (atomic write).""" - from pathlib import Path - settings_path = Path(config.MC_CONFIG_DIR) / ".webui_settings.json" - + """Save message retention settings to database.""" + db = _get_db() + if not db: + return False try: - settings = {} - if settings_path.exists(): - with open(settings_path, 'r', encoding='utf-8') as f: - settings = json.load(f) - - settings['retention_settings'] = retention_settings - - temp_file = settings_path.with_suffix('.tmp') - with open(temp_file, 'w', encoding='utf-8') as f: - json.dump(settings, f, indent=2, ensure_ascii=False) - temp_file.replace(settings_path) - + db.set_setting_json('retention_settings', retention_settings) return True except Exception as e: logger.error(f"Failed to save retention settings: {e}") @@ -837,13 +728,14 @@ def _filter_contacts_by_criteria(contacts: list, criteria: dict) -> list: current_time = int(time.time()) days_threshold = days * 86400 # Convert days to seconds - # Get protected contacts list (exclude from cleanup) - protected_contacts = get_protected_contacts() + # Get protected contacts (exclude from cleanup) + db = _get_db() + protected_keys = db.get_protected_keys() if db else set() filtered = [] for contact in contacts: # Skip protected contacts - if contact.get('public_key', '').lower() in protected_contacts: + if contact.get('public_key', '').lower() in protected_keys: continue # Filter by type @@ -2369,8 +2261,8 @@ def get_contacts_detailed_api(): contacts = [] # Get protected/ignored/blocked contacts for status fields - protected_contacts = get_protected_contacts() db = _get_db() + protected_keys = db.get_protected_keys() if db else set() ignored_keys = db.get_ignored_keys() if db else set() blocked_keys = db.get_blocked_keys() if db else set() @@ -2411,7 +2303,7 @@ def get_contacts_detailed_api(): 'type_label': type_labels.get(details.get('type'), 'UNKNOWN'), 'path_or_mode': path_or_mode, # For UI display 'last_seen': details.get('last_advert'), # Alias for compatibility - 'is_protected': public_key.lower() in protected_contacts, # Protection status + 'is_protected': public_key.lower() in protected_keys, # Protection status 'is_ignored': public_key.lower() in ignored_keys, 'is_blocked': public_key.lower() in blocked_keys, } @@ -2586,57 +2478,37 @@ def toggle_contact_protection(public_key): }), 400 public_key = public_key.lower() - - # Get current protected list - protected_contacts = get_protected_contacts() + db = _get_db() + if not db: + return jsonify({'success': False, 'error': 'Database unavailable'}), 500 # Find matching full public_key if prefix provided if len(public_key) < 64: - # Fetch contacts to resolve prefix to full key - success, contacts_dict, error = cli.get_contacts_with_last_seen() - if not success: - return jsonify({ - 'success': False, - 'error': error or 'Failed to get contacts' - }), 500 - - # Find matching contact - full_key = None - for pk in contacts_dict.keys(): - if pk.lower().startswith(public_key): - full_key = pk.lower() - break - - if not full_key: + contact = db.get_contact_by_prefix(public_key) + if not contact: return jsonify({ 'success': False, 'error': f'Contact not found with public_key prefix: {public_key}' }), 404 - - public_key = full_key + public_key = contact['public_key'] + else: + contact = db.get_contact(public_key) # Check if explicit protected value provided data = request.get_json() or {} if 'protected' in data: - should_protect = data['protected'] + should_protect = bool(data['protected']) else: # Toggle current state - should_protect = public_key not in protected_contacts + is_currently_protected = contact['is_protected'] == 1 if contact else False + should_protect = not is_currently_protected - # Update protected list - if should_protect: - if public_key not in protected_contacts: - protected_contacts.append(public_key) + # Update in database + if contact: + db.set_contact_protected(public_key, should_protect) else: - if public_key in protected_contacts: - protected_contacts.remove(public_key) - - # Save updated list - if not save_protected_contacts(protected_contacts): - return jsonify({ - 'success': False, - 'error': 'Failed to save protected contacts' - }), 500 + # Contact not in DB - create minimal record + db.upsert_contact(public_key, name='', is_protected=1 if should_protect else 0, source='advert') return jsonify({ 'success': True, @@ -3185,7 +3057,7 @@ def update_device_settings_api(): } This setting is: - 1. Saved to .webui_settings.json for persistence across container restarts + 1. Saved to database for persistence across container restarts 2. Applied immediately to the running meshcli session Returns: diff --git a/app/schema.sql b/app/schema.sql index 076371b..595aa12 100644 --- a/app/schema.sql +++ b/app/schema.sql @@ -155,6 +155,13 @@ CREATE TABLE IF NOT EXISTS blocked_names ( created_at TEXT NOT NULL DEFAULT (datetime('now')) ); +-- Application settings (key-value store, replaces .webui_settings.json) +CREATE TABLE IF NOT EXISTS app_settings ( + key TEXT PRIMARY KEY, + value TEXT NOT NULL DEFAULT '', -- JSON-encoded value + updated_at TEXT NOT NULL DEFAULT (datetime('now')) +); + -- ============================================================ -- Indexes -- ============================================================ diff --git a/tests/test_database.py b/tests/test_database.py index d84f1fd..a4dba8c 100644 --- a/tests/test_database.py +++ b/tests/test_database.py @@ -134,6 +134,41 @@ class TestContacts: assert abs(contact['adv_lat'] - 52.23) < 0.001 assert abs(contact['adv_lon'] - 21.01) < 0.001 + def test_get_protected_keys(self, db): + db.upsert_contact('AA', name='Alice') + db.upsert_contact('BB', name='Bob') + db.set_contact_protected('AA', True) + keys = db.get_protected_keys() + assert 'aa' in keys + assert 'bb' not in keys + + +# ================================================================ +# App Settings +# ================================================================ + +class TestAppSettings: + def test_set_and_get_setting(self, db): + db.set_setting('test_key', 'test_value') + assert db.get_setting('test_key') == 'test_value' + + def test_get_nonexistent_setting(self, db): + assert db.get_setting('nonexistent') is None + + def test_set_and_get_json(self, db): + db.set_setting_json('cleanup', {'enabled': True, 'days': 7}) + result = db.get_setting_json('cleanup') + assert result == {'enabled': True, 'days': 7} + + def test_get_json_default(self, db): + result = db.get_setting_json('missing', {'default': True}) + assert result == {'default': True} + + def test_setting_upsert(self, db): + db.set_setting_json('key', 'old') + db.set_setting_json('key', 'new') + assert db.get_setting_json('key') == 'new' + # ================================================================ # Channels