fix(retention): enable retention by default and unblock the scheduler

The retention/cleanup scheduler was throwing "Working outside of application
context" on every boot because init_*_schedule() and the APScheduler jobs
themselves call api.get_*_settings(), which reads current_app.db. Pass the
Flask app into archiver.manager via set_flask_app() and decorate every
scheduled job with @_with_app_context so the context is active when the
worker thread runs. Wrap the init calls in main.py in app.app_context() too.

Extend cleanup_old_messages() to also delete from echoes, paths and acks
(the diagnostic tables — 191k echo rows account for the bulk of the 85 MB
DB). Each diagnostic table has its own retention window, defaulting to
min(days, 30) so debug data is purged on a tighter schedule than
user-visible messages.

Switch RETENTION_DEFAULTS to enabled=True with 90/90/60/30 days for
channel_messages/DMs/adverts/diagnostics; user opted in explicitly. First
run scheduled for 03:30 local. SQLite DELETE only marks pages free —
file size won't shrink until a manual VACUUM is run separately.

Archive job stopped working the day the device was renamed to include an
emoji ("MarWoj 💡"): the meshcore library strips non-ASCII when writing
the .msgs file, but our archiver still built /data/{device_name}.msgs and
hit ENOENT every night at 00:00. Add a tolerant fallback that globs the
data dir for a single non-archive .msgs file when the expected path is
missing — mirroring how migrate_v1 already handles this.

Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>

app/archiver/manager.py

@@ -5,6 +5,7 @@ Archive manager - handles message archiving and scheduling
 import os
 import shutil
 import logging
+from functools import wraps
 from pathlib import Path
 from datetime import datetime, time
 from typing import List, Dict, Optional
@@ -23,8 +24,69 @@ CLEANUP_JOB_ID = 'daily_cleanup'
 RETENTION_JOB_ID = 'daily_retention'
 BACKUP_JOB_ID = 'daily_backup'
 
-# Module-level db reference (set by init_retention_schedule)
+# Module-level references (set by set_flask_app / init_retention_schedule)
 _db = None
+_app = None
+
+
+def set_flask_app(app):
+    """Store Flask app so scheduled jobs can push app_context.
+
+    Without this, _cleanup_job/_retention_job hit `current_app.db` from
+    api.get_*_settings() and raise "Working outside of application context".
+    """
+    global _app
+    _app = app
+
+
+def _with_app_context(fn):
+    """Decorator: push Flask app_context around scheduled jobs.
+
+    APScheduler runs job functions in worker threads with no Flask context.
+    Anything that reaches current_app (e.g. api.get_cleanup_settings -> _get_db)
+    needs the context to be active.
+    """
+    @wraps(fn)
+    def wrapper(*args, **kwargs):
+        if _app is not None:
+            with _app.app_context():
+                return fn(*args, **kwargs)
+        logger.warning(f"{fn.__name__}: no Flask app registered, running without context")
+        return fn(*args, **kwargs)
+    return wrapper
+
+
+def _find_live_msgs_file(device_name: str) -> Optional[Path]:
+    """Locate the live (non-archive) .msgs file.
+
+    Tries the configured device-name-based path first. If that file does not
+    exist (e.g. because the device name contains emoji or whitespace the
+    meshcore library strips when writing the file), falls back to an
+    unambiguous glob in the data dir, excluding archive files
+    (which have a .YYYY-MM-DD. segment in their name).
+    """
+    candidate = Path(config.MC_CONFIG_DIR) / f"{device_name}.msgs"
+    if candidate.exists():
+        return candidate
+
+    data_dir = Path(config.MC_CONFIG_DIR)
+    if not data_dir.exists():
+        return None
+
+    # Live file pattern: name.msgs (no date segment in stem)
+    live_files = [f for f in data_dir.glob("*.msgs") if f.stem.count('.') == 0]
+    if len(live_files) == 1:
+        logger.info(
+            f"Live .msgs file resolved via fallback glob: {live_files[0].name} "
+            f"(expected {candidate.name})"
+        )
+        return live_files[0]
+    if len(live_files) > 1:
+        logger.warning(
+            f"Multiple .msgs files found in {data_dir}, cannot pick one: "
+            f"{[f.name for f in live_files]}"
+        )
+    return None
 
 
 def get_local_timezone_name() -> str:
@@ -104,13 +166,15 @@ def archive_messages(archive_date: Optional[str] = None) -> Dict[str, any]:
         archive_dir = config.archive_dir_path
         archive_dir.mkdir(parents=True, exist_ok=True)
 
-        # Get source .msgs file
-        source_file = runtime_config.get_msgs_file_path()
-        if not source_file.exists():
-            logger.warning(f"Source messages file not found: {source_file}")
+        # Get source .msgs file. Use tolerant lookup because meshcore lib may
+        # strip non-ASCII / whitespace from the device-derived filename.
+        source_file = _find_live_msgs_file(runtime_config.get_device_name())
+        if source_file is None or not source_file.exists():
+            expected = runtime_config.get_msgs_file_path()
+            logger.warning(f"Source messages file not found (expected {expected})")
             return {
                 'success': False,
-                'error': f'Messages file not found: {source_file}'
+                'error': f'Messages file not found: {expected}'
             }
 
         # Get destination archive file
@@ -245,6 +309,7 @@ def _count_messages_in_file(file_path: Path) -> int:
     return count
 
 
+@_with_app_context
 def _archive_job():
     """
     Background job that runs daily to archive messages.
@@ -264,6 +329,7 @@ def _archive_job():
         logger.error(f"Archive job failed: {result.get('error', 'Unknown error')}")
 
 
+@_with_app_context
 def _cleanup_job():
     """
     Background job that runs daily to clean up contacts.
@@ -465,6 +531,7 @@ def init_cleanup_schedule():
         logger.error(f"Error initializing cleanup schedule: {e}", exc_info=True)
 
 
+@_with_app_context
 def _retention_job():
     """Background job that runs daily to delete old messages from DB."""
     logger.info("Running daily retention job...")
@@ -482,14 +549,14 @@ def _retention_job():
             logger.error("Database not available for retention job")
             return
 
-        days = settings.get('days', 90)
-        include_dms = settings.get('include_dms', False)
-        include_adverts = settings.get('include_adverts', False)
-
         result = _db.cleanup_old_messages(
-            days=days,
-            include_dms=include_dms,
-            include_adverts=include_adverts
+            days=settings.get('days', 90),
+            include_dms=settings.get('include_dms', True),
+            include_adverts=settings.get('include_adverts', True),
+            days_dms=settings.get('days_dms'),
+            days_adverts=settings.get('days_adverts'),
+            include_diagnostics=settings.get('include_diagnostics', True),
+            days_diagnostics=settings.get('days_diagnostics'),
         )
 
         total = sum(result.values())
@@ -640,6 +707,7 @@ def init_backup_schedule():
         logger.error(f"Error scheduling backup: {e}", exc_info=True)
 
 
+@_with_app_context
 def _backup_job(backup_dir):
     """Execute daily backup and cleanup old backups."""
     global _db

app/database.py

@@ -1287,28 +1287,66 @@ class Database:
         return stats
 
     def cleanup_old_messages(self, days: int, include_dms: bool = False,
-                             include_adverts: bool = False) -> dict:
-        """Delete messages older than N days. Returns counts per table."""
-        cutoff = int((datetime.now() - timedelta(days=days)).timestamp())
+                             include_adverts: bool = False,
+                             days_dms: Optional[int] = None,
+                             days_adverts: Optional[int] = None,
+                             include_diagnostics: bool = True,
+                             days_diagnostics: Optional[int] = None) -> dict:
+        """Delete old rows from message + diagnostic tables. Returns counts per table.
+
+        - channel_messages, direct_messages, advertisements use unix `timestamp`.
+        - echoes, paths, acks use TEXT `received_at` (datetime('now') format).
+        Diagnostic tables are joined to messages but are mostly write-heavy
+        debug data, so they get a tighter default retention.
+        """
         result = {}
+        now = datetime.now()
+
+        def _cutoff_unix(d):
+            return int((now - timedelta(days=d)).timestamp())
+
+        def _cutoff_text(d):
+            return (now - timedelta(days=d)).strftime('%Y-%m-%d %H:%M:%S')
+
         with self._connect() as conn:
             cursor = conn.execute(
-                "DELETE FROM channel_messages WHERE timestamp < ?", (cutoff,)
+                "DELETE FROM channel_messages WHERE timestamp < ?",
+                (_cutoff_unix(days),)
             )
             result['channel_messages'] = cursor.rowcount
 
             if include_dms:
+                dm_days = days_dms if days_dms is not None else days
                 cursor = conn.execute(
-                    "DELETE FROM direct_messages WHERE timestamp < ?", (cutoff,)
+                    "DELETE FROM direct_messages WHERE timestamp < ?",
+                    (_cutoff_unix(dm_days),)
                 )
                 result['direct_messages'] = cursor.rowcount
 
             if include_adverts:
+                adv_days = days_adverts if days_adverts is not None else days
                 cursor = conn.execute(
-                    "DELETE FROM advertisements WHERE timestamp < ?", (cutoff,)
+                    "DELETE FROM advertisements WHERE timestamp < ?",
+                    (_cutoff_unix(adv_days),)
                 )
                 result['advertisements'] = cursor.rowcount
 
+            if include_diagnostics:
+                diag_days = days_diagnostics if days_diagnostics is not None else min(days, 30)
+                diag_cutoff = _cutoff_text(diag_days)
+                cursor = conn.execute(
+                    "DELETE FROM echoes WHERE received_at < ?", (diag_cutoff,)
+                )
+                result['echoes'] = cursor.rowcount
+                cursor = conn.execute(
+                    "DELETE FROM paths WHERE received_at < ?", (diag_cutoff,)
+                )
+                result['paths'] = cursor.rowcount
+                cursor = conn.execute(
+                    "DELETE FROM acks WHERE received_at < ?", (diag_cutoff,)
+                )
+                result['acks'] = cursor.rowcount
+
         return result
 
     # ================================================================

app/main.py

@@ -338,10 +338,19 @@ def create_app():
 
     threading.Thread(target=_wait_for_device_name, daemon=True).start()
 
-    # Start background scheduler (archiving, contact cleanup, message retention)
-    from app.archiver.manager import schedule_daily_archiving, init_retention_schedule
-    schedule_daily_archiving()
-    init_retention_schedule(db=db)
+    # Start background scheduler (archiving, contact cleanup, message retention).
+    # init_*_schedule and the jobs themselves call api.get_*_settings(), which
+    # touches current_app.db — so the scheduler module needs an app reference
+    # to push app_context, and the init paths must run inside a context too.
+    from app.archiver.manager import (
+        schedule_daily_archiving,
+        init_retention_schedule,
+        set_flask_app as _archiver_set_app,
+    )
+    _archiver_set_app(app)
+    with app.app_context():
+        schedule_daily_archiving()
+        init_retention_schedule(db=db)
 
     logger.info(f"mc-webui v2 started — transport: {config.transport_type}")
     logger.info(f"Database: {db.db_path}")

app/routes/api.py

@@ -212,11 +212,15 @@ CLEANUP_DEFAULTS = {
 }
 
 RETENTION_DEFAULTS = {
-    'enabled': False,
-    'days': 90,
-    'include_dms': False,
-    'include_adverts': False,
-    'hour': 2
+    'enabled': True,
+    'days': 90,                  # channel_messages
+    'days_dms': 90,              # direct_messages
+    'days_adverts': 60,          # advertisements
+    'days_diagnostics': 30,      # echoes, paths, acks (high-volume debug data)
+    'include_dms': True,
+    'include_adverts': True,
+    'include_diagnostics': True,
+    'hour': 3
 }