diff --git a/app/archiver/manager.py b/app/archiver/manager.py index c3d94c9..dedd6c9 100644 --- a/app/archiver/manager.py +++ b/app/archiver/manager.py @@ -5,6 +5,7 @@ Archive manager - handles message archiving and scheduling import os import shutil import logging +from functools import wraps from pathlib import Path from datetime import datetime, time from typing import List, Dict, Optional @@ -23,8 +24,69 @@ CLEANUP_JOB_ID = 'daily_cleanup' RETENTION_JOB_ID = 'daily_retention' BACKUP_JOB_ID = 'daily_backup' -# Module-level db reference (set by init_retention_schedule) +# Module-level references (set by set_flask_app / init_retention_schedule) _db = None +_app = None + + +def set_flask_app(app): + """Store Flask app so scheduled jobs can push app_context. + + Without this, _cleanup_job/_retention_job hit `current_app.db` from + api.get_*_settings() and raise "Working outside of application context". + """ + global _app + _app = app + + +def _with_app_context(fn): + """Decorator: push Flask app_context around scheduled jobs. + + APScheduler runs job functions in worker threads with no Flask context. + Anything that reaches current_app (e.g. api.get_cleanup_settings -> _get_db) + needs the context to be active. + """ + @wraps(fn) + def wrapper(*args, **kwargs): + if _app is not None: + with _app.app_context(): + return fn(*args, **kwargs) + logger.warning(f"{fn.__name__}: no Flask app registered, running without context") + return fn(*args, **kwargs) + return wrapper + + +def _find_live_msgs_file(device_name: str) -> Optional[Path]: + """Locate the live (non-archive) .msgs file. + + Tries the configured device-name-based path first. If that file does not + exist (e.g. because the device name contains emoji or whitespace the + meshcore library strips when writing the file), falls back to an + unambiguous glob in the data dir, excluding archive files + (which have a .YYYY-MM-DD. segment in their name). + """ + candidate = Path(config.MC_CONFIG_DIR) / f"{device_name}.msgs" + if candidate.exists(): + return candidate + + data_dir = Path(config.MC_CONFIG_DIR) + if not data_dir.exists(): + return None + + # Live file pattern: name.msgs (no date segment in stem) + live_files = [f for f in data_dir.glob("*.msgs") if f.stem.count('.') == 0] + if len(live_files) == 1: + logger.info( + f"Live .msgs file resolved via fallback glob: {live_files[0].name} " + f"(expected {candidate.name})" + ) + return live_files[0] + if len(live_files) > 1: + logger.warning( + f"Multiple .msgs files found in {data_dir}, cannot pick one: " + f"{[f.name for f in live_files]}" + ) + return None def get_local_timezone_name() -> str: @@ -104,13 +166,15 @@ def archive_messages(archive_date: Optional[str] = None) -> Dict[str, any]: archive_dir = config.archive_dir_path archive_dir.mkdir(parents=True, exist_ok=True) - # Get source .msgs file - source_file = runtime_config.get_msgs_file_path() - if not source_file.exists(): - logger.warning(f"Source messages file not found: {source_file}") + # Get source .msgs file. Use tolerant lookup because meshcore lib may + # strip non-ASCII / whitespace from the device-derived filename. + source_file = _find_live_msgs_file(runtime_config.get_device_name()) + if source_file is None or not source_file.exists(): + expected = runtime_config.get_msgs_file_path() + logger.warning(f"Source messages file not found (expected {expected})") return { 'success': False, - 'error': f'Messages file not found: {source_file}' + 'error': f'Messages file not found: {expected}' } # Get destination archive file @@ -245,6 +309,7 @@ def _count_messages_in_file(file_path: Path) -> int: return count +@_with_app_context def _archive_job(): """ Background job that runs daily to archive messages. @@ -264,6 +329,7 @@ def _archive_job(): logger.error(f"Archive job failed: {result.get('error', 'Unknown error')}") +@_with_app_context def _cleanup_job(): """ Background job that runs daily to clean up contacts. @@ -465,6 +531,7 @@ def init_cleanup_schedule(): logger.error(f"Error initializing cleanup schedule: {e}", exc_info=True) +@_with_app_context def _retention_job(): """Background job that runs daily to delete old messages from DB.""" logger.info("Running daily retention job...") @@ -482,14 +549,14 @@ def _retention_job(): logger.error("Database not available for retention job") return - days = settings.get('days', 90) - include_dms = settings.get('include_dms', False) - include_adverts = settings.get('include_adverts', False) - result = _db.cleanup_old_messages( - days=days, - include_dms=include_dms, - include_adverts=include_adverts + days=settings.get('days', 90), + include_dms=settings.get('include_dms', True), + include_adverts=settings.get('include_adverts', True), + days_dms=settings.get('days_dms'), + days_adverts=settings.get('days_adverts'), + include_diagnostics=settings.get('include_diagnostics', True), + days_diagnostics=settings.get('days_diagnostics'), ) total = sum(result.values()) @@ -640,6 +707,7 @@ def init_backup_schedule(): logger.error(f"Error scheduling backup: {e}", exc_info=True) +@_with_app_context def _backup_job(backup_dir): """Execute daily backup and cleanup old backups.""" global _db diff --git a/app/database.py b/app/database.py index ceff526..f4bf292 100644 --- a/app/database.py +++ b/app/database.py @@ -1287,28 +1287,66 @@ class Database: return stats def cleanup_old_messages(self, days: int, include_dms: bool = False, - include_adverts: bool = False) -> dict: - """Delete messages older than N days. Returns counts per table.""" - cutoff = int((datetime.now() - timedelta(days=days)).timestamp()) + include_adverts: bool = False, + days_dms: Optional[int] = None, + days_adverts: Optional[int] = None, + include_diagnostics: bool = True, + days_diagnostics: Optional[int] = None) -> dict: + """Delete old rows from message + diagnostic tables. Returns counts per table. + + - channel_messages, direct_messages, advertisements use unix `timestamp`. + - echoes, paths, acks use TEXT `received_at` (datetime('now') format). + Diagnostic tables are joined to messages but are mostly write-heavy + debug data, so they get a tighter default retention. + """ result = {} + now = datetime.now() + + def _cutoff_unix(d): + return int((now - timedelta(days=d)).timestamp()) + + def _cutoff_text(d): + return (now - timedelta(days=d)).strftime('%Y-%m-%d %H:%M:%S') + with self._connect() as conn: cursor = conn.execute( - "DELETE FROM channel_messages WHERE timestamp < ?", (cutoff,) + "DELETE FROM channel_messages WHERE timestamp < ?", + (_cutoff_unix(days),) ) result['channel_messages'] = cursor.rowcount if include_dms: + dm_days = days_dms if days_dms is not None else days cursor = conn.execute( - "DELETE FROM direct_messages WHERE timestamp < ?", (cutoff,) + "DELETE FROM direct_messages WHERE timestamp < ?", + (_cutoff_unix(dm_days),) ) result['direct_messages'] = cursor.rowcount if include_adverts: + adv_days = days_adverts if days_adverts is not None else days cursor = conn.execute( - "DELETE FROM advertisements WHERE timestamp < ?", (cutoff,) + "DELETE FROM advertisements WHERE timestamp < ?", + (_cutoff_unix(adv_days),) ) result['advertisements'] = cursor.rowcount + if include_diagnostics: + diag_days = days_diagnostics if days_diagnostics is not None else min(days, 30) + diag_cutoff = _cutoff_text(diag_days) + cursor = conn.execute( + "DELETE FROM echoes WHERE received_at < ?", (diag_cutoff,) + ) + result['echoes'] = cursor.rowcount + cursor = conn.execute( + "DELETE FROM paths WHERE received_at < ?", (diag_cutoff,) + ) + result['paths'] = cursor.rowcount + cursor = conn.execute( + "DELETE FROM acks WHERE received_at < ?", (diag_cutoff,) + ) + result['acks'] = cursor.rowcount + return result # ================================================================ diff --git a/app/main.py b/app/main.py index a3449b2..11220d3 100644 --- a/app/main.py +++ b/app/main.py @@ -338,10 +338,19 @@ def create_app(): threading.Thread(target=_wait_for_device_name, daemon=True).start() - # Start background scheduler (archiving, contact cleanup, message retention) - from app.archiver.manager import schedule_daily_archiving, init_retention_schedule - schedule_daily_archiving() - init_retention_schedule(db=db) + # Start background scheduler (archiving, contact cleanup, message retention). + # init_*_schedule and the jobs themselves call api.get_*_settings(), which + # touches current_app.db — so the scheduler module needs an app reference + # to push app_context, and the init paths must run inside a context too. + from app.archiver.manager import ( + schedule_daily_archiving, + init_retention_schedule, + set_flask_app as _archiver_set_app, + ) + _archiver_set_app(app) + with app.app_context(): + schedule_daily_archiving() + init_retention_schedule(db=db) logger.info(f"mc-webui v2 started — transport: {config.transport_type}") logger.info(f"Database: {db.db_path}") diff --git a/app/routes/api.py b/app/routes/api.py index 7a46b34..c172ce7 100644 --- a/app/routes/api.py +++ b/app/routes/api.py @@ -212,11 +212,15 @@ CLEANUP_DEFAULTS = { } RETENTION_DEFAULTS = { - 'enabled': False, - 'days': 90, - 'include_dms': False, - 'include_adverts': False, - 'hour': 2 + 'enabled': True, + 'days': 90, # channel_messages + 'days_dms': 90, # direct_messages + 'days_adverts': 60, # advertisements + 'days_diagnostics': 30, # echoes, paths, acks (high-volume debug data) + 'include_dms': True, + 'include_adverts': True, + 'include_diagnostics': True, + 'hour': 3 }