From bc70b5c39db557b91356ebff0778a8c8df3f4169 Mon Sep 17 00:00:00 2001 From: Joel Krauska Date: Tue, 4 Nov 2025 20:51:46 -0800 Subject: [PATCH] DB Backups --- .dockerignore | 7 ++++ .gitignore | 4 +++ sample.config.ini | 4 +++ startdb.py | 84 +++++++++++++++++++++++++++++++++++++++++++++-- 4 files changed, 97 insertions(+), 2 deletions(-) diff --git a/.dockerignore b/.dockerignore index 41540f3..d83d076 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,3 +1,10 @@ # This keeps Docker from including hostOS virtual environment folders env/ .venv/ + +# Database files and backups +*.db +*.db-shm +*.db-wal +backups/ +*.db.gz diff --git a/.gitignore b/.gitignore index 6812cb1..43e9b55 100644 --- a/.gitignore +++ b/.gitignore @@ -11,6 +11,10 @@ packets*.db *.db-shm *.db-wal +# Database backups +backups/ +*.db.gz + # Process files meshview-db.pid meshview-web.pid diff --git a/sample.config.ini b/sample.config.ini index 0e64980..50f982a 100644 --- a/sample.config.ini +++ b/sample.config.ini @@ -98,6 +98,10 @@ hour = 2 minute = 00 # Run VACUUM after cleanup vacuum = False +# Enable database backups before cleanup +backup_enabled = False +# Directory to store database backups (relative or absolute path) +backup_dir = ./backups # ------------------------- diff --git a/startdb.py b/startdb.py index 46267fe..eabfade 100644 --- a/startdb.py +++ b/startdb.py @@ -1,7 +1,10 @@ import asyncio import datetime +import gzip import json import logging +import shutil +from pathlib import Path from sqlalchemy import delete @@ -50,11 +53,75 @@ def get_int(config, section, key, default=0): db_lock = asyncio.Lock() +# ------------------------- +# Database backup function +# ------------------------- +async def backup_database(database_url: str, backup_dir: str = ".") -> None: + """ + Create a compressed backup of the database file. + + Args: + database_url: SQLAlchemy connection string + backup_dir: Directory to store backups (default: current directory) + """ + try: + # Extract database file path from connection string + # Format: sqlite+aiosqlite:///path/to/db.db + if not database_url.startswith("sqlite"): + cleanup_logger.warning("Backup only supported for SQLite databases") + return + + db_path = database_url.split("///", 1)[1] if "///" in database_url else None + if not db_path: + cleanup_logger.error("Could not extract database path from connection string") + return + + db_file = Path(db_path) + if not db_file.exists(): + cleanup_logger.error(f"Database file not found: {db_file}") + return + + # Create backup directory if it doesn't exist + backup_path = Path(backup_dir) + backup_path.mkdir(parents=True, exist_ok=True) + + # Generate backup filename with timestamp + timestamp = datetime.datetime.now().strftime("%Y%m%d_%H%M%S") + backup_filename = f"{db_file.stem}_backup_{timestamp}.db.gz" + backup_file = backup_path / backup_filename + + cleanup_logger.info(f"Creating backup: {backup_file}") + + # Copy and compress the database file + with open(db_file, 'rb') as f_in: + with gzip.open(backup_file, 'wb', compresslevel=9) as f_out: + shutil.copyfileobj(f_in, f_out) + + # Get file sizes for logging + original_size = db_file.stat().st_size / (1024 * 1024) # MB + compressed_size = backup_file.stat().st_size / (1024 * 1024) # MB + compression_ratio = (1 - compressed_size / original_size) * 100 if original_size > 0 else 0 + + cleanup_logger.info( + f"Backup created successfully: {backup_file.name} " + f"({original_size:.2f} MB -> {compressed_size:.2f} MB, " + f"{compression_ratio:.1f}% compression)" + ) + + except Exception as e: + cleanup_logger.error(f"Error creating database backup: {e}") + + # ------------------------- # Database cleanup using ORM # ------------------------- async def daily_cleanup_at( - hour: int = 2, minute: int = 0, days_to_keep: int = 14, vacuum_db: bool = True + hour: int = 2, + minute: int = 0, + days_to_keep: int = 14, + vacuum_db: bool = True, + backup_enabled: bool = False, + backup_dir: str = "." ): while True: now = datetime.datetime.now() @@ -72,6 +139,11 @@ async def daily_cleanup_at( cleanup_logger.info(f"Running cleanup for records older than {cutoff}...") try: + # Create backup before cleanup if enabled + if backup_enabled: + database_url = CONFIG["database"]["connection_string"] + await backup_database(database_url, backup_dir) + async with db_lock: # Pause ingestion cleanup_logger.info("Ingestion paused for cleanup.") @@ -187,10 +259,14 @@ async def main(): vacuum_db = get_bool(CONFIG, "cleanup", "vacuum", False) cleanup_hour = get_int(CONFIG, "cleanup", "hour", 2) cleanup_minute = get_int(CONFIG, "cleanup", "minute", 0) + backup_enabled = get_bool(CONFIG, "cleanup", "backup_enabled", False) + backup_dir = CONFIG.get("cleanup", {}).get("backup_dir", "./backups") logger.info(f"Starting MQTT ingestion from {CONFIG['mqtt']['server']}:{CONFIG['mqtt']['port']}") if cleanup_enabled: logger.info(f"Daily cleanup enabled: keeping {cleanup_days} days of data") + if backup_enabled: + logger.info(f"Database backups enabled: storing in {backup_dir}") async with asyncio.TaskGroup() as tg: tg.create_task( @@ -204,7 +280,11 @@ async def main(): ) if cleanup_enabled: - tg.create_task(daily_cleanup_at(cleanup_hour, cleanup_minute, cleanup_days, vacuum_db)) + tg.create_task( + daily_cleanup_at( + cleanup_hour, cleanup_minute, cleanup_days, vacuum_db, backup_enabled, backup_dir + ) + ) else: cleanup_logger.info("Daily cleanup is disabled by configuration.")