From f72f6d418ab21c7ef1f2d49feede91fbb7c54841 Mon Sep 17 00:00:00 2001 From: MarekWo Date: Sun, 7 Jun 2026 10:55:23 +0200 Subject: [PATCH] feat(db): VACUUM after retention and an Optimize button in Backup modal MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit SQLite DELETE marks pages free but doesn't shrink the file, so the new retention job would keep DBs at their bloated size forever without a follow-up VACUUM. Add db.vacuum() that runs PRAGMA-free VACUUM and reports size_before/size_after/elapsed so callers can surface results. The retention job now calls vacuum() automatically when it deleted at least 1000 rows. Threshold avoids the multi-second VACUUM cost on quiet days. Failure is logged, not raised — a missed VACUUM never crashes the scheduler. Power-user override: new "Optimize now" button in the Database Backup modal triggers VACUUM on demand via POST /api/db/vacuum, alongside a GET /api/db/size that drives the live "Current size" label. This way users don't have to wait until 03:30 to reclaim space after the first big retention pass. Co-Authored-By: Claude Opus 4.7 --- app/archiver/manager.py | 16 +++++++++++ app/database.py | 24 ++++++++++++++++ app/routes/api.py | 38 +++++++++++++++++++++++++ app/static/js/app.js | 62 ++++++++++++++++++++++++++++++++++++++++- app/templates/base.html | 12 ++++++++ 5 files changed, 151 insertions(+), 1 deletion(-) diff --git a/app/archiver/manager.py b/app/archiver/manager.py index dedd6c9..cb99b6f 100644 --- a/app/archiver/manager.py +++ b/app/archiver/manager.py @@ -562,6 +562,22 @@ def _retention_job(): total = sum(result.values()) logger.info(f"Retention job completed: {total} rows deleted ({result})") + # Reclaim free pages so the DB file actually shrinks. SQLite DELETE + # only marks pages free; without VACUUM the file size never drops. + # Threshold avoids a multi-second VACUUM on quiet days when nothing + # meaningful was deleted. + VACUUM_THRESHOLD = 1000 + if total >= VACUUM_THRESHOLD: + try: + stats = _db.vacuum() + logger.info( + f"Post-retention VACUUM: {stats['size_before']:,} → " + f"{stats['size_after']:,} bytes (freed {stats['freed']:,} " + f"in {stats['elapsed_seconds']}s)" + ) + except Exception as e: + logger.warning(f"Post-retention VACUUM failed: {e}", exc_info=True) + except Exception as e: logger.error(f"Retention job failed: {e}", exc_info=True) diff --git a/app/database.py b/app/database.py index f4bf292..5f0b4ea 100644 --- a/app/database.py +++ b/app/database.py @@ -1349,6 +1349,30 @@ class Database: return result + def vacuum(self) -> dict: + """Run VACUUM to reclaim space freed by DELETEs. + + VACUUM rewrites the entire database file, so it briefly takes an + exclusive lock — writers wait, readers continue in WAL mode. Caller + gets back the size before/after and the duration so it can be + surfaced to the user. + """ + size_before = self.db_path.stat().st_size if self.db_path.exists() else 0 + started = time.time() + conn = sqlite3.connect(str(self.db_path), timeout=30, isolation_level=None) + try: + conn.execute("VACUUM") + finally: + conn.close() + elapsed = time.time() - started + size_after = self.db_path.stat().st_size if self.db_path.exists() else 0 + return { + 'size_before': size_before, + 'size_after': size_after, + 'freed': size_before - size_after, + 'elapsed_seconds': round(elapsed, 2), + } + # ================================================================ # Backup # ================================================================ diff --git a/app/routes/api.py b/app/routes/api.py index c172ce7..0754dee 100644 --- a/app/routes/api.py +++ b/app/routes/api.py @@ -4389,6 +4389,44 @@ def update_retention_settings_api(): return jsonify({'success': False, 'error': str(e)}), 500 +@api_bp.route('/db/vacuum', methods=['POST']) +def vacuum_database_api(): + """Run SQLite VACUUM to reclaim space freed by DELETE statements. + + Returned payload includes size_before / size_after / freed (bytes) and + elapsed_seconds so the UI can show how much space was reclaimed. + """ + try: + db = _get_db() + if db is None: + return jsonify({'success': False, 'error': 'Database not available'}), 500 + stats = db.vacuum() + logger.info( + f"Manual VACUUM: {stats['size_before']:,} -> {stats['size_after']:,} " + f"bytes (freed {stats['freed']:,} in {stats['elapsed_seconds']}s)" + ) + return jsonify({'success': True, **stats}) + except Exception as e: + logger.error(f"VACUUM failed: {e}", exc_info=True) + return jsonify({'success': False, 'error': str(e)}), 500 + + +@api_bp.route('/db/size', methods=['GET']) +def get_database_size_api(): + """Return current database file size in bytes (for the Optimize DB UI).""" + try: + db = _get_db() + if db is None: + return jsonify({'success': False, 'error': 'Database not available'}), 500 + from pathlib import Path + path = Path(db.db_path) + size = path.stat().st_size if path.exists() else 0 + return jsonify({'success': True, 'size': size, 'path': str(path)}) + except Exception as e: + logger.error(f"Failed to read DB size: {e}") + return jsonify({'success': False, 'error': str(e)}), 500 + + # ============================================================================= # Read Status (Server-side message read tracking) # ============================================================================= diff --git a/app/static/js/app.js b/app/static/js/app.js index a532458..e81834f 100644 --- a/app/static/js/app.js +++ b/app/static/js/app.js @@ -6373,7 +6373,67 @@ document.addEventListener('DOMContentLoaded', initializeSearch); // ============================================================================= function initializeBackup() { - document.getElementById('backupModal')?.addEventListener('shown.bs.modal', loadBackupList); + const modal = document.getElementById('backupModal'); + if (!modal) return; + modal.addEventListener('shown.bs.modal', () => { + loadBackupList(); + loadDatabaseSize(); + }); +} + +function _formatBytes(n) { + if (!Number.isFinite(n) || n < 0) return '?'; + if (n < 1024) return `${n} B`; + if (n < 1024 * 1024) return `${(n / 1024).toFixed(1)} KB`; + if (n < 1024 * 1024 * 1024) return `${(n / (1024 * 1024)).toFixed(1)} MB`; + return `${(n / (1024 * 1024 * 1024)).toFixed(2)} GB`; +} + +async function loadDatabaseSize() { + const statusEl = document.getElementById('vacuumDbStatus'); + if (!statusEl) return; + try { + const response = await fetch('/api/db/size'); + const data = await response.json(); + if (data.success) { + statusEl.textContent = `Current size: ${_formatBytes(data.size)}`; + } else { + statusEl.textContent = 'Size: unknown'; + } + } catch (error) { + statusEl.textContent = 'Size: unknown'; + } +} + +async function optimizeDatabase() { + const btn = document.getElementById('vacuumDbBtn'); + const statusEl = document.getElementById('vacuumDbStatus'); + if (!btn) return; + + btn.disabled = true; + btn.innerHTML = ' Optimizing…'; + if (statusEl) statusEl.textContent = 'Running VACUUM…'; + + try { + const response = await fetch('/api/db/vacuum', { method: 'POST' }); + const data = await response.json(); + + if (data.success) { + const freed = data.freed > 0 ? `freed ${_formatBytes(data.freed)}` : 'no space to reclaim'; + showNotification(`Optimized: ${freed} in ${data.elapsed_seconds}s`, 'success'); + if (statusEl) statusEl.textContent = `Current size: ${_formatBytes(data.size_after)}`; + } else { + showNotification('Optimize failed: ' + (data.error || 'unknown'), 'danger'); + loadDatabaseSize(); + } + } catch (error) { + console.error('Error running VACUUM:', error); + showNotification('Optimize failed', 'danger'); + loadDatabaseSize(); + } finally { + btn.disabled = false; + btn.innerHTML = ' Optimize now'; + } } async function loadBackupList() { diff --git a/app/templates/base.html b/app/templates/base.html index 6573d85..34d6f0f 100644 --- a/app/templates/base.html +++ b/app/templates/base.html @@ -1212,6 +1212,18 @@ Loading... +
+
Optimize database
+

+ Reclaim space freed by message retention. Runs SQLite VACUUM; + readers keep working but writes are paused for a few seconds. +

+
+ + Current size: … +