mirror of
https://github.com/MarekWo/mc-webui.git
synced 2026-07-06 09:51:14 +02:00
feat(v2): Import archive .msgs files + DB-based message history
Migration now imports all archive files (oldest first) in addition to the live .msgs file, with deduplication. Archives endpoint and message history now query SQLite by date instead of reading .msgs files. Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
MarekWo
+124
-23
@@ -5,22 +5,22 @@ Runs automatically on startup if .msgs file exists and database is empty.
|
||||
Can also be run manually: python -m app.migrate_v1
|
||||
|
||||
Migrates:
|
||||
- Live .msgs file (today's messages)
|
||||
- Archive .msgs files (historical messages)
|
||||
- Channel messages (CHAN, SENT_CHAN)
|
||||
- Direct messages (PRIV, SENT_MSG)
|
||||
"""
|
||||
|
||||
import json
|
||||
import logging
|
||||
import time
|
||||
from pathlib import Path
|
||||
from datetime import datetime
|
||||
from typing import Optional
|
||||
from typing import Optional, List
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _find_msgs_file(data_dir: Path, device_name: str) -> Optional[Path]:
|
||||
"""Find the .msgs file for the given device name."""
|
||||
"""Find the live .msgs file for the given device name."""
|
||||
msgs_file = data_dir / f"{device_name}.msgs"
|
||||
if msgs_file.exists():
|
||||
return msgs_file
|
||||
@@ -35,9 +35,41 @@ def _find_msgs_file(data_dir: Path, device_name: str) -> Optional[Path]:
|
||||
return None
|
||||
|
||||
|
||||
def _find_archive_files(data_dir: Path, device_name: str) -> List[Path]:
|
||||
"""Find all archive .msgs files, sorted oldest first."""
|
||||
archive_files = []
|
||||
|
||||
# Check common archive locations
|
||||
archive_dirs = [
|
||||
data_dir / 'archive', # /data/archive/
|
||||
data_dir.parent / 'archive', # sibling archive dir
|
||||
]
|
||||
|
||||
for archive_dir in archive_dirs:
|
||||
if archive_dir.exists():
|
||||
# Pattern: DeviceName.YYYY-MM-DD.msgs
|
||||
for f in archive_dir.glob(f"{device_name}.*.msgs"):
|
||||
# Validate it's an archive file (has date in name)
|
||||
parts = f.stem.split('.')
|
||||
if len(parts) >= 2:
|
||||
archive_files.append(f)
|
||||
|
||||
# Also check data_dir itself for archives
|
||||
for f in data_dir.glob(f"{device_name}.*.msgs"):
|
||||
parts = f.stem.split('.')
|
||||
if len(parts) >= 2 and f not in archive_files:
|
||||
archive_files.append(f)
|
||||
|
||||
# Sort by filename (which sorts by date since format is Name.YYYY-MM-DD)
|
||||
archive_files.sort(key=lambda f: f.name)
|
||||
|
||||
return archive_files
|
||||
|
||||
|
||||
def migrate_v1_data(db, data_dir: Path, device_name: str) -> dict:
|
||||
"""
|
||||
Import v1 .msgs data into v2 SQLite database.
|
||||
Imports both live .msgs file and all archive files.
|
||||
|
||||
Args:
|
||||
db: Database instance
|
||||
@@ -47,26 +79,69 @@ def migrate_v1_data(db, data_dir: Path, device_name: str) -> dict:
|
||||
Returns:
|
||||
dict with migration stats
|
||||
"""
|
||||
msgs_file = _find_msgs_file(data_dir, device_name)
|
||||
if not msgs_file:
|
||||
logger.info("No .msgs file found, skipping v1 migration")
|
||||
return {'status': 'skipped', 'reason': 'no_msgs_file'}
|
||||
|
||||
stats = {
|
||||
'channel_messages': 0,
|
||||
'direct_messages': 0,
|
||||
'skipped': 0,
|
||||
'errors': 0,
|
||||
'file': str(msgs_file),
|
||||
'files_processed': 0,
|
||||
}
|
||||
|
||||
logger.info(f"Starting v1 data migration from {msgs_file}")
|
||||
# Collect all files to import: archives first (oldest), then live
|
||||
files_to_import = []
|
||||
|
||||
archive_files = _find_archive_files(data_dir, device_name)
|
||||
if archive_files:
|
||||
files_to_import.extend(archive_files)
|
||||
logger.info(f"Found {len(archive_files)} archive files to migrate")
|
||||
|
||||
live_file = _find_msgs_file(data_dir, device_name)
|
||||
if live_file:
|
||||
files_to_import.append(live_file)
|
||||
|
||||
if not files_to_import:
|
||||
logger.info("No .msgs files found, skipping v1 migration")
|
||||
return {'status': 'skipped', 'reason': 'no_msgs_files'}
|
||||
|
||||
logger.info(f"Starting v1 data migration: {len(files_to_import)} files to process")
|
||||
|
||||
# Track seen timestamps+text to avoid duplicates across archive and live file
|
||||
seen_channel = set()
|
||||
seen_dm = set()
|
||||
|
||||
for msgs_file in files_to_import:
|
||||
file_stats = _import_msgs_file(
|
||||
db, msgs_file, device_name, seen_channel, seen_dm
|
||||
)
|
||||
stats['channel_messages'] += file_stats['channel_messages']
|
||||
stats['direct_messages'] += file_stats['direct_messages']
|
||||
stats['skipped'] += file_stats['skipped']
|
||||
stats['errors'] += file_stats['errors']
|
||||
stats['files_processed'] += 1
|
||||
|
||||
stats['status'] = 'completed'
|
||||
logger.info(
|
||||
f"v1 migration complete: {stats['files_processed']} files, "
|
||||
f"{stats['channel_messages']} channel msgs, "
|
||||
f"{stats['direct_messages']} DMs, {stats['skipped']} skipped, "
|
||||
f"{stats['errors']} errors"
|
||||
)
|
||||
return stats
|
||||
|
||||
|
||||
def _import_msgs_file(db, msgs_file: Path, device_name: str,
|
||||
seen_channel: set, seen_dm: set) -> dict:
|
||||
"""Import a single .msgs file. Returns per-file stats."""
|
||||
stats = {'channel_messages': 0, 'direct_messages': 0, 'skipped': 0, 'errors': 0}
|
||||
|
||||
logger.info(f"Importing {msgs_file.name}...")
|
||||
|
||||
try:
|
||||
lines = msgs_file.read_text(encoding='utf-8', errors='replace').splitlines()
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to read .msgs file: {e}")
|
||||
return {'status': 'error', 'reason': str(e)}
|
||||
logger.error(f"Failed to read {msgs_file}: {e}")
|
||||
stats['errors'] += 1
|
||||
return stats
|
||||
|
||||
for line_num, raw_line in enumerate(lines, 1):
|
||||
raw_line = raw_line.strip()
|
||||
@@ -83,13 +158,38 @@ def migrate_v1_data(db, data_dir: Path, device_name: str) -> dict:
|
||||
|
||||
try:
|
||||
if msg_type in ('CHAN', 'SENT_CHAN'):
|
||||
# Dedup key: timestamp + first 50 chars of text
|
||||
ts = entry.get('timestamp', 0)
|
||||
text = entry.get('text', '')[:50]
|
||||
dedup = (ts, text)
|
||||
if dedup in seen_channel:
|
||||
stats['skipped'] += 1
|
||||
continue
|
||||
seen_channel.add(dedup)
|
||||
|
||||
_migrate_channel_msg(db, entry, device_name)
|
||||
stats['channel_messages'] += 1
|
||||
elif msg_type == 'PRIV':
|
||||
ts = entry.get('timestamp', 0)
|
||||
text = entry.get('text', '')[:50]
|
||||
dedup = (ts, text)
|
||||
if dedup in seen_dm:
|
||||
stats['skipped'] += 1
|
||||
continue
|
||||
seen_dm.add(dedup)
|
||||
|
||||
_migrate_dm_incoming(db, entry)
|
||||
stats['direct_messages'] += 1
|
||||
elif msg_type == 'SENT_MSG':
|
||||
if entry.get('txt_type', 0) == 0: # Only private messages
|
||||
ts = entry.get('timestamp', 0)
|
||||
text = entry.get('text', '')[:50]
|
||||
dedup = (ts, text)
|
||||
if dedup in seen_dm:
|
||||
stats['skipped'] += 1
|
||||
continue
|
||||
seen_dm.add(dedup)
|
||||
|
||||
_migrate_dm_outgoing(db, entry, device_name)
|
||||
stats['direct_messages'] += 1
|
||||
else:
|
||||
@@ -99,13 +199,12 @@ def migrate_v1_data(db, data_dir: Path, device_name: str) -> dict:
|
||||
except Exception as e:
|
||||
stats['errors'] += 1
|
||||
if stats['errors'] <= 5:
|
||||
logger.warning(f"Migration error on line {line_num}: {e}")
|
||||
logger.warning(f"Migration error in {msgs_file.name} line {line_num}: {e}")
|
||||
|
||||
stats['status'] = 'completed'
|
||||
logger.info(
|
||||
f"v1 migration complete: {stats['channel_messages']} channel msgs, "
|
||||
f"{stats['direct_messages']} DMs, {stats['skipped']} skipped, "
|
||||
f"{stats['errors']} errors"
|
||||
f" {msgs_file.name}: {stats['channel_messages']} chan, "
|
||||
f"{stats['direct_messages']} DMs, {stats['skipped']} skip, "
|
||||
f"{stats['errors']} err"
|
||||
)
|
||||
return stats
|
||||
|
||||
@@ -182,8 +281,6 @@ def _migrate_dm_outgoing(db, entry: dict, device_name: str):
|
||||
return
|
||||
|
||||
# For outgoing DMs, we don't have recipient pubkey in v1 data.
|
||||
# Use empty string — the messages will appear once a contact with
|
||||
# matching name is found.
|
||||
# In v1, conversation_id was "name_{recipient}" — we store the name
|
||||
# in raw_json for reference.
|
||||
recipient = entry.get('recipient', entry.get('name', ''))
|
||||
@@ -235,9 +332,13 @@ def _lookup_pubkey_by_name(db, name: str) -> Optional[str]:
|
||||
|
||||
|
||||
def should_migrate(db, data_dir: Path, device_name: str) -> bool:
|
||||
"""Check if migration is needed: .msgs exists and DB has no messages."""
|
||||
msgs_file = _find_msgs_file(data_dir, device_name)
|
||||
if not msgs_file:
|
||||
"""Check if migration is needed: .msgs files exist and DB has no messages."""
|
||||
# Check for live file
|
||||
has_live = _find_msgs_file(data_dir, device_name) is not None
|
||||
# Check for archive files
|
||||
has_archives = len(_find_archive_files(data_dir, device_name)) > 0
|
||||
|
||||
if not has_live and not has_archives:
|
||||
return False
|
||||
|
||||
# Only migrate if DB is empty (no channel messages and no DMs)
|
||||
|
||||
Reference in New Issue
Block a user