From c643c50378ef5c18acf1c3ad668b8093b6ce53b1 Mon Sep 17 00:00:00 2001 From: Louis King Date: Sun, 30 Aug 2026 22:20:16 +0100 Subject: [PATCH] chore(db): remove v0.19 Postgres-transition shims Remove the upgrade-compatibility items scheduled for v0.20 in the v0.19 release notes: - the `meshcore-hub db migrate-to-postgres` command and its `common/db_migrate.py` module (with its dedicated tests) - the `database_backend` settings field and its legacy-backend validator (a leftover DATABASE_BACKEND is now silently ignored) - the `[postgres]` pip extra alias (Dockerfile installs `.` now) `tests/test_main.py` is rewritten as a minimal CLI smoke test to keep entry-point coverage. Docs updated: v0.20.0 upgrade section notes the removals (SQLite deployments must upgrade through v0.19 first), README and database.md pointers reworded, stale .env.example comment dropped. --- .env.example | 3 +- .gitignore | 1 - Dockerfile | 2 +- README.md | 2 +- docs/database.md | 8 +- .../tasks.md | 2 +- docs/upgrading.md | 16 +- pyproject.toml | 4 - src/meshcore_hub/__main__.py | 95 -------- src/meshcore_hub/common/config.py | 31 +-- src/meshcore_hub/common/db_migrate.py | 216 ------------------ tests/test_common/test_config.py | 25 -- tests/test_common/test_db_migrate.py | 207 ----------------- tests/test_main.py | 126 ++-------- 14 files changed, 34 insertions(+), 704 deletions(-) delete mode 100644 src/meshcore_hub/common/db_migrate.py delete mode 100644 tests/test_common/test_db_migrate.py diff --git a/.env.example b/.env.example index 4501953..5ce7018 100644 --- a/.env.example +++ b/.env.example @@ -80,8 +80,7 @@ DATA_HOME=./data # # See docs/database.md for the full reference: production role/database # provisioning, managed/external Postgres, and schema-per-instance (search_path) -# isolation for multiple instances sharing one cluster. Upgrading from an old -# SQLite deployment? See docs/upgrading.md (v0.19 section). +# isolation for multiple instances sharing one cluster. # # DATABASE_HOST=postgres # DATABASE_PORT=5432 diff --git a/.gitignore b/.gitignore index 1e7c69c..0438ffa 100644 --- a/.gitignore +++ b/.gitignore @@ -224,7 +224,6 @@ __marimo__/ # MeshCore Hub specific *.db -meshcore.db # Frontend build artifacts node_modules/ diff --git a/Dockerfile b/Dockerfile index dfcf530..3a10c67 100644 --- a/Dockerfile +++ b/Dockerfile @@ -55,7 +55,7 @@ ARG BUILD_VERSION=dev # Set version in _version.py and install the package RUN sed -i "s|__version__ = \"dev\"|__version__ = \"${BUILD_VERSION}\"|" src/meshcore_hub/_version.py && \ pip install --upgrade pip && \ - pip install ".[postgres]" + pip install "." # ============================================================================= # Stage 3: Runtime - Final production image diff --git a/README.md b/README.md index 878d36a..cce3cac 100644 --- a/README.md +++ b/README.md @@ -8,7 +8,7 @@ Python 3.13+ platform for managing and orchestrating MeshCore mesh networks. > [!WARNING] -> **PostgreSQL-only since v0.19** — MeshCore Hub runs on PostgreSQL (bundled container by default; `docker compose up` is still zero-effort). SQLite support was removed in v0.19; existing deployments can migrate in place with `meshcore-hub db migrate-to-postgres` — see [docs/upgrading.md](docs/upgrading.md) and [docs/database.md](docs/database.md). +> **PostgreSQL-only since v0.19** — MeshCore Hub runs on PostgreSQL (bundled container by default; `docker compose up` is still zero-effort). SQLite support was removed in v0.19 and the in-place migration tool was removed in v0.20 — deployments still on SQLite must upgrade through v0.19 and migrate there first; see [docs/upgrading.md](docs/upgrading.md) and [docs/database.md](docs/database.md). ![MeshCore Hub Web Dashboard](docs/images/web.png) diff --git a/docs/database.md b/docs/database.md index 720b739..c4b2862 100644 --- a/docs/database.md +++ b/docs/database.md @@ -3,7 +3,7 @@ MeshCore Hub runs on **PostgreSQL** — the only supported database backend since v0.19. A bundled container ships with the stack for development (it joins the `core` compose profile via `docker-compose.dev.yml`, so a standard dev `compose up` works with zero configuration); managed/external Postgres and multiple schema-isolated instances sharing one cluster are equally supported — and are the norm in production. > [!NOTE] -> SQLite support (the pre-v0.14 default) was **removed in v0.19**. Existing SQLite deployments can migrate in place with the built-in `meshcore-hub db migrate-to-postgres` command — see [Upgrading from SQLite](#upgrading-from-sqlite) below and the [v0.19 upgrade guide](upgrading.md). +> SQLite support (the pre-v0.14 default) was **removed in v0.19**. Deployments still on SQLite must upgrade through v0.19 and run the in-place migration there (the `meshcore-hub db migrate-to-postgres` command was itself removed in v0.20) — see the [v0.19 upgrade guide](upgrading.md). ## Docker (bundled container) @@ -72,9 +72,3 @@ DATABASE_SCHEMA=meshcorehub_stg The schema is created automatically on `db upgrade` if it does not exist, so no manual `CREATE SCHEMA` is required. Connect both instances to the same `DATABASE_HOST` / `DATABASE_NAME` / `DATABASE_USER`; only `DATABASE_SCHEMA` (and `COMPOSE_PROJECT_NAME`) differ. > **Note:** This is the database-level isolation for instances sharing a Postgres cluster. For running multiple instances on the same Docker host (separate volumes, Traefik routing), see [Multi-Instance Deployments](deployment.md#multi-instance-deployments). - -## Upgrading from SQLite - -Deployments still on SQLite (the pre-v0.14 default, deprecated in v0.14 and removed in v0.19) can be moved to Postgres in place with a single built-in command (`meshcore-hub db migrate-to-postgres`), which copies every table in foreign-key order through the ORM and prints a per-table row-count reconciliation. Downtime is required while writers are stopped; the source SQLite file is never modified. The command is retained through v0.19 and scheduled for removal in v0.20. - -See the **v0.19 upgrade guide** in [upgrading.md](upgrading.md) for the full step-by-step runbook (backup, stop writers, bring up Postgres, run the migration, restart). diff --git a/docs/plans/20260829-2312-remove-sqlite-support/tasks.md b/docs/plans/20260829-2312-remove-sqlite-support/tasks.md index abe7897..e814c30 100644 --- a/docs/plans/20260829-2312-remove-sqlite-support/tasks.md +++ b/docs/plans/20260829-2312-remove-sqlite-support/tasks.md @@ -113,5 +113,5 @@ ## Follow-ups & Coordination (not part of this implementation) -- [ ] Track for v0.20: remove `db migrate-to-postgres` + `common/db_migrate.py` + its tests, the `database_backend` settings field, and the `[postgres]` extra alias (announced in the v0.19 `docs/upgrading.md`) +- [x] Track for v0.20: remove `db migrate-to-postgres` + `common/db_migrate.py` + its tests, the `database_backend` settings field, and the `[postgres]` extra alias (announced in the v0.19 `docs/upgrading.md`) - [ ] When taskifying `docs/plans/20260705-2306-mesh-link-monitoring` (future session): note its SQLite branches (`sqlite_insert` arm, batch-mode DDL, `SQLITE_MAX_VARIABLE_NUMBER` avoidance) are superseded by this plan — implementation targets Postgres-only diff --git a/docs/upgrading.md b/docs/upgrading.md index 4cb0a46..d4325dd 100644 --- a/docs/upgrading.md +++ b/docs/upgrading.md @@ -22,6 +22,14 @@ Upgrading: 2. Ensure a Redis server is reachable from the `api` service (set `REDIS_*` accordingly — in Docker use the `cache` profile or an external instance). 3. Restart the stack and verify `/health/ready` reports `"redis": "connected"`. +### v0.19 Postgres-transition shims removed + +The three upgrade-compatibility items retained through v0.19 (see the v0.19 notes below) are removed: + +- **The `meshcore-hub db migrate-to-postgres` command** (and its `common/db_migrate.py` module) — the in-place SQLite→Postgres data copy. **Still on SQLite?** Upgrade to v0.19 first and run the migration there (see the [v0.19 runbook](#migrating-an-existing-sqlite-deployment-in-place)), then upgrade to v0.20 — or copy the data manually. +- **The `database_backend` settings field** — a leftover `DATABASE_BACKEND` in your environment is now **silently ignored** (previously `sqlite` failed with a targeted upgrade error; any other value than `postgres` was rejected). Remove the variable — it has had no effect since v0.19. +- **The `[postgres]` pip extra** — a deprecated no-op since v0.19 (the Postgres drivers are core dependencies). Update external scripts that install `meshcore-hub[postgres]` to drop the extra. + ## v0.19.0 ### PostgreSQL-only (SQLite support removed) @@ -83,14 +91,6 @@ If you ran a v0.19 pre-release with `docker-compose.prod.yml` while pointing `DA > If you intended to keep the bundled container as this deployment's database, start it explicitly with `--profile postgres` instead of relying on `core`. -#### Scheduled for removal in v0.20 - -The following are retained through v0.19 for upgrade compatibility and removed in v0.20: - -- the `meshcore-hub db migrate-to-postgres` command and its `common/db_migrate.py` module, -- the `database_backend` settings field (the `DATABASE_BACKEND` rejection), -- the `[postgres]` pip extra alias (the drivers are core dependencies since v0.19). - ## v0.18.0 ### Security hardening (fail-closed defaults) diff --git a/pyproject.toml b/pyproject.toml index 2bdb9ab..cf7c666 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -62,10 +62,6 @@ dev = [ "types-paho-mqtt>=1.6.0", "types-PyYAML>=6.0.0", ] -# Deprecated no-op alias: the Postgres drivers are core dependencies since -# v0.19. Kept one release for external install instructions that reference -# `.[postgres]`; removed in v0.20. -postgres = [] [project.scripts] meshcore-hub = "meshcore_hub.__main__:main" diff --git a/src/meshcore_hub/__main__.py b/src/meshcore_hub/__main__.py index a978c6e..43e8f75 100644 --- a/src/meshcore_hub/__main__.py +++ b/src/meshcore_hub/__main__.py @@ -85,101 +85,6 @@ def db_upgrade(revision: str, database_url: str | None) -> None: click.echo("Database upgrade complete.") -@db.command("migrate-to-postgres") -@click.option( - "--source", - type=str, - default=None, - help="Source SQLite URL (default: sqlite:///{DATA_HOME}/collector/meshcore.db)", -) -@click.option( - "--target", - type=str, - default=None, - help="Target Postgres URL (default: the configured DATABASE_* connection)", -) -@click.option("--batch-size", type=int, default=2000, help="Rows per insert batch") -@click.option( - "--truncate", - is_flag=True, - default=False, - help="Delete existing rows from target tables before loading", -) -@click.option( - "--dry-run", - is_flag=True, - default=False, - help="Report source/target row counts without writing", -) -@click.option( - "--no-replication-role", - is_flag=True, - default=False, - help="Don't disable FK triggers via session_replication_role (managed Postgres)", -) -def db_migrate_to_postgres( - source: str | None, - target: str | None, - batch_size: int, - truncate: bool, - dry_run: bool, - no_replication_role: bool, -) -> None: - """Copy data from an existing SQLite database into PostgreSQL. - - Run 'db upgrade' against the target first to create the schema. This command - only moves data and never modifies the source. - """ - from pathlib import Path - - from meshcore_hub.common.config import CollectorSettings - from meshcore_hub.common.db_migrate import migrate_sqlite_to_postgres - - settings = CollectorSettings() - source_url = ( - source or f"sqlite:///{Path(settings.data_home) / 'collector' / 'meshcore.db'}" - ) - target_url = target or settings.effective_database_url - target_schema = settings.effective_database_schema - - click.echo(f"Source: {source_url}") - click.echo(f"Target: {target_url} (schema: {target_schema})") - if dry_run: - click.echo("Mode: dry-run (no writes)") - - try: - result = migrate_sqlite_to_postgres( - source_url, - target_url, - target_schema=target_schema, - batch_size=batch_size, - truncate=truncate, - dry_run=dry_run, - disable_replication_role=no_replication_role, - ) - except (RuntimeError, ValueError) as exc: - raise click.ClickException(str(exc)) from exc - - click.echo("") - if dry_run: - # Preview only: the target is expected to be empty, so no OK/MISMATCH judgement. - click.echo("table (source rows -> current target rows)") - for t in result.tables: - click.echo(f" {t.name:28} {t.source_rows:>8} -> {t.target_rows:>8}") - click.echo("") - click.echo("Dry run complete.") - return - - click.echo("table (source -> target)") - for t in result.tables: - status = "OK" if t.ok else "MISMATCH" - click.echo(f" {t.name:28} {t.source_rows:>8} -> {t.target_rows:>8} {status}") - if not result.ok: - raise click.ClickException("Row-count mismatch between source and target") - click.echo("") - click.echo("Migration complete.") - - @db.command("downgrade") @click.option( "--revision", diff --git a/src/meshcore_hub/common/config.py b/src/meshcore_hub/common/config.py index 74860c4..b2e8686 100644 --- a/src/meshcore_hub/common/config.py +++ b/src/meshcore_hub/common/config.py @@ -3,7 +3,7 @@ from enum import Enum from typing import Optional -from pydantic import Field, field_validator +from pydantic import Field from pydantic_settings import BaseSettings, SettingsConfigDict @@ -40,17 +40,7 @@ class CommonSettings(BaseSettings): ) # Database connection. PostgreSQL is the only backend since v0.19: either - # an explicit DATABASE_URL or the DATABASE_* components below. The legacy - # DATABASE_BACKEND selector is kept for one release purely so a leftover - # `sqlite` setting fails with a targeted upgrade error instead of being - # silently ignored; the field is removed in v0.20. - database_backend: str = Field( - default="postgres", - description=( - "Legacy backend selector retained for one release; 'postgres' is " - "accepted (no-op) and 'sqlite' is rejected (support removed in v0.19)" - ), - ) + # an explicit DATABASE_URL or the DATABASE_* components below. database_url: Optional[str] = Field( default=None, description=( @@ -78,23 +68,6 @@ class CommonSettings(BaseSettings): description="PostgreSQL password (required unless DATABASE_URL is set)", ) - @field_validator("database_backend", mode="before") - @classmethod - def _validate_legacy_backend(cls, value: object) -> str: - """Reject the removed SQLite backend with a targeted upgrade error.""" - raw = "postgres" if value is None else str(value).strip().lower() - if raw == "sqlite": - raise ValueError( - "SQLite support was removed in v0.19 — migrate with " - "`meshcore-hub db migrate-to-postgres`, see docs/upgrading.md" - ) - if raw != "postgres": - raise ValueError( - f"DATABASE_BACKEND must be 'postgres' (the only supported " - f"backend), got {value!r}" - ) - return raw - @property def effective_database_url(self) -> str: """Resolve the SQLAlchemy PostgreSQL URL. diff --git a/src/meshcore_hub/common/db_migrate.py b/src/meshcore_hub/common/db_migrate.py deleted file mode 100644 index a0f8e60..0000000 --- a/src/meshcore_hub/common/db_migrate.py +++ /dev/null @@ -1,216 +0,0 @@ -"""Copy all data from a source (SQLite) database into a target (Postgres) database. - -This powers ``meshcore-hub db migrate-to-postgres`` — the v0.19 upgrade path for -existing SQLite deployments. It is scheduled for removal in v0.20. It operates at -the SQLAlchemy Core level, iterating ``Base.metadata.sorted_tables`` and copying -each table through the ORM's typed columns. The round-trip ``SQLite value -> -Python object -> Postgres value`` is what makes the conversion correct (e.g. -integer ``0/1`` -> ``bool``, JSON ``TEXT`` -> ``dict``, datetime string -> -``timestamptz``) without any per-model code. SQLite is read via Python's stdlib -``sqlite3`` driver — it is not a supported runtime backend. - -The schema must already exist in the target (created by ``db upgrade``); this module -only moves data and never mutates the source. -""" - -from __future__ import annotations - -import logging -from dataclasses import dataclass, field -from datetime import timezone -from typing import Optional - -import sqlalchemy as sa -from sqlalchemy import func, insert, select, text -from sqlalchemy.engine import Engine - -from meshcore_hub.common.database import create_database_engine -from meshcore_hub.common.models.base import Base - -logger = logging.getLogger(__name__) - - -@dataclass -class TableResult: - """Per-table outcome of a migration run.""" - - name: str - source_rows: int - target_rows: int - - @property - def ok(self) -> bool: - return self.source_rows == self.target_rows - - -@dataclass -class MigrationResult: - """Aggregate outcome of a migration run.""" - - tables: list[TableResult] = field(default_factory=list) - dry_run: bool = False - - @property - def ok(self) -> bool: - return all(t.ok for t in self.tables) - - -def _tz_aware_columns(table: sa.Table) -> list[str]: - """Names of timezone-aware DateTime columns on a table. - - SQLite does not persist tzinfo, so these values read back naive and must be - stamped UTC before insert into Postgres ``timestamptz`` (the app always writes - UTC via utc_now()). - """ - return [ - col.name - for col in table.columns - if isinstance(col.type, sa.DateTime) - and bool(getattr(col.type, "timezone", False)) - ] - - -def _count(engine: Engine, table: sa.Table) -> int: - with engine.connect() as conn: - return int(conn.execute(select(func.count()).select_from(table)).scalar() or 0) - - -def _is_superuser(engine: Engine) -> bool: - """Whether the target role can SET session_replication_role (superuser-only).""" - if engine.dialect.name != "postgresql": - return False - try: - with engine.connect() as conn: - value = conn.execute( - text("SELECT current_setting('is_superuser')") - ).scalar() - return str(value).lower() == "on" - except Exception: # pragma: no cover - defensive - return False - - -def _copy_table( - source_engine: Engine, - tgt_conn: sa.engine.Connection, - table: sa.Table, - batch_size: int, -) -> int: - """Stream rows from source and bulk-insert into the target connection.""" - tz_cols = _tz_aware_columns(table) - copied = 0 - with source_engine.connect().execution_options(stream_results=True) as src: - result = src.execute(select(table)) - for partition in result.partitions(batch_size): - rows = [] - for row in partition: - data = dict(row._mapping) - for col in tz_cols: - value = data.get(col) - if value is not None and value.tzinfo is None: - data[col] = value.replace(tzinfo=timezone.utc) - rows.append(data) - if rows: - tgt_conn.execute(insert(table), rows) - copied += len(rows) - return copied - - -def migrate_sqlite_to_postgres( - source_url: str, - target_url: str, - *, - target_schema: Optional[str] = None, - batch_size: int = 2000, - truncate: bool = False, - dry_run: bool = False, - disable_replication_role: bool = False, -) -> MigrationResult: - """Copy every table from ``source_url`` into ``target_url``. - - Args: - source_url: SQLAlchemy URL of the source (SQLite) database. - target_url: SQLAlchemy URL of the target (Postgres) database. - target_schema: Postgres schema to load into (search_path). Defaults to the - DATABASE_SCHEMA env var via the engine. - batch_size: Rows per insert batch. - truncate: Delete existing rows from target tables before loading. - dry_run: Report source/target counts without writing. - disable_replication_role: Skip the session_replication_role trick even if the - target role is a superuser (e.g. to mirror managed-Postgres behaviour). - - Returns: - MigrationResult with per-table source/target row counts. - """ - if not target_url.startswith(("postgresql", "postgres")): - raise ValueError("Target must be a PostgreSQL database URL") - - source_engine = create_database_engine(source_url) - target_engine = create_database_engine(target_url, schema=target_schema) - tables = list(Base.metadata.sorted_tables) # parent-first; excludes alembic_version - - try: - # Pre-flight: schema present + (unless truncating) target empty. - for table in tables: - try: - existing = _count(target_engine, table) - except Exception as exc: # table missing -> schema not initialised - raise RuntimeError( - f"Target table {table.name!r} not found. Run 'meshcore-hub db " - f"upgrade' against the target first." - ) from exc - if existing and not (truncate or dry_run): - raise RuntimeError( - f"Target table {table.name!r} is not empty ({existing} rows). " - f"Refusing to load; pass --truncate to overwrite." - ) - - if dry_run: - result = MigrationResult(dry_run=True) - for table in tables: - result.tables.append( - TableResult( - table.name, - _count(source_engine, table), - _count(target_engine, table), - ) - ) - return result - - use_replica = ( - target_engine.dialect.name == "postgresql" - and not disable_replication_role - and _is_superuser(target_engine) - ) - if not use_replica: - logger.info( - "Not disabling FK triggers (session_replication_role); relying on " - "parent-first table order." - ) - - # Single transaction: all-or-nothing. A failure leaves the target empty. - with target_engine.begin() as tgt: - if use_replica: - tgt.execute(text("SET session_replication_role = replica")) - if truncate: - for table in reversed(tables): # children first - tgt.execute(table.delete()) - for table in tables: - copied = _copy_table(source_engine, tgt, table, batch_size) - logger.info("Copied %s rows into %s", copied, table.name) - if use_replica: - tgt.execute(text("SET session_replication_role = DEFAULT")) - - # Reconcile. - result = MigrationResult() - for table in tables: - result.tables.append( - TableResult( - table.name, - _count(source_engine, table), - _count(target_engine, table), - ) - ) - return result - finally: - source_engine.dispose() - target_engine.dispose() diff --git a/tests/test_common/test_config.py b/tests/test_common/test_config.py index b0035b3..0b3388b 100644 --- a/tests/test_common/test_config.py +++ b/tests/test_common/test_config.py @@ -1,7 +1,6 @@ """Tests for configuration settings.""" import pytest -from pydantic import ValidationError from meshcore_hub.common.config import ( CommonSettings, @@ -376,27 +375,3 @@ class TestDatabaseUrlResolution: ) assert settings.effective_database_url == "postgresql+psycopg2://u:p@h/db" - - def test_backend_defaults_to_postgres_noop(self) -> None: - """DATABASE_BACKEND=postgres (the default) is accepted as a no-op.""" - settings = CommonSettings( - _env_file=None, - database_backend="postgres", - database_host="pg", - database_password="pw", - ) - - assert settings.database_backend == "postgres" - assert settings.effective_database_url.startswith("postgresql+psycopg2://") - - def test_backend_sqlite_is_rejected_with_upgrade_hint(self) -> None: - """DATABASE_BACKEND=sqlite fails with the targeted v0.19 upgrade error.""" - with pytest.raises( - ValidationError, match="SQLite support was removed in v0.19" - ): - _ = CommonSettings(_env_file=None, database_backend="sqlite") - - def test_backend_unknown_value_is_rejected(self) -> None: - """Anything other than 'postgres' is rejected.""" - with pytest.raises(ValidationError, match="must be 'postgres'"): - _ = CommonSettings(_env_file=None, database_backend="mysql") diff --git a/tests/test_common/test_db_migrate.py b/tests/test_common/test_db_migrate.py deleted file mode 100644 index d886679..0000000 --- a/tests/test_common/test_db_migrate.py +++ /dev/null @@ -1,207 +0,0 @@ -"""Tests for the SQLite -> Postgres data migration helper. - -The full round-trip is validated against a live Postgres; here we cover the -dialect-agnostic pieces (the Postgres-target guard, tz-aware column detection, and -the Core copy/stream/normalize logic exercised SQLite -> SQLite) so they run in CI. -""" - -from datetime import datetime - -import pytest -from sqlalchemy import create_engine, func, select - -from meshcore_hub.common import db_migrate -from meshcore_hub.common.db_migrate import ( - _copy_table, - _is_superuser, - _tz_aware_columns, - migrate_sqlite_to_postgres, -) -from meshcore_hub.common.models import Base - -# Import models so their tables register on Base.metadata. -import meshcore_hub.common.models.node # noqa: F401 -import meshcore_hub.common.models.raw_packet # noqa: F401 - - -def test_target_must_be_postgres() -> None: - """The command refuses a non-Postgres target.""" - with pytest.raises(ValueError, match="PostgreSQL"): - migrate_sqlite_to_postgres("sqlite:///a.db", "sqlite:///b.db") - - -def test_tz_aware_columns_detects_timestamptz() -> None: - """Timezone-aware DateTime columns are identified for UTC normalization.""" - cols = _tz_aware_columns(Base.metadata.tables["raw_packets"]) - - assert "received_at" in cols - assert "created_at" in cols - assert "packet_hash" not in cols # not a datetime - - -def test_copy_table_roundtrips_rows_and_boolean() -> None: - """_copy_table streams rows across engines, preserving the boolean value. - - (UTC normalization of naive datetimes is validated against Postgres, where - timestamptz round-trips reliably; SQLite does not retain tzinfo.) - """ - nodes = Base.metadata.tables["nodes"] - src = create_engine("sqlite:///:memory:") - dst = create_engine("sqlite:///:memory:") - Base.metadata.create_all(src) - Base.metadata.create_all(dst) - - now = datetime(2026, 6, 13, 10, 0) - with src.begin() as conn: - conn.execute( - nodes.insert(), - [ - { - "id": f"n{i}", - "public_key": f"key{i}", - "is_observer": i == 0, - "first_seen": now, - "created_at": now, - "updated_at": now, - } - for i in range(3) - ], - ) - - with dst.begin() as conn: - copied = _copy_table(src, conn, nodes, batch_size=2) # forces >1 batch - - assert copied == 3 - with dst.connect() as conn: - assert conn.execute(select(func.count()).select_from(nodes)).scalar() == 3 - observers = conn.execute( - select(func.count()).select_from(nodes).where(nodes.c.is_observer.is_(True)) - ).scalar() - assert observers == 1 # boolean preserved across the copy - - src.dispose() - dst.dispose() - - -def test_is_superuser_false_for_non_postgres() -> None: - """session_replication_role is Postgres-only; SQLite is never a superuser target.""" - engine = create_engine("sqlite:///:memory:") - try: - assert _is_superuser(engine) is False - finally: - engine.dispose() - - -def _seed_nodes(engine, count: int) -> None: - nodes = Base.metadata.tables["nodes"] - now = datetime(2026, 6, 13, 10, 0) - with engine.begin() as conn: - conn.execute( - nodes.insert(), - [ - { - "id": f"n{i}", - "public_key": f"key{i}", - "is_observer": i == 0, - "first_seen": now, - "created_at": now, - "updated_at": now, - } - for i in range(count) - ], - ) - - -@pytest.fixture -def _patch_engines(monkeypatch, tmp_path): - """Route create_database_engine to SQLite files keyed by URL. - - Lets the Postgres-targeting migration flow run end-to-end SQLite -> SQLite in CI: - a ``postgresql://`` target URL satisfies the guard while the real engine is a - local SQLite file, so the schema/empty/truncate/copy logic is exercised without - a live Postgres. - """ - src_url = f"sqlite:///{tmp_path / 'src.db'}" - target_url = "postgresql://fake/target" # satisfies the Postgres guard - src_engine = create_engine(src_url) - tgt_engine = create_engine(f"sqlite:///{tmp_path / 'tgt.db'}") - Base.metadata.create_all(src_engine) - - def fake_create_engine(url, echo=False, schema=None): - if url == src_url: - return src_engine - if url == target_url: - return tgt_engine - raise AssertionError(f"unexpected url {url!r}") - - monkeypatch.setattr(db_migrate, "create_database_engine", fake_create_engine) - return src_url, target_url, src_engine, tgt_engine - - -def _count_rows(engine) -> int: - nodes = Base.metadata.tables["nodes"] - with engine.connect() as conn: - return int(conn.execute(select(func.count()).select_from(nodes)).scalar() or 0) - - -def test_migrate_dry_run_reports_counts_without_writing(_patch_engines) -> None: - """Dry run reports source/target counts and leaves the target untouched.""" - src_url, target_url, src_engine, tgt_engine = _patch_engines - Base.metadata.create_all(tgt_engine) - _seed_nodes(src_engine, 3) - - result = migrate_sqlite_to_postgres(src_url, target_url, dry_run=True) - - assert result.dry_run is True - nodes_result = next(t for t in result.tables if t.name == "nodes") - assert nodes_result.source_rows == 3 - assert nodes_result.target_rows == 0 - assert _count_rows(tgt_engine) == 0 # nothing written - - -def test_migrate_copies_all_rows(_patch_engines) -> None: - """A full run copies rows and reconciles source/target counts as OK.""" - src_url, target_url, src_engine, tgt_engine = _patch_engines - Base.metadata.create_all(tgt_engine) - _seed_nodes(src_engine, 5) - - result = migrate_sqlite_to_postgres(src_url, target_url, batch_size=2) - - assert result.ok is True - assert _count_rows(tgt_engine) == 5 - nodes_result = next(t for t in result.tables if t.name == "nodes") - assert nodes_result.source_rows == nodes_result.target_rows == 5 - - -def test_migrate_refuses_non_empty_target(_patch_engines) -> None: - """Without --truncate, a non-empty target is refused before any write.""" - src_url, target_url, src_engine, tgt_engine = _patch_engines - Base.metadata.create_all(tgt_engine) - _seed_nodes(src_engine, 2) - _seed_nodes(tgt_engine, 1) - - with pytest.raises(RuntimeError, match="not empty"): - migrate_sqlite_to_postgres(src_url, target_url) - - -def test_migrate_truncate_overwrites_target(_patch_engines) -> None: - """--truncate clears existing target rows before loading from source.""" - src_url, target_url, src_engine, tgt_engine = _patch_engines - Base.metadata.create_all(tgt_engine) - _seed_nodes(src_engine, 2) - _seed_nodes(tgt_engine, 4) - - result = migrate_sqlite_to_postgres(src_url, target_url, truncate=True) - - assert result.ok is True - assert _count_rows(tgt_engine) == 2 - - -def test_migrate_errors_when_target_schema_missing(_patch_engines) -> None: - """A target without the schema (no create_all) fails with a clear message.""" - src_url, target_url, src_engine, tgt_engine = _patch_engines - # Note: target schema intentionally not created. - _seed_nodes(src_engine, 1) - - with pytest.raises(RuntimeError, match="Run 'meshcore-hub db upgrade'"): - migrate_sqlite_to_postgres(src_url, target_url) diff --git a/tests/test_main.py b/tests/test_main.py index 8a1618e..965e3d7 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -1,127 +1,39 @@ -"""Tests for the top-level CLI, focused on ``db migrate-to-postgres``. +"""Smoke tests for the top-level CLI entry point. -The migration engine itself is covered in test_common/test_db_migrate.py; here we -verify the command's wiring: option plumbing, dry-run vs. real output, and how the -MigrationResult (or an error) maps onto exit codes and messages. +Verify the ``cli`` group wiring: version reporting, help output, and the +registered component subcommands (collector, api, web, db). """ -from unittest.mock import patch - from click.testing import CliRunner +from meshcore_hub import __version__ from meshcore_hub.__main__ import cli -from meshcore_hub.common.db_migrate import MigrationResult, TableResult -def _result(*tables: TableResult, dry_run: bool = False) -> MigrationResult: - return MigrationResult(tables=list(tables), dry_run=dry_run) - - -def test_migrate_to_postgres_success() -> None: - """A matching run reports OK per table and exits 0.""" +def test_version_reports_package_version() -> None: + """--version prints the package version and exits 0.""" runner = CliRunner() - fake = _result(TableResult("nodes", 5, 5)) - - with patch( - "meshcore_hub.common.db_migrate.migrate_sqlite_to_postgres", - return_value=fake, - ) as mock_migrate: - result = runner.invoke( - cli, - [ - "db", - "migrate-to-postgres", - "--source", - "sqlite:///src.db", - "--target", - "postgresql://u@h/db", - ], - ) + result = runner.invoke(cli, ["--version"]) assert result.exit_code == 0 - assert "nodes" in result.output - assert "OK" in result.output - assert "Migration complete." in result.output - # Flags thread through to the engine call. - _, kwargs = mock_migrate.call_args - assert kwargs["dry_run"] is False - assert kwargs["truncate"] is False + assert __version__ in result.output -def test_migrate_to_postgres_dry_run() -> None: - """Dry run prints a preview and never renders OK/MISMATCH judgements.""" +def test_help_lists_component_commands() -> None: + """--help advertises the component subcommands.""" runner = CliRunner() - fake = _result(TableResult("nodes", 3, 0), dry_run=True) - - with patch( - "meshcore_hub.common.db_migrate.migrate_sqlite_to_postgres", - return_value=fake, - ) as mock_migrate: - result = runner.invoke( - cli, - [ - "db", - "migrate-to-postgres", - "--source", - "sqlite:///src.db", - "--target", - "postgresql://u@h/db", - "--dry-run", - ], - ) + result = runner.invoke(cli, ["--help"]) assert result.exit_code == 0 - assert "dry-run" in result.output - assert "Dry run complete." in result.output - assert "OK" not in result.output - assert mock_migrate.call_args.kwargs["dry_run"] is True + for command in ("collector", "api", "web", "db"): + assert command in result.output -def test_migrate_to_postgres_mismatch_exits_nonzero() -> None: - """A row-count mismatch surfaces as a ClickException (non-zero exit).""" +def test_db_help_lists_migration_commands() -> None: + """db --help advertises the Alembic migration commands.""" runner = CliRunner() - fake = _result(TableResult("nodes", 5, 4)) # ok == False + result = runner.invoke(cli, ["db", "--help"]) - with patch( - "meshcore_hub.common.db_migrate.migrate_sqlite_to_postgres", - return_value=fake, - ): - result = runner.invoke( - cli, - [ - "db", - "migrate-to-postgres", - "--source", - "sqlite:///src.db", - "--target", - "postgresql://u@h/db", - ], - ) - - assert result.exit_code != 0 - assert "MISMATCH" in result.output - assert "mismatch" in result.output.lower() - - -def test_migrate_to_postgres_value_error_becomes_click_exception() -> None: - """A ValueError from the engine (e.g. bad target) maps to a clean CLI error.""" - runner = CliRunner() - - with patch( - "meshcore_hub.common.db_migrate.migrate_sqlite_to_postgres", - side_effect=ValueError("Target must be a PostgreSQL database URL"), - ): - result = runner.invoke( - cli, - [ - "db", - "migrate-to-postgres", - "--source", - "sqlite:///src.db", - "--target", - "sqlite:///bad.db", - ], - ) - - assert result.exit_code != 0 - assert "PostgreSQL" in result.output + assert result.exit_code == 0 + assert "upgrade" in result.output + assert "downgrade" in result.output