From f8c2a7bb405419f099c877ea452dc571e79bdafc Mon Sep 17 00:00:00 2001 From: Louis King Date: Thu, 4 Jun 2026 14:07:12 +0100 Subject: [PATCH] Rename channel visibility 'public' to 'community' - Rename ChannelVisibility.PUBLIC to ChannelVisibility.COMMUNITY - Update stored value from 'public' to 'community' across model, schema, API, CLI, and frontend - Add Alembic migration to update existing database rows - Consolidate upgrade docs: merge v0.11.0, v0.12.0, v0.13.0 into single v0.11.0 section - Add i18n visibility level translation keys (en, nl) - Update section headings on channels page to use t() for i18n - Keep visibility badges lowercase per UI design --- .env.example | 10 - SCHEMAS.md | 2 +- ...0260604_1200_rename_public_to_community.py | 29 ++ docs/i18n.md | 4 + docs/seeding.md | 2 +- docs/upgrading.md | 47 +-- src/meshcore_hub/api/channel_visibility.py | 4 +- src/meshcore_hub/collector/cli.py | 8 +- src/meshcore_hub/common/database.py | 39 ++- src/meshcore_hub/common/models/channel.py | 6 +- src/meshcore_hub/common/schemas/channels.py | 6 +- src/meshcore_hub/web/app.py | 22 +- .../web/static/js/spa/pages/channels.js | 10 +- src/meshcore_hub/web/static/locales/en.json | 4 + src/meshcore_hub/web/static/locales/nl.json | 6 + tests/test_api/conftest.py | 51 +++ tests/test_api/test_channel_visibility.py | 312 +++++++++++++++++ tests/test_api/test_channels.py | 323 ++++++++++++++++++ tests/test_api/test_dashboard.py | 138 +++++++- tests/test_api/test_messages.py | 126 ++++++- tests/test_collector/test_subscriber.py | 201 +++++++++++ tests/test_common/test_channel_model.py | 241 +++++++++++++ tests/test_common/test_config.py | 33 ++ tests/test_common/test_i18n.py | 7 + 24 files changed, 1545 insertions(+), 86 deletions(-) create mode 100644 alembic/versions/20260604_1200_rename_public_to_community.py create mode 100644 tests/test_api/test_channel_visibility.py create mode 100644 tests/test_api/test_channels.py create mode 100644 tests/test_common/test_channel_model.py diff --git a/.env.example b/.env.example index 57881f0..80aad59 100644 --- a/.env.example +++ b/.env.example @@ -194,16 +194,6 @@ PACKETCAPTURE_EXIT_ON_RECONNECT_FAIL=true # ============================================================================= # The collector subscribes to MQTT events and stores them in the database -# LetsMesh decoder support -# The native Python decoder is always enabled. -# Optional: channel secret keys (comma or space separated) used to decrypt GroupText -# ------------------- -# Channel Settings -# ------------------- -# Channel keys are now managed via the database (channels table). -# Use `meshcore-hub collector channel add --name X --key HEX` or seed via channels.yaml. -# See docs/seeding.md for the channels.yaml format. -# # Refresh interval for reloading channel keys from the database (seconds). # CHANNEL_REFRESH_INTERVAL_SECONDS=300 diff --git a/SCHEMAS.md b/SCHEMAS.md index e270a0b..7b77d1e 100644 --- a/SCHEMAS.md +++ b/SCHEMAS.md @@ -154,7 +154,7 @@ Group/broadcast messages on specific channels. **Field Descriptions**: - `channel_idx`: Channel number (0-255) when available -- `channel_name`: Channel display label (e.g., `"Public"`, `"#test"`) when available +- `channel_name`: Channel display label (e.g., `"Public"`, `"Community"`, `"#test"`) when available - `pubkey_prefix`: First 12 characters of the source public key prefix, used for message identification when available - `path_len`: Number of hops message traveled - `txt_type`: Message type indicator (0=plain, 2=signed, etc.) diff --git a/alembic/versions/20260604_1200_rename_public_to_community.py b/alembic/versions/20260604_1200_rename_public_to_community.py new file mode 100644 index 0000000..cce821d --- /dev/null +++ b/alembic/versions/20260604_1200_rename_public_to_community.py @@ -0,0 +1,29 @@ +"""rename channel visibility public to community + +Revision ID: 20260604_1200 +Revises: 82dff87d6576 +Create Date: 2026-06-04 12:00:00.000000+00:00 + +""" + +from typing import Sequence, Union + +from alembic import op + +# revision identifiers, used by Alembic. +revision: str = "20260604_1200" +down_revision: Union[str, None] = "82dff87d6576" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + op.execute( + "UPDATE channels SET visibility = 'community' WHERE visibility = 'public'" + ) + + +def downgrade() -> None: + op.execute( + "UPDATE channels SET visibility = 'public' WHERE visibility = 'community'" + ) diff --git a/docs/i18n.md b/docs/i18n.md index 1bc4749..6618653 100644 --- a/docs/i18n.md +++ b/docs/i18n.md @@ -390,6 +390,10 @@ Channel management and filter UI: | `name_label` | Channel Name | Form label | | `key_label` | Channel Key (hex) | Form label | | `visibility_label` | Visibility | Form label | +| `visibility_community` | Community | Community visibility section heading | +| `visibility_member` | Member | Member visibility section heading | +| `visibility_operator` | Operator | Operator visibility section heading | +| `visibility_admin` | Admin | Admin visibility section heading | | `enabled_label` | Enabled | Form label | | `channel_hash_label` | Hash | Column header | | `disabled` | Disabled | Disabled channel badge | diff --git a/docs/seeding.md b/docs/seeding.md index 5e19a68..3af2e8e 100644 --- a/docs/seeding.md +++ b/docs/seeding.md @@ -86,7 +86,7 @@ MyChannel: Key rules: - Keys must be uppercase hex, 32 characters (AES-128) or 64 characters (AES-256) -- Seeded channels always have `visibility: public` — to set member/operator/admin visibility, use the CLI or API +- Seeded channels always have `visibility: community` — to set member/operator/admin visibility, use the CLI or API - The `Public` and `test` built-in keys are always loaded into the decoder regardless of database contents - Test channel messages are only stored when a `test` channel row exists in the database with `enabled: true` diff --git a/docs/upgrading.md b/docs/upgrading.md index 9368347..4d93d85 100644 --- a/docs/upgrading.md +++ b/docs/upgrading.md @@ -2,7 +2,15 @@ This guide covers upgrading from a previous MeshCore Hub release to the current version. Check the relevant version section below before upgrading. -## v0.13.0 +## v0.11.0 + +### Channel Visibility Rename: "public" → "community" + +The channel visibility level `"public"` has been renamed to `"community"` to avoid confusion with MeshCore's concept of public channels. All MeshCore channels are private (encrypted) in protocol terms, so "community" better reflects the access level. + +The Alembic migration automatically updates existing `visibility='public'` rows to `visibility='community'`. No manual database changes are required. + +API consumers that filter channels by `visibility=public` must update to `visibility=community`. ### Database-Backed Channel Keys @@ -16,7 +24,7 @@ Channel decryption keys are now managed via the `channels` database table instea | `name` | `VARCHAR(100), UNIQUE` | Channel display name | | `key_hex` | `VARCHAR(64), UNIQUE` | Uppercase hex key (32 or 64 chars) | | `channel_hash` | `VARCHAR(2)` | First byte of SHA-256 of key | -| `visibility` | `VARCHAR(20)` | `public`, `member`, `operator`, or `admin` | +| `visibility` | `VARCHAR(20)` | `community`, `member`, `operator`, or `admin` | | `enabled` | `BOOLEAN` | Whether the channel is active | | `created_at`, `updated_at` | `DATETIME` | Timestamps | @@ -30,7 +38,7 @@ Channel decryption keys are now managed via the `channels` database table instea **Migration steps:** -1. Run `meshcore-hub db upgrade` to create the `channels` table +1. Run `meshcore-hub db upgrade` to create the `channels` table and update visibility values 2. Convert any `COLLECTOR_CHANNEL_KEYS` values to either: - A `channels.yaml` seed file in `SEED_HOME` (see `docs/seeding.md`) - Database rows via CLI: `meshcore-hub collector channel add --name X --key HEX` @@ -39,38 +47,9 @@ Channel decryption keys are now managed via the `channels` database table instea **Test channel behavior change:** Test channel messages (channel_idx 217) are now discarded by default unless a `test` channel row exists in the database with `enabled=true`. Previously this was controlled by `COLLECTOR_INCLUDE_TEST_CHANNEL`. -## v0.12.0 +### Advertisement Route Type & Deduplication -### Advertisement Route Type & Deduplication Improvements - -This release adds route type tracking and improves advertisement deduplication to better distinguish between flood and zero-hop (local) advertisements. - -**New database columns on `advertisements` table:** - -| Column | Type | Description | -|--------|------|-------------| -| `route_type` | `VARCHAR(20), nullable` | Route type: `flood`, `transport_flood`, `direct`, `transport_direct` | -| `advert_timestamp` | `DATETIME, nullable` | Node's own Unix timestamp from the advert payload | - -Both columns are nullable — existing records will have `NULL` values. The Alembic migration adds these columns automatically. - -**Default API filter change:** - -`GET /api/v1/advertisements` now defaults to `route_type=flood,transport_flood`, showing only flood advertisements. Existing records with `route_type=NULL` are included in all default queries to avoid hiding historical data. Pass `route_type=all` to see all types. - -**Dashboard metrics now flood-only:** - -All dashboard advertisement counts (`total_advertisements`, `advertisements_24h`, `advertisements_7d`, `recent_advertisements`, and `/activity`) now count only flood/transport_flood adverts plus NULL (historical records). - -**Deduplication bucket increased from 120s to 300s:** - -Both `compute_advertisement_hash()` and `compute_telemetry_hash()` now use a 5-minute (300-second) deduplication bucket instead of the previous 2-minute (120-second) bucket. This reduces duplicate records when multiple observers report the same event within a 5-minute window. - -**Advertisement deduplication now uses node timestamp:** - -When available, the node's own `advert_timestamp` is used for deduplication bucketing instead of `received_at`. This means the same flood advertisement observed by multiple receivers will correctly deduplicate even if received several minutes apart. Node timestamps that deviate by more than 4 hours from `received_at` are rejected for bucketing (the raw value is still stored). - -## v0.11.0 +Advertisement route type tracking and improved deduplication are included. New `route_type` and `advert_timestamp` columns are added to the `advertisements` table automatically by the migration. The API defaults to showing flood advertisements only. Deduplication uses a 5-minute bucket with node timestamps when available. ### Async SQLite Foreign Key Fix diff --git a/src/meshcore_hub/api/channel_visibility.py b/src/meshcore_hub/api/channel_visibility.py index 022a5b1..7f729a9 100644 --- a/src/meshcore_hub/api/channel_visibility.py +++ b/src/meshcore_hub/api/channel_visibility.py @@ -10,7 +10,7 @@ from sqlalchemy import select from meshcore_hub.api.dependencies import DbSession from meshcore_hub.common.models.channel import Channel -VISIBILITY_LEVELS = {"public": 0, "member": 1, "operator": 2, "admin": 3} +VISIBILITY_LEVELS = {"community": 0, "member": 1, "operator": 2, "admin": 3} def resolve_user_role(request: Request) -> str | None: @@ -34,7 +34,7 @@ def resolve_user_role(request: Request) -> str | None: def get_max_visibility_level(role: str | None) -> int: """Get the maximum visibility level for a given role. - Returns 0 for anonymous users (public only). + Returns 0 for anonymous users (community only). """ if role is None: return 0 diff --git a/src/meshcore_hub/collector/cli.py b/src/meshcore_hub/collector/cli.py index c7f2478..271b19d 100644 --- a/src/meshcore_hub/collector/cli.py +++ b/src/meshcore_hub/collector/cli.py @@ -348,9 +348,9 @@ def channel_list_cmd(ctx: click.Context) -> None: ) @click.option( "--visibility", - type=click.Choice(["public", "member", "operator", "admin"]), - default="public", - help="Channel visibility level (default: public)", + type=click.Choice(["community", "member", "operator", "admin"]), + default="community", + help="Channel visibility level (default: community)", ) @click.pass_context def channel_add_cmd( @@ -633,7 +633,7 @@ def _import_channels( name=name, key_hex=key_hex, channel_hash=Channel.compute_channel_hash(key_hex), - visibility="public", + visibility="community", enabled=enabled, ) session.add(channel) diff --git a/src/meshcore_hub/common/database.py b/src/meshcore_hub/common/database.py index 17b72ae..bb5cb73 100644 --- a/src/meshcore_hub/common/database.py +++ b/src/meshcore_hub/common/database.py @@ -1,11 +1,11 @@ """Database connection and session management.""" from contextlib import asynccontextmanager, contextmanager -from typing import AsyncGenerator, Generator +from typing import Any, AsyncGenerator, Generator from sqlalchemy import create_engine, event from sqlalchemy.engine import Engine -from sqlalchemy.ext.asyncio import AsyncSession, create_async_engine +from sqlalchemy.ext.asyncio import AsyncEngine, AsyncSession, create_async_engine from sqlalchemy.orm import Session, sessionmaker from meshcore_hub.common.models.base import Base @@ -88,6 +88,8 @@ class DatabaseManager: """Database connection manager. Manages database engine and session creation for a component. + The async engine is created lazily on first async session access + to avoid leaking connections when only sync operations are needed. """ def __init__(self, database_url: str, echo: bool = False): @@ -98,6 +100,7 @@ class DatabaseManager: echo: Enable SQL query logging """ self.database_url = database_url + self._echo = echo # Ensure parent directory exists for SQLite databases if database_url.startswith("sqlite:///"): @@ -110,14 +113,24 @@ class DatabaseManager: self.engine = create_database_engine(database_url, echo=echo) self.session_factory = create_session_factory(self.engine) - # Create async engine for async operations - async_url = database_url.replace("sqlite://", "sqlite+aiosqlite://") - self.async_engine = create_async_engine(async_url, echo=echo) + # Lazy-initialized async engine (created on first async_session call) + self._async_engine: AsyncEngine | None = None + self._async_session_factory: Any = None + + def _ensure_async_engine(self) -> None: + """Create the async engine and session factory on first use.""" + if self._async_engine is not None: + return + + from sqlalchemy.ext.asyncio import async_sessionmaker + + async_url = self.database_url.replace("sqlite://", "sqlite+aiosqlite://") + self._async_engine = create_async_engine(async_url, echo=self._echo) # Enable foreign keys for async SQLite engine - if database_url.startswith("sqlite"): + if self.database_url.startswith("sqlite"): - @event.listens_for(self.async_engine.sync_engine, "connect") + @event.listens_for(self._async_engine.sync_engine, "connect") def set_sqlite_pragma_async( dbapi_connection: object, connection_record: object ) -> None: @@ -125,10 +138,8 @@ class DatabaseManager: cursor.execute("PRAGMA foreign_keys=ON") cursor.close() - from sqlalchemy.ext.asyncio import async_sessionmaker - - self.async_session_factory = async_sessionmaker( - self.async_engine, + self._async_session_factory = async_sessionmaker( + self._async_engine, class_=AsyncSession, expire_on_commit=False, ) @@ -183,12 +194,16 @@ class DatabaseManager: result = await session.execute(select(Node)) await session.commit() """ - async with self.async_session_factory() as session: + self._ensure_async_engine() + assert self._async_session_factory is not None + async with self._async_session_factory() as session: yield session def dispose(self) -> None: """Dispose of the database engine and connection pool.""" self.engine.dispose() + if self._async_engine is not None: + self._async_engine.sync_engine.dispose() # Global database manager instance (initialized at runtime) diff --git a/src/meshcore_hub/common/models/channel.py b/src/meshcore_hub/common/models/channel.py index 1347e7c..78f059d 100644 --- a/src/meshcore_hub/common/models/channel.py +++ b/src/meshcore_hub/common/models/channel.py @@ -12,7 +12,7 @@ from meshcore_hub.common.models.base import Base, TimestampMixin, UUIDMixin class ChannelVisibility(str, Enum): """Channel visibility/permission levels.""" - PUBLIC = "public" + COMMUNITY = "community" MEMBER = "member" OPERATOR = "operator" ADMIN = "admin" @@ -26,7 +26,7 @@ class Channel(Base, UUIDMixin, TimestampMixin): name: Channel display name (unique, non-empty) key_hex: Secret key as uppercase hex (supports AES-128 and AES-256) channel_hash: First byte of SHA-256 of key_hex (2-char uppercase hex) - visibility: Permission level (public, member, operator, admin) + visibility: Permission level (community, member, operator, admin) enabled: Whether the channel is active created_at: Record creation timestamp updated_at: Record update timestamp @@ -40,7 +40,7 @@ class Channel(Base, UUIDMixin, TimestampMixin): key_hex: Mapped[str] = mapped_column(String(64), unique=True, nullable=False) channel_hash: Mapped[str] = mapped_column(String(2), nullable=False) visibility: Mapped[str] = mapped_column( - String(20), default=ChannelVisibility.PUBLIC.value, nullable=False + String(20), default=ChannelVisibility.COMMUNITY.value, nullable=False ) enabled: Mapped[bool] = mapped_column(Boolean, default=True, nullable=False) diff --git a/src/meshcore_hub/common/schemas/channels.py b/src/meshcore_hub/common/schemas/channels.py index d428e55..4db6ad4 100644 --- a/src/meshcore_hub/common/schemas/channels.py +++ b/src/meshcore_hub/common/schemas/channels.py @@ -22,8 +22,8 @@ class ChannelCreate(BaseModel): max_length=64, description="Channel secret key as uppercase hex (32 or 64 chars)", ) - visibility: Literal["public", "member", "operator", "admin"] = Field( - default="public", + visibility: Literal["community", "member", "operator", "admin"] = Field( + default="community", description="Channel visibility/permission level", ) enabled: bool = Field( @@ -52,7 +52,7 @@ class ChannelUpdate(BaseModel): max_length=64, description="Channel secret key as uppercase hex", ) - visibility: Optional[Literal["public", "member", "operator", "admin"]] = Field( + visibility: Optional[Literal["community", "member", "operator", "admin"]] = Field( default=None, description="Channel visibility/permission level", ) diff --git a/src/meshcore_hub/web/app.py b/src/meshcore_hub/web/app.py index 6905eb3..c0d671f 100644 --- a/src/meshcore_hub/web/app.py +++ b/src/meshcore_hub/web/app.py @@ -180,16 +180,20 @@ def _build_channel_labels() -> dict[str, str]: from meshcore_hub.common.database import DatabaseManager db = DatabaseManager(settings.effective_database_url) - from meshcore_hub.common.models.channel import Channel + try: + from meshcore_hub.common.models.channel import Channel - with db.session_scope() as session: - channels = session.query(Channel).filter(Channel.enabled.is_(True)).all() - db_decoder = LetsMeshPacketDecoder( - channel_keys=[f"{ch.name}={ch.key_hex}" for ch in channels] - ) - db_labels = db_decoder.channel_labels_by_index() - labels.update(db_labels) - db.dispose() + with db.session_scope() as session: + channels = ( + session.query(Channel).filter(Channel.enabled.is_(True)).all() + ) + db_decoder = LetsMeshPacketDecoder( + channel_keys=[f"{ch.name}={ch.key_hex}" for ch in channels] + ) + db_labels = db_decoder.channel_labels_by_index() + labels.update(db_labels) + finally: + db.dispose() except Exception as e: logger.warning("Failed to load channel labels from database: %s", e) diff --git a/src/meshcore_hub/web/static/js/spa/pages/channels.js b/src/meshcore_hub/web/static/js/spa/pages/channels.js index 22bd1c6..79a5862 100644 --- a/src/meshcore_hub/web/static/js/spa/pages/channels.js +++ b/src/meshcore_hub/web/static/js/spa/pages/channels.js @@ -2,7 +2,7 @@ import { apiGet, apiPost, apiPut, apiDelete } from '../api.js'; import { html, litRender, nothing, t, errorAlert, getConfig, hasRole } from '../components.js'; import { iconChannel, iconPlus, iconEdit, iconTrash, iconLock } from '../icons.js'; -const VISIBILITY_ORDER = ['public', 'member', 'operator', 'admin']; +const VISIBILITY_ORDER = ['community', 'member', 'operator', 'admin']; function renderVisibilityBadge(visibility, oidcEnabled) { if (!oidcEnabled) return nothing; @@ -83,7 +83,7 @@ function renderChannelModal({ channel, isEdit, onSave, onCancel }) { pattern="[0-9A-Fa-f]{32,64}" />` : nothing}