Merge branch 'main' of github.com:ipnet-mesh/meshcore-hub into feat/postgres-support

This commit is contained in:
Louis King
2026-06-14 22:53:20 +01:00
13 changed files with 335 additions and 122 deletions
+4
View File
@@ -1,3 +1,7 @@
if has nix && declare -F use_nix >/dev/null; then
use nix
fi
if [ -f .venv/bin/activate ]; then
source .venv/bin/activate
fi
+27 -1
View File
@@ -3,13 +3,33 @@ name: CI
on:
push:
branches: [main]
paths-ignore:
- "**/*.md"
- "docs/**"
- ".env.example"
- "LICENSE"
- "FUNDING.yml"
pull_request:
branches: [main]
paths-ignore:
- "**/*.md"
- "docs/**"
- ".env.example"
- "LICENSE"
- "FUNDING.yml"
permissions:
contents: read
concurrency:
group: ${{ github.workflow }}-${{ github.ref }}
cancel-in-progress: ${{ github.event_name == 'pull_request' }}
jobs:
lint:
name: Lint
runs-on: ubuntu-latest
timeout-minutes: 15
steps:
- uses: actions/checkout@v6
@@ -17,6 +37,7 @@ jobs:
uses: actions/setup-python@v6
with:
python-version-file: ".python-version"
cache: pip
- name: Run pre-commit
uses: pre-commit/action@v3.0.1
@@ -24,6 +45,7 @@ jobs:
test:
name: Test
runs-on: ubuntu-latest
timeout-minutes: 30
steps:
- uses: actions/checkout@v6
@@ -31,6 +53,7 @@ jobs:
uses: actions/setup-python@v6
with:
python-version-file: ".python-version"
cache: pip
- name: Install dependencies
run: |
@@ -39,7 +62,7 @@ jobs:
- name: Run tests with pytest
run: |
pytest --cov=meshcore_hub --cov-report=xml --cov-report=term-missing --junitxml=junit.xml -o junit_family=legacy
pytest -nauto --cov=meshcore_hub --cov-report=xml --cov-report=term-missing --junitxml=junit.xml -o junit_family=legacy
- name: Upload coverage to Codecov
uses: codecov/codecov-action@v7
@@ -60,7 +83,9 @@ jobs:
build:
name: Build Package
runs-on: ubuntu-latest
timeout-minutes: 15
needs: [lint, test]
if: github.event_name == 'push'
steps:
- uses: actions/checkout@v6
@@ -68,6 +93,7 @@ jobs:
uses: actions/setup-python@v6
with:
python-version-file: ".python-version"
cache: pip
- name: Install build tools
run: |
+50 -3
View File
@@ -15,38 +15,79 @@ env:
UPSTREAM_REPO: michaelhart/meshcore-mqtt-broker
IMAGE_NAME: ipnet-mesh/meshcore-mqtt-broker
permissions:
contents: read
packages: write
concurrency:
group: ${{ github.workflow }}
cancel-in-progress: false
jobs:
build:
name: Build and Push MQTT Broker
runs-on: ubuntu-latest
permissions:
contents: read
packages: write
timeout-minutes: 60
steps:
- name: Check upstream for new commits
id: check
run: |
set -euo pipefail
UPSTREAM_SHA="$(git ls-remote "https://github.com/${UPSTREAM_REPO}.git" "refs/heads/${{ inputs.ref || 'main' }}" | awk '{print $1}')"
if [ -z "${UPSTREAM_SHA}" ]; then
UPSTREAM_SHA="$(git ls-remote "https://github.com/${UPSTREAM_REPO}.git" "${{ inputs.ref || 'main' }}" | awk '{print $1}')"
fi
echo "upstream_sha=${UPSTREAM_SHA}" >> "$GITHUB_OUTPUT"
echo "Latest upstream SHA: ${UPSTREAM_SHA}"
- name: Cache last-built upstream SHA
id: cache
uses: actions/cache@v5
with:
path: .last-upstream-sha
key: mqtt-broker-upstream-${{ steps.check.outputs.upstream_sha }}
- name: Decide whether to build
id: decide
if: steps.cache.outputs.cache-hit != 'true'
run: echo "changed=true" >> "$GITHUB_OUTPUT"
- name: Persist upstream SHA for cache write
if: steps.decide.outputs.changed == 'true'
run: echo "${{ steps.check.outputs.upstream_sha }}" > .last-upstream-sha
- name: Checkout this repo (sparse)
if: steps.decide.outputs.changed == 'true'
uses: actions/checkout@v6
with:
persist-credentials: false
sparse-checkout: |
etc/docker/meshcore-mqtt-broker
- name: Checkout upstream source
if: steps.decide.outputs.changed == 'true'
uses: actions/checkout@v6
with:
persist-credentials: false
repository: ${{ env.UPSTREAM_REPO }}
ref: ${{ inputs.ref || 'main' }}
path: upstream
- name: Copy Dockerfile into upstream source
if: steps.decide.outputs.changed == 'true'
run: cp etc/docker/meshcore-mqtt-broker/Dockerfile upstream/Dockerfile
- name: Set up QEMU
if: steps.decide.outputs.changed == 'true'
uses: docker/setup-qemu-action@v4
- name: Set up Docker Buildx
if: steps.decide.outputs.changed == 'true'
uses: docker/setup-buildx-action@v4
- name: Log in to Container Registry
if: steps.decide.outputs.changed == 'true'
uses: docker/login-action@v4
with:
registry: ${{ env.REGISTRY }}
@@ -54,6 +95,7 @@ jobs:
password: ${{ secrets.GITHUB_TOKEN }}
- name: Extract metadata
if: steps.decide.outputs.changed == 'true'
id: meta
uses: docker/metadata-action@v6
with:
@@ -63,6 +105,7 @@ jobs:
type=sha
- name: Build and push Docker image
if: steps.decide.outputs.changed == 'true'
uses: docker/build-push-action@v7
with:
context: ./upstream
@@ -73,3 +116,7 @@ jobs:
labels: ${{ steps.meta.outputs.labels }}
cache-from: type=gha
cache-to: type=gha,mode=max
- name: Skip build (unchanged)
if: steps.decide.outputs.changed != 'true'
run: echo "Upstream ${{ steps.check.outputs.upstream_sha }} already built; skipping."
+11 -3
View File
@@ -10,16 +10,24 @@ env:
REGISTRY: ghcr.io
IMAGE_NAME: ${{ github.repository }}
permissions:
contents: read
packages: write
concurrency:
group: ${{ github.workflow }}-${{ github.ref }}
cancel-in-progress: ${{ github.event_name == 'pull_request' }}
jobs:
build:
name: Build Docker Image
runs-on: ubuntu-latest
permissions:
contents: read
packages: write
timeout-minutes: 60
steps:
- uses: actions/checkout@v6
with:
persist-credentials: false
- name: Set up QEMU
uses: docker/setup-qemu-action@v4
+16 -5
View File
@@ -6,17 +6,28 @@ on:
pull_request_review_comment:
types: [created]
permissions:
contents: read
pull-requests: read
issues: read
id-token: write
concurrency:
group: opencode-${{ github.event.issue.number || github.event.pull_request.number || github.run_id }}
cancel-in-progress: false
jobs:
opencode:
if: |
(contains(github.event.comment.author_association, 'OWNER') ||
contains(github.event.comment.author_association, 'MEMBER') ||
contains(github.event.comment.author_association, 'COLLABORATOR')) &&
(contains(github.event.comment.body, ' /oc') ||
startsWith(github.event.comment.body, '/oc') ||
contains(github.event.comment.body, ' /opencode') ||
startsWith(github.event.comment.body, '/opencode'))
(startsWith(github.event.comment.body, '/oc') ||
contains(github.event.comment.body, '\n/oc') ||
startsWith(github.event.comment.body, '/opencode') ||
contains(github.event.comment.body, '\n/opencode'))
runs-on: ubuntu-latest
timeout-minutes: 30
permissions:
id-token: write
contents: read
@@ -29,7 +40,7 @@ jobs:
persist-credentials: false
- name: Run opencode
uses: anomalyco/opencode/github@latest
uses: anomalyco/opencode/github@v1.17.7
env:
ZHIPU_API_KEY: ${{ secrets.ZHIPU_API_KEY }}
with:
+9 -2
View File
@@ -40,9 +40,16 @@ docker compose -f docker-compose.yml -f docker-compose.dev.yml --profile core ex
## Tests & Quality
Coverage is **opt-in**; add `--cov=meshcore_hub` (or `make test-cov`) when you want it. The dev loop defaults to no coverage and parallel across CPU cores.
```bash
# Canonical: run tests and surface the pass/fail summary
pytest --no-cov 2>&1 | grep -iE "passed|failed" | tail -3
# Canonical: run tests in parallel, no coverage, surface the pass/fail summary
pytest -nauto --no-cov 2>&1 | grep -iE "passed|failed" | tail -3
# Makefile shorthands
make test # pytest -nauto --no-cov (parallel dev loop)
make test-cov # full run with coverage report
make test-unit # parallel, fast unit suites only (skips e2e)
# Targeted by component (run only what you changed)
pytest --no-cov tests/test_web/ # templates, static JS, web routes
+1 -1
View File
@@ -65,7 +65,7 @@ FROM python:3.14-slim AS runtime
# Labels
LABEL org.opencontainers.image.title="MeshCore Hub" \
org.opencontainers.image.description="Python monorepo for managing MeshCore mesh networks" \
org.opencontainers.image.source="https://github.com/meshcore-dev/meshcore-hub"
org.opencontainers.image.source="https://github.com/ipnet-mesh/meshcore-hub"
# Set environment variables
ENV PYTHONDONTWRITEBYTECODE=1 \
+12 -6
View File
@@ -1,15 +1,10 @@
ifneq (,$(wildcard ./.env))
include .env
export
endif
COMPOSE_PROJECT_NAME ?= hub
PROFILES ?= mqtt core
COMPOSE_FILES = -f docker-compose.yml -f docker-compose.dev.yml
VOLUMES = $(COMPOSE_PROJECT_NAME)_data $(COMPOSE_PROJECT_NAME)_mqtt_data \
$(COMPOSE_PROJECT_NAME)_observer_data
.PHONY: build up down logs backup restore
.PHONY: build up down logs backup restore test test-cov test-unit
build:
docker compose $(COMPOSE_FILES) --profile all build --no-cache
@@ -38,3 +33,14 @@ restore:
echo "Restoring $$vol from $(FILE)..."; \
docker run --rm -v $$vol:/data -v $(PWD)/backup:/backup \
alpine sh -c "cd / && tar xzf /backup/$$(basename $(FILE))"
# --- Tests ---------------------------------------------------------------
# Coverage is opt-in (use test-cov). Dev loop runs in parallel across cores.
test:
pytest -nauto --no-cov
test-cov:
pytest --cov=meshcore_hub --cov-report=term-missing
test-unit:
pytest -nauto --no-cov tests/test_common/ tests/test_api/ tests/test_collector/ tests/test_web/
+8 -2
View File
@@ -56,6 +56,7 @@ dev = [
"pytest>=7.4.0",
"pytest-asyncio>=0.21.0",
"pytest-cov>=4.1.0",
"pytest-xdist>=3.5.0",
"black>=23.0.0",
"flake8>=6.1.0",
"mypy>=1.5.0",
@@ -137,6 +138,10 @@ module = [
]
disallow_untyped_defs = false
disallow_incomplete_defs = false
# Single-file mypy checks (e.g. pre-commit commit hook) lack full project
# context and report false-positive unused-ignore on type:ignore comments
# that are required in full-project mode.
warn_unused_ignores = false
[[tool.mypy.overrides]]
module = [
@@ -156,8 +161,9 @@ addopts = [
"-ra",
"-q",
"--strict-markers",
"--cov=meshcore_hub",
"--cov-report=term-missing",
]
markers = [
"e2e: end-to-end tests requiring Docker services (skipped unless --e2e)",
]
filterwarnings = [
"ignore::DeprecationWarning",
+83
View File
@@ -47,6 +47,89 @@ def _ignore_dotenv(monkeypatch):
monkeypatch.setattr(cls, "model_config", cfg)
def _settings_classes():
"""CommonSettings and every subclass (recursively)."""
seen: set[type] = set()
stack = [config_module.CommonSettings]
while stack:
cls = stack.pop()
if cls in seen:
continue
seen.add(cls)
stack.extend(cls.__subclasses__())
return seen
def _cli_envvars() -> set[str]:
"""Collect Click envvar names from CLI commands (best-effort).
CLI options read env vars via ``envvar=`` independently of pydantic
Settings, so ``_settings_classes`` alone misses them (e.g. ``API_WORKERS``).
"""
import importlib
import click
envvars: set[str] = set()
def _collect(cmd: click.BaseCommand) -> None:
if isinstance(cmd, click.Group):
for subcmd in cmd.commands.values():
_collect(subcmd)
if isinstance(cmd, click.Command):
for param in cmd.params:
if isinstance(param, click.Option) and param.envvar:
ev = param.envvar
if isinstance(ev, str):
envvars.add(ev)
else:
envvars.update(ev)
for module_path in (
"meshcore_hub.api.cli",
"meshcore_hub.collector.cli",
"meshcore_hub.web.cli",
):
try:
mod = importlib.import_module(module_path)
for attr in vars(mod).values():
if isinstance(attr, click.BaseCommand):
_collect(attr)
except Exception:
pass
return envvars
@pytest.fixture(autouse=True)
def _ignore_dotenv(monkeypatch):
"""Stop pydantic-settings and Click from reading ``.env`` or leaked env vars.
Three-pronged defence:
1. Disable ``env_file`` on every settings subclass so pydantic-settings
won't read the ``.env`` file itself.
2. Delete any env vars matching a settings field name from ``os.environ``
for the duration of the test.
3. Delete any env vars matching a Click CLI ``envvar=`` name (e.g.
``API_WORKERS``) that aren't settings fields.
This catches vars exported into the shell via direnv, Makefile, CI, etc.
before pytest started. Tests must depend only on defaults and explicit
env overrides (``monkeypatch.setenv``).
"""
for cls in _settings_classes():
cfg = dict(cls.model_config)
cfg["env_file"] = None
monkeypatch.setattr(cls, "model_config", cfg)
for field_name in cls.model_fields:
monkeypatch.delenv(field_name.upper(), raising=False)
for ev in _cli_envvars():
monkeypatch.delenv(ev, raising=False)
@pytest.fixture
def db_engine():
"""Create an in-memory SQLite database engine for testing."""
+99 -88
View File
@@ -4,7 +4,6 @@ import os
import tempfile
from contextlib import contextmanager
from datetime import datetime, timezone
from unittest.mock import MagicMock, patch
import pytest
from fastapi.testclient import TestClient
@@ -33,20 +32,27 @@ from meshcore_hub.common.models import (
)
@pytest.fixture
@pytest.fixture(scope="session")
def test_db_path():
"""Create a temporary database file path."""
"""Session-scoped temporary database file path.
One file per pytest session; the engine below builds schema on it once.
"""
fd, path = tempfile.mkstemp(suffix=".db")
os.close(fd)
yield path
# Cleanup
if os.path.exists(path):
os.unlink(path)
@pytest.fixture
@pytest.fixture(scope="session")
def api_db_engine(test_db_path):
"""Create a SQLite database engine for API testing."""
"""Session-scoped SQLite engine. Schema is built once per pytest session.
Previously this was function-scoped and rebuilt ~15 tables for every test,
costing ~0.2s/test. Promoting it to session scope eliminates that. Per-test
isolation is handled by truncation in ``api_db_session``.
"""
db_url = f"sqlite:///{test_db_path}"
engine = create_engine(
db_url,
@@ -65,18 +71,33 @@ def api_db_engine(test_db_path):
engine.dispose()
def _truncate_all(engine) -> None:
"""Delete rows from every table in child-first order (FK-safe)."""
with engine.begin() as conn:
for table in reversed(Base.metadata.sorted_tables):
conn.execute(table.delete())
@pytest.fixture
def api_db_session(api_db_engine):
"""Create a database session for API testing."""
"""Per-test session bound to the shared session-scoped engine.
Rows are truncated at teardown so each test starts with empty tables
without paying schema-build cost. Tests must ``commit()`` their seed
data before invoking the test client (the sample_* fixtures already do).
"""
Session = sessionmaker(bind=api_db_engine)
session = Session()
yield session
session.close()
_truncate_all(api_db_engine)
@pytest.fixture
@pytest.fixture(scope="session")
def mock_mqtt():
"""Create a mock MQTT client."""
"""Session-scoped mock MQTT client (no per-test state)."""
from unittest.mock import MagicMock
mock = MagicMock()
mock.connect.return_value = None
mock.start_background.return_value = None
@@ -86,9 +107,11 @@ def mock_mqtt():
return mock
@pytest.fixture
@pytest.fixture(scope="session")
def mock_db_manager(api_db_engine):
"""Create a mock database manager using the test engine."""
"""Session-scoped mock database manager backed by the shared engine."""
from unittest.mock import MagicMock
manager = MagicMock(spec=DatabaseManager)
Session = sessionmaker(bind=api_db_engine)
manager.get_session = lambda: Session()
@@ -109,95 +132,83 @@ def mock_db_manager(api_db_engine):
return manager
@pytest.fixture
@pytest.fixture(autouse=True)
def _isolate_db_global(monkeypatch: pytest.MonkeyPatch, mock_db_manager) -> None:
"""Pin ``meshcore_hub.api.app._db_manager`` to the mock for every test.
Function-scoped autouse so tests that mutate the global (e.g. the lifespan
tests in ``test_cache.py``) cannot leak state to siblings. ``monkeypatch``
restores the prior value on exit.
"""
import meshcore_hub.api.app as app_module
monkeypatch.setattr(app_module, "_db_manager", mock_db_manager)
def _wire_overrides(app, api_db_engine, mock_mqtt, mock_db_manager) -> None:
"""Install the standard DB/MQTT dependency overrides on ``app``."""
Session = sessionmaker(bind=api_db_engine)
def override_get_db_manager(request=None):
return mock_db_manager
def override_get_db_session():
session = Session()
try:
yield session
finally:
session.close()
def override_get_mqtt_client(request=None):
return mock_mqtt
app.dependency_overrides[get_db_manager] = override_get_db_manager
app.dependency_overrides[get_db_session] = override_get_db_session
app.dependency_overrides[get_mqtt_client] = override_get_mqtt_client
@pytest.fixture(scope="module")
def app_no_auth(test_db_path, api_db_engine, mock_mqtt, mock_db_manager):
"""Create a FastAPI app with no authentication required."""
"""Module-scoped FastAPI app with no authentication.
Built once per test module; ``create_app`` is the second-most expensive
setup step (~0.3s), so sharing it across a module is a major win. The
``_isolate_db_global`` autouse fixture handles the global ``_db_manager``
so this fixture doesn't need a ``with patch(...)`` context.
"""
db_url = f"sqlite:///{test_db_path}"
# Patch the global db_manager to avoid lifespan issues
with patch("meshcore_hub.api.app._db_manager", mock_db_manager):
app = create_app(
database_url=db_url,
read_key=None,
admin_key=None,
)
# Create session maker for this test engine
Session = sessionmaker(bind=api_db_engine)
def override_get_db_manager(request=None):
return mock_db_manager
def override_get_db_session():
session = Session()
try:
yield session
finally:
session.close()
def override_get_mqtt_client(request=None):
return mock_mqtt
app.dependency_overrides[get_db_manager] = override_get_db_manager
app.dependency_overrides[get_db_session] = override_get_db_session
app.dependency_overrides[get_mqtt_client] = override_get_mqtt_client
yield app
app = create_app(
database_url=db_url,
read_key=None,
admin_key=None,
)
_wire_overrides(app, api_db_engine, mock_mqtt, mock_db_manager)
yield app
@pytest.fixture
@pytest.fixture(scope="module")
def app_with_auth(test_db_path, api_db_engine, mock_mqtt, mock_db_manager):
"""Create a FastAPI app with authentication enabled."""
"""Module-scoped FastAPI app with authentication enabled."""
db_url = f"sqlite:///{test_db_path}"
with patch("meshcore_hub.api.app._db_manager", mock_db_manager):
app = create_app(
database_url=db_url,
read_key="test-read-key",
admin_key="test-admin-key",
)
Session = sessionmaker(bind=api_db_engine)
def override_get_db_manager(request=None):
return mock_db_manager
def override_get_db_session():
session = Session()
try:
yield session
finally:
session.close()
def override_get_mqtt_client(request=None):
return mock_mqtt
app.dependency_overrides[get_db_manager] = override_get_db_manager
app.dependency_overrides[get_db_session] = override_get_db_session
app.dependency_overrides[get_mqtt_client] = override_get_mqtt_client
yield app
app = create_app(
database_url=db_url,
read_key="test-read-key",
admin_key="test-admin-key",
)
_wire_overrides(app, api_db_engine, mock_mqtt, mock_db_manager)
yield app
@pytest.fixture
def client_no_auth(app_no_auth, mock_db_manager):
"""Create a test client with no authentication.
Uses raise_server_exceptions=False to skip lifespan events.
"""
# Don't use context manager to skip lifespan
client = TestClient(app_no_auth, raise_server_exceptions=True)
yield client
def client_no_auth(app_no_auth) -> TestClient:
"""Test client with no authentication."""
return TestClient(app_no_auth, raise_server_exceptions=True)
@pytest.fixture
def client_with_auth(app_with_auth, mock_db_manager):
"""Create a test client with authentication enabled.
Uses raise_server_exceptions=False to skip lifespan events.
"""
client = TestClient(app_with_auth, raise_server_exceptions=True)
yield client
def client_with_auth(app_with_auth) -> TestClient:
"""Test client with authentication enabled."""
return TestClient(app_with_auth, raise_server_exceptions=True)
@pytest.fixture
+13 -10
View File
@@ -49,8 +49,9 @@ class TestSubscriber:
def test_stop_disconnects_mqtt(self, subscriber, mock_mqtt_client):
"""Test that stop disconnects MQTT."""
subscriber.start()
subscriber.stop()
with patch("meshcore_hub.collector.subscriber.time.sleep"):
subscriber.start()
subscriber.stop()
mock_mqtt_client.stop.assert_called_once()
mock_mqtt_client.disconnect.assert_called_once()
@@ -1229,13 +1230,14 @@ class TestChannelKeyRefresh:
mock_mqtt_client, db_manager, channel_refresh_interval_seconds=300
)
subscriber._running = True
subscriber._start_channel_refresh_scheduler()
with patch("meshcore_hub.collector.subscriber.time.sleep"):
subscriber._start_channel_refresh_scheduler()
assert subscriber._channel_refresh_thread is not None
assert subscriber._channel_refresh_thread.daemon is True
assert subscriber._channel_refresh_thread is not None
assert subscriber._channel_refresh_thread.daemon is True
subscriber._running = False
subscriber._channel_refresh_thread.join(timeout=2.0)
subscriber._running = False
subscriber._channel_refresh_thread.join(timeout=2.0)
def test_channel_refresh_scheduler_disabled(self, mock_mqtt_client, db_manager):
"""Test channel refresh scheduler is disabled when interval is 0."""
@@ -1255,10 +1257,11 @@ class TestChannelKeyRefresh:
mock_mqtt_client, db_manager, channel_refresh_interval_seconds=300
)
subscriber._running = True
subscriber._start_channel_refresh_scheduler()
subscriber._running = False
with patch("meshcore_hub.collector.subscriber.time.sleep"):
subscriber._start_channel_refresh_scheduler()
subscriber._running = False
subscriber._stop_channel_refresh_scheduler()
subscriber._stop_channel_refresh_scheduler()
assert subscriber._channel_refresh_thread is not None
assert not subscriber._channel_refresh_thread.is_alive()
+2 -1
View File
@@ -5,6 +5,7 @@ import hashlib
import pytest
from pydantic import ValidationError
from sqlalchemy import create_engine
from sqlalchemy.exc import IntegrityError
from sqlalchemy.orm import sessionmaker
from meshcore_hub.common.models import Base, Channel, ChannelVisibility
@@ -118,7 +119,7 @@ class TestChannelModel:
db_session.commit()
db_session.add(ch2)
with pytest.raises(Exception, match=""):
with pytest.raises(IntegrityError):
db_session.commit()
def test_channel_default_values(self, db_session) -> None: