diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..c375148 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,93 @@ +name: CI + +on: + push: + branches: [main, master] + pull_request: + branches: [main, master] + +jobs: + lint: + name: Lint + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.11" + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install black flake8 mypy + pip install -e ".[dev]" + + - name: Check formatting with black + run: black --check src/ tests/ + + - name: Lint with flake8 + run: flake8 src/ tests/ + + - name: Type check with mypy + run: mypy src/ + + test: + name: Test (Python ${{ matrix.python-version }}) + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + python-version: ["3.11", "3.12"] + + steps: + - uses: actions/checkout@v4 + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install -e ".[dev]" + + - name: Run tests with pytest + run: | + pytest --cov=meshcore_hub --cov-report=xml --cov-report=term-missing + + - name: Upload coverage to Codecov + uses: codecov/codecov-action@v4 + if: matrix.python-version == '3.11' + with: + files: ./coverage.xml + fail_ci_if_error: false + verbose: true + + build: + name: Build Package + runs-on: ubuntu-latest + needs: [lint, test] + steps: + - uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.11" + + - name: Install build tools + run: | + python -m pip install --upgrade pip + pip install build + + - name: Build package + run: python -m build + + - name: Upload artifacts + uses: actions/upload-artifact@v4 + with: + name: dist + path: dist/ diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml new file mode 100644 index 0000000..bd35981 --- /dev/null +++ b/.github/workflows/docker.yml @@ -0,0 +1,65 @@ +name: Docker + +on: + push: + branches: [main, master] + tags: + - "v*" + pull_request: + branches: [main, master] + +env: + REGISTRY: ghcr.io + IMAGE_NAME: ${{ github.repository }} + +jobs: + build: + name: Build Docker Image + runs-on: ubuntu-latest + permissions: + contents: read + packages: write + + steps: + - uses: actions/checkout@v4 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Log in to Container Registry + if: github.event_name != 'pull_request' + uses: docker/login-action@v3 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Extract metadata for Docker + id: meta + uses: docker/metadata-action@v5 + with: + images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} + tags: | + type=ref,event=branch + type=ref,event=pr + type=semver,pattern={{version}} + type=semver,pattern={{major}}.{{minor}} + type=sha + + - name: Build and push Docker image + uses: docker/build-push-action@v5 + with: + context: . + file: docker/Dockerfile + push: ${{ github.event_name != 'pull_request' }} + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + cache-from: type=gha + cache-to: type=gha,mode=max + + - name: Test Docker image + if: github.event_name == 'pull_request' + run: | + docker build -t meshcore-hub-test -f docker/Dockerfile . + docker run --rm meshcore-hub-test --version + docker run --rm meshcore-hub-test --help diff --git a/.gitignore b/.gitignore index 9ee5c5c..0e2a9aa 100644 --- a/.gitignore +++ b/.gitignore @@ -210,4 +210,3 @@ __marimo__/ # MeshCore Hub specific *.db meshcore.db -members.json diff --git a/README.md b/README.md index 4a7490b..8f567b2 100644 --- a/README.md +++ b/README.md @@ -66,16 +66,61 @@ MeshCore Hub provides a complete solution for monitoring, collecting, and intera ### Using Docker Compose (Recommended) +Docker Compose supports **profiles** to selectively enable/disable components: + +| Profile | Services | +|---------|----------| +| `mqtt` | Eclipse Mosquitto MQTT broker | +| `interface-receiver` | MeshCore device receiver (events to MQTT) | +| `interface-sender` | MeshCore device sender (MQTT to device) | +| `collector` | MQTT subscriber + database storage | +| `api` | REST API server | +| `web` | Web dashboard | +| `mock` | All services with mock device (for testing) | +| `all` | All production services | + ```bash # Clone the repository git clone https://github.com/your-org/meshcore-hub.git -cd meshcore-hub +cd meshcore-hub/docker -# Start all services -docker compose -f docker/docker-compose.yml up -d +# Copy and configure environment +cp .env.example .env +# Edit .env with your settings (API keys, serial port, network info) + +# Option 1: Start all services with mock device (for testing) +docker compose --profile mock up -d + +# Option 2: Start specific services for production +docker compose --profile mqtt --profile collector --profile api --profile web up -d + +# Option 3: Start all production services (requires real MeshCore device) +docker compose --profile all up -d # View logs -docker compose -f docker/docker-compose.yml logs -f +docker compose logs -f + +# Run database migrations +docker compose --profile migrate up + +# Stop services +docker compose --profile mock down +``` + +#### Serial Device Access + +For production with real MeshCore devices, ensure the serial port is accessible: + +```bash +# Check device path +ls -la /dev/ttyUSB* + +# Add user to dialout group (Linux) +sudo usermod -aG dialout $USER + +# Configure in .env +SERIAL_PORT=/dev/ttyUSB0 +SERIAL_PORT_SENDER=/dev/ttyUSB1 # If using separate sender device ``` ### Manual Installation @@ -177,9 +222,14 @@ meshcore-hub db current # Show current revision When running, the API provides interactive documentation at: -- **Swagger UI**: http://localhost:8000/docs -- **ReDoc**: http://localhost:8000/redoc -- **OpenAPI JSON**: http://localhost:8000/openapi.json +- **Swagger UI**: http://localhost:8000/api/docs +- **ReDoc**: http://localhost:8000/api/redoc +- **OpenAPI JSON**: http://localhost:8000/api/openapi.json + +Health check endpoints are also available: + +- **Health**: http://localhost:8000/health +- **Ready**: http://localhost:8000/health/ready (includes database check) ### Authentication diff --git a/docker/.dockerignore b/docker/.dockerignore new file mode 100644 index 0000000..0e6914a --- /dev/null +++ b/docker/.dockerignore @@ -0,0 +1,58 @@ +# MeshCore Hub Docker Ignore File + +# Git +.git +.gitignore + +# Python +__pycache__ +*.py[cod] +*$py.class +*.so +.Python +.venv +venv +.env +*.egg-info +dist +build +.eggs +*.egg + +# IDE +.idea +.vscode +*.swp +*.swo + +# Testing +.pytest_cache +.coverage +htmlcov +.tox +.nox + +# Documentation +docs/_build +*.md +!README.md + +# Docker (avoid recursive) +docker/ + +# Development files +.pre-commit-config.yaml +.flake8 +mypy.ini + +# Data files +*.db +*.sqlite +*.sqlite3 + +# Logs +*.log + +# OS files +.DS_Store +Thumbs.db diff --git a/docker/.env.example b/docker/.env.example new file mode 100644 index 0000000..8f8de42 --- /dev/null +++ b/docker/.env.example @@ -0,0 +1,66 @@ +# MeshCore Hub - Docker Compose Environment Configuration +# Copy this file to .env and customize values + +# =================== +# Common Settings +# =================== + +# Logging level (DEBUG, INFO, WARNING, ERROR, CRITICAL) +LOG_LEVEL=INFO + +# MQTT Broker Settings (internal use) +MQTT_USERNAME= +MQTT_PASSWORD= +MQTT_PREFIX=meshcore + +# External MQTT port mapping +MQTT_EXTERNAL_PORT=1883 +MQTT_WS_PORT=9001 + +# =================== +# Interface Settings +# =================== + +# Serial port for receiver device +SERIAL_PORT=/dev/ttyUSB0 + +# Serial port for sender device (if separate) +SERIAL_PORT_SENDER=/dev/ttyUSB1 + +# Baud rate for serial communication +SERIAL_BAUD=115200 + +# Optional node address override (64-char hex string) +NODE_ADDRESS= +NODE_ADDRESS_SENDER= + +# =================== +# API Settings +# =================== + +# External API port +API_PORT=8000 + +# API Keys for authentication (generate secure keys for production!) +# Example: openssl rand -hex 32 +API_READ_KEY= +API_ADMIN_KEY= + +# =================== +# Web Dashboard Settings +# =================== + +# External web port +WEB_PORT=8080 + +# Network Information (displayed on web dashboard) +NETWORK_NAME=MeshCore Network +NETWORK_CITY= +NETWORK_COUNTRY= +NETWORK_LOCATION= +NETWORK_RADIO_CONFIG= +NETWORK_CONTACT_EMAIL= +NETWORK_CONTACT_DISCORD= + +# Path to members JSON file (mounted into container) +MEMBERS_FILE_PATH=./members.json diff --git a/docker/Dockerfile b/docker/Dockerfile new file mode 100644 index 0000000..d3327d5 --- /dev/null +++ b/docker/Dockerfile @@ -0,0 +1,94 @@ +# MeshCore Hub - Multi-stage Dockerfile +# Build and run MeshCore Hub components + +# ============================================================================= +# Stage 1: Builder - Install dependencies and build package +# ============================================================================= +FROM python:3.11-slim AS builder + +# Set environment variables +ENV PYTHONDONTWRITEBYTECODE=1 \ + PYTHONUNBUFFERED=1 \ + PIP_NO_CACHE_DIR=1 \ + PIP_DISABLE_PIP_VERSION_CHECK=1 + +# Install build dependencies +RUN apt-get update && apt-get install -y --no-install-recommends \ + build-essential \ + && rm -rf /var/lib/apt/lists/* + +# Create and use virtual environment +RUN python -m venv /opt/venv +ENV PATH="/opt/venv/bin:$PATH" + +# Copy project files +WORKDIR /app +COPY pyproject.toml README.md ./ +COPY src/ ./src/ +COPY alembic/ ./alembic/ +COPY alembic.ini ./ + +# Install the package +RUN pip install --upgrade pip && \ + pip install . + +# ============================================================================= +# Stage 2: Runtime - Final production image +# ============================================================================= +FROM python:3.11-slim AS runtime + +# Labels +LABEL org.opencontainers.image.title="MeshCore Hub" \ + org.opencontainers.image.description="Python monorepo for managing MeshCore mesh networks" \ + org.opencontainers.image.source="https://github.com/meshcore-dev/meshcore-hub" + +# Set environment variables +ENV PYTHONDONTWRITEBYTECODE=1 \ + PYTHONUNBUFFERED=1 \ + # Default configuration + LOG_LEVEL=INFO \ + MQTT_HOST=mqtt \ + MQTT_PORT=1883 \ + MQTT_PREFIX=meshcore \ + DATABASE_URL=sqlite:////data/meshcore.db \ + API_HOST=0.0.0.0 \ + API_PORT=8000 \ + WEB_HOST=0.0.0.0 \ + WEB_PORT=8080 \ + API_BASE_URL=http://api:8000 + +# Install runtime dependencies +RUN apt-get update && apt-get install -y --no-install-recommends \ + # For serial port access + udev \ + && rm -rf /var/lib/apt/lists/* \ + && mkdir -p /data + +# Copy virtual environment from builder +COPY --from=builder /opt/venv /opt/venv +ENV PATH="/opt/venv/bin:$PATH" + +# Copy alembic configuration for migrations +WORKDIR /app +COPY --from=builder /app/alembic.ini ./ +COPY --from=builder /app/alembic/ ./alembic/ + +# Create non-root user +RUN useradd --create-home --shell /bin/bash meshcore && \ + chown -R meshcore:meshcore /data /app + +# Default to non-root user (can be overridden for device access) +USER meshcore + +# Expose common ports +EXPOSE 8000 8080 + +# Health check - uses the API health endpoint by default +HEALTHCHECK --interval=30s --timeout=10s --start-period=5s --retries=3 \ + CMD python -c "import urllib.request; urllib.request.urlopen('http://localhost:8000/health')" || exit 1 + +# Set entrypoint to the CLI +ENTRYPOINT ["meshcore-hub"] + +# Default command shows help +CMD ["--help"] diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml new file mode 100644 index 0000000..aa46885 --- /dev/null +++ b/docker/docker-compose.yml @@ -0,0 +1,290 @@ +# MeshCore Hub - Docker Compose Configuration +# +# Usage with profiles: +# docker compose --profile mqtt --profile collector --profile api up +# docker compose --profile all up +# docker compose --profile mock up # For testing with mock devices +# +# Available profiles: +# - mqtt: MQTT broker (Eclipse Mosquitto) +# - interface-receiver: Interface in RECEIVER mode +# - interface-sender: Interface in SENDER mode +# - collector: Event collector and database storage +# - api: REST API server +# - web: Web dashboard +# - mock: All components with mock device (for testing) +# - all: All production components (requires real device) + +services: + # ========================================================================== + # MQTT Broker - Eclipse Mosquitto + # ========================================================================== + mqtt: + image: eclipse-mosquitto:2 + container_name: meshcore-mqtt + profiles: + - mqtt + - all + - mock + restart: unless-stopped + ports: + - "${MQTT_EXTERNAL_PORT:-1883}:1883" + - "${MQTT_WS_PORT:-9001}:9001" + volumes: + - ./mosquitto.conf:/mosquitto/config/mosquitto.conf:ro + - mosquitto_data:/mosquitto/data + - mosquitto_log:/mosquitto/log + healthcheck: + test: ["CMD", "mosquitto_sub", "-t", "$$SYS/#", "-C", "1", "-i", "healthcheck", "-W", "3"] + interval: 30s + timeout: 10s + retries: 3 + start_period: 10s + + # ========================================================================== + # Interface Receiver - MeshCore device to MQTT bridge (events) + # ========================================================================== + interface-receiver: + build: + context: .. + dockerfile: docker/Dockerfile + container_name: meshcore-interface-receiver + profiles: + - interface-receiver + - all + restart: unless-stopped + depends_on: + mqtt: + condition: service_healthy + devices: + - "${SERIAL_PORT:-/dev/ttyUSB0}:${SERIAL_PORT:-/dev/ttyUSB0}" + user: root # Required for device access + environment: + - LOG_LEVEL=${LOG_LEVEL:-INFO} + - MQTT_HOST=mqtt + - MQTT_PORT=1883 + - MQTT_USERNAME=${MQTT_USERNAME:-} + - MQTT_PASSWORD=${MQTT_PASSWORD:-} + - MQTT_PREFIX=${MQTT_PREFIX:-meshcore} + - SERIAL_PORT=${SERIAL_PORT:-/dev/ttyUSB0} + - SERIAL_BAUD=${SERIAL_BAUD:-115200} + - NODE_ADDRESS=${NODE_ADDRESS:-} + command: ["interface", "receiver"] + healthcheck: + test: ["CMD", "pgrep", "-f", "meshcore-hub"] + interval: 30s + timeout: 10s + retries: 3 + start_period: 30s + + # ========================================================================== + # Interface Sender - MQTT to MeshCore device bridge (commands) + # ========================================================================== + interface-sender: + build: + context: .. + dockerfile: docker/Dockerfile + container_name: meshcore-interface-sender + profiles: + - interface-sender + - all + restart: unless-stopped + depends_on: + mqtt: + condition: service_healthy + devices: + - "${SERIAL_PORT_SENDER:-/dev/ttyUSB1}:${SERIAL_PORT_SENDER:-/dev/ttyUSB1}" + user: root # Required for device access + environment: + - LOG_LEVEL=${LOG_LEVEL:-INFO} + - MQTT_HOST=mqtt + - MQTT_PORT=1883 + - MQTT_USERNAME=${MQTT_USERNAME:-} + - MQTT_PASSWORD=${MQTT_PASSWORD:-} + - MQTT_PREFIX=${MQTT_PREFIX:-meshcore} + - SERIAL_PORT=${SERIAL_PORT_SENDER:-/dev/ttyUSB1} + - SERIAL_BAUD=${SERIAL_BAUD:-115200} + - NODE_ADDRESS=${NODE_ADDRESS_SENDER:-} + command: ["interface", "sender"] + healthcheck: + test: ["CMD", "pgrep", "-f", "meshcore-hub"] + interval: 30s + timeout: 10s + retries: 3 + start_period: 30s + + # ========================================================================== + # Interface Mock Receiver - For testing without real devices + # ========================================================================== + interface-mock-receiver: + build: + context: .. + dockerfile: docker/Dockerfile + container_name: meshcore-interface-mock-receiver + profiles: + - mock + restart: unless-stopped + depends_on: + mqtt: + condition: service_healthy + environment: + - LOG_LEVEL=${LOG_LEVEL:-INFO} + - MQTT_HOST=mqtt + - MQTT_PORT=1883 + - MQTT_USERNAME=${MQTT_USERNAME:-} + - MQTT_PASSWORD=${MQTT_PASSWORD:-} + - MQTT_PREFIX=${MQTT_PREFIX:-meshcore} + - MOCK_DEVICE=true + - NODE_ADDRESS=${NODE_ADDRESS:-0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef} + command: ["interface", "receiver", "--mock"] + healthcheck: + test: ["CMD", "pgrep", "-f", "meshcore-hub"] + interval: 30s + timeout: 10s + retries: 3 + start_period: 10s + + # ========================================================================== + # Collector - MQTT subscriber and database storage + # ========================================================================== + collector: + build: + context: .. + dockerfile: docker/Dockerfile + container_name: meshcore-collector + profiles: + - collector + - all + - mock + restart: unless-stopped + depends_on: + mqtt: + condition: service_healthy + volumes: + - meshcore_data:/data + environment: + - LOG_LEVEL=${LOG_LEVEL:-INFO} + - MQTT_HOST=mqtt + - MQTT_PORT=1883 + - MQTT_USERNAME=${MQTT_USERNAME:-} + - MQTT_PASSWORD=${MQTT_PASSWORD:-} + - MQTT_PREFIX=${MQTT_PREFIX:-meshcore} + - DATABASE_URL=sqlite:////data/meshcore.db + command: ["collector"] + healthcheck: + test: ["CMD", "pgrep", "-f", "meshcore-hub"] + interval: 30s + timeout: 10s + retries: 3 + start_period: 10s + + # ========================================================================== + # API Server - REST API for querying data and sending commands + # ========================================================================== + api: + build: + context: .. + dockerfile: docker/Dockerfile + container_name: meshcore-api + profiles: + - api + - all + - mock + restart: unless-stopped + depends_on: + mqtt: + condition: service_healthy + collector: + condition: service_started + ports: + - "${API_PORT:-8000}:8000" + volumes: + - meshcore_data:/data + environment: + - LOG_LEVEL=${LOG_LEVEL:-INFO} + - MQTT_HOST=mqtt + - MQTT_PORT=1883 + - MQTT_USERNAME=${MQTT_USERNAME:-} + - MQTT_PASSWORD=${MQTT_PASSWORD:-} + - MQTT_PREFIX=${MQTT_PREFIX:-meshcore} + - DATABASE_URL=sqlite:////data/meshcore.db + - API_HOST=0.0.0.0 + - API_PORT=8000 + - API_READ_KEY=${API_READ_KEY:-} + - API_ADMIN_KEY=${API_ADMIN_KEY:-} + command: ["api"] + healthcheck: + test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://localhost:8000/health')"] + interval: 30s + timeout: 10s + retries: 3 + start_period: 10s + + # ========================================================================== + # Web Dashboard - Web interface for network visualization + # ========================================================================== + web: + build: + context: .. + dockerfile: docker/Dockerfile + container_name: meshcore-web + profiles: + - web + - all + - mock + restart: unless-stopped + depends_on: + api: + condition: service_healthy + ports: + - "${WEB_PORT:-8080}:8080" + environment: + - LOG_LEVEL=${LOG_LEVEL:-INFO} + - API_BASE_URL=http://api:8000 + - API_KEY=${API_READ_KEY:-} + - WEB_HOST=0.0.0.0 + - WEB_PORT=8080 + - NETWORK_NAME=${NETWORK_NAME:-MeshCore Network} + - NETWORK_CITY=${NETWORK_CITY:-} + - NETWORK_COUNTRY=${NETWORK_COUNTRY:-} + - NETWORK_LOCATION=${NETWORK_LOCATION:-} + - NETWORK_RADIO_CONFIG=${NETWORK_RADIO_CONFIG:-} + - NETWORK_CONTACT_EMAIL=${NETWORK_CONTACT_EMAIL:-} + - NETWORK_CONTACT_DISCORD=${NETWORK_CONTACT_DISCORD:-} + - MEMBERS_FILE=${MEMBERS_FILE:-} + volumes: + - ${MEMBERS_FILE_PATH:-./members.json}:/app/members.json:ro + command: ["web"] + healthcheck: + test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://localhost:8080/health')"] + interval: 30s + timeout: 10s + retries: 3 + start_period: 10s + + # ========================================================================== + # Database Migrations - Run Alembic migrations + # ========================================================================== + db-migrate: + build: + context: .. + dockerfile: docker/Dockerfile + container_name: meshcore-db-migrate + profiles: + - migrate + volumes: + - meshcore_data:/data + environment: + - DATABASE_URL=sqlite:////data/meshcore.db + command: ["db", "upgrade"] + +# ========================================================================== +# Volumes +# ========================================================================== +volumes: + mosquitto_data: + name: meshcore_mosquitto_data + mosquitto_log: + name: meshcore_mosquitto_log + meshcore_data: + name: meshcore_data diff --git a/docker/members.json b/docker/members.json new file mode 100644 index 0000000..89857ff --- /dev/null +++ b/docker/members.json @@ -0,0 +1,10 @@ +{ + "members": [ + { + "name": "Example Member", + "callsign": "N0CALL", + "role": "Network Operator", + "description": "Example member entry" + } + ] +} diff --git a/docker/mosquitto.conf b/docker/mosquitto.conf new file mode 100644 index 0000000..5ceba1f --- /dev/null +++ b/docker/mosquitto.conf @@ -0,0 +1,94 @@ +# MeshCore Hub - Mosquitto MQTT Broker Configuration +# Eclipse Mosquitto 2.x configuration file + +# ============================================================================= +# General Settings +# ============================================================================= + +# Persistence for retained messages and subscriptions +persistence true +persistence_location /mosquitto/data/ + +# Logging configuration +log_dest file /mosquitto/log/mosquitto.log +log_dest stdout +log_type error +log_type warning +log_type notice +log_type information +log_timestamp true +log_timestamp_format %Y-%m-%dT%H:%M:%S + +# Connection messages +connection_messages true + +# ============================================================================= +# Default Listener (MQTT over TCP) +# ============================================================================= + +# Listen on all interfaces, port 1883 +listener 1883 +protocol mqtt + +# Allow anonymous connections (for development/testing) +# For production, set to false and configure authentication +allow_anonymous true + +# ============================================================================= +# WebSocket Listener (optional, for browser clients) +# ============================================================================= + +listener 9001 +protocol websockets + +# ============================================================================= +# Security Settings +# ============================================================================= + +# Maximum packet size (default 268435455 bytes) +# Uncomment to limit: +# message_size_limit 1048576 + +# Maximum number of client connections (0 = unlimited) +max_connections 100 + +# Maximum queued messages per client +max_queued_messages 1000 + +# Maximum number of QoS 1 and 2 messages in flight +max_inflight_messages 20 + +# ============================================================================= +# Authentication (uncomment for production) +# ============================================================================= + +# Password file authentication +# password_file /mosquitto/config/passwd + +# To create password file: +# mosquitto_passwd -c /mosquitto/config/passwd username + +# ============================================================================= +# Access Control (uncomment for production) +# ============================================================================= + +# ACL file for topic-level access control +# acl_file /mosquitto/config/acl + +# Example ACL file content: +# user readonly +# topic read meshcore/# +# +# user admin +# topic readwrite meshcore/# + +# ============================================================================= +# TLS/SSL (uncomment for production) +# ============================================================================= + +# listener 8883 +# protocol mqtt +# cafile /mosquitto/config/ca.crt +# certfile /mosquitto/config/server.crt +# keyfile /mosquitto/config/server.key +# require_certificate false diff --git a/src/meshcore_hub/collector/subscriber.py b/src/meshcore_hub/collector/subscriber.py index 456c9cc..89fcef7 100644 --- a/src/meshcore_hub/collector/subscriber.py +++ b/src/meshcore_hub/collector/subscriber.py @@ -42,6 +42,30 @@ class Subscriber: self._running = False self._shutdown_event = threading.Event() self._handlers: dict[str, EventHandler] = {} + self._mqtt_connected = False + self._db_connected = False + + @property + def is_healthy(self) -> bool: + """Check if the subscriber is healthy. + + Returns: + True if MQTT and database are connected + """ + return self._running and self._mqtt_connected and self._db_connected + + def get_health_status(self) -> dict[str, Any]: + """Get detailed health status. + + Returns: + Dictionary with health status details + """ + return { + "healthy": self.is_healthy, + "running": self._running, + "mqtt_connected": self._mqtt_connected, + "database_connected": self._db_connected, + } def register_handler(self, event_type: str, handler: EventHandler) -> None: """Register a handler for an event type. @@ -96,14 +120,23 @@ class Subscriber: logger.info("Starting collector subscriber") # Create database tables if needed - self.db.create_tables() + try: + self.db.create_tables() + self._db_connected = True + logger.info("Database initialized") + except Exception as e: + self._db_connected = False + logger.error(f"Failed to initialize database: {e}") + raise # Connect to MQTT broker try: self.mqtt.connect() self.mqtt.start_background() + self._mqtt_connected = True logger.info("Connected to MQTT broker") except Exception as e: + self._mqtt_connected = False logger.error(f"Failed to connect to MQTT broker: {e}") raise @@ -141,6 +174,7 @@ class Subscriber: # Stop MQTT self.mqtt.stop() self.mqtt.disconnect() + self._mqtt_connected = False logger.info("Collector subscriber stopped") diff --git a/src/meshcore_hub/interface/receiver.py b/src/meshcore_hub/interface/receiver.py index 4adcca6..b5a7746 100644 --- a/src/meshcore_hub/interface/receiver.py +++ b/src/meshcore_hub/interface/receiver.py @@ -43,6 +43,31 @@ class Receiver: self.mqtt = mqtt_client self._running = False self._shutdown_event = threading.Event() + self._device_connected = False + self._mqtt_connected = False + + @property + def is_healthy(self) -> bool: + """Check if the receiver is healthy. + + Returns: + True if device and MQTT are connected + """ + return self._running and self._device_connected and self._mqtt_connected + + def get_health_status(self) -> dict[str, Any]: + """Get detailed health status. + + Returns: + Dictionary with health status details + """ + return { + "healthy": self.is_healthy, + "running": self._running, + "device_connected": self._device_connected, + "mqtt_connected": self._mqtt_connected, + "device_public_key": self.device.public_key, + } def _initialize_device(self) -> None: """Initialize device after connection. @@ -108,18 +133,23 @@ class Receiver: try: self.mqtt.connect() self.mqtt.start_background() + self._mqtt_connected = True logger.info("Connected to MQTT broker") except Exception as e: + self._mqtt_connected = False logger.error(f"Failed to connect to MQTT broker: {e}") raise # Connect to device if not self.device.connect(): + self._device_connected = False logger.error("Failed to connect to MeshCore device") self.mqtt.stop() self.mqtt.disconnect() + self._mqtt_connected = False raise RuntimeError("Failed to connect to MeshCore device") + self._device_connected = True logger.info(f"Connected to MeshCore device: {self.device.public_key}") # Initialize device: set time and send local advertisement @@ -154,10 +184,12 @@ class Receiver: # Stop device self.device.stop() self.device.disconnect() + self._device_connected = False # Stop MQTT self.mqtt.stop() self.mqtt.disconnect() + self._mqtt_connected = False logger.info("Receiver stopped") diff --git a/src/meshcore_hub/interface/sender.py b/src/meshcore_hub/interface/sender.py index 14a7e4c..53993b7 100644 --- a/src/meshcore_hub/interface/sender.py +++ b/src/meshcore_hub/interface/sender.py @@ -42,6 +42,31 @@ class Sender: self.mqtt = mqtt_client self._running = False self._shutdown_event = threading.Event() + self._device_connected = False + self._mqtt_connected = False + + @property + def is_healthy(self) -> bool: + """Check if the sender is healthy. + + Returns: + True if device and MQTT are connected + """ + return self._running and self._device_connected and self._mqtt_connected + + def get_health_status(self) -> dict[str, Any]: + """Get detailed health status. + + Returns: + Dictionary with health status details + """ + return { + "healthy": self.is_healthy, + "running": self._running, + "device_connected": self._device_connected, + "mqtt_connected": self._mqtt_connected, + "device_public_key": self.device.public_key, + } def _handle_mqtt_message( self, @@ -175,19 +200,24 @@ class Sender: # Connect to device first if not self.device.connect(): + self._device_connected = False logger.error("Failed to connect to MeshCore device") raise RuntimeError("Failed to connect to MeshCore device") + self._device_connected = True logger.info(f"Connected to MeshCore device: {self.device.public_key}") # Connect to MQTT broker try: self.mqtt.connect() self.mqtt.start_background() + self._mqtt_connected = True logger.info("Connected to MQTT broker") except Exception as e: + self._mqtt_connected = False logger.error(f"Failed to connect to MQTT broker: {e}") self.device.disconnect() + self._device_connected = False raise # Subscribe to command topics @@ -225,10 +255,12 @@ class Sender: # Stop MQTT self.mqtt.stop() self.mqtt.disconnect() + self._mqtt_connected = False # Stop device self.device.stop() self.device.disconnect() + self._device_connected = False logger.info("Sender stopped") diff --git a/tests/e2e/__init__.py b/tests/e2e/__init__.py new file mode 100644 index 0000000..bd2b902 --- /dev/null +++ b/tests/e2e/__init__.py @@ -0,0 +1 @@ +"""End-to-end tests for MeshCore Hub.""" diff --git a/tests/e2e/conftest.py b/tests/e2e/conftest.py new file mode 100644 index 0000000..678f47b --- /dev/null +++ b/tests/e2e/conftest.py @@ -0,0 +1,109 @@ +"""Fixtures for end-to-end tests.""" + +import os +import time +from typing import Generator + +import httpx +import pytest + +# E2E test configuration +E2E_API_URL = os.environ.get("E2E_API_URL", "http://localhost:18000") +E2E_WEB_URL = os.environ.get("E2E_WEB_URL", "http://localhost:18080") +E2E_MQTT_HOST = os.environ.get("E2E_MQTT_HOST", "localhost") +E2E_MQTT_PORT = int(os.environ.get("E2E_MQTT_PORT", "11883")) +E2E_READ_KEY = os.environ.get("E2E_READ_KEY", "test-read-key") +E2E_ADMIN_KEY = os.environ.get("E2E_ADMIN_KEY", "test-admin-key") + + +def wait_for_service(url: str, timeout: int = 60) -> bool: + """Wait for a service to become available. + + Args: + url: Health check URL + timeout: Maximum seconds to wait + + Returns: + True if service is available, False if timeout + """ + start = time.time() + while time.time() - start < timeout: + try: + response = httpx.get(url, timeout=5.0) + if response.status_code == 200: + return True + except httpx.RequestError: + pass + time.sleep(1) + return False + + +@pytest.fixture(scope="session") +def api_url() -> str: + """Get API base URL.""" + return E2E_API_URL + + +@pytest.fixture(scope="session") +def web_url() -> str: + """Get Web dashboard URL.""" + return E2E_WEB_URL + + +@pytest.fixture(scope="session") +def read_key() -> str: + """Get read API key.""" + return E2E_READ_KEY + + +@pytest.fixture(scope="session") +def admin_key() -> str: + """Get admin API key.""" + return E2E_ADMIN_KEY + + +@pytest.fixture(scope="session") +def api_client(api_url: str, read_key: str) -> Generator[httpx.Client, None, None]: + """Create an API client with read access. + + This fixture waits for the API to be available before returning. + """ + health_url = f"{api_url}/health" + if not wait_for_service(health_url): + pytest.skip(f"API not available at {api_url}") + + with httpx.Client( + base_url=api_url, + headers={"Authorization": f"Bearer {read_key}"}, + timeout=30.0, + ) as client: + yield client + + +@pytest.fixture(scope="session") +def admin_client(api_url: str, admin_key: str) -> Generator[httpx.Client, None, None]: + """Create an API client with admin access.""" + health_url = f"{api_url}/health" + if not wait_for_service(health_url): + pytest.skip(f"API not available at {api_url}") + + with httpx.Client( + base_url=api_url, + headers={"Authorization": f"Bearer {admin_key}"}, + timeout=30.0, + ) as client: + yield client + + +@pytest.fixture(scope="session") +def web_client(web_url: str) -> Generator[httpx.Client, None, None]: + """Create a web dashboard client.""" + health_url = f"{web_url}/health" + if not wait_for_service(health_url): + pytest.skip(f"Web dashboard not available at {web_url}") + + with httpx.Client( + base_url=web_url, + timeout=30.0, + ) as client: + yield client diff --git a/tests/e2e/docker-compose.test.yml b/tests/e2e/docker-compose.test.yml new file mode 100644 index 0000000..de6dedc --- /dev/null +++ b/tests/e2e/docker-compose.test.yml @@ -0,0 +1,136 @@ +# MeshCore Hub - End-to-End Test Docker Compose +# +# This configuration runs all services with a mock device for integration testing. +# +# Usage: +# docker compose -f tests/e2e/docker-compose.test.yml up -d +# pytest tests/e2e/ +# docker compose -f tests/e2e/docker-compose.test.yml down -v + +services: + # MQTT Broker + mqtt: + image: eclipse-mosquitto:2 + container_name: meshcore-test-mqtt + ports: + - "11883:1883" + volumes: + - ../../docker/mosquitto.conf:/mosquitto/config/mosquitto.conf:ro + healthcheck: + test: ["CMD", "mosquitto_sub", "-t", "$$SYS/#", "-C", "1", "-i", "healthcheck", "-W", "3"] + interval: 5s + timeout: 5s + retries: 3 + start_period: 5s + + # Interface with mock device + interface-mock: + build: + context: ../.. + dockerfile: docker/Dockerfile + container_name: meshcore-test-interface + depends_on: + mqtt: + condition: service_healthy + environment: + - LOG_LEVEL=DEBUG + - MQTT_HOST=mqtt + - MQTT_PORT=1883 + - MQTT_PREFIX=test + - MOCK_DEVICE=true + - NODE_ADDRESS=0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef + command: ["interface", "receiver", "--mock"] + healthcheck: + test: ["CMD", "pgrep", "-f", "meshcore-hub"] + interval: 5s + timeout: 5s + retries: 3 + start_period: 10s + + # Collector + collector: + build: + context: ../.. + dockerfile: docker/Dockerfile + container_name: meshcore-test-collector + depends_on: + mqtt: + condition: service_healthy + volumes: + - test_data:/data + environment: + - LOG_LEVEL=DEBUG + - MQTT_HOST=mqtt + - MQTT_PORT=1883 + - MQTT_PREFIX=test + - DATABASE_URL=sqlite:////data/test.db + command: ["collector"] + healthcheck: + test: ["CMD", "pgrep", "-f", "meshcore-hub"] + interval: 5s + timeout: 5s + retries: 3 + start_period: 10s + + # API Server + api: + build: + context: ../.. + dockerfile: docker/Dockerfile + container_name: meshcore-test-api + depends_on: + mqtt: + condition: service_healthy + collector: + condition: service_started + ports: + - "18000:8000" + volumes: + - test_data:/data + environment: + - LOG_LEVEL=DEBUG + - MQTT_HOST=mqtt + - MQTT_PORT=1883 + - MQTT_PREFIX=test + - DATABASE_URL=sqlite:////data/test.db + - API_HOST=0.0.0.0 + - API_PORT=8000 + - API_READ_KEY=test-read-key + - API_ADMIN_KEY=test-admin-key + command: ["api"] + healthcheck: + test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://localhost:8000/health')"] + interval: 5s + timeout: 5s + retries: 3 + start_period: 10s + + # Web Dashboard + web: + build: + context: ../.. + dockerfile: docker/Dockerfile + container_name: meshcore-test-web + depends_on: + api: + condition: service_healthy + ports: + - "18080:8080" + environment: + - LOG_LEVEL=DEBUG + - API_BASE_URL=http://api:8000 + - API_KEY=test-read-key + - WEB_HOST=0.0.0.0 + - WEB_PORT=8080 + - NETWORK_NAME=Test Network + command: ["web"] + healthcheck: + test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://localhost:8080/health')"] + interval: 5s + timeout: 5s + retries: 3 + start_period: 10s + +volumes: + test_data: + name: meshcore_test_data diff --git a/tests/e2e/test_full_flow.py b/tests/e2e/test_full_flow.py new file mode 100644 index 0000000..6365aa5 --- /dev/null +++ b/tests/e2e/test_full_flow.py @@ -0,0 +1,198 @@ +"""End-to-end tests for the full MeshCore Hub flow. + +These tests require Docker Compose services to be running: + + docker compose -f tests/e2e/docker-compose.test.yml up -d + pytest tests/e2e/ + docker compose -f tests/e2e/docker-compose.test.yml down -v + +The tests verify: +1. API health endpoints +2. Web dashboard health endpoints +3. Node listing and retrieval +4. Message listing +5. Statistics endpoint +6. Command sending (admin only) +""" + +import httpx + + +class TestHealthEndpoints: + """Test health check endpoints.""" + + def test_api_health(self, api_client: httpx.Client) -> None: + """Test API basic health endpoint.""" + response = api_client.get("/health") + assert response.status_code == 200 + data = response.json() + assert data["status"] == "healthy" + assert "version" in data + + def test_api_ready(self, api_client: httpx.Client) -> None: + """Test API readiness endpoint with database check.""" + response = api_client.get("/health/ready") + assert response.status_code == 200 + data = response.json() + assert data["status"] == "ready" + assert data["database"] == "connected" + + def test_web_health(self, web_client: httpx.Client) -> None: + """Test Web dashboard health endpoint.""" + response = web_client.get("/health") + assert response.status_code == 200 + data = response.json() + assert data["status"] == "healthy" + + def test_web_ready(self, web_client: httpx.Client) -> None: + """Test Web dashboard readiness with API connectivity.""" + response = web_client.get("/health/ready") + assert response.status_code == 200 + data = response.json() + assert data["status"] == "ready" + assert data["api"] == "connected" + + +class TestAPIEndpoints: + """Test API data endpoints.""" + + def test_list_nodes(self, api_client: httpx.Client) -> None: + """Test listing nodes.""" + response = api_client.get("/api/v1/nodes") + assert response.status_code == 200 + data = response.json() + assert "items" in data + assert "total" in data + assert isinstance(data["items"], list) + + def test_list_messages(self, api_client: httpx.Client) -> None: + """Test listing messages.""" + response = api_client.get("/api/v1/messages") + assert response.status_code == 200 + data = response.json() + assert "items" in data + assert "total" in data + + def test_list_advertisements(self, api_client: httpx.Client) -> None: + """Test listing advertisements.""" + response = api_client.get("/api/v1/advertisements") + assert response.status_code == 200 + data = response.json() + assert "items" in data + assert "total" in data + + def test_get_stats(self, api_client: httpx.Client) -> None: + """Test getting network statistics.""" + response = api_client.get("/api/v1/stats") + assert response.status_code == 200 + data = response.json() + assert "total_nodes" in data + assert "active_nodes" in data + assert "total_messages" in data + + def test_list_telemetry(self, api_client: httpx.Client) -> None: + """Test listing telemetry records.""" + response = api_client.get("/api/v1/telemetry") + assert response.status_code == 200 + data = response.json() + assert "items" in data + + def test_list_trace_paths(self, api_client: httpx.Client) -> None: + """Test listing trace paths.""" + response = api_client.get("/api/v1/trace-paths") + assert response.status_code == 200 + data = response.json() + assert "items" in data + + +class TestWebDashboard: + """Test web dashboard pages.""" + + def test_home_page(self, web_client: httpx.Client) -> None: + """Test home page loads.""" + response = web_client.get("/") + assert response.status_code == 200 + assert "text/html" in response.headers.get("content-type", "") + + def test_network_page(self, web_client: httpx.Client) -> None: + """Test network overview page loads.""" + response = web_client.get("/network") + assert response.status_code == 200 + assert "text/html" in response.headers.get("content-type", "") + + def test_nodes_page(self, web_client: httpx.Client) -> None: + """Test nodes listing page loads.""" + response = web_client.get("/nodes") + assert response.status_code == 200 + assert "text/html" in response.headers.get("content-type", "") + + def test_messages_page(self, web_client: httpx.Client) -> None: + """Test messages page loads.""" + response = web_client.get("/messages") + assert response.status_code == 200 + assert "text/html" in response.headers.get("content-type", "") + + def test_map_page(self, web_client: httpx.Client) -> None: + """Test map page loads.""" + response = web_client.get("/map") + assert response.status_code == 200 + assert "text/html" in response.headers.get("content-type", "") + + def test_members_page(self, web_client: httpx.Client) -> None: + """Test members page loads.""" + response = web_client.get("/members") + assert response.status_code == 200 + assert "text/html" in response.headers.get("content-type", "") + + +class TestAuthentication: + """Test API authentication.""" + + def test_read_access_with_read_key(self, api_client: httpx.Client) -> None: + """Test read access works with read key.""" + response = api_client.get("/api/v1/nodes") + assert response.status_code == 200 + + def test_admin_access_with_admin_key(self, admin_client: httpx.Client) -> None: + """Test admin access works with admin key.""" + # Admin key should have read access + response = admin_client.get("/api/v1/nodes") + assert response.status_code == 200 + + +class TestCommands: + """Test command endpoints (requires admin access).""" + + def test_send_message_requires_admin(self, api_client: httpx.Client) -> None: + """Test that send message requires admin key.""" + response = api_client.post( + "/api/v1/commands/send-message", + json={ + "destination": "0" * 64, + "text": "Test message", + }, + ) + # Read key should not have admin access + assert response.status_code == 403 + + def test_send_channel_message_admin(self, admin_client: httpx.Client) -> None: + """Test sending channel message with admin key.""" + response = admin_client.post( + "/api/v1/commands/send-channel-message", + json={ + "channel_idx": 0, + "text": "Test channel message", + }, + ) + # Should succeed with admin key (202 = accepted for processing) + assert response.status_code == 202 + + def test_send_advertisement_admin(self, admin_client: httpx.Client) -> None: + """Test sending advertisement with admin key.""" + response = admin_client.post( + "/api/v1/commands/send-advertisement", + json={ + "flood": False, + }, + ) + assert response.status_code == 202