diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..158bd86 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,42 @@ +# Git +.git +.gitignore + +# Python +__pycache__ +*.py[cod] +*$py.class +*.so +.Python +.venv +venv +ENV +env + +# IDE +.idea +.vscode +*.swp +*.swo +*~ + +# Project directories (generated/runtime) +data/ +out/ + +# Configuration (use environment variables in Docker) +.envrc +.direnv +meshcore.conf + +# Documentation +docs/ +*.md +!README.md + +# Development files +.claude/ +*.log + +# macOS +.DS_Store diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml new file mode 100644 index 0000000..877eebc --- /dev/null +++ b/.github/workflows/docker-publish.yml @@ -0,0 +1,233 @@ +# Build and publish Docker images to GitHub Container Registry +# +# Triggers: +# - On release: Build with version tags (X.Y.Z, X.Y, latest) +# - On schedule: Rebuild all tags with fresh base image (OS patches) +# - Manual: For testing, optional push +# +# Security: +# - All actions pinned by SHA +# - Vulnerability scanning with Trivy +# - SBOM and provenance attestation + +name: Docker Build and Publish + +on: + release: + types: [published] + + schedule: + # Daily at 4 AM UTC - rebuild with fresh base image + - cron: "0 4 * * *" + + workflow_dispatch: + inputs: + push: + description: "Push image to registry" + required: false + default: false + type: boolean + +permissions: + contents: read + packages: write + id-token: write + attestations: write + +concurrency: + group: docker-${{ github.ref }} + cancel-in-progress: true + +env: + REGISTRY: ghcr.io + IMAGE_NAME: ${{ github.repository }} + +jobs: + build: + runs-on: ubuntu-latest + timeout-minutes: 30 + + steps: + - name: Checkout repository + uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 + + # For nightly builds, get the latest release version + - name: Get latest release version + id: get-version + if: github.event_name == 'schedule' + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + # Get latest release tag + LATEST_TAG=$(gh release view --json tagName -q '.tagName' 2>/dev/null || echo "") + if [ -z "$LATEST_TAG" ]; then + echo "No releases found, skipping nightly build" + echo "skip=true" >> $GITHUB_OUTPUT + exit 0 + fi + # Strip 'v' prefix if present + VERSION=$(echo "$LATEST_TAG" | sed 's/^v//') + echo "version=$VERSION" >> $GITHUB_OUTPUT + echo "skip=false" >> $GITHUB_OUTPUT + + - name: Skip if no releases + if: github.event_name == 'schedule' && steps.get-version.outputs.skip == 'true' + run: | + echo "No releases found, skipping nightly build" + exit 0 + + - name: Set up QEMU + if: "!(github.event_name == 'schedule' && steps.get-version.outputs.skip == 'true')" + uses: docker/setup-qemu-action@c7c53464625b32c7a7e944ae62b3e17d2b600130 # v3.7.0 + + - name: Set up Docker Buildx + if: "!(github.event_name == 'schedule' && steps.get-version.outputs.skip == 'true')" + uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3.12.0 + + - name: Log in to Container Registry + if: "!(github.event_name == 'schedule' && steps.get-version.outputs.skip == 'true')" + uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef # v3.6.0 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + # Generate tags based on event type + - name: Extract metadata (release) + id: meta-release + if: github.event_name == 'release' + uses: docker/metadata-action@c299e40c65443455700f0fdfc63efafe5b349051 # v5.10.0 + with: + images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} + tags: | + # X.Y.Z + type=semver,pattern={{version}} + # X.Y + type=semver,pattern={{major}}.{{minor}} + # latest + type=raw,value=latest + + - name: Extract metadata (nightly) + id: meta-nightly + if: github.event_name == 'schedule' && steps.get-version.outputs.skip != 'true' + uses: docker/metadata-action@c299e40c65443455700f0fdfc63efafe5b349051 # v5.10.0 + with: + images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} + tags: | + # Rebuild version tags with OS patches + type=raw,value=${{ steps.get-version.outputs.version }} + # Nightly tags + type=raw,value=nightly + type=raw,value=nightly-{{date 'YYYYMMDD'}} + # Also update latest with security patches + type=raw,value=latest + + - name: Extract metadata (manual) + id: meta-manual + if: github.event_name == 'workflow_dispatch' + uses: docker/metadata-action@c299e40c65443455700f0fdfc63efafe5b349051 # v5.10.0 + with: + images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} + tags: | + type=sha,prefix=sha- + + # Build image (release - with cache) + - name: Build and push (release) + id: build-release + if: github.event_name == 'release' + uses: docker/build-push-action@263435318d21b8e681c14492fe198d362a7d2c83 # v6.18.0 + with: + context: . + platforms: linux/amd64,linux/arm64 + push: true + tags: ${{ steps.meta-release.outputs.tags }} + labels: ${{ steps.meta-release.outputs.labels }} + cache-from: type=gha + cache-to: type=gha,mode=max + provenance: true + sbom: true + + # Build image (nightly - no cache, fresh base) + - name: Build and push (nightly) + id: build-nightly + if: github.event_name == 'schedule' && steps.get-version.outputs.skip != 'true' + uses: docker/build-push-action@263435318d21b8e681c14492fe198d362a7d2c83 # v6.18.0 + with: + context: . + platforms: linux/amd64,linux/arm64 + push: true + tags: ${{ steps.meta-nightly.outputs.tags }} + labels: ${{ steps.meta-nightly.outputs.labels }} + pull: true + no-cache: true + provenance: true + sbom: true + + # Build image (manual) + - name: Build and push (manual) + id: build-manual + if: github.event_name == 'workflow_dispatch' + uses: docker/build-push-action@263435318d21b8e681c14492fe198d362a7d2c83 # v6.18.0 + with: + context: . + platforms: linux/amd64,linux/arm64 + push: ${{ inputs.push }} + tags: ${{ steps.meta-manual.outputs.tags }} + labels: ${{ steps.meta-manual.outputs.labels }} + cache-from: type=gha + cache-to: type=gha,mode=max + provenance: true + sbom: true + + # Determine image tag for scanning and testing + - name: Determine image tag + id: image-tag + if: "!(github.event_name == 'schedule' && steps.get-version.outputs.skip == 'true')" + run: | + if [ "${{ github.event_name }}" = "release" ]; then + # Strip 'v' prefix to match semver tag format from metadata-action + echo "tag=$(echo '${{ github.event.release.tag_name }}' | sed 's/^v//')" >> $GITHUB_OUTPUT + elif [ "${{ github.event_name }}" = "schedule" ]; then + echo "tag=nightly" >> $GITHUB_OUTPUT + else + echo "tag=sha-${{ github.sha }}" >> $GITHUB_OUTPUT + fi + + # Vulnerability scanning + - name: Run Trivy vulnerability scanner + if: "!(github.event_name == 'schedule' && steps.get-version.outputs.skip == 'true')" + uses: aquasecurity/trivy-action@b6643a29fecd7f34b3597bc6acb0a98b03d33ff8 # v0.33.1 + with: + image-ref: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ steps.image-tag.outputs.tag }} + format: "sarif" + output: "trivy-results.sarif" + severity: "CRITICAL,HIGH" + continue-on-error: true + + - name: Upload Trivy scan results + if: "!(github.event_name == 'schedule' && steps.get-version.outputs.skip == 'true')" + uses: github/codeql-action/upload-sarif@6e4b8622b82fab3c6ad2a7814fad1effc7615bc8 # v3.28.4 + with: + sarif_file: "trivy-results.sarif" + continue-on-error: true + + # Smoke test - verify image runs correctly + - name: Smoke test + if: "!(github.event_name == 'schedule' && steps.get-version.outputs.skip == 'true')" + run: | + IMAGE_TAG="${{ steps.image-tag.outputs.tag }}" + + docker pull ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${IMAGE_TAG} + + # Test that Python and key modules are available + docker run --rm ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${IMAGE_TAG} \ + python -c "from meshmon.db import init_db; from meshmon.env import get_config; print('Smoke test passed')" + + # Attestation (releases only) + - name: Generate attestation + if: github.event_name == 'release' + uses: actions/attest-build-provenance@46a583fd92dfbf46b772907a9740f888f4324bb9 # v3.1.0 + with: + subject-name: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} + subject-digest: ${{ steps.build-release.outputs.digest }} + push-to-registry: true diff --git a/.gitignore b/.gitignore index 17934f4..c7a4d03 100644 --- a/.gitignore +++ b/.gitignore @@ -17,6 +17,9 @@ build/ .env meshcore.conf +# Docker local overrides +docker-compose.override.yml + # Data directories (keep structure, ignore content) data/snapshots/companion/**/*.json data/snapshots/repeater/**/*.json diff --git a/CLAUDE.md b/CLAUDE.md index f37422e..5fdc7bd 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -171,6 +171,106 @@ Phase 3: Render → Static HTML site (inline SVG) Phase 4: Render → Reports (monthly/yearly statistics) ``` +## Docker Architecture + +The project provides Docker containerization for easy deployment. Two containers work together: + +``` +┌─────────────────────────────────────────────────────────────────┐ +│ Docker Compose │ +│ ┌─────────────────────┐ ┌─────────────────────────────────┐ │ +│ │ meshcore-stats │ │ nginx │ │ +│ │ ┌───────────────┐ │ │ │ │ +│ │ │ Ofelia │ │ │ Serves static site on :8080 │ │ +│ │ │ (scheduler) │ │ │ │ │ +│ │ └───────┬───────┘ │ └──────────────▲──────────────────┘ │ +│ │ │ │ │ │ +│ │ ┌──────▼──────┐ │ ┌─────────┴─────────┐ │ +│ │ │ Python │ │ │ output_data │ │ +│ │ │ Scripts │───┼────────►│ (named volume) │ │ +│ │ └─────────────┘ │ └───────────────────┘ │ +│ │ │ │ │ +│ └──────────┼──────────┘ │ +│ │ │ +│ ┌────────▼────────┐ │ +│ │ ./data/state │ │ +│ │ (bind mount) │ │ +│ └─────────────────┘ │ +└─────────────────────────────────────────────────────────────────┘ +``` + +### Container Files + +| File | Purpose | +|------|---------| +| `Dockerfile` | Multi-stage build: Python + Ofelia scheduler | +| `docker-compose.yml` | Production deployment using published ghcr.io image | +| `docker-compose.dev.yml` | Development override for local builds | +| `docker-compose.override.yml` | Local overrides (gitignored) | +| `docker/ofelia.ini` | Scheduler configuration (cron jobs) | +| `docker/nginx.conf` | nginx configuration for static site serving | +| `.dockerignore` | Files excluded from Docker build context | + +### Docker Compose Files + +**Production** (`docker-compose.yml`): +- Uses published image from `ghcr.io/jorijn/meshcore-stats` +- Image version managed by release-please via `x-release-please-version` placeholder +- Suitable for end users + +**Development** (`docker-compose.dev.yml`): +- Override file that builds locally instead of pulling from registry +- Mounts `src/` and `scripts/` for live code changes +- Usage: `docker compose -f docker-compose.yml -f docker-compose.dev.yml up --build` + +**Local overrides** (`docker-compose.override.yml`): +- Gitignored file for local customizations (e.g., device paths, env_file) +- Automatically merged when running `docker compose up` + +### Ofelia Scheduler + +[Ofelia](https://github.com/mcuadros/ofelia) is a lightweight job scheduler designed for Docker. It replaces cron for container environments. + +Jobs configured in `docker/ofelia.ini`: +- `collect-companion`: Every minute (with `no-overlap=true`) +- `collect-repeater`: Every 15 minutes at :01, :16, :31, :46 (with `no-overlap=true`) +- `render-charts`: Every 5 minutes +- `render-site`: Every 5 minutes +- `render-reports`: Daily at midnight +- `db-maintenance`: Monthly at 3 AM on the 1st + +### GitHub Actions Workflow + +`.github/workflows/docker-publish.yml` builds and publishes Docker images: + +| Trigger | Tags Created | +|---------|--------------| +| Release | `X.Y.Z`, `X.Y`, `latest` | +| Nightly (4 AM UTC) | Rebuilds all version tags + `nightly`, `nightly-YYYYMMDD` | +| Manual | `sha-xxxxxx` | + +**Nightly rebuilds** ensure version tags always include the latest OS security patches. This is a common pattern used by official Docker images (nginx, postgres, node). Users needing reproducibility should pin by SHA digest or use dated nightly tags. + +All GitHub Actions are pinned by full SHA for security. Dependabot can be configured to update these automatically. + +### Version Placeholder + +The version in `docker-compose.yml` uses release-please's placeholder syntax: +```yaml +image: ghcr.io/jorijn/meshcore-stats:0.3.0 # x-release-please-version +``` + +This is automatically updated when a new release is created. + +### Agent Review Guidelines + +When reviewing Docker-related changes, always provide the **full plan or implementation** to review agents. Do not summarize or abbreviate - agents need complete context to provide accurate feedback. + +Relevant agents for Docker reviews: +- **k8s-security-reviewer**: Container security, RBAC, secrets handling +- **cicd-pipeline-specialist**: GitHub Actions workflows, build pipelines +- **python-code-reviewer**: Dockerfile Python-specific issues (venv PATH, runtime libs) + ## Directory Structure ``` diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..625015b --- /dev/null +++ b/Dockerfile @@ -0,0 +1,112 @@ +# ============================================================================= +# Stage 1: Build dependencies +# ============================================================================= +FROM python:3.12-slim-bookworm AS builder + +# Ofelia version and checksums (verified from GitHub releases) +ARG OFELIA_VERSION=0.3.12 +ARG TARGETARCH +ARG OFELIA_SHA256_AMD64=cf06d2199abafbd3aa5afe0f8266e478818faacd11555b99200707321035c931 +ARG OFELIA_SHA256_ARM64=57760ef7f17a2cd55b5b1e1946f79b91b24bde40d47e81a0d75fd1470d883c1a + +# Install build dependencies for Python packages +RUN apt-get update && apt-get install -y --no-install-recommends \ + gcc \ + libfreetype6-dev \ + libpng-dev \ + curl \ + && rm -rf /var/lib/apt/lists/* + +# Download and verify Ofelia binary in builder stage (keeps curl out of runtime) +RUN set -ex; \ + if [ "$TARGETARCH" = "amd64" ]; then \ + OFELIA_SHA256="$OFELIA_SHA256_AMD64"; \ + elif [ "$TARGETARCH" = "arm64" ]; then \ + OFELIA_SHA256="$OFELIA_SHA256_ARM64"; \ + else \ + echo "Unsupported architecture: $TARGETARCH" && exit 1; \ + fi; \ + curl -fsSL "https://github.com/mcuadros/ofelia/releases/download/v${OFELIA_VERSION}/ofelia_${OFELIA_VERSION}_linux_${TARGETARCH}.tar.gz" -o /tmp/ofelia.tar.gz \ + && echo "${OFELIA_SHA256} /tmp/ofelia.tar.gz" | sha256sum -c - \ + && tar -xzf /tmp/ofelia.tar.gz -C /usr/local/bin ofelia \ + && rm /tmp/ofelia.tar.gz \ + && chmod +x /usr/local/bin/ofelia + +# Create virtual environment +RUN python -m venv /opt/venv +ENV PATH="/opt/venv/bin:$PATH" + +# Install Python dependencies +COPY requirements.txt . +RUN pip install --no-cache-dir --upgrade pip && \ + pip install --no-cache-dir -r requirements.txt + +# ============================================================================= +# Stage 2: Runtime +# ============================================================================= +FROM python:3.12-slim-bookworm + +# OCI Labels +LABEL org.opencontainers.image.source="https://github.com/jorijn/meshcore-stats" +LABEL org.opencontainers.image.description="MeshCore Stats - LoRa mesh network monitoring" +LABEL org.opencontainers.image.licenses="MIT" + +# Install runtime dependencies +# - tini: init system for proper signal handling +# - libfreetype6, libpng16-16: matplotlib runtime libraries +# - fontconfig, fonts-dejavu-core: fonts for chart text rendering +RUN apt-get update && apt-get install -y --no-install-recommends \ + tini \ + libfreetype6 \ + libpng16-16 \ + fontconfig \ + fonts-dejavu-core \ + && rm -rf /var/lib/apt/lists/* \ + # Build font cache for matplotlib + && fc-cache -f \ + # Remove setuid/setgid binaries for security + && find / -perm /6000 -type f -exec chmod a-s {} \; 2>/dev/null || true + +# Create non-root user with dialout group for serial access +RUN groupadd -g 1000 meshmon \ + && useradd -u 1000 -g meshmon -G dialout -s /sbin/nologin meshmon \ + && mkdir -p /data/state /out /tmp/matplotlib \ + && chown -R meshmon:meshmon /data /out /tmp/matplotlib + +# Copy Ofelia binary from builder (keeps curl out of runtime image) +COPY --from=builder /usr/local/bin/ofelia /usr/local/bin/ofelia + +# Copy virtual environment from builder +COPY --from=builder /opt/venv /opt/venv + +# Copy application code +COPY --chown=meshmon:meshmon src/ /app/src/ +COPY --chown=meshmon:meshmon scripts/ /app/scripts/ +COPY --chown=meshmon:meshmon docker/ofelia.ini /app/ofelia.ini + +# Environment configuration +# - PATH: Include venv so Ofelia can run Python +# - PYTHONPATH: Allow imports from src/meshmon +# - PYTHONUNBUFFERED: Ensure logs are output immediately +# - PYTHONDONTWRITEBYTECODE: Don't create .pyc files +# - MPLCONFIGDIR: Matplotlib font cache directory +ENV PATH="/opt/venv/bin:$PATH" \ + PYTHONPATH=/app/src \ + PYTHONUNBUFFERED=1 \ + PYTHONDONTWRITEBYTECODE=1 \ + MPLCONFIGDIR=/tmp/matplotlib + +WORKDIR /app + +# Run as non-root user +USER meshmon + +# Use tini as init system for proper signal handling +ENTRYPOINT ["/usr/bin/tini", "--"] + +# Run Ofelia scheduler +CMD ["ofelia", "daemon", "--config=/app/ofelia.ini"] + +# Health check - verify database is accessible +HEALTHCHECK --interval=5m --timeout=30s --start-period=60s --retries=3 \ + CMD python -c "import sqlite3; sqlite3.connect('/data/state/metrics.db').execute('SELECT 1')" || exit 1 diff --git a/README.md b/README.md index ac6acc6..810098b 100644 --- a/README.md +++ b/README.md @@ -112,14 +112,138 @@ MESHCORE=/home/user/meshcore-stats - `cd $MESHCORE` is required because paths in the config are relative to the project root - `flock` prevents USB serial conflicts when companion and repeater collection overlap -### Docker / Container Usage +### Docker Installation -When running in Docker, you can skip the config file and pass environment variables directly: +The recommended way to run MeshCore Stats is with Docker Compose. This provides automatic scheduling of all collection and rendering tasks. + +#### Quick Start ```bash -docker run -e MESH_SERIAL_PORT=/dev/ttyUSB0 -e REPEATER_NAME="My Repeater" ... +# Clone the repository +git clone https://github.com/jorijn/meshcore-stats.git +cd meshcore-stats + +# Create configuration +cp meshcore.conf.example meshcore.conf +# Edit meshcore.conf with your settings + +# Create data directories with correct ownership for container (UID 1000) +mkdir -p data/state out +sudo chown -R 1000:1000 data out +# Alternative: chmod -R 777 data out (less secure, use chown if possible) + +# Start the containers +docker compose up -d + +# View logs +docker compose logs -f ``` +The web interface will be available at `http://localhost:8080`. + +#### Architecture + +The Docker setup uses two containers: + +| Container | Purpose | +|-----------|---------| +| `meshcore-stats` | Runs Ofelia scheduler for data collection and rendering | +| `nginx` | Serves the static website | + +#### Configuration + +Configuration is loaded from `meshcore.conf` via the `env_file` directive. Key settings: + +```bash +# Required: Serial device for companion node +MESH_SERIAL_PORT=/dev/ttyUSB0 # Adjust for your system + +# Required: Repeater identity +REPEATER_NAME="Your Repeater Name" +REPEATER_PASSWORD="your-password" + +# Display names (shown in UI) +REPEATER_DISPLAY_NAME="My Repeater" +COMPANION_DISPLAY_NAME="My Companion" +``` + +See `meshcore.conf.example` for all available options. + +#### Serial Device Access + +The container needs access to your USB serial device. To customize the device path without modifying the tracked `docker-compose.yml`, create a `docker-compose.override.yml` file (gitignored): + +```yaml +# docker-compose.override.yml - Local overrides (not tracked in git) +services: + meshcore-stats: + devices: + - /dev/ttyACM0:/dev/ttyACM0:rw # Your device path +``` + +This file is automatically merged with `docker-compose.yml` when running `docker compose up`. + +On the host, ensure the device is accessible: + +```bash +# Add user to dialout group (Linux) +sudo usermod -a -G dialout $USER +# Log out and back in for changes to take effect +``` + +#### Development Mode + +For local development with live code changes: + +```bash +docker compose -f docker-compose.yml -f docker-compose.dev.yml up --build +``` + +This mounts `src/` and `scripts/` into the container, so changes take effect immediately without rebuilding. + +#### Image Tags + +Images are published to `ghcr.io/jorijn/meshcore-stats`: + +| Tag | Description | +|-----|-------------| +| `X.Y.Z` | Specific version (e.g., `0.3.0`) | +| `latest` | Latest release | +| `nightly` | Latest release rebuilt with OS patches | +| `nightly-YYYYMMDD` | Dated nightly build | + +Version tags are rebuilt nightly to include OS security patches. For reproducible deployments, pin by SHA digest: + +```yaml +image: ghcr.io/jorijn/meshcore-stats@sha256:abc123... +``` + +#### Volumes + +| Path | Purpose | +|------|---------| +| `./data/state` | SQLite database and circuit breaker state | +| `./out` | Generated static site (served by nginx) | + +Both directories must be writable by UID 1000 (the container user). See Quick Start for setup. + +#### Resource Limits + +Default resource limits in `docker-compose.yml`: + +| Container | CPU | Memory | +|-----------|-----|--------| +| meshcore-stats | 1.0 | 512MB | +| nginx | 0.5 | 64MB | + +Adjust in `docker-compose.yml` if needed. + +#### Important Notes + +- **Single instance only**: SQLite uses WAL mode which requires exclusive access. Do not run multiple container instances. +- **Persistent storage**: Mount `./data/state` to preserve your database across container restarts. +- **Health checks**: Both containers have health checks. Use `docker compose ps` to verify status. + Environment variables always take precedence over `meshcore.conf`. ### Serving the Site @@ -243,6 +367,34 @@ If repeater collection shows "cooldown active": rm data/state/repeater_circuit.json ``` +### Docker on macOS: Serial Devices Not Available + +Docker on macOS (including Docker Desktop and OrbStack) runs containers inside a Linux virtual machine. USB and serial devices connected to the Mac host cannot be passed through to this VM, so the `devices:` section in docker-compose.yml will fail with: + +``` +error gathering device information while adding custom device "/dev/cu.usbserial-0001": no such file or directory +``` + +**Workarounds:** + +1. **Use TCP transport**: Run a serial-to-TCP bridge on the host and configure the container to connect via TCP: + ```bash + # On macOS host, expose serial port over TCP (install socat via Homebrew) + socat TCP-LISTEN:5000,fork,reuseaddr OPEN:/dev/cu.usbserial-0001,rawer,nonblock,ispeed=115200,ospeed=115200 + ``` + Then configure in meshcore.conf: + ```bash + MESH_TRANSPORT=tcp + MESH_TCP_HOST=host.docker.internal + MESH_TCP_PORT=5000 + ``` + +2. **Run natively on macOS**: Use the cron-based setup instead of Docker (see "Cron Setup" section). + +3. **Use a Linux host**: Docker on Linux can pass through USB devices directly. + +Note: OrbStack has [USB passthrough on their roadmap](https://github.com/orbstack/orbstack/issues/89) but it is not yet available. + ## Environment Variables Reference | Variable | Default | Description | @@ -260,7 +412,6 @@ If repeater collection shows "cooldown active": | `REPEATER_NAME` | - | Repeater advertised name | | `REPEATER_KEY_PREFIX` | - | Repeater public key prefix | | `REPEATER_PASSWORD` | - | Repeater login password | -| `REPEATER_FETCH_ACL` | 0 | Also fetch ACL from repeater | | **Display Names** | | | | `REPEATER_DISPLAY_NAME` | Repeater Node | Display name for repeater in UI | | `COMPANION_DISPLAY_NAME` | Companion Node | Display name for companion in UI | diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml new file mode 100644 index 0000000..c1334a4 --- /dev/null +++ b/docker-compose.dev.yml @@ -0,0 +1,29 @@ +# MeshCore Stats - Development Override +# +# Use this file for local development with live code changes. +# +# Usage: +# docker compose -f docker-compose.yml -f docker-compose.development.yml up --build +# +# This override: +# - Builds the image locally instead of pulling from ghcr.io +# - Mounts src/ and scripts/ for live code changes (no rebuild needed) + +services: + meshcore-stats: + # Build locally instead of using published image + build: + context: . + dockerfile: Dockerfile + + # Remove the image reference (use built image) + image: meshcore-stats:development + + # Mount source code for live development + # Changes to Python files take effect immediately (no rebuild needed) + volumes: + - ./data/state:/data/state + - ./out:/out + # Development mounts (read-only to prevent accidental writes) + - ./src:/app/src:ro + - ./scripts:/app/scripts:ro diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..467aada --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,135 @@ +# MeshCore Stats - Docker Compose Configuration +# +# Production deployment using published container image. +# For local development, use: docker compose -f docker-compose.yml -f docker-compose.development.yml up +# +# Prerequisites: +# 1. Copy meshcore.conf.example to meshcore.conf and configure +# 2. Ensure your user has access to the serial device (dialout group) +# 3. Create data directories with correct ownership: +# mkdir -p ./data/state ./out && sudo chown -R 1000:1000 ./data ./out + +services: + # ========================================================================== + # MeshCore Stats - Data collection and rendering + # ========================================================================== + meshcore-stats: + image: ghcr.io/jorijn/meshcore-stats:0.3.0 # x-release-please-version + container_name: meshcore-stats + restart: unless-stopped + + # Load configuration from meshcore.conf + env_file: + - meshcore.conf + + # Serial device for companion node communication + devices: + # Update to match your serial device (e.g., /dev/ttyACM0, /dev/cu.usbserial-*) + - /dev/ttyUSB0:/dev/ttyUSB0:rw + + volumes: + # Persistent storage for SQLite database and circuit breaker state + - ./data/state:/data/state + # Generated static site (served by nginx) + - ./out:/out + + # Run as meshmon user (UID 1000) + user: "1000:1000" + + # Add dialout group for serial port access + group_add: + - dialout + + # Security hardening + security_opt: + - no-new-privileges:true + cap_drop: + - ALL + read_only: true + tmpfs: + - /tmp:noexec,nosuid,size=64m + + # Resource limits + deploy: + resources: + limits: + cpus: "1.0" + memory: 512M + reservations: + cpus: "0.1" + memory: 128M + + # Logging limits to prevent disk exhaustion + logging: + driver: json-file + options: + max-size: "10m" + max-file: "3" + + # Health check + healthcheck: + test: ["CMD", "python", "-c", "import sqlite3; sqlite3.connect('/data/state/metrics.db').execute('SELECT 1')"] + interval: 5m + timeout: 30s + start_period: 60s + retries: 3 + + # ========================================================================== + # nginx - Static site server + # ========================================================================== + nginx: + image: nginx:1.27-alpine + container_name: meshcore-stats-nginx + restart: unless-stopped + + # Run as nginx user (UID 101 in Alpine nginx image) + user: "101:101" + + ports: + - "8080:8080" + + volumes: + # Mount generated static site from meshcore-stats container + - ./out:/usr/share/nginx/html:ro + # Custom nginx configuration + - ./docker/nginx.conf:/etc/nginx/conf.d/default.conf:ro + + # Security hardening + security_opt: + - no-new-privileges:true + cap_drop: + - ALL + # NET_BIND_SERVICE not needed for port 8080 (unprivileged) + read_only: true + tmpfs: + - /var/cache/nginx:noexec,nosuid,size=16m,uid=101,gid=101 + - /var/run:noexec,nosuid,size=1m,uid=101,gid=101 + + # Resource limits + deploy: + resources: + limits: + cpus: "0.5" + memory: 64M + reservations: + cpus: "0.05" + memory: 16M + + # Logging limits + logging: + driver: json-file + options: + max-size: "10m" + max-file: "3" + + # Health check + healthcheck: + test: ["CMD", "wget", "--quiet", "--tries=1", "--spider", "http://localhost:8080/health"] + interval: 30s + timeout: 10s + start_period: 5s + retries: 3 + + depends_on: + meshcore-stats: + condition: service_healthy diff --git a/docker/nginx.conf b/docker/nginx.conf new file mode 100644 index 0000000..b3d77bb --- /dev/null +++ b/docker/nginx.conf @@ -0,0 +1,67 @@ +# nginx configuration for MeshCore Stats static site +# This file is used by the nginx container in docker-compose.yml + +server { + listen 8080; + server_name _; + + root /usr/share/nginx/html; + index day.html index.html; + + # UTF-8 charset for all text files + charset utf-8; + charset_types text/plain text/css text/javascript application/json image/svg+xml; + + # Gzip compression + gzip on; + gzip_vary on; + gzip_min_length 1024; + gzip_types text/plain text/css text/javascript application/json image/svg+xml; + + # Security headers + add_header X-Frame-Options "SAMEORIGIN" always; + add_header X-Content-Type-Options "nosniff" always; + add_header Referrer-Policy "strict-origin-when-cross-origin" always; + + # HTML, JSON, TXT files - no cache (frequently updated) + location ~* \.(html|json|txt)$ { + add_header Cache-Control "no-cache, no-store, must-revalidate" always; + add_header Pragma "no-cache" always; + add_header Expires "0" always; + # Re-add security headers (add_header in location blocks replaces parent) + add_header X-Frame-Options "SAMEORIGIN" always; + add_header X-Content-Type-Options "nosniff" always; + add_header Referrer-Policy "strict-origin-when-cross-origin" always; + } + + # PNG files - no cache (charts are regenerated frequently) + location ~* \.png$ { + add_header Cache-Control "no-cache, no-store, must-revalidate" always; + add_header Pragma "no-cache" always; + add_header Expires "0" always; + add_header X-Frame-Options "SAMEORIGIN" always; + add_header X-Content-Type-Options "nosniff" always; + add_header Referrer-Policy "strict-origin-when-cross-origin" always; + } + + # CSS, JS, SVG files - short cache (5 minutes) + location ~* \.(css|js|svg)$ { + expires 5m; + add_header Cache-Control "public, max-age=300" always; + add_header X-Frame-Options "SAMEORIGIN" always; + add_header X-Content-Type-Options "nosniff" always; + add_header Referrer-Policy "strict-origin-when-cross-origin" always; + } + + # Default location + location / { + try_files $uri $uri/ =404; + } + + # Health check endpoint + location /health { + access_log off; + return 200 "OK\n"; + add_header Content-Type text/plain; + } +} diff --git a/docker/ofelia.ini b/docker/ofelia.ini new file mode 100644 index 0000000..3a293a2 --- /dev/null +++ b/docker/ofelia.ini @@ -0,0 +1,51 @@ +# Ofelia Job Scheduler Configuration +# https://github.com/mcuadros/ofelia +# +# This file defines the cron-like schedule for all MeshCore Stats tasks. +# Jobs run inside the same container (job-local). + +# ============================================================================= +# Data Collection Jobs +# ============================================================================= + +[job-local "collect-companion"] +# Collect metrics from companion node (USB serial) +schedule = @every 1m +command = python /app/scripts/collect_companion.py +no-overlap = true + +[job-local "collect-repeater"] +# Collect metrics from repeater node (via LoRa) +# Offset by 1 second to avoid USB serial conflicts with companion collection +schedule = 1 1,16,31,46 * * * * +command = python /app/scripts/collect_repeater.py +no-overlap = true + +# ============================================================================= +# Rendering Jobs +# ============================================================================= + +[job-local "render-charts"] +# Generate SVG charts from database +schedule = @every 5m +command = python /app/scripts/render_charts.py + +[job-local "render-site"] +# Generate static HTML site +schedule = @every 5m +command = python /app/scripts/render_site.py + +[job-local "render-reports"] +# Generate monthly/yearly statistics reports +schedule = @daily +command = python /app/scripts/render_reports.py + +# ============================================================================= +# Maintenance Jobs +# ============================================================================= + +[job-local "db-maintenance"] +# Database VACUUM and ANALYZE for optimal performance +# Runs at 3 AM on the 1st of each month +schedule = 0 3 1 * * +command = python -c "import sqlite3; db=sqlite3.connect('/data/state/metrics.db'); db.execute('VACUUM'); db.execute('ANALYZE'); db.close(); print('Database maintenance complete')" diff --git a/meshcore.conf.example b/meshcore.conf.example index 0dc8bb3..f13f3c9 100644 --- a/meshcore.conf.example +++ b/meshcore.conf.example @@ -2,55 +2,66 @@ # Copy this file to meshcore.conf and customize for your setup: # cp meshcore.conf.example meshcore.conf # -# This file is automatically loaded by the scripts. No need to source it manually. -# Environment variables always take precedence over this file (useful for Docker). +# Format: KEY=value (no 'export' keyword, no spaces around '=') +# This format is compatible with both Docker env_file and shell 'source' command. +# Comments start with # and blank lines are ignored. # ============================================================================= # Connection Settings # ============================================================================= -export MESH_TRANSPORT=serial -export MESH_SERIAL_PORT=/dev/ttyUSB0 # Adjust for your system (e.g., /dev/ttyACM0, /dev/cu.usbserial-*) -export MESH_SERIAL_BAUD=115200 -export MESH_DEBUG=0 # Set to 1 for verbose meshcore debug output +MESH_TRANSPORT=serial +MESH_SERIAL_PORT=/dev/ttyUSB0 +# MESH_SERIAL_BAUD=115200 +# MESH_DEBUG=0 + +# TCP transport (for macOS Docker or remote serial servers) +# MESH_TRANSPORT=tcp +# MESH_TCP_HOST=host.docker.internal +# MESH_TCP_PORT=5000 + +# BLE transport (Bluetooth Low Energy) +# MESH_TRANSPORT=ble +# MESH_BLE_ADDR=AA:BB:CC:DD:EE:FF +# MESH_BLE_PIN=123456 # ============================================================================= # Remote Repeater Identity # ============================================================================= # At least REPEATER_NAME or REPEATER_KEY_PREFIX is required to identify your repeater -export REPEATER_NAME="Your Repeater Name" # Advertised name shown in contacts -# export REPEATER_KEY_PREFIX="a1b2c3" # Alternative: hex prefix of public key -export REPEATER_PASSWORD="your-password" # Admin password for repeater login +REPEATER_NAME=Your Repeater Name +# REPEATER_KEY_PREFIX=a1b2c3 +REPEATER_PASSWORD=your-password # ============================================================================= # Display Names (shown in UI) # ============================================================================= -export REPEATER_DISPLAY_NAME="My Repeater" -export COMPANION_DISPLAY_NAME="My Companion" +REPEATER_DISPLAY_NAME=My Repeater +COMPANION_DISPLAY_NAME=My Companion # Public key prefixes (shown below node name in sidebar, e.g., "!a1b2c3d4") -# export REPEATER_PUBKEY_PREFIX="!a1b2c3d4" -# export COMPANION_PUBKEY_PREFIX="!e5f6g7h8" +# REPEATER_PUBKEY_PREFIX=!a1b2c3d4 +# COMPANION_PUBKEY_PREFIX=!e5f6g7h8 # ============================================================================= # Location Metadata (for reports and sidebar display) # ============================================================================= -export REPORT_LOCATION_NAME="City, Country" # Full location name for reports -export REPORT_LOCATION_SHORT="City, XX" # Short version for sidebar/meta -export REPORT_LAT=0.0 # Latitude in decimal degrees -export REPORT_LON=0.0 # Longitude in decimal degrees -export REPORT_ELEV=0 # Elevation -export REPORT_ELEV_UNIT=m # "m" for meters, "ft" for feet +REPORT_LOCATION_NAME=City, Country +REPORT_LOCATION_SHORT=City, XX +REPORT_LAT=0.0 +REPORT_LON=0.0 +REPORT_ELEV=0 +REPORT_ELEV_UNIT=m # ============================================================================= # Hardware Info (shown in sidebar) # ============================================================================= -export REPEATER_HARDWARE="Your Repeater Model" # e.g., "SenseCAP P1-Pro", "LILYGO T-Beam" -export COMPANION_HARDWARE="Your Companion Model" # e.g., "Elecrow ThinkNode-M1", "Heltec V3" +REPEATER_HARDWARE=Your Repeater Model +COMPANION_HARDWARE=Your Companion Model # ============================================================================= # Radio Configuration Presets @@ -59,58 +70,54 @@ export COMPANION_HARDWARE="Your Companion Model" # e.g., "Elecrow ThinkNode-M1", # or set custom values. These are for display purposes only. # MeshCore EU/UK Narrow (default) -export RADIO_FREQUENCY="869.618 MHz" -export RADIO_BANDWIDTH="62.5 kHz" -export RADIO_SPREAD_FACTOR="SF8" -export RADIO_CODING_RATE="CR8" +RADIO_FREQUENCY=869.618 MHz +RADIO_BANDWIDTH=62.5 kHz +RADIO_SPREAD_FACTOR=SF8 +RADIO_CODING_RATE=CR8 -# # MeshCore EU/UK Wide -# export RADIO_FREQUENCY="869.525 MHz" -# export RADIO_BANDWIDTH="250 kHz" -# export RADIO_SPREAD_FACTOR="SF10" -# export RADIO_CODING_RATE="CR5" +# MeshCore EU/UK Wide +# RADIO_FREQUENCY=869.525 MHz +# RADIO_BANDWIDTH=250 kHz +# RADIO_SPREAD_FACTOR=SF10 +# RADIO_CODING_RATE=CR5 -# # MeshCore US Standard -# export RADIO_FREQUENCY="906.875 MHz" -# export RADIO_BANDWIDTH="250 kHz" -# export RADIO_SPREAD_FACTOR="SF10" -# export RADIO_CODING_RATE="CR5" +# MeshCore US Standard +# RADIO_FREQUENCY=906.875 MHz +# RADIO_BANDWIDTH=250 kHz +# RADIO_SPREAD_FACTOR=SF10 +# RADIO_CODING_RATE=CR5 -# # MeshCore US Fast -# export RADIO_FREQUENCY="906.875 MHz" -# export RADIO_BANDWIDTH="500 kHz" -# export RADIO_SPREAD_FACTOR="SF7" -# export RADIO_CODING_RATE="CR5" +# MeshCore US Fast +# RADIO_FREQUENCY=906.875 MHz +# RADIO_BANDWIDTH=500 kHz +# RADIO_SPREAD_FACTOR=SF7 +# RADIO_CODING_RATE=CR5 -# # MeshCore ANZ (Australia/New Zealand) -# export RADIO_FREQUENCY="917.0 MHz" -# export RADIO_BANDWIDTH="250 kHz" -# export RADIO_SPREAD_FACTOR="SF10" -# export RADIO_CODING_RATE="CR5" +# MeshCore ANZ (Australia/New Zealand) +# RADIO_FREQUENCY=917.0 MHz +# RADIO_BANDWIDTH=250 kHz +# RADIO_SPREAD_FACTOR=SF10 +# RADIO_CODING_RATE=CR5 # ============================================================================= # Intervals and Timeouts # ============================================================================= -export COMPANION_STEP=60 # Collection interval for companion (seconds) -export REPEATER_STEP=900 # Collection interval for repeater (seconds, 15min default) -export REMOTE_TIMEOUT_S=10 # Minimum timeout for LoRa requests -export REMOTE_RETRY_ATTEMPTS=2 # Number of retry attempts -export REMOTE_RETRY_BACKOFF_S=4 # Seconds between retries +# COMPANION_STEP=60 +# REPEATER_STEP=900 +# REMOTE_TIMEOUT_S=10 +# REMOTE_RETRY_ATTEMPTS=2 +# REMOTE_RETRY_BACKOFF_S=4 # Circuit breaker settings (prevents spamming LoRa when repeater is unreachable) -export REMOTE_CB_FAILS=6 # Failures before circuit breaker opens -export REMOTE_CB_COOLDOWN_S=3600 # Cooldown period in seconds (1 hour) +# REMOTE_CB_FAILS=6 +# REMOTE_CB_COOLDOWN_S=3600 # ============================================================================= -# Paths +# Paths (Native installation only) # ============================================================================= +# Docker: Leave these commented. The container uses /data/state and /out by default. +# Native: Uncomment for local cron-based installation: +# STATE_DIR=./data/state +# OUT_DIR=./out -export STATE_DIR=./data/state # SQLite database and circuit breaker state -export OUT_DIR=./out # Generated static site output - -# ============================================================================= -# Optional -# ============================================================================= - -export REPEATER_FETCH_ACL=0 # Set to 1 to fetch ACL from repeater diff --git a/release-please-config.json b/release-please-config.json index d9cd985..7d18b94 100644 --- a/release-please-config.json +++ b/release-please-config.json @@ -9,6 +9,11 @@ "type": "generic", "path": "src/meshmon/__init__.py", "glob": false + }, + { + "type": "generic", + "path": "docker-compose.yml", + "glob": false } ], "changelog-sections": [ diff --git a/scripts/collect_repeater.py b/scripts/collect_repeater.py index ba1b57e..ef8c2ab 100755 --- a/scripts/collect_repeater.py +++ b/scripts/collect_repeater.py @@ -233,20 +233,6 @@ async def collect_repeater() -> int: else: log.warn(f"req_status_sync failed: {err}") - # Optional ACL query (using _sync version) - if cfg.repeater_fetch_acl: - log.debug("Querying repeater ACL...") - success, payload, err = await query_repeater_with_retry( - mc, - contact, - "req_acl_sync", - lambda: cmd.req_acl_sync(contact, timeout=0, min_timeout=cfg.remote_timeout_s), - ) - if success: - log.debug(f"req_acl_sync: {payload}") - else: - log.debug(f"req_acl_sync failed: {err}") - # Update circuit breaker if status_ok: cb.record_success() diff --git a/src/meshmon/env.py b/src/meshmon/env.py index f2c662a..1561cda 100644 --- a/src/meshmon/env.py +++ b/src/meshmon/env.py @@ -145,7 +145,6 @@ class Config: self.repeater_name = get_str("REPEATER_NAME") self.repeater_key_prefix = get_str("REPEATER_KEY_PREFIX") self.repeater_password = get_str("REPEATER_PASSWORD") - self.repeater_fetch_acl = get_bool("REPEATER_FETCH_ACL", False) # Intervals and timeouts self.companion_step = get_int("COMPANION_STEP", 60) @@ -156,9 +155,9 @@ class Config: self.remote_cb_fails = get_int("REMOTE_CB_FAILS", 6) self.remote_cb_cooldown_s = get_int("REMOTE_CB_COOLDOWN_S", 3600) - # Paths - self.state_dir = get_path("STATE_DIR", "./data/state") - self.out_dir = get_path("OUT_DIR", "./out") + # Paths (defaults are Docker container paths; native installs override via config) + self.state_dir = get_path("STATE_DIR", "/data/state") + self.out_dir = get_path("OUT_DIR", "/out") # Report location metadata self.report_location_name = get_str(