From 058a6e2c95abe4807259a3ced67bab910f36d320 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 4 Dec 2025 15:31:04 +0000 Subject: [PATCH] Update docs: Docker profiles, seed data, and Members model - Update Docker Compose section: core services run by default (mqtt, collector, api, web), optional profiles for interfaces and utilities - Document automatic seeding on collector startup - Add SEED_HOME environment variable documentation - Document new Members model and YAML seed file format - Update node_tags format to YAML with public_key-keyed structure - Update project structure to reflect seed/ and data/ directories - Add CLI reference for collector seed commands --- AGENTS.md | 86 ++++++++++++++-------- README.md | 212 +++++++++++++++++++++++++++++++++++------------------- 2 files changed, 197 insertions(+), 101 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index ba82af0..a5e5be8 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -114,17 +114,15 @@ class NodeRead(BaseModel): ### SQLAlchemy Models ```python -from sqlalchemy import String, DateTime, ForeignKey +from sqlalchemy import String, DateTime, Text from sqlalchemy.orm import Mapped, mapped_column, relationship -from datetime import datetime -from uuid import uuid4 +from typing import Optional -from meshcore_hub.common.models.base import Base +from meshcore_hub.common.models.base import Base, TimestampMixin, UUIDMixin -class Node(Base): +class Node(Base, UUIDMixin, TimestampMixin): __tablename__ = "nodes" - id: Mapped[str] = mapped_column(String(36), primary_key=True, default=lambda: str(uuid4())) public_key: Mapped[str] = mapped_column(String(64), unique=True, index=True) name: Mapped[str | None] = mapped_column(String(255), nullable=True) adv_type: Mapped[str | None] = mapped_column(String(20), nullable=True) @@ -132,6 +130,18 @@ class Node(Base): # Relationships tags: Mapped[list["NodeTag"]] = relationship(back_populates="node", cascade="all, delete-orphan") + + +class Member(Base, UUIDMixin, TimestampMixin): + """Network member model - stores info about network operators.""" + __tablename__ = "members" + + name: Mapped[str] = mapped_column(String(255), nullable=False) + callsign: Mapped[Optional[str]] = mapped_column(String(20), nullable=True) + role: Mapped[Optional[str]] = mapped_column(String(100), nullable=True) + description: Mapped[Optional[str]] = mapped_column(Text, nullable=True) + contact: Mapped[Optional[str]] = mapped_column(String(255), nullable=True) + public_key: Mapped[Optional[str]] = mapped_column(String(64), nullable=True, index=True) ``` ### FastAPI Routes @@ -239,7 +249,12 @@ meshcore-hub/ │ │ ├── mqtt.py # MQTT utilities │ │ ├── logging.py # Logging config │ │ ├── models/ # SQLAlchemy models +│ │ │ ├── node.py # Node model +│ │ │ ├── member.py # Network member model +│ │ │ └── ... │ │ └── schemas/ # Pydantic schemas +│ │ ├── members.py # Member API schemas +│ │ └── ... │ ├── interface/ │ │ ├── cli.py │ │ ├── device.py # MeshCore device wrapper @@ -247,9 +262,10 @@ meshcore-hub/ │ │ ├── receiver.py # RECEIVER mode │ │ └── sender.py # SENDER mode │ ├── collector/ -│ │ ├── cli.py +│ │ ├── cli.py # Collector CLI with seed commands │ │ ├── subscriber.py # MQTT subscriber -│ │ ├── tag_import.py # Tag import from JSON +│ │ ├── tag_import.py # Tag import from YAML +│ │ ├── member_import.py # Member import from YAML │ │ ├── handlers/ # Event handlers │ │ └── webhook.py # Webhook dispatcher │ ├── api/ @@ -258,11 +274,15 @@ meshcore-hub/ │ │ ├── auth.py # Authentication │ │ ├── dependencies.py │ │ ├── routes/ # API routes +│ │ │ ├── members.py # Member CRUD endpoints +│ │ │ └── ... │ │ └── templates/ # Dashboard HTML │ └── web/ │ ├── cli.py │ ├── app.py # FastAPI app │ ├── routes/ # Page routes +│ │ ├── members.py # Members page +│ │ └── ... │ ├── templates/ # Jinja2 templates │ └── static/ # CSS, JS ├── tests/ @@ -278,20 +298,17 @@ meshcore-hub/ ├── etc/ │ └── mosquitto.conf # MQTT broker configuration ├── example/ -│ └── data/ -│ ├── collector/ -│ │ └── tags.json # Example node tags data -│ └── web/ -│ └── members.json # Example network members data +│ └── seed/ # Example seed data files +│ ├── node_tags.yaml # Example node tags +│ └── members.yaml # Example network members +├── seed/ # Seed data directory (SEED_HOME) +│ ├── node_tags.yaml # Node tags for import +│ └── members.yaml # Network members for import ├── data/ # Runtime data (gitignored, DATA_HOME default) -│ ├── collector/ # Collector data -│ │ ├── meshcore.db # SQLite database -│ │ └── tags.json # Node tags for import -│ └── web/ # Web data -│ └── members.json # Network members list +│ └── collector/ # Collector data +│ └── meshcore.db # SQLite database ├── Dockerfile # Docker build configuration -├── docker-compose.yml # Docker Compose services (gitignored) -└── docker-compose.yml.example # Docker Compose template +└── docker-compose.yml # Docker Compose services ``` ## MQTT Topic Structure @@ -436,26 +453,39 @@ meshcore-hub interface --mode receiver --mock See [PLAN.md](PLAN.md#configuration-environment-variables) for complete list. Key variables: -- `DATA_HOME` - Base directory for all service data (default: `./data`) +- `DATA_HOME` - Base directory for runtime data (default: `./data`) +- `SEED_HOME` - Directory containing seed data files (default: `./seed`) - `MQTT_HOST`, `MQTT_PORT`, `MQTT_PREFIX` - MQTT broker connection - `DATABASE_URL` - SQLAlchemy database URL (default: `sqlite:///{DATA_HOME}/collector/meshcore.db`) - `API_READ_KEY`, `API_ADMIN_KEY` - API authentication keys - `LOG_LEVEL` - Logging verbosity -### Data Directory Structure +### Directory Structure -The `DATA_HOME` environment variable controls where all service data is stored: +**Seed Data (`SEED_HOME`)** - Contains initial data files for database seeding: +``` +${SEED_HOME}/ +├── node_tags.yaml # Node tags (keyed by public_key) +└── members.yaml # Network members list +``` + +**Runtime Data (`DATA_HOME`)** - Contains runtime data (gitignored): ``` ${DATA_HOME}/ -├── collector/ -│ ├── meshcore.db # SQLite database -│ └── tags.json # Node tags for import -└── web/ - └── members.json # Network members list +└── collector/ + └── meshcore.db # SQLite database ``` Services automatically create their subdirectories if they don't exist. +### Automatic Seeding + +The collector automatically imports seed data on startup if YAML files exist in `SEED_HOME`: +- `node_tags.yaml` - Node tag definitions (keyed by public_key) +- `members.yaml` - Network member definitions + +Manual seeding can be triggered with: `meshcore-hub collector seed` + ### Webhook Configuration The collector supports forwarding events to external HTTP endpoints: diff --git a/README.md b/README.md index f686855..3049ca7 100644 --- a/README.md +++ b/README.md @@ -66,45 +66,56 @@ MeshCore Hub provides a complete solution for monitoring, collecting, and intera ### Using Docker Compose (Recommended) -Docker Compose supports **profiles** to selectively enable/disable components: +Docker Compose runs core services by default and uses **profiles** for optional components: + +**Default Services (always run):** + +| Service | Description | +|---------|-------------| +| `mqtt` | Eclipse Mosquitto MQTT broker | +| `collector` | MQTT subscriber + database storage (auto-seeds on startup) | +| `api` | REST API server | +| `web` | Web dashboard | + +**Optional Profiles:** | Profile | Services | |---------|----------| -| `mqtt` | Eclipse Mosquitto MQTT broker | | `interface-receiver` | MeshCore device receiver (events to MQTT) | | `interface-sender` | MeshCore device sender (MQTT to device) | -| `collector` | MQTT subscriber + database storage | -| `api` | REST API server | -| `web` | Web dashboard | -| `mock` | All services with mock device (for testing) | -| `all` | All production services | +| `mock` | Mock device receiver (for testing without hardware) | +| `migrate` | One-time database migration runner | +| `seed` | One-time seed data import (also runs automatically on collector startup) | ```bash # Clone the repository git clone https://github.com/your-org/meshcore-hub.git -cd meshcore-hub/docker +cd meshcore-hub # Copy and configure environment cp .env.example .env # Edit .env with your settings (API keys, serial port, network info) -# Option 1: Start all services with mock device (for testing) +# Option 1: Start core services (mqtt, collector, api, web) +docker compose up -d + +# Option 2: Start with mock device for testing docker compose --profile mock up -d -# Option 2: Start specific services for production -docker compose --profile mqtt --profile collector --profile api --profile web up -d - -# Option 3: Start all production services (requires real MeshCore device) -docker compose --profile all up -d +# Option 3: Start with real MeshCore device +docker compose --profile interface-receiver up -d # View logs docker compose logs -f -# Run database migrations +# Run database migrations (one-time) docker compose --profile migrate up +# Import seed data manually (also runs on collector startup) +docker compose --profile seed up + # Stop services -docker compose --profile mock down +docker compose down ``` #### Serial Device Access @@ -170,7 +181,8 @@ All components are configured via environment variables. Create a `.env` file or | Variable | Default | Description | |----------|---------|-------------| -| `DATABASE_URL` | `sqlite:///./meshcore.db` | SQLAlchemy database URL | +| `DATABASE_URL` | `sqlite:///{data_home}/collector/meshcore.db` | SQLAlchemy database URL | +| `SEED_HOME` | `./seed` | Directory containing seed data files (node_tags.yaml, members.yaml) | #### Webhook Configuration @@ -229,10 +241,12 @@ meshcore-hub interface --mode receiver --port /dev/ttyUSB0 meshcore-hub interface --mode sender --mock # Use mock device # Collector component -meshcore-hub collector --database-url sqlite:///./data.db - -# Import node tags from JSON file -meshcore-hub collector import-tags /path/to/tags.json +meshcore-hub collector # Run collector (auto-seeds on startup) +meshcore-hub collector seed # Import all seed data from SEED_HOME +meshcore-hub collector import-tags # Import node tags from SEED_HOME/node_tags.yaml +meshcore-hub collector import-tags /path/to/file.yaml # Import from specific file +meshcore-hub collector import-members # Import members from SEED_HOME/members.yaml +meshcore-hub collector import-members /path/to/file.yaml # Import from specific file # API component meshcore-hub api --host 0.0.0.0 --port 8000 @@ -246,74 +260,124 @@ meshcore-hub db downgrade # Rollback one migration meshcore-hub db current # Show current revision ``` +## Seed Data + +The collector supports seeding the database with node tags and network members on startup. Seed files are read from the `SEED_HOME` directory (default: `./seed`). + +### Automatic Seeding + +When the collector starts, it automatically imports seed data from YAML files if they exist: +- `{SEED_HOME}/node_tags.yaml` - Node tag definitions +- `{SEED_HOME}/members.yaml` - Network member definitions + +### Manual Seeding + +```bash +# Native CLI +meshcore-hub collector seed + +# With Docker Compose +docker compose --profile seed up +``` + +### Directory Structure + +``` +seed/ # SEED_HOME (seed data files) +├── node_tags.yaml # Node tags for import +└── members.yaml # Network members for import + +data/ # DATA_HOME (runtime data) +└── collector/ + └── meshcore.db # SQLite database +``` + +Example seed files are provided in `example/seed/`. + ## Node Tags Node tags allow you to attach custom metadata to nodes (e.g., location, role, owner). Tags are stored in the database and returned with node data via the API. -### Importing Tags from JSON +### Node Tags YAML Format -Tags can be bulk imported from a JSON file: +Tags are keyed by public key in YAML format: -```bash -# Native CLI -meshcore-hub collector import-tags /path/to/tags.json +```yaml +# Each key is a 64-character hex public key +0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef: + friendly_name: Gateway Node + role: gateway + lat: 37.7749 + lon: -122.4194 + is_online: true -# With Docker Compose -docker compose --profile import-tags run --rm import-tags +fedcba9876543210fedcba9876543210fedcba9876543210fedcba9876543210: + friendly_name: Oakland Repeater + altitude: 150 + location: + value: "37.8044,-122.2712" + type: coordinate ``` -### Tags JSON Format +Tag values can be: +- **YAML primitives** (auto-detected type): strings, numbers, booleans +- **Explicit type** (for special types like coordinate): + ```yaml + location: + value: "37.7749,-122.4194" + type: coordinate + ``` -```json -{ - "tags": [ - { - "public_key": "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef", - "key": "location", - "value": "San Francisco, CA", - "value_type": "string" - }, - { - "public_key": "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef", - "key": "altitude", - "value": "150", - "value_type": "number" - } - ] -} +Supported types: `string`, `number`, `boolean`, `coordinate` + +### Import Tags Manually + +```bash +# Import from default location ({SEED_HOME}/node_tags.yaml) +meshcore-hub collector import-tags + +# Import from specific file +meshcore-hub collector import-tags /path/to/node_tags.yaml + +# Skip tags for nodes that don't exist +meshcore-hub collector import-tags --no-create-nodes +``` + +## Network Members + +Network members represent the people operating nodes in your network. Members can optionally be linked to nodes via their public key. + +### Members YAML Format + +```yaml +members: + - name: John Doe + callsign: N0CALL + role: Network Operator + description: Example member entry + contact: john@example.com + public_key: 0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef ``` | Field | Required | Description | |-------|----------|-------------| -| `public_key` | Yes | 64-character hex public key of the node | -| `key` | Yes | Tag name (max 100 characters) | -| `value` | No | Tag value (stored as text) | -| `value_type` | No | Type hint: `string`, `number`, `boolean`, or `coordinate` (default: `string`) | +| `name` | Yes | Member's display name | +| `callsign` | No | Amateur radio callsign | +| `role` | No | Member's role in the network | +| `description` | No | Additional description | +| `contact` | No | Contact information | +| `public_key` | No | Associated node public key (64-char hex) | -### Import Options +### Import Members Manually ```bash -# Create nodes if they don't exist (default behavior) -meshcore-hub collector import-tags tags.json +# Import from default location ({SEED_HOME}/members.yaml) +meshcore-hub collector import-members -# Skip tags for nodes that don't exist -meshcore-hub collector import-tags --no-create-nodes tags.json +# Import from specific file +meshcore-hub collector import-members /path/to/members.yaml ``` -### Data Directory Structure - -For Docker deployments, organize your data files: - -``` -data/ -├── collector/ -│ └── tags.json # Node tags for import -└── web/ - └── members.json # Network members list -``` - -Example files are provided in `example/data/`. - ### Managing Tags via API Tags can also be managed via the REST API: @@ -459,11 +523,13 @@ meshcore-hub/ ├── alembic/ # Database migrations ├── etc/ # Configuration files (mosquitto.conf) ├── example/ # Example files for testing -│ └── data/ # Example data files (members.json) -├── data/ # Runtime data (gitignored) +│ └── seed/ # Example seed data files +│ ├── node_tags.yaml # Example node tags +│ └── members.yaml # Example network members +├── seed/ # Seed data directory (SEED_HOME) +├── data/ # Runtime data directory (DATA_HOME, gitignored) ├── Dockerfile # Docker build configuration -├── docker-compose.yml # Docker Compose services (gitignored) -├── docker-compose.yml.example # Docker Compose template +├── docker-compose.yml # Docker Compose services ├── PROMPT.md # Project specification ├── SCHEMAS.md # Event schema documentation ├── PLAN.md # Implementation plan