From db86b3198e91f3605b70d98789933c303f3ff648 Mon Sep 17 00:00:00 2001 From: Louis King Date: Mon, 8 Dec 2025 23:06:04 +0000 Subject: [PATCH] Some minor UI improvements, updated env.example, and docs --- .env.example | 198 +++++++++++-------- AGENTS.md | 29 ++- README.md | 75 ++++--- src/meshcore_hub/web/routes/home.py | 21 +- src/meshcore_hub/web/static/img/meshcore.svg | 12 ++ src/meshcore_hub/web/templates/home.html | 111 +++++------ 6 files changed, 266 insertions(+), 180 deletions(-) create mode 100644 src/meshcore_hub/web/static/img/meshcore.svg diff --git a/.env.example b/.env.example index da98afe..4f4367a 100644 --- a/.env.example +++ b/.env.example @@ -1,17 +1,40 @@ -# MeshCore Hub - Docker Compose Environment Configuration +# MeshCore Hub - Environment Configuration # Copy this file to .env and customize values +# +# Configuration is grouped by service. Most deployments only need: +# - Common Settings (always required) +# - MQTT Settings (always required) +# - Interface Settings (for receiver/sender services) +# +# The Collector, API, and Web services typically run as a combined "core" +# profile and share the same data directory. +# +# ----------------------------------------------------------------------------- +# QUICK START: Receiver/Sender Only +# ----------------------------------------------------------------------------- +# For a minimal receiver or sender setup, you only need these settings: +# +# MQTT_HOST=your-mqtt-broker.example.com +# MQTT_PORT=1883 +# MQTT_USERNAME=your_username +# MQTT_PASSWORD=your_password +# MQTT_TLS=false +# SERIAL_PORT=/dev/ttyUSB0 +# +# Serial ports are typically /dev/ttyUSB[0-9] or /dev/ttyACM[0-9] on Linux. +# ----------------------------------------------------------------------------- -# =================== -# Docker Image -# =================== +# ============================================================================= +# COMMON SETTINGS +# ============================================================================= +# These settings apply to all services # Docker image version tag to use # Options: latest, main, v1.0.0, etc. IMAGE_VERSION=latest -# =================== -# Data & Seed Directories -# =================== +# Logging level (DEBUG, INFO, WARNING, ERROR, CRITICAL) +LOG_LEVEL=INFO # Base directory for runtime data (database, etc.) # Default: ./data (relative to docker-compose.yml location) @@ -19,7 +42,8 @@ IMAGE_VERSION=latest # # Structure: # ${DATA_HOME}/ -# └── meshcore.db # SQLite database +# └── collector/ +# └── meshcore.db # SQLite database DATA_HOME=./data # Directory containing seed data files for import @@ -32,43 +56,43 @@ DATA_HOME=./data # └── members.yaml # Network members for import SEED_HOME=./seed -# =================== -# Common Settings -# =================== +# ============================================================================= +# MQTT SETTINGS +# ============================================================================= +# MQTT broker connection settings for interface, collector, and API services -# Logging level (DEBUG, INFO, WARNING, ERROR, CRITICAL) -LOG_LEVEL=INFO - -# =================== -# MQTT Settings -# =================== - -# MQTT Broker connection (for interface/collector/api services) -# When using the local MQTT broker (--profile mqtt), use "mqtt" as host +# MQTT Broker host +# When using the local MQTT broker (--profile mqtt), use "mqtt" # When using an external broker, set the hostname/IP MQTT_HOST=mqtt + +# MQTT Broker port (default: 1883, or 8883 for TLS) MQTT_PORT=1883 + +# MQTT authentication (optional) MQTT_USERNAME= MQTT_PASSWORD= + +# MQTT topic prefix for all MeshCore messages MQTT_PREFIX=meshcore -# Enable TLS/SSL for MQTT connection (default: false) +# Enable TLS/SSL for MQTT connection # When enabled, uses TLS with system CA certificates (e.g., for Let's Encrypt) -# Set to true for secure MQTT connections (port 8883) MQTT_TLS=false # External port mappings for local MQTT broker (--profile mqtt only) MQTT_EXTERNAL_PORT=1883 MQTT_WS_PORT=9001 -# =================== -# Interface Settings -# =================== +# ============================================================================= +# INTERFACE SETTINGS (Receiver/Sender) +# ============================================================================= +# Settings for the MeshCore device interface services # Serial port for receiver device SERIAL_PORT=/dev/ttyUSB0 -# Serial port for sender device (if separate) +# Serial port for sender device (if using separate device) SERIAL_PORT_SENDER=/dev/ttyUSB1 # Baud rate for serial communication @@ -83,55 +107,21 @@ MESHCORE_DEVICE_NAME= NODE_ADDRESS= NODE_ADDRESS_SENDER= -# =================== -# API Settings -# =================== +# ============================================================================= +# COLLECTOR SETTINGS +# ============================================================================= +# The collector subscribes to MQTT events and stores them in the database -# External API port -API_PORT=8000 - -# API Keys for authentication (generate secure keys for production!) -# Example: openssl rand -hex 32 -API_READ_KEY= -API_ADMIN_KEY= - -# =================== -# Web Dashboard Settings -# =================== - -# External web port -WEB_PORT=8080 - -# Network Information (displayed on web dashboard) -NETWORK_NAME=MeshCore Network -NETWORK_CITY= -NETWORK_COUNTRY= - -# Radio configuration (comma-delimited) -# Format: ,,,,, -# Example: EU/UK Narrow,869.618MHz,62.5kHz,8,8,22dBm -NETWORK_RADIO_CONFIG= - -# Contact information -NETWORK_CONTACT_EMAIL= -NETWORK_CONTACT_DISCORD= -NETWORK_CONTACT_GITHUB= - -# Welcome text displayed on the homepage (plain text, optional) -# If not set, a default welcome message is shown -NETWORK_WELCOME_TEXT= - -# =================== +# ------------------- # Webhook Settings -# =================== +# ------------------- +# Webhooks forward mesh events to external HTTP endpoints as POST requests # Webhook for advertisement events (node discovery) -# Events are sent as POST requests with JSON payload WEBHOOK_ADVERTISEMENT_URL= WEBHOOK_ADVERTISEMENT_SECRET= # Webhook for all message events (channel and direct messages) -# Use this for a single endpoint handling all messages WEBHOOK_MESSAGE_URL= WEBHOOK_MESSAGE_SECRET= @@ -147,34 +137,82 @@ WEBHOOK_TIMEOUT=10.0 WEBHOOK_MAX_RETRIES=3 WEBHOOK_RETRY_BACKOFF=2.0 -# =================== +# ------------------- # Data Retention Settings -# =================== +# ------------------- +# Automatic cleanup of old event data (advertisements, messages, telemetry, etc.) # Enable automatic cleanup of old event data -# When enabled, the collector runs periodic cleanup to delete old events -# Default: true DATA_RETENTION_ENABLED=true -# Number of days to retain event data (advertisements, messages, telemetry, etc.) +# Number of days to retain event data # Events older than this are deleted during cleanup -# Default: 30 days DATA_RETENTION_DAYS=30 -# Hours between automatic cleanup runs (applies to both events and nodes) -# Default: 24 hours (once per day) +# Hours between automatic cleanup runs +# Applies to both event data and node cleanup DATA_RETENTION_INTERVAL_HOURS=24 -# =================== +# ------------------- # Node Cleanup Settings -# =================== +# ------------------- +# Automatic removal of inactive nodes # Enable automatic cleanup of inactive nodes -# Nodes that haven't been seen (last_seen) for the specified period are removed # Nodes with last_seen=NULL (never seen on network) are NOT removed -# Default: true NODE_CLEANUP_ENABLED=true # Remove nodes not seen for this many days (based on last_seen field) -# Default: 7 days NODE_CLEANUP_DAYS=7 + +# ============================================================================= +# API SETTINGS +# ============================================================================= +# REST API for querying data and sending commands + +# External API port +API_PORT=8000 + +# API Keys for authentication +# Generate secure keys for production: openssl rand -hex 32 +# Leave empty to disable authentication (not recommended for production) +API_READ_KEY= +API_ADMIN_KEY= + +# ============================================================================= +# WEB DASHBOARD SETTINGS +# ============================================================================= +# Web interface for visualizing network status + +# External web port +WEB_PORT=8080 + +# ------------------- +# Network Information +# ------------------- +# Displayed on the web dashboard homepage + +# Network display name +NETWORK_NAME=MeshCore Network + +# Network location +NETWORK_CITY= +NETWORK_COUNTRY= + +# Radio configuration (comma-delimited) +# Format: ,,,,, +# Example: EU/UK Narrow,869.618MHz,62.5kHz,SF8,CR8,22dBm +NETWORK_RADIO_CONFIG= + +# Welcome text displayed on the homepage (optional, plain text) +# If not set, a default welcome message is shown +NETWORK_WELCOME_TEXT= + +# ------------------- +# Contact Information +# ------------------- +# Contact links displayed in the footer + +NETWORK_CONTACT_EMAIL= +NETWORK_CONTACT_DISCORD= +NETWORK_CONTACT_GITHUB= diff --git a/AGENTS.md b/AGENTS.md index 09a5f76..3740ea8 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -274,10 +274,9 @@ meshcore-hub/ │ │ ├── app.py # FastAPI app │ │ ├── auth.py # Authentication │ │ ├── dependencies.py -│ │ ├── routes/ # API routes -│ │ │ ├── members.py # Member CRUD endpoints -│ │ │ └── ... -│ │ └── templates/ # Dashboard HTML +│ │ └── routes/ # API routes +│ │ ├── members.py # Member CRUD endpoints +│ │ └── ... │ └── web/ │ ├── cli.py │ ├── app.py # FastAPI app @@ -457,10 +456,12 @@ Key variables: - `DATA_HOME` - Base directory for runtime data (default: `./data`) - `SEED_HOME` - Directory containing seed data files (default: `./seed`) - `MQTT_HOST`, `MQTT_PORT`, `MQTT_PREFIX` - MQTT broker connection -- `DATABASE_URL` - SQLAlchemy database URL (default: `sqlite:///{DATA_HOME}/collector/meshcore.db`) +- `MQTT_TLS` - Enable TLS/SSL for MQTT (default: `false`) - `API_READ_KEY`, `API_ADMIN_KEY` - API authentication keys - `LOG_LEVEL` - Logging verbosity +The database defaults to `sqlite:///{DATA_HOME}/collector/meshcore.db` and does not typically need to be configured. + ### Directory Structure **Seed Data (`SEED_HOME`)** - Contains initial data files for database seeding: @@ -479,13 +480,21 @@ ${DATA_HOME}/ Services automatically create their subdirectories if they don't exist. -### Automatic Seeding +### Seeding -The collector automatically imports seed data on startup if YAML files exist in `SEED_HOME`: +The database can be seeded with node tags and network members from YAML files in `SEED_HOME`: - `node_tags.yaml` - Node tag definitions (keyed by public_key) - `members.yaml` - Network member definitions -Manual seeding can be triggered with: `meshcore-hub collector seed` +Seeding is a separate process from the collector and must be run explicitly: + +```bash +# Native CLI +meshcore-hub collector seed + +# With Docker Compose +docker compose --profile seed up +``` ### Webhook Configuration @@ -559,9 +568,9 @@ Webhook payload structure: ### Common Issues 1. **MQTT Connection Failed**: Check broker is running and `MQTT_HOST`/`MQTT_PORT` are correct -2. **Database Migration Errors**: Ensure `DATABASE_URL` is correct, run `alembic upgrade head` +2. **Database Migration Errors**: Ensure `DATA_HOME` is writable, run `meshcore-hub db upgrade` 3. **Import Errors**: Ensure package is installed with `pip install -e .` -4. **Type Errors**: Run `mypy src/` to check type annotations +4. **Type Errors**: Run `pre-commit run --all-files` to check type annotations and other issues ### Debugging diff --git a/README.md b/README.md index f50d5fa..3799e10 100644 --- a/README.md +++ b/README.md @@ -297,26 +297,26 @@ All components are configured via environment variables. Create a `.env` file or | Variable | Default | Description | |----------|---------|-------------| | `LOG_LEVEL` | `INFO` | Logging level (DEBUG, INFO, WARNING, ERROR) | +| `DATA_HOME` | `./data` | Base directory for runtime data | +| `SEED_HOME` | `./seed` | Directory containing seed data files | | `MQTT_HOST` | `localhost` | MQTT broker hostname | | `MQTT_PORT` | `1883` | MQTT broker port | +| `MQTT_USERNAME` | *(none)* | MQTT username (optional) | +| `MQTT_PASSWORD` | *(none)* | MQTT password (optional) | | `MQTT_PREFIX` | `meshcore` | Topic prefix for all MQTT messages | +| `MQTT_TLS` | `false` | Enable TLS/SSL for MQTT connection | ### Interface Settings | Variable | Default | Description | |----------|---------|-------------| -| `INTERFACE_MODE` | `RECEIVER` | Operating mode (RECEIVER or SENDER) | | `SERIAL_PORT` | `/dev/ttyUSB0` | Serial port for MeshCore device | | `SERIAL_BAUD` | `115200` | Serial baud rate | | `MESHCORE_DEVICE_NAME` | *(none)* | Device/node name set on startup (broadcast in advertisements) | -| `MOCK_DEVICE` | `false` | Use mock device for testing | ### Collector Settings -| Variable | Default | Description | -|----------|---------|-------------| -| `DATABASE_URL` | `sqlite:///{data_home}/collector/meshcore.db` | SQLAlchemy database URL | -| `SEED_HOME` | `./seed` | Directory containing seed data files (node_tags.yaml, members.yaml) | +The database is stored in `{DATA_HOME}/collector/meshcore.db` by default. #### Webhook Configuration @@ -343,6 +343,18 @@ Webhook payload format: } ``` +#### Data Retention + +The collector automatically cleans up old event data and inactive nodes: + +| Variable | Default | Description | +|----------|---------|-------------| +| `DATA_RETENTION_ENABLED` | `true` | Enable automatic cleanup of old events | +| `DATA_RETENTION_DAYS` | `30` | Days to retain event data | +| `DATA_RETENTION_INTERVAL_HOURS` | `24` | Hours between cleanup runs | +| `NODE_CLEANUP_ENABLED` | `true` | Enable removal of inactive nodes | +| `NODE_CLEANUP_DAYS` | `7` | Remove nodes not seen for this many days | + ### API Settings | Variable | Default | Description | @@ -362,6 +374,11 @@ Webhook payload format: | `NETWORK_NAME` | `MeshCore Network` | Display name for the network | | `NETWORK_CITY` | *(none)* | City where network is located | | `NETWORK_COUNTRY` | *(none)* | Country code (ISO 3166-1 alpha-2) | +| `NETWORK_RADIO_CONFIG` | *(none)* | Radio config (comma-delimited: profile,freq,bw,sf,cr,power) | +| `NETWORK_WELCOME_TEXT` | *(none)* | Custom welcome text for homepage | +| `NETWORK_CONTACT_EMAIL` | *(none)* | Contact email address | +| `NETWORK_CONTACT_DISCORD` | *(none)* | Discord server link | +| `NETWORK_CONTACT_GITHUB` | *(none)* | GitHub repository URL | ## CLI Reference @@ -375,7 +392,7 @@ meshcore-hub interface --mode receiver --device-name "Gateway Node" # Set devic meshcore-hub interface --mode sender --mock # Use mock device # Collector component -meshcore-hub collector # Run collector (auto-seeds on startup) +meshcore-hub collector # Run collector meshcore-hub collector seed # Import all seed data from SEED_HOME meshcore-hub collector import-tags # Import node tags from SEED_HOME/node_tags.yaml meshcore-hub collector import-tags /path/to/file.yaml # Import from specific file @@ -396,15 +413,11 @@ meshcore-hub db current # Show current revision ## Seed Data -The collector supports seeding the database with node tags and network members on startup. Seed files are read from the `SEED_HOME` directory (default: `./seed`). +The database can be seeded with node tags and network members from YAML files in the `SEED_HOME` directory (default: `./seed`). -### Automatic Seeding +### Running the Seed Process -When the collector starts, it automatically imports seed data from YAML files if they exist: -- `{SEED_HOME}/node_tags.yaml` - Node tag definitions -- `{SEED_HOME}/members.yaml` - Network member definitions - -### Manual Seeding +Seeding is a separate process and must be run explicitly: ```bash # Native CLI @@ -414,6 +427,10 @@ meshcore-hub collector seed docker compose --profile seed up ``` +This imports data from the following files (if they exist): +- `{SEED_HOME}/node_tags.yaml` - Node tag definitions +- `{SEED_HOME}/members.yaml` - Network member definitions + ### Directory Structure ``` @@ -484,17 +501,21 @@ Network members represent the people operating nodes in your network. Members ca ### Members YAML Format ```yaml -members: - - name: John Doe - callsign: N0CALL - role: Network Operator - description: Example member entry - contact: john@example.com - public_key: 0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef +- member_id: walshie86 + name: Walshie + callsign: Walshie86 + role: member + description: IPNet Member +- member_id: craig + name: Craig + callsign: M7XCN + role: member + description: IPNet Member ``` | Field | Required | Description | |-------|----------|-------------| +| `member_id` | Yes | Unique identifier for the member | | `name` | Yes | Member's display name | | `callsign` | No | Amateur radio callsign | | `role` | No | Member's role in the network | @@ -620,14 +641,8 @@ pytest -k "test_list" ### Code Quality ```bash -# Format code -black src/ tests/ - -# Lint -flake8 src/ tests/ - -# Type check -mypy src/ +# Run all code quality checks (formatting, linting, type checking) +pre-commit run --all-files ``` ### Creating Database Migrations @@ -684,7 +699,7 @@ meshcore-hub/ 1. Fork the repository 2. Create a feature branch (`git checkout -b feature/amazing-feature`) 3. Make your changes -4. Run tests and linting (`pytest && black . && flake8`) +4. Run tests and quality checks (`pytest && pre-commit run --all-files`) 5. Commit your changes (`git commit -m 'Add amazing feature'`) 6. Push to the branch (`git push origin feature/amazing-feature`) 7. Open a Pull Request diff --git a/src/meshcore_hub/web/routes/home.py b/src/meshcore_hub/web/routes/home.py index 01657f5..9169d0a 100644 --- a/src/meshcore_hub/web/routes/home.py +++ b/src/meshcore_hub/web/routes/home.py @@ -29,8 +29,9 @@ async def home(request: Request) -> HTMLResponse: "advertisements_24h": 0, } - # Fetch activity data for chart - activity = {"days": 7, "data": []} + # Fetch activity data for charts + advert_activity = {"days": 7, "data": []} + message_activity = {"days": 7, "data": []} try: response = await request.app.state.http_client.get("/api/v1/dashboard/stats") @@ -45,12 +46,22 @@ async def home(request: Request) -> HTMLResponse: "/api/v1/dashboard/activity", params={"days": 7} ) if response.status_code == 200: - activity = response.json() + advert_activity = response.json() except Exception as e: logger.warning(f"Failed to fetch activity from API: {e}") + try: + response = await request.app.state.http_client.get( + "/api/v1/dashboard/message-activity", params={"days": 7} + ) + if response.status_code == 200: + message_activity = response.json() + except Exception as e: + logger.warning(f"Failed to fetch message activity from API: {e}") + context["stats"] = stats - # Pass activity data as JSON string for the chart - context["activity_json"] = json.dumps(activity) + # Pass activity data as JSON strings for the chart + context["advert_activity_json"] = json.dumps(advert_activity) + context["message_activity_json"] = json.dumps(message_activity) return templates.TemplateResponse("home.html", context) diff --git a/src/meshcore_hub/web/static/img/meshcore.svg b/src/meshcore_hub/web/static/img/meshcore.svg new file mode 100644 index 0000000..eb1b7bc --- /dev/null +++ b/src/meshcore_hub/web/static/img/meshcore.svg @@ -0,0 +1,12 @@ + + + + + + + + + + + + diff --git a/src/meshcore_hub/web/templates/home.html b/src/meshcore_hub/web/templates/home.html index 36ee5cb..cb1a618 100644 --- a/src/meshcore_hub/web/templates/home.html +++ b/src/meshcore_hub/web/templates/home.html @@ -111,18 +111,18 @@ {{ network_radio_config.frequency }} {% endif %} - {% if network_radio_config.spreading_factor %} -
- Spreading Factor: - {{ network_radio_config.spreading_factor }} -
- {% endif %} {% if network_radio_config.bandwidth %}
Bandwidth: {{ network_radio_config.bandwidth }}
{% endif %} + {% if network_radio_config.spreading_factor %} +
+ Spreading Factor: + {{ network_radio_config.spreading_factor }} +
+ {% endif %} {% if network_radio_config.coding_rate %}
Coding Rate: @@ -146,6 +146,31 @@
+ +
+
+

Our local off-grid mesh network is made possible by

+ + MeshCore + +

Connecting people and things, without using the internet

+
+ + + + + Website + + + + + + GitHub + +
+
+
+
@@ -155,53 +180,12 @@ Network Activity -

Advertisements received per day (last 7 days)

+

Activity per day (last 7 days)

- - -
-
-

- - - - Contact -

-
- {% if network_contact_email %} - - - - - {{ network_contact_email }} - - {% endif %} - {% if network_contact_discord %} - - - - - Discord - - {% endif %} - {% if network_contact_github %} - - - - - GitHub - - {% endif %} - {% if not network_contact_email and not network_contact_discord and not network_contact_github %} -

No contact information configured.

- {% endif %} -
-
-
{% endblock %} @@ -209,16 +193,18 @@