Merge pull request #76 from ipnet-mesh/chore/add-dynamic-sitemap-xml

Added dynamic XML sitemap for SEO

README.md

@@ -80,9 +80,13 @@ The quickest way to get started is running the entire stack on a single machine
 
 **Steps:**
 ```bash
-# Clone the repository
-git clone https://github.com/ipnet-mesh/meshcore-hub.git
+# Create a directory, download the Docker Compose file and
+# example environment configuration file
+
+mkdir meshcore-hub
 cd meshcore-hub
+wget https://raw.githubusercontent.com/ipnet-mesh/meshcore-hub/refs/heads/main/docker-compose.yml
+wget https://raw.githubusercontent.com/ipnet-mesh/meshcore-hub/refs/heads/main/.env.example
 
 # Copy and configure environment
 cp .env.example .env
@@ -156,9 +160,9 @@ This architecture allows:
 - Community members to contribute coverage with minimal setup
 - The central server to be hosted anywhere with internet access
 
-## Quick Start
+## Deployment
 
-### Using Docker Compose (Recommended)
+### Docker Compose Profiles
 
 Docker Compose uses **profiles** to select which services to run:
 
@@ -175,14 +179,6 @@ Docker Compose uses **profiles** to select which services to run:
 **Note:** Most deployments connect to an external MQTT broker. Add `--profile mqtt` only if you need a local broker.
 
 ```bash
-# Clone the repository
-git clone https://github.com/ipnet-mesh/meshcore-hub.git
-cd meshcore-hub
-
-# Copy and configure environment
-cp .env.example .env
-# Edit .env with your settings (API keys, serial port, network info)
-
 # Create database schema
 docker compose --profile migrate run --rm db-migrate
 
@@ -205,7 +201,7 @@ docker compose logs -f
 docker compose down
 ```
 
-#### Serial Device Access
+### Serial Device Access
 
 For production with real MeshCore devices, ensure the serial port is accessible:
 
@@ -226,8 +222,7 @@ SERIAL_PORT_SENDER=/dev/ttyUSB1  # If using separate sender device
 ```bash
 # Create virtual environment
 python -m venv .venv
-source .venv/bin/activate  # Linux/macOS
-# .venv\Scripts\activate   # Windows
+source .venv/bin/activate
 
 # Install the package
 pip install -e ".[dev]"
@@ -242,57 +237,6 @@ meshcore-hub api
 meshcore-hub web
 ```
 
-## Updating an Existing Installation
-
-To update MeshCore Hub to the latest version:
-
-```bash
-# Navigate to your installation directory
-cd meshcore-hub
-
-# Pull the latest code
-git pull
-
-# Pull latest Docker images
-docker compose --profile all pull
-
-# Recreate and restart services
-# For receiver/sender only installs:
-docker compose --profile receiver up -d --force-recreate
-
-# For core services with MQTT:
-docker compose --profile mqtt --profile core up -d --force-recreate
-
-# For core services without local MQTT:
-docker compose --profile core up -d --force-recreate
-
-# For complete stack (all services):
-docker compose --profile mqtt --profile core --profile receiver up -d --force-recreate
-
-# View logs to verify update
-docker compose logs -f
-```
-
-**Note:** Database migrations run automatically on collector startup, so no manual migration step is needed when using Docker.
-
-For manual installations:
-
-```bash
-# Pull latest code
-git pull
-
-# Activate virtual environment
-source .venv/bin/activate
-
-# Update dependencies
-pip install -e ".[dev]"
-
-# Run database migrations
-meshcore-hub db upgrade
-
-# Restart your services
-```
-
 ## Configuration
 
 All components are configured via environment variables. Create a `.env` file or export variables:
@@ -319,11 +263,7 @@ All components are configured via environment variables. Create a `.env` file or
 | `SERIAL_BAUD` | `115200` | Serial baud rate |
 | `MESHCORE_DEVICE_NAME` | *(none)* | Device/node name set on startup (broadcast in advertisements) |
 
-### Collector Settings
-
-The database is stored in `{DATA_HOME}/collector/meshcore.db` by default.
-
-#### Webhook Configuration
+### Webhooks
 
 The collector can forward certain events to external HTTP endpoints:
 
@@ -348,7 +288,7 @@ Webhook payload format:
 }
 ```
 
-#### Data Retention
+### Data Retention
 
 The collector automatically cleans up old event data and inactive nodes:
 
@@ -386,50 +326,15 @@ The collector automatically cleans up old event data and inactive nodes:
 | `NETWORK_CONTACT_DISCORD` | *(none)* | Discord server link |
 | `NETWORK_CONTACT_GITHUB` | *(none)* | GitHub repository URL |
 
-## CLI Reference
-
-```bash
-# Show help
-meshcore-hub --help
-
-# Interface component
-meshcore-hub interface --mode receiver --port /dev/ttyUSB0
-meshcore-hub interface --mode receiver --device-name "Gateway Node"  # Set device name
-meshcore-hub interface --mode sender --mock  # Use mock device
-
-# Collector component
-meshcore-hub collector                          # Run collector
-meshcore-hub collector seed                     # Import all seed data from SEED_HOME
-meshcore-hub collector import-tags              # Import node tags from SEED_HOME/node_tags.yaml
-meshcore-hub collector import-tags /path/to/file.yaml  # Import from specific file
-meshcore-hub collector import-members           # Import members from SEED_HOME/members.yaml
-meshcore-hub collector import-members /path/to/file.yaml  # Import from specific file
-
-# API component
-meshcore-hub api --host 0.0.0.0 --port 8000
-
-# Web dashboard
-meshcore-hub web --port 8080 --network-name "My Network"
-
-# Database management
-meshcore-hub db upgrade      # Run migrations
-meshcore-hub db downgrade    # Rollback one migration
-meshcore-hub db current      # Show current revision
-```
-
 ## Seed Data
 
 The database can be seeded with node tags and network members from YAML files in the `SEED_HOME` directory (default: `./seed`).
 
-### Running the Seed Process
+#### Running the Seed Process
 
 Seeding is a separate process and must be run explicitly:
 
 ```bash
-# Native CLI
-meshcore-hub collector seed
-
-# With Docker Compose
 docker compose --profile seed up
 ```
 
@@ -437,7 +342,7 @@ This imports data from the following files (if they exist):
 - `{SEED_HOME}/node_tags.yaml` - Node tag definitions
 - `{SEED_HOME}/members.yaml` - Network member definitions
 
-### Directory Structure
+#### Directory Structure
 
 ```
 seed/                          # SEED_HOME (seed data files)
@@ -451,11 +356,11 @@ data/                          # DATA_HOME (runtime data)
 
 Example seed files are provided in `example/seed/`.
 
-## Node Tags
+### Node Tags
 
 Node tags allow you to attach custom metadata to nodes (e.g., location, role, owner). Tags are stored in the database and returned with node data via the API.
 
-### Node Tags YAML Format
+#### Node Tags YAML Format
 
 Tags are keyed by public key in YAML format:
 
@@ -484,24 +389,11 @@ Tag values can be:
 
 Supported types: `string`, `number`, `boolean`
 
-### Import Tags Manually
-
-```bash
-# Import from default location ({SEED_HOME}/node_tags.yaml)
-meshcore-hub collector import-tags
-
-# Import from specific file
-meshcore-hub collector import-tags /path/to/node_tags.yaml
-
-# Skip tags for nodes that don't exist
-meshcore-hub collector import-tags --no-create-nodes
-```
-
-## Network Members
+### Network Members
 
 Network members represent the people operating nodes in your network. Members can optionally be linked to nodes via their public key.
 
-### Members YAML Format
+#### Members YAML Format
 
 ```yaml
 - member_id: walshie86
@@ -526,44 +418,6 @@ Network members represent the people operating nodes in your network. Members ca
 | `contact` | No | Contact information |
 | `public_key` | No | Associated node public key (64-char hex) |
 
-### Import Members Manually
-
-```bash
-# Import from default location ({SEED_HOME}/members.yaml)
-meshcore-hub collector import-members
-
-# Import from specific file
-meshcore-hub collector import-members /path/to/members.yaml
-```
-
-### Managing Tags via API
-
-Tags can also be managed via the REST API:
-
-```bash
-# List tags for a node
-curl http://localhost:8000/api/v1/nodes/{public_key}/tags
-
-# Create a tag (requires admin key)
-curl -X POST \
-  -H "Authorization: Bearer <API_ADMIN_KEY>" \
-  -H "Content-Type: application/json" \
-  -d '{"key": "location", "value": "Building A"}' \
-  http://localhost:8000/api/v1/nodes/{public_key}/tags
-
-# Update a tag
-curl -X PUT \
-  -H "Authorization: Bearer <API_ADMIN_KEY>" \
-  -H "Content-Type: application/json" \
-  -d '{"value": "Building B"}' \
-  http://localhost:8000/api/v1/nodes/{public_key}/tags/location
-
-# Delete a tag
-curl -X DELETE \
-  -H "Authorization: Bearer <API_ADMIN_KEY>" \
-  http://localhost:8000/api/v1/nodes/{public_key}/tags/location
-```
-
 ## API Documentation
 
 When running, the API provides interactive documentation at:

docker-compose.yml

@@ -14,7 +14,7 @@ services:
       - "${MQTT_EXTERNAL_PORT:-1883}:1883"
       - "${MQTT_WS_PORT:-9001}:9001"
     volumes:
-      - ./etc/mosquitto.conf:/mosquitto/config/mosquitto.conf:ro
+      # - ./etc/mosquitto.conf:/mosquitto/config/mosquitto.conf:ro
       - mosquitto_data:/mosquitto/data
       - mosquitto_log:/mosquitto/log
     healthcheck:
@@ -142,7 +142,7 @@ services:
       db-migrate:
         condition: service_completed_successfully
     volumes:
-      - ${DATA_HOME:-./data}:/data
+      - hub_data:/data
       - ${SEED_HOME:-./seed}:/seed
     environment:
       - LOG_LEVEL=${LOG_LEVEL:-INFO}
@@ -154,8 +154,6 @@ services:
       - MQTT_TLS=${MQTT_TLS:-false}
       - DATA_HOME=/data
       - SEED_HOME=/seed
-      # Explicitly unset to use DATA_HOME-based default path
-      - DATABASE_URL=
       # Webhook configuration
       - WEBHOOK_ADVERTISEMENT_URL=${WEBHOOK_ADVERTISEMENT_URL:-}
       - WEBHOOK_ADVERTISEMENT_SECRET=${WEBHOOK_ADVERTISEMENT_SECRET:-}
@@ -203,8 +201,7 @@ services:
     ports:
       - "${API_PORT:-8000}:8000"
     volumes:
-      # Mount data directory (uses collector/meshcore.db)
-      - ${DATA_HOME:-./data}:/data
+      - hub_data:/data
     environment:
       - LOG_LEVEL=${LOG_LEVEL:-INFO}
       - MQTT_HOST=${MQTT_HOST:-mqtt}
@@ -214,8 +211,6 @@ services:
       - MQTT_PREFIX=${MQTT_PREFIX:-meshcore}
       - MQTT_TLS=${MQTT_TLS:-false}
       - DATA_HOME=/data
-      # Explicitly unset to use DATA_HOME-based default path
-      - DATABASE_URL=
       - API_HOST=0.0.0.0
       - API_PORT=8000
       - API_READ_KEY=${API_READ_KEY:-}
@@ -286,12 +281,9 @@ services:
       - migrate
     restart: "no"
     volumes:
-      # Mount data directory (uses collector/meshcore.db)
-      - ${DATA_HOME:-./data}:/data
+      - hub_data:/data
     environment:
       - DATA_HOME=/data
-      # Explicitly unset to use DATA_HOME-based default path
-      - DATABASE_URL=
     command: ["db", "upgrade"]
 
   # ==========================================================================
@@ -309,20 +301,13 @@ services:
     profiles:
       - seed
     restart: "no"
-    depends_on:
-      db-migrate:
-        condition: service_completed_successfully
     volumes:
-      # Mount data directory for database (read-write)
-      - ${DATA_HOME:-./data}:/data
-      # Mount seed directory for seed files (read-only)
+      - hub_data:/data
       - ${SEED_HOME:-./seed}:/seed:ro
     environment:
       - DATA_HOME=/data
       - SEED_HOME=/seed
       - LOG_LEVEL=${LOG_LEVEL:-INFO}
-      # Explicitly unset to use DATA_HOME-based default path
-      - DATABASE_URL=
     # Imports both node_tags.yaml and members.yaml if they exist
     command: ["collector", "seed"]
 
@@ -330,6 +315,8 @@ services:
 # Volumes
 # ==========================================================================
 volumes:
+  hub_data:
+    name: meshcore_hub_data
   mosquitto_data:
     name: meshcore_mosquitto_data
   mosquitto_log:

src/meshcore_hub/api/routes/nodes.py

@@ -23,6 +23,7 @@ async def list_nodes(
     ),
     adv_type: Optional[str] = Query(None, description="Filter by advertisement type"),
     member_id: Optional[str] = Query(None, description="Filter by member_id tag value"),
+    role: Optional[str] = Query(None, description="Filter by role tag value"),
     limit: int = Query(50, ge=1, le=500, description="Page size"),
     offset: int = Query(0, ge=0, description="Page offset"),
 ) -> NodeList:
@@ -59,6 +60,16 @@ async def list_nodes(
             )
         )
 
+    if role:
+        # Filter nodes that have a role tag with the specified value
+        query = query.where(
+            Node.id.in_(
+                select(NodeTag.node_id).where(
+                    NodeTag.key == "role", NodeTag.value == role
+                )
+            )
+        )
+
     # Get total count
     count_query = select(func.count()).select_from(query.subquery())
     total = session.execute(count_query).scalar() or 0

src/meshcore_hub/web/app.py

@@ -7,7 +7,7 @@ from typing import AsyncGenerator
 
 import httpx
 from fastapi import FastAPI, Request
-from fastapi.responses import HTMLResponse
+from fastapi.responses import HTMLResponse, PlainTextResponse, Response
 from fastapi.staticfiles import StaticFiles
 from fastapi.templating import Jinja2Templates
 from starlette.exceptions import HTTPException as StarletteHTTPException
@@ -152,6 +152,72 @@ def create_app(
         except Exception as e:
             return {"status": "not_ready", "api": str(e)}
 
+    @app.get("/robots.txt", response_class=PlainTextResponse)
+    async def robots_txt(request: Request) -> str:
+        """Serve robots.txt to control search engine crawling."""
+        base_url = str(request.base_url).rstrip("/")
+        return f"User-agent: *\nAllow: /\n\nSitemap: {base_url}/sitemap.xml\n"
+
+    @app.get("/sitemap.xml")
+    async def sitemap_xml(request: Request) -> Response:
+        """Generate dynamic sitemap including all node pages."""
+        base_url = str(request.base_url).rstrip("/")
+
+        # Static pages
+        static_pages = [
+            ("", "daily", "1.0"),
+            ("/network", "hourly", "0.9"),
+            ("/nodes", "hourly", "0.9"),
+            ("/advertisements", "hourly", "0.8"),
+            ("/messages", "hourly", "0.8"),
+            ("/map", "daily", "0.7"),
+            ("/members", "weekly", "0.6"),
+        ]
+
+        urls = []
+        for path, changefreq, priority in static_pages:
+            urls.append(
+                f"  <url>\n"
+                f"    <loc>{base_url}{path}</loc>\n"
+                f"    <changefreq>{changefreq}</changefreq>\n"
+                f"    <priority>{priority}</priority>\n"
+                f"  </url>"
+            )
+
+        # Fetch infrastructure nodes for dynamic pages
+        try:
+            response = await request.app.state.http_client.get(
+                "/api/v1/nodes", params={"limit": 500, "role": "infra"}
+            )
+            if response.status_code == 200:
+                nodes = response.json().get("items", [])
+                for node in nodes:
+                    public_key = node.get("public_key")
+                    if public_key:
+                        # Use 8-char prefix (route handles redirect to full key)
+                        urls.append(
+                            f"  <url>\n"
+                            f"    <loc>{base_url}/nodes/{public_key[:8]}</loc>\n"
+                            f"    <changefreq>daily</changefreq>\n"
+                            f"    <priority>0.5</priority>\n"
+                            f"  </url>"
+                        )
+            else:
+                logger.warning(
+                    f"Failed to fetch nodes for sitemap: {response.status_code}"
+                )
+        except Exception as e:
+            logger.warning(f"Failed to fetch nodes for sitemap: {e}")
+
+        xml = (
+            '<?xml version="1.0" encoding="UTF-8"?>\n'
+            '<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">\n'
+            + "\n".join(urls)
+            + "\n</urlset>"
+        )
+
+        return Response(content=xml, media_type="application/xml")
+
     @app.exception_handler(StarletteHTTPException)
     async def http_exception_handler(
         request: Request, exc: StarletteHTTPException

src/meshcore_hub/web/templates/base.html

@@ -5,6 +5,27 @@
     <meta name="viewport" content="width=device-width, initial-scale=1.0">
     <title>{% block title %}{{ network_name }}{% endblock %}</title>
 
+    <!-- SEO Meta Tags -->
+    {% set default_description = network_name ~ " - MeshCore off-grid LoRa mesh network dashboard. Monitor nodes, messages, and network activity." %}
+    <meta name="description" content="{% block meta_description %}{{ default_description }}{% endblock %}">
+    <meta name="generator" content="MeshCore Hub {{ version }}">
+    <link rel="canonical" href="{{ request.url }}">
+
+    <!-- Open Graph / Facebook -->
+    <meta property="og:type" content="website">
+    <meta property="og:url" content="{{ request.url }}">
+    <meta property="og:title" content="{% block og_title %}{{ self.title() }}{% endblock %}">
+    <meta property="og:description" content="{% block og_description %}{{ self.meta_description() }}{% endblock %}">
+    <meta property="og:site_name" content="{{ network_name }}">
+
+    <!-- Twitter Card -->
+    <meta name="twitter:card" content="summary">
+    <meta name="twitter:title" content="{% block twitter_title %}{{ self.title() }}{% endblock %}">
+    <meta name="twitter:description" content="{% block twitter_description %}{{ self.meta_description() }}{% endblock %}">
+
+    <!-- Favicon -->
+    <link rel="icon" type="image/svg+xml" href="/static/img/meshcore.svg">
+
     <!-- Tailwind CSS with DaisyUI -->
     <link href="https://cdn.jsdelivr.net/npm/daisyui@4.4.19/dist/full.min.css" rel="stylesheet" type="text/css" />
     <script src="https://cdn.tailwindcss.com"></script>