diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..55d418a --- /dev/null +++ b/.dockerignore @@ -0,0 +1,88 @@ +# Git +.git +.gitignore + +# Python +__pycache__/ +*.py[cod] +*$py.class +*.so +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# Virtual environments +venv/ +env/ +ENV/ + +# IDE +.vscode/ +.idea/ +*.swp +*.swo +*~ + +# Testing +.pytest_cache/ +.coverage +htmlcov/ +.tox/ +.cache +nosetests.xml +coverage.xml +*.cover +.hypothesis/ + +# Documentation +docs/_build/ +.readthedocs.yml + +# OS +.DS_Store +.DS_Store? +._* +.Spotlight-V100 +.Trashes +ehthumbs.db +Thumbs.db + +# Logs +*.log +logs/ + +# Temporary files +*.tmp +*.temp + +# Development files that aren't needed in container +.pre-commit-config.yaml +.flake8 +.mypy_cache/ +tasks.md +spec.md +CLAUDE.md + +# GitHub Actions +.github/ + +# Example files (not needed in production) +config.example.* +=3.6.0 + +# Claude commands +.claude/ diff --git a/.env.docker.example b/.env.docker.example new file mode 100644 index 0000000..c855bad --- /dev/null +++ b/.env.docker.example @@ -0,0 +1,32 @@ +# Docker environment file example for MeshCore MQTT Bridge +# Copy this file to .env.docker and modify the values as needed + +# Logging Configuration +LOG_LEVEL=INFO + +# MQTT Broker Configuration +MQTT_BROKER=localhost +MQTT_PORT=1883 +MQTT_USERNAME= +MQTT_PASSWORD= +MQTT_TOPIC_PREFIX=meshcore +MQTT_QOS=1 +MQTT_RETAIN=true + +# MeshCore Device Configuration (serial default) +MESHCORE_CONNECTION=serial +MESHCORE_ADDRESS=/dev/ttyUSB0 +MESHCORE_BAUDRATE=115200 +MESHCORE_TIMEOUT=30 + +# Configurable Events (comma-separated list) +MESHCORE_EVENTS=CONTACT_MSG_RECV,CHANNEL_MSG_RECV,CONNECTED,DISCONNECTED,LOGIN_SUCCESS,LOGIN_FAILED,MESSAGES_WAITING,DEVICE_INFO,BATTERY,NEW_CONTACT,ADVERTISEMENT + +# TCP Connection Example (for network connections) +# MESHCORE_CONNECTION=tcp +# MESHCORE_ADDRESS=192.168.1.100 +# MESHCORE_PORT=4403 + +# BLE Connection Example (use MAC address as address) +# MESHCORE_CONNECTION=ble +# MESHCORE_ADDRESS=AA:BB:CC:DD:EE:FF diff --git a/.flake8 b/.flake8 new file mode 100644 index 0000000..22e8544 --- /dev/null +++ b/.flake8 @@ -0,0 +1,13 @@ +[flake8] +max-line-length = 88 +extend-ignore = E203, W503 +exclude = + .git, + __pycache__, + .venv, + venv, + build, + dist, + *.egg-info +per-file-ignores = + __init__.py:F401 diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..9225b7a --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,149 @@ +name: CI + +on: + push: + branches: [ main, develop ] + pull_request: + branches: [ main, develop ] + +jobs: + code-quality: + name: Code Quality + runs-on: ubuntu-latest + strategy: + matrix: + python-version: ["3.11", "3.12"] + + steps: + - uses: actions/checkout@v4 + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + + - name: Cache pip dependencies + uses: actions/cache@v4 + with: + path: ~/.cache/pip + key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements*.txt', 'pyproject.toml') }} + restore-keys: | + ${{ runner.os }}-pip- + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install -e ".[dev]" + + - name: Code formatting with Black + run: | + black --check --diff meshcore_mqtt/ tests/ + + - name: Linting with Flake8 + run: | + flake8 meshcore_mqtt/ tests/ + + - name: Type checking with MyPy + run: | + mypy meshcore_mqtt/ tests/ + + - name: Import sorting with isort + run: | + isort --check-only --diff meshcore_mqtt/ tests/ + + test: + name: Tests + runs-on: ubuntu-latest + strategy: + matrix: + python-version: ["3.11", "3.12"] + + steps: + - uses: actions/checkout@v4 + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + + - name: Cache pip dependencies + uses: actions/cache@v4 + with: + path: ~/.cache/pip + key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements*.txt', 'pyproject.toml') }} + restore-keys: | + ${{ runner.os }}-pip- + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install -e ".[dev]" + + - name: Run tests with pytest + run: | + pytest -v --cov=meshcore_mqtt --cov-report=xml --cov-report=term-missing + + - name: Upload coverage to Codecov + uses: codecov/codecov-action@v3 + with: + file: ./coverage.xml + flags: unittests + name: codecov-umbrella + fail_ci_if_error: false + + security: + name: Security Scan + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.12" + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install -e ".[dev]" + + - name: Run Bandit security scan + run: | + bandit -r meshcore_mqtt/ -f json -o bandit-report.json || true + bandit -r meshcore_mqtt/ + + # - name: Run Safety check + # run: | + # safety check --json --output safety-report.json || true + # safety check + + build-test: + name: Build Test + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.12" + + - name: Install build dependencies + run: | + python -m pip install --upgrade pip + pip install build twine + + - name: Build package + run: | + python -m build + + - name: Check package + run: | + twine check dist/* + + - name: Test installation + run: | + pip install dist/*.whl + python -c "import meshcore_mqtt; print('Package installed successfully')" diff --git a/.github/workflows/code-quality.yml b/.github/workflows/code-quality.yml new file mode 100644 index 0000000..a45d8c8 --- /dev/null +++ b/.github/workflows/code-quality.yml @@ -0,0 +1,72 @@ +name: Code Quality + +on: + push: + branches: [ main, develop ] + pull_request: + branches: [ main, develop ] + +jobs: + lint: + name: Lint and Format Check + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.12" + + - name: Cache pip dependencies + uses: actions/cache@v4 + with: + path: ~/.cache/pip + key: ${{ runner.os }}-pip-lint-${{ hashFiles('**/requirements-dev.txt') }} + restore-keys: | + ${{ runner.os }}-pip-lint- + + - name: Install linting dependencies + run: | + python -m pip install --upgrade pip + pip install black flake8 mypy isort bandit + + - name: Check code formatting with Black + run: | + echo "::group::Black formatting check" + black --check --diff --color meshcore_mqtt/ tests/ + echo "::endgroup::" + + - name: Check import sorting with isort + run: | + echo "::group::Import sorting check" + isort --check-only --diff meshcore_mqtt/ tests/ + echo "::endgroup::" + + - name: Lint with Flake8 + run: | + echo "::group::Flake8 linting" + flake8 meshcore_mqtt/ tests/ --statistics + echo "::endgroup::" + + - name: Type check with MyPy + run: | + echo "::group::MyPy type checking" + # Install package dependencies for type checking + pip install -r requirements-dev.txt + mypy meshcore_mqtt/ tests/ + echo "::endgroup::" + + - name: Security check with Bandit + run: | + echo "::group::Bandit security scan" + bandit -r meshcore_mqtt/ -ll + echo "::endgroup::" + + # - name: Dependency security check with Safety + # run: | + # echo "::group::Safety dependency check" + # pip install -r requirements.txt + # safety check + # echo "::endgroup::" diff --git a/.github/workflows/docker-build.yml b/.github/workflows/docker-build.yml new file mode 100644 index 0000000..e8f7ec8 --- /dev/null +++ b/.github/workflows/docker-build.yml @@ -0,0 +1,94 @@ +name: Docker Build and Push + +on: + push: + branches: [ main, develop ] + tags: [ 'v*' ] + pull_request: + branches: [ main, develop ] + release: + types: [ published ] + +env: + REGISTRY: ghcr.io + IMAGE_NAME: ${{ github.repository }} + +jobs: + build-and-push: + name: Build and Push Docker Image + runs-on: ubuntu-latest + permissions: + contents: read + packages: write + attestations: write + id-token: write + security-events: write + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Log in to Container Registry + if: github.event_name != 'pull_request' + uses: docker/login-action@v3 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Extract metadata + id: meta + uses: docker/metadata-action@v5 + with: + images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} + tags: | + type=ref,event=branch + type=ref,event=pr + type=semver,pattern={{version}} + type=semver,pattern={{major}}.{{minor}} + type=semver,pattern={{major}} + type=raw,value=latest,enable=${{ startsWith(github.ref, 'refs/tags/') }} + labels: | + org.opencontainers.image.title=MeshCore MQTT Bridge + org.opencontainers.image.description=A robust bridge service that connects MeshCore devices to MQTT brokers + org.opencontainers.image.vendor=MeshCore MQTT Bridge Team + org.opencontainers.image.licenses=GPL-3.0 + + - name: Build and push Docker image + id: build + uses: docker/build-push-action@v6 + with: + context: . + platforms: linux/amd64,linux/arm64 + push: ${{ github.event_name != 'pull_request' }} + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + cache-from: type=gha + cache-to: type=gha,mode=max + provenance: true + sbom: true + + - name: Generate artifact attestation + if: github.event_name != 'pull_request' + uses: actions/attest-build-provenance@v1 + with: + subject-name: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} + subject-digest: ${{ steps.build.outputs.digest }} + push-to-registry: true + + - name: Run Trivy vulnerability scanner + if: github.event_name != 'pull_request' + uses: aquasecurity/trivy-action@master + with: + image-ref: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ steps.meta.outputs.version }} + format: 'sarif' + output: 'trivy-results.sarif' + + - name: Upload Trivy scan results to GitHub Security tab + if: github.event_name != 'pull_request' + uses: github/codeql-action/upload-sarif@v3 + with: + sarif_file: 'trivy-results.sarif' diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml new file mode 100644 index 0000000..ead4ddf --- /dev/null +++ b/.github/workflows/test.yml @@ -0,0 +1,133 @@ +name: Tests + +on: + push: + branches: [ main, develop ] + pull_request: + branches: [ main, develop ] + +jobs: + test: + name: Test Suite + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest] + python-version: ["3.11", "3.12"] + + steps: + - uses: actions/checkout@v4 + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + + - name: Cache pip dependencies + uses: actions/cache@v4 + with: + path: | + ~/.cache/pip + ~/.local/share/virtualenvs + key: ${{ runner.os }}-pip-${{ matrix.python-version }}-${{ hashFiles('**/requirements*.txt') }} + restore-keys: | + ${{ runner.os }}-pip-${{ matrix.python-version }}- + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install -r requirements.txt + pip install -r requirements-dev.txt + + - name: Run tests with pytest + run: | + pytest -v --tb=short --cov=meshcore_mqtt --cov-report=xml --cov-report=term-missing --cov-report=html + + - name: Generate coverage report + if: matrix.os == 'ubuntu-latest' && matrix.python-version == '3.12' + run: | + echo "## Test Coverage Report" >> $GITHUB_STEP_SUMMARY + echo "```" >> $GITHUB_STEP_SUMMARY + coverage report --show-missing >> $GITHUB_STEP_SUMMARY + echo "```" >> $GITHUB_STEP_SUMMARY + + - name: Upload coverage to Codecov + if: matrix.os == 'ubuntu-latest' && matrix.python-version == '3.12' + uses: codecov/codecov-action@v3 + with: + file: ./coverage.xml + flags: unittests + name: codecov-umbrella + fail_ci_if_error: false + + - name: Archive coverage reports + if: matrix.os == 'ubuntu-latest' && matrix.python-version == '3.12' + uses: actions/upload-artifact@v4 + with: + name: coverage-report + path: htmlcov/ + + - name: Test CLI functionality + run: | + # Test CLI help + python -m meshcore_mqtt.main --help + + # Test version info (if available) + python -c "import meshcore_mqtt; print('Package version check passed')" + + test-examples: + name: Test Configuration Examples + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.12" + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install -r requirements.txt + pip install pyyaml + + - name: Validate JSON configuration examples + run: | + python -c " + import json + with open('config.example.json', 'r') as f: + config = json.load(f) + print('✓ config.example.json is valid JSON') + print(f'✓ Contains {len(config)} top-level keys') + " + + - name: Validate YAML configuration examples + run: | + python -c " + import yaml + with open('config.example.yaml', 'r') as f: + config = yaml.safe_load(f) + print('✓ config.example.yaml is valid YAML') + print(f'✓ Contains {len(config)} top-level keys') + " + + - name: Test configuration loading + run: | + python -c " + from meshcore_mqtt.config import Config + + # Test JSON config loading + config_json = Config.from_file('config.example.json') + print('✓ JSON configuration loads successfully') + print(f'✓ MQTT broker: {config_json.mqtt.broker}') + print(f'✓ Events configured: {len(config_json.meshcore.events)}') + + # Test YAML config loading + config_yaml = Config.from_file('config.example.yaml') + print('✓ YAML configuration loads successfully') + print(f'✓ MQTT broker: {config_yaml.mqtt.broker}') + print(f'✓ Events configured: {len(config_yaml.meshcore.events)}') + " diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..36fbdd4 --- /dev/null +++ b/.gitignore @@ -0,0 +1,196 @@ +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ +cover/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +.pybuilder/ +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +# For a library or package, you might want to ignore these files since the code is +# intended to run in multiple environments; otherwise, check them in: +# .python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# poetry +# Similar to Pipenv.lock, it is generally recommended to include poetry.lock in version control. +# This is especially recommended for binary packages to ensure reproducibility, and is more +# commonly ignored for libraries. +# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control +#poetry.lock + +# pdm +# Similar to Pipenv.lock, it is generally recommended to include pdm.lock in version control. +#pdm.lock +# pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it +# in version control. +# https://pdm.fming.dev/#use-with-ide +.pdm.toml + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +# pytype static type analyzer +.pytype/ + +# Cython debug symbols +cython_debug/ + +# PyCharm +# JetBrains specific template is maintained in a separate JetBrains.gitignore that can +# be added to the global gitignore or merged into this project gitignore. For a PyCharm +# project, it is recommended to use the following gitignore template: +# https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore +.idea/ + +# VS Code +.vscode/ + +# macOS +.DS_Store +.AppleDouble +.LSOverride + +# Windows +Thumbs.db +ehthumbs.db +Desktop.ini +$RECYCLE.BIN/ +*.cab +*.msi +*.msix +*.msm +*.msp +*.lnk + +# Linux +*~ + +# Project specific +*.log +config.json +config.yaml +config.yml +.meshcore/ +logs/ +temp/ + +# Personal config files (keep examples but ignore actual configs) +!config.example.json +!config.example.yaml diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000..510b972 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,43 @@ + +repos: + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v5.0.0 + hooks: + - id: trailing-whitespace + - id: end-of-file-fixer + - id: check-yaml + - id: check-added-large-files + - id: check-json + - id: check-toml + + - repo: https://github.com/pycqa/isort + rev: 6.0.1 + hooks: + - id: isort + args: ["--profile", "black"] + + - repo: https://github.com/psf/black + rev: 25.1.0 + hooks: + - id: black + language_version: python3 + + - repo: https://github.com/pycqa/flake8 + rev: 7.3.0 + hooks: + - id: flake8 + + - repo: https://github.com/pre-commit/mirrors-mypy + rev: v1.17.1 + hooks: + - id: mypy + additional_dependencies: [pydantic, click, paho-mqtt] + + - repo: local + hooks: + - id: pytest + name: pytest + entry: pytest + language: system + pass_filenames: false + always_run: true diff --git a/CLAUDE.md b/CLAUDE.md new file mode 100644 index 0000000..5fe00c0 --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1,399 @@ +# MeshCore MQTT Bridge - Claude Context + +## Instructions for Claude + +- Run `pre-commit run --all-files` to ensure code quality and tests pass after making changes. +- Activate and use Python virtual environment located at `./venv` before any development or running commands. + +## Project Overview + +This project is a **MeshCore MQTT Bridge** - a robust Python application that bridges MeshCore mesh networking devices to MQTT brokers, enabling seamless integration with IoT platforms and message processing systems. + +### Key Features +- **Inbox/Outbox Architecture**: Independent MeshCore and MQTT workers with message bus coordination +- **Multi-connection support**: TCP, Serial, and BLE connections to MeshCore devices +- **TLS/SSL Support**: Secure MQTT connections with configurable certificates +- **Flexible configuration**: JSON/YAML files, environment variables, CLI arguments +- **Configurable event system**: Subscribe to specific MeshCore event types +- **Robust MQTT integration**: Authentication, QoS, retention, auto-reconnection +- **Health monitoring**: Built-in health checks and automatic recovery for both workers +- **Async architecture**: Built with Python asyncio for high performance +- **Type safety**: Full type annotations with mypy support +- **Comprehensive testing**: 59+ tests with pytest and pytest-asyncio + +## Architecture + +The bridge uses an **Inbox/Outbox Architecture** with independent workers coordinated by a message bus. + +### Core Components + +1. **Bridge Coordinator** (`meshcore_mqtt/bridge_coordinator.py`) + - Coordinates independent MeshCore and MQTT workers + - Manages shared message bus for inter-worker communication + - Provides health monitoring and system statistics + - Handles graceful startup and shutdown + +2. **Message Bus System** (`meshcore_mqtt/message_queue.py`) + - Async message queue system for worker communication + - Thread-safe inbox/outbox pattern with asyncio.Queue + - Component status tracking and health monitoring + - Message types: MESHCORE_EVENT, MQTT_COMMAND, STATUS, HEALTH_CHECK, SHUTDOWN + +3. **MeshCore Worker** (`meshcore_mqtt/meshcore_worker.py`) + - Independent worker managing MeshCore device connection + - Handles device commands forwarded from MQTT worker + - Auto-reconnection with exponential backoff and health monitoring + - Forwards MeshCore events to MQTT worker via message bus + - Manages auto-fetch restart after NO_MORE_MSGS events + +4. **MQTT Worker** (`meshcore_mqtt/mqtt_worker.py`) + - Independent worker managing MQTT broker connection + - Subscribes to command topics and publishes events/status + - TLS/SSL support with configurable certificates + - Auto-reconnection with complete client recreation + - Forwards MQTT commands to MeshCore worker via message bus + +5. **Configuration System** (`meshcore_mqtt/config.py`) + - Pydantic-based configuration with validation + - Support for multiple input methods (file, env, CLI) + - Event type validation and normalization + - TLS/SSL configuration validation + +6. **CLI Interface** (`meshcore_mqtt/main.py`) + - Click-based command line interface + - Logging configuration for third-party libraries + - Configuration loading with precedence handling + +### Event System + +The bridge supports configurable event subscriptions: + +**Default Events**: +- `CONTACT_MSG_RECV`, `CHANNEL_MSG_RECV` (messages) +- `CONNECTED`, `DISCONNECTED` (connection status) +- `LOGIN_SUCCESS`, `LOGIN_FAILED` (authentication) +- `DEVICE_INFO`, `BATTERY`, `NEW_CONTACT` (device info) +- `MESSAGES_WAITING` (notifications) + +**Additional Events**: +- `ADVERTISEMENT`, `TELEMETRY`, `POSITION` +- `ROUTING`, `ADMIN`, `USER` +- `TEXT_MESSAGE_RX`, `TEXT_MESSAGE_TX` +- `WAYPOINT`, `NEIGHBOR_INFO`, `TRACEROUTE` +- `NODE_LIST_CHANGED`, `CONFIG_CHANGED` + +### Auto-Fetch Restart Feature + +The bridge automatically handles `NO_MORE_MSGS` events from MeshCore by restarting the auto-fetch mechanism after a configurable delay: + +- **Purpose**: Prevents message fetching from stopping when MeshCore reports no more messages +- **Configuration**: `auto_fetch_restart_delay` (1-60 seconds, default: 5) +- **Behavior**: When `NO_MORE_MSGS` is received, waits the configured delay then restarts auto-fetching +- **Environment Variable**: `MESHCORE_AUTO_FETCH_RESTART_DELAY=10` +- **CLI Argument**: `--meshcore-auto-fetch-restart-delay 10` + +### MQTT Topics + +The bridge publishes to structured MQTT topics: +- `{prefix}/message/channel/{channel_idx}` - Channel messages (public channels) +- `{prefix}/message/direct/{pubkey_prefix}` - Direct messages (private messages) +- `{prefix}/status` - Connection status +- `{prefix}/advertisement` - Device advertisements +- `{prefix}/battery` - Battery updates +- `{prefix}/device_info` - Device information +- `{prefix}/new_contact` - Contact discovery +- `{prefix}/login` - Authentication status +- `{prefix}/command/{type}` - Commands (subscribed) + +**Message Topic Structure:** +- For channel messages: `{prefix}/message/channel/{channel_idx}` where `channel_idx` is the numeric channel identifier +- For direct messages: `{prefix}/message/direct/{pubkey_prefix}` where `pubkey_prefix` is the sender's 6-byte public key prefix (hex encoded) +- This allows subscribers to: + - Subscribe to all messages: `{prefix}/message/+/+` + - Subscribe to all channel messages: `{prefix}/message/channel/+` + - Subscribe to specific channel: `{prefix}/message/channel/0` + - Subscribe to all direct messages: `{prefix}/message/direct/+` + - Subscribe to messages from specific node: `{prefix}/message/direct/{specific_pubkey_prefix}` + +### MQTT Command System + +The bridge supports bidirectional communication via MQTT commands. Send commands to `{prefix}/command/{command_type}` with JSON payloads: + +**Available Commands** (implemented in `meshcore_worker.py:_handle_mqtt_command`): + +| Command | Description | Required Fields | MeshCore Method | +|---------|-------------|-----------------|------------------| +| `send_msg` | Send direct message | `destination`, `message` | `meshcore.commands.send_msg()` | +| `send_chan_msg` | Send channel message | `channel`, `message` | `meshcore.commands.send_chan_msg()` | +| `device_query` | Query device information | None | `meshcore.commands.send_device_query()` | +| `get_battery` | Get battery status | None | `meshcore.commands.get_bat()` | +| `set_name` | Set device name | `name` | `meshcore.commands.set_name()` | +| `ping` | Ping a node | `destination` | `meshcore.commands.ping()` | +| `send_advert` | Send device advertisement | None (optional: `flood`) | `meshcore.commands.send_advert()` | + +**Command Examples**: +```json +// Send direct message +{"destination": "node_id_or_contact_name", "message": "Hello!"} + +// Send channel message +{"channel": 0, "message": "Hello channel!"} + +// Device query +{} + +// Get battery +{} + +// Set device name +{"name": "MyDevice"} + +// Ping node +{"destination": "node_id"} + +// Send advertisement +{} + +// Send advertisement with flood +{"flood": true} +``` + +**Command Examples**: +```bash +# Send direct message +mosquitto_pub -h localhost -t "meshcore/command/send_msg" \ + -m '{"destination": "Alice", "message": "Hello Alice!"}' + +# Send channel message +mosquitto_pub -h localhost -t "meshcore/command/send_chan_msg" \ + -m '{"channel": 0, "message": "Hello everyone on channel 0!"}' + +# Ping a node +mosquitto_pub -h localhost -t "meshcore/command/ping" \ + -m '{"destination": "node123"}' + +# Get device info +mosquitto_pub -h localhost -t "meshcore/command/device_query" -m '{}' + +# Get battery status +mosquitto_pub -h localhost -t "meshcore/command/get_battery" -m '{}' + +# Set device name +mosquitto_pub -h localhost -t "meshcore/command/set_name" \ + -m '{"name": "MyMeshDevice"}' + +# Send device advertisement +mosquitto_pub -h localhost -t "meshcore/command/send_advert" -m '{}' + +# Send device advertisement with flood +mosquitto_pub -h localhost -t "meshcore/command/send_advert" \ + -m '{"flood": true}' +``` + +## Development Guidelines + +### Code Quality Tools + +The project uses these tools (configured in `pyproject.toml`): +- **Black**: Code formatting (line length: 88) +- **Flake8**: Linting with custom rules +- **MyPy**: Type checking with strict settings +- **Pytest**: Testing with asyncio support +- **Pre-commit**: Automated code quality checks + +### Testing Strategy + +**Test Structure**: +- `tests/test_config.py` - Configuration system (18 tests) +- `tests/test_bridge.py` - Bridge coordinator functionality (10 tests) +- `tests/test_configurable_events.py` - Event configuration (13 tests) +- `tests/test_json_serialization.py` - JSON handling (10 tests) +- `tests/test_logging.py` - Logging configuration (6 tests) + +**Key Test Areas**: +- Configuration validation and loading +- Worker coordination and message bus functionality +- Event handler mapping and subscription +- JSON serialization edge cases +- MQTT topic generation and command handling +- Health monitoring and recovery mechanisms +- Logging setup and third-party library control + +### Architecture Benefits + +**Inbox/Outbox Pattern**: +- **Resilience**: Workers operate independently, one can fail without affecting the other +- **Scalability**: Easy to add new workers or modify existing ones +- **Testability**: Each worker can be tested in isolation +- **Monitoring**: Built-in health checks and statistics for each component +- **Recovery**: Automatic reconnection and error recovery for both workers + +**Message Bus Design**: +- **Thread-safe**: Uses asyncio.Queue for safe async operations +- **Typed messages**: Structured message types with validation +- **Component tracking**: Status monitoring for all registered components +- **Graceful shutdown**: Coordinated shutdown with cleanup + +### Running Commands + +**Development Setup**: +```bash +python -m venv venv +source venv/bin/activate +pip install -r requirements-dev.txt +pre-commit install +``` + +**Testing**: +```bash +pytest # Run all tests +pytest -v # Verbose output +pytest --cov=meshcore_mqtt # With coverage +pytest tests/test_config.py # Specific test file +``` + +**Code Quality**: +```bash +black meshcore_mqtt/ tests/ # Format code +flake8 meshcore_mqtt/ tests/ # Lint code +mypy meshcore_mqtt/ tests/ # Type check +pre-commit run --all-files # Run all checks +``` + +**Running the Application**: +```bash +# With config file +python -m meshcore_mqtt.main --config-file config.yaml + +# With CLI arguments +python -m meshcore_mqtt.main \ + --mqtt-broker localhost \ + --meshcore-connection tcp \ + --meshcore-address 192.168.1.100 \ + --meshcore-events "CONNECTED,BATTERY,ADVERTISEMENT" + +# With environment variables +python -m meshcore_mqtt.main --env +``` + +## Recent Development History + +### Major Features Implemented + +1. **Inbox/Outbox Architecture (Latest)** + - Complete restructure to independent worker pattern + - Message bus system for inter-worker communication + - Enhanced health monitoring and recovery + - Improved resilience and testability + +2. **TLS/SSL Support** + - Secure MQTT connections with configurable certificates + - Support for custom CA, client certificates, and private keys + - Optional certificate verification bypass for testing + +3. **Enhanced Command System** + - Full bidirectional MQTT ↔ MeshCore command support + - Structured command handling with proper error reporting + - Support for all major MeshCore operations + +4. **JSON Serialization (6b492db)** + - Robust JSON serialization handling all Python data types + - Fallback mechanisms for complex objects + - Comprehensive error handling and validation + +5. **Configurable Events (f7c10cc)** + - Made MeshCore event types configurable + - Support for config files, env vars, and CLI args + - Case-insensitive event parsing with validation + - Enhanced logging configuration for third-party libraries + +### Code Patterns + +**Worker Message Handling**: +```python +async def _handle_inbox_message(self, message: Message) -> None: + """Handle messages from the inbox.""" + if message.message_type == MessageType.MQTT_COMMAND: + await self._handle_mqtt_command(message) + elif message.message_type == MessageType.MESHCORE_EVENT: + await self._handle_meshcore_event(message) +``` + +**Event Forwarding Pattern**: +```python +def _on_meshcore_event(self, event_data: Any) -> None: + """Forward MeshCore events to MQTT worker.""" + message = Message.create( + message_type=MessageType.MESHCORE_EVENT, + source=self.component_name, + target="mqtt", + payload={"event_data": event_data, "timestamp": time.time()} + ) + asyncio.create_task(self.message_bus.send_message(message)) +``` + +**Configuration Validation**: +```python +@field_validator("field_name") +@classmethod +def validate_field(cls, v: Type) -> Type: + """Validate field with custom logic.""" + # Validation logic here + return v +``` + +## Important Notes for Claude + +### When Making Changes + +1. **Always run tests** after changes: `pytest -v` +2. **Follow worker patterns** for new functionality +3. **Update message bus** if adding new message types +4. **Use type hints** for all new code +5. **Handle errors gracefully** with proper logging +6. **Test worker isolation** and message passing + +### Configuration Precedence +1. Command-line arguments (highest) +2. Configuration file +3. Environment variables (lowest) + +### Worker Development Guidelines +- **Message Handling**: Use inbox/outbox pattern for all inter-worker communication +- **Health Monitoring**: Implement health checks in `_perform_health_check()` +- **Recovery Logic**: Add automatic reconnection with exponential backoff +- **Status Updates**: Send status updates via message bus +- **Graceful Shutdown**: Handle shutdown messages and clean up resources + +### Command Implementation +- Add new commands to `meshcore_worker.py:_handle_mqtt_command()` +- Validate required fields before calling MeshCore methods +- Handle command results and errors appropriately +- Update activity timestamp on successful operations + +### Testing Requirements +- Test worker components in isolation +- Test message bus communication +- Test configuration validation +- Test error conditions and recovery scenarios +- Test health monitoring and status updates +- Maintain high test coverage + +### Logging Best Practices +- Use structured logging with proper levels +- Configure third-party library logging appropriately +- Provide meaningful log messages for debugging +- Use `self.logger` instance for consistency +- Log worker status changes and health events + +This project demonstrates modern Python development practices with async programming, inbox/outbox architecture, comprehensive testing, and robust error handling. The codebase is well-structured and maintainable, with clear separation of concerns, independent workers, and extensive documentation. + +### Key Architectural Decisions + +1. **Inbox/Outbox Pattern**: Ensures worker independence and resilience +2. **Message Bus**: Provides typed, thread-safe communication between components +3. **Health Monitoring**: Built-in health checks and automatic recovery +4. **TLS/SSL Support**: Secure MQTT connections for production deployments +5. **Async-First Design**: All I/O operations are asynchronous for maximum performance +6. **Type Safety**: Comprehensive type annotations with mypy validation +7. **Configuration Flexibility**: Multiple input methods with proper validation diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..2d5b7bc --- /dev/null +++ b/Dockerfile @@ -0,0 +1,98 @@ +# Multi-stage Dockerfile for MeshCore MQTT Bridge +# Stage 1: Build stage with development dependencies +FROM python:3.12-alpine AS builder + +LABEL maintainer="MeshCore MQTT Bridge Team" +LABEL description="Build stage for MeshCore MQTT Bridge" + +# Install build dependencies +RUN apk add --no-cache \ + gcc \ + musl-dev \ + linux-headers \ + libffi-dev \ + openssl-dev \ + cargo \ + rust + +# Set working directory +WORKDIR /app + +# Copy requirements files +COPY requirements.txt requirements-dev.txt ./ + +# Install Python dependencies in a virtual environment +RUN python -m venv /opt/venv +ENV PATH="/opt/venv/bin:$PATH" + +# Install production dependencies +RUN pip install --no-cache-dir --upgrade pip wheel setuptools && \ + pip install --no-cache-dir -r requirements.txt + +# Copy source code +COPY meshcore_mqtt/ ./meshcore_mqtt/ +COPY pyproject.toml ./ + +# Install the package +RUN pip install --no-cache-dir . + +# Stage 2: Runtime stage with minimal Alpine image +FROM python:3.12-alpine AS runtime + +LABEL maintainer="MeshCore MQTT Bridge Team" +LABEL description="Production runtime for MeshCore MQTT Bridge" +LABEL version="1.0.0" + +# Install runtime dependencies only +RUN apk add --no-cache \ + ca-certificates \ + tzdata \ + tini + +# Create non-root user for security +RUN addgroup -g 1000 meshcore && \ + adduser -D -u 1000 -G meshcore -s /bin/sh meshcore + +# Copy virtual environment from builder stage +COPY --from=builder /opt/venv /opt/venv + +# Set environment variables +ENV PATH="/opt/venv/bin:$PATH" \ + PYTHONUNBUFFERED=1 \ + PYTHONDONTWRITEBYTECODE=1 \ + PYTHONPATH="/app" + +# Create directories for logs only +RUN mkdir -p /app/logs && \ + chown -R meshcore:meshcore /app + +# Set working directory +WORKDIR /app + +# Add user to dialout group for serial port access +RUN adduser meshcore dialout + +# Switch to non-root user +USER meshcore + +# Health check +HEALTHCHECK --interval=30s --timeout=10s --start-period=5s --retries=3 \ + CMD python -c "import meshcore_mqtt; print('Health check passed')" || exit 1 + +# Default environment variables (can be overridden) +ENV LOG_LEVEL=INFO \ + MQTT_BROKER=localhost \ + MQTT_PORT=1883 \ + MQTT_TOPIC_PREFIX=meshcore \ + MESHCORE_CONNECTION=serial \ + MESHCORE_ADDRESS=/dev/ttyUSB0 \ + MESHCORE_BAUDRATE=115200 + +# Expose default MQTT port (informational) +EXPOSE 1883 + +# Use tini as init system for proper signal handling +ENTRYPOINT ["/sbin/tini", "--"] + +# Default command - can be overridden +CMD ["python", "-m", "meshcore_mqtt.main", "--env"] diff --git a/README.md b/README.md new file mode 100644 index 0000000..2f10d0e --- /dev/null +++ b/README.md @@ -0,0 +1,672 @@ +# MeshCore MQTT Bridge + +[![License](https://img.shields.io/badge/License-GPL_v3-blue.svg)](https://www.gnu.org/licenses/gpl-3.0) +[![Python](https://img.shields.io/badge/Python-3.11+-blue.svg)](https://www.python.org/) +[![CI](https://github.com/jinglemansweep/meshcore-mqtt/actions/workflows/ci.yml/badge.svg)](https://github.com/jinglemansweep/meshcore-mqtt/actions/workflows/ci.yml) +[![Code Quality](https://github.com/jinglemansweep/meshcore-mqtt/actions/workflows/code-quality.yml/badge.svg)](https://github.com/jinglemansweep/meshcore-mqtt/actions/workflows/code-quality.yml) +[![Tests](https://github.com/jinglemansweep/meshcore-mqtt/actions/workflows/test.yml/badge.svg)](https://github.com/jinglemansweep/meshcore-mqtt/actions/workflows/test.yml) +[![Docker Build and Push](https://github.com/jinglemansweep/meshcore-mqtt/actions/workflows/docker-build.yml/badge.svg)](https://github.com/jinglemansweep/meshcore-mqtt/actions/workflows/docker-build.yml) +[![Code style: black](https://img.shields.io/badge/Code%20style-black-000000.svg)](https://github.com/psf/black) +[![Typing: mypy](https://img.shields.io/badge/Typing-mypy-blue.svg)](https://mypy.readthedocs.io/) +[![Security: bandit](https://img.shields.io/badge/Security-bandit-yellow.svg)](https://github.com/PyCQA/bandit) + +A robust bridge service that connects MeshCore devices to MQTT brokers, enabling seamless integration with IoT platforms and message processing systems. + +## Features + +- **Inbox/Outbox Architecture**: Independent MeshCore and MQTT workers with message bus coordination +- **Multiple Connection Types**: Support for TCP, Serial, and BLE connections to MeshCore devices +- **Full Command Support**: Send messages, query device information, and network operations via MQTT +- **TLS/SSL Support**: Secure MQTT connections with configurable certificates +- **Flexible Configuration**: JSON, YAML, environment variables, and command-line configuration options +- **MQTT Integration**: Full MQTT client with authentication, QoS, retention, and auto-reconnection +- **Configurable Event Monitoring**: Subscribe to specific MeshCore event types for optimal performance +- **Health Monitoring**: Built-in health checks and automatic recovery for both workers +- **Async Architecture**: Built with Python asyncio for high performance and concurrent operations +- **Type Safety**: Full type annotations with mypy support +- **Comprehensive Testing**: 59+ unit tests with pytest and pytest-asyncio +- **Code Quality**: Pre-commit hooks with black formatting, flake8 linting, and automated testing + +## Installation + +### From Source + +```bash +git clone +cd meshcore-mqtt +python -m venv venv +source venv/bin/activate # On Windows: venv\Scripts\activate +pip install -r requirements.txt +``` + +### Development Installation + +```bash +pip install -r requirements-dev.txt +pre-commit install +``` + +## Configuration + +The bridge supports multiple configuration methods with the following precedence: +1. Command-line arguments (highest priority) +2. Configuration file (JSON or YAML) +3. Environment variables (lowest priority) + +### Configuration Options + +#### MQTT Settings +- `mqtt_broker`: MQTT broker address (required) +- `mqtt_port`: MQTT broker port (default: 1883) +- `mqtt_username`: MQTT authentication username (optional) +- `mqtt_password`: MQTT authentication password (optional) +- `mqtt_topic_prefix`: MQTT topic prefix (default: "meshcore") +- `mqtt_qos`: Quality of Service level 0-2 (default: 0) +- `mqtt_retain`: Message retention flag (default: false) + +#### MeshCore Settings +- `meshcore_connection`: Connection type (serial, ble, tcp) +- `meshcore_address`: Device address (required) +- `meshcore_port`: Device port for TCP connections (default: 12345) +- `meshcore_baudrate`: Baudrate for serial connections (default: 115200) +- `meshcore_timeout`: Operation timeout in seconds (default: 5) +- `meshcore_auto_fetch_restart_delay`: Delay in seconds before restarting auto-fetch after NO_MORE_MSGS (default: 5, range: 1-60) +- `meshcore_events`: List of MeshCore event types to subscribe to (see [Event Types](#event-types)) + +#### General Settings +- `log_level`: Logging level (DEBUG, INFO, WARNING, ERROR, CRITICAL) + +### Configuration Examples + +#### JSON Configuration (config.json) +```json +{ + "mqtt": { + "broker": "mqtt.example.com", + "port": 1883, + "username": "myuser", + "password": "mypass", + "topic_prefix": "meshcore", + "qos": 1, + "retain": false + }, + "meshcore": { + "connection_type": "tcp", + "address": "192.168.1.100", + "port": 12345, + "baudrate": 115200, + "timeout": 10, + "auto_fetch_restart_delay": 10, + "events": [ + "CONTACT_MSG_RECV", + "CHANNEL_MSG_RECV", + "CONNECTED", + "DISCONNECTED", + "BATTERY", + "DEVICE_INFO" + ] + }, + "log_level": "INFO" +} +``` + +#### YAML Configuration (config.yaml) +```yaml +mqtt: + broker: mqtt.example.com + port: 1883 + username: myuser + password: mypass + topic_prefix: meshcore + qos: 1 + retain: false + +meshcore: + connection_type: tcp + address: "192.168.1.100" + port: 12345 + baudrate: 115200 + timeout: 10 + auto_fetch_restart_delay: 10 + events: + - CONTACT_MSG_RECV + - CHANNEL_MSG_RECV + - CONNECTED + - DISCONNECTED + - BATTERY + - DEVICE_INFO + +log_level: INFO +``` + +#### Environment Variables +```bash +export MQTT_BROKER=mqtt.example.com +export MQTT_PORT=1883 +export MQTT_USERNAME=myuser +export MQTT_PASSWORD=mypass +export MESHCORE_CONNECTION=tcp +export MESHCORE_ADDRESS=192.168.1.100 +export MESHCORE_PORT=12345 +export MESHCORE_BAUDRATE=115200 +export MESHCORE_AUTO_FETCH_RESTART_DELAY=10 +export MESHCORE_EVENTS="CONNECTED,DISCONNECTED,BATTERY,DEVICE_INFO" +export LOG_LEVEL=INFO +``` + +## Usage + +### Command Line Interface + +#### Using Configuration File +```bash +python -m meshcore_mqtt.main --config-file config.json +``` + +#### Using Command Line Arguments +```bash +python -m meshcore_mqtt.main \ + --mqtt-broker mqtt.example.com \ + --mqtt-username myuser \ + --mqtt-password mypass \ + --meshcore-connection tcp \ + --meshcore-address 192.168.1.100 \ + --meshcore-port 12345 \ + --meshcore-auto-fetch-restart-delay 10 \ + --meshcore-events "CONNECTED,DISCONNECTED,BATTERY" +``` + +#### Using Environment Variables +```bash +python -m meshcore_mqtt.main --env +``` + +### Connection Types + +#### TCP Connection +```bash +python -m meshcore_mqtt.main \ + --mqtt-broker localhost \ + --meshcore-connection tcp \ + --meshcore-address 192.168.1.100 \ + --meshcore-port 12345 +``` + +#### Serial Connection +```bash +python -m meshcore_mqtt.main \ + --mqtt-broker localhost \ + --meshcore-connection serial \ + --meshcore-address /dev/ttyUSB0 \ + --meshcore-baudrate 9600 +``` + +#### BLE Connection +```bash +python -m meshcore_mqtt.main \ + --mqtt-broker localhost \ + --meshcore-connection ble \ + --meshcore-address AA:BB:CC:DD:EE:FF +``` + +## Event Types + +The bridge can subscribe to various MeshCore event types. You can configure which events to monitor using the `events` configuration option. + +### Default Events +If no events are specified, the bridge subscribes to these default events: +- `CONTACT_MSG_RECV` - Contact messages received +- `CHANNEL_MSG_RECV` - Channel messages received +- `CONNECTED` - Device connection events +- `DISCONNECTED` - Device disconnection events +- `LOGIN_SUCCESS` - Successful authentication +- `LOGIN_FAILED` - Failed authentication +- `MESSAGES_WAITING` - Pending messages notification +- `DEVICE_INFO` - Device information updates +- `BATTERY` - Battery status updates +- `NEW_CONTACT` - New contact discovered + +### Additional Supported Events +You can also subscribe to these additional event types: +- `TELEMETRY` - Telemetry data +- `POSITION` - Position/GPS updates +- `USER` - User-related events +- `ROUTING` - Mesh routing events +- `ADMIN` - Administrative messages +- `TEXT_MESSAGE_RX` - Text messages received +- `TEXT_MESSAGE_TX` - Text messages transmitted +- `WAYPOINT` - Waypoint data +- `NEIGHBOR_INFO` - Neighbor node information +- `TRACEROUTE` - Network trace information +- `NODE_LIST_CHANGED` - Node list updates +- `CONFIG_CHANGED` - Configuration changes +- `ADVERTISEMENT` - Device advertisement broadcasts + +### Configuration Examples + +#### Minimal Events (Performance Optimized) +```yaml +meshcore: + events: + - CONNECTED + - DISCONNECTED + - BATTERY +``` + +#### Message-Focused Events +```yaml +meshcore: + events: + - CONTACT_MSG_RECV + - CHANNEL_MSG_RECV + - TEXT_MESSAGE_RX +``` + +#### Full Monitoring +```yaml +meshcore: + events: + - CONTACT_MSG_RECV + - CHANNEL_MSG_RECV + - CONNECTED + - DISCONNECTED + - TELEMETRY + - POSITION + - BATTERY + - DEVICE_INFO + - ADVERTISEMENT +``` + +**Note**: Event names are case-insensitive. You can use `connected`, `CONNECTED`, or `Connected` - they will all be normalized to uppercase. + +## MQTT Topics + +The bridge provides full bidirectional communication between MQTT and MeshCore devices. Using the configured topic prefix (default: "meshcore"): + +### Published Topics (MeshCore → MQTT) +The bridge publishes to these topics based on configured MeshCore events: + +- `{prefix}/message/channel/{channel_idx}` - Channel messages from CHANNEL_MSG_RECV events +- `{prefix}/message/direct/{pubkey_prefix}` - Direct messages from CONTACT_MSG_RECV events +- `{prefix}/status` - Connection status from CONNECTED/DISCONNECTED events + +**Message Topic Subscription Patterns:** +- Subscribe to all messages: `{prefix}/message/+/+` +- Subscribe to all channel messages: `{prefix}/message/channel/+` +- Subscribe to specific channel: `{prefix}/message/channel/0` +- Subscribe to all direct messages: `{prefix}/message/direct/+` +- Subscribe to messages from specific node: `{prefix}/message/direct/{specific_pubkey_prefix}` + +Where: +- `{channel_idx}` is the numeric channel identifier (e.g., 0, 1, 2) +- `{pubkey_prefix}` is the sender's 6-byte public key prefix (hex encoded, e.g., `a1b2c3d4e5f6`) + +**Other Published Topics:** +- `{prefix}/login` - Authentication status from LOGIN_SUCCESS/LOGIN_FAILED events +- `{prefix}/device_info` - Device information from DEVICE_INFO events +- `{prefix}/battery` - Battery status from BATTERY events +- `{prefix}/new_contact` - Contact discovery from NEW_CONTACT events +- `{prefix}/advertisement` - Device advertisements from ADVERTISEMENT events + +### Command Topics (MQTT → MeshCore) +Send commands to MeshCore devices via MQTT using `{prefix}/command/{command_type}` with JSON payloads: + +#### Message Commands +- `{prefix}/command/send_msg` - Send direct message to contact/node + ```json + {"destination": "contact_name_or_node_id", "message": "Hello!"} + ``` + +- `{prefix}/command/send_chan_msg` - Send channel message + ```json + {"channel": 0, "message": "Hello channel!"} + ``` + +#### Device Management Commands +- `{prefix}/command/device_query` - Query device information + ```json + {} + ``` +- `{prefix}/command/get_battery` - Get battery status + ```json + {} + ``` +- `{prefix}/command/set_name` - Set device name + ```json + {"name": "MyDevice"} + ``` + +#### Network Commands +- `{prefix}/command/ping` - Ping a specific node + ```json + {"destination": "node_id"} + ``` + +### MQTT Command Examples + +Using `mosquitto_pub` client: + +```bash +# Send direct message +mosquitto_pub -h localhost -t "meshcore/command/send_msg" \ + -m '{"destination": "Alice", "message": "Hello Alice!"}' + +# Send channel message +mosquitto_pub -h localhost -t "meshcore/command/send_chan_msg" \ + -m '{"channel": 0, "message": "Hello everyone on channel 0!"}' + +# Ping a node +mosquitto_pub -h localhost -t "meshcore/command/ping" \ + -m '{"destination": "node123"}' + +# Get device information +mosquitto_pub -h localhost -t "meshcore/command/device_query" -m '{}' + +# Set device name +mosquitto_pub -h localhost -t "meshcore/command/set_name" \ + -m '{"name": "MyMeshDevice"}' + +# Send device advertisement +mosquitto_pub -h localhost -t "meshcore/command/send_advert" -m '{}' + +# Send device advertisement with flood +mosquitto_pub -h localhost -t "meshcore/command/send_advert" \ + -m '{"flood": true}' +``` + +### Available MeshCore Commands + +The bridge supports these MeshCore commands via MQTT: + +| Command | Description | Required Fields | Example | +|---------|-------------|-----------------|----------| +| `send_msg` | Send direct message | `destination`, `message` | `{"destination": "Alice", "message": "Hello!"}` | +| `send_chan_msg` | Send channel message | `channel`, `message` | `{"channel": 0, "message": "Hello channel!"}` | +| `device_query` | Query device information | None | `{}` | +| `get_battery` | Get battery status | None | `{}` | +| `set_name` | Set device name | `name` | `{"name": "MyDevice"}` | +| `ping` | Ping a node | `destination` | `{"destination": "node123"}` | +| `send_advert` | Send device advertisement | None (optional: `flood`) | `{}` or `{"flood": true}` | + +### Topic Examples +- `meshcore/message/channel/0` - Channel 0 messages +- `meshcore/message/direct/a1b2c3` - Direct messages from node a1b2c3 +- `meshcore/status` - Bridge connection status ("connected"/"disconnected") +- `meshcore/events/connection` - Raw MeshCore connection events (JSON) +- `meshcore/battery` - Battery level updates +- `meshcore/device_info` - Device specifications and capabilities +- `meshcore/advertisement` - Device advertisement broadcasts +- `meshcore/command/send_msg` - Send message command (subscribed) +- `meshcore/command/ping` - Ping command (subscribed) + +## Docker Deployment + +The MeshCore MQTT Bridge provides multi-stage Docker support with Alpine Linux for minimal image size and enhanced security. Following 12-factor app principles, the Docker container is configured entirely through environment variables. + +### Docker Features + +- **Multi-stage build**: Optimized Alpine-based images with minimal attack surface +- **Non-root user**: Runs as dedicated `meshcore` user for security +- **Environment variables**: Full configuration via environment variables (12-factor app) +- **Health checks**: Built-in container health monitoring +- **Signal handling**: Proper init system with tini for clean shutdowns +- **Container logging**: Logs output to stdout/stderr for Docker log drivers + +### Quick Start with Docker + +#### Using Pre-built Images from GHCR + +Pre-built Docker images are available from GitHub Container Registry: + +**Available Tags:** +- `latest` - Latest stable release from main branch +- `develop` - Latest development build +- `v1.0.0` - Specific version tags +- `v1.0` - Major.minor version tags +- `v1` - Major version tags + +```bash +# Pull the latest image +docker pull ghcr.io/jinglemansweep/meshcore-mqtt:latest + +# Run with serial connection (default for MeshCore devices) +docker run -d \ + --name meshcore-mqtt-bridge \ + --restart unless-stopped \ + --device=/dev/ttyUSB0:/dev/ttyUSB0 \ + -e MQTT_BROKER=192.168.1.100 \ + -e MQTT_USERNAME=meshcore \ + -e MQTT_PASSWORD=meshcore123 \ + -e MESHCORE_CONNECTION=serial \ + -e MESHCORE_ADDRESS=/dev/ttyUSB0 \ + ghcr.io/jinglemansweep/meshcore-mqtt:latest +``` + +#### Building Locally + +```bash +# Build the image locally +docker build -t meshcore-mqtt:latest . + +# Run with local image +docker run -d \ + --name meshcore-mqtt-bridge \ + --restart unless-stopped \ + --device=/dev/ttyUSB0:/dev/ttyUSB0 \ + -e MQTT_BROKER=192.168.1.100 \ + -e MQTT_USERNAME=meshcore \ + -e MQTT_PASSWORD=meshcore123 \ + -e MESHCORE_CONNECTION=serial \ + -e MESHCORE_ADDRESS=/dev/ttyUSB0 \ + meshcore-mqtt:latest +``` + +#### Using Environment File + +```bash +# Create environment file from example +cp .env.docker.example .env.docker +# Edit .env.docker with your configuration + +# Run with environment file (includes device mount for serial) +docker run -d \ + --name meshcore-mqtt-bridge \ + --restart unless-stopped \ + --device=/dev/ttyUSB0:/dev/ttyUSB0 \ + --env-file .env.docker \ + ghcr.io/jinglemansweep/meshcore-mqtt:latest +``` + +#### Option 3: Using Docker Compose +```bash +# Start the entire stack with MQTT broker +docker-compose up -d + +# View logs +docker-compose logs -f meshcore-mqtt + +# Stop the stack +docker-compose down +``` + + +### Docker Environment Variables + +All configuration options can be set via environment variables: + +```bash +# Logging Configuration +LOG_LEVEL=INFO + +# MQTT Broker Configuration +MQTT_BROKER=localhost +MQTT_PORT=1883 +MQTT_USERNAME=user +MQTT_PASSWORD=pass +MQTT_TOPIC_PREFIX=meshcore +MQTT_QOS=1 +MQTT_RETAIN=true + +# MQTT TLS/SSL Configuration (optional) +MQTT_TLS_ENABLED=false +MQTT_TLS_CA_CERT=/path/to/ca.crt +MQTT_TLS_CLIENT_CERT=/path/to/client.crt +MQTT_TLS_CLIENT_KEY=/path/to/client.key +MQTT_TLS_INSECURE=false + +# MeshCore Device Configuration (serial default) +MESHCORE_CONNECTION=serial +MESHCORE_ADDRESS=/dev/ttyUSB0 # Serial port, IP address, or BLE MAC address +MESHCORE_BAUDRATE=115200 # For serial connections +MESHCORE_PORT=4403 # Only for TCP connections +MESHCORE_TIMEOUT=30 +MESHCORE_AUTO_FETCH_RESTART_DELAY=10 # Restart delay after NO_MORE_MSGS (1-60 seconds) + +# Event Configuration (comma-separated) +MESHCORE_EVENTS=CONNECTED,DISCONNECTED,BATTERY,DEVICE_INFO +``` + +#### Connection Type Examples + +**Serial Connection (Default):** +```bash +MESHCORE_CONNECTION=serial +MESHCORE_ADDRESS=/dev/ttyUSB0 +MESHCORE_BAUDRATE=115200 +# Note: Use --device=/dev/ttyUSB0 in docker run for device access +``` + +**TCP Connection:** +```bash +MESHCORE_CONNECTION=tcp +MESHCORE_ADDRESS=192.168.1.100 +MESHCORE_PORT=4403 +``` + +**BLE Connection:** +```bash +MESHCORE_CONNECTION=ble +MESHCORE_ADDRESS=AA:BB:CC:DD:EE:FF +``` + + +### Health Monitoring + +The container includes health checks: + +```bash +# Check container health +docker inspect --format='{{.State.Health.Status}}' meshcore-mqtt-bridge + +# View health check logs +docker inspect --format='{{range .State.Health.Log}}{{.Output}}{{end}}' meshcore-mqtt-bridge +``` + +### Container Management + +```bash +# View container logs +docker logs -f meshcore-mqtt-bridge + +# Execute commands in container +docker exec -it meshcore-mqtt-bridge sh + +# Stop container +docker stop meshcore-mqtt-bridge + +# Remove container +docker rm meshcore-mqtt-bridge +``` + +## Development + +### Running Tests +```bash +# Run all tests +pytest + +# Run with coverage +pytest --cov=meshcore_mqtt + +# Run specific test file +pytest tests/test_config.py -v +``` + +### Code Quality +```bash +# Format code +black meshcore_mqtt/ tests/ + +# Lint code +flake8 meshcore_mqtt/ tests/ + +# Type checking +mypy meshcore_mqtt/ tests/ + +# Run pre-commit hooks +pre-commit run --all-files +``` + +## Architecture + +The bridge uses an **Inbox/Outbox Architecture** with independent workers: + +### Core Components + +1. **Bridge Coordinator** (`bridge_coordinator.py`) + - Coordinates independent MeshCore and MQTT workers + - Manages shared message bus for inter-worker communication + - Provides health monitoring and system statistics + - Handles graceful startup and shutdown + +2. **Message Bus System** (`message_queue.py`) + - Async message queue system for worker communication + - Thread-safe inbox/outbox pattern + - Component status tracking and health monitoring + - Message types: events, commands, status updates + +3. **MeshCore Worker** (`meshcore_worker.py`) + - Independent worker managing MeshCore device connection + - Handles device commands and event subscriptions + - Auto-reconnection and health monitoring + - Forwards events to MQTT worker via message bus + +4. **MQTT Worker** (`mqtt_worker.py`) + - Independent worker managing MQTT broker connection + - Subscribes to command topics and publishes events + - TLS/SSL support with configurable certificates + - Auto-reconnection and connection recovery + +5. **Configuration System** (`config.py`) + - Pydantic-based configuration with validation + - Support for JSON, YAML, environment variables, and CLI args + - Type-safe configuration with proper defaults + +6. **CLI Interface** (`main.py`) + - Click-based command line interface + - Configuration loading and validation + - Logging setup and error handling + +### Benefits of Inbox/Outbox Architecture + +- **Resilience**: Workers operate independently, one can fail without affecting the other +- **Scalability**: Easy to add new workers or modify existing ones +- **Testability**: Each worker can be tested in isolation +- **Monitoring**: Built-in health checks and statistics for each component +- **Recovery**: Automatic reconnection and error recovery for both workers + +## License + +This project is licensed under the GNU General Public License v3.0 - see the [LICENSE](LICENSE) file for details. + +## Contributing + +1. Fork the repository +2. Create a feature branch (`git checkout -b feature/amazing-feature`) +3. Commit your changes (`git commit -m 'Add some amazing feature'`) +4. Push to the branch (`git push origin feature/amazing-feature`) +5. Open a Pull Request + +## Support + +For issues and questions, please open an issue on the GitHub repository. diff --git a/config.example.json b/config.example.json new file mode 100644 index 0000000..7964f0e --- /dev/null +++ b/config.example.json @@ -0,0 +1,33 @@ +{ + "mqtt": { + "broker": "localhost", + "port": 1883, + "username": null, + "password": null, + "topic_prefix": "meshcore", + "qos": 0, + "retain": false + }, + "meshcore": { + "connection_type": "tcp", + "address": "192.168.1.100", + "port": 12345, + "baudrate": 115200, + "timeout": 5, + "auto_fetch_restart_delay": 5, + "events": [ + "CONTACT_MSG_RECV", + "CHANNEL_MSG_RECV", + "CONNECTED", + "DISCONNECTED", + "LOGIN_SUCCESS", + "LOGIN_FAILED", + "MESSAGES_WAITING", + "DEVICE_INFO", + "BATTERY", + "NEW_CONTACT", + "ADVERTISEMENT" + ] + }, + "log_level": "INFO" +} diff --git a/config.example.yaml b/config.example.yaml new file mode 100644 index 0000000..b1e4321 --- /dev/null +++ b/config.example.yaml @@ -0,0 +1,30 @@ +mqtt: + broker: localhost + port: 1883 + username: null + password: null + topic_prefix: meshcore + qos: 0 + retain: false + +meshcore: + connection_type: tcp + address: "192.168.1.100" + port: 12345 + baudrate: 115200 + timeout: 5 + auto_fetch_restart_delay: 5 + events: + - CONTACT_MSG_RECV + - CHANNEL_MSG_RECV + - CONNECTED + - DISCONNECTED + - LOGIN_SUCCESS + - LOGIN_FAILED + - MESSAGES_WAITING + - DEVICE_INFO + - BATTERY + - NEW_CONTACT + - ADVERTISEMENT + +log_level: INFO diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..a023fb3 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,86 @@ +services: + meshcore-mqtt: + image: ghcr.io/jinglemansweep/meshcore-mqtt:latest + # Alternative: build locally + # build: . + container_name: meshcore-mqtt-bridge + restart: unless-stopped + + # Environment variables configuration + environment: + # Logging configuration + - LOG_LEVEL=INFO + + # MQTT broker configuration + - MQTT_BROKER=mqtt-broker + - MQTT_PORT=1883 + - MQTT_USERNAME=meshcore + - MQTT_PASSWORD=meshcore123 + - MQTT_TOPIC_PREFIX=meshcore + - MQTT_QOS=1 + - MQTT_RETAIN=true + + # MeshCore device configuration (serial default) + - MESHCORE_CONNECTION=serial + - MESHCORE_ADDRESS=/dev/ttyUSB0 + - MESHCORE_BAUDRATE=115200 + - MESHCORE_TIMEOUT=30 + + # Configurable events (comma-separated) + - MESHCORE_EVENTS=CONTACT_MSG_RECV,CHANNEL_MSG_RECV,CONNECTED,DISCONNECTED,LOGIN_SUCCESS,DEVICE_INFO,BATTERY,ADVERTISEMENT + + # Device access for serial communication + devices: + - "/dev/ttyUSB0:/dev/ttyUSB0" + + # Network configuration + networks: + - meshcore-network + + # Dependencies + depends_on: + - mqtt-broker + + # Health check + healthcheck: + test: ["CMD", "python", "-c", "import meshcore_mqtt; print('Health check passed')"] + interval: 30s + timeout: 10s + retries: 3 + start_period: 10s + + # Example MQTT broker (Eclipse Mosquitto) + mqtt-broker: + image: eclipse-mosquitto:2.0-openssl + container_name: mqtt-broker + restart: unless-stopped + + ports: + - "1883:1883" + - "9001:9001" + + volumes: + - mqtt-config:/mosquitto/config + - mqtt-data:/mosquitto/data + - mqtt-logs:/mosquitto/log + + networks: + - meshcore-network + + # Simple mosquitto config + command: | + sh -c 'echo "listener 1883 + allow_anonymous true + persistence true + persistence_location /mosquitto/data/ + log_dest file /mosquitto/log/mosquitto.log" > /mosquitto/config/mosquitto.conf && + /usr/sbin/mosquitto -c /mosquitto/config/mosquitto.conf' + +volumes: + mqtt-config: + mqtt-data: + mqtt-logs: + +networks: + meshcore-network: + driver: bridge diff --git a/docker-run.example.sh b/docker-run.example.sh new file mode 100755 index 0000000..fe53d06 --- /dev/null +++ b/docker-run.example.sh @@ -0,0 +1,48 @@ +#!/bin/bash + +# Example script for running MeshCore MQTT Bridge with Docker +# Docker containers use environment variables for configuration (12-factor app) + +echo "Building MeshCore MQTT Bridge Docker image..." +docker build -t meshcore-mqtt:latest . + +echo "Running MeshCore MQTT Bridge with serial connection (default)..." + +# Option 1: Serial connection (default) - most common for MeshCore devices +docker run -d \ + --name meshcore-mqtt-bridge \ + --restart unless-stopped \ + --device=/dev/ttyUSB0:/dev/ttyUSB0 \ + -e LOG_LEVEL=INFO \ + -e MQTT_BROKER=192.168.1.100 \ + -e MQTT_USERNAME=meshcore \ + -e MQTT_PASSWORD=meshcore123 \ + -e MESHCORE_CONNECTION=serial \ + -e MESHCORE_ADDRESS=/dev/ttyUSB0 \ + -e MESHCORE_BAUDRATE=115200 \ + -e MESHCORE_EVENTS=CONTACT_MSG_RECV,CHANNEL_MSG_RECV,CONNECTED,DISCONNECTED,DEVICE_INFO \ + meshcore-mqtt:latest + +echo "Container started. Check logs with: docker logs -f meshcore-mqtt-bridge" + +# Option 2: Using environment file (recommended for production) +# docker run -d \ +# --name meshcore-mqtt-bridge \ +# --restart unless-stopped \ +# --device=/dev/ttyUSB0:/dev/ttyUSB0 \ +# --env-file .env.docker \ +# meshcore-mqtt:latest + +# Option 3: Using Docker Compose (recommended for multi-container setup) +# docker-compose up -d + +# Option 4: TCP connection example (for network-connected MeshCore devices) +# docker run -d \ +# --name meshcore-mqtt-bridge \ +# --restart unless-stopped \ +# -e LOG_LEVEL=INFO \ +# -e MQTT_BROKER=192.168.1.100 \ +# -e MESHCORE_CONNECTION=tcp \ +# -e MESHCORE_ADDRESS=192.168.1.200 \ +# -e MESHCORE_PORT=4403 \ +# meshcore-mqtt:latest diff --git a/meshcore_mqtt/__init__.py b/meshcore_mqtt/__init__.py new file mode 100644 index 0000000..a1212ac --- /dev/null +++ b/meshcore_mqtt/__init__.py @@ -0,0 +1,3 @@ +"""MeshCore to MQTT Bridge.""" + +__version__ = "0.1.0" diff --git a/meshcore_mqtt/bridge_coordinator.py b/meshcore_mqtt/bridge_coordinator.py new file mode 100644 index 0000000..c7a4011 --- /dev/null +++ b/meshcore_mqtt/bridge_coordinator.py @@ -0,0 +1,309 @@ +"""Bridge coordinator managing independent MeshCore and MQTT workers.""" + +import asyncio +import logging +from typing import Any, Optional + +from .config import Config +from .meshcore_worker import MeshCoreWorker +from .message_queue import ( + ComponentStatus, + MessageBus, + get_message_bus, + reset_message_bus, +) +from .mqtt_worker import MQTTWorker + + +class BridgeCoordinator: + """Coordinates independent MeshCore and MQTT workers with shared messaging.""" + + def __init__(self, config: Config) -> None: + """Initialize the bridge coordinator.""" + self.config = config + self.logger = logging.getLogger(__name__) + + # Reset message bus for clean start + reset_message_bus() + + # Message bus + self.message_bus: MessageBus = get_message_bus() + + # Workers + self.meshcore_worker: Optional[MeshCoreWorker] = None + self.mqtt_worker: Optional[MQTTWorker] = None + + # Coordinator state + self._running = False + self._shutdown_event = asyncio.Event() + self._worker_tasks: list[asyncio.Task[Any]] = [] + + async def start(self) -> None: + """Start the bridge coordinator and all workers.""" + if self._running: + self.logger.warning("Bridge coordinator is already running") + return + + self.logger.info("Starting MeshCore MQTT Bridge Coordinator") + self._running = True + + try: + # Initialize workers + self.meshcore_worker = MeshCoreWorker(self.config) + self.mqtt_worker = MQTTWorker(self.config) + + # Start MQTT worker first and wait for it to be connected + self.logger.info("Starting MQTT worker first...") + mqtt_task = asyncio.create_task( + self.mqtt_worker.start(), name="mqtt_worker" + ) + self._worker_tasks.append(mqtt_task) + + # Wait for MQTT to be connected before starting MeshCore + await self._wait_for_mqtt_connection() + + # Now start MeshCore worker + self.logger.info("Starting MeshCore worker...") + meshcore_task = asyncio.create_task( + self.meshcore_worker.start(), name="meshcore_worker" + ) + self._worker_tasks.append(meshcore_task) + + # Start coordinator monitoring + monitor_task = asyncio.create_task( + self._monitor_workers(), name="coordinator_monitor" + ) + self._worker_tasks.append(monitor_task) + + self.logger.info("Bridge coordinator started successfully") + + # Wait for shutdown signal + await self._shutdown_event.wait() + + except Exception as e: + self.logger.error(f"Error starting bridge coordinator: {e}") + raise + finally: + await self.stop() + + async def stop(self) -> None: + """Stop the bridge coordinator and all workers.""" + if not self._running: + return + + self.logger.info("Stopping MeshCore MQTT Bridge Coordinator") + self._running = False + self._shutdown_event.set() + + try: + # Signal shutdown to message bus + await self.message_bus.shutdown() + + # Cancel all worker tasks + for task in self._worker_tasks: + if not task.done(): + task.cancel() + + # Wait for tasks to complete + if self._worker_tasks: + await asyncio.gather(*self._worker_tasks, return_exceptions=True) + self._worker_tasks.clear() + + # Stop workers explicitly if they exist + if self.meshcore_worker: + await self.meshcore_worker.stop() + if self.mqtt_worker: + await self.mqtt_worker.stop() + + except Exception as e: + self.logger.error(f"Error during coordinator shutdown: {e}") + + self.logger.info("Bridge coordinator stopped") + + async def _monitor_workers(self) -> None: + """Monitor worker health and overall system status.""" + self.logger.info("Starting coordinator monitoring") + + last_stats_log = 0.0 + stats_interval = 60.0 # Log stats every 60 seconds + + while self._running: + try: + current_time = asyncio.get_event_loop().time() + + # Check worker status + meshcore_status = self.message_bus.get_component_status("meshcore") + mqtt_status = self.message_bus.get_component_status("mqtt") + + # Log status changes + if meshcore_status == ComponentStatus.ERROR: + self.logger.error("MeshCore worker is in error state") + if mqtt_status == ComponentStatus.ERROR: + self.logger.error("MQTT worker is in error state") + + # Log periodic stats + if current_time - last_stats_log >= stats_interval: + self._log_system_stats() + last_stats_log = current_time + + # Check if both workers are in error state + if ( + meshcore_status == ComponentStatus.ERROR + and mqtt_status == ComponentStatus.ERROR + ): + self.logger.critical( + "Both workers are in error state, shutting down" + ) + self._shutdown_event.set() + break + + await asyncio.sleep(10) # Monitor every 10 seconds + + except Exception as e: + self.logger.error(f"Error in coordinator monitoring: {e}") + await asyncio.sleep(30) + + def _log_system_stats(self) -> None: + """Log system statistics.""" + try: + stats = self.message_bus.get_stats() + + self.logger.info("=== Bridge System Status ===") + self.logger.info(f"Total components: {stats['total_components']}") + + for name, component_info in stats["components"].items(): + status = component_info["status"] + queue_stats = component_info["queue"] + + self.logger.info( + f"{name.upper()}: {status} | " + f"Queue: {queue_stats['size']}/{queue_stats['max_size']} | " + f"Dropped: {queue_stats['dropped_messages']}" + ) + + self.logger.info("========================") + + except Exception as e: + self.logger.error(f"Error logging system stats: {e}") + + async def _wait_for_mqtt_connection(self, timeout: float = 30.0) -> None: + """Wait for MQTT worker to be connected before proceeding.""" + self.logger.info("Waiting for MQTT worker to connect...") + start_time = asyncio.get_event_loop().time() + + while self._running: + mqtt_status = self.message_bus.get_component_status("mqtt") + + if mqtt_status == ComponentStatus.CONNECTED: + self.logger.info( + "MQTT worker is connected, proceeding with MeshCore startup" + ) + return + elif mqtt_status == ComponentStatus.ERROR: + self.logger.warning("MQTT worker is in error state, proceeding anyway") + return + + # Check timeout + elapsed = asyncio.get_event_loop().time() - start_time + if elapsed > timeout: + self.logger.warning( + f"Timeout waiting for MQTT connection after {timeout}s, " + "proceeding with MeshCore startup" + ) + return + + await asyncio.sleep(0.5) + + async def health_check(self) -> dict[str, Any]: + """Perform overall system health check.""" + try: + stats = self.message_bus.get_stats() + + # Get component statuses + meshcore_status = self.message_bus.get_component_status("meshcore") + mqtt_status = self.message_bus.get_component_status("mqtt") + + # Determine overall health + healthy_statuses = {ComponentStatus.RUNNING, ComponentStatus.CONNECTED} + meshcore_healthy = meshcore_status in healthy_statuses + mqtt_healthy = mqtt_status in healthy_statuses + overall_healthy = meshcore_healthy and mqtt_healthy + + return { + "healthy": overall_healthy, + "running": self._running, + "components": { + "meshcore": { + "status": ( + meshcore_status.value if meshcore_status else "unknown" + ), + "healthy": meshcore_healthy, + }, + "mqtt": { + "status": mqtt_status.value if mqtt_status else "unknown", + "healthy": mqtt_healthy, + }, + }, + "message_bus": stats, + } + + except Exception as e: + self.logger.error(f"Error in health check: {e}") + return {"healthy": False, "error": str(e), "running": self._running} + + def get_stats(self) -> dict[str, Any]: + """Get comprehensive system statistics.""" + try: + bus_stats = self.message_bus.get_stats() + + return { + "coordinator": { + "running": self._running, + "worker_tasks": len(self._worker_tasks), + "active_tasks": sum(1 for t in self._worker_tasks if not t.done()), + }, + "message_bus": bus_stats, + "config": { + "mqtt_broker": f"{self.config.mqtt.broker}:{self.config.mqtt.port}", + "mqtt_topic_prefix": self.config.mqtt.topic_prefix, + "meshcore_connection": { + "type": self.config.meshcore.connection_type.value, + "address": self.config.meshcore.address, + "port": getattr(self.config.meshcore, "port", None), + }, + "events": self.config.meshcore.events, + }, + } + + except Exception as e: + self.logger.error(f"Error getting stats: {e}") + return {"error": str(e)} + + # Compatibility methods for existing tests + @property + def meshcore(self) -> Any: + """Compatibility property for tests.""" + return self.meshcore_worker.meshcore if self.meshcore_worker else None + + @property + def connection_manager(self) -> Any: + """Compatibility property for tests.""" + if self.meshcore_worker and self.meshcore_worker.meshcore: + return self.meshcore_worker.meshcore.connection_manager + return None + + @property + def mqtt_client(self) -> Any: + """Compatibility property for tests.""" + return self.mqtt_worker.client if self.mqtt_worker else None + + def _serialize_to_json(self, data: Any) -> str: + """Compatibility method for tests.""" + if self.meshcore_worker: + return self.meshcore_worker.serialize_to_json(data) + return "{}" + + async def _setup_mqtt(self) -> None: + """Compatibility method for tests.""" + if self.mqtt_worker: + await self.mqtt_worker._setup_connection() diff --git a/meshcore_mqtt/config.py b/meshcore_mqtt/config.py new file mode 100644 index 0000000..4d3e829 --- /dev/null +++ b/meshcore_mqtt/config.py @@ -0,0 +1,253 @@ +"""Configuration system for MeshCore MQTT Bridge.""" + +import json +import os +from enum import Enum +from pathlib import Path +from typing import Any, List, Optional, Union + +import yaml +from pydantic import BaseModel, Field, field_validator + + +class ConnectionType(str, Enum): + """Supported MeshCore connection types.""" + + SERIAL = "serial" + BLE = "ble" + TCP = "tcp" + + +class MQTTConfig(BaseModel): + """MQTT broker configuration.""" + + broker: str = Field(..., description="MQTT broker address") + port: int = Field(default=1883, description="MQTT broker port") + username: Optional[str] = Field(default=None, description="MQTT username") + password: Optional[str] = Field(default=None, description="MQTT password") + topic_prefix: str = Field(default="meshcore", description="MQTT topic prefix") + qos: int = Field(default=0, ge=0, le=2, description="Quality of Service level") + retain: bool = Field(default=False, description="Message retention flag") + + # TLS configuration + tls_enabled: bool = Field(default=False, description="Enable TLS/SSL connection") + tls_ca_cert: Optional[str] = Field( + default=None, description="Path to CA certificate file" + ) + tls_client_cert: Optional[str] = Field( + default=None, description="Path to client certificate file" + ) + tls_client_key: Optional[str] = Field( + default=None, description="Path to client private key file" + ) + tls_insecure: bool = Field( + default=False, description="Disable certificate verification" + ) + + @field_validator("port") + @classmethod + def validate_port(cls, v: int) -> int: + """Validate MQTT port number.""" + if not 1 <= v <= 65535: + raise ValueError("Port must be between 1 and 65535") + return v + + @field_validator("tls_ca_cert", "tls_client_cert", "tls_client_key") + @classmethod + def validate_tls_files(cls, v: Optional[str], info: Any) -> Optional[str]: + """Validate TLS certificate and key file paths.""" + if v is not None and v.strip(): + file_path = Path(v.strip()) + if not file_path.exists(): + raise ValueError(f"TLS file not found: {v}") + if not file_path.is_file(): + raise ValueError(f"TLS path is not a file: {v}") + return v + + +class MeshCoreConfig(BaseModel): + """MeshCore device configuration.""" + + connection_type: ConnectionType = Field(..., description="Connection type") + address: str = Field(..., description="Device address") + port: Optional[int] = Field(default=None, description="Device port for TCP") + baudrate: int = Field(default=115200, description="Baudrate for serial connections") + timeout: int = Field(default=5, gt=0, description="Operation timeout in seconds") + auto_fetch_restart_delay: int = Field( + default=5, + ge=1, + le=60, + description="Delay in seconds before restarting auto-fetch after NO_MORE_MSGS", + ) + events: List[str] = Field( + default=[ + "CONTACT_MSG_RECV", + "CHANNEL_MSG_RECV", + "CONNECTED", + "DISCONNECTED", + "LOGIN_SUCCESS", + "LOGIN_FAILED", + "DEVICE_INFO", + "BATTERY", + "NEW_CONTACT", + "ADVERTISEMENT", + ], + description="List of MeshCore event types to subscribe to", + ) + + @field_validator("port") + @classmethod + def validate_port(cls, v: Optional[int], info: Any) -> Optional[int]: + """Validate port is provided for TCP connections.""" + # Get connection_type from the validation context + connection_type = info.data.get("connection_type") if info.data else None + + # Set default port for TCP if None provided + if connection_type == ConnectionType.TCP and v is None: + v = 12345 + + if v is not None and not 1 <= v <= 65535: + raise ValueError("Port must be between 1 and 65535") + + return v + + @field_validator("events") + @classmethod + def validate_events(cls, v: List[str]) -> List[str]: + """Validate event types are valid MeshCore EventType names.""" + # Normalize to uppercase for case-insensitive validation + normalized_events = [event.upper() for event in v] + + # Common MeshCore event types (based on typical mesh networking events) + valid_events = { + "CONTACT_MSG_RECV", + "CHANNEL_MSG_RECV", + "CONNECTED", + "DISCONNECTED", + "LOGIN_SUCCESS", + "LOGIN_FAILED", + "MESSAGES_WAITING", + "DEVICE_INFO", + "BATTERY", + "NEW_CONTACT", + "NODE_LIST_CHANGED", + "CONFIG_CHANGED", + "TELEMETRY", + "POSITION", + "USER", + "ROUTING", + "ADMIN", + "TEXT_MESSAGE_RX", + "TEXT_MESSAGE_TX", + "WAYPOINT", + "NEIGHBOR_INFO", + "TRACEROUTE", + "ADVERTISEMENT", + } + + invalid_events = [ + event for event in normalized_events if event not in valid_events + ] + if invalid_events: + raise ValueError( + f"Invalid event types: {invalid_events}. " + f"Valid events: {sorted(valid_events)}" + ) + + return normalized_events + + +class Config(BaseModel): + """Main application configuration.""" + + mqtt: MQTTConfig + meshcore: MeshCoreConfig + log_level: str = Field(default="INFO", description="Logging level") + + @field_validator("log_level") + @classmethod + def validate_log_level(cls, v: str) -> str: + """Validate logging level.""" + valid_levels = {"DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"} + if v.upper() not in valid_levels: + raise ValueError(f"log_level must be one of {valid_levels}") + return v.upper() + + @classmethod + def from_file(cls, config_path: Union[str, Path]) -> "Config": + """Load configuration from a file (JSON or YAML).""" + config_path = Path(config_path) + + if not config_path.exists(): + raise FileNotFoundError(f"Configuration file not found: {config_path}") + + with open(config_path, "r") as f: + if config_path.suffix.lower() in [".yaml", ".yml"]: + try: + data = yaml.safe_load(f) + except yaml.YAMLError as e: + raise ValueError(f"Invalid YAML configuration: {e}") + else: + try: + data = json.load(f) + except json.JSONDecodeError as e: + raise ValueError(f"Invalid JSON configuration: {e}") + + return cls(**data) + + @classmethod + def parse_events_string(cls, events_str: str) -> List[str]: + """Parse comma-separated event string into list.""" + if not events_str or events_str.strip() == "": + return [] + return [ + event.strip().upper() for event in events_str.split(",") if event.strip() + ] + + @classmethod + def from_env(cls) -> "Config": + """Load configuration from environment variables.""" + mqtt_config = MQTTConfig( + broker=os.getenv("MQTT_BROKER", ""), + port=int(os.getenv("MQTT_PORT", "1883")), + username=os.getenv("MQTT_USERNAME"), + password=os.getenv("MQTT_PASSWORD"), + topic_prefix=os.getenv("MQTT_TOPIC_PREFIX", "meshcore"), + qos=int(os.getenv("MQTT_QOS", "0")), + retain=os.getenv("MQTT_RETAIN", "false").lower() == "true", + tls_enabled=os.getenv("MQTT_TLS_ENABLED", "false").lower() == "true", + tls_ca_cert=os.getenv("MQTT_TLS_CA_CERT"), + tls_client_cert=os.getenv("MQTT_TLS_CLIENT_CERT"), + tls_client_key=os.getenv("MQTT_TLS_CLIENT_KEY"), + tls_insecure=os.getenv("MQTT_TLS_INSECURE", "false").lower() == "true", + ) + + # Parse events from environment variable if provided + events_env = os.getenv("MESHCORE_EVENTS") + events = cls.parse_events_string(events_env) if events_env else None + + meshcore_config = MeshCoreConfig( + connection_type=ConnectionType(os.getenv("MESHCORE_CONNECTION", "tcp")), + address=os.getenv("MESHCORE_ADDRESS", ""), + port=( + int(os.getenv("MESHCORE_PORT", "12345")) + if os.getenv("MESHCORE_PORT") + else None + ), + baudrate=int(os.getenv("MESHCORE_BAUDRATE", "115200")), + timeout=int(os.getenv("MESHCORE_TIMEOUT", "5")), + auto_fetch_restart_delay=int( + os.getenv("MESHCORE_AUTO_FETCH_RESTART_DELAY", "5") + ), + events=( + events + if events is not None + else MeshCoreConfig.model_fields["events"].default + ), + ) + + return cls( + mqtt=mqtt_config, + meshcore=meshcore_config, + log_level=os.getenv("LOG_LEVEL", "INFO"), + ) diff --git a/meshcore_mqtt/main.py b/meshcore_mqtt/main.py new file mode 100644 index 0000000..32b74e6 --- /dev/null +++ b/meshcore_mqtt/main.py @@ -0,0 +1,321 @@ +"""Main entry point for MeshCore MQTT Bridge.""" + +import asyncio +import logging +import sys +from pathlib import Path +from typing import Optional + +import click + +from .config import Config, ConnectionType + + +def setup_logging(level: str) -> None: + """Set up logging configuration.""" + log_level = getattr(logging, level) + + # Configure root logger + logging.basicConfig( + level=log_level, + format="%(asctime)s - %(name)s - %(levelname)s - %(message)s", + handlers=[logging.StreamHandler(sys.stdout)], + force=True, # Override any existing configuration + ) + + # Ensure MeshCore and other third-party libraries respect our log level + # Set common third-party library loggers + third_party_loggers = [ + "meshcore", + "paho", + "paho.mqtt", + "paho.mqtt.client", + "asyncio", + ] + + for logger_name in third_party_loggers: + logging.getLogger(logger_name).setLevel(log_level) + + # Set urllib3 and requests to WARNING to reduce noise unless we're in DEBUG mode + if level != "DEBUG": + logging.getLogger("urllib3").setLevel(logging.WARNING) + logging.getLogger("requests").setLevel(logging.WARNING) + else: + # In DEBUG mode, let urllib3 and requests use the same log level + logging.getLogger("urllib3").setLevel(log_level) + logging.getLogger("requests").setLevel(log_level) + + +@click.command() +@click.option( + "--config-file", + "-c", + type=click.Path(exists=True, path_type=Path), + help="Path to configuration file (JSON or YAML)", +) +@click.option( + "--mqtt-broker", + help="MQTT broker address", +) +@click.option( + "--mqtt-port", + type=int, + default=1883, + help="MQTT broker port (default: 1883)", +) +@click.option( + "--mqtt-username", + help="MQTT username", +) +@click.option( + "--mqtt-password", + help="MQTT password", +) +@click.option( + "--mqtt-topic-prefix", + default="meshcore", + help="MQTT topic prefix (default: meshcore)", +) +@click.option( + "--mqtt-qos", + type=click.IntRange(0, 2), + default=0, + help="MQTT QoS level (default: 0)", +) +@click.option( + "--mqtt-retain/--no-mqtt-retain", + default=False, + help="Enable MQTT message retention (default: disabled)", +) +@click.option( + "--mqtt-tls/--no-mqtt-tls", + default=False, + help="Enable MQTT TLS/SSL connection (default: disabled)", +) +@click.option( + "--mqtt-tls-ca-cert", + help="Path to CA certificate file for TLS", +) +@click.option( + "--mqtt-tls-client-cert", + help="Path to client certificate file for TLS", +) +@click.option( + "--mqtt-tls-client-key", + help="Path to client private key file for TLS", +) +@click.option( + "--mqtt-tls-insecure/--no-mqtt-tls-insecure", + default=False, + help="Disable TLS certificate verification (default: disabled)", +) +@click.option( + "--meshcore-connection", + type=click.Choice([conn.value for conn in ConnectionType]), + help="MeshCore connection type", +) +@click.option( + "--meshcore-address", + help="MeshCore device address", +) +@click.option( + "--meshcore-port", + type=int, + help="MeshCore device port (for TCP connections)", +) +@click.option( + "--meshcore-baudrate", + type=int, + default=115200, + help="MeshCore baudrate for serial connections (default: 115200)", +) +@click.option( + "--meshcore-timeout", + type=int, + default=5, + help="MeshCore operation timeout in seconds (default: 5)", +) +@click.option( + "--meshcore-auto-fetch-restart-delay", + type=click.IntRange(1, 60), + default=5, + help="Delay in seconds before restarting auto-fetch after NO_MORE_MSGS " + "(default: 5)", +) +@click.option( + "--meshcore-events", + help="Comma-separated list of MeshCore event types to subscribe to", +) +@click.option( + "--log-level", + type=click.Choice(["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"]), + default="INFO", + help="Logging level (default: INFO)", +) +@click.option( + "--env", + is_flag=True, + help="Load configuration from environment variables", +) +def main( + config_file: Optional[Path], + mqtt_broker: Optional[str], + mqtt_port: int, + mqtt_username: Optional[str], + mqtt_password: Optional[str], + mqtt_topic_prefix: str, + mqtt_qos: int, + mqtt_retain: bool, + mqtt_tls: bool, + mqtt_tls_ca_cert: Optional[str], + mqtt_tls_client_cert: Optional[str], + mqtt_tls_client_key: Optional[str], + mqtt_tls_insecure: bool, + meshcore_connection: Optional[str], + meshcore_address: Optional[str], + meshcore_port: Optional[int], + meshcore_baudrate: int, + meshcore_timeout: int, + meshcore_auto_fetch_restart_delay: int, + meshcore_events: Optional[str], + log_level: str, + env: bool, +) -> None: + """MeshCore to MQTT Bridge. + + Bridge messages between MeshCore devices and MQTT brokers. + Configuration can be provided via command-line arguments, + configuration file, or environment variables. + """ + try: + # Load configuration in order of precedence: + # 1. Command line arguments (highest priority) + # 2. Configuration file + # 3. Environment variables (lowest priority) + + if config_file: + config = Config.from_file(config_file) + elif env: + config = Config.from_env() + else: + # Build config from command line arguments + if not mqtt_broker or not meshcore_connection or not meshcore_address: + click.echo( + "Error: --mqtt-broker, --meshcore-connection, and " + "--meshcore-address are required when not using a config file", + err=True, + ) + sys.exit(1) + + from .config import MeshCoreConfig, MQTTConfig + + mqtt_config = MQTTConfig( + broker=mqtt_broker, + port=mqtt_port, + username=mqtt_username, + password=mqtt_password, + topic_prefix=mqtt_topic_prefix, + qos=mqtt_qos, + retain=mqtt_retain, + tls_enabled=mqtt_tls, + tls_ca_cert=mqtt_tls_ca_cert, + tls_client_cert=mqtt_tls_client_cert, + tls_client_key=mqtt_tls_client_key, + tls_insecure=mqtt_tls_insecure, + ) + + # Parse events if provided + events = ( + Config.parse_events_string(meshcore_events) if meshcore_events else None + ) + + meshcore_config = MeshCoreConfig( + connection_type=ConnectionType(meshcore_connection), + address=meshcore_address, + port=meshcore_port, + baudrate=meshcore_baudrate, + timeout=meshcore_timeout, + auto_fetch_restart_delay=meshcore_auto_fetch_restart_delay, + events=( + events + if events is not None + else MeshCoreConfig.model_fields["events"].default + ), + ) + + config = Config( + mqtt=mqtt_config, + meshcore=meshcore_config, + log_level=log_level, + ) + + # Override config with any provided command line arguments + if mqtt_broker: + config.mqtt.broker = mqtt_broker + if mqtt_username: + config.mqtt.username = mqtt_username + if mqtt_password: + config.mqtt.password = mqtt_password + if meshcore_connection: + config.meshcore.connection_type = ConnectionType(meshcore_connection) + if meshcore_address: + config.meshcore.address = meshcore_address + if meshcore_port: + config.meshcore.port = meshcore_port + if meshcore_baudrate != 115200: # Only override if different from default + config.meshcore.baudrate = meshcore_baudrate + if meshcore_timeout != 5: # Only override if different from default + config.meshcore.timeout = meshcore_timeout + if ( + meshcore_auto_fetch_restart_delay != 5 + ): # Only override if different from default + config.meshcore.auto_fetch_restart_delay = meshcore_auto_fetch_restart_delay + if meshcore_events: + config.meshcore.events = Config.parse_events_string(meshcore_events) + + # Set up logging + setup_logging(config.log_level) + logger = logging.getLogger(__name__) + + logger.info("Starting MeshCore MQTT Bridge") + logger.info(f"MQTT Broker: {config.mqtt.broker}:{config.mqtt.port}") + logger.info( + f"MeshCore: {config.meshcore.connection_type.value}://" + f"{config.meshcore.address}" + ) + + # Run the bridge application + asyncio.run(run_bridge(config)) + + except Exception as e: + click.echo(f"Error: {e}", err=True) + sys.exit(1) + + +async def run_bridge(config: Config) -> None: + """Run the MeshCore MQTT bridge.""" + from .bridge_coordinator import BridgeCoordinator + + logger = logging.getLogger(__name__) + bridge = BridgeCoordinator(config) + + try: + # Start the bridge + await bridge.start() + + # Keep running until interrupted + while True: + await asyncio.sleep(1) + + except KeyboardInterrupt: + logger.info("Bridge interrupted by user") + except Exception as e: + logger.error(f"Bridge error: {e}") + raise + finally: + # Clean shutdown + await bridge.stop() + + +if __name__ == "__main__": + main() diff --git a/meshcore_mqtt/meshcore_client.py b/meshcore_mqtt/meshcore_client.py new file mode 100644 index 0000000..3d48f31 --- /dev/null +++ b/meshcore_mqtt/meshcore_client.py @@ -0,0 +1,620 @@ +"""MeshCore client management for bridge.""" + +import asyncio +import json +import logging +import time +from datetime import datetime, timezone +from typing import TYPE_CHECKING, Any, Callable, Dict, Optional + +if TYPE_CHECKING: + import serial +else: + try: + import serial + except ImportError: + serial = None + +from meshcore import ( + BLEConnection, + ConnectionManager, + EventType, + MeshCore, + SerialConnection, + TCPConnection, +) + +from .config import Config, ConnectionType + + +class MeshCoreClientManager: + """Manages MeshCore device connection, event handling, and recovery.""" + + def __init__(self, config: Config) -> None: + """Initialize MeshCore client manager.""" + self.config = config + self.logger = logging.getLogger(__name__) + + # MeshCore components + self.meshcore: Optional[MeshCore] = None + self.connection_manager: Optional[ConnectionManager] = None + + # Connection state + self._connected = False + self._reconnect_attempts = 0 + self._max_reconnect_attempts = 10 + self._last_activity: Optional[float] = None + self._auto_fetch_running = False + self._last_health_check: Optional[float] = None + self._consecutive_health_failures = 0 + self._max_health_failures = 3 + + # Event handlers + self._event_handlers: Dict[str, Callable[[Any], None]] = {} + + # Running state + self._running = False + + async def start(self) -> None: + """Start the MeshCore client.""" + if self._running: + self.logger.warning("MeshCore client is already running") + return + + self.logger.info("Starting MeshCore client") + self._running = True + + # Setup MeshCore connection + await self._setup_connection() + + async def stop(self) -> None: + """Stop the MeshCore client.""" + if not self._running: + return + + self.logger.info("Stopping MeshCore client") + self._running = False + + if self.meshcore: + try: + await self.meshcore.stop_auto_message_fetching() + await self.meshcore.disconnect() + except Exception as e: + self.logger.error(f"Error stopping MeshCore client: {e}") + + self.logger.info("MeshCore client stopped") + + async def _setup_connection(self) -> None: + """Set up MeshCore connection.""" + self.logger.info("Setting up MeshCore connection") + + # Create appropriate connection based on configuration + if self.config.meshcore.connection_type == ConnectionType.TCP: + connection = TCPConnection( + self.config.meshcore.address, self.config.meshcore.port or 12345 + ) + elif self.config.meshcore.connection_type == ConnectionType.SERIAL: + connection = SerialConnection( + self.config.meshcore.address, self.config.meshcore.baudrate + ) + elif self.config.meshcore.connection_type == ConnectionType.BLE: + connection = BLEConnection(self.config.meshcore.address) + else: + raise ValueError( + f"Unsupported connection type: {self.config.meshcore.connection_type}" + ) + + # Initialize connection manager and MeshCore + self.connection_manager = ConnectionManager(connection) + + # Enable debug logging only if log level is DEBUG + debug_logging = self.config.log_level == "DEBUG" + + self.meshcore = MeshCore( + self.connection_manager, + debug=debug_logging, + auto_reconnect=True, + default_timeout=self.config.meshcore.timeout, + ) + + # Configure MeshCore logger + meshcore_logger = logging.getLogger("meshcore") + meshcore_logger.setLevel(getattr(logging, self.config.log_level)) + + # Set up event subscriptions + await self._setup_event_subscriptions() + + # Connect to MeshCore device + try: + await self.meshcore.connect() + self.logger.info("Connected to MeshCore device") + self._connected = True + self._last_activity = time.time() + + # Start auto message fetching + await self.meshcore.start_auto_message_fetching() + self.logger.info("Started auto message fetching") + self._auto_fetch_running = True + except Exception as e: + raise RuntimeError(f"Failed to connect to MeshCore device: {e}") + + async def _setup_event_subscriptions(self) -> None: + """Set up MeshCore event subscriptions.""" + self.logger.info("Setting up MeshCore event subscriptions") + configured_events = set(self.config.meshcore.events) + + # Always subscribe to NO_MORE_MSGS to restart auto-fetching + if self.meshcore: + try: + no_more_msgs_event = getattr(EventType, "NO_MORE_MSGS") + self.meshcore.subscribe(no_more_msgs_event, self._on_no_more_msgs) + self.logger.info( + "Subscribed to NO_MORE_MSGS event for auto-fetch restart" + ) + except AttributeError: + self.logger.warning("NO_MORE_MSGS event type not available") + + # Subscribe to configured events + subscribed_events = set() + if self.meshcore: + for event_name in configured_events: + try: + event_type = getattr(EventType, event_name) + # Use registered handler or default debug handler + handler = self._event_handlers.get(event_name, self._on_debug_event) + self.meshcore.subscribe(event_type, handler) + subscribed_events.add(event_name) + self.logger.info(f"Subscribed to event: {event_name}") + except AttributeError: + self.logger.warning(f"Unknown event type: {event_name}") + + async def _recover_connection(self) -> None: + """Recover MeshCore connection.""" + if self._reconnect_attempts >= self._max_reconnect_attempts: + self.logger.error("Max MeshCore reconnection attempts reached") + return + + self._reconnect_attempts += 1 + self.logger.warning( + f"Starting MeshCore recovery (attempt " + f"{self._reconnect_attempts}/{self._max_reconnect_attempts})" + ) + + try: + # Stop existing connection + if self.meshcore: + try: + await self.meshcore.stop_auto_message_fetching() + await self.meshcore.disconnect() + except Exception as e: + self.logger.debug(f"Error stopping old MeshCore connection: {e}") + + # Wait before attempting reconnection with exponential backoff + delay = min(2 ** (self._reconnect_attempts - 1), 300) # Max 5 minutes + self.logger.info( + f"Waiting {delay}s before MeshCore reconnection (exponential backoff)" + ) + await asyncio.sleep(delay) + + # Re-setup connection + await self._setup_connection() + self._reconnect_attempts = 0 + self.logger.info("MeshCore connection recovery successful") + + except Exception as e: + self.logger.error( + f"MeshCore recovery attempt {self._reconnect_attempts} failed: {e}" + ) + if self._reconnect_attempts < self._max_reconnect_attempts: + retry_delay = min( + 2**self._reconnect_attempts, 300 + ) # Exponential backoff, max 5 minutes + self.logger.info( + f"Scheduling MeshCore retry in {retry_delay}s (exponential backoff)" + ) + await asyncio.sleep(retry_delay) + if self._running: + asyncio.create_task(self._recover_connection()) + else: + self.logger.error("🚨 MeshCore recovery failed permanently") + + async def _maintain_auto_fetch(self) -> None: + """Continuously maintain auto-fetch, restarting if it stops.""" + self.logger.info("Starting persistent auto-fetch maintenance") + + while self._running: + try: + if self.meshcore and self._connected and not self._auto_fetch_running: + self.logger.info("Starting/restarting MeshCore auto-fetch") + try: + await self.meshcore.start_auto_message_fetching() + self._auto_fetch_running = True + self._last_activity = time.time() + except Exception as e: + self.logger.error(f"Failed to start auto-fetch: {e}") + self._auto_fetch_running = False + + await asyncio.sleep(60) # Check every minute + + except Exception as e: + self.logger.error(f"Error in auto-fetch maintenance: {e}") + await asyncio.sleep(60) + + def _on_no_more_msgs(self, event_data: Any) -> None: + """Handle NO_MORE_MSGS events - mark auto-fetch as stopped.""" + self.logger.info(f"Received NO_MORE_MSGS event: {event_data}") + self.logger.info( + "Auto-fetch has stopped - persistent maintenance will restart it" + ) + + self._auto_fetch_running = False + self._last_activity = time.time() + + def _on_debug_event(self, event_data: Any) -> None: + """Handle any other MeshCore events for debugging.""" + event_info = f"type: {type(event_data)}, data: {event_data}" + self.logger.debug(f"MeshCore debug event: {event_info}") + print(f"DEBUG EVENT: {event_info}") + + def serialize_to_json(self, data: Any) -> str: + """Safely serialize any data to JSON string.""" + try: + # Handle common data types + if isinstance(data, (dict, list, str, int, float, bool)) or data is None: + return json.dumps(data, ensure_ascii=False) + + # Handle objects with custom serialization + if hasattr(data, "__dict__"): + obj_dict = { + key: value + for key, value in data.__dict__.items() + if not key.startswith("_") + } + if obj_dict: + return json.dumps(obj_dict, ensure_ascii=False, default=str) + + # Handle iterables + if hasattr(data, "__iter__") and not isinstance(data, (str, bytes)): + try: + return json.dumps(list(data), ensure_ascii=False, default=str) + except (TypeError, ValueError): + pass + + # Fallback: structured JSON with metadata + return json.dumps( + { + "type": type(data).__name__, + "value": str(data), + "timestamp": datetime.now(timezone.utc).isoformat(), + }, + ensure_ascii=False, + ) + + except Exception as e: + self.logger.warning(f"Failed to serialize data to JSON: {e}") + return json.dumps( + { + "error": f"Serialization failed: {str(e)}", + "raw_value": str(data)[:1000], + "type": type(data).__name__, + "timestamp": datetime.now(timezone.utc).isoformat(), + }, + ensure_ascii=False, + ) + + def register_event_handler( + self, event_name: str, handler: Callable[[Any], None] + ) -> None: + """Register an event handler for a specific MeshCore event.""" + self._event_handlers[event_name] = handler + self.logger.debug(f"Registered handler for event: {event_name}") + + async def _safe_command_call( + self, + command_func: Callable[..., Any], + command_type: str, + *args: Any, + **kwargs: Any, + ) -> Any: + """Safely call a MeshCore command with proper error handling.""" + try: + return await command_func(*args, **kwargs) + except AttributeError as e: + self.logger.error( + f"Command '{command_type}' not supported by MeshCore version: {e}" + ) + return None + except Exception as e: + self.logger.error(f"Error executing command '{command_type}': {e}") + return None + + async def send_command(self, command_type: str, command_data: Any) -> None: + """Send command to MeshCore device.""" + if not self.meshcore: + self.logger.error("MeshCore not initialized") + return + + try: + self.logger.info( + f"Sending command to MeshCore: {command_type} -> {command_data}" + ) + + result = None + + if command_type == "send_msg": + # Send direct message + destination = command_data.get("destination") + message = command_data.get("message", "") + if not destination or not message: + self.logger.error( + "send_msg requires 'destination' and 'message' fields" + ) + return + result = await self._safe_command_call( + self.meshcore.commands.send_msg, "send_msg", destination, message + ) + + elif command_type == "device_query": + # Query device information + result = await self._safe_command_call( + self.meshcore.commands.send_device_query, "device_query" + ) + + elif command_type == "get_battery": + # Get battery status + result = await self._safe_command_call( + self.meshcore.commands.get_bat, "get_battery" + ) + + elif command_type == "set_name": + # Set device name + name = command_data.get("name", "") + if not name: + self.logger.error("set_name requires 'name' field") + return + result = await self._safe_command_call( + self.meshcore.commands.set_name, "set_name", name + ) + + elif command_type == "send_chan_msg": + # Send channel message + channel = command_data.get("channel") + message = command_data.get("message", "") + if channel is None or not message: + self.logger.error( + "send_chan_msg requires 'channel' and 'message' fields" + ) + return + result = await self._safe_command_call( + self.meshcore.commands.send_chan_msg, + "send_chan_msg", + channel, + message, + ) + + elif command_type == "ping": + # Ping a node + destination = command_data.get("destination") + if not destination: + self.logger.error("ping requires 'destination' field") + return + result = await self._safe_command_call( + self.meshcore.commands.ping, "ping", destination + ) + + else: + self.logger.warning(f"Unknown command type: {command_type}") + return + + # Handle result + if result and hasattr(result, "type"): + if result.type == EventType.ERROR: + self.logger.error( + f"MeshCore command '{command_type}' failed: {result.payload}" + ) + else: + self.logger.info(f"MeshCore command '{command_type}' successful") + # Update activity timestamp on successful command + self.update_activity() + else: + self.logger.info(f"MeshCore command '{command_type}' completed") + self.update_activity() + + except AttributeError as e: + # Handle case where commands attribute doesn't exist + self.logger.error(f"MeshCore command '{command_type}' unavailable: {e}") + except Exception as e: + self.logger.error( + f"Error sending command '{command_type}' to MeshCore: {e}" + ) + + def is_connected(self) -> bool: + """Check if MeshCore client is connected.""" + return self._connected + + def is_stale(self, timeout_seconds: int = 300) -> bool: + """Check if connection appears stale.""" + if not self._last_activity: + return False + return time.time() - self._last_activity > timeout_seconds + + async def health_check(self) -> bool: + """Perform health check and trigger recovery if needed.""" + # Basic health check + if not self.meshcore: + self._consecutive_health_failures += 1 + return False + + try: + # Check if MeshCore and connection manager exist + basic_healthy = ( + hasattr(self.meshcore, "connection_manager") + and self.meshcore.connection_manager is not None + ) + + # Enhanced health check for serial connections + connection_healthy = await self._check_connection_health() + + healthy = basic_healthy and connection_healthy + + if healthy: + # Reset failure counter on successful health check + self._consecutive_health_failures = 0 + else: + self._consecutive_health_failures += 1 + self.logger.debug( + f"Health check failed (attempt {self._consecutive_health_failures}" + f"/{self._max_health_failures})" + ) + + # Only trigger recovery after multiple consecutive failures + if ( + self._consecutive_health_failures >= self._max_health_failures + and self._connected + ): + self.logger.warning( + f"MeshCore connection lost after " + f"{self._consecutive_health_failures} " + "failed health checks, attempting recovery" + ) + self._connected = False + self._consecutive_health_failures = 0 + asyncio.create_task(self._recover_connection()) + return False + + # Check for stale connections with more aggressive detection + if healthy and self.is_stale(timeout_seconds=180): # 3 minutes + self.logger.warning( + "MeshCore connection appears stale, forcing reconnection" + ) + self._consecutive_health_failures = 0 + asyncio.create_task(self._recover_connection()) + return False + + return healthy + + except Exception as e: + self.logger.debug(f"Health check exception: {e}") + self._consecutive_health_failures += 1 + return False + + async def _check_connection_health(self) -> bool: + """Check if the underlying connection is healthy, especially for serial.""" + if not self.meshcore or not self.meshcore.connection_manager: + return False + + try: + connection = self.meshcore.connection_manager.connection + current_time = time.time() + + # Check if we should perform an intensive health check + should_deep_check = ( + self._last_health_check is None + or (current_time - self._last_health_check) + > 10 # Deep check every 10 seconds + ) + + # Basic connection manager check + if not hasattr(self.meshcore.connection_manager, "is_connected"): + # Check if connection manager has basic connectivity indicators + try: + # Try to access connection manager state + if hasattr(self.meshcore.connection_manager, "connection"): + conn = self.meshcore.connection_manager.connection + if conn is None: + self.logger.warning("Connection manager has no connection") + return False + except Exception as e: + self.logger.warning(f"Connection manager check failed: {e}") + return False + + # For serial connections, perform more rigorous checks + if hasattr(connection, "port") and hasattr(connection, "is_open"): + # This is likely a SerialConnection + if not connection.is_open: + self.logger.warning("Serial connection is closed") + return False + + if should_deep_check: + # Check if the serial port still exists in the system + try: + import serial.tools.list_ports + + available_ports = [ + port.device for port in serial.tools.list_ports.comports() + ] + if connection.port not in available_ports: + self.logger.warning( + f"Serial port {connection.port} no longer available" + ) + return False + except ImportError: + # pyserial not available for port checking, skip this test + pass + except Exception as e: + self.logger.debug( + f"Error checking serial port availability: {e}" + ) + + # Try to check if the serial connection is actually responsive + try: + # Check serial port properties that indicate connection health + if hasattr(connection, "in_waiting"): + # This property access can fail if device is unplugged + _ = connection.in_waiting + if hasattr(connection, "out_waiting"): + _ = connection.out_waiting + except OSError as e: + self.logger.warning(f"Serial port appears disconnected: {e}") + return False + except Exception as e: + self.logger.debug(f"Serial health check error: {e}") + # Don't fail on unexpected errors, but log them + pass + + self._last_health_check = current_time + + # For TCP connections, check socket state + elif hasattr(connection, "host") and hasattr(connection, "port"): + # This is likely a TCPConnection + if should_deep_check: + try: + # Check if the underlying socket exists and is connected + if hasattr(connection, "_socket") and connection._socket: + # Try to check socket state + pass # The meshcore library should handle this + except Exception as e: + self.logger.debug(f"TCP health check error: {e}") + + self._last_health_check = current_time + + # For BLE connections + elif hasattr(connection, "address"): + # This is likely a BLEConnection + if should_deep_check: + # BLE connection health is handled by the meshcore library + self._last_health_check = current_time + + # Check activity staleness more aggressively + if self._last_activity and should_deep_check: + time_since_activity = current_time - self._last_activity + if time_since_activity > 120: # 2 minutes of no activity + self.logger.warning( + f"No MeshCore activity for {time_since_activity:.1f}s, " + "connection may be stale" + ) + return False + + return True + + except Exception as e: + self.logger.debug(f"Connection health check failed: {e}") + return False + + def get_auto_fetch_task(self) -> asyncio.Task[None]: + """Get the auto-fetch maintenance task.""" + return asyncio.create_task(self._maintain_auto_fetch()) + + def update_activity(self) -> None: + """Update the last activity timestamp.""" + self._last_activity = time.time() diff --git a/meshcore_mqtt/meshcore_worker.py b/meshcore_mqtt/meshcore_worker.py new file mode 100644 index 0000000..2f14431 --- /dev/null +++ b/meshcore_mqtt/meshcore_worker.py @@ -0,0 +1,686 @@ +"""Independent MeshCore worker with inbox/outbox message handling.""" + +import asyncio +import logging +import time +from typing import TYPE_CHECKING, Any, Optional + +if TYPE_CHECKING: + import serial +else: + try: + import serial + except ImportError: + serial = None + +from meshcore import ( + BLEConnection, + ConnectionManager, + EventType, + MeshCore, + SerialConnection, + TCPConnection, +) + +from .config import Config, ConnectionType +from .message_queue import ( + ComponentStatus, + Message, + MessageBus, + MessageQueue, + MessageType, + get_message_bus, +) + + +class MeshCoreWorker: + """Independent MeshCore worker managing device connection and messaging.""" + + def __init__(self, config: Config) -> None: + """Initialize MeshCore worker.""" + self.config = config + self.logger = logging.getLogger(__name__) + + # Component identification + self.component_name = "meshcore" + + # Message bus + self.message_bus: MessageBus = get_message_bus() + self.inbox: MessageQueue = self.message_bus.register_component( + self.component_name, queue_size=1000 + ) + + # MeshCore components + self.meshcore: Optional[MeshCore] = None + self.connection_manager: Optional[ConnectionManager] = None + + # Connection state + self._connected = False + self._reconnect_attempts = 0 + self._max_reconnect_attempts = 10 + self._last_activity: Optional[float] = None + self._auto_fetch_running = False + self._last_health_check: Optional[float] = None + self._consecutive_health_failures = 0 + self._max_health_failures = 3 + + # Worker state + self._running = False + self._shutdown_event = asyncio.Event() + self._tasks: list[asyncio.Task[Any]] = [] + + async def start(self) -> None: + """Start the MeshCore worker.""" + if self._running: + self.logger.warning("MeshCore worker is already running") + return + + self.logger.info("Starting MeshCore worker") + self._running = True + + # Update status + self.message_bus.update_component_status( + self.component_name, ComponentStatus.STARTING + ) + + try: + # Setup MeshCore connection + await self._setup_connection() + + # Start worker tasks + tasks = [ + asyncio.create_task( + self._message_processor(), name="meshcore_processor" + ), + asyncio.create_task(self._health_monitor(), name="meshcore_health"), + asyncio.create_task( + self._auto_fetch_monitor(), name="meshcore_autofetch" + ), + ] + self._tasks.extend(tasks) + + # Update status to running + self.message_bus.update_component_status( + self.component_name, ComponentStatus.RUNNING + ) + + self.logger.info("MeshCore worker started successfully") + + # Wait for shutdown + await self._shutdown_event.wait() + + except Exception as e: + self.logger.error(f"Error starting MeshCore worker: {e}") + self.message_bus.update_component_status( + self.component_name, ComponentStatus.ERROR + ) + raise + finally: + await self.stop() + + async def stop(self) -> None: + """Stop the MeshCore worker.""" + if not self._running: + return + + self.logger.info("Stopping MeshCore worker") + self.message_bus.update_component_status( + self.component_name, ComponentStatus.STOPPING + ) + + self._running = False + self._shutdown_event.set() + + # Cancel all tasks + for task in self._tasks: + if not task.done(): + task.cancel() + + if self._tasks: + await asyncio.gather(*self._tasks, return_exceptions=True) + self._tasks.clear() + + # Stop MeshCore connection + if self.meshcore: + try: + await self.meshcore.stop_auto_message_fetching() + await self.meshcore.disconnect() + except Exception as e: + self.logger.error(f"Error stopping MeshCore connection: {e}") + + self.message_bus.update_component_status( + self.component_name, ComponentStatus.STOPPED + ) + self.logger.info("MeshCore worker stopped") + + async def _setup_connection(self) -> None: + """Set up MeshCore connection.""" + self.logger.info("Setting up MeshCore connection") + + # Create appropriate connection based on configuration + if self.config.meshcore.connection_type == ConnectionType.TCP: + connection = TCPConnection( + self.config.meshcore.address, self.config.meshcore.port or 12345 + ) + elif self.config.meshcore.connection_type == ConnectionType.SERIAL: + connection = SerialConnection( + self.config.meshcore.address, self.config.meshcore.baudrate + ) + elif self.config.meshcore.connection_type == ConnectionType.BLE: + connection = BLEConnection(self.config.meshcore.address) + else: + raise ValueError( + f"Unsupported connection type: {self.config.meshcore.connection_type}" + ) + + # Initialize connection manager and MeshCore + self.connection_manager = ConnectionManager(connection) + + # Enable debug logging only if log level is DEBUG + debug_logging = self.config.log_level == "DEBUG" + + self.meshcore = MeshCore( + self.connection_manager, + debug=debug_logging, + auto_reconnect=True, + default_timeout=self.config.meshcore.timeout, + ) + + # Configure MeshCore logger + meshcore_logger = logging.getLogger("meshcore") + meshcore_logger.setLevel(getattr(logging, self.config.log_level)) + + # Set up event subscriptions + await self._setup_event_subscriptions() + + # Connect to MeshCore device + try: + await self.meshcore.connect() + self.logger.info("Connected to MeshCore device") + self._connected = True + self._last_activity = time.time() + + # Send connection status + await self._send_status_update(ComponentStatus.CONNECTED, "connected") + + # Start auto message fetching + await self.meshcore.start_auto_message_fetching() + self.logger.info("Started auto message fetching") + self._auto_fetch_running = True + + except Exception as e: + await self._send_status_update( + ComponentStatus.ERROR, f"connection_failed: {e}" + ) + raise RuntimeError(f"Failed to connect to MeshCore device: {e}") + + async def _setup_event_subscriptions(self) -> None: + """Set up MeshCore event subscriptions.""" + self.logger.info("Setting up MeshCore event subscriptions") + configured_events = set(self.config.meshcore.events) + + # Always subscribe to NO_MORE_MSGS to restart auto-fetching + if self.meshcore: + try: + no_more_msgs_event = getattr(EventType, "NO_MORE_MSGS") + self.meshcore.subscribe(no_more_msgs_event, self._on_no_more_msgs) + self.logger.info( + "Subscribed to NO_MORE_MSGS event for auto-fetch restart" + ) + except AttributeError: + self.logger.warning("NO_MORE_MSGS event type not available") + + # Subscribe to configured events + if self.meshcore: + for event_name in configured_events: + try: + event_type = getattr(EventType, event_name) + self.meshcore.subscribe(event_type, self._on_meshcore_event) + self.logger.info(f"Subscribed to event: {event_name}") + except AttributeError: + self.logger.warning(f"Unknown event type: {event_name}") + + async def _message_processor(self) -> None: + """Process messages from the inbox.""" + self.logger.info("Starting MeshCore message processor") + + while self._running: + try: + # Get message from inbox with timeout + message = await self.inbox.get(timeout=1.0) + if message is None: + continue + + await self._handle_inbox_message(message) + + except Exception as e: + self.logger.error(f"Error in message processor: {e}") + await asyncio.sleep(1) + + async def _handle_inbox_message(self, message: Message) -> None: + """Handle a message from the inbox.""" + self.logger.debug(f"Processing message: {message.message_type.value}") + + try: + if message.message_type == MessageType.MQTT_COMMAND: + await self._handle_mqtt_command(message) + elif message.message_type == MessageType.HEALTH_CHECK: + await self._handle_health_check(message) + elif message.message_type == MessageType.SHUTDOWN: + self.logger.info("Received shutdown message") + self._shutdown_event.set() + else: + self.logger.warning( + f"Unknown message type: {message.message_type.value}" + ) + + except Exception as e: + self.logger.error(f"Error handling message {message.id}: {e}") + + async def _handle_mqtt_command(self, message: Message) -> None: + """Handle MQTT command forwarded from MQTT worker.""" + if not self.meshcore: + self.logger.error("MeshCore not initialized, cannot process command") + return + + command_data = message.payload + command_type = command_data.get("command_type", "") + + self.logger.info(f"Processing MQTT command: {command_type}") + + try: + result = None + + if command_type == "send_msg": + destination = command_data.get("destination") + msg_text = command_data.get("message", "") + if not destination or not msg_text: + self.logger.error( + "send_msg requires 'destination' and 'message' fields" + ) + return + result = await self.meshcore.commands.send_msg(destination, msg_text) + + elif command_type == "device_query": + result = await self.meshcore.commands.send_device_query() + + elif command_type == "get_battery": + result = await self.meshcore.commands.get_bat() + + elif command_type == "set_name": + name = command_data.get("name", "") + if not name: + self.logger.error("set_name requires 'name' field") + return + result = await self.meshcore.commands.set_name(name) + + elif command_type == "send_chan_msg": + channel = command_data.get("channel") + msg_text = command_data.get("message", "") + if channel is None or not msg_text: + self.logger.error( + "send_chan_msg requires 'channel' and 'message' fields" + ) + return + result = await self.meshcore.commands.send_chan_msg(channel, msg_text) + + elif command_type == "ping": + destination = command_data.get("destination") + if not destination: + self.logger.error("ping requires 'destination' field") + return + result = await self.meshcore.commands.ping(destination) + + elif command_type == "send_advert": + flood = command_data.get("flood", False) + result = await self.meshcore.commands.send_advert(flood=flood) + + else: + self.logger.warning(f"Unknown command type: {command_type}") + return + + # Handle result and update activity + if result and hasattr(result, "type"): + if result.type == EventType.ERROR: + self.logger.error( + f"MeshCore command '{command_type}' failed: {result.payload}" + ) + else: + self.logger.info(f"MeshCore command '{command_type}' successful") + self.update_activity() + else: + self.logger.info(f"MeshCore command '{command_type}' completed") + self.update_activity() + + except AttributeError as e: + self.logger.error(f"MeshCore command '{command_type}' unavailable: {e}") + except Exception as e: + self.logger.error(f"Error executing command '{command_type}': {e}") + + async def _handle_health_check(self, message: Message) -> None: + """Handle health check request.""" + healthy = await self._perform_health_check() + + # Send health status back + response = Message.create( + message_type=MessageType.HEALTH_CHECK, + source=self.component_name, + target=message.source, + payload={ + "healthy": healthy, + "connected": self._connected, + "last_activity": self._last_activity, + "auto_fetch_running": self._auto_fetch_running, + }, + ) + await self.message_bus.send_message(response) + + async def _health_monitor(self) -> None: + """Monitor MeshCore connection health.""" + self.logger.info("Starting MeshCore health monitor") + + # Wait for initial connection to stabilize + await asyncio.sleep(5) + + while self._running: + try: + healthy = await self._perform_health_check() + + if not healthy and self._connected: + self.logger.warning( + "MeshCore health check failed, attempting recovery" + ) + await self._recover_connection() + + await asyncio.sleep(10) # Health check every 10 seconds + + except Exception as e: + self.logger.error(f"Error in health monitor: {e}") + await asyncio.sleep(30) + + async def _perform_health_check(self) -> bool: + """Perform comprehensive health check.""" + if not self.meshcore: + self._consecutive_health_failures += 1 + return False + + try: + # Check basic connectivity + basic_healthy = ( + hasattr(self.meshcore, "connection_manager") + and self.meshcore.connection_manager is not None + ) + + # Enhanced health check for different connection types + connection_healthy = await self._check_connection_health() + + healthy = basic_healthy and connection_healthy + + if healthy: + self._consecutive_health_failures = 0 + else: + self._consecutive_health_failures += 1 + + # Check for stale connections + if healthy and self._is_stale(timeout_seconds=180): + self.logger.warning("MeshCore connection appears stale") + return False + + return healthy + + except Exception as e: + self.logger.debug(f"Health check exception: {e}") + self._consecutive_health_failures += 1 + return False + + async def _check_connection_health(self) -> bool: + """Check if the underlying connection is healthy.""" + if not self.meshcore or not self.meshcore.connection_manager: + return False + + try: + connection = self.meshcore.connection_manager.connection + current_time = time.time() + + # Check if we should perform an intensive health check + should_deep_check = ( + self._last_health_check is None + or (current_time - self._last_health_check) > 10 + ) + + # For serial connections, perform more rigorous checks + if hasattr(connection, "port") and hasattr(connection, "is_open"): + if not connection.is_open: + self.logger.warning("Serial connection is closed") + return False + + if should_deep_check: + try: + # Check if serial port still exists + if serial: + import serial.tools.list_ports + + available_ports = [ + port.device + for port in serial.tools.list_ports.comports() + ] + if connection.port not in available_ports: + self.logger.warning( + f"Serial port {connection.port} no longer available" + ) + return False + + # Try to check if the serial connection is responsive + if hasattr(connection, "in_waiting"): + _ = connection.in_waiting + if hasattr(connection, "out_waiting"): + _ = connection.out_waiting + + except (ImportError, OSError) as e: + self.logger.warning(f"Serial port health check failed: {e}") + return False + + self._last_health_check = current_time + + # For TCP connections + elif hasattr(connection, "host") and hasattr(connection, "port"): + if should_deep_check: + self._last_health_check = current_time + + # For BLE connections + elif hasattr(connection, "address"): + if should_deep_check: + self._last_health_check = current_time + + return True + + except Exception as e: + self.logger.debug(f"Connection health check failed: {e}") + return False + + async def _auto_fetch_monitor(self) -> None: + """Monitor and maintain auto-fetch functionality.""" + self.logger.info("Starting auto-fetch monitor") + + while self._running: + try: + if self.meshcore and self._connected and not self._auto_fetch_running: + self.logger.info("Restarting MeshCore auto-fetch") + try: + await self.meshcore.start_auto_message_fetching() + self._auto_fetch_running = True + self.update_activity() + except Exception as e: + self.logger.error(f"Failed to restart auto-fetch: {e}") + + await asyncio.sleep(60) # Check every minute + + except Exception as e: + self.logger.error(f"Error in auto-fetch monitor: {e}") + await asyncio.sleep(60) + + async def _recover_connection(self) -> None: + """Recover MeshCore connection.""" + if self._reconnect_attempts >= self._max_reconnect_attempts: + self.logger.error("Max MeshCore reconnection attempts reached") + await self._send_status_update( + ComponentStatus.ERROR, "max_reconnect_attempts" + ) + return + + self._reconnect_attempts += 1 + self.logger.warning( + f"Starting MeshCore recovery (attempt " + f"{self._reconnect_attempts}/{self._max_reconnect_attempts})" + ) + + # Update status + await self._send_status_update(ComponentStatus.DISCONNECTED, "reconnecting") + + try: + # Stop existing connection + if self.meshcore: + try: + await self.meshcore.stop_auto_message_fetching() + await self.meshcore.disconnect() + except Exception as e: + self.logger.debug(f"Error stopping old MeshCore connection: {e}") + + # Wait before attempting reconnection with exponential backoff + delay = min(2 ** (self._reconnect_attempts - 1), 300) + self.logger.info(f"Waiting {delay}s before MeshCore reconnection") + await asyncio.sleep(delay) + + # Re-setup connection + await self._setup_connection() + self._reconnect_attempts = 0 + self.logger.info("MeshCore connection recovery successful") + + except Exception as e: + self.logger.error( + f"MeshCore recovery attempt {self._reconnect_attempts} failed: {e}" + ) + await self._send_status_update( + ComponentStatus.ERROR, f"recovery_failed: {e}" + ) + + if self._reconnect_attempts < self._max_reconnect_attempts: + # Schedule retry + retry_delay = min(2**self._reconnect_attempts, 300) + self.logger.info(f"Scheduling MeshCore retry in {retry_delay}s") + await asyncio.sleep(retry_delay) + if self._running: + asyncio.create_task(self._recover_connection()) + else: + self.logger.error("🚨 MeshCore recovery failed permanently") + + def _on_meshcore_event(self, event_data: Any) -> None: + """Handle MeshCore events and forward them to MQTT.""" + try: + self.update_activity() + + # Log the event for debugging + event_type_name = getattr(event_data, "type", "UNKNOWN") + event_name = ( + str(event_type_name).split(".")[-1] if event_type_name else "UNKNOWN" + ) + + # Add extra logging for connection events to help debug + if event_name in ["CONNECTED", "DISCONNECTED"]: + self.logger.info(f"MeshCore {event_name} event received: {event_data}") + + # Create message for MQTT worker + message = Message.create( + message_type=MessageType.MESHCORE_EVENT, + source=self.component_name, + target="mqtt", + payload={"event_data": event_data, "timestamp": time.time()}, + ) + + # Send to message bus (non-blocking) + asyncio.create_task(self.message_bus.send_message(message)) + + except Exception as e: + self.logger.error(f"Error processing MeshCore event: {e}") + + def _on_no_more_msgs(self, event_data: Any) -> None: + """Handle NO_MORE_MSGS events.""" + self.logger.debug(f"Received NO_MORE_MSGS event: {event_data}") + self._auto_fetch_running = False + self.update_activity() + + async def _send_status_update(self, status: ComponentStatus, details: str) -> None: + """Send status update to other components.""" + self.message_bus.update_component_status(self.component_name, status) + + message = Message.create( + message_type=MessageType.MESHCORE_STATUS, + source=self.component_name, + target="mqtt", + payload={ + "status": status.value, + "details": details, + "connected": self._connected, + "timestamp": time.time(), + }, + ) + await self.message_bus.send_message(message) + + def update_activity(self) -> None: + """Update the last activity timestamp.""" + self._last_activity = time.time() + + def _is_stale(self, timeout_seconds: int = 300) -> bool: + """Check if connection appears stale.""" + if not self._last_activity: + return False + return time.time() - self._last_activity > timeout_seconds + + def serialize_to_json(self, data: Any) -> str: + """Safely serialize any data to JSON string.""" + import json + from datetime import datetime, timezone + + try: + # Handle common data types + if isinstance(data, (dict, list, str, int, float, bool)) or data is None: + return json.dumps(data, ensure_ascii=False) + + # Handle objects with custom serialization + if hasattr(data, "__dict__"): + obj_dict = { + key: value + for key, value in data.__dict__.items() + if not key.startswith("_") + } + if obj_dict: + return json.dumps(obj_dict, ensure_ascii=False, default=str) + + # Handle iterables + if hasattr(data, "__iter__") and not isinstance(data, (str, bytes)): + try: + return json.dumps(list(data), ensure_ascii=False, default=str) + except (TypeError, ValueError): + pass + + # Fallback: structured JSON with metadata + return json.dumps( + { + "type": type(data).__name__, + "value": str(data), + "timestamp": datetime.now(timezone.utc).isoformat(), + }, + ensure_ascii=False, + ) + + except Exception as e: + self.logger.warning(f"Failed to serialize data to JSON: {e}") + return json.dumps( + { + "error": f"Serialization failed: {str(e)}", + "raw_value": str(data)[:1000], + "type": type(data).__name__, + "timestamp": datetime.now(timezone.utc).isoformat(), + }, + ensure_ascii=False, + ) diff --git a/meshcore_mqtt/message_queue.py b/meshcore_mqtt/message_queue.py new file mode 100644 index 0000000..fed87c3 --- /dev/null +++ b/meshcore_mqtt/message_queue.py @@ -0,0 +1,305 @@ +"""Message queue system for inbox/outbox communication between threads.""" + +import asyncio +import logging +import time +from dataclasses import dataclass +from enum import Enum +from typing import Any, Dict, Optional + + +class MessageType(Enum): + """Types of messages that can be sent between components.""" + + # MeshCore to MQTT + MESHCORE_MESSAGE = "meshcore_message" + MESHCORE_STATUS = "meshcore_status" + MESHCORE_EVENT = "meshcore_event" + + # MQTT to MeshCore + MQTT_COMMAND = "mqtt_command" + MQTT_STATUS = "mqtt_status" + + # Control messages + SHUTDOWN = "shutdown" + HEALTH_CHECK = "health_check" + + +class ComponentStatus(Enum): + """Status of a component.""" + + STARTING = "starting" + RUNNING = "running" + CONNECTED = "connected" + DISCONNECTED = "disconnected" + ERROR = "error" + STOPPING = "stopping" + STOPPED = "stopped" + + +@dataclass +class Message: + """A message in the inbox/outbox system.""" + + id: str + message_type: MessageType + source: str + target: str + payload: Any + timestamp: float + metadata: Optional[Dict[str, Any]] = None + + @classmethod + def create( + cls, + message_type: MessageType, + source: str, + target: str, + payload: Any, + metadata: Optional[Dict[str, Any]] = None, + ) -> "Message": + """Create a new message with auto-generated ID and timestamp.""" + import uuid + + return cls( + id=uuid.uuid4().hex[:8], + message_type=message_type, + source=source, + target=target, + payload=payload, + timestamp=time.time(), + metadata=metadata or {}, + ) + + +class MessageQueue: + """Thread-safe message queue for component communication.""" + + def __init__(self, name: str, max_size: int = 1000) -> None: + """Initialize the message queue.""" + self.name = name + self.max_size = max_size + self.logger = logging.getLogger(f"{__name__}.{name}") + + # Use asyncio.Queue for thread-safe async operations + self._queue: asyncio.Queue[Message] = asyncio.Queue(maxsize=max_size) + self._dropped_messages = 0 + + async def put(self, message: Message, timeout: Optional[float] = None) -> bool: + """Put a message in the queue. Returns True if successful.""" + try: + if timeout is None: + await self._queue.put(message) + else: + await asyncio.wait_for(self._queue.put(message), timeout=timeout) + + self.logger.debug( + f"Queued {message.message_type.value} from {message.source} " + f"to {message.target} (id: {message.id})" + ) + return True + + except asyncio.TimeoutError: + self.logger.warning( + f"Timeout queueing message {message.id} " + f"({message.message_type.value})" + ) + return False + except asyncio.QueueFull: + self._dropped_messages += 1 + self.logger.warning( + f"Queue {self.name} full, dropping message {message.id} " + f"(total dropped: {self._dropped_messages})" + ) + return False + except Exception as e: + self.logger.error(f"Error queueing message {message.id}: {e}") + return False + + async def get(self, timeout: Optional[float] = None) -> Optional[Message]: + """Get a message from the queue. Returns None on timeout or error.""" + try: + if timeout is None: + message = await self._queue.get() + else: + message = await asyncio.wait_for(self._queue.get(), timeout=timeout) + + self.logger.debug( + f"Dequeued {message.message_type.value} from {message.source} " + f"to {message.target} (id: {message.id})" + ) + return message + + except asyncio.TimeoutError: + return None + except Exception as e: + self.logger.error(f"Error getting message from queue: {e}") + return None + + async def get_nowait(self) -> Optional[Message]: + """Get a message without waiting. Returns None if queue is empty.""" + try: + message = self._queue.get_nowait() + self.logger.debug( + f"Dequeued (nowait) {message.message_type.value} from " + f"{message.source} to {message.target} (id: {message.id})" + ) + return message + except asyncio.QueueEmpty: + return None + except Exception as e: + self.logger.error(f"Error getting message nowait: {e}") + return None + + def qsize(self) -> int: + """Get the approximate size of the queue.""" + return self._queue.qsize() + + def empty(self) -> bool: + """Check if the queue is empty.""" + return self._queue.empty() + + def full(self) -> bool: + """Check if the queue is full.""" + return self._queue.full() + + def stats(self) -> Dict[str, Any]: + """Get queue statistics.""" + return { + "name": self.name, + "size": self.qsize(), + "max_size": self.max_size, + "dropped_messages": self._dropped_messages, + "empty": self.empty(), + "full": self.full(), + } + + +class MessageBus: + """Central message bus coordinating inbox/outbox between components.""" + + def __init__(self) -> None: + """Initialize the message bus.""" + self.logger = logging.getLogger(__name__) + + # Component queues + self._queues: Dict[str, MessageQueue] = {} + + # Component status tracking + self._component_status: Dict[str, ComponentStatus] = {} + + # Running state + self._running = False + self._shutdown_event = asyncio.Event() + + def register_component(self, name: str, queue_size: int = 1000) -> MessageQueue: + """Register a component and get its message queue.""" + if name in self._queues: + self.logger.warning(f"Component {name} already registered") + return self._queues[name] + + queue = MessageQueue(name, max_size=queue_size) + self._queues[name] = queue + self._component_status[name] = ComponentStatus.STARTING + + self.logger.info(f"Registered component: {name}") + return queue + + def update_component_status(self, name: str, status: ComponentStatus) -> None: + """Update the status of a component.""" + old_status = self._component_status.get(name, ComponentStatus.STARTING) + self._component_status[name] = status + + if old_status != status: + self.logger.info(f"Component {name}: {old_status.value} -> {status.value}") + + def get_component_status(self, name: str) -> Optional[ComponentStatus]: + """Get the status of a component.""" + return self._component_status.get(name) + + async def send_message( + self, message: Message, timeout: Optional[float] = 1.0 + ) -> bool: + """Send a message to the target component.""" + target_queue = self._queues.get(message.target) + if not target_queue: + self.logger.error(f"Unknown target component: {message.target}") + return False + + return await target_queue.put(message, timeout=timeout) + + async def broadcast_message( + self, + message_type: MessageType, + source: str, + payload: Any, + exclude: Optional[list[str]] = None, + metadata: Optional[Dict[str, Any]] = None, + ) -> int: + """Broadcast a message to all components except excluded ones.""" + exclude = exclude or [] + exclude.append(source) # Don't send to self + + sent_count = 0 + for target in self._queues.keys(): + if target not in exclude: + message = Message.create( + message_type=message_type, + source=source, + target=target, + payload=payload, + metadata=metadata, + ) + if await self.send_message(message): + sent_count += 1 + + return sent_count + + async def shutdown(self) -> None: + """Shutdown the message bus and notify all components.""" + self.logger.info("Shutting down message bus") + self._running = False + + # Send shutdown messages to all components + await self.broadcast_message( + MessageType.SHUTDOWN, + source="message_bus", + payload={"reason": "shutdown_requested"}, + ) + + self._shutdown_event.set() + + def get_stats(self) -> Dict[str, Any]: + """Get statistics for all queues and components.""" + return { + "components": { + name: {"status": status.value, "queue": queue.stats()} + for name, (status, queue) in zip( + self._component_status.keys(), + [ + (self._component_status[name], self._queues[name]) + for name in self._component_status.keys() + ], + ) + }, + "total_components": len(self._queues), + "running": self._running, + } + + +# Global message bus instance +_message_bus: Optional[MessageBus] = None + + +def get_message_bus() -> MessageBus: + """Get the global message bus instance.""" + global _message_bus + if _message_bus is None: + _message_bus = MessageBus() + return _message_bus + + +def reset_message_bus() -> None: + """Reset the global message bus (mainly for testing).""" + global _message_bus + _message_bus = None diff --git a/meshcore_mqtt/mqtt_client.py b/meshcore_mqtt/mqtt_client.py new file mode 100644 index 0000000..665a29f --- /dev/null +++ b/meshcore_mqtt/mqtt_client.py @@ -0,0 +1,493 @@ +"""MQTT client management for MeshCore bridge.""" + +import asyncio +import logging +import time +import uuid +from typing import Any, Callable, Dict, Optional + +import paho.mqtt.client as mqtt + +from .config import Config + + +class MQTTClientManager: + """Manages MQTT client connection, reconnection, and message handling.""" + + def __init__(self, config: Config) -> None: + """Initialize MQTT client manager.""" + self.config = config + self.logger = logging.getLogger(__name__) + + # MQTT client + self.client: Optional[mqtt.Client] = None + + # Connection state + self._connected = False + self._reconnecting = False + self._reconnect_attempts = 0 + self._max_reconnect_attempts = 10 + self._last_activity: Optional[float] = None + + # Message handlers + self._message_handlers: Dict[str, Callable[[str, str], None]] = {} + self._default_command_handler: Optional[Callable[[str, str], None]] = None + + # Running state + self._running = False + + async def start(self) -> None: + """Start the MQTT client.""" + if self._running: + self.logger.warning("MQTT client is already running") + return + + self.logger.info("Starting MQTT client") + self._running = True + + # Create and configure client + self.client = self._create_client() + + # Connect with retry logic + try: + await self._connect_with_retry() + except Exception as e: + self.logger.error(f"Initial MQTT connection failed: {e}") + await self._recover_connection() + + # Start client loop + if self.client: + self.client.loop_start() + + async def stop(self) -> None: + """Stop the MQTT client.""" + if not self._running: + return + + self.logger.info("Stopping MQTT client") + self._running = False + + if self.client: + try: + # Stop the loop first + if hasattr(self.client, "_loop_started"): + self.client.loop_stop() + delattr(self.client, "_loop_started") + + # Then disconnect + if self.client.is_connected(): + self.client.disconnect() + except Exception as e: + self.logger.error(f"Error stopping MQTT client: {e}") + + self.logger.info("MQTT client stopped") + + def _create_client(self) -> mqtt.Client: + """Create and configure a new MQTT client.""" + # Generate a unique client ID + client_id = f"meshcore-mqtt-{uuid.uuid4().hex[:8]}" + self.logger.debug(f"Using MQTT client ID: {client_id}") + + client = mqtt.Client( + callback_api_version=mqtt.CallbackAPIVersion.VERSION2, + client_id=client_id, + clean_session=True, + reconnect_on_failure=True, + ) + + # Set up callbacks + client.on_connect = self._on_connect + client.on_disconnect = self._on_disconnect # type: ignore + client.on_message = self._on_message + client.on_publish = self._on_publish + client.on_log = self._on_log + + # Set authentication if provided + if self.config.mqtt.username and self.config.mqtt.password: + client.username_pw_set(self.config.mqtt.username, self.config.mqtt.password) + + # Configure TLS if enabled + if self.config.mqtt.tls_enabled: + self._configure_tls(client) + + # Set connection parameters + client.keepalive = 60 + client.max_inflight_messages_set(1) + client.max_queued_messages_set(100) + client.reconnect_delay_set(min_delay=1, max_delay=30) + + return client + + def _configure_tls(self, client: mqtt.Client) -> None: + """Configure TLS settings for MQTT client.""" + self.logger.info("Configuring MQTT TLS connection") + + try: + # Use paho-mqtt's tls_set() method + # For simple TLS (like Let's Encrypt), no arguments needed + if ( + not self.config.mqtt.tls_ca_cert + and not self.config.mqtt.tls_client_cert + and not self.config.mqtt.tls_client_key + ): + # Simple TLS setup (e.g., Let's Encrypt) + client.tls_set() + self.logger.info("Using default TLS configuration") + else: + # Custom certificate setup + client.tls_set( + ca_certs=self.config.mqtt.tls_ca_cert, + certfile=self.config.mqtt.tls_client_cert, + keyfile=self.config.mqtt.tls_client_key, + ) + self.logger.info("Using custom TLS certificates") + + # Handle insecure mode + if self.config.mqtt.tls_insecure: + self.logger.warning("TLS certificate verification disabled") + client.tls_insecure_set(True) + + self.logger.info("MQTT TLS configuration completed successfully") + + except Exception as e: + self.logger.error(f"Failed to configure MQTT TLS: {e}") + raise RuntimeError(f"TLS configuration failed: {e}") + + async def _connect_with_retry(self, max_retries: int = 5) -> None: + """Connect to MQTT broker with retry logic.""" + for attempt in range(max_retries): + try: + if self.client: + await asyncio.get_event_loop().run_in_executor( + None, + lambda: self.client.connect( # type: ignore + self.config.mqtt.broker, self.config.mqtt.port, 60 + ), + ) + self.logger.info( + f"Connected to MQTT broker on attempt {attempt + 1}" + ) + return + except Exception as e: + self.logger.warning( + f"MQTT connection attempt {attempt + 1} failed: {e}" + ) + if attempt < max_retries - 1: + delay = min(2**attempt, 30) + self.logger.info(f"Retrying MQTT connection in {delay} seconds") + await asyncio.sleep(delay) + else: + raise RuntimeError( + f"Failed to connect to MQTT broker after " + f"{max_retries} attempts: {e}" + ) + + async def _recover_connection(self) -> None: + """Recover MQTT connection with complete client recreation.""" + if self._reconnecting: + self.logger.debug("MQTT recovery already in progress") + return + + if self._reconnect_attempts >= self._max_reconnect_attempts: + self.logger.error("Max MQTT reconnection attempts reached") + return + + self._reconnecting = True + self._reconnect_attempts += 1 + + self.logger.warning( + f"Starting MQTT recovery (attempt " + f"{self._reconnect_attempts}/{self._max_reconnect_attempts})" + ) + + try: + # Destroy old client + await self._destroy_client() + + # Wait with exponential backoff + delay = min(2**self._reconnect_attempts, 30) + self.logger.info(f"Waiting {delay}s before MQTT reconnection") + await asyncio.sleep(delay) + + # Create fresh client and connection + await self._create_fresh_connection() + + # Success - reset counters + self._reconnect_attempts = 0 + self._reconnecting = False + self._connected = True + self._last_activity = time.time() + + self.logger.info("MQTT connection recovery successful") + + except Exception as e: + self.logger.error( + f"MQTT recovery attempt {self._reconnect_attempts} failed: {e}" + ) + self._reconnecting = False + + # Schedule retry if we haven't hit max attempts + if self._reconnect_attempts < self._max_reconnect_attempts: + retry_delay = min(5 * self._reconnect_attempts, 60) + self.logger.info(f"Scheduling MQTT retry in {retry_delay}s") + await asyncio.sleep(retry_delay) + if self._running: + asyncio.create_task(self._recover_connection()) + else: + self.logger.error("🚨 MQTT recovery failed permanently") + + async def _destroy_client(self) -> None: + """Destroy the existing MQTT client.""" + if not self.client: + return + + self.logger.debug("Destroying old MQTT client") + + try: + if hasattr(self.client, "_loop_started"): + self.client.loop_stop() + delattr(self.client, "_loop_started") + + if self.client.is_connected(): + self.client.disconnect() + + # Remove callbacks + self.client.on_connect = None + self.client.on_disconnect = None + self.client.on_message = None + self.client.on_publish = None + self.client.on_log = None + + except Exception as e: + self.logger.debug(f"Error during MQTT client destruction: {e}") + finally: + self.client = None + self._connected = False + + async def _create_fresh_connection(self) -> None: + """Create fresh client and establish connection.""" + self.logger.info("Creating fresh MQTT client") + + # Create and configure new client + self.client = self._create_client() + # We handle reconnection manually + + # Connect + self.logger.debug( + f"Connecting to {self.config.mqtt.broker}:{self.config.mqtt.port}" + ) + + if self.client: + await asyncio.get_event_loop().run_in_executor( + None, + lambda: self.client.connect( # type: ignore + self.config.mqtt.broker, self.config.mqtt.port, 60 + ), + ) + + # Start client loop + self.client.loop_start() + + # Wait for connection + await asyncio.sleep(2) + + if not self.client.is_connected(): + raise RuntimeError("MQTT client failed to connect") + + self.logger.info("Fresh MQTT client connected") + + def _on_connect( + self, + client: mqtt.Client, + userdata: Any, + flags: Dict[str, Any], + rc: int, + properties: Any = None, + ) -> None: + """Handle MQTT connection.""" + if rc == 0: + self.logger.info("Connected to MQTT broker") + self._connected = True + self._last_activity = time.time() + + # Subscribe to command topics + command_topic = f"{self.config.mqtt.topic_prefix}/command/+" + client.subscribe(command_topic, self.config.mqtt.qos) + self.logger.info(f"Subscribed to MQTT topic: {command_topic}") + else: + self.logger.error(f"Failed to connect to MQTT broker: {rc}") + self._connected = False + + def _on_disconnect( + self, + client: mqtt.Client, + userdata: Any, + flags: Dict[str, Any], + rc: int, + properties: Any = None, + ) -> None: + """Handle MQTT disconnection.""" + self._connected = False + if rc != 0: + self.logger.warning( + f"🔴 Unexpected MQTT disconnection: {mqtt.error_string(rc)} (code: {rc})" + ) + if self._running and not self._reconnecting: + self.logger.info("Triggering MQTT recovery from disconnect callback") + asyncio.create_task(self._recover_connection()) + else: + self.logger.info("MQTT client disconnected cleanly") + + def _on_message( + self, client: mqtt.Client, userdata: Any, message: mqtt.MQTTMessage + ) -> None: + """Handle incoming MQTT messages.""" + try: + topic_parts = message.topic.split("/") + if len(topic_parts) >= 3 and topic_parts[1] == "command": + command_type = topic_parts[2] + payload = message.payload.decode("utf-8") + + self.logger.info(f"Received MQTT command: {command_type} = {payload}") + + # Call registered handler if available + if command_type in self._message_handlers: + self._message_handlers[command_type](command_type, payload) + elif self._default_command_handler: + self._default_command_handler(command_type, payload) + else: + self.logger.warning( + f"No handler registered for command: {command_type}" + ) + + except Exception as e: + self.logger.error(f"Error processing MQTT message: {e}") + + def _on_publish( + self, + client: mqtt.Client, + userdata: Any, + mid: int, + reason_codes: Any = None, + properties: Any = None, + ) -> None: + """Handle MQTT publish confirmation.""" + self.logger.debug(f"MQTT message published: {mid}") + + def _on_log(self, client: mqtt.Client, userdata: Any, level: int, buf: str) -> None: + """Handle MQTT logging.""" + if level == mqtt.MQTT_LOG_DEBUG: + self.logger.debug(f"MQTT: {buf}") + elif level == mqtt.MQTT_LOG_INFO: + self.logger.info(f"MQTT: {buf}") + elif level == mqtt.MQTT_LOG_NOTICE: + self.logger.info(f"MQTT: {buf}") + elif level == mqtt.MQTT_LOG_WARNING: + self.logger.warning(f"MQTT: {buf}") + elif level == mqtt.MQTT_LOG_ERR: + self.logger.error(f"MQTT: {buf}") + else: + self.logger.debug(f"MQTT ({level}): {buf}") + + def register_command_handler( + self, command_type: str, handler: Callable[[str, str], None] + ) -> None: + """Register a handler for MQTT commands.""" + if command_type == "*": + # Special case: register handler for all command types + self._default_command_handler = handler + else: + self._message_handlers[command_type] = handler + self.logger.debug(f"Registered handler for command: {command_type}") + + def publish( + self, + topic: str, + payload: str, + qos: Optional[int] = None, + retain: Optional[bool] = None, + ) -> bool: + """Publish message to MQTT broker.""" + if not self.client: + self.logger.error("MQTT client not initialized") + return False + + try: + if not self.client.is_connected(): + self.logger.warning( + f"MQTT client not connected, skipping publish to {topic}" + ) + if self._running: + self.logger.debug( + "Triggering MQTT reconnection from publish method" + ) + asyncio.create_task(self._recover_connection()) + return False + + qos = qos if qos is not None else self.config.mqtt.qos + retain = retain if retain is not None else self.config.mqtt.retain + + self.logger.debug( + f"Publishing to MQTT: topic={topic}, qos={qos}, retain={retain}, " + f"payload_length={len(payload)}" + ) + + result = self.client.publish(topic, payload, qos=qos, retain=retain) + + if result.rc == mqtt.MQTT_ERR_SUCCESS: + self.logger.info(f"Successfully published to MQTT topic: {topic}") + self._last_activity = time.time() + if qos > 0: + result.wait_for_publish(timeout=5.0) + return True + elif result.rc == mqtt.MQTT_ERR_NO_CONN: + self.logger.warning(f"MQTT not connected while publishing to {topic}") + if self._running: + asyncio.create_task(self._recover_connection()) + return False + else: + self.logger.error( + f"Failed to publish to MQTT topic {topic}: " + f"{mqtt.error_string(result.rc)} ({result.rc})" + ) + return False + + except (ConnectionError, OSError, BrokenPipeError) as e: + self.logger.error(f"Connection error during MQTT publish to {topic}: {e}") + if self._running: + self.logger.info("Triggering MQTT reconnection due to connection error") + asyncio.create_task(self._recover_connection()) + return False + except Exception as e: + self.logger.error( + f"Unexpected exception during MQTT publish to {topic}: {e}" + ) + return False + + def is_connected(self) -> bool: + """Check if MQTT client is connected.""" + return ( + self._connected and self.client is not None and self.client.is_connected() + ) + + def is_stale(self, timeout_seconds: int = 300) -> bool: + """Check if connection appears stale.""" + if not self._last_activity: + return False + return time.time() - self._last_activity > timeout_seconds + + async def health_check(self) -> bool: + """Perform health check and trigger recovery if needed.""" + if not self.is_connected(): + if self._connected and not self._reconnecting: + self.logger.warning("🔴 MQTT connection lost, starting recovery") + self._connected = False + asyncio.create_task(self._recover_connection()) + return False + + if self.is_stale(): + self.logger.warning("MQTT connection appears stale, forcing reconnection") + asyncio.create_task(self._recover_connection()) + return False + + return True diff --git a/meshcore_mqtt/mqtt_worker.py b/meshcore_mqtt/mqtt_worker.py new file mode 100644 index 0000000..67c1a0c --- /dev/null +++ b/meshcore_mqtt/mqtt_worker.py @@ -0,0 +1,832 @@ +"""Independent MQTT worker with inbox/outbox message handling.""" + +import asyncio +import json +import logging +import time +import uuid +from typing import Any, Dict, Optional + +import paho.mqtt.client as mqtt + +from .config import Config +from .message_queue import ( + ComponentStatus, + Message, + MessageBus, + MessageQueue, + MessageType, + get_message_bus, +) + + +class MQTTWorker: + """Independent MQTT worker managing broker connection and messaging.""" + + def __init__(self, config: Config) -> None: + """Initialize MQTT worker.""" + self.config = config + self.logger = logging.getLogger(__name__) + + # Component identification + self.component_name = "mqtt" + + # Message bus + self.message_bus: MessageBus = get_message_bus() + self.inbox: MessageQueue = self.message_bus.register_component( + self.component_name, queue_size=1000 + ) + + # MQTT client + self.client: Optional[mqtt.Client] = None + + # Connection state + self._connected = False + self._reconnecting = False + self._reconnect_attempts = 0 + self._max_reconnect_attempts = 10 + self._last_activity: Optional[float] = None + + # Worker state + self._running = False + self._shutdown_event = asyncio.Event() + self._tasks: list[asyncio.Task[Any]] = [] + self._event_loop: Optional[asyncio.AbstractEventLoop] = None + + async def start(self) -> None: + """Start the MQTT worker.""" + if self._running: + self.logger.warning("MQTT worker is already running") + return + + self.logger.info("Starting MQTT worker") + self._running = True + + # Capture the current event loop for use in callbacks + self._event_loop = asyncio.get_running_loop() + + # Update status + self.message_bus.update_component_status( + self.component_name, ComponentStatus.STARTING + ) + + try: + # Setup MQTT connection + await self._setup_connection() + + # Start worker tasks + tasks = [ + asyncio.create_task(self._message_processor(), name="mqtt_processor"), + asyncio.create_task(self._health_monitor(), name="mqtt_health"), + ] + self._tasks.extend(tasks) + + # Update status to running + self.message_bus.update_component_status( + self.component_name, ComponentStatus.RUNNING + ) + + self.logger.info("MQTT worker started successfully") + + # Wait for shutdown + await self._shutdown_event.wait() + + except Exception as e: + self.logger.error(f"Error starting MQTT worker: {e}") + self.message_bus.update_component_status( + self.component_name, ComponentStatus.ERROR + ) + raise + finally: + await self.stop() + + async def stop(self) -> None: + """Stop the MQTT worker.""" + if not self._running: + return + + self.logger.info("Stopping MQTT worker") + self.message_bus.update_component_status( + self.component_name, ComponentStatus.STOPPING + ) + + self._running = False + self._shutdown_event.set() + + # Cancel all tasks + for task in self._tasks: + if not task.done(): + task.cancel() + + if self._tasks: + await asyncio.gather(*self._tasks, return_exceptions=True) + self._tasks.clear() + + # Stop MQTT connection + if self.client: + try: + if hasattr(self.client, "_loop_started"): + self.client.loop_stop() + delattr(self.client, "_loop_started") + + if self.client.is_connected(): + self.client.disconnect() + except Exception as e: + self.logger.error(f"Error stopping MQTT client: {e}") + + self.message_bus.update_component_status( + self.component_name, ComponentStatus.STOPPED + ) + self.logger.info("MQTT worker stopped") + + async def _setup_connection(self) -> None: + """Set up MQTT connection.""" + self.logger.info("Setting up MQTT connection") + + # Create and configure client + self.client = self._create_client() + + # Connect with retry logic + try: + await self._connect_with_retry() + # Note: Don't set _connected=True here - wait for _on_connect callback + # Note: Don't send CONNECTED status here - wait for _on_connect callback + + except Exception as e: + await self._send_status_update( + ComponentStatus.ERROR, f"connection_failed: {e}" + ) + raise RuntimeError(f"Failed to connect to MQTT broker: {e}") + + # Start client loop + if self.client: + self.client.loop_start() + + def _create_client(self) -> mqtt.Client: + """Create and configure a new MQTT client.""" + # Generate a unique client ID + client_id = f"meshcore-mqtt-{uuid.uuid4().hex[:8]}" + self.logger.debug(f"Using MQTT client ID: {client_id}") + + client = mqtt.Client( + callback_api_version=mqtt.CallbackAPIVersion.VERSION2, + client_id=client_id, + clean_session=True, + reconnect_on_failure=True, + ) + + # Set up callbacks + client.on_connect = self._on_connect + client.on_disconnect = self._on_disconnect # type: ignore + client.on_message = self._on_message + client.on_publish = self._on_publish + client.on_log = self._on_log + + # Set authentication if provided + if self.config.mqtt.username and self.config.mqtt.password: + client.username_pw_set(self.config.mqtt.username, self.config.mqtt.password) + + # Configure TLS if enabled + if self.config.mqtt.tls_enabled: + self._configure_tls(client) + + # Set connection parameters + client.keepalive = 60 + client.max_inflight_messages_set(1) + client.max_queued_messages_set(100) + client.reconnect_delay_set(min_delay=1, max_delay=30) + + return client + + def _configure_tls(self, client: mqtt.Client) -> None: + """Configure TLS settings for MQTT client.""" + self.logger.info("Configuring MQTT TLS connection") + + try: + if ( + not self.config.mqtt.tls_ca_cert + and not self.config.mqtt.tls_client_cert + and not self.config.mqtt.tls_client_key + ): + # Simple TLS setup (e.g., Let's Encrypt) + client.tls_set() + self.logger.info("Using default TLS configuration") + else: + # Custom certificate setup + client.tls_set( + ca_certs=self.config.mqtt.tls_ca_cert, + certfile=self.config.mqtt.tls_client_cert, + keyfile=self.config.mqtt.tls_client_key, + ) + self.logger.info("Using custom TLS certificates") + + # Handle insecure mode + if self.config.mqtt.tls_insecure: + self.logger.warning("TLS certificate verification disabled") + client.tls_insecure_set(True) + + self.logger.info("MQTT TLS configuration completed successfully") + + except Exception as e: + self.logger.error(f"Failed to configure MQTT TLS: {e}") + raise RuntimeError(f"TLS configuration failed: {e}") + + async def _connect_with_retry(self, max_retries: int = 5) -> None: + """Connect to MQTT broker with retry logic.""" + for attempt in range(max_retries): + try: + if self.client: + await asyncio.get_event_loop().run_in_executor( + None, + lambda: self.client.connect( # type: ignore + self.config.mqtt.broker, self.config.mqtt.port, 60 + ), + ) + self.logger.info( + f"MQTT connection initiated on attempt {attempt + 1}" + ) + return + except Exception as e: + self.logger.warning( + f"MQTT connection attempt {attempt + 1} failed: {e}" + ) + if attempt < max_retries - 1: + delay = min(2**attempt, 30) + self.logger.info(f"Retrying MQTT connection in {delay} seconds") + await asyncio.sleep(delay) + else: + raise RuntimeError( + f"Failed to connect to MQTT broker after {max_retries} " + f"attempts: {e}" + ) + + async def _message_processor(self) -> None: + """Process messages from the inbox.""" + self.logger.info("Starting MQTT message processor") + + while self._running: + try: + # Get message from inbox with timeout + message = await self.inbox.get(timeout=1.0) + if message is None: + continue + + await self._handle_inbox_message(message) + + except Exception as e: + self.logger.error(f"Error in message processor: {e}") + await asyncio.sleep(1) + + async def _handle_inbox_message(self, message: Message) -> None: + """Handle a message from the inbox.""" + self.logger.debug(f"Processing message: {message.message_type.value}") + + try: + if message.message_type == MessageType.MESHCORE_EVENT: + await self._handle_meshcore_event(message) + elif message.message_type == MessageType.MESHCORE_STATUS: + await self._handle_meshcore_status(message) + elif message.message_type == MessageType.HEALTH_CHECK: + await self._handle_health_check(message) + elif message.message_type == MessageType.SHUTDOWN: + self.logger.info("Received shutdown message") + self._shutdown_event.set() + else: + self.logger.warning( + f"Unknown message type: {message.message_type.value}" + ) + + except Exception as e: + self.logger.error(f"Error handling message {message.id}: {e}") + + async def _handle_meshcore_event(self, message: Message) -> None: + """Handle MeshCore event and publish to MQTT.""" + # Check if MQTT is connected before processing + if not self._connected: + self.logger.debug("MQTT not connected, queuing event for later") + # Could implement a retry queue here if needed + return + + event_payload = message.payload + event_data = event_payload.get("event_data") + + if not event_data: + self.logger.warning("Received empty MeshCore event data") + return + + try: + # Determine message type and create appropriate topic structure + topic = self._determine_mqtt_topic(event_data) + payload = self._serialize_to_json(event_data) + + # Publish to MQTT + self.logger.debug(f"Publishing MeshCore event to MQTT topic: {topic}") + success = await self._safe_mqtt_publish(topic, payload) + + if success: + self.logger.debug(f"Published MeshCore event to MQTT: {topic}") + else: + self.logger.warning( + f"Failed to publish MeshCore event to MQTT: {topic}" + ) + + except Exception as e: + self.logger.error(f"Error processing MeshCore event: {e}") + + def _determine_mqtt_topic(self, event_data: Any) -> str: + """Determine the appropriate MQTT topic for the event data.""" + try: + # Check if this is a message event + if hasattr(event_data, "payload") and isinstance(event_data.payload, dict): + message_data = event_data.payload + message_type = message_data.get("type", "") + + if message_type == "CHAN": + # Channel message - use channel identifier + channel_idx = message_data.get("channel_idx", 0) + return ( + f"{self.config.mqtt.topic_prefix}/message/channel/{channel_idx}" + ) + elif message_type == "PRIV": + # Direct message - use sender's public key prefix + pubkey_prefix = message_data.get("pubkey_prefix", "unknown") + return ( + f"{self.config.mqtt.topic_prefix}/message/direct/" + f"{pubkey_prefix}" + ) + + # Check event type for non-message events + event_type_name = getattr(event_data, "type", None) + if event_type_name: + event_name = str(event_type_name).split(".")[-1] # Get enum name + + if event_name in ["CONNECTED", "DISCONNECTED"]: + return f"{self.config.mqtt.topic_prefix}/events/connection" + elif event_name in ["LOGIN_SUCCESS", "LOGIN_FAILED"]: + return f"{self.config.mqtt.topic_prefix}/login" + elif event_name == "DEVICE_INFO": + return f"{self.config.mqtt.topic_prefix}/device_info" + elif event_name == "BATTERY": + return f"{self.config.mqtt.topic_prefix}/battery" + elif event_name == "NEW_CONTACT": + return f"{self.config.mqtt.topic_prefix}/new_contact" + elif event_name == "ADVERTISEMENT": + return f"{self.config.mqtt.topic_prefix}/advertisement" + + # Fallback for unknown event types + return f"{self.config.mqtt.topic_prefix}/event" + + except Exception as e: + self.logger.warning(f"Error determining MQTT topic: {e}") + return f"{self.config.mqtt.topic_prefix}/event" + + async def _handle_meshcore_status(self, message: Message) -> None: + """Handle MeshCore status update.""" + status_payload = message.payload + status = status_payload.get("status") + details = status_payload.get("details", "") + + self.logger.info(f"MeshCore status update: {status} - {details}") + + # Only publish to MQTT if we're connected + if not self._connected: + self.logger.debug("MQTT not connected, skipping status publish") + return + + # Publish status to MQTT + topic = f"{self.config.mqtt.topic_prefix}/status" + + if status == "connected": + payload = "connected" + elif status == "disconnected": + payload = "disconnected" + elif status == "error": + payload = f"error: {details}" + else: + payload = f"{status}: {details}" + + await self._safe_mqtt_publish(topic, payload, retain=True) + + async def _handle_health_check(self, message: Message) -> None: + """Handle health check request.""" + healthy = self._is_healthy() + + # Send health status back + response = Message.create( + message_type=MessageType.HEALTH_CHECK, + source=self.component_name, + target=message.source, + payload={ + "healthy": healthy, + "connected": self._connected, + "last_activity": self._last_activity, + }, + ) + await self.message_bus.send_message(response) + + async def _health_monitor(self) -> None: + """Monitor MQTT connection health.""" + self.logger.info("Starting MQTT health monitor") + + # Wait for initial connection to stabilize + await asyncio.sleep(5) + + while self._running: + try: + healthy = self._is_healthy() + + if not healthy and self._connected: + self.logger.warning("MQTT health check failed, attempting recovery") + await self._recover_connection() + + await asyncio.sleep(10) # Health check every 10 seconds + + except Exception as e: + self.logger.error(f"Error in health monitor: {e}") + await asyncio.sleep(30) + + def _is_healthy(self) -> bool: + """Check if MQTT connection is healthy.""" + if not self.client: + return False + + # Check basic connectivity + if not self.client.is_connected(): + return False + + # Check for stale connections + if self._is_stale(): + return False + + return True + + def _is_stale(self, timeout_seconds: int = 300) -> bool: + """Check if connection appears stale.""" + if not self._last_activity: + return False + return time.time() - self._last_activity > timeout_seconds + + async def _recover_connection(self) -> None: + """Recover MQTT connection with complete client recreation.""" + if self._reconnecting: + self.logger.debug("MQTT recovery already in progress") + return + + if self._reconnect_attempts >= self._max_reconnect_attempts: + self.logger.error("Max MQTT reconnection attempts reached") + await self._send_status_update( + ComponentStatus.ERROR, "max_reconnect_attempts" + ) + return + + self._reconnecting = True + self._reconnect_attempts += 1 + + self.logger.warning( + f"Starting MQTT recovery (attempt " + f"{self._reconnect_attempts}/{self._max_reconnect_attempts})" + ) + + # Update status + await self._send_status_update(ComponentStatus.DISCONNECTED, "reconnecting") + + try: + # Destroy old client + await self._destroy_client() + + # Wait with exponential backoff + delay = min(2**self._reconnect_attempts, 30) + self.logger.info(f"Waiting {delay}s before MQTT reconnection") + await asyncio.sleep(delay) + + # Create fresh client and connection + await self._create_fresh_connection() + + # Success - reset counters + self._reconnect_attempts = 0 + self._reconnecting = False + # Note: Don't set _connected=True here - wait for _on_connect callback + # Note: Don't send CONNECTED status here - wait for _on_connect callback + + self.logger.info("MQTT connection recovery initiated") + + except Exception as e: + self.logger.error( + f"MQTT recovery attempt {self._reconnect_attempts} failed: {e}" + ) + self._reconnecting = False + await self._send_status_update( + ComponentStatus.ERROR, f"recovery_failed: {e}" + ) + + # Schedule retry if we haven't hit max attempts + if self._reconnect_attempts < self._max_reconnect_attempts: + retry_delay = min(5 * self._reconnect_attempts, 60) + self.logger.info(f"Scheduling MQTT retry in {retry_delay}s") + await asyncio.sleep(retry_delay) + if self._running: + asyncio.create_task(self._recover_connection()) + else: + self.logger.error("🚨 MQTT recovery failed permanently") + + async def _destroy_client(self) -> None: + """Destroy the existing MQTT client.""" + if not self.client: + return + + self.logger.debug("Destroying old MQTT client") + + try: + if hasattr(self.client, "_loop_started"): + self.client.loop_stop() + delattr(self.client, "_loop_started") + + if self.client.is_connected(): + self.client.disconnect() + + # Remove callbacks + self.client.on_connect = None + self.client.on_disconnect = None + self.client.on_message = None + self.client.on_publish = None + self.client.on_log = None + + except Exception as e: + self.logger.debug(f"Error during MQTT client destruction: {e}") + finally: + self.client = None + self._connected = False + + async def _create_fresh_connection(self) -> None: + """Create fresh client and establish connection.""" + self.logger.info("Creating fresh MQTT client") + + # Create and configure new client + self.client = self._create_client() + + # Connect + self.logger.debug( + f"Connecting to {self.config.mqtt.broker}:{self.config.mqtt.port}" + ) + + if self.client: + await asyncio.get_event_loop().run_in_executor( + None, + lambda: self.client.connect( # type: ignore + self.config.mqtt.broker, self.config.mqtt.port, 60 + ), + ) + + # Start client loop + self.client.loop_start() + + # Wait for connection + await asyncio.sleep(2) + + if not self.client.is_connected(): + raise RuntimeError("MQTT client failed to connect") + + self.logger.info("Fresh MQTT client connected") + + async def _safe_mqtt_publish( + self, topic: str, payload: str, retain: bool = False + ) -> bool: + """Safely publish to MQTT broker.""" + if not self.client: + self.logger.error("MQTT client not initialized") + return False + + try: + if not self.client.is_connected(): + self.logger.warning( + f"MQTT client not connected, skipping publish to {topic}" + ) + if self._running: + asyncio.create_task(self._recover_connection()) + return False + + qos = self.config.mqtt.qos + retain = retain or self.config.mqtt.retain + + self.logger.debug( + f"Publishing to MQTT: topic={topic}, qos={qos}, retain={retain}, " + f"payload_length={len(payload)}" + ) + + result = self.client.publish(topic, payload, qos=qos, retain=retain) + + if result.rc == mqtt.MQTT_ERR_SUCCESS: + self._last_activity = time.time() + if qos > 0: + result.wait_for_publish(timeout=5.0) + return True + elif result.rc == mqtt.MQTT_ERR_NO_CONN: + self.logger.warning(f"MQTT not connected while publishing to {topic}") + if self._running: + asyncio.create_task(self._recover_connection()) + return False + else: + self.logger.error( + f"Failed to publish to MQTT topic {topic}: " + f"{mqtt.error_string(result.rc)} ({result.rc})" + ) + return False + + except (ConnectionError, OSError, BrokenPipeError) as e: + self.logger.error(f"Connection error during MQTT publish to {topic}: {e}") + if self._running: + asyncio.create_task(self._recover_connection()) + return False + except Exception as e: + self.logger.error( + f"Unexpected exception during MQTT publish to {topic}: {e}" + ) + return False + + def _on_connect( + self, + client: mqtt.Client, + userdata: Any, + flags: Dict[str, Any], + rc: int, + properties: Any = None, + ) -> None: + """Handle MQTT connection.""" + if rc == 0: + self.logger.info("Connected to MQTT broker") + self._connected = True + self._last_activity = time.time() + + # Update component status in message bus + self.message_bus.update_component_status( + self.component_name, ComponentStatus.CONNECTED + ) + + # Subscribe to command topics + command_topic = f"{self.config.mqtt.topic_prefix}/command/+" + client.subscribe(command_topic, self.config.mqtt.qos) + self.logger.info(f"Subscribed to MQTT topic: {command_topic}") + else: + self.logger.error(f"Failed to connect to MQTT broker: {rc}") + self._connected = False + self.message_bus.update_component_status( + self.component_name, ComponentStatus.ERROR + ) + + def _on_disconnect( + self, + client: mqtt.Client, + userdata: Any, + flags: Dict[str, Any], + rc: int, + properties: Any = None, + ) -> None: + """Handle MQTT disconnection.""" + self._connected = False + + # Update component status in message bus + self.message_bus.update_component_status( + self.component_name, ComponentStatus.DISCONNECTED + ) + + if rc != 0: + self.logger.warning( + f"🔴 Unexpected MQTT disconnection: {mqtt.error_string(rc)} (code: {rc})" + ) + if self._running and not self._reconnecting: + self.logger.info("Triggering MQTT recovery from disconnect callback") + asyncio.create_task(self._recover_connection()) + else: + self.logger.info("MQTT client disconnected cleanly") + + def _on_message( + self, client: mqtt.Client, userdata: Any, message: mqtt.MQTTMessage + ) -> None: + """Handle incoming MQTT messages.""" + try: + topic_parts = message.topic.split("/") + if len(topic_parts) >= 3 and topic_parts[1] == "command": + command_type = topic_parts[2] + payload = message.payload.decode("utf-8") + + self.logger.info(f"Received MQTT command: {command_type} = {payload}") + + # Forward command to MeshCore worker + self._forward_command_to_meshcore(command_type, payload) + + except Exception as e: + self.logger.error(f"Error processing MQTT message: {e}") + + def _forward_command_to_meshcore(self, command_type: str, payload: str) -> None: + """Forward MQTT command to MeshCore worker.""" + try: + # Parse command payload + if payload.startswith("{"): + command_data = json.loads(payload) + else: + command_data = {"data": payload} + + # Add command type to payload + command_data["command_type"] = command_type + + # Create message for MeshCore worker + message = Message.create( + message_type=MessageType.MQTT_COMMAND, + source=self.component_name, + target="meshcore", + payload=command_data, + ) + + # Send to message bus (non-blocking) + # Use run_coroutine_threadsafe since called from MQTT callback thread + if self._event_loop and not self._event_loop.is_closed(): + asyncio.run_coroutine_threadsafe( + self.message_bus.send_message(message), self._event_loop + ) + else: + self.logger.warning( + "No valid event loop available, dropping MQTT command" + ) + + except Exception as e: + self.logger.error(f"Error forwarding MQTT command to MeshCore: {e}") + + def _on_publish( + self, + client: mqtt.Client, + userdata: Any, + mid: int, + reason_codes: Any = None, + properties: Any = None, + ) -> None: + """Handle MQTT publish confirmation.""" + self.logger.debug(f"MQTT message published: {mid}") + + def _on_log(self, client: mqtt.Client, userdata: Any, level: int, buf: str) -> None: + """Handle MQTT logging.""" + if level == mqtt.MQTT_LOG_DEBUG: + self.logger.debug(f"MQTT: {buf}") + elif level == mqtt.MQTT_LOG_INFO: + self.logger.info(f"MQTT: {buf}") + elif level == mqtt.MQTT_LOG_NOTICE: + self.logger.info(f"MQTT: {buf}") + elif level == mqtt.MQTT_LOG_WARNING: + self.logger.warning(f"MQTT: {buf}") + elif level == mqtt.MQTT_LOG_ERR: + self.logger.error(f"MQTT: {buf}") + else: + self.logger.debug(f"MQTT ({level}): {buf}") + + async def _send_status_update(self, status: ComponentStatus, details: str) -> None: + """Send status update to other components.""" + self.message_bus.update_component_status(self.component_name, status) + + self.logger.info(f"MQTT status update: {status.value} - {details}") + + def _serialize_to_json(self, data: Any) -> str: + """Safely serialize any data to JSON string.""" + from datetime import datetime, timezone + + try: + # Handle common data types + if isinstance(data, (dict, list, str, int, float, bool)) or data is None: + return json.dumps(data, ensure_ascii=False) + + # Handle objects with custom serialization + if hasattr(data, "__dict__"): + obj_dict = { + key: value + for key, value in data.__dict__.items() + if not key.startswith("_") + } + if obj_dict: + return json.dumps(obj_dict, ensure_ascii=False, default=str) + + # Handle iterables + if hasattr(data, "__iter__") and not isinstance(data, (str, bytes)): + try: + return json.dumps(list(data), ensure_ascii=False, default=str) + except (TypeError, ValueError): + pass + + # Fallback: structured JSON with metadata + return json.dumps( + { + "type": type(data).__name__, + "value": str(data), + "timestamp": datetime.now(timezone.utc).isoformat(), + }, + ensure_ascii=False, + ) + + except Exception as e: + self.logger.warning(f"Failed to serialize data to JSON: {e}") + return json.dumps( + { + "error": f"Serialization failed: {str(e)}", + "raw_value": str(data)[:1000], + "type": type(data).__name__, + "timestamp": datetime.now(timezone.utc).isoformat(), + }, + ensure_ascii=False, + ) diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..3742b3c --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,131 @@ +[build-system] +requires = ["setuptools>=61.0", "wheel"] +build-backend = "setuptools.build_meta" + +[project] +name = "meshcore-mqtt" +version = "0.1.0" +description = "A bridge between MeshCore devices and MQTT brokers" +readme = "README.md" +authors = [{name = "MeshCore MQTT Bridge", email = "contact@example.com"}] +license = {text = "GPL-3.0"} +classifiers = [ + "Development Status :: 3 - Alpha", + "Intended Audience :: Developers", + "License :: OSI Approved :: GNU General Public License v3 (GPLv3)", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", +] +requires-python = ">=3.11" +dependencies = [ + "paho-mqtt>=2.0.0", + "click>=8.0.0", + "pydantic>=2.0.0", + "meshcore>=2.0.0", + "PyYAML>=6.0.0", +] + +[project.optional-dependencies] +dev = [ + "pytest>=7.0.0", + "pytest-asyncio>=1.0.0", + "pytest-cov>=4.0.0", + "flake8>=6.0.0", + "black>=23.0.0", + "mypy>=1.0.0", + "isort>=5.0.0", + "bandit>=1.7.0", + "safety>=2.0.0", + "pre-commit>=3.0.0", + "build>=0.10.0", + "twine>=4.0.0", + "types-PyYAML>=6.0.0", + "types-pyserial>=3.5.0", +] + +[project.scripts] +meshcore-mqtt = "meshcore_mqtt.main:main" + +[tool.black] +line-length = 88 +target-version = ['py311'] +include = '\.pyi?$' +extend-exclude = ''' +/( + # directories + \.eggs + | \.git + | \.hg + | \.mypy_cache + | \.tox + | \.venv + | build + | dist +)/ +''' + +[tool.mypy] +python_version = "3.11" +warn_return_any = true +warn_unused_configs = true +disallow_untyped_defs = true +disallow_incomplete_defs = true +check_untyped_defs = true +disallow_untyped_decorators = true +no_implicit_optional = true +warn_redundant_casts = true +warn_unused_ignores = true +warn_no_return = true +warn_unreachable = true +strict_equality = true + +[[tool.mypy.overrides]] +module = "meshcore.*" +ignore_missing_imports = true + +[[tool.mypy.overrides]] +module = "yaml.*" +ignore_missing_imports = true + +[[tool.mypy.overrides]] +module = "tests.*" +disallow_untyped_decorators = false + +[tool.pytest.ini_options] +testpaths = ["tests"] +python_files = ["test_*.py", "*_test.py"] +python_classes = ["Test*"] +python_functions = ["test_*"] +addopts = "-v --tb=short" +asyncio_mode = "auto" + +[tool.isort] +profile = "black" +multi_line_output = 3 +line_length = 88 +known_first_party = ["meshcore_mqtt"] +known_third_party = ["paho", "meshcore", "click", "pydantic", "yaml"] + +[tool.bandit] +exclude_dirs = ["tests", "venv", ".venv"] +skips = ["B101"] # Skip assert_used test + +[tool.coverage.run] +source = ["meshcore_mqtt"] +omit = [ + "*/tests/*", + "*/venv/*", + "*/.venv/*", +] + +[tool.coverage.report] +exclude_lines = [ + "pragma: no cover", + "def __repr__", + "raise AssertionError", + "raise NotImplementedError", + "if __name__ == .__main__.:", +] +show_missing = true +precision = 2 diff --git a/requirements-dev.txt b/requirements-dev.txt new file mode 100644 index 0000000..5ffe22f --- /dev/null +++ b/requirements-dev.txt @@ -0,0 +1,27 @@ +-r requirements.txt + +# Testing +pytest==8.4.1 +pytest-asyncio==1.1.0 +pytest-cov==6.0.0 + +# Code Quality +flake8==7.3.0 +black==25.1.0 +mypy==1.17.1 +isort==5.13.2 + +# Security +bandit==1.8.0 +# safety>=3.6.0 # Temporarily disabled due to pydantic version conflict + +# Development Tools +pre-commit==4.2.0 + +# Type Stubs +types-PyYAML==6.0.12.20250516 +types-pyserial==3.5.0.20240311 + +# Build Tools +build==1.2.2.post1 +twine==6.0.1 diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..91ec8d5 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,5 @@ +paho-mqtt==2.1.0 +click==8.2.2 +pydantic==2.11.7 +meshcore==2.0.4 +PyYAML==6.0.2 diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000..edd5fda --- /dev/null +++ b/tests/__init__.py @@ -0,0 +1 @@ +"""Tests for MeshCore MQTT Bridge.""" diff --git a/tests/test_bridge.py b/tests/test_bridge.py new file mode 100644 index 0000000..9a31aa4 --- /dev/null +++ b/tests/test_bridge.py @@ -0,0 +1,184 @@ +"""Tests for the MeshCore MQTT bridge.""" + +from unittest.mock import MagicMock, patch + +import pytest + +from meshcore_mqtt.bridge_coordinator import BridgeCoordinator +from meshcore_mqtt.config import Config, ConnectionType, MeshCoreConfig, MQTTConfig + + +@pytest.fixture +def test_config() -> Config: + """Create a test configuration.""" + return Config( + mqtt=MQTTConfig( + broker="localhost", + port=1883, + topic_prefix="test", + ), + meshcore=MeshCoreConfig( + connection_type=ConnectionType.TCP, + address="127.0.0.1", + port=12345, + ), + ) + + +@pytest.fixture +def bridge(test_config: Config) -> BridgeCoordinator: + """Create a bridge instance for testing.""" + return BridgeCoordinator(test_config) + + +class TestBridgeCoordinator: + """Test the MeshCore MQTT bridge.""" + + def test_init(self, bridge: BridgeCoordinator, test_config: Config) -> None: + """Test bridge initialization.""" + assert bridge.config == test_config + assert bridge.meshcore is None + assert bridge.connection_manager is None + assert bridge.mqtt_client is None + assert not bridge._running + assert bridge._worker_tasks == [] + + async def test_start_stop_basic(self, bridge: BridgeCoordinator) -> None: + """Test basic start/stop functionality without actual connections.""" + # Initial state + assert not bridge._running + assert bridge.mqtt_client is None + assert bridge.connection_manager is None + assert bridge.meshcore is None + + # Test that stop works even when not started + await bridge.stop() + assert not bridge._running + + def test_tcp_connection_config(self, test_config: Config) -> None: + """Test TCP connection configuration.""" + bridge = BridgeCoordinator(test_config) + assert bridge.config.meshcore.connection_type == ConnectionType.TCP + assert bridge.config.meshcore.address == "127.0.0.1" + assert bridge.config.meshcore.port == 12345 + + def test_serial_connection_config(self) -> None: + """Test serial connection configuration.""" + config = Config( + mqtt=MQTTConfig(broker="localhost"), + meshcore=MeshCoreConfig( + connection_type=ConnectionType.SERIAL, + address="/dev/ttyUSB0", + ), + ) + bridge = BridgeCoordinator(config) + assert bridge.config.meshcore.connection_type == ConnectionType.SERIAL + assert bridge.config.meshcore.address == "/dev/ttyUSB0" + assert bridge.config.meshcore.port is None + + def test_ble_connection_config(self) -> None: + """Test BLE connection configuration.""" + config = Config( + mqtt=MQTTConfig(broker="localhost"), + meshcore=MeshCoreConfig( + connection_type=ConnectionType.BLE, + address="AA:BB:CC:DD:EE:FF", + ), + ) + bridge = BridgeCoordinator(config) + assert bridge.config.meshcore.connection_type == ConnectionType.BLE + assert bridge.config.meshcore.address == "AA:BB:CC:DD:EE:FF" + assert bridge.config.meshcore.port is None + + @patch("meshcore_mqtt.mqtt_worker.mqtt.Client") + async def test_mqtt_setup( + self, mock_mqtt_client: MagicMock, bridge: BridgeCoordinator + ) -> None: + """Test MQTT setup.""" + # Setup mock + mock_mqtt_instance = MagicMock() + mock_mqtt_client.return_value = mock_mqtt_instance + mock_mqtt_instance.connect = MagicMock(return_value=0) + mock_mqtt_instance.is_connected = MagicMock(return_value=True) + mock_mqtt_instance.loop_start = MagicMock() + + # Initialize workers first + from meshcore_mqtt.mqtt_worker import MQTTWorker + + bridge.mqtt_worker = MQTTWorker(bridge.config) + + # Test MQTT setup + await bridge._setup_mqtt() + + # Verify MQTT client setup + assert bridge.mqtt_client is not None + + def test_mqtt_auth_setup(self) -> None: + """Test MQTT setup with authentication.""" + config = Config( + mqtt=MQTTConfig( + broker="localhost", + username="testuser", + password="testpass", + ), + meshcore=MeshCoreConfig( + connection_type=ConnectionType.TCP, + address="127.0.0.1", + port=12345, + ), + ) + bridge = BridgeCoordinator(config) + + # Test that authentication config is stored + assert bridge.config.mqtt.username == "testuser" + assert bridge.config.mqtt.password == "testpass" + + def test_mqtt_topic_generation(self, bridge: BridgeCoordinator) -> None: + """Test MQTT topic generation.""" + # Test message topic + expected_message_topic = f"{bridge.config.mqtt.topic_prefix}/message" + assert expected_message_topic == "test/message" + + # Test status topic + expected_status_topic = f"{bridge.config.mqtt.topic_prefix}/status" + assert expected_status_topic == "test/status" + + # Test command topic pattern + expected_command_topic = f"{bridge.config.mqtt.topic_prefix}/command/+" + assert expected_command_topic == "test/command/+" + + def test_advertisement_event_handler_mapping(self) -> None: + """Test that ADVERTISEMENT events are mapped to the correct handler.""" + config = Config( + mqtt=MQTTConfig(broker="localhost"), + meshcore=MeshCoreConfig( + connection_type=ConnectionType.TCP, + address="127.0.0.1", + port=12345, + events=["ADVERTISEMENT"], + ), + ) + + bridge = BridgeCoordinator(config) + + # Verify that ADVERTISEMENT is configured in events + assert "ADVERTISEMENT" in bridge.config.meshcore.events + + # In the new architecture, event handling is done by workers + # Verify the configuration is properly set up + assert bridge.config.meshcore.events is not None + + def test_advertisement_mqtt_topic(self) -> None: + """Test that ADVERTISEMENT events generate the correct MQTT topic.""" + config = Config( + mqtt=MQTTConfig(broker="localhost", topic_prefix="meshtest"), + meshcore=MeshCoreConfig( + connection_type=ConnectionType.TCP, address="127.0.0.1", port=12345 + ), + ) + + bridge = BridgeCoordinator(config) + + # Test advertisement topic generation + expected_topic = f"{bridge.config.mqtt.topic_prefix}/advertisement" + assert expected_topic == "meshtest/advertisement" diff --git a/tests/test_command_forwarding.py b/tests/test_command_forwarding.py new file mode 100644 index 0000000..32bf0cf --- /dev/null +++ b/tests/test_command_forwarding.py @@ -0,0 +1,222 @@ +"""Tests for MQTT command forwarding to MeshCore.""" + +from typing import Any +from unittest.mock import AsyncMock, MagicMock + +import pytest + +from meshcore_mqtt.config import Config, ConnectionType, MeshCoreConfig, MQTTConfig +from meshcore_mqtt.meshcore_client import MeshCoreClientManager + + +@pytest.fixture +def test_config() -> Config: + """Create a test configuration.""" + return Config( + mqtt=MQTTConfig(broker="localhost"), + meshcore=MeshCoreConfig( + connection_type=ConnectionType.TCP, + address="127.0.0.1", + port=12345, + ), + ) + + +@pytest.fixture +def meshcore_manager(test_config: Config) -> MeshCoreClientManager: + """Create a MeshCore manager instance for testing.""" + return MeshCoreClientManager(test_config) + + +class TestCommandForwarding: + """Test MQTT command forwarding to MeshCore.""" + + async def test_send_msg_command( + self, meshcore_manager: MeshCoreClientManager + ) -> None: + """Test send_msg command forwarding.""" + # Setup mock MeshCore instance + mock_meshcore = MagicMock() + mock_meshcore.commands = MagicMock() + mock_meshcore.commands.send_msg = AsyncMock() + meshcore_manager.meshcore = mock_meshcore + + # Test send_msg command + command_data = {"destination": "Alice", "message": "Hello!"} + await meshcore_manager.send_command("send_msg", command_data) + + # Verify the command was called + mock_meshcore.commands.send_msg.assert_called_once_with("Alice", "Hello!") + + async def test_device_query_command( + self, meshcore_manager: MeshCoreClientManager + ) -> None: + """Test device_query command forwarding.""" + # Setup mock MeshCore instance + mock_meshcore = MagicMock() + mock_meshcore.commands = MagicMock() + mock_meshcore.commands.send_device_query = AsyncMock() + meshcore_manager.meshcore = mock_meshcore + + # Test device_query command + await meshcore_manager.send_command("device_query", {}) + + # Verify the command was called + mock_meshcore.commands.send_device_query.assert_called_once() + + async def test_ping_command(self, meshcore_manager: MeshCoreClientManager) -> None: + """Test ping command forwarding.""" + # Setup mock MeshCore instance + mock_meshcore = MagicMock() + mock_meshcore.commands = MagicMock() + mock_meshcore.commands.ping = AsyncMock() + meshcore_manager.meshcore = mock_meshcore + + # Test ping command + command_data = {"destination": "node123"} + await meshcore_manager.send_command("ping", command_data) + + # Verify the command was called + mock_meshcore.commands.ping.assert_called_once_with("node123") + + async def test_set_name_command( + self, meshcore_manager: MeshCoreClientManager + ) -> None: + """Test set_name command forwarding.""" + # Setup mock MeshCore instance + mock_meshcore = MagicMock() + mock_meshcore.commands = MagicMock() + mock_meshcore.commands.set_name = AsyncMock() + meshcore_manager.meshcore = mock_meshcore + + # Test set_name command + command_data = {"name": "MyDevice"} + await meshcore_manager.send_command("set_name", command_data) + + # Verify the command was called + mock_meshcore.commands.set_name.assert_called_once_with("MyDevice") + + async def test_send_chan_msg_command( + self, meshcore_manager: MeshCoreClientManager + ) -> None: + """Test send_chan_msg command forwarding.""" + # Setup mock MeshCore instance + mock_meshcore = MagicMock() + mock_meshcore.commands = MagicMock() + mock_meshcore.commands.send_chan_msg = AsyncMock() + meshcore_manager.meshcore = mock_meshcore + + # Test send_chan_msg command + command_data = {"channel": 0, "message": "Hello channel!"} + await meshcore_manager.send_command("send_chan_msg", command_data) + + # Verify the command was called + mock_meshcore.commands.send_chan_msg.assert_called_once_with( + 0, "Hello channel!" + ) + + async def test_missing_required_fields( + self, meshcore_manager: MeshCoreClientManager + ) -> None: + """Test command validation for missing required fields.""" + # Setup mock MeshCore instance + mock_meshcore = MagicMock() + mock_meshcore.commands = MagicMock() + mock_meshcore.commands.send_msg = AsyncMock() + meshcore_manager.meshcore = mock_meshcore + + # Test send_msg without required fields + command_data = {"message": "Hello!"} # Missing destination + await meshcore_manager.send_command("send_msg", command_data) + + # Verify the command was NOT called due to validation + mock_meshcore.commands.send_msg.assert_not_called() + + # Setup mock for send_chan_msg + mock_meshcore.commands.send_chan_msg = AsyncMock() + + # Test send_chan_msg without required fields + command_data = {"message": "Hello channel!"} # Missing channel + await meshcore_manager.send_command("send_chan_msg", command_data) + + # Verify the command was NOT called due to validation + mock_meshcore.commands.send_chan_msg.assert_not_called() + + # Test send_chan_msg with None channel + command_data_none: dict[str, Any] = { + "channel": None, + "message": "Hello channel!", + } + await meshcore_manager.send_command("send_chan_msg", command_data_none) + + # Verify the command was NOT called due to validation + mock_meshcore.commands.send_chan_msg.assert_not_called() + + # Test send_chan_msg without message + command_data_no_msg: dict[str, Any] = {"channel": 0} # Missing message + await meshcore_manager.send_command("send_chan_msg", command_data_no_msg) + + # Verify the command was NOT called due to validation + mock_meshcore.commands.send_chan_msg.assert_not_called() + + async def test_unknown_command_type( + self, meshcore_manager: MeshCoreClientManager + ) -> None: + """Test handling of unknown command types.""" + # Setup mock MeshCore instance + mock_meshcore = MagicMock() + meshcore_manager.meshcore = mock_meshcore + + # Test unknown command + await meshcore_manager.send_command("unknown_command", {}) + + # Should not raise an exception, just log a warning + + async def test_no_meshcore_instance( + self, meshcore_manager: MeshCoreClientManager + ) -> None: + """Test command handling when MeshCore instance is None.""" + # Ensure meshcore is None + meshcore_manager.meshcore = None + + # Test command - should not raise exception + await meshcore_manager.send_command( + "send_msg", {"destination": "test", "message": "test"} + ) + + async def test_command_error_handling( + self, meshcore_manager: MeshCoreClientManager + ) -> None: + """Test error handling when MeshCore command fails.""" + # Setup mock MeshCore instance that raises an exception + mock_meshcore = MagicMock() + mock_meshcore.commands = MagicMock() + mock_meshcore.commands.send_msg = AsyncMock(side_effect=Exception("Test error")) + meshcore_manager.meshcore = mock_meshcore + + # Test command - should not raise exception, just log error + command_data = {"destination": "Alice", "message": "Hello!"} + await meshcore_manager.send_command("send_msg", command_data) + + # Verify the command was attempted + mock_meshcore.commands.send_msg.assert_called_once_with("Alice", "Hello!") + + async def test_activity_update_on_successful_command( + self, meshcore_manager: MeshCoreClientManager + ) -> None: + """Test that activity timestamp is updated on successful commands.""" + # Setup mock MeshCore instance + mock_meshcore = MagicMock() + mock_meshcore.commands = MagicMock() + mock_meshcore.commands.send_device_query = AsyncMock(return_value=None) + meshcore_manager.meshcore = mock_meshcore + + # Mock the update_activity method using setattr to avoid mypy error + mock_update_activity = MagicMock() + setattr(meshcore_manager, "update_activity", mock_update_activity) + + # Test device_query command (no result object) + await meshcore_manager.send_command("device_query", {}) + + # Verify activity was updated + mock_update_activity.assert_called_once() diff --git a/tests/test_config.py b/tests/test_config.py new file mode 100644 index 0000000..de23e47 --- /dev/null +++ b/tests/test_config.py @@ -0,0 +1,320 @@ +"""Tests for configuration system.""" + +import json +import tempfile +from pathlib import Path + +import pytest +import yaml +from pydantic import ValidationError + +from meshcore_mqtt.config import Config, ConnectionType, MeshCoreConfig, MQTTConfig + + +class TestMQTTConfig: + """Test MQTT configuration.""" + + def test_valid_config(self) -> None: + """Test valid MQTT configuration.""" + config = MQTTConfig(broker="localhost") + assert config.broker == "localhost" + assert config.port == 1883 + assert config.username is None + assert config.password is None + assert config.topic_prefix == "meshcore" + assert config.qos == 0 + assert config.retain is False + + def test_invalid_port(self) -> None: + """Test invalid port numbers.""" + with pytest.raises(ValidationError): + MQTTConfig(broker="localhost", port=0) + + with pytest.raises(ValidationError): + MQTTConfig(broker="localhost", port=65536) + + def test_invalid_qos(self) -> None: + """Test invalid QoS values.""" + with pytest.raises(ValidationError): + MQTTConfig(broker="localhost", qos=-1) + + with pytest.raises(ValidationError): + MQTTConfig(broker="localhost", qos=3) + + def test_tls_config_default(self) -> None: + """Test default TLS configuration.""" + config = MQTTConfig(broker="localhost") + assert config.tls_enabled is False + assert config.tls_ca_cert is None + assert config.tls_client_cert is None + assert config.tls_client_key is None + assert config.tls_insecure is False + + def test_tls_enabled(self) -> None: + """Test TLS enabled configuration.""" + config = MQTTConfig(broker="localhost", tls_enabled=True) + assert config.tls_enabled is True + + def test_tls_with_certificates(self) -> None: + """Test TLS configuration with certificates.""" + # Create temporary certificate files for testing + with tempfile.NamedTemporaryFile( + mode="w", suffix=".pem", delete=False + ) as ca_cert: + ca_cert.write("dummy ca cert") + ca_cert_path = ca_cert.name + + with tempfile.NamedTemporaryFile( + mode="w", suffix=".pem", delete=False + ) as client_cert: + client_cert.write("dummy client cert") + client_cert_path = client_cert.name + + with tempfile.NamedTemporaryFile( + mode="w", suffix=".key", delete=False + ) as client_key: + client_key.write("dummy client key") + client_key_path = client_key.name + + try: + config = MQTTConfig( + broker="localhost", + tls_enabled=True, + tls_ca_cert=ca_cert_path, + tls_client_cert=client_cert_path, + tls_client_key=client_key_path, + ) + assert config.tls_enabled is True + assert config.tls_ca_cert == ca_cert_path + assert config.tls_client_cert == client_cert_path + assert config.tls_client_key == client_key_path + finally: + # Clean up temporary files + Path(ca_cert_path).unlink() + Path(client_cert_path).unlink() + Path(client_key_path).unlink() + + def test_tls_invalid_certificate_path(self) -> None: + """Test TLS configuration with invalid certificate paths.""" + with pytest.raises(ValidationError): + MQTTConfig( + broker="localhost", + tls_enabled=True, + tls_ca_cert="/nonexistent/path/ca.pem", + ) + + +class TestMeshCoreConfig: + """Test MeshCore configuration.""" + + def test_valid_tcp_config(self) -> None: + """Test valid TCP configuration.""" + config = MeshCoreConfig( + connection_type=ConnectionType.TCP, address="192.168.1.100", port=12345 + ) + assert config.connection_type == ConnectionType.TCP + assert config.address == "192.168.1.100" + assert config.port == 12345 + assert config.timeout == 5 + + def test_valid_serial_config(self) -> None: + """Test valid serial configuration.""" + config = MeshCoreConfig( + connection_type=ConnectionType.SERIAL, address="/dev/ttyUSB0" + ) + assert config.connection_type == ConnectionType.SERIAL + assert config.address == "/dev/ttyUSB0" + assert config.port is None + assert config.baudrate == 115200 + + def test_valid_ble_config(self) -> None: + """Test valid BLE configuration.""" + config = MeshCoreConfig( + connection_type=ConnectionType.BLE, address="AA:BB:CC:DD:EE:FF" + ) + assert config.connection_type == ConnectionType.BLE + assert config.address == "AA:BB:CC:DD:EE:FF" + assert config.port is None + assert config.baudrate == 115200 + + def test_tcp_sets_default_port(self) -> None: + """Test that TCP connections get a default port.""" + config = MeshCoreConfig( + connection_type=ConnectionType.TCP, address="192.168.1.100", port=None + ) + assert config.port == 12345 + + def test_custom_baudrate(self) -> None: + """Test custom baudrate for serial connections.""" + config = MeshCoreConfig( + connection_type=ConnectionType.SERIAL, + address="/dev/ttyUSB0", + baudrate=9600, + ) + assert config.baudrate == 9600 + + def test_invalid_port(self) -> None: + """Test invalid port numbers.""" + with pytest.raises(ValidationError): + MeshCoreConfig( + connection_type=ConnectionType.TCP, address="192.168.1.100", port=0 + ) + + with pytest.raises(ValidationError): + MeshCoreConfig( + connection_type=ConnectionType.TCP, address="192.168.1.100", port=65536 + ) + + def test_invalid_timeout(self) -> None: + """Test invalid timeout values.""" + with pytest.raises(ValidationError): + MeshCoreConfig( + connection_type=ConnectionType.SERIAL, address="/dev/ttyUSB0", timeout=0 + ) + + +class TestConfig: + """Test main configuration.""" + + def test_valid_config(self) -> None: + """Test valid complete configuration.""" + config = Config( + mqtt=MQTTConfig(broker="localhost"), + meshcore=MeshCoreConfig( + connection_type=ConnectionType.TCP, address="192.168.1.100", port=12345 + ), + ) + assert config.mqtt.broker == "localhost" + assert config.meshcore.connection_type == ConnectionType.TCP + assert config.log_level == "INFO" + + def test_invalid_log_level(self) -> None: + """Test invalid log level.""" + with pytest.raises(ValidationError, match="log_level must be one of"): + Config( + mqtt=MQTTConfig(broker="localhost"), + meshcore=MeshCoreConfig( + connection_type=ConnectionType.TCP, + address="192.168.1.100", + port=12345, + ), + log_level="INVALID", + ) + + def test_log_level_case_normalization(self) -> None: + """Test that log levels are normalized to uppercase.""" + config = Config( + mqtt=MQTTConfig(broker="localhost"), + meshcore=MeshCoreConfig( + connection_type=ConnectionType.TCP, address="192.168.1.100", port=12345 + ), + log_level="debug", + ) + assert config.log_level == "DEBUG" + + def test_from_json_file(self) -> None: + """Test loading configuration from JSON file.""" + config_data = { + "mqtt": { + "broker": "test-broker", + "port": 1883, + "topic_prefix": "test", + "qos": 1, + "retain": True, + }, + "meshcore": { + "connection_type": "serial", + "address": "/dev/ttyUSB0", + "timeout": 10, + }, + "log_level": "DEBUG", + } + + with tempfile.NamedTemporaryFile(mode="w", suffix=".json", delete=False) as f: + json.dump(config_data, f) + temp_path = Path(f.name) + + try: + config = Config.from_file(temp_path) + assert config.mqtt.broker == "test-broker" + assert config.mqtt.port == 1883 + assert config.mqtt.topic_prefix == "test" + assert config.mqtt.qos == 1 + assert config.mqtt.retain is True + assert config.meshcore.connection_type == ConnectionType.SERIAL + assert config.meshcore.address == "/dev/ttyUSB0" + assert config.meshcore.timeout == 10 + assert config.log_level == "DEBUG" + finally: + temp_path.unlink() + + def test_from_yaml_file(self) -> None: + """Test loading configuration from YAML file.""" + config_data = { + "mqtt": { + "broker": "yaml-broker", + "port": 8883, + }, + "meshcore": { + "connection_type": "ble", + "address": "AA:BB:CC:DD:EE:FF", + }, + } + + with tempfile.NamedTemporaryFile(mode="w", suffix=".yaml", delete=False) as f: + yaml.dump(config_data, f) + temp_path = Path(f.name) + + try: + config = Config.from_file(temp_path) + assert config.mqtt.broker == "yaml-broker" + assert config.mqtt.port == 8883 + assert config.meshcore.connection_type == ConnectionType.BLE + assert config.meshcore.address == "AA:BB:CC:DD:EE:FF" + finally: + temp_path.unlink() + + def test_from_nonexistent_file(self) -> None: + """Test loading from non-existent file.""" + with pytest.raises(FileNotFoundError): + Config.from_file("/nonexistent/config.json") + + def test_from_invalid_json(self) -> None: + """Test loading from invalid JSON file.""" + with tempfile.NamedTemporaryFile(mode="w", suffix=".json", delete=False) as f: + f.write("invalid json {") + temp_path = Path(f.name) + + try: + with pytest.raises(ValueError, match="Invalid JSON configuration"): + Config.from_file(temp_path) + finally: + temp_path.unlink() + + def test_from_invalid_yaml(self) -> None: + """Test loading from invalid YAML file.""" + with tempfile.NamedTemporaryFile(mode="w", suffix=".yaml", delete=False) as f: + f.write("invalid: yaml: content: [") + temp_path = Path(f.name) + + try: + with pytest.raises(ValueError, match="Invalid YAML configuration"): + Config.from_file(temp_path) + finally: + temp_path.unlink() + + +class TestConnectionType: + """Test connection type enum.""" + + def test_valid_values(self) -> None: + """Test valid connection type values.""" + assert ConnectionType.SERIAL.value == "serial" + assert ConnectionType.BLE.value == "ble" + assert ConnectionType.TCP.value == "tcp" + + def test_from_string(self) -> None: + """Test creating connection type from string.""" + assert ConnectionType("serial") == ConnectionType.SERIAL + assert ConnectionType("ble") == ConnectionType.BLE + assert ConnectionType("tcp") == ConnectionType.TCP diff --git a/tests/test_configurable_events.py b/tests/test_configurable_events.py new file mode 100644 index 0000000..3a16732 --- /dev/null +++ b/tests/test_configurable_events.py @@ -0,0 +1,247 @@ +"""Tests for configurable MeshCore events functionality.""" + +import json +import tempfile +from pathlib import Path + +import pytest +import yaml +from pydantic import ValidationError + +from meshcore_mqtt.config import Config, ConnectionType, MeshCoreConfig + + +class TestConfigurableEvents: + """Test configurable MeshCore events feature.""" + + def test_default_events(self) -> None: + """Test that default events are properly configured.""" + config = MeshCoreConfig( + connection_type=ConnectionType.TCP, address="127.0.0.1", port=12345 + ) + + expected_events = [ + "CONTACT_MSG_RECV", + "CHANNEL_MSG_RECV", + "CONNECTED", + "DISCONNECTED", + "LOGIN_SUCCESS", + "LOGIN_FAILED", + "DEVICE_INFO", + "BATTERY", + "NEW_CONTACT", + "ADVERTISEMENT", + ] + + assert config.events == expected_events + + def test_custom_events_list(self) -> None: + """Test custom events list configuration.""" + custom_events = ["CONNECTED", "DISCONNECTED", "BATTERY"] + + config = MeshCoreConfig( + connection_type=ConnectionType.TCP, + address="127.0.0.1", + port=12345, + events=custom_events, + ) + + assert config.events == custom_events + + def test_valid_event_types(self) -> None: + """Test validation accepts valid event types.""" + valid_events = [ + "CONTACT_MSG_RECV", + "CHANNEL_MSG_RECV", + "CONNECTED", + "DISCONNECTED", + "LOGIN_SUCCESS", + "LOGIN_FAILED", + "MESSAGES_WAITING", + "DEVICE_INFO", + "BATTERY", + "NEW_CONTACT", + "NODE_LIST_CHANGED", + "CONFIG_CHANGED", + "TELEMETRY", + "POSITION", + "USER", + "ROUTING", + "ADMIN", + ] + + config = MeshCoreConfig( + connection_type=ConnectionType.TCP, + address="127.0.0.1", + port=12345, + events=valid_events, + ) + + assert config.events == valid_events + + def test_invalid_event_types(self) -> None: + """Test validation rejects invalid event types.""" + invalid_events = ["CONNECTED", "INVALID_EVENT", "ANOTHER_INVALID"] + + with pytest.raises(ValidationError, match="Invalid event types"): + MeshCoreConfig( + connection_type=ConnectionType.TCP, + address="127.0.0.1", + port=12345, + events=invalid_events, + ) + + def test_empty_events_list(self) -> None: + """Test empty events list is allowed.""" + config = MeshCoreConfig( + connection_type=ConnectionType.TCP, + address="127.0.0.1", + port=12345, + events=[], + ) + + assert config.events == [] + + def test_parse_events_string(self) -> None: + """Test parsing comma-separated event strings.""" + test_cases = [ + ( + "CONNECTED,DISCONNECTED,BATTERY", + ["CONNECTED", "DISCONNECTED", "BATTERY"], + ), + ( + "connected, disconnected , battery ", + ["CONNECTED", "DISCONNECTED", "BATTERY"], + ), + ("CONNECTED", ["CONNECTED"]), + ("", []), + (" ", []), + ] + + for input_str, expected in test_cases: + result = Config.parse_events_string(input_str) + assert result == expected + + def test_parse_events_string_with_empty_parts(self) -> None: + """Test parsing handles empty parts correctly.""" + result = Config.parse_events_string("CONNECTED,,DISCONNECTED,") + assert result == ["CONNECTED", "DISCONNECTED"] + + def test_events_from_json_config(self) -> None: + """Test loading events from JSON configuration file.""" + config_data = { + "mqtt": { + "broker": "localhost", + "port": 1883, + }, + "meshcore": { + "connection_type": "tcp", + "address": "127.0.0.1", + "port": 12345, + "events": ["CONNECTED", "DISCONNECTED", "BATTERY"], + }, + } + + with tempfile.NamedTemporaryFile(mode="w", suffix=".json", delete=False) as f: + json.dump(config_data, f) + temp_path = Path(f.name) + + try: + config = Config.from_file(temp_path) + assert config.meshcore.events == ["CONNECTED", "DISCONNECTED", "BATTERY"] + finally: + temp_path.unlink() + + def test_events_from_yaml_config(self) -> None: + """Test loading events from YAML configuration file.""" + config_data = { + "mqtt": { + "broker": "localhost", + "port": 1883, + }, + "meshcore": { + "connection_type": "tcp", + "address": "127.0.0.1", + "port": 12345, + "events": ["LOGIN_SUCCESS", "LOGIN_FAILED", "DEVICE_INFO"], + }, + } + + with tempfile.NamedTemporaryFile(mode="w", suffix=".yaml", delete=False) as f: + yaml.dump(config_data, f) + temp_path = Path(f.name) + + try: + config = Config.from_file(temp_path) + assert config.meshcore.events == [ + "LOGIN_SUCCESS", + "LOGIN_FAILED", + "DEVICE_INFO", + ] + finally: + temp_path.unlink() + + def test_events_from_environment_variable( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + """Test loading events from environment variable.""" + # Set up environment variables + monkeypatch.setenv("MQTT_BROKER", "localhost") + monkeypatch.setenv("MESHCORE_CONNECTION", "tcp") + monkeypatch.setenv("MESHCORE_ADDRESS", "127.0.0.1") + monkeypatch.setenv( + "MESHCORE_EVENTS", "CONNECTED,BATTERY,NEW_CONTACT,ADVERTISEMENT" + ) + + config = Config.from_env() + assert config.meshcore.events == [ + "CONNECTED", + "BATTERY", + "NEW_CONTACT", + "ADVERTISEMENT", + ] + + def test_events_environment_variable_fallback( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + """Test that default events are used when env var is not set.""" + # Set up required environment variables without MESHCORE_EVENTS + monkeypatch.setenv("MQTT_BROKER", "localhost") + monkeypatch.setenv("MESHCORE_CONNECTION", "tcp") + monkeypatch.setenv("MESHCORE_ADDRESS", "127.0.0.1") + + config = Config.from_env() + # Should use default events + expected_events = [ + "CONTACT_MSG_RECV", + "CHANNEL_MSG_RECV", + "CONNECTED", + "DISCONNECTED", + "LOGIN_SUCCESS", + "LOGIN_FAILED", + "DEVICE_INFO", + "BATTERY", + "NEW_CONTACT", + "ADVERTISEMENT", + ] + assert config.meshcore.events == expected_events + + def test_case_insensitive_event_parsing(self) -> None: + """Test that event parsing is case insensitive.""" + result = Config.parse_events_string("connected,Disconnected,BATTERY") + assert result == ["CONNECTED", "DISCONNECTED", "BATTERY"] + + def test_mixed_case_events_in_config(self) -> None: + """Test that mixed case events in config are normalized.""" + config = MeshCoreConfig( + connection_type=ConnectionType.TCP, + address="127.0.0.1", + port=12345, + events=["connected", "Disconnected", "BATTERY"], # Mixed case + ) + + # The validator should normalize these to uppercase + assert all(event.isupper() for event in config.events) + assert "CONNECTED" in config.events + assert "DISCONNECTED" in config.events + assert "BATTERY" in config.events diff --git a/tests/test_json_serialization.py b/tests/test_json_serialization.py new file mode 100644 index 0000000..c52881d --- /dev/null +++ b/tests/test_json_serialization.py @@ -0,0 +1,180 @@ +"""Tests for JSON serialization functionality.""" + +import json + +import pytest + +from meshcore_mqtt.config import Config, ConnectionType, MeshCoreConfig, MQTTConfig +from meshcore_mqtt.meshcore_worker import MeshCoreWorker + + +class TestJSONSerialization: + """Test JSON serialization helper.""" + + @pytest.fixture + def worker(self) -> MeshCoreWorker: + """Create a MeshCore worker instance for testing.""" + config = Config( + mqtt=MQTTConfig(broker="localhost"), + meshcore=MeshCoreConfig( + connection_type=ConnectionType.TCP, address="127.0.0.1", port=12345 + ), + ) + return MeshCoreWorker(config) + + def test_serialize_dict(self, worker: MeshCoreWorker) -> None: + """Test serialization of dictionary data.""" + data = {"message": "hello", "id": 123, "active": True} + result = worker.serialize_to_json(data) + + # Should be valid JSON + parsed = json.loads(result) + assert parsed == data + + def test_serialize_list(self, worker: MeshCoreWorker) -> None: + """Test serialization of list data.""" + data = [1, 2, "three", {"nested": True}] + result = worker.serialize_to_json(data) + + # Should be valid JSON + parsed = json.loads(result) + assert parsed == data + + def test_serialize_primitive_types(self, worker: MeshCoreWorker) -> None: + """Test serialization of primitive types.""" + test_cases = [ + "string", + 123, + 45.67, + True, + False, + None, + ] + + for data in test_cases: + result = worker.serialize_to_json(data) + parsed = json.loads(result) + assert parsed == data + + def test_serialize_object_with_attributes(self, worker: MeshCoreWorker) -> None: + """Test serialization of objects with attributes.""" + + class TestObject: + def __init__(self) -> None: + self.public_attr = "visible" + self.number = 42 + self._private_attr = "hidden" + + obj = TestObject() + result = worker.serialize_to_json(obj) + + # Should be valid JSON + parsed = json.loads(result) + assert "public_attr" in parsed + assert "number" in parsed + assert "_private_attr" not in parsed # Private attributes excluded + assert parsed["public_attr"] == "visible" + assert parsed["number"] == 42 + + def test_serialize_complex_object_fallback(self, worker: MeshCoreWorker) -> None: + """Test serialization fallback for complex objects.""" + + class ComplexObject: + def __str__(self) -> str: + return "complex_representation" + + obj = ComplexObject() + result = worker.serialize_to_json(obj) + + # Should be valid JSON with metadata structure + parsed = json.loads(result) + assert "type" in parsed + assert "value" in parsed + assert "timestamp" in parsed + assert parsed["type"] == "ComplexObject" + assert parsed["value"] == "complex_representation" + + def test_serialize_iterable_object(self, worker: MeshCoreWorker) -> None: + """Test serialization of iterable objects.""" + data = range(3) # Creates an iterable but not list/dict + result = worker.serialize_to_json(data) + + # Should be valid JSON + parsed = json.loads(result) + assert parsed == [0, 1, 2] + + def test_serialize_exception_handling(self, worker: MeshCoreWorker) -> None: + """Test serialization with objects that cause exceptions.""" + + class ProblematicObject: + def __init__(self) -> None: + self.circular_ref = self + + def __str__(self) -> str: + return "problematic" + + obj = ProblematicObject() + result = worker.serialize_to_json(obj) + + # Should still produce valid JSON - circular refs are handled by default=str + parsed = json.loads(result) + + # Result should always be valid JSON + assert isinstance(parsed, dict) + + # Should contain the circular reference as a string representation + assert "circular_ref" in parsed + assert isinstance(parsed["circular_ref"], str) + + def test_serialize_long_string_truncation(self, worker: MeshCoreWorker) -> None: + """Test that very long strings are properly handled.""" + + class LongStringObject: + def __str__(self) -> str: + return "x" * 2000 # Very long string + + obj = LongStringObject() + result = worker.serialize_to_json(obj) + + # Should be valid JSON + parsed = json.loads(result) + + # If it falls back to error handling, raw_value should be truncated + if "raw_value" in parsed: + assert len(parsed["raw_value"]) <= 1000 + + def test_serialize_unicode_handling(self, worker: MeshCoreWorker) -> None: + """Test serialization handles Unicode correctly.""" + data = {"message": "Hello 世界", "emoji": "🚀"} + result = worker.serialize_to_json(data) + + # Should be valid JSON with Unicode preserved + parsed = json.loads(result) + assert parsed["message"] == "Hello 世界" + assert parsed["emoji"] == "🚀" + + def test_all_results_are_valid_json(self, worker: MeshCoreWorker) -> None: + """Test that all serialization results are valid JSON.""" + test_data = [ + {"normal": "dict"}, + [1, 2, 3], + "simple string", + 42, + True, + None, + range(5), + set([1, 2, 3]), # Non-JSON-serializable by default + complex(1, 2), # Non-JSON-serializable type + ] + + for data in test_data: + result = worker.serialize_to_json(data) + + # Every result should be valid JSON + try: + parsed = json.loads(result) + assert isinstance( + parsed, (dict, list, str, int, float, bool, type(None)) + ) + except json.JSONDecodeError: + pytest.fail(f"Invalid JSON produced for {type(data)}: {result}") diff --git a/tests/test_logging.py b/tests/test_logging.py new file mode 100644 index 0000000..46a530e --- /dev/null +++ b/tests/test_logging.py @@ -0,0 +1,110 @@ +"""Tests for logging configuration.""" + +import logging + +from meshcore_mqtt.config import Config, ConnectionType, MeshCoreConfig, MQTTConfig +from meshcore_mqtt.main import setup_logging + + +class TestLoggingConfiguration: + """Test logging setup and configuration.""" + + def test_setup_logging_info_level(self) -> None: + """Test logging setup with INFO level.""" + setup_logging("INFO") + + # Check root logger level + root_logger = logging.getLogger() + assert root_logger.level == logging.INFO + + # Check that third-party loggers are configured + assert logging.getLogger("meshcore").level == logging.INFO + assert logging.getLogger("paho").level == logging.INFO + + # Check that urllib3 is set to WARNING to reduce noise + assert logging.getLogger("urllib3").level == logging.WARNING + + def test_setup_logging_debug_level(self) -> None: + """Test logging setup with DEBUG level.""" + setup_logging("DEBUG") + + # Check root logger level + root_logger = logging.getLogger() + assert root_logger.level == logging.DEBUG + + # Check that third-party loggers are configured + assert logging.getLogger("meshcore").level == logging.DEBUG + assert logging.getLogger("paho").level == logging.DEBUG + + # In DEBUG mode, urllib3 should not be suppressed + assert logging.getLogger("urllib3").level == logging.DEBUG + + def test_setup_logging_warning_level(self) -> None: + """Test logging setup with WARNING level.""" + setup_logging("WARNING") + + # Check root logger level + root_logger = logging.getLogger() + assert root_logger.level == logging.WARNING + + # Check that third-party loggers are configured + assert logging.getLogger("meshcore").level == logging.WARNING + assert logging.getLogger("paho").level == logging.WARNING + + def test_bridge_respects_debug_setting(self) -> None: + """Test that bridge respects debug setting based on log level.""" + from meshcore_mqtt.bridge_coordinator import BridgeCoordinator + + # Test with DEBUG level + debug_config = Config( + mqtt=MQTTConfig(broker="localhost"), + meshcore=MeshCoreConfig( + connection_type=ConnectionType.TCP, address="127.0.0.1", port=12345 + ), + log_level="DEBUG", + ) + + debug_bridge = BridgeCoordinator(debug_config) + # The bridge should be initialized without errors + assert debug_bridge.config.log_level == "DEBUG" + + # Test with INFO level + info_config = Config( + mqtt=MQTTConfig(broker="localhost"), + meshcore=MeshCoreConfig( + connection_type=ConnectionType.TCP, address="127.0.0.1", port=12345 + ), + log_level="INFO", + ) + + info_bridge = BridgeCoordinator(info_config) + assert info_bridge.config.log_level == "INFO" + + def test_third_party_loggers_configured(self) -> None: + """Test that all expected third-party loggers are configured.""" + setup_logging("INFO") + + expected_loggers = [ + "meshcore", + "paho", + "paho.mqtt", + "paho.mqtt.client", + "asyncio", + ] + + for logger_name in expected_loggers: + logger = logging.getLogger(logger_name) + assert ( + logger.level == logging.INFO + ), f"Logger {logger_name} not configured correctly" + + def test_logging_force_override(self) -> None: + """Test that logging configuration overrides existing settings.""" + # Set up initial logging + logging.basicConfig(level=logging.ERROR) + + # Our setup should override this + setup_logging("DEBUG") + + root_logger = logging.getLogger() + assert root_logger.level == logging.DEBUG