From 56730ddc7cfe4661d83d26c49c291831b34d3f13 Mon Sep 17 00:00:00 2001 From: Ben Allfree Date: Sat, 29 Nov 2025 06:17:21 -0800 Subject: [PATCH] feat: introduce Mesh Plugin Manager and dynamic module registration system with initial plugin structure --- .github/workflows/custom_build.yml | 37 +++ firmware-patch.diff | 352 +++++++++++++++++++++++++++++ package.json | 3 +- 3 files changed, 391 insertions(+), 1 deletion(-) create mode 100644 firmware-patch.diff diff --git a/.github/workflows/custom_build.yml b/.github/workflows/custom_build.yml index f93f451..c84fbe3 100644 --- a/.github/workflows/custom_build.yml +++ b/.github/workflows/custom_build.yml @@ -53,6 +53,17 @@ jobs: } EOF chmod +x /tmp/update_status.sh + + - name: Update Status - Fetching Mesh Forge + shell: bash + run: | + source /tmp/update_status.sh + update_status checking_out_meshforge + + - name: Checkout Repository + uses: actions/checkout@v4 + with: + path: meshforge - name: Update Status - Fetching Web Flasher shell: bash @@ -84,6 +95,32 @@ jobs: submodules: recursive fetch-depth: 1 + - name: Apply Firmware Patch + shell: bash + run: | + cd firmware + git apply ../meshforge/firmware-patch.diff + + - name: Update Status - Fetching Mesh Package Manager + shell: bash + run: | + source /tmp/update_status.sh + update_status checking_out_mpm + + - name: Checkout Mesh Package Manager + uses: actions/checkout@v4 + with: + repository: MeshEnvy/mpm + ref: ${{ inputs.version }} + path: mpm + submodules: recursive + fetch-depth: 1 + + - name: Install Python Requirements + working-directory: firmware + run: | + python -m pip install -r requirements.txt -t pyvendor + - name: Update Status - Downloading PlatformIO Cache shell: bash run: | diff --git a/firmware-patch.diff b/firmware-patch.diff new file mode 100644 index 0000000..4a6f8bf --- /dev/null +++ b/firmware-patch.diff @@ -0,0 +1,352 @@ +diff --git a/.gitignore b/.gitignore +index cc742c6c1..545b0923a 100644 +--- a/.gitignore ++++ b/.gitignore +@@ -41,3 +41,11 @@ src/mesh/raspihttp/private_key.pem + + # Ignore logo (set at build time with platformio-custom.py) + data/boot/logo.* ++ ++# Ignore 3rd party plugins ++plugins/* ++!plugins/README.md ++!plugins/sample-plugin ++ ++# Ignore Python vendor directory ++pyvendor +\ No newline at end of file +diff --git a/bin/mpm_pio.py b/bin/mpm_pio.py +new file mode 100644 +index 000000000..48c088acc +--- /dev/null ++++ b/bin/mpm_pio.py +@@ -0,0 +1,55 @@ ++#!/usr/bin/env python3 ++""" ++Mesh Plugin Manager (MPM) - PlatformIO build integration shim. ++ ++This script is intended to be used only as a PlatformIO extra_script. ++It imports the real `mpm` package (vendored via `pyvendor/`) and ++exposes the build helpers expected by the firmware build. ++""" ++ ++import os ++import sys ++ ++try: ++ # Provided by PlatformIO/SCons when used as `pre:` extra_script ++ Import("env") # type: ignore[name-defined] # noqa: F821 ++ IS_PLATFORMIO = True ++except Exception: ++ IS_PLATFORMIO = False ++ ++# Add vendored Python dependencies (installed with `pip install -r requirements.txt -t pyvendor`) ++if IS_PLATFORMIO: ++ # Use PROJECT_DIR from PlatformIO env (__file__ is not available when exec'd by SCons) ++ project_dir = env["PROJECT_DIR"] # type: ignore[name-defined] # noqa: F821 ++ _pyvendor = os.path.join(project_dir, "pyvendor") ++else: ++ # Standalone execution: use __file__ if available, otherwise cwd ++ try: ++ _here = os.path.dirname(__file__) ++ _pyvendor = os.path.abspath(os.path.join(_here, "..", "pyvendor")) ++ except NameError: ++ _pyvendor = os.path.join(os.getcwd(), "pyvendor") ++ ++if os.path.isdir(_pyvendor) and _pyvendor not in sys.path: ++ sys.path.insert(0, _pyvendor) ++ ++if IS_PLATFORMIO: ++ # Use the installed `mpm` package ++ from mpm.build import init_plugins # type: ignore[import] ++ from mpm.build_utils import scan_plugins # type: ignore[import] ++ from mpm.proto import generate_all_protobuf_files # type: ignore[import] ++ ++ # Auto-initialize when imported by PlatformIO (not when run as __main__) ++ if __name__ != "__main__": ++ try: ++ init_plugins(env) # type: ignore[name-defined] # noqa: F821 ++ except NameError: ++ # If env is missing for some reason, just skip auto-init ++ pass ++else: ++ # Standalone execution: delegate to the real CLI for convenience ++ if __name__ == "__main__": ++ from mpm.cli import main # type: ignore[import] ++ ++ main() ++ +diff --git a/platformio.ini b/platformio.ini +index d6ff155e4..b92615db3 100644 +--- a/platformio.ini ++++ b/platformio.ini +@@ -14,7 +14,9 @@ description = Meshtastic + + [env] + test_build_src = true +-extra_scripts = bin/platformio-custom.py ++extra_scripts = ++ bin/platformio-custom.py ++ pre:bin/mpm_pio.py + ; note: we add src to our include search path so that lmic_project_config can override + ; note: TINYGPS_OPTION_NO_CUSTOM_FIELDS is VERY important. We don't use custom fields and somewhere in that pile + ; of code is a heap corruption bug! +@@ -24,6 +26,7 @@ build_flags = -Wno-missing-field-initializers + + -Wno-format + -Isrc -Isrc/mesh -Isrc/mesh/generated -Isrc/gps -Isrc/buzz -Wl,-Map,"${platformio.build_dir}"/output.map ++ -Isrc/modules + -DUSE_THREAD_NAMES + -DTINYGPS_OPTION_NO_CUSTOM_FIELDS + -DPB_ENABLE_MALLOC=1 +diff --git a/plugins/README.md b/plugins/README.md +new file mode 100644 +index 000000000..84e8e8fdf +--- /dev/null ++++ b/plugins/README.md +@@ -0,0 +1,92 @@ ++# Plugin Development Guide ++ ++This directory houses plugins that extend the Meshtastic firmware. Plugins are automatically discovered and integrated into the build system. ++ ++## Plugin Structure ++ ++The only requirement for a plugin is that it must have a `./src` directory: ++ ++``` ++src/plugins/ ++└── myplugin/ ++ └── src/ ++ ├── MyModule.h ++ ├── MyModule.cpp ++ └── mymodule.proto ++``` ++ ++- Plugin directory name can be anything ++- All source files must be placed in `./src` ++- Only files in `./src` are compiled (the root plugin directory and all other subdirectories are excluded from the build) ++ ++## Python Dependencies ++ ++Before building or working with plugins, install the Python tooling into a local vendor directory so PlatformIO can import it: ++ ++```bash ++# From the firmware repo root (directory containing platformio.ini) ++python -m pip install -r requirements.txt -t pyvendor ++``` ++ ++This vendors the Mesh Plugin Manager (MPM) and its dependencies (including `nanopb`) into `pyvendor/`. The build scripts automatically add `pyvendor/` to `sys.path` when PlatformIO runs. ++ ++## Automatic Protobuf Generation ++ ++For convenience, the Meshtastic Plugin Manager (MPM) automatically scans for and generates protobuf files: ++ ++- **Discovery**: MPM recursively scans plugin directories for `.proto` files ++- **Options file**: Auto-detects matching `.options` files (e.g., `mymodule.proto` → `mymodule.options`) ++- **Generation**: Uses the vendored `nanopb` tooling from `pyvendor/` to generate C++ files ++- **Output**: Generated files are placed in the same directory as the `.proto` file ++- **Timing**: Runs during PlatformIO pre-build phase (configured in `platformio.ini`) ++ ++**Note**: Once `pyvendor/` is populated as described above, you can also use the Mesh Plugin Manager CLI from a Python environment that has `pyvendor/` on its `PYTHONPATH` to inspect or manage plugins. ++ ++Example protobuf structure: ++ ++``` ++src/plugins/myplugin/src/ ++├── mymodule.proto # Protobuf definition ++├── mymodule.options # Nanopb options (optional) ++├── mymodule.pb.h # Generated header ++└── mymodule.pb.c # Generated implementation ++``` ++ ++## Include Path Setup ++ ++The plugin's `src/` directory is automatically added to the compiler's include path (`CPPPATH`) during build: ++ ++- Headers in `src/` can be included directly: `#include "MyModule.h"` ++- No need to specify relative paths from other plugin files ++- The build system handles this automatically via `bin/mpm.py` ++ ++## Module Registration ++ ++If your plugin implements a Meshtastic module, you can use the automatic registration system: ++ ++1. Include `ModuleRegistry.h` in your module `.cpp` file ++2. Place `MESHTASTIC_REGISTER_MODULE(ModuleClassName)` at the end of your implementation file ++3. Your module will be automatically initialized when the firmware starts ++ ++Example: ++ ++```cpp ++#include "MyModule.h" ++#include "ModuleRegistry.h" ++ ++// ... module implementation ... ++ ++MESHTASTIC_REGISTER_MODULE(MyModule); ++``` ++ ++**Note**: Module registration is optional. Plugins that don't implement Meshtastic modules (e.g., utility libraries) don't need this. ++ ++For details on writing Meshtastic modules, see the [Module API documentation](https://meshtastic.org/docs/development/device/module-api/). ++ ++## Example Plugin ++ ++See the `lobbs` plugin for a complete example that demonstrates: ++ ++- Protobuf definitions with options file ++- Module implementation with automatic registration ++- Proper source file organization +diff --git a/plugins/sample-plugin/README.md b/plugins/sample-plugin/README.md +new file mode 100644 +index 000000000..f60ee9b9c +--- /dev/null ++++ b/plugins/sample-plugin/README.md +@@ -0,0 +1 @@ ++Sample plugin +\ No newline at end of file +diff --git a/plugins/sample-plugin/src/SampleModule.cpp b/plugins/sample-plugin/src/SampleModule.cpp +new file mode 100644 +index 000000000..647c78bd5 +--- /dev/null ++++ b/plugins/sample-plugin/src/SampleModule.cpp +@@ -0,0 +1,13 @@ ++#include "ModuleRegistry.h" ++#include "SinglePortModule.h" ++ ++class MySampleModule : public SinglePortModule ++{ ++ public: ++ MySampleModule() : SinglePortModule("my_sample_module", meshtastic_PortNum_REPLY_APP) { ++ LOG_INFO("MySampleModule constructor"); ++ } ++}; ++ ++ ++MESHTASTIC_REGISTER_MODULE(MySampleModule) +\ No newline at end of file +diff --git a/plugins/sample-plugin/src/SampleModule.h b/plugins/sample-plugin/src/SampleModule.h +new file mode 100644 +index 000000000..4faef7774 +--- /dev/null ++++ b/plugins/sample-plugin/src/SampleModule.h +@@ -0,0 +1,12 @@ ++#ifndef SAMPLE_MODULE_H ++#define SAMPLE_MODULE_H ++ ++#include "SinglePortModule.h" ++ ++class MySampleModule : public SinglePortModule ++{ ++ public: ++ MySampleModule() : SinglePortModule("my_sample_module", meshtastic_PortNum_REPLY_APP); ++}; ++ ++#endif +\ No newline at end of file +diff --git a/requirements.txt b/requirements.txt +new file mode 100644 +index 000000000..e45155ab7 +--- /dev/null ++++ b/requirements.txt +@@ -0,0 +1,2 @@ ++../mpm ++ +diff --git a/src/modules/ModuleRegistry.cpp b/src/modules/ModuleRegistry.cpp +new file mode 100644 +index 000000000..5c83dc70f +--- /dev/null ++++ b/src/modules/ModuleRegistry.cpp +@@ -0,0 +1,30 @@ ++// src/modules/ModuleRegistry.cpp ++ ++#include "ModuleRegistry.h" ++#include "DebugConfiguration.h" ++ ++// Initialize the global vector in static storage. ++// This vector will be populated by the constructor-attributed functions. ++std::vector g_module_init_functions; ++ ++/** ++ * @brief Called by a module's constructor-attributed function to add ++ * its setup routine to the central list. ++ */ ++void register_module_initializer(ModuleInitFunc func) { ++ // This push_back happens during C++ static initialization, before main(). ++ g_module_init_functions.push_back(func); ++} ++ ++/** ++ * @brief Initializes all modules that have self-registered. ++ * Called once by the core Meshtastic firmware setup routine. ++ */ ++void init_dynamic_modules() { ++ LOG_INFO("Initializing dynamic modules via vector...\n"); ++ ++ // Loop through the collected pointers and execute the setup functions ++ for (ModuleInitFunc func : g_module_init_functions) { ++ func(); // Executes the module's initialization code (e.g., new MyModule()) ++ } ++} +\ No newline at end of file +diff --git a/src/modules/ModuleRegistry.h b/src/modules/ModuleRegistry.h +new file mode 100644 +index 000000000..82f571a5e +--- /dev/null ++++ b/src/modules/ModuleRegistry.h +@@ -0,0 +1,31 @@ ++// src/modules/ModuleRegistry.h ++ ++#ifndef MODULE_REGISTRY_H ++#define MODULE_REGISTRY_H ++ ++#include // Required for std::vector ++ ++// Define the function pointer type for module initialization ++typedef void (*ModuleInitFunc)(void); ++ ++// The central list to hold pointers to the initialization functions. ++// This is defined externally in the CPP file. ++extern std::vector g_module_init_functions; ++ ++// Function that all modules will call to register themselves ++void register_module_initializer(ModuleInitFunc func); ++ ++// Function called by the core firmware setup to initialize all modules ++void init_dynamic_modules(); ++ ++/** ++ * @brief Macro used by module authors to self-register a new Meshtastic Module. ++ * This creates a lambda that instantiates the module and automatically applies the constructor attribute. ++ * * @param ModuleClassName The name of the module's C++ class (e.g., MySensorModule). ++ */ ++#define MESHTASTIC_REGISTER_MODULE(ModuleClassName) \ ++ static void __attribute__((constructor)) register_##ModuleClassName() { \ ++ register_module_initializer([]() { new ModuleClassName(); }); \ ++ } ++ ++#endif // MODULE_REGISTRY_H +\ No newline at end of file +diff --git a/src/modules/Modules.cpp b/src/modules/Modules.cpp +index e477574dd..49df725dd 100644 +--- a/src/modules/Modules.cpp ++++ b/src/modules/Modules.cpp +@@ -9,6 +9,8 @@ + #include "input/UpDownInterruptImpl1.h" + #include "input/i2cButton.h" + #include "modules/SystemCommandsModule.h" ++#include "modules/ModuleRegistry.h" ++ + #if HAS_TRACKBALL + #include "input/TrackballInterruptImpl1.h" + #endif +@@ -298,6 +300,9 @@ void setupModules() + if (moduleConfig.has_range_test && moduleConfig.range_test.enabled) + new RangeTestModule(); + #endif ++ ++ init_dynamic_modules(); ++ + // NOTE! This module must be added LAST because it likes to check for replies from other modules and avoid sending extra + // acks + routingModule = new RoutingModule(); diff --git a/package.json b/package.json index b75e411..2cfe2f6 100644 --- a/package.json +++ b/package.json @@ -10,7 +10,8 @@ "lint": "biome lint", "lint:fix": "biome lint --fix && biome format --write", "preview": "vite preview", - "deploy": "npx convex deploy --cmd 'bun run build' && wrangler deploy && wrangler deploy --config wrangler-registry.json" + "deploy": "npx convex deploy --cmd 'bun run build' && wrangler deploy && wrangler deploy --config wrangler-registry.json", + "generate:firmware-patch": "cd vendor/firmware && git diff develop..meshenvy/module-registry > ../../firmware-patch.diff" }, "dependencies": { "@aws-sdk/client-s3": "^3.937.0",