diff --git a/meshforge-sideload/LICENSE b/meshforge-sideload/LICENSE new file mode 100644 index 0000000..03d8242 --- /dev/null +++ b/meshforge-sideload/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2026 MeshEnvy + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/meshforge-sideload/README.md b/meshforge-sideload/README.md new file mode 100644 index 0000000..a072f02 --- /dev/null +++ b/meshforge-sideload/README.md @@ -0,0 +1,117 @@ +# meshforge-sideload + +A PlatformIO plugin used by the [MeshForge](https://meshforge.org) web flasher to sideload data files to Meshtastic devices after flashing. + +**This is not a C++ library.** It adds no code to device firmware. Instead it: +1. Registers a `sideload` PlatformIO build target that transfers data files to a connected device via Meshtastic's built-in XModem file transfer protocol +2. Ships the Meshtastic firmware patches that enable `/__ext__/` and `/__int__/` path routing in XModem (see `patches/`) + +## Protocol + +Frames are delimited by `0xBB` and coexist safely with Meshtastic's serial API (which uses `0x94C3`) and KISS framing (which uses `0xC0`). + +``` +Frame: 0xBB +Response: 0xBB 0x80 (0x00 = OK, 0x01 = ERR) + +Commands: + 0x00 PING payload=[] → OK + "MESHFORGE-SIDELOAD:0.1\n" + 0x01 OPEN payload=[pathLen:1][path][size:4 LE] + 0x02 DATA payload=[bytes, max 256] + 0x03 CLOSE payload=[] + 0x04 MKDIR payload=[path] +``` + +## Path routing + +| Destination prefix | nRF52 | ESP32 | +|--------------------|-------|-------| +| `/ext/foo` | External QSPI LittleFS | LittleFS (prefix stripped) | +| `/int/foo` | InternalFS | LittleFS (prefix stripped) | +| `/foo` (bare) | InternalFS | LittleFS | + +On ESP32 both prefixes are equivalent — the device has one filesystem. The prefix convention lets `meshforge.yaml` use a single universal path (e.g. `/ext/bbs/kb`) that works correctly on both platforms. + +## Installation + +```ini +; platformio.ini +[env:your_target] +lib_deps = + meshenvy/meshforge-sideload@^0.1.0 +``` + +## Meshtastic integration + +The library has no dependency on Meshtastic. Integrate it by calling `begin()` from your module's `setup()` and `poll()` from `runOnce()`. + +### 1. Add the library + +```ini +lib_deps = + meshenvy/meshforge-sideload@^0.1.0 +``` + +### 2. Override the external FS (nRF52 only) + +By default `/ext/` paths fall back to `InternalFS`. If your firmware has an external QSPI LittleFS, override the weak function anywhere in your source: + +```cpp +// In your module .cpp, after including your QSPI flash header: +#if defined(MFSL_PLATFORM_NRF52) +Adafruit_LittleFS_Namespace::Adafruit_LittleFS& meshforgeSideloadExtFS() { + return myQspiFs; // your Adafruit_LittleFS instance, already mounted +} +#endif +``` + +The library never calls `begin()` on this FS — mount it before calling `MeshForgeSideload::begin()`. + +### 3. Wire into your module + +```cpp +#include "MeshForgeSideload.h" + +static MeshForgeSideload meshForgeSideload; + +void YourModule::setup() { + // ... your existing setup ... + meshForgeSideload.begin(); +} + +int32_t YourModule::runOnce() { + meshForgeSideload.poll(); + // ... your existing logic ... + return interval; +} +``` + +`poll()` is non-blocking and returns immediately when no `0xBB` frame is present on Serial. + +### 4. Declare data files in meshforge.yaml + +Add a `meshforge.yaml` to your firmware repo root: + +```yaml +meshforge: + data: + - my-data/*.bin:/ext/my/path +``` + +The MeshForge CI bundler includes these files in the firmware `.tar.gz`. The MeshForge web flasher reads the yaml from the bundle, waits for the device to boot after flashing, and sideloads the files automatically. + +## MeshCore + +The library is firmware-agnostic — it depends only on the Arduino `Serial` object and standard Adafruit nRF52 BSP / ESP-IDF Arduino filesystem APIs. MeshCore runs on the same Adafruit nRF52 BSP, so the integration should work identically: override `meshforgeSideloadExtFS()` if needed and call `poll()` from your main loop. Formal MeshCore testing has not been done yet. + +## Platforms + +| Platform | Support | +|----------|---------| +| nRF52840 (Adafruit BSP) | Supported — InternalFS + optional QSPI ext FS | +| ESP32 / ESP32-S3 | Supported — LittleFS | +| Other Arduino targets | Compile-time no-op with `#pragma message` warning | + +## License + +MIT — see [LICENSE](LICENSE). diff --git a/meshforge-sideload/extra_scripts/register_sideload_target.py b/meshforge-sideload/extra_scripts/register_sideload_target.py new file mode 100644 index 0000000..5c79b9b --- /dev/null +++ b/meshforge-sideload/extra_scripts/register_sideload_target.py @@ -0,0 +1,239 @@ +""" +register_sideload_target.py — PlatformIO extra_script that registers the +`sideload` custom target for the meshforge-sideload library. + +Transfers data files to the device via Meshtastic StreamAPI + XModem, using +the /ext/ and /int/ path prefix convention supported by the xmodem-ext-fs patch. + +Usage: + pio run -t sideload + pio run -t upload -t sideload + pio run -t sideload --upload-port /dev/cu.usbmodem101 + MESHFORGE_PORT=/dev/cu.usbmodemXXXX pio run -t sideload + MESHFORGE_BOOT_WAIT=5 pio run -t sideload +""" + +Import('env') # noqa: F821 — PlatformIO SCons environment + +import glob as globmod +import os +import re +import struct +import sys +import time + +# ── Meshtastic StreamAPI + XModem protocol (inline) ────────────────────────── + +XC_SOH = 1; XC_STX = 2; XC_EOT = 4; XC_ACK = 6; XC_NAK = 21; XC_CAN = 24 +XMODEM_BUF = 128; MAX_RETRY = 10; ACK_TIMEOUT = 5.0 + + +def _varint(n): + out = [] + while n > 0x7F: out.append((n & 0x7F) | 0x80); n >>= 7 + out.append(n); return bytes(out) + + +def _pb_bytes(field, data): + tag = _varint((field << 3) | 2) + return tag + _varint(len(data)) + data + + +def _encode_xm(control, seq, crc16, buf): + msg = _varint((1 << 3) | 0) + _varint(control) + if seq: msg += _varint((2 << 3) | 0) + _varint(seq) + if crc16: msg += _varint((3 << 3) | 0) + _varint(crc16) + if buf: msg += _pb_bytes(4, buf) + return msg + + +def _to_radio(xm): return _pb_bytes(5, xm) +def _frame(payload): return b'\x94\xc3' + struct.pack('>H', len(payload)) + payload + + +def _crc16(data): + crc = 0 + for b in data: + crc = ((crc >> 8) | (crc << 8)) & 0xFFFF; crc ^= b + crc ^= ((crc & 0xFF) >> 4) & 0xFFFF + crc ^= ((crc << 8) << 4) & 0xFFFF + crc ^= (((crc & 0xFF) << 4) << 1) & 0xFFFF + return crc & 0xFFFF + + +def _read_varint(data, pos): + val = shift = 0 + while pos < len(data): + b = data[pos]; pos += 1; val |= (b & 0x7F) << shift; shift += 7 + if not (b & 0x80): break + return val, pos + + +def _parse_xm(data): + i, r = 0, {'control': 0, 'seq': 0, 'crc16': 0, 'buffer': b''} + while i < len(data): + tag = data[i]; i += 1; fn, wt = tag >> 3, tag & 0x7 + if wt == 0: + val, i = _read_varint(data, i) + if fn == 1: r['control'] = val + elif fn == 2: r['seq'] = val + elif fn == 3: r['crc16'] = val + elif wt == 2: + l, i = _read_varint(data, i) + if fn == 4: r['buffer'] = data[i:i+l] + i += l + else: break + return r + + +def _parse_from_radio(data): + i = 0 + while i < len(data): + tag = data[i]; i += 1; fn, wt = tag >> 3, tag & 0x7 + if wt == 0: _, i = _read_varint(data, i) + elif wt == 2: + l, i = _read_varint(data, i); payload = data[i:i+l]; i += l + if fn == 12: return _parse_xm(payload) + else: break + return None + + +def _read_xm_resp(port, timeout_s=ACK_TIMEOUT): + buf = bytearray(); deadline = time.time() + timeout_s + while time.time() < deadline: + n = port.in_waiting + if n: buf += port.read(n) + for i in range(len(buf) - 1): + if buf[i] == 0x94 and buf[i+1] == 0xC3 and len(buf) >= i + 4: + length = (buf[i+2] << 8) | buf[i+3] + if len(buf) >= i + 4 + length: + payload = bytes(buf[i+4:i+4+length]); del buf[:i+4+length] + xm = _parse_from_radio(payload) + if xm: return xm + break + time.sleep(0.01) + return None + + +def _send_file(port, dest, data, on_progress=None): + # SOH — filename + fn = dest.encode('ascii') + for attempt in range(MAX_RETRY): + port.write(_frame(_to_radio(_encode_xm(XC_SOH, 0, 0, fn)))) + r = _read_xm_resp(port) + if r and r['control'] == XC_ACK: break + if attempt == MAX_RETRY - 1: raise IOError(f'XModem OPEN rejected: {dest}') + # STX data + seq = 1; off = 0 + while off < len(data): + chunk = data[off:off+XMODEM_BUF]; crc = _crc16(chunk); acked = False + for retry in range(MAX_RETRY): + port.write(_frame(_to_radio(_encode_xm(XC_STX, seq, crc, chunk)))) + r = _read_xm_resp(port) + if r and r['control'] == XC_ACK: acked = True; break + if r and r['control'] == XC_CAN: raise IOError(f'Transfer cancelled at {off}') + if not acked: raise IOError(f'No ACK for seq {seq} at offset {off}') + off += len(chunk); seq = (seq & 0xFF) + 1 + if on_progress: on_progress(off, len(data)) + # EOT + for attempt in range(MAX_RETRY): + port.write(_frame(_to_radio(_encode_xm(XC_EOT, 0, 0, b'')))) + r = _read_xm_resp(port) + if r and r['control'] == XC_ACK: return + if attempt == MAX_RETRY - 1: raise IOError(f'EOT not acked: {dest}') + + +def _parse_data_entries(yaml_text): + entries = []; in_mf = in_data = False + for raw in yaml_text.splitlines(): + line = re.sub(r'#.*$', '', raw).rstrip() + if not line.strip(): continue + indent = len(line) - len(line.lstrip()); content = line.strip() + if indent == 0: in_mf = (content == 'meshforge:'); in_data = False + elif in_mf and indent == 2: in_data = (content == 'data:') + elif in_mf and in_data and indent == 4: + m = re.match(r'^-\s+(.+)$', content) + if m: + entry = m.group(1).strip().strip('"\''); colon = entry.find(':') + if colon > 0: entries.append((entry[:colon].strip(), entry[colon+1:].strip())) + return entries + + +def _autodetect_port(): + try: + import serial.tools.list_ports + KNOWN = ['RAK', 'nRF52', 'Adafruit', 'Nordic', 'Meshtastic', 'WisMesh', + 'T-Echo', 'CP210', 'CH340', 'FTDI', 'USB Serial', 'JTAG', 'LilyGO', 'Espressif'] + ports = list(serial.tools.list_ports.comports()) + for p in ports: + desc = (p.description or '') + ' ' + (p.manufacturer or '') + if any(k.lower() in desc.lower() for k in KNOWN): return p.device + for p in ports: + if 'usbmodem' in p.device or 'usbserial' in p.device: return p.device + except ImportError: pass + return None + + +# ── PlatformIO target action ────────────────────────────────────────────────── + +def _sideload(source, target, env): # noqa: F821 + try: + import serial + except ImportError: + print('meshforge-sideload: ERROR — pyserial not installed. Run: pip install pyserial') + raise SystemExit(1) + + project_dir = env['PROJECT_DIR'] # noqa: F821 + yaml_path = os.path.join(project_dir, 'meshforge.yaml') + if not os.path.isfile(yaml_path): + print('meshforge-sideload: no meshforge.yaml — nothing to sideload'); return + + with open(yaml_path, encoding='utf-8') as f: + entries = _parse_data_entries(f.read()) + if not entries: + print('meshforge-sideload: no data: entries — nothing to sideload'); return + + transfers = [] + for glob_pat, dest in entries: + for src in sorted(globmod.glob(os.path.join(project_dir, glob_pat), recursive=True)): + if os.path.isfile(src): + transfers.append((src, dest.rstrip('/') + '/' + os.path.basename(src))) + if not transfers: + print('meshforge-sideload: no files matched — nothing to sideload'); return + + port_name = env.get('UPLOAD_PORT') or os.environ.get('MESHFORGE_PORT') or _autodetect_port() + if not port_name: + print('meshforge-sideload: ERROR — no serial port found.\n' + ' Pass one: pio run -t sideload --upload-port /dev/cu.usbmodemXXXX') + raise SystemExit(1) + + boot_wait = float(os.environ.get('MESHFORGE_BOOT_WAIT', '3')) + baud = int(env.get('MONITOR_SPEED', 115200)) + + print(f'meshforge-sideload: waiting {boot_wait}s for device to boot...') + time.sleep(boot_wait) + print(f'meshforge-sideload: connecting to {port_name} @ {baud}...') + + with serial.Serial(port_name, baud, timeout=ACK_TIMEOUT) as port: + time.sleep(0.5) + for i, (src, device_path) in enumerate(transfers): + data = open(src, 'rb').read() + name = os.path.basename(src) + print(f' [{i+1}/{len(transfers)}] {name} ({len(data)/1024:.1f} KB) → {device_path}') + def progress(sent, total, _name=name): + pct = 100 * sent // total + bar = '#' * (pct // 5) + '.' * (20 - pct // 5) + print(f'\r [{bar}] {pct}%', end='', flush=True) + _send_file(port, device_path, data, on_progress=progress) + print(f'\r [{"#"*20}] 100% done') + + print(f'meshforge-sideload: {len(transfers)} file(s) uploaded successfully') + + +env.AddCustomTarget( # noqa: F821 + name='sideload', + dependencies=None, + actions=_sideload, + title='MeshForge Sideload', + description='Upload meshforge.yaml data files via Meshtastic StreamAPI + XModem', +) diff --git a/meshforge-sideload/library.json b/meshforge-sideload/library.json new file mode 100644 index 0000000..db29a99 --- /dev/null +++ b/meshforge-sideload/library.json @@ -0,0 +1,22 @@ +{ + "name": "meshforge-sideload", + "version": "0.1.0", + "description": "PlatformIO plugin that registers a 'sideload' build target for uploading data files to Meshtastic devices via XModem. Also ships the Meshtastic firmware patches required to enable /__ext__/ and /__int__/ path routing.", + "keywords": ["meshtastic", "meshtastic-module", "sideload", "xmodem"], + "authors": [ + { + "name": "MeshEnvy", + "url": "https://github.com/meshenvy" + } + ], + "repository": { + "type": "git", + "url": "https://github.com/meshenvy/mesh-forge" + }, + "license": "MIT", + "frameworks": ["arduino"], + "platforms": ["nordicnrf52", "espressif32"], + "build": { + "extraScript": "extra_scripts/register_sideload_target.py" + } +} diff --git a/meshforge-sideload/patches/01-nrf-external-flash.patch b/meshforge-sideload/patches/01-nrf-external-flash.patch new file mode 100644 index 0000000..b6c5350 --- /dev/null +++ b/meshforge-sideload/patches/01-nrf-external-flash.patch @@ -0,0 +1,354 @@ +From a577f8b8774f3a517c72b526dc8dc2b67cd0aea5 Mon Sep 17 00:00:00 2001 +From: Ben Allfree +Date: Sun, 12 Apr 2026 11:29:40 -0700 +Subject: [PATCH] Add littleFS support for external flash on nRF + +--- + src/FSCommon.cpp | 22 +++ + src/FSCommon.h | 18 ++ + src/platform/nrf52/extfs-nrf52.cpp | 198 +++++++++++++++++++++ + variants/nrf52840/nano-g2-ultra/variant.h | 1 + + variants/nrf52840/rak_wismeshtap/variant.h | 1 + + variants/nrf52840/t-echo-lite/variant.h | 1 + + variants/nrf52840/t-echo-plus/variant.h | 1 + + variants/nrf52840/t-echo/variant.h | 1 + + 8 files changed, 243 insertions(+) + create mode 100644 src/platform/nrf52/extfs-nrf52.cpp + +diff --git a/src/FSCommon.cpp b/src/FSCommon.cpp +index f215be80f..2c14f308c 100644 +--- a/src/FSCommon.cpp ++++ b/src/FSCommon.cpp +@@ -284,6 +284,22 @@ void rmDir(const char *dirname) + */ + __attribute__((weak, noinline)) void preFSBegin() {} + ++#if defined(ARCH_NRF52) ++// Default null; set by extFSInit() when external flash is available. ++Adafruit_LittleFS *extFS = nullptr; ++ ++/** ++ * Default weak implementation — no external filesystem. ++ * Override in your firmware to mount a QSPI/SPI flash chip and assign extFS: ++ * ++ * void extFSInit() { ++ * static MyExternalFS myFS; ++ * if (myFS.begin()) extFS = &myFS; ++ * } ++ */ ++__attribute__((weak, noinline)) void extFSInit() {} ++#endif ++ + void fsInit() + { + #ifdef FSCom +@@ -293,6 +309,12 @@ void fsInit() + LOG_ERROR("Filesystem mount failed"); + // assert(0); This auto-formats the partition, so no need to fail here. + } ++#if defined(ARCH_NRF52) ++ extFSInit(); ++ if (extFS) { ++ LOG_DEBUG("External filesystem mounted OK"); ++ } ++#endif + #if defined(ARCH_ESP32) + LOG_DEBUG("Filesystem files (%d/%d Bytes):", FSCom.usedBytes(), FSCom.totalBytes()); + #else +diff --git a/src/FSCommon.h b/src/FSCommon.h +index fdc0b76ec..9b3de20c2 100644 +--- a/src/FSCommon.h ++++ b/src/FSCommon.h +@@ -45,7 +45,25 @@ using namespace STM32_LittleFS_Namespace; + #include "InternalFileSystem.h" + #define FSCom InternalFS + #define FSBegin() FSCom.begin() // InternalFS formats on failure ++#include "Adafruit_LittleFS.h" + using namespace Adafruit_LittleFS_Namespace; ++ ++/** ++ * Optional external LittleFS instance (e.g. QSPI flash on boards that define ++ * EXTERNAL_FLASH_USE_QSPI in their variant.h). ++ * ++ * Null by default. Set by extFSInit() when a board or firmware module ++ * initialises an external flash filesystem. XModem uses this pointer to ++ * route "/ext/" paths to external storage instead of InternalFS. ++ */ ++extern Adafruit_LittleFS *extFS; ++ ++/** ++ * Called from fsInit() to initialise the external filesystem. ++ * The default weak implementation is a no-op; override in a platform or ++ * firmware module to mount the QSPI (or SPI) flash and assign extFS. ++ */ ++void extFSInit(); + #endif + + void fsInit(); +diff --git a/src/platform/nrf52/extfs-nrf52.cpp b/src/platform/nrf52/extfs-nrf52.cpp +new file mode 100644 +index 000000000..f316da5fc +--- /dev/null ++++ b/src/platform/nrf52/extfs-nrf52.cpp +@@ -0,0 +1,198 @@ ++/** ++ * @file extfs-nrf52.cpp ++ * @brief Optional external QSPI LittleFS for nRF52840 boards. ++ * ++ * Compiled only when BOTH of the following are defined in a board's variant.h: ++ * ++ * EXTERNAL_FLASH_USE_QSPI — chip is physically wired (hardware fact) ++ * MESHTASTIC_EXTERNAL_FLASH_FS — board opts into LittleFS on that chip ++ * ++ * The two-define pattern lets boards that use external flash for other purposes ++ * (raw storage, FAT, custom formats) declare the hardware without getting an ++ * unwanted LittleFS mount. If either define is absent this file is a no-op ++ * and extFS stays null. ++ * ++ * Required variant.h defines (already on QSPI-capable boards): ++ * PIN_QSPI_SCK, PIN_QSPI_CS, PIN_QSPI_IO0..IO3 ++ * EXTERNAL_FLASH_USE_QSPI (hardware capability) ++ * MESHTASTIC_EXTERNAL_FLASH_FS (intent to use as LittleFS — add this) ++ * ++ * Optional build flag: ++ * EXTFLASH_SIZE_BYTES — total chip size in bytes (defaults to 2 MB) ++ * ++ * This file is intentionally self-contained so it can be proposed as a ++ * pull request to Meshtastic firmware without touching other platform files ++ * beyond the weak extFSInit() hook in FSCommon.cpp. ++ */ ++ ++#if defined(ARCH_NRF52) && defined(EXTERNAL_FLASH_USE_QSPI) && defined(MESHTASTIC_EXTERNAL_FLASH_FS) ++ ++#include "FSCommon.h" ++#include "configuration.h" ++#include ++#include ++#include ++ ++// ── Chip geometry ───────────────────────────────────────────────────────────── ++ ++#ifndef EXTFLASH_SIZE_BYTES ++#define EXTFLASH_SIZE_BYTES (2 * 1024 * 1024) // Default: 2 MB (MX25R1635F / GD25Q16C) ++#endif ++ ++#define EXTFLASH_PAGE_SIZE 256 ++#define EXTFLASH_SECTOR_SIZE 4096 ++#define EXTFLASH_BLOCK_COUNT (EXTFLASH_SIZE_BYTES / EXTFLASH_SECTOR_SIZE) ++ ++// ── QSPI hardware init ──────────────────────────────────────────────────────── ++ ++static uint8_t _qspi_scratch[EXTFLASH_PAGE_SIZE] __attribute__((aligned(4))); ++static bool _qspi_ready = false; ++ ++static bool _qspi_init() ++{ ++ if (_qspi_ready) return true; ++ ++#ifndef NRFX_QSPI_DEFAULT_CONFIG_IRQ_PRIORITY ++#define NRFX_QSPI_DEFAULT_CONFIG_IRQ_PRIORITY 6 ++#endif ++ ++ nrfx_qspi_config_t cfg = NRFX_QSPI_DEFAULT_CONFIG( ++ PIN_QSPI_SCK, PIN_QSPI_CS, ++ PIN_QSPI_IO0, PIN_QSPI_IO1, PIN_QSPI_IO2, PIN_QSPI_IO3); ++ ++ // Conservative 8 MHz clock; avoids needing the QE status-register bit set ++ // for true quad I/O. Works reliably on all boards with QSPI flash. ++ cfg.phy_if.sck_freq = NRF_QSPI_FREQ_DIV8; ++ ++ nrfx_err_t err = nrfx_qspi_init(&cfg, NULL, NULL); // blocking mode ++ if (err == NRFX_ERROR_INVALID_STATE) { ++ // Already initialised (e.g. by bootloader hand-off) — that's fine. ++ _qspi_ready = true; ++ return true; ++ } ++ if (err != NRFX_SUCCESS) { ++ LOG_ERROR("QSPI init failed: %d", (int)err); ++ return false; ++ } ++ ++ _qspi_ready = true; ++ LOG_DEBUG("External QSPI flash ready (8 MHz)"); ++ return true; ++} ++ ++// ── LittleFS I/O callbacks ──────────────────────────────────────────────────── ++ ++static int _ef_read(const struct lfs_config *c, lfs_block_t block, ++ lfs_off_t off, void *buf, lfs_size_t size) ++{ ++ (void)c; ++ uint32_t addr = block * EXTFLASH_SECTOR_SIZE + off; ++ ++ if (((uintptr_t)buf & 3) == 0 && (size & 3) == 0) { ++ return (nrfx_qspi_read(buf, size, addr) == NRFX_SUCCESS) ? LFS_ERR_OK : LFS_ERR_IO; ++ } ++ ++ // Unaligned: go through scratch ++ uint8_t *dst = (uint8_t *)buf; ++ while (size > 0) { ++ uint32_t chunk = (size > sizeof(_qspi_scratch)) ? sizeof(_qspi_scratch) : size; ++ uint32_t qchunk = (chunk + 3) & ~3u; ++ if (nrfx_qspi_read(_qspi_scratch, qchunk, addr) != NRFX_SUCCESS) return LFS_ERR_IO; ++ memcpy(dst, _qspi_scratch, chunk); ++ dst += chunk; addr += chunk; size -= chunk; ++ } ++ return LFS_ERR_OK; ++} ++ ++static int _ef_prog(const struct lfs_config *c, lfs_block_t block, ++ lfs_off_t off, const void *buf, lfs_size_t size) ++{ ++ (void)c; ++ uint32_t addr = block * EXTFLASH_SECTOR_SIZE + off; ++ ++ if (((uintptr_t)buf & 3) == 0 && (size & 3) == 0) { ++ if (nrfx_qspi_write(buf, size, addr) != NRFX_SUCCESS) return LFS_ERR_IO; ++ } else { ++ const uint8_t *src = (const uint8_t *)buf; ++ while (size > 0) { ++ uint32_t chunk = (size > sizeof(_qspi_scratch)) ? sizeof(_qspi_scratch) : size; ++ uint32_t qchunk = (chunk + 3) & ~3u; ++ memcpy(_qspi_scratch, src, chunk); ++ for (uint32_t i = chunk; i < qchunk; i++) _qspi_scratch[i] = 0xFF; ++ if (nrfx_qspi_write(_qspi_scratch, qchunk, addr) != NRFX_SUCCESS) return LFS_ERR_IO; ++ src += chunk; addr += chunk; size -= chunk; ++ } ++ } ++ ++ while (nrfx_qspi_mem_busy_check() == NRFX_ERROR_BUSY) yield(); ++ return LFS_ERR_OK; ++} ++ ++static int _ef_erase(const struct lfs_config *c, lfs_block_t block) ++{ ++ (void)c; ++ uint32_t addr = block * EXTFLASH_SECTOR_SIZE; ++ if (nrfx_qspi_erase(NRF_QSPI_ERASE_LEN_4KB, addr) != NRFX_SUCCESS) return LFS_ERR_IO; ++ while (nrfx_qspi_mem_busy_check() == NRFX_ERROR_BUSY) yield(); ++ return LFS_ERR_OK; ++} ++ ++static int _ef_sync(const struct lfs_config *c) ++{ ++ (void)c; ++ while (nrfx_qspi_mem_busy_check() == NRFX_ERROR_BUSY) yield(); ++ return LFS_ERR_OK; ++} ++ ++// ── LittleFS instance ───────────────────────────────────────────────────────── ++ ++static uint8_t _read_buf [EXTFLASH_PAGE_SIZE] __attribute__((aligned(4))); ++static uint8_t _prog_buf [EXTFLASH_PAGE_SIZE] __attribute__((aligned(4))); ++static uint8_t _lookahead_buf[64] __attribute__((aligned(4))); ++ ++static struct lfs_config _cfg = { ++ .context = NULL, ++ .read = _ef_read, ++ .prog = _ef_prog, ++ .erase = _ef_erase, ++ .sync = _ef_sync, ++ ++ .read_size = EXTFLASH_PAGE_SIZE, ++ .prog_size = EXTFLASH_PAGE_SIZE, ++ .block_size = EXTFLASH_SECTOR_SIZE, ++ .block_count = EXTFLASH_BLOCK_COUNT, ++ .lookahead = 512, ++ ++ .read_buffer = _read_buf, ++ .prog_buffer = _prog_buf, ++ .lookahead_buffer = _lookahead_buf, ++ .file_buffer = NULL, ++}; ++ ++static Adafruit_LittleFS _extLittleFS(&_cfg); ++ ++// ── extFSInit() — overrides the weak no-op in FSCommon.cpp ─────────────────── ++ ++void extFSInit() ++{ ++ if (!_qspi_init()) return; ++ ++ if (_extLittleFS.begin()) { ++ LOG_INFO("External QSPI LittleFS mounted (%u blocks x %u B)", ++ EXTFLASH_BLOCK_COUNT, EXTFLASH_SECTOR_SIZE); ++ extFS = &_extLittleFS; ++ return; ++ } ++ ++ // First boot or corrupted — format then remount ++ LOG_WARN("External QSPI LittleFS mount failed, formatting..."); ++ if (!_extLittleFS.format() || !_extLittleFS.begin()) { ++ LOG_ERROR("External QSPI LittleFS format/mount failed"); ++ return; ++ } ++ ++ LOG_INFO("External QSPI LittleFS formatted and mounted"); ++ extFS = &_extLittleFS; ++} ++ ++#endif // ARCH_NRF52 && EXTERNAL_FLASH_USE_QSPI && MESHTASTIC_EXTERNAL_FLASH_FS +diff --git a/variants/nrf52840/nano-g2-ultra/variant.h b/variants/nrf52840/nano-g2-ultra/variant.h +index 631af72d8..93f5e4011 100644 +--- a/variants/nrf52840/nano-g2-ultra/variant.h ++++ b/variants/nrf52840/nano-g2-ultra/variant.h +@@ -90,6 +90,7 @@ External serial flash W25Q16JV_IQ + // On-board QSPI Flash + #define EXTERNAL_FLASH_DEVICES W25Q16JV_IQ + #define EXTERNAL_FLASH_USE_QSPI ++#define MESHTASTIC_EXTERNAL_FLASH_FS + + /* + * Lora radio +diff --git a/variants/nrf52840/rak_wismeshtap/variant.h b/variants/nrf52840/rak_wismeshtap/variant.h +index 358117cd5..106fce89d 100644 +--- a/variants/nrf52840/rak_wismeshtap/variant.h ++++ b/variants/nrf52840/rak_wismeshtap/variant.h +@@ -154,6 +154,7 @@ static const uint8_t SCK = PIN_SPI_SCK; + // On-board QSPI Flash + #define EXTERNAL_FLASH_DEVICES IS25LP080D + #define EXTERNAL_FLASH_USE_QSPI ++#define MESHTASTIC_EXTERNAL_FLASH_FS + + /* @note RAK5005-O GPIO mapping to RAK4631 GPIO ports + RAK5005-O <-> nRF52840 +diff --git a/variants/nrf52840/t-echo-lite/variant.h b/variants/nrf52840/t-echo-lite/variant.h +index 54c7bdfb5..01d26f2d4 100644 +--- a/variants/nrf52840/t-echo-lite/variant.h ++++ b/variants/nrf52840/t-echo-lite/variant.h +@@ -98,6 +98,7 @@ static const uint8_t A0 = PIN_A0; + // On-board QSPI Flash + #define EXTERNAL_FLASH_DEVICES ZD25WQ32CEIGR + #define EXTERNAL_FLASH_USE_QSPI ++#define MESHTASTIC_EXTERNAL_FLASH_FS + + // Lora radio + +diff --git a/variants/nrf52840/t-echo-plus/variant.h b/variants/nrf52840/t-echo-plus/variant.h +index 7ebdf48c0..0cca07efc 100644 +--- a/variants/nrf52840/t-echo-plus/variant.h ++++ b/variants/nrf52840/t-echo-plus/variant.h +@@ -77,6 +77,7 @@ static const uint8_t A0 = PIN_A0; + // On-board QSPI Flash + #define EXTERNAL_FLASH_DEVICES MX25R1635F + #define EXTERNAL_FLASH_USE_QSPI ++#define MESHTASTIC_EXTERNAL_FLASH_FS + + // LoRa SX1262 + #define USE_SX1262 +diff --git a/variants/nrf52840/t-echo/variant.h b/variants/nrf52840/t-echo/variant.h +index f4644c6de..b13046121 100644 +--- a/variants/nrf52840/t-echo/variant.h ++++ b/variants/nrf52840/t-echo/variant.h +@@ -121,6 +121,7 @@ External serial flash WP25R1635FZUIL0 + // On-board QSPI Flash + #define EXTERNAL_FLASH_DEVICES MX25R1635F + #define EXTERNAL_FLASH_USE_QSPI ++#define MESHTASTIC_EXTERNAL_FLASH_FS + + /* + * Lora radio +-- +2.50.1 + diff --git a/meshforge-sideload/patches/02-xmodem-vfs-routing.patch b/meshforge-sideload/patches/02-xmodem-vfs-routing.patch new file mode 100644 index 0000000..ad64e84 --- /dev/null +++ b/meshforge-sideload/patches/02-xmodem-vfs-routing.patch @@ -0,0 +1,230 @@ +From 697b1317e012c74751ac995794b42bfbce967b14 Mon Sep 17 00:00:00 2001 +From: Ben Allfree +Date: Sun, 12 Apr 2026 12:04:17 -0700 +Subject: [PATCH] Add FS routing support for virtual mount points in FSCommon + +--- + src/FSCommon.cpp | 94 ++++++++++++++++++++++++++++++++++++++++++++++++ + src/FSCommon.h | 42 +++++++++++++++++++++- + src/xmodem.cpp | 9 +++-- + src/xmodem.h | 3 +- + 4 files changed, 141 insertions(+), 7 deletions(-) + +diff --git a/src/FSCommon.cpp b/src/FSCommon.cpp +index 2c14f308c..79e927e64 100644 +--- a/src/FSCommon.cpp ++++ b/src/FSCommon.cpp +@@ -324,6 +324,100 @@ void fsInit() + #endif + } + ++// ── FSRoute virtual mount-point routing ────────────────────────────────────── ++ ++#ifdef FSCom ++ ++FSRoute fsRoute(const char *path) ++{ ++ FSRoute r; ++ if (strncmp(path, "/__ext__/", 9) == 0) { ++ r.mount = FsMount::External; ++ r.path[0] = '/'; ++ strlcpy(r.path + 1, path + 9, sizeof(r.path) - 1); ++ } else if (strncmp(path, "/__int__/", 9) == 0) { ++ r.mount = FsMount::Internal; ++ r.path[0] = '/'; ++ strlcpy(r.path + 1, path + 9, sizeof(r.path) - 1); ++ } else if (strncmp(path, "/__sd__/", 8) == 0) { ++ r.mount = FsMount::SD; ++ r.path[0] = '/'; ++ strlcpy(r.path + 1, path + 8, sizeof(r.path) - 1); ++ } else { ++ r.mount = FsMount::Internal; ++ strlcpy(r.path, path, sizeof(r.path)); ++ } ++ return r; ++} ++ ++// ── FS selection helper ─────────────────────────────────────────────────────── ++// Returns the correct FS object for the given mount, falling back to FSCom ++// when the requested mount is unavailable. ++ ++#if defined(ARCH_NRF52) ++static Adafruit_LittleFS &_fsForMount(FsMount mount) ++{ ++ if (mount == FsMount::External && extFS != nullptr) ++ return *extFS; ++ // SD: future ++ return (Adafruit_LittleFS &)FSCom; ++} ++#endif ++ ++// ── Public helpers ──────────────────────────────────────────────────────────── ++ ++File fsOpenRead(const FSRoute &r) ++{ ++#if defined(ARCH_NRF52) ++ return _fsForMount(r.mount).open(r.path, FILE_O_READ); ++#else ++ (void)r.mount; ++ return FSCom.open(r.path, FILE_O_READ); ++#endif ++} ++ ++File fsOpenWrite(const FSRoute &r) ++{ ++#if defined(ARCH_NRF52) ++ return _fsForMount(r.mount).open(r.path, FILE_O_WRITE); ++#else ++ (void)r.mount; ++ return FSCom.open(r.path, FILE_O_WRITE); ++#endif ++} ++ ++bool fsRemove(const FSRoute &r) ++{ ++#if defined(ARCH_NRF52) ++ return _fsForMount(r.mount).remove(r.path); ++#else ++ (void)r.mount; ++ return FSCom.remove(r.path); ++#endif ++} ++ ++bool fsMkdir(const FSRoute &r) ++{ ++#if defined(ARCH_NRF52) ++ return _fsForMount(r.mount).mkdir(r.path); ++#else ++ (void)r.mount; ++ return FSCom.mkdir(r.path); ++#endif ++} ++ ++bool fsExists(const FSRoute &r) ++{ ++#if defined(ARCH_NRF52) ++ return _fsForMount(r.mount).exists(r.path); ++#else ++ (void)r.mount; ++ return FSCom.exists(r.path); ++#endif ++} ++ ++#endif // FSCom ++ + /** + * Initializes the SD card and mounts the file system. + */ +diff --git a/src/FSCommon.h b/src/FSCommon.h +index 9b3de20c2..1ac0d04a1 100644 +--- a/src/FSCommon.h ++++ b/src/FSCommon.h +@@ -73,4 +73,44 @@ bool renameFile(const char *pathFrom, const char *pathTo); + std::vector getFiles(const char *dirname, uint8_t levels); + void listDir(const char *dirname, uint8_t levels, bool del = false); + void rmDir(const char *dirname); +-void setupSDCard(); +\ No newline at end of file ++void setupSDCard(); ++ ++#ifdef FSCom ++/** ++ * Virtual filesystem mount-point routing. ++ * ++ * Path-prefix convention (double-underscore delimiters avoid collisions): ++ * /__int__/foo → internal flash (InternalFS / LittleFS) ++ * /__ext__/foo → external flash (QSPI LittleFS if mounted, else internal) ++ * /__sd__/foo → SD card (if mounted, else internal) ++ * /foo → internal flash (bare paths passed through unchanged) ++ * ++ * This provides a lightweight mount-point convention without a full VFS. ++ * All consumers call fsRoute() + the helpers below; when Meshtastic gains a ++ * proper VFS layer these functions become the adapter to it. ++ */ ++enum class FsMount { Internal, External, SD }; ++ ++struct FSRoute { ++ FsMount mount = FsMount::Internal; ++ char path[128] = {}; // real path after prefix stripped ++}; ++ ++/** Resolve a path string to an FSRoute (mount + stripped path). */ ++FSRoute fsRoute(const char *path); ++ ++/** Open a file for reading on the routed filesystem. */ ++File fsOpenRead(const FSRoute &r); ++ ++/** Open a file for writing on the routed filesystem. */ ++File fsOpenWrite(const FSRoute &r); ++ ++/** Remove a file on the routed filesystem. Returns true on success. */ ++bool fsRemove(const FSRoute &r); ++ ++/** Create a directory (and parents) on the routed filesystem. */ ++bool fsMkdir(const FSRoute &r); ++ ++/** Return true if the path exists on the routed filesystem. */ ++bool fsExists(const FSRoute &r); ++#endif // FSCom +\ No newline at end of file +diff --git a/src/xmodem.cpp b/src/xmodem.cpp +index 1d8c77760..7c67d7fbf 100644 +--- a/src/xmodem.cpp ++++ b/src/xmodem.cpp +@@ -120,10 +120,11 @@ void XModemAdapter::handlePacket(meshtastic_XModem xmodemPacket) + if ((xmodemPacket.seq == 0) && !isReceiving && !isTransmitting) { + // NULL packet has the destination filename + memcpy(filename, &xmodemPacket.buffer.bytes, xmodemPacket.buffer.size); ++ activeRoute_ = fsRoute(filename); + + if (xmodemPacket.control == meshtastic_XModem_Control_SOH) { // Receive this file and put to Flash + spiLock->lock(); +- file = FSCom.open(filename, FILE_O_WRITE); ++ file = fsOpenWrite(activeRoute_); + spiLock->unlock(); + if (file) { + sendControl(meshtastic_XModem_Control_ACK); +@@ -137,7 +138,7 @@ void XModemAdapter::handlePacket(meshtastic_XModem xmodemPacket) + } else { // Transmit this file from Flash + LOG_INFO("XModem: Transmit file %s", filename); + spiLock->lock(); +- file = FSCom.open(filename, FILE_O_READ); ++ file = fsOpenRead(activeRoute_); + spiLock->unlock(); + if (file) { + packetno = 1; +@@ -200,8 +201,7 @@ void XModemAdapter::handlePacket(meshtastic_XModem xmodemPacket) + spiLock->lock(); + file.flush(); + file.close(); +- +- FSCom.remove(filename); ++ fsRemove(activeRoute_); + spiLock->unlock(); + isReceiving = false; + break; +@@ -255,7 +255,6 @@ void XModemAdapter::handlePacket(meshtastic_XModem xmodemPacket) + xmodemStore.seq = packetno; + spiLock->lock(); + file.seek((packetno - 1) * sizeof(meshtastic_XModem_buffer_t::bytes)); +- + xmodemStore.buffer.size = file.read(xmodemStore.buffer.bytes, sizeof(meshtastic_XModem_buffer_t::bytes)); + spiLock->unlock(); + xmodemStore.crc16 = crc16_ccitt(xmodemStore.buffer.bytes, xmodemStore.buffer.size); +diff --git a/src/xmodem.h b/src/xmodem.h +index 4cfcb43e1..48ff25fbb 100644 +--- a/src/xmodem.h ++++ b/src/xmodem.h +@@ -67,7 +67,8 @@ class XModemAdapter + File file; + #endif + +- char filename[sizeof(meshtastic_XModem_buffer_t::bytes)] = {0}; ++ char filename[sizeof(meshtastic_XModem_buffer_t::bytes)] = {0}; ++ FSRoute activeRoute_; // resolved once at SOH, reused for DATA/EOT/CAN + + protected: + meshtastic_XModem xmodemStore = meshtastic_XModem_init_zero; +-- +2.50.1 + diff --git a/meshforge-sideload/patches/README.md b/meshforge-sideload/patches/README.md new file mode 100644 index 0000000..538c373 --- /dev/null +++ b/meshforge-sideload/patches/README.md @@ -0,0 +1,68 @@ +# MeshForge Meshtastic patches + +Two patches against Meshtastic firmware that enable MeshForge data sideloading. +Both are pending upstream merge. Once merged, these patches and the apply script +become unnecessary. + +--- + +## 01-nrf-external-flash.patch + +**PR title:** `feat(nrf52): optional external QSPI LittleFS filesystem` + +Adds infrastructure for mounting the external QSPI flash chip as a LittleFS +volume on nRF52840 boards that have it wired: + +- `src/FSCommon.h` — `extern Adafruit_LittleFS* extFS` pointer + `extFSInit()` declaration +- `src/FSCommon.cpp` — `extFS = nullptr` global, weak `extFSInit()` no-op, called from `fsInit()` +- `src/platform/nrf52/extfs-nrf52.cpp` — actual QSPI init + LittleFS mount using `nrfx_qspi` +- `variants/nrf52840/{t-echo,t-echo-lite,t-echo-plus,nano-g2-ultra,rak_wismeshtap}/variant.h` — add `MESHTASTIC_EXTERNAL_FLASH_FS` + +**Sentinel:** `extFSInit` + +--- + +## 02-xmodem-vfs-routing.patch + +**PR title:** `feat(nrf52): virtual filesystem mount points + XModem path routing` + +Depends on patch 01. Adds a lightweight VFS routing layer and wires XModem into it: + +- `src/FSCommon.h` — `FsMount` enum, `FSRoute` struct, `fsRoute()` + helpers +- `src/FSCommon.cpp` — routing and helper implementations +- `src/xmodem.h` / `src/xmodem.cpp` — XModem uses `fsRoute()` for `/__ext__/` and `/__int__/` paths + +Path prefix convention: + +| Prefix | Destination | +|--------|-------------| +| `/__ext__/foo` | `extFS` if mounted, else internal | +| `/__int__/foo` | Internal flash (InternalFS / LittleFS) | +| `/__sd__/foo` | SD card (reserved, falls back to internal) | +| `/foo` | Internal flash — bare paths unchanged | + +**Sentinel:** `fsRoute` + +--- + +## Applying + +```bash +# From within a Meshtastic firmware checkout: +git apply meshforge-sideload/patches/01-nrf-external-flash.patch +git apply meshforge-sideload/patches/02-xmodem-vfs-routing.patch +``` + +Or via PlatformIO extra_scripts (applied automatically at build time): + +```ini +[env] +extra_scripts = pre:path/to/meshforge-sideload/patches/apply_patches.py +``` + +## Reverting + +```bash +git apply -R meshforge-sideload/patches/02-xmodem-vfs-routing.patch +git apply -R meshforge-sideload/patches/01-nrf-external-flash.patch +``` diff --git a/meshforge-sideload/patches/apply_patches.py b/meshforge-sideload/patches/apply_patches.py new file mode 100644 index 0000000..eabb4dc --- /dev/null +++ b/meshforge-sideload/patches/apply_patches.py @@ -0,0 +1,85 @@ +""" +apply_patches.py — PlatformIO pre: extra_script. + +Applies the MeshForge Meshtastic patches to the firmware source tree in order: + + 01-nrf-external-flash.patch — extFSInit hook, extFS pointer, extfs-nrf52.cpp + 02-xmodem-vfs-routing.patch — FSRoute, fsRoute(), XModem /__ext__/ routing + +Each patch is skipped if its sentinel string is already present (idempotent). +Safe to run on every build. + +Add to your firmware's platformio.ini: + + extra_scripts = + pre:path/to/apply_patches.py +""" + +Import('env') # noqa: F821 + +import os +import subprocess +import sys + +_PATCHES = [ + ('01-nrf-external-flash.patch', 'extFSInit'), # sentinel unique to patch 1 + ('02-xmodem-vfs-routing.patch', 'fsRoute'), # sentinel unique to patch 2 +] + +_PATCH_DIR = os.path.dirname(os.path.abspath(__file__)) + + +def _project_dir(): + return env.subst('$PROJECT_DIR') # noqa: F821 + + +def _sentinel_present(sentinel: str) -> bool: + """Return True if the sentinel string already exists in the source tree.""" + for dirpath, _, filenames in os.walk(os.path.join(_project_dir(), 'src')): + for fname in filenames: + if not (fname.endswith('.h') or fname.endswith('.cpp')): + continue + try: + with open(os.path.join(dirpath, fname), encoding='utf-8', errors='replace') as f: + if sentinel in f.read(): + return True + except OSError: + pass + return False + + +def _apply(patch_file: str) -> None: + patch_path = os.path.join(_PATCH_DIR, patch_file) + if not os.path.isfile(patch_path): + print(f'[meshforge-patches] WARNING: {patch_file} not found') + return + + # Dry-run first + check = subprocess.run( + ['git', 'apply', '--check', '--whitespace=nowarn', patch_path], + cwd=_project_dir(), + capture_output=True, + ) + if check.returncode != 0: + print(f'[meshforge-patches] WARNING: {patch_file} cannot apply cleanly — skipping ' + f'(already applied or conflict)') + return + + result = subprocess.run( + ['git', 'apply', '--whitespace=nowarn', patch_path], + cwd=_project_dir(), + capture_output=True, + ) + if result.returncode == 0: + print(f'[meshforge-patches] Applied {patch_file}') + else: + print(f'[meshforge-patches] ERROR applying {patch_file}:\n{result.stderr.decode()}') + sys.exit(1) + + +for patch_file, sentinel in _PATCHES: + if _sentinel_present(sentinel): + print(f'[meshforge-patches] {patch_file} already applied — skipping') + else: + print(f'[meshforge-patches] Applying {patch_file}...') + _apply(patch_file) diff --git a/meshforge-sideload/tools/mf_protocol.py b/meshforge-sideload/tools/mf_protocol.py new file mode 100644 index 0000000..2286055 --- /dev/null +++ b/meshforge-sideload/tools/mf_protocol.py @@ -0,0 +1,296 @@ +""" +mf_protocol.py — Meshtastic StreamAPI + XModem file transfer (Python host side). + +Matches the patched xmodem.cpp on the firmware side. + +Meshtastic StreamAPI framing: + 0x94 0xC3 + +ToRadio.xmodemPacket = field 5 +FromRadio.xmodemPacket = field 12 + +meshtastic_XModem fields: + control (1, varint) — XModem Control enum + seq (2, varint) — packet sequence + crc16 (3, varint) — CRC-16-CCITT of buffer + buffer (4, bytes) — up to 128 bytes +""" + +import glob as globmod +import os +import re +import struct +import sys +import time + +# XModem Control enum values +XC_NUL = 0 +XC_SOH = 1 +XC_STX = 2 +XC_EOT = 4 +XC_ACK = 6 +XC_NAK = 21 +XC_CAN = 24 + +XMODEM_BUFFER_SIZE = 128 +MAX_RETRIES = 10 +ACK_TIMEOUT_S = 5.0 + + +# ── CRC-16-CCITT ────────────────────────────────────────────────────────────── + +def crc16_ccitt(data: bytes) -> int: + crc = 0 + for b in data: + crc = ((crc >> 8) | (crc << 8)) & 0xFFFF + crc ^= b + crc ^= ((crc & 0xFF) >> 4) & 0xFFFF + crc ^= ((crc << 8) << 4) & 0xFFFF + crc ^= (((crc & 0xFF) << 4) << 1) & 0xFFFF + return crc & 0xFFFF + + +# ── Minimal protobuf encoding ───────────────────────────────────────────────── + +def varint(n: int) -> bytes: + out = [] + while n > 0x7F: + out.append((n & 0x7F) | 0x80) + n >>= 7 + out.append(n) + return bytes(out) + + +def pb_bytes_field(field: int, data: bytes) -> bytes: + tag = varint((field << 3) | 2) + return tag + varint(len(data)) + data + + +def encode_xmodem(control: int, seq: int, crc16: int, buffer: bytes) -> bytes: + msg = varint((1 << 3) | 0) + varint(control) + if seq: msg += varint((2 << 3) | 0) + varint(seq) + if crc16: msg += varint((3 << 3) | 0) + varint(crc16) + if buffer: msg += pb_bytes_field(4, buffer) + return msg + + +def encode_toradio(xmodem_bytes: bytes) -> bytes: + return pb_bytes_field(5, xmodem_bytes) + + +def stream_frame(payload: bytes) -> bytes: + return b'\x94\xc3' + struct.pack('>H', len(payload)) + payload + + +# ── FromRadio response parsing ──────────────────────────────────────────────── + +def _parse_varint(data: bytes, pos: int): + val, shift = 0, 0 + while pos < len(data): + b = data[pos]; pos += 1 + val |= (b & 0x7F) << shift; shift += 7 + if not (b & 0x80): break + return val, pos + + +def parse_xmodem(data: bytes) -> dict: + i, result = 0, {'control': 0, 'seq': 0, 'crc16': 0, 'buffer': b''} + while i < len(data): + tag = data[i]; i += 1 + fn, wt = tag >> 3, tag & 0x7 + if wt == 0: + val, i = _parse_varint(data, i) + if fn == 1: result['control'] = val + elif fn == 2: result['seq'] = val + elif fn == 3: result['crc16'] = val + elif wt == 2: + l, i = _parse_varint(data, i) + if fn == 4: result['buffer'] = data[i:i+l] + i += l + else: break + return result + + +def parse_from_radio(data: bytes) -> dict | None: + i = 0 + while i < len(data): + tag = data[i]; i += 1 + fn, wt = tag >> 3, tag & 0x7 + if wt == 0: + _, i = _parse_varint(data, i) + elif wt == 2: + l, i = _parse_varint(data, i) + payload = data[i:i+l]; i += l + if fn == 12: + return parse_xmodem(payload) + else: break + return None + + +# ── StreamAPI frame reader ──────────────────────────────────────────────────── + +def read_xmodem_response(port, timeout_s: float = ACK_TIMEOUT_S) -> dict | None: + buf = bytearray() + deadline = time.time() + timeout_s + while time.time() < deadline: + n = port.in_waiting + if n: buf += port.read(n) + # Scan for 0x94 0xC3 frame + start = -1 + for i in range(len(buf) - 1): + if buf[i] == 0x94 and buf[i+1] == 0xC3: + start = i; break + if start >= 0 and len(buf) >= start + 4: + length = (buf[start+2] << 8) | buf[start+3] + if len(buf) >= start + 4 + length: + payload = bytes(buf[start+4:start+4+length]) + del buf[:start+4+length] + xm = parse_from_radio(payload) + if xm: return xm + # Not an XModem frame — keep reading + continue + time.sleep(0.01) + return None + + +# ── File transfer ───────────────────────────────────────────────────────────── + +def send_xmodem_file(port, dest_path: str, data: bytes, + on_progress=None) -> None: + # SOH seq=0 — filename handshake + fn_bytes = dest_path.encode('ascii') + for attempt in range(MAX_RETRIES): + port.write(stream_frame(encode_toradio(encode_xmodem(XC_SOH, 0, 0, fn_bytes)))) + resp = read_xmodem_response(port) + if resp and resp['control'] == XC_ACK: + break + if attempt == MAX_RETRIES - 1: + raise IOError(f'XModem OPEN rejected for {dest_path}') + + # STX data packets + seq = 1 + offset = 0 + while offset < len(data): + chunk = data[offset:offset + XMODEM_BUFFER_SIZE] + crc = crc16_ccitt(chunk) + acked = False + for retry in range(MAX_RETRIES): + port.write(stream_frame(encode_toradio(encode_xmodem(XC_STX, seq, crc, chunk)))) + resp = read_xmodem_response(port) + if resp and resp['control'] == XC_ACK: + acked = True; break + if resp and resp['control'] == XC_CAN: + raise IOError(f'XModem transfer cancelled at offset {offset}') + if not acked: + raise IOError(f'XModem: no ACK for seq {seq} at offset {offset}') + offset += len(chunk) + seq = (seq & 0xFF) + 1 + if on_progress: on_progress(offset, len(data)) + + # EOT + for attempt in range(MAX_RETRIES): + port.write(stream_frame(encode_toradio(encode_xmodem(XC_EOT, 0, 0, b'')))) + resp = read_xmodem_response(port) + if resp and resp['control'] == XC_ACK: + return + if attempt == MAX_RETRIES - 1: + raise IOError(f'XModem EOT not acknowledged for {dest_path}') + + +# ── meshforge.yaml parser (minimal) ────────────────────────────────────────── + +def parse_data_entries(yaml_text: str) -> list: + entries = [] + in_mf = in_data = False + for raw in yaml_text.splitlines(): + line = re.sub(r'#.*$', '', raw).rstrip() + if not line.strip(): continue + indent = len(line) - len(line.lstrip()) + content = line.strip() + if indent == 0: + in_mf = (content == 'meshforge:'); in_data = False + elif in_mf and indent == 2: + in_data = (content == 'data:') + elif in_mf and in_data and indent == 4: + m = re.match(r'^-\s+(.+)$', content) + if m: + entry = m.group(1).strip().strip('"\'') + colon = entry.find(':') + if colon > 0: + entries.append((entry[:colon].strip(), entry[colon+1:].strip())) + return entries + + +# ── Autodetect port ─────────────────────────────────────────────────────────── + +def autodetect_port(): + try: + import serial.tools.list_ports + KNOWN = ['RAK', 'nRF52', 'Adafruit', 'Nordic', 'Meshtastic', + 'WisMesh', 'T-Echo', 'CP210', 'CH340', 'FTDI', 'USB Serial', + 'JTAG', 'LilyGO', 'Espressif'] + ports = list(serial.tools.list_ports.comports()) + for p in ports: + desc = (p.description or '') + ' ' + (p.manufacturer or '') + if any(k.lower() in desc.lower() for k in KNOWN): + return p.device + for p in ports: + if 'usbmodem' in p.device or 'usbserial' in p.device: + return p.device + except ImportError: + pass + return None + + +# ── Main sideload routine ───────────────────────────────────────────────────── + +def run_sideload(project_dir: str, port_name: str, baud: int = 115200, + boot_wait_s: float = 3.0) -> None: + try: + import serial + except ImportError: + print('ERROR: pyserial not installed. Run: pip install pyserial') + sys.exit(1) + + yaml_path = os.path.join(project_dir, 'meshforge.yaml') + if not os.path.isfile(yaml_path): + print('meshforge-sideload: no meshforge.yaml — nothing to sideload') + return + + with open(yaml_path, encoding='utf-8') as f: + entries = parse_data_entries(f.read()) + + if not entries: + print('meshforge-sideload: no data: entries — nothing to sideload') + return + + transfers = [] + for glob_pat, dest in entries: + for src in sorted(globmod.glob(os.path.join(project_dir, glob_pat), recursive=True)): + if os.path.isfile(src): + transfers.append((src, dest.rstrip('/') + '/' + os.path.basename(src))) + + if not transfers: + print('meshforge-sideload: no files matched globs — nothing to sideload') + return + + print(f'meshforge-sideload: waiting {boot_wait_s}s for device to boot...') + time.sleep(boot_wait_s) + + print(f'meshforge-sideload: connecting to {port_name} @ {baud}...') + with serial.Serial(port_name, baud, timeout=ACK_TIMEOUT_S) as port: + time.sleep(0.5) + for i, (src, device_path) in enumerate(transfers): + data = open(src, 'rb').read() + name = os.path.basename(src) + print(f' [{i+1}/{len(transfers)}] {name} ({len(data)/1024:.1f} KB) → {device_path}') + + def progress(sent, total, _name=name): + pct = 100 * sent // total + bar = '#' * (pct // 5) + '.' * (20 - pct // 5) + print(f'\r [{bar}] {pct}%', end='', flush=True) + + send_xmodem_file(port, device_path, data, on_progress=progress) + print(f'\r [{"#"*20}] 100% done') + + print(f'meshforge-sideload: {len(transfers)} file(s) uploaded successfully')