diff --git a/.gitmodules b/.gitmodules index 95f93f5..82a669e 100644 --- a/.gitmodules +++ b/.gitmodules @@ -31,3 +31,6 @@ [submodule "vendor/TinyBBS"] path = vendor/TinyBBS url = git@github.com:MeshEnvy/TinyBBS.git +[submodule "vendor/meshtastic-python"] + path = vendor/meshtastic-python + url = git@github.com:MeshEnvy/meshtastic-python.git diff --git a/meshforge-sideload/extra_scripts/register_sideload_target.py b/meshforge-sideload/extra_scripts/register_sideload_target.py index 5c79b9b..293c297 100644 --- a/meshforge-sideload/extra_scripts/register_sideload_target.py +++ b/meshforge-sideload/extra_scripts/register_sideload_target.py @@ -2,8 +2,8 @@ register_sideload_target.py — PlatformIO extra_script that registers the `sideload` custom target for the meshforge-sideload library. -Transfers data files to the device via Meshtastic StreamAPI + XModem, using -the /ext/ and /int/ path prefix convention supported by the xmodem-ext-fs patch. +Transfers data files declared in meshforge.yaml to the device using +Node.uploadFile() from the meshtastic Python library. Usage: pio run -t sideload @@ -13,221 +13,79 @@ Usage: MESHFORGE_BOOT_WAIT=5 pio run -t sideload """ -Import('env') # noqa: F821 — PlatformIO SCons environment +Import('env') # noqa: F821 -import glob as globmod import os -import re -import struct import sys import time -# ── Meshtastic StreamAPI + XModem protocol (inline) ────────────────────────── -XC_SOH = 1; XC_STX = 2; XC_EOT = 4; XC_ACK = 6; XC_NAK = 21; XC_CAN = 24 -XMODEM_BUF = 128; MAX_RETRY = 10; ACK_TIMEOUT = 5.0 - - -def _varint(n): - out = [] - while n > 0x7F: out.append((n & 0x7F) | 0x80); n >>= 7 - out.append(n); return bytes(out) - - -def _pb_bytes(field, data): - tag = _varint((field << 3) | 2) - return tag + _varint(len(data)) + data - - -def _encode_xm(control, seq, crc16, buf): - msg = _varint((1 << 3) | 0) + _varint(control) - if seq: msg += _varint((2 << 3) | 0) + _varint(seq) - if crc16: msg += _varint((3 << 3) | 0) + _varint(crc16) - if buf: msg += _pb_bytes(4, buf) - return msg - - -def _to_radio(xm): return _pb_bytes(5, xm) -def _frame(payload): return b'\x94\xc3' + struct.pack('>H', len(payload)) + payload - - -def _crc16(data): - crc = 0 - for b in data: - crc = ((crc >> 8) | (crc << 8)) & 0xFFFF; crc ^= b - crc ^= ((crc & 0xFF) >> 4) & 0xFFFF - crc ^= ((crc << 8) << 4) & 0xFFFF - crc ^= (((crc & 0xFF) << 4) << 1) & 0xFFFF - return crc & 0xFFFF - - -def _read_varint(data, pos): - val = shift = 0 - while pos < len(data): - b = data[pos]; pos += 1; val |= (b & 0x7F) << shift; shift += 7 - if not (b & 0x80): break - return val, pos - - -def _parse_xm(data): - i, r = 0, {'control': 0, 'seq': 0, 'crc16': 0, 'buffer': b''} - while i < len(data): - tag = data[i]; i += 1; fn, wt = tag >> 3, tag & 0x7 - if wt == 0: - val, i = _read_varint(data, i) - if fn == 1: r['control'] = val - elif fn == 2: r['seq'] = val - elif fn == 3: r['crc16'] = val - elif wt == 2: - l, i = _read_varint(data, i) - if fn == 4: r['buffer'] = data[i:i+l] - i += l - else: break - return r - - -def _parse_from_radio(data): - i = 0 - while i < len(data): - tag = data[i]; i += 1; fn, wt = tag >> 3, tag & 0x7 - if wt == 0: _, i = _read_varint(data, i) - elif wt == 2: - l, i = _read_varint(data, i); payload = data[i:i+l]; i += l - if fn == 12: return _parse_xm(payload) - else: break - return None - - -def _read_xm_resp(port, timeout_s=ACK_TIMEOUT): - buf = bytearray(); deadline = time.time() + timeout_s - while time.time() < deadline: - n = port.in_waiting - if n: buf += port.read(n) - for i in range(len(buf) - 1): - if buf[i] == 0x94 and buf[i+1] == 0xC3 and len(buf) >= i + 4: - length = (buf[i+2] << 8) | buf[i+3] - if len(buf) >= i + 4 + length: - payload = bytes(buf[i+4:i+4+length]); del buf[:i+4+length] - xm = _parse_from_radio(payload) - if xm: return xm - break - time.sleep(0.01) - return None - - -def _send_file(port, dest, data, on_progress=None): - # SOH — filename - fn = dest.encode('ascii') - for attempt in range(MAX_RETRY): - port.write(_frame(_to_radio(_encode_xm(XC_SOH, 0, 0, fn)))) - r = _read_xm_resp(port) - if r and r['control'] == XC_ACK: break - if attempt == MAX_RETRY - 1: raise IOError(f'XModem OPEN rejected: {dest}') - # STX data - seq = 1; off = 0 - while off < len(data): - chunk = data[off:off+XMODEM_BUF]; crc = _crc16(chunk); acked = False - for retry in range(MAX_RETRY): - port.write(_frame(_to_radio(_encode_xm(XC_STX, seq, crc, chunk)))) - r = _read_xm_resp(port) - if r and r['control'] == XC_ACK: acked = True; break - if r and r['control'] == XC_CAN: raise IOError(f'Transfer cancelled at {off}') - if not acked: raise IOError(f'No ACK for seq {seq} at offset {off}') - off += len(chunk); seq = (seq & 0xFF) + 1 - if on_progress: on_progress(off, len(data)) - # EOT - for attempt in range(MAX_RETRY): - port.write(_frame(_to_radio(_encode_xm(XC_EOT, 0, 0, b'')))) - r = _read_xm_resp(port) - if r and r['control'] == XC_ACK: return - if attempt == MAX_RETRY - 1: raise IOError(f'EOT not acked: {dest}') - - -def _parse_data_entries(yaml_text): - entries = []; in_mf = in_data = False - for raw in yaml_text.splitlines(): - line = re.sub(r'#.*$', '', raw).rstrip() - if not line.strip(): continue - indent = len(line) - len(line.lstrip()); content = line.strip() - if indent == 0: in_mf = (content == 'meshforge:'); in_data = False - elif in_mf and indent == 2: in_data = (content == 'data:') - elif in_mf and in_data and indent == 4: - m = re.match(r'^-\s+(.+)$', content) - if m: - entry = m.group(1).strip().strip('"\''); colon = entry.find(':') - if colon > 0: entries.append((entry[:colon].strip(), entry[colon+1:].strip())) - return entries +def _ensure_vendor_meshtastic(): + """Add vendor/meshtastic-python to sys.path so we can import it.""" + project_dir = env.subst('$PROJECT_DIR') # noqa: F821 + # firmware/ → TinyBBS/ → vendor/ → mesh-forge root + candidate = os.path.normpath(os.path.join(project_dir, '..', '..', '..', 'vendor', 'meshtastic-python')) + if os.path.isdir(os.path.join(candidate, 'meshtastic')) and candidate not in sys.path: + sys.path.insert(0, candidate) + # Also add tools/ dir of meshforge-sideload for mf_protocol + libdeps = env.subst('$PROJECT_LIBDEPS_DIR') # noqa: F821 + pioenv = env.subst('$PIOENV') # noqa: F821 + tools_dir = os.path.join(libdeps, pioenv, 'meshforge-sideload', 'tools') + if os.path.isdir(tools_dir) and tools_dir not in sys.path: + sys.path.insert(0, tools_dir) def _autodetect_port(): try: import serial.tools.list_ports KNOWN = ['RAK', 'nRF52', 'Adafruit', 'Nordic', 'Meshtastic', 'WisMesh', - 'T-Echo', 'CP210', 'CH340', 'FTDI', 'USB Serial', 'JTAG', 'LilyGO', 'Espressif'] + 'T-Echo', 'CP210', 'CH340', 'FTDI', 'USB Serial', 'JTAG', + 'LilyGO', 'Espressif'] ports = list(serial.tools.list_ports.comports()) for p in ports: desc = (p.description or '') + ' ' + (p.manufacturer or '') - if any(k.lower() in desc.lower() for k in KNOWN): return p.device + if any(k.lower() in desc.lower() for k in KNOWN): + return p.device for p in ports: - if 'usbmodem' in p.device or 'usbserial' in p.device: return p.device - except ImportError: pass + if 'usbmodem' in p.device or 'usbserial' in p.device: + return p.device + except ImportError: + pass return None -# ── PlatformIO target action ────────────────────────────────────────────────── - def _sideload(source, target, env): # noqa: F821 + _ensure_vendor_meshtastic() + try: - import serial - except ImportError: - print('meshforge-sideload: ERROR — pyserial not installed. Run: pip install pyserial') + from mf_protocol import run_sideload + except ImportError as e: + print(f'meshforge-sideload: ERROR importing mf_protocol: {e}') raise SystemExit(1) project_dir = env['PROJECT_DIR'] # noqa: F821 - yaml_path = os.path.join(project_dir, 'meshforge.yaml') - if not os.path.isfile(yaml_path): - print('meshforge-sideload: no meshforge.yaml — nothing to sideload'); return - - with open(yaml_path, encoding='utf-8') as f: - entries = _parse_data_entries(f.read()) - if not entries: - print('meshforge-sideload: no data: entries — nothing to sideload'); return - - transfers = [] - for glob_pat, dest in entries: - for src in sorted(globmod.glob(os.path.join(project_dir, glob_pat), recursive=True)): - if os.path.isfile(src): - transfers.append((src, dest.rstrip('/') + '/' + os.path.basename(src))) - if not transfers: - print('meshforge-sideload: no files matched — nothing to sideload'); return - - port_name = env.get('UPLOAD_PORT') or os.environ.get('MESHFORGE_PORT') or _autodetect_port() + port_name = ( + env.get('UPLOAD_PORT') + or os.environ.get('MESHFORGE_PORT') + or _autodetect_port() + ) if not port_name: - print('meshforge-sideload: ERROR — no serial port found.\n' - ' Pass one: pio run -t sideload --upload-port /dev/cu.usbmodemXXXX') + print( + 'meshforge-sideload: ERROR — no serial port found.\n' + ' Pass one: pio run -t sideload --upload-port /dev/cu.usbmodemXXXX\n' + ' Or set: MESHFORGE_PORT=/dev/cu.usbmodemXXXX pio run -t sideload' + ) raise SystemExit(1) boot_wait = float(os.environ.get('MESHFORGE_BOOT_WAIT', '3')) - baud = int(env.get('MONITOR_SPEED', 115200)) + baud = int(env.get('MONITOR_SPEED', 115200)) - print(f'meshforge-sideload: waiting {boot_wait}s for device to boot...') - time.sleep(boot_wait) - print(f'meshforge-sideload: connecting to {port_name} @ {baud}...') - - with serial.Serial(port_name, baud, timeout=ACK_TIMEOUT) as port: - time.sleep(0.5) - for i, (src, device_path) in enumerate(transfers): - data = open(src, 'rb').read() - name = os.path.basename(src) - print(f' [{i+1}/{len(transfers)}] {name} ({len(data)/1024:.1f} KB) → {device_path}') - def progress(sent, total, _name=name): - pct = 100 * sent // total - bar = '#' * (pct // 5) + '.' * (20 - pct // 5) - print(f'\r [{bar}] {pct}%', end='', flush=True) - _send_file(port, device_path, data, on_progress=progress) - print(f'\r [{"#"*20}] 100% done') - - print(f'meshforge-sideload: {len(transfers)} file(s) uploaded successfully') + run_sideload( + project_dir=project_dir, + port_name=port_name, + baud=baud, + boot_wait_s=boot_wait, + ) env.AddCustomTarget( # noqa: F821 @@ -235,5 +93,5 @@ env.AddCustomTarget( # noqa: F821 dependencies=None, actions=_sideload, title='MeshForge Sideload', - description='Upload meshforge.yaml data files via Meshtastic StreamAPI + XModem', + description='Upload meshforge.yaml data files to device via Meshtastic XModem', ) diff --git a/meshforge-sideload/patches/00-xmodem-truncate-fix.patch b/meshforge-sideload/patches/00-xmodem-truncate-fix.patch new file mode 100644 index 0000000..e8395f1 --- /dev/null +++ b/meshforge-sideload/patches/00-xmodem-truncate-fix.patch @@ -0,0 +1,30 @@ +From 488d40597572d54b19245e23282fcf2b1cd2140c Mon Sep 17 00:00:00 2001 +From: Ben Allfree +Date: Sun, 12 Apr 2026 13:33:44 -0700 +Subject: [PATCH] fix(xmodem): truncate file on open instead of appending + +FILE_O_WRITE on nRF52 (Adafruit LittleFS) appends to existing files +rather than truncating. Remove the file before opening for write so +repeated XModem uploads always produce the correct file size. + +Made-with: Cursor +--- + src/xmodem.cpp | 2 ++ + 1 file changed, 2 insertions(+) + +diff --git a/src/xmodem.cpp b/src/xmodem.cpp +index 1d8c77760..cdbbaefe1 100644 +--- a/src/xmodem.cpp ++++ b/src/xmodem.cpp +@@ -123,6 +123,8 @@ void XModemAdapter::handlePacket(meshtastic_XModem xmodemPacket) + + if (xmodemPacket.control == meshtastic_XModem_Control_SOH) { // Receive this file and put to Flash + spiLock->lock(); ++ // Remove existing file first so we truncate rather than append ++ if (FSCom.exists(filename)) FSCom.remove(filename); + file = FSCom.open(filename, FILE_O_WRITE); + spiLock->unlock(); + if (file) { +-- +2.50.1 + diff --git a/meshforge-sideload/patches/02-xmodem-vfs-routing.patch b/meshforge-sideload/patches/02-xmodem-vfs-routing.patch index ad64e84..4102463 100644 --- a/meshforge-sideload/patches/02-xmodem-vfs-routing.patch +++ b/meshforge-sideload/patches/02-xmodem-vfs-routing.patch @@ -1,15 +1,3 @@ -From 697b1317e012c74751ac995794b42bfbce967b14 Mon Sep 17 00:00:00 2001 -From: Ben Allfree -Date: Sun, 12 Apr 2026 12:04:17 -0700 -Subject: [PATCH] Add FS routing support for virtual mount points in FSCommon - ---- - src/FSCommon.cpp | 94 ++++++++++++++++++++++++++++++++++++++++++++++++ - src/FSCommon.h | 42 +++++++++++++++++++++- - src/xmodem.cpp | 9 +++-- - src/xmodem.h | 3 +- - 4 files changed, 141 insertions(+), 7 deletions(-) - diff --git a/src/FSCommon.cpp b/src/FSCommon.cpp index 2c14f308c..79e927e64 100644 --- a/src/FSCommon.cpp @@ -168,10 +156,10 @@ index 9b3de20c2..1ac0d04a1 100644 +#endif // FSCom \ No newline at end of file diff --git a/src/xmodem.cpp b/src/xmodem.cpp -index 1d8c77760..7c67d7fbf 100644 +index 1d8c77760..847fe49c8 100644 --- a/src/xmodem.cpp +++ b/src/xmodem.cpp -@@ -120,10 +120,11 @@ void XModemAdapter::handlePacket(meshtastic_XModem xmodemPacket) +@@ -120,10 +120,13 @@ void XModemAdapter::handlePacket(meshtastic_XModem xmodemPacket) if ((xmodemPacket.seq == 0) && !isReceiving && !isTransmitting) { // NULL packet has the destination filename memcpy(filename, &xmodemPacket.buffer.bytes, xmodemPacket.buffer.size); @@ -180,11 +168,13 @@ index 1d8c77760..7c67d7fbf 100644 if (xmodemPacket.control == meshtastic_XModem_Control_SOH) { // Receive this file and put to Flash spiLock->lock(); - file = FSCom.open(filename, FILE_O_WRITE); ++ // Remove existing file first so we truncate rather than append ++ if (FSCom.exists(filename)) FSCom.remove(filename); + file = fsOpenWrite(activeRoute_); spiLock->unlock(); if (file) { sendControl(meshtastic_XModem_Control_ACK); -@@ -137,7 +138,7 @@ void XModemAdapter::handlePacket(meshtastic_XModem xmodemPacket) +@@ -137,7 +140,7 @@ void XModemAdapter::handlePacket(meshtastic_XModem xmodemPacket) } else { // Transmit this file from Flash LOG_INFO("XModem: Transmit file %s", filename); spiLock->lock(); @@ -193,7 +183,7 @@ index 1d8c77760..7c67d7fbf 100644 spiLock->unlock(); if (file) { packetno = 1; -@@ -200,8 +201,7 @@ void XModemAdapter::handlePacket(meshtastic_XModem xmodemPacket) +@@ -200,8 +203,7 @@ void XModemAdapter::handlePacket(meshtastic_XModem xmodemPacket) spiLock->lock(); file.flush(); file.close(); @@ -203,7 +193,7 @@ index 1d8c77760..7c67d7fbf 100644 spiLock->unlock(); isReceiving = false; break; -@@ -255,7 +255,6 @@ void XModemAdapter::handlePacket(meshtastic_XModem xmodemPacket) +@@ -255,7 +257,6 @@ void XModemAdapter::handlePacket(meshtastic_XModem xmodemPacket) xmodemStore.seq = packetno; spiLock->lock(); file.seek((packetno - 1) * sizeof(meshtastic_XModem_buffer_t::bytes)); @@ -225,6 +215,3 @@ index 4cfcb43e1..48ff25fbb 100644 protected: meshtastic_XModem xmodemStore = meshtastic_XModem_init_zero; --- -2.50.1 - diff --git a/meshforge-sideload/patches/03-xmodem-esp32-serial-polling.patch b/meshforge-sideload/patches/03-xmodem-esp32-serial-polling.patch new file mode 100644 index 0000000..9aa1cef --- /dev/null +++ b/meshforge-sideload/patches/03-xmodem-esp32-serial-polling.patch @@ -0,0 +1,83 @@ +commit b5de3323b3e106edb28efcda815d86a066ef34f4 +Author: Ben Allfree +Date: Sun Apr 12 21:31:37 2026 -0700 + + tdeck fixes + +diff --git a/src/SerialConsole.cpp b/src/SerialConsole.cpp +index e24aa3c57..c81854a7b 100644 +--- a/src/SerialConsole.cpp ++++ b/src/SerialConsole.cpp +@@ -5,6 +5,7 @@ + #include "Throttle.h" + #include "configuration.h" + #include "time.h" ++#include "xmodem.h" + + #if defined(ARDUINO_USB_CDC_ON_BOOT) && ARDUINO_USB_CDC_ON_BOOT + #define IS_USB_SERIAL +@@ -92,7 +93,7 @@ int32_t SerialConsole::runOnce() + #if defined(SERIAL_HAS_ON_RECEIVE) || defined(CONFIG_IDF_TARGET_ESP32S2) + return Port.available() ? delay : INT32_MAX; + #elif defined(IS_USB_SERIAL) +- return HWCDC::isPlugged() ? delay : (1000 * 20); ++ return (HWCDC::isPlugged() || xModem.isActive() || delay < 250) ? delay : (1000 * 20); + #else + return delay; + #endif +diff --git a/src/xmodem.cpp b/src/xmodem.cpp +index ccd6a1b95..3155a6526 100644 +--- a/src/xmodem.cpp ++++ b/src/xmodem.cpp +@@ -152,8 +152,12 @@ void XModemAdapter::handlePacket(meshtastic_XModem xmodemPacket) + case meshtastic_XModem_Control_SOH: + case meshtastic_XModem_Control_STX: + if ((xmodemPacket.seq == 0) && !isReceiving && !isTransmitting) { +- // NULL packet has the destination filename +- memcpy(filename, &xmodemPacket.buffer.bytes, xmodemPacket.buffer.size); ++ // NULL packet has the destination filename (protobuf bytes are not NUL-terminated) ++ size_t n = xmodemPacket.buffer.size; ++ if (n >= sizeof(filename)) ++ n = sizeof(filename) - 1; ++ memcpy(filename, xmodemPacket.buffer.bytes, n); ++ filename[n] = '\0'; + activeRoute_ = fsRoute(filename); + + if (xmodemPacket.control == meshtastic_XModem_Control_SOH) { // Receive this file and put to Flash +@@ -201,6 +205,16 @@ void XModemAdapter::handlePacket(meshtastic_XModem xmodemPacket) + } + } else { + if (isReceiving) { ++ if (xmodemPacket.seq == 0) { ++ // Duplicate OPEN retry (client re-sent while already receiving) — re-ACK so Python proceeds. ++ sendControl(meshtastic_XModem_Control_ACK); ++ break; ++ } ++ if (xmodemPacket.seq + 1 == packetno) { ++ // Already-delivered packet still in flight (stale serial buffer retry) — re-ACK. ++ sendControl(meshtastic_XModem_Control_ACK); ++ break; ++ } + // normal file data packet + if ((xmodemPacket.seq == packetno) && + check(xmodemPacket.buffer.bytes, xmodemPacket.buffer.size, xmodemPacket.crc16)) { +diff --git a/src/xmodem.h b/src/xmodem.h +index 48ff25fbb..5b8c4930d 100644 +--- a/src/xmodem.h ++++ b/src/xmodem.h +@@ -51,6 +51,7 @@ class XModemAdapter + void handlePacket(meshtastic_XModem xmodemPacket); + meshtastic_XModem getForPhone(); + void resetForPhone(); ++ bool isActive() const { return isReceiving || isTransmitting; } + + private: + bool isReceiving = false; +@@ -61,6 +62,7 @@ class XModemAdapter + + uint16_t packetno = 0; + ++// Adafruit nRF/STM32 File can be constructed bound to FSCom; Arduino-ESP32 fs::File cannot. + #if defined(ARCH_NRF52) || defined(ARCH_STM32WL) + File file = File(FSCom); + #else diff --git a/meshforge-sideload/patches/README.md b/meshforge-sideload/patches/README.md index 538c373..8f99db0 100644 --- a/meshforge-sideload/patches/README.md +++ b/meshforge-sideload/patches/README.md @@ -6,6 +6,19 @@ become unnecessary. --- +## 00-xmodem-truncate-fix.patch + +**PR title:** `fix(xmodem): truncate file on open instead of appending` + +Standalone 2-line fix. `FILE_O_WRITE` on nRF52 (Adafruit LittleFS) appends to +existing files rather than truncating. Removes the file before opening for write +so repeated XModem uploads always produce the correct file size. Independent of +the other two patches — can be reviewed and merged on its own. + +**Sentinel:** `Remove existing file first so we truncate` + +--- + ## 01-nrf-external-flash.patch **PR title:** `feat(nrf52): optional external QSPI LittleFS filesystem` diff --git a/meshforge-sideload/patches/apply_patches.py b/meshforge-sideload/patches/apply_patches.py index eabb4dc..cd7f288 100644 --- a/meshforge-sideload/patches/apply_patches.py +++ b/meshforge-sideload/patches/apply_patches.py @@ -22,8 +22,9 @@ import subprocess import sys _PATCHES = [ - ('01-nrf-external-flash.patch', 'extFSInit'), # sentinel unique to patch 1 - ('02-xmodem-vfs-routing.patch', 'fsRoute'), # sentinel unique to patch 2 + ('00-xmodem-truncate-fix.patch', 'Remove existing file first so we truncate'), # standalone fix + ('01-nrf-external-flash.patch', 'extFSInit'), # PR1: external flash FS + ('02-xmodem-vfs-routing.patch', 'fsRoute'), # PR2: VFS routing + XModem ] _PATCH_DIR = os.path.dirname(os.path.abspath(__file__)) diff --git a/meshforge-sideload/tools/mf_protocol.py b/meshforge-sideload/tools/mf_protocol.py index 2286055..711d3f7 100644 --- a/meshforge-sideload/tools/mf_protocol.py +++ b/meshforge-sideload/tools/mf_protocol.py @@ -1,214 +1,51 @@ """ -mf_protocol.py — Meshtastic StreamAPI + XModem file transfer (Python host side). +mf_protocol.py — MeshForge sideload via meshtastic-python XModem. -Matches the patched xmodem.cpp on the firmware side. +Uses Node.uploadFile() from the meshtastic Python library (vendored at +vendor/meshtastic-python) for all file transfers. -Meshtastic StreamAPI framing: - 0x94 0xC3 - -ToRadio.xmodemPacket = field 5 -FromRadio.xmodemPacket = field 12 - -meshtastic_XModem fields: - control (1, varint) — XModem Control enum - seq (2, varint) — packet sequence - crc16 (3, varint) — CRC-16-CCITT of buffer - buffer (4, bytes) — up to 128 bytes +Requires: pip install pyserial (meshtastic is loaded from the vendor submodule) """ import glob as globmod import os import re -import struct import sys import time -# XModem Control enum values -XC_NUL = 0 -XC_SOH = 1 -XC_STX = 2 -XC_EOT = 4 -XC_ACK = 6 -XC_NAK = 21 -XC_CAN = 24 -XMODEM_BUFFER_SIZE = 128 -MAX_RETRIES = 10 -ACK_TIMEOUT_S = 5.0 +# ── Resolve the vendored meshtastic-python library ──────────────────────────── +# Walk up from this file to find the mesh-forge repo root, then add the +# vendored library to sys.path. - -# ── CRC-16-CCITT ────────────────────────────────────────────────────────────── - -def crc16_ccitt(data: bytes) -> int: - crc = 0 - for b in data: - crc = ((crc >> 8) | (crc << 8)) & 0xFFFF - crc ^= b - crc ^= ((crc & 0xFF) >> 4) & 0xFFFF - crc ^= ((crc << 8) << 4) & 0xFFFF - crc ^= (((crc & 0xFF) << 4) << 1) & 0xFFFF - return crc & 0xFFFF - - -# ── Minimal protobuf encoding ───────────────────────────────────────────────── - -def varint(n: int) -> bytes: - out = [] - while n > 0x7F: - out.append((n & 0x7F) | 0x80) - n >>= 7 - out.append(n) - return bytes(out) - - -def pb_bytes_field(field: int, data: bytes) -> bytes: - tag = varint((field << 3) | 2) - return tag + varint(len(data)) + data - - -def encode_xmodem(control: int, seq: int, crc16: int, buffer: bytes) -> bytes: - msg = varint((1 << 3) | 0) + varint(control) - if seq: msg += varint((2 << 3) | 0) + varint(seq) - if crc16: msg += varint((3 << 3) | 0) + varint(crc16) - if buffer: msg += pb_bytes_field(4, buffer) - return msg - - -def encode_toradio(xmodem_bytes: bytes) -> bytes: - return pb_bytes_field(5, xmodem_bytes) - - -def stream_frame(payload: bytes) -> bytes: - return b'\x94\xc3' + struct.pack('>H', len(payload)) + payload - - -# ── FromRadio response parsing ──────────────────────────────────────────────── - -def _parse_varint(data: bytes, pos: int): - val, shift = 0, 0 - while pos < len(data): - b = data[pos]; pos += 1 - val |= (b & 0x7F) << shift; shift += 7 - if not (b & 0x80): break - return val, pos - - -def parse_xmodem(data: bytes) -> dict: - i, result = 0, {'control': 0, 'seq': 0, 'crc16': 0, 'buffer': b''} - while i < len(data): - tag = data[i]; i += 1 - fn, wt = tag >> 3, tag & 0x7 - if wt == 0: - val, i = _parse_varint(data, i) - if fn == 1: result['control'] = val - elif fn == 2: result['seq'] = val - elif fn == 3: result['crc16'] = val - elif wt == 2: - l, i = _parse_varint(data, i) - if fn == 4: result['buffer'] = data[i:i+l] - i += l - else: break - return result - - -def parse_from_radio(data: bytes) -> dict | None: - i = 0 - while i < len(data): - tag = data[i]; i += 1 - fn, wt = tag >> 3, tag & 0x7 - if wt == 0: - _, i = _parse_varint(data, i) - elif wt == 2: - l, i = _parse_varint(data, i) - payload = data[i:i+l]; i += l - if fn == 12: - return parse_xmodem(payload) - else: break +def _find_vendor_meshtastic(): + here = os.path.dirname(os.path.abspath(__file__)) + # tools/ → meshforge-sideload/ → mesh-forge root + candidate = os.path.normpath(os.path.join(here, '..', '..', 'vendor', 'meshtastic-python')) + if os.path.isdir(os.path.join(candidate, 'meshtastic')): + return candidate return None - -# ── StreamAPI frame reader ──────────────────────────────────────────────────── - -def read_xmodem_response(port, timeout_s: float = ACK_TIMEOUT_S) -> dict | None: - buf = bytearray() - deadline = time.time() + timeout_s - while time.time() < deadline: - n = port.in_waiting - if n: buf += port.read(n) - # Scan for 0x94 0xC3 frame - start = -1 - for i in range(len(buf) - 1): - if buf[i] == 0x94 and buf[i+1] == 0xC3: - start = i; break - if start >= 0 and len(buf) >= start + 4: - length = (buf[start+2] << 8) | buf[start+3] - if len(buf) >= start + 4 + length: - payload = bytes(buf[start+4:start+4+length]) - del buf[:start+4+length] - xm = parse_from_radio(payload) - if xm: return xm - # Not an XModem frame — keep reading - continue - time.sleep(0.01) - return None +_vendor_path = _find_vendor_meshtastic() +if _vendor_path and _vendor_path not in sys.path: + sys.path.insert(0, _vendor_path) -# ── File transfer ───────────────────────────────────────────────────────────── - -def send_xmodem_file(port, dest_path: str, data: bytes, - on_progress=None) -> None: - # SOH seq=0 — filename handshake - fn_bytes = dest_path.encode('ascii') - for attempt in range(MAX_RETRIES): - port.write(stream_frame(encode_toradio(encode_xmodem(XC_SOH, 0, 0, fn_bytes)))) - resp = read_xmodem_response(port) - if resp and resp['control'] == XC_ACK: - break - if attempt == MAX_RETRIES - 1: - raise IOError(f'XModem OPEN rejected for {dest_path}') - - # STX data packets - seq = 1 - offset = 0 - while offset < len(data): - chunk = data[offset:offset + XMODEM_BUFFER_SIZE] - crc = crc16_ccitt(chunk) - acked = False - for retry in range(MAX_RETRIES): - port.write(stream_frame(encode_toradio(encode_xmodem(XC_STX, seq, crc, chunk)))) - resp = read_xmodem_response(port) - if resp and resp['control'] == XC_ACK: - acked = True; break - if resp and resp['control'] == XC_CAN: - raise IOError(f'XModem transfer cancelled at offset {offset}') - if not acked: - raise IOError(f'XModem: no ACK for seq {seq} at offset {offset}') - offset += len(chunk) - seq = (seq & 0xFF) + 1 - if on_progress: on_progress(offset, len(data)) - - # EOT - for attempt in range(MAX_RETRIES): - port.write(stream_frame(encode_toradio(encode_xmodem(XC_EOT, 0, 0, b'')))) - resp = read_xmodem_response(port) - if resp and resp['control'] == XC_ACK: - return - if attempt == MAX_RETRIES - 1: - raise IOError(f'XModem EOT not acknowledged for {dest_path}') - - -# ── meshforge.yaml parser (minimal) ────────────────────────────────────────── +# ── meshforge.yaml parser ───────────────────────────────────────────────────── def parse_data_entries(yaml_text: str) -> list: + """Return list of (glob_pattern, device_dest) from meshforge.yaml data: section.""" entries = [] in_mf = in_data = False for raw in yaml_text.splitlines(): line = re.sub(r'#.*$', '', raw).rstrip() - if not line.strip(): continue + if not line.strip(): + continue indent = len(line) - len(line.lstrip()) content = line.strip() if indent == 0: - in_mf = (content == 'meshforge:'); in_data = False + in_mf = (content == 'meshforge:') + in_data = False elif in_mf and indent == 2: in_data = (content == 'data:') elif in_mf and in_data and indent == 4: @@ -217,7 +54,7 @@ def parse_data_entries(yaml_text: str) -> list: entry = m.group(1).strip().strip('"\'') colon = entry.find(':') if colon > 0: - entries.append((entry[:colon].strip(), entry[colon+1:].strip())) + entries.append((entry[:colon].strip(), entry[colon + 1:].strip())) return entries @@ -226,9 +63,9 @@ def parse_data_entries(yaml_text: str) -> list: def autodetect_port(): try: import serial.tools.list_ports - KNOWN = ['RAK', 'nRF52', 'Adafruit', 'Nordic', 'Meshtastic', - 'WisMesh', 'T-Echo', 'CP210', 'CH340', 'FTDI', 'USB Serial', - 'JTAG', 'LilyGO', 'Espressif'] + KNOWN = ['RAK', 'nRF52', 'Adafruit', 'Nordic', 'Meshtastic', 'WisMesh', + 'T-Echo', 'CP210', 'CH340', 'FTDI', 'USB Serial', 'JTAG', + 'LilyGO', 'Espressif'] ports = list(serial.tools.list_ports.comports()) for p in ports: desc = (p.description or '') + ' ' + (p.manufacturer or '') @@ -247,9 +84,11 @@ def autodetect_port(): def run_sideload(project_dir: str, port_name: str, baud: int = 115200, boot_wait_s: float = 3.0) -> None: try: - import serial + import meshtastic.serial_interface except ImportError: - print('ERROR: pyserial not installed. Run: pip install pyserial') + print('ERROR: meshtastic library not found.\n' + 'Install with: pip install meshtastic\n' + 'Or ensure vendor/meshtastic-python is in sys.path') sys.exit(1) yaml_path = os.path.join(project_dir, 'meshforge.yaml') @@ -271,26 +110,33 @@ def run_sideload(project_dir: str, port_name: str, baud: int = 115200, transfers.append((src, dest.rstrip('/') + '/' + os.path.basename(src))) if not transfers: - print('meshforge-sideload: no files matched globs — nothing to sideload') + print('meshforge-sideload: no files matched — nothing to sideload') return print(f'meshforge-sideload: waiting {boot_wait_s}s for device to boot...') time.sleep(boot_wait_s) - print(f'meshforge-sideload: connecting to {port_name} @ {baud}...') - with serial.Serial(port_name, baud, timeout=ACK_TIMEOUT_S) as port: - time.sleep(0.5) + print(f'meshforge-sideload: connecting to {port_name}...') + iface = meshtastic.serial_interface.SerialInterface(port_name) + + try: for i, (src, device_path) in enumerate(transfers): - data = open(src, 'rb').read() + size_kb = os.path.getsize(src) / 1024 name = os.path.basename(src) - print(f' [{i+1}/{len(transfers)}] {name} ({len(data)/1024:.1f} KB) → {device_path}') + print(f' [{i + 1}/{len(transfers)}] {name} ({size_kb:.1f} KB) → {device_path}') def progress(sent, total, _name=name): pct = 100 * sent // total bar = '#' * (pct // 5) + '.' * (20 - pct // 5) print(f'\r [{bar}] {pct}%', end='', flush=True) - send_xmodem_file(port, device_path, data, on_progress=progress) - print(f'\r [{"#"*20}] 100% done') + ok = iface.localNode.uploadFile(src, device_path, on_progress=progress) + if ok: + print(f'\r [{"#" * 20}] 100% done') + else: + print(f'\r FAILED') + raise RuntimeError(f'Upload failed: {src} → {device_path}') + finally: + iface.close() print(f'meshforge-sideload: {len(transfers)} file(s) uploaded successfully') diff --git a/vendor/meshtastic-python b/vendor/meshtastic-python new file mode 160000 index 0000000..cec79a7 --- /dev/null +++ b/vendor/meshtastic-python @@ -0,0 +1 @@ +Subproject commit cec79a7c1f10c189588f3aae2a7370f9ce9904de