adam/mesh-forge
1
0
Fork 0
mirror of https://github.com/MeshEnvy/mesh-forge.git synced 2026-08-07 01:13:05 +02:00
Code Issues Packages Projects Releases Wiki Activity

sideload updates

Browse Source
This commit is contained in:
Ben Allfree
2026-04-13 04:18:31 -07:00
parent 8c35bfd5d7
commit 07482650da
9 changed files with 232 additions and 410 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.gitmodules
+3
View File
@@ -31,3 +31,6 @@
[submodule "vendor/TinyBBS"]
path = vendor/TinyBBS
url = git@github.com:MeshEnvy/TinyBBS.git
[submodule "vendor/meshtastic-python"]
path = vendor/meshtastic-python
url = git@github.com:MeshEnvy/meshtastic-python.git
meshforge-sideload/extra_scripts/register_sideload_target.py
+47 -189
View File
@@ -2,8 +2,8 @@
register_sideload_target.py — PlatformIO extra_script that registers the
`sideload` custom target for the meshforge-sideload library.
Transfers data files to the device via Meshtastic StreamAPI + XModem, using
the /ext/ and /int/ path prefix convention supported by the xmodem-ext-fs patch.
Transfers data files declared in meshforge.yaml to the device using
Node.uploadFile() from the meshtastic Python library.
Usage:
pio run -t sideload
@@ -13,221 +13,79 @@ Usage:
MESHFORGE_BOOT_WAIT=5 pio run -t sideload
"""
Import('env') # noqa: F821 — PlatformIO SCons environment
Import('env') # noqa: F821
import glob as globmod
import os
import re
import struct
import sys
import time
# ── Meshtastic StreamAPI + XModem protocol (inline) ──────────────────────────
XC_SOH = 1; XC_STX = 2; XC_EOT = 4; XC_ACK = 6; XC_NAK = 21; XC_CAN = 24
XMODEM_BUF = 128; MAX_RETRY = 10; ACK_TIMEOUT = 5.0
def _varint(n):
out = []
while n > 0x7F: out.append((n & 0x7F) | 0x80); n >>= 7
out.append(n); return bytes(out)
def _pb_bytes(field, data):
tag = _varint((field << 3) | 2)
return tag + _varint(len(data)) + data
def _encode_xm(control, seq, crc16, buf):
msg = _varint((1 << 3) | 0) + _varint(control)
if seq: msg += _varint((2 << 3) | 0) + _varint(seq)
if crc16: msg += _varint((3 << 3) | 0) + _varint(crc16)
if buf: msg += _pb_bytes(4, buf)
return msg
def _to_radio(xm): return _pb_bytes(5, xm)
def _frame(payload): return b'\x94\xc3' + struct.pack('>H', len(payload)) + payload
def _crc16(data):
crc = 0
for b in data:
crc = ((crc >> 8) | (crc << 8)) & 0xFFFF; crc ^= b
crc ^= ((crc & 0xFF) >> 4) & 0xFFFF
crc ^= ((crc << 8) << 4) & 0xFFFF
crc ^= (((crc & 0xFF) << 4) << 1) & 0xFFFF
return crc & 0xFFFF
def _read_varint(data, pos):
val = shift = 0
while pos < len(data):
b = data[pos]; pos += 1; val |= (b & 0x7F) << shift; shift += 7
if not (b & 0x80): break
return val, pos
def _parse_xm(data):
i, r = 0, {'control': 0, 'seq': 0, 'crc16': 0, 'buffer': b''}
while i < len(data):
tag = data[i]; i += 1; fn, wt = tag >> 3, tag & 0x7
if wt == 0:
val, i = _read_varint(data, i)
if fn == 1: r['control'] = val
elif fn == 2: r['seq'] = val
elif fn == 3: r['crc16'] = val
elif wt == 2:
l, i = _read_varint(data, i)
if fn == 4: r['buffer'] = data[i:i+l]
i += l
else: break
return r
def _parse_from_radio(data):
i = 0
while i < len(data):
tag = data[i]; i += 1; fn, wt = tag >> 3, tag & 0x7
if wt == 0: _, i = _read_varint(data, i)
elif wt == 2:
l, i = _read_varint(data, i); payload = data[i:i+l]; i += l
if fn == 12: return _parse_xm(payload)
else: break
return None
def _read_xm_resp(port, timeout_s=ACK_TIMEOUT):
buf = bytearray(); deadline = time.time() + timeout_s
while time.time() < deadline:
n = port.in_waiting
if n: buf += port.read(n)
for i in range(len(buf) - 1):
if buf[i] == 0x94 and buf[i+1] == 0xC3 and len(buf) >= i + 4:
length = (buf[i+2] << 8) | buf[i+3]
if len(buf) >= i + 4 + length:
payload = bytes(buf[i+4:i+4+length]); del buf[:i+4+length]
xm = _parse_from_radio(payload)
if xm: return xm
break
time.sleep(0.01)
return None
def _send_file(port, dest, data, on_progress=None):
# SOH — filename
fn = dest.encode('ascii')
for attempt in range(MAX_RETRY):
port.write(_frame(_to_radio(_encode_xm(XC_SOH, 0, 0, fn))))
r = _read_xm_resp(port)
if r and r['control'] == XC_ACK: break
if attempt == MAX_RETRY - 1: raise IOError(f'XModem OPEN rejected: {dest}')
# STX data
seq = 1; off = 0
while off < len(data):
chunk = data[off:off+XMODEM_BUF]; crc = _crc16(chunk); acked = False
for retry in range(MAX_RETRY):
port.write(_frame(_to_radio(_encode_xm(XC_STX, seq, crc, chunk))))
r = _read_xm_resp(port)
if r and r['control'] == XC_ACK: acked = True; break
if r and r['control'] == XC_CAN: raise IOError(f'Transfer cancelled at {off}')
if not acked: raise IOError(f'No ACK for seq {seq} at offset {off}')
off += len(chunk); seq = (seq & 0xFF) + 1
if on_progress: on_progress(off, len(data))
# EOT
for attempt in range(MAX_RETRY):
port.write(_frame(_to_radio(_encode_xm(XC_EOT, 0, 0, b''))))
r = _read_xm_resp(port)
if r and r['control'] == XC_ACK: return
if attempt == MAX_RETRY - 1: raise IOError(f'EOT not acked: {dest}')
def _parse_data_entries(yaml_text):
entries = []; in_mf = in_data = False
for raw in yaml_text.splitlines():
line = re.sub(r'#.*$', '', raw).rstrip()
if not line.strip(): continue
indent = len(line) - len(line.lstrip()); content = line.strip()
if indent == 0: in_mf = (content == 'meshforge:'); in_data = False
elif in_mf and indent == 2: in_data = (content == 'data:')
elif in_mf and in_data and indent == 4:
m = re.match(r'^-\s+(.+)$', content)
if m:
entry = m.group(1).strip().strip('"\''); colon = entry.find(':')
if colon > 0: entries.append((entry[:colon].strip(), entry[colon+1:].strip()))
return entries
def _ensure_vendor_meshtastic():
"""Add vendor/meshtastic-python to sys.path so we can import it."""
project_dir = env.subst('$PROJECT_DIR') # noqa: F821
# firmware/ → TinyBBS/ → vendor/ → mesh-forge root
candidate = os.path.normpath(os.path.join(project_dir, '..', '..', '..', 'vendor', 'meshtastic-python'))
if os.path.isdir(os.path.join(candidate, 'meshtastic')) and candidate not in sys.path:
sys.path.insert(0, candidate)
# Also add tools/ dir of meshforge-sideload for mf_protocol
libdeps = env.subst('$PROJECT_LIBDEPS_DIR') # noqa: F821
pioenv = env.subst('$PIOENV') # noqa: F821
tools_dir = os.path.join(libdeps, pioenv, 'meshforge-sideload', 'tools')
if os.path.isdir(tools_dir) and tools_dir not in sys.path:
sys.path.insert(0, tools_dir)
def _autodetect_port():
try:
import serial.tools.list_ports
KNOWN = ['RAK', 'nRF52', 'Adafruit', 'Nordic', 'Meshtastic', 'WisMesh',
'T-Echo', 'CP210', 'CH340', 'FTDI', 'USB Serial', 'JTAG', 'LilyGO', 'Espressif']
'T-Echo', 'CP210', 'CH340', 'FTDI', 'USB Serial', 'JTAG',
'LilyGO', 'Espressif']
ports = list(serial.tools.list_ports.comports())
for p in ports:
desc = (p.description or '') + ' ' + (p.manufacturer or '')
if any(k.lower() in desc.lower() for k in KNOWN): return p.device
if any(k.lower() in desc.lower() for k in KNOWN):
return p.device
for p in ports:
if 'usbmodem' in p.device or 'usbserial' in p.device: return p.device
except ImportError: pass
if 'usbmodem' in p.device or 'usbserial' in p.device:
return p.device
except ImportError:
pass
return None
# ── PlatformIO target action ──────────────────────────────────────────────────
def _sideload(source, target, env): # noqa: F821
_ensure_vendor_meshtastic()
try:
import serial
except ImportError:
print('meshforge-sideload: ERROR — pyserial not installed. Run: pip install pyserial')
from mf_protocol import run_sideload
except ImportError as e:
print(f'meshforge-sideload: ERROR importing mf_protocol: {e}')
raise SystemExit(1)
project_dir = env['PROJECT_DIR'] # noqa: F821
yaml_path = os.path.join(project_dir, 'meshforge.yaml')
if not os.path.isfile(yaml_path):
print('meshforge-sideload: no meshforge.yaml — nothing to sideload'); return
with open(yaml_path, encoding='utf-8') as f:
entries = _parse_data_entries(f.read())
if not entries:
print('meshforge-sideload: no data: entries — nothing to sideload'); return
transfers = []
for glob_pat, dest in entries:
for src in sorted(globmod.glob(os.path.join(project_dir, glob_pat), recursive=True)):
if os.path.isfile(src):
transfers.append((src, dest.rstrip('/') + '/' + os.path.basename(src)))
if not transfers:
print('meshforge-sideload: no files matched — nothing to sideload'); return
port_name = env.get('UPLOAD_PORT') or os.environ.get('MESHFORGE_PORT') or _autodetect_port()
port_name = (
env.get('UPLOAD_PORT')
or os.environ.get('MESHFORGE_PORT')
or _autodetect_port()
)
if not port_name:
print('meshforge-sideload: ERROR — no serial port found.\n'
' Pass one: pio run -t sideload --upload-port /dev/cu.usbmodemXXXX')
print(
'meshforge-sideload: ERROR — no serial port found.\n'
' Pass one: pio run -t sideload --upload-port /dev/cu.usbmodemXXXX\n'
' Or set: MESHFORGE_PORT=/dev/cu.usbmodemXXXX pio run -t sideload'
)
raise SystemExit(1)
boot_wait = float(os.environ.get('MESHFORGE_BOOT_WAIT', '3'))
baud = int(env.get('MONITOR_SPEED', 115200))
baud = int(env.get('MONITOR_SPEED', 115200))
print(f'meshforge-sideload: waiting {boot_wait}s for device to boot...')
time.sleep(boot_wait)
print(f'meshforge-sideload: connecting to {port_name} @ {baud}...')
with serial.Serial(port_name, baud, timeout=ACK_TIMEOUT) as port:
time.sleep(0.5)
for i, (src, device_path) in enumerate(transfers):
data = open(src, 'rb').read()
name = os.path.basename(src)
print(f' [{i+1}/{len(transfers)}] {name} ({len(data)/1024:.1f} KB) → {device_path}')
def progress(sent, total, _name=name):
pct = 100 * sent // total
bar = '#' * (pct // 5) + '.' * (20 - pct // 5)
print(f'\r [{bar}] {pct}%', end='', flush=True)
_send_file(port, device_path, data, on_progress=progress)
print(f'\r [{"#"*20}] 100% done')
print(f'meshforge-sideload: {len(transfers)} file(s) uploaded successfully')
run_sideload(
project_dir=project_dir,
port_name=port_name,
baud=baud,
boot_wait_s=boot_wait,
)
env.AddCustomTarget( # noqa: F821
@@ -235,5 +93,5 @@ env.AddCustomTarget( # noqa: F821
dependencies=None,
actions=_sideload,
title='MeshForge Sideload',
description='Upload meshforge.yaml data files via Meshtastic StreamAPI + XModem',
description='Upload meshforge.yaml data files to device via Meshtastic XModem',
)
meshforge-sideload/patches/00-xmodem-truncate-fix.patch
+30
View File
@@ -0,0 +1,30 @@
From 488d40597572d54b19245e23282fcf2b1cd2140c Mon Sep 17 00:00:00 2001
From: Ben Allfree <ben@benallfree.com>
Date: Sun, 12 Apr 2026 13:33:44 -0700
Subject: [PATCH] fix(xmodem): truncate file on open instead of appending
FILE_O_WRITE on nRF52 (Adafruit LittleFS) appends to existing files
rather than truncating. Remove the file before opening for write so
repeated XModem uploads always produce the correct file size.
Made-with: Cursor
---
src/xmodem.cpp | 2 ++
1 file changed, 2 insertions(+)
diff --git a/src/xmodem.cpp b/src/xmodem.cpp
index 1d8c77760..cdbbaefe1 100644
--- a/src/xmodem.cpp
+++ b/src/xmodem.cpp
@@ -123,6 +123,8 @@ void XModemAdapter::handlePacket(meshtastic_XModem xmodemPacket)
if (xmodemPacket.control == meshtastic_XModem_Control_SOH) { // Receive this file and put to Flash
spiLock->lock();
+ // Remove existing file first so we truncate rather than append
+ if (FSCom.exists(filename)) FSCom.remove(filename);
file = FSCom.open(filename, FILE_O_WRITE);
spiLock->unlock();
if (file) {
--
2.50.1
meshforge-sideload/patches/02-xmodem-vfs-routing.patch
+7 -20
View File
@@ -1,15 +1,3 @@
From 697b1317e012c74751ac995794b42bfbce967b14 Mon Sep 17 00:00:00 2001
From: Ben Allfree <ben@benallfree.com>
Date: Sun, 12 Apr 2026 12:04:17 -0700
Subject: [PATCH] Add FS routing support for virtual mount points in FSCommon
---
src/FSCommon.cpp | 94 ++++++++++++++++++++++++++++++++++++++++++++++++
src/FSCommon.h | 42 +++++++++++++++++++++-
src/xmodem.cpp | 9 +++--
src/xmodem.h | 3 +-
4 files changed, 141 insertions(+), 7 deletions(-)
diff --git a/src/FSCommon.cpp b/src/FSCommon.cpp
index 2c14f308c..79e927e64 100644
--- a/src/FSCommon.cpp
@@ -168,10 +156,10 @@ index 9b3de20c2..1ac0d04a1 100644
+#endif // FSCom
\ No newline at end of file
diff --git a/src/xmodem.cpp b/src/xmodem.cpp
index 1d8c77760..7c67d7fbf 100644
index 1d8c77760..847fe49c8 100644
--- a/src/xmodem.cpp
+++ b/src/xmodem.cpp
@@ -120,10 +120,11 @@ void XModemAdapter::handlePacket(meshtastic_XModem xmodemPacket)
@@ -120,10 +120,13 @@ void XModemAdapter::handlePacket(meshtastic_XModem xmodemPacket)
if ((xmodemPacket.seq == 0) && !isReceiving && !isTransmitting) {
// NULL packet has the destination filename
memcpy(filename, &xmodemPacket.buffer.bytes, xmodemPacket.buffer.size);
@@ -180,11 +168,13 @@ index 1d8c77760..7c67d7fbf 100644
if (xmodemPacket.control == meshtastic_XModem_Control_SOH) { // Receive this file and put to Flash
spiLock->lock();
- file = FSCom.open(filename, FILE_O_WRITE);
+ // Remove existing file first so we truncate rather than append
+ if (FSCom.exists(filename)) FSCom.remove(filename);
+ file = fsOpenWrite(activeRoute_);
spiLock->unlock();
if (file) {
sendControl(meshtastic_XModem_Control_ACK);
@@ -137,7 +138,7 @@ void XModemAdapter::handlePacket(meshtastic_XModem xmodemPacket)
@@ -137,7 +140,7 @@ void XModemAdapter::handlePacket(meshtastic_XModem xmodemPacket)
} else { // Transmit this file from Flash
LOG_INFO("XModem: Transmit file %s", filename);
spiLock->lock();
@@ -193,7 +183,7 @@ index 1d8c77760..7c67d7fbf 100644
spiLock->unlock();
if (file) {
packetno = 1;
@@ -200,8 +201,7 @@ void XModemAdapter::handlePacket(meshtastic_XModem xmodemPacket)
@@ -200,8 +203,7 @@ void XModemAdapter::handlePacket(meshtastic_XModem xmodemPacket)
spiLock->lock();
file.flush();
file.close();
@@ -203,7 +193,7 @@ index 1d8c77760..7c67d7fbf 100644
spiLock->unlock();
isReceiving = false;
break;
@@ -255,7 +255,6 @@ void XModemAdapter::handlePacket(meshtastic_XModem xmodemPacket)
@@ -255,7 +257,6 @@ void XModemAdapter::handlePacket(meshtastic_XModem xmodemPacket)
xmodemStore.seq = packetno;
spiLock->lock();
file.seek((packetno - 1) * sizeof(meshtastic_XModem_buffer_t::bytes));
@@ -225,6 +215,3 @@ index 4cfcb43e1..48ff25fbb 100644
protected:
meshtastic_XModem xmodemStore = meshtastic_XModem_init_zero;
--
2.50.1
meshforge-sideload/patches/03-xmodem-esp32-serial-polling.patch
+83
View File
@@ -0,0 +1,83 @@
commit b5de3323b3e106edb28efcda815d86a066ef34f4
Author: Ben Allfree <ben@benallfree.com>
Date: Sun Apr 12 21:31:37 2026 -0700
tdeck fixes
diff --git a/src/SerialConsole.cpp b/src/SerialConsole.cpp
index e24aa3c57..c81854a7b 100644
--- a/src/SerialConsole.cpp
+++ b/src/SerialConsole.cpp
@@ -5,6 +5,7 @@
#include "Throttle.h"
#include "configuration.h"
#include "time.h"
+#include "xmodem.h"
#if defined(ARDUINO_USB_CDC_ON_BOOT) && ARDUINO_USB_CDC_ON_BOOT
#define IS_USB_SERIAL
@@ -92,7 +93,7 @@ int32_t SerialConsole::runOnce()
#if defined(SERIAL_HAS_ON_RECEIVE) || defined(CONFIG_IDF_TARGET_ESP32S2)
return Port.available() ? delay : INT32_MAX;
#elif defined(IS_USB_SERIAL)
- return HWCDC::isPlugged() ? delay : (1000 * 20);
+ return (HWCDC::isPlugged() || xModem.isActive() || delay < 250) ? delay : (1000 * 20);
#else
return delay;
#endif
diff --git a/src/xmodem.cpp b/src/xmodem.cpp
index ccd6a1b95..3155a6526 100644
--- a/src/xmodem.cpp
+++ b/src/xmodem.cpp
@@ -152,8 +152,12 @@ void XModemAdapter::handlePacket(meshtastic_XModem xmodemPacket)
case meshtastic_XModem_Control_SOH:
case meshtastic_XModem_Control_STX:
if ((xmodemPacket.seq == 0) && !isReceiving && !isTransmitting) {
- // NULL packet has the destination filename
- memcpy(filename, &xmodemPacket.buffer.bytes, xmodemPacket.buffer.size);
+ // NULL packet has the destination filename (protobuf bytes are not NUL-terminated)
+ size_t n = xmodemPacket.buffer.size;
+ if (n >= sizeof(filename))
+ n = sizeof(filename) - 1;
+ memcpy(filename, xmodemPacket.buffer.bytes, n);
+ filename[n] = '\0';
activeRoute_ = fsRoute(filename);
if (xmodemPacket.control == meshtastic_XModem_Control_SOH) { // Receive this file and put to Flash
@@ -201,6 +205,16 @@ void XModemAdapter::handlePacket(meshtastic_XModem xmodemPacket)
}
} else {
if (isReceiving) {
+ if (xmodemPacket.seq == 0) {
+ // Duplicate OPEN retry (client re-sent while already receiving) — re-ACK so Python proceeds.
+ sendControl(meshtastic_XModem_Control_ACK);
+ break;
+ }
+ if (xmodemPacket.seq + 1 == packetno) {
+ // Already-delivered packet still in flight (stale serial buffer retry) — re-ACK.
+ sendControl(meshtastic_XModem_Control_ACK);
+ break;
+ }
// normal file data packet
if ((xmodemPacket.seq == packetno) &&
check(xmodemPacket.buffer.bytes, xmodemPacket.buffer.size, xmodemPacket.crc16)) {
diff --git a/src/xmodem.h b/src/xmodem.h
index 48ff25fbb..5b8c4930d 100644
--- a/src/xmodem.h
+++ b/src/xmodem.h
@@ -51,6 +51,7 @@ class XModemAdapter
void handlePacket(meshtastic_XModem xmodemPacket);
meshtastic_XModem getForPhone();
void resetForPhone();
+ bool isActive() const { return isReceiving || isTransmitting; }
private:
bool isReceiving = false;
@@ -61,6 +62,7 @@ class XModemAdapter
uint16_t packetno = 0;
+// Adafruit nRF/STM32 File can be constructed bound to FSCom; Arduino-ESP32 fs::File cannot.
#if defined(ARCH_NRF52) || defined(ARCH_STM32WL)
File file = File(FSCom);
#else
meshforge-sideload/patches/README.md
+13
View File
@@ -6,6 +6,19 @@ become unnecessary.
---
## 00-xmodem-truncate-fix.patch
**PR title:** `fix(xmodem): truncate file on open instead of appending`
Standalone 2-line fix. `FILE_O_WRITE` on nRF52 (Adafruit LittleFS) appends to
existing files rather than truncating. Removes the file before opening for write
so repeated XModem uploads always produce the correct file size. Independent of
the other two patches — can be reviewed and merged on its own.
**Sentinel:** `Remove existing file first so we truncate`
---
## 01-nrf-external-flash.patch
**PR title:** `feat(nrf52): optional external QSPI LittleFS filesystem`
meshforge-sideload/patches/apply_patches.py
+3 -2
View File
@@ -22,8 +22,9 @@ import subprocess
import sys
_PATCHES = [
('01-nrf-external-flash.patch', 'extFSInit'), # sentinel unique to patch 1
('02-xmodem-vfs-routing.patch', 'fsRoute'), # sentinel unique to patch 2
('00-xmodem-truncate-fix.patch', 'Remove existing file first so we truncate'), # standalone fix
('01-nrf-external-flash.patch', 'extFSInit'), # PR1: external flash FS
('02-xmodem-vfs-routing.patch', 'fsRoute'), # PR2: VFS routing + XModem
]
_PATCH_DIR = os.path.dirname(os.path.abspath(__file__))
meshforge-sideload/tools/mf_protocol.py
+45 -199
View File
@@ -1,214 +1,51 @@
"""
mf_protocol.py — Meshtastic StreamAPI + XModem file transfer (Python host side).
mf_protocol.py — MeshForge sideload via meshtastic-python XModem.
Matches the patched xmodem.cpp on the firmware side.
Uses Node.uploadFile() from the meshtastic Python library (vendored at
vendor/meshtastic-python) for all file transfers.
Meshtastic StreamAPI framing:
0x94 0xC3 <len:2 BE> <protobuf payload>
ToRadio.xmodemPacket = field 5
FromRadio.xmodemPacket = field 12
meshtastic_XModem fields:
control (1, varint) — XModem Control enum
seq (2, varint) — packet sequence
crc16 (3, varint) — CRC-16-CCITT of buffer
buffer (4, bytes) — up to 128 bytes
Requires: pip install pyserial (meshtastic is loaded from the vendor submodule)
"""
import glob as globmod
import os
import re
import struct
import sys
import time
# XModem Control enum values
XC_NUL = 0
XC_SOH = 1
XC_STX = 2
XC_EOT = 4
XC_ACK = 6
XC_NAK = 21
XC_CAN = 24
XMODEM_BUFFER_SIZE = 128
MAX_RETRIES = 10
ACK_TIMEOUT_S = 5.0
# ── Resolve the vendored meshtastic-python library ────────────────────────────
# Walk up from this file to find the mesh-forge repo root, then add the
# vendored library to sys.path.
# ── CRC-16-CCITT ──────────────────────────────────────────────────────────────
def crc16_ccitt(data: bytes) -> int:
crc = 0
for b in data:
crc = ((crc >> 8) | (crc << 8)) & 0xFFFF
crc ^= b
crc ^= ((crc & 0xFF) >> 4) & 0xFFFF
crc ^= ((crc << 8) << 4) & 0xFFFF
crc ^= (((crc & 0xFF) << 4) << 1) & 0xFFFF
return crc & 0xFFFF
# ── Minimal protobuf encoding ─────────────────────────────────────────────────
def varint(n: int) -> bytes:
out = []
while n > 0x7F:
out.append((n & 0x7F) | 0x80)
n >>= 7
out.append(n)
return bytes(out)
def pb_bytes_field(field: int, data: bytes) -> bytes:
tag = varint((field << 3) | 2)
return tag + varint(len(data)) + data
def encode_xmodem(control: int, seq: int, crc16: int, buffer: bytes) -> bytes:
msg = varint((1 << 3) | 0) + varint(control)
if seq: msg += varint((2 << 3) | 0) + varint(seq)
if crc16: msg += varint((3 << 3) | 0) + varint(crc16)
if buffer: msg += pb_bytes_field(4, buffer)
return msg
def encode_toradio(xmodem_bytes: bytes) -> bytes:
return pb_bytes_field(5, xmodem_bytes)
def stream_frame(payload: bytes) -> bytes:
return b'\x94\xc3' + struct.pack('>H', len(payload)) + payload
# ── FromRadio response parsing ────────────────────────────────────────────────
def _parse_varint(data: bytes, pos: int):
val, shift = 0, 0
while pos < len(data):
b = data[pos]; pos += 1
val |= (b & 0x7F) << shift; shift += 7
if not (b & 0x80): break
return val, pos
def parse_xmodem(data: bytes) -> dict:
i, result = 0, {'control': 0, 'seq': 0, 'crc16': 0, 'buffer': b''}
while i < len(data):
tag = data[i]; i += 1
fn, wt = tag >> 3, tag & 0x7
if wt == 0:
val, i = _parse_varint(data, i)
if fn == 1: result['control'] = val
elif fn == 2: result['seq'] = val
elif fn == 3: result['crc16'] = val
elif wt == 2:
l, i = _parse_varint(data, i)
if fn == 4: result['buffer'] = data[i:i+l]
i += l
else: break
return result
def parse_from_radio(data: bytes) -> dict | None:
i = 0
while i < len(data):
tag = data[i]; i += 1
fn, wt = tag >> 3, tag & 0x7
if wt == 0:
_, i = _parse_varint(data, i)
elif wt == 2:
l, i = _parse_varint(data, i)
payload = data[i:i+l]; i += l
if fn == 12:
return parse_xmodem(payload)
else: break
def _find_vendor_meshtastic():
here = os.path.dirname(os.path.abspath(__file__))
# tools/ → meshforge-sideload/ → mesh-forge root
candidate = os.path.normpath(os.path.join(here, '..', '..', 'vendor', 'meshtastic-python'))
if os.path.isdir(os.path.join(candidate, 'meshtastic')):
return candidate
return None
# ── StreamAPI frame reader ────────────────────────────────────────────────────
def read_xmodem_response(port, timeout_s: float = ACK_TIMEOUT_S) -> dict | None:
buf = bytearray()
deadline = time.time() + timeout_s
while time.time() < deadline:
n = port.in_waiting
if n: buf += port.read(n)
# Scan for 0x94 0xC3 frame
start = -1
for i in range(len(buf) - 1):
if buf[i] == 0x94 and buf[i+1] == 0xC3:
start = i; break
if start >= 0 and len(buf) >= start + 4:
length = (buf[start+2] << 8) | buf[start+3]
if len(buf) >= start + 4 + length:
payload = bytes(buf[start+4:start+4+length])
del buf[:start+4+length]
xm = parse_from_radio(payload)
if xm: return xm
# Not an XModem frame — keep reading
continue
time.sleep(0.01)
return None
_vendor_path = _find_vendor_meshtastic()
if _vendor_path and _vendor_path not in sys.path:
sys.path.insert(0, _vendor_path)
# ── File transfer ─────────────────────────────────────────────────────────────
def send_xmodem_file(port, dest_path: str, data: bytes,
on_progress=None) -> None:
# SOH seq=0 — filename handshake
fn_bytes = dest_path.encode('ascii')
for attempt in range(MAX_RETRIES):
port.write(stream_frame(encode_toradio(encode_xmodem(XC_SOH, 0, 0, fn_bytes))))
resp = read_xmodem_response(port)
if resp and resp['control'] == XC_ACK:
break
if attempt == MAX_RETRIES - 1:
raise IOError(f'XModem OPEN rejected for {dest_path}')
# STX data packets
seq = 1
offset = 0
while offset < len(data):
chunk = data[offset:offset + XMODEM_BUFFER_SIZE]
crc = crc16_ccitt(chunk)
acked = False
for retry in range(MAX_RETRIES):
port.write(stream_frame(encode_toradio(encode_xmodem(XC_STX, seq, crc, chunk))))
resp = read_xmodem_response(port)
if resp and resp['control'] == XC_ACK:
acked = True; break
if resp and resp['control'] == XC_CAN:
raise IOError(f'XModem transfer cancelled at offset {offset}')
if not acked:
raise IOError(f'XModem: no ACK for seq {seq} at offset {offset}')
offset += len(chunk)
seq = (seq & 0xFF) + 1
if on_progress: on_progress(offset, len(data))
# EOT
for attempt in range(MAX_RETRIES):
port.write(stream_frame(encode_toradio(encode_xmodem(XC_EOT, 0, 0, b''))))
resp = read_xmodem_response(port)
if resp and resp['control'] == XC_ACK:
return
if attempt == MAX_RETRIES - 1:
raise IOError(f'XModem EOT not acknowledged for {dest_path}')
# ── meshforge.yaml parser (minimal) ──────────────────────────────────────────
# ── meshforge.yaml parser ─────────────────────────────────────────────────────
def parse_data_entries(yaml_text: str) -> list:
"""Return list of (glob_pattern, device_dest) from meshforge.yaml data: section."""
entries = []
in_mf = in_data = False
for raw in yaml_text.splitlines():
line = re.sub(r'#.*$', '', raw).rstrip()
if not line.strip(): continue
if not line.strip():
continue
indent = len(line) - len(line.lstrip())
content = line.strip()
if indent == 0:
in_mf = (content == 'meshforge:'); in_data = False
in_mf = (content == 'meshforge:')
in_data = False
elif in_mf and indent == 2:
in_data = (content == 'data:')
elif in_mf and in_data and indent == 4:
@@ -217,7 +54,7 @@ def parse_data_entries(yaml_text: str) -> list:
entry = m.group(1).strip().strip('"\'')
colon = entry.find(':')
if colon > 0:
entries.append((entry[:colon].strip(), entry[colon+1:].strip()))
entries.append((entry[:colon].strip(), entry[colon + 1:].strip()))
return entries
@@ -226,9 +63,9 @@ def parse_data_entries(yaml_text: str) -> list:
def autodetect_port():
try:
import serial.tools.list_ports
KNOWN = ['RAK', 'nRF52', 'Adafruit', 'Nordic', 'Meshtastic',
'WisMesh', 'T-Echo', 'CP210', 'CH340', 'FTDI', 'USB Serial',
'JTAG', 'LilyGO', 'Espressif']
KNOWN = ['RAK', 'nRF52', 'Adafruit', 'Nordic', 'Meshtastic', 'WisMesh',
'T-Echo', 'CP210', 'CH340', 'FTDI', 'USB Serial', 'JTAG',
'LilyGO', 'Espressif']
ports = list(serial.tools.list_ports.comports())
for p in ports:
desc = (p.description or '') + ' ' + (p.manufacturer or '')
@@ -247,9 +84,11 @@ def autodetect_port():
def run_sideload(project_dir: str, port_name: str, baud: int = 115200,
boot_wait_s: float = 3.0) -> None:
try:
import serial
import meshtastic.serial_interface
except ImportError:
print('ERROR: pyserial not installed. Run: pip install pyserial')
print('ERROR: meshtastic library not found.\n'
'Install with: pip install meshtastic\n'
'Or ensure vendor/meshtastic-python is in sys.path')
sys.exit(1)
yaml_path = os.path.join(project_dir, 'meshforge.yaml')
@@ -271,26 +110,33 @@ def run_sideload(project_dir: str, port_name: str, baud: int = 115200,
transfers.append((src, dest.rstrip('/') + '/' + os.path.basename(src)))
if not transfers:
print('meshforge-sideload: no files matched globs — nothing to sideload')
print('meshforge-sideload: no files matched — nothing to sideload')
return
print(f'meshforge-sideload: waiting {boot_wait_s}s for device to boot...')
time.sleep(boot_wait_s)
print(f'meshforge-sideload: connecting to {port_name} @ {baud}...')
with serial.Serial(port_name, baud, timeout=ACK_TIMEOUT_S) as port:
time.sleep(0.5)
print(f'meshforge-sideload: connecting to {port_name}...')
iface = meshtastic.serial_interface.SerialInterface(port_name)
try:
for i, (src, device_path) in enumerate(transfers):
data = open(src, 'rb').read()
size_kb = os.path.getsize(src) / 1024
name = os.path.basename(src)
print(f' [{i+1}/{len(transfers)}] {name} ({len(data)/1024:.1f} KB) → {device_path}')
print(f' [{i + 1}/{len(transfers)}] {name} ({size_kb:.1f} KB) → {device_path}')
def progress(sent, total, _name=name):
pct = 100 * sent // total
bar = '#' * (pct // 5) + '.' * (20 - pct // 5)
print(f'\r [{bar}] {pct}%', end='', flush=True)
send_xmodem_file(port, device_path, data, on_progress=progress)
print(f'\r [{"#"*20}] 100% done')
ok = iface.localNode.uploadFile(src, device_path, on_progress=progress)
if ok:
print(f'\r [{"#" * 20}] 100% done')
else:
print(f'\r FAILED')
raise RuntimeError(f'Upload failed: {src} → {device_path}')
finally:
iface.close()
print(f'meshforge-sideload: {len(transfers)} file(s) uploaded successfully')
vendor/meshtastic-python
Vendored Submodule
+1
Submodule vendor/meshtastic-python added at cec79a7c1f