From 1ec72c81b49d40fa6c7c85d792c1f583ed8aaefc Mon Sep 17 00:00:00 2001 From: Ben Allfree Date: Sat, 11 Apr 2026 05:09:59 -0700 Subject: [PATCH] consolidate --- .github/workflows/custom_build.yml | 34 +- .github/workflows/custom_build_test.yml | 34 +- scripts/emit-flash-manifest.py | 577 ------------------------ scripts/extend-merged-with-ota.py | 73 +++ src/components/DeviceFlasher.tsx | 41 +- src/lib/espFlashLayout.ts | 155 +------ src/lib/flashTargetFamily.ts | 57 ++- src/lib/nrfFlashRun.ts | 59 +-- src/lib/untarGz.ts | 42 -- 9 files changed, 212 insertions(+), 860 deletions(-) delete mode 100644 scripts/emit-flash-manifest.py create mode 100644 scripts/extend-merged-with-ota.py diff --git a/.github/workflows/custom_build.yml b/.github/workflows/custom_build.yml index d426755..c14d389 100644 --- a/.github/workflows/custom_build.yml +++ b/.github/workflows/custom_build.yml @@ -165,6 +165,21 @@ jobs: exit 1 fi + - name: Generate merged factory binary for ESP32 targets (if applicable) + shell: bash + run: | + ROOT=$(cat /tmp/fw-src-root.txt) + BUILD_DIR="$ROOT/.pio/build/${{ inputs.target_env }}" + if [ -f "$BUILD_DIR/bootloader.bin" ]; then + echo "ESP32 target detected; running mergebin → firmware-merged.factory.bin" + cd "$ROOT" + export MERGED_BIN_PATH="$BUILD_DIR/firmware-merged.factory.bin" + pio run -t mergebin -e "${{ inputs.target_env }}" 2>&1 || \ + echo "WARNING: mergebin target failed; factory binary will not be available" + else + echo "No bootloader.bin found; skipping mergebin (non-ESP32 target)" + fi + - name: Download Meshtastic ESP OTA companion (if applicable) shell: bash run: | @@ -174,6 +189,21 @@ jobs: BUILD_DIR=".pio/build/${{ inputs.target_env }}" bash "${{ github.workspace }}/scripts/download-meshtastic-ota.sh" "$ROOT/$BUILD_DIR" + - name: Extend merged binary with Meshtastic OTA companion (if applicable) + shell: bash + run: | + ROOT=$(cat /tmp/fw-src-root.txt) + BUILD_DIR="$ROOT/.pio/build/${{ inputs.target_env }}" + MERGED="$BUILD_DIR/firmware-merged.factory.bin" + PARTS="$BUILD_DIR/partitions.bin" + OTA_BIN=$(ls "$BUILD_DIR"/mt-*.ota.bin "$BUILD_DIR"/bleota-c3.bin 2>/dev/null | head -1) + if [ -f "$MERGED" ] && [ -f "$PARTS" ] && [ -n "$OTA_BIN" ]; then + python3 "${{ github.workspace }}/scripts/extend-merged-with-ota.py" \ + "$MERGED" "$PARTS" "$OTA_BIN" + else + echo "Skipping OTA extension (merged=$( [ -f "$MERGED" ] && echo yes || echo no), ota=$( [ -n "$OTA_BIN" ] && echo yes || echo no))" + fi + - name: Stage or generate nRF52 DFU files (if applicable) shell: bash run: | @@ -227,7 +257,6 @@ jobs: ROOT=$(cat /tmp/fw-src-root.txt) cd "$ROOT" BUILD_DIR=".pio/build/${{ inputs.target_env }}" - python3 "${{ github.workspace }}/scripts/emit-flash-manifest.py" "$ROOT/$BUILD_DIR" "$ROOT" "${{ inputs.target_env }}" ARTIFACT_NAME="firmware-${{ inputs.build_key }}-${{ github.run_id }}.tar.gz" STAGE=/tmp/fw-bundle rm -rf "$STAGE" @@ -237,9 +266,6 @@ jobs: cp -a "$f" "$STAGE/" done shopt -u nullglob - if [ -f "$BUILD_DIR/flash-manifest.json" ]; then - cp -a "$BUILD_DIR/flash-manifest.json" "$STAGE/" - fi # Nordic DFU manifest (MeshCore and other nRF52 builds) if [ -f "$BUILD_DIR/manifest.json" ]; then cp -a "$BUILD_DIR/manifest.json" "$STAGE/" diff --git a/.github/workflows/custom_build_test.yml b/.github/workflows/custom_build_test.yml index d426755..c14d389 100644 --- a/.github/workflows/custom_build_test.yml +++ b/.github/workflows/custom_build_test.yml @@ -165,6 +165,21 @@ jobs: exit 1 fi + - name: Generate merged factory binary for ESP32 targets (if applicable) + shell: bash + run: | + ROOT=$(cat /tmp/fw-src-root.txt) + BUILD_DIR="$ROOT/.pio/build/${{ inputs.target_env }}" + if [ -f "$BUILD_DIR/bootloader.bin" ]; then + echo "ESP32 target detected; running mergebin → firmware-merged.factory.bin" + cd "$ROOT" + export MERGED_BIN_PATH="$BUILD_DIR/firmware-merged.factory.bin" + pio run -t mergebin -e "${{ inputs.target_env }}" 2>&1 || \ + echo "WARNING: mergebin target failed; factory binary will not be available" + else + echo "No bootloader.bin found; skipping mergebin (non-ESP32 target)" + fi + - name: Download Meshtastic ESP OTA companion (if applicable) shell: bash run: | @@ -174,6 +189,21 @@ jobs: BUILD_DIR=".pio/build/${{ inputs.target_env }}" bash "${{ github.workspace }}/scripts/download-meshtastic-ota.sh" "$ROOT/$BUILD_DIR" + - name: Extend merged binary with Meshtastic OTA companion (if applicable) + shell: bash + run: | + ROOT=$(cat /tmp/fw-src-root.txt) + BUILD_DIR="$ROOT/.pio/build/${{ inputs.target_env }}" + MERGED="$BUILD_DIR/firmware-merged.factory.bin" + PARTS="$BUILD_DIR/partitions.bin" + OTA_BIN=$(ls "$BUILD_DIR"/mt-*.ota.bin "$BUILD_DIR"/bleota-c3.bin 2>/dev/null | head -1) + if [ -f "$MERGED" ] && [ -f "$PARTS" ] && [ -n "$OTA_BIN" ]; then + python3 "${{ github.workspace }}/scripts/extend-merged-with-ota.py" \ + "$MERGED" "$PARTS" "$OTA_BIN" + else + echo "Skipping OTA extension (merged=$( [ -f "$MERGED" ] && echo yes || echo no), ota=$( [ -n "$OTA_BIN" ] && echo yes || echo no))" + fi + - name: Stage or generate nRF52 DFU files (if applicable) shell: bash run: | @@ -227,7 +257,6 @@ jobs: ROOT=$(cat /tmp/fw-src-root.txt) cd "$ROOT" BUILD_DIR=".pio/build/${{ inputs.target_env }}" - python3 "${{ github.workspace }}/scripts/emit-flash-manifest.py" "$ROOT/$BUILD_DIR" "$ROOT" "${{ inputs.target_env }}" ARTIFACT_NAME="firmware-${{ inputs.build_key }}-${{ github.run_id }}.tar.gz" STAGE=/tmp/fw-bundle rm -rf "$STAGE" @@ -237,9 +266,6 @@ jobs: cp -a "$f" "$STAGE/" done shopt -u nullglob - if [ -f "$BUILD_DIR/flash-manifest.json" ]; then - cp -a "$BUILD_DIR/flash-manifest.json" "$STAGE/" - fi # Nordic DFU manifest (MeshCore and other nRF52 builds) if [ -f "$BUILD_DIR/manifest.json" ]; then cp -a "$BUILD_DIR/manifest.json" "$STAGE/" diff --git a/scripts/emit-flash-manifest.py b/scripts/emit-flash-manifest.py deleted file mode 100644 index 37ec2d4..0000000 --- a/scripts/emit-flash-manifest.py +++ /dev/null @@ -1,577 +0,0 @@ -#!/usr/bin/env python3 -""" -Emit flash-manifest.json for Mesh Forge USB flasher. - -If BUILD_DIR/flash-manifest.json already exists (project-supplied), merge in -targetFamily from PlatformIO when missing. - -Otherwise, if Meshtastic-style *.mt.json is present, synthesize one JSON with: - - "update": app @ ota_0 + BLE OTA @ ota_1, eraseFlash false (Meshtastic “Update”). - - "factory": merged *.factory.bin @ 0 + OTA + LittleFS, eraseFlash true (Meshtastic “Erase device”), - omitted if no factory.bin in BUILD_DIR. - -OTA: mt-*-ota.bin (ESP32/S3) or bleota-c3.bin (C3/C6). - -Optional args: PROJECT_ROOT TARGET_ENV — merged PIO config for that env fills -targetFamily (and platform/board) for the USB flasher UI. -""" -from __future__ import annotations - -import glob -import json -import os -import re -import sys - - -def _normalize_platform(platform: str | None) -> str: - if not platform: - return "" - p = platform.strip().lower() - if "#" in p: - p = p.split("#", 1)[0].strip() - if "@" in p: - p = p.split("@", 1)[0].strip() - if "/" in p: - p = p.rsplit("/", 1)[-1].strip() - return p - - -def _platform_to_target_family(platform: str | None, board: str | None) -> str: - pl = _normalize_platform(platform) - b = (board or "").lower() - if "8266" in pl or "esp8266" in b: - return "esp8266" - if "nrf52" in pl or "nrf52840" in b or "nrf52833" in b or "nrf52832" in b: - return "nrf52" - if "rp2040" in pl or "rp2040" in b or "raspberrypi" in pl: - return "rp2040" - if "espressif32" in pl or "esp32" in pl or "esp32" in b or "esp32c3" in b or "esp32s3" in b: - return "esp32" - return "unknown" - - -def _resolve_pio_target_family(project_root: str, target_env: str) -> tuple[str, str | None, str | None]: - try: - from platformio.project.config import ProjectConfig - except ImportError: - print("platformio not installed; targetFamily will be unknown", file=sys.stderr) - return "unknown", None, None - - ini = os.path.join(project_root, "platformio.ini") - if not os.path.isfile(ini): - return "unknown", None, None - - old = os.getcwd() - try: - os.chdir(project_root) - config = ProjectConfig("platformio.ini") - section = f"env:{target_env}" - if section not in config.sections(): - print(f"PIO env not found: {target_env!r}", file=sys.stderr) - return "unknown", None, None - platform = config.get(section, "platform") - board = config.get(section, "board") - fam = _platform_to_target_family(platform, board) - return fam, platform, board - except Exception as e: - print(f"PIO targetFamily resolution failed: {e}", file=sys.stderr) - return "unknown", None, None - finally: - os.chdir(old) - - -def _merge_target_family_meta( - manifest: dict, - target_family: str, - platform: str | None, - board: str | None, -) -> dict: - """Add targetFamily / platform / board only when absent.""" - out = dict(manifest) - if "targetFamily" not in out: - out["targetFamily"] = target_family - if platform and "platform" not in out: - out["platform"] = platform.strip() if isinstance(platform, str) else platform - if board and "board" not in out: - out["board"] = board.strip() if isinstance(board, str) else board - return out - - -def _write_manifest(path: str, data: dict) -> None: - with open(path, "w", encoding="utf-8") as f: - json.dump(data, f, indent=2) - - -def parse_offset(v: object) -> int | None: - if v is None: - return None - if isinstance(v, bool): - return None - if isinstance(v, int): - return v - s = str(v).strip() - if not s: - return None - if s.lower().startswith("0x"): - return int(s, 16) - return int(s, 10) - - -def list_basenames(build_dir: str) -> set[str]: - out: set[str] = set() - for p in glob.glob(os.path.join(build_dir, "*")): - if os.path.isfile(p): - out.add(os.path.basename(p)) - return out - - -def part_offset_for_slot(parts: list[dict], part_name: str) -> int | None: - for p in parts: - if str(p.get("name", "")) == part_name: - o = parse_offset(p.get("offset")) - if o is not None: - return o - for p in parts: - if str(p.get("subtype", "")) == part_name: - o = parse_offset(p.get("offset")) - if o is not None: - return o - if part_name == "app0": - for p in parts: - if p.get("type") == "app" and p.get("subtype") == "ota_0": - o = parse_offset(p.get("offset")) - if o is not None: - return o - if part_name == "spiffs": - for p in parts: - if str(p.get("subtype", "")) == "spiffs": - o = parse_offset(p.get("offset")) - if o is not None: - return o - return None - - -def ota1_offset(parts: list[dict]) -> int | None: - for p in parts: - if str(p.get("subtype", "")) == "ota_1": - return parse_offset(p.get("offset")) - return None - - -def spiffs_offset(parts: list[dict]) -> int | None: - return part_offset_for_slot(parts, "spiffs") - - -def _offset_int(im: dict) -> int: - o = im["offset"] - return int(o) if isinstance(o, int) else parse_offset(o) or 0 - - -def _dedupe_same_offset(images: list[dict]) -> list[dict]: - """One esptool image per physical offset (prefer non-optional, then lexicographic file).""" - buckets: dict[int, list[dict]] = {} - for im in images: - buckets.setdefault(_offset_int(im), []).append(im) - - out: list[dict] = [] - for off in sorted(buckets): - group = buckets[off] - out.append( - group[0] - if len(group) == 1 - else min(group, key=lambda im: (im.get("optional") is True, str(im.get("file", "")))) - ) - return out - - -def _is_non_factory_firmware(name: str) -> bool: - return bool(re.match(r"^firmware-.+\.bin$", name, re.I) and not re.search(r"\.factory\.bin$", name, re.I)) - - -def pick_factory_bin(names: set[str]) -> str | None: - cands = sorted(n for n in names if re.match(r"^firmware-.+\.factory\.bin$", n, re.I)) - return cands[0] if cands else None - - -def pick_non_factory_firmware(names: set[str], mt: dict) -> str | None: - for entry in mt.get("files") or []: - fname = entry.get("name") - if not fname or fname not in names or entry.get("part_name") != "app0": - continue - if _is_non_factory_firmware(fname): - return fname - for n in sorted(names): - if _is_non_factory_firmware(n): - return n - return None - - -def emit_meshtastic_update(build_dir: str, mt: dict) -> dict | None: - """Official-style update: app @ ota_0 + OTA @ ota_1 only (no erase, no factory, no FS).""" - parts: list[dict] = mt.get("part") or [] - if not parts: - return None - names = list_basenames(build_dir) - images: list[dict] = [] - seen: set[str] = set() - - def add(file: str, offset: int, optional: bool = False) -> None: - if file not in names or file in seen: - return - row: dict = {"file": file, "offset": offset} - if optional: - row["optional"] = True - images.append(row) - seen.add(file) - - app_bin = pick_non_factory_firmware(names, mt) - if not app_bin: - return None - off0 = part_offset_for_slot(parts, "app0") - if off0 is None: - return None - add(app_bin, off0, optional=False) - - ota_added = False - for n in sorted(names): - if re.match(r"^mt-.+-ota\.bin$", n, re.I): - o = ota1_offset(parts) - if o is not None: - add(n, o, optional=False) - ota_added = True - break - if not ota_added and "bleota-c3.bin" in names: - o = ota1_offset(parts) - if o is not None: - add("bleota-c3.bin", o, optional=False) - - images = _dedupe_same_offset(images) - images.sort(key=_offset_int) - return {"images": images, "eraseFlash": False} - - -def emit_meshtastic_full(build_dir: str, mt: dict, factory_bin: str) -> dict | None: - """Official-style erase + factory: merged factory @ 0 + OTA + LittleFS.""" - parts: list[dict] = mt.get("part") or [] - if not parts: - return None - names = list_basenames(build_dir) - if factory_bin not in names: - return None - - images: list[dict] = [] - seen: set[str] = set() - - def add(file: str, offset: int, optional: bool = False) -> None: - if file not in names or file in seen: - return - row: dict = {"file": file, "offset": offset} - if optional: - row["optional"] = True - images.append(row) - seen.add(file) - - add(factory_bin, 0, optional=False) - - ota_added = False - for n in sorted(names): - if re.match(r"^mt-.+-ota\.bin$", n, re.I): - o = ota1_offset(parts) - if o is not None: - add(n, o, optional=False) - ota_added = True - break - if not ota_added and "bleota-c3.bin" in names: - o = ota1_offset(parts) - if o is not None: - add("bleota-c3.bin", o, optional=False) - - for n in sorted(names): - if n.startswith("littlefs-") and n.endswith(".bin"): - so = spiffs_offset(parts) - if so is not None: - add(n, so, optional=False) - - for entry in mt.get("files") or []: - fname = entry.get("name") - part_name = entry.get("part_name") - if not fname or part_name != "spiffs" or fname not in names: - continue - if not str(fname).lower().startswith("littlefs-"): - continue - so = spiffs_offset(parts) - if so is not None: - add(fname, so, optional=False) - - images = _dedupe_same_offset(images) - images.sort(key=_offset_int) - - o1 = ota1_offset(parts) - if o1 is not None and not any(_offset_int(im) == o1 for im in images): - return None - - return {"images": images, "eraseFlash": True} - - -def pick_mt_json(build_dir: str) -> str | None: - paths = glob.glob(os.path.join(build_dir, "*.mt.json")) - if not paths: - return None - if len(paths) == 1: - return paths[0] - fw = [p for p in paths if re.search(r"firmware-.+\.mt\.json$", os.path.basename(p), re.I)] - return fw[0] if fw else paths[0] - - -# --------------------------------------------------------------------------- -# Path 3 — generic nRF52 (UF2 bootloader, no *.mt.json) -# --------------------------------------------------------------------------- - -def _pick_firmware_uf2(names: set[str]) -> str | None: - """Return the best candidate firmware *.uf2 (exclude nuke.uf2).""" - cands = sorted( - n for n in names - if n.lower().endswith(".uf2") and n.lower() != "nuke.uf2" - ) - # Prefer names starting with "firmware" - fw = [n for n in cands if n.lower().startswith("firmware")] - return fw[0] if fw else (cands[0] if cands else None) - - -def emit_generic_nrf52(names: set[str]) -> dict | None: - """ - Emit a UF2-based manifest for nRF52 builds. - update: firmware *.uf2 with role "uf2" - factory (optional): nuke.uf2 (role "nuke") + firmware *.uf2, eraseFlash true - """ - fw_uf2 = _pick_firmware_uf2(names) - if not fw_uf2: - return None - - doc: dict = { - "update": { - "images": [{"file": fw_uf2, "offset": 0, "role": "uf2"}], - "eraseFlash": False, - } - } - - if "nuke.uf2" in names: - doc["factory"] = { - "images": [ - {"file": "nuke.uf2", "offset": 0, "role": "nuke"}, - {"file": fw_uf2, "offset": 0, "role": "uf2"}, - ], - "eraseFlash": True, - } - - return doc - - -# --------------------------------------------------------------------------- -# Path 4 — generic ESP32 (standard PlatformIO bins, no *.mt.json) -# --------------------------------------------------------------------------- - -# Chips where the secondary bootloader lives at 0x0 (not 0x1000). -# ESP32 and ESP32-S2 use 0x1000; S3, C3, C6, H2 use 0x0. -_ESP32_BOOTLOADER_OFFSET_ZERO_BOARDS = re.compile( - r"esp32[-_]?(s3|c3|c6|h2|c2|p4)", - re.I, -) - -_ESP32_OFFSETS_1000: dict[str, int] = { - "bootloader.bin": 0x1000, - "partitions.bin": 0x8000, - "boot_app0.bin": 0xE000, -} -_ESP32_OFFSETS_0000: dict[str, int] = { - "bootloader.bin": 0x0, - "partitions.bin": 0x8000, - "boot_app0.bin": 0xE000, -} -_ESP32_APP_OFFSET = 0x10000 - - -def _esp32_bootloader_at_zero(board: str | None) -> bool: - """Return True when the chip variant places its bootloader at 0x0 (S3, C3, C6, H2…).""" - return bool(_ESP32_BOOTLOADER_OFFSET_ZERO_BOARDS.search(board or "")) - - -def _pick_esp32_app_bin(names: set[str]) -> str | None: - """Return firmware app bin (versioned or exact), excluding factory images.""" - if "firmware.bin" in names: - return "firmware.bin" - cands = sorted( - n for n in names - if re.match(r"^firmware-.+\.bin$", n, re.I) and not re.search(r"\.factory\.bin$", n, re.I) - ) - return cands[0] if cands else None - - -def emit_generic_esp32(names: set[str], board: str | None = None) -> dict | None: - """ - Emit a standard ESP32 manifest from PlatformIO bin output. - update: bootloader + partitions + app (+ boot_app0 if present) - factory (optional): firmware-*.factory.bin @ 0x0, eraseFlash true - - The bootloader offset varies by chip: - ESP32 / ESP32-S2 → 0x1000 - ESP32-S3 / C3 / C6 / H2 → 0x0 - """ - bootloader = "bootloader.bin" if "bootloader.bin" in names else None - partitions = "partitions.bin" if "partitions.bin" in names else None - app_bin = _pick_esp32_app_bin(names) - - if not (bootloader and partitions and app_bin): - # Not enough for a canonical ESP32 layout - return None - - offsets = _ESP32_OFFSETS_0000 if _esp32_bootloader_at_zero(board) else _ESP32_OFFSETS_1000 - - update_images: list[dict] = [ - {"file": "bootloader.bin", "offset": offsets["bootloader.bin"]}, - {"file": "partitions.bin", "offset": offsets["partitions.bin"]}, - ] - if "boot_app0.bin" in names: - update_images.append({"file": "boot_app0.bin", "offset": offsets["boot_app0.bin"]}) - update_images.append({"file": app_bin, "offset": _ESP32_APP_OFFSET}) - update_images.sort(key=lambda im: im["offset"]) - - doc: dict = { - "update": {"images": update_images, "eraseFlash": False} - } - - factory_bin = pick_factory_bin(names) - if factory_bin: - doc["factory"] = { - "images": [{"file": factory_bin, "offset": 0}], - "eraseFlash": True, - } - - return doc - - -# --------------------------------------------------------------------------- -# Entry point -# --------------------------------------------------------------------------- - -def main() -> int: - if len(sys.argv) < 2: - print( - "usage: emit-flash-manifest.py BUILD_DIR [PROJECT_ROOT TARGET_ENV]", - file=sys.stderr, - ) - return 2 - build_dir = os.path.abspath(sys.argv[1]) - project_root: str | None = None - target_env: str | None = None - if len(sys.argv) >= 4: - project_root = os.path.abspath(sys.argv[2]) - target_env = sys.argv[3] - - pio_family, pio_platform, pio_board = ( - _resolve_pio_target_family(project_root, target_env) - if project_root and target_env - else ("unknown", None, None) - ) - - out_path = os.path.join(build_dir, "flash-manifest.json") - - # Path 1: existing manifest — merge targetFamily when absent - if os.path.isfile(out_path): - with open(out_path, encoding="utf-8") as f: - manifest = json.load(f) - if not isinstance(manifest, dict): - print(f"Invalid existing {out_path}", file=sys.stderr) - return 1 - ok_legacy = isinstance(manifest.get("images"), list) - up = manifest.get("update") - ok_dual = isinstance(up, dict) and isinstance(up.get("images"), list) - if not ok_legacy and not ok_dual: - print(f"Invalid existing {out_path} (need images[] or update.images[])", file=sys.stderr) - return 1 - merged = _merge_target_family_meta(manifest, pio_family, pio_platform, pio_board) - if merged != manifest: - _write_manifest(out_path, merged) - print(f"Merged targetFamily into {out_path}") - else: - print(f"Keeping existing {out_path} (targetFamily already set)") - return 0 - - names = list_basenames(build_dir) - - # Path 2: Meshtastic *.mt.json - mt_path = pick_mt_json(build_dir) - if mt_path: - with open(mt_path, encoding="utf-8") as f: - mt = json.load(f) - - manifest_update = emit_meshtastic_update(build_dir, mt) - if not manifest_update: - print("Could not synthesize flash-manifest.json (update) from mt.json") - return 0 - - doc: dict = {"update": manifest_update} - factory = pick_factory_bin(names) - if factory: - manifest_full = emit_meshtastic_full(build_dir, mt, factory) - if manifest_full: - doc["factory"] = manifest_full - else: - print("Could not build factory section; omitting factory key", file=sys.stderr) - else: - print("No firmware-*.factory.bin in BUILD_DIR; omitting factory section") - - merged_doc = _merge_target_family_meta(doc, pio_family, pio_platform, pio_board) - _write_manifest(out_path, merged_doc) - print(f"Wrote {out_path} (update + {'factory' if 'factory' in merged_doc else 'no factory'})") - return 0 - - # Path 3: generic nRF52 — UF2 bootloader builds or Nordic DFU (bin + dat) builds - if pio_family == "nrf52": - # 3a: UF2 output (Meshtastic and other UF2-bootloader boards) - doc = emit_generic_nrf52(names) - if doc: - merged_doc = _merge_target_family_meta(doc, pio_family, pio_platform, pio_board) - _write_manifest(out_path, merged_doc) - print(f"Wrote {out_path} (nRF52 UF2, update + {'factory' if 'factory' in merged_doc else 'no factory'})") - return 0 - - # 3b: Nordic DFU bin + dat output (MeshCore and similar PlatformIO nRF52 builds). - # The browser uses manifest.json (Nordic format) directly for the DFU protocol; - # flash-manifest.json here just records targetFamily for the UI. - if "firmware.bin" in names and "firmware.dat" in names: - doc = { - "update": { - "images": [ - {"file": "firmware.bin", "offset": 0, "role": "dfu-bin"}, - {"file": "firmware.dat", "offset": 0, "role": "dfu-dat"}, - ], - "eraseFlash": False, - } - } - merged_doc = _merge_target_family_meta(doc, pio_family, pio_platform, pio_board) - _write_manifest(out_path, merged_doc) - print(f"Wrote {out_path} (nRF52 Nordic DFU)") - return 0 - - print("nRF52 target but no flashable files found in BUILD_DIR; skipping manifest", file=sys.stderr) - return 0 - - # Path 4: generic ESP32 (standard PlatformIO bin output) - if pio_family == "esp32": - doc = emit_generic_esp32(names, board=pio_board) - if doc: - merged_doc = _merge_target_family_meta(doc, pio_family, pio_platform, pio_board) - _write_manifest(out_path, merged_doc) - print(f"Wrote {out_path} (ESP32 bins, update + {'factory' if 'factory' in merged_doc else 'no factory'})") - return 0 - print("ESP32 target but could not detect bootloader+partitions+firmware in BUILD_DIR; skipping manifest", file=sys.stderr) - return 0 - - print(f"No manifest source found for family={pio_family!r}; skipping flash-manifest.json") - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/scripts/extend-merged-with-ota.py b/scripts/extend-merged-with-ota.py new file mode 100644 index 0000000..bd54386 --- /dev/null +++ b/scripts/extend-merged-with-ota.py @@ -0,0 +1,73 @@ +#!/usr/bin/env python3 +""" +Extend a PlatformIO-merged ESP32 factory binary with the Meshtastic OTA +companion binary at the ota_1 partition offset. + +Reads the ota_1 offset from the partition table binary, pads the merged +binary to that offset with 0xFF, and writes the OTA binary in-place. + +Usage: extend-merged-with-ota.py MERGED_BIN PARTITIONS_BIN OTA_BIN +""" +from __future__ import annotations + +import struct +import sys + + +def find_ota1_offset(parts_data: bytes) -> int | None: + """Parse ESP partition table binary to find the ota_1 partition offset.""" + MAGIC = 0xAA50 + APP_TYPE = 0x00 + OTA1_SUBTYPE = 0x11 + for i in range(0, len(parts_data) - 31, 32): + magic, ptype, subtype = struct.unpack_from(' int: + if len(sys.argv) != 4: + print( + 'usage: extend-merged-with-ota.py MERGED_BIN PARTITIONS_BIN OTA_BIN', + file=sys.stderr, + ) + return 2 + + merged_path, parts_path, ota_path = sys.argv[1], sys.argv[2], sys.argv[3] + + with open(parts_path, 'rb') as f: + parts_data = f.read() + + ota1_offset = find_ota1_offset(parts_data) + if ota1_offset is None: + print('No ota_1 partition found in partition table; skipping OTA extension') + return 0 + + with open(merged_path, 'rb') as f: + merged = bytearray(f.read()) + with open(ota_path, 'rb') as f: + ota = f.read() + + if len(merged) > ota1_offset: + print( + f'WARNING: merged binary ({len(merged)} bytes) already extends past ' + f'ota_1 offset (0x{ota1_offset:x}); overwriting', + file=sys.stderr, + ) + elif len(merged) < ota1_offset: + merged += b'\xff' * (ota1_offset - len(merged)) + + merged[ota1_offset:ota1_offset + len(ota)] = ota + + with open(merged_path, 'wb') as f: + f.write(merged) + + print( + f'Extended {merged_path}: OTA companion ({len(ota)} bytes) at 0x{ota1_offset:x}' + ) + return 0 + + +if __name__ == '__main__': + raise SystemExit(main()) diff --git a/src/components/DeviceFlasher.tsx b/src/components/DeviceFlasher.tsx index f38313c..cbd94bc 100644 --- a/src/components/DeviceFlasher.tsx +++ b/src/components/DeviceFlasher.tsx @@ -13,7 +13,7 @@ import { } from "lucide-react" import { useCallback, useEffect, useMemo, useRef, useState } from "react" import { toast } from "sonner" -import { buildFlashParts, manifestFromMap, manifestHasFactorySection } from "../lib/espFlashLayout" +import { buildFlashParts } from "../lib/espFlashLayout" import { ensureSerialPortClosed, ESP_FLASH_WEB_BAUD, @@ -23,8 +23,8 @@ import { type FlashPhase, } from "../lib/espFlashRun" import { runNrfFlash } from "../lib/nrfFlashRun" -import { resolveFlashTargetFamily } from "../lib/flashTargetFamily" -import type { FlashManifest, FlashTargetFamily } from "../lib/untarGz" +import { inferTargetFamilyFromBundle, inferTargetFamilyFromEnv } from "../lib/flashTargetFamily" +import type { FlashTargetFamily } from "../lib/untarGz" import { extractTarGz } from "../lib/untarGz" type FlashProgress = @@ -84,12 +84,14 @@ export default function DeviceFlasher({ const [busy, setBusy] = useState(false) const shareDialogRef = useRef(null) const [eraseFlashForFactory, setEraseFlashForFactory] = useState(false) - const [layoutPreview, setLayoutPreview] = useState(null) + const [bundleFamily, setBundleFamily] = useState( + inferTargetFamilyFromEnv(targetEnv) ?? "esp32" + ) + const [bundleCanErase, setBundleCanErase] = useState(false) const [flashProgress, setFlashProgress] = useState(null) const [bundleLoadError, setBundleLoadError] = useState(null) useEffect(() => { - setLayoutPreview(null) setBundleLoadError(null) let cancelled = false void (async () => { @@ -101,10 +103,11 @@ export default function DeviceFlasher({ } const buf = new Uint8Array(await res.arrayBuffer()) const files = extractTarGz(buf) - const m = manifestFromMap(files) + const { family, canErase } = inferTargetFamilyFromBundle(files, targetEnv) if (!cancelled) { - setLayoutPreview(m) - setEraseFlashForFactory(prev => (manifestHasFactorySection(m) ? prev : false)) + setBundleFamily(family) + setBundleCanErase(canErase) + setEraseFlashForFactory(prev => (canErase ? prev : false)) setBundleLoadError(null) } } catch (e) { @@ -116,25 +119,20 @@ export default function DeviceFlasher({ return () => { cancelled = true } - }, [bundleUrl]) + }, [bundleUrl, targetEnv]) - const resolvedFamily = resolveFlashTargetFamily(layoutPreview, targetEnv) + const resolvedFamily = bundleFamily const flashBlockedReason = unsupportedFlashMessage(resolvedFamily) const canEspFlash = flashBlockedReason === null - const hasFactorySection = useMemo(() => manifestHasFactorySection(layoutPreview), [layoutPreview]) - // ESP32 can always chip-erase regardless of whether the bundle has a factory section. - const canFullReset = resolvedFamily === "esp32" || hasFactorySection + // ESP32 can always chip-erase (merged binary supports it); nRF52 only when canErase is set. + const canFullReset = resolvedFamily === "esp32" || bundleCanErase const prepareBundle = useCallback(async () => { const res = await fetch(bundleUrl) if (!res.ok) throw new Error(`Download failed: ${res.status}`) const buf = new Uint8Array(await res.arrayBuffer()) - const files = extractTarGz(buf) - const m = manifestFromMap(files) - setLayoutPreview(m) - setEraseFlashForFactory(prev => (manifestHasFactorySection(m) ? prev : false)) - return files + return extractTarGz(buf) }, [bundleUrl]) const flash = useCallback(async () => { @@ -159,11 +157,9 @@ export default function DeviceFlasher({ const files = await prepareBundle() if (resolvedFamily === "nrf52") { - const manifest = manifestFromMap(files) await runNrfFlash({ port: port!, files, - manifest, factoryInstall: eraseFlashForFactory, onPhase: label => { setFlashProgress({ kind: "indeterminate", label }) @@ -173,10 +169,7 @@ export default function DeviceFlasher({ }, }) } else { - const plan = buildFlashParts(files, { - factoryInstall: eraseFlashForFactory && hasFactorySection, - resetDeviceStorage: false, - }) + const plan = buildFlashParts(files) if (!plan) { toast.error("Could not detect flash layout from bundle") return diff --git a/src/lib/espFlashLayout.ts b/src/lib/espFlashLayout.ts index e890d1a..3fd635a 100644 --- a/src/lib/espFlashLayout.ts +++ b/src/lib/espFlashLayout.ts @@ -1,10 +1,3 @@ -import { - findInTar, - parseFlashManifest, - type FlashManifest, - type FlashManifestSection, -} from './untarGz' - export type FlashPart = { data: Uint8Array; address: number; name: string } export type BuildFlashPlan = { @@ -12,154 +5,32 @@ export type BuildFlashPlan = { eraseAll: boolean } -export type BuildFlashPartsOptions = { - /** - * When true, use manifest `factory` section (chip erase + merged factory + OTA + filesystem). - * When false, use `update` section or legacy flat `images`. - */ - factoryInstall?: boolean - /** - * Legacy: when an image has optional:true (e.g. LittleFS), skip unless true. - * Ignored for Meshtastic dual manifests (update has no optional rows). - */ - resetDeviceStorage?: boolean -} - -function sortFlashParts(parts: FlashPart[]): FlashPart[] { - return [...parts].sort((a, b) => a.address - b.address) -} - function tarBasename(path: string): string { const parts = path.replace(/^\.\//, '').split('/') return parts[parts.length - 1] ?? path } -function isLittlefsManifestFile(file: string): boolean { - const base = tarBasename(file) - return base.toLowerCase().startsWith('littlefs-') && base.toLowerCase().endsWith('.bin') -} - -function activeSection(m: FlashManifest, factoryInstall: boolean): FlashManifestSection | null { - if (factoryInstall) { - const f = m.factory - if (f && Array.isArray(f.images) && f.images.length > 0) return f - return null - } - const u = m.update - if (u && Array.isArray(u.images) && u.images.length > 0) return u - if (Array.isArray(m.images) && m.images.length > 0) { - return { images: m.images, eraseFlash: m.eraseFlash } - } - return null -} - /** - * PlatformIO projects (e.g. Meshtastic) often emit versioned names like - * firmware-heltec-v3-2.7.20.bin (split app image; merged factory.bin is not bundled for USB flash). + * Build the flash plan from a firmware bundle (tar paths or bare filenames → bytes). + * + * For ESP32: expects a single merged binary (firmware-*.factory.bin) at address 0x0. + * PlatformIO's mergebin target handles all chip-specific offsets during the build. + * + * For custom firmware drag-and-drop (no manifest): finds any .bin that is not a + * sub-component and flashes it at 0x0. */ -function resolveVersionedFirmwareApp( - files: Map -): { data: Uint8Array; name: string } | undefined { - type Entry = { base: string; data: Uint8Array } - const list: Entry[] = [] +export function buildFlashParts(files: Map): BuildFlashPlan | null { for (const [path, data] of files) { const base = tarBasename(path) const lower = base.toLowerCase() - if (lower.startsWith('littlefs-')) continue if ( - lower === 'bootloader.bin' || - lower === 'partitions.bin' || - lower === 'boot_app0.bin' + lower.endsWith('.bin') && + lower !== 'bootloader.bin' && + lower !== 'partitions.bin' && + lower !== 'boot_app0.bin' ) { - continue - } - list.push({ base, data }) - } - - const app = list.find( - e => /^firmware-.+\.bin$/i.test(e.base) && !/\.factory\.bin$/i.test(e.base) - ) - if (app) return { data: app.data, name: app.base } - - return undefined -} - -/** Build ordered flash parts + erase policy from a flat map (tar paths or bare filenames → bytes). */ -export function buildFlashParts( - files: Map, - options: BuildFlashPartsOptions = {} -): BuildFlashPlan | null { - const resetDeviceStorage = options.resetDeviceStorage ?? false - const factoryInstall = options.factoryInstall ?? false - - const manifestRaw = findInTar(files, 'flash-manifest.json') - if (manifestRaw) { - const text = new TextDecoder().decode(manifestRaw) - const m = parseFlashManifest(text) - if (m) { - const section = activeSection(m, factoryInstall) - if (!section) return null - const out: FlashPart[] = [] - for (const img of section.images) { - if (img.optional === true && isLittlefsManifestFile(img.file) && !resetDeviceStorage) continue - const data = findInTar(files, img.file) - if (!data) return null - const addr = typeof img.offset === 'string' ? parseInt(img.offset, 0) : Number(img.offset) - if (!Number.isFinite(addr)) return null - out.push({ data, address: addr, name: img.file }) - } - if (out.length) { - return { - parts: sortFlashParts(out), - eraseAll: Boolean(section.eraseFlash), - } - } + return { parts: [{ data, address: 0x0, name: base }], eraseAll: false } } } - - const bootloader = findInTar(files, 'bootloader.bin') - const partitions = findInTar(files, 'partitions.bin') - const bootApp0 = findInTar(files, 'boot_app0.bin') - const firmwareExact = findInTar(files, 'firmware.bin') - const versioned = resolveVersionedFirmwareApp(files) - - let app: Uint8Array | undefined - let appName: string | undefined - if (firmwareExact) { - app = firmwareExact - appName = 'firmware.bin' - } else if (versioned) { - app = versioned.data - appName = versioned.name - } - - if (bootloader && partitions && app && appName) { - const arr: FlashPart[] = [ - { data: bootloader, address: 0x1000, name: 'bootloader.bin' }, - { data: partitions, address: 0x8000, name: 'partitions.bin' }, - { data: app, address: 0x10000, name: appName }, - ] - if (bootApp0) arr.push({ data: bootApp0, address: 0xe000, name: 'boot_app0.bin' }) - return { parts: sortFlashParts(arr), eraseAll: false } - } - - if (app && appName && !bootloader && !partitions) { - return { parts: [{ data: app, address: 0x0, name: appName }], eraseAll: false } - } - return null } - -export function manifestFromMap( - files: Map, - manifestFile = 'flash-manifest.json' -): FlashManifest | null { - const raw = findInTar(files, manifestFile) - if (!raw) return null - return parseFlashManifest(new TextDecoder().decode(raw)) -} - -/** True if manifest includes a factory (erase + merged image) section with images. */ -export function manifestHasFactorySection(m: FlashManifest | null): boolean { - return Boolean(m?.factory?.images && m.factory.images.length > 0) -} diff --git a/src/lib/flashTargetFamily.ts b/src/lib/flashTargetFamily.ts index a446fd2..016aa19 100644 --- a/src/lib/flashTargetFamily.ts +++ b/src/lib/flashTargetFamily.ts @@ -1,13 +1,7 @@ -import type { FlashManifest, FlashTargetFamily } from "./untarGz" +import type { FlashTargetFamily } from "./untarGz" -const KNOWN: readonly FlashTargetFamily[] = ["esp32", "esp8266", "nrf52", "rp2040", "unknown"] -function coerceTargetFamily(v: unknown): FlashTargetFamily | undefined { - if (typeof v !== "string") return undefined - return KNOWN.includes(v as FlashTargetFamily) ? (v as FlashTargetFamily) : undefined -} - -/** Heuristic when manifest lacks targetFamily (older bundles). */ +/** Heuristic when bundle file types are ambiguous (e.g. UF2 shared by nRF52 and RP2040). */ export function inferTargetFamilyFromEnv(env: string | null | undefined): FlashTargetFamily | null { if (!env?.trim()) return null const e = env.toLowerCase() @@ -27,22 +21,39 @@ export function inferTargetFamilyFromEnv(env: string | null | undefined): FlashT } /** - * Prefer manifest.targetFamily from CI; else env-name heuristic; else esp32 for legacy ESP-only bundles. + * Derive target family and erase capability from the files in a firmware bundle. + * + * Rules (in priority order): + * *.factory.bin present → esp32 (always supports chip erase) + * firmware.dat present → nrf52 Nordic DFU (always supports erase) + * *.uf2 present (not nuke.uf2) → nrf52 or rp2040 (use env name to disambiguate) + * + * canErase: + * esp32 → always true + * nrf52 DFU → always true + * nrf52 UF2 → true only when nuke.uf2 is also in the bundle */ -export function resolveFlashTargetFamily( - manifest: FlashManifest | null, - targetEnv: string | null | undefined -): FlashTargetFamily { - const fromManifest = coerceTargetFamily(manifest?.targetFamily) - if (fromManifest && fromManifest !== "unknown") { - return fromManifest +export function inferTargetFamilyFromBundle( + files: Map, + targetEnv?: string | null +): { family: FlashTargetFamily; canErase: boolean } { + let hasUf2 = false + let hasNukeUf2 = false + + for (const path of files.keys()) { + const base = path.replace(/^.*\//, "").toLowerCase() + if (base.endsWith(".factory.bin")) return { family: "esp32", canErase: true } + if (base === "firmware.dat") return { family: "nrf52", canErase: true } + if (base === "nuke.uf2") hasNukeUf2 = true + else if (base.endsWith(".uf2")) hasUf2 = true } - const fromEnv = inferTargetFamilyFromEnv(targetEnv ?? null) - if (fromEnv) { - return fromEnv + + if (hasUf2) { + const family = inferTargetFamilyFromEnv(targetEnv) ?? "nrf52" + return { family, canErase: hasNukeUf2 } } - if (fromManifest === "unknown") { - return "unknown" - } - return "esp32" + + // No clear signal — fall back to env name heuristic or default to esp32. + const family = inferTargetFamilyFromEnv(targetEnv) ?? "esp32" + return { family, canErase: family === "esp32" } } diff --git a/src/lib/nrfFlashRun.ts b/src/lib/nrfFlashRun.ts index 7dee190..ad740ec 100644 --- a/src/lib/nrfFlashRun.ts +++ b/src/lib/nrfFlashRun.ts @@ -1,5 +1,4 @@ import { findInTar } from './untarGz' -import type { FlashManifest } from './untarGz' import { buildNordicDfuPlan, runNordicDfu } from './nrfDfuRun' export type NrfFlashPlan = { @@ -30,59 +29,32 @@ function pickDirectory(): Promise { } /** - * Build an nRF52 UF2 flash plan from the bundle tarball + parsed manifest. - * Reads the `update` section for a normal flash, or `factory` when factoryInstall is true. - * Images with role "uf2" are the firmware; role "nuke" is the erase-all UF2. - * - * Falls back to scanning the bundle directly for *.uf2 when no manifest is present - * (covers pre-manifest bundles and builds where emit-flash-manifest.py did not run). + * Build an nRF52 UF2 flash plan from the bundle files. + * Scans for *.uf2 (preferring names starting with "firmware", excluding nuke.uf2). + * When factoryInstall is true, also includes nuke.uf2 if present in the bundle. */ export function buildNrfPlan( files: Map, - manifest: FlashManifest | null, factoryInstall: boolean ): NrfFlashPlan | null { - const section = factoryInstall ? (manifest?.factory ?? manifest?.update) : manifest?.update + let firmwareFile: Uint8Array | undefined + let firmwareName: string | undefined - if (section?.images?.length) { - let firmwareFile: Uint8Array | undefined - let firmwareName: string | undefined - let nukeFile: Uint8Array | undefined - - for (const img of section.images) { - const role = img.role?.toLowerCase() - if (role === 'nuke') { - nukeFile = findInTar(files, img.file) - } else if (role === 'uf2' || img.file.toLowerCase().endsWith('.uf2')) { - if (!firmwareFile) { - firmwareFile = findInTar(files, img.file) - firmwareName = img.file - } - } - } - - if (firmwareFile && firmwareName) { - return { firmwareFile, firmwareName, nukeFile } - } - } - - // Fallback: scan bundle files directly for *.uf2 (no manifest or manifest had no UF2 images). - // Prefer names starting with "firmware", exclude nuke.uf2. - let fallbackName: string | undefined - let fallbackData: Uint8Array | undefined for (const [path, data] of files) { const base = path.replace(/^.*\//, '') const baseLower = base.toLowerCase() if (!baseLower.endsWith('.uf2') || baseLower === 'nuke.uf2') continue - if (!fallbackName || baseLower.startsWith('firmware')) { - fallbackName = base - fallbackData = data + if (!firmwareName || baseLower.startsWith('firmware')) { + firmwareName = base + firmwareFile = data if (baseLower.startsWith('firmware')) break } } - if (!fallbackData || !fallbackName) return null - return { firmwareFile: fallbackData, firmwareName: fallbackName } + if (!firmwareFile || !firmwareName) return null + + const nukeFile = factoryInstall ? findInTar(files, 'nuke.uf2') : undefined + return { firmwareFile, firmwareName, nukeFile } } /** @@ -153,7 +125,7 @@ async function runUf2Flash(options: { /** * Flash an nRF52 device. Routes to Nordic Serial DFU (seamless, progress bar) when the bundle - * contains a Nordic DFU manifest (firmware.bin + firmware.dat), otherwise falls back to UF2 + * contains a Nordic DFU package (firmware.bin + firmware.dat), otherwise falls back to UF2 * via the File System Access API drive picker. * * The port must have already received the 1200-baud CDC bootloader pulse and be closed. @@ -161,12 +133,11 @@ async function runUf2Flash(options: { export async function runNrfFlash(options: { port: SerialPort files: Map - manifest: FlashManifest | null factoryInstall: boolean onPhase: (label: string) => void onWriteProgress: (p: NrfWriteProgressPayload) => void }): Promise { - const { port, files, manifest, factoryInstall, onPhase, onWriteProgress } = options + const { port, files, factoryInstall, onPhase, onWriteProgress } = options // Prefer Nordic Serial DFU when the bundle contains a Nordic DFU package (bin + dat). const dfuPlan = buildNordicDfuPlan(files) @@ -182,7 +153,7 @@ export async function runNrfFlash(options: { } // Fall back to UF2 drive picker (Meshtastic nRF52 and other UF2 bootloader boards). - const uf2Plan = buildNrfPlan(files, manifest, factoryInstall) + const uf2Plan = buildNrfPlan(files, factoryInstall) if (!uf2Plan) { throw new Error('No flashable firmware found in bundle (expected Nordic DFU .bin/.dat or a .uf2 file)') } diff --git a/src/lib/untarGz.ts b/src/lib/untarGz.ts index 0aeafe3..bd90328 100644 --- a/src/lib/untarGz.ts +++ b/src/lib/untarGz.ts @@ -45,47 +45,5 @@ export function findInTar(files: Map, filename: string): Uin return undefined } -export type FlashManifestImage = { - file: string - offset: number | string - /** When true on LittleFS rows, Mesh Forge skips unless optional handling passes (legacy flat manifests). */ - optional?: boolean - role?: string -} - -/** One flash plan (update or factory) inside flash-manifest.json. */ -export type FlashManifestSection = { - images: FlashManifestImage[] - /** When true, flasher performs full chip erase before write. */ - eraseFlash?: boolean -} - /** Coarse MCU family for USB flasher entry + tool selection (from CI / PlatformIO). */ export type FlashTargetFamily = "esp32" | "esp8266" | "nrf52" | "rp2040" | "unknown" - -/** - * Root flash-manifest.json: Meshtastic-style `update` + optional `factory`, or legacy flat `images`. - */ -export type FlashManifest = { - update?: FlashManifestSection - factory?: FlashManifestSection - /** Legacy single-layout manifest. */ - images?: FlashManifestImage[] - eraseFlash?: boolean - targetFamily?: FlashTargetFamily - platform?: string - board?: string -} - -export function parseFlashManifest(json: string): FlashManifest | null { - try { - const o = JSON.parse(json) as FlashManifest - if (!o || typeof o !== "object") return null - if (o.update && Array.isArray(o.update.images) && o.update.images.length > 0) return o - if (o.factory && Array.isArray(o.factory.images) && o.factory.images.length > 0) return o - if (Array.isArray(o.images) && o.images.length > 0) return o - return null - } catch { - return null - } -}