From ca43841da8c340acb5237e73de149b7ac260179d Mon Sep 17 00:00:00 2001 From: Ben Allfree Date: Fri, 10 Apr 2026 18:36:44 -0700 Subject: [PATCH] Refactor EspFlasher component to implement factory reset functionality and update flash manifest handling. Modify buildFlashParts to support factory and update sections in manifests, ensuring proper erase policies. Enhance user interface to reflect changes in reset behavior and provide clearer warnings regarding data loss during flashing. --- .github/workflows/custom_build.yml | 1 - .github/workflows/custom_build_test.yml | 1 - scripts/emit-flash-manifest.py | 179 +++++++++++++++++------- src/components/EspFlasher.tsx | 51 ++++--- src/lib/espFlashLayout.ts | 68 +++++++-- src/lib/untarGz.ts | 27 +++- 6 files changed, 240 insertions(+), 87 deletions(-) diff --git a/.github/workflows/custom_build.yml b/.github/workflows/custom_build.yml index 29b83dc..dc18e32 100644 --- a/.github/workflows/custom_build.yml +++ b/.github/workflows/custom_build.yml @@ -194,7 +194,6 @@ jobs: mkdir -p "$STAGE" shopt -s nullglob for f in "$BUILD_DIR"/*.bin "$BUILD_DIR"/*.uf2 "$BUILD_DIR"/*.hex; do - case "$f" in *.factory.bin) continue ;; esac cp -a "$f" "$STAGE/" done shopt -u nullglob diff --git a/.github/workflows/custom_build_test.yml b/.github/workflows/custom_build_test.yml index 29b83dc..dc18e32 100644 --- a/.github/workflows/custom_build_test.yml +++ b/.github/workflows/custom_build_test.yml @@ -194,7 +194,6 @@ jobs: mkdir -p "$STAGE" shopt -s nullglob for f in "$BUILD_DIR"/*.bin "$BUILD_DIR"/*.uf2 "$BUILD_DIR"/*.hex; do - case "$f" in *.factory.bin) continue ;; esac cp -a "$f" "$STAGE/" done shopt -u nullglob diff --git a/scripts/emit-flash-manifest.py b/scripts/emit-flash-manifest.py index 7507e21..37f9721 100644 --- a/scripts/emit-flash-manifest.py +++ b/scripts/emit-flash-manifest.py @@ -5,10 +5,12 @@ Emit flash-manifest.json for Mesh Forge USB flasher. If BUILD_DIR/flash-manifest.json already exists (project-supplied), merge in targetFamily from PlatformIO when missing. -Otherwise, if Meshtastic-style *.mt.json is present, synthesize a manifest -from partition table + on-disk split artifacts (no *.factory.bin — USB bundles -omit the merged image). Only LittleFS rows use optional:true (Mesh Forge: -Reset device storage). OTA slots accept mt-*-ota.bin (ESP32/S3) or bleota-c3.bin (C3/C6). +Otherwise, if Meshtastic-style *.mt.json is present, synthesize one JSON with: + - "update": app @ ota_0 + BLE OTA @ ota_1, eraseFlash false (Meshtastic “Update”). + - "factory": merged *.factory.bin @ 0 + OTA + LittleFS, eraseFlash true (Meshtastic “Erase device”), + omitted if no factory.bin in BUILD_DIR. + +OTA: mt-*-ota.bin (ESP32/S3) or bleota-c3.bin (C3/C6). Optional args: PROJECT_ROOT TARGET_ENV — merged PIO config for that env fills targetFamily (and platform/board) for the USB flasher UI. @@ -183,11 +185,33 @@ def _dedupe_same_offset(images: list[dict]) -> list[dict]: return out -def emit_from_mt(build_dir: str, mt: dict) -> dict | None: +def _is_non_factory_firmware(name: str) -> bool: + return bool(re.match(r"^firmware-.+\.bin$", name, re.I) and not re.search(r"\.factory\.bin$", name, re.I)) + + +def pick_factory_bin(names: set[str]) -> str | None: + cands = sorted(n for n in names if re.match(r"^firmware-.+\.factory\.bin$", n, re.I)) + return cands[0] if cands else None + + +def pick_non_factory_firmware(names: set[str], mt: dict) -> str | None: + for entry in mt.get("files") or []: + fname = entry.get("name") + if not fname or fname not in names or entry.get("part_name") != "app0": + continue + if _is_non_factory_firmware(fname): + return fname + for n in sorted(names): + if _is_non_factory_firmware(n): + return n + return None + + +def emit_meshtastic_update(build_dir: str, mt: dict) -> dict | None: + """Official-style update: app @ ota_0 + OTA @ ota_1 only (no erase, no factory, no FS).""" parts: list[dict] = mt.get("part") or [] if not parts: return None - names = list_basenames(build_dir) images: list[dict] = [] seen: set[str] = set() @@ -201,51 +225,93 @@ def emit_from_mt(build_dir: str, mt: dict) -> dict | None: images.append(row) seen.add(file) - add("bootloader.bin", 0x1000) - add("partitions.bin", 0x8000) - add("boot_app0.bin", 0xE000) + app_bin = pick_non_factory_firmware(names, mt) + if not app_bin: + return None + off0 = part_offset_for_slot(parts, "app0") + if off0 is None: + return None + add(app_bin, off0, optional=False) + + ota_added = False + for n in sorted(names): + if re.match(r"^mt-.+-ota\.bin$", n, re.I): + o = ota1_offset(parts) + if o is not None: + add(n, o, optional=False) + ota_added = True + break + if not ota_added and "bleota-c3.bin" in names: + o = ota1_offset(parts) + if o is not None: + add("bleota-c3.bin", o, optional=False) + + images = _dedupe_same_offset(images) + images.sort(key=_offset_int) + return {"images": images, "eraseFlash": False} + + +def emit_meshtastic_full(build_dir: str, mt: dict, factory_bin: str) -> dict | None: + """Official-style erase + factory: merged factory @ 0 + OTA + LittleFS.""" + parts: list[dict] = mt.get("part") or [] + if not parts: + return None + names = list_basenames(build_dir) + if factory_bin not in names: + return None + + images: list[dict] = [] + seen: set[str] = set() + + def add(file: str, offset: int, optional: bool = False) -> None: + if file not in names or file in seen: + return + row: dict = {"file": file, "offset": offset} + if optional: + row["optional"] = True + images.append(row) + seen.add(file) + + add(factory_bin, 0, optional=False) + + ota_added = False + for n in sorted(names): + if re.match(r"^mt-.+-ota\.bin$", n, re.I): + o = ota1_offset(parts) + if o is not None: + add(n, o, optional=False) + ota_added = True + break + if not ota_added and "bleota-c3.bin" in names: + o = ota1_offset(parts) + if o is not None: + add("bleota-c3.bin", o, optional=False) + + for n in sorted(names): + if n.startswith("littlefs-") and n.endswith(".bin"): + so = spiffs_offset(parts) + if so is not None: + add(n, so, optional=False) for entry in mt.get("files") or []: fname = entry.get("name") part_name = entry.get("part_name") - if not fname or not part_name or fname not in names: + if not fname or part_name != "spiffs" or fname not in names: continue - off = part_offset_for_slot(parts, str(part_name)) - if off is None: + if not str(fname).lower().startswith("littlefs-"): continue - # Only LittleFS is optional in the bundle; Mesh Forge flashes it when the user enables - # "Reset device storage". Bootloader, partitions, app, and BLE OTA are always flashed when present. - opt = bool(fname.startswith("littlefs-")) - add(fname, off, optional=opt) - - for n in sorted(names): - if n.startswith("littlefs-") and n.endswith(".bin"): - off = spiffs_offset(parts) - if off is not None: - add(n, off, optional=True) - - for n in sorted(names): - if re.match(r"^mt-.+-ota\.bin$", n, re.I): - off = ota1_offset(parts) - if off is not None: - add(n, off, optional=False) - - # ESP32-C3/C6 Meshtastic workflow uses bleota-c3.bin at ota_1 (see build_firmware.yml). - if "bleota-c3.bin" in names: - off = ota1_offset(parts) - if off is not None: - add("bleota-c3.bin", off, optional=False) - - if not images: - return None + so = spiffs_offset(parts) + if so is not None: + add(fname, so, optional=False) images = _dedupe_same_offset(images) + images.sort(key=_offset_int) - if not any(re.match(r"^firmware-.+\.bin$", im["file"], re.I) for im in images): + o1 = ota1_offset(parts) + if o1 is not None and not any(_offset_int(im) == o1 for im in images): return None - images.sort(key=_offset_int) - return {"images": images} + return {"images": images, "eraseFlash": True} def pick_mt_json(build_dir: str) -> str | None: @@ -283,9 +349,15 @@ def main() -> int: if os.path.isfile(out_path): with open(out_path, encoding="utf-8") as f: manifest = json.load(f) - if not isinstance(manifest, dict) or not isinstance(manifest.get("images"), list): + if not isinstance(manifest, dict): print(f"Invalid existing {out_path}", file=sys.stderr) return 1 + ok_legacy = isinstance(manifest.get("images"), list) + up = manifest.get("update") + ok_dual = isinstance(up, dict) and isinstance(up.get("images"), list) + if not ok_legacy and not ok_dual: + print(f"Invalid existing {out_path} (need images[] or update.images[])", file=sys.stderr) + return 1 merged = _merge_target_family_meta(manifest, pio_family, pio_platform, pio_board) if merged != manifest: _write_manifest(out_path, merged) @@ -302,14 +374,27 @@ def main() -> int: with open(mt_path, encoding="utf-8") as f: mt = json.load(f) - manifest = emit_from_mt(build_dir, mt) - if not manifest: - print("Could not synthesize flash-manifest.json from mt.json") + names = list_basenames(build_dir) + manifest_update = emit_meshtastic_update(build_dir, mt) + if not manifest_update: + print("Could not synthesize flash-manifest.json (update) from mt.json") return 0 - manifest = _merge_target_family_meta(manifest, pio_family, pio_platform, pio_board) - _write_manifest(out_path, manifest) - print(f"Wrote {out_path}") + doc: dict = {"update": manifest_update} + factory = pick_factory_bin(names) + if factory: + manifest_full = emit_meshtastic_full(build_dir, mt, factory) + if manifest_full: + doc["factory"] = manifest_full + else: + print("Could not build factory section; omitting factory key", file=sys.stderr) + else: + print("No firmware-*.factory.bin in BUILD_DIR; omitting factory section") + + merged_doc = _merge_target_family_meta(doc, pio_family, pio_platform, pio_board) + _write_manifest(out_path, merged_doc) + print(f"Wrote {out_path} (update + {'factory' if 'factory' in merged_doc else 'no factory'})") + return 0 diff --git a/src/components/EspFlasher.tsx b/src/components/EspFlasher.tsx index 71c5556..3dcf20d 100644 --- a/src/components/EspFlasher.tsx +++ b/src/components/EspFlasher.tsx @@ -13,7 +13,7 @@ import { } from "lucide-react" import { useCallback, useEffect, useMemo, useRef, useState } from "react" import { toast } from "sonner" -import { buildFlashParts, manifestFromMap } from "../lib/espFlashLayout" +import { buildFlashParts, manifestFromMap, manifestHasFactorySection } from "../lib/espFlashLayout" import { ensureSerialPortClosed, ESP_FLASH_WEB_BAUD, @@ -85,7 +85,7 @@ export default function EspFlasher({ }: EspFlasherProps) { const [busy, setBusy] = useState(false) const shareDialogRef = useRef(null) - const [resetDeviceStorage, setResetDeviceStorage] = useState(false) + const [eraseFlashForFactory, setEraseFlashForFactory] = useState(false) const [layoutPreview, setLayoutPreview] = useState(null) const [flashProgress, setFlashProgress] = useState(null) const [bundleLoadError, setBundleLoadError] = useState(null) @@ -106,6 +106,7 @@ export default function EspFlasher({ const m = manifestFromMap(files) if (!cancelled) { setLayoutPreview(m) + setEraseFlashForFactory(prev => (manifestHasFactorySection(m) ? prev : false)) setBundleLoadError(null) } } catch (e) { @@ -123,6 +124,8 @@ export default function EspFlasher({ const flashBlockedReason = unsupportedFlashMessage(resolvedFamily) const canEspFlash = flashBlockedReason === null + const hasFactorySection = useMemo(() => manifestHasFactorySection(layoutPreview), [layoutPreview]) + const prepareBundle = useCallback(async () => { const res = await fetch(bundleUrl) if (!res.ok) throw new Error(`Download failed: ${res.status}`) @@ -130,6 +133,7 @@ export default function EspFlasher({ const files = extractTarGz(buf) const m = manifestFromMap(files) setLayoutPreview(m) + setEraseFlashForFactory(prev => (manifestHasFactorySection(m) ? prev : false)) return files }, [bundleUrl]) @@ -153,17 +157,20 @@ export default function EspFlasher({ setFlashProgress({ kind: "indeterminate", label: "Downloading firmware…" }) const files = await prepareBundle() - const parts = buildFlashParts(files, { resetDeviceStorage }) - if (!parts) { + const plan = buildFlashParts(files, { + factoryInstall: eraseFlashForFactory, + resetDeviceStorage: false, + }) + if (!plan) { toast.error("Could not detect flash layout from bundle") return } await runEspFlash({ port, - parts, + parts: plan.parts, baud: ESP_FLASH_WEB_BAUD, - eraseAll: false, + eraseAll: plan.eraseAll, onPhase: phase => { setFlashProgress({ kind: "indeterminate", label: PHASE_LABEL[phase] }) }, @@ -193,7 +200,7 @@ export default function EspFlasher({ setFlashProgress(null) } } - }, [resetDeviceStorage, prepareBundle, canEspFlash, flashBlockedReason]) + }, [eraseFlashForFactory, prepareBundle, canEspFlash, flashBlockedReason]) const shareUrlTrimmed = useMemo(() => sharePageUrl?.trim() ?? "", [sharePageUrl]) const canNativeShare = typeof navigator !== "undefined" && typeof navigator.share === "function" @@ -264,31 +271,39 @@ export default function EspFlasher({ - Reset device storage + Full device reset - {resetDeviceStorage ? ( + {!hasFactorySection ? ( +

+ Full device reset is not available for this bundle—only an update. Typical reasons: an older download, a + build that did not include a factory image, or firmware not produced by this app’s usual ESP32 flow. +

+ ) : null} + + {eraseFlashForFactory ? (

- Warning: all user data on the device will be deleted - when you flash (channels, preferences, and stored files). + You will lose everything on the radio: channels and + settings, private keys, node info, message history, and any stored maps or telemetry. Only use this if you are + deliberately starting over or recovering a bad state.

) : null} diff --git a/src/lib/espFlashLayout.ts b/src/lib/espFlashLayout.ts index 644e74e..e890d1a 100644 --- a/src/lib/espFlashLayout.ts +++ b/src/lib/espFlashLayout.ts @@ -1,16 +1,26 @@ -import { findInTar, parseFlashManifest, type FlashManifest, type FlashManifestImage } from './untarGz' +import { + findInTar, + parseFlashManifest, + type FlashManifest, + type FlashManifestSection, +} from './untarGz' export type FlashPart = { data: Uint8Array; address: number; name: string } -function manifestImageOffset(im: FlashManifestImage): number | null { - const addr = typeof im.offset === 'string' ? parseInt(im.offset, 0) : Number(im.offset) - return Number.isFinite(addr) ? addr : null +export type BuildFlashPlan = { + parts: FlashPart[] + eraseAll: boolean } export type BuildFlashPartsOptions = { /** - * When true, flash optional LittleFS images from the manifest (wipes Meshtastic storage on device). - * Bootloader, partitions, firmware, and other non-LittleFS images are always included when present. + * When true, use manifest `factory` section (chip erase + merged factory + OTA + filesystem). + * When false, use `update` section or legacy flat `images`. + */ + factoryInstall?: boolean + /** + * Legacy: when an image has optional:true (e.g. LittleFS), skip unless true. + * Ignored for Meshtastic dual manifests (update has no optional rows). */ resetDeviceStorage?: boolean } @@ -29,6 +39,20 @@ function isLittlefsManifestFile(file: string): boolean { return base.toLowerCase().startsWith('littlefs-') && base.toLowerCase().endsWith('.bin') } +function activeSection(m: FlashManifest, factoryInstall: boolean): FlashManifestSection | null { + if (factoryInstall) { + const f = m.factory + if (f && Array.isArray(f.images) && f.images.length > 0) return f + return null + } + const u = m.update + if (u && Array.isArray(u.images) && u.images.length > 0) return u + if (Array.isArray(m.images) && m.images.length > 0) { + return { images: m.images, eraseFlash: m.eraseFlash } + } + return null +} + /** * PlatformIO projects (e.g. Meshtastic) often emit versioned names like * firmware-heltec-v3-2.7.20.bin (split app image; merged factory.bin is not bundled for USB flash). @@ -60,20 +84,23 @@ function resolveVersionedFirmwareApp( return undefined } -/** Build ordered flash parts from a flat map (tar paths or bare filenames → bytes). */ +/** Build ordered flash parts + erase policy from a flat map (tar paths or bare filenames → bytes). */ export function buildFlashParts( files: Map, options: BuildFlashPartsOptions = {} -): FlashPart[] | null { +): BuildFlashPlan | null { const resetDeviceStorage = options.resetDeviceStorage ?? false + const factoryInstall = options.factoryInstall ?? false const manifestRaw = findInTar(files, 'flash-manifest.json') if (manifestRaw) { const text = new TextDecoder().decode(manifestRaw) const m = parseFlashManifest(text) if (m) { + const section = activeSection(m, factoryInstall) + if (!section) return null const out: FlashPart[] = [] - for (const img of m.images) { + for (const img of section.images) { if (img.optional === true && isLittlefsManifestFile(img.file) && !resetDeviceStorage) continue const data = findInTar(files, img.file) if (!data) return null @@ -81,7 +108,12 @@ export function buildFlashParts( if (!Number.isFinite(addr)) return null out.push({ data, address: addr, name: img.file }) } - if (out.length) return sortFlashParts(out) + if (out.length) { + return { + parts: sortFlashParts(out), + eraseAll: Boolean(section.eraseFlash), + } + } } } @@ -108,18 +140,26 @@ export function buildFlashParts( { data: app, address: 0x10000, name: appName }, ] if (bootApp0) arr.push({ data: bootApp0, address: 0xe000, name: 'boot_app0.bin' }) - return sortFlashParts(arr) + return { parts: sortFlashParts(arr), eraseAll: false } } if (app && appName && !bootloader && !partitions) { - return [{ data: app, address: 0x0, name: appName }] + return { parts: [{ data: app, address: 0x0, name: appName }], eraseAll: false } } return null } -export function manifestFromMap(files: Map): FlashManifest | null { - const raw = findInTar(files, 'flash-manifest.json') +export function manifestFromMap( + files: Map, + manifestFile = 'flash-manifest.json' +): FlashManifest | null { + const raw = findInTar(files, manifestFile) if (!raw) return null return parseFlashManifest(new TextDecoder().decode(raw)) } + +/** True if manifest includes a factory (erase + merged image) section with images. */ +export function manifestHasFactorySection(m: FlashManifest | null): boolean { + return Boolean(m?.factory?.images && m.factory.images.length > 0) +} diff --git a/src/lib/untarGz.ts b/src/lib/untarGz.ts index cc625e4..0aeafe3 100644 --- a/src/lib/untarGz.ts +++ b/src/lib/untarGz.ts @@ -48,28 +48,43 @@ export function findInTar(files: Map, filename: string): Uin export type FlashManifestImage = { file: string offset: number | string - /** When true on LittleFS rows, Mesh Forge skips unless the user enables Reset device storage. */ + /** When true on LittleFS rows, Mesh Forge skips unless optional handling passes (legacy flat manifests). */ optional?: boolean role?: string } +/** One flash plan (update or factory) inside flash-manifest.json. */ +export type FlashManifestSection = { + images: FlashManifestImage[] + /** When true, flasher performs full chip erase before write. */ + eraseFlash?: boolean +} + /** Coarse MCU family for USB flasher entry + tool selection (from CI / PlatformIO). */ export type FlashTargetFamily = "esp32" | "esp8266" | "nrf52" | "rp2040" | "unknown" +/** + * Root flash-manifest.json: Meshtastic-style `update` + optional `factory`, or legacy flat `images`. + */ export type FlashManifest = { - images: FlashManifestImage[] + update?: FlashManifestSection + factory?: FlashManifestSection + /** Legacy single-layout manifest. */ + images?: FlashManifestImage[] + eraseFlash?: boolean targetFamily?: FlashTargetFamily - /** Raw PlatformIO `platform` (debug / advanced UI). */ platform?: string - /** Raw PlatformIO `board` (debug / advanced UI). */ board?: string } export function parseFlashManifest(json: string): FlashManifest | null { try { const o = JSON.parse(json) as FlashManifest - if (!o || !Array.isArray(o.images)) return null - return o + if (!o || typeof o !== "object") return null + if (o.update && Array.isArray(o.update.images) && o.update.images.length > 0) return o + if (o.factory && Array.isArray(o.factory.images) && o.factory.images.length > 0) return o + if (Array.isArray(o.images) && o.images.length > 0) return o + return null } catch { return null }