diff --git a/.github/workflows/custom_build.yml b/.github/workflows/custom_build.yml index 29b83dc..dc18e32 100644 --- a/.github/workflows/custom_build.yml +++ b/.github/workflows/custom_build.yml @@ -194,7 +194,6 @@ jobs: mkdir -p "$STAGE" shopt -s nullglob for f in "$BUILD_DIR"/*.bin "$BUILD_DIR"/*.uf2 "$BUILD_DIR"/*.hex; do - case "$f" in *.factory.bin) continue ;; esac cp -a "$f" "$STAGE/" done shopt -u nullglob diff --git a/.github/workflows/custom_build_test.yml b/.github/workflows/custom_build_test.yml index 29b83dc..dc18e32 100644 --- a/.github/workflows/custom_build_test.yml +++ b/.github/workflows/custom_build_test.yml @@ -194,7 +194,6 @@ jobs: mkdir -p "$STAGE" shopt -s nullglob for f in "$BUILD_DIR"/*.bin "$BUILD_DIR"/*.uf2 "$BUILD_DIR"/*.hex; do - case "$f" in *.factory.bin) continue ;; esac cp -a "$f" "$STAGE/" done shopt -u nullglob diff --git a/scripts/emit-flash-manifest.py b/scripts/emit-flash-manifest.py index 7507e21..37f9721 100644 --- a/scripts/emit-flash-manifest.py +++ b/scripts/emit-flash-manifest.py @@ -5,10 +5,12 @@ Emit flash-manifest.json for Mesh Forge USB flasher. If BUILD_DIR/flash-manifest.json already exists (project-supplied), merge in targetFamily from PlatformIO when missing. -Otherwise, if Meshtastic-style *.mt.json is present, synthesize a manifest -from partition table + on-disk split artifacts (no *.factory.bin — USB bundles -omit the merged image). Only LittleFS rows use optional:true (Mesh Forge: -Reset device storage). OTA slots accept mt-*-ota.bin (ESP32/S3) or bleota-c3.bin (C3/C6). +Otherwise, if Meshtastic-style *.mt.json is present, synthesize one JSON with: + - "update": app @ ota_0 + BLE OTA @ ota_1, eraseFlash false (Meshtastic “Update”). + - "factory": merged *.factory.bin @ 0 + OTA + LittleFS, eraseFlash true (Meshtastic “Erase device”), + omitted if no factory.bin in BUILD_DIR. + +OTA: mt-*-ota.bin (ESP32/S3) or bleota-c3.bin (C3/C6). Optional args: PROJECT_ROOT TARGET_ENV — merged PIO config for that env fills targetFamily (and platform/board) for the USB flasher UI. @@ -183,11 +185,33 @@ def _dedupe_same_offset(images: list[dict]) -> list[dict]: return out -def emit_from_mt(build_dir: str, mt: dict) -> dict | None: +def _is_non_factory_firmware(name: str) -> bool: + return bool(re.match(r"^firmware-.+\.bin$", name, re.I) and not re.search(r"\.factory\.bin$", name, re.I)) + + +def pick_factory_bin(names: set[str]) -> str | None: + cands = sorted(n for n in names if re.match(r"^firmware-.+\.factory\.bin$", n, re.I)) + return cands[0] if cands else None + + +def pick_non_factory_firmware(names: set[str], mt: dict) -> str | None: + for entry in mt.get("files") or []: + fname = entry.get("name") + if not fname or fname not in names or entry.get("part_name") != "app0": + continue + if _is_non_factory_firmware(fname): + return fname + for n in sorted(names): + if _is_non_factory_firmware(n): + return n + return None + + +def emit_meshtastic_update(build_dir: str, mt: dict) -> dict | None: + """Official-style update: app @ ota_0 + OTA @ ota_1 only (no erase, no factory, no FS).""" parts: list[dict] = mt.get("part") or [] if not parts: return None - names = list_basenames(build_dir) images: list[dict] = [] seen: set[str] = set() @@ -201,51 +225,93 @@ def emit_from_mt(build_dir: str, mt: dict) -> dict | None: images.append(row) seen.add(file) - add("bootloader.bin", 0x1000) - add("partitions.bin", 0x8000) - add("boot_app0.bin", 0xE000) + app_bin = pick_non_factory_firmware(names, mt) + if not app_bin: + return None + off0 = part_offset_for_slot(parts, "app0") + if off0 is None: + return None + add(app_bin, off0, optional=False) + + ota_added = False + for n in sorted(names): + if re.match(r"^mt-.+-ota\.bin$", n, re.I): + o = ota1_offset(parts) + if o is not None: + add(n, o, optional=False) + ota_added = True + break + if not ota_added and "bleota-c3.bin" in names: + o = ota1_offset(parts) + if o is not None: + add("bleota-c3.bin", o, optional=False) + + images = _dedupe_same_offset(images) + images.sort(key=_offset_int) + return {"images": images, "eraseFlash": False} + + +def emit_meshtastic_full(build_dir: str, mt: dict, factory_bin: str) -> dict | None: + """Official-style erase + factory: merged factory @ 0 + OTA + LittleFS.""" + parts: list[dict] = mt.get("part") or [] + if not parts: + return None + names = list_basenames(build_dir) + if factory_bin not in names: + return None + + images: list[dict] = [] + seen: set[str] = set() + + def add(file: str, offset: int, optional: bool = False) -> None: + if file not in names or file in seen: + return + row: dict = {"file": file, "offset": offset} + if optional: + row["optional"] = True + images.append(row) + seen.add(file) + + add(factory_bin, 0, optional=False) + + ota_added = False + for n in sorted(names): + if re.match(r"^mt-.+-ota\.bin$", n, re.I): + o = ota1_offset(parts) + if o is not None: + add(n, o, optional=False) + ota_added = True + break + if not ota_added and "bleota-c3.bin" in names: + o = ota1_offset(parts) + if o is not None: + add("bleota-c3.bin", o, optional=False) + + for n in sorted(names): + if n.startswith("littlefs-") and n.endswith(".bin"): + so = spiffs_offset(parts) + if so is not None: + add(n, so, optional=False) for entry in mt.get("files") or []: fname = entry.get("name") part_name = entry.get("part_name") - if not fname or not part_name or fname not in names: + if not fname or part_name != "spiffs" or fname not in names: continue - off = part_offset_for_slot(parts, str(part_name)) - if off is None: + if not str(fname).lower().startswith("littlefs-"): continue - # Only LittleFS is optional in the bundle; Mesh Forge flashes it when the user enables - # "Reset device storage". Bootloader, partitions, app, and BLE OTA are always flashed when present. - opt = bool(fname.startswith("littlefs-")) - add(fname, off, optional=opt) - - for n in sorted(names): - if n.startswith("littlefs-") and n.endswith(".bin"): - off = spiffs_offset(parts) - if off is not None: - add(n, off, optional=True) - - for n in sorted(names): - if re.match(r"^mt-.+-ota\.bin$", n, re.I): - off = ota1_offset(parts) - if off is not None: - add(n, off, optional=False) - - # ESP32-C3/C6 Meshtastic workflow uses bleota-c3.bin at ota_1 (see build_firmware.yml). - if "bleota-c3.bin" in names: - off = ota1_offset(parts) - if off is not None: - add("bleota-c3.bin", off, optional=False) - - if not images: - return None + so = spiffs_offset(parts) + if so is not None: + add(fname, so, optional=False) images = _dedupe_same_offset(images) + images.sort(key=_offset_int) - if not any(re.match(r"^firmware-.+\.bin$", im["file"], re.I) for im in images): + o1 = ota1_offset(parts) + if o1 is not None and not any(_offset_int(im) == o1 for im in images): return None - images.sort(key=_offset_int) - return {"images": images} + return {"images": images, "eraseFlash": True} def pick_mt_json(build_dir: str) -> str | None: @@ -283,9 +349,15 @@ def main() -> int: if os.path.isfile(out_path): with open(out_path, encoding="utf-8") as f: manifest = json.load(f) - if not isinstance(manifest, dict) or not isinstance(manifest.get("images"), list): + if not isinstance(manifest, dict): print(f"Invalid existing {out_path}", file=sys.stderr) return 1 + ok_legacy = isinstance(manifest.get("images"), list) + up = manifest.get("update") + ok_dual = isinstance(up, dict) and isinstance(up.get("images"), list) + if not ok_legacy and not ok_dual: + print(f"Invalid existing {out_path} (need images[] or update.images[])", file=sys.stderr) + return 1 merged = _merge_target_family_meta(manifest, pio_family, pio_platform, pio_board) if merged != manifest: _write_manifest(out_path, merged) @@ -302,14 +374,27 @@ def main() -> int: with open(mt_path, encoding="utf-8") as f: mt = json.load(f) - manifest = emit_from_mt(build_dir, mt) - if not manifest: - print("Could not synthesize flash-manifest.json from mt.json") + names = list_basenames(build_dir) + manifest_update = emit_meshtastic_update(build_dir, mt) + if not manifest_update: + print("Could not synthesize flash-manifest.json (update) from mt.json") return 0 - manifest = _merge_target_family_meta(manifest, pio_family, pio_platform, pio_board) - _write_manifest(out_path, manifest) - print(f"Wrote {out_path}") + doc: dict = {"update": manifest_update} + factory = pick_factory_bin(names) + if factory: + manifest_full = emit_meshtastic_full(build_dir, mt, factory) + if manifest_full: + doc["factory"] = manifest_full + else: + print("Could not build factory section; omitting factory key", file=sys.stderr) + else: + print("No firmware-*.factory.bin in BUILD_DIR; omitting factory section") + + merged_doc = _merge_target_family_meta(doc, pio_family, pio_platform, pio_board) + _write_manifest(out_path, merged_doc) + print(f"Wrote {out_path} (update + {'factory' if 'factory' in merged_doc else 'no factory'})") + return 0 diff --git a/src/components/EspFlasher.tsx b/src/components/EspFlasher.tsx index 71c5556..3dcf20d 100644 --- a/src/components/EspFlasher.tsx +++ b/src/components/EspFlasher.tsx @@ -13,7 +13,7 @@ import { } from "lucide-react" import { useCallback, useEffect, useMemo, useRef, useState } from "react" import { toast } from "sonner" -import { buildFlashParts, manifestFromMap } from "../lib/espFlashLayout" +import { buildFlashParts, manifestFromMap, manifestHasFactorySection } from "../lib/espFlashLayout" import { ensureSerialPortClosed, ESP_FLASH_WEB_BAUD, @@ -85,7 +85,7 @@ export default function EspFlasher({ }: EspFlasherProps) { const [busy, setBusy] = useState(false) const shareDialogRef = useRef(null) - const [resetDeviceStorage, setResetDeviceStorage] = useState(false) + const [eraseFlashForFactory, setEraseFlashForFactory] = useState(false) const [layoutPreview, setLayoutPreview] = useState(null) const [flashProgress, setFlashProgress] = useState(null) const [bundleLoadError, setBundleLoadError] = useState(null) @@ -106,6 +106,7 @@ export default function EspFlasher({ const m = manifestFromMap(files) if (!cancelled) { setLayoutPreview(m) + setEraseFlashForFactory(prev => (manifestHasFactorySection(m) ? prev : false)) setBundleLoadError(null) } } catch (e) { @@ -123,6 +124,8 @@ export default function EspFlasher({ const flashBlockedReason = unsupportedFlashMessage(resolvedFamily) const canEspFlash = flashBlockedReason === null + const hasFactorySection = useMemo(() => manifestHasFactorySection(layoutPreview), [layoutPreview]) + const prepareBundle = useCallback(async () => { const res = await fetch(bundleUrl) if (!res.ok) throw new Error(`Download failed: ${res.status}`) @@ -130,6 +133,7 @@ export default function EspFlasher({ const files = extractTarGz(buf) const m = manifestFromMap(files) setLayoutPreview(m) + setEraseFlashForFactory(prev => (manifestHasFactorySection(m) ? prev : false)) return files }, [bundleUrl]) @@ -153,17 +157,20 @@ export default function EspFlasher({ setFlashProgress({ kind: "indeterminate", label: "Downloading firmware…" }) const files = await prepareBundle() - const parts = buildFlashParts(files, { resetDeviceStorage }) - if (!parts) { + const plan = buildFlashParts(files, { + factoryInstall: eraseFlashForFactory, + resetDeviceStorage: false, + }) + if (!plan) { toast.error("Could not detect flash layout from bundle") return } await runEspFlash({ port, - parts, + parts: plan.parts, baud: ESP_FLASH_WEB_BAUD, - eraseAll: false, + eraseAll: plan.eraseAll, onPhase: phase => { setFlashProgress({ kind: "indeterminate", label: PHASE_LABEL[phase] }) }, @@ -193,7 +200,7 @@ export default function EspFlasher({ setFlashProgress(null) } } - }, [resetDeviceStorage, prepareBundle, canEspFlash, flashBlockedReason]) + }, [eraseFlashForFactory, prepareBundle, canEspFlash, flashBlockedReason]) const shareUrlTrimmed = useMemo(() => sharePageUrl?.trim() ?? "", [sharePageUrl]) const canNativeShare = typeof navigator !== "undefined" && typeof navigator.share === "function" @@ -264,31 +271,39 @@ export default function EspFlasher({ - Reset device storage + Full device reset - {resetDeviceStorage ? ( + {!hasFactorySection ? ( +

+ Full device reset is not available for this bundle—only an update. Typical reasons: an older download, a + build that did not include a factory image, or firmware not produced by this app’s usual ESP32 flow. +

+ ) : null} + + {eraseFlashForFactory ? (

- Warning: all user data on the device will be deleted - when you flash (channels, preferences, and stored files). + You will lose everything on the radio: channels and + settings, private keys, node info, message history, and any stored maps or telemetry. Only use this if you are + deliberately starting over or recovering a bad state.

) : null} diff --git a/src/lib/espFlashLayout.ts b/src/lib/espFlashLayout.ts index 644e74e..e890d1a 100644 --- a/src/lib/espFlashLayout.ts +++ b/src/lib/espFlashLayout.ts @@ -1,16 +1,26 @@ -import { findInTar, parseFlashManifest, type FlashManifest, type FlashManifestImage } from './untarGz' +import { + findInTar, + parseFlashManifest, + type FlashManifest, + type FlashManifestSection, +} from './untarGz' export type FlashPart = { data: Uint8Array; address: number; name: string } -function manifestImageOffset(im: FlashManifestImage): number | null { - const addr = typeof im.offset === 'string' ? parseInt(im.offset, 0) : Number(im.offset) - return Number.isFinite(addr) ? addr : null +export type BuildFlashPlan = { + parts: FlashPart[] + eraseAll: boolean } export type BuildFlashPartsOptions = { /** - * When true, flash optional LittleFS images from the manifest (wipes Meshtastic storage on device). - * Bootloader, partitions, firmware, and other non-LittleFS images are always included when present. + * When true, use manifest `factory` section (chip erase + merged factory + OTA + filesystem). + * When false, use `update` section or legacy flat `images`. + */ + factoryInstall?: boolean + /** + * Legacy: when an image has optional:true (e.g. LittleFS), skip unless true. + * Ignored for Meshtastic dual manifests (update has no optional rows). */ resetDeviceStorage?: boolean } @@ -29,6 +39,20 @@ function isLittlefsManifestFile(file: string): boolean { return base.toLowerCase().startsWith('littlefs-') && base.toLowerCase().endsWith('.bin') } +function activeSection(m: FlashManifest, factoryInstall: boolean): FlashManifestSection | null { + if (factoryInstall) { + const f = m.factory + if (f && Array.isArray(f.images) && f.images.length > 0) return f + return null + } + const u = m.update + if (u && Array.isArray(u.images) && u.images.length > 0) return u + if (Array.isArray(m.images) && m.images.length > 0) { + return { images: m.images, eraseFlash: m.eraseFlash } + } + return null +} + /** * PlatformIO projects (e.g. Meshtastic) often emit versioned names like * firmware-heltec-v3-2.7.20.bin (split app image; merged factory.bin is not bundled for USB flash). @@ -60,20 +84,23 @@ function resolveVersionedFirmwareApp( return undefined } -/** Build ordered flash parts from a flat map (tar paths or bare filenames → bytes). */ +/** Build ordered flash parts + erase policy from a flat map (tar paths or bare filenames → bytes). */ export function buildFlashParts( files: Map, options: BuildFlashPartsOptions = {} -): FlashPart[] | null { +): BuildFlashPlan | null { const resetDeviceStorage = options.resetDeviceStorage ?? false + const factoryInstall = options.factoryInstall ?? false const manifestRaw = findInTar(files, 'flash-manifest.json') if (manifestRaw) { const text = new TextDecoder().decode(manifestRaw) const m = parseFlashManifest(text) if (m) { + const section = activeSection(m, factoryInstall) + if (!section) return null const out: FlashPart[] = [] - for (const img of m.images) { + for (const img of section.images) { if (img.optional === true && isLittlefsManifestFile(img.file) && !resetDeviceStorage) continue const data = findInTar(files, img.file) if (!data) return null @@ -81,7 +108,12 @@ export function buildFlashParts( if (!Number.isFinite(addr)) return null out.push({ data, address: addr, name: img.file }) } - if (out.length) return sortFlashParts(out) + if (out.length) { + return { + parts: sortFlashParts(out), + eraseAll: Boolean(section.eraseFlash), + } + } } } @@ -108,18 +140,26 @@ export function buildFlashParts( { data: app, address: 0x10000, name: appName }, ] if (bootApp0) arr.push({ data: bootApp0, address: 0xe000, name: 'boot_app0.bin' }) - return sortFlashParts(arr) + return { parts: sortFlashParts(arr), eraseAll: false } } if (app && appName && !bootloader && !partitions) { - return [{ data: app, address: 0x0, name: appName }] + return { parts: [{ data: app, address: 0x0, name: appName }], eraseAll: false } } return null } -export function manifestFromMap(files: Map): FlashManifest | null { - const raw = findInTar(files, 'flash-manifest.json') +export function manifestFromMap( + files: Map, + manifestFile = 'flash-manifest.json' +): FlashManifest | null { + const raw = findInTar(files, manifestFile) if (!raw) return null return parseFlashManifest(new TextDecoder().decode(raw)) } + +/** True if manifest includes a factory (erase + merged image) section with images. */ +export function manifestHasFactorySection(m: FlashManifest | null): boolean { + return Boolean(m?.factory?.images && m.factory.images.length > 0) +} diff --git a/src/lib/untarGz.ts b/src/lib/untarGz.ts index cc625e4..0aeafe3 100644 --- a/src/lib/untarGz.ts +++ b/src/lib/untarGz.ts @@ -48,28 +48,43 @@ export function findInTar(files: Map, filename: string): Uin export type FlashManifestImage = { file: string offset: number | string - /** When true on LittleFS rows, Mesh Forge skips unless the user enables Reset device storage. */ + /** When true on LittleFS rows, Mesh Forge skips unless optional handling passes (legacy flat manifests). */ optional?: boolean role?: string } +/** One flash plan (update or factory) inside flash-manifest.json. */ +export type FlashManifestSection = { + images: FlashManifestImage[] + /** When true, flasher performs full chip erase before write. */ + eraseFlash?: boolean +} + /** Coarse MCU family for USB flasher entry + tool selection (from CI / PlatformIO). */ export type FlashTargetFamily = "esp32" | "esp8266" | "nrf52" | "rp2040" | "unknown" +/** + * Root flash-manifest.json: Meshtastic-style `update` + optional `factory`, or legacy flat `images`. + */ export type FlashManifest = { - images: FlashManifestImage[] + update?: FlashManifestSection + factory?: FlashManifestSection + /** Legacy single-layout manifest. */ + images?: FlashManifestImage[] + eraseFlash?: boolean targetFamily?: FlashTargetFamily - /** Raw PlatformIO `platform` (debug / advanced UI). */ platform?: string - /** Raw PlatformIO `board` (debug / advanced UI). */ board?: string } export function parseFlashManifest(json: string): FlashManifest | null { try { const o = JSON.parse(json) as FlashManifest - if (!o || !Array.isArray(o.images)) return null - return o + if (!o || typeof o !== "object") return null + if (o.update && Array.isArray(o.update.images) && o.update.images.length > 0) return o + if (o.factory && Array.isArray(o.factory.images) && o.factory.images.length > 0) return o + if (Array.isArray(o.images) && o.images.length > 0) return o + return null } catch { return null }