Update CI workflows to include actions/checkout@v4 and enhance flash layout functionality. Added optional handling for flash manifest images and improved sorting of flash parts. Updated .gitignore to include __pycache__ and .wrangler.

.github/workflows/custom_build.yml

@@ -36,6 +36,8 @@ jobs:
       REPO_BUILD_ID: ${{ inputs.repo_build_id }}
       CONVEX_BUILD_TOKEN: ${{ secrets.CONVEX_BUILD_TOKEN }}
     steps:
+      - uses: actions/checkout@v4
+
       - name: Notify Convex — running
         shell: bash
         run: |
@@ -85,6 +87,7 @@ jobs:
             echo "Build output directory missing"
             exit 1
           fi
+          python3 "${{ github.workspace }}/scripts/emit-flash-manifest.py" "$ROOT/$BUILD_DIR"
           ARTIFACT_NAME="firmware-${{ inputs.build_key }}-${{ github.run_id }}.tar.gz"
           STAGE=/tmp/fw-bundle
           rm -rf "$STAGE"

.github/workflows/custom_build_test.yml

@@ -36,6 +36,8 @@ jobs:
       REPO_BUILD_ID: ${{ inputs.repo_build_id }}
       CONVEX_BUILD_TOKEN: ${{ secrets.CONVEX_BUILD_TOKEN }}
     steps:
+      - uses: actions/checkout@v4
+
       - name: Notify Convex — running
         shell: bash
         run: |
@@ -85,6 +87,7 @@ jobs:
             echo "Build output directory missing"
             exit 1
           fi
+          python3 "${{ github.workspace }}/scripts/emit-flash-manifest.py" "$ROOT/$BUILD_DIR"
           ARTIFACT_NAME="firmware-${{ inputs.build_key }}-${{ github.run_id }}.tar.gz"
           STAGE=/tmp/fw-bundle
           rm -rf "$STAGE"

.gitignore

@@ -23,4 +23,6 @@ dist-ssr
 *.njsproj
 *.sln
 *.sw?
-.wrangler
+.wrangler
+
+__pycache__/

scripts/emit-flash-manifest.py

@@ -0,0 +1,203 @@
+#!/usr/bin/env python3
+"""
+Emit flash-manifest.json for Mesh Forge USB flasher.
+
+If BUILD_DIR/flash-manifest.json already exists (project-supplied), leave it.
+Otherwise, if Meshtastic-style *.mt.json is present, synthesize a manifest
+from partition table + on-disk artifacts.
+"""
+from __future__ import annotations
+
+import glob
+import json
+import os
+import re
+import sys
+
+
+def parse_offset(v: object) -> int | None:
+    if v is None:
+        return None
+    if isinstance(v, bool):
+        return None
+    if isinstance(v, int):
+        return v
+    s = str(v).strip()
+    if not s:
+        return None
+    if s.lower().startswith("0x"):
+        return int(s, 16)
+    return int(s, 10)
+
+
+def list_basenames(build_dir: str) -> set[str]:
+    out: set[str] = set()
+    for p in glob.glob(os.path.join(build_dir, "*")):
+        if os.path.isfile(p):
+            out.add(os.path.basename(p))
+    return out
+
+
+def part_offset_for_slot(parts: list[dict], part_name: str) -> int | None:
+    for p in parts:
+        if str(p.get("name", "")) == part_name:
+            o = parse_offset(p.get("offset"))
+            if o is not None:
+                return o
+    for p in parts:
+        if str(p.get("subtype", "")) == part_name:
+            o = parse_offset(p.get("offset"))
+            if o is not None:
+                return o
+    if part_name == "app0":
+        for p in parts:
+            if p.get("type") == "app" and p.get("subtype") == "ota_0":
+                o = parse_offset(p.get("offset"))
+                if o is not None:
+                    return o
+    if part_name == "spiffs":
+        for p in parts:
+            if str(p.get("subtype", "")) == "spiffs":
+                o = parse_offset(p.get("offset"))
+                if o is not None:
+                    return o
+    return None
+
+
+def factory_app_offset(parts: list[dict]) -> int:
+    for p in parts:
+        if str(p.get("subtype", "")) == "factory":
+            o = parse_offset(p.get("offset"))
+            if o is not None:
+                return o
+    for p in parts:
+        if p.get("type") == "app" and p.get("subtype") == "ota_0":
+            o = parse_offset(p.get("offset"))
+            if o is not None:
+                return o
+    return 0x10000
+
+
+def ota1_offset(parts: list[dict]) -> int | None:
+    for p in parts:
+        if str(p.get("subtype", "")) == "ota_1":
+            return parse_offset(p.get("offset"))
+    return None
+
+
+def spiffs_offset(parts: list[dict]) -> int | None:
+    return part_offset_for_slot(parts, "spiffs")
+
+
+def emit_from_mt(build_dir: str, mt: dict) -> dict | None:
+    parts: list[dict] = mt.get("part") or []
+    if not parts:
+        return None
+
+    names = list_basenames(build_dir)
+    images: list[dict] = []
+    seen: set[str] = set()
+
+    def add(file: str, offset: int, optional: bool = False) -> None:
+        if file not in names or file in seen:
+            return
+        row: dict = {"file": file, "offset": offset}
+        if optional:
+            row["optional"] = True
+        images.append(row)
+        seen.add(file)
+
+    add("bootloader.bin", 0x1000)
+    add("partitions.bin", 0x8000)
+    add("boot_app0.bin", 0xE000)
+
+    for entry in mt.get("files") or []:
+        fname = entry.get("name")
+        part_name = entry.get("part_name")
+        if not fname or not part_name or fname not in names:
+            continue
+        off = part_offset_for_slot(parts, str(part_name))
+        if off is None:
+            continue
+        opt = bool(
+            fname.startswith("littlefs-")
+            or re.match(r"^mt-.+-ota\.bin$", fname, re.I)
+            or (
+                re.match(r"^firmware-.+\.bin$", fname, re.I)
+                and not re.search(r"\.factory\.bin$", fname, re.I)
+            )
+        )
+        add(fname, off, optional=opt)
+
+    factory_bins = sorted(n for n in names if re.match(r"^firmware-.+\.factory\.bin$", n, re.I))
+    if factory_bins:
+        add(factory_bins[0], factory_app_offset(parts))
+
+    for n in sorted(names):
+        if n.startswith("littlefs-") and n.endswith(".bin"):
+            off = spiffs_offset(parts)
+            if off is not None:
+                add(n, off, optional=True)
+
+    for n in sorted(names):
+        if re.match(r"^mt-.+-ota\.bin$", n, re.I):
+            off = ota1_offset(parts)
+            if off is not None:
+                add(n, off, optional=True)
+
+    if not images:
+        return None
+
+    if not any(re.match(r"^firmware-.+\.bin$", im["file"], re.I) for im in images):
+        return None
+
+    def sort_key(im: dict) -> int:
+        o = im["offset"]
+        return int(o) if isinstance(o, int) else parse_offset(o) or 0
+
+    images.sort(key=sort_key)
+    return {"images": images}
+
+
+def pick_mt_json(build_dir: str) -> str | None:
+    paths = glob.glob(os.path.join(build_dir, "*.mt.json"))
+    if not paths:
+        return None
+    if len(paths) == 1:
+        return paths[0]
+    fw = [p for p in paths if re.search(r"firmware-.+\.mt\.json$", os.path.basename(p), re.I)]
+    return fw[0] if fw else paths[0]
+
+
+def main() -> int:
+    if len(sys.argv) < 2:
+        print("usage: emit-flash-manifest.py BUILD_DIR", file=sys.stderr)
+        return 2
+    build_dir = os.path.abspath(sys.argv[1])
+    out_path = os.path.join(build_dir, "flash-manifest.json")
+
+    if os.path.isfile(out_path):
+        print(f"Keeping existing {out_path}")
+        return 0
+
+    mt_path = pick_mt_json(build_dir)
+    if not mt_path:
+        print("No *.mt.json; not emitting flash-manifest.json")
+        return 0
+
+    with open(mt_path, encoding="utf-8") as f:
+        mt = json.load(f)
+
+    manifest = emit_from_mt(build_dir, mt)
+    if not manifest:
+        print("Could not synthesize flash-manifest.json from mt.json")
+        return 0
+
+    with open(out_path, "w", encoding="utf-8") as f:
+        json.dump(manifest, f, indent=2)
+    print(f"Wrote {out_path}")
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

src/lib/espFlashLayout.ts

@@ -2,6 +2,15 @@ import { findInTar, parseFlashManifest, type FlashManifest } from './untarGz'
 
 export type FlashPart = { data: Uint8Array; address: number; name: string }
 
+export type BuildFlashPartsOptions = {
+  /** When false, manifest rows with optional:true are omitted. */
+  eraseAll?: boolean
+}
+
+function sortFlashParts(parts: FlashPart[]): FlashPart[] {
+  return [...parts].sort((a, b) => a.address - b.address)
+}
+
 function tarBasename(path: string): string {
   const parts = path.replace(/^\.\//, '').split('/')
   return parts[parts.length - 1] ?? path
@@ -42,11 +51,19 @@ function resolveVersionedFirmwareApp(
 }
 
 export function layoutPreviewFromManifest(m: FlashManifest): string[] {
-  return m.images.map(im => `${im.file} @ ${String(im.offset)}`)
+  return m.images.map(im => {
+    const tag = im.optional ? ' — optional unless full chip erase' : ''
+    return `${im.file} @ ${String(im.offset)}${tag}`
+  })
 }
 
 /** Build ordered flash parts from a flat map (tar paths or bare filenames → bytes). */
-export function buildFlashParts(files: Map<string, Uint8Array>): FlashPart[] | null {
+export function buildFlashParts(
+  files: Map<string, Uint8Array>,
+  options: BuildFlashPartsOptions = {}
+): FlashPart[] | null {
+  const eraseAll = options.eraseAll ?? false
+
   const manifestRaw = findInTar(files, 'flash-manifest.json')
   if (manifestRaw) {
     const text = new TextDecoder().decode(manifestRaw)
@@ -54,13 +71,14 @@ export function buildFlashParts(files: Map<string, Uint8Array>): FlashPart[] | n
     if (m) {
       const out: FlashPart[] = []
       for (const img of m.images) {
+        if (!eraseAll && img.optional === true) continue
         const data = findInTar(files, img.file)
         if (!data) return null
         const addr = typeof img.offset === 'string' ? parseInt(img.offset, 0) : Number(img.offset)
         if (!Number.isFinite(addr)) return null
         out.push({ data, address: addr, name: img.file })
       }
-      if (out.length) return out
+      if (out.length) return sortFlashParts(out)
     }
   }
 
@@ -91,7 +109,7 @@ export function buildFlashParts(files: Map<string, Uint8Array>): FlashPart[] | n
       { data: app, address: 0x10000, name: appName },
     ]
     if (bootApp0) arr.push({ data: bootApp0, address: 0xe000, name: 'boot_app0.bin' })
-    return arr
+    return sortFlashParts(arr)
   }
 
   if (app && appName && !bootloader && !partitions) {

src/lib/untarGz.ts

@@ -45,7 +45,13 @@ export function findInTar(files: Map<string, Uint8Array>, filename: string): Uin
   return undefined
 }
 
-export type FlashManifestImage = { file: string; offset: number | string }
+export type FlashManifestImage = {
+  file: string
+  offset: number | string
+  /** When true, Mesh Forge skips this image unless the user enables full chip erase. */
+  optional?: boolean
+  role?: string
+}
 
 export type FlashManifest = { images: FlashManifestImage[] }