consolidate

This commit is contained in:
Ben Allfree
2026-04-11 05:09:59 -07:00
parent c28955c920
commit 1ec72c81b4
9 changed files with 212 additions and 860 deletions
+30 -4
View File
@@ -165,6 +165,21 @@ jobs:
exit 1
fi
- name: Generate merged factory binary for ESP32 targets (if applicable)
shell: bash
run: |
ROOT=$(cat /tmp/fw-src-root.txt)
BUILD_DIR="$ROOT/.pio/build/${{ inputs.target_env }}"
if [ -f "$BUILD_DIR/bootloader.bin" ]; then
echo "ESP32 target detected; running mergebin → firmware-merged.factory.bin"
cd "$ROOT"
export MERGED_BIN_PATH="$BUILD_DIR/firmware-merged.factory.bin"
pio run -t mergebin -e "${{ inputs.target_env }}" 2>&1 || \
echo "WARNING: mergebin target failed; factory binary will not be available"
else
echo "No bootloader.bin found; skipping mergebin (non-ESP32 target)"
fi
- name: Download Meshtastic ESP OTA companion (if applicable)
shell: bash
run: |
@@ -174,6 +189,21 @@ jobs:
BUILD_DIR=".pio/build/${{ inputs.target_env }}"
bash "${{ github.workspace }}/scripts/download-meshtastic-ota.sh" "$ROOT/$BUILD_DIR"
- name: Extend merged binary with Meshtastic OTA companion (if applicable)
shell: bash
run: |
ROOT=$(cat /tmp/fw-src-root.txt)
BUILD_DIR="$ROOT/.pio/build/${{ inputs.target_env }}"
MERGED="$BUILD_DIR/firmware-merged.factory.bin"
PARTS="$BUILD_DIR/partitions.bin"
OTA_BIN=$(ls "$BUILD_DIR"/mt-*.ota.bin "$BUILD_DIR"/bleota-c3.bin 2>/dev/null | head -1)
if [ -f "$MERGED" ] && [ -f "$PARTS" ] && [ -n "$OTA_BIN" ]; then
python3 "${{ github.workspace }}/scripts/extend-merged-with-ota.py" \
"$MERGED" "$PARTS" "$OTA_BIN"
else
echo "Skipping OTA extension (merged=$( [ -f "$MERGED" ] && echo yes || echo no), ota=$( [ -n "$OTA_BIN" ] && echo yes || echo no))"
fi
- name: Stage or generate nRF52 DFU files (if applicable)
shell: bash
run: |
@@ -227,7 +257,6 @@ jobs:
ROOT=$(cat /tmp/fw-src-root.txt)
cd "$ROOT"
BUILD_DIR=".pio/build/${{ inputs.target_env }}"
python3 "${{ github.workspace }}/scripts/emit-flash-manifest.py" "$ROOT/$BUILD_DIR" "$ROOT" "${{ inputs.target_env }}"
ARTIFACT_NAME="firmware-${{ inputs.build_key }}-${{ github.run_id }}.tar.gz"
STAGE=/tmp/fw-bundle
rm -rf "$STAGE"
@@ -237,9 +266,6 @@ jobs:
cp -a "$f" "$STAGE/"
done
shopt -u nullglob
if [ -f "$BUILD_DIR/flash-manifest.json" ]; then
cp -a "$BUILD_DIR/flash-manifest.json" "$STAGE/"
fi
# Nordic DFU manifest (MeshCore and other nRF52 builds)
if [ -f "$BUILD_DIR/manifest.json" ]; then
cp -a "$BUILD_DIR/manifest.json" "$STAGE/"
+30 -4
View File
@@ -165,6 +165,21 @@ jobs:
exit 1
fi
- name: Generate merged factory binary for ESP32 targets (if applicable)
shell: bash
run: |
ROOT=$(cat /tmp/fw-src-root.txt)
BUILD_DIR="$ROOT/.pio/build/${{ inputs.target_env }}"
if [ -f "$BUILD_DIR/bootloader.bin" ]; then
echo "ESP32 target detected; running mergebin → firmware-merged.factory.bin"
cd "$ROOT"
export MERGED_BIN_PATH="$BUILD_DIR/firmware-merged.factory.bin"
pio run -t mergebin -e "${{ inputs.target_env }}" 2>&1 || \
echo "WARNING: mergebin target failed; factory binary will not be available"
else
echo "No bootloader.bin found; skipping mergebin (non-ESP32 target)"
fi
- name: Download Meshtastic ESP OTA companion (if applicable)
shell: bash
run: |
@@ -174,6 +189,21 @@ jobs:
BUILD_DIR=".pio/build/${{ inputs.target_env }}"
bash "${{ github.workspace }}/scripts/download-meshtastic-ota.sh" "$ROOT/$BUILD_DIR"
- name: Extend merged binary with Meshtastic OTA companion (if applicable)
shell: bash
run: |
ROOT=$(cat /tmp/fw-src-root.txt)
BUILD_DIR="$ROOT/.pio/build/${{ inputs.target_env }}"
MERGED="$BUILD_DIR/firmware-merged.factory.bin"
PARTS="$BUILD_DIR/partitions.bin"
OTA_BIN=$(ls "$BUILD_DIR"/mt-*.ota.bin "$BUILD_DIR"/bleota-c3.bin 2>/dev/null | head -1)
if [ -f "$MERGED" ] && [ -f "$PARTS" ] && [ -n "$OTA_BIN" ]; then
python3 "${{ github.workspace }}/scripts/extend-merged-with-ota.py" \
"$MERGED" "$PARTS" "$OTA_BIN"
else
echo "Skipping OTA extension (merged=$( [ -f "$MERGED" ] && echo yes || echo no), ota=$( [ -n "$OTA_BIN" ] && echo yes || echo no))"
fi
- name: Stage or generate nRF52 DFU files (if applicable)
shell: bash
run: |
@@ -227,7 +257,6 @@ jobs:
ROOT=$(cat /tmp/fw-src-root.txt)
cd "$ROOT"
BUILD_DIR=".pio/build/${{ inputs.target_env }}"
python3 "${{ github.workspace }}/scripts/emit-flash-manifest.py" "$ROOT/$BUILD_DIR" "$ROOT" "${{ inputs.target_env }}"
ARTIFACT_NAME="firmware-${{ inputs.build_key }}-${{ github.run_id }}.tar.gz"
STAGE=/tmp/fw-bundle
rm -rf "$STAGE"
@@ -237,9 +266,6 @@ jobs:
cp -a "$f" "$STAGE/"
done
shopt -u nullglob
if [ -f "$BUILD_DIR/flash-manifest.json" ]; then
cp -a "$BUILD_DIR/flash-manifest.json" "$STAGE/"
fi
# Nordic DFU manifest (MeshCore and other nRF52 builds)
if [ -f "$BUILD_DIR/manifest.json" ]; then
cp -a "$BUILD_DIR/manifest.json" "$STAGE/"
-577
View File
@@ -1,577 +0,0 @@
#!/usr/bin/env python3
"""
Emit flash-manifest.json for Mesh Forge USB flasher.
If BUILD_DIR/flash-manifest.json already exists (project-supplied), merge in
targetFamily from PlatformIO when missing.
Otherwise, if Meshtastic-style *.mt.json is present, synthesize one JSON with:
- "update": app @ ota_0 + BLE OTA @ ota_1, eraseFlash false (Meshtastic “Update”).
- "factory": merged *.factory.bin @ 0 + OTA + LittleFS, eraseFlash true (Meshtastic “Erase device”),
omitted if no factory.bin in BUILD_DIR.
OTA: mt-*-ota.bin (ESP32/S3) or bleota-c3.bin (C3/C6).
Optional args: PROJECT_ROOT TARGET_ENV — merged PIO config for that env fills
targetFamily (and platform/board) for the USB flasher UI.
"""
from __future__ import annotations
import glob
import json
import os
import re
import sys
def _normalize_platform(platform: str | None) -> str:
if not platform:
return ""
p = platform.strip().lower()
if "#" in p:
p = p.split("#", 1)[0].strip()
if "@" in p:
p = p.split("@", 1)[0].strip()
if "/" in p:
p = p.rsplit("/", 1)[-1].strip()
return p
def _platform_to_target_family(platform: str | None, board: str | None) -> str:
pl = _normalize_platform(platform)
b = (board or "").lower()
if "8266" in pl or "esp8266" in b:
return "esp8266"
if "nrf52" in pl or "nrf52840" in b or "nrf52833" in b or "nrf52832" in b:
return "nrf52"
if "rp2040" in pl or "rp2040" in b or "raspberrypi" in pl:
return "rp2040"
if "espressif32" in pl or "esp32" in pl or "esp32" in b or "esp32c3" in b or "esp32s3" in b:
return "esp32"
return "unknown"
def _resolve_pio_target_family(project_root: str, target_env: str) -> tuple[str, str | None, str | None]:
try:
from platformio.project.config import ProjectConfig
except ImportError:
print("platformio not installed; targetFamily will be unknown", file=sys.stderr)
return "unknown", None, None
ini = os.path.join(project_root, "platformio.ini")
if not os.path.isfile(ini):
return "unknown", None, None
old = os.getcwd()
try:
os.chdir(project_root)
config = ProjectConfig("platformio.ini")
section = f"env:{target_env}"
if section not in config.sections():
print(f"PIO env not found: {target_env!r}", file=sys.stderr)
return "unknown", None, None
platform = config.get(section, "platform")
board = config.get(section, "board")
fam = _platform_to_target_family(platform, board)
return fam, platform, board
except Exception as e:
print(f"PIO targetFamily resolution failed: {e}", file=sys.stderr)
return "unknown", None, None
finally:
os.chdir(old)
def _merge_target_family_meta(
manifest: dict,
target_family: str,
platform: str | None,
board: str | None,
) -> dict:
"""Add targetFamily / platform / board only when absent."""
out = dict(manifest)
if "targetFamily" not in out:
out["targetFamily"] = target_family
if platform and "platform" not in out:
out["platform"] = platform.strip() if isinstance(platform, str) else platform
if board and "board" not in out:
out["board"] = board.strip() if isinstance(board, str) else board
return out
def _write_manifest(path: str, data: dict) -> None:
with open(path, "w", encoding="utf-8") as f:
json.dump(data, f, indent=2)
def parse_offset(v: object) -> int | None:
if v is None:
return None
if isinstance(v, bool):
return None
if isinstance(v, int):
return v
s = str(v).strip()
if not s:
return None
if s.lower().startswith("0x"):
return int(s, 16)
return int(s, 10)
def list_basenames(build_dir: str) -> set[str]:
out: set[str] = set()
for p in glob.glob(os.path.join(build_dir, "*")):
if os.path.isfile(p):
out.add(os.path.basename(p))
return out
def part_offset_for_slot(parts: list[dict], part_name: str) -> int | None:
for p in parts:
if str(p.get("name", "")) == part_name:
o = parse_offset(p.get("offset"))
if o is not None:
return o
for p in parts:
if str(p.get("subtype", "")) == part_name:
o = parse_offset(p.get("offset"))
if o is not None:
return o
if part_name == "app0":
for p in parts:
if p.get("type") == "app" and p.get("subtype") == "ota_0":
o = parse_offset(p.get("offset"))
if o is not None:
return o
if part_name == "spiffs":
for p in parts:
if str(p.get("subtype", "")) == "spiffs":
o = parse_offset(p.get("offset"))
if o is not None:
return o
return None
def ota1_offset(parts: list[dict]) -> int | None:
for p in parts:
if str(p.get("subtype", "")) == "ota_1":
return parse_offset(p.get("offset"))
return None
def spiffs_offset(parts: list[dict]) -> int | None:
return part_offset_for_slot(parts, "spiffs")
def _offset_int(im: dict) -> int:
o = im["offset"]
return int(o) if isinstance(o, int) else parse_offset(o) or 0
def _dedupe_same_offset(images: list[dict]) -> list[dict]:
"""One esptool image per physical offset (prefer non-optional, then lexicographic file)."""
buckets: dict[int, list[dict]] = {}
for im in images:
buckets.setdefault(_offset_int(im), []).append(im)
out: list[dict] = []
for off in sorted(buckets):
group = buckets[off]
out.append(
group[0]
if len(group) == 1
else min(group, key=lambda im: (im.get("optional") is True, str(im.get("file", ""))))
)
return out
def _is_non_factory_firmware(name: str) -> bool:
return bool(re.match(r"^firmware-.+\.bin$", name, re.I) and not re.search(r"\.factory\.bin$", name, re.I))
def pick_factory_bin(names: set[str]) -> str | None:
cands = sorted(n for n in names if re.match(r"^firmware-.+\.factory\.bin$", n, re.I))
return cands[0] if cands else None
def pick_non_factory_firmware(names: set[str], mt: dict) -> str | None:
for entry in mt.get("files") or []:
fname = entry.get("name")
if not fname or fname not in names or entry.get("part_name") != "app0":
continue
if _is_non_factory_firmware(fname):
return fname
for n in sorted(names):
if _is_non_factory_firmware(n):
return n
return None
def emit_meshtastic_update(build_dir: str, mt: dict) -> dict | None:
"""Official-style update: app @ ota_0 + OTA @ ota_1 only (no erase, no factory, no FS)."""
parts: list[dict] = mt.get("part") or []
if not parts:
return None
names = list_basenames(build_dir)
images: list[dict] = []
seen: set[str] = set()
def add(file: str, offset: int, optional: bool = False) -> None:
if file not in names or file in seen:
return
row: dict = {"file": file, "offset": offset}
if optional:
row["optional"] = True
images.append(row)
seen.add(file)
app_bin = pick_non_factory_firmware(names, mt)
if not app_bin:
return None
off0 = part_offset_for_slot(parts, "app0")
if off0 is None:
return None
add(app_bin, off0, optional=False)
ota_added = False
for n in sorted(names):
if re.match(r"^mt-.+-ota\.bin$", n, re.I):
o = ota1_offset(parts)
if o is not None:
add(n, o, optional=False)
ota_added = True
break
if not ota_added and "bleota-c3.bin" in names:
o = ota1_offset(parts)
if o is not None:
add("bleota-c3.bin", o, optional=False)
images = _dedupe_same_offset(images)
images.sort(key=_offset_int)
return {"images": images, "eraseFlash": False}
def emit_meshtastic_full(build_dir: str, mt: dict, factory_bin: str) -> dict | None:
"""Official-style erase + factory: merged factory @ 0 + OTA + LittleFS."""
parts: list[dict] = mt.get("part") or []
if not parts:
return None
names = list_basenames(build_dir)
if factory_bin not in names:
return None
images: list[dict] = []
seen: set[str] = set()
def add(file: str, offset: int, optional: bool = False) -> None:
if file not in names or file in seen:
return
row: dict = {"file": file, "offset": offset}
if optional:
row["optional"] = True
images.append(row)
seen.add(file)
add(factory_bin, 0, optional=False)
ota_added = False
for n in sorted(names):
if re.match(r"^mt-.+-ota\.bin$", n, re.I):
o = ota1_offset(parts)
if o is not None:
add(n, o, optional=False)
ota_added = True
break
if not ota_added and "bleota-c3.bin" in names:
o = ota1_offset(parts)
if o is not None:
add("bleota-c3.bin", o, optional=False)
for n in sorted(names):
if n.startswith("littlefs-") and n.endswith(".bin"):
so = spiffs_offset(parts)
if so is not None:
add(n, so, optional=False)
for entry in mt.get("files") or []:
fname = entry.get("name")
part_name = entry.get("part_name")
if not fname or part_name != "spiffs" or fname not in names:
continue
if not str(fname).lower().startswith("littlefs-"):
continue
so = spiffs_offset(parts)
if so is not None:
add(fname, so, optional=False)
images = _dedupe_same_offset(images)
images.sort(key=_offset_int)
o1 = ota1_offset(parts)
if o1 is not None and not any(_offset_int(im) == o1 for im in images):
return None
return {"images": images, "eraseFlash": True}
def pick_mt_json(build_dir: str) -> str | None:
paths = glob.glob(os.path.join(build_dir, "*.mt.json"))
if not paths:
return None
if len(paths) == 1:
return paths[0]
fw = [p for p in paths if re.search(r"firmware-.+\.mt\.json$", os.path.basename(p), re.I)]
return fw[0] if fw else paths[0]
# ---------------------------------------------------------------------------
# Path 3 — generic nRF52 (UF2 bootloader, no *.mt.json)
# ---------------------------------------------------------------------------
def _pick_firmware_uf2(names: set[str]) -> str | None:
"""Return the best candidate firmware *.uf2 (exclude nuke.uf2)."""
cands = sorted(
n for n in names
if n.lower().endswith(".uf2") and n.lower() != "nuke.uf2"
)
# Prefer names starting with "firmware"
fw = [n for n in cands if n.lower().startswith("firmware")]
return fw[0] if fw else (cands[0] if cands else None)
def emit_generic_nrf52(names: set[str]) -> dict | None:
"""
Emit a UF2-based manifest for nRF52 builds.
update: firmware *.uf2 with role "uf2"
factory (optional): nuke.uf2 (role "nuke") + firmware *.uf2, eraseFlash true
"""
fw_uf2 = _pick_firmware_uf2(names)
if not fw_uf2:
return None
doc: dict = {
"update": {
"images": [{"file": fw_uf2, "offset": 0, "role": "uf2"}],
"eraseFlash": False,
}
}
if "nuke.uf2" in names:
doc["factory"] = {
"images": [
{"file": "nuke.uf2", "offset": 0, "role": "nuke"},
{"file": fw_uf2, "offset": 0, "role": "uf2"},
],
"eraseFlash": True,
}
return doc
# ---------------------------------------------------------------------------
# Path 4 — generic ESP32 (standard PlatformIO bins, no *.mt.json)
# ---------------------------------------------------------------------------
# Chips where the secondary bootloader lives at 0x0 (not 0x1000).
# ESP32 and ESP32-S2 use 0x1000; S3, C3, C6, H2 use 0x0.
_ESP32_BOOTLOADER_OFFSET_ZERO_BOARDS = re.compile(
r"esp32[-_]?(s3|c3|c6|h2|c2|p4)",
re.I,
)
_ESP32_OFFSETS_1000: dict[str, int] = {
"bootloader.bin": 0x1000,
"partitions.bin": 0x8000,
"boot_app0.bin": 0xE000,
}
_ESP32_OFFSETS_0000: dict[str, int] = {
"bootloader.bin": 0x0,
"partitions.bin": 0x8000,
"boot_app0.bin": 0xE000,
}
_ESP32_APP_OFFSET = 0x10000
def _esp32_bootloader_at_zero(board: str | None) -> bool:
"""Return True when the chip variant places its bootloader at 0x0 (S3, C3, C6, H2…)."""
return bool(_ESP32_BOOTLOADER_OFFSET_ZERO_BOARDS.search(board or ""))
def _pick_esp32_app_bin(names: set[str]) -> str | None:
"""Return firmware app bin (versioned or exact), excluding factory images."""
if "firmware.bin" in names:
return "firmware.bin"
cands = sorted(
n for n in names
if re.match(r"^firmware-.+\.bin$", n, re.I) and not re.search(r"\.factory\.bin$", n, re.I)
)
return cands[0] if cands else None
def emit_generic_esp32(names: set[str], board: str | None = None) -> dict | None:
"""
Emit a standard ESP32 manifest from PlatformIO bin output.
update: bootloader + partitions + app (+ boot_app0 if present)
factory (optional): firmware-*.factory.bin @ 0x0, eraseFlash true
The bootloader offset varies by chip:
ESP32 / ESP32-S2 → 0x1000
ESP32-S3 / C3 / C6 / H2 → 0x0
"""
bootloader = "bootloader.bin" if "bootloader.bin" in names else None
partitions = "partitions.bin" if "partitions.bin" in names else None
app_bin = _pick_esp32_app_bin(names)
if not (bootloader and partitions and app_bin):
# Not enough for a canonical ESP32 layout
return None
offsets = _ESP32_OFFSETS_0000 if _esp32_bootloader_at_zero(board) else _ESP32_OFFSETS_1000
update_images: list[dict] = [
{"file": "bootloader.bin", "offset": offsets["bootloader.bin"]},
{"file": "partitions.bin", "offset": offsets["partitions.bin"]},
]
if "boot_app0.bin" in names:
update_images.append({"file": "boot_app0.bin", "offset": offsets["boot_app0.bin"]})
update_images.append({"file": app_bin, "offset": _ESP32_APP_OFFSET})
update_images.sort(key=lambda im: im["offset"])
doc: dict = {
"update": {"images": update_images, "eraseFlash": False}
}
factory_bin = pick_factory_bin(names)
if factory_bin:
doc["factory"] = {
"images": [{"file": factory_bin, "offset": 0}],
"eraseFlash": True,
}
return doc
# ---------------------------------------------------------------------------
# Entry point
# ---------------------------------------------------------------------------
def main() -> int:
if len(sys.argv) < 2:
print(
"usage: emit-flash-manifest.py BUILD_DIR [PROJECT_ROOT TARGET_ENV]",
file=sys.stderr,
)
return 2
build_dir = os.path.abspath(sys.argv[1])
project_root: str | None = None
target_env: str | None = None
if len(sys.argv) >= 4:
project_root = os.path.abspath(sys.argv[2])
target_env = sys.argv[3]
pio_family, pio_platform, pio_board = (
_resolve_pio_target_family(project_root, target_env)
if project_root and target_env
else ("unknown", None, None)
)
out_path = os.path.join(build_dir, "flash-manifest.json")
# Path 1: existing manifest — merge targetFamily when absent
if os.path.isfile(out_path):
with open(out_path, encoding="utf-8") as f:
manifest = json.load(f)
if not isinstance(manifest, dict):
print(f"Invalid existing {out_path}", file=sys.stderr)
return 1
ok_legacy = isinstance(manifest.get("images"), list)
up = manifest.get("update")
ok_dual = isinstance(up, dict) and isinstance(up.get("images"), list)
if not ok_legacy and not ok_dual:
print(f"Invalid existing {out_path} (need images[] or update.images[])", file=sys.stderr)
return 1
merged = _merge_target_family_meta(manifest, pio_family, pio_platform, pio_board)
if merged != manifest:
_write_manifest(out_path, merged)
print(f"Merged targetFamily into {out_path}")
else:
print(f"Keeping existing {out_path} (targetFamily already set)")
return 0
names = list_basenames(build_dir)
# Path 2: Meshtastic *.mt.json
mt_path = pick_mt_json(build_dir)
if mt_path:
with open(mt_path, encoding="utf-8") as f:
mt = json.load(f)
manifest_update = emit_meshtastic_update(build_dir, mt)
if not manifest_update:
print("Could not synthesize flash-manifest.json (update) from mt.json")
return 0
doc: dict = {"update": manifest_update}
factory = pick_factory_bin(names)
if factory:
manifest_full = emit_meshtastic_full(build_dir, mt, factory)
if manifest_full:
doc["factory"] = manifest_full
else:
print("Could not build factory section; omitting factory key", file=sys.stderr)
else:
print("No firmware-*.factory.bin in BUILD_DIR; omitting factory section")
merged_doc = _merge_target_family_meta(doc, pio_family, pio_platform, pio_board)
_write_manifest(out_path, merged_doc)
print(f"Wrote {out_path} (update + {'factory' if 'factory' in merged_doc else 'no factory'})")
return 0
# Path 3: generic nRF52 — UF2 bootloader builds or Nordic DFU (bin + dat) builds
if pio_family == "nrf52":
# 3a: UF2 output (Meshtastic and other UF2-bootloader boards)
doc = emit_generic_nrf52(names)
if doc:
merged_doc = _merge_target_family_meta(doc, pio_family, pio_platform, pio_board)
_write_manifest(out_path, merged_doc)
print(f"Wrote {out_path} (nRF52 UF2, update + {'factory' if 'factory' in merged_doc else 'no factory'})")
return 0
# 3b: Nordic DFU bin + dat output (MeshCore and similar PlatformIO nRF52 builds).
# The browser uses manifest.json (Nordic format) directly for the DFU protocol;
# flash-manifest.json here just records targetFamily for the UI.
if "firmware.bin" in names and "firmware.dat" in names:
doc = {
"update": {
"images": [
{"file": "firmware.bin", "offset": 0, "role": "dfu-bin"},
{"file": "firmware.dat", "offset": 0, "role": "dfu-dat"},
],
"eraseFlash": False,
}
}
merged_doc = _merge_target_family_meta(doc, pio_family, pio_platform, pio_board)
_write_manifest(out_path, merged_doc)
print(f"Wrote {out_path} (nRF52 Nordic DFU)")
return 0
print("nRF52 target but no flashable files found in BUILD_DIR; skipping manifest", file=sys.stderr)
return 0
# Path 4: generic ESP32 (standard PlatformIO bin output)
if pio_family == "esp32":
doc = emit_generic_esp32(names, board=pio_board)
if doc:
merged_doc = _merge_target_family_meta(doc, pio_family, pio_platform, pio_board)
_write_manifest(out_path, merged_doc)
print(f"Wrote {out_path} (ESP32 bins, update + {'factory' if 'factory' in merged_doc else 'no factory'})")
return 0
print("ESP32 target but could not detect bootloader+partitions+firmware in BUILD_DIR; skipping manifest", file=sys.stderr)
return 0
print(f"No manifest source found for family={pio_family!r}; skipping flash-manifest.json")
return 0
if __name__ == "__main__":
raise SystemExit(main())
+73
View File
@@ -0,0 +1,73 @@
#!/usr/bin/env python3
"""
Extend a PlatformIO-merged ESP32 factory binary with the Meshtastic OTA
companion binary at the ota_1 partition offset.
Reads the ota_1 offset from the partition table binary, pads the merged
binary to that offset with 0xFF, and writes the OTA binary in-place.
Usage: extend-merged-with-ota.py MERGED_BIN PARTITIONS_BIN OTA_BIN
"""
from __future__ import annotations
import struct
import sys
def find_ota1_offset(parts_data: bytes) -> int | None:
"""Parse ESP partition table binary to find the ota_1 partition offset."""
MAGIC = 0xAA50
APP_TYPE = 0x00
OTA1_SUBTYPE = 0x11
for i in range(0, len(parts_data) - 31, 32):
magic, ptype, subtype = struct.unpack_from('<HBB', parts_data, i)
if magic == MAGIC and ptype == APP_TYPE and subtype == OTA1_SUBTYPE:
return struct.unpack_from('<I', parts_data, i + 4)[0]
return None
def main() -> int:
if len(sys.argv) != 4:
print(
'usage: extend-merged-with-ota.py MERGED_BIN PARTITIONS_BIN OTA_BIN',
file=sys.stderr,
)
return 2
merged_path, parts_path, ota_path = sys.argv[1], sys.argv[2], sys.argv[3]
with open(parts_path, 'rb') as f:
parts_data = f.read()
ota1_offset = find_ota1_offset(parts_data)
if ota1_offset is None:
print('No ota_1 partition found in partition table; skipping OTA extension')
return 0
with open(merged_path, 'rb') as f:
merged = bytearray(f.read())
with open(ota_path, 'rb') as f:
ota = f.read()
if len(merged) > ota1_offset:
print(
f'WARNING: merged binary ({len(merged)} bytes) already extends past '
f'ota_1 offset (0x{ota1_offset:x}); overwriting',
file=sys.stderr,
)
elif len(merged) < ota1_offset:
merged += b'\xff' * (ota1_offset - len(merged))
merged[ota1_offset:ota1_offset + len(ota)] = ota
with open(merged_path, 'wb') as f:
f.write(merged)
print(
f'Extended {merged_path}: OTA companion ({len(ota)} bytes) at 0x{ota1_offset:x}'
)
return 0
if __name__ == '__main__':
raise SystemExit(main())
+17 -24
View File
@@ -13,7 +13,7 @@ import {
} from "lucide-react"
import { useCallback, useEffect, useMemo, useRef, useState } from "react"
import { toast } from "sonner"
import { buildFlashParts, manifestFromMap, manifestHasFactorySection } from "../lib/espFlashLayout"
import { buildFlashParts } from "../lib/espFlashLayout"
import {
ensureSerialPortClosed,
ESP_FLASH_WEB_BAUD,
@@ -23,8 +23,8 @@ import {
type FlashPhase,
} from "../lib/espFlashRun"
import { runNrfFlash } from "../lib/nrfFlashRun"
import { resolveFlashTargetFamily } from "../lib/flashTargetFamily"
import type { FlashManifest, FlashTargetFamily } from "../lib/untarGz"
import { inferTargetFamilyFromBundle, inferTargetFamilyFromEnv } from "../lib/flashTargetFamily"
import type { FlashTargetFamily } from "../lib/untarGz"
import { extractTarGz } from "../lib/untarGz"
type FlashProgress =
@@ -84,12 +84,14 @@ export default function DeviceFlasher({
const [busy, setBusy] = useState(false)
const shareDialogRef = useRef<HTMLDialogElement>(null)
const [eraseFlashForFactory, setEraseFlashForFactory] = useState(false)
const [layoutPreview, setLayoutPreview] = useState<FlashManifest | null>(null)
const [bundleFamily, setBundleFamily] = useState<FlashTargetFamily>(
inferTargetFamilyFromEnv(targetEnv) ?? "esp32"
)
const [bundleCanErase, setBundleCanErase] = useState(false)
const [flashProgress, setFlashProgress] = useState<FlashProgress | null>(null)
const [bundleLoadError, setBundleLoadError] = useState<string | null>(null)
useEffect(() => {
setLayoutPreview(null)
setBundleLoadError(null)
let cancelled = false
void (async () => {
@@ -101,10 +103,11 @@ export default function DeviceFlasher({
}
const buf = new Uint8Array(await res.arrayBuffer())
const files = extractTarGz(buf)
const m = manifestFromMap(files)
const { family, canErase } = inferTargetFamilyFromBundle(files, targetEnv)
if (!cancelled) {
setLayoutPreview(m)
setEraseFlashForFactory(prev => (manifestHasFactorySection(m) ? prev : false))
setBundleFamily(family)
setBundleCanErase(canErase)
setEraseFlashForFactory(prev => (canErase ? prev : false))
setBundleLoadError(null)
}
} catch (e) {
@@ -116,25 +119,20 @@ export default function DeviceFlasher({
return () => {
cancelled = true
}
}, [bundleUrl])
}, [bundleUrl, targetEnv])
const resolvedFamily = resolveFlashTargetFamily(layoutPreview, targetEnv)
const resolvedFamily = bundleFamily
const flashBlockedReason = unsupportedFlashMessage(resolvedFamily)
const canEspFlash = flashBlockedReason === null
const hasFactorySection = useMemo(() => manifestHasFactorySection(layoutPreview), [layoutPreview])
// ESP32 can always chip-erase regardless of whether the bundle has a factory section.
const canFullReset = resolvedFamily === "esp32" || hasFactorySection
// ESP32 can always chip-erase (merged binary supports it); nRF52 only when canErase is set.
const canFullReset = resolvedFamily === "esp32" || bundleCanErase
const prepareBundle = useCallback(async () => {
const res = await fetch(bundleUrl)
if (!res.ok) throw new Error(`Download failed: ${res.status}`)
const buf = new Uint8Array(await res.arrayBuffer())
const files = extractTarGz(buf)
const m = manifestFromMap(files)
setLayoutPreview(m)
setEraseFlashForFactory(prev => (manifestHasFactorySection(m) ? prev : false))
return files
return extractTarGz(buf)
}, [bundleUrl])
const flash = useCallback(async () => {
@@ -159,11 +157,9 @@ export default function DeviceFlasher({
const files = await prepareBundle()
if (resolvedFamily === "nrf52") {
const manifest = manifestFromMap(files)
await runNrfFlash({
port: port!,
files,
manifest,
factoryInstall: eraseFlashForFactory,
onPhase: label => {
setFlashProgress({ kind: "indeterminate", label })
@@ -173,10 +169,7 @@ export default function DeviceFlasher({
},
})
} else {
const plan = buildFlashParts(files, {
factoryInstall: eraseFlashForFactory && hasFactorySection,
resetDeviceStorage: false,
})
const plan = buildFlashParts(files)
if (!plan) {
toast.error("Could not detect flash layout from bundle")
return
+13 -142
View File
@@ -1,10 +1,3 @@
import {
findInTar,
parseFlashManifest,
type FlashManifest,
type FlashManifestSection,
} from './untarGz'
export type FlashPart = { data: Uint8Array; address: number; name: string }
export type BuildFlashPlan = {
@@ -12,154 +5,32 @@ export type BuildFlashPlan = {
eraseAll: boolean
}
export type BuildFlashPartsOptions = {
/**
* When true, use manifest `factory` section (chip erase + merged factory + OTA + filesystem).
* When false, use `update` section or legacy flat `images`.
*/
factoryInstall?: boolean
/**
* Legacy: when an image has optional:true (e.g. LittleFS), skip unless true.
* Ignored for Meshtastic dual manifests (update has no optional rows).
*/
resetDeviceStorage?: boolean
}
function sortFlashParts(parts: FlashPart[]): FlashPart[] {
return [...parts].sort((a, b) => a.address - b.address)
}
function tarBasename(path: string): string {
const parts = path.replace(/^\.\//, '').split('/')
return parts[parts.length - 1] ?? path
}
function isLittlefsManifestFile(file: string): boolean {
const base = tarBasename(file)
return base.toLowerCase().startsWith('littlefs-') && base.toLowerCase().endsWith('.bin')
}
function activeSection(m: FlashManifest, factoryInstall: boolean): FlashManifestSection | null {
if (factoryInstall) {
const f = m.factory
if (f && Array.isArray(f.images) && f.images.length > 0) return f
return null
}
const u = m.update
if (u && Array.isArray(u.images) && u.images.length > 0) return u
if (Array.isArray(m.images) && m.images.length > 0) {
return { images: m.images, eraseFlash: m.eraseFlash }
}
return null
}
/**
* PlatformIO projects (e.g. Meshtastic) often emit versioned names like
* firmware-heltec-v3-2.7.20.bin (split app image; merged factory.bin is not bundled for USB flash).
* Build the flash plan from a firmware bundle (tar paths or bare filenames → bytes).
*
* For ESP32: expects a single merged binary (firmware-*.factory.bin) at address 0x0.
* PlatformIO's mergebin target handles all chip-specific offsets during the build.
*
* For custom firmware drag-and-drop (no manifest): finds any .bin that is not a
* sub-component and flashes it at 0x0.
*/
function resolveVersionedFirmwareApp(
files: Map<string, Uint8Array>
): { data: Uint8Array; name: string } | undefined {
type Entry = { base: string; data: Uint8Array }
const list: Entry[] = []
export function buildFlashParts(files: Map<string, Uint8Array>): BuildFlashPlan | null {
for (const [path, data] of files) {
const base = tarBasename(path)
const lower = base.toLowerCase()
if (lower.startsWith('littlefs-')) continue
if (
lower === 'bootloader.bin' ||
lower === 'partitions.bin' ||
lower === 'boot_app0.bin'
lower.endsWith('.bin') &&
lower !== 'bootloader.bin' &&
lower !== 'partitions.bin' &&
lower !== 'boot_app0.bin'
) {
continue
}
list.push({ base, data })
}
const app = list.find(
e => /^firmware-.+\.bin$/i.test(e.base) && !/\.factory\.bin$/i.test(e.base)
)
if (app) return { data: app.data, name: app.base }
return undefined
}
/** Build ordered flash parts + erase policy from a flat map (tar paths or bare filenames → bytes). */
export function buildFlashParts(
files: Map<string, Uint8Array>,
options: BuildFlashPartsOptions = {}
): BuildFlashPlan | null {
const resetDeviceStorage = options.resetDeviceStorage ?? false
const factoryInstall = options.factoryInstall ?? false
const manifestRaw = findInTar(files, 'flash-manifest.json')
if (manifestRaw) {
const text = new TextDecoder().decode(manifestRaw)
const m = parseFlashManifest(text)
if (m) {
const section = activeSection(m, factoryInstall)
if (!section) return null
const out: FlashPart[] = []
for (const img of section.images) {
if (img.optional === true && isLittlefsManifestFile(img.file) && !resetDeviceStorage) continue
const data = findInTar(files, img.file)
if (!data) return null
const addr = typeof img.offset === 'string' ? parseInt(img.offset, 0) : Number(img.offset)
if (!Number.isFinite(addr)) return null
out.push({ data, address: addr, name: img.file })
}
if (out.length) {
return {
parts: sortFlashParts(out),
eraseAll: Boolean(section.eraseFlash),
}
}
return { parts: [{ data, address: 0x0, name: base }], eraseAll: false }
}
}
const bootloader = findInTar(files, 'bootloader.bin')
const partitions = findInTar(files, 'partitions.bin')
const bootApp0 = findInTar(files, 'boot_app0.bin')
const firmwareExact = findInTar(files, 'firmware.bin')
const versioned = resolveVersionedFirmwareApp(files)
let app: Uint8Array | undefined
let appName: string | undefined
if (firmwareExact) {
app = firmwareExact
appName = 'firmware.bin'
} else if (versioned) {
app = versioned.data
appName = versioned.name
}
if (bootloader && partitions && app && appName) {
const arr: FlashPart[] = [
{ data: bootloader, address: 0x1000, name: 'bootloader.bin' },
{ data: partitions, address: 0x8000, name: 'partitions.bin' },
{ data: app, address: 0x10000, name: appName },
]
if (bootApp0) arr.push({ data: bootApp0, address: 0xe000, name: 'boot_app0.bin' })
return { parts: sortFlashParts(arr), eraseAll: false }
}
if (app && appName && !bootloader && !partitions) {
return { parts: [{ data: app, address: 0x0, name: appName }], eraseAll: false }
}
return null
}
export function manifestFromMap(
files: Map<string, Uint8Array>,
manifestFile = 'flash-manifest.json'
): FlashManifest | null {
const raw = findInTar(files, manifestFile)
if (!raw) return null
return parseFlashManifest(new TextDecoder().decode(raw))
}
/** True if manifest includes a factory (erase + merged image) section with images. */
export function manifestHasFactorySection(m: FlashManifest | null): boolean {
return Boolean(m?.factory?.images && m.factory.images.length > 0)
}
+34 -23
View File
@@ -1,13 +1,7 @@
import type { FlashManifest, FlashTargetFamily } from "./untarGz"
import type { FlashTargetFamily } from "./untarGz"
const KNOWN: readonly FlashTargetFamily[] = ["esp32", "esp8266", "nrf52", "rp2040", "unknown"]
function coerceTargetFamily(v: unknown): FlashTargetFamily | undefined {
if (typeof v !== "string") return undefined
return KNOWN.includes(v as FlashTargetFamily) ? (v as FlashTargetFamily) : undefined
}
/** Heuristic when manifest lacks targetFamily (older bundles). */
/** Heuristic when bundle file types are ambiguous (e.g. UF2 shared by nRF52 and RP2040). */
export function inferTargetFamilyFromEnv(env: string | null | undefined): FlashTargetFamily | null {
if (!env?.trim()) return null
const e = env.toLowerCase()
@@ -27,22 +21,39 @@ export function inferTargetFamilyFromEnv(env: string | null | undefined): FlashT
}
/**
* Prefer manifest.targetFamily from CI; else env-name heuristic; else esp32 for legacy ESP-only bundles.
* Derive target family and erase capability from the files in a firmware bundle.
*
* Rules (in priority order):
* *.factory.bin present → esp32 (always supports chip erase)
* firmware.dat present → nrf52 Nordic DFU (always supports erase)
* *.uf2 present (not nuke.uf2) → nrf52 or rp2040 (use env name to disambiguate)
*
* canErase:
* esp32 → always true
* nrf52 DFU → always true
* nrf52 UF2 → true only when nuke.uf2 is also in the bundle
*/
export function resolveFlashTargetFamily(
manifest: FlashManifest | null,
targetEnv: string | null | undefined
): FlashTargetFamily {
const fromManifest = coerceTargetFamily(manifest?.targetFamily)
if (fromManifest && fromManifest !== "unknown") {
return fromManifest
export function inferTargetFamilyFromBundle(
files: Map<string, Uint8Array>,
targetEnv?: string | null
): { family: FlashTargetFamily; canErase: boolean } {
let hasUf2 = false
let hasNukeUf2 = false
for (const path of files.keys()) {
const base = path.replace(/^.*\//, "").toLowerCase()
if (base.endsWith(".factory.bin")) return { family: "esp32", canErase: true }
if (base === "firmware.dat") return { family: "nrf52", canErase: true }
if (base === "nuke.uf2") hasNukeUf2 = true
else if (base.endsWith(".uf2")) hasUf2 = true
}
const fromEnv = inferTargetFamilyFromEnv(targetEnv ?? null)
if (fromEnv) {
return fromEnv
if (hasUf2) {
const family = inferTargetFamilyFromEnv(targetEnv) ?? "nrf52"
return { family, canErase: hasNukeUf2 }
}
if (fromManifest === "unknown") {
return "unknown"
}
return "esp32"
// No clear signal — fall back to env name heuristic or default to esp32.
const family = inferTargetFamilyFromEnv(targetEnv) ?? "esp32"
return { family, canErase: family === "esp32" }
}
+15 -44
View File
@@ -1,5 +1,4 @@
import { findInTar } from './untarGz'
import type { FlashManifest } from './untarGz'
import { buildNordicDfuPlan, runNordicDfu } from './nrfDfuRun'
export type NrfFlashPlan = {
@@ -30,59 +29,32 @@ function pickDirectory(): Promise<FileSystemDirectoryHandle> {
}
/**
* Build an nRF52 UF2 flash plan from the bundle tarball + parsed manifest.
* Reads the `update` section for a normal flash, or `factory` when factoryInstall is true.
* Images with role "uf2" are the firmware; role "nuke" is the erase-all UF2.
*
* Falls back to scanning the bundle directly for *.uf2 when no manifest is present
* (covers pre-manifest bundles and builds where emit-flash-manifest.py did not run).
* Build an nRF52 UF2 flash plan from the bundle files.
* Scans for *.uf2 (preferring names starting with "firmware", excluding nuke.uf2).
* When factoryInstall is true, also includes nuke.uf2 if present in the bundle.
*/
export function buildNrfPlan(
files: Map<string, Uint8Array>,
manifest: FlashManifest | null,
factoryInstall: boolean
): NrfFlashPlan | null {
const section = factoryInstall ? (manifest?.factory ?? manifest?.update) : manifest?.update
let firmwareFile: Uint8Array | undefined
let firmwareName: string | undefined
if (section?.images?.length) {
let firmwareFile: Uint8Array | undefined
let firmwareName: string | undefined
let nukeFile: Uint8Array | undefined
for (const img of section.images) {
const role = img.role?.toLowerCase()
if (role === 'nuke') {
nukeFile = findInTar(files, img.file)
} else if (role === 'uf2' || img.file.toLowerCase().endsWith('.uf2')) {
if (!firmwareFile) {
firmwareFile = findInTar(files, img.file)
firmwareName = img.file
}
}
}
if (firmwareFile && firmwareName) {
return { firmwareFile, firmwareName, nukeFile }
}
}
// Fallback: scan bundle files directly for *.uf2 (no manifest or manifest had no UF2 images).
// Prefer names starting with "firmware", exclude nuke.uf2.
let fallbackName: string | undefined
let fallbackData: Uint8Array | undefined
for (const [path, data] of files) {
const base = path.replace(/^.*\//, '')
const baseLower = base.toLowerCase()
if (!baseLower.endsWith('.uf2') || baseLower === 'nuke.uf2') continue
if (!fallbackName || baseLower.startsWith('firmware')) {
fallbackName = base
fallbackData = data
if (!firmwareName || baseLower.startsWith('firmware')) {
firmwareName = base
firmwareFile = data
if (baseLower.startsWith('firmware')) break
}
}
if (!fallbackData || !fallbackName) return null
return { firmwareFile: fallbackData, firmwareName: fallbackName }
if (!firmwareFile || !firmwareName) return null
const nukeFile = factoryInstall ? findInTar(files, 'nuke.uf2') : undefined
return { firmwareFile, firmwareName, nukeFile }
}
/**
@@ -153,7 +125,7 @@ async function runUf2Flash(options: {
/**
* Flash an nRF52 device. Routes to Nordic Serial DFU (seamless, progress bar) when the bundle
* contains a Nordic DFU manifest (firmware.bin + firmware.dat), otherwise falls back to UF2
* contains a Nordic DFU package (firmware.bin + firmware.dat), otherwise falls back to UF2
* via the File System Access API drive picker.
*
* The port must have already received the 1200-baud CDC bootloader pulse and be closed.
@@ -161,12 +133,11 @@ async function runUf2Flash(options: {
export async function runNrfFlash(options: {
port: SerialPort
files: Map<string, Uint8Array>
manifest: FlashManifest | null
factoryInstall: boolean
onPhase: (label: string) => void
onWriteProgress: (p: NrfWriteProgressPayload) => void
}): Promise<void> {
const { port, files, manifest, factoryInstall, onPhase, onWriteProgress } = options
const { port, files, factoryInstall, onPhase, onWriteProgress } = options
// Prefer Nordic Serial DFU when the bundle contains a Nordic DFU package (bin + dat).
const dfuPlan = buildNordicDfuPlan(files)
@@ -182,7 +153,7 @@ export async function runNrfFlash(options: {
}
// Fall back to UF2 drive picker (Meshtastic nRF52 and other UF2 bootloader boards).
const uf2Plan = buildNrfPlan(files, manifest, factoryInstall)
const uf2Plan = buildNrfPlan(files, factoryInstall)
if (!uf2Plan) {
throw new Error('No flashable firmware found in bundle (expected Nordic DFU .bin/.dat or a .uf2 file)')
}
-42
View File
@@ -45,47 +45,5 @@ export function findInTar(files: Map<string, Uint8Array>, filename: string): Uin
return undefined
}
export type FlashManifestImage = {
file: string
offset: number | string
/** When true on LittleFS rows, Mesh Forge skips unless optional handling passes (legacy flat manifests). */
optional?: boolean
role?: string
}
/** One flash plan (update or factory) inside flash-manifest.json. */
export type FlashManifestSection = {
images: FlashManifestImage[]
/** When true, flasher performs full chip erase before write. */
eraseFlash?: boolean
}
/** Coarse MCU family for USB flasher entry + tool selection (from CI / PlatformIO). */
export type FlashTargetFamily = "esp32" | "esp8266" | "nrf52" | "rp2040" | "unknown"
/**
* Root flash-manifest.json: Meshtastic-style `update` + optional `factory`, or legacy flat `images`.
*/
export type FlashManifest = {
update?: FlashManifestSection
factory?: FlashManifestSection
/** Legacy single-layout manifest. */
images?: FlashManifestImage[]
eraseFlash?: boolean
targetFamily?: FlashTargetFamily
platform?: string
board?: string
}
export function parseFlashManifest(json: string): FlashManifest | null {
try {
const o = JSON.parse(json) as FlashManifest
if (!o || typeof o !== "object") return null
if (o.update && Array.isArray(o.update.images) && o.update.images.length > 0) return o
if (o.factory && Array.isArray(o.factory.images) && o.factory.images.length > 0) return o
if (Array.isArray(o.images) && o.images.length > 0) return o
return null
} catch {
return null
}
}