From 1002219ae25660c22772a9314ee115e64c2c7924 Mon Sep 17 00:00:00 2001 From: Ben Allfree Date: Sat, 18 Apr 2026 10:19:37 -0700 Subject: [PATCH] refactor: update CI scripts to use new reporting script and improve firmware packaging process --- .github/workflows/custom_build.yml | 130 ++---------- .github/workflows/custom_build_test.yml | 130 ++---------- .../common/copy-esp32-app-bin-for-bundle.sh | 48 +++++ .../common}/download-meshtastic-ota.sh | 0 .../{ => ci/common}/extend-merged-with-ota.py | 0 .../extend-meshtastic-merged-if-present.sh | 27 +++ scripts/ci/common/fw-bundle-pipeline.sh | 66 +++++++ scripts/ci/common/mergebin-esp32.sh | 28 +++ .../common/normalize-esp32-merged-factory.sh | 56 ++++++ .../ci/common/package-firmware-to-stage.sh | 45 +++++ .../{ => ci/common}/stage-fw-bundle-docs.sh | 0 scripts/ci/common/stage-nrf-dfu.sh | 41 ++++ .../common}/stage-nrf52-dfu-from-build.sh | 40 +++- scripts/ci/common/verify-fw-bundle-stage.sh | 186 ++++++++++++++++++ scripts/{ => ci}/report-convex-ci-progress.py | 0 scripts/ci/test/fw-pio-smoke-envs.json | 35 ++++ scripts/ci/test/run-fw-pio-smoke-local.sh | 72 +++++++ scripts/ci/test/test-pio-fw-staging-local.sh | 38 ++++ 18 files changed, 706 insertions(+), 236 deletions(-) create mode 100755 scripts/ci/common/copy-esp32-app-bin-for-bundle.sh rename scripts/{ => ci/common}/download-meshtastic-ota.sh (100%) rename scripts/{ => ci/common}/extend-merged-with-ota.py (100%) create mode 100755 scripts/ci/common/extend-meshtastic-merged-if-present.sh create mode 100755 scripts/ci/common/fw-bundle-pipeline.sh create mode 100755 scripts/ci/common/mergebin-esp32.sh create mode 100755 scripts/ci/common/normalize-esp32-merged-factory.sh create mode 100755 scripts/ci/common/package-firmware-to-stage.sh rename scripts/{ => ci/common}/stage-fw-bundle-docs.sh (100%) create mode 100755 scripts/ci/common/stage-nrf-dfu.sh rename scripts/{ => ci/common}/stage-nrf52-dfu-from-build.sh (72%) create mode 100755 scripts/ci/common/verify-fw-bundle-stage.sh rename scripts/{ => ci}/report-convex-ci-progress.py (100%) create mode 100644 scripts/ci/test/fw-pio-smoke-envs.json create mode 100755 scripts/ci/test/run-fw-pio-smoke-local.sh create mode 100755 scripts/ci/test/test-pio-fw-staging-local.sh diff --git a/.github/workflows/custom_build.yml b/.github/workflows/custom_build.yml index 3a0d13e..4b882de 100644 --- a/.github/workflows/custom_build.yml +++ b/.github/workflows/custom_build.yml @@ -53,7 +53,7 @@ jobs: env: STEP_INDEX: 1 LABEL: Build started on Actions - run: python3 "${{ github.workspace }}/scripts/report-convex-ci-progress.py" + run: python3 "${{ github.workspace }}/scripts/ci/report-convex-ci-progress.py" - name: Set up Python uses: actions/setup-python@v5 @@ -70,7 +70,7 @@ jobs: env: STEP_INDEX: 2 LABEL: Runner tooling ready - run: python3 "${{ github.workspace }}/scripts/report-convex-ci-progress.py" + run: python3 "${{ github.workspace }}/scripts/ci/report-convex-ci-progress.py" - name: Download source archive shell: bash @@ -79,7 +79,7 @@ jobs: REPO: ${{ inputs.repo }} REF: ${{ inputs.ref }} run: | - STEP_INDEX=3 LABEL="Downloading source archive" python3 "${{ github.workspace }}/scripts/report-convex-ci-progress.py" + STEP_INDEX=3 LABEL="Downloading source archive" python3 "${{ github.workspace }}/scripts/ci/report-convex-ci-progress.py" ENC_REF=$(python3 -c "import urllib.parse,os; print(urllib.parse.quote(os.environ['REF'], safe=''))") curl -fsSL -o /tmp/src.zip "https://codeload.github.com/${OWNER}/${REPO}/zip/${ENC_REF}" @@ -88,7 +88,7 @@ jobs: env: STEP_INDEX: 4 LABEL: Restoring PlatformIO global cache (toolchains) - run: python3 "${{ github.workspace }}/scripts/report-convex-ci-progress.py" + run: python3 "${{ github.workspace }}/scripts/ci/report-convex-ci-progress.py" - name: Cache PlatformIO global store uses: actions/cache@v4 @@ -105,7 +105,7 @@ jobs: env: STEP_INDEX: 5 LABEL: Restoring PlatformIO libdeps cache - run: python3 "${{ github.workspace }}/scripts/report-convex-ci-progress.py" + run: python3 "${{ github.workspace }}/scripts/ci/report-convex-ci-progress.py" # Key is owner/repo/env only (not source SHA); bump …-v2 if lib_deps get out of sync. - name: Cache PlatformIO libdeps (per repo + target) @@ -117,7 +117,7 @@ jobs: - name: Extract firmware source shell: bash run: | - STEP_INDEX=6 LABEL="Extracting firmware source" python3 "${{ github.workspace }}/scripts/report-convex-ci-progress.py" + STEP_INDEX=6 LABEL="Extracting firmware source" python3 "${{ github.workspace }}/scripts/ci/report-convex-ci-progress.py" set -e sudo apt-get update -qq && sudo apt-get install -y -qq unzip rm -rf /tmp/src @@ -130,7 +130,7 @@ jobs: - name: Install PlatformIO shell: bash run: | - STEP_INDEX=7 LABEL="Installing PlatformIO" python3 "${{ github.workspace }}/scripts/report-convex-ci-progress.py" + STEP_INDEX=7 LABEL="Installing PlatformIO" python3 "${{ github.workspace }}/scripts/ci/report-convex-ci-progress.py" set -e ROOT=$(cat /tmp/fw-src-root.txt) cd "$ROOT" @@ -140,7 +140,7 @@ jobs: - name: Install PlatformIO dependencies shell: bash run: | - STEP_INDEX=8 LABEL="Installing firmware dependencies (PlatformIO)" python3 "${{ github.workspace }}/scripts/report-convex-ci-progress.py" + STEP_INDEX=8 LABEL="Installing firmware dependencies (PlatformIO)" python3 "${{ github.workspace }}/scripts/ci/report-convex-ci-progress.py" set -e ROOT=$(cat /tmp/fw-src-root.txt) cd "$ROOT" @@ -149,7 +149,7 @@ jobs: - name: Compile firmware (PlatformIO) shell: bash run: | - STEP_INDEX=9 LABEL="Compiling firmware (PlatformIO)" python3 "${{ github.workspace }}/scripts/report-convex-ci-progress.py" + STEP_INDEX=9 LABEL="Compiling firmware (PlatformIO)" python3 "${{ github.workspace }}/scripts/ci/report-convex-ci-progress.py" set -e ROOT=$(cat /tmp/fw-src-root.txt) cd "$ROOT" @@ -165,87 +165,6 @@ jobs: exit 1 fi - - name: Generate merged factory binary for ESP32 targets (if applicable) - shell: bash - run: | - ROOT=$(cat /tmp/fw-src-root.txt) - BUILD_DIR="$ROOT/.pio/build/${{ inputs.target_env }}" - if [ -f "$BUILD_DIR/bootloader.bin" ]; then - echo "ESP32 target detected; running mergebin → firmware-merged.factory.bin" - cd "$ROOT" - export MERGED_BIN_PATH="$BUILD_DIR/firmware-merged.factory.bin" - pio run -t mergebin -e "${{ inputs.target_env }}" 2>&1 || \ - echo "WARNING: mergebin target failed; factory binary will not be available" - else - echo "No bootloader.bin found; skipping mergebin (non-ESP32 target)" - fi - - - name: Download Meshtastic ESP OTA companion (if applicable) - shell: bash - run: | - set -e - ROOT=$(cat /tmp/fw-src-root.txt) - cd "$ROOT" - BUILD_DIR=".pio/build/${{ inputs.target_env }}" - bash "${{ github.workspace }}/scripts/download-meshtastic-ota.sh" "$ROOT/$BUILD_DIR" - - - name: Extend merged binary with Meshtastic OTA companion (if applicable) - shell: bash - run: | - ROOT=$(cat /tmp/fw-src-root.txt) - BUILD_DIR="$ROOT/.pio/build/${{ inputs.target_env }}" - MERGED="$BUILD_DIR/firmware-merged.factory.bin" - PARTS="$BUILD_DIR/partitions.bin" - OTA_BIN="" - for candidate in "$BUILD_DIR"/mt-*.ota.bin "$BUILD_DIR/bleota-c3.bin"; do - if [ -f "$candidate" ]; then OTA_BIN="$candidate"; break; fi - done - if [ -f "$MERGED" ] && [ -f "$PARTS" ] && [ -n "$OTA_BIN" ]; then - python3 "${{ github.workspace }}/scripts/extend-merged-with-ota.py" \ - "$MERGED" "$PARTS" "$OTA_BIN" - else - echo "Skipping OTA extension (merged=$([ -f "$MERGED" ] && echo yes || echo no), ota=$([ -n "$OTA_BIN" ] && echo yes || echo no))" - fi - - - name: Stage or generate nRF52 DFU files (if applicable) - shell: bash - run: | - set -e - ROOT=$(cat /tmp/fw-src-root.txt) - BUILD_DIR="$ROOT/.pio/build/${{ inputs.target_env }}" - - # Path 1: unpack any DFU-shaped *.zip from the PIO build dir, normalize to firmware.bin/dat. - bash "${{ github.workspace }}/scripts/stage-nrf52-dfu-from-build.sh" "$BUILD_DIR" - - # Path 2: firmware/ subdirectory fallback (local builds or alternate PIO layouts). - if [ -d "$BUILD_DIR/firmware" ] && ! ls "$BUILD_DIR"/*.dat >/dev/null 2>&1; then - echo "Found firmware/ subdirectory; staging nRF52 DFU files to build root" - shopt -s nullglob - for f in "$BUILD_DIR/firmware/"*.bin "$BUILD_DIR/firmware/"*.dat; do - cp -a "$f" "$BUILD_DIR/" - done - shopt -u nullglob - [ -f "$BUILD_DIR/firmware/manifest.json" ] && cp -a "$BUILD_DIR/firmware/manifest.json" "$BUILD_DIR/" - fi - - # Path 3: firmware.bin present but still no .dat — nRF52 only. - # Guard against ESP32 builds which also produce firmware.bin; bootloader.bin is - # ESP32-specific and is never present in nRF52 build output. - if [ -f "$BUILD_DIR/firmware.bin" ] && \ - ! [ -f "$BUILD_DIR/bootloader.bin" ] && \ - ! ls "$BUILD_DIR"/*.dat >/dev/null 2>&1; then - echo "firmware.bin found but no .dat (nRF52); generating DFU init packet" - adafruit-nrfutil dfu genpkg \ - --dev-type 0xFFFF \ - --dev-revision 0xFFFF \ - --application-version 0xFFFFFFFF \ - --sd-req 0xFFFE \ - --application "$BUILD_DIR/firmware.bin" \ - /tmp/nrf52-dfu.zip - unzip -j /tmp/nrf52-dfu.zip "*.dat" -d "$BUILD_DIR/" - echo "DFU init packet written: $(ls "$BUILD_DIR"/*.dat 2>/dev/null || echo '(none)')" - fi - - name: Package and upload firmware bundle id: pio env: @@ -254,38 +173,15 @@ jobs: R2_BUCKET_NAME: ${{ secrets.R2_BUCKET_NAME }} shell: bash run: | - STEP_INDEX=10 LABEL="Packaging and uploading firmware bundle" python3 "${{ github.workspace }}/scripts/report-convex-ci-progress.py" + STEP_INDEX=10 LABEL="Packaging and uploading firmware bundle" python3 "${{ github.workspace }}/scripts/ci/report-convex-ci-progress.py" set -e ROOT=$(cat /tmp/fw-src-root.txt) - cd "$ROOT" - BUILD_DIR=".pio/build/${{ inputs.target_env }}" + BUILD_DIR_ABS="$ROOT/.pio/build/${{ inputs.target_env }}" ARTIFACT_NAME="firmware-${{ inputs.build_key }}-${{ github.run_id }}.tar.gz" - STAGE=/tmp/fw-bundle - rm -rf "$STAGE" - mkdir -p "$STAGE" - shopt -s nullglob - if ls "$BUILD_DIR"/firmware-*.factory.bin >/dev/null 2>&1; then - # ESP32: only the merged factory binary — sub-components are not needed - for f in "$BUILD_DIR"/firmware-*.factory.bin; do cp -a "$f" "$STAGE/"; done - else - # nRF52 / RP2040: firmware binary, DFU init packet, UF2 files - for f in "$BUILD_DIR"/firmware.bin "$BUILD_DIR"/firmware.dat \ - "$BUILD_DIR"/*.uf2 "$BUILD_DIR"/*.hex; do - [ -f "$f" ] && cp -a "$f" "$STAGE/" - done - # Nordic DFU manifest (MeshCore and other nRF52 builds) - [ -f "$BUILD_DIR/manifest.json" ] && cp -a "$BUILD_DIR/manifest.json" "$STAGE/" - fi - shopt -u nullglob - bash "${{ github.workspace }}/scripts/stage-fw-bundle-docs.sh" "$ROOT" "$STAGE" - if [ -z "$(find "$STAGE" -mindepth 1 -maxdepth 1 -type f -print -quit)" ]; then - echo "No firmware artifacts found in $BUILD_DIR" - exit 1 - fi - tar -czf "/tmp/$ARTIFACT_NAME" -C "$STAGE" . + MESHTASTIC_OTA=1 bash "${{ github.workspace }}/scripts/ci/common/fw-bundle-pipeline.sh" "$ROOT" "$BUILD_DIR_ABS" "/tmp/$ARTIFACT_NAME" OBJECT_PATH="${R2_BUCKET_NAME}/${ARTIFACT_NAME}" bunx wrangler r2 object put "$OBJECT_PATH" --file "/tmp/$ARTIFACT_NAME" --remote - STEP_INDEX=10 LABEL="Firmware bundle uploaded" python3 "${{ github.workspace }}/scripts/report-convex-ci-progress.py" + STEP_INDEX=10 LABEL="Firmware bundle uploaded" python3 "${{ github.workspace }}/scripts/ci/report-convex-ci-progress.py" echo "r2_key=${ARTIFACT_NAME}" >> "$GITHUB_OUTPUT" - name: Notify Convex — success diff --git a/.github/workflows/custom_build_test.yml b/.github/workflows/custom_build_test.yml index 3a0d13e..4b882de 100644 --- a/.github/workflows/custom_build_test.yml +++ b/.github/workflows/custom_build_test.yml @@ -53,7 +53,7 @@ jobs: env: STEP_INDEX: 1 LABEL: Build started on Actions - run: python3 "${{ github.workspace }}/scripts/report-convex-ci-progress.py" + run: python3 "${{ github.workspace }}/scripts/ci/report-convex-ci-progress.py" - name: Set up Python uses: actions/setup-python@v5 @@ -70,7 +70,7 @@ jobs: env: STEP_INDEX: 2 LABEL: Runner tooling ready - run: python3 "${{ github.workspace }}/scripts/report-convex-ci-progress.py" + run: python3 "${{ github.workspace }}/scripts/ci/report-convex-ci-progress.py" - name: Download source archive shell: bash @@ -79,7 +79,7 @@ jobs: REPO: ${{ inputs.repo }} REF: ${{ inputs.ref }} run: | - STEP_INDEX=3 LABEL="Downloading source archive" python3 "${{ github.workspace }}/scripts/report-convex-ci-progress.py" + STEP_INDEX=3 LABEL="Downloading source archive" python3 "${{ github.workspace }}/scripts/ci/report-convex-ci-progress.py" ENC_REF=$(python3 -c "import urllib.parse,os; print(urllib.parse.quote(os.environ['REF'], safe=''))") curl -fsSL -o /tmp/src.zip "https://codeload.github.com/${OWNER}/${REPO}/zip/${ENC_REF}" @@ -88,7 +88,7 @@ jobs: env: STEP_INDEX: 4 LABEL: Restoring PlatformIO global cache (toolchains) - run: python3 "${{ github.workspace }}/scripts/report-convex-ci-progress.py" + run: python3 "${{ github.workspace }}/scripts/ci/report-convex-ci-progress.py" - name: Cache PlatformIO global store uses: actions/cache@v4 @@ -105,7 +105,7 @@ jobs: env: STEP_INDEX: 5 LABEL: Restoring PlatformIO libdeps cache - run: python3 "${{ github.workspace }}/scripts/report-convex-ci-progress.py" + run: python3 "${{ github.workspace }}/scripts/ci/report-convex-ci-progress.py" # Key is owner/repo/env only (not source SHA); bump …-v2 if lib_deps get out of sync. - name: Cache PlatformIO libdeps (per repo + target) @@ -117,7 +117,7 @@ jobs: - name: Extract firmware source shell: bash run: | - STEP_INDEX=6 LABEL="Extracting firmware source" python3 "${{ github.workspace }}/scripts/report-convex-ci-progress.py" + STEP_INDEX=6 LABEL="Extracting firmware source" python3 "${{ github.workspace }}/scripts/ci/report-convex-ci-progress.py" set -e sudo apt-get update -qq && sudo apt-get install -y -qq unzip rm -rf /tmp/src @@ -130,7 +130,7 @@ jobs: - name: Install PlatformIO shell: bash run: | - STEP_INDEX=7 LABEL="Installing PlatformIO" python3 "${{ github.workspace }}/scripts/report-convex-ci-progress.py" + STEP_INDEX=7 LABEL="Installing PlatformIO" python3 "${{ github.workspace }}/scripts/ci/report-convex-ci-progress.py" set -e ROOT=$(cat /tmp/fw-src-root.txt) cd "$ROOT" @@ -140,7 +140,7 @@ jobs: - name: Install PlatformIO dependencies shell: bash run: | - STEP_INDEX=8 LABEL="Installing firmware dependencies (PlatformIO)" python3 "${{ github.workspace }}/scripts/report-convex-ci-progress.py" + STEP_INDEX=8 LABEL="Installing firmware dependencies (PlatformIO)" python3 "${{ github.workspace }}/scripts/ci/report-convex-ci-progress.py" set -e ROOT=$(cat /tmp/fw-src-root.txt) cd "$ROOT" @@ -149,7 +149,7 @@ jobs: - name: Compile firmware (PlatformIO) shell: bash run: | - STEP_INDEX=9 LABEL="Compiling firmware (PlatformIO)" python3 "${{ github.workspace }}/scripts/report-convex-ci-progress.py" + STEP_INDEX=9 LABEL="Compiling firmware (PlatformIO)" python3 "${{ github.workspace }}/scripts/ci/report-convex-ci-progress.py" set -e ROOT=$(cat /tmp/fw-src-root.txt) cd "$ROOT" @@ -165,87 +165,6 @@ jobs: exit 1 fi - - name: Generate merged factory binary for ESP32 targets (if applicable) - shell: bash - run: | - ROOT=$(cat /tmp/fw-src-root.txt) - BUILD_DIR="$ROOT/.pio/build/${{ inputs.target_env }}" - if [ -f "$BUILD_DIR/bootloader.bin" ]; then - echo "ESP32 target detected; running mergebin → firmware-merged.factory.bin" - cd "$ROOT" - export MERGED_BIN_PATH="$BUILD_DIR/firmware-merged.factory.bin" - pio run -t mergebin -e "${{ inputs.target_env }}" 2>&1 || \ - echo "WARNING: mergebin target failed; factory binary will not be available" - else - echo "No bootloader.bin found; skipping mergebin (non-ESP32 target)" - fi - - - name: Download Meshtastic ESP OTA companion (if applicable) - shell: bash - run: | - set -e - ROOT=$(cat /tmp/fw-src-root.txt) - cd "$ROOT" - BUILD_DIR=".pio/build/${{ inputs.target_env }}" - bash "${{ github.workspace }}/scripts/download-meshtastic-ota.sh" "$ROOT/$BUILD_DIR" - - - name: Extend merged binary with Meshtastic OTA companion (if applicable) - shell: bash - run: | - ROOT=$(cat /tmp/fw-src-root.txt) - BUILD_DIR="$ROOT/.pio/build/${{ inputs.target_env }}" - MERGED="$BUILD_DIR/firmware-merged.factory.bin" - PARTS="$BUILD_DIR/partitions.bin" - OTA_BIN="" - for candidate in "$BUILD_DIR"/mt-*.ota.bin "$BUILD_DIR/bleota-c3.bin"; do - if [ -f "$candidate" ]; then OTA_BIN="$candidate"; break; fi - done - if [ -f "$MERGED" ] && [ -f "$PARTS" ] && [ -n "$OTA_BIN" ]; then - python3 "${{ github.workspace }}/scripts/extend-merged-with-ota.py" \ - "$MERGED" "$PARTS" "$OTA_BIN" - else - echo "Skipping OTA extension (merged=$([ -f "$MERGED" ] && echo yes || echo no), ota=$([ -n "$OTA_BIN" ] && echo yes || echo no))" - fi - - - name: Stage or generate nRF52 DFU files (if applicable) - shell: bash - run: | - set -e - ROOT=$(cat /tmp/fw-src-root.txt) - BUILD_DIR="$ROOT/.pio/build/${{ inputs.target_env }}" - - # Path 1: unpack any DFU-shaped *.zip from the PIO build dir, normalize to firmware.bin/dat. - bash "${{ github.workspace }}/scripts/stage-nrf52-dfu-from-build.sh" "$BUILD_DIR" - - # Path 2: firmware/ subdirectory fallback (local builds or alternate PIO layouts). - if [ -d "$BUILD_DIR/firmware" ] && ! ls "$BUILD_DIR"/*.dat >/dev/null 2>&1; then - echo "Found firmware/ subdirectory; staging nRF52 DFU files to build root" - shopt -s nullglob - for f in "$BUILD_DIR/firmware/"*.bin "$BUILD_DIR/firmware/"*.dat; do - cp -a "$f" "$BUILD_DIR/" - done - shopt -u nullglob - [ -f "$BUILD_DIR/firmware/manifest.json" ] && cp -a "$BUILD_DIR/firmware/manifest.json" "$BUILD_DIR/" - fi - - # Path 3: firmware.bin present but still no .dat — nRF52 only. - # Guard against ESP32 builds which also produce firmware.bin; bootloader.bin is - # ESP32-specific and is never present in nRF52 build output. - if [ -f "$BUILD_DIR/firmware.bin" ] && \ - ! [ -f "$BUILD_DIR/bootloader.bin" ] && \ - ! ls "$BUILD_DIR"/*.dat >/dev/null 2>&1; then - echo "firmware.bin found but no .dat (nRF52); generating DFU init packet" - adafruit-nrfutil dfu genpkg \ - --dev-type 0xFFFF \ - --dev-revision 0xFFFF \ - --application-version 0xFFFFFFFF \ - --sd-req 0xFFFE \ - --application "$BUILD_DIR/firmware.bin" \ - /tmp/nrf52-dfu.zip - unzip -j /tmp/nrf52-dfu.zip "*.dat" -d "$BUILD_DIR/" - echo "DFU init packet written: $(ls "$BUILD_DIR"/*.dat 2>/dev/null || echo '(none)')" - fi - - name: Package and upload firmware bundle id: pio env: @@ -254,38 +173,15 @@ jobs: R2_BUCKET_NAME: ${{ secrets.R2_BUCKET_NAME }} shell: bash run: | - STEP_INDEX=10 LABEL="Packaging and uploading firmware bundle" python3 "${{ github.workspace }}/scripts/report-convex-ci-progress.py" + STEP_INDEX=10 LABEL="Packaging and uploading firmware bundle" python3 "${{ github.workspace }}/scripts/ci/report-convex-ci-progress.py" set -e ROOT=$(cat /tmp/fw-src-root.txt) - cd "$ROOT" - BUILD_DIR=".pio/build/${{ inputs.target_env }}" + BUILD_DIR_ABS="$ROOT/.pio/build/${{ inputs.target_env }}" ARTIFACT_NAME="firmware-${{ inputs.build_key }}-${{ github.run_id }}.tar.gz" - STAGE=/tmp/fw-bundle - rm -rf "$STAGE" - mkdir -p "$STAGE" - shopt -s nullglob - if ls "$BUILD_DIR"/firmware-*.factory.bin >/dev/null 2>&1; then - # ESP32: only the merged factory binary — sub-components are not needed - for f in "$BUILD_DIR"/firmware-*.factory.bin; do cp -a "$f" "$STAGE/"; done - else - # nRF52 / RP2040: firmware binary, DFU init packet, UF2 files - for f in "$BUILD_DIR"/firmware.bin "$BUILD_DIR"/firmware.dat \ - "$BUILD_DIR"/*.uf2 "$BUILD_DIR"/*.hex; do - [ -f "$f" ] && cp -a "$f" "$STAGE/" - done - # Nordic DFU manifest (MeshCore and other nRF52 builds) - [ -f "$BUILD_DIR/manifest.json" ] && cp -a "$BUILD_DIR/manifest.json" "$STAGE/" - fi - shopt -u nullglob - bash "${{ github.workspace }}/scripts/stage-fw-bundle-docs.sh" "$ROOT" "$STAGE" - if [ -z "$(find "$STAGE" -mindepth 1 -maxdepth 1 -type f -print -quit)" ]; then - echo "No firmware artifacts found in $BUILD_DIR" - exit 1 - fi - tar -czf "/tmp/$ARTIFACT_NAME" -C "$STAGE" . + MESHTASTIC_OTA=1 bash "${{ github.workspace }}/scripts/ci/common/fw-bundle-pipeline.sh" "$ROOT" "$BUILD_DIR_ABS" "/tmp/$ARTIFACT_NAME" OBJECT_PATH="${R2_BUCKET_NAME}/${ARTIFACT_NAME}" bunx wrangler r2 object put "$OBJECT_PATH" --file "/tmp/$ARTIFACT_NAME" --remote - STEP_INDEX=10 LABEL="Firmware bundle uploaded" python3 "${{ github.workspace }}/scripts/report-convex-ci-progress.py" + STEP_INDEX=10 LABEL="Firmware bundle uploaded" python3 "${{ github.workspace }}/scripts/ci/report-convex-ci-progress.py" echo "r2_key=${ARTIFACT_NAME}" >> "$GITHUB_OUTPUT" - name: Notify Convex — success diff --git a/scripts/ci/common/copy-esp32-app-bin-for-bundle.sh b/scripts/ci/common/copy-esp32-app-bin-for-bundle.sh new file mode 100755 index 0000000..07382f7 --- /dev/null +++ b/scripts/ci/common/copy-esp32-app-bin-for-bundle.sh @@ -0,0 +1,48 @@ +#!/usr/bin/env bash +# When no merged *.factory.bin exists, stage the main application .bin for Web Serial flashing +# (single image at 0x0). Picks the largest .bin excluding common ESP-IDF / partition artifacts. +# +# Usage: copy-esp32-app-bin-for-bundle.sh BUILD_DIR STAGE_DIR +set -euo pipefail + +BUILD_DIR="${1:?usage: copy-esp32-app-bin-for-bundle.sh BUILD_DIR STAGE_DIR}" +STAGE_DIR="${2:?usage: copy-esp32-app-bin-for-bundle.sh BUILD_DIR STAGE_DIR}" + +is_excluded() { + local bl + bl=$(echo "$1" | tr '[:upper:]' '[:lower:]') + case "$bl" in + bootloader.bin|partitions.bin|boot_app0.bin) return 0 ;; + esac + return 1 +} + +shopt -s nullglob +bins=( "$BUILD_DIR"/*.bin ) +shopt -u nullglob + +eligible=() +for f in "${bins[@]}"; do + [ -f "$f" ] || continue + is_excluded "$(basename "$f")" && continue + eligible+=("$f") +done + +if [ "${#eligible[@]}" -eq 0 ]; then + echo "No eligible application .bin under $BUILD_DIR" + exit 1 +fi + +best="" +bestz=-1 +for f in "${eligible[@]}"; do + z=$(wc -c <"$f" | tr -d ' ') + if [ "$z" -gt "$bestz" ]; then + bestz=$z + best=$f + fi +done + +bn=$(basename "$best") +echo "Staging ESP32 application binary (no factory merge): $bn" +cp -a "$best" "$STAGE_DIR/$bn" diff --git a/scripts/download-meshtastic-ota.sh b/scripts/ci/common/download-meshtastic-ota.sh similarity index 100% rename from scripts/download-meshtastic-ota.sh rename to scripts/ci/common/download-meshtastic-ota.sh diff --git a/scripts/extend-merged-with-ota.py b/scripts/ci/common/extend-merged-with-ota.py similarity index 100% rename from scripts/extend-merged-with-ota.py rename to scripts/ci/common/extend-merged-with-ota.py diff --git a/scripts/ci/common/extend-meshtastic-merged-if-present.sh b/scripts/ci/common/extend-meshtastic-merged-if-present.sh new file mode 100755 index 0000000..2f70564 --- /dev/null +++ b/scripts/ci/common/extend-meshtastic-merged-if-present.sh @@ -0,0 +1,27 @@ +#!/usr/bin/env bash +# If merged factory + partition table + a Meshtastic OTA companion bin exist, extend the merged image. +# No-op when inputs are missing (same behavior as custom_build firmware workflow). +# +# Usage: extend-meshtastic-merged-if-present.sh BUILD_DIR +set -euo pipefail + +SCRIPT_DIR=$(cd "$(dirname "$0")" && pwd) +MESH_FORGE_ROOT=$(cd "$SCRIPT_DIR/../../.." && pwd) + +BUILD_DIR=${1:?usage: extend-meshtastic-merged-if-present.sh BUILD_DIR} + +MERGED="$BUILD_DIR/firmware-merged.factory.bin" +PARTS="$BUILD_DIR/partitions.bin" +OTA_BIN="" +shopt -s nullglob +for candidate in "$BUILD_DIR"/mt-*.ota.bin "$BUILD_DIR"/bleota-c3.bin; do + if [ -f "$candidate" ]; then OTA_BIN="$candidate"; break; fi +done +shopt -u nullglob + +if [ -f "$MERGED" ] && [ -f "$PARTS" ] && [ -n "$OTA_BIN" ]; then + python3 "$MESH_FORGE_ROOT/scripts/ci/common/extend-merged-with-ota.py" \ + "$MERGED" "$PARTS" "$OTA_BIN" +else + echo "Skipping OTA extension (merged=$([ -f "$MERGED" ] && echo yes || echo no), ota=$([ -n "$OTA_BIN" ] && echo yes || echo no))" +fi diff --git a/scripts/ci/common/fw-bundle-pipeline.sh b/scripts/ci/common/fw-bundle-pipeline.sh new file mode 100755 index 0000000..8a54328 --- /dev/null +++ b/scripts/ci/common/fw-bundle-pipeline.sh @@ -0,0 +1,66 @@ +#!/usr/bin/env bash +# Single pipeline: PlatformIO build dir → staged artifacts → gzip tarball → verify tarball. +# Used by GitHub Actions (with Meshtastic OTA) and local smoke (without OTA). Do not duplicate +# these steps in workflows — call this script only. +# +# Env: +# MESHTASTIC_OTA Set to 1 for CI (download OTA companion + extend merged bin when applicable). +# Omit or 0 for local parity tests (skip download/extend). +# FW_BUNDLE_VERBOSE Set to 1 to print ls of stage dir before tarring (local UX). +# +# Usage: fw-bundle-pipeline.sh FIRMWARE_ROOT BUILD_DIR_ABS OUTPUT_TAR_GZ +# FIRMWARE_ROOT = PlatformIO project root (platformio.ini directory) +# BUILD_DIR_ABS = absolute path to .pio/build/ +# OUTPUT_TAR_GZ = path to write the verified .tar.gz (parent dir must exist) +set -euo pipefail + +SCRIPT_DIR=$(cd "$(dirname "$0")" && pwd) +MESH_FORGE_ROOT=$(cd "$SCRIPT_DIR/../../.." && pwd) + +FIRMWARE_ROOT=${1:?usage: fw-bundle-pipeline.sh FIRMWARE_ROOT BUILD_DIR_ABS OUTPUT_TAR_GZ} +BUILD_DIR_ABS=${2:?usage: fw-bundle-pipeline.sh FIRMWARE_ROOT BUILD_DIR_ABS OUTPUT_TAR_GZ} +OUTPUT_TAR_GZ=${3:?usage: fw-bundle-pipeline.sh FIRMWARE_ROOT BUILD_DIR_ABS OUTPUT_TAR_GZ} + +MESHTASTIC_OTA=${MESHTASTIC_OTA:-0} +FW_BUNDLE_VERBOSE=${FW_BUNDLE_VERBOSE:-0} + +if [ ! -d "$FIRMWARE_ROOT" ] || [ ! -d "$BUILD_DIR_ABS" ]; then + echo "FIRMWARE_ROOT or BUILD_DIR_ABS is not a directory" >&2 + exit 1 +fi + +PIO_ENV=$(basename "$BUILD_DIR_ABS") + +echo "== fw-bundle-pipeline: FIRMWARE_ROOT=$FIRMWARE_ROOT PIO_ENV=$PIO_ENV OUTPUT=$OUTPUT_TAR_GZ MESHTASTIC_OTA=$MESHTASTIC_OTA ==" + +bash "$MESH_FORGE_ROOT/scripts/ci/common/mergebin-esp32.sh" "$FIRMWARE_ROOT" "$PIO_ENV" +bash "$MESH_FORGE_ROOT/scripts/ci/common/normalize-esp32-merged-factory.sh" "$BUILD_DIR_ABS" + +if [ "$MESHTASTIC_OTA" = 1 ]; then + bash "$MESH_FORGE_ROOT/scripts/ci/common/download-meshtastic-ota.sh" "$BUILD_DIR_ABS" + bash "$MESH_FORGE_ROOT/scripts/ci/common/extend-meshtastic-merged-if-present.sh" "$BUILD_DIR_ABS" +else + echo "(Skipping Meshtastic OTA download/extend — set MESHTASTIC_OTA=1 for CI parity)" +fi + +bash "$MESH_FORGE_ROOT/scripts/ci/common/stage-nrf-dfu.sh" "$BUILD_DIR_ABS" + +STAGE_DIR=$(mktemp -d "${TMPDIR:-/tmp}/fw-bundle-stage.XXXXXX") +cleanup() { + rm -rf "$STAGE_DIR" +} +trap cleanup EXIT + +bash "$MESH_FORGE_ROOT/scripts/ci/common/package-firmware-to-stage.sh" "$FIRMWARE_ROOT" "$BUILD_DIR_ABS" "$STAGE_DIR" + +if [ "$FW_BUNDLE_VERBOSE" = 1 ]; then + echo "" + echo "--- Staged files (tarball root) ---" + ls -la "$STAGE_DIR" + echo "" +fi + +tar -czf "$OUTPUT_TAR_GZ" -C "$STAGE_DIR" . +bash "$MESH_FORGE_ROOT/scripts/ci/common/verify-fw-bundle-stage.sh" "$BUILD_DIR_ABS" "$OUTPUT_TAR_GZ" + +echo "fw-bundle-pipeline: OK -> $OUTPUT_TAR_GZ" diff --git a/scripts/ci/common/mergebin-esp32.sh b/scripts/ci/common/mergebin-esp32.sh new file mode 100755 index 0000000..89d1c16 --- /dev/null +++ b/scripts/ci/common/mergebin-esp32.sh @@ -0,0 +1,28 @@ +#!/usr/bin/env bash +# Run PlatformIO mergebin only when the env exposes that target (avoids CI noise when absent). +# +# Usage: mergebin-esp32.sh FIRMWARE_ROOT PIO_ENV_NAME +# FIRMWARE_ROOT = PlatformIO project root (where platformio.ini lives). +set -euo pipefail + +FIRMWARE_ROOT=${1:?usage: mergebin-esp32.sh FIRMWARE_ROOT PIO_ENV} +PIO_ENV=${2:?usage: mergebin-esp32.sh FIRMWARE_ROOT PIO_ENV} + +BUILD_DIR="$FIRMWARE_ROOT/.pio/build/$PIO_ENV" + +if [ ! -f "$BUILD_DIR/bootloader.bin" ]; then + echo "No bootloader.bin in $BUILD_DIR; skipping mergebin (non-ESP32 target)" + exit 0 +fi + +echo "ESP32 target detected ($PIO_ENV)" +cd "$FIRMWARE_ROOT" + +if pio run -e "$PIO_ENV" --list-targets 2>/dev/null | grep -qw mergebin; then + echo "Running mergebin -> firmware-merged.factory.bin" + export MERGED_BIN_PATH="$BUILD_DIR/firmware-merged.factory.bin" + pio run -t mergebin -e "$PIO_ENV" 2>&1 || \ + echo "WARNING: mergebin failed; merged factory may be missing" +else + echo "No mergebin target for $PIO_ENV; skipping (packaging uses app .bin or *.factory.bin if present)" +fi diff --git a/scripts/ci/common/normalize-esp32-merged-factory.sh b/scripts/ci/common/normalize-esp32-merged-factory.sh new file mode 100755 index 0000000..265ac12 --- /dev/null +++ b/scripts/ci/common/normalize-esp32-merged-factory.sh @@ -0,0 +1,56 @@ +#!/usr/bin/env bash +# After `pio run -t mergebin`, PlatformIO may write any *.factory.bin name (often tied to +# PROGNAME). Downstream steps expect a single merged image at firmware-merged.factory.bin. +# +# Usage: normalize-esp32-merged-factory.sh BUILD_DIR +set -euo pipefail + +BUILD_DIR="${1:?usage: normalize-esp32-merged-factory.sh BUILD_DIR}" + +if [ ! -f "$BUILD_DIR/bootloader.bin" ]; then + exit 0 +fi + +TARGET="$BUILD_DIR/firmware-merged.factory.bin" +if [ -f "$TARGET" ]; then + echo "ESP32 merged factory already present: $TARGET" + exit 0 +fi + +shopt -s nullglob +candidates=( "$BUILD_DIR"/*.factory.bin ) +shopt -u nullglob + +if [ "${#candidates[@]}" -eq 0 ]; then + echo "No *.factory.bin in $BUILD_DIR (mergebin may have failed or not apply to this env)" + exit 0 +fi + +pick_merged() { + local best="" f b + for f in "${candidates[@]}"; do + b=$(basename "$f" | tr '[:upper:]' '[:lower:]') + if [[ "$b" == *merged* ]]; then + echo "$f" + return 0 + fi + done + local largest="" maxz=-1 z + for f in "${candidates[@]}"; do + z=$(wc -c <"$f" | tr -d ' ') + if [ "$z" -gt "$maxz" ]; then + maxz=$z + largest=$f + fi + done + echo "$largest" +} + +chosen="$(pick_merged)" +if [ -z "$chosen" ] || [ ! -f "$chosen" ]; then + echo "Could not pick a merged factory binary" + exit 0 +fi + +cp -a "$chosen" "$TARGET" +echo "ESP32 merged factory normalized: $(basename "$chosen") -> $(basename "$TARGET")" diff --git a/scripts/ci/common/package-firmware-to-stage.sh b/scripts/ci/common/package-firmware-to-stage.sh new file mode 100755 index 0000000..adec492 --- /dev/null +++ b/scripts/ci/common/package-firmware-to-stage.sh @@ -0,0 +1,45 @@ +#!/usr/bin/env bash +# Copy flashable artifacts from a PIO build dir into STAGE and add bundle docs (CI package step). +# +# Usage: package-firmware-to-stage.sh FIRMWARE_ROOT BUILD_DIR STAGE_DIR +# FIRMWARE_ROOT = PlatformIO project root +# BUILD_DIR = absolute path to .pio/build/ +# STAGE_DIR = output directory (caller creates/clears as needed) +set -euo pipefail + +SCRIPT_DIR=$(cd "$(dirname "$0")" && pwd) +MESH_FORGE_ROOT=$(cd "$SCRIPT_DIR/../../.." && pwd) + +FIRMWARE_ROOT=${1:?usage: package-firmware-to-stage.sh FIRMWARE_ROOT BUILD_DIR STAGE_DIR} +BUILD_DIR=${2:?usage: package-firmware-to-stage.sh FIRMWARE_ROOT BUILD_DIR STAGE_DIR} +STAGE_DIR=${3:?usage: package-firmware-to-stage.sh FIRMWARE_ROOT BUILD_DIR STAGE_DIR} + +mkdir -p "$STAGE_DIR" + +shopt -s nullglob +if [ -f "$BUILD_DIR/bootloader.bin" ]; then + if [ -f "$BUILD_DIR/firmware-merged.factory.bin" ]; then + cp -a "$BUILD_DIR/firmware-merged.factory.bin" "$STAGE_DIR/" + else + fz=( "$BUILD_DIR"/*.factory.bin ) + if [ "${#fz[@]}" -gt 0 ]; then + for f in "${fz[@]}"; do cp -a "$f" "$STAGE_DIR/"; done + else + bash "$MESH_FORGE_ROOT/scripts/ci/common/copy-esp32-app-bin-for-bundle.sh" "$BUILD_DIR" "$STAGE_DIR" + fi + fi +else + for f in "$BUILD_DIR"/firmware.bin "$BUILD_DIR"/firmware.dat \ + "$BUILD_DIR"/*.uf2 "$BUILD_DIR"/*.hex; do + [ -f "$f" ] && cp -a "$f" "$STAGE_DIR/" + done + [ -f "$BUILD_DIR/manifest.json" ] && cp -a "$BUILD_DIR/manifest.json" "$STAGE_DIR/" +fi +shopt -u nullglob + +bash "$MESH_FORGE_ROOT/scripts/ci/common/stage-fw-bundle-docs.sh" "$FIRMWARE_ROOT" "$STAGE_DIR" + +if [ -z "$(find "$STAGE_DIR" -mindepth 1 -maxdepth 1 -type f -print -quit)" ]; then + echo "No firmware artifacts found in $BUILD_DIR" >&2 + exit 1 +fi diff --git a/scripts/stage-fw-bundle-docs.sh b/scripts/ci/common/stage-fw-bundle-docs.sh similarity index 100% rename from scripts/stage-fw-bundle-docs.sh rename to scripts/ci/common/stage-fw-bundle-docs.sh diff --git a/scripts/ci/common/stage-nrf-dfu.sh b/scripts/ci/common/stage-nrf-dfu.sh new file mode 100755 index 0000000..439a42b --- /dev/null +++ b/scripts/ci/common/stage-nrf-dfu.sh @@ -0,0 +1,41 @@ +#!/usr/bin/env bash +# Unpack / normalize Nordic DFU artifacts and fallbacks (same logic as custom_build firmware workflow). +# +# Usage: stage-nrf-dfu.sh BUILD_DIR +# BUILD_DIR = absolute path to .pio/build/ +set -euo pipefail + +SCRIPT_DIR=$(cd "$(dirname "$0")" && pwd) +MESH_FORGE_ROOT=$(cd "$SCRIPT_DIR/../../.." && pwd) + +BUILD_DIR=${1:?usage: stage-nrf-dfu.sh BUILD_DIR} + +bash "$MESH_FORGE_ROOT/scripts/ci/common/stage-nrf52-dfu-from-build.sh" "$BUILD_DIR" + +# firmware/ subdirectory fallback (local builds or alternate PIO layouts). +if [ -d "$BUILD_DIR/firmware" ] && ! ls "$BUILD_DIR"/*.dat >/dev/null 2>&1; then + echo "Found firmware/ subdirectory; staging nRF52 DFU files to build root" + shopt -s nullglob + for f in "$BUILD_DIR/firmware/"*.bin "$BUILD_DIR/firmware/"*.dat; do + [ -f "$f" ] && cp -a "$f" "$BUILD_DIR/" + done + shopt -u nullglob + [ -f "$BUILD_DIR/firmware/manifest.json" ] && cp -a "$BUILD_DIR/firmware/manifest.json" "$BUILD_DIR/" +fi + +# firmware.bin present but still no .dat — nRF52 only (never when bootloader.bin exists). +if [ -f "$BUILD_DIR/firmware.bin" ] && \ + ! [ -f "$BUILD_DIR/bootloader.bin" ] && \ + ! ls "$BUILD_DIR"/*.dat >/dev/null 2>&1; then + echo "firmware.bin found but no .dat (nRF52); generating DFU init packet" + adafruit-nrfutil dfu genpkg \ + --dev-type 0xFFFF \ + --dev-revision 0xFFFF \ + --application-version 0xFFFFFFFF \ + --sd-req 0xFFFE \ + --application "$BUILD_DIR/firmware.bin" \ + /tmp/nrf52-dfu-ci.zip + unzip -jo /tmp/nrf52-dfu-ci.zip "*.dat" -d "$BUILD_DIR/" + rm -f /tmp/nrf52-dfu-ci.zip + echo "DFU init packet written: $(ls "$BUILD_DIR"/*.dat 2>/dev/null || echo '(none)')" +fi diff --git a/scripts/stage-nrf52-dfu-from-build.sh b/scripts/ci/common/stage-nrf52-dfu-from-build.sh similarity index 72% rename from scripts/stage-nrf52-dfu-from-build.sh rename to scripts/ci/common/stage-nrf52-dfu-from-build.sh index 88c5eca..bd79efd 100755 --- a/scripts/stage-nrf52-dfu-from-build.sh +++ b/scripts/ci/common/stage-nrf52-dfu-from-build.sh @@ -10,8 +10,8 @@ set -euo pipefail BUILD_DIR="${1:?usage: stage-nrf52-dfu-from-build.sh BUILD_DIR}" zip_looks_like_dfu() { - # Heuristic: DFU package includes a .dat init packet (member path ends in .dat). - unzip -l "$1" 2>/dev/null | grep -qE '[.]dat$' + # Heuristic: DFU package includes a .dat init packet (allow trailing spaces on listing lines). + unzip -l "$1" 2>/dev/null | grep -qE '[.]dat[[:space:]]*$' } normalize_pair() { @@ -77,3 +77,39 @@ else fi fi fi + +# Standalone application .bin (no DFU zip / no package): map largest eligible .bin to +# firmware.bin so CI genpkg and bundle steps can run. Skip ESP32 and UF2-first layouts. +if [ -f "$BUILD_DIR/bootloader.bin" ]; then + exit 0 +fi +if [ -f "$BUILD_DIR/firmware.bin" ]; then + exit 0 +fi +if ls "$BUILD_DIR"/*.dat >/dev/null 2>&1; then + exit 0 +fi +shopt -s nullglob +uf2s=( "$BUILD_DIR"/*.uf2 ) +bins=( "$BUILD_DIR"/*.bin ) +shopt -u nullglob +if [ "${#uf2s[@]}" -gt 0 ]; then + exit 0 +fi +if [ "${#bins[@]}" -eq 0 ]; then + exit 0 +fi +best="" +bestz=-1 +for f in "${bins[@]}"; do + [ -f "$f" ] || continue + z=$(wc -c <"$f" | tr -d ' ') + if [ "$z" -gt "$bestz" ]; then + bestz=$z + best=$f + fi +done +if [ -n "$best" ]; then + echo "Promoting standalone application .bin -> firmware.bin: $(basename "$best")" + cp -a "$best" "$BUILD_DIR/firmware.bin" +fi diff --git a/scripts/ci/common/verify-fw-bundle-stage.sh b/scripts/ci/common/verify-fw-bundle-stage.sh new file mode 100755 index 0000000..a75570d --- /dev/null +++ b/scripts/ci/common/verify-fw-bundle-stage.sh @@ -0,0 +1,186 @@ +#!/usr/bin/env bash +# Fail if a firmware bundle (staging directory OR final .tar.gz/.tgz) is missing expected +# flashable artifacts or sizes look wrong. R2 uploads use the tarball — verify that file +# so docs-only or empty archives cannot slip through. +# +# Usage: verify-fw-bundle-stage.sh BUILD_DIR STAGE_DIR_OR_TARBALL +# BUILD_DIR = absolute path to .pio/build/ (ESP32 vs nRF52-class) +# STAGE_DIR_OR_TARBALL = directory passed to scripts/ci/common/package-firmware-to-stage.sh, OR path to +# the gzip tarball produced by: tar -czf out.tgz -C "$STAGE" . +set -euo pipefail + +BUILD_DIR=${1:?usage: verify-fw-bundle-stage.sh BUILD_DIR STAGE_DIR_OR_TARBALL} +ARTIFACT=${2:?usage: verify-fw-bundle-stage.sh BUILD_DIR STAGE_DIR_OR_TARBALL} + +if [ ! -d "$BUILD_DIR" ]; then + echo "BUILD_DIR is not a directory: $BUILD_DIR" >&2 + exit 1 +fi + +EXTRACT="" +cleanup() { + [ -n "$EXTRACT" ] && rm -rf "$EXTRACT" +} +trap cleanup EXIT + +if [ -f "$ARTIFACT" ]; then + case "$ARTIFACT" in + *.tar.gz|*.tgz) ;; + *) + echo "When second arg is a file, it must be .tar.gz or .tgz: $ARTIFACT" >&2 + exit 1 + ;; + esac + if ! tar -tzf "$ARTIFACT" >/dev/null 2>&1; then + echo "Not a readable gzip tarball: $ARTIFACT" >&2 + exit 1 + fi + arch_bytes=$(wc -c <"$ARTIFACT" | tr -d ' ') + if [ "$arch_bytes" -lt 2048 ]; then + echo "Tarball file too small ($arch_bytes bytes), likely corrupt or empty: $ARTIFACT" >&2 + exit 1 + fi + EXTRACT=$(mktemp -d "${TMPDIR:-/tmp}/fw-bundle-verify.XXXXXX") + tar -xzf "$ARTIFACT" -C "$EXTRACT" + STAGE_DIR="$EXTRACT" + echo "Verifying tarball ($(basename "$ARTIFACT"), $arch_bytes bytes) -> extracted under $STAGE_DIR" + members=$(tar -tzf "$ARTIFACT" | grep -c . || true) + if [ "${members:-0}" -lt 1 ]; then + echo "Tarball lists zero members: $ARTIFACT" >&2 + exit 1 + fi +elif [ -d "$ARTIFACT" ]; then + STAGE_DIR="$ARTIFACT" + echo "Verifying staged directory: $STAGE_DIR" +else + echo "Second arg must be a stage directory or a .tar.gz/.tgz file: $ARTIFACT" >&2 + exit 1 +fi + +bytes() { + wc -c <"$1" | tr -d ' ' +} + +largest_staged_bin() { + local best="" bestz=-1 f z + shopt -s nullglob + for f in "$STAGE_DIR"/*.bin; do + [ -f "$f" ] || continue + z=$(bytes "$f") + if [ "$z" -gt "$bestz" ]; then + bestz=$z + best=$f + fi + done + shopt -u nullglob + printf '%s' "$best" +} + +largest_staged_uf2() { + local best="" f z bestz=-1 + shopt -s nullglob + for f in "$STAGE_DIR"/*.uf2; do + [ -f "$f" ] || continue + z=$(bytes "$f") + if [ "$z" -gt "$bestz" ]; then + bestz=$z + best=$f + fi + done + shopt -u nullglob + printf '%s' "$best" +} + +largest_staged_hex() { + local best="" f z bestz=-1 + shopt -s nullglob + for f in "$STAGE_DIR"/*.hex; do + [ -f "$f" ] || continue + z=$(bytes "$f") + if [ "$z" -gt "$bestz" ]; then + bestz=$z + best=$f + fi + done + shopt -u nullglob + printf '%s' "$best" +} + +# Tunable minima (bytes) — catch empty/truncated outputs without being brittle across releases. +MIN_ESP32_BIN=$((300 * 1024)) +MIN_NRF52_APP=$((128 * 1024)) +MIN_NRF52_DAT=8 +MIN_MANIFEST=80 +MIN_UF2=$((200 * 1024)) +MIN_HEX=$((64 * 1024)) + +echo "Bundle check: BUILD_DIR=$BUILD_DIR" + +if [ -f "$BUILD_DIR/bootloader.bin" ]; then + echo "Bundle class: ESP32 (bootloader.bin in build tree)" + best=$(largest_staged_bin) + if [ -z "$best" ]; then + echo "ESP32 bundle: no *.bin under archive/stage (R2 tarball would be unusable for flash)" >&2 + ls -la "$STAGE_DIR" >&2 + exit 1 + fi + z=$(bytes "$best") + if [ "$z" -lt "$MIN_ESP32_BIN" ]; then + echo "ESP32 bundle: largest .bin too small ($z bytes < $MIN_ESP32_BIN): $best" >&2 + exit 1 + fi + echo "ESP32 bundle OK: primary image $(basename "$best") size=$z bytes" + exit 0 +fi + +echo "Bundle class: non-ESP32 (expect nRF52-style DFU, UF2, or HEX in archive/stage)" + +if [ -f "$STAGE_DIR/firmware.bin" ] && [ -f "$STAGE_DIR/firmware.dat" ]; then + zb=$(bytes "$STAGE_DIR/firmware.bin") + zd=$(bytes "$STAGE_DIR/firmware.dat") + if [ "$zb" -lt "$MIN_NRF52_APP" ]; then + echo "nRF52 bundle: firmware.bin too small ($zb < $MIN_NRF52_APP)" >&2 + exit 1 + fi + if [ "$zd" -lt "$MIN_NRF52_DAT" ]; then + echo "nRF52 bundle: firmware.dat too small ($zd < $MIN_NRF52_DAT)" >&2 + exit 1 + fi + if [ ! -f "$STAGE_DIR/manifest.json" ]; then + echo "nRF52 bundle: missing manifest.json" >&2 + exit 1 + fi + zm=$(bytes "$STAGE_DIR/manifest.json") + if [ "$zm" -lt "$MIN_MANIFEST" ]; then + echo "nRF52 bundle: manifest.json too small ($zm < $MIN_MANIFEST)" >&2 + exit 1 + fi + echo "nRF52 DFU bundle OK: firmware.bin=$zb firmware.dat=$zd manifest.json=$zm bytes" + exit 0 +fi + +uf2=$(largest_staged_uf2) +if [ -n "$uf2" ]; then + zu=$(bytes "$uf2") + if [ "$zu" -lt "$MIN_UF2" ]; then + echo "UF2 bundle: $(basename "$uf2") too small ($zu < $MIN_UF2)" >&2 + exit 1 + fi + echo "UF2 bundle OK: $(basename "$uf2") size=$zu bytes" + exit 0 +fi + +hex=$(largest_staged_hex) +if [ -n "$hex" ]; then + zh=$(bytes "$hex") + if [ "$zh" -lt "$MIN_HEX" ]; then + echo "HEX bundle: $(basename "$hex") too small ($zh < $MIN_HEX)" >&2 + exit 1 + fi + echo "HEX-only bundle OK: $(basename "$hex") size=$zh bytes" + exit 0 +fi + +echo "No recognized firmware payload in bundle (expected ESP32 .bin, or nRF52 firmware.bin+.dat+manifest, or UF2/HEX)" >&2 +ls -la "$STAGE_DIR" >&2 +exit 1 diff --git a/scripts/report-convex-ci-progress.py b/scripts/ci/report-convex-ci-progress.py similarity index 100% rename from scripts/report-convex-ci-progress.py rename to scripts/ci/report-convex-ci-progress.py diff --git a/scripts/ci/test/fw-pio-smoke-envs.json b/scripts/ci/test/fw-pio-smoke-envs.json new file mode 100644 index 0000000..e566925 --- /dev/null +++ b/scripts/ci/test/fw-pio-smoke-envs.json @@ -0,0 +1,35 @@ +{ + "description": "PlatformIO environment names for Heltec WiFi LoRa 32 V3 and RAK WisBlock 4631, per vendored tree. Names taken from PIO UnknownEnvNamesError listings (pio run -e __invalid__).", + "projects": [ + { + "path": "vendor/microReticulum-Firmware", + "heltec_v3": "heltec_wifi_lora_32_V3", + "rak_4631": "wiscore_rak4631" + }, + { + "path": "vendor/meshtastic-firmware", + "heltec_v3": "heltec-v3", + "rak_4631": "rak4631" + }, + { + "path": "vendor/lobbs-meshtastic-firmware", + "heltec_v3": "heltec-v3", + "rak_4631": "rak4631" + }, + { + "path": "vendor/meshtastic-baymesh", + "heltec_v3": "heltec-v3", + "rak_4631": "rak4631" + }, + { + "path": "vendor/meshcore-firmware", + "heltec_v3": "Heltec_v3_companion_radio_ble", + "rak_4631": "RAK_4631_companion_radio_ble" + }, + { + "path": "vendor/meshcore-lotato", + "heltec_v3": "Heltec_v3_companion_radio_ble", + "rak_4631": "RAK_4631_companion_radio_ble" + } + ] +} diff --git a/scripts/ci/test/run-fw-pio-smoke-local.sh b/scripts/ci/test/run-fw-pio-smoke-local.sh new file mode 100755 index 0000000..c095e4f --- /dev/null +++ b/scripts/ci/test/run-fw-pio-smoke-local.sh @@ -0,0 +1,72 @@ +#!/usr/bin/env bash +# End-to-end local smoke for flashable gzip tarballs (same layout as CI → R2): +# 1) pio run -e -e per vendored project (test/fw-pio-smoke-envs.json) +# 2) Per env: test/test-pio-fw-staging-local.sh → common/fw-bundle-pipeline.sh (MESHTASTIC_OTA=0; CI uses 1) +# Meshtastic OTA download/extend is skipped locally (set MESHTASTIC_OTA=1 to match Actions exactly). +# +# Requires: jq, platformio, bash +# Usage (from mesh-forge repo root): +# ./scripts/ci/test/run-fw-pio-smoke-local.sh [path/to/fw-pio-smoke-envs.json] +set -euo pipefail + +SCRIPT_DIR=$(cd "$(dirname "$0")" && pwd) +MESH_FORGE_ROOT=$(cd "$SCRIPT_DIR/../../.." && pwd) +CONFIG_JSON=${1:-"$SCRIPT_DIR/fw-pio-smoke-envs.json"} + +if ! command -v jq >/dev/null 2>&1; then + echo "jq is required to read $CONFIG_JSON" >&2 + exit 1 +fi + +if [ ! -f "$CONFIG_JSON" ]; then + echo "Config not found: $CONFIG_JSON" >&2 + exit 1 +fi + +# Same helper as CI for LittleFS-heavy Meshtastic-family builds. +MKLITTLEFS_BIN=$(find "${HOME}/.platformio/packages" -type f -name mklittlefs 2>/dev/null | head -1) +if [ -n "${MKLITTLEFS_BIN:-}" ]; then + export PATH + PATH="$(dirname "$MKLITTLEFS_BIN"):$PATH" +fi +command -v mklittlefs >/dev/null 2>&1 || echo "WARNING: mklittlefs not on PATH; some envs may fail to build" >&2 + +echo "== MeshForge firmware smoke: full PIO build + flashable .tar.gz verification ==" +echo "MESH_FORGE_ROOT=$MESH_FORGE_ROOT" +echo "CONFIG_JSON=$CONFIG_JSON" +echo "" + +N=$(jq '.projects | length' "$CONFIG_JSON") +i=0 +while [ "$i" -lt "$N" ]; do + rel=$(jq -r --argjson idx "$i" '.projects[$idx].path' "$CONFIG_JSON") + heltec=$(jq -r --argjson idx "$i" '.projects[$idx].heltec_v3' "$CONFIG_JSON") + rak=$(jq -r --argjson idx "$i" '.projects[$idx].rak_4631' "$CONFIG_JSON") + i=$((i + 1)) + + proj="$MESH_FORGE_ROOT/$rel" + if [ ! -f "$proj/platformio.ini" ]; then + echo "SKIP (no platformio.ini): $rel" >&2 + continue + fi + + echo "-------------------------------------------------------------------" + echo "[$i/$N] PROJECT $rel — phase: PlatformIO compile" + echo " pio run -e $heltec -e $rak" + echo "-------------------------------------------------------------------" + (cd "$proj" && pio run -e "$heltec" -e "$rak") + + for env in "$heltec" "$rak"; do + build_dir="$proj/.pio/build/$env" + if [ ! -d "$build_dir" ]; then + echo "Missing build dir: $build_dir" >&2 + exit 1 + fi + echo "" + echo "--- Phase: CI package + gzip tarball verify (R2-shaped artifact): $rel / $env ---" + bash "$MESH_FORGE_ROOT/scripts/ci/test/test-pio-fw-staging-local.sh" "$build_dir" + echo "" + done +done + +echo "OK: end-to-end smoke finished — every env built with PIO and passed flashable .tar.gz checks." diff --git a/scripts/ci/test/test-pio-fw-staging-local.sh b/scripts/ci/test/test-pio-fw-staging-local.sh new file mode 100755 index 0000000..2286d7f --- /dev/null +++ b/scripts/ci/test/test-pio-fw-staging-local.sh @@ -0,0 +1,38 @@ +#!/usr/bin/env bash +# Local parity with CI firmware bundle: runs common/fw-bundle-pipeline.sh (no Meshtastic OTA). +# Same mergebin → normalize → DFU stage → package → tar → verify path as Actions. +# +# Usage (from repo root): +# ./scripts/ci/test/test-pio-fw-staging-local.sh /path/to/.pio/build/ENV +set -euo pipefail + +MESH_FORGE_ROOT=$(cd "$(dirname "$0")/../../.." && pwd) +BUILD_DIR_INPUT=${1:?usage: $0 /path/to/.pio/build/ENV} + +if [ ! -d "$BUILD_DIR_INPUT" ]; then + echo "Not a directory: $BUILD_DIR_INPUT" >&2 + exit 1 +fi + +BUILD_DIR=$(cd "$BUILD_DIR_INPUT" && pwd) +FIRMWARE_ROOT=$(cd "$BUILD_DIR/../../.." && pwd) +PIO_ENV=$(basename "$BUILD_DIR") + +TAR_LOCAL=/tmp/fw-bundle-local-test.tar.gz + +echo "== MeshForge local firmware bundle (fw-bundle-pipeline.sh, no OTA) ==" +echo "FIRMWARE_ROOT=$FIRMWARE_ROOT" +echo "BUILD_DIR=$BUILD_DIR" +echo "PIO_ENV=$PIO_ENV" +echo "" + +FW_BUNDLE_VERBOSE=1 MESHTASTIC_OTA=0 bash "$MESH_FORGE_ROOT/scripts/ci/common/fw-bundle-pipeline.sh" \ + "$FIRMWARE_ROOT" "$BUILD_DIR" "$TAR_LOCAL" + +echo "" +echo "--- Tarball listing ---" +tar -tzf "$TAR_LOCAL" +rm -f "$TAR_LOCAL" + +echo "" +echo "OK: fw-bundle-pipeline.sh produced a verified tarball (same core path as CI)."