From dfea7a358d12e5c257b6b1306a00f1eed098a324 Mon Sep 17 00:00:00 2001 From: Ben Allfree Date: Sun, 23 Nov 2025 03:50:22 -0800 Subject: [PATCH] ci: artifact uploading to r2 --- .github/workflows/custom_build.yml | 36 ++++- convex/actions.ts | 2 + convex/builds.ts | 227 ++++++++++++++++++++++++++--- convex/http.ts | 1 + convex/schema.ts | 9 ++ 5 files changed, 247 insertions(+), 28 deletions(-) diff --git a/.github/workflows/custom_build.yml b/.github/workflows/custom_build.yml index ae5aeb5..ec19430 100644 --- a/.github/workflows/custom_build.yml +++ b/.github/workflows/custom_build.yml @@ -18,6 +18,10 @@ on: description: 'Convex Build ID' required: true type: string + build_hash: + description: 'Build hash for artifact naming' + required: true + type: string convex_url: description: 'Convex Site URL' required: true @@ -110,18 +114,36 @@ jobs: -H "Content-Type: application/json" \ -d '{"buildId": "${{ inputs.build_id }}", "logs": "✅ Build completed successfully!\n"}' - - name: Upload Artifact - uses: actions/upload-artifact@v4 - with: - name: firmware-${{ inputs.target }} - path: firmware/.pio/build/${{ inputs.target }}/firmware.bin + - name: Install AWS CLI (for R2) + if: success() + run: | + pip install awscli - - name: Log - Artifact uploaded + - name: Upload to R2 + if: success() + env: + AWS_ACCESS_KEY_ID: ${{ secrets.R2_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.R2_SECRET_ACCESS_KEY }} + AWS_ENDPOINT_URL: ${{ secrets.R2_ENDPOINT_URL }} + run: | + # Determine file extension based on target (most are .bin, some might be .uf2) + BUILD_FILE="firmware/.pio/build/${{ inputs.target }}/firmware.bin" + if [ ! -f "$BUILD_FILE" ]; then + BUILD_FILE="firmware/.pio/build/${{ inputs.target }}/firmware.uf2" + fi + + # Upload to R2 with hash as filename + aws s3 cp "$BUILD_FILE" "s3://${{ secrets.R2_BUCKET_NAME }}/${{ inputs.build_hash }}.uf2" \ + --endpoint-url "$AWS_ENDPOINT_URL" + + echo "✅ Uploaded to R2: ${{ inputs.build_hash }}.uf2" + + - name: Log - Artifact uploaded to R2 if: success() run: | curl -X POST "${{ inputs.convex_url }}/api/logs" \ -H "Content-Type: application/json" \ - -d '{"buildId": "${{ inputs.build_id }}", "logs": "📤 Artifact uploaded: firmware-${{ inputs.target }}\n"}' + -d '{"buildId": "${{ inputs.build_id }}", "logs": "📤 Artifact uploaded to R2: ${{ inputs.build_hash }}.uf2\n"}' - name: Log - Build failed if: failure() diff --git a/convex/actions.ts b/convex/actions.ts index c2033b5..a6a57df 100644 --- a/convex/actions.ts +++ b/convex/actions.ts @@ -8,6 +8,7 @@ export const dispatchGithubBuild = action({ target: v.string(), flags: v.string(), version: v.string(), + buildHash: v.string(), }, handler: async (ctx, args) => { const githubToken = process.env.GITHUB_TOKEN; @@ -32,6 +33,7 @@ export const dispatchGithubBuild = action({ flags: args.flags, version: args.version, build_id: args.buildId, + build_hash: args.buildHash, convex_url: process.env.CONVEX_SITE_URL, }, }), diff --git a/convex/builds.ts b/convex/builds.ts index 9b5040a..f975a55 100644 --- a/convex/builds.ts +++ b/convex/builds.ts @@ -3,6 +3,84 @@ import { v } from "convex/values"; import { api, } from "./_generated/api"; import { internalMutation, mutation, query } from "./_generated/server"; +/** + * Normalizes a config object to a stable JSON string for hashing. + * Sorts keys and handles values consistently. + */ +function normalizeConfig(config: any): string { + const normalized: Record = {}; + + // Sort keys and process values + const sortedKeys = Object.keys(config || {}).sort(); + + for (const key of sortedKeys) { + const value = config[key]; + // Only include non-null, non-undefined values + if (value !== null && value !== undefined) { + // Normalize boolean, number, and string values + if (typeof value === "boolean") { + normalized[key] = value; + } else if (typeof value === "number") { + normalized[key] = value; + } else if (typeof value === "string") { + // Only include non-empty strings + if (value.trim() !== "") { + normalized[key] = value.trim(); + } + } + } + } + + return JSON.stringify(normalized); +} + +/** + * Computes a stable SHA-256 hash from version, target, and config. + * This hash uniquely identifies a build configuration. + */ +async function computeBuildHash( + version: string, + target: string, + config: any, +): Promise { + const normalizedConfig = normalizeConfig(config); + const input = JSON.stringify({ + version, + target, + config: normalizedConfig, + }); + + // Use Web Crypto API for SHA-256 hashing + const encoder = new TextEncoder(); + const data = encoder.encode(input); + const hashBuffer = await crypto.subtle.digest("SHA-256", data); + const hashArray = Array.from(new Uint8Array(hashBuffer)); + const hashHex = hashArray.map((b) => b.toString(16).padStart(2, "0")).join(""); + + return hashHex; +} + +/** + * Constructs the R2 artifact URL from a build hash. + * Uses the R2 public URL pattern: https://..r2.cloudflarestorage.com/.uf2 + * Or custom domain if R2_PUBLIC_URL is set. + */ +function getR2ArtifactUrl(buildHash: string): string { + const r2PublicUrl = process.env.R2_PUBLIC_URL; + if (r2PublicUrl) { + // Custom domain configured + return `${r2PublicUrl}/${buildHash}.uf2`; + } + // Default R2 public URL pattern (requires public bucket) + const bucketName = process.env.R2_BUCKET_NAME || "firmware-builds"; + const accountId = process.env.R2_ACCOUNT_ID || ""; + if (accountId) { + return `https://${bucketName}.${accountId}.r2.cloudflarestorage.com/${buildHash}.uf2`; + } + // Fallback: assume custom domain or public bucket URL + return `https://${bucketName}.r2.cloudflarestorage.com/${buildHash}.uf2`; +} + export const triggerBuild = mutation({ args: { profileId: v.id("profiles"), @@ -18,28 +96,68 @@ export const triggerBuild = mutation({ // Convert config object to flags string const flags = Object.entries(profile.config) - .filter(([_, value]) => value === true) - .map(([key, _]) => `-D${key}`) + .map(([key, value]) => { + if (value === true) return `-D${key}`; + if (typeof value === "number") return `-D${key}=${value}`; + if (typeof value === "string" && value.trim() !== "") + return `-D${key}=${value}`; + return null; + }) + .filter(Boolean) .join(" "); // Create build records for each target for (const target of profile.targets) { - const buildId = await ctx.db.insert("builds", { - profileId: profile._id, - target: target, - githubRunId: 0, - status: "queued", - logs: "Build queued...", - startedAt: Date.now(), - }); + // Compute build hash + const buildHash = await computeBuildHash( + profile.version, + target, + profile.config, + ); - // Schedule the action to dispatch GitHub workflow - await ctx.scheduler.runAfter(0, api.actions.dispatchGithubBuild, { - buildId: buildId, - target: target, - flags: flags, - version: profile.version, - }); + // Check cache for existing build + const cached = await ctx.db + .query("buildCache") + .withIndex("by_hash_target", (q) => + q.eq("buildHash", buildHash).eq("target", target), + ) + .first(); + + if (cached) { + // Use cached artifact, skip GitHub workflow + const artifactUrl = getR2ArtifactUrl(buildHash); + const buildId = await ctx.db.insert("builds", { + profileId: profile._id, + target: target, + githubRunId: 0, + status: "success", + artifactUrl: artifactUrl, + logs: "Build completed from cache", + startedAt: Date.now(), + completedAt: Date.now(), + buildHash: buildHash, + }); + } else { + // Not cached, proceed with normal build flow + const buildId = await ctx.db.insert("builds", { + profileId: profile._id, + target: target, + githubRunId: 0, + status: "queued", + logs: "Build queued...", + startedAt: Date.now(), + buildHash: buildHash, + }); + + // Schedule the action to dispatch GitHub workflow + await ctx.scheduler.runAfter(0, api.actions.dispatchGithubBuild, { + buildId: buildId, + target: target, + flags: flags, + version: profile.version, + buildHash: buildHash, + }); + } } }, }); @@ -71,6 +189,14 @@ export const get = query({ }, }); +// Internal query to get build without auth checks (for webhooks) +export const getInternal = internalMutation({ + args: { buildId: v.id("builds") }, + handler: async (ctx, args) => { + return await ctx.db.get(args.buildId); + }, +}); + export const deleteBuild = mutation({ args: { buildId: v.id("builds") }, handler: async (ctx, args) => { @@ -113,15 +239,29 @@ export const retryBuild = mutation({ // Retry the build const flags = Object.entries(profile.config) - .filter(([_, value]) => value === true) - .map(([key, _]) => `-D${key}`) + .map(([key, value]) => { + if (value === true) return `-D${key}`; + if (typeof value === "number") return `-D${key}=${value}`; + if (typeof value === "string" && value.trim() !== "") + return `-D${key}=${value}`; + return null; + }) + .filter(Boolean) .join(" "); + // Compute build hash for retry + const buildHash = await computeBuildHash( + profile.version, + build.target, + profile.config, + ); + await ctx.scheduler.runAfter(0, api.actions.dispatchGithubBuild, { buildId: args.buildId, target: build.target, flags: flags, version: profile.version, + buildHash: buildHash, }); }, }); @@ -162,11 +302,56 @@ export const updateBuildStatus = internalMutation({ args: { buildId: v.id("builds"), status: v.union(v.literal("success"), v.literal("failure")), + artifactUrl: v.optional(v.string()), }, handler: async (ctx, args) => { - await ctx.db.patch(args.buildId, { + const build = await ctx.db.get(args.buildId); + if (!build) return; + + const updateData: any = { status: args.status, completedAt: Date.now(), - }); + }; + + if (args.artifactUrl) { + updateData.artifactUrl = args.artifactUrl; + } + + await ctx.db.patch(args.buildId, updateData); + + // If build succeeded and we have buildHash, store in cache with R2 URL + if (args.status === "success" && build.buildHash && build.target) { + // Get version from profile + const profile = await ctx.db.get(build.profileId); + if (profile) { + // Construct R2 URL from hash + const artifactUrl = getR2ArtifactUrl(build.buildHash); + + // Update build with R2 URL if not already set + if (!args.artifactUrl) { + updateData.artifactUrl = artifactUrl; + await ctx.db.patch(args.buildId, { artifactUrl }); + } + + // Check if cache entry already exists + const existing = await ctx.db + .query("buildCache") + .withIndex("by_hash_target", (q) => + q.eq("buildHash", build.buildHash!).eq("target", build.target), + ) + .first(); + + if (!existing) { + // Store in cache + await ctx.db.insert("buildCache", { + buildHash: build.buildHash, + target: build.target, + artifactUrl: artifactUrl, + version: profile.version, + createdAt: Date.now(), + }); + } + } + } }, }); diff --git a/convex/http.ts b/convex/http.ts index f1eeeec..59d08b3 100644 --- a/convex/http.ts +++ b/convex/http.ts @@ -19,6 +19,7 @@ http.route({ if (payload.action === "completed" && payload.build_id) { const status = payload.status === "success" ? "success" : "failure"; + // Update build status - R2 URL will be constructed automatically from buildHash await ctx.runMutation(internal.builds.updateBuildStatus, { buildId: payload.build_id, status, diff --git a/convex/schema.ts b/convex/schema.ts index 3a55360..104e2d7 100644 --- a/convex/schema.ts +++ b/convex/schema.ts @@ -22,5 +22,14 @@ export default defineSchema({ logs: v.optional(v.string()), startedAt: v.number(), completedAt: v.optional(v.number()), + buildHash: v.optional(v.string()), }).index("by_profile", ["profileId"]), + + buildCache: defineTable({ + buildHash: v.string(), + target: v.string(), + artifactUrl: v.string(), + version: v.string(), + createdAt: v.number(), + }).index("by_hash_target", ["buildHash", "target"]), });