From 4fb5c8e62e59d16759e011d0233dc3189b1eafe0 Mon Sep 17 00:00:00 2001 From: Ben Allfree Date: Thu, 23 Apr 2026 15:53:57 -0700 Subject: [PATCH] chore: enhance GitHub Actions workflow to support optional platform root for monorepo projects and update related functions for platform-specific handling --- .github/workflows/custom_build.yml | 39 ++- bun.lock | 2 +- convex/_generated/ai/ai-files.state.json | 5 +- convex/_generated/ai/guidelines.md | 175 ++++++++------ convex/_generated/api.d.ts | 6 + convex/actions.ts | 1 + convex/lib/githubRepoScan.ts | 290 +++++++++++++++++++++++ convex/lib/githubSubmodule.ts | 44 ++++ convex/lib/meshforgeYaml.ts | 200 ++++++++++------ convex/lib/platformioScan.ts | 27 +-- convex/repoBuilds.ts | 12 +- convex/repoScanGitAction.ts | 108 +++++++++ convex/repoScans.ts | 147 ++++++------ convex/repoTags.ts | 2 +- convex/schema.ts | 10 +- samples/projects/lotato | 2 +- src/lib/buildKey.ts | 12 +- src/lib/repoTreeUrl.ts | 44 +++- src/pages/RepoPage.tsx | 273 +++++++++++++++++---- 19 files changed, 1101 insertions(+), 298 deletions(-) create mode 100644 convex/lib/githubRepoScan.ts create mode 100644 convex/lib/githubSubmodule.ts create mode 100644 convex/repoScanGitAction.ts diff --git a/.github/workflows/custom_build.yml b/.github/workflows/custom_build.yml index 37a3d5b..be60ad2 100644 --- a/.github/workflows/custom_build.yml +++ b/.github/workflows/custom_build.yml @@ -27,6 +27,11 @@ on: convex_url: required: true type: string + platform_root: + description: Monorepo subdirectory (git submodule path) containing the PlatformIO project; empty = zip at repo root + required: false + type: string + default: "" jobs: build: @@ -36,7 +41,7 @@ jobs: REPO_BUILD_ID: ${{ inputs.repo_build_id }} CONVEX_BUILD_TOKEN: ${{ secrets.CONVEX_BUILD_TOKEN }} CI_PROGRESS_TOTAL: "10" - PLATFORMIO_LIBDEPS_DIR: ${{ github.workspace }}/.pio-libdeps/${{ inputs.owner }}/${{ inputs.repo }}/${{ inputs.target_env }} + PLATFORMIO_LIBDEPS_DIR: ${{ github.workspace }}/.pio-libdeps/${{ inputs.owner }}/${{ inputs.repo }}/${{ inputs.platform_root == '' && '__root__' || inputs.platform_root }}/${{ inputs.target_env }} steps: - uses: actions/checkout@v4 @@ -73,6 +78,7 @@ jobs: run: python3 "${{ github.workspace }}/scripts/ci/report-convex-ci-progress.py" - name: Download source archive + if: ${{ inputs.platform_root == '' }} shell: bash env: OWNER: ${{ inputs.owner }} @@ -83,6 +89,32 @@ jobs: ENC_REF=$(python3 -c "import urllib.parse,os; print(urllib.parse.quote(os.environ['REF'], safe=''))") curl -fsSL -o /tmp/src.zip "https://codeload.github.com/${OWNER}/${REPO}/zip/${ENC_REF}" + - name: Clone monorepo with submodules + if: ${{ inputs.platform_root != '' }} + shell: bash + env: + OWNER: ${{ inputs.owner }} + REPO: ${{ inputs.repo }} + REF: ${{ inputs.ref }} + PLATFORM_ROOT: ${{ inputs.platform_root }} + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + STEP_INDEX=3 LABEL="Cloning monorepo with submodules" python3 "${{ github.workspace }}/scripts/ci/report-convex-ci-progress.py" + set -e + sudo apt-get update -qq && sudo apt-get install -y -qq git + rm -rf /tmp/fw-checkout + git clone "https://x-access-token:${GITHUB_TOKEN}@github.com/${OWNER}/${REPO}.git" /tmp/fw-checkout + cd /tmp/fw-checkout + git checkout "${REF}" + git submodule update --init --recursive + SUB="$(pwd)/${PLATFORM_ROOT}" + if [ ! -d "$SUB" ]; then + echo "platform_root directory missing: ${PLATFORM_ROOT}" + exit 1 + fi + printf '%s\n' "$SUB" > /tmp/fw-src-root.txt + mkdir -p "$PLATFORMIO_LIBDEPS_DIR" + - name: CI progress — restoring PlatformIO global cache shell: bash env: @@ -111,10 +143,11 @@ jobs: - name: Cache PlatformIO libdeps (per repo + target) uses: actions/cache@v4 with: - path: ${{ github.workspace }}/.pio-libdeps/${{ inputs.owner }}/${{ inputs.repo }}/${{ inputs.target_env }} - key: pio-libdeps-${{ inputs.owner }}-${{ inputs.repo }}-${{ inputs.target_env }}-v1 + path: ${{ github.workspace }}/.pio-libdeps/${{ inputs.owner }}/${{ inputs.repo }}/${{ inputs.platform_root == '' && '__root__' || inputs.platform_root }}/${{ inputs.target_env }} + key: pio-libdeps-${{ inputs.owner }}-${{ inputs.repo }}-${{ inputs.platform_root == '' && 'root' || inputs.platform_root }}-${{ inputs.target_env }}-v1 - name: Extract firmware source + if: ${{ inputs.platform_root == '' }} shell: bash run: | STEP_INDEX=6 LABEL="Extracting firmware source" python3 "${{ github.workspace }}/scripts/ci/report-convex-ci-progress.py" diff --git a/bun.lock b/bun.lock index 0df0d49..9d6dd0a 100644 --- a/bun.lock +++ b/bun.lock @@ -22,6 +22,7 @@ "rehype-sanitize": "^6.0.0", "semver": "^7.7.4", "sonner": "^2.0.7", + "yaml": "^2.8.3", }, "devDependencies": { "@mdx-js/rollup": "^3.1.1", @@ -48,7 +49,6 @@ "typescript": "^5.9.3", "vite": "^7.2.6", "wrangler": "^4.51.0", - "yaml": "^2.8.3", }, }, }, diff --git a/convex/_generated/ai/ai-files.state.json b/convex/_generated/ai/ai-files.state.json index a8f6e5f..a65f4f2 100644 --- a/convex/_generated/ai/ai-files.state.json +++ b/convex/_generated/ai/ai-files.state.json @@ -1,9 +1,10 @@ { - "guidelinesHash": "294b619f8246c26bd6bfb6a57122503f0e2149872fc6b26609b7a95bfefaf2b8", + "guidelinesHash": "62d72acb9afcc18f658d88dd772f34b5b1da5fa60ef0402e57a784d97c458e57", "agentsMdSectionHash": "bbf30bd25ceea0aefd279d62e1cb2b4c207fcb712b69adf26f3d02b296ffc7b2", "claudeMdHash": "bbf30bd25ceea0aefd279d62e1cb2b4c207fcb712b69adf26f3d02b296ffc7b2", - "agentSkillsSha": "dc8ff761cfe4da450af2ea8a9ec708f737064bed", + "agentSkillsSha": "231a67aa8a5b29cc2794cbc8298335a71aaa6d0e", "installedSkillNames": [ + "convex", "convex-create-component", "convex-migration-helper", "convex-performance-audit", diff --git a/convex/_generated/ai/guidelines.md b/convex/_generated/ai/guidelines.md index 151cdf7..e41bedd 100644 --- a/convex/_generated/ai/guidelines.md +++ b/convex/_generated/ai/guidelines.md @@ -1,78 +1,90 @@ # Convex guidelines + ## Function guidelines + ### Http endpoint syntax + - HTTP endpoints are defined in `convex/http.ts` and require an `httpAction` decorator. For example: + ```typescript import { httpRouter } from "convex/server"; import { httpAction } from "./_generated/server"; const http = httpRouter(); http.route({ - path: "/echo", - method: "POST", - handler: httpAction(async (ctx, req) => { + path: "/echo", + method: "POST", + handler: httpAction(async (ctx, req) => { const body = await req.bytes(); return new Response(body, { status: 200 }); - }), + }), }); ``` + - HTTP endpoints are always registered at the exact path you specify in the `path` field. For example, if you specify `/api/someRoute`, the endpoint will be registered at `/api/someRoute`. ### Validators + - Below is an example of an array validator: + ```typescript import { mutation } from "./_generated/server"; import { v } from "convex/values"; export default mutation({ -args: { + args: { simpleArray: v.array(v.union(v.string(), v.number())), -}, -handler: async (ctx, args) => { + }, + handler: async (ctx, args) => { //... -}, + }, }); ``` + - Below is an example of a schema with validators that codify a discriminated union type: + ```typescript import { defineSchema, defineTable } from "convex/server"; import { v } from "convex/values"; export default defineSchema({ - results: defineTable( - v.union( - v.object({ - kind: v.literal("error"), - errorMessage: v.string(), - }), - v.object({ - kind: v.literal("success"), - value: v.number(), - }), - ), - ) + results: defineTable( + v.union( + v.object({ + kind: v.literal("error"), + errorMessage: v.string(), + }), + v.object({ + kind: v.literal("success"), + value: v.number(), + }), + ), + ), }); ``` + - Here are the valid Convex types along with their respective validators: -Convex Type | TS/JS type | Example Usage | Validator for argument validation and schemas | Notes | -| ----------- | ------------| -----------------------| -----------------------------------------------| ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| Id | string | `doc._id` | `v.id(tableName)` | | -| Null | null | `null` | `v.null()` | JavaScript's `undefined` is not a valid Convex value. Functions the return `undefined` or do not return will return `null` when called from a client. Use `null` instead. | -| Int64 | bigint | `3n` | `v.int64()` | Int64s only support BigInts between -2^63 and 2^63-1. Convex supports `bigint`s in most modern browsers. | -| Float64 | number | `3.1` | `v.number()` | Convex supports all IEEE-754 double-precision floating point numbers (such as NaNs). Inf and NaN are JSON serialized as strings. | -| Boolean | boolean | `true` | `v.boolean()` | -| String | string | `"abc"` | `v.string()` | Strings are stored as UTF-8 and must be valid Unicode sequences. Strings must be smaller than the 1MB total size limit when encoded as UTF-8. | -| Bytes | ArrayBuffer | `new ArrayBuffer(8)` | `v.bytes()` | Convex supports first class bytestrings, passed in as `ArrayBuffer`s. Bytestrings must be smaller than the 1MB total size limit for Convex types. | -| Array | Array | `[1, 3.2, "abc"]` | `v.array(values)` | Arrays can have at most 8192 values. | -| Object | Object | `{a: "abc"}` | `v.object({property: value})` | Convex only supports "plain old JavaScript objects" (objects that do not have a custom prototype). Objects can have at most 1024 entries. Field names must be nonempty and not start with "$" or "_". | -| Record | Record | `{"a": "1", "b": "2"}` | `v.record(keys, values)` | Records are objects at runtime, but can have dynamic keys. Keys must be only ASCII characters, nonempty, and not start with "$" or "_". | + Convex Type | TS/JS type | Example Usage | Validator for argument validation and schemas | Notes | + | ----------- | ------------| -----------------------| -----------------------------------------------| ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| + | Id | string | `doc._id` | `v.id(tableName)` | | + | Null | null | `null` | `v.null()` | JavaScript's `undefined` is not a valid Convex value. Functions the return `undefined` or do not return will return `null` when called from a client. Use `null` instead. | + | Int64 | bigint | `3n` | `v.int64()` | Int64s only support BigInts between -2^63 and 2^63-1. Convex supports `bigint`s in most modern browsers. | + | Float64 | number | `3.1` | `v.number()` | Convex supports all IEEE-754 double-precision floating point numbers (such as NaNs). Inf and NaN are JSON serialized as strings. | + | Boolean | boolean | `true` | `v.boolean()` | + | String | string | `"abc"` | `v.string()` | Strings are stored as UTF-8 and must be valid Unicode sequences. Strings must be smaller than the 1MB total size limit when encoded as UTF-8. | + | Bytes | ArrayBuffer | `new ArrayBuffer(8)` | `v.bytes()` | Convex supports first class bytestrings, passed in as `ArrayBuffer`s. Bytestrings must be smaller than the 1MB total size limit for Convex types. | + | Array | Array | `[1, 3.2, "abc"]` | `v.array(values)` | Arrays can have at most 8192 values. | + | Object | Object | `{a: "abc"}` | `v.object({property: value})` | Convex only supports "plain old JavaScript objects" (objects that do not have a custom prototype). Objects can have at most 1024 entries. Field names must be nonempty and not start with "$" or "_". | +| Record | Record | `{"a": "1", "b": "2"}` | `v.record(keys, values)` | Records are objects at runtime, but can have dynamic keys. Keys must be only ASCII characters, nonempty, and not start with "$" or "\_". | ### Function registration + - Use `internalQuery`, `internalMutation`, and `internalAction` to register internal functions. These functions are private and aren't part of an app's API. They can only be called by other Convex functions. These functions are always imported from `./_generated/server`. - Use `query`, `mutation`, and `action` to register public functions. These functions are part of the public API and are exposed to the public Internet. Do NOT use `query`, `mutation`, or `action` to register sensitive internal functions that should be kept private. - You CANNOT register a function through the `api` or `internal` objects. - ALWAYS include argument validators for all Convex functions. This includes all of `query`, `internalQuery`, `mutation`, `internalMutation`, `action`, and `internalAction`. ### Function calling + - Use `ctx.runQuery` to call a query from a query, mutation, or action. - Use `ctx.runMutation` to call a mutation from a mutation or action. - Use `ctx.runAction` to call an action from an action. @@ -80,6 +92,7 @@ Convex Type | TS/JS type | Example Usage | Validator for argument val - Try to use as few calls from actions to queries and mutations as possible. Queries and mutations are transactions, so splitting logic up into multiple calls introduces the risk of race conditions. - All of these calls take in a `FunctionReference`. Do NOT try to pass the callee function directly into one of these calls. - When using `ctx.runQuery`, `ctx.runMutation`, or `ctx.runAction` to call a function in the same file, specify a type annotation on the return value to work around TypeScript circularity limitations. For example, + ``` export const f = query({ args: { name: v.string() }, @@ -98,6 +111,7 @@ export const g = query({ ``` ### Function references + - Use the `api` object defined by the framework in `convex/_generated/api.ts` to call public functions registered with `query`, `mutation`, or `action`. - Use the `internal` object defined by the framework in `convex/_generated/api.ts` to call internal (or private) functions registered with `internalQuery`, `internalMutation`, or `internalAction`. - Convex uses file-based routing, so a public function defined in `convex/example.ts` named `f` has a function reference of `api.example.f`. @@ -105,6 +119,7 @@ export const g = query({ - Functions can also registered within directories nested within the `convex/` folder. For example, a public function `h` defined in `convex/messages/access.ts` has a function reference of `api.messages.access.h`. ### Pagination + - Define pagination using the following syntax: ```ts @@ -112,17 +127,19 @@ import { v } from "convex/values"; import { query, mutation } from "./_generated/server"; import { paginationOptsValidator } from "convex/server"; export const listWithExtraArg = query({ - args: { paginationOpts: paginationOptsValidator, author: v.string() }, - handler: async (ctx, args) => { - return await ctx.db - .query("messages") - .withIndex("by_author", (q) => q.eq("author", args.author)) - .order("desc") - .paginate(args.paginationOpts); - }, + args: { paginationOpts: paginationOptsValidator, author: v.string() }, + handler: async (ctx, args) => { + return await ctx.db + .query("messages") + .withIndex("by_author", (q) => q.eq("author", args.author)) + .order("desc") + .paginate(args.paginationOpts); + }, }); ``` + Note: `paginationOpts` is an object with the following properties: + - `numItems`: the maximum number of documents to return (the validator is `v.number()`) - `cursor`: the cursor to use to fetch the next page of documents (the validator is `v.union(v.string(), v.null())`) - A query that ends in `.paginate()` returns an object that has the following properties: @@ -130,8 +147,8 @@ Note: `paginationOpts` is an object with the following properties: - isDone (a boolean that represents whether or not this is the last page of documents) - continueCursor (a string that represents the cursor to use to fetch the next page of documents) - ## Schema guidelines + - Always define your schema in `convex/schema.ts`. - Always import the schema definition functions from `convex/server`. - System fields are automatically added to all documents and are prefixed with an underscore. The two system fields that are automatically added to all documents are `_creationTime` which has the validator `v.number()` and `_id` which has the validator `v.id(tableName)`. @@ -141,8 +158,10 @@ Note: `paginationOpts` is an object with the following properties: - Separate high-churn operational data (e.g. heartbeats, online status, typing indicators) from stable profile data. Storing frequently updated fields on a shared document forces every write to contend with reads of the entire document. Instead, create a dedicated table for the high-churn data with a foreign key back to the parent record. ## Authentication guidelines + - Convex supports JWT-based authentication through `convex/auth.config.ts`. ALWAYS create this file when using authentication. Without it, `ctx.auth.getUserIdentity()` will always return `null`. - Example `convex/auth.config.ts`: + ```typescript export default { providers: [ @@ -153,11 +172,14 @@ export default { ], }; ``` + The `domain` must be the issuer URL of the JWT provider. Convex fetches `{domain}/.well-known/openid-configuration` to discover the JWKS endpoint. The `applicationID` is checked against the JWT `aud` (audience) claim. + - Use `ctx.auth.getUserIdentity()` to get the authenticated user's identity in any query, mutation, or action. This returns `null` if the user is not authenticated, or a `UserIdentity` object with fields like `subject`, `issuer`, `name`, `email`, etc. The `subject` field is the unique user identifier. - In Convex `UserIdentity`, `tokenIdentifier` is guaranteed and is the canonical stable identifier for the authenticated identity. For any auth-linked database lookup or ownership check, prefer `identity.tokenIdentifier` over `identity.subject`. Do NOT use `identity.subject` alone as a global identity key. - NEVER accept a `userId` or any user identifier as a function argument for authorization purposes. Always derive the user identity server-side via `ctx.auth.getUserIdentity()`. - When using an external auth provider with Convex on the client, use `ConvexProviderWithAuth` instead of `ConvexProvider`: + ```tsx import { ConvexProviderWithAuth, ConvexReactClient } from "convex/react"; @@ -171,45 +193,51 @@ function App({ children }: { children: React.ReactNode }) { ); } ``` + The `useAuth` prop must return `{ isLoading, isAuthenticated, fetchAccessToken }`. Do NOT use plain `ConvexProvider` when authentication is needed — it will not send tokens with requests. ## Typescript guidelines -- You can use the helper typescript type `Id` imported from './_generated/dataModel' to get the type of the id for a given table. For example if there is a table called 'users' you can use `Id<'users'>` to get the type of the id for that table. + +- You can use the helper typescript type `Id` imported from './\_generated/dataModel' to get the type of the id for a given table. For example if there is a table called 'users' you can use `Id<'users'>` to get the type of the id for that table. - Use `Doc<"tableName">` from `./_generated/dataModel` to get the full document type for a table. - Use `QueryCtx`, `MutationCtx`, `ActionCtx` from `./_generated/server` for typing function contexts. NEVER use `any` for ctx parameters — always use the proper context type. - If you need to define a `Record` make sure that you correctly provide the type of the key and value in the type. For example a validator `v.record(v.id('users'), v.string())` would have the type `Record, string>`. Below is an example of using `Record` with an `Id` type in a query: + ```ts import { query } from "./_generated/server"; import { Doc, Id } from "./_generated/dataModel"; export const exampleQuery = query({ - args: { userIds: v.array(v.id("users")) }, - handler: async (ctx, args) => { - const idToUsername: Record, string> = {}; - for (const userId of args.userIds) { - const user = await ctx.db.get("users", userId); - if (user) { - idToUsername[user._id] = user.username; - } - } + args: { userIds: v.array(v.id("users")) }, + handler: async (ctx, args) => { + const idToUsername: Record, string> = {}; + for (const userId of args.userIds) { + const user = await ctx.db.get("users", userId); + if (user) { + idToUsername[user._id] = user.username; + } + } - return idToUsername; - }, + return idToUsername; + }, }); ``` + - Be strict with types, particularly around id's of documents. For example, if a function takes in an id for a document in the 'users' table, take in `Id<'users'>` rather than `string`. ## Full text search guidelines + - A query for "10 messages in channel '#general' that best match the query 'hello hi' in their body" would look like: const messages = await ctx.db - .query("messages") - .withSearchIndex("search_body", (q) => - q.search("body", "hello hi").eq("channel", "#general"), - ) - .take(10); +.query("messages") +.withSearchIndex("search_body", (q) => +q.search("body", "hello hi").eq("channel", "#general"), +) +.take(10); ## Query guidelines + - Do NOT use `filter` in queries. Instead, define an index in the schema and use `withIndex` instead. - If the user does not explicitly tell you to return all results from a query you should ALWAYS return a bounded collection instead. So that is instead of using `.collect()` you should use `.take()` or paginate on database queries. This prevents future performance issues when tables grow in an unbounded way. - Never use `.collect().length` to count rows. Convex has no built-in count operator, so if you need a count that stays efficient at scale, maintain a denormalized counter in a separate document and update it in your mutations. @@ -217,39 +245,46 @@ const messages = await ctx.db - Convex mutations are transactions with limits on the number of documents read and written. If a mutation needs to process more documents than fit in a single transaction (e.g. bulk deletion on a large table), process a batch with `.take(n)` and then call `ctx.scheduler.runAfter(0, api.myModule.myMutation, args)` to schedule itself to continue. This way each invocation stays within transaction limits. - Use `.unique()` to get a single document from a query. This method will throw an error if there are multiple documents that match the query. - When using async iteration, don't use `.collect()` or `.take(n)` on the result of a query. Instead, use the `for await (const row of query)` syntax. + ### Ordering + - By default Convex always returns documents in ascending `_creationTime` order. - You can use `.order('asc')` or `.order('desc')` to pick whether a query is in ascending or descending order. If the order isn't specified, it defaults to ascending. - Document queries that use indexes will be ordered based on the columns in the index and can avoid slow table scans. - ## Mutation guidelines + - Use `ctx.db.replace` to fully replace an existing document. This method will throw an error if the document does not exist. Syntax: `await ctx.db.replace('tasks', taskId, { name: 'Buy milk', completed: false })` - Use `ctx.db.patch` to shallow merge updates into an existing document. This method will throw an error if the document does not exist. Syntax: `await ctx.db.patch('tasks', taskId, { completed: true })` ## Action guidelines + - Always add `"use node";` to the top of files containing actions that use Node.js built-in modules. - Never add `"use node";` to a file that also exports queries or mutations. Only actions can run in the Node.js runtime; queries and mutations must stay in the default Convex runtime. If you need Node.js built-ins alongside queries or mutations, put the action in a separate file. - `fetch()` is available in the default Convex runtime. You do NOT need `"use node";` just to use `fetch()`. - Never use `ctx.db` inside of an action. Actions don't have access to the database. - Below is an example of the syntax for an action: + ```ts import { action } from "./_generated/server"; export const exampleAction = action({ - args: {}, - handler: async (ctx, args) => { - console.log("This action does not return anything"); - return null; - }, + args: {}, + handler: async (ctx, args) => { + console.log("This action does not return anything"); + return null; + }, }); ``` ## Scheduling guidelines + ### Cron guidelines + - Only use the `crons.interval` or `crons.cron` methods to schedule cron jobs. Do NOT use the `crons.hourly`, `crons.daily`, or `crons.weekly` helpers. - Both cron methods take in a FunctionReference. Do NOT try to pass the function directly into one of these methods. - Define crons by declaring the top-level `crons` object, calling some methods on it, and then exporting it as default. For example, + ```ts import { cronJobs } from "convex/server"; import { internal } from "./_generated/api"; @@ -269,14 +304,16 @@ crons.interval("delete inactive users", { hours: 2 }, internal.crons.empty, {}); export default crons; ``` -- You can register Convex functions within `crons.ts` just like any other file. -- If a cron calls an internal function, always import the `internal` object from '_generated/api', even if the internal function is registered in the same file. +- You can register Convex functions within `crons.ts` just like any other file. +- If a cron calls an internal function, always import the `internal` object from '\_generated/api', even if the internal function is registered in the same file. ## Testing guidelines + - Use `convex-test` with `vitest` and `@edge-runtime/vm` to test Convex functions. Always install the latest versions of these packages. Configure vitest with `environment: "edge-runtime"` in `vitest.config.ts`. Test files go inside the `convex/` directory. You must pass a module map from `import.meta.glob` to `convexTest`: + ```typescript /// import { convexTest } from "convex-test"; @@ -293,13 +330,16 @@ test("some behavior", async () => { expect(messages).toMatchObject([{ body: "Hi!", author: "Sarah" }]); }); ``` + The `modules` argument is required so convex-test can discover and load function files. The `/// ` directive is needed for TypeScript to recognize `import.meta.glob`. ## File storage guidelines + - The `ctx.storage.getUrl()` method returns a signed URL for a given file. It returns `null` if the file doesn't exist. - Do NOT use the deprecated `ctx.storage.getMetadata` call for loading a file's metadata. Instead, query the `_storage` system table. For example, you can use `ctx.db.system.get` to get an `Id<"_storage">`. + ``` import { query } from "./_generated/server"; import { Id } from "./_generated/dataModel"; @@ -321,6 +361,5 @@ export const exampleQuery = query({ }, }); ``` + - Convex storage stores items as `Blob` objects. You must convert all items to/from a `Blob` when using Convex storage. - - diff --git a/convex/_generated/api.d.ts b/convex/_generated/api.d.ts index ed8d3fe..83214a8 100644 --- a/convex/_generated/api.d.ts +++ b/convex/_generated/api.d.ts @@ -14,12 +14,15 @@ import type * as auth from "../auth.js"; import type * as deviceReports from "../deviceReports.js"; import type * as helpers from "../helpers.js"; import type * as http from "../http.js"; +import type * as lib_githubRepoScan from "../lib/githubRepoScan.js"; +import type * as lib_githubSubmodule from "../lib/githubSubmodule.js"; import type * as lib_meshforgeYaml from "../lib/meshforgeYaml.js"; import type * as lib_platformioScan from "../lib/platformioScan.js"; import type * as lib_r2 from "../lib/r2.js"; import type * as lib_tagSemver from "../lib/tagSemver.js"; import type * as repoBuildDownloads from "../repoBuildDownloads.js"; import type * as repoBuilds from "../repoBuilds.js"; +import type * as repoScanGitAction from "../repoScanGitAction.js"; import type * as repoScans from "../repoScans.js"; import type * as repoTags from "../repoTags.js"; @@ -36,12 +39,15 @@ declare const fullApi: ApiFromModules<{ deviceReports: typeof deviceReports; helpers: typeof helpers; http: typeof http; + "lib/githubRepoScan": typeof lib_githubRepoScan; + "lib/githubSubmodule": typeof lib_githubSubmodule; "lib/meshforgeYaml": typeof lib_meshforgeYaml; "lib/platformioScan": typeof lib_platformioScan; "lib/r2": typeof lib_r2; "lib/tagSemver": typeof lib_tagSemver; repoBuildDownloads: typeof repoBuildDownloads; repoBuilds: typeof repoBuilds; + repoScanGitAction: typeof repoScanGitAction; repoScans: typeof repoScans; repoTags: typeof repoTags; }>; diff --git a/convex/actions.ts b/convex/actions.ts index 367f86d..91a686e 100644 --- a/convex/actions.ts +++ b/convex/actions.ts @@ -32,6 +32,7 @@ export const dispatchRepoBuild = action({ repo: doc.repo, ref: doc.ref, target_env: doc.targetEnv, + platform_root: doc.platformRoot ?? "", repo_build_id: doc._id, build_key: doc.buildKey, resolved_source_sha: doc.resolvedSourceSha, diff --git a/convex/lib/githubRepoScan.ts b/convex/lib/githubRepoScan.ts new file mode 100644 index 0000000..d5623a4 --- /dev/null +++ b/convex/lib/githubRepoScan.ts @@ -0,0 +1,290 @@ +import { githubRepoFromRemote, parseGitmodulesPathToUrl } from './githubSubmodule' +import { unzipSync } from 'fflate' +import type { VirtualFileMap } from './platformioScan' + +const GITHUB_API = 'https://api.github.com' + +type TreeEntry = { + path?: string + mode?: string + type?: string + sha?: string + size?: number +} + +type TreeResponse = { + tree?: TreeEntry[] + truncated?: boolean +} + +type SubmodulePointer = { + path: string + owner: string + repo: string + commitSha: string +} + +function jsonHeaders(headers: Record): Record { + return { + Accept: 'application/vnd.github+json', + 'X-GitHub-Api-Version': '2022-11-28', + ...headers, + } +} + +async function fetchTreeRecursive(args: { + owner: string + repo: string + commitSha: string + headers: Record +}): Promise<{ entries: TreeEntry[]; truncated: boolean }> { + const url = `${GITHUB_API}/repos/${args.owner}/${args.repo}/git/trees/${args.commitSha}?recursive=1` + const res = await fetch(url, { headers: jsonHeaders(args.headers) }) + if (!res.ok) { + throw new Error( + `github tree ${args.owner}/${args.repo}@${args.commitSha}: ${res.status} ${await res.text()}` + ) + } + const j = (await res.json()) as TreeResponse + return { entries: j.tree ?? [], truncated: Boolean(j.truncated) } +} + +async function fetchRepoZipArchive(args: { + owner: string + repo: string + commitSha: string + headers: Record +}): Promise { + const url = `${GITHUB_API}/repos/${args.owner}/${args.repo}/zipball/${args.commitSha}` + const res = await fetch(url, { headers: jsonHeaders(args.headers) }) + if (!res.ok) { + throw new Error( + `github zip ${args.owner}/${args.repo}@${args.commitSha}: ${res.status} ${await res.text()}` + ) + } + return await res.arrayBuffer() +} + +async function fetchBlobText(args: { + owner: string + repo: string + sha: string + headers: Record +}): Promise { + const url = `${GITHUB_API}/repos/${args.owner}/${args.repo}/git/blobs/${args.sha}` + const res = await fetch(url, { + headers: { + Accept: 'application/vnd.github.raw', + 'X-GitHub-Api-Version': '2022-11-28', + ...args.headers, + }, + }) + if (!res.ok) { + throw new Error(`github blob ${args.owner}/${args.repo}@${args.sha}: ${res.status} ${await res.text()}`) + } + return await res.text() +} + +function isScanFile(path: string): boolean { + if (path.endsWith('.ini')) return true + if (path === 'meshforge.yaml' || path.endsWith('/meshforge.yaml')) return true + return false +} + +function stripZipRoot(path: string): string { + const normalized = path.replace(/\\/g, '/') + const i = normalized.indexOf('/') + if (i < 0) return '' + return normalized.slice(i + 1) +} + +function decodeArchiveFiles(zip: ArrayBuffer): VirtualFileMap { + const map: VirtualFileMap = {} + const entries = unzipSync(new Uint8Array(zip)) + const decoder = new TextDecoder('utf-8') + for (const [zipPath, bytes] of Object.entries(entries)) { + const path = stripZipRoot(zipPath) + if (!path || path.endsWith('/')) continue + if (path !== '.gitmodules' && !isScanFile(path)) continue + map[path] = decoder.decode(bytes) + } + return map +} + +function shouldFetchSubmodule(path: string, platformRoot: string): boolean { + const p = path.trim().replace(/\/+$/, '') + const root = platformRoot.trim().replace(/\/+$/, '') + if (!root) return true + if (p === root) return true + if (p.startsWith(`${root}/`)) return true + if (root.startsWith(`${p}/`)) return true + return false +} + +function mapLimit(items: readonly T[], limit: number, worker: (item: T) => Promise): Promise { + if (items.length === 0) return Promise.resolve([]) + const max = Math.max(1, Math.min(limit, items.length)) + const out = new Array(items.length) + let next = 0 + async function runWorker(): Promise { + while (true) { + const i = next + next += 1 + if (i >= items.length) return + out[i] = await worker(items[i]) + } + } + return Promise.all(Array.from({ length: max }, () => runWorker())).then(() => out) +} + +async function resolveSubmodulePointers(args: { + owner: string + repo: string + commitSha: string + headers: Record + gitmodulesText: string + platformRoot: string +}): Promise { + const pathToUrl = parseGitmodulesPathToUrl(args.gitmodulesText) + if (Object.keys(pathToUrl).length === 0) return [] + const { entries, truncated } = await fetchTreeRecursive(args) + if (truncated) { + console.warn(`github tree truncated for ${args.owner}/${args.repo}@${args.commitSha}; submodule scan may miss entries`) + } + const gitlinkByPath: Record = {} + for (const entry of entries) { + if (!entry.path || !entry.sha) continue + if (entry.type === 'commit' && entry.mode === '160000') { + gitlinkByPath[entry.path] = entry.sha + } + } + const out: SubmodulePointer[] = [] + for (const [path, url] of Object.entries(pathToUrl)) { + if (!shouldFetchSubmodule(path, args.platformRoot)) continue + const commitSha = gitlinkByPath[path] + if (!commitSha) continue + const gh = githubRepoFromRemote(url) + if (!gh) continue + out.push({ path, owner: gh.owner, repo: gh.repo, commitSha }) + } + return out +} + +/** + * Archive-first scanner: + * 1) Download root zipball and parse scan files from it. + * 2) Read `.gitmodules` + root tree gitlinks for pinned submodule SHAs. + * 3) Download relevant submodule zipballs and merge scan files. + */ +export async function collectScanFilesFromGithubArchives(args: { + owner: string + repo: string + commitSha: string + headers: Record + platformRoot: string +}): Promise { + const rootZip = await fetchRepoZipArchive(args) + const rootFiles = decodeArchiveFiles(rootZip) + const out: VirtualFileMap = {} + for (const [path, content] of Object.entries(rootFiles)) { + if (isScanFile(path)) out[path] = content + } + const gitmodulesText = rootFiles['.gitmodules'] + if (!gitmodulesText) return out + + const pointers = await resolveSubmodulePointers({ + owner: args.owner, + repo: args.repo, + commitSha: args.commitSha, + headers: args.headers, + gitmodulesText, + platformRoot: args.platformRoot, + }) + const archives = await mapLimit(pointers, 4, async p => { + const zip = await fetchRepoZipArchive({ + owner: p.owner, + repo: p.repo, + commitSha: p.commitSha, + headers: args.headers, + }) + return { basePath: p.path, files: decodeArchiveFiles(zip) } + }) + for (const { basePath, files } of archives) { + for (const [path, content] of Object.entries(files)) { + if (!isScanFile(path)) continue + out[`${basePath}/${path}`] = content + } + } + return out +} + +/** + * Walk the GitHub tree for `owner/repo@commitSha` and return all *.ini / meshforge.yaml files + * (paths relative to this repo root). Recurses into submodules listed in `.gitmodules` — + * resolving each pinned commit via the parent tree's `type: "commit"` entries, then fetching the + * submodule's own recursive tree. Pure HTTP — runs in Convex's default V8 runtime (no `"use node"`). + */ +export async function collectScanFilesFromGithub(args: { + owner: string + repo: string + commitSha: string + headers: Record +}): Promise { + const { entries, truncated } = await fetchTreeRecursive(args) + if (truncated) { + console.warn( + `github tree truncated for ${args.owner}/${args.repo}@${args.commitSha}; PlatformIO scan may miss files` + ) + } + + const files: VirtualFileMap = {} + const submoduleEntries: TreeEntry[] = [] + let gitmodulesText: string | null = null + + for (const entry of entries) { + if (!entry.path) continue + if (entry.type === 'commit' && entry.mode === '160000' && entry.sha) { + submoduleEntries.push(entry) + continue + } + if (entry.type !== 'blob' || !entry.sha) continue + if (entry.path === '.gitmodules') { + gitmodulesText = await fetchBlobText({ + owner: args.owner, + repo: args.repo, + sha: entry.sha, + headers: args.headers, + }) + continue + } + if (isScanFile(entry.path)) { + files[entry.path] = await fetchBlobText({ + owner: args.owner, + repo: args.repo, + sha: entry.sha, + headers: args.headers, + }) + } + } + + if (submoduleEntries.length === 0) return files + + const pathToUrl = gitmodulesText ? parseGitmodulesPathToUrl(gitmodulesText) : {} + for (const sub of submoduleEntries) { + const url = pathToUrl[sub.path!] + if (!url) continue + const gh = githubRepoFromRemote(url) + if (!gh) continue + const subFiles = await collectScanFilesFromGithub({ + owner: gh.owner, + repo: gh.repo, + commitSha: sub.sha!, + headers: args.headers, + }) + for (const [k, v] of Object.entries(subFiles)) { + files[`${sub.path}/${k}`] = v + } + } + + return files +} diff --git a/convex/lib/githubSubmodule.ts b/convex/lib/githubSubmodule.ts new file mode 100644 index 0000000..90c496f --- /dev/null +++ b/convex/lib/githubSubmodule.ts @@ -0,0 +1,44 @@ +/** + * Parsers for `.gitmodules` and GitHub remote URLs. Pure string logic. + */ + +export function parseGitmodulesPathToUrl(gitmodulesText: string): Record { + const map: Record = {} + let pendingPath: string | null = null + for (const line of gitmodulesText.split(/\r?\n/)) { + const t = line.trim() + if (!t || t.startsWith('#')) continue + if (t.startsWith('[') && t.endsWith(']')) { + pendingPath = null + continue + } + const pathM = t.match(/^path\s*=\s*(.+)$/) + if (pathM) { + pendingPath = stripQuotes(pathM[1].trim()) + continue + } + const urlM = t.match(/^url\s*=\s*(.+)$/) + if (urlM && pendingPath) { + map[pendingPath] = stripQuotes(urlM[1].trim()) + pendingPath = null + } + } + return map +} + +function stripQuotes(s: string): string { + if ((s.startsWith('"') && s.endsWith('"')) || (s.startsWith("'") && s.endsWith("'"))) { + return s.slice(1, -1) + } + return s +} + +/** Parse `https://github.com/o/r.git` or `git@github.com:o/r.git` → { owner, repo }. */ +export function githubRepoFromRemote(url: string): { owner: string; repo: string } | null { + const u = url.trim() + const https = u.match(/github\.com\/([^/]+)\/([^/.]+?)(?:\.git)?(?:\/|$)/i) + if (https) return { owner: https[1], repo: https[2] } + const ssh = u.match(/git@github\.com:([^/]+)\/([^/.]+?)(?:\.git)?$/i) + if (ssh) return { owner: ssh[1], repo: ssh[2] } + return null +} diff --git a/convex/lib/meshforgeYaml.ts b/convex/lib/meshforgeYaml.ts index 879dd77..f606e32 100644 --- a/convex/lib/meshforgeYaml.ts +++ b/convex/lib/meshforgeYaml.ts @@ -1,3 +1,5 @@ +import YAML from 'yaml' + export interface MeshforgeTagsConfig { /** JS regex tested against each tag name. Tags not matching are hidden. */ include?: string @@ -20,88 +22,152 @@ export interface MeshforgeTargetsConfig { require_capabilities?: string[] } +/** Per-platform overlay (submodule name → profile fragment). */ +export interface MeshforgePlatformFragment { + tags?: MeshforgeTagsConfig + targets?: MeshforgeTargetsConfig + /** Same semantics as meshforge root `require_capabilities`; merged into effective targets filter. */ + require_capabilities?: string[] +} + export interface MeshforgeConfig { tags?: MeshforgeTagsConfig targets?: MeshforgeTargetsConfig + /** MeshForge root-level capability filter (merged into effective `targets.require_capabilities`). */ + require_capabilities?: string[] + /** Submodule directory names → overlay merged on top of root when that platform is selected. */ + platforms?: Record +} + +function isRecord(x: unknown): x is Record { + return x !== null && typeof x === 'object' && !Array.isArray(x) +} + +function readTags(obj: Record): MeshforgeTagsConfig | undefined { + const t = obj.tags + if (!isRecord(t)) return undefined + const tags: MeshforgeTagsConfig = {} + if (typeof t.include === 'string') tags.include = t.include + return Object.keys(tags).length ? tags : undefined +} + +function readTargets(obj: Record): MeshforgeTargetsConfig | undefined { + const t = obj.targets + if (!isRecord(t)) return undefined + const targets: MeshforgeTargetsConfig = {} + if (typeof t.include === 'string') targets.include = t.include + if (typeof t.include_template === 'string') targets.include_template = t.include_template + if (Array.isArray(t.require_capabilities)) { + const caps = t.require_capabilities.filter((c): c is string => typeof c === 'string') + if (caps.length) targets.require_capabilities = caps + } + return Object.keys(targets).length ? targets : undefined +} + +function readPlatformFragment(obj: Record): MeshforgePlatformFragment { + const frag: MeshforgePlatformFragment = {} + const tags = readTags(obj) + const targets = readTargets(obj) + if (tags) frag.tags = tags + if (targets) frag.targets = targets + if (Array.isArray(obj.require_capabilities)) { + const caps = obj.require_capabilities.filter((c): c is string => typeof c === 'string') + if (caps.length) frag.require_capabilities = caps + } + return frag } /** - * Minimal parser for the meshforge.yaml format. Only the known schema is handled; - * unknown keys are silently ignored. - * - * Supports: - * - 2-space YAML-like indentation (0 / 2 / 4 spaces) - * - Quoted ("…" or '…') and unquoted scalar values - * - Inline flow lists [a, b, c] - * - Line comments starting with # + * Parse meshforge.yaml using a real YAML parser (nested `platforms:` and root keys). */ export function parseMeshforgeYaml(raw: string): MeshforgeConfig | null { + let doc: unknown + try { + doc = YAML.parse(raw) + } catch { + return null + } + if (!isRecord(doc)) return null + const mf = doc.meshforge + if (!isRecord(mf)) return null + const config: MeshforgeConfig = {} - let inMeshforge = false - let section: 'tags' | 'targets' | null = null + const tags = readTags(mf) + const targets = readTargets(mf) + if (tags) config.tags = tags + if (targets) config.targets = targets - for (const rawLine of raw.split(/\r?\n/)) { - const stripped = rawLine.replace(/#.*$/, '').trimEnd() - if (!stripped.trim()) continue - - const indent = stripped.length - stripped.trimStart().length - const content = stripped.trimStart() - - if (indent === 0) { - inMeshforge = content === 'meshforge:' - section = null - continue - } - - if (!inMeshforge) continue - - if (indent === 2) { - if (content === 'tags:') { - section = 'tags' - if (!config.tags) config.tags = {} - } else if (content === 'targets:') { - section = 'targets' - if (!config.targets) config.targets = {} - } else { - section = null - } - continue - } - - if (indent === 4 && section) { - const kv = content.match(/^(\w+):\s*(.*)$/) - if (!kv) continue - const [, key, rawVal] = kv - if (section === 'tags') { - if (key === 'include') config.tags!.include = parseScalar(rawVal) - } else if (section === 'targets') { - if (key === 'include') config.targets!.include = parseScalar(rawVal) - else if (key === 'include_template') config.targets!.include_template = parseScalar(rawVal) - else if (key === 'require_capabilities') config.targets!.require_capabilities = parseInlineList(rawVal) - } - } + if (Array.isArray(mf.require_capabilities)) { + const caps = mf.require_capabilities.filter((c): c is string => typeof c === 'string') + if (caps.length) config.require_capabilities = caps } - if (!config.tags && !config.targets) return null + if (isRecord(mf.platforms)) { + const platforms: Record = {} + for (const [name, fragRaw] of Object.entries(mf.platforms)) { + const key = name.trim() + if (!key) continue + if (fragRaw === null || fragRaw === undefined) { + platforms[key] = {} + } else if (isRecord(fragRaw)) { + platforms[key] = readPlatformFragment(fragRaw) + } + } + if (Object.keys(platforms).length) config.platforms = platforms + } + + if (!config.tags && !config.targets && !config.platforms && !config.require_capabilities) return null return config } -function parseScalar(raw: string): string { - const s = raw.trim() - if ((s.startsWith('"') && s.endsWith('"')) || (s.startsWith("'") && s.endsWith("'"))) { - return s.slice(1, -1) - } - return s +function rootCapabilityUnion(config: MeshforgeConfig): string[] { + const a = config.targets?.require_capabilities ?? [] + const b = config.require_capabilities ?? [] + return [...new Set([...a, ...b])] } -function parseInlineList(raw: string): string[] { - const s = raw.trim() - if (s.startsWith('[') && s.endsWith(']')) { - return s - .slice(1, -1) - .split(',') - .map(p => parseScalar(p.trim())) - .filter(Boolean) - } - return s ? [parseScalar(s)] : [] +/** Sorted submodule / platform keys from `meshforge.platforms`. */ +export function meshforgePlatformKeys(config: MeshforgeConfig | null | undefined): string[] { + if (!config?.platforms) return [] + return Object.keys(config.platforms) + .map(k => k.trim()) + .filter(Boolean) + .sort((x, y) => x.localeCompare(y)) +} + +/** + * Root meshforge plus optional platform overlay (for tag/target filtering in UI and CI context). + * Strips `platforms` from the result — callers use {@link meshforgePlatformKeys} for the menu. + */ +export function mergeEffectiveMeshforgeConfig( + config: MeshforgeConfig | null, + platformKey: string | null +): MeshforgeConfig | null { + if (!config) return null + const baseCaps = rootCapabilityUnion(config) + const stripPlatforms = (): MeshforgeConfig => { + const { platforms: _p, ...rest } = config + const out: MeshforgeConfig = { ...rest } + if (baseCaps.length) { + out.targets = { ...out.targets, require_capabilities: baseCaps } + } + return out + } + + if (!platformKey || !config.platforms?.[platformKey]) { + return stripPlatforms() + } + + const frag = config.platforms[platformKey] + const fragCaps = [...(frag.targets?.require_capabilities ?? []), ...(frag.require_capabilities ?? [])] + const mergedCaps = [...new Set([...baseCaps, ...fragCaps])] + + return { + tags: { ...config.tags, ...frag.tags }, + targets: { + ...config.targets, + ...frag.targets, + ...(mergedCaps.length ? { require_capabilities: mergedCaps } : {}), + }, + } } diff --git a/convex/lib/platformioScan.ts b/convex/lib/platformioScan.ts index b943db8..ee1884b 100644 --- a/convex/lib/platformioScan.ts +++ b/convex/lib/platformioScan.ts @@ -1,10 +1,12 @@ /** * Parse PlatformIO-style INI content and collect [env:...] section names from file contents. + * Pure string logic — safe to import from either Convex runtime. */ export function parseIniSections(content: string): Record> { const sections: Record> = {} let current: string | null = null + let lastKey: string | null = null for (const line of content.split(/\r?\n/)) { const trimmed = line.trim() if (!trimmed || trimmed.startsWith(';') || trimmed.startsWith('#')) continue @@ -12,6 +14,7 @@ export function parseIniSections(content: string): Record -/** Strip first path segment (GitHub zip root folder). Captures *.ini and meshforge.yaml. */ -export function normalizeZipPaths(files: Record, decode: (u: Uint8Array) => string): VirtualFileMap { - const out: VirtualFileMap = {} - for (const path of Object.keys(files)) { - const parts = path.split('/').filter(Boolean) - if (parts.length < 2) continue - const rel = parts.slice(1).join('/') - if (!rel.endsWith('.ini') && rel !== 'meshforge.yaml') continue - try { - out[rel] = decode(files[path]) - } catch { - // skip binary - } - } - return out -} - /** Aggregate all PlatformIO sections from every .ini file in the virtual file map. */ function aggregateIniSections(files: VirtualFileMap): Record> { const allSections: Record> = {} diff --git a/convex/repoBuilds.ts b/convex/repoBuilds.ts index 0fef469..56a556d 100644 --- a/convex/repoBuilds.ts +++ b/convex/repoBuilds.ts @@ -2,9 +2,11 @@ import { v } from "convex/values" import { api, internal } from "./_generated/api" import { internalMutation, internalQuery, mutation, query } from "./_generated/server" -export function normalizeBuildKey(resolvedSourceSha: string, targetEnv: string): string { +export function normalizeBuildKey(resolvedSourceSha: string, targetEnv: string, platformRoot?: string): string { + const p = (platformRoot ?? "").trim().replace(/\//g, "_") const t = targetEnv.replace(/\//g, "_") - return `${resolvedSourceSha}_${t}` + if (!p) return `${resolvedSourceSha}_${t}` + return `${resolvedSourceSha}_${p}_${t}` } export const getById = query({ @@ -34,9 +36,11 @@ export const ensureBuild = mutation({ ref: v.string(), resolvedSourceSha: v.string(), targetEnv: v.string(), + platformRoot: v.optional(v.string()), }, handler: async (ctx, args) => { - const buildKey = normalizeBuildKey(args.resolvedSourceSha, args.targetEnv) + const platformRoot = args.platformRoot ?? "" + const buildKey = normalizeBuildKey(args.resolvedSourceSha, args.targetEnv, platformRoot) const existing = await ctx.db .query("repoBuilds") .withIndex("by_buildKey", q => q.eq("buildKey", buildKey)) @@ -60,6 +64,7 @@ export const ensureBuild = mutation({ repo: args.repo, ref: args.ref, resolvedSourceSha: args.resolvedSourceSha, + platformRoot: platformRoot || undefined, targetEnv: args.targetEnv, buildKey, status: "queued", @@ -159,6 +164,7 @@ export const retryBuild = mutation({ repo: doc.repo, ref: doc.ref, resolvedSourceSha: doc.resolvedSourceSha, + platformRoot: doc.platformRoot, targetEnv: doc.targetEnv, buildKey: doc.buildKey, status: "queued", diff --git a/convex/repoScanGitAction.ts b/convex/repoScanGitAction.ts new file mode 100644 index 0000000..d7a344e --- /dev/null +++ b/convex/repoScanGitAction.ts @@ -0,0 +1,108 @@ +import { v } from 'convex/values' +import { internal } from './_generated/api' +import { action } from './_generated/server' +import { collectScanFilesFromGithub, collectScanFilesFromGithubArchives } from './lib/githubRepoScan' +import { parseMeshforgeYaml } from './lib/meshforgeYaml' +import { collectPlatformioEnvsFromFiles, type VirtualFileMap } from './lib/platformioScan' + +function githubHeaders(token: string | undefined): Record { + const h: Record = {} + if (token) h.Authorization = `Bearer ${token}` + return h +} + +function scopeFiles(files: VirtualFileMap, platformRoot: string): VirtualFileMap { + if (!platformRoot) return files + const prefix = `${platformRoot}/` + const out: VirtualFileMap = {} + for (const [k, v] of Object.entries(files)) { + if (!k.startsWith(prefix)) continue + out[k.slice(prefix.length)] = v + } + return out +} + +/** + * Walk the GitHub tree (recursing into submodules) and scan `*.ini` for PlatformIO envs. + * Runs in Convex's default V8 runtime — no git binary, no disk, no `"use node"`. + */ +export const runArchiveScan = action({ + args: { + scanId: v.id('repoRefScan'), + owner: v.string(), + repo: v.string(), + ref: v.string(), + resolvedSourceSha: v.string(), + platformRoot: v.optional(v.string()), + }, + handler: async (ctx, args) => { + const headers = githubHeaders(process.env.GITHUB_TOKEN) + const platformRoot = (args.platformRoot ?? '').trim() + + try { + let allFiles: VirtualFileMap + try { + allFiles = await collectScanFilesFromGithubArchives({ + owner: args.owner, + repo: args.repo, + commitSha: args.resolvedSourceSha, + headers, + platformRoot, + }) + } catch (archiveError) { + console.warn( + `archive scan path failed for ${args.owner}/${args.repo}@${args.resolvedSourceSha}; falling back to tree/blob scan`, + archiveError + ) + allFiles = await collectScanFilesFromGithub({ + owner: args.owner, + repo: args.repo, + commitSha: args.resolvedSourceSha, + headers, + }) + } + + const files = scopeFiles(allFiles, platformRoot) + const hasAnyIni = Object.keys(files).some(k => k.endsWith('.ini')) + if (!hasAnyIni) { + throw new Error( + platformRoot + ? `No PlatformIO files under "${platformRoot}" (no *.ini found).` + : 'No PlatformIO files in repository (no *.ini found).' + ) + } + + const r = collectPlatformioEnvsFromFiles(files) + if (r.envNames.length === 0) { + throw new Error( + platformRoot + ? `No PlatformIO environments under "${platformRoot}" (no *.ini env sections found).` + : 'No PlatformIO environments in repository (no *.ini env sections found).' + ) + } + + let meshforgeConfig: ReturnType = null + const yamlRaw = allFiles['meshforge.yaml'] + if (yamlRaw) { + try { + meshforgeConfig = parseMeshforgeYaml(yamlRaw) + } catch { + // ignore malformed yaml + } + } + + await ctx.runMutation(internal.repoScans.completeScanInternal, { + scanId: args.scanId, + envNames: r.envNames, + grouped: r.grouped, + envCapabilities: r.envCapabilities, + meshforgeConfig: meshforgeConfig ?? undefined, + }) + } catch (e) { + await ctx.runMutation(internal.repoScans.failScanInternal, { + scanId: args.scanId, + message: String(e), + }) + } + }, +}) diff --git a/convex/repoScans.ts b/convex/repoScans.ts index 6288a1a..6878ac6 100644 --- a/convex/repoScans.ts +++ b/convex/repoScans.ts @@ -1,23 +1,62 @@ -import { unzipSync, strFromU8 } from "fflate" import { v } from "convex/values" import { api, internal } from "./_generated/api" -import { collectPlatformioEnvsFromFiles, normalizeZipPaths } from "./lib/platformioScan" -import { parseMeshforgeYaml } from "./lib/meshforgeYaml" +import type { MutationCtx } from "./_generated/server" import { action, internalMutation, mutation, query } from "./_generated/server" +async function findRepoRefScan( + ctx: Pick, + args: { owner: string; repo: string; resolvedSourceSha: string; platformRoot: string } +) { + const pr = args.platformRoot + let existing = await ctx.db + .query("repoRefScan") + .withIndex("by_repo_sha_platform", q => + q + .eq("owner", args.owner) + .eq("repo", args.repo) + .eq("resolvedSourceSha", args.resolvedSourceSha) + .eq("platformRoot", pr) + ) + .first() + if (!existing && pr === "") { + existing = await ctx.db + .query("repoRefScan") + .withIndex("by_repo_sha", q => + q.eq("owner", args.owner).eq("repo", args.repo).eq("resolvedSourceSha", args.resolvedSourceSha) + ) + .first() + } + return existing +} + export const getByRepoSha = query({ args: { owner: v.string(), repo: v.string(), resolvedSourceSha: v.string(), + platformRoot: v.optional(v.string()), }, handler: async (ctx, args) => { - return await ctx.db + const pr = args.platformRoot ?? "" + let row = await ctx.db .query("repoRefScan") - .withIndex("by_repo_sha", q => - q.eq("owner", args.owner).eq("repo", args.repo).eq("resolvedSourceSha", args.resolvedSourceSha) + .withIndex("by_repo_sha_platform", q => + q + .eq("owner", args.owner) + .eq("repo", args.repo) + .eq("resolvedSourceSha", args.resolvedSourceSha) + .eq("platformRoot", pr) ) .first() + if (!row && pr === "") { + row = await ctx.db + .query("repoRefScan") + .withIndex("by_repo_sha", q => + q.eq("owner", args.owner).eq("repo", args.repo).eq("resolvedSourceSha", args.resolvedSourceSha) + ) + .first() + } + return row }, }) @@ -48,14 +87,16 @@ export const ensureScan = mutation({ repo: v.string(), ref: v.string(), resolvedSourceSha: v.string(), + platformRoot: v.optional(v.string()), }, handler: async (ctx, args) => { - const existing = await ctx.db - .query("repoRefScan") - .withIndex("by_repo_sha", q => - q.eq("owner", args.owner).eq("repo", args.repo).eq("resolvedSourceSha", args.resolvedSourceSha) - ) - .first() + const platformRoot = args.platformRoot ?? "" + const existing = await findRepoRefScan(ctx, { + owner: args.owner, + repo: args.repo, + resolvedSourceSha: args.resolvedSourceSha, + platformRoot, + }) if (existing?.scanStatus === "complete") { const names = existing.envNames ?? [] @@ -67,7 +108,6 @@ export const ensureScan = mutation({ if (capsComplete) { return { scanId: existing._id, status: "complete" as const } } - // Completed before envCapabilities existed (or partial write) — rescan same SHA. await ctx.db.patch(existing._id, { scanStatus: "in_progress", envNames: undefined, @@ -76,13 +116,15 @@ export const ensureScan = mutation({ meshforgeConfig: undefined, scanError: undefined, updatedAt: Date.now(), + platformRoot: existing.platformRoot ?? platformRoot, }) - await ctx.scheduler.runAfter(0, api.repoScans.runArchiveScan, { + await ctx.scheduler.runAfter(0, api.repoScanGitAction.runArchiveScan, { scanId: existing._id, owner: args.owner, repo: args.repo, ref: args.ref, resolvedSourceSha: args.resolvedSourceSha, + platformRoot, }) return { scanId: existing._id, status: "in_progress" as const } } @@ -90,23 +132,43 @@ export const ensureScan = mutation({ return { scanId: existing._id, status: "in_progress" as const } } if (existing?.scanStatus === "failed") { - return { scanId: existing._id, status: "failed" as const } + await ctx.db.patch(existing._id, { + scanStatus: "in_progress", + envNames: undefined, + grouped: undefined, + envCapabilities: undefined, + meshforgeConfig: undefined, + scanError: undefined, + updatedAt: Date.now(), + platformRoot: existing.platformRoot ?? platformRoot, + }) + await ctx.scheduler.runAfter(0, api.repoScanGitAction.runArchiveScan, { + scanId: existing._id, + owner: args.owner, + repo: args.repo, + ref: args.ref, + resolvedSourceSha: args.resolvedSourceSha, + platformRoot, + }) + return { scanId: existing._id, status: "in_progress" as const } } const scanId = await ctx.db.insert("repoRefScan", { owner: args.owner, repo: args.repo, resolvedSourceSha: args.resolvedSourceSha, + platformRoot, scanStatus: "in_progress", updatedAt: Date.now(), }) - await ctx.scheduler.runAfter(0, api.repoScans.runArchiveScan, { + await ctx.scheduler.runAfter(0, api.repoScanGitAction.runArchiveScan, { scanId, owner: args.owner, repo: args.repo, ref: args.ref, resolvedSourceSha: args.resolvedSourceSha, + platformRoot, }) return { scanId, status: "in_progress" as const } @@ -148,56 +210,3 @@ export const failScanInternal = internalMutation({ }) }, }) - -export const runArchiveScan = action({ - args: { - scanId: v.id("repoRefScan"), - owner: v.string(), - repo: v.string(), - ref: v.string(), - resolvedSourceSha: v.string(), - }, - handler: async (ctx, args) => { - const token = process.env.GITHUB_TOKEN - const zipUrl = `https://codeload.github.com/${args.owner}/${args.repo}/zip/${encodeURIComponent(args.ref)}` - const headers: Record = {} - if (token) headers.Authorization = `Bearer ${token}` - - try { - const zipRes = await fetch(zipUrl, { headers, redirect: "follow" }) - if (!zipRes.ok) { - throw new Error(`Archive fetch ${zipRes.status}: ${await zipRes.text()}`) - } - const buf = new Uint8Array(await zipRes.arrayBuffer()) - if (buf.byteLength > 80 * 1024 * 1024) { - throw new Error("Archive too large for inline scan (max 80MB)") - } - const files = unzipSync(buf) - const virtual = normalizeZipPaths(files, u => strFromU8(u, true)) - const { envNames, grouped, envCapabilities } = collectPlatformioEnvsFromFiles(virtual) - - let meshforgeConfig: ReturnType = null - const yamlContent = virtual['meshforge.yaml'] - if (yamlContent) { - try { - meshforgeConfig = parseMeshforgeYaml(yamlContent) - } catch { - // ignore malformed yaml - } - } - - await ctx.runMutation(internal.repoScans.completeScanInternal, { - scanId: args.scanId, - envNames, - grouped, - envCapabilities, - meshforgeConfig: meshforgeConfig ?? undefined, - }) - } catch (e) { - await ctx.runMutation(internal.repoScans.failScanInternal, { - scanId: args.scanId, - message: String(e), - }) - } - }, -}) diff --git a/convex/repoTags.ts b/convex/repoTags.ts index 4169cf4..17954dc 100644 --- a/convex/repoTags.ts +++ b/convex/repoTags.ts @@ -96,7 +96,7 @@ export const refresh = action({ const homepage = (repoJson.homepage ?? "").trim() const defaultBranch = (repoJson.default_branch ?? "").trim() || undefined - // Fetch meshforge.yaml from the default branch (no ref = default branch). + // meshforge.yaml from the repo default branch (Contents API with no `ref` = default). let meshforgeConfig: MeshforgeConfig | null = null const yamlRes = await fetch( `https://api.github.com/repos/${args.owner}/${args.repo}/contents/meshforge.yaml`, diff --git a/convex/schema.ts b/convex/schema.ts index dd50e54..9e558cc 100644 --- a/convex/schema.ts +++ b/convex/schema.ts @@ -28,12 +28,14 @@ export const repoRefScanFields = { owner: v.string(), repo: v.string(), resolvedSourceSha: v.string(), + /** Submodule directory containing the PlatformIO project (`""` = repo root). */ + platformRoot: v.optional(v.string()), scanStatus: v.union(v.literal("in_progress"), v.literal("complete"), v.literal("failed")), envNames: v.optional(v.array(v.string())), grouped: v.optional(v.any()), /** Detected capability sets keyed by env name, e.g. { "LilyGo_TDeck_repeater": ["wifi","ble"] }. */ envCapabilities: v.optional(v.any()), - /** Parsed meshforge.yaml config from the scanned source tree, if present. */ + /** Parsed meshforge.yaml from the scanned ref tree (repo root), when present. */ meshforgeConfig: v.optional(v.any()), scanError: v.optional(v.string()), scannedAt: v.optional(v.number()), @@ -48,6 +50,8 @@ export const repoBuildsFields = { repo: v.string(), ref: v.string(), resolvedSourceSha: v.string(), + /** Same as repoRefScan: PlatformIO project lives under this path in the monorepo. */ + platformRoot: v.optional(v.string()), targetEnv: v.string(), buildKey: v.string(), status: v.union(v.literal("queued"), v.literal("running"), v.literal("succeeded"), v.literal("failed")), @@ -81,7 +85,9 @@ export const userSettingsFields = { export const schema = defineSchema({ ...authTables, repoTagList: defineTable(repoTagListFields).index("by_owner_repo", ["owner", "repo"]), - repoRefScan: defineTable(repoRefScanFields).index("by_repo_sha", ["owner", "repo", "resolvedSourceSha"]), + repoRefScan: defineTable(repoRefScanFields) + .index("by_repo_sha", ["owner", "repo", "resolvedSourceSha"]) + .index("by_repo_sha_platform", ["owner", "repo", "resolvedSourceSha", "platformRoot"]), repoBuilds: defineTable(repoBuildsFields) .index("by_buildKey", ["buildKey"]) .index("by_owner_repo", ["owner", "repo"]) diff --git a/samples/projects/lotato b/samples/projects/lotato index 576578b..7425c36 160000 --- a/samples/projects/lotato +++ b/samples/projects/lotato @@ -1 +1 @@ -Subproject commit 576578b0b6e39402dbaf1ccab3a9f35129683d78 +Subproject commit 7425c365881296cbd7e36943f5e0cb68e6f8a0fc diff --git a/src/lib/buildKey.ts b/src/lib/buildKey.ts index 0bd5ad5..1fd7a4b 100644 --- a/src/lib/buildKey.ts +++ b/src/lib/buildKey.ts @@ -1,4 +1,10 @@ -export function normalizeBuildKey(resolvedSourceSha: string, targetEnv: string): string { - const t = targetEnv.replace(/\//g, '_') - return `${resolvedSourceSha}_${t}` +export function normalizeBuildKey( + resolvedSourceSha: string, + targetEnv: string, + platformRoot?: string +): string { + const p = (platformRoot ?? "").trim().replace(/\//g, "_") + const t = targetEnv.replace(/\//g, "_") + if (!p) return `${resolvedSourceSha}_${t}` + return `${resolvedSourceSha}_${p}_${t}` } diff --git a/src/lib/repoTreeUrl.ts b/src/lib/repoTreeUrl.ts index 6a4a4ab..0017384 100644 --- a/src/lib/repoTreeUrl.ts +++ b/src/lib/repoTreeUrl.ts @@ -1,36 +1,49 @@ /** - * MeshForge tree URLs: `/owner/repo/tree//target/` with optional `/flash` for the flasher-only view. - * Source ref may contain `/` (nested tags are rare but allowed). `target` is a reserved final segment pair. + * MeshForge tree URLs: + * `/owner/repo/tree//platform//target/` (monorepo + submodule PlatformIO root) + * or legacy `/owner/repo/tree//target/`. + * Optional `/flash` after the target segment for the flasher-only view. */ const TARGET_TAIL = /\/target\/([^/]+)$/ +const PLATFORM_TAIL = /\/platform\/([^/]+)$/ const FLASH_AFTER_TARGET = /\/target\/[^/]+\/flash$/ export function parseTreeSplat(treePath: string | undefined): { sourceRef: string | null + platformKey: string | null targetEnv: string | null flash: boolean } { - if (!treePath?.trim()) return { sourceRef: null, targetEnv: null, flash: false } + if (!treePath?.trim()) return { sourceRef: null, platformKey: null, targetEnv: null, flash: false } const segments = treePath.split("/").filter(Boolean) - if (segments.length === 0) return { sourceRef: null, targetEnv: null, flash: false } + if (segments.length === 0) return { sourceRef: null, platformKey: null, targetEnv: null, flash: false } let joined = segments.map(s => decodeURIComponent(s)).join("/") let flash = false if (FLASH_AFTER_TARGET.test(joined)) { flash = true joined = joined.slice(0, -"/flash".length) } - const m = TARGET_TAIL.exec(joined) - if (!m) return { sourceRef: joined, targetEnv: null, flash } - const sourceRef = joined.slice(0, m.index).replace(/\/$/, "") || null - const targetEnv = m[1] ? decodeURIComponent(m[1]) : null - return { sourceRef, targetEnv, flash } + const tm = TARGET_TAIL.exec(joined) + let targetEnv: string | null = null + if (tm) { + targetEnv = decodeURIComponent(tm[1]) + joined = joined.slice(0, tm.index) + } + const pm = PLATFORM_TAIL.exec(joined) + let platformKey: string | null = null + if (pm) { + platformKey = decodeURIComponent(pm[1]) + joined = joined.slice(0, pm.index) + } + const sourceRef = joined.replace(/\/$/, "").trim() || null + return { sourceRef, platformKey, targetEnv, flash } } /** Path after `/tree/` (no leading slash). Empty string means no ref — short `/owner/repo` redirects to latest tag. */ export function buildTreeSplatPath( sourceRef: string | null, targetEnv: string | null, - opts?: { flash?: boolean } + opts?: { flash?: boolean; platform?: string | null } ): string { const b = sourceRef?.trim() if (!b) return "" @@ -38,9 +51,16 @@ export function buildTreeSplatPath( .split("/") .map(s => encodeURIComponent(s)) .join("/") + let mid = enc + const plat = opts?.platform?.trim() + if (plat) { + mid = `${enc}/platform/${encodeURIComponent(plat)}` + } const t = targetEnv?.trim() - if (!t) return enc - const base = `${enc}/target/${encodeURIComponent(t)}` + if (!t) { + return mid + } + const base = `${mid}/target/${encodeURIComponent(t)}` if (opts?.flash) return `${base}/flash` return base } diff --git a/src/pages/RepoPage.tsx b/src/pages/RepoPage.tsx index b62398b..994b766 100644 --- a/src/pages/RepoPage.tsx +++ b/src/pages/RepoPage.tsx @@ -23,6 +23,7 @@ import DeviceFlasher from "../components/DeviceFlasher" import { normalizeBuildKey } from "../lib/buildKey" import { buildFailurePresentation } from "../lib/formatBuildErrorSummary" import { homepageHref } from "../lib/githubHomepage" +import { mergeEffectiveMeshforgeConfig, meshforgePlatformKeys } from "@/convex/lib/meshforgeYaml" import { filterEnvNames, filterTagNames, type MeshforgeConfig } from "../lib/meshforgeApplyProfile" import { resolveReadmeRelativeUrl } from "../lib/readmeAssetUrl" import { buildTreeSplatPath, parseTreeSplat } from "../lib/repoTreeUrl" @@ -60,6 +61,7 @@ export default function RepoPage() { const { sourceRef, targetEnv: targetFromUrl, + platformKey: platformFromUrl, flash: flashFromUrl, } = useMemo(() => parseTreeSplat(treePath), [treePath]) const isFlashView = flashFromUrl @@ -81,6 +83,8 @@ export default function RepoPage() { /** Git ref from URL only (null = short `/owner/repo` → redirect to latest SemVer tag). */ const effectiveRef = sourceRef + const tagRowReady = tagData !== undefined && tagData.row !== null + useEffect(() => { if (!owner || !repo || tagData === undefined) return if (tagData.row !== null && !tagData.isStale) return @@ -99,23 +103,34 @@ export default function RepoPage() { const candidates = cfg ? filterTagNames(allSorted, cfg) : allSorted const latest = candidates[0] if (latest) { - navigate(`/${ownerParam}/${repoParam}/tree/${buildTreeSplatPath(latest, null)}`, { replace: true }) + const pk = meshforgePlatformKeys(cfg as MeshforgeConfig) + navigate( + `/${ownerParam}/${repoParam}/tree/${buildTreeSplatPath(latest, null, pk[0] ? { platform: pk[0] } : undefined)}`, + { replace: true } + ) return } // Profile filtered out all tags; use default branch when available. if (defaultBranch) { - navigate(`/${ownerParam}/${repoParam}/tree/${buildTreeSplatPath(defaultBranch, null)}`, { replace: true }) + const pk = meshforgePlatformKeys(cfg as MeshforgeConfig) + navigate( + `/${ownerParam}/${repoParam}/tree/${buildTreeSplatPath(defaultBranch, null, pk[0] ? { platform: pk[0] } : undefined)}`, + { replace: true } + ) } } else { // No tags — redirect to the repo's default branch if known if (!defaultBranch) return - navigate(`/${ownerParam}/${repoParam}/tree/${buildTreeSplatPath(defaultBranch, null)}`, { replace: true }) + const pk = meshforgePlatformKeys((tagData.row?.meshforgeConfig ?? null) as MeshforgeConfig | null) + navigate( + `/${ownerParam}/${repoParam}/tree/${buildTreeSplatPath(defaultBranch, null, pk[0] ? { platform: pk[0] } : undefined)}`, + { replace: true } + ) } }, [owner, repo, sourceRef, tagData, navigate, ownerParam, repoParam]) const [resolvedSha, setResolvedSha] = useState(null) const [refError, setRefError] = useState(null) - const [pendingTagRefreshValidation, setPendingTagRefreshValidation] = useState(false) const [isRefreshingRepo, setIsRefreshingRepo] = useState(false) const [refreshResolveTick, setRefreshResolveTick] = useState(0) const [readmeRefreshTick, setReadmeRefreshTick] = useState(0) @@ -141,32 +156,114 @@ export default function RepoPage() { } }, [owner, repo, effectiveRef, resolveRef, navigate, ownerParam, repoParam, refreshResolveTick]) - useEffect(() => { - if (!pendingTagRefreshValidation || !sourceRef || tagData === undefined) return - setPendingTagRefreshValidation(false) - const tags = tagData.row?.tags ?? [] - const defaultBranch = (tagData.row as { defaultBranch?: string } | null | undefined)?.defaultBranch - const normalizedSourceRef = sourceRef.toLowerCase() - const refStillExists = - tags.some(t => t.name.toLowerCase() === normalizedSourceRef) || - defaultBranch?.toLowerCase() === normalizedSourceRef - if (!refStillExists) { - navigate(`/${ownerParam}/${repoParam}`, { replace: true }) - } - }, [pendingTagRefreshValidation, sourceRef, tagData, navigate, ownerParam, repoParam]) - - useEffect(() => { - if (!owner || !repo || !effectiveRef || !resolvedSha) return - void ensureScan({ owner, repo, ref: effectiveRef, resolvedSourceSha: resolvedSha }).catch(e => - toast.error(String(e)) - ) - }, [owner, repo, effectiveRef, resolvedSha, ensureScan]) + /** Submodule/platform segment in the URL — drives scan row identity (no circular wait on meshforge). */ + const scanPlatformRoot = useMemo(() => (platformFromUrl ?? "").trim(), [platformFromUrl]) const scan = useQuery( api.repoScans.getByRepoSha, - resolvedSha ? { owner, repo, resolvedSourceSha: resolvedSha } : "skip" + resolvedSha && tagRowReady + ? { owner, repo, resolvedSourceSha: resolvedSha, platformRoot: scanPlatformRoot } + : "skip" ) + useEffect(() => { + if (!isRefreshingRepo) return + // Keep refresh active until the current ref re-resolves and scan settles. + if (refError) { + setIsRefreshingRepo(false) + return + } + if (!effectiveRef) { + setIsRefreshingRepo(false) + return + } + if (!resolvedSha) return + if (scan == null || scan.scanStatus === "in_progress") return + setIsRefreshingRepo(false) + }, [isRefreshingRepo, refError, effectiveRef, resolvedSha, scan]) + + const tagBootstrapMeshforge = (tagData?.row?.meshforgeConfig ?? null) as MeshforgeConfig | null + const scanMeshforgeReady = scan?.scanStatus === "complete" + const hasAuthoritativeProfile = scanMeshforgeReady + const meshforgeConfig = useMemo(() => { + if (scanMeshforgeReady) { + // Once a ref/platform scan completes, prefer its meshforge.yaml snapshot even if null. + return (scan?.meshforgeConfig ?? null) as MeshforgeConfig | null + } + return tagBootstrapMeshforge + }, [scanMeshforgeReady, scan?.meshforgeConfig, tagBootstrapMeshforge]) + + const platformMenuKeys = useMemo(() => meshforgePlatformKeys(meshforgeConfig), [meshforgeConfig]) + const resolvedPlatformKey = useMemo(() => { + if (!platformMenuKeys.length) return "" + const p = (platformFromUrl ?? "").trim() + if (p && platformMenuKeys.includes(p)) return p + return "" + }, [platformMenuKeys, platformFromUrl]) + + const effectiveMeshforgeConfig = useMemo(() => { + const m = mergeEffectiveMeshforgeConfig(meshforgeConfig, resolvedPlatformKey || null) + return m ?? ({} as MeshforgeConfig) + }, [meshforgeConfig, resolvedPlatformKey]) + + useLayoutEffect(() => { + if (!owner || !repo || !sourceRef) return + // Avoid URL auto-correction while scan is still bootstrapping from default-branch profile data. + if (!scanMeshforgeReady) return + if (!platformMenuKeys.length) return + if (resolvedPlatformKey) return + const defaultP = platformMenuKeys[0] + navigate( + `/${ownerParam}/${repoParam}/tree/${buildTreeSplatPath(sourceRef, targetFromUrl, { platform: defaultP })}`, + { replace: true } + ) + }, [ + owner, + repo, + sourceRef, + targetFromUrl, + scanMeshforgeReady, + platformMenuKeys, + resolvedPlatformKey, + navigate, + ownerParam, + repoParam, + ]) + + /** Plain PlatformIO repos: meshforge has no `platforms` — drop a stray `/platform/...` from the URL. */ + useLayoutEffect(() => { + if (!owner || !repo || !sourceRef) return + if (platformMenuKeys.length > 0) return + if (!platformFromUrl) return + navigate( + `/${ownerParam}/${repoParam}/tree/${buildTreeSplatPath(sourceRef, targetFromUrl, flashFromUrl ? { flash: true } : undefined)}`, + { replace: true } + ) + }, [ + owner, + repo, + sourceRef, + targetFromUrl, + platformFromUrl, + platformMenuKeys.length, + flashFromUrl, + navigate, + ownerParam, + repoParam, + ]) + + useEffect(() => { + if (!owner || !repo || !effectiveRef || !resolvedSha) return + if (!tagRowReady) return + void ensureScan({ + owner, + repo, + ref: effectiveRef, + resolvedSourceSha: resolvedSha, + platformRoot: scanPlatformRoot, + }).catch(e => toast.error(String(e))) + }, [owner, repo, effectiveRef, resolvedSha, ensureScan, tagRowReady, scanPlatformRoot]) + const [readmeMd, setReadmeMd] = useState(null) const [readmeDownloadUrl, setReadmeDownloadUrl] = useState(null) useEffect(() => { @@ -260,15 +357,16 @@ export default function RepoPage() { return sortTagNames([...raw, ...extra]) }, [tagData?.row, sourceRef]) - // meshforgeConfig comes from the default branch (stored on the tag list, available before - // a tag is selected so the tag dropdown itself can be filtered). - const meshforgeConfig = (tagData?.row?.meshforgeConfig ?? null) as MeshforgeConfig | null const envCapabilities = (scan?.scanStatus === "complete" ? scan.envCapabilities : null) as Record< string, string[] > | null + const profileMatchedTagOptions = useMemo( + () => filterTagNames(tagOptions, effectiveMeshforgeConfig), + [tagOptions, effectiveMeshforgeConfig] + ) const filteredTagOptions = useMemo(() => { - const filtered = filterTagNames(tagOptions, meshforgeConfig ?? {}) + const filtered = profileMatchedTagOptions // Always keep sourceRef and defaultBranch selectable even if the profile filter drops them const defaultBranch = (tagData?.row as { defaultBranch?: string } | null | undefined)?.defaultBranch const reinjected = new Set(filtered.map(n => n.toLowerCase())) @@ -288,7 +386,7 @@ export default function RepoPage() { } } return extras.length > 0 ? sortTagNames([...filtered, ...extras]) : filtered - }, [tagOptions, meshforgeConfig, sourceRef, tagData?.row]) + }, [profileMatchedTagOptions, sourceRef, tagData?.row]) const [refShaByName, setRefShaByName] = useState>({}) useEffect(() => { setRefShaByName({}) @@ -332,8 +430,8 @@ export default function RepoPage() { return sha ? `${name} (${sha.slice(0, 7)})` : name } const filteredEnvNames = useMemo( - () => filterEnvNames(envNames, meshforgeConfig, envCapabilities ?? {}, tagDraft), - [envNames, meshforgeConfig, envCapabilities, tagDraft] + () => filterEnvNames(envNames, effectiveMeshforgeConfig, envCapabilities ?? {}, tagDraft), + [envNames, effectiveMeshforgeConfig, envCapabilities, tagDraft] ) const resolvedTargetEnv = @@ -354,11 +452,28 @@ export default function RepoPage() { if (!sourceRef || !targetFromUrl) return if (scan?.scanStatus !== "complete") return if (envNames.length === 0 || !envNames.includes(targetFromUrl)) { - navigate(`/${ownerParam}/${repoParam}/tree/${buildTreeSplatPath(sourceRef, null)}`, { replace: true }) + navigate( + `/${ownerParam}/${repoParam}/tree/${buildTreeSplatPath(sourceRef, null, { + platform: resolvedPlatformKey || undefined, + })}`, + { replace: true } + ) } - }, [sourceRef, targetFromUrl, envNames, scan?.scanStatus, navigate, ownerParam, repoParam]) + }, [ + sourceRef, + targetFromUrl, + envNames, + scan?.scanStatus, + navigate, + ownerParam, + repoParam, + resolvedPlatformKey, + ]) - const buildKey = resolvedSha && resolvedTargetEnv ? normalizeBuildKey(resolvedSha, resolvedTargetEnv) : null + const buildKey = + resolvedSha && resolvedTargetEnv && (!platformMenuKeys.length || resolvedPlatformKey) + ? normalizeBuildKey(resolvedSha, resolvedTargetEnv, resolvedPlatformKey) + : null const build = useQuery(api.repoBuilds.getByBuildKey, buildKey && isFlashView ? { buildKey } : "skip") /** Only show CI failure UI if this tab saw the build move into `failed` (not for stale failures on load). */ @@ -430,12 +545,14 @@ export default function RepoPage() { useEffect(() => { if (!isFlashView || !owner || !repo || !effectiveRef || !resolvedSha || !resolvedTargetEnv) return if (!(hasRef && resolvedSha && scan?.scanStatus === "complete" && envNames.length > 0)) return + if (platformMenuKeys.length > 0 && !resolvedPlatformKey) return void ensureBuild({ owner, repo, ref: effectiveRef, resolvedSourceSha: resolvedSha, targetEnv: resolvedTargetEnv, + platformRoot: resolvedPlatformKey, }).catch(e => toast.error(String(e))) }, [ isFlashView, @@ -448,13 +565,18 @@ export default function RepoPage() { scan, envNames.length, ensureBuild, + platformMenuKeys.length, + resolvedPlatformKey, ]) const queueFlashArtifacts = () => { if (!effectiveRef || !resolvedSha || !resolvedTargetEnv) return const goFlash = () => navigate( - `/${ownerParam}/${repoParam}/tree/${buildTreeSplatPath(effectiveRef, resolvedTargetEnv, { flash: true })}` + `/${ownerParam}/${repoParam}/tree/${buildTreeSplatPath(effectiveRef, resolvedTargetEnv, { + flash: true, + platform: resolvedPlatformKey || undefined, + })}` ) void ensureBuild({ owner, @@ -462,6 +584,7 @@ export default function RepoPage() { ref: effectiveRef, resolvedSourceSha: resolvedSha, targetEnv: resolvedTargetEnv, + platformRoot: resolvedPlatformKey, }) .then(res => { if (res.status === "failed") { @@ -516,7 +639,13 @@ export default function RepoPage() { const ghAboutDescription = tagData.row.description?.trim() ?? "" const ghAboutHomepage = tagData.row.homepage?.trim() ?? "" - const scanReady = Boolean(hasRef && resolvedSha && scan?.scanStatus === "complete" && envNames.length > 0) + const scanReady = Boolean( + hasRef && + resolvedSha && + scan?.scanStatus === "complete" && + envNames.length > 0 && + (!platformMenuKeys.length || Boolean(resolvedPlatformKey)) + ) const buildInProgress = Boolean(build && (build.status === "queued" || build.status === "running")) const flashPrimaryDisabled = !hasRef || @@ -542,11 +671,16 @@ export default function RepoPage() { : "No targets match profile" : "--target--" - const backToRepoPath = `/${ownerParam}/${repoParam}/tree/${buildTreeSplatPath(sourceRef, resolvedTargetEnv || null)}` + const backToRepoPath = `/${ownerParam}/${repoParam}/tree/${buildTreeSplatPath(sourceRef, resolvedTargetEnv || null, { + platform: resolvedPlatformKey || undefined, + })}` const statusStripEl = (
{tagData?.isStale ? Tag list may be stale. : null} + {hasAuthoritativeProfile && tagOptions.length > 0 && profileMatchedTagOptions.length === 0 ? ( + No refs match current profile. + ) : null} {refError ? {refError} : null} {!refError && hasRef && !resolvedSha ? Resolving tag… : null} {resolvedSha && (scan == null || scan.scanStatus === "in_progress") ? Scanning PlatformIO… : null} @@ -720,6 +854,14 @@ export default function RepoPage() {
{effectiveRef}
+ {resolvedPlatformKey ? ( +
+
+ Platform +
+
{resolvedPlatformKey}
+
+ ) : null}
Target @@ -758,11 +900,37 @@ export default function RepoPage() { return } if (filteredTagOptions.includes(v)) { - navigate(`/${ownerParam}/${repoParam}/tree/${buildTreeSplatPath(v, targetFromUrl)}`) + navigate( + `/${ownerParam}/${repoParam}/tree/${buildTreeSplatPath(v, targetFromUrl, { + platform: resolvedPlatformKey || undefined, + })}` + ) } }} disabled={filteredTagOptions.length === 0} /> + {hasRef && platformMenuKeys.length > 0 ? ( + { + if (!sourceRef) return + if (v === "") return + if (platformMenuKeys.includes(v)) { + navigate( + `/${ownerParam}/${repoParam}/tree/${buildTreeSplatPath(sourceRef, targetFromUrl, { + platform: v, + })}` + ) + } + }} + disabled={!resolvedSha} + /> + ) : null} {hasRef && scanReady && filteredEnvNames.length > 0 ? ( { if (isRefreshingRepo) return // Force re-resolve current ref so moving branches pick up latest SHA. + setResolvedSha(null) + setRefError(null) setRefreshResolveTick(t => t + 1) setReadmeRefreshTick(t => t + 1) setIsRefreshingRepo(true) - void refreshTags({ owner, repo }) - .then(() => { - setPendingTagRefreshValidation(true) - }) - .catch(e => toast.error(String(e))) - .finally(() => setIsRefreshingRepo(false)) + void refreshTags({ owner, repo }).catch(e => { + toast.error(String(e)) + setIsRefreshingRepo(false) + }) }} >