diff --git a/README.md b/README.md index 5500ad1..775b462 100644 --- a/README.md +++ b/README.md @@ -31,9 +31,9 @@ First user signup becomes admin. Configure GitHub and Gitea/Forgejo through the ## ✨ Features - 🔁 Mirror public, private, and starred GitHub repos to Gitea/Forgejo -- 🦊 **GitLab (beta) and Gitea/Forgejo sources** - Pick the source in the configuration card. gitlab.com, Codeberg and self hosted instances mirror code, tags, wiki and LFS (issues, pull requests and releases need a GitHub source). See [docs/SOURCE_PROVIDERS.md](docs/SOURCE_PROVIDERS.md) +- 🦊 **GitLab and Gitea/Forgejo sources (beta)** - Pick the source in the configuration card. gitlab.com, Codeberg and self hosted instances mirror code, tags, wiki and LFS (issues, pull requests and releases need a GitHub source). GitHub to Gitea is the supported default; every other source or destination is beta. See [docs/SOURCE_PROVIDERS.md](docs/SOURCE_PROVIDERS.md) - 🏛️ **GitHub Enterprise support** - Works with GHES and GHEC with data residency via `GH_API_URL` -- 🚀 **GitHub and GitLab destinations (beta)** - Pick GitHub or GitLab as the destination and the app keeps a bare clone of each repository and pushes its branches and tags there. Code only, and the target is overwritten on every sync. See [docs/PUSH_TARGETS.md](docs/PUSH_TARGETS.md) +- 🚀 **Forgejo, GitHub and GitLab destinations (beta)** - Forgejo works like Gitea. Pick GitHub or GitLab and the app keeps a bare clone of each repository and pushes its branches and tags there. Code only, and the target is overwritten on every sync. See [docs/PUSH_TARGETS.md](docs/PUSH_TARGETS.md) - 🏢 Mirror entire organizations with flexible strategies - 🎯 Custom destination control for repos and organizations - 📦 **Git LFS support** - Mirror large files with Git LFS @@ -205,7 +205,7 @@ bun run dev 1. **First Time Setup** - Navigate to http://localhost:4321 - Create admin account (first user signup) - - Configure the source and destination connections. The destination is Gitea or Forgejo (pull mirrors), or GitHub or GitLab (push targets, beta) + - Configure the source and destination connections. GitHub to Gitea is the default. Forgejo works like Gitea, and GitHub or GitLab as destination uses push targets. Everything other than GitHub to Gitea is beta 2. **Mirror Strategies** - **Preserve Structure**: Maintains GitHub organization structure diff --git a/docs/API.md b/docs/API.md index c863913..6124da0 100644 --- a/docs/API.md +++ b/docs/API.md @@ -144,6 +144,7 @@ Nothing is deleted or archived on either side by this call; the cleanup service "notManaged": [{ "location": "me/notes", "reason": "not a mirror" }], "unverified": [], "healthyCount": 41, + "elsewhereCount": 0, "scannedOwners": ["e2e_admin", "github-mirrors"], "skippedOwners": ["starred"], "totalOnDestination": 43 diff --git a/docs/ENVIRONMENT_VARIABLES.md b/docs/ENVIRONMENT_VARIABLES.md index 7f97841..27e5a6b 100644 --- a/docs/ENVIRONMENT_VARIABLES.md +++ b/docs/ENVIRONMENT_VARIABLES.md @@ -69,13 +69,13 @@ Notes: ## GitHub Configuration -Settings for connecting to the source host. GitHub is the default. GitLab and Gitea/Forgejo (including Codeberg) are selected with `SOURCE_PROVIDER`; the username and token variables below then hold the account and token for that host. See [SOURCE_PROVIDERS.md](SOURCE_PROVIDERS.md) for what each source supports. +Settings for connecting to the source host. GitHub is the default and, together with a Gitea destination, the supported path; every other source or destination is beta. GitLab and Gitea/Forgejo (including Codeberg) are selected with `SOURCE_PROVIDER`; the username and token variables below then hold the account and token for that host. See [SOURCE_PROVIDERS.md](SOURCE_PROVIDERS.md) for what each source supports. ### Basic Settings | Variable | Description | Default | Options | |----------|-------------|---------|---------| -| `SOURCE_PROVIDER` | Which host repositories are pulled from. `gitea` also covers Forgejo and Codeberg. | `github` | `github`, `gitlab`, `gitea` | +| `SOURCE_PROVIDER` | Which host repositories are pulled from. `gitea` also covers Forgejo and Codeberg. Anything other than `github` is beta. | `github` | `github`, `gitlab`, `gitea` | | `SOURCE_URL` | Base URL of the GitLab or Gitea/Forgejo instance. Ignored for GitHub, which uses `GH_API_URL`. | `https://gitlab.com` for GitLab, `https://codeberg.org` for Gitea | e.g. `https://gitlab.example.com` | | `GITHUB_USERNAME` | Your username on the source host | - | - | | `GITHUB_TOKEN` | Personal access token for the source host. GitHub needs the `repo` and `admin:org` scopes, GitLab `read_api` and `read_repository`, Gitea/Forgejo `read:repository`, `read:user` and `read:organization`. | - | - | @@ -155,7 +155,7 @@ Settings for the destination. Gitea and Forgejo are pull mirrors; GitHub and Git | Variable | Description | Default | Options | |----------|-------------|---------|---------| -| `DESTINATION_PROVIDER` | Which host mirrors are created on. Gitea and Forgejo share one API (labels and hints differ). GitHub and GitLab (beta) receive `git push` of branches and tags from this app. | `gitea` | `gitea`, `forgejo`, `github`, `gitlab` | +| `DESTINATION_PROVIDER` | Which host mirrors are created on. Gitea and Forgejo share one API (labels and hints differ). GitHub and GitLab receive `git push` of branches and tags from this app. Anything other than `gitea` is beta. | `gitea` | `gitea`, `forgejo`, `github`, `gitlab` | | `GITEA_URL` | Destination instance URL. Optional for GitHub and GitLab, which default to github.com and gitlab.com; set it for GitHub Enterprise Server or a self hosted GitLab. | - | Valid URL | | `GITEA_EXTERNAL_URL` | Optional external/browser URL used for dashboard links. API and mirroring still use `GITEA_URL`. | - | Valid URL | | `GITEA_TOKEN` | Gitea access token | - | - | diff --git a/docs/PUSH_TARGETS.md b/docs/PUSH_TARGETS.md index 35d744b..83476cc 100644 --- a/docs/PUSH_TARGETS.md +++ b/docs/PUSH_TARGETS.md @@ -2,7 +2,7 @@ Gitea and Forgejo are pull mirrors: the app asks them to fetch from the source, and they keep the copy up to date themselves. GitHub has no pull mirror API and GitLab only offers one on paid tiers, so for those two the app runs a push engine instead. This page explains what that engine does, what it needs, and what it does not do. -Push targets are marked **beta** in the destination dropdown. +Push targets are marked **beta** in the destination dropdown, like every source and destination other than GitHub to Gitea. They are tested end to end against github.com and gitlab.com, but have had less time in the field than the default path. ## How it works diff --git a/docs/SOURCE_PROVIDERS.md b/docs/SOURCE_PROVIDERS.md index 70e63be..549b8c1 100644 --- a/docs/SOURCE_PROVIDERS.md +++ b/docs/SOURCE_PROVIDERS.md @@ -1,6 +1,6 @@ # Source Providers -The destination card has a matching **Destination** dropdown with Gitea and Forgejo. They share the same API, so it only changes the name, icon and hints shown on the card (`DESTINATION_PROVIDER` sets it from the environment). +GitHub to Gitea is the supported default. Every other source and every other destination is marked **beta** in the dropdowns: they are tested end to end against real hosts, but have had less time in the field. The destination card has a matching **Destination** dropdown with Gitea, Forgejo (beta, same API as Gitea), GitHub (beta) and GitLab (beta); `DESTINATION_PROVIDER` sets it from the environment, and [PUSH_TARGETS.md](PUSH_TARGETS.md) covers the GitHub and GitLab targets. Gitea Mirror pulls repositories from one source host per user. GitHub is the default. GitLab and Gitea/Forgejo are available from the **Source** dropdown at the top of the connection card on the Configuration page. @@ -8,9 +8,9 @@ Gitea Mirror pulls repositories from one source host per user. GitHub is the def |--------|--------|----------------------| | GitHub | github.com, GitHub Enterprise (via `GH_API_URL`) | `https://github.com` | | GitLab (beta) | gitlab.com, self hosted GitLab | `https://gitlab.com` | -| Gitea / Forgejo | Codeberg, self hosted Gitea or Forgejo | `https://codeberg.org` | +| Gitea / Forgejo (beta) | Codeberg, self hosted Gitea or Forgejo | `https://codeberg.org` | -GitLab support is marked beta: the adapter is tested against the public API and mocked responses, not yet against a private self hosted instance. Picking GitLab or Gitea/Forgejo shows an **Instance URL** field. Leave it empty for the default instance, or enter the base URL of your own (for example `https://gitlab.example.com` or `http://gitea.local:3000/gitea`). The username and token fields hold the account and token for the selected host. +Picking GitLab or Gitea/Forgejo shows an **Instance URL** field. Leave it empty for the default instance, or enter the base URL of your own (for example `https://gitlab.example.com` or `http://gitea.local:3000/gitea`). The username and token fields hold the account and token for the selected host. ## Tokens diff --git a/src/components/config/GitHubConfigForm.tsx b/src/components/config/GitHubConfigForm.tsx index bf1b721..882edd7 100644 --- a/src/components/config/GitHubConfigForm.tsx +++ b/src/components/config/GitHubConfigForm.tsx @@ -36,6 +36,7 @@ import { import { SOURCE_PROVIDER_DEFAULT_URLS, SOURCE_PROVIDER_LABELS, + isBetaSourceProvider, } from "@/lib/source-providers/kinds"; import { GitHubMirrorSettings } from "./GitHubMirrorSettings"; import { HostLockNotice } from "./HostLockNotice"; @@ -142,7 +143,7 @@ const sourceProviders: SourceProviderMeta[] = [ value: "gitlab", label: SOURCE_PROVIDER_LABELS.gitlab, icon: SiGitlab, - badge: "BETA", + badge: isBetaSourceProvider("gitlab") ? "BETA" : undefined, usernamePlaceholder: "Your GitLab username", tokenPlaceholder: "Your GitLab personal access token", tokenHint: "Needed for private projects, groups, and starred projects", @@ -158,6 +159,7 @@ const sourceProviders: SourceProviderMeta[] = [ value: "gitea", label: SOURCE_PROVIDER_LABELS.gitea, icon: SiGitea, + badge: isBetaSourceProvider("gitea") ? "BETA" : undefined, usernamePlaceholder: "Your Gitea or Forgejo username", tokenPlaceholder: "Your access token", tokenHint: "Needed for private repos, organizations, and starred repositories", @@ -324,7 +326,7 @@ export function GitHubConfigForm({ ? "Where your repositories are pulled from" : provider === "gitlab" ? "Beta. Code, tags, wiki and LFS are mirrored. Issues, merge requests and releases need a GitHub source." - : "Code, tags, wiki and LFS are mirrored. Issues, pull requests and releases need a GitHub source."} + : "Beta. Code, tags, wiki and LFS are mirrored. Issues, pull requests and releases need a GitHub source."}

{sourceLock?.locked && ( ({ ...option, badge: isBetaDestinationProvider(option.value) ? "BETA" : undefined })); /** The public instance URLs a hosted target is prefilled with. */ const HOSTED_DEFAULT_URLS = new Set(Object.values(DESTINATION_PROVIDER_DEFAULT_URLS)); @@ -318,7 +319,9 @@ export function GiteaConfigForm({ config, setConfig, onAutoSave, isAutoSaving, g

{isPushTarget ? `${providerLabel} has no pull mirror API, so this app keeps a bare clone of each source repository and pushes its branches and tags there.` - : "Where mirrors are created. Gitea and Forgejo share the same API."} + : provider === "forgejo" + ? "Beta. Where mirrors are created. Forgejo shares Gitea's API, so everything Gitea supports works here." + : "Where mirrors are created."}

{destinationLock?.locked && ( ; unverified: Array<{ fullName: string; location: string; error: string }>; healthyCount: number; + /** Rows mirrored to another destination host, left out of the comparison. */ + elsewhereCount?: number; scannedOwners: string[]; skippedOwners: string[]; totalOnDestination: number; @@ -194,6 +196,13 @@ export function ReconcileDialog({ open, onOpenChange }: ReconcileDialogProps) { {report.totalOnDestination} repositor{report.totalOnDestination === 1 ? "y" : "ies"} under{" "} {report.scannedOwners.length} owner{report.scannedOwners.length === 1 ? "" : "s"} on the destination,{" "} {report.healthyCount} tracked and present. + {(report.elsewhereCount ?? 0) > 0 && ( + <> + {" "} + {report.elsewhereCount} row{report.elsewhereCount === 1 ? " is" : "s are"} mirrored to a previous + destination and {report.elsewhereCount === 1 ? "was" : "were"} left out. + + )} {report.skippedOwners.length > 0 && ( <> Not found there: {report.skippedOwners.join(", ")}. )} diff --git a/src/lib/destination-kinds.test.ts b/src/lib/destination-kinds.test.ts index 3e6052f..0e875e9 100644 --- a/src/lib/destination-kinds.test.ts +++ b/src/lib/destination-kinds.test.ts @@ -1,5 +1,6 @@ import { describe, expect, test } from "bun:test"; import { + isBetaDestinationProvider, describeDestination, destinationHostOf, getRepositoryDestination, @@ -11,6 +12,13 @@ import { } from "./destination-kinds"; describe("destination kinds", () => { + test("Gitea is the supported default; every other destination is beta", () => { + expect(isBetaDestinationProvider("gitea")).toBe(false); + expect(isBetaDestinationProvider("forgejo")).toBe(true); + expect(isBetaDestinationProvider("github")).toBe(true); + expect(isBetaDestinationProvider("gitlab")).toBe(true); + }); + test("recognizes the four kinds and maps codeberg to forgejo", () => { expect(isDestinationProviderKind("gitea")).toBe(true); expect(isDestinationProviderKind("forgejo")).toBe(true); diff --git a/src/lib/destination-kinds.ts b/src/lib/destination-kinds.ts index ea7c9ce..b094f88 100644 --- a/src/lib/destination-kinds.ts +++ b/src/lib/destination-kinds.ts @@ -19,6 +19,11 @@ export type PushDestinationKind = (typeof PUSH_DESTINATION_KINDS)[number]; export const DEFAULT_DESTINATION_PROVIDER: DestinationProviderKind = "gitea"; +/** Destinations other than Gitea are beta: they work end to end but have had less time in the field. */ +export function isBetaDestinationProvider(kind: DestinationProviderKind): boolean { + return kind !== DEFAULT_DESTINATION_PROVIDER; +} + export const DESTINATION_PROVIDER_LABELS: Record = { gitea: "Gitea", forgejo: "Forgejo", diff --git a/src/lib/destination-reconcile-service.ts b/src/lib/destination-reconcile-service.ts index 40b62ef..88b01c3 100644 --- a/src/lib/destination-reconcile-service.ts +++ b/src/lib/destination-reconcile-service.ts @@ -18,6 +18,7 @@ import { getDecryptedGiteaToken } from "@/lib/utils/config-encryption"; import { createSourceProviderFromConfig, resolveSourceConnection } from "@/lib/source-providers"; import { githubApiBaseUrl } from "@/lib/source-providers/github-source"; import { getGiteaRepoOwnerAsync } from "@/lib/gitea"; +import { resolveDestinationIdentity } from "@/lib/destination-connection"; import { normalizeGitRepoToInsert } from "@/lib/repo-utils"; import { createMirrorJob } from "@/lib/helpers"; import { processInParallel } from "@/lib/utils/concurrency"; @@ -28,6 +29,7 @@ import { knownSourceHosts, parseRepoUrl, type DestinationRepo, + splitRowsByDestination, } from "@/lib/destination-reconcile"; const PAGE_SIZE = 50; @@ -55,6 +57,8 @@ export interface ReconcileReport { /** Rows whose presence could not be confirmed because the check itself failed. */ unverified: Array<{ fullName: string; location: string; error: string }>; healthyCount: number; + /** Rows mirrored to another destination host; they are expected to be absent here. */ + elsewhereCount: number; scannedOwners: string[]; skippedOwners: string[]; totalOnDestination: number; @@ -229,7 +233,10 @@ export async function reconcileDestination( cloneUrls: rows.map((row) => row.cloneUrl), }); - const classified = classifyDestinationRepos({ destinationRepos, rows, knownHosts }); + // Rows mirrored to a previous destination are neither healthy nor missing + // here; they are left out of the comparison and only counted. + const { here: rowsHere, elsewhere } = splitRowsByDestination(rows, resolveDestinationIdentity(config)); + const classified = classifyDestinationRepos({ destinationRepos, rows: rowsHere, knownHosts }); const listedLocations = new Set(destinationRepos.map((repo) => repo.fullName.toLowerCase())); // Rows that claim a mirror the listing did not show. Confirm each one @@ -287,6 +294,7 @@ export async function reconcileDestination( notManaged: classified.notManaged, unverified, healthyCount: classified.matchedRowIds.size + confirmedPresent, + elsewhereCount: elsewhere.length, scannedOwners, skippedOwners, totalOnDestination: destinationRepos.length, diff --git a/src/lib/destination-reconcile.test.ts b/src/lib/destination-reconcile.test.ts index a2a59c3..fd96529 100644 --- a/src/lib/destination-reconcile.test.ts +++ b/src/lib/destination-reconcile.test.ts @@ -11,6 +11,7 @@ import { expectedLocation, knownSourceHosts, parseRepoUrl, + splitRowsByDestination, type DestinationRepo, type TrackedRepositoryRow, } from "./destination-reconcile"; @@ -259,3 +260,30 @@ describe("collectDestinationOwners", () => { ).toEqual(["Mirrors", "starred", "user"]); }); }); + +describe("splitRowsByDestination", () => { + const row = (id: string, extra: Partial = {}): TrackedRepositoryRow => ({ + id, + name: id, + fullName: `octo/${id}`, + cloneUrl: `https://github.com/octo/${id}.git`, + mirroredLocation: `mirror-org/${id}`, + status: "mirrored", + ...extra, + }); + const gitea = { provider: "gitea" as const, url: "http://gitea.local:3000" }; + + test("keeps rows recorded on the configured destination or with no recorded URL, sets the rest aside", () => { + const { here, elsewhere } = splitRowsByDestination( + [ + row("same", { destinationProvider: "gitea", destinationUrl: "http://gitea.local:3000" }), + row("legacy", { destinationProvider: "gitea", destinationUrl: null }), + row("pushed", { destinationProvider: "gitlab", destinationUrl: "https://gitlab.com" }), + row("other-gitea", { destinationProvider: "gitea", destinationUrl: "https://try.gitea.io" }), + ], + gitea + ); + expect(here.map((r) => r.id)).toEqual(["same", "legacy"]); + expect(elsewhere.map((r) => r.id)).toEqual(["pushed", "other-gitea"]); + }); +}); diff --git a/src/lib/destination-reconcile.ts b/src/lib/destination-reconcile.ts index 20aa0cf..eea832a 100644 --- a/src/lib/destination-reconcile.ts +++ b/src/lib/destination-reconcile.ts @@ -13,6 +13,11 @@ */ import { sourceHostOf } from "@/lib/source-providers/kinds"; +import { + isRepositoryOnConfiguredDestination, + type RepositoryDestination, + type RepositoryDestinationFields, +} from "@/lib/destination-kinds"; /** What the destination reports about one repository. */ export interface DestinationRepo { @@ -46,6 +51,30 @@ export interface TrackedRepositoryRow { cloneUrl: string; mirroredLocation: string | null; status: string; + destinationProvider?: RepositoryDestinationFields["destinationProvider"]; + destinationUrl?: RepositoryDestinationFields["destinationUrl"]; +} + +/** + * Split rows into the ones whose recorded destination is the configured + * one (or unknown) and the ones mirrored to another host. Only the first + * group can be missing from this destination; the second group is expected + * to be absent and must not be reset. + */ +export function splitRowsByDestination( + rows: T[], + destination: RepositoryDestination +): { here: T[]; elsewhere: T[] } { + const here: T[] = []; + const elsewhere: T[] = []; + for (const row of rows) { + const fields = { + destinationProvider: row.destinationProvider ?? null, + destinationUrl: row.destinationUrl ?? null, + }; + (isRepositoryOnConfiguredDestination(fields, destination) ? here : elsewhere).push(row); + } + return { here, elsewhere }; } export interface NotManagedRepo { diff --git a/src/lib/gitea.ts b/src/lib/gitea.ts index 4e91bbc..b76dcbb 100644 --- a/src/lib/gitea.ts +++ b/src/lib/gitea.ts @@ -30,6 +30,8 @@ import { resolveMirrorOptionsForRepository, type ResolvedMirrorOptions, } from "./utils/mirror-overrides"; +import { repositoryDestinationColumns } from "./repo-utils"; +import { resolveDestinationIdentity } from "./destination-connection"; /** * Helper function to get organization configuration including destination override @@ -814,6 +816,7 @@ export const mirrorGithubRepoToGitea = async ({ lastMirrored: new Date(), errorMessage: null, mirroredLocation: `${repoOwner}/${targetRepoName}`, + ...repositoryDestinationColumns(resolveDestinationIdentity(config)), }) .where(eq(repositories.id, repository.id!)); @@ -869,6 +872,7 @@ export const mirrorGithubRepoToGitea = async ({ .set({ status: repoStatusEnum.parse("mirroring"), mirroredLocation: targetLocation, + ...repositoryDestinationColumns(resolveDestinationIdentity(config)), updatedAt: new Date(), }) .where(eq(repositories.id, repository.id!)); @@ -1195,6 +1199,7 @@ export const mirrorGithubRepoToGitea = async ({ lastMirrored: new Date(), errorMessage: null, mirroredLocation: `${repoOwner}/${targetRepoName}`, + ...repositoryDestinationColumns(resolveDestinationIdentity(config)), metadata: metadataUpdated ? serializeRepositoryMetadataState(metadataState) : repository.metadata ?? null, @@ -1648,6 +1653,7 @@ export async function mirrorGitHubRepoToGiteaOrg({ lastMirrored: new Date(), errorMessage: null, mirroredLocation: `${targetOwner}/${targetRepoName}`, + ...repositoryDestinationColumns(resolveDestinationIdentity(config)), }) .where(eq(repositories.id, repository.id!)); @@ -1708,6 +1714,7 @@ export async function mirrorGitHubRepoToGiteaOrg({ .set({ status: repoStatusEnum.parse("mirroring"), mirroredLocation: targetLocation, + ...repositoryDestinationColumns(resolveDestinationIdentity(config)), updatedAt: new Date(), }) .where(eq(repositories.id, repository.id!)); @@ -1957,6 +1964,7 @@ export async function mirrorGitHubRepoToGiteaOrg({ lastMirrored: new Date(), errorMessage: null, mirroredLocation: `${orgName}/${targetRepoName}`, + ...repositoryDestinationColumns(resolveDestinationIdentity(config)), metadata: metadataUpdated ? serializeRepositoryMetadataState(metadataState) : repository.metadata ?? null, diff --git a/src/lib/push-engine/engine.test.ts b/src/lib/push-engine/engine.test.ts index 83448bb..6cbbdb6 100644 --- a/src/lib/push-engine/engine.test.ts +++ b/src/lib/push-engine/engine.test.ts @@ -10,6 +10,7 @@ import { countChanges, isHealthyBareClone, isRetryableInBatches, + isTargetNotReady, listRefs, parsePushPorcelain, removeClone, @@ -114,6 +115,48 @@ class LocalPushTarget implements PushTarget { } } +/** + * A target whose freshly created repository only exists after the first + * push attempt, the way GitLab answers "not found" for a moment after the + * create call returns. + */ +class LazyPushTarget extends LocalPushTarget { + pushAttempts = 0; + private pending: { owner: string; name: string } | null = null; + + constructor(root: string) { + super(root); + } + + override async ensureRepository(input: EnsureRepositoryInput): Promise { + const existing = await this.getRepository(input.owner, input.name); + if (existing) return existing; + this.pending = { owner: input.owner, name: input.name }; + const dir = path.join(this.baseUrl.replace(/^file:\/\//, ""), input.owner, `${input.name}.git`); + return { + owner: input.owner, + name: input.name, + pushUrl: `file://${dir}`, + htmlUrl: `file://${dir}`, + isPrivate: false, + archived: false, + created: true, + }; + } + + override pushCredentials() { + this.pushAttempts += 1; + if (this.pushAttempts === 2 && this.pending) { + const dir = path.join(this.baseUrl.replace(/^file:\/\//, ""), this.pending.owner, `${this.pending.name}.git`); + // Create the repository between the first and the second attempt. + Bun.spawnSync(["git", "init", "--bare", "--quiet", dir]); + this.createdRepositories.push(`${this.pending.owner}/${this.pending.name}`); + this.pending = null; + } + return null; + } +} + interface SourceRepo { bareDir: string; workDir: string; @@ -378,6 +421,26 @@ describe("helpers", () => { } }); + test("a repository that is not ready right after creation is pushed on a later attempt", async () => { + const lazy = new LazyPushTarget(path.join(root, "lazy-targets")); + const source = await createSourceRepo(root, "lazy"); + const outcome = await runPushMirror(planFor(source, "lazy", { target: lazy, readinessDelayMs: 10 })); + + expect(outcome.targetRepository.created).toBe(true); + expect(outcome.pushed).toBe(true); + expect(lazy.pushAttempts).toBe(2); + const targetRefs = await listRefs(path.join(root, "lazy-targets", "acme", "lazy.git")); + expect(targetRefs.size).toBe(3); + }); + + test("isTargetNotReady only matches the transient not-found answers", () => { + const notReady = new GitCommandError("git push failed", ["push"], 128, "remote: The project you were looking for could not be found or you don't have permission to view it."); + const auth = new GitCommandError("git push failed", ["push"], 128, "fatal: Authentication failed for 'https://x'"); + expect(isTargetNotReady(notReady)).toBe(true); + expect(isTargetNotReady(auth)).toBe(false); + expect(isTargetNotReady(new Error("not found"))).toBe(false); + }); + test("isRetryableInBatches keeps credential and missing repository failures out of the batch retry", () => { const auth = new GitCommandError("git push failed", ["push"], 128, "fatal: Authentication failed for 'https://x'"); const missing = new GitCommandError("git push failed", ["push"], 128, "remote: Repository not found."); diff --git a/src/lib/push-engine/engine.ts b/src/lib/push-engine/engine.ts index cd0b7ee..f7e2dd8 100644 --- a/src/lib/push-engine/engine.ts +++ b/src/lib/push-engine/engine.ts @@ -36,6 +36,8 @@ export interface PushMirrorPlan { description?: string | null; defaultBranch?: string | null; lockStaleAfterMs?: number; + /** Wait between pushes into a repository this run just created; defaults to READINESS_DELAY_MS. */ + readinessDelayMs?: number; log?: (message: string) => void; } @@ -86,7 +88,9 @@ async function execute(plan: PushMirrorPlan): Promise { log(`created ${targetRepository.owner}/${targetRepository.name} on ${plan.target.baseUrl}`); } - const { pushed, batches } = await pushAll(plan, targetRepository.pushUrl, refsAfter, log); + const { pushed, batches } = targetRepository.created + ? await pushWhenReady(plan, targetRepository.pushUrl, refsAfter, log) + : await pushAll(plan, targetRepository.pushUrl, refsAfter, log); log(pushed ? `pushed to ${targetRepository.htmlUrl}` : `${targetRepository.htmlUrl} was already up to date`); const wanted = plan.defaultBranch?.trim(); @@ -205,7 +209,8 @@ export function parsePushPorcelain(output: string): { updated: number; upToDate: return { updated, upToDate, rejected }; } -const NON_RETRYABLE_PUSH = /authentication failed|could not read username|permission|denied|not found|repository .* does not exist|403|401|archived|read-only/i; +const NON_RETRYABLE_PUSH = + /authentication failed|could not read username|permission|denied|not found|repository .* does not exist|does not appear to be a git repository|could not read from remote repository|403|401|archived|read-only/i; /** Failures that batching cannot help with: bad credentials, missing repository. */ export function isRetryableInBatches(error: unknown): boolean { @@ -261,6 +266,40 @@ async function pushAll( return { pushed: true, batches }; } +/** How a host answers a push that arrives before a just created repository is ready. */ +const TARGET_NOT_READY = /not found|could not be found|does not exist|does not appear to be a git repository/i; +export const READINESS_ATTEMPTS = 6; +export const READINESS_DELAY_MS = 2000; + +/** True for the transient "repository not found" a host returns right after creating it. */ +export function isTargetNotReady(error: unknown): boolean { + if (!(error instanceof GitCommandError)) return false; + return TARGET_NOT_READY.test(`${error.message}\n${error.stderr}`); +} + +/** + * First push into a repository this run created. GitLab (and sometimes + * GitHub) answer "not found" for a second or two after the create call + * returns, so a not-found here is retried a few times before it counts. + */ +async function pushWhenReady( + plan: PushMirrorPlan, + pushUrl: string, + refs: Map, + log: (message: string) => void +): Promise<{ pushed: boolean; batches: number }> { + for (let attempt = 1; ; attempt += 1) { + try { + return await pushAll(plan, pushUrl, refs, log); + } catch (error) { + if (attempt >= READINESS_ATTEMPTS || !isTargetNotReady(error)) throw error; + const delay = plan.readinessDelayMs ?? READINESS_DELAY_MS; + log(`the new repository is not ready for pushes yet (attempt ${attempt}); retrying in ${delay / 1000}s`); + await Bun.sleep(delay); + } + } +} + /** Remove a repository's bare clone and its lock file. */ export async function removeClone(clonePath: string): Promise { await rm(clonePath, { recursive: true, force: true }); diff --git a/src/lib/source-providers/kinds.test.ts b/src/lib/source-providers/kinds.test.ts index aa7ec41..94ab4b8 100644 --- a/src/lib/source-providers/kinds.test.ts +++ b/src/lib/source-providers/kinds.test.ts @@ -1,5 +1,6 @@ import { describe, expect, test } from "bun:test"; import { + isBetaSourceProvider, SOURCE_PROVIDER_DEFAULT_URLS, getRepositorySource, isRepositoryFromConfiguredSource, @@ -10,6 +11,14 @@ import { sourceHostOf, } from "./kinds"; +describe("isBetaSourceProvider", () => { + test("GitHub is the supported default; every other source is beta", () => { + expect(isBetaSourceProvider("github")).toBe(false); + expect(isBetaSourceProvider("gitlab")).toBe(true); + expect(isBetaSourceProvider("gitea")).toBe(true); + }); +}); + describe("normalizeSourceProviderKind", () => { test("accepts the three kinds and defaults everything else to GitHub", () => { expect(normalizeSourceProviderKind("github")).toBe("github"); diff --git a/src/lib/source-providers/kinds.ts b/src/lib/source-providers/kinds.ts index 95e6bbe..4a3b44f 100644 --- a/src/lib/source-providers/kinds.ts +++ b/src/lib/source-providers/kinds.ts @@ -11,6 +11,11 @@ export type SourceProviderKind = (typeof SOURCE_PROVIDER_KINDS)[number]; export const DEFAULT_SOURCE_PROVIDER: SourceProviderKind = "github"; +/** Sources other than GitHub are beta: they work end to end but have had less time in the field. */ +export function isBetaSourceProvider(kind: SourceProviderKind): boolean { + return kind !== DEFAULT_SOURCE_PROVIDER; +} + /** Base URL used when the config has no instance URL for the provider. */ export const SOURCE_PROVIDER_DEFAULT_URLS: Record = { github: "https://github.com", diff --git a/www/src/pages/docs/configuration.mdx b/www/src/pages/docs/configuration.mdx index 0bcb3bc..5ca9956 100644 --- a/www/src/pages/docs/configuration.mdx +++ b/www/src/pages/docs/configuration.mdx @@ -22,7 +22,7 @@ This is where the two ends of the mirror are set up. It contains two cards: **GitHub Connection** holds your GitHub username, personal access token, and account type (personal or organization). The token needs the `repo` scope for private repositories and `admin:org` to read organization data. This card also carries the repository selection options: whether to include private repositories, public repositories, archived repositories, forks, repositories where you are only a collaborator, and starred repositories. -**Gitea Connection** holds the Gitea or Forgejo instance URL, an access token, and the default owner. It also carries the destination settings: the mirror strategy, default organization, organization visibility, and the per-repository mirror interval that Gitea itself uses to re-pull from GitHub. +**Gitea Connection** holds the destination. Gitea is the default; Forgejo, GitHub and GitLab are available from the Destination dropdown and marked beta, as are the GitLab and Gitea / Forgejo sources on the GitHub card. GitHub to Gitea is the supported path. The card holds the instance URL, an access token, and the default owner. It also carries the destination settings: the mirror strategy, default organization, organization visibility, and the per-repository mirror interval that Gitea itself uses to re-pull from GitHub. Both cards have a test button that performs a live authenticated call, so you can confirm credentials before saving.