diff --git a/CHANGELOG.md b/CHANGELOG.md index 08910f1..87c9142 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] ### Fixed +- Issue and pull request passes over large repositories finish across rate limit resets (#449 follow-up) + - A pass stopped by the rate limit keeps the numbers it finished, and the next run repeats the same pass and skips them unless they changed since it started; before, every run started over from the first item, so a repository too large for one hour of budget never completed a pass + - The weekly full pass no longer fetches comments for issues, or the detail, commits and files of pull requests, that are already in the destination and have not changed since the last completed pass; it lists everything and corrects title, body, state and labels from the listing + - A sync paused by the rate limit logs one line instead of the error object with every partial result, and its activity entry says it was paused - No request leaves the process while GitHub is rate limited (#437, second report) - The v3.37.0 change stopped the scheduler run from holding its lock through a rate limit, but every other request already queued in the run still went to GitHub and came back 403, and the retry helper sent each of them three more times; GitHub counts requests made while limited toward abuse detection and one account was suspended that way - Every GitHub client now holds its requests while the source is paused: a limit that resets within two minutes is waited out inside the request, a longer one fails the request at once without contacting GitHub, and the rate limit probe endpoint stays available so the pause can end early diff --git a/src/lib/gitea-enhanced.ts b/src/lib/gitea-enhanced.ts index bd83dcb..f54c924 100644 --- a/src/lib/gitea-enhanced.ts +++ b/src/lib/gitea-enhanced.ts @@ -8,6 +8,7 @@ import type { Config } from "@/types/config"; import { isRateLimitError } from "@/lib/rate-limit-gate"; +import { persistMetadataPassProgress } from "./metadata-progress-store"; import { getRepositorySource, SOURCE_PROVIDER_LABELS, @@ -1101,6 +1102,11 @@ export async function syncGiteaRepoEnhanced({ giteaOwner: repoOwner, giteaRepoName: repoName, syncCursor: metadataState.syncCursors.issues, + passProgress: metadataState.passProgress.issues, + onPassProgress: async (progress) => { + metadataUpdated = true; + await persistMetadataPassProgress(repository.id, metadataState, "issues", progress); + }, }); if (issuesCursor) metadataState.syncCursors.issues = issuesCursor; metadataState.components.issues = true; @@ -1139,6 +1145,11 @@ export async function syncGiteaRepoEnhanced({ giteaOwner: repoOwner, giteaRepoName: repoName, syncCursor: metadataState.syncCursors.pullRequests, + passProgress: metadataState.passProgress.pullRequests, + onPassProgress: async (progress) => { + metadataUpdated = true; + await persistMetadataPassProgress(repository.id, metadataState, "pullRequests", progress); + }, }); if (pullRequestsCursor) metadataState.syncCursors.pullRequests = pullRequestsCursor; metadataState.components.pullRequests = true; @@ -1321,7 +1332,17 @@ export async function syncGiteaRepoEnhanced({ throw syncError; } } catch (error) { - console.error(`[Sync] Error while syncing repository ${repository.name}:`, error); + if (isRateLimitError(error)) { + // Expected and handled: the scheduler puts the row back and retries + // after the reset. One line instead of the whole error object. + console.warn( + `[Sync] Sync of ${repository.name} paused by the source rate limit: ${ + error instanceof Error ? error.message : String(error) + }` + ); + } else { + console.error(`[Sync] Error while syncing repository ${repository.name}:`, error); + } // Update repo with error status await db @@ -1338,7 +1359,9 @@ export async function syncGiteaRepoEnhanced({ userId: config.userId, repositoryId: repository.id, repositoryName: repository.name, - message: `Failed to sync repository: ${repository.name}`, + message: isRateLimitError(error) + ? `Sync paused by the source rate limit: ${repository.name}` + : `Failed to sync repository: ${repository.name}`, details: error instanceof Error ? error.message : "Unknown error", status: "failed", }); diff --git a/src/lib/gitea-incremental-metadata.test.ts b/src/lib/gitea-incremental-metadata.test.ts index 164db24..bb66a8a 100644 --- a/src/lib/gitea-incremental-metadata.test.ts +++ b/src/lib/gitea-incremental-metadata.test.ts @@ -15,7 +15,7 @@ import { afterEach, describe, expect, it, test } from "bun:test"; import type { Config } from "@/types/config"; import type { Repository } from "@/lib/db/schema"; -import type { MetadataSyncCursor } from "@/lib/metadata-state"; +import type { MetadataPassProgress, MetadataSyncCursor } from "@/lib/metadata-state"; const CHILD_FLAG = "GM_INCREMENTAL_METADATA_ISOLATED"; const isChild = !!process.env[CHILD_FLAG]; @@ -176,6 +176,14 @@ interface GitHubData { pullListing?: any[]; /** When set, pulls.get throws for these numbers. */ failDetailFor?: number[]; + /** GitHub refuses the comment listing of these issues with a rate limit. */ + rateLimitCommentsFor?: number[]; + /** GitHub refuses pulls.get for these numbers with a rate limit. */ + rateLimitDetailFor?: number[]; +} + +function rateLimitRefusal() { + return Object.assign(new Error("API rate limit exceeded for user ID 1."), { status: 403 }); } /** Fake Octokit that records every call made through it. */ @@ -189,6 +197,7 @@ function fakeOctokit(data: GitHubData) { }, listComments: async (params: any) => { calls.push({ endpoint: "issues.listComments", params }); + if (data.rateLimitCommentsFor?.includes(params.issue_number)) throw rateLimitRefusal(); return { data: [] }; }, }; @@ -199,6 +208,7 @@ function fakeOctokit(data: GitHubData) { }, get: async (params: any) => { calls.push({ endpoint: "pulls.get", params }); + if (data.rateLimitDetailFor?.includes(params.pull_number)) throw rateLimitRefusal(); if (data.failDetailFor?.includes(params.pull_number)) { throw new Error("detail unavailable"); } @@ -374,7 +384,272 @@ describe.skipIf(!isChild)("mirrorGitRepoIssuesToGitea incremental sync", () => { const listings = github.callsTo("issues.listForRepo"); expect(listings).toHaveLength(1); expect(listings[0].params.since).toBeUndefined(); - expect(github.callsTo("issues.listComments")).toHaveLength(2); + // #1 is in Gitea and has not changed since the last pass, so the full + // pass does not list its comments again; #2 is new. + expect(github.callsTo("issues.listComments").map((c) => c.params.issue_number)).toEqual([2]); + }); +}); + +describe.skipIf(!isChild)("full passes that revisit unchanged items (#449 follow-up)", () => { + const weekOldFullPass = (): MetadataSyncCursor => ({ + lastPassStartedAt: new Date(Date.now() - 60 * 60 * 1000).toISOString(), + lastFullPassStartedAt: new Date(Date.now() - 8 * 24 * 60 * 60 * 1000).toISOString(), + }); + const recently = () => new Date(Date.now() - 5 * 60 * 1000).toISOString(); + + it("only lists comments for issues that changed or are missing", async () => { + const gitea = fakeGitea([ + { number: 11, title: "[GH-ISSUE #1] Issue 1" }, + { number: 12, title: "[GH-ISSUE #2] Issue 2" }, + ]); + const github = fakeOctokit({ + issueListing: [ghIssue(1), ghIssue(2, { updated_at: recently() }), ghIssue(3)], + }); + + const cursor = await mirrorGitRepoIssuesToGitea({ + config, + octokit: github.octokit, + repository, + giteaOwner: DEST_OWNER, + giteaRepoName: DEST_REPO, + syncCursor: weekOldFullPass(), + }); + + expect(github.callsTo("issues.listComments").map((c) => c.params.issue_number)).toEqual([2, 3]); + // The unchanged issue is still brought in line from the listing. + expect( + gitea.requests.some((r) => r.method === "PATCH" && r.url === `${DEST_API}/issues/11`) + ).toBe(true); + expect(gitea.createdTitles()).toEqual(["[GH-ISSUE #3] Issue 3"]); + expect(cursor!.lastFullPassStartedAt).toBe(cursor!.lastPassStartedAt); + }); + + it("leaves an unchanged issue alone when the Gitea copy already matches", async () => { + const gitea = fakeGitea([]); + const issue = ghIssue(1); + const matching = { + number: 11, + title: "[GH-ISSUE #1] Issue 1", + body: + `Originally created by @alice on GitHub (${new Intl.DateTimeFormat("en-US", { + year: "numeric", + month: "short", + day: "numeric", + }).format(new Date(issue.created_at))}).\n` + + `Original GitHub issue: ${issue.html_url}\n\n${issue.body}`, + state: "open", + labels: [], + }; + // Serve an exact copy by overriding the listing response. + globalThis.fetch = ((original) => + (async (input: any, init?: any) => { + const url = typeof input === "string" ? input : String(input); + const method = String(init?.method ?? "GET").toUpperCase(); + if (method === "GET" && url.startsWith(`${DEST_API}/issues?`) && !url.includes("&q=")) { + const page = Number(new URL(url).searchParams.get("page") ?? "1"); + return new Response(JSON.stringify(page === 1 ? [matching] : []), { + status: 200, + headers: { "content-type": "application/json" }, + }); + } + return original(input, init); + }) as unknown as typeof fetch)(globalThis.fetch); + const github = fakeOctokit({ issueListing: [issue] }); + + await mirrorGitRepoIssuesToGitea({ + config, + octokit: github.octokit, + repository, + giteaOwner: DEST_OWNER, + giteaRepoName: DEST_REPO, + syncCursor: weekOldFullPass(), + }); + + expect(github.callsTo("issues.listComments")).toHaveLength(0); + // Nothing to write either: no edit, no label reset, no create. + expect(gitea.requests.filter((r) => r.method !== "GET")).toEqual([]); + }); + + it("skips the detail, commits and files calls for unchanged pull requests", async () => { + fakeGitea([ + { number: 21, title: "[PR #1] PR 1" }, + { number: 22, title: "[PR #2] PR 2" }, + ]); + const github = fakeOctokit({ + issueListing: [], + pullListing: [ + ghPull(1, { updated_at: "2026-09-01T00:00:00Z" }), + ghPull(2, { updated_at: recently() }), + ghPull(3, { updated_at: "2026-09-01T00:00:00Z" }), + ], + }); + + const cursor = await mirrorGitRepoPullRequestsToGitea({ + config, + octokit: github.octokit, + repository, + giteaOwner: DEST_OWNER, + giteaRepoName: DEST_REPO, + syncCursor: weekOldFullPass(), + }); + + // #1 is unchanged and mirrored; #2 changed; #3 is missing in Gitea. + expect(github.callsTo("pulls.get").map((c) => c.params.pull_number)).toEqual([2, 3]); + expect(github.callsTo("pulls.listCommits")).toHaveLength(2); + expect(github.callsTo("pulls.listFiles")).toHaveLength(2); + expect(cursor!.lastFullPassStartedAt).toBe(cursor!.lastPassStartedAt); + }); + + it("does not treat anything as unchanged on the first pass, when there is no watermark", async () => { + fakeGitea([{ number: 21, title: "[PR #1] PR 1" }]); + const github = fakeOctokit({ + issueListing: [], + pullListing: [ghPull(1, { updated_at: "2026-09-01T00:00:00Z" })], + }); + + await mirrorGitRepoPullRequestsToGitea({ + config, + octokit: github.octokit, + repository, + giteaOwner: DEST_OWNER, + giteaRepoName: DEST_REPO, + }); + + expect(github.callsTo("pulls.get").map((c) => c.params.pull_number)).toEqual([1]); + }); +}); + +describe.skipIf(!isChild)("passes stopped by the rate limit resume (#449 follow-up)", () => { + it("keeps the finished issues and continues from there on the next run", async () => { + fakeGitea([]); + const listing = [ghIssue(1), ghIssue(2), ghIssue(3), ghIssue(4)]; + const saved: Array = []; + + const first = fakeOctokit({ issueListing: listing, rateLimitCommentsFor: [3] }); + let caught: unknown; + try { + await mirrorGitRepoIssuesToGitea({ + config, + octokit: first.octokit, + repository, + giteaOwner: DEST_OWNER, + giteaRepoName: DEST_REPO, + onPassProgress: (progress) => void saved.push(progress), + }); + } catch (error) { + caught = error; + } + expect(String(caught)).toContain("rate limit"); + // #3 was refused and #4 never started. + expect(first.callsTo("issues.listComments").map((c) => c.params.issue_number)).toEqual([1, 2, 3]); + expect(saved).toHaveLength(1); + const progress = saved[0]!; + expect(progress.mode).toBe("full"); + expect(progress.done).toEqual([[1, 2]]); + + // Next run, budget back: #1 and #2 are not fetched again. + fakeGitea([ + { number: 11, title: "[GH-ISSUE #1] Issue 1" }, + { number: 12, title: "[GH-ISSUE #2] Issue 2" }, + { number: 13, title: "[GH-ISSUE #3] Issue 3" }, + ]); + const second = fakeOctokit({ issueListing: listing }); + const cursor = await mirrorGitRepoIssuesToGitea({ + config, + octokit: second.octokit, + repository, + giteaOwner: DEST_OWNER, + giteaRepoName: DEST_REPO, + passProgress: progress, + onPassProgress: (next) => void saved.push(next), + }); + + const [relisting] = second.callsTo("issues.listForRepo"); + expect(relisting.params.since).toBeUndefined(); + expect(second.callsTo("issues.listComments").map((c) => c.params.issue_number)).toEqual([3, 4]); + // Completed: the progress is cleared and the watermark is the start of + // the first attempt, so nothing changed in between is missed. + expect(saved[saved.length - 1]).toBeUndefined(); + expect(cursor!.lastPassStartedAt).toBe(progress.startedAt); + expect(cursor!.lastFullPassStartedAt).toBe(progress.startedAt); + }); + + it("redoes a finished issue that changed after the interrupted pass started", async () => { + fakeGitea([ + { number: 11, title: "[GH-ISSUE #1] Issue 1" }, + { number: 12, title: "[GH-ISSUE #2] Issue 2" }, + ]); + const startedAt = new Date(Date.now() - 2 * 60 * 60 * 1000).toISOString(); + const github = fakeOctokit({ + issueListing: [ghIssue(1), ghIssue(2, { updated_at: new Date().toISOString() })], + }); + + await mirrorGitRepoIssuesToGitea({ + config, + octokit: github.octokit, + repository, + giteaOwner: DEST_OWNER, + giteaRepoName: DEST_REPO, + passProgress: { mode: "full", startedAt, done: [[1, 2]] }, + }); + + expect(github.callsTo("issues.listComments").map((c) => c.params.issue_number)).toEqual([2]); + }); + + it("resumes an incremental pass with the same since", async () => { + fakeGitea([{ number: 11, title: "[GH-ISSUE #1] Issue 1" }]); + const since = new Date(Date.now() - 3 * 60 * 60 * 1000).toISOString(); + const startedAt = new Date(Date.now() - 2 * 60 * 60 * 1000).toISOString(); + const github = fakeOctokit({ issueListing: [ghIssue(1), ghIssue(2)] }); + + const cursor = await mirrorGitRepoIssuesToGitea({ + config, + octokit: github.octokit, + repository, + giteaOwner: DEST_OWNER, + giteaRepoName: DEST_REPO, + syncCursor: recentCursor(), + passProgress: { mode: "incremental", since, startedAt, done: [[1, 1]] }, + }); + + const [listing] = github.callsTo("issues.listForRepo"); + expect(listing.params.since).toBe(since); + expect(github.callsTo("issues.listComments").map((c) => c.params.issue_number)).toEqual([2]); + expect(cursor!.lastPassStartedAt).toBe(startedAt); + }); + + it("keeps the finished pull requests when the detail call is refused", async () => { + fakeGitea([]); + const pullListing = [ghPull(1), ghPull(2), ghPull(3)].map((pr) => ({ + ...pr, + updated_at: "2026-09-01T00:00:00Z", + })); + const saved: Array = []; + const first = fakeOctokit({ issueListing: [], pullListing, rateLimitDetailFor: [2] }); + + await expect( + mirrorGitRepoPullRequestsToGitea({ + config, + octokit: first.octokit, + repository, + giteaOwner: DEST_OWNER, + giteaRepoName: DEST_REPO, + onPassProgress: (progress) => void saved.push(progress), + }) + ).rejects.toThrow(/rate limit/i); + expect(saved[0]!.done).toEqual([[1, 1]]); + + fakeGitea([{ number: 21, title: "[PR #1] PR 1" }]); + const second = fakeOctokit({ issueListing: [], pullListing }); + await mirrorGitRepoPullRequestsToGitea({ + config, + octokit: second.octokit, + repository, + giteaOwner: DEST_OWNER, + giteaRepoName: DEST_REPO, + passProgress: saved[0], + }); + expect(second.callsTo("pulls.get").map((c) => c.params.pull_number)).toEqual([2, 3]); }); }); diff --git a/src/lib/gitea.ts b/src/lib/gitea.ts index 4d8b673..1a7738c 100644 --- a/src/lib/gitea.ts +++ b/src/lib/gitea.ts @@ -30,11 +30,15 @@ import { listSources, } from "./sources"; import { isRateLimitError } from "./rate-limit-gate"; +import { persistMetadataPassProgress } from "./metadata-progress-store"; import { advanceMetadataSyncCursor, + buildPassProgress, + classifyPassItem, parseRepositoryMetadataState, - planMetadataPass, + planMetadataPassWithProgress, serializeRepositoryMetadataState, + type MetadataPassProgress, type MetadataSyncCursor, } from "./metadata-state"; import { @@ -1193,6 +1197,11 @@ export const mirrorGithubRepoToGitea = async ({ giteaOwner: repoOwner, giteaRepoName: targetRepoName, syncCursor: metadataState.syncCursors.issues, + passProgress: metadataState.passProgress.issues, + onPassProgress: async (progress) => { + metadataUpdated = true; + await persistMetadataPassProgress(repository.id, metadataState, "issues", progress); + }, }); if (issuesCursor) metadataState.syncCursors.issues = issuesCursor; metadataState.components.issues = true; @@ -1230,6 +1239,11 @@ export const mirrorGithubRepoToGitea = async ({ giteaOwner: repoOwner, giteaRepoName: targetRepoName, syncCursor: metadataState.syncCursors.pullRequests, + passProgress: metadataState.passProgress.pullRequests, + onPassProgress: async (progress) => { + metadataUpdated = true; + await persistMetadataPassProgress(repository.id, metadataState, "pullRequests", progress); + }, }); if (pullRequestsCursor) metadataState.syncCursors.pullRequests = pullRequestsCursor; metadataState.components.pullRequests = true; @@ -1989,6 +2003,11 @@ export async function mirrorGitHubRepoToGiteaOrg({ giteaOwner: orgName, giteaRepoName: targetRepoName, syncCursor: metadataState.syncCursors.issues, + passProgress: metadataState.passProgress.issues, + onPassProgress: async (progress) => { + metadataUpdated = true; + await persistMetadataPassProgress(repository.id, metadataState, "issues", progress); + }, }); if (issuesCursor) metadataState.syncCursors.issues = issuesCursor; metadataState.components.issues = true; @@ -2026,6 +2045,11 @@ export async function mirrorGitHubRepoToGiteaOrg({ giteaOwner: orgName, giteaRepoName: targetRepoName, syncCursor: metadataState.syncCursors.pullRequests, + passProgress: metadataState.passProgress.pullRequests, + onPassProgress: async (progress) => { + metadataUpdated = true; + await persistMetadataPassProgress(repository.id, metadataState, "pullRequests", progress); + }, }); if (pullRequestsCursor) metadataState.syncCursors.pullRequests = pullRequestsCursor; metadataState.components.pullRequests = true; @@ -2657,6 +2681,8 @@ export const mirrorGitRepoIssuesToGitea = async ({ giteaOwner, giteaRepoName, syncCursor, + passProgress, + onPassProgress, }: { config: Partial; octokit: Octokit; @@ -2668,6 +2694,14 @@ export const mirrorGitRepoIssuesToGitea = async ({ * only issues updated since then are listed and reconciled. */ syncCursor?: MetadataSyncCursor; + /** Unfinished pass to pick up where it stopped (#449 follow-up). */ + passProgress?: MetadataPassProgress; + /** + * Called with the progress to keep when the pass does not finish, and + * with undefined once a resumed pass completes. The caller persists it + * right away, since a rate limit ends the sync with a throw. + */ + onPassProgress?: (progress: MetadataPassProgress | undefined) => Promise | void; }): Promise => { //things covered here are- issue, title, body, labels, comments and assignees // The source client arrives as `octokit` (anonymous for a public-only @@ -2712,8 +2746,24 @@ export const mirrorGitRepoIssuesToGitea = async ({ // milestone, title or body change all bump updated_at, so this is the // set that needs reconciling, and the per-issue comment listing below // only runs for it. The watermark is the time this listing started. - let plan = planMetadataPass(syncCursor); - const listingStartedAt = new Date(); + const planned = planMetadataPassWithProgress(syncCursor, passProgress); + let plan = planned.plan; + let resume = planned.resume; + const passStartedAt = planned.startedAt; + if (resume) { + console.log( + `[Issues] Resuming an unfinished ${resume.mode} pass for ${repository.fullName} started at ${resume.startedAt}` + ); + } + // GitHub numbers this pass has finished, kept if it does not complete. + const completedNumbers = new Set(); + const finishPass = async () => { + if (resume) await onPassProgress?.(undefined); + return advanceMetadataSyncCursor(syncCursor, plan.mode, passStartedAt); + }; + const keepPassProgress = async () => { + await onPassProgress?.(buildPassProgress(plan, passStartedAt, completedNumbers, resume)); + }; const listGitHubIssues = async (since?: string) => { const listed = await octokit.paginate( @@ -2774,7 +2824,7 @@ export const mirrorGitRepoIssuesToGitea = async ({ if (filteredIssues.length === 0 && plan.mode === "full") { console.log(`No issues to mirror for ${repository.fullName}`); - return advanceMetadataSyncCursor(syncCursor, plan.mode, listingStartedAt); + return finishPass(); } const ghIssueMarkerRegex = /\[GH-ISSUE #(\d+)\]/i; @@ -2846,6 +2896,7 @@ export const mirrorGitRepoIssuesToGitea = async ({ `[Issues] No mirrored issues found in Gitea for ${repoName}; ignoring the watermark and running a full pass` ); plan = { mode: "full", reason: "destination has no mirrored issues" }; + resume = undefined; filteredIssues = await listGitHubIssues(); console.log( `Mirroring ${filteredIssues.length} issues from ${repository.fullName}` @@ -2854,7 +2905,7 @@ export const mirrorGitRepoIssuesToGitea = async ({ if (filteredIssues.length === 0) { console.log(`No issues to mirror for ${repository.fullName}`); - return advanceMetadataSyncCursor(syncCursor, plan.mode, listingStartedAt); + return finishPass(); } // Get existing labels from Gitea @@ -2885,10 +2936,27 @@ export const mirrorGitRepoIssuesToGitea = async ({ ); } + let unchangedIssueCount = 0; + // Process issues in parallel with concurrency control - const mirroredIssues = await processWithRetry( + let mirroredIssues: unknown[]; + try { + mirroredIssues = await processWithRetry( filteredIssues, async (issue) => { + const itemClass = classifyPassItem({ + number: issue.number, + updatedAt: issue.updated_at, + plan, + cursor: syncCursor, + resume, + existsInDestination: giteaIssueByGitHubNumber.has(issue.number), + }); + if (itemClass === "skip") { + completedNumbers.add(issue.number); + return issue; + } + const githubLabelNames = issue.labels ?.map((l) => (typeof l === "string" ? l : l.name)) @@ -2966,6 +3034,48 @@ export const mirrorGitRepoIssuesToGitea = async ({ } } + // A full pass revisiting an issue that has not changed since the last + // completed pass: bring the Gitea copy in line from the listing alone + // (a renamed label, an edit made on the Gitea side) and skip the + // comment listing, which is the per-issue GitHub call. + if (itemClass === "unchanged" && existingIssue) { + const wantClosed = issue.state === "closed"; + if ( + existingIssue.title !== issuePayload.title || + String(existingIssue.body ?? "") !== issuePayload.body || + (existingIssue.state === "closed") !== wantClosed + ) { + await httpPatch( + `${config.giteaConfig!.url}/api/v1/repos/${giteaOwner}/${repoName}/issues/${existingIssue.number}`, + buildGiteaIssueEditPayload({ + title: issuePayload.title, + body: issuePayload.body, + closed: wantClosed, + }), + { + Authorization: `token ${decryptedConfig.giteaConfig!.token}`, + } + ); + } + const currentLabelIds = (Array.isArray(existingIssue.labels) ? existingIssue.labels : []) + .map((label: any) => Number(label?.id)) + .sort((a: number, b: number) => a - b); + const wantedLabelIds = [...giteaLabelIds].sort((a, b) => a - b); + if (currentLabelIds.join(",") !== wantedLabelIds.join(",")) { + await reconcileGiteaIssueLabels({ + config, + decryptedConfig, + giteaOwner, + repoName, + issueNumber: existingIssue.number, + labelIds: giteaLabelIds, + }); + } + unchangedIssueCount++; + completedNumbers.add(issue.number); + return issue; + } + let targetIssueNumber: number; if (existingIssue) { targetIssueNumber = existingIssue.number; @@ -3180,6 +3290,7 @@ export const mirrorGitRepoIssuesToGitea = async ({ ); } + completedNumbers.add(issue.number); return issue; }, { @@ -3201,9 +3312,23 @@ export const mirrorGitRepoIssuesToGitea = async ({ }, } ); + } catch (error) { + // Stopped by the rate limit: keep what this pass finished so the run + // after the reset carries on from here instead of starting over. + if (isRateLimitError(error)) { + await keepPassProgress(); + console.warn( + `[Issues] Pass for ${repository.fullName} stopped by the rate limit after ${completedNumbers.size} of ${filteredIssues.length} issue(s); the next run continues from there` + ); + } + throw error; + } console.log( - `Completed mirroring ${filteredIssues.length} issues for ${repository.fullName}` + `Completed mirroring ${filteredIssues.length} issues for ${repository.fullName}` + + (unchangedIssueCount > 0 + ? ` (${unchangedIssueCount} unchanged since the last pass, no GitHub calls)` + : "") ); // processWithRetry logs and drops items that still fail after their @@ -3213,10 +3338,11 @@ export const mirrorGitRepoIssuesToGitea = async ({ console.warn( `[Issues] ${failedIssueCount} issue(s) failed for ${repository.fullName}; keeping the previous watermark so they are retried next run` ); + await keepPassProgress(); return undefined; } - return advanceMetadataSyncCursor(syncCursor, plan.mode, listingStartedAt); + return finishPass(); }; /** @@ -4098,6 +4224,8 @@ export async function mirrorGitRepoPullRequestsToGitea({ giteaOwner, giteaRepoName, syncCursor, + passProgress, + onPassProgress, }: { config: Partial; octokit: Octokit; @@ -4109,6 +4237,9 @@ export async function mirrorGitRepoPullRequestsToGitea({ * When set, only pull requests updated since then are processed. */ syncCursor?: MetadataSyncCursor; + /** Unfinished pass to pick up where it stopped; see the issues pass. */ + passProgress?: MetadataPassProgress; + onPassProgress?: (progress: MetadataPassProgress | undefined) => Promise | void; }): Promise { // The source client arrives as `octokit`; see mirrorGitRepoIssuesToGitea. if ( @@ -4151,8 +4282,23 @@ export async function mirrorGitRepoPullRequestsToGitea({ // the pull request entries. The detail, commits and files calls below // then only run for those. The watermark is the time this listing // started. - let plan = planMetadataPass(syncCursor); - const listingStartedAt = new Date(); + const planned = planMetadataPassWithProgress(syncCursor, passProgress); + let plan = planned.plan; + let resume = planned.resume; + const passStartedAt = planned.startedAt; + if (resume) { + console.log( + `[Pull Requests] Resuming an unfinished ${resume.mode} pass for ${repository.fullName} started at ${resume.startedAt}` + ); + } + const completedNumbers = new Set(); + const finishPass = async () => { + if (resume) await onPassProgress?.(undefined); + return advanceMetadataSyncCursor(syncCursor, plan.mode, passStartedAt); + }; + const keepPassProgress = async () => { + await onPassProgress?.(buildPassProgress(plan, passStartedAt, completedNumbers, resume)); + }; type ListedPullRequest = Awaited< ReturnType >["data"][number]; @@ -4207,6 +4353,7 @@ export async function mirrorGitRepoPullRequestsToGitea({ html_url: entry.html_url, user: entry.user, created_at: entry.created_at, + updated_at: entry.updated_at, merged_at: (entry as any).pull_request?.merged_at ?? null, }) as unknown as ListedPullRequest ); @@ -4236,7 +4383,7 @@ export async function mirrorGitRepoPullRequestsToGitea({ if (pullRequests.length === 0 && plan.mode === "full") { console.log(`No pull requests to mirror for ${repository.fullName}`); - return advanceMetadataSyncCursor(syncCursor, plan.mode, listingStartedAt); + return finishPass(); } // Note: Gitea doesn't have a direct API to create pull requests from external sources @@ -4320,6 +4467,7 @@ export async function mirrorGitRepoPullRequestsToGitea({ `[Pull Requests] No mirrored pull requests found in Gitea for ${repoName}; ignoring the watermark and running a full pass` ); plan = { mode: "full", reason: "destination has no mirrored pull requests" }; + resume = undefined; pullRequests = await listAllPullRequests(); listedFromIssues = false; console.log( @@ -4329,7 +4477,7 @@ export async function mirrorGitRepoPullRequestsToGitea({ if (pullRequests.length === 0) { console.log(`No pull requests to mirror for ${repository.fullName}`); - return advanceMetadataSyncCursor(syncCursor, plan.mode, listingStartedAt); + return finishPass(); } const { processWithRetry } = await import("@/lib/utils/concurrency"); @@ -4348,11 +4496,31 @@ export async function mirrorGitRepoPullRequestsToGitea({ let successCount = 0; let failedCount = 0; + let unchangedCount = 0; - const processedPullRequests = await processWithRetry( + let processedPullRequests: unknown[]; + try { + processedPullRequests = await processWithRetry( pullRequests, async (listedPr) => { let pr = listedPr; + // Already finished by this pass, or a full pass revisiting a pull + // request that is in Gitea and has not changed since the last + // completed pass: its detail, commits and files are the same, so + // skip the three GitHub calls (#449 follow-up). + const itemClass = classifyPassItem({ + number: listedPr.number, + updatedAt: (listedPr as { updated_at?: string | null }).updated_at, + plan, + cursor: syncCursor, + resume, + existsInDestination: existingPrIssuesByNumber.has(listedPr.number), + }); + if (itemClass !== "sync") { + if (itemClass === "unchanged") unchangedCount++; + completedNumbers.add(listedPr.number); + return; + } try { // Fetch additional PR data for rich metadata const [prDetail, commits, files] = await Promise.all([ @@ -4526,6 +4694,7 @@ export async function mirrorGitRepoPullRequestsToGitea({ } successCount++; + completedNumbers.add(pr.number); console.log(`[Pull Requests] ✅ Successfully created issue for PR #${pr.number}`); } catch (apiError) { // A rate limit refusal is not a reason to write a stripped down @@ -4628,6 +4797,7 @@ export async function mirrorGitRepoPullRequestsToGitea({ } successCount++; + completedNumbers.add(pr.number); console.log(`[Pull Requests] ✅ Created basic issue for PR #${pr.number}`); } catch (error) { failedCount++; @@ -4643,8 +4813,23 @@ export async function mirrorGitRepoPullRequestsToGitea({ retryDelay: 1000, } ); + } catch (error) { + // Stopped by the rate limit: keep what this pass finished so the run + // after the reset carries on from here instead of starting over. + if (isRateLimitError(error)) { + await keepPassProgress(); + console.warn( + `[Pull Requests] Pass for ${repository.fullName} stopped by the rate limit after ${completedNumbers.size} of ${pullRequests.length} pull request(s); the next run continues from there` + ); + } + throw error; + } - console.log(`✅ Mirrored ${successCount}/${pullRequests.length} pull requests to Gitea as enriched issues (${failedCount} failed)`); + console.log( + `✅ Mirrored ${successCount}/${pullRequests.length} pull requests to Gitea as enriched issues (${failedCount} failed` + + (unchangedCount > 0 ? `, ${unchangedCount} unchanged since the last pass` : "") + + `)` + ); // Keep the old watermark when any pull request failed so the next // pass lists it again. @@ -4656,10 +4841,11 @@ export async function mirrorGitRepoPullRequestsToGitea({ console.warn( `[Pull Requests] ${unfinishedPullRequests} pull request(s) failed for ${repository.fullName}; keeping the previous watermark so they are retried next run` ); + await keepPassProgress(); return undefined; } - return advanceMetadataSyncCursor(syncCursor, plan.mode, listingStartedAt); + return finishPass(); } export async function mirrorGitRepoLabelsToGitea({ diff --git a/src/lib/metadata-progress-store.ts b/src/lib/metadata-progress-store.ts new file mode 100644 index 0000000..e23dc6f --- /dev/null +++ b/src/lib/metadata-progress-store.ts @@ -0,0 +1,44 @@ +/** + * Write the progress of an unfinished issues or pull request pass as soon + * as it is known (#449 follow-up). A rate limit ends the sync with a throw, + * before the usual metadata write at the end of it, so without this the + * next run would start the pass over from the first item. + */ + +import { eq } from "drizzle-orm"; +import { db, repositories } from "./db"; +import { + serializeRepositoryMetadataState, + type MetadataPassProgress, + type MetadataPassProgressByKind, + type RepositoryMetadataState, +} from "./metadata-state"; + +export async function persistMetadataPassProgress( + repositoryId: string | null | undefined, + metadataState: RepositoryMetadataState, + kind: keyof MetadataPassProgressByKind, + progress: MetadataPassProgress | undefined +): Promise { + if (progress) { + metadataState.passProgress[kind] = progress; + } else { + delete metadataState.passProgress[kind]; + } + if (!repositoryId) return; + + try { + await db + .update(repositories) + .set({ metadata: serializeRepositoryMetadataState(metadataState) }) + .where(eq(repositories.id, repositoryId)); + } catch (error) { + // The in-memory state still carries it for the write at the end of a + // sync that does not throw; only a rate limited run loses it. + console.warn( + `[Metadata] Could not save ${kind} pass progress: ${ + error instanceof Error ? error.message : String(error) + }` + ); + } +} diff --git a/src/lib/metadata-state.test.ts b/src/lib/metadata-state.test.ts index af79175..0549bfe 100644 --- a/src/lib/metadata-state.test.ts +++ b/src/lib/metadata-state.test.ts @@ -7,10 +7,15 @@ import { FULL_METADATA_PASS_INTERVAL_MS, INCREMENTAL_SYNC_SAFETY_MARGIN_MS, advanceMetadataSyncCursor, + buildPassProgress, + classifyPassItem, createDefaultMetadataState, + numberInRanges, parseRepositoryMetadataState, planMetadataPass, + planMetadataPassWithProgress, serializeRepositoryMetadataState, + toNumberRanges, } from "./metadata-state"; const HOUR = 60 * 60 * 1000; @@ -142,3 +147,119 @@ describe("advanceMetadataSyncCursor", () => { }); }); }); + +describe("unfinished pass progress (#449 follow-up)", () => { + const now = new Date("2026-09-25T12:00:00.000Z"); + + test("numbers collapse into ranges and can be looked up", () => { + const ranges = toNumberRanges([5, 1, 2, 3, 3, 9, 10, 7]); + expect(ranges).toEqual([[1, 3], [5, 5], [7, 7], [9, 10]]); + expect(numberInRanges(2, ranges)).toBe(true); + expect(numberInRanges(8, ranges)).toBe(false); + expect(toNumberRanges([])).toEqual([]); + }); + + test("progress survives a round trip and bad entries are dropped", () => { + const state = createDefaultMetadataState(); + state.passProgress.issues = { + mode: "full", + startedAt: "2026-09-25T10:00:00.000Z", + done: [[1, 164]], + }; + const reparsed = parseRepositoryMetadataState(serializeRepositoryMetadataState(state)); + expect(reparsed.passProgress).toEqual(state.passProgress); + + const junk = parseRepositoryMetadataState({ + passProgress: { + issues: { mode: "sideways", startedAt: "2026-09-25T10:00:00.000Z", done: [] }, + pullRequests: { mode: "incremental", startedAt: "2026-09-25T10:00:00.000Z", done: [] }, + }, + }); + // Unknown mode, and an incremental pass without its since. + expect(junk.passProgress).toEqual({}); + expect(parseRepositoryMetadataState(null).passProgress).toEqual({}); + }); + + test("a stored pass is resumed with its mode, since and start", () => { + const cursor = { + lastPassStartedAt: "2026-09-25T09:00:00.000Z", + lastFullPassStartedAt: "2026-09-20T09:00:00.000Z", + }; + const incremental = planMetadataPassWithProgress( + cursor, + { mode: "incremental", since: "2026-09-25T08:50:00.000Z", startedAt: "2026-09-25T10:00:00.000Z", done: [] }, + now + ); + expect(incremental.plan).toEqual({ mode: "incremental", since: "2026-09-25T08:50:00.000Z" }); + expect(incremental.startedAt.toISOString()).toBe("2026-09-25T10:00:00.000Z"); + expect(incremental.resume).toBeDefined(); + + const full = planMetadataPassWithProgress( + undefined, + { mode: "full", startedAt: "2026-09-25T10:00:00.000Z", done: [[1, 10]] }, + now + ); + expect(full.plan.mode).toBe("full"); + + const none = planMetadataPassWithProgress(cursor, undefined, now); + expect(none.plan.mode).toBe("incremental"); + expect(none.resume).toBeUndefined(); + expect(none.startedAt).toEqual(now); + }); + + test("progress stamped in the future is ignored", () => { + const planned = planMetadataPassWithProgress( + undefined, + { mode: "full", startedAt: "2026-10-25T10:00:00.000Z", done: [[1, 10]] }, + now + ); + expect(planned.resume).toBeUndefined(); + }); + + test("items are classified by what the pass already knows about them", () => { + const cursor = { + lastPassStartedAt: "2026-09-25T09:00:00.000Z", + lastFullPassStartedAt: "2026-09-17T09:00:00.000Z", + }; + const fullPlan = { mode: "full" as const, reason: "test" }; + const resume = { mode: "full" as const, startedAt: "2026-09-25T10:00:00.000Z", done: [[1, 5]] as Array<[number, number]> }; + const base = { plan: fullPlan, cursor, resume, existsInDestination: true }; + + expect(classifyPassItem({ ...base, number: 3, updatedAt: "2026-09-24T00:00:00Z" })).toBe("skip"); + // Changed after the interrupted pass started: done does not count. + expect(classifyPassItem({ ...base, number: 3, updatedAt: "2026-09-25T11:00:00Z" })).toBe("sync"); + // Not done yet, but unchanged since the last completed pass. + expect(classifyPassItem({ ...base, number: 8, updatedAt: "2026-09-24T00:00:00Z" })).toBe("unchanged"); + // Inside the safety margin of the last pass. + expect( + classifyPassItem({ ...base, number: 8, updatedAt: new Date(Date.parse(cursor.lastPassStartedAt) - INCREMENTAL_SYNC_SAFETY_MARGIN_MS + 1000).toISOString() }) + ).toBe("sync"); + // Missing in the destination, no watermark, or no updated_at: full treatment. + expect(classifyPassItem({ ...base, number: 8, updatedAt: "2026-09-24T00:00:00Z", existsInDestination: false })).toBe("sync"); + expect(classifyPassItem({ ...base, number: 8, updatedAt: "2026-09-24T00:00:00Z", cursor: undefined })).toBe("sync"); + expect(classifyPassItem({ ...base, number: 8, updatedAt: null })).toBe("sync"); + // An incremental pass never skips on the watermark alone. + expect( + classifyPassItem({ ...base, plan: { mode: "incremental", since: "x" }, resume: undefined, number: 8, updatedAt: "2026-09-24T00:00:00Z" }) + ).toBe("sync"); + }); + + test("progress keeps what an earlier attempt of the same pass finished", () => { + const earlier = { mode: "full" as const, startedAt: "2026-09-25T10:00:00.000Z", done: [[1, 3]] as Array<[number, number]> }; + const progress = buildPassProgress( + { mode: "full", reason: "resume" }, + new Date(earlier.startedAt), + [4, 5, 7], + earlier + ); + expect(progress).toEqual({ mode: "full", startedAt: earlier.startedAt, done: [[1, 5], [7, 7]] }); + + const incremental = buildPassProgress( + { mode: "incremental", since: "2026-09-25T08:50:00.000Z" }, + new Date("2026-09-25T10:00:00.000Z"), + [2], + undefined + ); + expect(incremental.since).toBe("2026-09-25T08:50:00.000Z"); + }); +}); diff --git a/src/lib/metadata-state.ts b/src/lib/metadata-state.ts index d3858d2..305d9da 100644 --- a/src/lib/metadata-state.ts +++ b/src/lib/metadata-state.ts @@ -36,11 +36,39 @@ export interface MetadataSyncCursors { pullRequests?: MetadataSyncCursor; } +/** + * Progress of an issues or pull request pass that did not finish, most + * often because the source ran out of rate limit budget (#449 follow-up). + * The next run repeats the same pass and skips the items listed in + * `done`, as long as they have not changed since `startedAt`, so a pass + * over a large repository gets further every run instead of starting + * over each time. + */ +export interface MetadataPassProgress { + mode: MetadataPassPlan["mode"]; + /** Listing start for an incremental pass; repeated as is on resume. */ + since?: string; + /** + * When the listing of the first attempt started. Used as the watermark + * once the pass completes, so nothing that changed while it was + * interrupted is missed. + */ + startedAt: string; + /** GitHub numbers finished by this pass, as inclusive ranges. */ + done: Array<[number, number]>; +} + +export interface MetadataPassProgressByKind { + issues?: MetadataPassProgress; + pullRequests?: MetadataPassProgress; +} + export interface RepositoryMetadataState { components: MetadataComponentsState; lastSyncedAt?: string; acknowledgedDeletions: AcknowledgedDeletion[]; syncCursors: MetadataSyncCursors; + passProgress: MetadataPassProgressByKind; } /** @@ -77,6 +105,7 @@ export function createDefaultMetadataState(): RepositoryMetadataState { components: { ...defaultComponents }, acknowledgedDeletions: [], syncCursors: {}, + passProgress: {}, }; } @@ -126,6 +155,147 @@ export function planMetadataPass( }; } +/** + * Collapse item numbers into sorted inclusive ranges, so a pass over + * thousands of issues stores a handful of pairs. + */ +export function toNumberRanges(numbers: Iterable): Array<[number, number]> { + const sorted = [...new Set(numbers)] + .filter((n) => Number.isInteger(n)) + .sort((a, b) => a - b); + const ranges: Array<[number, number]> = []; + for (const n of sorted) { + const last = ranges[ranges.length - 1]; + if (last && n === last[1] + 1) { + last[1] = n; + } else { + ranges.push([n, n]); + } + } + return ranges; +} + +export function numberInRanges(n: number, ranges: Array<[number, number]>): boolean { + return ranges.some(([from, to]) => n >= from && n <= to); +} + +function parsePassProgress(raw: unknown): MetadataPassProgress | undefined { + if (!raw || typeof raw !== "object") return undefined; + const { mode, since, startedAt, done } = raw as { + mode?: unknown; + since?: unknown; + startedAt?: unknown; + done?: unknown; + }; + if (mode !== "full" && mode !== "incremental") return undefined; + if (!isValidTimestamp(startedAt)) return undefined; + if (mode === "incremental" && !isValidTimestamp(since)) return undefined; + if (!Array.isArray(done)) return undefined; + const ranges = done.filter( + (entry): entry is [number, number] => + Array.isArray(entry) && + entry.length === 2 && + Number.isInteger(entry[0]) && + Number.isInteger(entry[1]) && + entry[0] <= entry[1] + ); + return mode === "incremental" + ? { mode, since: since as string, startedAt, done: ranges } + : { mode, startedAt, done: ranges }; +} + +/** + * Plan a pass, picking up an unfinished one when there is one. A resumed + * pass keeps its mode and `since`, and reports its original start so the + * watermark stored at the end covers the whole interrupted window. + */ +export function planMetadataPassWithProgress( + cursor: MetadataSyncCursor | undefined, + progress: MetadataPassProgress | undefined, + now: Date = new Date() +): { plan: MetadataPassPlan; startedAt: Date; resume?: MetadataPassProgress } { + const startedAtMs = progress ? Date.parse(progress.startedAt) : NaN; + if (progress && Number.isFinite(startedAtMs) && startedAtMs <= now.getTime()) { + const plan: MetadataPassPlan = + progress.mode === "incremental" && progress.since + ? { mode: "incremental", since: progress.since } + : { mode: "full", reason: "resuming a full pass that did not finish" }; + return { plan, startedAt: new Date(startedAtMs), resume: progress }; + } + return { plan: planMetadataPass(cursor, now), startedAt: now }; +} + +/** + * Decides how much work an item needs in the current pass. + * + * - "skip": a resumed pass already finished it and it has not changed + * since that pass started. + * - "unchanged": a full pass is revisiting an item that is already in + * Gitea and has not changed since the last completed pass. Its + * comments, commits and files cannot have changed either (each of + * those bumps updated_at), so no per-item source call is needed. + * - "sync": everything else gets the full treatment. + */ +export function classifyPassItem({ + number, + updatedAt, + plan, + cursor, + resume, + existsInDestination, +}: { + number: number; + updatedAt: string | null | undefined; + plan: MetadataPassPlan; + cursor: MetadataSyncCursor | undefined; + resume: MetadataPassProgress | undefined; + existsInDestination: boolean; +}): "skip" | "unchanged" | "sync" { + const updatedMs = updatedAt ? Date.parse(updatedAt) : NaN; + if (!Number.isFinite(updatedMs)) return "sync"; + + if (resume && numberInRanges(number, resume.done)) { + const resumeStartMs = Date.parse(resume.startedAt); + if (updatedMs < resumeStartMs - INCREMENTAL_SYNC_SAFETY_MARGIN_MS) { + return "skip"; + } + } + + if (plan.mode === "full" && cursor && existsInDestination) { + const lastPassMs = Date.parse(cursor.lastPassStartedAt); + if ( + Number.isFinite(lastPassMs) && + updatedMs < lastPassMs - INCREMENTAL_SYNC_SAFETY_MARGIN_MS + ) { + return "unchanged"; + } + } + + return "sync"; +} + +/** + * Progress to store for a pass that did not finish. Numbers finished by + * an earlier attempt of the same pass are kept. + */ +export function buildPassProgress( + plan: MetadataPassPlan, + startedAt: Date, + completed: Iterable, + resume: MetadataPassProgress | undefined +): MetadataPassProgress { + const numbers = new Set(completed); + for (const [from, to] of resume?.done ?? []) { + for (let n = from; n <= to; n++) numbers.add(n); + } + return { + mode: plan.mode, + ...(plan.mode === "incremental" ? { since: plan.since } : {}), + startedAt: startedAt.toISOString(), + done: toNumberRanges(numbers), + }; +} + /** * The watermark to store after a pass that completed without failures. * `startedAt` is the time the GitHub listing started. @@ -204,6 +374,13 @@ export function parseRepositoryMetadataState( if (pullRequests) base.syncCursors.pullRequests = pullRequests; } + if (parsed.passProgress && typeof parsed.passProgress === "object") { + const issues = parsePassProgress(parsed.passProgress.issues); + const pullRequests = parsePassProgress(parsed.passProgress.pullRequests); + if (issues) base.passProgress.issues = issues; + if (pullRequests) base.passProgress.pullRequests = pullRequests; + } + return base; } diff --git a/src/lib/utils/concurrency.ts b/src/lib/utils/concurrency.ts index b8044ed..aad6e85 100644 --- a/src/lib/utils/concurrency.ts +++ b/src/lib/utils/concurrency.ts @@ -8,8 +8,8 @@ import { isRateLimitError } from "@/lib/rate-limit-gate"; */ export class RateLimitedBatchError extends Error { readonly status = 403; - readonly results: R[]; - readonly cause: unknown; + declare readonly results: R[]; + declare readonly cause: unknown; constructor(cause: unknown, results: R[], remaining: number) { super( @@ -17,8 +17,10 @@ export class RateLimitedBatchError extends Error { `(${cause instanceof Error ? cause.message : String(cause)})` ); this.name = "RateLimitedBatchError"; - this.cause = cause; - this.results = results; + // Not enumerable, so logging the error does not print every partial + // result and the whole source response it carries. + Object.defineProperty(this, "cause", { value: cause, enumerable: false }); + Object.defineProperty(this, "results", { value: results, enumerable: false }); } } diff --git a/www/src/pages/docs/architecture.mdx b/www/src/pages/docs/architecture.mdx index fb7355e..d0fb3f8 100644 --- a/www/src/pages/docs/architecture.mdx +++ b/www/src/pages/docs/architecture.mdx @@ -158,6 +158,10 @@ The first issues pass and the first pull request pass list everything on GitHub. A full pass still runs when the last one is more than 7 days old, when the destination has no mirrored issues or pull requests (for example after it was recreated), and after Reset metadata. If any item fails, the watermark is not moved, so the next pass lists it again. Items deleted on GitHub are left on the destination, as before. +A full pass lists every issue and pull request, but only fetches comments, pull request details, commits and files for the items that are missing on the destination or changed since the last completed pass. Everything else is corrected from the listing alone, so a weekly full pass over a large tracker costs about one request per hundred items. + +When a pass stops because the source is rate limited, the items it finished are saved with the repository. The next run repeats the same pass and skips them unless they changed in the meantime, so a pass that needs more than one hour of budget still completes over several runs. Once it completes, the watermark is set to the start of its first attempt. + ### Ordering and concurrency Gitea assigns issue numbers on creation, so processing issues in parallel can produce numbering that does not match GitHub's. The concurrency settings default to 3 for issues and 5 for pull requests, trading some ordering fidelity for speed. Set both to 1 if you need the numbering to line up. diff --git a/www/src/pages/use-cases/github-api-rate-limits.mdx b/www/src/pages/use-cases/github-api-rate-limits.mdx index 6c3074f..cc6335c 100644 --- a/www/src/pages/use-cases/github-api-rate-limits.mdx +++ b/www/src/pages/use-cases/github-api-rate-limits.mdx @@ -33,7 +33,7 @@ Gitea Mirror uses this on every GET. Each response's ETag is cached; every subse The practical effect is that sync frequency mostly stops mattering. A `30m` interval on a stable account costs barely more than a `24h` one, because 47 of the 48 daily sweeps are nearly all cache hits. The requests you spend scale with how much changed, not how often you look. -Issues and pull requests go one step further. After the first complete pass, a scheduled sync asks GitHub only for the issues and pull requests updated since the previous one, and fetches comments, commits and changed files for those alone. A tracker with thousands of issues and a handful of changes costs a handful of requests. A full pass still runs once a week to catch anything the incremental listing cannot see. +Issues and pull requests go one step further. After the first complete pass, a scheduled sync asks GitHub only for the issues and pull requests updated since the previous one, and fetches comments, commits and changed files for those alone. A tracker with thousands of issues and a handful of changes costs a handful of requests. A full pass still runs once a week to catch anything the incremental listing cannot see, and it only fetches details for items that are missing or changed. A pass that runs out of budget picks up where it stopped after the reset. When the limit is hit anyway, nothing else is sent. Every GitHub client pauses until the reset GitHub reported: a reset within two minutes is waited out inside the request, a longer one fails the request at once without contacting GitHub, and the run ends so the next scheduled one starts a minute after the reset. Repositories that failed only because of the limit go back to their previous status and are retried then. This matters because GitHub counts requests made while limited toward abuse detection, which can suspend the account.