diff --git a/scripts/startup-recovery.ts b/scripts/startup-recovery.ts index 8d7b594..a31efca 100644 --- a/scripts/startup-recovery.ts +++ b/scripts/startup-recovery.ts @@ -13,6 +13,7 @@ */ import { initializeRecovery, hasJobsNeedingRecovery, getRecoveryStatus } from "../src/lib/recovery"; +import { resetStuckMirrorStatuses } from "../src/lib/stuck-status-recovery"; // Parse command line arguments const args = process.argv.slice(2); @@ -41,10 +42,29 @@ async function runStartupRecovery() { }, timeout); }); + // Reset repositories/organizations stuck in an in-flight status + // ("mirroring"/"syncing") from a previous run (issue #339). This must + // happen BEFORE the needsRecovery early-exit below: scheduler-driven + // syncs create no resilient job records, so a crash mid-scheduled-sync + // leaves stuck repo rows but NO interrupted jobs — the early exit would + // skip them forever. The app is not running while this script executes, + // so every in-flight row is an orphan by definition (this script's own + // process start is the cutoff). Never throws. + console.log('Checking for repositories stuck in an in-flight status...'); + const stuckReset = await resetStuckMirrorStatuses(); + if (stuckReset.repositories > 0 || stuckReset.organizations > 0) { + console.log( + `✅ Reset ${stuckReset.repositories} stuck repositor${stuckReset.repositories === 1 ? 'y' : 'ies'} ` + + `and ${stuckReset.organizations} stuck organization(s) to "failed" for retry.` + ); + } else { + console.log('✅ No stuck repository/organization statuses found.'); + } + // Check if recovery is needed first console.log('Checking if recovery is needed...'); const needsRecovery = await hasJobsNeedingRecovery(); - + if (!needsRecovery) { console.log('✅ No jobs need recovery. Startup can proceed.'); process.exit(0); diff --git a/src/lib/recovery.ts b/src/lib/recovery.ts index 04a2f61..9cbde7b 100644 --- a/src/lib/recovery.ts +++ b/src/lib/recovery.ts @@ -4,6 +4,7 @@ */ import { findInterruptedJobs, resumeInterruptedJob } from './helpers'; +import { resetStuckMirrorStatuses } from './stuck-status-recovery'; import { db, repositories, organizations, mirrorJobs, configs } from './db'; import { eq, and, lt, inArray, sql } from 'drizzle-orm'; import { mirrorGithubRepoToGitea, syncGiteaRepo } from './gitea'; @@ -121,6 +122,13 @@ export async function initializeRecovery(options: { // Clean up stale jobs first await cleanupStaleJobs(); + // Reset repositories/organizations stuck in an in-flight status + // ("mirroring"/"syncing") with no live process behind them (issue + // #339). Job-level recovery below only reconciles mirrorJobs rows; + // repository.status is never reconciled by it, so rows orphaned by + // a crash would otherwise stay "syncing" forever. Never throws. + await resetStuckMirrorStatuses(); + // Find interrupted jobs (with per-job logging — this is the // active recovery path that will immediately try to resume them) const interruptedJobs = await findInterruptedJobs({ logFound: true }); diff --git a/src/lib/scheduler-service.ts b/src/lib/scheduler-service.ts index f75737d..fac0b85 100644 --- a/src/lib/scheduler-service.ts +++ b/src/lib/scheduler-service.ts @@ -15,6 +15,7 @@ import { mergeGitReposPreferStarred, normalizeGitRepoToInsert, calcBatchSizeForI import { isMirrorableGitHubRepo } from '@/lib/repo-eligibility'; import { createMirrorJob } from '@/lib/helpers'; import { getNextScheduledRun, isCronExpression, normalizeTimezone } from '@/lib/utils/schedule-utils'; +import { resetStuckMirrorStatuses } from '@/lib/stuck-status-recovery'; let schedulerInterval: NodeJS.Timeout | null = null; let isSchedulerRunning = false; @@ -723,8 +724,20 @@ async function schedulerLoop(): Promise { } isSchedulerRunning = true; - + try { + // Heal repositories/organizations stuck in an in-flight status + // ("mirroring"/"syncing") from a crash or restart (issue #339). Runs on + // every tick, before the enabled-config filtering, so stuck rows are + // reset even for users without scheduling enabled. Never throws. + const stuckReset = await resetStuckMirrorStatuses(); + if (stuckReset.repositories > 0 || stuckReset.organizations > 0) { + console.log( + `[Scheduler] Reset ${stuckReset.repositories} stuck repositor${stuckReset.repositories === 1 ? 'y' : 'ies'} ` + + `and ${stuckReset.organizations} stuck organization(s) from in-flight status to "failed"` + ); + } + // Get all active configurations with scheduling enabled const activeConfigs = await db .select() diff --git a/src/lib/stuck-status-recovery.test.ts b/src/lib/stuck-status-recovery.test.ts new file mode 100644 index 0000000..3590c84 --- /dev/null +++ b/src/lib/stuck-status-recovery.test.ts @@ -0,0 +1,197 @@ +/** + * Tests for stuck in-flight status recovery (issue #339). + * + * Repositories interrupted mid-mirror/mid-sync (container restart, OOM, + * crash) were left stuck at "mirroring"/"syncing" forever: the scheduler's + * sync pool never selects those statuses, job-level recovery only reconciles + * mirrorJobs rows (and the scheduler path creates none), and the UI disables + * the Sync button for in-flight statuses. + * + * The decision logic is tested directly as pure functions. The wiring into + * the scheduler loop, initializeRecovery, and the startup-recovery script is + * asserted by reading the source (same pattern as + * gitea-mirror-failure-recovery.test.ts) because behavioral tests of those + * modules require process-wide module mocks that pollute other test files. + */ +import { describe, test, expect } from "bun:test"; +import { readFileSync } from "node:fs"; +import { join } from "node:path"; +import { + IN_FLIGHT_REPO_STATUSES, + IN_FLIGHT_ORG_STATUSES, + STUCK_IN_FLIGHT_THRESHOLD_MS, + computeStuckStatusCutoff, + isStuckInFlight, + buildStuckResetErrorMessage, + buildStuckResetUpdate, + resetStuckMirrorStatuses, +} from "./stuck-status-recovery"; + +const HOUR = 60 * 60 * 1000; + +describe("computeStuckStatusCutoff", () => { + test("uses process start as cutoff while the process is younger than the threshold", () => { + // Process started 10 minutes ago; now - 2h would reach back BEFORE the + // process started. The cutoff must clamp to process start so rows written + // by this process are never considered stuck. + const processStart = new Date("2026-07-16T10:00:00Z"); + const now = new Date("2026-07-16T10:10:00Z"); + + const cutoff = computeStuckStatusCutoff(now, processStart); + + expect(cutoff.getTime()).toBe(processStart.getTime()); + }); + + test("uses now - threshold once the process has been up longer than the threshold", () => { + const processStart = new Date("2026-07-16T00:00:00Z"); + const now = new Date("2026-07-16T10:00:00Z"); // up for 10 hours + + const cutoff = computeStuckStatusCutoff(now, processStart); + + expect(cutoff.getTime()).toBe(now.getTime() - STUCK_IN_FLIGHT_THRESHOLD_MS); + }); + + test("honors a custom threshold", () => { + const processStart = new Date("2026-07-16T00:00:00Z"); + const now = new Date("2026-07-16T10:00:00Z"); + + const cutoff = computeStuckStatusCutoff(now, processStart, 30 * 60 * 1000); + + expect(cutoff.getTime()).toBe(now.getTime() - 30 * 60 * 1000); + }); + + test("threshold matches the 2-hour staleness window from isRepoCurrentlyMirroring", () => { + expect(STUCK_IN_FLIGHT_THRESHOLD_MS).toBe(2 * HOUR); + }); +}); + +describe("isStuckInFlight", () => { + const cutoff = new Date("2026-07-16T08:00:00Z"); + const before = new Date("2026-07-16T05:00:00Z"); // older than cutoff + const after = new Date("2026-07-16T09:00:00Z"); // newer than cutoff + + test("a 'syncing' repo last updated before the cutoff is stuck", () => { + expect(isStuckInFlight({ status: "syncing", updatedAt: before }, cutoff)).toBe(true); + }); + + test("a 'mirroring' repo last updated before the cutoff is stuck", () => { + expect(isStuckInFlight({ status: "mirroring", updatedAt: before }, cutoff)).toBe(true); + }); + + test("an in-flight repo updated after the cutoff is NOT stuck (live work is protected)", () => { + expect(isStuckInFlight({ status: "syncing", updatedAt: after }, cutoff)).toBe(false); + expect(isStuckInFlight({ status: "mirroring", updatedAt: after }, cutoff)).toBe(false); + }); + + test("terminal or queued statuses are never stuck, no matter how old", () => { + for (const status of ["synced", "mirrored", "failed", "imported", "pending-approval", "archived", "ignored"]) { + expect(isStuckInFlight({ status, updatedAt: before }, cutoff)).toBe(false); + } + }); + + test("an in-flight repo with no updatedAt is treated as stuck", () => { + expect(isStuckInFlight({ status: "syncing", updatedAt: null }, cutoff)).toBe(true); + }); + + test("in-flight status lists cover exactly the statuses set before network work", () => { + // gitea-enhanced.ts sets "syncing"; gitea.ts sets "mirroring" for repos + // and orgs. Nothing else is written as an intermediate status. + expect([...IN_FLIGHT_REPO_STATUSES].sort()).toEqual(["mirroring", "syncing"]); + expect([...IN_FLIGHT_ORG_STATUSES]).toEqual(["mirroring"]); + }); +}); + +describe("buildStuckResetUpdate / buildStuckResetErrorMessage", () => { + test("resets to 'failed' with the provided timestamp", () => { + const now = new Date("2026-07-16T12:00:00Z"); + const update = buildStuckResetUpdate("syncing", now); + + expect(update.status).toBe("failed"); + expect(update.updatedAt).toBe(now); + expect(update.errorMessage.length).toBeGreaterThan(0); + }); + + test("error message names the stuck status and the interrupted operation", () => { + const syncMessage = buildStuckResetErrorMessage("syncing"); + expect(syncMessage).toContain('"syncing"'); + expect(syncMessage).toContain("interrupted sync"); + + const mirrorMessage = buildStuckResetErrorMessage("mirroring"); + expect(mirrorMessage).toContain('"mirroring"'); + expect(mirrorMessage).toContain("interrupted mirror"); + }); + + test("error message tells the user how work resumes", () => { + const message = buildStuckResetErrorMessage("syncing"); + expect(message).toContain("next scheduled run"); + expect(message).toContain("Retry"); + }); +}); + +describe("resetStuckMirrorStatuses", () => { + test("never throws even when the db layer misbehaves", async () => { + // The global test setup replaces @/lib/db with a stub whose select chain + // does not return arrays. The function must swallow that (it runs inside + // the scheduler loop and recovery — housekeeping must never block them) + // and report zero resets. + const result = await resetStuckMirrorStatuses({ + cutoff: new Date(), + now: new Date(), + }); + + expect(result).toEqual({ repositories: 0, organizations: 0 }); + }); +}); + +describe("wiring (source regression)", () => { + const read = (...segments: string[]) => + readFileSync(join(import.meta.dir, ...segments), "utf8"); + + test("scheduler loop resets stuck statuses on every tick, before config filtering", () => { + const source = read("scheduler-service.ts"); + + expect(source).toContain('from \'@/lib/stuck-status-recovery\''); + + const loopStart = source.indexOf("async function schedulerLoop"); + expect(loopStart).toBeGreaterThan(-1); + const loopBody = source.slice(loopStart); + + const resetCall = loopBody.indexOf("resetStuckMirrorStatuses("); + const configQuery = loopBody.indexOf("const activeConfigs"); + expect(resetCall).toBeGreaterThan(-1); + expect(configQuery).toBeGreaterThan(-1); + // Must run before the enabled-config filtering so stuck rows heal even + // for users without scheduling enabled. + expect(resetCall).toBeLessThan(configQuery); + }); + + test("initializeRecovery resets stuck statuses before resuming interrupted jobs", () => { + const source = read("recovery.ts"); + + expect(source).toContain("from './stuck-status-recovery'"); + + const initStart = source.indexOf("export async function initializeRecovery"); + expect(initStart).toBeGreaterThan(-1); + const initBody = source.slice(initStart); + + const resetCall = initBody.indexOf("resetStuckMirrorStatuses("); + const findJobs = initBody.indexOf("findInterruptedJobs("); + expect(resetCall).toBeGreaterThan(-1); + expect(findJobs).toBeGreaterThan(-1); + expect(resetCall).toBeLessThan(findJobs); + }); + + test("startup-recovery script resets stuck statuses BEFORE the no-jobs early exit", () => { + const source = read("..", "..", "scripts", "startup-recovery.ts"); + + const resetCall = source.indexOf("resetStuckMirrorStatuses("); + const needsRecoveryCheck = source.indexOf("hasJobsNeedingRecovery()"); + expect(resetCall).toBeGreaterThan(-1); + expect(needsRecoveryCheck).toBeGreaterThan(-1); + // Scheduler-driven syncs create no resilient job records, so a crash + // mid-scheduled-sync leaves stuck repos but NO interrupted jobs. If the + // reset ran after the early exit it would be skipped in exactly the case + // that matters (issue #339). + expect(resetCall).toBeLessThan(needsRecoveryCheck); + }); +}); diff --git a/src/lib/stuck-status-recovery.ts b/src/lib/stuck-status-recovery.ts new file mode 100644 index 0000000..089aa34 --- /dev/null +++ b/src/lib/stuck-status-recovery.ts @@ -0,0 +1,251 @@ +/** + * Stuck in-flight status recovery (issue #339). + * + * Repositories are marked "mirroring"/"syncing" (organizations: "mirroring") + * in the DB before long-running network work starts. Every in-process failure + * path resets the status via catch blocks, but a hard interruption (container + * restart, OOM kill, host reboot, deploy) kills the process before the catch + * runs and leaves the row stuck in an in-flight status forever: + * + * - the scheduler's sync pool only selects mirrored/synced/failed/pending + * (scheduler-service.ts), so a stuck repo is never picked up again; + * - the UI disables the Sync/Mirror button for in-flight statuses + * (RepositoryTable.tsx), so the user cannot restart it either; + * - job-level recovery (recovery.ts) only reconciles mirrorJobs rows — the + * scheduler's sync path does not create resilient job records at all, so + * an interrupted scheduled sync leaves nothing for recovery to find. + * + * This module resets those orphaned rows to "failed" (with an explanatory + * errorMessage) so the scheduler's next run and the UI's Retry button can + * pick them up again. It is invoked from: + * + * 1. the scheduler loop (every minute) — heals stuck rows at runtime; + * 2. initializeRecovery() — heals on the startup/middleware recovery path; + * 3. scripts/startup-recovery.ts — heals before the app starts serving. + * + * Cutoff semantics: a row is only reset when its updatedAt is older than + * max(process start, now - 2h). Rows written by the current process are + * therefore never touched while the process is younger than the threshold, + * and long-lived processes use the same 2-hour staleness window that + * isRepoCurrentlyMirroring (gitea.ts) already applies to in-flight statuses. + */ + +import { db, repositories, organizations } from "@/lib/db"; +import { inArray, eq } from "drizzle-orm"; +import { repoStatusEnum } from "@/types/Repository"; +import { createMirrorJob } from "@/lib/helpers"; + +/** Repository statuses that indicate in-flight work. */ +export const IN_FLIGHT_REPO_STATUSES = ["mirroring", "syncing"] as const; + +/** Organization statuses that indicate in-flight work. */ +export const IN_FLIGHT_ORG_STATUSES = ["mirroring"] as const; + +/** + * How long an in-flight status may go without an update before it is + * considered stuck. Matches the 2-hour staleness window used by + * isRepoCurrentlyMirroring in gitea.ts. + */ +export const STUCK_IN_FLIGHT_THRESHOLD_MS = 2 * 60 * 60 * 1000; + +/** + * Captured at module load, before any request handling can start a mirror or + * sync in this process. Any in-flight row older than this was written by a + * previous (crashed/restarted) process. + */ +const PROCESS_START = new Date(); + +export function getProcessStart(): Date { + return PROCESS_START; +} + +/** + * Compute the cutoff before which an in-flight status counts as stuck: + * max(processStart, now - threshold). + * + * - Early in the process lifetime the cutoff is the process start, so rows + * stuck by a PREVIOUS process are reset immediately after a restart while + * rows written by THIS process are never touched. + * - Once the process has been up longer than the threshold, the cutoff is + * now - threshold, healing operations that stalled at runtime. + */ +export function computeStuckStatusCutoff( + now: Date, + processStart: Date = PROCESS_START, + thresholdMs: number = STUCK_IN_FLIGHT_THRESHOLD_MS +): Date { + return new Date(Math.max(processStart.getTime(), now.getTime() - thresholdMs)); +} + +/** + * Whether a row with an in-flight status counts as stuck relative to the + * cutoff. A missing updatedAt is treated as stuck (nothing can prove the + * work is still alive). + */ +export function isStuckInFlight( + row: { status: string; updatedAt: Date | null }, + cutoff: Date +): boolean { + if ( + !(IN_FLIGHT_REPO_STATUSES as readonly string[]).includes(row.status) && + !(IN_FLIGHT_ORG_STATUSES as readonly string[]).includes(row.status) + ) { + return false; + } + if (!row.updatedAt) { + return true; + } + return new Date(row.updatedAt).getTime() < cutoff.getTime(); +} + +/** Human-readable explanation stored in errorMessage on reset. */ +export function buildStuckResetErrorMessage(previousStatus: string): string { + const operation = previousStatus === "syncing" ? "sync" : "mirror"; + return ( + `Detected interrupted ${operation}: status was stuck at "${previousStatus}" ` + + `(the application was likely restarted or crashed mid-operation). ` + + `The status was reset automatically; the next scheduled run will retry, ` + + `or you can use Retry to run it now.` + ); +} + +/** The DB update payload applied to a stuck row. */ +export function buildStuckResetUpdate( + previousStatus: string, + now: Date +): { status: "failed"; errorMessage: string; updatedAt: Date } { + return { + status: repoStatusEnum.parse("failed") as "failed", + errorMessage: buildStuckResetErrorMessage(previousStatus), + updatedAt: now, + }; +} + +export interface StuckStatusResetResult { + repositories: number; + organizations: number; +} + +/** + * Reset repositories and organizations stuck in an in-flight status to + * "failed" so the scheduler and the UI's Retry action can pick them up. + * + * Never throws: errors are logged and reflected as zero counts so callers + * (scheduler loop, recovery) are never blocked by this housekeeping step. + */ +export async function resetStuckMirrorStatuses(options: { + cutoff?: Date; + now?: Date; +} = {}): Promise { + const now = options.now ?? new Date(); + const cutoff = options.cutoff ?? computeStuckStatusCutoff(now); + const result: StuckStatusResetResult = { repositories: 0, organizations: 0 }; + + // --- Repositories --- + try { + const inFlightRepos = await db + .select({ + id: repositories.id, + userId: repositories.userId, + name: repositories.name, + fullName: repositories.fullName, + status: repositories.status, + updatedAt: repositories.updatedAt, + }) + .from(repositories) + .where(inArray(repositories.status, [...IN_FLIGHT_REPO_STATUSES])); + + const stuckRepos = inFlightRepos.filter((repo) => + isStuckInFlight(repo, cutoff) + ); + + for (const repo of stuckRepos) { + await db + .update(repositories) + .set(buildStuckResetUpdate(repo.status, now)) + .where(eq(repositories.id, repo.id)); + + // Activity-log entry + SSE event so the UI updates live. Push + // notifications are skipped: a restart can reset many rows at once + // and this is internal housekeeping, not a user-triggered failure. + await createMirrorJob({ + userId: repo.userId, + repositoryId: repo.id, + repositoryName: repo.name, + message: `Reset stuck repository status: ${repo.fullName ?? repo.name}`, + details: + `Repository was stuck at "${repo.status}" since ` + + `${repo.updatedAt ? new Date(repo.updatedAt).toISOString() : "an unknown time"} ` + + `with no active job — most likely an application restart or crash interrupted it. ` + + `Status was reset to "failed" so it can be retried.`, + status: "failed", + skipNotification: true, + }); + + console.log( + `[StuckStatusRecovery] Reset repository ${repo.fullName ?? repo.name} from "${repo.status}" to "failed" (stuck since ${ + repo.updatedAt ? new Date(repo.updatedAt).toISOString() : "unknown" + })` + ); + } + + result.repositories = stuckRepos.length; + } catch (error) { + console.error( + "[StuckStatusRecovery] Failed to reset stuck repository statuses:", + error + ); + } + + // --- Organizations --- + try { + const inFlightOrgs = await db + .select({ + id: organizations.id, + userId: organizations.userId, + name: organizations.name, + status: organizations.status, + updatedAt: organizations.updatedAt, + }) + .from(organizations) + .where(inArray(organizations.status, [...IN_FLIGHT_ORG_STATUSES])); + + const stuckOrgs = inFlightOrgs.filter((org) => isStuckInFlight(org, cutoff)); + + for (const org of stuckOrgs) { + await db + .update(organizations) + .set(buildStuckResetUpdate(org.status, now)) + .where(eq(organizations.id, org.id)); + + await createMirrorJob({ + userId: org.userId, + organizationId: org.id, + organizationName: org.name, + message: `Reset stuck organization status: ${org.name}`, + details: + `Organization was stuck at "${org.status}" since ` + + `${org.updatedAt ? new Date(org.updatedAt).toISOString() : "an unknown time"} ` + + `with no active job — most likely an application restart or crash interrupted it. ` + + `Status was reset to "failed" so it can be retried.`, + status: "failed", + skipNotification: true, + }); + + console.log( + `[StuckStatusRecovery] Reset organization ${org.name} from "${org.status}" to "failed" (stuck since ${ + org.updatedAt ? new Date(org.updatedAt).toISOString() : "unknown" + })` + ); + } + + result.organizations = stuckOrgs.length; + } catch (error) { + console.error( + "[StuckStatusRecovery] Failed to reset stuck organization statuses:", + error + ); + } + + return result; +}