fix: recover repositories stuck in syncing/mirroring after crashes (#339) (#347)

Adds stuck-status recovery: repositories (and orgs) stranded in an in-flight status by a crash/restart are reset to failed with an explanation, on container start and every scheduler tick, guarded by the existing 2h liveness window.

Fixes #339

scripts/startup-recovery.ts

@@ -13,6 +13,7 @@
  */
 
 import { initializeRecovery, hasJobsNeedingRecovery, getRecoveryStatus } from "../src/lib/recovery";
+import { resetStuckMirrorStatuses } from "../src/lib/stuck-status-recovery";
 
 // Parse command line arguments
 const args = process.argv.slice(2);
@@ -41,10 +42,29 @@ async function runStartupRecovery() {
       }, timeout);
     });
 
+    // Reset repositories/organizations stuck in an in-flight status
+    // ("mirroring"/"syncing") from a previous run (issue #339). This must
+    // happen BEFORE the needsRecovery early-exit below: scheduler-driven
+    // syncs create no resilient job records, so a crash mid-scheduled-sync
+    // leaves stuck repo rows but NO interrupted jobs — the early exit would
+    // skip them forever. The app is not running while this script executes,
+    // so every in-flight row is an orphan by definition (this script's own
+    // process start is the cutoff). Never throws.
+    console.log('Checking for repositories stuck in an in-flight status...');
+    const stuckReset = await resetStuckMirrorStatuses();
+    if (stuckReset.repositories > 0 || stuckReset.organizations > 0) {
+      console.log(
+        `✅ Reset ${stuckReset.repositories} stuck repositor${stuckReset.repositories === 1 ? 'y' : 'ies'} ` +
+        `and ${stuckReset.organizations} stuck organization(s) to "failed" for retry.`
+      );
+    } else {
+      console.log('✅ No stuck repository/organization statuses found.');
+    }
+
     // Check if recovery is needed first
     console.log('Checking if recovery is needed...');
     const needsRecovery = await hasJobsNeedingRecovery();
-    
+
     if (!needsRecovery) {
       console.log('✅ No jobs need recovery. Startup can proceed.');
       process.exit(0);

src/lib/recovery.ts

@@ -4,6 +4,7 @@
  */
 
 import { findInterruptedJobs, resumeInterruptedJob } from './helpers';
+import { resetStuckMirrorStatuses } from './stuck-status-recovery';
 import { db, repositories, organizations, mirrorJobs, configs } from './db';
 import { eq, and, lt, inArray, sql } from 'drizzle-orm';
 import { mirrorGithubRepoToGitea, syncGiteaRepo } from './gitea';
@@ -121,6 +122,13 @@ export async function initializeRecovery(options: {
       // Clean up stale jobs first
       await cleanupStaleJobs();
 
+      // Reset repositories/organizations stuck in an in-flight status
+      // ("mirroring"/"syncing") with no live process behind them (issue
+      // #339). Job-level recovery below only reconciles mirrorJobs rows;
+      // repository.status is never reconciled by it, so rows orphaned by
+      // a crash would otherwise stay "syncing" forever. Never throws.
+      await resetStuckMirrorStatuses();
+
       // Find interrupted jobs (with per-job logging — this is the
       // active recovery path that will immediately try to resume them)
       const interruptedJobs = await findInterruptedJobs({ logFound: true });

src/lib/scheduler-service.ts

@@ -15,6 +15,7 @@ import { mergeGitReposPreferStarred, normalizeGitRepoToInsert, calcBatchSizeForI
 import { isMirrorableGitHubRepo } from '@/lib/repo-eligibility';
 import { createMirrorJob } from '@/lib/helpers';
 import { getNextScheduledRun, isCronExpression, normalizeTimezone } from '@/lib/utils/schedule-utils';
+import { resetStuckMirrorStatuses } from '@/lib/stuck-status-recovery';
 
 let schedulerInterval: NodeJS.Timeout | null = null;
 let isSchedulerRunning = false;
@@ -723,8 +724,20 @@ async function schedulerLoop(): Promise<void> {
   }
   
   isSchedulerRunning = true;
-  
+
   try {
+    // Heal repositories/organizations stuck in an in-flight status
+    // ("mirroring"/"syncing") from a crash or restart (issue #339). Runs on
+    // every tick, before the enabled-config filtering, so stuck rows are
+    // reset even for users without scheduling enabled. Never throws.
+    const stuckReset = await resetStuckMirrorStatuses();
+    if (stuckReset.repositories > 0 || stuckReset.organizations > 0) {
+      console.log(
+        `[Scheduler] Reset ${stuckReset.repositories} stuck repositor${stuckReset.repositories === 1 ? 'y' : 'ies'} ` +
+        `and ${stuckReset.organizations} stuck organization(s) from in-flight status to "failed"`
+      );
+    }
+
     // Get all active configurations with scheduling enabled
     const activeConfigs = await db
       .select()

src/lib/stuck-status-recovery.test.ts

@@ -0,0 +1,197 @@
+/**
+ * Tests for stuck in-flight status recovery (issue #339).
+ *
+ * Repositories interrupted mid-mirror/mid-sync (container restart, OOM,
+ * crash) were left stuck at "mirroring"/"syncing" forever: the scheduler's
+ * sync pool never selects those statuses, job-level recovery only reconciles
+ * mirrorJobs rows (and the scheduler path creates none), and the UI disables
+ * the Sync button for in-flight statuses.
+ *
+ * The decision logic is tested directly as pure functions. The wiring into
+ * the scheduler loop, initializeRecovery, and the startup-recovery script is
+ * asserted by reading the source (same pattern as
+ * gitea-mirror-failure-recovery.test.ts) because behavioral tests of those
+ * modules require process-wide module mocks that pollute other test files.
+ */
+import { describe, test, expect } from "bun:test";
+import { readFileSync } from "node:fs";
+import { join } from "node:path";
+import {
+  IN_FLIGHT_REPO_STATUSES,
+  IN_FLIGHT_ORG_STATUSES,
+  STUCK_IN_FLIGHT_THRESHOLD_MS,
+  computeStuckStatusCutoff,
+  isStuckInFlight,
+  buildStuckResetErrorMessage,
+  buildStuckResetUpdate,
+  resetStuckMirrorStatuses,
+} from "./stuck-status-recovery";
+
+const HOUR = 60 * 60 * 1000;
+
+describe("computeStuckStatusCutoff", () => {
+  test("uses process start as cutoff while the process is younger than the threshold", () => {
+    // Process started 10 minutes ago; now - 2h would reach back BEFORE the
+    // process started. The cutoff must clamp to process start so rows written
+    // by this process are never considered stuck.
+    const processStart = new Date("2026-07-16T10:00:00Z");
+    const now = new Date("2026-07-16T10:10:00Z");
+
+    const cutoff = computeStuckStatusCutoff(now, processStart);
+
+    expect(cutoff.getTime()).toBe(processStart.getTime());
+  });
+
+  test("uses now - threshold once the process has been up longer than the threshold", () => {
+    const processStart = new Date("2026-07-16T00:00:00Z");
+    const now = new Date("2026-07-16T10:00:00Z"); // up for 10 hours
+
+    const cutoff = computeStuckStatusCutoff(now, processStart);
+
+    expect(cutoff.getTime()).toBe(now.getTime() - STUCK_IN_FLIGHT_THRESHOLD_MS);
+  });
+
+  test("honors a custom threshold", () => {
+    const processStart = new Date("2026-07-16T00:00:00Z");
+    const now = new Date("2026-07-16T10:00:00Z");
+
+    const cutoff = computeStuckStatusCutoff(now, processStart, 30 * 60 * 1000);
+
+    expect(cutoff.getTime()).toBe(now.getTime() - 30 * 60 * 1000);
+  });
+
+  test("threshold matches the 2-hour staleness window from isRepoCurrentlyMirroring", () => {
+    expect(STUCK_IN_FLIGHT_THRESHOLD_MS).toBe(2 * HOUR);
+  });
+});
+
+describe("isStuckInFlight", () => {
+  const cutoff = new Date("2026-07-16T08:00:00Z");
+  const before = new Date("2026-07-16T05:00:00Z"); // older than cutoff
+  const after = new Date("2026-07-16T09:00:00Z"); // newer than cutoff
+
+  test("a 'syncing' repo last updated before the cutoff is stuck", () => {
+    expect(isStuckInFlight({ status: "syncing", updatedAt: before }, cutoff)).toBe(true);
+  });
+
+  test("a 'mirroring' repo last updated before the cutoff is stuck", () => {
+    expect(isStuckInFlight({ status: "mirroring", updatedAt: before }, cutoff)).toBe(true);
+  });
+
+  test("an in-flight repo updated after the cutoff is NOT stuck (live work is protected)", () => {
+    expect(isStuckInFlight({ status: "syncing", updatedAt: after }, cutoff)).toBe(false);
+    expect(isStuckInFlight({ status: "mirroring", updatedAt: after }, cutoff)).toBe(false);
+  });
+
+  test("terminal or queued statuses are never stuck, no matter how old", () => {
+    for (const status of ["synced", "mirrored", "failed", "imported", "pending-approval", "archived", "ignored"]) {
+      expect(isStuckInFlight({ status, updatedAt: before }, cutoff)).toBe(false);
+    }
+  });
+
+  test("an in-flight repo with no updatedAt is treated as stuck", () => {
+    expect(isStuckInFlight({ status: "syncing", updatedAt: null }, cutoff)).toBe(true);
+  });
+
+  test("in-flight status lists cover exactly the statuses set before network work", () => {
+    // gitea-enhanced.ts sets "syncing"; gitea.ts sets "mirroring" for repos
+    // and orgs. Nothing else is written as an intermediate status.
+    expect([...IN_FLIGHT_REPO_STATUSES].sort()).toEqual(["mirroring", "syncing"]);
+    expect([...IN_FLIGHT_ORG_STATUSES]).toEqual(["mirroring"]);
+  });
+});
+
+describe("buildStuckResetUpdate / buildStuckResetErrorMessage", () => {
+  test("resets to 'failed' with the provided timestamp", () => {
+    const now = new Date("2026-07-16T12:00:00Z");
+    const update = buildStuckResetUpdate("syncing", now);
+
+    expect(update.status).toBe("failed");
+    expect(update.updatedAt).toBe(now);
+    expect(update.errorMessage.length).toBeGreaterThan(0);
+  });
+
+  test("error message names the stuck status and the interrupted operation", () => {
+    const syncMessage = buildStuckResetErrorMessage("syncing");
+    expect(syncMessage).toContain('"syncing"');
+    expect(syncMessage).toContain("interrupted sync");
+
+    const mirrorMessage = buildStuckResetErrorMessage("mirroring");
+    expect(mirrorMessage).toContain('"mirroring"');
+    expect(mirrorMessage).toContain("interrupted mirror");
+  });
+
+  test("error message tells the user how work resumes", () => {
+    const message = buildStuckResetErrorMessage("syncing");
+    expect(message).toContain("next scheduled run");
+    expect(message).toContain("Retry");
+  });
+});
+
+describe("resetStuckMirrorStatuses", () => {
+  test("never throws even when the db layer misbehaves", async () => {
+    // The global test setup replaces @/lib/db with a stub whose select chain
+    // does not return arrays. The function must swallow that (it runs inside
+    // the scheduler loop and recovery — housekeeping must never block them)
+    // and report zero resets.
+    const result = await resetStuckMirrorStatuses({
+      cutoff: new Date(),
+      now: new Date(),
+    });
+
+    expect(result).toEqual({ repositories: 0, organizations: 0 });
+  });
+});
+
+describe("wiring (source regression)", () => {
+  const read = (...segments: string[]) =>
+    readFileSync(join(import.meta.dir, ...segments), "utf8");
+
+  test("scheduler loop resets stuck statuses on every tick, before config filtering", () => {
+    const source = read("scheduler-service.ts");
+
+    expect(source).toContain('from \'@/lib/stuck-status-recovery\'');
+
+    const loopStart = source.indexOf("async function schedulerLoop");
+    expect(loopStart).toBeGreaterThan(-1);
+    const loopBody = source.slice(loopStart);
+
+    const resetCall = loopBody.indexOf("resetStuckMirrorStatuses(");
+    const configQuery = loopBody.indexOf("const activeConfigs");
+    expect(resetCall).toBeGreaterThan(-1);
+    expect(configQuery).toBeGreaterThan(-1);
+    // Must run before the enabled-config filtering so stuck rows heal even
+    // for users without scheduling enabled.
+    expect(resetCall).toBeLessThan(configQuery);
+  });
+
+  test("initializeRecovery resets stuck statuses before resuming interrupted jobs", () => {
+    const source = read("recovery.ts");
+
+    expect(source).toContain("from './stuck-status-recovery'");
+
+    const initStart = source.indexOf("export async function initializeRecovery");
+    expect(initStart).toBeGreaterThan(-1);
+    const initBody = source.slice(initStart);
+
+    const resetCall = initBody.indexOf("resetStuckMirrorStatuses(");
+    const findJobs = initBody.indexOf("findInterruptedJobs(");
+    expect(resetCall).toBeGreaterThan(-1);
+    expect(findJobs).toBeGreaterThan(-1);
+    expect(resetCall).toBeLessThan(findJobs);
+  });
+
+  test("startup-recovery script resets stuck statuses BEFORE the no-jobs early exit", () => {
+    const source = read("..", "..", "scripts", "startup-recovery.ts");
+
+    const resetCall = source.indexOf("resetStuckMirrorStatuses(");
+    const needsRecoveryCheck = source.indexOf("hasJobsNeedingRecovery()");
+    expect(resetCall).toBeGreaterThan(-1);
+    expect(needsRecoveryCheck).toBeGreaterThan(-1);
+    // Scheduler-driven syncs create no resilient job records, so a crash
+    // mid-scheduled-sync leaves stuck repos but NO interrupted jobs. If the
+    // reset ran after the early exit it would be skipped in exactly the case
+    // that matters (issue #339).
+    expect(resetCall).toBeLessThan(needsRecoveryCheck);
+  });
+});

src/lib/stuck-status-recovery.ts

@@ -0,0 +1,251 @@
+/**
+ * Stuck in-flight status recovery (issue #339).
+ *
+ * Repositories are marked "mirroring"/"syncing" (organizations: "mirroring")
+ * in the DB before long-running network work starts. Every in-process failure
+ * path resets the status via catch blocks, but a hard interruption (container
+ * restart, OOM kill, host reboot, deploy) kills the process before the catch
+ * runs and leaves the row stuck in an in-flight status forever:
+ *
+ *   - the scheduler's sync pool only selects mirrored/synced/failed/pending
+ *     (scheduler-service.ts), so a stuck repo is never picked up again;
+ *   - the UI disables the Sync/Mirror button for in-flight statuses
+ *     (RepositoryTable.tsx), so the user cannot restart it either;
+ *   - job-level recovery (recovery.ts) only reconciles mirrorJobs rows — the
+ *     scheduler's sync path does not create resilient job records at all, so
+ *     an interrupted scheduled sync leaves nothing for recovery to find.
+ *
+ * This module resets those orphaned rows to "failed" (with an explanatory
+ * errorMessage) so the scheduler's next run and the UI's Retry button can
+ * pick them up again. It is invoked from:
+ *
+ *   1. the scheduler loop (every minute) — heals stuck rows at runtime;
+ *   2. initializeRecovery() — heals on the startup/middleware recovery path;
+ *   3. scripts/startup-recovery.ts — heals before the app starts serving.
+ *
+ * Cutoff semantics: a row is only reset when its updatedAt is older than
+ * max(process start, now - 2h). Rows written by the current process are
+ * therefore never touched while the process is younger than the threshold,
+ * and long-lived processes use the same 2-hour staleness window that
+ * isRepoCurrentlyMirroring (gitea.ts) already applies to in-flight statuses.
+ */
+
+import { db, repositories, organizations } from "@/lib/db";
+import { inArray, eq } from "drizzle-orm";
+import { repoStatusEnum } from "@/types/Repository";
+import { createMirrorJob } from "@/lib/helpers";
+
+/** Repository statuses that indicate in-flight work. */
+export const IN_FLIGHT_REPO_STATUSES = ["mirroring", "syncing"] as const;
+
+/** Organization statuses that indicate in-flight work. */
+export const IN_FLIGHT_ORG_STATUSES = ["mirroring"] as const;
+
+/**
+ * How long an in-flight status may go without an update before it is
+ * considered stuck. Matches the 2-hour staleness window used by
+ * isRepoCurrentlyMirroring in gitea.ts.
+ */
+export const STUCK_IN_FLIGHT_THRESHOLD_MS = 2 * 60 * 60 * 1000;
+
+/**
+ * Captured at module load, before any request handling can start a mirror or
+ * sync in this process. Any in-flight row older than this was written by a
+ * previous (crashed/restarted) process.
+ */
+const PROCESS_START = new Date();
+
+export function getProcessStart(): Date {
+  return PROCESS_START;
+}
+
+/**
+ * Compute the cutoff before which an in-flight status counts as stuck:
+ * max(processStart, now - threshold).
+ *
+ * - Early in the process lifetime the cutoff is the process start, so rows
+ *   stuck by a PREVIOUS process are reset immediately after a restart while
+ *   rows written by THIS process are never touched.
+ * - Once the process has been up longer than the threshold, the cutoff is
+ *   now - threshold, healing operations that stalled at runtime.
+ */
+export function computeStuckStatusCutoff(
+  now: Date,
+  processStart: Date = PROCESS_START,
+  thresholdMs: number = STUCK_IN_FLIGHT_THRESHOLD_MS
+): Date {
+  return new Date(Math.max(processStart.getTime(), now.getTime() - thresholdMs));
+}
+
+/**
+ * Whether a row with an in-flight status counts as stuck relative to the
+ * cutoff. A missing updatedAt is treated as stuck (nothing can prove the
+ * work is still alive).
+ */
+export function isStuckInFlight(
+  row: { status: string; updatedAt: Date | null },
+  cutoff: Date
+): boolean {
+  if (
+    !(IN_FLIGHT_REPO_STATUSES as readonly string[]).includes(row.status) &&
+    !(IN_FLIGHT_ORG_STATUSES as readonly string[]).includes(row.status)
+  ) {
+    return false;
+  }
+  if (!row.updatedAt) {
+    return true;
+  }
+  return new Date(row.updatedAt).getTime() < cutoff.getTime();
+}
+
+/** Human-readable explanation stored in errorMessage on reset. */
+export function buildStuckResetErrorMessage(previousStatus: string): string {
+  const operation = previousStatus === "syncing" ? "sync" : "mirror";
+  return (
+    `Detected interrupted ${operation}: status was stuck at "${previousStatus}" ` +
+    `(the application was likely restarted or crashed mid-operation). ` +
+    `The status was reset automatically; the next scheduled run will retry, ` +
+    `or you can use Retry to run it now.`
+  );
+}
+
+/** The DB update payload applied to a stuck row. */
+export function buildStuckResetUpdate(
+  previousStatus: string,
+  now: Date
+): { status: "failed"; errorMessage: string; updatedAt: Date } {
+  return {
+    status: repoStatusEnum.parse("failed") as "failed",
+    errorMessage: buildStuckResetErrorMessage(previousStatus),
+    updatedAt: now,
+  };
+}
+
+export interface StuckStatusResetResult {
+  repositories: number;
+  organizations: number;
+}
+
+/**
+ * Reset repositories and organizations stuck in an in-flight status to
+ * "failed" so the scheduler and the UI's Retry action can pick them up.
+ *
+ * Never throws: errors are logged and reflected as zero counts so callers
+ * (scheduler loop, recovery) are never blocked by this housekeeping step.
+ */
+export async function resetStuckMirrorStatuses(options: {
+  cutoff?: Date;
+  now?: Date;
+} = {}): Promise<StuckStatusResetResult> {
+  const now = options.now ?? new Date();
+  const cutoff = options.cutoff ?? computeStuckStatusCutoff(now);
+  const result: StuckStatusResetResult = { repositories: 0, organizations: 0 };
+
+  // --- Repositories ---
+  try {
+    const inFlightRepos = await db
+      .select({
+        id: repositories.id,
+        userId: repositories.userId,
+        name: repositories.name,
+        fullName: repositories.fullName,
+        status: repositories.status,
+        updatedAt: repositories.updatedAt,
+      })
+      .from(repositories)
+      .where(inArray(repositories.status, [...IN_FLIGHT_REPO_STATUSES]));
+
+    const stuckRepos = inFlightRepos.filter((repo) =>
+      isStuckInFlight(repo, cutoff)
+    );
+
+    for (const repo of stuckRepos) {
+      await db
+        .update(repositories)
+        .set(buildStuckResetUpdate(repo.status, now))
+        .where(eq(repositories.id, repo.id));
+
+      // Activity-log entry + SSE event so the UI updates live. Push
+      // notifications are skipped: a restart can reset many rows at once
+      // and this is internal housekeeping, not a user-triggered failure.
+      await createMirrorJob({
+        userId: repo.userId,
+        repositoryId: repo.id,
+        repositoryName: repo.name,
+        message: `Reset stuck repository status: ${repo.fullName ?? repo.name}`,
+        details:
+          `Repository was stuck at "${repo.status}" since ` +
+          `${repo.updatedAt ? new Date(repo.updatedAt).toISOString() : "an unknown time"} ` +
+          `with no active job — most likely an application restart or crash interrupted it. ` +
+          `Status was reset to "failed" so it can be retried.`,
+        status: "failed",
+        skipNotification: true,
+      });
+
+      console.log(
+        `[StuckStatusRecovery] Reset repository ${repo.fullName ?? repo.name} from "${repo.status}" to "failed" (stuck since ${
+          repo.updatedAt ? new Date(repo.updatedAt).toISOString() : "unknown"
+        })`
+      );
+    }
+
+    result.repositories = stuckRepos.length;
+  } catch (error) {
+    console.error(
+      "[StuckStatusRecovery] Failed to reset stuck repository statuses:",
+      error
+    );
+  }
+
+  // --- Organizations ---
+  try {
+    const inFlightOrgs = await db
+      .select({
+        id: organizations.id,
+        userId: organizations.userId,
+        name: organizations.name,
+        status: organizations.status,
+        updatedAt: organizations.updatedAt,
+      })
+      .from(organizations)
+      .where(inArray(organizations.status, [...IN_FLIGHT_ORG_STATUSES]));
+
+    const stuckOrgs = inFlightOrgs.filter((org) => isStuckInFlight(org, cutoff));
+
+    for (const org of stuckOrgs) {
+      await db
+        .update(organizations)
+        .set(buildStuckResetUpdate(org.status, now))
+        .where(eq(organizations.id, org.id));
+
+      await createMirrorJob({
+        userId: org.userId,
+        organizationId: org.id,
+        organizationName: org.name,
+        message: `Reset stuck organization status: ${org.name}`,
+        details:
+          `Organization was stuck at "${org.status}" since ` +
+          `${org.updatedAt ? new Date(org.updatedAt).toISOString() : "an unknown time"} ` +
+          `with no active job — most likely an application restart or crash interrupted it. ` +
+          `Status was reset to "failed" so it can be retried.`,
+        status: "failed",
+        skipNotification: true,
+      });
+
+      console.log(
+        `[StuckStatusRecovery] Reset organization ${org.name} from "${org.status}" to "failed" (stuck since ${
+          org.updatedAt ? new Date(org.updatedAt).toISOString() : "unknown"
+        })`
+      );
+    }
+
+    result.organizations = stuckOrgs.length;
+  } catch (error) {
+    console.error(
+      "[StuckStatusRecovery] Failed to reset stuck organization statuses:",
+      error
+    );
+  }
+
+  return result;
+}