Files
gitea-mirror/src/lib/repo-backup.ts
Xyndra 2e00a610cb Add E2E testing (#201)
* feat: add E2E testing infrastructure with fake GitHub, Playwright, and CI workflow

- Add fake GitHub API server (tests/e2e/fake-github-server.ts) with
  management API for seeding test data
- Add Playwright E2E test suite covering full mirror workflow:
  service health checks, user registration, config, sync, verify
- Add Docker Compose for E2E Gitea instance
- Add orchestrator script (run-e2e.sh) with cleanup
- Add GitHub Actions workflow (e2e-tests.yml) with Gitea service container
- Make GITHUB_API_URL configurable via env var for testing
- Add npm scripts: test:e2e, test:e2e:ci, test:e2e:keep, test:e2e:cleanup

* feat: add real git repos + backup config testing to E2E suite

- Create programmatic test git repos (create-test-repos.ts) with real
  commits, branches (main, develop, feature/*), and tags (v1.0.0, v1.1.0)
- Add git-server container to docker-compose serving bare repos via
  dumb HTTP protocol so Gitea can actually clone them
- Update fake GitHub server to emit reachable clone_url fields pointing
  to the git-server container (configurable via GIT_SERVER_URL env var)
- Add management endpoint POST /___mgmt/set-clone-url for runtime config
- Update E2E spec with real mirroring verification:
  * Verify repos appear in Gitea with actual content
  * Check branches, tags, commits, file content
  * Verify 4/4 repos mirrored successfully
- Add backup configuration test suite:
  * Enable/disable backupBeforeSync config
  * Toggle blockSyncOnBackupFailure
  * Trigger re-sync with backup enabled and verify activities
  * Verify config persistence across changes
- Update CI workflow to use docker compose (not service containers)
  matching the local run-e2e.sh approach
- Update cleanup.sh for git-repos directory and git-server port
- All 22 tests passing with real git content verification

* refactor: split E2E tests into focused files + add force-push tests

Split the monolithic e2e.spec.ts (1335 lines) into 5 focused spec files
and a shared helpers module:

  helpers.ts                 — constants, GiteaAPI, auth, saveConfig, utilities
  01-health.spec.ts          — service health checks (4 tests)
  02-mirror-workflow.spec.ts — full first-mirror journey (8 tests)
  03-backup.spec.ts          — backup config toggling (6 tests)
  04-force-push.spec.ts      — force-push simulation & backup verification (9 tests)
  05-sync-verification.spec.ts — dynamic repos, content integrity, reset (5 tests)

The force-push tests are the critical addition:
  F0: Record original state (commit SHAs, file content)
  F1: Rewrite source repo history (simulate force-push)
  F2: Sync to Gitea WITHOUT backup
  F3: Verify data loss — LICENSE file gone, README overwritten
  F4: Restore source, re-mirror to clean state
  F5: Enable backup, force-push again, sync through app
  F6: Verify Gitea reflects the force-push
  F7: Verify backup system was invoked (snapshot activities logged)
  F8: Restore source repo for subsequent tests

Also added to helpers.ts:
  - GiteaAPI.getBranch(), .getCommit(), .triggerMirrorSync()
  - getRepositoryIds(), triggerMirrorJobs(), triggerSyncRepo()

All 32 tests passing.

* Try to fix actions

* Try to fix the other action

* Add debug info to check why e2e action is failing

* More debug info

* Even more debug info

* E2E fix attempt #1

* E2E fix attempt #2

* more debug again

* E2E fix attempt #3

* E2E fix attempt #4

* Remove a bunch of debug info

* Hopefully fix backup bug

* Force backups to succeed
2026-03-01 07:35:13 +05:30

171 lines
5.4 KiB
TypeScript

import { mkdir, mkdtemp, readdir, rm, stat } from "node:fs/promises";
import os from "node:os";
import path from "node:path";
import type { Config } from "@/types/config";
import { decryptConfigTokens } from "./utils/config-encryption";
const TRUE_VALUES = new Set(["1", "true", "yes", "on"]);
function parseBoolean(value: string | undefined, fallback: boolean): boolean {
if (value === undefined) return fallback;
return TRUE_VALUES.has(value.trim().toLowerCase());
}
function parsePositiveInt(value: string | undefined, fallback: number): number {
if (!value) return fallback;
const parsed = Number.parseInt(value, 10);
if (!Number.isFinite(parsed) || parsed <= 0) {
return fallback;
}
return parsed;
}
function sanitizePathSegment(input: string): string {
return input.replace(/[^a-zA-Z0-9._-]/g, "_");
}
function buildTimestamp(): string {
// Example: 2026-02-25T18-34-22-123Z
return new Date().toISOString().replace(/[:.]/g, "-");
}
function buildAuthenticatedCloneUrl(cloneUrl: string, token: string): string {
const parsed = new URL(cloneUrl);
if (parsed.protocol !== "http:" && parsed.protocol !== "https:") {
return cloneUrl;
}
parsed.username = process.env.PRE_SYNC_BACKUP_GIT_USERNAME || "oauth2";
parsed.password = token;
return parsed.toString();
}
function maskToken(text: string, token: string): string {
if (!token) return text;
return text.split(token).join("***");
}
async function runGit(args: string[], tokenToMask: string): Promise<void> {
const proc = Bun.spawn({
cmd: ["git", ...args],
stdout: "pipe",
stderr: "pipe",
});
const [stdout, stderr, exitCode] = await Promise.all([
new Response(proc.stdout).text(),
new Response(proc.stderr).text(),
proc.exited,
]);
if (exitCode !== 0) {
const details = [stdout, stderr].filter(Boolean).join("\n").trim();
const safeDetails = maskToken(details, tokenToMask);
throw new Error(`git command failed: ${safeDetails || "unknown git error"}`);
}
}
async function enforceRetention(repoBackupDir: string, keepCount: number): Promise<void> {
const entries = await readdir(repoBackupDir);
const bundleFiles = entries
.filter((name) => name.endsWith(".bundle"))
.map((name) => path.join(repoBackupDir, name));
if (bundleFiles.length <= keepCount) return;
const filesWithMtime = await Promise.all(
bundleFiles.map(async (filePath) => ({
filePath,
mtimeMs: (await stat(filePath)).mtimeMs,
}))
);
filesWithMtime.sort((a, b) => b.mtimeMs - a.mtimeMs);
const toDelete = filesWithMtime.slice(keepCount);
await Promise.all(toDelete.map((entry) => rm(entry.filePath, { force: true })));
}
export function isPreSyncBackupEnabled(): boolean {
return parseBoolean(process.env.PRE_SYNC_BACKUP_ENABLED, true);
}
export function shouldCreatePreSyncBackup(config: Partial<Config>): boolean {
const configSetting = config.giteaConfig?.backupBeforeSync;
const fallback = isPreSyncBackupEnabled();
return configSetting === undefined ? fallback : Boolean(configSetting);
}
export function shouldBlockSyncOnBackupFailure(config: Partial<Config>): boolean {
const configSetting = config.giteaConfig?.blockSyncOnBackupFailure;
return configSetting === undefined ? true : Boolean(configSetting);
}
export async function createPreSyncBundleBackup({
config,
owner,
repoName,
cloneUrl,
}: {
config: Partial<Config>;
owner: string;
repoName: string;
cloneUrl: string;
}): Promise<{ bundlePath: string }> {
if (!shouldCreatePreSyncBackup(config)) {
throw new Error("Pre-sync backup is disabled.");
}
if (!config.giteaConfig?.token) {
throw new Error("Gitea token is required for pre-sync backup.");
}
const decryptedConfig = decryptConfigTokens(config as Config);
const giteaToken = decryptedConfig.giteaConfig?.token;
if (!giteaToken) {
throw new Error("Decrypted Gitea token is required for pre-sync backup.");
}
let backupRoot =
config.giteaConfig?.backupDirectory?.trim() ||
process.env.PRE_SYNC_BACKUP_DIR?.trim() ||
path.join(process.cwd(), "data", "repo-backups");
// Ensure backupRoot is absolute - relative paths break git bundle creation
// because git runs with -C mirrorClonePath and interprets relative paths from there
if (!path.isAbsolute(backupRoot)) {
backupRoot = path.resolve(process.cwd(), backupRoot);
}
const retention = Math.max(
1,
Number.isFinite(config.giteaConfig?.backupRetentionCount)
? Number(config.giteaConfig?.backupRetentionCount)
: parsePositiveInt(process.env.PRE_SYNC_BACKUP_KEEP_COUNT, 20)
);
const repoBackupDir = path.join(
backupRoot,
sanitizePathSegment(config.userId || "unknown-user"),
sanitizePathSegment(owner),
sanitizePathSegment(repoName)
);
await mkdir(repoBackupDir, { recursive: true });
const tmpDir = await mkdtemp(path.join(os.tmpdir(), "gitea-mirror-backup-"));
const mirrorClonePath = path.join(tmpDir, "repo.git");
const bundlePath = path.join(repoBackupDir, `${buildTimestamp()}.bundle`);
try {
const authCloneUrl = buildAuthenticatedCloneUrl(cloneUrl, giteaToken);
await runGit(["clone", "--mirror", authCloneUrl, mirrorClonePath], giteaToken);
await runGit(["-C", mirrorClonePath, "bundle", "create", bundlePath, "--all"], giteaToken);
await enforceRetention(repoBackupDir, retention);
return { bundlePath };
} finally {
await rm(tmpDir, { recursive: true, force: true });
}
}