mirror of
https://github.com/RayLabsHQ/gitea-mirror.git
synced 2026-03-28 17:42:38 +01:00
* feat: add E2E testing infrastructure with fake GitHub, Playwright, and CI workflow - Add fake GitHub API server (tests/e2e/fake-github-server.ts) with management API for seeding test data - Add Playwright E2E test suite covering full mirror workflow: service health checks, user registration, config, sync, verify - Add Docker Compose for E2E Gitea instance - Add orchestrator script (run-e2e.sh) with cleanup - Add GitHub Actions workflow (e2e-tests.yml) with Gitea service container - Make GITHUB_API_URL configurable via env var for testing - Add npm scripts: test:e2e, test:e2e:ci, test:e2e:keep, test:e2e:cleanup * feat: add real git repos + backup config testing to E2E suite - Create programmatic test git repos (create-test-repos.ts) with real commits, branches (main, develop, feature/*), and tags (v1.0.0, v1.1.0) - Add git-server container to docker-compose serving bare repos via dumb HTTP protocol so Gitea can actually clone them - Update fake GitHub server to emit reachable clone_url fields pointing to the git-server container (configurable via GIT_SERVER_URL env var) - Add management endpoint POST /___mgmt/set-clone-url for runtime config - Update E2E spec with real mirroring verification: * Verify repos appear in Gitea with actual content * Check branches, tags, commits, file content * Verify 4/4 repos mirrored successfully - Add backup configuration test suite: * Enable/disable backupBeforeSync config * Toggle blockSyncOnBackupFailure * Trigger re-sync with backup enabled and verify activities * Verify config persistence across changes - Update CI workflow to use docker compose (not service containers) matching the local run-e2e.sh approach - Update cleanup.sh for git-repos directory and git-server port - All 22 tests passing with real git content verification * refactor: split E2E tests into focused files + add force-push tests Split the monolithic e2e.spec.ts (1335 lines) into 5 focused spec files and a shared helpers module: helpers.ts — constants, GiteaAPI, auth, saveConfig, utilities 01-health.spec.ts — service health checks (4 tests) 02-mirror-workflow.spec.ts — full first-mirror journey (8 tests) 03-backup.spec.ts — backup config toggling (6 tests) 04-force-push.spec.ts — force-push simulation & backup verification (9 tests) 05-sync-verification.spec.ts — dynamic repos, content integrity, reset (5 tests) The force-push tests are the critical addition: F0: Record original state (commit SHAs, file content) F1: Rewrite source repo history (simulate force-push) F2: Sync to Gitea WITHOUT backup F3: Verify data loss — LICENSE file gone, README overwritten F4: Restore source, re-mirror to clean state F5: Enable backup, force-push again, sync through app F6: Verify Gitea reflects the force-push F7: Verify backup system was invoked (snapshot activities logged) F8: Restore source repo for subsequent tests Also added to helpers.ts: - GiteaAPI.getBranch(), .getCommit(), .triggerMirrorSync() - getRepositoryIds(), triggerMirrorJobs(), triggerSyncRepo() All 32 tests passing. * Try to fix actions * Try to fix the other action * Add debug info to check why e2e action is failing * More debug info * Even more debug info * E2E fix attempt #1 * E2E fix attempt #2 * more debug again * E2E fix attempt #3 * E2E fix attempt #4 * Remove a bunch of debug info * Hopefully fix backup bug * Force backups to succeed
171 lines
5.4 KiB
TypeScript
171 lines
5.4 KiB
TypeScript
import { mkdir, mkdtemp, readdir, rm, stat } from "node:fs/promises";
|
|
import os from "node:os";
|
|
import path from "node:path";
|
|
import type { Config } from "@/types/config";
|
|
import { decryptConfigTokens } from "./utils/config-encryption";
|
|
|
|
const TRUE_VALUES = new Set(["1", "true", "yes", "on"]);
|
|
|
|
function parseBoolean(value: string | undefined, fallback: boolean): boolean {
|
|
if (value === undefined) return fallback;
|
|
return TRUE_VALUES.has(value.trim().toLowerCase());
|
|
}
|
|
|
|
function parsePositiveInt(value: string | undefined, fallback: number): number {
|
|
if (!value) return fallback;
|
|
const parsed = Number.parseInt(value, 10);
|
|
if (!Number.isFinite(parsed) || parsed <= 0) {
|
|
return fallback;
|
|
}
|
|
return parsed;
|
|
}
|
|
|
|
function sanitizePathSegment(input: string): string {
|
|
return input.replace(/[^a-zA-Z0-9._-]/g, "_");
|
|
}
|
|
|
|
function buildTimestamp(): string {
|
|
// Example: 2026-02-25T18-34-22-123Z
|
|
return new Date().toISOString().replace(/[:.]/g, "-");
|
|
}
|
|
|
|
function buildAuthenticatedCloneUrl(cloneUrl: string, token: string): string {
|
|
const parsed = new URL(cloneUrl);
|
|
if (parsed.protocol !== "http:" && parsed.protocol !== "https:") {
|
|
return cloneUrl;
|
|
}
|
|
|
|
parsed.username = process.env.PRE_SYNC_BACKUP_GIT_USERNAME || "oauth2";
|
|
parsed.password = token;
|
|
return parsed.toString();
|
|
}
|
|
|
|
function maskToken(text: string, token: string): string {
|
|
if (!token) return text;
|
|
return text.split(token).join("***");
|
|
}
|
|
|
|
async function runGit(args: string[], tokenToMask: string): Promise<void> {
|
|
const proc = Bun.spawn({
|
|
cmd: ["git", ...args],
|
|
stdout: "pipe",
|
|
stderr: "pipe",
|
|
});
|
|
|
|
const [stdout, stderr, exitCode] = await Promise.all([
|
|
new Response(proc.stdout).text(),
|
|
new Response(proc.stderr).text(),
|
|
proc.exited,
|
|
]);
|
|
|
|
if (exitCode !== 0) {
|
|
const details = [stdout, stderr].filter(Boolean).join("\n").trim();
|
|
const safeDetails = maskToken(details, tokenToMask);
|
|
throw new Error(`git command failed: ${safeDetails || "unknown git error"}`);
|
|
}
|
|
}
|
|
|
|
async function enforceRetention(repoBackupDir: string, keepCount: number): Promise<void> {
|
|
const entries = await readdir(repoBackupDir);
|
|
const bundleFiles = entries
|
|
.filter((name) => name.endsWith(".bundle"))
|
|
.map((name) => path.join(repoBackupDir, name));
|
|
|
|
if (bundleFiles.length <= keepCount) return;
|
|
|
|
const filesWithMtime = await Promise.all(
|
|
bundleFiles.map(async (filePath) => ({
|
|
filePath,
|
|
mtimeMs: (await stat(filePath)).mtimeMs,
|
|
}))
|
|
);
|
|
|
|
filesWithMtime.sort((a, b) => b.mtimeMs - a.mtimeMs);
|
|
const toDelete = filesWithMtime.slice(keepCount);
|
|
|
|
await Promise.all(toDelete.map((entry) => rm(entry.filePath, { force: true })));
|
|
}
|
|
|
|
export function isPreSyncBackupEnabled(): boolean {
|
|
return parseBoolean(process.env.PRE_SYNC_BACKUP_ENABLED, true);
|
|
}
|
|
|
|
export function shouldCreatePreSyncBackup(config: Partial<Config>): boolean {
|
|
const configSetting = config.giteaConfig?.backupBeforeSync;
|
|
const fallback = isPreSyncBackupEnabled();
|
|
return configSetting === undefined ? fallback : Boolean(configSetting);
|
|
}
|
|
|
|
export function shouldBlockSyncOnBackupFailure(config: Partial<Config>): boolean {
|
|
const configSetting = config.giteaConfig?.blockSyncOnBackupFailure;
|
|
return configSetting === undefined ? true : Boolean(configSetting);
|
|
}
|
|
|
|
export async function createPreSyncBundleBackup({
|
|
config,
|
|
owner,
|
|
repoName,
|
|
cloneUrl,
|
|
}: {
|
|
config: Partial<Config>;
|
|
owner: string;
|
|
repoName: string;
|
|
cloneUrl: string;
|
|
}): Promise<{ bundlePath: string }> {
|
|
if (!shouldCreatePreSyncBackup(config)) {
|
|
throw new Error("Pre-sync backup is disabled.");
|
|
}
|
|
|
|
if (!config.giteaConfig?.token) {
|
|
throw new Error("Gitea token is required for pre-sync backup.");
|
|
}
|
|
|
|
const decryptedConfig = decryptConfigTokens(config as Config);
|
|
const giteaToken = decryptedConfig.giteaConfig?.token;
|
|
if (!giteaToken) {
|
|
throw new Error("Decrypted Gitea token is required for pre-sync backup.");
|
|
}
|
|
|
|
let backupRoot =
|
|
config.giteaConfig?.backupDirectory?.trim() ||
|
|
process.env.PRE_SYNC_BACKUP_DIR?.trim() ||
|
|
path.join(process.cwd(), "data", "repo-backups");
|
|
|
|
// Ensure backupRoot is absolute - relative paths break git bundle creation
|
|
// because git runs with -C mirrorClonePath and interprets relative paths from there
|
|
if (!path.isAbsolute(backupRoot)) {
|
|
backupRoot = path.resolve(process.cwd(), backupRoot);
|
|
}
|
|
const retention = Math.max(
|
|
1,
|
|
Number.isFinite(config.giteaConfig?.backupRetentionCount)
|
|
? Number(config.giteaConfig?.backupRetentionCount)
|
|
: parsePositiveInt(process.env.PRE_SYNC_BACKUP_KEEP_COUNT, 20)
|
|
);
|
|
|
|
const repoBackupDir = path.join(
|
|
backupRoot,
|
|
sanitizePathSegment(config.userId || "unknown-user"),
|
|
sanitizePathSegment(owner),
|
|
sanitizePathSegment(repoName)
|
|
);
|
|
|
|
await mkdir(repoBackupDir, { recursive: true });
|
|
|
|
const tmpDir = await mkdtemp(path.join(os.tmpdir(), "gitea-mirror-backup-"));
|
|
const mirrorClonePath = path.join(tmpDir, "repo.git");
|
|
const bundlePath = path.join(repoBackupDir, `${buildTimestamp()}.bundle`);
|
|
|
|
try {
|
|
const authCloneUrl = buildAuthenticatedCloneUrl(cloneUrl, giteaToken);
|
|
|
|
await runGit(["clone", "--mirror", authCloneUrl, mirrorClonePath], giteaToken);
|
|
await runGit(["-C", mirrorClonePath, "bundle", "create", bundlePath, "--all"], giteaToken);
|
|
|
|
await enforceRetention(repoBackupDir, retention);
|
|
return { bundlePath };
|
|
} finally {
|
|
await rm(tmpDir, { recursive: true, force: true });
|
|
}
|
|
}
|