mirror of
https://github.com/RayLabsHQ/gitea-mirror.git
synced 2026-10-30 22:21:58 +01:00
feat(orgs): per-organization skip forks mirror option (#405)
Adds a Skip forks tri-state to the organization Mirror options dialog. An organization can opt out of importing and mirroring forks without touching the global switch, and the pin is honored by the bulk organization mirror, the add-organization route, bulk import and the scheduler.
This commit is contained in:
1 parent
67ca35d893
commit
34babd4d8c
17 files changed
+474
-32
No files matched your search
@@ -26,6 +26,7 @@ import {
|
||||
UI_MIRROR_OVERRIDE_KEYS,
|
||||
type InheritedMirrorOptions,
|
||||
type MirrorOverrideKey,
|
||||
type OrgSelectionOverrideKey,
|
||||
} from "@/lib/utils/mirror-overrides";
|
||||
|
||||
/**
|
||||
@@ -34,6 +35,8 @@ import {
|
||||
*/
|
||||
type TriState = "inherit" | "on" | "off";
|
||||
|
||||
type DialogOverrideKey = MirrorOverrideKey | OrgSelectionOverrideKey;
|
||||
|
||||
function toTriState(value: boolean | null | undefined): TriState {
|
||||
if (value === true) return "on";
|
||||
if (value === false) return "off";
|
||||
@@ -106,8 +109,18 @@ export function MirrorOverridesDialog({
|
||||
destinationProvider,
|
||||
onSave,
|
||||
}: MirrorOverridesDialogProps) {
|
||||
const [draft, setDraft] = useState<Record<MirrorOverrideKey, TriState>>(
|
||||
() => buildDraft(value)
|
||||
// Skip forks steers which repositories an organization imports and mirrors
|
||||
// at all, so it leads the list; it is meaningless for a single repository.
|
||||
const dialogKeys = useMemo<DialogOverrideKey[]>(
|
||||
() =>
|
||||
targetKind === "organization"
|
||||
? ["skipForks", ...UI_MIRROR_OVERRIDE_KEYS]
|
||||
: [...UI_MIRROR_OVERRIDE_KEYS],
|
||||
[targetKind]
|
||||
);
|
||||
|
||||
const [draft, setDraft] = useState<Record<DialogOverrideKey, TriState>>(
|
||||
() => buildDraft(value, dialogKeys)
|
||||
);
|
||||
const [releaseLimitDraft, setReleaseLimitDraft] = useState(() =>
|
||||
buildReleaseLimitDraft(value)
|
||||
@@ -121,11 +134,11 @@ export function MirrorOverridesDialog({
|
||||
// previous edit never leaks into the next one.
|
||||
useEffect(() => {
|
||||
if (open) {
|
||||
setDraft(buildDraft(value));
|
||||
setDraft(buildDraft(value, dialogKeys));
|
||||
setReleaseLimitDraft(buildReleaseLimitDraft(value));
|
||||
setReleaseAssetLimitDraft(buildReleaseAssetLimitDraft(value));
|
||||
}
|
||||
}, [open, value]);
|
||||
}, [open, value, dialogKeys]);
|
||||
|
||||
// What each flag currently resolves to, including the in-progress edit.
|
||||
// Drives the labels and release-limit gates so they react live as issues
|
||||
@@ -136,6 +149,8 @@ export function MirrorOverridesDialog({
|
||||
const pinned = fromTriState(draft[key]);
|
||||
next[key] = pinned ?? inheritedFrom?.[key] ?? false;
|
||||
}
|
||||
next.skipForks =
|
||||
fromTriState(draft.skipForks) ?? inheritedFrom?.skipForks ?? false;
|
||||
return next;
|
||||
}, [draft, inheritedFrom]);
|
||||
|
||||
@@ -178,6 +193,10 @@ export function MirrorOverridesDialog({
|
||||
const resolved = fromTriState(draft[key]);
|
||||
if (typeof resolved === "boolean") next[key] = resolved;
|
||||
}
|
||||
if (targetKind === "organization") {
|
||||
const resolved = fromTriState(draft.skipForks);
|
||||
if (typeof resolved === "boolean") next.skipForks = resolved;
|
||||
}
|
||||
if (pinnedReleaseLimit !== undefined) next.releaseLimit = pinnedReleaseLimit;
|
||||
if (pinnedReleaseAssetLimit !== undefined) {
|
||||
next.releaseAssetLimit = pinnedReleaseAssetLimit;
|
||||
@@ -190,7 +209,7 @@ export function MirrorOverridesDialog({
|
||||
};
|
||||
|
||||
const handleResetAll = () => {
|
||||
setDraft(buildDraft(null));
|
||||
setDraft(buildDraft(null, dialogKeys));
|
||||
setReleaseLimitDraft("");
|
||||
setReleaseAssetLimitDraft("");
|
||||
};
|
||||
@@ -229,10 +248,11 @@ export function MirrorOverridesDialog({
|
||||
</DialogHeader>
|
||||
|
||||
<div className="space-y-3 py-2">
|
||||
{UI_MIRROR_OVERRIDE_KEYS.map((key) => {
|
||||
{dialogKeys.map((key) => {
|
||||
const disabledReason = gating[key];
|
||||
const isDisabled = !!disabledReason;
|
||||
const inherited = inheritedFrom?.[key];
|
||||
const inherited =
|
||||
key === "skipForks" ? inheritedFrom?.skipForks : inheritedFrom?.[key];
|
||||
const showHint =
|
||||
!inheritedLoading &&
|
||||
!isDisabled &&
|
||||
@@ -424,10 +444,11 @@ export function MirrorOverridesDialog({
|
||||
}
|
||||
|
||||
function buildDraft(
|
||||
value: MirrorOverrides | null | undefined
|
||||
): Record<MirrorOverrideKey, TriState> {
|
||||
const draft = {} as Record<MirrorOverrideKey, TriState>;
|
||||
for (const key of UI_MIRROR_OVERRIDE_KEYS) {
|
||||
value: MirrorOverrides | null | undefined,
|
||||
keys: readonly DialogOverrideKey[]
|
||||
): Record<DialogOverrideKey, TriState> {
|
||||
const draft = {} as Record<DialogOverrideKey, TriState>;
|
||||
for (const key of keys) {
|
||||
draft[key] = toTriState(value?.[key]);
|
||||
}
|
||||
return draft;
|
||||
|
||||
@@ -69,7 +69,7 @@ export function OrganizationList({
|
||||
onRefresh,
|
||||
onDelete,
|
||||
}: OrganizationListProps) {
|
||||
const { giteaConfig, mirrorOptions } = useGiteaConfig();
|
||||
const { giteaConfig, mirrorOptions, advancedOptions } = useGiteaConfig();
|
||||
const [overridesTarget, setOverridesTarget] = useState<Organization | null>(null);
|
||||
|
||||
const handleUpdateMirrorOverrides = async (
|
||||
@@ -786,7 +786,10 @@ export function OrganizationList({
|
||||
targetName={overridesTarget?.name ?? ""}
|
||||
value={overridesTarget?.mirrorOverrides ?? null}
|
||||
destinationProvider={giteaConfig?.provider}
|
||||
inheritedFrom={mirrorOptionsToFlags(mirrorOptions)}
|
||||
inheritedFrom={{
|
||||
...mirrorOptionsToFlags(mirrorOptions),
|
||||
skipForks: !!advancedOptions?.skipForks,
|
||||
}}
|
||||
inheritedLabel="global settings"
|
||||
onSave={async (overrides) => {
|
||||
if (overridesTarget?.id) {
|
||||
|
||||
@@ -72,6 +72,13 @@ export const backupStrategyEnum = z.enum([
|
||||
* `releaseAssetLimit` follows the same rule (`normalizeReleaseAssetLimit`):
|
||||
* assets are uploaded only for the newest N mirrored releases. 0 is a real
|
||||
* value (notes only); null/absent means inherit.
|
||||
*
|
||||
* `skipForks` is organization-tier only: it opts an organization out of
|
||||
* importing and mirroring forked repositories without touching the global
|
||||
* `githubConfig.skipForks` switch. Null/absent inherits that global value.
|
||||
* It is deliberately NOT part of `MIRROR_OVERRIDE_KEYS` in
|
||||
* `mirror-overrides.ts` (those mirror `giteaConfigSchema` flags); see
|
||||
* `ORG_SELECTION_OVERRIDE_KEYS` there.
|
||||
*/
|
||||
export const mirrorOverridesSchema = z.object({
|
||||
lfs: z.boolean().nullable().optional(),
|
||||
@@ -84,6 +91,7 @@ export const mirrorOverridesSchema = z.object({
|
||||
mirrorMilestones: z.boolean().nullable().optional(),
|
||||
releaseLimit: z.number().nullable().optional(),
|
||||
releaseAssetLimit: z.number().nullable().optional(),
|
||||
skipForks: z.boolean().nullable().optional(),
|
||||
});
|
||||
|
||||
export type MirrorOverrides = z.infer<typeof mirrorOverridesSchema>;
|
||||
|
||||
@@ -373,3 +373,75 @@ describe.skipIf(!isChild)("mirrorGitHubOrgToGitea destination routing (#343)", (
|
||||
expect(orgCreateCalls.length).toBe(0);
|
||||
});
|
||||
});
|
||||
|
||||
describe.skipIf(!isChild)("mirrorGitHubOrgToGitea fork policy", () => {
|
||||
const forkRepo = makeRepo({ id: "repo-fork", name: "forked", fullName: "A/forked", isForked: true });
|
||||
const normalRepo = makeRepo({ id: "repo-2", name: "r2", fullName: "A/r2", cloneUrl: "https://github.com/A/r2.git" });
|
||||
|
||||
test("a skip-forks override drops forked repositories from the batch", async () => {
|
||||
const config = makeConfig({ githubConfig: { mirrorStrategy: "preserve" } });
|
||||
const organization = makeOrg({ mirrorOverrides: { skipForks: true } });
|
||||
orgRepoRows = [makeRepo(), forkRepo];
|
||||
orgConfigRows = [organization];
|
||||
|
||||
await mirrorGitHubOrgToGitea({ organization, octokit: fakeOctokit, config });
|
||||
|
||||
const migrates = migrateCalls();
|
||||
expect(migrates.length).toBe(1);
|
||||
expect(migrates[0].payload.repo_name).toBe("r1");
|
||||
});
|
||||
|
||||
test("the global skipForks switch applies to the org batch too", async () => {
|
||||
const config = makeConfig({
|
||||
githubConfig: { mirrorStrategy: "preserve", skipForks: true },
|
||||
});
|
||||
const organization = makeOrg();
|
||||
orgRepoRows = [makeRepo(), forkRepo];
|
||||
orgConfigRows = [organization];
|
||||
|
||||
await mirrorGitHubOrgToGitea({ organization, octokit: fakeOctokit, config });
|
||||
|
||||
const migrates = migrateCalls();
|
||||
expect(migrates.length).toBe(1);
|
||||
expect(migrates[0].payload.repo_name).toBe("r1");
|
||||
});
|
||||
|
||||
test("a false org pin keeps forks despite the global switch", async () => {
|
||||
const config = makeConfig({
|
||||
githubConfig: { mirrorStrategy: "preserve", skipForks: true },
|
||||
});
|
||||
const organization = makeOrg({ mirrorOverrides: { skipForks: false } });
|
||||
orgRepoRows = [makeRepo(), forkRepo];
|
||||
orgConfigRows = [organization];
|
||||
|
||||
await mirrorGitHubOrgToGitea({ organization, octokit: fakeOctokit, config });
|
||||
|
||||
const migrates = migrateCalls();
|
||||
expect(migrates.length).toBe(2);
|
||||
});
|
||||
|
||||
test("forks are mirrored when nothing opts out (regression)", async () => {
|
||||
const config = makeConfig({ githubConfig: { mirrorStrategy: "preserve" } });
|
||||
const organization = makeOrg();
|
||||
orgRepoRows = [normalRepo, forkRepo];
|
||||
orgConfigRows = [organization];
|
||||
|
||||
await mirrorGitHubOrgToGitea({ organization, octokit: fakeOctokit, config });
|
||||
|
||||
const migrates = migrateCalls();
|
||||
expect(migrates.length).toBe(2);
|
||||
});
|
||||
|
||||
test("a fork-only organization with skip forks mirrors nothing and still succeeds", async () => {
|
||||
const config = makeConfig({
|
||||
githubConfig: { mirrorStrategy: "preserve", skipForks: true },
|
||||
});
|
||||
const organization = makeOrg();
|
||||
orgRepoRows = [forkRepo];
|
||||
orgConfigRows = [organization];
|
||||
|
||||
await mirrorGitHubOrgToGitea({ organization, octokit: fakeOctokit, config });
|
||||
|
||||
expect(migrateCalls().length).toBe(0);
|
||||
});
|
||||
});
|
||||
+20
-3
@@ -28,6 +28,7 @@ import {
|
||||
normalizeReleaseLimit,
|
||||
releaseGetsAssets,
|
||||
resolveMirrorOptionsForRepository,
|
||||
resolveOrganizationSkipForks,
|
||||
type ResolvedMirrorOptions,
|
||||
} from "./utils/mirror-overrides";
|
||||
import { repositoryDestinationColumns } from "./repo-utils";
|
||||
@@ -2160,13 +2161,29 @@ export async function mirrorGitHubOrgToGitea({
|
||||
.from(repositories)
|
||||
.where(eq(repositories.organization, organization.name));
|
||||
|
||||
if (orgRepos.length === 0) {
|
||||
// The organization's fork policy (override -> global skipForks) drops
|
||||
// forked repositories from the batch so they are neither mirrored here
|
||||
// nor re-mirrored by later runs until the policy changes.
|
||||
const skipOrgForks = resolveOrganizationSkipForks({
|
||||
orgOverrides: organization.mirrorOverrides,
|
||||
config,
|
||||
});
|
||||
const mirrorableOrgRepos = skipOrgForks
|
||||
? orgRepos.filter((repo) => !repo.isForked)
|
||||
: orgRepos;
|
||||
if (skipOrgForks && orgRepos.length !== mirrorableOrgRepos.length) {
|
||||
console.log(
|
||||
`Skipping ${orgRepos.length - mirrorableOrgRepos.length} forked repositories for organization ${organization.name} (skip forks is on)`
|
||||
);
|
||||
}
|
||||
|
||||
if (mirrorableOrgRepos.length === 0) {
|
||||
console.log(
|
||||
`No repositories found for organization ${organization.name} - marking as successfully mirrored`
|
||||
);
|
||||
} else {
|
||||
console.log(
|
||||
`Mirroring ${orgRepos.length} repositories for organization ${organization.name}`
|
||||
`Mirroring ${mirrorableOrgRepos.length} repositories for organization ${organization.name}`
|
||||
);
|
||||
|
||||
// Import the processWithRetry function
|
||||
@@ -2174,7 +2191,7 @@ export async function mirrorGitHubOrgToGitea({
|
||||
|
||||
// Process repositories in parallel with concurrency control
|
||||
await processWithRetry(
|
||||
orgRepos,
|
||||
mirrorableOrgRepos,
|
||||
async (repo) => {
|
||||
// Prepare repository data
|
||||
const repoData = {
|
||||
|
||||
@@ -253,3 +253,47 @@ describe("getGithubRepositories - includeOrganizations allowlist", () => {
|
||||
expect(repos.map((r) => r.name)).toContain("noise");
|
||||
});
|
||||
});
|
||||
|
||||
describe("getGithubRepositories - per-organization fork pins", () => {
|
||||
const acmeFork = makeRepo({ name: "acme-fork", ownerLogin: "acme", ownerType: "Organization", fork: true });
|
||||
const globexFork = makeRepo({ name: "globex-fork", ownerLogin: "globex", ownerType: "Organization", fork: true });
|
||||
const ownFork = makeRepo({ name: "own-fork", ownerLogin: "octo", ownerType: "User", fork: true });
|
||||
const plain = makeRepo({ name: "plain" });
|
||||
|
||||
async function listed(
|
||||
config: Record<string, unknown>,
|
||||
orgForkOverrides?: Map<string, boolean>
|
||||
) {
|
||||
const { octokit } = makeOctokit([acmeFork, globexFork, ownFork, plain]);
|
||||
const repos = await getGithubRepositories({
|
||||
octokit,
|
||||
config: { githubConfig: { owner: "octo", ...config } as any },
|
||||
orgForkOverrides,
|
||||
});
|
||||
return repos.map((r) => r.name).sort();
|
||||
}
|
||||
|
||||
test("a pinned org drops only its own forks", async () => {
|
||||
expect(await listed({}, new Map([["acme", true]]))).toEqual([
|
||||
"globex-fork",
|
||||
"own-fork",
|
||||
"plain",
|
||||
]);
|
||||
});
|
||||
|
||||
test("a false pin keeps that org's forks when the global switch skips forks", async () => {
|
||||
expect(await listed({ skipForks: true }, new Map([["globex", false]]))).toEqual([
|
||||
"globex-fork",
|
||||
"plain",
|
||||
]);
|
||||
});
|
||||
|
||||
test("pins never apply to personal forks", async () => {
|
||||
expect(await listed({}, new Map([["octo", true]]))).toEqual([
|
||||
"acme-fork",
|
||||
"globex-fork",
|
||||
"own-fork",
|
||||
"plain",
|
||||
]);
|
||||
});
|
||||
});
|
||||
+9
-1
@@ -3,6 +3,7 @@ import type { GitRepo, RepoStatus } from "@/types/Repository";
|
||||
import { Octokit } from "@octokit/rest";
|
||||
import { throttling } from "@octokit/plugin-throttling";
|
||||
import type { Config } from "@/types/config";
|
||||
import { orgForkSkipDecision } from "@/lib/utils/mirror-overrides";
|
||||
import {
|
||||
applyConditionalRequests,
|
||||
conditionalRequestTokenScope,
|
||||
@@ -249,6 +250,7 @@ export async function getGithubRepositories({
|
||||
config,
|
||||
includeCollaboratorReposOverride,
|
||||
includeAllOrgsOverride,
|
||||
orgForkOverrides,
|
||||
}: {
|
||||
octokit: Octokit;
|
||||
config: Partial<Config>;
|
||||
@@ -260,6 +262,9 @@ export async function getGithubRepositories({
|
||||
// Used by the cleanup service so a previously-mirrored org repo isn't flagged
|
||||
// as orphaned just because the user narrowed the allowlist.
|
||||
includeAllOrgsOverride?: boolean;
|
||||
// Per-organization fork pins (normalized org name -> skip forks). A fork
|
||||
// owned by a pinned org follows the pin instead of the global skipForks.
|
||||
orgForkOverrides?: Map<string, boolean>;
|
||||
}): Promise<GitRepo[]> {
|
||||
try {
|
||||
const includeCollab =
|
||||
@@ -279,7 +284,7 @@ export async function getGithubRepositories({
|
||||
{ per_page: 100, affiliation },
|
||||
);
|
||||
|
||||
const skipForks = config.githubConfig?.skipForks ?? false;
|
||||
const forkSkipFor = orgForkSkipDecision(orgForkOverrides, config);
|
||||
const skipPersonalRepos = config.githubConfig?.skipPersonalRepos ?? false;
|
||||
// The authenticated user's login — used to identify personally-owned repos
|
||||
const authenticatedUserLogin = config.githubConfig?.owner ?? "";
|
||||
@@ -295,6 +300,9 @@ export async function getGithubRepositories({
|
||||
);
|
||||
|
||||
const filteredRepos = repos.filter((repo) => {
|
||||
const skipForks = forkSkipFor(
|
||||
repo.owner.type === "Organization" ? repo.owner.login : undefined
|
||||
);
|
||||
const isForkAllowed = !skipForks || !repo.fork;
|
||||
// When skipPersonalRepos is true, drop repos owned by the authenticated user
|
||||
// (owner.type === "User" and owner.login matches the configured GitHub username).
|
||||
|
||||
@@ -15,6 +15,7 @@ import type { Octokit } from '@octokit/rest';
|
||||
import { repoStatusEnum, repositoryVisibilityEnum } from '@/types/Repository';
|
||||
import { mergeGitReposPreferStarred, normalizeGitRepoToInsert, calcBatchSizeForInsert } from '@/lib/repo-utils';
|
||||
import { isMirrorableGitHubRepo } from '@/lib/repo-eligibility';
|
||||
import { loadOrganizationForkPolicies, orgForkSkipDecision } from '@/lib/utils/mirror-overrides';
|
||||
import { createMirrorJob } from '@/lib/helpers';
|
||||
import { getNextScheduledRun, isCronExpression, normalizeTimezone } from '@/lib/utils/schedule-utils';
|
||||
import { resetStuckMirrorStatuses } from '@/lib/stuck-status-recovery';
|
||||
@@ -108,9 +109,13 @@ async function runScheduledSync(config: any): Promise<void> {
|
||||
// The configured source host (GitHub honors GH_API_URL for GHES / GHEC)
|
||||
const sourceProvider = createSourceProviderFromConfig(config, { userId });
|
||||
|
||||
// Per-organization fork pins, so orgs opted out of forks stay that
|
||||
// way during scheduled auto-import.
|
||||
const orgForkOverrides = await loadOrganizationForkPolicies({ userId });
|
||||
|
||||
// Fetch source data
|
||||
const [basicAndForkedRepos, starredRepos] = await Promise.all([
|
||||
sourceProvider.listRepositories(config),
|
||||
sourceProvider.listRepositories(config, { orgForkOverrides }),
|
||||
config.githubConfig?.includeStarred
|
||||
? sourceProvider.listStarredRepositories(config)
|
||||
: Promise.resolve([]),
|
||||
@@ -123,7 +128,7 @@ async function runScheduledSync(config: any): Promise<void> {
|
||||
.select({ normalizedFullName: repositories.normalizedFullName })
|
||||
.from(repositories)
|
||||
.where(eq(repositories.userId, userId));
|
||||
|
||||
|
||||
const existingRepoNames = new Set(existingRepos.map(r => r.normalizedFullName));
|
||||
const newRepos = mirrorableGithubRepos.filter(r => !existingRepoNames.has(r.fullName.toLowerCase()));
|
||||
|
||||
@@ -238,6 +243,18 @@ async function runScheduledSync(config: any): Promise<void> {
|
||||
console.log(`[Scheduler] Skipped ${skippedCount} repositories from auto-mirror (autoMirror=${autoMirrorOwned}, autoMirrorStarred=${autoMirrorStarred})`);
|
||||
}
|
||||
|
||||
// Organization fork policies apply to already-imported rows too, so a
|
||||
// fork that slipped in before its org opted out is not auto-mirrored.
|
||||
const orgForkOverrides = await loadOrganizationForkPolicies({ userId });
|
||||
const forkSkipFor = orgForkSkipDecision(orgForkOverrides, config);
|
||||
const forkSkippedCount = reposNeedingMirror.length;
|
||||
reposNeedingMirror = reposNeedingMirror.filter(
|
||||
repo => !repo.isForked || !repo.organization || !forkSkipFor(repo.organization)
|
||||
);
|
||||
if (reposNeedingMirror.length !== forkSkippedCount) {
|
||||
console.log(`[Scheduler] Skipped ${forkSkippedCount - reposNeedingMirror.length} forked repositories from auto-mirror (organization skip forks)`);
|
||||
}
|
||||
|
||||
if (reposNeedingMirror.length > 0) {
|
||||
console.log(`[Scheduler] Found ${reposNeedingMirror.length} repositories that need initial mirroring`);
|
||||
|
||||
@@ -521,23 +538,27 @@ async function performInitialAutoStart(): Promise<void> {
|
||||
|
||||
// The configured source host (GitHub honors GH_API_URL for GHES / GHEC)
|
||||
const sourceProvider = createSourceProviderFromConfig(config, { userId: config.userId });
|
||||
|
||||
|
||||
// Per-organization fork pins, so orgs opted out of forks stay that
|
||||
// way during boot auto-start imports.
|
||||
const orgForkOverrides = await loadOrganizationForkPolicies({ userId: config.userId });
|
||||
|
||||
// Fetch source data
|
||||
const [basicAndForkedRepos, starredRepos] = await Promise.all([
|
||||
sourceProvider.listRepositories(config),
|
||||
sourceProvider.listRepositories(config, { orgForkOverrides }),
|
||||
config.githubConfig?.includeStarred
|
||||
? sourceProvider.listStarredRepositories(config)
|
||||
: Promise.resolve([]),
|
||||
]);
|
||||
const allGithubRepos = mergeGitReposPreferStarred(basicAndForkedRepos, starredRepos);
|
||||
const mirrorableGithubRepos = allGithubRepos.filter(isMirrorableGitHubRepo);
|
||||
|
||||
|
||||
// Check for new repositories
|
||||
const existingRepos = await db
|
||||
.select({ normalizedFullName: repositories.normalizedFullName })
|
||||
.from(repositories)
|
||||
.where(eq(repositories.userId, config.userId));
|
||||
|
||||
|
||||
const existingRepoNames = new Set(existingRepos.map(r => r.normalizedFullName));
|
||||
const reposToImport = mirrorableGithubRepos.filter(r => !existingRepoNames.has(r.fullName.toLowerCase()));
|
||||
|
||||
|
||||
@@ -46,7 +46,7 @@ describe("discovery and housekeeping go through the source provider", () => {
|
||||
test("scheduler auto import, auto mirror and boot auto start", () => {
|
||||
const source = read("scheduler-service.ts");
|
||||
expect(count(source, "createSourceProviderFromConfig(config")).toBe(2);
|
||||
expect(count(source, "sourceProvider.listRepositories(config)")).toBe(2);
|
||||
expect(count(source, "sourceProvider.listRepositories(")).toBe(2);
|
||||
expect(source).not.toContain("getGithubRepositories(");
|
||||
// A GitHub client is built only for GitHub sources, in both mirror phases.
|
||||
expect(count(source, "resolveSourceProviderKind(config) === 'github'")).toBe(2);
|
||||
|
||||
@@ -91,3 +91,48 @@ describe("filterSourceRepositories", () => {
|
||||
expect(kept).toHaveLength(5);
|
||||
});
|
||||
});
|
||||
|
||||
describe("filterSourceRepositories - per-organization fork pins", () => {
|
||||
const acmeFork = repo({ owner: "acme", name: "forked-tool", organization: "acme", isForked: true });
|
||||
const globexFork = repo({ owner: "globex", name: "forked-app", organization: "globex", isForked: true });
|
||||
const repos = [own, acme, acmeFork, globexFork];
|
||||
|
||||
test("a pinned org drops only its own forks", () => {
|
||||
const kept = filterSourceRepositories(repos, {
|
||||
config: {},
|
||||
options: { orgForkOverrides: new Map([["acme", true]]) },
|
||||
username: "me",
|
||||
});
|
||||
expect(kept.map((r) => r.name).sort()).toEqual(["forked-app", "mine", "tool"]);
|
||||
});
|
||||
|
||||
test("a false pin keeps that org's forks when the global switch skips forks", () => {
|
||||
const kept = filterSourceRepositories(repos, {
|
||||
config: { githubConfig: { skipForks: true } as any },
|
||||
options: { orgForkOverrides: new Map([["globex", false]]) },
|
||||
username: "me",
|
||||
});
|
||||
expect(kept.map((r) => r.name).sort()).toEqual(["forked-app", "mine", "tool"]);
|
||||
});
|
||||
|
||||
test("pins do not affect non-fork repositories", () => {
|
||||
const kept = filterSourceRepositories([acme, acmeFork], {
|
||||
config: {},
|
||||
options: { orgForkOverrides: new Map([["acme", true]]) },
|
||||
username: "me",
|
||||
});
|
||||
expect(kept.map((r) => r.name)).toEqual(["tool"]);
|
||||
});
|
||||
|
||||
test("a repository org name is matched against map keys case-insensitively", () => {
|
||||
// Map keys come from organizations.normalizedName (lowercase); the repo's
|
||||
// organization can carry any casing from the source host.
|
||||
const casedFork = repo({ owner: "Acme", name: "forked-tool", organization: "Acme", isForked: true });
|
||||
const kept = filterSourceRepositories([casedFork], {
|
||||
config: {},
|
||||
options: { orgForkOverrides: new Map([["acme", true]]) },
|
||||
username: "me",
|
||||
});
|
||||
expect(kept).toHaveLength(0);
|
||||
});
|
||||
});
|
||||
@@ -1,5 +1,6 @@
|
||||
import type { Config } from "@/lib/db/schema";
|
||||
import type { GitRepo } from "@/types/Repository";
|
||||
import { orgForkSkipDecision } from "@/lib/utils/mirror-overrides";
|
||||
import type { ListRepositoriesOptions } from "./types";
|
||||
|
||||
/**
|
||||
@@ -11,6 +12,8 @@ import type { ListRepositoriesOptions } from "./types";
|
||||
* - skipPersonalRepos drops repos the configured account owns
|
||||
* - includeCollaboratorRepos=false drops personal repos of other users
|
||||
* - includeOrganizations, when non-empty, keeps only org repos from listed orgs
|
||||
* - options.orgForkOverrides pins skipForks per organization: a fork owned by
|
||||
* a pinned org follows the pin instead of the global skipForks
|
||||
*/
|
||||
export function filterSourceRepositories(
|
||||
repos: GitRepo[],
|
||||
@@ -28,7 +31,7 @@ export function filterSourceRepositories(
|
||||
options.includeCollaboratorReposOverride ??
|
||||
config.githubConfig?.includeCollaboratorRepos ??
|
||||
true;
|
||||
const skipForks = config.githubConfig?.skipForks ?? false;
|
||||
const forkSkipFor = orgForkSkipDecision(options.orgForkOverrides, config);
|
||||
const skipPersonalRepos = config.githubConfig?.skipPersonalRepos ?? false;
|
||||
const includeOrgs = options.includeAllOrgsOverride
|
||||
? []
|
||||
@@ -39,13 +42,13 @@ export function filterSourceRepositories(
|
||||
const me = username.trim().toLowerCase();
|
||||
|
||||
return repos.filter((repo) => {
|
||||
if (skipForks && repo.isForked) return false;
|
||||
|
||||
const ownerKey = repo.owner.trim().toLowerCase();
|
||||
const orgKey = (repo.organization ?? "").trim().toLowerCase();
|
||||
const isOrgRepo = orgKey.length > 0;
|
||||
const isOwnRepo = !isOrgRepo && me.length > 0 && ownerKey === me;
|
||||
|
||||
if (forkSkipFor(repo.organization) && repo.isForked) return false;
|
||||
|
||||
if (skipPersonalRepos && isOwnRepo) return false;
|
||||
if (!includeCollab && !isOrgRepo && !isOwnRepo) return false;
|
||||
if (allowedOrgs.size > 0 && isOrgRepo && !allowedOrgs.has(orgKey)) return false;
|
||||
|
||||
@@ -138,6 +138,7 @@ export class GitHubSourceProvider implements SourceProvider {
|
||||
config,
|
||||
includeCollaboratorReposOverride: options.includeCollaboratorReposOverride,
|
||||
includeAllOrgsOverride: options.includeAllOrgsOverride,
|
||||
orgForkOverrides: options.orgForkOverrides,
|
||||
});
|
||||
return this.stamp(repos);
|
||||
}
|
||||
|
||||
@@ -21,6 +21,12 @@ export interface ListRepositoriesOptions {
|
||||
includeCollaboratorReposOverride?: boolean;
|
||||
/** Ignore the organization allowlist and return every org repository. */
|
||||
includeAllOrgsOverride?: boolean;
|
||||
/**
|
||||
* Per-organization fork pins (normalized org name -> skip forks), from
|
||||
* `loadOrganizationForkPolicies`. An org-repository fork follows its org's
|
||||
* pin; repositories of unpinned orgs follow the global `skipForks`.
|
||||
*/
|
||||
orgForkOverrides?: Map<string, boolean>;
|
||||
}
|
||||
|
||||
export interface SourceFailedOrganization {
|
||||
|
||||
@@ -12,9 +12,11 @@ import {
|
||||
normalizeMirrorOverrides,
|
||||
normalizeReleaseAssetLimit,
|
||||
normalizeReleaseLimit,
|
||||
orgForkSkipDecision,
|
||||
parseMirrorOverrides,
|
||||
releaseGetsAssets,
|
||||
resolveMirrorOptions,
|
||||
resolveOrganizationSkipForks,
|
||||
STARRED_CLAMPED_KEYS,
|
||||
UI_MIRROR_OVERRIDE_KEYS,
|
||||
} from "./mirror-overrides";
|
||||
@@ -1065,3 +1067,77 @@ describe("release asset limit (#311)", () => {
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe("organization skip forks policy", () => {
|
||||
test("resolveOrganizationSkipForks defaults to the global switch", () => {
|
||||
expect(
|
||||
resolveOrganizationSkipForks({ orgOverrides: null, config: makeConfig({}, {}) })
|
||||
).toBe(false);
|
||||
expect(
|
||||
resolveOrganizationSkipForks({
|
||||
orgOverrides: null,
|
||||
config: makeConfig({}, { skipForks: true }),
|
||||
})
|
||||
).toBe(true);
|
||||
});
|
||||
|
||||
test("an organization pin beats the global switch in either direction", () => {
|
||||
expect(
|
||||
resolveOrganizationSkipForks({
|
||||
orgOverrides: { skipForks: true },
|
||||
config: makeConfig({}, { skipForks: false }),
|
||||
})
|
||||
).toBe(true);
|
||||
expect(
|
||||
resolveOrganizationSkipForks({
|
||||
orgOverrides: { skipForks: false },
|
||||
config: makeConfig({}, { skipForks: true }),
|
||||
})
|
||||
).toBe(false);
|
||||
});
|
||||
|
||||
test("a null pin means inherit, not false", () => {
|
||||
expect(
|
||||
resolveOrganizationSkipForks({
|
||||
orgOverrides: { skipForks: null },
|
||||
config: makeConfig({}, { skipForks: true }),
|
||||
})
|
||||
).toBe(true);
|
||||
});
|
||||
|
||||
test("orgForkSkipDecision resolves per organization, case-insensitively", () => {
|
||||
const decide = orgForkSkipDecision(
|
||||
new Map([["acme", true], ["globex", false]]),
|
||||
makeConfig({}, { skipForks: false })
|
||||
);
|
||||
expect(decide("Acme")).toBe(true);
|
||||
expect(decide("GLOBEX")).toBe(false);
|
||||
expect(decide("umbrella")).toBe(false);
|
||||
expect(decide(undefined)).toBe(false);
|
||||
});
|
||||
|
||||
test("orgForkSkipDecision falls back to a global skip when no org is pinned", () => {
|
||||
const decide = orgForkSkipDecision(undefined, makeConfig({}, { skipForks: true }));
|
||||
expect(decide("acme")).toBe(true);
|
||||
expect(decide(undefined)).toBe(true);
|
||||
expect(decide("")).toBe(true);
|
||||
});
|
||||
|
||||
test("skip forks participates in the storage helpers like any other pin", () => {
|
||||
expect(hasMirrorOverrides({ skipForks: true })).toBe(true);
|
||||
expect(listOverriddenKeys({ skipForks: true })).toEqual(["skipForks"]);
|
||||
expect(normalizeMirrorOverrides({ skipForks: true, lfs: null })).toEqual({
|
||||
skipForks: true,
|
||||
});
|
||||
expect(MIRROR_OVERRIDE_LABELS.skipForks).toBe("Skip forks");
|
||||
});
|
||||
|
||||
test("skip forks is not part of the gitea mirror flag set the resolver walks", () => {
|
||||
expect(MIRROR_OVERRIDE_KEYS).not.toContain("skipForks");
|
||||
const resolved = resolveMirrorOptions({
|
||||
config: makeConfig({}),
|
||||
repository: makeRepo({ mirrorOverrides: { skipForks: true } as any }),
|
||||
});
|
||||
expect((resolved as any).skipForks).toBeUndefined();
|
||||
});
|
||||
});
|
||||
@@ -35,6 +35,18 @@ export const MIRROR_OVERRIDE_LIMIT_KEYS = ["releaseLimit", "releaseAssetLimit"]
|
||||
|
||||
export type MirrorOverrideLimitKey = (typeof MIRROR_OVERRIDE_LIMIT_KEYS)[number];
|
||||
|
||||
/**
|
||||
* Organization-tier selection overrides: per-organization switches that steer
|
||||
* which source repositories are imported and mirrored, not what a mirror
|
||||
* contains. They are kept apart from `MIRROR_OVERRIDE_KEYS` because that list
|
||||
* mirrors `giteaConfigSchema` flag names so the resolver can read the global
|
||||
* tier with the same key — the global tier for these lives on
|
||||
* `githubConfig` instead, and they are meaningless at the repository tier.
|
||||
*/
|
||||
export const ORG_SELECTION_OVERRIDE_KEYS = ["skipForks"] as const;
|
||||
|
||||
export type OrgSelectionOverrideKey = (typeof ORG_SELECTION_OVERRIDE_KEYS)[number];
|
||||
|
||||
/** Matches `giteaConfigSchema.releaseLimit`'s default. */
|
||||
export const DEFAULT_RELEASE_LIMIT = 10;
|
||||
|
||||
@@ -56,6 +68,7 @@ export type ResolvedMirrorOptions = Record<MirrorOverrideKey, boolean> & {
|
||||
export type InheritedMirrorOptions = Partial<Record<MirrorOverrideKey, boolean>> & {
|
||||
releaseLimit?: number;
|
||||
releaseAssetLimit?: number | null;
|
||||
skipForks?: boolean;
|
||||
};
|
||||
|
||||
/**
|
||||
@@ -256,7 +269,10 @@ export const MIRROR_GATING_REASONS = {
|
||||
|
||||
/** key -> reason it cannot take effect. Absent key means editable. */
|
||||
export type MirrorOverrideGating = Partial<
|
||||
Record<MirrorOverrideKey | MirrorOverrideLimitKey, string>
|
||||
Record<
|
||||
MirrorOverrideKey | MirrorOverrideLimitKey | OrgSelectionOverrideKey,
|
||||
string
|
||||
>
|
||||
>;
|
||||
|
||||
/**
|
||||
@@ -337,7 +353,7 @@ export function getMirrorOverrideGating({
|
||||
}
|
||||
|
||||
export const MIRROR_OVERRIDE_LABELS: Record<
|
||||
MirrorOverrideKey | MirrorOverrideLimitKey,
|
||||
MirrorOverrideKey | MirrorOverrideLimitKey | OrgSelectionOverrideKey,
|
||||
string
|
||||
> = {
|
||||
lfs: "Git LFS files",
|
||||
@@ -350,6 +366,7 @@ export const MIRROR_OVERRIDE_LABELS: Record<
|
||||
mirrorMilestones: "Milestones",
|
||||
releaseLimit: "Release limit",
|
||||
releaseAssetLimit: "Release asset limit",
|
||||
skipForks: "Skip forks",
|
||||
};
|
||||
|
||||
/**
|
||||
@@ -392,7 +409,7 @@ export function hasMirrorOverrides(value: unknown): boolean {
|
||||
/** The keys an overrides object pins, for badges and summaries. */
|
||||
export function listOverriddenKeys(
|
||||
value: unknown
|
||||
): (MirrorOverrideKey | MirrorOverrideLimitKey)[] {
|
||||
): (MirrorOverrideKey | MirrorOverrideLimitKey | OrgSelectionOverrideKey)[] {
|
||||
const overrides = parseMirrorOverrides(value);
|
||||
if (!overrides) return [];
|
||||
return [
|
||||
@@ -400,6 +417,7 @@ export function listOverriddenKeys(
|
||||
...MIRROR_OVERRIDE_LIMIT_KEYS.filter(
|
||||
(key) => LIMIT_NORMALIZERS[key](overrides[key]) !== undefined
|
||||
),
|
||||
...ORG_SELECTION_OVERRIDE_KEYS.filter((key) => overrides[key] != null),
|
||||
];
|
||||
}
|
||||
|
||||
@@ -424,6 +442,10 @@ export function normalizeMirrorOverrides(
|
||||
const limit = LIMIT_NORMALIZERS[key](overrides[key]);
|
||||
if (limit !== undefined) cleaned[key] = limit;
|
||||
}
|
||||
for (const key of ORG_SELECTION_OVERRIDE_KEYS) {
|
||||
const flag = overrides[key];
|
||||
if (typeof flag === "boolean") cleaned[key] = flag;
|
||||
}
|
||||
|
||||
return Object.keys(cleaned).length > 0 ? cleaned : null;
|
||||
}
|
||||
@@ -525,6 +547,89 @@ export function resolveMirrorOptions({
|
||||
return resolved;
|
||||
}
|
||||
|
||||
/**
|
||||
* Resolve the fork policy for one organization.
|
||||
*
|
||||
* Precedence: organization override -> global `githubConfig.skipForks` ->
|
||||
* false. An organization pinning `false` keeps its forks even when the
|
||||
* global switch skips forks everywhere else, and vice versa.
|
||||
*/
|
||||
export function resolveOrganizationSkipForks({
|
||||
orgOverrides,
|
||||
config,
|
||||
}: {
|
||||
orgOverrides: unknown;
|
||||
config: Partial<Config>;
|
||||
}): boolean {
|
||||
const org = parseMirrorOverrides(orgOverrides);
|
||||
if (typeof org?.skipForks === "boolean") return org.skipForks;
|
||||
return config.githubConfig?.skipForks ?? false;
|
||||
}
|
||||
|
||||
/**
|
||||
* Build the per-repository fork decision for bulk paths that hold the fork
|
||||
* pin map from `loadOrganizationForkPolicies` instead of one org row. The
|
||||
* returned predicate applies the same precedence as
|
||||
* `resolveOrganizationSkipForks`: a pinned org wins over the global switch,
|
||||
* unpinned orgs and personal repos follow it.
|
||||
*/
|
||||
export function orgForkSkipDecision(
|
||||
orgForkOverrides: Map<string, boolean> | undefined,
|
||||
config: Partial<Config>
|
||||
): (organizationName: string | null | undefined) => boolean {
|
||||
const globalSkipForks = config.githubConfig?.skipForks ?? false;
|
||||
return (organizationName) => {
|
||||
const key = (organizationName ?? "").trim().toLowerCase();
|
||||
if (key && orgForkOverrides?.has(key)) return orgForkOverrides.get(key)!;
|
||||
return globalSkipForks;
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Load every organization that pins a fork policy, keyed by normalized name.
|
||||
*
|
||||
* Bulk import paths consult this per repository: an org-repository fork is
|
||||
* dropped or kept according to its org's pin, while repos of unpinned orgs
|
||||
* fall back to the global `skipForks`. Failures degrade to an empty map so a
|
||||
* DB hiccup cannot fail the import.
|
||||
*/
|
||||
export async function loadOrganizationForkPolicies({
|
||||
userId,
|
||||
}: {
|
||||
userId?: string;
|
||||
}): Promise<Map<string, boolean>> {
|
||||
const policies = new Map<string, boolean>();
|
||||
if (!userId) return policies;
|
||||
|
||||
try {
|
||||
const { db, organizations } = await import("@/lib/db");
|
||||
const { eq } = await import("drizzle-orm");
|
||||
|
||||
const rows = await db
|
||||
.select({
|
||||
normalizedName: organizations.normalizedName,
|
||||
mirrorOverrides: organizations.mirrorOverrides,
|
||||
})
|
||||
.from(organizations)
|
||||
.where(eq(organizations.userId, userId));
|
||||
|
||||
for (const row of rows) {
|
||||
const overrides = parseMirrorOverrides(row.mirrorOverrides);
|
||||
if (typeof overrides?.skipForks === "boolean") {
|
||||
policies.set(row.normalizedName, overrides.skipForks);
|
||||
}
|
||||
}
|
||||
} catch (error) {
|
||||
console.error(
|
||||
`[MirrorOverrides] Failed to load organization fork policies: ${
|
||||
error instanceof Error ? error.message : String(error)
|
||||
}`
|
||||
);
|
||||
}
|
||||
|
||||
return policies;
|
||||
}
|
||||
|
||||
/**
|
||||
* Fetch the organization-tier overrides for a repository, if it belongs to one.
|
||||
*
|
||||
|
||||
@@ -10,6 +10,7 @@ import {
|
||||
normalizeGitRepoToInsert,
|
||||
calcBatchSizeForInsert,
|
||||
} from "@/lib/repo-utils";
|
||||
import { loadOrganizationForkPolicies } from "@/lib/utils/mirror-overrides";
|
||||
import { requireAuthenticatedUserId } from "@/lib/auth-guards";
|
||||
import { isMirrorableGitHubRepo } from "@/lib/repo-eligibility";
|
||||
|
||||
@@ -53,9 +54,13 @@ export const POST: APIRoute = async ({ request, locals }) => {
|
||||
.where(and(eq(organizations.userId, userId), eq(organizations.status, "ignored")));
|
||||
const ignoredOrgNames = new Set(ignoredOrgRows.map((o) => o.normalizedName));
|
||||
|
||||
// Per-organization fork pins, so an org opted out of forks is not
|
||||
// re-imported by this bulk pass even when the global switch is off.
|
||||
const orgForkOverrides = await loadOrganizationForkPolicies({ userId });
|
||||
|
||||
// Fetch source data in parallel
|
||||
const [basicAndForkedRepos, starredRepos, orgResult] = await Promise.all([
|
||||
sourceProvider.listRepositories(config),
|
||||
sourceProvider.listRepositories(config, { orgForkOverrides }),
|
||||
config.githubConfig?.includeStarred
|
||||
? sourceProvider.listStarredRepositories(config)
|
||||
: Promise.resolve([]),
|
||||
|
||||
@@ -10,6 +10,7 @@ import type { RepoStatus } from "@/types/Repository";
|
||||
import { v4 as uuidv4 } from "uuid";
|
||||
import { createSourceProviderFromConfig } from "@/lib/source-providers";
|
||||
import { normalizeGitRepoToInsert, calcBatchSizeForInsert } from "@/lib/repo-utils";
|
||||
import { resolveOrganizationSkipForks } from "@/lib/utils/mirror-overrides";
|
||||
import { requireAuthenticatedUserId } from "@/lib/auth-guards";
|
||||
|
||||
export const POST: APIRoute = async ({ request, locals }) => {
|
||||
@@ -123,7 +124,13 @@ export const POST: APIRoute = async ({ request, locals }) => {
|
||||
|
||||
// Fetch every repository the token can see in the organization
|
||||
const orgRepos = await sourceProvider.listOrganizationRepositories(trimmedOrg);
|
||||
const mirrorableRepos = orgRepos.filter((repo) => repo.isDisabled !== true);
|
||||
|
||||
// Both existing-org branches above returned, so this organization is new
|
||||
// and has no overrides: the fork policy resolves from the global switch.
|
||||
const skipOrgForks = resolveOrganizationSkipForks({ orgOverrides: null, config });
|
||||
const mirrorableRepos = orgRepos.filter(
|
||||
(repo) => repo.isDisabled !== true && !(skipOrgForks && repo.isForked)
|
||||
);
|
||||
|
||||
// Insert repositories. The normalizer stamps the source provider and URL.
|
||||
const repoRecords = mirrorableRepos.map((repo) =>
|
||||
|
||||
Reference in new issue
Block a user