diff --git a/src/components/config/MirrorOverridesDialog.tsx b/src/components/config/MirrorOverridesDialog.tsx index 637a83f..c99496a 100644 --- a/src/components/config/MirrorOverridesDialog.tsx +++ b/src/components/config/MirrorOverridesDialog.tsx @@ -26,6 +26,7 @@ import { UI_MIRROR_OVERRIDE_KEYS, type InheritedMirrorOptions, type MirrorOverrideKey, + type OrgSelectionOverrideKey, } from "@/lib/utils/mirror-overrides"; /** @@ -34,6 +35,8 @@ import { */ type TriState = "inherit" | "on" | "off"; +type DialogOverrideKey = MirrorOverrideKey | OrgSelectionOverrideKey; + function toTriState(value: boolean | null | undefined): TriState { if (value === true) return "on"; if (value === false) return "off"; @@ -106,8 +109,18 @@ export function MirrorOverridesDialog({ destinationProvider, onSave, }: MirrorOverridesDialogProps) { - const [draft, setDraft] = useState>( - () => buildDraft(value) + // Skip forks steers which repositories an organization imports and mirrors + // at all, so it leads the list; it is meaningless for a single repository. + const dialogKeys = useMemo( + () => + targetKind === "organization" + ? ["skipForks", ...UI_MIRROR_OVERRIDE_KEYS] + : [...UI_MIRROR_OVERRIDE_KEYS], + [targetKind] + ); + + const [draft, setDraft] = useState>( + () => buildDraft(value, dialogKeys) ); const [releaseLimitDraft, setReleaseLimitDraft] = useState(() => buildReleaseLimitDraft(value) @@ -121,11 +134,11 @@ export function MirrorOverridesDialog({ // previous edit never leaks into the next one. useEffect(() => { if (open) { - setDraft(buildDraft(value)); + setDraft(buildDraft(value, dialogKeys)); setReleaseLimitDraft(buildReleaseLimitDraft(value)); setReleaseAssetLimitDraft(buildReleaseAssetLimitDraft(value)); } - }, [open, value]); + }, [open, value, dialogKeys]); // What each flag currently resolves to, including the in-progress edit. // Drives the labels and release-limit gates so they react live as issues @@ -136,6 +149,8 @@ export function MirrorOverridesDialog({ const pinned = fromTriState(draft[key]); next[key] = pinned ?? inheritedFrom?.[key] ?? false; } + next.skipForks = + fromTriState(draft.skipForks) ?? inheritedFrom?.skipForks ?? false; return next; }, [draft, inheritedFrom]); @@ -178,6 +193,10 @@ export function MirrorOverridesDialog({ const resolved = fromTriState(draft[key]); if (typeof resolved === "boolean") next[key] = resolved; } + if (targetKind === "organization") { + const resolved = fromTriState(draft.skipForks); + if (typeof resolved === "boolean") next.skipForks = resolved; + } if (pinnedReleaseLimit !== undefined) next.releaseLimit = pinnedReleaseLimit; if (pinnedReleaseAssetLimit !== undefined) { next.releaseAssetLimit = pinnedReleaseAssetLimit; @@ -190,7 +209,7 @@ export function MirrorOverridesDialog({ }; const handleResetAll = () => { - setDraft(buildDraft(null)); + setDraft(buildDraft(null, dialogKeys)); setReleaseLimitDraft(""); setReleaseAssetLimitDraft(""); }; @@ -229,10 +248,11 @@ export function MirrorOverridesDialog({
- {UI_MIRROR_OVERRIDE_KEYS.map((key) => { + {dialogKeys.map((key) => { const disabledReason = gating[key]; const isDisabled = !!disabledReason; - const inherited = inheritedFrom?.[key]; + const inherited = + key === "skipForks" ? inheritedFrom?.skipForks : inheritedFrom?.[key]; const showHint = !inheritedLoading && !isDisabled && @@ -424,10 +444,11 @@ export function MirrorOverridesDialog({ } function buildDraft( - value: MirrorOverrides | null | undefined -): Record { - const draft = {} as Record; - for (const key of UI_MIRROR_OVERRIDE_KEYS) { + value: MirrorOverrides | null | undefined, + keys: readonly DialogOverrideKey[] +): Record { + const draft = {} as Record; + for (const key of keys) { draft[key] = toTriState(value?.[key]); } return draft; diff --git a/src/components/organizations/OrganizationsList.tsx b/src/components/organizations/OrganizationsList.tsx index b98d660..6ec9b79 100644 --- a/src/components/organizations/OrganizationsList.tsx +++ b/src/components/organizations/OrganizationsList.tsx @@ -69,7 +69,7 @@ export function OrganizationList({ onRefresh, onDelete, }: OrganizationListProps) { - const { giteaConfig, mirrorOptions } = useGiteaConfig(); + const { giteaConfig, mirrorOptions, advancedOptions } = useGiteaConfig(); const [overridesTarget, setOverridesTarget] = useState(null); const handleUpdateMirrorOverrides = async ( @@ -786,7 +786,10 @@ export function OrganizationList({ targetName={overridesTarget?.name ?? ""} value={overridesTarget?.mirrorOverrides ?? null} destinationProvider={giteaConfig?.provider} - inheritedFrom={mirrorOptionsToFlags(mirrorOptions)} + inheritedFrom={{ + ...mirrorOptionsToFlags(mirrorOptions), + skipForks: !!advancedOptions?.skipForks, + }} inheritedLabel="global settings" onSave={async (overrides) => { if (overridesTarget?.id) { diff --git a/src/lib/db/schema.ts b/src/lib/db/schema.ts index 4883fd9..971b7a8 100644 --- a/src/lib/db/schema.ts +++ b/src/lib/db/schema.ts @@ -72,6 +72,13 @@ export const backupStrategyEnum = z.enum([ * `releaseAssetLimit` follows the same rule (`normalizeReleaseAssetLimit`): * assets are uploaded only for the newest N mirrored releases. 0 is a real * value (notes only); null/absent means inherit. + * + * `skipForks` is organization-tier only: it opts an organization out of + * importing and mirroring forked repositories without touching the global + * `githubConfig.skipForks` switch. Null/absent inherits that global value. + * It is deliberately NOT part of `MIRROR_OVERRIDE_KEYS` in + * `mirror-overrides.ts` (those mirror `giteaConfigSchema` flags); see + * `ORG_SELECTION_OVERRIDE_KEYS` there. */ export const mirrorOverridesSchema = z.object({ lfs: z.boolean().nullable().optional(), @@ -84,6 +91,7 @@ export const mirrorOverridesSchema = z.object({ mirrorMilestones: z.boolean().nullable().optional(), releaseLimit: z.number().nullable().optional(), releaseAssetLimit: z.number().nullable().optional(), + skipForks: z.boolean().nullable().optional(), }); export type MirrorOverrides = z.infer; diff --git a/src/lib/gitea-org-mirror-destination.test.ts b/src/lib/gitea-org-mirror-destination.test.ts index 930fb0d..60b437b 100644 --- a/src/lib/gitea-org-mirror-destination.test.ts +++ b/src/lib/gitea-org-mirror-destination.test.ts @@ -373,3 +373,75 @@ describe.skipIf(!isChild)("mirrorGitHubOrgToGitea destination routing (#343)", ( expect(orgCreateCalls.length).toBe(0); }); }); + +describe.skipIf(!isChild)("mirrorGitHubOrgToGitea fork policy", () => { + const forkRepo = makeRepo({ id: "repo-fork", name: "forked", fullName: "A/forked", isForked: true }); + const normalRepo = makeRepo({ id: "repo-2", name: "r2", fullName: "A/r2", cloneUrl: "https://github.com/A/r2.git" }); + + test("a skip-forks override drops forked repositories from the batch", async () => { + const config = makeConfig({ githubConfig: { mirrorStrategy: "preserve" } }); + const organization = makeOrg({ mirrorOverrides: { skipForks: true } }); + orgRepoRows = [makeRepo(), forkRepo]; + orgConfigRows = [organization]; + + await mirrorGitHubOrgToGitea({ organization, octokit: fakeOctokit, config }); + + const migrates = migrateCalls(); + expect(migrates.length).toBe(1); + expect(migrates[0].payload.repo_name).toBe("r1"); + }); + + test("the global skipForks switch applies to the org batch too", async () => { + const config = makeConfig({ + githubConfig: { mirrorStrategy: "preserve", skipForks: true }, + }); + const organization = makeOrg(); + orgRepoRows = [makeRepo(), forkRepo]; + orgConfigRows = [organization]; + + await mirrorGitHubOrgToGitea({ organization, octokit: fakeOctokit, config }); + + const migrates = migrateCalls(); + expect(migrates.length).toBe(1); + expect(migrates[0].payload.repo_name).toBe("r1"); + }); + + test("a false org pin keeps forks despite the global switch", async () => { + const config = makeConfig({ + githubConfig: { mirrorStrategy: "preserve", skipForks: true }, + }); + const organization = makeOrg({ mirrorOverrides: { skipForks: false } }); + orgRepoRows = [makeRepo(), forkRepo]; + orgConfigRows = [organization]; + + await mirrorGitHubOrgToGitea({ organization, octokit: fakeOctokit, config }); + + const migrates = migrateCalls(); + expect(migrates.length).toBe(2); + }); + + test("forks are mirrored when nothing opts out (regression)", async () => { + const config = makeConfig({ githubConfig: { mirrorStrategy: "preserve" } }); + const organization = makeOrg(); + orgRepoRows = [normalRepo, forkRepo]; + orgConfigRows = [organization]; + + await mirrorGitHubOrgToGitea({ organization, octokit: fakeOctokit, config }); + + const migrates = migrateCalls(); + expect(migrates.length).toBe(2); + }); + + test("a fork-only organization with skip forks mirrors nothing and still succeeds", async () => { + const config = makeConfig({ + githubConfig: { mirrorStrategy: "preserve", skipForks: true }, + }); + const organization = makeOrg(); + orgRepoRows = [forkRepo]; + orgConfigRows = [organization]; + + await mirrorGitHubOrgToGitea({ organization, octokit: fakeOctokit, config }); + + expect(migrateCalls().length).toBe(0); + }); +}); diff --git a/src/lib/gitea.ts b/src/lib/gitea.ts index b76dcbb..cbc91c2 100644 --- a/src/lib/gitea.ts +++ b/src/lib/gitea.ts @@ -28,6 +28,7 @@ import { normalizeReleaseLimit, releaseGetsAssets, resolveMirrorOptionsForRepository, + resolveOrganizationSkipForks, type ResolvedMirrorOptions, } from "./utils/mirror-overrides"; import { repositoryDestinationColumns } from "./repo-utils"; @@ -2160,13 +2161,29 @@ export async function mirrorGitHubOrgToGitea({ .from(repositories) .where(eq(repositories.organization, organization.name)); - if (orgRepos.length === 0) { + // The organization's fork policy (override -> global skipForks) drops + // forked repositories from the batch so they are neither mirrored here + // nor re-mirrored by later runs until the policy changes. + const skipOrgForks = resolveOrganizationSkipForks({ + orgOverrides: organization.mirrorOverrides, + config, + }); + const mirrorableOrgRepos = skipOrgForks + ? orgRepos.filter((repo) => !repo.isForked) + : orgRepos; + if (skipOrgForks && orgRepos.length !== mirrorableOrgRepos.length) { + console.log( + `Skipping ${orgRepos.length - mirrorableOrgRepos.length} forked repositories for organization ${organization.name} (skip forks is on)` + ); + } + + if (mirrorableOrgRepos.length === 0) { console.log( `No repositories found for organization ${organization.name} - marking as successfully mirrored` ); } else { console.log( - `Mirroring ${orgRepos.length} repositories for organization ${organization.name}` + `Mirroring ${mirrorableOrgRepos.length} repositories for organization ${organization.name}` ); // Import the processWithRetry function @@ -2174,7 +2191,7 @@ export async function mirrorGitHubOrgToGitea({ // Process repositories in parallel with concurrency control await processWithRetry( - orgRepos, + mirrorableOrgRepos, async (repo) => { // Prepare repository data const repoData = { diff --git a/src/lib/github-affiliation.test.ts b/src/lib/github-affiliation.test.ts index d8f5288..7c330a6 100644 --- a/src/lib/github-affiliation.test.ts +++ b/src/lib/github-affiliation.test.ts @@ -253,3 +253,47 @@ describe("getGithubRepositories - includeOrganizations allowlist", () => { expect(repos.map((r) => r.name)).toContain("noise"); }); }); + +describe("getGithubRepositories - per-organization fork pins", () => { + const acmeFork = makeRepo({ name: "acme-fork", ownerLogin: "acme", ownerType: "Organization", fork: true }); + const globexFork = makeRepo({ name: "globex-fork", ownerLogin: "globex", ownerType: "Organization", fork: true }); + const ownFork = makeRepo({ name: "own-fork", ownerLogin: "octo", ownerType: "User", fork: true }); + const plain = makeRepo({ name: "plain" }); + + async function listed( + config: Record, + orgForkOverrides?: Map + ) { + const { octokit } = makeOctokit([acmeFork, globexFork, ownFork, plain]); + const repos = await getGithubRepositories({ + octokit, + config: { githubConfig: { owner: "octo", ...config } as any }, + orgForkOverrides, + }); + return repos.map((r) => r.name).sort(); + } + + test("a pinned org drops only its own forks", async () => { + expect(await listed({}, new Map([["acme", true]]))).toEqual([ + "globex-fork", + "own-fork", + "plain", + ]); + }); + + test("a false pin keeps that org's forks when the global switch skips forks", async () => { + expect(await listed({ skipForks: true }, new Map([["globex", false]]))).toEqual([ + "globex-fork", + "plain", + ]); + }); + + test("pins never apply to personal forks", async () => { + expect(await listed({}, new Map([["octo", true]]))).toEqual([ + "acme-fork", + "globex-fork", + "own-fork", + "plain", + ]); + }); +}); diff --git a/src/lib/github.ts b/src/lib/github.ts index b1dc7b3..7502fee 100644 --- a/src/lib/github.ts +++ b/src/lib/github.ts @@ -3,6 +3,7 @@ import type { GitRepo, RepoStatus } from "@/types/Repository"; import { Octokit } from "@octokit/rest"; import { throttling } from "@octokit/plugin-throttling"; import type { Config } from "@/types/config"; +import { orgForkSkipDecision } from "@/lib/utils/mirror-overrides"; import { applyConditionalRequests, conditionalRequestTokenScope, @@ -249,6 +250,7 @@ export async function getGithubRepositories({ config, includeCollaboratorReposOverride, includeAllOrgsOverride, + orgForkOverrides, }: { octokit: Octokit; config: Partial; @@ -260,6 +262,9 @@ export async function getGithubRepositories({ // Used by the cleanup service so a previously-mirrored org repo isn't flagged // as orphaned just because the user narrowed the allowlist. includeAllOrgsOverride?: boolean; + // Per-organization fork pins (normalized org name -> skip forks). A fork + // owned by a pinned org follows the pin instead of the global skipForks. + orgForkOverrides?: Map; }): Promise { try { const includeCollab = @@ -279,7 +284,7 @@ export async function getGithubRepositories({ { per_page: 100, affiliation }, ); - const skipForks = config.githubConfig?.skipForks ?? false; + const forkSkipFor = orgForkSkipDecision(orgForkOverrides, config); const skipPersonalRepos = config.githubConfig?.skipPersonalRepos ?? false; // The authenticated user's login — used to identify personally-owned repos const authenticatedUserLogin = config.githubConfig?.owner ?? ""; @@ -295,6 +300,9 @@ export async function getGithubRepositories({ ); const filteredRepos = repos.filter((repo) => { + const skipForks = forkSkipFor( + repo.owner.type === "Organization" ? repo.owner.login : undefined + ); const isForkAllowed = !skipForks || !repo.fork; // When skipPersonalRepos is true, drop repos owned by the authenticated user // (owner.type === "User" and owner.login matches the configured GitHub username). diff --git a/src/lib/scheduler-service.ts b/src/lib/scheduler-service.ts index 1cd20ae..96e90f0 100644 --- a/src/lib/scheduler-service.ts +++ b/src/lib/scheduler-service.ts @@ -15,6 +15,7 @@ import type { Octokit } from '@octokit/rest'; import { repoStatusEnum, repositoryVisibilityEnum } from '@/types/Repository'; import { mergeGitReposPreferStarred, normalizeGitRepoToInsert, calcBatchSizeForInsert } from '@/lib/repo-utils'; import { isMirrorableGitHubRepo } from '@/lib/repo-eligibility'; +import { loadOrganizationForkPolicies, orgForkSkipDecision } from '@/lib/utils/mirror-overrides'; import { createMirrorJob } from '@/lib/helpers'; import { getNextScheduledRun, isCronExpression, normalizeTimezone } from '@/lib/utils/schedule-utils'; import { resetStuckMirrorStatuses } from '@/lib/stuck-status-recovery'; @@ -108,9 +109,13 @@ async function runScheduledSync(config: any): Promise { // The configured source host (GitHub honors GH_API_URL for GHES / GHEC) const sourceProvider = createSourceProviderFromConfig(config, { userId }); + // Per-organization fork pins, so orgs opted out of forks stay that + // way during scheduled auto-import. + const orgForkOverrides = await loadOrganizationForkPolicies({ userId }); + // Fetch source data const [basicAndForkedRepos, starredRepos] = await Promise.all([ - sourceProvider.listRepositories(config), + sourceProvider.listRepositories(config, { orgForkOverrides }), config.githubConfig?.includeStarred ? sourceProvider.listStarredRepositories(config) : Promise.resolve([]), @@ -123,7 +128,7 @@ async function runScheduledSync(config: any): Promise { .select({ normalizedFullName: repositories.normalizedFullName }) .from(repositories) .where(eq(repositories.userId, userId)); - + const existingRepoNames = new Set(existingRepos.map(r => r.normalizedFullName)); const newRepos = mirrorableGithubRepos.filter(r => !existingRepoNames.has(r.fullName.toLowerCase())); @@ -238,6 +243,18 @@ async function runScheduledSync(config: any): Promise { console.log(`[Scheduler] Skipped ${skippedCount} repositories from auto-mirror (autoMirror=${autoMirrorOwned}, autoMirrorStarred=${autoMirrorStarred})`); } + // Organization fork policies apply to already-imported rows too, so a + // fork that slipped in before its org opted out is not auto-mirrored. + const orgForkOverrides = await loadOrganizationForkPolicies({ userId }); + const forkSkipFor = orgForkSkipDecision(orgForkOverrides, config); + const forkSkippedCount = reposNeedingMirror.length; + reposNeedingMirror = reposNeedingMirror.filter( + repo => !repo.isForked || !repo.organization || !forkSkipFor(repo.organization) + ); + if (reposNeedingMirror.length !== forkSkippedCount) { + console.log(`[Scheduler] Skipped ${forkSkippedCount - reposNeedingMirror.length} forked repositories from auto-mirror (organization skip forks)`); + } + if (reposNeedingMirror.length > 0) { console.log(`[Scheduler] Found ${reposNeedingMirror.length} repositories that need initial mirroring`); @@ -521,23 +538,27 @@ async function performInitialAutoStart(): Promise { // The configured source host (GitHub honors GH_API_URL for GHES / GHEC) const sourceProvider = createSourceProviderFromConfig(config, { userId: config.userId }); - + + // Per-organization fork pins, so orgs opted out of forks stay that + // way during boot auto-start imports. + const orgForkOverrides = await loadOrganizationForkPolicies({ userId: config.userId }); + // Fetch source data const [basicAndForkedRepos, starredRepos] = await Promise.all([ - sourceProvider.listRepositories(config), + sourceProvider.listRepositories(config, { orgForkOverrides }), config.githubConfig?.includeStarred ? sourceProvider.listStarredRepositories(config) : Promise.resolve([]), ]); const allGithubRepos = mergeGitReposPreferStarred(basicAndForkedRepos, starredRepos); const mirrorableGithubRepos = allGithubRepos.filter(isMirrorableGitHubRepo); - + // Check for new repositories const existingRepos = await db .select({ normalizedFullName: repositories.normalizedFullName }) .from(repositories) .where(eq(repositories.userId, config.userId)); - + const existingRepoNames = new Set(existingRepos.map(r => r.normalizedFullName)); const reposToImport = mirrorableGithubRepos.filter(r => !existingRepoNames.has(r.fullName.toLowerCase())); diff --git a/src/lib/source-provider-wiring.test.ts b/src/lib/source-provider-wiring.test.ts index 50d9f55..2ae3404 100644 --- a/src/lib/source-provider-wiring.test.ts +++ b/src/lib/source-provider-wiring.test.ts @@ -46,7 +46,7 @@ describe("discovery and housekeeping go through the source provider", () => { test("scheduler auto import, auto mirror and boot auto start", () => { const source = read("scheduler-service.ts"); expect(count(source, "createSourceProviderFromConfig(config")).toBe(2); - expect(count(source, "sourceProvider.listRepositories(config)")).toBe(2); + expect(count(source, "sourceProvider.listRepositories(")).toBe(2); expect(source).not.toContain("getGithubRepositories("); // A GitHub client is built only for GitHub sources, in both mirror phases. expect(count(source, "resolveSourceProviderKind(config) === 'github'")).toBe(2); diff --git a/src/lib/source-providers/filters.test.ts b/src/lib/source-providers/filters.test.ts index e5e40c2..6e99723 100644 --- a/src/lib/source-providers/filters.test.ts +++ b/src/lib/source-providers/filters.test.ts @@ -91,3 +91,48 @@ describe("filterSourceRepositories", () => { expect(kept).toHaveLength(5); }); }); + +describe("filterSourceRepositories - per-organization fork pins", () => { + const acmeFork = repo({ owner: "acme", name: "forked-tool", organization: "acme", isForked: true }); + const globexFork = repo({ owner: "globex", name: "forked-app", organization: "globex", isForked: true }); + const repos = [own, acme, acmeFork, globexFork]; + + test("a pinned org drops only its own forks", () => { + const kept = filterSourceRepositories(repos, { + config: {}, + options: { orgForkOverrides: new Map([["acme", true]]) }, + username: "me", + }); + expect(kept.map((r) => r.name).sort()).toEqual(["forked-app", "mine", "tool"]); + }); + + test("a false pin keeps that org's forks when the global switch skips forks", () => { + const kept = filterSourceRepositories(repos, { + config: { githubConfig: { skipForks: true } as any }, + options: { orgForkOverrides: new Map([["globex", false]]) }, + username: "me", + }); + expect(kept.map((r) => r.name).sort()).toEqual(["forked-app", "mine", "tool"]); + }); + + test("pins do not affect non-fork repositories", () => { + const kept = filterSourceRepositories([acme, acmeFork], { + config: {}, + options: { orgForkOverrides: new Map([["acme", true]]) }, + username: "me", + }); + expect(kept.map((r) => r.name)).toEqual(["tool"]); + }); + + test("a repository org name is matched against map keys case-insensitively", () => { + // Map keys come from organizations.normalizedName (lowercase); the repo's + // organization can carry any casing from the source host. + const casedFork = repo({ owner: "Acme", name: "forked-tool", organization: "Acme", isForked: true }); + const kept = filterSourceRepositories([casedFork], { + config: {}, + options: { orgForkOverrides: new Map([["acme", true]]) }, + username: "me", + }); + expect(kept).toHaveLength(0); + }); +}); diff --git a/src/lib/source-providers/filters.ts b/src/lib/source-providers/filters.ts index 3de208c..05d740b 100644 --- a/src/lib/source-providers/filters.ts +++ b/src/lib/source-providers/filters.ts @@ -1,5 +1,6 @@ import type { Config } from "@/lib/db/schema"; import type { GitRepo } from "@/types/Repository"; +import { orgForkSkipDecision } from "@/lib/utils/mirror-overrides"; import type { ListRepositoriesOptions } from "./types"; /** @@ -11,6 +12,8 @@ import type { ListRepositoriesOptions } from "./types"; * - skipPersonalRepos drops repos the configured account owns * - includeCollaboratorRepos=false drops personal repos of other users * - includeOrganizations, when non-empty, keeps only org repos from listed orgs + * - options.orgForkOverrides pins skipForks per organization: a fork owned by + * a pinned org follows the pin instead of the global skipForks */ export function filterSourceRepositories( repos: GitRepo[], @@ -28,7 +31,7 @@ export function filterSourceRepositories( options.includeCollaboratorReposOverride ?? config.githubConfig?.includeCollaboratorRepos ?? true; - const skipForks = config.githubConfig?.skipForks ?? false; + const forkSkipFor = orgForkSkipDecision(options.orgForkOverrides, config); const skipPersonalRepos = config.githubConfig?.skipPersonalRepos ?? false; const includeOrgs = options.includeAllOrgsOverride ? [] @@ -39,13 +42,13 @@ export function filterSourceRepositories( const me = username.trim().toLowerCase(); return repos.filter((repo) => { - if (skipForks && repo.isForked) return false; - const ownerKey = repo.owner.trim().toLowerCase(); const orgKey = (repo.organization ?? "").trim().toLowerCase(); const isOrgRepo = orgKey.length > 0; const isOwnRepo = !isOrgRepo && me.length > 0 && ownerKey === me; + if (forkSkipFor(repo.organization) && repo.isForked) return false; + if (skipPersonalRepos && isOwnRepo) return false; if (!includeCollab && !isOrgRepo && !isOwnRepo) return false; if (allowedOrgs.size > 0 && isOrgRepo && !allowedOrgs.has(orgKey)) return false; diff --git a/src/lib/source-providers/github-source.ts b/src/lib/source-providers/github-source.ts index f0dc0e3..21e3689 100644 --- a/src/lib/source-providers/github-source.ts +++ b/src/lib/source-providers/github-source.ts @@ -138,6 +138,7 @@ export class GitHubSourceProvider implements SourceProvider { config, includeCollaboratorReposOverride: options.includeCollaboratorReposOverride, includeAllOrgsOverride: options.includeAllOrgsOverride, + orgForkOverrides: options.orgForkOverrides, }); return this.stamp(repos); } diff --git a/src/lib/source-providers/types.ts b/src/lib/source-providers/types.ts index 86a0acb..96fd01b 100644 --- a/src/lib/source-providers/types.ts +++ b/src/lib/source-providers/types.ts @@ -21,6 +21,12 @@ export interface ListRepositoriesOptions { includeCollaboratorReposOverride?: boolean; /** Ignore the organization allowlist and return every org repository. */ includeAllOrgsOverride?: boolean; + /** + * Per-organization fork pins (normalized org name -> skip forks), from + * `loadOrganizationForkPolicies`. An org-repository fork follows its org's + * pin; repositories of unpinned orgs follow the global `skipForks`. + */ + orgForkOverrides?: Map; } export interface SourceFailedOrganization { diff --git a/src/lib/utils/mirror-overrides.test.ts b/src/lib/utils/mirror-overrides.test.ts index 88a53d1..3093a9f 100644 --- a/src/lib/utils/mirror-overrides.test.ts +++ b/src/lib/utils/mirror-overrides.test.ts @@ -12,9 +12,11 @@ import { normalizeMirrorOverrides, normalizeReleaseAssetLimit, normalizeReleaseLimit, + orgForkSkipDecision, parseMirrorOverrides, releaseGetsAssets, resolveMirrorOptions, + resolveOrganizationSkipForks, STARRED_CLAMPED_KEYS, UI_MIRROR_OVERRIDE_KEYS, } from "./mirror-overrides"; @@ -1065,3 +1067,77 @@ describe("release asset limit (#311)", () => { }); }); }); + +describe("organization skip forks policy", () => { + test("resolveOrganizationSkipForks defaults to the global switch", () => { + expect( + resolveOrganizationSkipForks({ orgOverrides: null, config: makeConfig({}, {}) }) + ).toBe(false); + expect( + resolveOrganizationSkipForks({ + orgOverrides: null, + config: makeConfig({}, { skipForks: true }), + }) + ).toBe(true); + }); + + test("an organization pin beats the global switch in either direction", () => { + expect( + resolveOrganizationSkipForks({ + orgOverrides: { skipForks: true }, + config: makeConfig({}, { skipForks: false }), + }) + ).toBe(true); + expect( + resolveOrganizationSkipForks({ + orgOverrides: { skipForks: false }, + config: makeConfig({}, { skipForks: true }), + }) + ).toBe(false); + }); + + test("a null pin means inherit, not false", () => { + expect( + resolveOrganizationSkipForks({ + orgOverrides: { skipForks: null }, + config: makeConfig({}, { skipForks: true }), + }) + ).toBe(true); + }); + + test("orgForkSkipDecision resolves per organization, case-insensitively", () => { + const decide = orgForkSkipDecision( + new Map([["acme", true], ["globex", false]]), + makeConfig({}, { skipForks: false }) + ); + expect(decide("Acme")).toBe(true); + expect(decide("GLOBEX")).toBe(false); + expect(decide("umbrella")).toBe(false); + expect(decide(undefined)).toBe(false); + }); + + test("orgForkSkipDecision falls back to a global skip when no org is pinned", () => { + const decide = orgForkSkipDecision(undefined, makeConfig({}, { skipForks: true })); + expect(decide("acme")).toBe(true); + expect(decide(undefined)).toBe(true); + expect(decide("")).toBe(true); + }); + + test("skip forks participates in the storage helpers like any other pin", () => { + expect(hasMirrorOverrides({ skipForks: true })).toBe(true); + expect(listOverriddenKeys({ skipForks: true })).toEqual(["skipForks"]); + expect(normalizeMirrorOverrides({ skipForks: true, lfs: null })).toEqual({ + skipForks: true, + }); + expect(MIRROR_OVERRIDE_LABELS.skipForks).toBe("Skip forks"); + }); + + test("skip forks is not part of the gitea mirror flag set the resolver walks", () => { + expect(MIRROR_OVERRIDE_KEYS).not.toContain("skipForks"); + const resolved = resolveMirrorOptions({ + config: makeConfig({}), + repository: makeRepo({ mirrorOverrides: { skipForks: true } as any }), + }); + expect((resolved as any).skipForks).toBeUndefined(); + }); +}); diff --git a/src/lib/utils/mirror-overrides.ts b/src/lib/utils/mirror-overrides.ts index 36b8565..523e6e2 100644 --- a/src/lib/utils/mirror-overrides.ts +++ b/src/lib/utils/mirror-overrides.ts @@ -35,6 +35,18 @@ export const MIRROR_OVERRIDE_LIMIT_KEYS = ["releaseLimit", "releaseAssetLimit"] export type MirrorOverrideLimitKey = (typeof MIRROR_OVERRIDE_LIMIT_KEYS)[number]; +/** + * Organization-tier selection overrides: per-organization switches that steer + * which source repositories are imported and mirrored, not what a mirror + * contains. They are kept apart from `MIRROR_OVERRIDE_KEYS` because that list + * mirrors `giteaConfigSchema` flag names so the resolver can read the global + * tier with the same key — the global tier for these lives on + * `githubConfig` instead, and they are meaningless at the repository tier. + */ +export const ORG_SELECTION_OVERRIDE_KEYS = ["skipForks"] as const; + +export type OrgSelectionOverrideKey = (typeof ORG_SELECTION_OVERRIDE_KEYS)[number]; + /** Matches `giteaConfigSchema.releaseLimit`'s default. */ export const DEFAULT_RELEASE_LIMIT = 10; @@ -56,6 +68,7 @@ export type ResolvedMirrorOptions = Record & { export type InheritedMirrorOptions = Partial> & { releaseLimit?: number; releaseAssetLimit?: number | null; + skipForks?: boolean; }; /** @@ -256,7 +269,10 @@ export const MIRROR_GATING_REASONS = { /** key -> reason it cannot take effect. Absent key means editable. */ export type MirrorOverrideGating = Partial< - Record + Record< + MirrorOverrideKey | MirrorOverrideLimitKey | OrgSelectionOverrideKey, + string + > >; /** @@ -337,7 +353,7 @@ export function getMirrorOverrideGating({ } export const MIRROR_OVERRIDE_LABELS: Record< - MirrorOverrideKey | MirrorOverrideLimitKey, + MirrorOverrideKey | MirrorOverrideLimitKey | OrgSelectionOverrideKey, string > = { lfs: "Git LFS files", @@ -350,6 +366,7 @@ export const MIRROR_OVERRIDE_LABELS: Record< mirrorMilestones: "Milestones", releaseLimit: "Release limit", releaseAssetLimit: "Release asset limit", + skipForks: "Skip forks", }; /** @@ -392,7 +409,7 @@ export function hasMirrorOverrides(value: unknown): boolean { /** The keys an overrides object pins, for badges and summaries. */ export function listOverriddenKeys( value: unknown -): (MirrorOverrideKey | MirrorOverrideLimitKey)[] { +): (MirrorOverrideKey | MirrorOverrideLimitKey | OrgSelectionOverrideKey)[] { const overrides = parseMirrorOverrides(value); if (!overrides) return []; return [ @@ -400,6 +417,7 @@ export function listOverriddenKeys( ...MIRROR_OVERRIDE_LIMIT_KEYS.filter( (key) => LIMIT_NORMALIZERS[key](overrides[key]) !== undefined ), + ...ORG_SELECTION_OVERRIDE_KEYS.filter((key) => overrides[key] != null), ]; } @@ -424,6 +442,10 @@ export function normalizeMirrorOverrides( const limit = LIMIT_NORMALIZERS[key](overrides[key]); if (limit !== undefined) cleaned[key] = limit; } + for (const key of ORG_SELECTION_OVERRIDE_KEYS) { + const flag = overrides[key]; + if (typeof flag === "boolean") cleaned[key] = flag; + } return Object.keys(cleaned).length > 0 ? cleaned : null; } @@ -525,6 +547,89 @@ export function resolveMirrorOptions({ return resolved; } +/** + * Resolve the fork policy for one organization. + * + * Precedence: organization override -> global `githubConfig.skipForks` -> + * false. An organization pinning `false` keeps its forks even when the + * global switch skips forks everywhere else, and vice versa. + */ +export function resolveOrganizationSkipForks({ + orgOverrides, + config, +}: { + orgOverrides: unknown; + config: Partial; +}): boolean { + const org = parseMirrorOverrides(orgOverrides); + if (typeof org?.skipForks === "boolean") return org.skipForks; + return config.githubConfig?.skipForks ?? false; +} + +/** + * Build the per-repository fork decision for bulk paths that hold the fork + * pin map from `loadOrganizationForkPolicies` instead of one org row. The + * returned predicate applies the same precedence as + * `resolveOrganizationSkipForks`: a pinned org wins over the global switch, + * unpinned orgs and personal repos follow it. + */ +export function orgForkSkipDecision( + orgForkOverrides: Map | undefined, + config: Partial +): (organizationName: string | null | undefined) => boolean { + const globalSkipForks = config.githubConfig?.skipForks ?? false; + return (organizationName) => { + const key = (organizationName ?? "").trim().toLowerCase(); + if (key && orgForkOverrides?.has(key)) return orgForkOverrides.get(key)!; + return globalSkipForks; + }; +} + +/** + * Load every organization that pins a fork policy, keyed by normalized name. + * + * Bulk import paths consult this per repository: an org-repository fork is + * dropped or kept according to its org's pin, while repos of unpinned orgs + * fall back to the global `skipForks`. Failures degrade to an empty map so a + * DB hiccup cannot fail the import. + */ +export async function loadOrganizationForkPolicies({ + userId, +}: { + userId?: string; +}): Promise> { + const policies = new Map(); + if (!userId) return policies; + + try { + const { db, organizations } = await import("@/lib/db"); + const { eq } = await import("drizzle-orm"); + + const rows = await db + .select({ + normalizedName: organizations.normalizedName, + mirrorOverrides: organizations.mirrorOverrides, + }) + .from(organizations) + .where(eq(organizations.userId, userId)); + + for (const row of rows) { + const overrides = parseMirrorOverrides(row.mirrorOverrides); + if (typeof overrides?.skipForks === "boolean") { + policies.set(row.normalizedName, overrides.skipForks); + } + } + } catch (error) { + console.error( + `[MirrorOverrides] Failed to load organization fork policies: ${ + error instanceof Error ? error.message : String(error) + }` + ); + } + + return policies; +} + /** * Fetch the organization-tier overrides for a repository, if it belongs to one. * diff --git a/src/pages/api/sync/index.ts b/src/pages/api/sync/index.ts index de28e62..e04835d 100644 --- a/src/pages/api/sync/index.ts +++ b/src/pages/api/sync/index.ts @@ -10,6 +10,7 @@ import { normalizeGitRepoToInsert, calcBatchSizeForInsert, } from "@/lib/repo-utils"; +import { loadOrganizationForkPolicies } from "@/lib/utils/mirror-overrides"; import { requireAuthenticatedUserId } from "@/lib/auth-guards"; import { isMirrorableGitHubRepo } from "@/lib/repo-eligibility"; @@ -53,9 +54,13 @@ export const POST: APIRoute = async ({ request, locals }) => { .where(and(eq(organizations.userId, userId), eq(organizations.status, "ignored"))); const ignoredOrgNames = new Set(ignoredOrgRows.map((o) => o.normalizedName)); + // Per-organization fork pins, so an org opted out of forks is not + // re-imported by this bulk pass even when the global switch is off. + const orgForkOverrides = await loadOrganizationForkPolicies({ userId }); + // Fetch source data in parallel const [basicAndForkedRepos, starredRepos, orgResult] = await Promise.all([ - sourceProvider.listRepositories(config), + sourceProvider.listRepositories(config, { orgForkOverrides }), config.githubConfig?.includeStarred ? sourceProvider.listStarredRepositories(config) : Promise.resolve([]), diff --git a/src/pages/api/sync/organization.ts b/src/pages/api/sync/organization.ts index 297a56d..f1c4b7a 100644 --- a/src/pages/api/sync/organization.ts +++ b/src/pages/api/sync/organization.ts @@ -10,6 +10,7 @@ import type { RepoStatus } from "@/types/Repository"; import { v4 as uuidv4 } from "uuid"; import { createSourceProviderFromConfig } from "@/lib/source-providers"; import { normalizeGitRepoToInsert, calcBatchSizeForInsert } from "@/lib/repo-utils"; +import { resolveOrganizationSkipForks } from "@/lib/utils/mirror-overrides"; import { requireAuthenticatedUserId } from "@/lib/auth-guards"; export const POST: APIRoute = async ({ request, locals }) => { @@ -123,7 +124,13 @@ export const POST: APIRoute = async ({ request, locals }) => { // Fetch every repository the token can see in the organization const orgRepos = await sourceProvider.listOrganizationRepositories(trimmedOrg); - const mirrorableRepos = orgRepos.filter((repo) => repo.isDisabled !== true); + + // Both existing-org branches above returned, so this organization is new + // and has no overrides: the fork policy resolves from the global switch. + const skipOrgForks = resolveOrganizationSkipForks({ orgOverrides: null, config }); + const mirrorableRepos = orgRepos.filter( + (repo) => repo.isDisabled !== true && !(skipOrgForks && repo.isForked) + ); // Insert repositories. The normalizer stamps the source provider and URL. const repoRecords = mirrorableRepos.map((repo) =>