From 088467a57db64f7a1bcfb2ab7c9e62f89bc1451a Mon Sep 17 00:00:00 2001 From: ARUNAVO RAY Date: Mon, 4 May 2026 14:00:10 +0530 Subject: [PATCH] feat: add option to exclude collaborator repos from import (closes #279) (#283) GitHub's listForAuthenticatedUser defaults to returning every repo the user has access to (owner + collaborator + organization_member), which imports a lot of noise for users who only want their own repos. Adds an `includeCollaboratorRepos` toggle, defaulting to true to preserve existing behavior. When disabled, the affiliation filter scopes the API call to "owner" only. The cleanup service overrides the filter to always include collaborator repos when computing the "what's still on GitHub" list. Without this, toggling the option off would mark previously-mirrored collab repos as orphaned and archive/delete them from Gitea. Wired through the schema, both UI<->DB mappers, the env-config loader (with new INCLUDE_COLLABORATOR_REPOS env var), and the settings UI. --- docs/ENVIRONMENT_VARIABLES.md | 1 + .../config/GitHubMirrorSettings.tsx | 21 +++++ src/lib/db/schema.ts | 1 + src/lib/env-config-loader.ts | 7 ++ src/lib/github-affiliation.test.ts | 75 +++++++++++++++ src/lib/github.ts | 13 ++- src/lib/repository-cleanup-service.ts | 7 +- src/lib/utils/config-mapper.test.ts | 94 ++++++++++++++++++- src/lib/utils/config-mapper.ts | 2 + src/types/config.ts | 1 + 10 files changed, 217 insertions(+), 5 deletions(-) create mode 100644 src/lib/github-affiliation.test.ts diff --git a/docs/ENVIRONMENT_VARIABLES.md b/docs/ENVIRONMENT_VARIABLES.md index 38d37c2..0fc2541 100644 --- a/docs/ENVIRONMENT_VARIABLES.md +++ b/docs/ENVIRONMENT_VARIABLES.md @@ -100,6 +100,7 @@ Standard GitHub Enterprise Cloud on `github.com` works with the default — no o | `PRIVATE_REPOSITORIES` | Include private repositories | `false` | `true`, `false` | | `PUBLIC_REPOSITORIES` | Include public repositories | `true` | `true`, `false` | | `INCLUDE_ARCHIVED` | Include archived repositories | `false` | `true`, `false` | +| `INCLUDE_COLLABORATOR_REPOS` | Include repositories where you are a collaborator (not just owned). Set to `false` to limit imports to repos you own. | `true` | `true`, `false` | | `SKIP_FORKS` | Skip forked repositories | `false` | `true`, `false` | | `MIRROR_STARRED` | Mirror starred repositories | `false` | `true`, `false` | | `MIRROR_STARRED_LISTS` | Optional comma-separated GitHub Star List names to mirror (only used when `MIRROR_STARRED=true`) | - | Comma-separated list names (empty = all starred repos) | diff --git a/src/components/config/GitHubMirrorSettings.tsx b/src/components/config/GitHubMirrorSettings.tsx index 64bf005..0137410 100644 --- a/src/components/config/GitHubMirrorSettings.tsx +++ b/src/components/config/GitHubMirrorSettings.tsx @@ -35,6 +35,7 @@ import { HardDrive, FileCode2, Plus, + Users, X } from "lucide-react"; import type { GitHubConfig, MirrorOptions, AdvancedOptions, DuplicateNameStrategy } from "@/types/config"; @@ -244,6 +245,26 @@ export function GitHubMirrorSettings({ +
+ handleGitHubChange('includeCollaboratorRepos', !!checked)} + /> +
+ +

+ Also mirror repos where you're a collaborator but not the owner. Turn off to limit imports to repos you own. +

+
+
+
{ includeArchived: envConfig.github.includeArchived ?? existingConfig?.[0]?.githubConfig?.includeArchived ?? false, includePrivate: envConfig.github.privateRepositories ?? existingConfig?.[0]?.githubConfig?.includePrivate ?? false, includePublic: envConfig.github.publicRepositories ?? existingConfig?.[0]?.githubConfig?.includePublic ?? true, + includeCollaboratorRepos: envConfig.github.includeCollaboratorRepos ?? existingConfig?.[0]?.githubConfig?.includeCollaboratorRepos ?? true, includeOrganizations: envConfig.github.mirrorOrganizations ? [] : (existingConfig?.[0]?.githubConfig?.includeOrganizations ?? []), starredReposOrg: envConfig.github.starredReposOrg || existingConfig?.[0]?.githubConfig?.starredReposOrg || 'starred', starredReposMode: envConfig.github.starredReposMode || existingConfig?.[0]?.githubConfig?.starredReposMode || 'dedicated-org', diff --git a/src/lib/github-affiliation.test.ts b/src/lib/github-affiliation.test.ts new file mode 100644 index 0000000..dca5f44 --- /dev/null +++ b/src/lib/github-affiliation.test.ts @@ -0,0 +1,75 @@ +import { describe, expect, test, mock } from "bun:test"; +import { getGithubRepositories } from "@/lib/github"; + +function makeRepo() { + return { + name: "demo", + full_name: "octo/demo", + html_url: "https://github.com/octo/demo", + clone_url: "https://github.com/octo/demo.git", + owner: { login: "octo", type: "User" }, + private: false, + fork: false, + has_issues: true, + archived: false, + size: 1, + language: "TypeScript", + description: "", + default_branch: "main", + visibility: "public", + disabled: false, + created_at: "2024-01-01T00:00:00Z", + updated_at: "2024-01-02T00:00:00Z", + }; +} + +function makeOctokit() { + let captured: Record | null = null; + const paginate = mock(async (_method: unknown, options?: Record) => { + captured = options ?? null; + return [makeRepo()]; + }); + return { + octokit: { + paginate, + repos: { listForAuthenticatedUser: () => {} }, + } as any, + getCaptured: () => captured, + }; +} + +describe("getGithubRepositories - affiliation", () => { + test("defaults to owner+collaborator when field is unset (backward compat)", async () => { + const { octokit, getCaptured } = makeOctokit(); + await getGithubRepositories({ octokit, config: { githubConfig: { owner: "octo" } as any } }); + expect(getCaptured()?.affiliation).toBe("owner,collaborator"); + }); + + test("uses owner only when includeCollaboratorRepos is false", async () => { + const { octokit, getCaptured } = makeOctokit(); + await getGithubRepositories({ + octokit, + config: { githubConfig: { owner: "octo", includeCollaboratorRepos: false } as any }, + }); + expect(getCaptured()?.affiliation).toBe("owner"); + }); + + test("uses owner+collaborator when includeCollaboratorRepos is true", async () => { + const { octokit, getCaptured } = makeOctokit(); + await getGithubRepositories({ + octokit, + config: { githubConfig: { owner: "octo", includeCollaboratorRepos: true } as any }, + }); + expect(getCaptured()?.affiliation).toBe("owner,collaborator"); + }); + + test("override forces owner+collaborator regardless of config (used by cleanup)", async () => { + const { octokit, getCaptured } = makeOctokit(); + await getGithubRepositories({ + octokit, + config: { githubConfig: { owner: "octo", includeCollaboratorRepos: false } as any }, + includeCollaboratorReposOverride: true, + }); + expect(getCaptured()?.affiliation).toBe("owner,collaborator"); + }); +}); diff --git a/src/lib/github.ts b/src/lib/github.ts index dc3db64..680a54e 100644 --- a/src/lib/github.ts +++ b/src/lib/github.ts @@ -235,14 +235,25 @@ export async function getGithubRepoCloneUrl({ export async function getGithubRepositories({ octokit, config, + includeCollaboratorReposOverride, }: { octokit: Octokit; config: Partial; + // Force-include collaborator repos regardless of user setting. Used by the + // cleanup service so we never mark a collab repo as orphaned just because + // the import filter is currently off. + includeCollaboratorReposOverride?: boolean; }): Promise { try { + const includeCollab = + includeCollaboratorReposOverride ?? + config.githubConfig?.includeCollaboratorRepos ?? + true; + const affiliation = includeCollab ? "owner,collaborator" : "owner"; + const repos = await octokit.paginate( octokit.repos.listForAuthenticatedUser, - { per_page: 100 }, + { per_page: 100, affiliation }, ); const skipForks = config.githubConfig?.skipForks ?? false; diff --git a/src/lib/repository-cleanup-service.ts b/src/lib/repository-cleanup-service.ts index df7e6a4..ea5ef92 100644 --- a/src/lib/repository-cleanup-service.ts +++ b/src/lib/repository-cleanup-service.ts @@ -33,9 +33,12 @@ async function identifyOrphanedRepositories(config: any): Promise { let githubApiAccessible = true; try { - // Fetch GitHub data + // Fetch GitHub data. Always include collaborator repos here regardless + // of the user's import filter, otherwise repos previously mirrored as a + // collaborator would be flagged as orphaned and archived/deleted as soon + // as the user disables the filter. const [basicAndForkedRepos, starredRepos] = await Promise.all([ - getGithubRepositories({ octokit, config }), + getGithubRepositories({ octokit, config, includeCollaboratorReposOverride: true }), config.githubConfig?.includeStarred ? getGithubStarredRepositories({ octokit, config }) : Promise.resolve([]), diff --git a/src/lib/utils/config-mapper.test.ts b/src/lib/utils/config-mapper.test.ts index 666785d..cb7de4e 100644 --- a/src/lib/utils/config-mapper.test.ts +++ b/src/lib/utils/config-mapper.test.ts @@ -1,6 +1,53 @@ import { expect, test } from "bun:test"; -import { mapDbScheduleToUi, mapUiScheduleToDb } from "./config-mapper"; -import { scheduleConfigSchema } from "@/lib/db/schema"; +import { + mapDbScheduleToUi, + mapDbToUiConfig, + mapUiScheduleToDb, + mapUiToDbConfig, +} from "./config-mapper"; +import { githubConfigSchema, scheduleConfigSchema } from "@/lib/db/schema"; +import type { + AdvancedOptions, + GitHubConfig, + GiteaConfig, + MirrorOptions, +} from "@/types/config"; + +function buildMinimalUiConfigs(overrides: { includeCollaboratorRepos?: boolean } = {}) { + const githubConfig: GitHubConfig = { + username: "octo", + token: "ghp_x", + privateRepositories: false, + mirrorStarred: false, + ...overrides, + }; + const giteaConfig: GiteaConfig = { + url: "https://gitea.example", + username: "octo", + token: "g_x", + organization: "github-mirrors", + visibility: "public", + starredReposOrg: "starred", + preserveOrgStructure: false, + }; + const mirrorOptions: MirrorOptions = { + mirrorReleases: false, + mirrorLFS: false, + mirrorMetadata: false, + metadataComponents: { + issues: false, + pullRequests: false, + labels: false, + milestones: false, + wiki: false, + }, + }; + const advancedOptions: AdvancedOptions = { + skipForks: false, + starredCodeOnly: false, + }; + return { githubConfig, giteaConfig, mirrorOptions, advancedOptions }; +} test("mapUiScheduleToDb - builds cron from start time + frequency", () => { const existing = scheduleConfigSchema.parse({}); @@ -34,3 +81,46 @@ test("mapDbScheduleToUi - infers clock mode for generated cron", () => { expect(mapped.startTime).toBe("22:15"); expect(mapped.timezone).toBe("Asia/Kolkata"); }); + +test("includeCollaboratorRepos round-trips through UI -> DB -> UI when true", () => { + const ui = buildMinimalUiConfigs({ includeCollaboratorRepos: true }); + const db = mapUiToDbConfig( + ui.githubConfig, + ui.giteaConfig, + ui.mirrorOptions, + ui.advancedOptions, + ); + expect(db.githubConfig.includeCollaboratorRepos).toBe(true); + + const roundTripped = mapDbToUiConfig({ githubConfig: db.githubConfig, giteaConfig: db.giteaConfig }); + expect(roundTripped.githubConfig.includeCollaboratorRepos).toBe(true); +}); + +test("includeCollaboratorRepos round-trips through UI -> DB -> UI when false", () => { + const ui = buildMinimalUiConfigs({ includeCollaboratorRepos: false }); + const db = mapUiToDbConfig( + ui.githubConfig, + ui.giteaConfig, + ui.mirrorOptions, + ui.advancedOptions, + ); + expect(db.githubConfig.includeCollaboratorRepos).toBe(false); + + const roundTripped = mapDbToUiConfig({ githubConfig: db.githubConfig, giteaConfig: db.giteaConfig }); + expect(roundTripped.githubConfig.includeCollaboratorRepos).toBe(false); +}); + +test("DB row missing includeCollaboratorRepos defaults to true on read", () => { + // Existing rows from before this field existed have no value stored. + const ui = mapDbToUiConfig({ githubConfig: { owner: "octo", token: "" } }); + expect(ui.githubConfig.includeCollaboratorRepos).toBe(true); +}); + +test("githubConfigSchema parses includeCollaboratorRepos with true default", () => { + const parsed = githubConfigSchema.parse({ + owner: "octo", + type: "personal", + token: "", + }); + expect(parsed.includeCollaboratorRepos).toBe(true); +}); diff --git a/src/lib/utils/config-mapper.ts b/src/lib/utils/config-mapper.ts index 00b0c56..af8bb53 100644 --- a/src/lib/utils/config-mapper.ts +++ b/src/lib/utils/config-mapper.ts @@ -50,6 +50,7 @@ export function mapUiToDbConfig( // Map checkbox fields with proper names includeStarred: githubConfig.mirrorStarred, includePrivate: githubConfig.privateRepositories, + includeCollaboratorRepos: githubConfig.includeCollaboratorRepos ?? true, includeForks: !advancedOptions.skipForks, // Note: UI has skipForks, DB has includeForks skipForks: advancedOptions.skipForks, // Add skipForks field includeArchived: false, // Not in UI yet, default to false @@ -142,6 +143,7 @@ export function mapDbToUiConfig(dbConfig: any): { username: dbConfig.githubConfig?.owner || "", // Map owner to username token: dbConfig.githubConfig?.token || "", privateRepositories: dbConfig.githubConfig?.includePrivate || false, // Map includePrivate to privateRepositories + includeCollaboratorRepos: dbConfig.githubConfig?.includeCollaboratorRepos ?? true, mirrorStarred: dbConfig.githubConfig?.includeStarred || false, // Map includeStarred to mirrorStarred starredLists: normalizeStarredLists(dbConfig.githubConfig?.starredLists), }; diff --git a/src/types/config.ts b/src/types/config.ts index b9e969b..e552564 100644 --- a/src/types/config.ts +++ b/src/types/config.ts @@ -61,6 +61,7 @@ export interface GitHubConfig { username: string; token: string; privateRepositories: boolean; + includeCollaboratorRepos?: boolean; mirrorStarred: boolean; starredLists?: string[]; starredDuplicateStrategy?: DuplicateNameStrategy;