mirror of
https://github.com/RayLabsHQ/gitea-mirror.git
synced 2026-08-07 01:13:38 +02:00
GitHub's listForAuthenticatedUser defaults to returning every repo the user has access to (owner + collaborator + organization_member), which imports a lot of noise for users who only want their own repos. Adds an `includeCollaboratorRepos` toggle, defaulting to true to preserve existing behavior. When disabled, the affiliation filter scopes the API call to "owner" only. The cleanup service overrides the filter to always include collaborator repos when computing the "what's still on GitHub" list. Without this, toggling the option off would mark previously-mirrored collab repos as orphaned and archive/delete them from Gitea. Wired through the schema, both UI<->DB mappers, the env-config loader (with new INCLUDE_COLLABORATOR_REPOS env var), and the settings UI.
This commit is contained in:
@@ -100,6 +100,7 @@ Standard GitHub Enterprise Cloud on `github.com` works with the default — no o
|
||||
| `PRIVATE_REPOSITORIES` | Include private repositories | `false` | `true`, `false` |
|
||||
| `PUBLIC_REPOSITORIES` | Include public repositories | `true` | `true`, `false` |
|
||||
| `INCLUDE_ARCHIVED` | Include archived repositories | `false` | `true`, `false` |
|
||||
| `INCLUDE_COLLABORATOR_REPOS` | Include repositories where you are a collaborator (not just owned). Set to `false` to limit imports to repos you own. | `true` | `true`, `false` |
|
||||
| `SKIP_FORKS` | Skip forked repositories | `false` | `true`, `false` |
|
||||
| `MIRROR_STARRED` | Mirror starred repositories | `false` | `true`, `false` |
|
||||
| `MIRROR_STARRED_LISTS` | Optional comma-separated GitHub Star List names to mirror (only used when `MIRROR_STARRED=true`) | - | Comma-separated list names (empty = all starred repos) |
|
||||
|
||||
@@ -35,6 +35,7 @@ import {
|
||||
HardDrive,
|
||||
FileCode2,
|
||||
Plus,
|
||||
Users,
|
||||
X
|
||||
} from "lucide-react";
|
||||
import type { GitHubConfig, MirrorOptions, AdvancedOptions, DuplicateNameStrategy } from "@/types/config";
|
||||
@@ -244,6 +245,26 @@ export function GitHubMirrorSettings({
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div className="flex items-start space-x-3">
|
||||
<Checkbox
|
||||
id="collaborator-repos"
|
||||
checked={githubConfig.includeCollaboratorRepos ?? true}
|
||||
onCheckedChange={(checked) => handleGitHubChange('includeCollaboratorRepos', !!checked)}
|
||||
/>
|
||||
<div className="space-y-0.5 flex-1">
|
||||
<Label
|
||||
htmlFor="collaborator-repos"
|
||||
className="text-sm font-normal cursor-pointer flex items-center gap-2"
|
||||
>
|
||||
<Users className="h-3.5 w-3.5" />
|
||||
Include collaborator repositories
|
||||
</Label>
|
||||
<p className="text-xs text-muted-foreground">
|
||||
Also mirror repos where you're a collaborator but not the owner. Turn off to limit imports to repos you own.
|
||||
</p>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div className="grid grid-cols-1 md:grid-cols-2 gap-4">
|
||||
<div className="flex items-start space-x-3">
|
||||
<Checkbox
|
||||
|
||||
@@ -23,6 +23,7 @@ export const githubConfigSchema = z.object({
|
||||
includeArchived: z.boolean().default(false),
|
||||
includePrivate: z.boolean().default(true),
|
||||
includePublic: z.boolean().default(true),
|
||||
includeCollaboratorRepos: z.boolean().default(true),
|
||||
includeOrganizations: z.array(z.string()).default([]),
|
||||
starredReposOrg: z.string().optional(),
|
||||
starredReposMode: z.enum(["dedicated-org", "preserve-owner"]).default("dedicated-org"),
|
||||
|
||||
@@ -15,6 +15,7 @@ interface EnvConfig {
|
||||
type?: 'personal' | 'organization';
|
||||
privateRepositories?: boolean;
|
||||
publicRepositories?: boolean;
|
||||
includeCollaboratorRepos?: boolean;
|
||||
mirrorStarred?: boolean;
|
||||
skipForks?: boolean;
|
||||
includeArchived?: boolean;
|
||||
@@ -111,6 +112,11 @@ function parseEnvConfig(): EnvConfig {
|
||||
type: process.env.GITHUB_TYPE as 'personal' | 'organization',
|
||||
privateRepositories: process.env.PRIVATE_REPOSITORIES === 'true',
|
||||
publicRepositories: process.env.PUBLIC_REPOSITORIES === 'true',
|
||||
// Tri-state parse so unset falls through to existingConfig / schema default (true).
|
||||
includeCollaboratorRepos:
|
||||
process.env.INCLUDE_COLLABORATOR_REPOS === 'true' ? true
|
||||
: process.env.INCLUDE_COLLABORATOR_REPOS === 'false' ? false
|
||||
: undefined,
|
||||
mirrorStarred: process.env.MIRROR_STARRED === 'true',
|
||||
skipForks: process.env.SKIP_FORKS === 'true',
|
||||
includeArchived: process.env.INCLUDE_ARCHIVED === 'true',
|
||||
@@ -270,6 +276,7 @@ export async function initializeConfigFromEnv(): Promise<void> {
|
||||
includeArchived: envConfig.github.includeArchived ?? existingConfig?.[0]?.githubConfig?.includeArchived ?? false,
|
||||
includePrivate: envConfig.github.privateRepositories ?? existingConfig?.[0]?.githubConfig?.includePrivate ?? false,
|
||||
includePublic: envConfig.github.publicRepositories ?? existingConfig?.[0]?.githubConfig?.includePublic ?? true,
|
||||
includeCollaboratorRepos: envConfig.github.includeCollaboratorRepos ?? existingConfig?.[0]?.githubConfig?.includeCollaboratorRepos ?? true,
|
||||
includeOrganizations: envConfig.github.mirrorOrganizations ? [] : (existingConfig?.[0]?.githubConfig?.includeOrganizations ?? []),
|
||||
starredReposOrg: envConfig.github.starredReposOrg || existingConfig?.[0]?.githubConfig?.starredReposOrg || 'starred',
|
||||
starredReposMode: envConfig.github.starredReposMode || existingConfig?.[0]?.githubConfig?.starredReposMode || 'dedicated-org',
|
||||
|
||||
@@ -0,0 +1,75 @@
|
||||
import { describe, expect, test, mock } from "bun:test";
|
||||
import { getGithubRepositories } from "@/lib/github";
|
||||
|
||||
function makeRepo() {
|
||||
return {
|
||||
name: "demo",
|
||||
full_name: "octo/demo",
|
||||
html_url: "https://github.com/octo/demo",
|
||||
clone_url: "https://github.com/octo/demo.git",
|
||||
owner: { login: "octo", type: "User" },
|
||||
private: false,
|
||||
fork: false,
|
||||
has_issues: true,
|
||||
archived: false,
|
||||
size: 1,
|
||||
language: "TypeScript",
|
||||
description: "",
|
||||
default_branch: "main",
|
||||
visibility: "public",
|
||||
disabled: false,
|
||||
created_at: "2024-01-01T00:00:00Z",
|
||||
updated_at: "2024-01-02T00:00:00Z",
|
||||
};
|
||||
}
|
||||
|
||||
function makeOctokit() {
|
||||
let captured: Record<string, unknown> | null = null;
|
||||
const paginate = mock(async (_method: unknown, options?: Record<string, unknown>) => {
|
||||
captured = options ?? null;
|
||||
return [makeRepo()];
|
||||
});
|
||||
return {
|
||||
octokit: {
|
||||
paginate,
|
||||
repos: { listForAuthenticatedUser: () => {} },
|
||||
} as any,
|
||||
getCaptured: () => captured,
|
||||
};
|
||||
}
|
||||
|
||||
describe("getGithubRepositories - affiliation", () => {
|
||||
test("defaults to owner+collaborator when field is unset (backward compat)", async () => {
|
||||
const { octokit, getCaptured } = makeOctokit();
|
||||
await getGithubRepositories({ octokit, config: { githubConfig: { owner: "octo" } as any } });
|
||||
expect(getCaptured()?.affiliation).toBe("owner,collaborator");
|
||||
});
|
||||
|
||||
test("uses owner only when includeCollaboratorRepos is false", async () => {
|
||||
const { octokit, getCaptured } = makeOctokit();
|
||||
await getGithubRepositories({
|
||||
octokit,
|
||||
config: { githubConfig: { owner: "octo", includeCollaboratorRepos: false } as any },
|
||||
});
|
||||
expect(getCaptured()?.affiliation).toBe("owner");
|
||||
});
|
||||
|
||||
test("uses owner+collaborator when includeCollaboratorRepos is true", async () => {
|
||||
const { octokit, getCaptured } = makeOctokit();
|
||||
await getGithubRepositories({
|
||||
octokit,
|
||||
config: { githubConfig: { owner: "octo", includeCollaboratorRepos: true } as any },
|
||||
});
|
||||
expect(getCaptured()?.affiliation).toBe("owner,collaborator");
|
||||
});
|
||||
|
||||
test("override forces owner+collaborator regardless of config (used by cleanup)", async () => {
|
||||
const { octokit, getCaptured } = makeOctokit();
|
||||
await getGithubRepositories({
|
||||
octokit,
|
||||
config: { githubConfig: { owner: "octo", includeCollaboratorRepos: false } as any },
|
||||
includeCollaboratorReposOverride: true,
|
||||
});
|
||||
expect(getCaptured()?.affiliation).toBe("owner,collaborator");
|
||||
});
|
||||
});
|
||||
+12
-1
@@ -235,14 +235,25 @@ export async function getGithubRepoCloneUrl({
|
||||
export async function getGithubRepositories({
|
||||
octokit,
|
||||
config,
|
||||
includeCollaboratorReposOverride,
|
||||
}: {
|
||||
octokit: Octokit;
|
||||
config: Partial<Config>;
|
||||
// Force-include collaborator repos regardless of user setting. Used by the
|
||||
// cleanup service so we never mark a collab repo as orphaned just because
|
||||
// the import filter is currently off.
|
||||
includeCollaboratorReposOverride?: boolean;
|
||||
}): Promise<GitRepo[]> {
|
||||
try {
|
||||
const includeCollab =
|
||||
includeCollaboratorReposOverride ??
|
||||
config.githubConfig?.includeCollaboratorRepos ??
|
||||
true;
|
||||
const affiliation = includeCollab ? "owner,collaborator" : "owner";
|
||||
|
||||
const repos = await octokit.paginate(
|
||||
octokit.repos.listForAuthenticatedUser,
|
||||
{ per_page: 100 },
|
||||
{ per_page: 100, affiliation },
|
||||
);
|
||||
|
||||
const skipForks = config.githubConfig?.skipForks ?? false;
|
||||
|
||||
@@ -33,9 +33,12 @@ async function identifyOrphanedRepositories(config: any): Promise<any[]> {
|
||||
let githubApiAccessible = true;
|
||||
|
||||
try {
|
||||
// Fetch GitHub data
|
||||
// Fetch GitHub data. Always include collaborator repos here regardless
|
||||
// of the user's import filter, otherwise repos previously mirrored as a
|
||||
// collaborator would be flagged as orphaned and archived/deleted as soon
|
||||
// as the user disables the filter.
|
||||
const [basicAndForkedRepos, starredRepos] = await Promise.all([
|
||||
getGithubRepositories({ octokit, config }),
|
||||
getGithubRepositories({ octokit, config, includeCollaboratorReposOverride: true }),
|
||||
config.githubConfig?.includeStarred
|
||||
? getGithubStarredRepositories({ octokit, config })
|
||||
: Promise.resolve([]),
|
||||
|
||||
@@ -1,6 +1,53 @@
|
||||
import { expect, test } from "bun:test";
|
||||
import { mapDbScheduleToUi, mapUiScheduleToDb } from "./config-mapper";
|
||||
import { scheduleConfigSchema } from "@/lib/db/schema";
|
||||
import {
|
||||
mapDbScheduleToUi,
|
||||
mapDbToUiConfig,
|
||||
mapUiScheduleToDb,
|
||||
mapUiToDbConfig,
|
||||
} from "./config-mapper";
|
||||
import { githubConfigSchema, scheduleConfigSchema } from "@/lib/db/schema";
|
||||
import type {
|
||||
AdvancedOptions,
|
||||
GitHubConfig,
|
||||
GiteaConfig,
|
||||
MirrorOptions,
|
||||
} from "@/types/config";
|
||||
|
||||
function buildMinimalUiConfigs(overrides: { includeCollaboratorRepos?: boolean } = {}) {
|
||||
const githubConfig: GitHubConfig = {
|
||||
username: "octo",
|
||||
token: "ghp_x",
|
||||
privateRepositories: false,
|
||||
mirrorStarred: false,
|
||||
...overrides,
|
||||
};
|
||||
const giteaConfig: GiteaConfig = {
|
||||
url: "https://gitea.example",
|
||||
username: "octo",
|
||||
token: "g_x",
|
||||
organization: "github-mirrors",
|
||||
visibility: "public",
|
||||
starredReposOrg: "starred",
|
||||
preserveOrgStructure: false,
|
||||
};
|
||||
const mirrorOptions: MirrorOptions = {
|
||||
mirrorReleases: false,
|
||||
mirrorLFS: false,
|
||||
mirrorMetadata: false,
|
||||
metadataComponents: {
|
||||
issues: false,
|
||||
pullRequests: false,
|
||||
labels: false,
|
||||
milestones: false,
|
||||
wiki: false,
|
||||
},
|
||||
};
|
||||
const advancedOptions: AdvancedOptions = {
|
||||
skipForks: false,
|
||||
starredCodeOnly: false,
|
||||
};
|
||||
return { githubConfig, giteaConfig, mirrorOptions, advancedOptions };
|
||||
}
|
||||
|
||||
test("mapUiScheduleToDb - builds cron from start time + frequency", () => {
|
||||
const existing = scheduleConfigSchema.parse({});
|
||||
@@ -34,3 +81,46 @@ test("mapDbScheduleToUi - infers clock mode for generated cron", () => {
|
||||
expect(mapped.startTime).toBe("22:15");
|
||||
expect(mapped.timezone).toBe("Asia/Kolkata");
|
||||
});
|
||||
|
||||
test("includeCollaboratorRepos round-trips through UI -> DB -> UI when true", () => {
|
||||
const ui = buildMinimalUiConfigs({ includeCollaboratorRepos: true });
|
||||
const db = mapUiToDbConfig(
|
||||
ui.githubConfig,
|
||||
ui.giteaConfig,
|
||||
ui.mirrorOptions,
|
||||
ui.advancedOptions,
|
||||
);
|
||||
expect(db.githubConfig.includeCollaboratorRepos).toBe(true);
|
||||
|
||||
const roundTripped = mapDbToUiConfig({ githubConfig: db.githubConfig, giteaConfig: db.giteaConfig });
|
||||
expect(roundTripped.githubConfig.includeCollaboratorRepos).toBe(true);
|
||||
});
|
||||
|
||||
test("includeCollaboratorRepos round-trips through UI -> DB -> UI when false", () => {
|
||||
const ui = buildMinimalUiConfigs({ includeCollaboratorRepos: false });
|
||||
const db = mapUiToDbConfig(
|
||||
ui.githubConfig,
|
||||
ui.giteaConfig,
|
||||
ui.mirrorOptions,
|
||||
ui.advancedOptions,
|
||||
);
|
||||
expect(db.githubConfig.includeCollaboratorRepos).toBe(false);
|
||||
|
||||
const roundTripped = mapDbToUiConfig({ githubConfig: db.githubConfig, giteaConfig: db.giteaConfig });
|
||||
expect(roundTripped.githubConfig.includeCollaboratorRepos).toBe(false);
|
||||
});
|
||||
|
||||
test("DB row missing includeCollaboratorRepos defaults to true on read", () => {
|
||||
// Existing rows from before this field existed have no value stored.
|
||||
const ui = mapDbToUiConfig({ githubConfig: { owner: "octo", token: "" } });
|
||||
expect(ui.githubConfig.includeCollaboratorRepos).toBe(true);
|
||||
});
|
||||
|
||||
test("githubConfigSchema parses includeCollaboratorRepos with true default", () => {
|
||||
const parsed = githubConfigSchema.parse({
|
||||
owner: "octo",
|
||||
type: "personal",
|
||||
token: "",
|
||||
});
|
||||
expect(parsed.includeCollaboratorRepos).toBe(true);
|
||||
});
|
||||
|
||||
@@ -50,6 +50,7 @@ export function mapUiToDbConfig(
|
||||
// Map checkbox fields with proper names
|
||||
includeStarred: githubConfig.mirrorStarred,
|
||||
includePrivate: githubConfig.privateRepositories,
|
||||
includeCollaboratorRepos: githubConfig.includeCollaboratorRepos ?? true,
|
||||
includeForks: !advancedOptions.skipForks, // Note: UI has skipForks, DB has includeForks
|
||||
skipForks: advancedOptions.skipForks, // Add skipForks field
|
||||
includeArchived: false, // Not in UI yet, default to false
|
||||
@@ -142,6 +143,7 @@ export function mapDbToUiConfig(dbConfig: any): {
|
||||
username: dbConfig.githubConfig?.owner || "", // Map owner to username
|
||||
token: dbConfig.githubConfig?.token || "",
|
||||
privateRepositories: dbConfig.githubConfig?.includePrivate || false, // Map includePrivate to privateRepositories
|
||||
includeCollaboratorRepos: dbConfig.githubConfig?.includeCollaboratorRepos ?? true,
|
||||
mirrorStarred: dbConfig.githubConfig?.includeStarred || false, // Map includeStarred to mirrorStarred
|
||||
starredLists: normalizeStarredLists(dbConfig.githubConfig?.starredLists),
|
||||
};
|
||||
|
||||
@@ -61,6 +61,7 @@ export interface GitHubConfig {
|
||||
username: string;
|
||||
token: string;
|
||||
privateRepositories: boolean;
|
||||
includeCollaboratorRepos?: boolean;
|
||||
mirrorStarred: boolean;
|
||||
starredLists?: string[];
|
||||
starredDuplicateStrategy?: DuplicateNameStrategy;
|
||||
|
||||
Reference in New Issue
Block a user