mirror of
https://github.com/RayLabsHQ/gitea-mirror.git
synced 2026-06-28 14:01:02 +02:00
feat(github): add organization allowlist to mirror only selected orgs (#327)
Repository discovery requested the `organization_member` affiliation unconditionally, so repos from every org a user belongs to were imported — even orgs they never explicitly added. `skipPersonalRepos` only dropped user-owned repos and left org repos unfiltered, which surprised users who expected "only mirror org repos" to mean "only the orgs I chose" (reported on #304). Wire up the previously-dormant `includeOrganizations` config field as an opt-in allowlist: when non-empty, only repos owned by the listed organizations are imported. Empty = all org repos (backward-compatible). Owned and collaborator repos are never restricted, so it composes cleanly with `skipPersonalRepos`. - Filter org repos by the allowlist in getGithubRepositories - Add includeAllOrgsOverride so the cleanup service bypasses the allowlist and never false-orphans a previously-mirrored repo from an org the user later removes from the list - UI control under Filtering & Behavior; INCLUDE_ORGANIZATIONS env var - Case-insensitive dedup/trim in the UI<->DB mapper round-trip - 7 unit tests covering the filter, composition, and the cleanup override
This commit is contained in:
@@ -115,6 +115,7 @@ Standard GitHub Enterprise Cloud on `github.com` works with the default — no o
|
||||
| `MIRROR_ORGANIZATIONS` | Mirror organization repositories | `false` | `true`, `false` |
|
||||
| `PRESERVE_ORG_STRUCTURE` | Preserve GitHub organization structure in Gitea | `false` | `true`, `false` |
|
||||
| `ONLY_MIRROR_ORGS` | Only mirror organization repos (skip personal); sets `skipPersonalRepos: true` in GitHub config | `false` | `true`, `false` |
|
||||
| `INCLUDE_ORGANIZATIONS` | Opt-in allowlist: only mirror repos from these organizations (empty = all orgs you belong to). Sets `includeOrganizations` in GitHub config | - | Comma-separated org names |
|
||||
| `MIRROR_STRATEGY` | Repository organization strategy | `preserve` | `preserve`, `single-org`, `flat-user`, `mixed` |
|
||||
|
||||
### Advanced Settings
|
||||
|
||||
@@ -77,6 +77,7 @@ export function GitHubMirrorSettings({
|
||||
const [starListsOpen, setStarListsOpen] = React.useState(false);
|
||||
const [starListSearch, setStarListSearch] = React.useState("");
|
||||
const [customStarListName, setCustomStarListName] = React.useState("");
|
||||
const [customOrgName, setCustomOrgName] = React.useState("");
|
||||
const [availableStarLists, setAvailableStarLists] = React.useState<string[]>([]);
|
||||
const [loadingStarLists, setLoadingStarLists] = React.useState(false);
|
||||
const [loadedStarLists, setLoadedStarLists] = React.useState(false);
|
||||
@@ -138,6 +139,38 @@ export function GitHubMirrorSettings({
|
||||
});
|
||||
}, [githubConfig, normalizeStarListNames, onGitHubConfigChange]);
|
||||
|
||||
const includedOrgs = React.useMemo(
|
||||
() => githubConfig.includeOrganizations ?? [],
|
||||
[githubConfig.includeOrganizations],
|
||||
);
|
||||
|
||||
const addIncludedOrg = React.useCallback(() => {
|
||||
const trimmed = customOrgName.trim();
|
||||
if (!trimmed) return;
|
||||
const exists = includedOrgs.some(
|
||||
(org) => org.toLowerCase() === trimmed.toLowerCase(),
|
||||
);
|
||||
if (!exists) {
|
||||
onGitHubConfigChange({
|
||||
...githubConfig,
|
||||
includeOrganizations: [...includedOrgs, trimmed],
|
||||
});
|
||||
}
|
||||
setCustomOrgName("");
|
||||
}, [customOrgName, includedOrgs, githubConfig, onGitHubConfigChange]);
|
||||
|
||||
const removeIncludedOrg = React.useCallback(
|
||||
(name: string) => {
|
||||
onGitHubConfigChange({
|
||||
...githubConfig,
|
||||
includeOrganizations: includedOrgs.filter(
|
||||
(org) => org.toLowerCase() !== name.toLowerCase(),
|
||||
),
|
||||
});
|
||||
},
|
||||
[includedOrgs, githubConfig, onGitHubConfigChange],
|
||||
);
|
||||
|
||||
const loadStarLists = React.useCallback(async () => {
|
||||
if (
|
||||
loadingStarLists ||
|
||||
@@ -947,6 +980,64 @@ export function GitHubMirrorSettings({
|
||||
</p>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div className="flex items-start space-x-3">
|
||||
<Users className="h-4 w-4 mt-0.5 text-muted-foreground" />
|
||||
<div className="space-y-2 flex-1">
|
||||
<div className="space-y-0.5">
|
||||
<Label className="text-sm font-normal flex items-center gap-2">
|
||||
Limit to specific organizations
|
||||
</Label>
|
||||
<p className="text-xs text-muted-foreground">
|
||||
Leave empty to mirror repos from every organization you belong to.
|
||||
Add one or more organizations to mirror only their repositories.
|
||||
</p>
|
||||
</div>
|
||||
|
||||
{includedOrgs.length > 0 && (
|
||||
<div className="flex flex-wrap gap-1.5">
|
||||
{includedOrgs.map((org) => (
|
||||
<Badge key={org} variant="secondary" className="gap-1">
|
||||
<span>{org}</span>
|
||||
<button
|
||||
type="button"
|
||||
onClick={() => removeIncludedOrg(org)}
|
||||
className="rounded-sm hover:text-foreground/80"
|
||||
aria-label={`Remove ${org} organization`}
|
||||
>
|
||||
<X className="h-3 w-3" />
|
||||
</button>
|
||||
</Badge>
|
||||
))}
|
||||
</div>
|
||||
)}
|
||||
|
||||
<div className="flex items-center gap-2">
|
||||
<Input
|
||||
value={customOrgName}
|
||||
onChange={(event) => setCustomOrgName(event.target.value)}
|
||||
onKeyDown={(event) => {
|
||||
if (event.key === "Enter") {
|
||||
event.preventDefault();
|
||||
addIncludedOrg();
|
||||
}
|
||||
}}
|
||||
placeholder="Add organization name"
|
||||
className="h-8 text-xs"
|
||||
/>
|
||||
<Button
|
||||
type="button"
|
||||
variant="outline"
|
||||
size="sm"
|
||||
className="h-8"
|
||||
onClick={addIncludedOrg}
|
||||
disabled={!customOrgName.trim()}
|
||||
>
|
||||
Add
|
||||
</Button>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
@@ -20,6 +20,7 @@ interface EnvConfig {
|
||||
skipForks?: boolean;
|
||||
includeArchived?: boolean;
|
||||
mirrorOrganizations?: boolean;
|
||||
includeOrganizations?: string[];
|
||||
preserveOrgStructure?: boolean;
|
||||
onlyMirrorOrgs?: boolean;
|
||||
starredCodeOnly?: boolean;
|
||||
@@ -101,6 +102,9 @@ function parseEnvConfig(): EnvConfig {
|
||||
const protectedRepos = process.env.CLEANUP_PROTECTED_REPOS
|
||||
? process.env.CLEANUP_PROTECTED_REPOS.split(',').map(r => r.trim()).filter(Boolean)
|
||||
: undefined;
|
||||
const includeOrganizations = process.env.INCLUDE_ORGANIZATIONS
|
||||
? process.env.INCLUDE_ORGANIZATIONS.split(',').map((org) => org.trim()).filter(Boolean)
|
||||
: undefined;
|
||||
const starredLists = process.env.MIRROR_STARRED_LISTS
|
||||
? process.env.MIRROR_STARRED_LISTS.split(',').map((list) => list.trim()).filter(Boolean)
|
||||
: undefined;
|
||||
@@ -121,6 +125,7 @@ function parseEnvConfig(): EnvConfig {
|
||||
skipForks: process.env.SKIP_FORKS === 'true',
|
||||
includeArchived: process.env.INCLUDE_ARCHIVED === 'true',
|
||||
mirrorOrganizations: process.env.MIRROR_ORGANIZATIONS === 'true',
|
||||
includeOrganizations,
|
||||
preserveOrgStructure: process.env.PRESERVE_ORG_STRUCTURE === 'true',
|
||||
onlyMirrorOrgs: process.env.ONLY_MIRROR_ORGS === 'true',
|
||||
starredCodeOnly: process.env.SKIP_STARRED_ISSUES === 'true',
|
||||
@@ -277,7 +282,9 @@ export async function initializeConfigFromEnv(): Promise<void> {
|
||||
includePrivate: envConfig.github.privateRepositories ?? existingConfig?.[0]?.githubConfig?.includePrivate ?? false,
|
||||
includePublic: envConfig.github.publicRepositories ?? existingConfig?.[0]?.githubConfig?.includePublic ?? true,
|
||||
includeCollaboratorRepos: envConfig.github.includeCollaboratorRepos ?? existingConfig?.[0]?.githubConfig?.includeCollaboratorRepos ?? true,
|
||||
includeOrganizations: envConfig.github.mirrorOrganizations ? [] : (existingConfig?.[0]?.githubConfig?.includeOrganizations ?? []),
|
||||
// Opt-in org allowlist from INCLUDE_ORGANIZATIONS (comma-separated). Falls
|
||||
// back to existing config so the UI-managed list isn't clobbered on restart.
|
||||
includeOrganizations: envConfig.github.includeOrganizations ?? existingConfig?.[0]?.githubConfig?.includeOrganizations ?? [],
|
||||
starredReposOrg: envConfig.github.starredReposOrg || existingConfig?.[0]?.githubConfig?.starredReposOrg || 'starred',
|
||||
starredReposMode: envConfig.github.starredReposMode || existingConfig?.[0]?.githubConfig?.starredReposMode || 'dedicated-org',
|
||||
mirrorStrategy,
|
||||
|
||||
@@ -164,3 +164,92 @@ describe("getGithubRepositories - skipPersonalRepos", () => {
|
||||
expect(repos.map((r) => r.name)).toContain("org-lib");
|
||||
});
|
||||
});
|
||||
|
||||
describe("getGithubRepositories - includeOrganizations allowlist", () => {
|
||||
const personalRepo = makeRepo({ name: "my-lib", ownerLogin: "octo", ownerType: "User" });
|
||||
const wantedOrgRepo = makeRepo({ name: "wanted", ownerLogin: "wanted-org", ownerType: "Organization" });
|
||||
const otherOrgRepo = makeRepo({ name: "noise", ownerLogin: "noise-org", ownerType: "Organization" });
|
||||
const collabRepo = makeRepo({ name: "collab", ownerLogin: "other-user", ownerType: "User" });
|
||||
|
||||
test("empty allowlist — keeps repos from all orgs (backward compat)", async () => {
|
||||
const { octokit } = makeOctokit([wantedOrgRepo, otherOrgRepo]);
|
||||
const repos = await getGithubRepositories({
|
||||
octokit,
|
||||
config: { githubConfig: { owner: "octo", includeOrganizations: [] } as any },
|
||||
});
|
||||
expect(repos.map((r) => r.name)).toContain("wanted");
|
||||
expect(repos.map((r) => r.name)).toContain("noise");
|
||||
});
|
||||
|
||||
test("non-empty allowlist — keeps only listed orgs, drops other orgs", async () => {
|
||||
const { octokit } = makeOctokit([wantedOrgRepo, otherOrgRepo]);
|
||||
const repos = await getGithubRepositories({
|
||||
octokit,
|
||||
config: { githubConfig: { owner: "octo", includeOrganizations: ["wanted-org"] } as any },
|
||||
});
|
||||
expect(repos.map((r) => r.name)).toContain("wanted");
|
||||
expect(repos.map((r) => r.name)).not.toContain("noise");
|
||||
});
|
||||
|
||||
test("allowlist match is case-insensitive", async () => {
|
||||
const { octokit } = makeOctokit([wantedOrgRepo, otherOrgRepo]);
|
||||
const repos = await getGithubRepositories({
|
||||
octokit,
|
||||
config: { githubConfig: { owner: "octo", includeOrganizations: ["Wanted-Org"] } as any },
|
||||
});
|
||||
expect(repos.map((r) => r.name)).toContain("wanted");
|
||||
expect(repos.map((r) => r.name)).not.toContain("noise");
|
||||
});
|
||||
|
||||
test("allowlist never restricts personal or collaborator repos", async () => {
|
||||
const { octokit } = makeOctokit([personalRepo, collabRepo, wantedOrgRepo, otherOrgRepo]);
|
||||
const repos = await getGithubRepositories({
|
||||
octokit,
|
||||
config: { githubConfig: { owner: "octo", includeOrganizations: ["wanted-org"] } as any },
|
||||
});
|
||||
// User-owned and collaborator repos pass through regardless of the allowlist
|
||||
expect(repos.map((r) => r.name)).toContain("my-lib");
|
||||
expect(repos.map((r) => r.name)).toContain("collab");
|
||||
expect(repos.map((r) => r.name)).toContain("wanted");
|
||||
expect(repos.map((r) => r.name)).not.toContain("noise");
|
||||
});
|
||||
|
||||
test("composes with skipPersonalRepos — drops personal, keeps only listed org", async () => {
|
||||
const { octokit } = makeOctokit([personalRepo, wantedOrgRepo, otherOrgRepo]);
|
||||
const repos = await getGithubRepositories({
|
||||
octokit,
|
||||
config: {
|
||||
githubConfig: {
|
||||
owner: "octo",
|
||||
skipPersonalRepos: true,
|
||||
includeOrganizations: ["wanted-org"],
|
||||
} as any,
|
||||
},
|
||||
});
|
||||
expect(repos.map((r) => r.name)).not.toContain("my-lib");
|
||||
expect(repos.map((r) => r.name)).toContain("wanted");
|
||||
expect(repos.map((r) => r.name)).not.toContain("noise");
|
||||
});
|
||||
|
||||
test("blank/whitespace entries are ignored (treated as empty allowlist)", async () => {
|
||||
const { octokit } = makeOctokit([wantedOrgRepo, otherOrgRepo]);
|
||||
const repos = await getGithubRepositories({
|
||||
octokit,
|
||||
config: { githubConfig: { owner: "octo", includeOrganizations: [" ", ""] } as any },
|
||||
});
|
||||
expect(repos.map((r) => r.name)).toContain("wanted");
|
||||
expect(repos.map((r) => r.name)).toContain("noise");
|
||||
});
|
||||
|
||||
test("includeAllOrgsOverride bypasses allowlist (cleanup safety)", async () => {
|
||||
const { octokit } = makeOctokit([wantedOrgRepo, otherOrgRepo]);
|
||||
const repos = await getGithubRepositories({
|
||||
octokit,
|
||||
config: { githubConfig: { owner: "octo", includeOrganizations: ["wanted-org"] } as any },
|
||||
includeAllOrgsOverride: true,
|
||||
});
|
||||
// Override returns all org repos so cleanup never false-orphans excluded orgs
|
||||
expect(repos.map((r) => r.name)).toContain("wanted");
|
||||
expect(repos.map((r) => r.name)).toContain("noise");
|
||||
});
|
||||
});
|
||||
|
||||
+22
-1
@@ -236,6 +236,7 @@ export async function getGithubRepositories({
|
||||
octokit,
|
||||
config,
|
||||
includeCollaboratorReposOverride,
|
||||
includeAllOrgsOverride,
|
||||
}: {
|
||||
octokit: Octokit;
|
||||
config: Partial<Config>;
|
||||
@@ -243,6 +244,10 @@ export async function getGithubRepositories({
|
||||
// cleanup service so we never mark a collab repo as orphaned just because
|
||||
// the import filter is currently off.
|
||||
includeCollaboratorReposOverride?: boolean;
|
||||
// Bypass the includeOrganizations allowlist so all org repos are returned.
|
||||
// Used by the cleanup service so a previously-mirrored org repo isn't flagged
|
||||
// as orphaned just because the user narrowed the allowlist.
|
||||
includeAllOrgsOverride?: boolean;
|
||||
}): Promise<GitRepo[]> {
|
||||
try {
|
||||
const includeCollab =
|
||||
@@ -266,6 +271,16 @@ export async function getGithubRepositories({
|
||||
const skipPersonalRepos = config.githubConfig?.skipPersonalRepos ?? false;
|
||||
// The authenticated user's login — used to identify personally-owned repos
|
||||
const authenticatedUserLogin = config.githubConfig?.owner ?? "";
|
||||
// Opt-in organization allowlist. When non-empty, only repos owned by the
|
||||
// listed organizations are imported; org repos from any other org the user
|
||||
// happens to be a member of are dropped. Empty = import all org repos
|
||||
// (backward-compatible default). Owned/collaborator repos are unaffected.
|
||||
const includeOrgs = includeAllOrgsOverride
|
||||
? []
|
||||
: config.githubConfig?.includeOrganizations ?? [];
|
||||
const allowedOrgs = new Set(
|
||||
includeOrgs.map((org) => org.trim().toLowerCase()).filter(Boolean),
|
||||
);
|
||||
|
||||
const filteredRepos = repos.filter((repo) => {
|
||||
const isForkAllowed = !skipForks || !repo.fork;
|
||||
@@ -277,7 +292,13 @@ export async function getGithubRepositories({
|
||||
authenticatedUserLogin.length > 0 &&
|
||||
repo.owner.login === authenticatedUserLogin &&
|
||||
repo.owner.type === "User";
|
||||
return isForkAllowed && !isPersonalRepo;
|
||||
// When an allowlist is configured, only keep org repos whose owning org
|
||||
// is listed. Non-org repos (owned/collaborator) are never restricted here.
|
||||
const isOrgAllowed =
|
||||
allowedOrgs.size === 0 ||
|
||||
repo.owner.type !== "Organization" ||
|
||||
allowedOrgs.has(repo.owner.login.toLowerCase());
|
||||
return isForkAllowed && !isPersonalRepo && isOrgAllowed;
|
||||
});
|
||||
|
||||
return filteredRepos.map((repo) => ({
|
||||
|
||||
@@ -33,12 +33,18 @@ async function identifyOrphanedRepositories(config: any): Promise<any[]> {
|
||||
let githubApiAccessible = true;
|
||||
|
||||
try {
|
||||
// Fetch GitHub data. Always include collaborator repos here regardless
|
||||
// of the user's import filter, otherwise repos previously mirrored as a
|
||||
// collaborator would be flagged as orphaned and archived/deleted as soon
|
||||
// as the user disables the filter.
|
||||
// Fetch GitHub data. Always include collaborator repos and bypass the
|
||||
// organization allowlist here regardless of the user's import filters,
|
||||
// otherwise repos previously mirrored as a collaborator or from an org the
|
||||
// user later removed from the allowlist would be flagged as orphaned and
|
||||
// archived/deleted as soon as the user narrows those filters.
|
||||
const [basicAndForkedRepos, starredRepos] = await Promise.all([
|
||||
getGithubRepositories({ octokit, config, includeCollaboratorReposOverride: true }),
|
||||
getGithubRepositories({
|
||||
octokit,
|
||||
config,
|
||||
includeCollaboratorReposOverride: true,
|
||||
includeAllOrgsOverride: true,
|
||||
}),
|
||||
config.githubConfig?.includeStarred
|
||||
? getGithubStarredRepositories({ octokit, config })
|
||||
: Promise.resolve([]),
|
||||
|
||||
@@ -31,6 +31,24 @@ function normalizeStarredLists(lists: string[] | undefined): string[] {
|
||||
return [...deduped];
|
||||
}
|
||||
|
||||
// Trim, drop blanks, and de-duplicate case-insensitively while preserving the
|
||||
// first-seen casing of each organization name.
|
||||
function normalizeOrgList(orgs: string[] | undefined): string[] {
|
||||
if (!Array.isArray(orgs)) return [];
|
||||
const seen = new Set<string>();
|
||||
const result: string[] = [];
|
||||
for (const org of orgs) {
|
||||
if (typeof org !== "string") continue;
|
||||
const trimmed = org.trim();
|
||||
if (!trimmed) continue;
|
||||
const key = trimmed.toLowerCase();
|
||||
if (seen.has(key)) continue;
|
||||
seen.add(key);
|
||||
result.push(trimmed);
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
/**
|
||||
* Maps UI config structure to database schema structure
|
||||
*/
|
||||
@@ -56,8 +74,8 @@ export function mapUiToDbConfig(
|
||||
includeArchived: false, // Not in UI yet, default to false
|
||||
includePublic: true, // Not in UI yet, default to true
|
||||
|
||||
// Organization related fields
|
||||
includeOrganizations: [], // Not in UI yet
|
||||
// Organization related fields — opt-in allowlist (empty = all org repos)
|
||||
includeOrganizations: normalizeOrgList(githubConfig.includeOrganizations),
|
||||
|
||||
// Starred repos organization
|
||||
starredReposOrg: giteaConfig.starredReposOrg,
|
||||
@@ -145,6 +163,7 @@ export function mapDbToUiConfig(dbConfig: any): {
|
||||
token: dbConfig.githubConfig?.token || "",
|
||||
privateRepositories: dbConfig.githubConfig?.includePrivate || false, // Map includePrivate to privateRepositories
|
||||
includeCollaboratorRepos: dbConfig.githubConfig?.includeCollaboratorRepos ?? true,
|
||||
includeOrganizations: normalizeOrgList(dbConfig.githubConfig?.includeOrganizations),
|
||||
mirrorStarred: dbConfig.githubConfig?.includeStarred || false, // Map includeStarred to mirrorStarred
|
||||
starredLists: normalizeStarredLists(dbConfig.githubConfig?.starredLists),
|
||||
};
|
||||
|
||||
@@ -62,6 +62,7 @@ export interface GitHubConfig {
|
||||
token: string;
|
||||
privateRepositories: boolean;
|
||||
includeCollaboratorRepos?: boolean;
|
||||
includeOrganizations?: string[];
|
||||
mirrorStarred: boolean;
|
||||
starredLists?: string[];
|
||||
starredDuplicateStrategy?: DuplicateNameStrategy;
|
||||
|
||||
Reference in New Issue
Block a user