diff --git a/README.md b/README.md index f539f97..4269b23 100644 --- a/README.md +++ b/README.md @@ -31,6 +31,7 @@ First user signup becomes admin. Configure GitHub and Gitea/Forgejo through the ## ✨ Features - 🔁 Mirror public, private, and starred GitHub repos to Gitea/Forgejo +- 🦊 **GitLab (beta) and Gitea/Forgejo sources** - Pick the source in the configuration card. gitlab.com, Codeberg and self hosted instances mirror code, tags, wiki and LFS (issues, pull requests and releases need a GitHub source). See [docs/SOURCE_PROVIDERS.md](docs/SOURCE_PROVIDERS.md) - 🏛️ **GitHub Enterprise support** - Works with GHES and GHEC with data residency via `GH_API_URL` - 🏢 Mirror entire organizations with flexible strategies - 🎯 Custom destination control for repos and organizations diff --git a/docs/ENVIRONMENT_VARIABLES.md b/docs/ENVIRONMENT_VARIABLES.md index aa42f77..d6a92d9 100644 --- a/docs/ENVIRONMENT_VARIABLES.md +++ b/docs/ENVIRONMENT_VARIABLES.md @@ -69,17 +69,38 @@ Notes: ## GitHub Configuration -Settings for connecting to and configuring GitHub repository sources. +Settings for connecting to the source host. GitHub is the default. GitLab and Gitea/Forgejo (including Codeberg) are selected with `SOURCE_PROVIDER`; the username and token variables below then hold the account and token for that host. See [SOURCE_PROVIDERS.md](SOURCE_PROVIDERS.md) for what each source supports. ### Basic Settings | Variable | Description | Default | Options | |----------|-------------|---------|---------| -| `GITHUB_USERNAME` | Your GitHub username | - | - | -| `GITHUB_TOKEN` | GitHub personal access token (requires repo and admin:org scopes) | - | - | +| `SOURCE_PROVIDER` | Which host repositories are pulled from. `gitea` also covers Forgejo and Codeberg. | `github` | `github`, `gitlab`, `gitea` | +| `SOURCE_URL` | Base URL of the GitLab or Gitea/Forgejo instance. Ignored for GitHub, which uses `GH_API_URL`. | `https://gitlab.com` for GitLab, `https://codeberg.org` for Gitea | e.g. `https://gitlab.example.com` | +| `GITHUB_USERNAME` | Your username on the source host | - | - | +| `GITHUB_TOKEN` | Personal access token for the source host. GitHub needs the `repo` and `admin:org` scopes, GitLab `read_api` and `read_repository`, Gitea/Forgejo `read:repository`, `read:user` and `read:organization`. | - | - | | `GITHUB_TYPE` | GitHub account type | `personal` | `personal`, `organization` | | `GH_API_URL` | GitHub API base URL. Override this to point at GitHub Enterprise Server or Enterprise Cloud with data residency. | `https://api.github.com` | e.g. `https://ghe.example.com/api/v3`, `https://api.TENANT.ghe.com` | +### GitLab and Gitea/Forgejo sources + +```bash +# Mirror from a self hosted GitLab +SOURCE_PROVIDER=gitlab +SOURCE_URL=https://gitlab.example.com +GITHUB_USERNAME=my-gitlab-user +GITHUB_TOKEN=glpat-... + +# Mirror from Codeberg (Forgejo) +SOURCE_PROVIDER=gitea +GITHUB_USERNAME=my-codeberg-user +GITHUB_TOKEN=... +``` + +Code, tags, wiki and LFS are mirrored from every source. Issues, pull requests, releases, labels, milestones and star lists need a GitHub source and are skipped for the others. + +Once repositories have been imported the source is locked, and once anything has been mirrored the Gitea server URL is locked. A `SOURCE_PROVIDER`, `SOURCE_URL` or `GITEA_URL` value that disagrees with a locked host is ignored on boot with a warning; change the host on the Configuration page, where the change asks for confirmation. + ### GitHub Enterprise (GHES / GHEC with data residency) Set `GH_API_URL` to point Octokit at a non-`github.com` API endpoint: @@ -134,6 +155,7 @@ Settings for the destination Gitea instance. | Variable | Description | Default | Options | |----------|-------------|---------|---------| +| `DESTINATION_PROVIDER` | Whether the destination is Gitea or Forgejo. Same API; changes labels and hints only. | `gitea` | `gitea`, `forgejo` | | `GITEA_URL` | Gitea instance URL | - | Valid URL | | `GITEA_EXTERNAL_URL` | Optional external/browser URL used for dashboard links. API and mirroring still use `GITEA_URL`. | - | Valid URL | | `GITEA_TOKEN` | Gitea access token | - | - | diff --git a/docs/SOURCE_PROVIDERS.md b/docs/SOURCE_PROVIDERS.md new file mode 100644 index 0000000..70e63be --- /dev/null +++ b/docs/SOURCE_PROVIDERS.md @@ -0,0 +1,69 @@ +# Source Providers + +The destination card has a matching **Destination** dropdown with Gitea and Forgejo. They share the same API, so it only changes the name, icon and hints shown on the card (`DESTINATION_PROVIDER` sets it from the environment). + +Gitea Mirror pulls repositories from one source host per user. GitHub is the default. GitLab and Gitea/Forgejo are available from the **Source** dropdown at the top of the connection card on the Configuration page. + +| Source | Covers | Default instance URL | +|--------|--------|----------------------| +| GitHub | github.com, GitHub Enterprise (via `GH_API_URL`) | `https://github.com` | +| GitLab (beta) | gitlab.com, self hosted GitLab | `https://gitlab.com` | +| Gitea / Forgejo | Codeberg, self hosted Gitea or Forgejo | `https://codeberg.org` | + +GitLab support is marked beta: the adapter is tested against the public API and mocked responses, not yet against a private self hosted instance. Picking GitLab or Gitea/Forgejo shows an **Instance URL** field. Leave it empty for the default instance, or enter the base URL of your own (for example `https://gitlab.example.com` or `http://gitea.local:3000/gitea`). The username and token fields hold the account and token for the selected host. + +## Tokens + +| Source | Where to create the token | Scopes | +|--------|---------------------------|--------| +| GitHub | Settings, Developer settings, Personal access tokens (classic) | `repo`, `admin:org` | +| GitLab | Preferences, Access tokens | `read_api`, `read_repository` | +| Gitea / Forgejo | Settings, Applications | `read:repository`, `read:user`, `read:organization` | + +The **Test** button on the card checks the token against the selected host. + +## What is mirrored + +The mirror itself is done by Gitea's pull mirror, which treats every source as a plain git remote. The differences are in what the source API offers. + +| | GitHub | GitLab | Gitea / Forgejo | +|---|:---:|:---:|:---:| +| Code, branches, tags | yes | yes | yes | +| Wiki | yes | yes | yes | +| LFS objects | yes | yes | yes | +| Scheduled sync | yes | yes | yes | +| Auto import of new repositories | yes | yes | yes | +| Cleanup of repositories deleted upstream | yes | yes | yes | +| Starred repositories | yes | yes | yes | +| Private repositories | yes | yes | yes | +| Add a single repository by URL | yes | yes | yes | +| Issues, pull requests, labels, milestones | yes | no | no | +| Releases with assets | yes | no | no | +| Star lists | yes | no | no | +| Force push detection | yes | no | no | + +The GitHub only rows read the GitHub API. For other sources the corresponding switches are disabled on the Configuration page, and the mirror step skips them. + +## Behaviour to know about + +- **The source locks once repositories are imported, and the destination locks once anything is mirrored.** The Source dropdown, the instance URL and the Gitea server URL then show a lock note and a **Change** button. Changing one is still possible, but only after confirming a dialog that spells out what happens to the existing repositories. Saves that try to switch a locked host without that confirmation are refused by the API, and an environment variable that disagrees with a locked host is ignored on boot with a warning. +- **One source at a time.** Switching the source does not touch repositories that were imported from the previous host. They keep syncing, because Gitea already holds their clone credentials. The cleanup service ignores them, and mirroring one of them again is refused with an error naming both hosts. Remove the repository and add it again from the current source if you need to re-mirror it. +- **GitLab groups are flattened.** A project under `group/subgroup/project` lands in the Gitea organization `group` (with the preserve strategy) and keeps its full path as the repository's full name. The "Limit to specific groups" filter matches the top level group. +- **GitLab internal projects are treated as private** when the mirror is created, because they are not visible to anonymous users on the source either. +- **Repository names come from the URL path**, not the display name, so a GitLab project called "My Widget" with path `my-widget` mirrors as `my-widget`. +- **Without a token** only the configured user's public repositories and starred repositories are visible on GitLab and Gitea/Forgejo. Adding a public repository by URL works without a token on every source. + +## Environment variables + +| Variable | Description | +|----------|-------------| +| `SOURCE_PROVIDER` | `github` (default), `gitlab` or `gitea` | +| `SOURCE_URL` | Instance base URL for GitLab and Gitea/Forgejo | +| `GITHUB_USERNAME` | Username on the selected host | +| `GITHUB_TOKEN` | Token for the selected host | + +See [ENVIRONMENT_VARIABLES.md](ENVIRONMENT_VARIABLES.md) for the full list. + +## Not yet + +Several sources at once (the tracking issue is #375, the follow up is a list of sources in the card), targets other than Gitea/Forgejo (see the `feature/matrix` branch), and hosts without a usable API. diff --git a/drizzle/0015_source_provider.sql b/drizzle/0015_source_provider.sql new file mode 100644 index 0000000..10369c7 --- /dev/null +++ b/drizzle/0015_source_provider.sql @@ -0,0 +1,2 @@ +ALTER TABLE `repositories` ADD `source_provider` text DEFAULT 'github' NOT NULL;--> statement-breakpoint +ALTER TABLE `repositories` ADD `source_url` text DEFAULT 'https://github.com' NOT NULL; \ No newline at end of file diff --git a/drizzle/meta/0015_snapshot.json b/drizzle/meta/0015_snapshot.json new file mode 100644 index 0000000..92b9146 --- /dev/null +++ b/drizzle/meta/0015_snapshot.json @@ -0,0 +1,2405 @@ +{ + "version": "6", + "dialect": "sqlite", + "id": "03397b55-257d-46bc-b099-c2839002763d", + "prevId": "367fa60e-8b76-4b55-8be8-a41a099a5c3c", + "tables": { + "accounts": { + "name": "accounts", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "account_id": { + "name": "account_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "provider_id": { + "name": "provider_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "provider_user_id": { + "name": "provider_user_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "access_token": { + "name": "access_token", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "refresh_token": { + "name": "refresh_token", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "id_token": { + "name": "id_token", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "access_token_expires_at": { + "name": "access_token_expires_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "refresh_token_expires_at": { + "name": "refresh_token_expires_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "scope": { + "name": "scope", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "expires_at": { + "name": "expires_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "password": { + "name": "password", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch())" + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch())" + } + }, + "indexes": { + "idx_accounts_account_id": { + "name": "idx_accounts_account_id", + "columns": [ + "account_id" + ], + "isUnique": false + }, + "idx_accounts_user_id": { + "name": "idx_accounts_user_id", + "columns": [ + "user_id" + ], + "isUnique": false + }, + "idx_accounts_provider": { + "name": "idx_accounts_provider", + "columns": [ + "provider_id", + "provider_user_id" + ], + "isUnique": false + } + }, + "foreignKeys": { + "accounts_user_id_users_id_fk": { + "name": "accounts_user_id_users_id_fk", + "tableFrom": "accounts", + "tableTo": "users", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "configs": { + "name": "configs", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "is_active": { + "name": "is_active", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": true + }, + "github_config": { + "name": "github_config", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "gitea_config": { + "name": "gitea_config", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "include": { + "name": "include", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'[\"*\"]'" + }, + "exclude": { + "name": "exclude", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'[]'" + }, + "schedule_config": { + "name": "schedule_config", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "cleanup_config": { + "name": "cleanup_config", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "notification_config": { + "name": "notification_config", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'{\"enabled\":false,\"provider\":\"ntfy\",\"notifyOnSyncError\":true,\"notifyOnSyncSuccess\":false,\"notifyOnNewRepo\":false}'" + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch())" + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch())" + } + }, + "indexes": {}, + "foreignKeys": { + "configs_user_id_users_id_fk": { + "name": "configs_user_id_users_id_fk", + "tableFrom": "configs", + "tableTo": "users", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "events": { + "name": "events", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "channel": { + "name": "channel", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "payload": { + "name": "payload", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "read": { + "name": "read", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch())" + } + }, + "indexes": { + "idx_events_user_channel": { + "name": "idx_events_user_channel", + "columns": [ + "user_id", + "channel" + ], + "isUnique": false + }, + "idx_events_created_at": { + "name": "idx_events_created_at", + "columns": [ + "created_at" + ], + "isUnique": false + }, + "idx_events_read": { + "name": "idx_events_read", + "columns": [ + "read" + ], + "isUnique": false + } + }, + "foreignKeys": { + "events_user_id_users_id_fk": { + "name": "events_user_id_users_id_fk", + "tableFrom": "events", + "tableTo": "users", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "jwks": { + "name": "jwks", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "public_key": { + "name": "public_key", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "private_key": { + "name": "private_key", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch())" + }, + "expires_at": { + "name": "expires_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "mirror_jobs": { + "name": "mirror_jobs", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "repository_id": { + "name": "repository_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "repository_name": { + "name": "repository_name", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "organization_id": { + "name": "organization_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "organization_name": { + "name": "organization_name", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "details": { + "name": "details", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'imported'" + }, + "message": { + "name": "message", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "timestamp": { + "name": "timestamp", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch())" + }, + "job_type": { + "name": "job_type", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'mirror'" + }, + "batch_id": { + "name": "batch_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "total_items": { + "name": "total_items", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "completed_items": { + "name": "completed_items", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": 0 + }, + "item_ids": { + "name": "item_ids", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "completed_item_ids": { + "name": "completed_item_ids", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": "'[]'" + }, + "in_progress": { + "name": "in_progress", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": false + }, + "started_at": { + "name": "started_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "completed_at": { + "name": "completed_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "last_checkpoint": { + "name": "last_checkpoint", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": { + "idx_mirror_jobs_user_id": { + "name": "idx_mirror_jobs_user_id", + "columns": [ + "user_id" + ], + "isUnique": false + }, + "idx_mirror_jobs_batch_id": { + "name": "idx_mirror_jobs_batch_id", + "columns": [ + "batch_id" + ], + "isUnique": false + }, + "idx_mirror_jobs_in_progress": { + "name": "idx_mirror_jobs_in_progress", + "columns": [ + "in_progress" + ], + "isUnique": false + }, + "idx_mirror_jobs_job_type": { + "name": "idx_mirror_jobs_job_type", + "columns": [ + "job_type" + ], + "isUnique": false + }, + "idx_mirror_jobs_timestamp": { + "name": "idx_mirror_jobs_timestamp", + "columns": [ + "timestamp" + ], + "isUnique": false + } + }, + "foreignKeys": { + "mirror_jobs_user_id_users_id_fk": { + "name": "mirror_jobs_user_id_users_id_fk", + "tableFrom": "mirror_jobs", + "tableTo": "users", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "oauth_access_tokens": { + "name": "oauth_access_tokens", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "token": { + "name": "token", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "client_id": { + "name": "client_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "session_id": { + "name": "session_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "reference_id": { + "name": "reference_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "refresh_id": { + "name": "refresh_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "expires_at": { + "name": "expires_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": "(unixepoch())" + }, + "scopes": { + "name": "scopes", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "oauth_access_tokens_token_unique": { + "name": "oauth_access_tokens_token_unique", + "columns": [ + "token" + ], + "isUnique": true + }, + "idx_oauth_access_tokens_token": { + "name": "idx_oauth_access_tokens_token", + "columns": [ + "token" + ], + "isUnique": false + }, + "idx_oauth_access_tokens_client_id": { + "name": "idx_oauth_access_tokens_client_id", + "columns": [ + "client_id" + ], + "isUnique": false + }, + "idx_oauth_access_tokens_user_id": { + "name": "idx_oauth_access_tokens_user_id", + "columns": [ + "user_id" + ], + "isUnique": false + } + }, + "foreignKeys": { + "oauth_access_tokens_user_id_users_id_fk": { + "name": "oauth_access_tokens_user_id_users_id_fk", + "tableFrom": "oauth_access_tokens", + "tableTo": "users", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "oauth_clients": { + "name": "oauth_clients", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "client_id": { + "name": "client_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "client_secret": { + "name": "client_secret", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "disabled": { + "name": "disabled", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": false + }, + "skip_consent": { + "name": "skip_consent", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "enable_end_session": { + "name": "enable_end_session", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "subject_type": { + "name": "subject_type", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "scopes": { + "name": "scopes", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "uri": { + "name": "uri", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "icon": { + "name": "icon", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "contacts": { + "name": "contacts", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "tos": { + "name": "tos", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "policy": { + "name": "policy", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "software_id": { + "name": "software_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "software_version": { + "name": "software_version", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "software_statement": { + "name": "software_statement", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "redirect_uris": { + "name": "redirect_uris", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "post_logout_redirect_uris": { + "name": "post_logout_redirect_uris", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "token_endpoint_auth_method": { + "name": "token_endpoint_auth_method", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "grant_types": { + "name": "grant_types", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "response_types": { + "name": "response_types", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "public": { + "name": "public", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "type": { + "name": "type", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "require_pkce": { + "name": "require_pkce", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "reference_id": { + "name": "reference_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "metadata": { + "name": "metadata", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": "(unixepoch())" + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": "(unixepoch())" + } + }, + "indexes": { + "oauth_clients_client_id_unique": { + "name": "oauth_clients_client_id_unique", + "columns": [ + "client_id" + ], + "isUnique": true + }, + "idx_oauth_clients_client_id": { + "name": "idx_oauth_clients_client_id", + "columns": [ + "client_id" + ], + "isUnique": false + }, + "idx_oauth_clients_user_id": { + "name": "idx_oauth_clients_user_id", + "columns": [ + "user_id" + ], + "isUnique": false + } + }, + "foreignKeys": { + "oauth_clients_user_id_users_id_fk": { + "name": "oauth_clients_user_id_users_id_fk", + "tableFrom": "oauth_clients", + "tableTo": "users", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "oauth_consents": { + "name": "oauth_consents", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "client_id": { + "name": "client_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "reference_id": { + "name": "reference_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "scopes": { + "name": "scopes", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": "(unixepoch())" + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": "(unixepoch())" + } + }, + "indexes": { + "idx_oauth_consents_client_id": { + "name": "idx_oauth_consents_client_id", + "columns": [ + "client_id" + ], + "isUnique": false + }, + "idx_oauth_consents_user_id": { + "name": "idx_oauth_consents_user_id", + "columns": [ + "user_id" + ], + "isUnique": false + } + }, + "foreignKeys": { + "oauth_consents_user_id_users_id_fk": { + "name": "oauth_consents_user_id_users_id_fk", + "tableFrom": "oauth_consents", + "tableTo": "users", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "oauth_refresh_tokens": { + "name": "oauth_refresh_tokens", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "token": { + "name": "token", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "client_id": { + "name": "client_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "session_id": { + "name": "session_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "reference_id": { + "name": "reference_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "expires_at": { + "name": "expires_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": "(unixepoch())" + }, + "revoked": { + "name": "revoked", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "auth_time": { + "name": "auth_time", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "scopes": { + "name": "scopes", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "oauth_refresh_tokens_token_unique": { + "name": "oauth_refresh_tokens_token_unique", + "columns": [ + "token" + ], + "isUnique": true + }, + "idx_oauth_refresh_tokens_token": { + "name": "idx_oauth_refresh_tokens_token", + "columns": [ + "token" + ], + "isUnique": false + }, + "idx_oauth_refresh_tokens_client_id": { + "name": "idx_oauth_refresh_tokens_client_id", + "columns": [ + "client_id" + ], + "isUnique": false + }, + "idx_oauth_refresh_tokens_user_id": { + "name": "idx_oauth_refresh_tokens_user_id", + "columns": [ + "user_id" + ], + "isUnique": false + } + }, + "foreignKeys": { + "oauth_refresh_tokens_user_id_users_id_fk": { + "name": "oauth_refresh_tokens_user_id_users_id_fk", + "tableFrom": "oauth_refresh_tokens", + "tableTo": "users", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "organizations": { + "name": "organizations", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "config_id": { + "name": "config_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "normalized_name": { + "name": "normalized_name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "avatar_url": { + "name": "avatar_url", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "membership_role": { + "name": "membership_role", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'member'" + }, + "is_included": { + "name": "is_included", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": true + }, + "destination_org": { + "name": "destination_org", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "mirror_overrides": { + "name": "mirror_overrides", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'imported'" + }, + "last_mirrored": { + "name": "last_mirrored", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "error_message": { + "name": "error_message", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "repository_count": { + "name": "repository_count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "public_repository_count": { + "name": "public_repository_count", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "private_repository_count": { + "name": "private_repository_count", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "fork_repository_count": { + "name": "fork_repository_count", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch())" + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch())" + } + }, + "indexes": { + "idx_organizations_user_id": { + "name": "idx_organizations_user_id", + "columns": [ + "user_id" + ], + "isUnique": false + }, + "idx_organizations_config_id": { + "name": "idx_organizations_config_id", + "columns": [ + "config_id" + ], + "isUnique": false + }, + "idx_organizations_status": { + "name": "idx_organizations_status", + "columns": [ + "status" + ], + "isUnique": false + }, + "idx_organizations_is_included": { + "name": "idx_organizations_is_included", + "columns": [ + "is_included" + ], + "isUnique": false + }, + "uniq_organizations_user_normalized_name": { + "name": "uniq_organizations_user_normalized_name", + "columns": [ + "user_id", + "normalized_name" + ], + "isUnique": true + } + }, + "foreignKeys": { + "organizations_user_id_users_id_fk": { + "name": "organizations_user_id_users_id_fk", + "tableFrom": "organizations", + "tableTo": "users", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + }, + "organizations_config_id_configs_id_fk": { + "name": "organizations_config_id_configs_id_fk", + "tableFrom": "organizations", + "tableTo": "configs", + "columnsFrom": [ + "config_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "rate_limits": { + "name": "rate_limits", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "provider": { + "name": "provider", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'github'" + }, + "limit": { + "name": "limit", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "remaining": { + "name": "remaining", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "used": { + "name": "used", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "reset": { + "name": "reset", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "retry_after": { + "name": "retry_after", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'ok'" + }, + "last_checked": { + "name": "last_checked", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch())" + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch())" + } + }, + "indexes": { + "idx_rate_limits_user_provider": { + "name": "idx_rate_limits_user_provider", + "columns": [ + "user_id", + "provider" + ], + "isUnique": false + }, + "idx_rate_limits_status": { + "name": "idx_rate_limits_status", + "columns": [ + "status" + ], + "isUnique": false + } + }, + "foreignKeys": { + "rate_limits_user_id_users_id_fk": { + "name": "rate_limits_user_id_users_id_fk", + "tableFrom": "rate_limits", + "tableTo": "users", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "repositories": { + "name": "repositories", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "config_id": { + "name": "config_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "full_name": { + "name": "full_name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "normalized_full_name": { + "name": "normalized_full_name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "url": { + "name": "url", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "clone_url": { + "name": "clone_url", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "source_provider": { + "name": "source_provider", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'github'" + }, + "source_url": { + "name": "source_url", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'https://github.com'" + }, + "owner": { + "name": "owner", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "organization": { + "name": "organization", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "mirrored_location": { + "name": "mirrored_location", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": "''" + }, + "is_private": { + "name": "is_private", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": false + }, + "is_fork": { + "name": "is_fork", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": false + }, + "forked_from": { + "name": "forked_from", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "has_issues": { + "name": "has_issues", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": false + }, + "is_starred": { + "name": "is_starred", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": false + }, + "is_archived": { + "name": "is_archived", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": false + }, + "size": { + "name": "size", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "has_lfs": { + "name": "has_lfs", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": false + }, + "has_submodules": { + "name": "has_submodules", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": false + }, + "language": { + "name": "language", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "default_branch": { + "name": "default_branch", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "visibility": { + "name": "visibility", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'public'" + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'imported'" + }, + "last_mirrored": { + "name": "last_mirrored", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "error_message": { + "name": "error_message", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "destination_org": { + "name": "destination_org", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "mirror_overrides": { + "name": "mirror_overrides", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "metadata": { + "name": "metadata", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "imported_at": { + "name": "imported_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch())" + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch())" + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch())" + } + }, + "indexes": { + "idx_repositories_user_id": { + "name": "idx_repositories_user_id", + "columns": [ + "user_id" + ], + "isUnique": false + }, + "idx_repositories_config_id": { + "name": "idx_repositories_config_id", + "columns": [ + "config_id" + ], + "isUnique": false + }, + "idx_repositories_status": { + "name": "idx_repositories_status", + "columns": [ + "status" + ], + "isUnique": false + }, + "idx_repositories_owner": { + "name": "idx_repositories_owner", + "columns": [ + "owner" + ], + "isUnique": false + }, + "idx_repositories_organization": { + "name": "idx_repositories_organization", + "columns": [ + "organization" + ], + "isUnique": false + }, + "idx_repositories_is_fork": { + "name": "idx_repositories_is_fork", + "columns": [ + "is_fork" + ], + "isUnique": false + }, + "idx_repositories_is_starred": { + "name": "idx_repositories_is_starred", + "columns": [ + "is_starred" + ], + "isUnique": false + }, + "idx_repositories_user_imported_at": { + "name": "idx_repositories_user_imported_at", + "columns": [ + "user_id", + "imported_at" + ], + "isUnique": false + }, + "uniq_repositories_user_full_name": { + "name": "uniq_repositories_user_full_name", + "columns": [ + "user_id", + "full_name" + ], + "isUnique": true + }, + "uniq_repositories_user_normalized_full_name": { + "name": "uniq_repositories_user_normalized_full_name", + "columns": [ + "user_id", + "normalized_full_name" + ], + "isUnique": true + }, + "idx_repositories_mirrored_location": { + "name": "idx_repositories_mirrored_location", + "columns": [ + "user_id", + "mirrored_location" + ], + "isUnique": false + } + }, + "foreignKeys": { + "repositories_user_id_users_id_fk": { + "name": "repositories_user_id_users_id_fk", + "tableFrom": "repositories", + "tableTo": "users", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + }, + "repositories_config_id_configs_id_fk": { + "name": "repositories_config_id_configs_id_fk", + "tableFrom": "repositories", + "tableTo": "configs", + "columnsFrom": [ + "config_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "sessions": { + "name": "sessions", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "token": { + "name": "token", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "expires_at": { + "name": "expires_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "ip_address": { + "name": "ip_address", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "user_agent": { + "name": "user_agent", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch())" + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch())" + } + }, + "indexes": { + "sessions_token_unique": { + "name": "sessions_token_unique", + "columns": [ + "token" + ], + "isUnique": true + }, + "idx_sessions_user_id": { + "name": "idx_sessions_user_id", + "columns": [ + "user_id" + ], + "isUnique": false + }, + "idx_sessions_token": { + "name": "idx_sessions_token", + "columns": [ + "token" + ], + "isUnique": false + }, + "idx_sessions_expires_at": { + "name": "idx_sessions_expires_at", + "columns": [ + "expires_at" + ], + "isUnique": false + } + }, + "foreignKeys": { + "sessions_user_id_users_id_fk": { + "name": "sessions_user_id_users_id_fk", + "tableFrom": "sessions", + "tableTo": "users", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "sso_providers": { + "name": "sso_providers", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "issuer": { + "name": "issuer", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "domain": { + "name": "domain", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "oidc_config": { + "name": "oidc_config", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "saml_config": { + "name": "saml_config", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "domain_verified": { + "name": "domain_verified", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "provider_id": { + "name": "provider_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "organization_id": { + "name": "organization_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch())" + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch())" + } + }, + "indexes": { + "sso_providers_provider_id_unique": { + "name": "sso_providers_provider_id_unique", + "columns": [ + "provider_id" + ], + "isUnique": true + }, + "idx_sso_providers_provider_id": { + "name": "idx_sso_providers_provider_id", + "columns": [ + "provider_id" + ], + "isUnique": false + }, + "idx_sso_providers_domain": { + "name": "idx_sso_providers_domain", + "columns": [ + "domain" + ], + "isUnique": false + }, + "idx_sso_providers_issuer": { + "name": "idx_sso_providers_issuer", + "columns": [ + "issuer" + ], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "users": { + "name": "users", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "email": { + "name": "email", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "email_verified": { + "name": "email_verified", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": false + }, + "image": { + "name": "image", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch())" + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch())" + }, + "username": { + "name": "username", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": { + "users_email_unique": { + "name": "users_email_unique", + "columns": [ + "email" + ], + "isUnique": true + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "verification_tokens": { + "name": "verification_tokens", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "token": { + "name": "token", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "identifier": { + "name": "identifier", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "type": { + "name": "type", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "expires_at": { + "name": "expires_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch())" + } + }, + "indexes": { + "verification_tokens_token_unique": { + "name": "verification_tokens_token_unique", + "columns": [ + "token" + ], + "isUnique": true + }, + "idx_verification_tokens_token": { + "name": "idx_verification_tokens_token", + "columns": [ + "token" + ], + "isUnique": false + }, + "idx_verification_tokens_identifier": { + "name": "idx_verification_tokens_identifier", + "columns": [ + "identifier" + ], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "verifications": { + "name": "verifications", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "identifier": { + "name": "identifier", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "value": { + "name": "value", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "expires_at": { + "name": "expires_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch())" + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch())" + } + }, + "indexes": { + "idx_verifications_identifier": { + "name": "idx_verifications_identifier", + "columns": [ + "identifier" + ], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + } + }, + "views": {}, + "enums": {}, + "_meta": { + "schemas": {}, + "tables": {}, + "columns": {} + }, + "internal": { + "indexes": {} + } +} \ No newline at end of file diff --git a/drizzle/meta/_journal.json b/drizzle/meta/_journal.json index 01e5cd9..4d0b046 100644 --- a/drizzle/meta/_journal.json +++ b/drizzle/meta/_journal.json @@ -106,6 +106,13 @@ "when": 1787182504975, "tag": "0014_needy_white_tiger", "breakpoints": true + }, + { + "idx": 15, + "version": "6", + "when": 1788333701612, + "tag": "0015_source_provider", + "breakpoints": true } ] } \ No newline at end of file diff --git a/scripts/validate-migrations.ts b/scripts/validate-migrations.ts index 3cf1046..6711d40 100644 --- a/scripts/validate-migrations.ts +++ b/scripts/validate-migrations.ts @@ -354,6 +354,60 @@ function verify0014Migration(db: any) { assert(parsed.lfs === false, `Expected persisted lfs override false, got ${updated.mirror_overrides}`); } +function seedPre0015Database(db: any) { + // Migrations 0000-0014 have run, so repositories exist but carry no source + // columns. Seed one so the column-add can be verified against a + // pre-existing row: everything before 0015 came from GitHub. + db.run("INSERT INTO users (id, email, username, name) VALUES ('u-src', 'src@example.com', 'src', 'Source User')"); + db.run("INSERT INTO configs (id, user_id, name, is_active, github_config, gitea_config, schedule_config, cleanup_config) VALUES ('cfg-pre15', 'u-src', 'Default', 1, '{}', '{}', '{}', '{}')"); + db.run( + "INSERT INTO repositories (id, user_id, config_id, name, full_name, normalized_full_name, url, clone_url, owner, default_branch) " + + "VALUES ('repo-pre15', 'u-src', 'cfg-pre15', 'tool', 'src/tool', 'src/tool', 'https://github.com/src/tool', 'https://github.com/src/tool.git', 'src', 'main')", + ); +} + +function verify0015Migration(db: any) { + const cols = db + .query("PRAGMA table_info(repositories)") + .all() as Array<{ name: string; type: string; notnull: number; dflt_value: string | null }>; + + for (const [name, expectedDefault] of [ + ["source_provider", "'github'"], + ["source_url", "'https://github.com'"], + ] as const) { + const col = cols.find((c) => c.name === name); + assert(col, `Expected repositories.${name} column to exist`); + assert(col.notnull === 1, `Expected repositories.${name} to be NOT NULL`); + assert( + col.dflt_value === expectedDefault, + `Expected repositories.${name} default ${expectedDefault}, got ${col.dflt_value}`, + ); + } + + // Pre-existing rows are backfilled as GitHub, which is where they came from. + const existing = db + .query("SELECT source_provider, source_url FROM repositories WHERE id = 'repo-pre15'") + .get() as { source_provider: string; source_url: string } | null; + assert(existing, "Expected pre-existing repository row to survive migration"); + assert( + existing.source_provider === "github" && existing.source_url === "https://github.com", + `Expected existing repository to default to GitHub, got ${JSON.stringify(existing)}`, + ); + + // New rows can record another host. + db.run( + "INSERT INTO repositories (id, user_id, config_id, name, full_name, normalized_full_name, url, clone_url, owner, default_branch, source_provider, source_url) " + + "VALUES ('repo-gitlab', 'u-src', 'cfg-pre15', 'tool', 'group/tool', 'group/tool', 'https://gitlab.com/group/tool', 'https://gitlab.com/group/tool.git', 'group', 'main', 'gitlab', 'https://gitlab.com')", + ); + const inserted = db + .query("SELECT source_provider, source_url FROM repositories WHERE id = 'repo-gitlab'") + .get() as { source_provider: string; source_url: string } | null; + assert( + inserted?.source_provider === "gitlab" && inserted.source_url === "https://gitlab.com", + `Expected GitLab source to round-trip, got ${JSON.stringify(inserted)}`, + ); +} + const MIGRATION_0012_TIMESTAMP = 1774062000000; const MIGRATION_0013_TIMESTAMP = 1780377747526; @@ -473,6 +527,10 @@ const latestUpgradeFixtures: Record = { seed: seedPre0014Database, verify: verify0014Migration, }, + "0015_source_provider": { + seed: seedPre0015Database, + verify: verify0015Migration, + }, }; function lintMigrations(selectedMigrations: Migration[]) { diff --git a/src/components/config/ConfigTabs.tsx b/src/components/config/ConfigTabs.tsx index ad103ee..f5d25dc 100644 --- a/src/components/config/ConfigTabs.tsx +++ b/src/components/config/ConfigTabs.tsx @@ -16,6 +16,7 @@ import type { MirrorOptions, AdvancedOptions, NotificationConfig, + ConfigLockState, } from '@/types/config'; import { Button } from '../ui/button'; import { useAuth } from '@/hooks/useAuth'; @@ -26,6 +27,7 @@ import { Skeleton } from '@/components/ui/skeleton'; import { invalidateConfigCache } from '@/hooks/useConfigStatus'; import { Tabs, TabsContent, TabsList, TabsTrigger } from '@/components/ui/tabs'; import { withBase } from '@/lib/base-path'; +import { SOURCE_PROVIDER_LABELS } from '@/lib/source-providers/kinds'; type ConfigState = { githubConfig: GitHubConfig; @@ -42,6 +44,8 @@ const CONFIG_API_PATH = withBase('/api/config'); export function ConfigTabs() { const [config, setConfig] = useState({ githubConfig: { + provider: 'github', + url: '', username: '', token: '', privateRepositories: false, @@ -49,6 +53,7 @@ export function ConfigTabs() { starredLists: [], }, giteaConfig: { + provider: 'gitea', url: '', externalUrl: '', username: '', @@ -105,6 +110,8 @@ export function ConfigTabs() { const { user } = useAuth(); const [isLoading, setIsLoading] = useState(true); const [isSyncing, setIsSyncing] = useState(false); + // Source and destination locks from the API; null until the config is loaded. + const [locks, setLocks] = useState(null); const [isAutoSavingSchedule, setIsAutoSavingSchedule] = useState(false); const [isAutoSavingCleanup, setIsAutoSavingCleanup] = useState(false); @@ -138,6 +145,8 @@ export function ConfigTabs() { // Removed the problematic useEffect that was causing circular dependencies // The lastRun and nextRun should be managed by the backend and fetched via API + const sourceLabel = SOURCE_PROVIDER_LABELS[config.githubConfig.provider ?? 'github']; + const handleImportGitHubData = async () => { if (!user?.id) return; setIsSyncing(true); @@ -148,7 +157,7 @@ export function ConfigTabs() { ); if (result.success) { toast.success( - 'GitHub data imported successfully! Head to the Repositories page to start mirroring.', + `${sourceLabel} data imported successfully! Head to the Repositories page to start mirroring.`, ); if (result.failedOrgs && result.failedOrgs.length > 0) { toast.warning( @@ -162,14 +171,14 @@ export function ConfigTabs() { } } else { toast.error( - `Failed to import GitHub data: ${ + `Failed to import ${sourceLabel} data: ${ result.message || 'Unknown error' }`, ); } } catch (error) { toast.error( - `Error importing GitHub data: ${ + `Error importing ${sourceLabel} data: ${ error instanceof Error ? error.message : String(error) }`, ); @@ -310,7 +319,10 @@ export function ConfigTabs() { }, [user?.id, config.githubConfig, config.giteaConfig, config.scheduleConfig]); // Auto-save function specifically for GitHub config changes - const autoSaveGitHubConfig = useCallback(async (githubConfig: GitHubConfig) => { + const autoSaveGitHubConfig = useCallback(async ( + githubConfig: GitHubConfig, + options: { confirmSourceChange?: boolean } = {}, + ) => { if (!user?.id) return; // Clear any existing timeout @@ -330,6 +342,7 @@ export function ConfigTabs() { cleanupConfig: config.cleanupConfig, mirrorOptions: config.mirrorOptions, advancedOptions: config.advancedOptions, + ...(options.confirmSourceChange ? { confirmSourceChange: true } : {}), }; try { @@ -359,7 +372,10 @@ export function ConfigTabs() { }, [user?.id, config.giteaConfig, config.scheduleConfig, config.cleanupConfig]); // Auto-save function specifically for Gitea config changes - const autoSaveGiteaConfig = useCallback(async (giteaConfig: GiteaConfig) => { + const autoSaveGiteaConfig = useCallback(async ( + giteaConfig: GiteaConfig, + options: { confirmDestinationChange?: boolean } = {}, + ) => { if (!user?.id) return; // Clear any existing timeout @@ -379,6 +395,7 @@ export function ConfigTabs() { cleanupConfig: config.cleanupConfig, mirrorOptions: config.mirrorOptions, advancedOptions: config.advancedOptions, + ...(options.confirmDestinationChange ? { confirmDestinationChange: true } : {}), }; try { @@ -558,9 +575,12 @@ export function ConfigTabs() { { method: 'GET' }, ); if (response && !response.error) { + setLocks(response.locks ?? null); setConfig({ - githubConfig: - response.githubConfig || config.githubConfig, + githubConfig: { + ...config.githubConfig, + ...response.githubConfig, // Merge so a response without provider keeps the default + }, giteaConfig: response.giteaConfig || config.giteaConfig, scheduleConfig: @@ -692,12 +712,12 @@ export function ConfigTabs() { {isSyncing ? ( <> - Import GitHub Data + Import {sourceLabel} Data ) : ( <> - Import GitHub Data + Import {sourceLabel} Data )} @@ -757,6 +777,7 @@ export function ConfigTabs() { onAdvancedOptionsAutoSave: autoSaveAdvancedOptions, onGiteaAutoSave: autoSaveGiteaConfig, isAutoSaving: isAutoSavingGitHub, + sourceLock: locks?.source, }; const giteaFormProps = { config: config.giteaConfig, @@ -771,6 +792,7 @@ export function ConfigTabs() { onAutoSave: autoSaveGiteaConfig, isAutoSaving: isAutoSavingGitea, githubUsername: config.githubConfig.username, + destinationLock: locks?.destination, }; return ( <> diff --git a/src/components/config/GitHubConfigForm.tsx b/src/components/config/GitHubConfigForm.tsx index f7afb23..bf1b721 100644 --- a/src/components/config/GitHubConfigForm.tsx +++ b/src/components/config/GitHubConfigForm.tsx @@ -1,7 +1,15 @@ import React, { useState } from "react"; import { Button } from "@/components/ui/button"; import { githubApi } from "@/lib/api"; -import type { GitHubConfig, MirrorOptions, AdvancedOptions, GiteaConfig, BackupStrategy } from "@/types/config"; +import type { + GitHubConfig, + MirrorOptions, + AdvancedOptions, + GiteaConfig, + BackupStrategy, + SourceProvider, + ConfigLockState, +} from "@/types/config"; import { Input } from "../ui/input"; import { Label } from "../ui/label"; import { toast } from "sonner"; @@ -17,8 +25,20 @@ import { ShieldCheck, Sparkles, } from "lucide-react"; -import { SiGithub } from "react-icons/si"; +import { SiGitea, SiGithub, SiGitlab } from "react-icons/si"; +import { + Select, + SelectContent, + SelectItem, + SelectTrigger, + SelectValue, +} from "@/components/ui/select"; +import { + SOURCE_PROVIDER_DEFAULT_URLS, + SOURCE_PROVIDER_LABELS, +} from "@/lib/source-providers/kinds"; import { GitHubMirrorSettings } from "./GitHubMirrorSettings"; +import { HostLockNotice } from "./HostLockNotice"; import { SettingsCard, SectionTitle, @@ -38,11 +58,19 @@ interface GitHubConfigFormProps { setAdvancedOptions: React.Dispatch>; giteaConfig?: GiteaConfig; setGiteaConfig?: React.Dispatch>; - onAutoSave?: (githubConfig: GitHubConfig) => Promise; + onAutoSave?: ( + githubConfig: GitHubConfig, + options?: { confirmSourceChange?: boolean } + ) => Promise; onMirrorOptionsAutoSave?: (mirrorOptions: MirrorOptions) => Promise; onAdvancedOptionsAutoSave?: (advancedOptions: AdvancedOptions) => Promise; - onGiteaAutoSave?: (giteaConfig: GiteaConfig) => Promise; + onGiteaAutoSave?: ( + giteaConfig: GiteaConfig, + options?: { confirmDestinationChange?: boolean } + ) => Promise; isAutoSaving?: boolean; + /** Set once repositories were imported: the source can only change with confirmation. */ + sourceLock?: ConfigLockState["source"]; /** Which card group to render: the connection card, or the settings stack * (repository selection + destructive update protection). */ part?: "connection" | "settings"; @@ -79,6 +107,70 @@ const backupStrategies = [ }, ]; +type SourceProviderMeta = { + value: SourceProvider; + label: string; + icon: React.ComponentType<{ className?: string }>; + usernamePlaceholder: string; + tokenPlaceholder: string; + tokenHint: string; + /** Appended to the instance URL to reach the token settings page. */ + tokenSettingsPath: string; + tokenSteps: string[]; + scopes: string[]; + /** Short pill shown next to the option, e.g. BETA. */ + badge?: string; +}; + +const sourceProviders: SourceProviderMeta[] = [ + { + value: "github", + label: SOURCE_PROVIDER_LABELS.github, + icon: SiGithub, + usernamePlaceholder: "Your GitHub username", + tokenPlaceholder: "Your GitHub token (classic)", + tokenHint: "Needed for private repos, organizations, and starred repositories", + tokenSettingsPath: "/settings/tokens", + tokenSteps: [ + "GitHub → Settings → Developer settings", + "Personal access tokens → Generate new token (classic)", + "Select the scopes below and paste the token here", + ], + scopes: ["repo", "admin:org"], + }, + { + value: "gitlab", + label: SOURCE_PROVIDER_LABELS.gitlab, + icon: SiGitlab, + badge: "BETA", + usernamePlaceholder: "Your GitLab username", + tokenPlaceholder: "Your GitLab personal access token", + tokenHint: "Needed for private projects, groups, and starred projects", + tokenSettingsPath: "/-/user_settings/personal_access_tokens", + tokenSteps: [ + "GitLab → Preferences → Access tokens", + "Add new token with the scopes below", + "Paste the token here", + ], + scopes: ["read_api", "read_repository"], + }, + { + value: "gitea", + label: SOURCE_PROVIDER_LABELS.gitea, + icon: SiGitea, + usernamePlaceholder: "Your Gitea or Forgejo username", + tokenPlaceholder: "Your access token", + tokenHint: "Needed for private repos, organizations, and starred repositories", + tokenSettingsPath: "/user/settings/applications", + tokenSteps: [ + "Gitea → Settings → Applications", + "Generate a token with the permissions below", + "Paste the token here", + ], + scopes: ["read:repository", "read:user", "read:organization"], + }, +]; + export function GitHubConfigForm({ config, setConfig, @@ -93,9 +185,34 @@ export function GitHubConfigForm({ onAdvancedOptionsAutoSave, onGiteaAutoSave, isAutoSaving, + sourceLock, part = "connection" }: GitHubConfigFormProps) { const [isLoading, setIsLoading] = useState(false); + const [sourceUnlocked, setSourceUnlocked] = useState(false); + const sourceLocked = Boolean(sourceLock?.locked) && !sourceUnlocked; + + const provider: SourceProvider = config.provider ?? "github"; + const providerMeta = + sourceProviders.find((option) => option.value === provider) ?? sourceProviders[0]; + const providerLabel = providerMeta.label; + const defaultInstanceUrl = SOURCE_PROVIDER_DEFAULT_URLS[provider]; + const instanceUrl = (config.url ?? "").trim() || defaultInstanceUrl; + const tokenSettingsUrl = `${instanceUrl.replace(/\/+$/, "")}${providerMeta.tokenSettingsPath}`; + + const handleProviderChange = (value: string) => { + const nextProvider = value as SourceProvider; + const newConfig: GitHubConfig = { + ...config, + provider: nextProvider, + // GitHub has no per-config instance URL (GHES routing uses GH_API_URL). + url: nextProvider === "github" ? "" : config.url ?? "", + }; + setConfig(newConfig); + if (onAutoSave) { + onAutoSave(newConfig, { confirmSourceChange: sourceUnlocked }); + } + }; const handleChange = (e: React.ChangeEvent) => { const { name, value, type, checked } = e.target; @@ -107,26 +224,32 @@ export function GitHubConfigForm({ setConfig(newConfig); - // Auto-save for all field changes + // Auto-save for all field changes. Once the source was unlocked through + // the dialog, the save carries the confirmation the API requires. if (onAutoSave) { - onAutoSave(newConfig); + onAutoSave(newConfig, { confirmSourceChange: sourceUnlocked }); } }; const testConnection = async () => { if (!config.token) { - toast.error("GitHub token is required to test the connection"); + toast.error(`${providerLabel} token is required to test the connection`); return; } setIsLoading(true); try { - const result = await githubApi.testConnection(config.token); + const result = await githubApi.testConnection(config.token, { + provider, + url: config.url, + }); if (result.success) { - toast.success("Successfully connected to GitHub!"); + toast.success(`Successfully connected to ${providerLabel}!`); } else { - toast.error("Failed to connect to GitHub. Please check your token."); + toast.error( + result.message || `Failed to connect to ${providerLabel}. Please check your token.` + ); } } catch (error) { toast.error( @@ -148,8 +271,8 @@ export function GitHubConfigForm({ if (part === "connection") { return ( {isAutoSaving && ( @@ -169,6 +292,81 @@ export function GitHubConfigForm({ } > +
+ + +

+ {provider === "github" + ? "Where your repositories are pulled from" + : provider === "gitlab" + ? "Beta. Code, tags, wiki and LFS are mirrored. Issues, merge requests and releases need a GitHub source." + : "Code, tags, wiki and LFS are mirrored. Issues, pull requests and releases need a GitHub source."} +

+ {sourceLock?.locked && ( + setSourceUnlocked(true)} + /> + )} +
+ + {provider !== "github" && ( +
+ + +

+ {`Leave empty for ${defaultInstanceUrl.replace(/^https?:\/\//, "")}, or enter the base URL of your own instance`} +

+
+ )} +
@@ -200,10 +398,10 @@ export function GitHubConfigForm({ type="password" value={config.token} onChange={handleChange} - placeholder="Your GitHub token (classic)" + placeholder={providerMeta.tokenPlaceholder} />

- Needed for private repos, organizations, and starred repositories + {providerMeta.tokenHint}

@@ -216,28 +414,30 @@ export function GitHubConfigForm({
    -
  1. GitHub → Settings → Developer settings
    2. -
  2. Personal access tokens → Generate new token (classic)
    3. -
  3. Select the scopes below and paste the token here
    4. + {providerMeta.tokenSteps.map((step) => ( +
  4. {step}
    5. + ))}
-
- - repo - - - admin:org - +
+ {providerMeta.scopes.map((scope) => ( + + {scope} + + ))}
diff --git a/src/components/config/GitHubMirrorSettings.tsx b/src/components/config/GitHubMirrorSettings.tsx index 8d2dc35..e9d31c5 100644 --- a/src/components/config/GitHubMirrorSettings.tsx +++ b/src/components/config/GitHubMirrorSettings.tsx @@ -57,6 +57,10 @@ import { } from "@/components/ui/command"; import { cn } from "@/lib/utils"; import { githubApi } from "@/lib/api"; +import { + SOURCE_PROVIDER_LABELS, + SOURCE_PROVIDER_ORG_NOUNS, +} from "@/lib/source-providers/kinds"; import { SettingsCard, SectionTitle, @@ -87,6 +91,12 @@ export function GitHubMirrorSettings({ onAdvancedOptionsChange, part = "both", }: GitHubMirrorSettingsProps) { + // Star lists, issues, pull requests, releases, labels and milestones all + // read the GitHub API, so they are only offered for GitHub sources. + const sourceProvider = githubConfig.provider ?? "github"; + const isGithubSource = sourceProvider === "github"; + const sourceLabel = SOURCE_PROVIDER_LABELS[sourceProvider]; + const orgNoun = SOURCE_PROVIDER_ORG_NOUNS[sourceProvider]; const [starListsOpen, setStarListsOpen] = React.useState(false); const [starListSearch, setStarListSearch] = React.useState(""); const [customStarListName, setCustomStarListName] = React.useState(""); @@ -410,6 +420,7 @@ export function GitHubMirrorSettings({ variant="ghost" size="sm" className="h-auto px-2 py-1 text-xs font-normal text-primary hover:text-primary/80" + disabled={!isGithubSource} onClick={() => { const allSelected = Object.values(mirrorOptions.metadataComponents).every(Boolean); const newValue = !allSelected; @@ -437,6 +448,7 @@ export function GitHubMirrorSettings({
handleMetadataComponentChange('issues', !!checked)} /> @@ -452,6 +464,7 @@ export function GitHubMirrorSettings({
handleMetadataComponentChange('pullRequests', !!checked)} /> @@ -492,6 +505,7 @@ export function GitHubMirrorSettings({
handleMetadataComponentChange('labels', !!checked)} /> @@ -507,6 +521,7 @@ export function GitHubMirrorSettings({
handleMetadataComponentChange('milestones', !!checked)} /> @@ -575,7 +590,7 @@ export function GitHubMirrorSettings({ handleGitHubChange('mirrorStarred', checked)} /> @@ -596,6 +611,7 @@ export function GitHubMirrorSettings({ right={starredContentPopover} /> + {isGithubSource && (
+ )} handleAdvancedChange('skipPersonalRepos', checked)} /> @@ -781,8 +798,8 @@ export function GitHubMirrorSettings({
{includedOrgs.length > 0 && ( @@ -813,7 +830,7 @@ export function GitHubMirrorSettings({ addIncludedOrg(); } }} - placeholder="Add organization name" + placeholder={`Add ${orgNoun} name`} className="h-8 text-xs" />
@@ -281,11 +369,11 @@ export function GiteaConfigForm({ config, setConfig, onAutoSave, isAutoSaving, g type="password" value={config.token} onChange={handleChange} - placeholder="Your Gitea access token" + placeholder={`Your ${providerLabel} access token`} required />

- Create one in Gitea under Settings → Applications + {`Create one in ${providerLabel} under Settings → Applications`}

diff --git a/src/components/config/HostLockNotice.tsx b/src/components/config/HostLockNotice.tsx new file mode 100644 index 0000000..e5cc6b0 --- /dev/null +++ b/src/components/config/HostLockNotice.tsx @@ -0,0 +1,97 @@ +import { useState } from "react"; +import { Lock, LockOpen } from "lucide-react"; +import { Button } from "@/components/ui/button"; +import { + Dialog, + DialogContent, + DialogDescription, + DialogFooter, + DialogHeader, + DialogTitle, +} from "@/components/ui/dialog"; + +interface HostLockNoticeProps { + /** One line shown next to the lock icon, e.g. "12 repositories imported from GitHub". */ + summary: string; + /** Dialog title, e.g. "Change the source?". */ + title: string; + /** What happens to existing repositories when the host changes. */ + consequences: string[]; + /** Label of the confirm button and of the inline button. */ + changeLabel: string; + unlocked: boolean; + onUnlock: () => void; +} + +/** + * Inline note for a locked source or destination, with the confirmation + * dialog that unlocks the fields for a deliberate change. + */ +export function HostLockNotice({ + summary, + title, + consequences, + changeLabel, + unlocked, + onUnlock, +}: HostLockNoticeProps) { + const [open, setOpen] = useState(false); + + if (unlocked) { + return ( +
+ + Unlocked for this edit. Existing repositories stay tied to the previous host. +
+ ); + } + + return ( + <> +
+
+ + {summary} +
+ +
+ + + + + {title} + {summary}. + +
    + {consequences.map((line) => ( +
  • {line}
    • + ))} +
+ + + + + + + + ); +} diff --git a/src/components/dashboard/RepositoryList.tsx b/src/components/dashboard/RepositoryList.tsx index 0c835c3..158c6cc 100644 --- a/src/components/dashboard/RepositoryList.tsx +++ b/src/components/dashboard/RepositoryList.tsx @@ -1,4 +1,5 @@ import { Card, CardContent, CardHeader, CardTitle } from "@/components/ui/card"; +import { SourceIcon, repositorySourceInfo } from "@/components/source/SourceIcon"; import { Button } from "@/components/ui/button"; import { GitFork } from "lucide-react"; import { SiGithub, SiGitea } from "react-icons/si"; @@ -128,9 +129,9 @@ export function RepositoryList({ repositories }: RepositoryListProps) { href={repo.url} target="_blank" rel="noopener noreferrer" - title="View on GitHub" + title={`View on ${repositorySourceInfo(repo).label}`} > - +
diff --git a/src/components/repositories/AddRepositoryDialog.tsx b/src/components/repositories/AddRepositoryDialog.tsx index 80c8c9f..f2924da 100644 --- a/src/components/repositories/AddRepositoryDialog.tsx +++ b/src/components/repositories/AddRepositoryDialog.tsx @@ -10,7 +10,11 @@ import { DialogTrigger, } from "../ui/dialog"; import { LoaderCircle, Plus } from "lucide-react"; -import { parseGitHubRepoReference } from "@/lib/utils/github-url"; +import { + parseGitHubRepoReference, + parseRepoReferenceParts, + type RepoReferenceParts, +} from "@/lib/utils/github-url"; const inputClassName = "w-full rounded-md border border-input bg-background px-3 py-2 text-sm shadow-sm transition-colors placeholder:text-muted-foreground focus-visible:outline-none focus-visible:ring-1 focus-visible:ring-ring"; @@ -28,6 +32,8 @@ interface AddRepositoryDialogProps { owner: string; force?: boolean; destinationOrg?: string; + host?: string; + path?: string[]; }) => Promise; } @@ -40,6 +46,8 @@ export default function AddRepositoryDialog({ const [repo, setRepo] = useState(""); const [owner, setOwner] = useState(""); const [destinationOrg, setDestinationOrg] = useState(""); + /** Host and path of the pasted URL, resolved per source on the server. */ + const [reference, setReference] = useState(null); const [isLoading, setIsLoading] = useState(false); const [error, setError] = useState(""); @@ -49,6 +57,7 @@ export default function AddRepositoryDialog({ setRepo(""); setOwner(""); setDestinationOrg(""); + setReference(null); }; useEffect(() => { @@ -63,6 +72,7 @@ export default function AddRepositoryDialog({ if (!parsed) return false; setOwner(parsed.owner); setRepo(parsed.repo); + setReference(parseRepoReferenceParts(value)); setError(""); return true; }; @@ -73,6 +83,7 @@ export default function AddRepositoryDialog({ if (!applyReference(value)) { setOwner(""); setRepo(""); + setReference(null); } }; @@ -98,7 +109,7 @@ export default function AddRepositoryDialog({ if (!repo || !owner || repo.trim() === "" || owner.trim() === "") { setError( urlIsUnparsed - ? "That does not look like a GitHub repository URL." + ? "That does not look like a repository URL." : "Please enter a valid repository name and owner." ); return; @@ -111,6 +122,8 @@ export default function AddRepositoryDialog({ repo, owner, destinationOrg: destinationOrg.trim() || undefined, + host: reference?.host ?? undefined, + path: reference?.segments, }); resetForm(); @@ -145,7 +158,7 @@ export default function AddRepositoryDialog({ htmlFor="repositoryUrl" className="block text-sm font-medium mb-1.5" > - GitHub URL + Repository URL { if (!user || !user.id) { return; @@ -819,6 +823,8 @@ export default function Repository() { owner: trimmedOwner, force, ...(destinationOrg ? { destinationOrg } : {}), + ...(host ? { host } : {}), + ...(path && path.length > 0 ? { path } : {}), }; const response = await apiRequest( diff --git a/src/components/repositories/RepositoryTable.tsx b/src/components/repositories/RepositoryTable.tsx index 868581a..f301ec9 100644 --- a/src/components/repositories/RepositoryTable.tsx +++ b/src/components/repositories/RepositoryTable.tsx @@ -1,4 +1,5 @@ import { useEffect, useMemo, useRef, useState } from "react"; +import { SourceIcon, repositorySourceInfo } from "@/components/source/SourceIcon"; import { getCoreRowModel, getFilteredRowModel, @@ -645,11 +646,11 @@ export default function RepositoryTable({ href={repo.url} target="_blank" rel="noopener noreferrer" - title="View on GitHub" + title={`View on ${repositorySourceInfo(repo).label}`} className="flex items-center justify-center gap-2" > - - GitHub + + {repositorySourceInfo(repo).label} {giteaUrl ? ( @@ -1131,9 +1132,9 @@ export default function RepositoryTable({ href={repo.url} target="_blank" rel="noopener noreferrer" - title="View on GitHub" + title={`View on ${repositorySourceInfo(repo).label}`} > - +
diff --git a/src/components/source/SourceIcon.tsx b/src/components/source/SourceIcon.tsx new file mode 100644 index 0000000..23157d7 --- /dev/null +++ b/src/components/source/SourceIcon.tsx @@ -0,0 +1,34 @@ +import { SiGitea, SiGithub, SiGitlab } from "react-icons/si"; +import { + SOURCE_PROVIDER_LABELS, + getRepositorySource, + type RepositorySourceFields, + type SourceProviderKind, +} from "@/lib/source-providers/kinds"; + +const icons: Record> = { + github: SiGithub, + gitlab: SiGitlab, + gitea: SiGitea, +}; + +/** Brand icon for the host a repository came from. */ +export function SourceIcon({ + provider, + className, +}: { + provider: SourceProviderKind; + className?: string; +}) { + const Icon = icons[provider] ?? SiGithub; + return ; +} + +/** Provider kind and label for a stored repository row. */ +export function repositorySourceInfo(repo: RepositorySourceFields): { + provider: SourceProviderKind; + label: string; +} { + const { provider } = getRepositorySource(repo); + return { provider, label: SOURCE_PROVIDER_LABELS[provider] }; +} diff --git a/src/lib/api.ts b/src/lib/api.ts index 6b17a66..74ac848 100644 --- a/src/lib/api.ts +++ b/src/lib/api.ts @@ -1,4 +1,5 @@ import { withBase } from "@/lib/base-path"; +import type { SourceProvider } from "@/types/config"; // Base API URL const API_BASE = withBase("/api"); @@ -75,10 +76,14 @@ export const authApi = { // GitHub API export const githubApi = { - testConnection: (token: string) => - apiRequest<{ success: boolean }>("/github/test-connection", { + /** Test the source connection. The route serves every source provider. */ + testConnection: ( + token: string, + options: { provider?: SourceProvider; url?: string; username?: string } = {} + ) => + apiRequest<{ success: boolean; message?: string }>("/github/test-connection", { method: "POST", - body: JSON.stringify({ token }), + body: JSON.stringify({ token, ...options }), }), getStarredLists: () => apiRequest<{ success: boolean; lists: string[] }>("/github/starred-lists", { diff --git a/src/lib/config-locks.test.ts b/src/lib/config-locks.test.ts new file mode 100644 index 0000000..e68d41a --- /dev/null +++ b/src/lib/config-locks.test.ts @@ -0,0 +1,150 @@ +import { describe, expect, test } from "bun:test"; +import { + computeConfigLocks, + evaluateConfigChange, + hasDestinationChanged, + hasSourceChanged, + loadConfigLocks, + normalizeDestinationUrl, + sourceIdentity, +} from "./config-locks"; + +describe("computeConfigLocks", () => { + test("locks the source once anything is imported and the destination once anything is mirrored", () => { + expect(computeConfigLocks({ repositoryCount: 0, mirroredCount: 0 })).toEqual({ + source: { locked: false, repositoryCount: 0 }, + destination: { locked: false, mirroredCount: 0 }, + }); + expect(computeConfigLocks({ repositoryCount: 3, mirroredCount: 0 })).toMatchObject({ + source: { locked: true }, + destination: { locked: false }, + }); + expect(computeConfigLocks({ repositoryCount: 3, mirroredCount: 1 }).destination.locked).toBe(true); + }); +}); + +describe("hasSourceChanged", () => { + test("legacy rows without a provider are GitHub and match a GitHub payload with any URL", () => { + expect(hasSourceChanged({}, { provider: "github", url: "https://github.com" })).toBe(false); + expect(hasSourceChanged({ provider: "github" }, { provider: "github", url: "" })).toBe(false); + }); + + test("cosmetic URL edits are not a change, another host is", () => { + expect( + hasSourceChanged( + { provider: "gitlab", url: "https://gitlab.example.com" }, + { provider: "gitlab", url: "GitLab.example.com/" } + ) + ).toBe(false); + expect(hasSourceChanged({ provider: "gitlab" }, { provider: "gitlab", url: "https://gitlab.example.com" })).toBe(true); + expect(hasSourceChanged({ provider: "github" }, { provider: "gitea" })).toBe(true); + }); + + test("sourceIdentity describes GitHub with its fixed URL", () => { + expect(sourceIdentity({ provider: "github", url: "https://ghe.example.com" })).toEqual({ + provider: "github", + url: "https://github.com", + }); + }); +}); + +describe("hasDestinationChanged", () => { + test("ignores trailing slashes and host case, and never fires while a side is empty", () => { + expect(hasDestinationChanged("https://Gitea.example.com/", "https://gitea.example.com")).toBe(false); + expect(hasDestinationChanged("https://gitea.example.com", "https://other.example.com")).toBe(true); + expect(hasDestinationChanged("", "https://gitea.example.com")).toBe(false); + expect(hasDestinationChanged("https://gitea.example.com", "")).toBe(false); + expect(normalizeDestinationUrl(" https://gitea.example.com/gitea// ")).toBe("https://gitea.example.com/gitea"); + }); +}); + +describe("evaluateConfigChange", () => { + const locked = computeConfigLocks({ repositoryCount: 12, mirroredCount: 4 }); + const unlocked = computeConfigLocks({ repositoryCount: 0, mirroredCount: 0 }); + const base = { + existingSource: { provider: "github" }, + incomingSource: { provider: "github", url: "" }, + existingDestinationUrl: "https://gitea.example.com", + incomingDestinationUrl: "https://gitea.example.com", + }; + + test("an unchanged save always passes", () => { + expect(evaluateConfigChange({ locks: locked, ...base })).toEqual({ ok: true }); + }); + + test("everything passes while nothing is imported", () => { + expect( + evaluateConfigChange({ + locks: unlocked, + ...base, + incomingSource: { provider: "gitlab" }, + incomingDestinationUrl: "https://other.example.com", + }) + ).toEqual({ ok: true }); + }); + + test("a locked source refuses a provider change without confirmation and names both hosts", () => { + const verdict = evaluateConfigChange({ + locks: locked, + ...base, + incomingSource: { provider: "gitlab", url: "https://gitlab.example.com" }, + }); + expect(verdict.ok).toBe(false); + if (verdict.ok) return; + expect(verdict.lock).toBe("source"); + expect(verdict.message).toContain("12 repositories were imported from GitHub (https://github.com)"); + expect(verdict.message).toContain("GitLab (https://gitlab.example.com)"); + }); + + test("the confirm flag lets a locked source change through", () => { + expect( + evaluateConfigChange({ + locks: locked, + ...base, + incomingSource: { provider: "gitlab" }, + confirmSourceChange: true, + }) + ).toEqual({ ok: true }); + }); + + test("a locked destination refuses a server URL change without confirmation", () => { + const verdict = evaluateConfigChange({ + locks: locked, + ...base, + incomingDestinationUrl: "https://other.example.com/", + }); + expect(verdict).toMatchObject({ ok: false, lock: "destination" }); + if (verdict.ok) return; + expect(verdict.message).toContain("4 repositories are mirrored to https://gitea.example.com"); + + expect( + evaluateConfigChange({ + locks: locked, + ...base, + incomingDestinationUrl: "https://other.example.com/", + confirmDestinationChange: true, + }) + ).toEqual({ ok: true }); + }); + + test("the source is reported before the destination when both change", () => { + const verdict = evaluateConfigChange({ + locks: locked, + ...base, + incomingSource: { provider: "gitea" }, + incomingDestinationUrl: "https://other.example.com", + }); + expect(verdict).toMatchObject({ ok: false, lock: "source" }); + }); +}); + +describe("loadConfigLocks", () => { + test("resolves to unlocked when the database layer misbehaves", async () => { + // The global test setup stubs @/lib/db with a select chain that never + // yields rows, so the count cannot be read. That must not throw. + expect(await loadConfigLocks("user-1")).toEqual({ + source: { locked: false, repositoryCount: 0 }, + destination: { locked: false, mirroredCount: 0 }, + }); + }); +}); diff --git a/src/lib/config-locks.ts b/src/lib/config-locks.ts new file mode 100644 index 0000000..ffe909c --- /dev/null +++ b/src/lib/config-locks.ts @@ -0,0 +1,199 @@ +/** + * Source and destination locks. + * + * Once repositories have been imported, the source (provider and instance + * URL) is locked; once anything has been mirrored, the destination (Gitea + * server URL) is locked. Changing either afterwards is still possible, but + * only with an explicit confirmation, so an autosave, a script or an + * environment variable cannot switch hosts by accident. + * + * The decision logic is pure; the database lookup lives in loadConfigLocks. + */ +import type { ConfigLockState } from "@/types/config"; +import { + SOURCE_PROVIDER_DEFAULT_URLS, + SOURCE_PROVIDER_LABELS, + normalizeSourceProviderKind, + normalizeSourceUrl, + type SourceProviderKind, +} from "@/lib/source-providers/kinds"; + +export interface SourceIdentity { + provider: SourceProviderKind; + url: string; +} + +export interface SourceLike { + provider?: unknown; + url?: string | null; +} + +/** Statuses that mean a repository exists (or is being created) on the destination. */ +export const MIRRORED_STATUSES = ["mirroring", "mirrored", "syncing", "synced"] as const; + +export function computeConfigLocks({ + repositoryCount, + mirroredCount, +}: { + repositoryCount: number; + mirroredCount: number; +}): ConfigLockState { + return { + source: { locked: repositoryCount > 0, repositoryCount }, + destination: { locked: mirroredCount > 0, mirroredCount }, + }; +} + +export const UNLOCKED_CONFIG: ConfigLockState = computeConfigLocks({ + repositoryCount: 0, + mirroredCount: 0, +}); + +/** The host a source config points at, normalized so cosmetic edits do not count as a change. */ +export function sourceIdentity(config: SourceLike | null | undefined): SourceIdentity { + const provider = normalizeSourceProviderKind(config?.provider); + return { + provider, + url: + provider === "github" + ? SOURCE_PROVIDER_DEFAULT_URLS.github + : normalizeSourceUrl(config?.url, provider), + }; +} + +export function describeSourceIdentity(identity: SourceIdentity): string { + return `${SOURCE_PROVIDER_LABELS[identity.provider]} (${identity.url})`; +} + +export function hasSourceChanged( + existing: SourceLike | null | undefined, + incoming: SourceLike | null | undefined +): boolean { + const a = sourceIdentity(existing); + const b = sourceIdentity(incoming); + return a.provider !== b.provider || a.url !== b.url; +} + +/** Normalize a Gitea server URL for comparison. Empty stays empty. */ +export function normalizeDestinationUrl(url: string | null | undefined): string { + const trimmed = typeof url === "string" ? url.trim() : ""; + if (!trimmed) return ""; + try { + const parsed = new URL(trimmed); + return `${parsed.protocol}//${parsed.host}${parsed.pathname.replace(/\/+$/, "")}`; + } catch { + return trimmed.replace(/\/+$/, "").toLowerCase(); + } +} + +/** An empty side means "not set yet", which is never a change. */ +export function hasDestinationChanged( + existingUrl: string | null | undefined, + incomingUrl: string | null | undefined +): boolean { + const a = normalizeDestinationUrl(existingUrl); + const b = normalizeDestinationUrl(incomingUrl); + if (!a || !b) return false; + return a !== b; +} + +export type ConfigChangeVerdict = + | { ok: true } + | { ok: false; lock: "source" | "destination"; message: string }; + +/** + * Decide whether a save may go through. The source is checked first: a + * request that changes both without confirming either is refused for the + * source, and the destination is reported on the next attempt. + */ +export function evaluateConfigChange({ + locks, + existingSource, + incomingSource, + existingDestinationUrl, + incomingDestinationUrl, + confirmSourceChange = false, + confirmDestinationChange = false, +}: { + locks: ConfigLockState; + existingSource: SourceLike | null | undefined; + incomingSource: SourceLike | null | undefined; + existingDestinationUrl: string | null | undefined; + incomingDestinationUrl: string | null | undefined; + confirmSourceChange?: boolean; + confirmDestinationChange?: boolean; +}): ConfigChangeVerdict { + if ( + locks.source.locked && + !confirmSourceChange && + hasSourceChanged(existingSource, incomingSource) + ) { + const count = locks.source.repositoryCount; + return { + ok: false, + lock: "source", + message: + `The source is locked: ${count} ${count === 1 ? "repository was" : "repositories were"} imported from ` + + `${describeSourceIdentity(sourceIdentity(existingSource))}. ` + + `Confirm the change to switch to ${describeSourceIdentity(sourceIdentity(incomingSource))}.`, + }; + } + + if ( + locks.destination.locked && + !confirmDestinationChange && + hasDestinationChanged(existingDestinationUrl, incomingDestinationUrl) + ) { + const count = locks.destination.mirroredCount; + return { + ok: false, + lock: "destination", + message: + `The destination is locked: ${count} ${count === 1 ? "repository is" : "repositories are"} mirrored to ` + + `${normalizeDestinationUrl(existingDestinationUrl)}. ` + + `Confirm the change to switch to ${normalizeDestinationUrl(incomingDestinationUrl)}.`, + }; + } + + return { ok: true }; +} + +/** + * Count the user's repositories to derive the locks. Any failure resolves + * to "unlocked" with a warning: a broken count must not block the + * Configuration page, and the mirror step still refuses cross-host tokens. + */ +export async function loadConfigLocks(userId: string): Promise { + try { + const { db, repositories } = await import("@/lib/db"); + const { and, eq, sql } = await import("drizzle-orm"); + + const [all] = await db + .select({ count: sql`count(*)` }) + .from(repositories) + .where(eq(repositories.userId, userId)); + + const statusList = MIRRORED_STATUSES.map((status) => `'${status}'`).join(", "); + const [mirrored] = await db + .select({ count: sql`count(*)` }) + .from(repositories) + .where( + and( + eq(repositories.userId, userId), + sql`(${repositories.status} in (${sql.raw(statusList)}) or coalesce(${repositories.mirroredLocation}, '') <> '')` + ) + ); + + return computeConfigLocks({ + repositoryCount: Number(all?.count ?? 0), + mirroredCount: Number(mirrored?.count ?? 0), + }); + } catch (error) { + console.warn( + `[Config] Could not compute source/destination locks for user ${userId}; treating as unlocked: ${ + error instanceof Error ? error.message : String(error) + }` + ); + return UNLOCKED_CONFIG; + } +} diff --git a/src/lib/db/schema.ts b/src/lib/db/schema.ts index 6a8d25e..414ca44 100644 --- a/src/lib/db/schema.ts +++ b/src/lib/db/schema.ts @@ -1,4 +1,8 @@ import { z } from "zod"; +import { + SOURCE_PROVIDER_KINDS, + type SourceProviderKind, +} from "../source-providers/kinds"; import { sqliteTable, text, integer, index, uniqueIndex } from "drizzle-orm/sqlite-core"; import { sql } from "drizzle-orm"; @@ -17,6 +21,10 @@ export const githubConfigSchema = z.object({ owner: z.string(), type: z.enum(["personal", "organization"]), token: z.string(), + // Which host the repositories come from. "gitea" also covers Forgejo. + provider: z.enum(SOURCE_PROVIDER_KINDS).default("github"), + // Base URL of the instance for GitLab and Gitea sources. + url: z.string().optional(), includeStarred: z.boolean().default(false), includeForks: z.boolean().default(true), skipForks: z.boolean().default(false), @@ -76,6 +84,8 @@ export type MirrorOverrides = z.infer; export const giteaConfigSchema = z.object({ url: z.url(), + // Gitea or Forgejo. Same API; only labels and hints differ. + provider: z.enum(["gitea", "forgejo"]).default("gitea"), externalUrl: z.url().optional(), token: z.string(), defaultOwner: z.string(), @@ -220,6 +230,8 @@ export const repositorySchema = z.object({ url: z.url(), cloneUrl: z.url(), owner: z.string(), + sourceProvider: z.string().optional(), + sourceUrl: z.string().optional(), organization: z.string().optional().nullable(), mirroredLocation: z.string().default(""), isPrivate: z.boolean().default(false), @@ -439,6 +451,14 @@ export const repositories = sqliteTable("repositories", { normalizedFullName: text("normalized_full_name").notNull(), url: text("url").notNull(), cloneUrl: text("clone_url").notNull(), + // Which host the repository was imported from, and that host's base + // URL. The mirror step picks clone credentials from these, and the + // cleanup service only judges repositories from the configured source. + sourceProvider: text("source_provider") + .$type() + .notNull() + .default("github"), + sourceUrl: text("source_url").notNull().default("https://github.com"), owner: text("owner").notNull(), organization: text("organization"), mirroredLocation: text("mirrored_location").default(""), diff --git a/src/lib/destination-kinds.ts b/src/lib/destination-kinds.ts new file mode 100644 index 0000000..5beb7bc --- /dev/null +++ b/src/lib/destination-kinds.ts @@ -0,0 +1,29 @@ +/** + * The hosts Gitea Mirror can mirror into. Gitea and Forgejo share the same + * API, so this only changes labels and hints; kept free of imports so client + * components can use it. + */ +export const DESTINATION_PROVIDER_KINDS = ["gitea", "forgejo"] as const; + +export type DestinationProviderKind = (typeof DESTINATION_PROVIDER_KINDS)[number]; + +export const DEFAULT_DESTINATION_PROVIDER: DestinationProviderKind = "gitea"; + +export const DESTINATION_PROVIDER_LABELS: Record = { + gitea: "Gitea", + forgejo: "Forgejo", +}; + +export function isDestinationProviderKind(value: unknown): value is DestinationProviderKind { + return ( + typeof value === "string" && + (DESTINATION_PROVIDER_KINDS as readonly string[]).includes(value) + ); +} + +/** Coerce an untrusted value to a destination kind, defaulting to Gitea. Codeberg is Forgejo. */ +export function normalizeDestinationProviderKind(value: unknown): DestinationProviderKind { + if (isDestinationProviderKind(value)) return value; + if (value === "codeberg") return "forgejo"; + return DEFAULT_DESTINATION_PROVIDER; +} diff --git a/src/lib/env-config-loader.ts b/src/lib/env-config-loader.ts index d0627bd..1ca6c9d 100644 --- a/src/lib/env-config-loader.ts +++ b/src/lib/env-config-loader.ts @@ -7,9 +7,14 @@ import { db, configs, users } from '@/lib/db'; import { eq, and, sql } from 'drizzle-orm'; import { v4 as uuidv4 } from 'uuid'; import { encrypt } from '@/lib/utils/encryption'; +import { isSourceProviderKind, type SourceProviderKind } from '@/lib/source-providers/kinds'; +import { isDestinationProviderKind, type DestinationProviderKind } from '@/lib/destination-kinds'; +import { hasDestinationChanged, hasSourceChanged, loadConfigLocks } from '@/lib/config-locks'; interface EnvConfig { github: { + provider?: SourceProviderKind; + url?: string; username?: string; token?: string; type?: 'personal' | 'organization'; @@ -32,6 +37,7 @@ interface EnvConfig { mirrorStrategy?: 'preserve' | 'single-org' | 'flat-user' | 'mixed'; }; gitea: { + provider?: DestinationProviderKind; url?: string; externalUrl?: string; username?: string; @@ -112,6 +118,13 @@ function parseEnvConfig(): EnvConfig { return { github: { + // SOURCE_PROVIDER picks GitHub, GitLab or Gitea/Forgejo; SOURCE_URL is + // the instance base URL for the last two. Username and token still come + // from GITHUB_USERNAME and GITHUB_TOKEN whatever the provider. + provider: isSourceProviderKind(process.env.SOURCE_PROVIDER) + ? process.env.SOURCE_PROVIDER + : undefined, + url: process.env.SOURCE_URL, username: process.env.GITHUB_USERNAME, token: process.env.GITHUB_TOKEN, type: process.env.GITHUB_TYPE as 'personal' | 'organization', @@ -138,6 +151,10 @@ function parseEnvConfig(): EnvConfig { mirrorStrategy: process.env.MIRROR_STRATEGY as 'preserve' | 'single-org' | 'flat-user' | 'mixed', }, gitea: { + // DESTINATION_PROVIDER picks Gitea or Forgejo (labels only, same API). + provider: isDestinationProviderKind(process.env.DESTINATION_PROVIDER) + ? process.env.DESTINATION_PROVIDER + : undefined, url: process.env.GITEA_URL, externalUrl: process.env.GITEA_EXTERNAL_URL, username: process.env.GITEA_USERNAME, @@ -262,6 +279,32 @@ export async function initializeConfigFromEnv(): Promise { .orderBy(sql`${configs.isActive} DESC`, sql`${configs.updatedAt} DESC`) .limit(1); + // A source with imported repositories, or a destination with mirrors, is + // locked: an environment variable that disagrees is ignored with a + // warning rather than switching hosts underneath existing mirrors. The + // Configuration page can still change them, with confirmation. + if (existingConfig[0]) { + const stored = existingConfig[0]; + const locks = await loadConfigLocks(userId); + const incomingSource = { + provider: envConfig.github.provider ?? stored.githubConfig?.provider, + url: envConfig.github.url ?? stored.githubConfig?.url, + }; + if (locks.source.locked && hasSourceChanged(stored.githubConfig, incomingSource)) { + console.warn( + `[ENV Config Loader] Ignoring SOURCE_PROVIDER/SOURCE_URL: the source is locked because ${locks.source.repositoryCount} repositories were imported from it. Change it on the Configuration page.` + ); + envConfig.github.provider = undefined; + envConfig.github.url = undefined; + } + if (locks.destination.locked && hasDestinationChanged(stored.giteaConfig?.url, envConfig.gitea.url)) { + console.warn( + `[ENV Config Loader] Ignoring GITEA_URL: the destination is locked because ${locks.destination.mirroredCount} repositories are mirrored to it. Change it on the Configuration page.` + ); + envConfig.gitea.url = undefined; + } + } + // Determine mirror strategy based on environment variables or use explicit value let mirrorStrategy: 'preserve' | 'single-org' | 'flat-user' | 'mixed' = 'preserve'; if (envConfig.github.mirrorStrategy) { @@ -276,6 +319,8 @@ export async function initializeConfigFromEnv(): Promise { const githubConfig = { owner: envConfig.github.username || existingConfig?.[0]?.githubConfig?.owner || '', type: envConfig.github.type || existingConfig?.[0]?.githubConfig?.type || 'personal', + provider: envConfig.github.provider || existingConfig?.[0]?.githubConfig?.provider || 'github', + url: envConfig.github.url || existingConfig?.[0]?.githubConfig?.url || undefined, token: envConfig.github.token ? encrypt(envConfig.github.token) : existingConfig?.[0]?.githubConfig?.token || '', includeStarred: envConfig.github.mirrorStarred ?? existingConfig?.[0]?.githubConfig?.includeStarred ?? false, includeForks: !(envConfig.github.skipForks ?? false), @@ -302,6 +347,7 @@ export async function initializeConfigFromEnv(): Promise { // Build Gitea config const giteaConfig = { url: envConfig.gitea.url || existingConfig?.[0]?.giteaConfig?.url || '', + provider: envConfig.gitea.provider || existingConfig?.[0]?.giteaConfig?.provider || 'gitea', externalUrl: envConfig.gitea.externalUrl || existingConfig?.[0]?.giteaConfig?.externalUrl || undefined, token: envConfig.gitea.token ? encrypt(envConfig.gitea.token) : existingConfig?.[0]?.giteaConfig?.token || '', defaultOwner: envConfig.gitea.username || existingConfig?.[0]?.giteaConfig?.defaultOwner || '', diff --git a/src/lib/gitea-enhanced.ts b/src/lib/gitea-enhanced.ts index a2d5e01..74cb18c 100644 --- a/src/lib/gitea-enhanced.ts +++ b/src/lib/gitea-enhanced.ts @@ -7,6 +7,7 @@ */ import type { Config } from "@/types/config"; +import { normalizeSourceProviderKind } from "@/lib/source-providers/kinds"; import type { Repository } from "./db/schema"; import type { Octokit } from "@octokit/rest"; import { createGitHubClient } from "./github"; @@ -530,7 +531,9 @@ export async function syncGiteaRepoEnhanced({ if (strategyNeedsDetection(backupStrategy) && !skipForcePushDetection) { try { const decryptedGithubToken = decryptedConfig.githubConfig?.token; - if (decryptedGithubToken) { + // Force-push detection compares branches through the GitHub API, + // so it only runs for GitHub sources. + if (decryptedGithubToken && normalizeSourceProviderKind(config.githubConfig?.provider) === "github") { const fpOctokit = createGitHubClient(decryptedGithubToken); const detectionResult = await detectForcePush({ giteaUrl: config.giteaConfig.url, @@ -753,7 +756,12 @@ export async function syncGiteaRepoEnhanced({ if (metadataOctokit) { return metadataOctokit; } - if (!decryptedConfig.githubConfig?.token) { + // The token belongs to whichever host is configured; only GitHub's + // token can drive Octokit. + if ( + !decryptedConfig.githubConfig?.token || + normalizeSourceProviderKind(config.githubConfig?.provider) !== "github" + ) { return null; } metadataOctokit = createGitHubClient(decryptedConfig.githubConfig.token); diff --git a/src/lib/gitea.ts b/src/lib/gitea.ts index e1818c6..a8e7510 100644 --- a/src/lib/gitea.ts +++ b/src/lib/gitea.ts @@ -13,7 +13,11 @@ import { db, organizations, repositories } from "./db"; import { eq, and, ne } from "drizzle-orm"; import { decryptConfigTokens } from "./utils/config-encryption"; import { formatDateShort } from "./utils"; -import { buildGithubSourceAuthPayload } from "./utils/mirror-source-auth"; +import { buildSourceAuthPayload } from "./utils/mirror-source-auth"; +import { + assertRepositoryMatchesConfiguredSource, + resolveSourceConnection, +} from "./source-providers"; import { parseRepositoryMetadataState, serializeRepositoryMetadataState, @@ -620,7 +624,7 @@ export const mirrorGithubRepoToGitea = async ({ repository, config, }: { - octokit: Octokit; + octokit: Octokit | null; repository: Repository; config: Partial; }): Promise => { @@ -641,6 +645,10 @@ export const mirrorGithubRepoToGitea = async ({ // Decrypt config tokens for API usage const decryptedConfig = decryptConfigTokens(config as Config); + // Never send another host's token: the repository must come from the + // source that is configured now. + assertRepositoryMatchesConfiguredSource({ repository, config }); + // Get the correct owner based on the strategy (with organization overrides) let repoOwner = await getGiteaRepoOwnerAsync({ config, repository }); const mirrorStrategy = config.githubConfig.mirrorStrategy || @@ -968,19 +976,13 @@ export const mirrorGithubRepoToGitea = async ({ // for subsequent mirror fetches. This prevents "terminal prompts disabled" // errors on public repos and raises GitHub API rate limits. { - const githubOwner = - ( - config.githubConfig as typeof config.githubConfig & { - owner?: string; - } - ).owner || ""; - + const sourceConnection = resolveSourceConnection(config); Object.assign( migratePayload, - buildGithubSourceAuthPayload({ - token: decryptedConfig.githubConfig.token, - githubOwner, - githubUsername: config.githubConfig.username, + buildSourceAuthPayload({ + provider: sourceConnection.provider, + token: decryptedConfig.githubConfig?.token, + username: sourceConnection.username, repositoryOwner: repository.owner, }) ); @@ -1010,7 +1012,7 @@ export const mirrorGithubRepoToGitea = async ({ // Mirror releases if enabled (always allowed to rerun for updates). // mirrorOptions already accounts for org/repo overrides and starredCodeOnly. - const shouldMirrorReleases = mirrorOptions.mirrorReleases; + const shouldMirrorReleases = mirrorOptions.mirrorReleases && octokit !== null; console.log( `[Metadata] Release mirroring check: mirrorReleases=${mirrorOptions.mirrorReleases}, isStarred=${repository.isStarred}, starredCodeOnly=${config.githubConfig?.starredCodeOnly}, shouldMirrorReleases=${shouldMirrorReleases}` @@ -1045,7 +1047,7 @@ export const mirrorGithubRepoToGitea = async ({ // The underlying mirror* functions are idempotent: issues/PRs are // matched via [GH-ISSUE #N] / [GH-PR #N] markers and PATCHed in place, // labels are deduped by name, milestones by title. - const shouldMirrorIssuesThisRun = mirrorOptions.mirrorIssues; + const shouldMirrorIssuesThisRun = mirrorOptions.mirrorIssues && octokit !== null; console.log( `[Metadata] Issue mirroring check: mirrorIssues=${mirrorOptions.mirrorIssues}, isStarred=${repository.isStarred}, starredCodeOnly=${config.githubConfig?.starredCodeOnly}, shouldMirrorIssues=${shouldMirrorIssuesThisRun}` @@ -1076,7 +1078,7 @@ export const mirrorGithubRepoToGitea = async ({ } } - const shouldMirrorPullRequests = mirrorOptions.mirrorPullRequests; + const shouldMirrorPullRequests = mirrorOptions.mirrorPullRequests && octokit !== null; console.log( `[Metadata] Pull request mirroring check: mirrorPullRequests=${mirrorOptions.mirrorPullRequests}, isStarred=${repository.isStarred}, starredCodeOnly=${config.githubConfig?.starredCodeOnly}, shouldMirrorPullRequests=${shouldMirrorPullRequests}` @@ -1108,7 +1110,7 @@ export const mirrorGithubRepoToGitea = async ({ // Labels-only path; issues run above already creates/reconciles labels. const shouldMirrorLabels = - mirrorOptions.mirrorLabels && !shouldMirrorIssuesThisRun; + mirrorOptions.mirrorLabels && !shouldMirrorIssuesThisRun && octokit !== null; console.log( `[Metadata] Label mirroring check: mirrorLabels=${mirrorOptions.mirrorLabels}, issuesRunning=${shouldMirrorIssuesThisRun}, isStarred=${repository.isStarred}, starredCodeOnly=${config.githubConfig?.starredCodeOnly}, shouldMirrorLabels=${shouldMirrorLabels}` @@ -1138,7 +1140,7 @@ export const mirrorGithubRepoToGitea = async ({ } } - const shouldMirrorMilestones = mirrorOptions.mirrorMilestones; + const shouldMirrorMilestones = mirrorOptions.mirrorMilestones && octokit !== null; console.log( `[Metadata] Milestone mirroring check: mirrorMilestones=${mirrorOptions.mirrorMilestones}, isStarred=${repository.isStarred}, starredCodeOnly=${config.githubConfig?.starredCodeOnly}, shouldMirrorMilestones=${shouldMirrorMilestones}` @@ -1469,7 +1471,7 @@ export async function mirrorGitHubRepoToGiteaOrg({ giteaOrgId, orgName, }: { - octokit: Octokit; + octokit: Octokit | null; config: Partial; repository: Repository; giteaOrgId: number; @@ -1490,6 +1492,10 @@ export async function mirrorGitHubRepoToGiteaOrg({ // Decrypt config tokens for API usage const decryptedConfig = decryptConfigTokens(config as Config); + // Never send another host's token: the repository must come from the + // source that is configured now. + assertRepositoryMatchesConfiguredSource({ repository, config }); + // Determine the actual repository name to use (handle duplicates for starred repos) let targetRepoName = repository.name; // The org we will record/reuse for. Stays === orgName on the create path @@ -1727,19 +1733,13 @@ export async function mirrorGitHubRepoToGiteaOrg({ // for subsequent mirror fetches. This prevents "terminal prompts disabled" // errors on public repos and raises GitHub API rate limits. { - const githubOwner = - ( - config.githubConfig as typeof config.githubConfig & { - owner?: string; - } - )?.owner || ""; - + const sourceConnection = resolveSourceConnection(config); Object.assign( migratePayload, - buildGithubSourceAuthPayload({ + buildSourceAuthPayload({ + provider: sourceConnection.provider, token: decryptedConfig.githubConfig?.token, - githubOwner, - githubUsername: config.githubConfig?.username, + username: sourceConnection.username, repositoryOwner: repository.owner, }) ); @@ -1768,7 +1768,7 @@ export async function mirrorGitHubRepoToGiteaOrg({ let metadataUpdated = false; // mirrorOptions already accounts for org/repo overrides and starredCodeOnly. - const shouldMirrorReleases = mirrorOptions.mirrorReleases; + const shouldMirrorReleases = mirrorOptions.mirrorReleases && octokit !== null; console.log( `[Metadata] Release mirroring check: mirrorReleases=${mirrorOptions.mirrorReleases}, isStarred=${repository.isStarred}, starredCodeOnly=${config.githubConfig?.starredCodeOnly}, shouldMirrorReleases=${shouldMirrorReleases}` @@ -1801,7 +1801,7 @@ export async function mirrorGitHubRepoToGiteaOrg({ // Reconcile metadata on every sync. See note in mirrorGithubRepoToGitea // above. The underlying mirror* functions are idempotent. - const shouldMirrorIssuesThisRun = mirrorOptions.mirrorIssues; + const shouldMirrorIssuesThisRun = mirrorOptions.mirrorIssues && octokit !== null; console.log( `[Metadata] Issue mirroring check: mirrorIssues=${mirrorOptions.mirrorIssues}, isStarred=${repository.isStarred}, starredCodeOnly=${config.githubConfig?.starredCodeOnly}, shouldMirrorIssues=${shouldMirrorIssuesThisRun}` @@ -1832,7 +1832,7 @@ export async function mirrorGitHubRepoToGiteaOrg({ } } - const shouldMirrorPullRequests = mirrorOptions.mirrorPullRequests; + const shouldMirrorPullRequests = mirrorOptions.mirrorPullRequests && octokit !== null; console.log( `[Metadata] Pull request mirroring check: mirrorPullRequests=${mirrorOptions.mirrorPullRequests}, isStarred=${repository.isStarred}, starredCodeOnly=${config.githubConfig?.starredCodeOnly}, shouldMirrorPullRequests=${shouldMirrorPullRequests}` @@ -1864,7 +1864,7 @@ export async function mirrorGitHubRepoToGiteaOrg({ // Labels-only path; issues run above already creates/reconciles labels. const shouldMirrorLabels = - mirrorOptions.mirrorLabels && !shouldMirrorIssuesThisRun; + mirrorOptions.mirrorLabels && !shouldMirrorIssuesThisRun && octokit !== null; console.log( `[Metadata] Label mirroring check: mirrorLabels=${mirrorOptions.mirrorLabels}, issuesRunning=${shouldMirrorIssuesThisRun}, isStarred=${repository.isStarred}, starredCodeOnly=${config.githubConfig?.starredCodeOnly}, shouldMirrorLabels=${shouldMirrorLabels}` @@ -1894,7 +1894,7 @@ export async function mirrorGitHubRepoToGiteaOrg({ } } - const shouldMirrorMilestones = mirrorOptions.mirrorMilestones; + const shouldMirrorMilestones = mirrorOptions.mirrorMilestones && octokit !== null; console.log( `[Metadata] Milestone mirroring check: mirrorMilestones=${mirrorOptions.mirrorMilestones}, isStarred=${repository.isStarred}, starredCodeOnly=${config.githubConfig?.starredCodeOnly}, shouldMirrorMilestones=${shouldMirrorMilestones}` @@ -2008,7 +2008,7 @@ export async function mirrorGitHubOrgRepoToGiteaOrg({ orgName, }: { config: Partial; - octokit: Octokit; + octokit: Octokit | null; repository: Repository; orgName: string; }) { @@ -2043,7 +2043,7 @@ export async function mirrorGitHubOrgToGitea({ config, }: { organization: Organization; - octokit: Octokit; + octokit: Octokit | null; config: Partial; }) { try { diff --git a/src/lib/recovery.ts b/src/lib/recovery.ts index 9cbde7b..14cbe30 100644 --- a/src/lib/recovery.ts +++ b/src/lib/recovery.ts @@ -9,6 +9,8 @@ import { db, repositories, organizations, mirrorJobs, configs } from './db'; import { eq, and, lt, inArray, sql } from 'drizzle-orm'; import { mirrorGithubRepoToGitea, syncGiteaRepo } from './gitea'; import { createGitHubClient } from './github'; +import type { Octokit } from '@octokit/rest'; +import { resolveSourceProviderKind } from './source-providers'; import { processWithResilience } from './utils/concurrency'; import { repositoryVisibilityEnum, repoStatusEnum } from '@/types/Repository'; import type { Repository } from './db/schema'; @@ -271,15 +273,18 @@ async function recoverMirrorJob(job: any, remainingItemIds: string[]) { throw new Error('GitHub token not found in configuration'); } - // Create GitHub client with error handling and rate limit tracking - let octokit; - try { - const decryptedToken = getDecryptedGitHubToken(config); - const githubUsername = config.githubConfig?.owner || undefined; - const userId = config.userId || undefined; - octokit = createGitHubClient(decryptedToken, userId, githubUsername); - } catch (error) { - throw new Error(`Failed to create GitHub client: ${error instanceof Error ? error.message : String(error)}`); + // Only GitHub sources use an API client while mirroring (metadata). + // Other hosts get code-only mirrors through Gitea, so no client is built. + let octokit: Octokit | null = null; + if (resolveSourceProviderKind(config) === 'github') { + try { + const decryptedToken = getDecryptedGitHubToken(config); + const githubUsername = config.githubConfig?.owner || undefined; + const userId = config.userId || undefined; + octokit = createGitHubClient(decryptedToken, userId, githubUsername); + } catch (error) { + throw new Error(`Failed to create GitHub client: ${error instanceof Error ? error.message : String(error)}`); + } } // Process repositories with resilience and reduced concurrency for recovery diff --git a/src/lib/repo-utils.ts b/src/lib/repo-utils.ts index 2591285..4880a55 100644 --- a/src/lib/repo-utils.ts +++ b/src/lib/repo-utils.ts @@ -1,6 +1,7 @@ import { v4 as uuidv4 } from 'uuid'; import type { GitRepo } from '@/types/Repository'; import { repositories } from '@/lib/db/schema'; +import { getRepositorySource } from '@/lib/source-providers/kinds'; export type RepoInsert = typeof repositories.$inferInsert; @@ -39,6 +40,7 @@ export function normalizeGitRepoToInsert( owner: repo.owner, organization: repo.organization ?? null, mirroredLocation: repo.mirroredLocation || '', + ...repositorySourceColumns(repo), destinationOrg: repo.destinationOrg || null, isPrivate: repo.isPrivate, isForked: repo.isForked, @@ -62,6 +64,14 @@ export function normalizeGitRepoToInsert( }; } +// The source columns for an insert, defaulting to GitHub for legacy producers +export function repositorySourceColumns( + repo: Pick +): Pick { + const source = getRepositorySource(repo); + return { sourceProvider: source.provider, sourceUrl: source.url }; +} + // Compute a safe batch size based on SQLite 999-parameter limit export function calcBatchSizeForInsert(columnCount: number, maxParams = 999): number { if (columnCount <= 0) return 1; diff --git a/src/lib/repository-cleanup-service.ts b/src/lib/repository-cleanup-service.ts index 92f2d82..044b6bf 100644 --- a/src/lib/repository-cleanup-service.ts +++ b/src/lib/repository-cleanup-service.ts @@ -6,9 +6,14 @@ import { db, configs, repositories } from '@/lib/db'; import { eq, and, or, sql, not, inArray } from 'drizzle-orm'; -import { createGitHubClient, getGithubRepositories, getGithubStarredRepositories } from '@/lib/github'; +import { + createSourceProviderFromConfig, + describeSource, + getRepositorySource, + isRepositoryFromConfiguredSource, +} from '@/lib/source-providers'; import { createGiteaClient, deleteGiteaRepo, archiveGiteaRepo, getGiteaRepoOwnerAsync, checkRepoLocation } from '@/lib/gitea'; -import { getDecryptedGitHubToken, getDecryptedGiteaToken } from '@/lib/utils/config-encryption'; +import { getDecryptedGiteaToken } from '@/lib/utils/config-encryption'; import { publishEvent } from '@/lib/events'; import { isMirrorableGitHubRepo } from '@/lib/repo-eligibility'; @@ -79,14 +84,30 @@ export function planOrphanedRepoAction( * These are repositories that exist in our database (and likely in Gitea) * but are no longer in GitHub based on current criteria */ +/** + * Split a stored full name into the owner path and repository name the + * source host expects. Falls back to the stored owner/name for rows whose + * full name does not contain a slash. + */ +export function splitFullName( + fullName: string, + fallbackOwner: string, + fallbackName: string +): { owner: string; name: string } { + const segments = fullName.split('/').filter(Boolean); + if (segments.length < 2) { + return { owner: fallbackOwner, name: fallbackName }; + } + return { owner: segments.slice(0, -1).join('/'), name: segments[segments.length - 1] }; +} + async function identifyOrphanedRepositories(config: any): Promise { const userId = config.userId; try { - // Get current GitHub repositories with rate limit tracking - const decryptedToken = getDecryptedGitHubToken(config); - const githubUsername = config.githubConfig?.owner || undefined; - const octokit = createGitHubClient(decryptedToken, userId, githubUsername); + // The configured source host (GitHub honors GH_API_URL and tracks rate limits) + const sourceProvider = createSourceProviderFromConfig(config, { userId }); + const sourceConnection = sourceProvider.connection; let allGithubRepos = []; let githubApiAccessible = true; @@ -98,14 +119,12 @@ async function identifyOrphanedRepositories(config: any): Promise { // user later removed from the allowlist would be flagged as orphaned and // archived/deleted as soon as the user narrows those filters. const [basicAndForkedRepos, starredRepos] = await Promise.all([ - getGithubRepositories({ - octokit, - config, + sourceProvider.listRepositories(config, { includeCollaboratorReposOverride: true, includeAllOrgsOverride: true, }), config.githubConfig?.includeStarred - ? getGithubStarredRepositories({ octokit, config }) + ? sourceProvider.listStarredRepositories(config) : Promise.resolve([]), ]); @@ -151,6 +170,15 @@ async function identifyOrphanedRepositories(config: any): Promise { return false; } + // Only the configured source can vouch for a repository. Rows imported + // from another host (the source was switched since) are left alone. + if (!isRepositoryFromConfiguredSource(repo, sourceConnection)) { + console.log( + `[Repository Cleanup] Skipping ${repo.fullName} - imported from ${describeSource(getRepositorySource(repo))}, not the configured source` + ); + return false; + } + // If starred repos are not being fetched from GitHub, we can't determine // if a starred repo is orphaned - skip it to prevent data loss if (repo.isStarred && !config.githubConfig?.includeStarred) { @@ -185,19 +213,21 @@ async function identifyOrphanedRepositories(config: any): Promise { // one repo's verification failure can't block the others. const verificationOutcomes = await Promise.allSettled( candidateOrphans.map(async (repo) => { + // Look the repository up by its full name: a GitLab project under a + // subgroup is stored with the top level group as owner, so owner/name + // would not resolve on the source host. + const upstreamPath = splitFullName(repo.fullName, repo.owner, repo.name); + if (repo.isStarred) { try { - await octokit.rest.activity.checkRepoIsStarredByAuthenticatedUser({ - owner: repo.owner, - repo: repo.name, - }); - // Resolves (no throw) => still starred; the bulk star fetch - // missed it. Fail safe: do not treat as orphaned. - return { repo, directCheckConfirmsGone: false }; + const stillStarred = await sourceProvider.isRepositoryStarred( + upstreamPath.owner, + upstreamPath.name + ); + // Still starred => the bulk star fetch missed it. Fail safe: do + // not treat as orphaned. A clean "not starred" confirms it. + return { repo, directCheckConfirmsGone: !stillStarred }; } catch (starError: any) { - if (starError?.status === 404) { - return { repo, directCheckConfirmsGone: true }; - } console.warn( `[Repository Cleanup] Direct star-check for ${repo.fullName} failed with a non-404 error; skipping this cycle to be safe: ${ starError instanceof Error ? starError.message : String(starError) @@ -208,14 +238,14 @@ async function identifyOrphanedRepositories(config: any): Promise { } try { - await octokit.rest.repos.get({ owner: repo.owner, repo: repo.name }); - // Resolves (no throw) => repo still exists; the bulk fetch missed - // it (e.g. an org-allowlist edge case). Fail safe: not orphaned. - return { repo, directCheckConfirmsGone: false }; + const upstream = await sourceProvider.getRepository( + upstreamPath.owner, + upstreamPath.name + ); + // Present => the bulk fetch missed it (e.g. an org-allowlist edge + // case). Fail safe: not orphaned. A clean 404 (null) confirms it. + return { repo, directCheckConfirmsGone: upstream === null }; } catch (repoError: any) { - if (repoError?.status === 404) { - return { repo, directCheckConfirmsGone: true }; - } console.warn( `[Repository Cleanup] Direct existence check for ${repo.fullName} failed with a non-404 error; skipping this cycle to be safe: ${ repoError instanceof Error ? repoError.message : String(repoError) diff --git a/src/lib/scheduler-service.ts b/src/lib/scheduler-service.ts index fac0b85..78b472e 100644 --- a/src/lib/scheduler-service.ts +++ b/src/lib/scheduler-service.ts @@ -10,6 +10,7 @@ import { syncGiteaRepo, mirrorGithubRepoToGitea } from '@/lib/gitea'; import { getDecryptedGitHubToken } from '@/lib/utils/config-encryption'; import { formatDuration } from '@/lib/utils/duration-parser'; import type { Repository } from '@/lib/db/schema'; +import type { Octokit } from '@octokit/rest'; import { repoStatusEnum, repositoryVisibilityEnum } from '@/types/Repository'; import { mergeGitReposPreferStarred, normalizeGitRepoToInsert, calcBatchSizeForInsert } from '@/lib/repo-utils'; import { isMirrorableGitHubRepo } from '@/lib/repo-eligibility'; @@ -98,21 +99,19 @@ async function runScheduledSync(config: any): Promise { // Auto-discovery: Check for new GitHub repositories if (scheduleConfig.autoImport !== false) { - console.log(`[Scheduler] Checking for new GitHub repositories for user ${userId}...`); + console.log(`[Scheduler] Checking for new source repositories for user ${userId}...`); try { - const { getGithubRepositories, getGithubStarredRepositories, createGitHubClient } = await import('@/lib/github'); + const { createSourceProviderFromConfig } = await import('@/lib/source-providers'); const { v4: uuidv4 } = await import('uuid'); - const { getDecryptedGitHubToken } = await import('@/lib/utils/config-encryption'); - // Create GitHub client (honors GH_API_URL for GHES / GHEC data residency) - const decryptedToken = getDecryptedGitHubToken(config); - const octokit = createGitHubClient(decryptedToken, userId, config.githubConfig?.owner); + // The configured source host (GitHub honors GH_API_URL for GHES / GHEC) + const sourceProvider = createSourceProviderFromConfig(config, { userId }); - // Fetch GitHub data + // Fetch source data const [basicAndForkedRepos, starredRepos] = await Promise.all([ - getGithubRepositories({ octokit, config }), + sourceProvider.listRepositories(config), config.githubConfig?.includeStarred - ? getGithubStarredRepositories({ octokit, config }) + ? sourceProvider.listStarredRepositories(config) : Promise.resolve([]), ]); const allGithubRepos = mergeGitReposPreferStarred(basicAndForkedRepos, starredRepos); @@ -241,10 +240,15 @@ async function runScheduledSync(config: any): Promise { if (reposNeedingMirror.length > 0) { console.log(`[Scheduler] Found ${reposNeedingMirror.length} repositories that need initial mirroring`); - // Prepare Octokit client (honors GH_API_URL for GHES / GHEC data residency) - const decryptedToken = getDecryptedGitHubToken(config); - const { createGitHubClient } = await import('@/lib/github'); - const octokit = createGitHubClient(decryptedToken, userId, config.githubConfig?.owner); + // Only GitHub sources need an API client while mirroring (metadata). + // Other hosts get code-only mirrors through Gitea. + const { resolveSourceProviderKind } = await import('@/lib/source-providers'); + let octokit: Octokit | null = null; + if (resolveSourceProviderKind(config) === 'github') { + const decryptedToken = getDecryptedGitHubToken(config); + const { createGitHubClient } = await import('@/lib/github'); + octokit = createGitHubClient(decryptedToken, userId, config.githubConfig?.owner); + } // Process repositories in batches const batchSize = scheduleConfig.batchSize || 10; @@ -510,19 +514,18 @@ async function performInitialAutoStart(): Promise { try { // Step 1: Import repositories from GitHub - console.log(`[Scheduler] Step 1: Importing repositories from GitHub for user ${config.userId}...`); - const { getGithubRepositories, getGithubStarredRepositories, createGitHubClient } = await import('@/lib/github'); + console.log(`[Scheduler] Step 1: Importing repositories from the configured source for user ${config.userId}...`); + const { createSourceProviderFromConfig } = await import('@/lib/source-providers'); const { v4: uuidv4 } = await import('uuid'); - // Create GitHub client (honors GH_API_URL for GHES / GHEC data residency) - const decryptedToken = getDecryptedGitHubToken(config); - const octokit = createGitHubClient(decryptedToken, config.userId, config.githubConfig?.owner); + // The configured source host (GitHub honors GH_API_URL for GHES / GHEC) + const sourceProvider = createSourceProviderFromConfig(config, { userId: config.userId }); - // Fetch GitHub data + // Fetch source data const [basicAndForkedRepos, starredRepos] = await Promise.all([ - getGithubRepositories({ octokit, config }), + sourceProvider.listRepositories(config), config.githubConfig?.includeStarred - ? getGithubStarredRepositories({ octokit, config }) + ? sourceProvider.listStarredRepositories(config) : Promise.resolve([]), ]); const allGithubRepos = mergeGitReposPreferStarred(basicAndForkedRepos, starredRepos); @@ -649,8 +652,15 @@ async function performInitialAutoStart(): Promise { if (reposNeedingMirror.length > 0) { console.log(`[Scheduler] Found ${reposNeedingMirror.length} repositories that need mirroring`); - // Reuse the octokit instance from above - // (octokit was already created in the import phase) + // Only GitHub sources need an API client while mirroring (metadata). + // Other hosts get code-only mirrors through Gitea. + const { resolveSourceProviderKind } = await import('@/lib/source-providers'); + let octokit: Octokit | null = null; + if (resolveSourceProviderKind(config) === 'github') { + const decryptedToken = getDecryptedGitHubToken(config); + const { createGitHubClient } = await import('@/lib/github'); + octokit = createGitHubClient(decryptedToken, config.userId, config.githubConfig?.owner); + } // Process repositories in batches const batchSize = config.scheduleConfig?.batchSize || 5; diff --git a/src/lib/source-provider-wiring.test.ts b/src/lib/source-provider-wiring.test.ts new file mode 100644 index 0000000..50d9f55 --- /dev/null +++ b/src/lib/source-provider-wiring.test.ts @@ -0,0 +1,112 @@ +/** + * Source regression tests for the source provider wiring (issue #375). + * + * The mirror pipeline, the scheduler, the cleanup service and the discovery + * routes must go through the SourceProvider built from the configuration + * instead of calling GitHub directly, and the two migrate sites must pick + * clone credentials per provider and refuse a repository from another host. + * These files need process-wide module mocks to exercise behaviorally, so + * the wiring is asserted by reading the source (same convention as + * stuck-status-recovery.test.ts). + */ +import { describe, expect, test } from "bun:test"; +import { readFileSync } from "node:fs"; +import { join } from "node:path"; + +const read = (...segments: string[]) => + readFileSync(join(import.meta.dir, ...segments), "utf8"); + +const count = (haystack: string, needle: string) => haystack.split(needle).length - 1; + +describe("gitea.ts migrate sites", () => { + const source = read("gitea.ts"); + + test("both migrate payloads take per-provider clone credentials", () => { + expect(count(source, "buildSourceAuthPayload({")).toBe(2); + expect(source).not.toContain("buildGithubSourceAuthPayload"); + expect(count(source, "provider: sourceConnection.provider,")).toBe(2); + }); + + test("both mirror paths refuse a repository from a different source before any side effect", () => { + expect(count(source, "assertRepositoryMatchesConfiguredSource({ repository, config });")).toBe(2); + const userPath = source.indexOf("export const mirrorGithubRepoToGitea"); + const guard = source.indexOf("assertRepositoryMatchesConfiguredSource({ repository, config });", userPath); + const mirroringWrite = source.indexOf('repoStatusEnum.parse("mirroring")', userPath); + expect(guard).toBeGreaterThan(userPath); + expect(guard).toBeLessThan(mirroringWrite); + }); + + test("the mirror entry points accept a null GitHub client and skip metadata without one", () => { + expect(count(source, "octokit: Octokit | null;")).toBeGreaterThanOrEqual(4); + expect(count(source, "&& octokit !== null;")).toBe(10); + }); +}); + +describe("discovery and housekeeping go through the source provider", () => { + test("scheduler auto import, auto mirror and boot auto start", () => { + const source = read("scheduler-service.ts"); + expect(count(source, "createSourceProviderFromConfig(config")).toBe(2); + expect(count(source, "sourceProvider.listRepositories(config)")).toBe(2); + expect(source).not.toContain("getGithubRepositories("); + // A GitHub client is built only for GitHub sources, in both mirror phases. + expect(count(source, "resolveSourceProviderKind(config) === 'github'")).toBe(2); + }); + + test("cleanup lists, filters and verifies through the provider", () => { + const source = read("repository-cleanup-service.ts"); + expect(source).toContain("createSourceProviderFromConfig(config, { userId })"); + expect(source).toContain("isRepositoryFromConfiguredSource(repo, sourceConnection)"); + expect(source).toContain("sourceProvider.getRepository("); + expect(source).toContain("sourceProvider.isRepositoryStarred("); + expect(source).not.toContain("octokit.rest"); + // The repository existence check goes by full name, not owner/name. + expect(source).toContain("splitFullName(repo.fullName, repo.owner, repo.name)"); + }); + + test("recovery and the sync path only build a GitHub client for GitHub sources", () => { + expect(read("recovery.ts")).toContain("resolveSourceProviderKind(config) === 'github'"); + const enhanced = read("gitea-enhanced.ts"); + expect(count(enhanced, 'normalizeSourceProviderKind(config.githubConfig?.provider)')).toBe(2); + }); + + test("the metadata resolver clamps GitHub only options for other sources", () => { + const source = read("utils", "mirror-overrides.ts"); + expect(source).toContain("GITHUB_ONLY_METADATA_KEYS"); + expect(source).toContain('normalizeSourceProviderKind(repository.sourceProvider) !== "github"'); + }); + + test("the discovery routes use the provider and stamp the source on inserted rows", () => { + for (const route of ["index.ts", "organization.ts", "repository.ts"]) { + const source = read("..", "pages", "api", "sync", route); + expect(source).toContain("createSourceProviderFromConfig(config, { userId })"); + expect(source).not.toContain("createGitHubClient("); + } + expect(read("repo-utils.ts")).toContain("repositorySourceColumns(repo)"); + }); +}); + +describe("source and destination locks", () => { + test("the config API reports locks and refuses unconfirmed changes to a locked host", () => { + const source = read("..", "pages", "api", "config", "index.ts"); + expect(source).toContain("loadConfigLocks(userId)"); + expect(source).toContain("evaluateConfigChange({"); + expect(source).toContain("status: 409"); + // Every GET branch carries the locks so the page can disable the fields. + expect(count(source, "locks,\n")).toBeGreaterThanOrEqual(3); + }); + + test("environment variables cannot switch a locked host on boot", () => { + const source = read("env-config-loader.ts"); + expect(source).toContain("hasSourceChanged(stored.githubConfig, incomingSource)"); + expect(source).toContain("hasDestinationChanged(stored.giteaConfig?.url, envConfig.gitea.url)"); + }); + + test("both connection cards disable the host fields while locked and confirm through the dialog", () => { + const github = read("..", "components", "config", "GitHubConfigForm.tsx"); + expect(github).toContain("disabled={sourceLocked}"); + expect(github).toContain("confirmSourceChange: sourceUnlocked"); + const gitea = read("..", "components", "config", "GiteaConfigForm.tsx"); + expect(gitea).toContain("disabled={destinationLocked}"); + expect(gitea).toContain("confirmDestinationChange: destinationUnlocked"); + }); +}); diff --git a/src/lib/source-providers/filters.test.ts b/src/lib/source-providers/filters.test.ts new file mode 100644 index 0000000..e5e40c2 --- /dev/null +++ b/src/lib/source-providers/filters.test.ts @@ -0,0 +1,93 @@ +import { describe, expect, test } from "bun:test"; +import type { GitRepo } from "@/types/Repository"; +import { filterSourceRepositories } from "./filters"; + +function repo(overrides: Partial & { owner: string; name: string }): GitRepo { + return { + fullName: `${overrides.owner}/${overrides.name}`, + url: `https://example.com/${overrides.owner}/${overrides.name}`, + cloneUrl: `https://example.com/${overrides.owner}/${overrides.name}.git`, + isPrivate: false, + isForked: false, + hasIssues: true, + isStarred: false, + isArchived: false, + size: 0, + hasLFS: false, + hasSubmodules: false, + defaultBranch: "main", + visibility: "public", + status: "imported", + importedAt: new Date(), + createdAt: new Date(), + updatedAt: new Date(), + ...overrides, + }; +} + +const own = repo({ owner: "me", name: "mine" }); +const collaborator = repo({ owner: "friend", name: "shared" }); +const acme = repo({ owner: "acme", name: "tool", organization: "acme" }); +const globex = repo({ owner: "globex", name: "app", organization: "globex" }); +const fork = repo({ owner: "me", name: "forked", isForked: true }); +const all = [own, collaborator, acme, globex, fork]; + +describe("filterSourceRepositories", () => { + test("keeps everything with the default config", () => { + const kept = filterSourceRepositories(all, { config: {}, username: "me" }); + expect(kept).toHaveLength(5); + }); + + test("skipForks drops forks", () => { + const kept = filterSourceRepositories(all, { + config: { githubConfig: { skipForks: true } as any }, + username: "me", + }); + expect(kept.map((r) => r.name)).not.toContain("forked"); + expect(kept).toHaveLength(4); + }); + + test("skipPersonalRepos drops the configured account's own repos, not org repos", () => { + const kept = filterSourceRepositories(all, { + config: { githubConfig: { skipPersonalRepos: true } as any }, + username: "ME", + }); + expect(kept.map((r) => r.name).sort()).toEqual(["app", "shared", "tool"]); + }); + + test("includeCollaboratorRepos=false drops other users' personal repos only", () => { + const kept = filterSourceRepositories(all, { + config: { githubConfig: { includeCollaboratorRepos: false } as any }, + username: "me", + }); + expect(kept.map((r) => r.name)).not.toContain("shared"); + expect(kept.map((r) => r.name)).toContain("tool"); + expect(kept.map((r) => r.name)).toContain("mine"); + }); + + test("the collaborator override wins over the config", () => { + const kept = filterSourceRepositories(all, { + config: { githubConfig: { includeCollaboratorRepos: false } as any }, + options: { includeCollaboratorReposOverride: true }, + username: "me", + }); + expect(kept.map((r) => r.name)).toContain("shared"); + }); + + test("an organization allowlist keeps only listed org repos and leaves personal repos alone", () => { + const kept = filterSourceRepositories(all, { + config: { githubConfig: { includeOrganizations: [" Acme "] } as any }, + username: "me", + }); + expect(kept.map((r) => r.name).sort()).toEqual(["forked", "mine", "shared", "tool"]); + }); + + test("includeAllOrgsOverride ignores the allowlist", () => { + const kept = filterSourceRepositories(all, { + config: { githubConfig: { includeOrganizations: ["acme"] } as any }, + options: { includeAllOrgsOverride: true }, + username: "me", + }); + expect(kept).toHaveLength(5); + }); +}); diff --git a/src/lib/source-providers/filters.ts b/src/lib/source-providers/filters.ts new file mode 100644 index 0000000..3de208c --- /dev/null +++ b/src/lib/source-providers/filters.ts @@ -0,0 +1,55 @@ +import type { Config } from "@/lib/db/schema"; +import type { GitRepo } from "@/types/Repository"; +import type { ListRepositoriesOptions } from "./types"; + +/** + * Apply the repository selection filters from the source config to a list of + * already mapped repositories. Mirrors the rules getGithubRepositories applies + * inline for GitHub so GitLab and Gitea sources behave the same way: + * + * - skipForks drops forks + * - skipPersonalRepos drops repos the configured account owns + * - includeCollaboratorRepos=false drops personal repos of other users + * - includeOrganizations, when non-empty, keeps only org repos from listed orgs + */ +export function filterSourceRepositories( + repos: GitRepo[], + { + config, + options = {}, + username, + }: { + config: Partial; + options?: ListRepositoriesOptions; + username: string; + } +): GitRepo[] { + const includeCollab = + options.includeCollaboratorReposOverride ?? + config.githubConfig?.includeCollaboratorRepos ?? + true; + const skipForks = config.githubConfig?.skipForks ?? false; + const skipPersonalRepos = config.githubConfig?.skipPersonalRepos ?? false; + const includeOrgs = options.includeAllOrgsOverride + ? [] + : config.githubConfig?.includeOrganizations ?? []; + const allowedOrgs = new Set( + includeOrgs.map((org) => org.trim().toLowerCase()).filter(Boolean) + ); + const me = username.trim().toLowerCase(); + + return repos.filter((repo) => { + if (skipForks && repo.isForked) return false; + + const ownerKey = repo.owner.trim().toLowerCase(); + const orgKey = (repo.organization ?? "").trim().toLowerCase(); + const isOrgRepo = orgKey.length > 0; + const isOwnRepo = !isOrgRepo && me.length > 0 && ownerKey === me; + + if (skipPersonalRepos && isOwnRepo) return false; + if (!includeCollab && !isOrgRepo && !isOwnRepo) return false; + if (allowedOrgs.size > 0 && isOrgRepo && !allowedOrgs.has(orgKey)) return false; + + return true; + }); +} diff --git a/src/lib/source-providers/gitea-source.test.ts b/src/lib/source-providers/gitea-source.test.ts new file mode 100644 index 0000000..14ea3fa --- /dev/null +++ b/src/lib/source-providers/gitea-source.test.ts @@ -0,0 +1,200 @@ +import { afterEach, beforeEach, describe, expect, mock, test } from "bun:test"; +import { + GiteaSourceProvider, + mapGiteaRepository, + resolveFlatRepositoryPath, + type GiteaRepository, +} from "./gitea-source"; + +type Call = { url: string; headers: Record }; + +function json(body: unknown, init: { status?: number; headers?: Record } = {}) { + return new Response(body === undefined ? null : JSON.stringify(body), { + status: init.status ?? 200, + headers: { "content-type": "application/json", ...(init.headers ?? {}) }, + }); +} + +function repo(overrides: Partial & { name: string; owner: string }): GiteaRepository { + const { owner, ...rest } = overrides; + return { + id: 1, + full_name: `${owner}/${overrides.name}`, + html_url: `https://codeberg.org/${owner}/${overrides.name}`, + clone_url: `https://codeberg.org/${owner}/${overrides.name}.git`, + owner: { login: owner, username: owner }, + private: false, + fork: false, + has_issues: true, + archived: false, + size: 42, + default_branch: "main", + created_at: "2024-01-01T00:00:00Z", + updated_at: "2024-06-01T00:00:00Z", + ...rest, + }; +} + +const connection = { + provider: "gitea" as const, + url: "https://codeberg.org", + username: "me", + token: "cb-secret", +}; + +let originalFetch: typeof globalThis.fetch; +let calls: Call[]; + +function installFetch(route: (url: string) => Response | Promise) { + globalThis.fetch = mock(async (input: RequestInfo | URL, init?: RequestInit) => { + calls.push({ url: String(input), headers: (init?.headers as Record) ?? {} }); + return route(String(input)); + }) as unknown as typeof globalThis.fetch; +} + +beforeEach(() => { + originalFetch = globalThis.fetch; + calls = []; +}); + +afterEach(() => { + globalThis.fetch = originalFetch; +}); + +describe("mapGiteaRepository", () => { + test("maps the repository payload and stamps the source", () => { + const mapped = mapGiteaRepository( + repo({ name: "tool", owner: "acme", fork: true, parent: { full_name: "upstream/tool" }, language: "Go" }), + connection, + { isOrganization: true } + ); + expect(mapped.fullName).toBe("acme/tool"); + expect(mapped.owner).toBe("acme"); + expect(mapped.organization).toBe("acme"); + expect(mapped.isForked).toBe(true); + expect(mapped.forkedFrom).toBe("upstream/tool"); + expect(mapped.language).toBe("Go"); + expect(mapped.size).toBe(42); + expect(mapped.sourceProvider).toBe("gitea"); + expect(mapped.sourceUrl).toBe("https://codeberg.org"); + }); + + test("private and internal repositories are both private for the mirror", () => { + expect(mapGiteaRepository(repo({ name: "a", owner: "me", private: true }), connection).visibility).toBe("private"); + const internal = mapGiteaRepository(repo({ name: "b", owner: "me", internal: true }), connection); + expect(internal.visibility).toBe("internal"); + expect(internal.isPrivate).toBe(true); + expect(mapGiteaRepository(repo({ name: "c", owner: "me" }), connection).isPrivate).toBe(false); + }); +}); + +describe("resolveFlatRepositoryPath", () => { + test("takes the first two segments and drops .git", () => { + expect(resolveFlatRepositoryPath(["forgejo", "forgejo.git", "src", "branch", "main"])).toEqual({ + owner: "forgejo", + repo: "forgejo", + }); + expect(resolveFlatRepositoryPath(["only"])).toBeNull(); + }); +}); + +describe("GiteaSourceProvider", () => { + test("lists the account's repositories, telling org owners apart with one lookup per owner", async () => { + installFetch((url) => { + if (url.includes("/api/v1/user/orgs?")) return json([{ username: "acme" }], { headers: { "x-total-count": "1" } }); + if (url.includes("/api/v1/user/repos?")) { + return json( + [ + repo({ name: "mine", owner: "me" }), + repo({ name: "tool", owner: "acme" }), + repo({ name: "shared", owner: "bob" }), + repo({ name: "shared2", owner: "bob" }), + ], + { headers: { "x-total-count": "4" } } + ); + } + if (url.endsWith("/api/v1/orgs/bob")) return json({ message: "not found" }, { status: 404 }); + return json({ message: "unexpected" }, { status: 500 }); + }); + + const repos = await new GiteaSourceProvider(connection).listRepositories({}); + + expect(calls[0].headers.Authorization).toBe("token cb-secret"); + expect(calls[1].url).toBe("https://codeberg.org/api/v1/user/repos?limit=50&page=1"); + expect(repos.map((r) => [r.fullName, r.organization ?? null])).toEqual([ + ["me/mine", null], + ["acme/tool", "acme"], + ["bob/shared", null], + ["bob/shared2", null], + ]); + expect(calls.filter((c) => c.url.endsWith("/api/v1/orgs/bob"))).toHaveLength(1); + }); + + test("pages through starred repositories until x-total-count is reached", async () => { + const page = (start: number, count: number) => + Array.from({ length: count }, (_, i) => repo({ name: `r${start + i}`, owner: "me" })); + installFetch((url) => { + if (url.includes("/api/v1/user/starred?limit=50&page=1")) return json(page(0, 50), { headers: { "x-total-count": "60" } }); + if (url.includes("/api/v1/user/starred?limit=50&page=2")) return json(page(50, 10), { headers: { "x-total-count": "60" } }); + return json({ message: "unexpected" }, { status: 500 }); + }); + + const repos = await new GiteaSourceProvider(connection).listStarredRepositories({}); + expect(repos).toHaveLength(60); + expect(repos.every((r) => r.isStarred)).toBe(true); + expect(calls).toHaveLength(2); + }); + + test("without a token it lists the user's public repositories", async () => { + installFetch((url) => { + if (url.includes("/api/v1/users/me/orgs?")) return json([]); + if (url.includes("/api/v1/users/me/repos?")) return json([repo({ name: "pub", owner: "me" })]); + return json({ message: "unexpected" }, { status: 500 }); + }); + const repos = await new GiteaSourceProvider({ ...connection, token: "" }).listRepositories({}); + expect(repos.map((r) => r.fullName)).toEqual(["me/pub"]); + expect(calls.every((c) => c.headers.Authorization === undefined)).toBe(true); + }); + + test("getRepository returns null on 404 and marks org owned repositories", async () => { + installFetch((url) => { + if (url.endsWith("/api/v1/repos/acme/tool")) return json(repo({ name: "tool", owner: "acme" })); + if (url.endsWith("/api/v1/orgs/acme")) return json({ username: "acme" }); + if (url.endsWith("/api/v1/repos/acme/missing")) return json({ message: "not found" }, { status: 404 }); + return json({ message: "unexpected" }, { status: 500 }); + }); + const provider = new GiteaSourceProvider(connection); + const found = await provider.getRepository("acme", "tool"); + expect(found?.organization).toBe("acme"); + expect(await provider.getRepository("acme", "missing")).toBeNull(); + }); + + test("isRepositoryStarred maps 204 to true and 404 to false", async () => { + installFetch((url) => { + if (url.endsWith("/api/v1/user/starred/acme/tool")) return new Response(null, { status: 204 }); + return json({ message: "not found" }, { status: 404 }); + }); + const provider = new GiteaSourceProvider(connection); + expect(await provider.isRepositoryStarred("acme", "tool")).toBe(true); + expect(await provider.isRepositoryStarred("acme", "other")).toBe(false); + }); + + test("lists organizations with their repository counts", async () => { + installFetch((url) => { + if (url.includes("/api/v1/user/orgs?")) return json([{ username: "acme", avatar_url: "https://a/acme.png" }]); + if (url.includes("/api/v1/orgs/acme/repos?limit=1")) return json([{}], { headers: { "x-total-count": "7" } }); + return json({ message: "unexpected" }, { status: 500 }); + }); + const { organizations } = await new GiteaSourceProvider(connection).listOrganizations({}); + expect(organizations).toEqual([ + expect.objectContaining({ name: "acme", avatarUrl: "https://a/acme.png", repositoryCount: 7 }), + ]); + }); + + test("testConnection reads /user", async () => { + installFetch(() => json({ login: "me", full_name: "Me", avatar_url: "https://a/me.png" })); + const account = await new GiteaSourceProvider(connection).testConnection(); + expect(calls[0].url).toBe("https://codeberg.org/api/v1/user"); + expect(account).toEqual({ login: "me", name: "Me", avatarUrl: "https://a/me.png" }); + }); +}); diff --git a/src/lib/source-providers/gitea-source.ts b/src/lib/source-providers/gitea-source.ts new file mode 100644 index 0000000..d36e88d --- /dev/null +++ b/src/lib/source-providers/gitea-source.ts @@ -0,0 +1,384 @@ +/** + * Gitea and Forgejo source adapter (Codeberg, self hosted instances). + * + * Talks to the v1 REST API with a token in the Authorization header. Gitea's + * repository payload does not say whether the owner is a user or an + * organization, so the adapter checks /orgs/{name} once per distinct owner + * and remembers the answer for the rest of the run. + */ +import type { Config } from "@/lib/db/schema"; +import type { GitRepo, RepositoryVisibility } from "@/types/Repository"; +import type { GitOrg } from "@/types/organizations"; +import { filterSourceRepositories } from "./filters"; +import { + isSourceNotFound, + sourceFetch, + stripGitSuffix, + toDate, + withQuery, +} from "./http"; +import type { + ListRepositoriesOptions, + SourceAccount, + SourceConnection, + SourceFailedOrganization, + SourceOrganizationResult, + SourceProvider, + SourceRepositoryPath, +} from "./types"; + +// Gitea caps page size at MAX_RESPONSE_ITEMS, 50 by default. +const PAGE_SIZE = 50; +const MAX_PAGES = 1000; + +export interface GiteaUser { + id?: number; + login?: string; + username?: string; + full_name?: string | null; + avatar_url?: string | null; +} + +export interface GiteaRepository { + id?: number; + name: string; + full_name?: string; + html_url?: string; + clone_url?: string; + owner?: GiteaUser; + private?: boolean; + internal?: boolean; + fork?: boolean; + parent?: { full_name?: string } | null; + has_issues?: boolean; + archived?: boolean; + size?: number; + language?: string | null; + description?: string | null; + default_branch?: string; + created_at?: string; + updated_at?: string; +} + +export interface GiteaOrganization { + id?: number; + username?: string; + name?: string; + full_name?: string; + avatar_url?: string | null; +} + +export function mapGiteaRepository( + repo: GiteaRepository, + connection: Pick, + options: { isStarred?: boolean; isOrganization?: boolean } = {} +): GitRepo { + const owner = + repo.owner?.login || repo.owner?.username || repo.full_name?.split("/")[0] || "unknown"; + const fullName = repo.full_name || `${owner}/${repo.name}`; + const visibility: RepositoryVisibility = repo.private + ? "private" + : repo.internal + ? "internal" + : "public"; + + return { + name: repo.name || fullName.split("/").at(-1) || "repository", + fullName, + url: repo.html_url || `${connection.url}/${fullName}`, + cloneUrl: repo.clone_url || `${connection.url}/${fullName}.git`, + + owner, + organization: options.isOrganization ? owner : undefined, + mirroredLocation: "", + destinationOrg: null, + + isPrivate: visibility !== "public", + isForked: Boolean(repo.fork), + forkedFrom: repo.parent?.full_name ?? undefined, + + hasIssues: repo.has_issues !== false, + isStarred: options.isStarred ?? false, + isArchived: Boolean(repo.archived), + + size: Number(repo.size) || 0, + hasLFS: false, + hasSubmodules: false, + + language: repo.language ?? null, + description: repo.description ?? null, + defaultBranch: repo.default_branch || "main", + visibility, + + status: "imported", + isDisabled: false, + + importedAt: new Date(), + createdAt: toDate(repo.created_at), + updatedAt: toDate(repo.updated_at), + + sourceProvider: "gitea", + sourceUrl: connection.url, + }; +} + +export function mapGiteaOrganization(org: GiteaOrganization, repositoryCount: number): GitOrg { + return { + name: org.username || org.name || "", + avatarUrl: org.avatar_url || "", + membershipRole: "member", + isIncluded: false, + status: "imported", + repositoryCount, + createdAt: new Date(), + updatedAt: new Date(), + }; +} + +/** Gitea namespaces are flat: the first two segments name the repository. */ +export function resolveFlatRepositoryPath(segments: string[]): SourceRepositoryPath | null { + const [owner, rawRepo] = segments.map((segment) => segment.trim()); + if (!owner || !rawRepo) return null; + const repo = stripGitSuffix(rawRepo); + if (!repo) return null; + return { owner, repo }; +} + +export class GiteaSourceProvider implements SourceProvider { + readonly kind = "gitea" as const; + private readonly apiRoot: string; + private readonly organizationCache = new Map(); + + constructor(readonly connection: SourceConnection) { + this.apiRoot = `${connection.url}/api/v1`; + } + + private get hasToken(): boolean { + return this.connection.token.trim().length > 0; + } + + private headers(): Record { + return this.hasToken ? { Authorization: `token ${this.connection.token}` } : {}; + } + + private encodedUser(): string { + return encodeURIComponent(this.connection.username); + } + + private async paginate( + path: string, + params: Record = {} + ): Promise { + const items: T[] = []; + for (let page = 1; page <= MAX_PAGES; page += 1) { + const url = withQuery(`${this.apiRoot}${path}`, { + ...params, + limit: PAGE_SIZE, + page, + }); + const { data, response } = await sourceFetch(url, { headers: this.headers() }); + if (!Array.isArray(data) || data.length === 0) break; + items.push(...data); + + const header = response.headers.get("x-total-count"); + const total = header === null ? Number.NaN : Number(header); + if (Number.isFinite(total)) { + if (items.length >= total) break; + } else if (data.length < PAGE_SIZE) { + break; + } + } + return items; + } + + /** Total item count for a list endpoint, from the x-total-count header. */ + private async total(path: string): Promise { + const url = withQuery(`${this.apiRoot}${path}`, { limit: 1, page: 1 }); + const { data, response } = await sourceFetch(url, { headers: this.headers() }); + const header = response.headers.get("x-total-count"); + const parsed = header === null ? Number.NaN : Number(header); + if (Number.isFinite(parsed)) return parsed; + return Array.isArray(data) ? data.length : 0; + } + + /** Whether a repository owner is an organization, cached per run. */ + private async isOrganization(login: string): Promise { + const key = login.trim().toLowerCase(); + if (!key) return false; + if (key === this.connection.username.trim().toLowerCase()) return false; + + const cached = this.organizationCache.get(key); + if (cached !== undefined) return cached; + + try { + await sourceFetch( + `${this.apiRoot}/orgs/${encodeURIComponent(login)}`, + { headers: this.headers() } + ); + this.organizationCache.set(key, true); + return true; + } catch (error) { + if (isSourceNotFound(error)) { + this.organizationCache.set(key, false); + } + return false; + } + } + + /** Load the account's organizations and seed the owner cache with them. */ + private async loadOrganizations(): Promise { + const orgs = this.hasToken + ? await this.paginate("/user/orgs") + : await this.paginate(`/users/${this.encodedUser()}/orgs`); + for (const org of orgs) { + const name = org.username || org.name; + if (name) this.organizationCache.set(name.toLowerCase(), true); + } + return orgs; + } + + private async mapRepositories( + repos: GiteaRepository[], + options: { isStarred?: boolean } = {} + ): Promise { + const mapped: GitRepo[] = []; + for (const repo of repos) { + const owner = repo.owner?.login || repo.owner?.username || ""; + const isOrganization = await this.isOrganization(owner); + mapped.push(mapGiteaRepository(repo, this.connection, { ...options, isOrganization })); + } + return mapped; + } + + async listRepositories( + config: Partial, + options: ListRepositoriesOptions = {} + ): Promise { + await this.loadOrganizations(); + const repos = this.hasToken + ? await this.paginate("/user/repos") + : await this.paginate(`/users/${this.encodedUser()}/repos`); + + const mapped = await this.mapRepositories(repos); + return filterSourceRepositories(mapped, { + config, + options, + username: this.connection.username, + }); + } + + async listStarredRepositories(_config: Partial): Promise { + const repos = this.hasToken + ? await this.paginate("/user/starred") + : await this.paginate(`/users/${this.encodedUser()}/starred`); + return this.mapRepositories(repos, { isStarred: true }); + } + + async listOrganizations( + _config: Partial, + skipOrgNames?: Set + ): Promise { + const orgs = await this.loadOrganizations(); + const organizations: GitOrg[] = []; + const failedOrgs: SourceFailedOrganization[] = []; + + for (const org of orgs) { + const name = org.username || org.name || ""; + if (!name) continue; + if (skipOrgNames?.has(name.toLowerCase())) { + console.log(`Skipping organization ${name} - ignored by user`); + continue; + } + + try { + const count = await this.total(`/orgs/${encodeURIComponent(name)}/repos`); + organizations.push(mapGiteaOrganization(org, count)); + } catch (error) { + const reason = error instanceof Error ? error.message : String(error); + console.warn(`Failed to import organization ${name} - ${reason}`); + failedOrgs.push({ name, avatarUrl: org.avatar_url || "", reason }); + } + } + + return { organizations, failedOrgs }; + } + + async getOrganization(name: string): Promise { + try { + const { data } = await sourceFetch( + `${this.apiRoot}/orgs/${encodeURIComponent(name)}`, + { headers: this.headers() } + ); + this.organizationCache.set(name.toLowerCase(), true); + let count = 0; + try { + count = await this.total(`/orgs/${encodeURIComponent(name)}/repos`); + } catch { + // The count is informational only. + } + return mapGiteaOrganization(data, count); + } catch (error) { + if (isSourceNotFound(error)) return null; + throw error; + } + } + + async listOrganizationRepositories(name: string): Promise { + const repos = await this.paginate( + `/orgs/${encodeURIComponent(name)}/repos` + ); + return repos.map((repo) => + mapGiteaRepository(repo, this.connection, { isOrganization: true }) + ); + } + + async getRepository(owner: string, name: string): Promise { + try { + const { data } = await sourceFetch( + `${this.apiRoot}/repos/${encodeURIComponent(owner)}/${encodeURIComponent(name)}`, + { headers: this.headers() } + ); + const isOrganization = await this.isOrganization( + data.owner?.login || data.owner?.username || owner + ); + return mapGiteaRepository(data, this.connection, { isOrganization }); + } catch (error) { + if (isSourceNotFound(error)) return null; + throw error; + } + } + + async isRepositoryStarred(owner: string, name: string): Promise { + if (!this.hasToken) { + throw new Error("A token is required to check starred repositories."); + } + try { + await sourceFetch( + `${this.apiRoot}/user/starred/${encodeURIComponent(owner)}/${encodeURIComponent(name)}`, + { headers: this.headers() } + ); + return true; + } catch (error) { + if (isSourceNotFound(error)) return false; + throw error; + } + } + + async testConnection(): Promise { + if (!this.hasToken) { + throw new Error("An access token is required."); + } + const { data } = await sourceFetch(`${this.apiRoot}/user`, { + headers: this.headers(), + }); + return { + login: data.login || data.username || "", + name: data.full_name ?? null, + avatarUrl: data.avatar_url ?? null, + }; + } + + resolveRepositoryPath(segments: string[]): SourceRepositoryPath | null { + return resolveFlatRepositoryPath(segments); + } +} diff --git a/src/lib/source-providers/github-source.ts b/src/lib/source-providers/github-source.ts new file mode 100644 index 0000000..f0dc0e3 --- /dev/null +++ b/src/lib/source-providers/github-source.ts @@ -0,0 +1,229 @@ +/** + * GitHub source adapter. + * + * Wraps the existing Octokit based discovery code in src/lib/github.ts so the + * pipeline can treat GitHub like any other source. Without a token it falls + * back to an unauthenticated client, which is enough to add public + * repositories by URL. + */ +import { Octokit } from "@octokit/rest"; +import type { Config } from "@/lib/db/schema"; +import { + createGitHubClient, + getGithubOrganizations, + getGithubRepositories, + getGithubStarredRepositories, +} from "@/lib/github"; +import type { GitRepo, RepositoryVisibility } from "@/types/Repository"; +import type { GitOrg } from "@/types/organizations"; +import { toDate } from "./http"; +import { resolveFlatRepositoryPath } from "./gitea-source"; +import type { + ListRepositoriesOptions, + SourceAccount, + SourceConnection, + SourceOrganizationResult, + SourceProvider, + SourceRepositoryPath, +} from "./types"; + +/** The REST API base, honoring GH_API_URL / GITHUB_API_URL for GHES and GHEC. */ +export function githubApiBaseUrl(): string { + return process.env.GH_API_URL || process.env.GITHUB_API_URL || "https://api.github.com"; +} + +export interface GithubRestRepository { + name: string; + full_name: string; + html_url: string; + clone_url?: string | null; + owner: { login: string; type?: string }; + private?: boolean; + fork?: boolean; + parent?: { full_name?: string } | null; + has_issues?: boolean; + archived?: boolean; + size?: number; + language?: string | null; + description?: string | null; + default_branch?: string; + visibility?: string; + disabled?: boolean; + created_at?: string | null; + updated_at?: string | null; +} + +export function mapGithubRestRepository( + repo: GithubRestRepository, + connection: Pick, + options: { isStarred?: boolean } = {} +): GitRepo { + const isOrganization = repo.owner.type === "Organization"; + return { + name: repo.name, + fullName: repo.full_name, + url: repo.html_url, + cloneUrl: repo.clone_url ?? "", + + owner: repo.owner.login, + organization: isOrganization ? repo.owner.login : undefined, + mirroredLocation: "", + destinationOrg: null, + + isPrivate: Boolean(repo.private), + isForked: Boolean(repo.fork), + forkedFrom: repo.parent?.full_name ?? undefined, + + hasIssues: repo.has_issues ?? false, + isStarred: options.isStarred ?? false, + isArchived: repo.archived ?? false, + + size: repo.size ?? 0, + hasLFS: false, + hasSubmodules: false, + + language: repo.language ?? null, + description: repo.description ?? null, + defaultBranch: repo.default_branch ?? "main", + visibility: (repo.visibility ?? "public") as RepositoryVisibility, + + status: "imported", + isDisabled: repo.disabled ?? false, + + importedAt: new Date(), + createdAt: toDate(repo.created_at), + updatedAt: toDate(repo.updated_at), + + sourceProvider: "github", + sourceUrl: connection.url, + }; +} + +function statusOf(error: unknown): number | undefined { + if (error && typeof error === "object" && "status" in error) { + const status = (error as { status?: unknown }).status; + return typeof status === "number" ? status : undefined; + } + return undefined; +} + +export class GitHubSourceProvider implements SourceProvider { + readonly kind = "github" as const; + readonly octokit: Octokit; + + constructor(readonly connection: SourceConnection) { + this.octokit = connection.token.trim() + ? createGitHubClient( + connection.token, + connection.userId, + connection.username || undefined + ) + : new Octokit({ baseUrl: githubApiBaseUrl() }); + } + + private stamp(repos: GitRepo[]): GitRepo[] { + return repos.map((repo) => ({ + ...repo, + sourceProvider: "github" as const, + sourceUrl: this.connection.url, + })); + } + + async listRepositories( + config: Partial, + options: ListRepositoriesOptions = {} + ): Promise { + const repos = await getGithubRepositories({ + octokit: this.octokit, + config, + includeCollaboratorReposOverride: options.includeCollaboratorReposOverride, + includeAllOrgsOverride: options.includeAllOrgsOverride, + }); + return this.stamp(repos); + } + + async listStarredRepositories(config: Partial): Promise { + const repos = await getGithubStarredRepositories({ octokit: this.octokit, config }); + return this.stamp(repos); + } + + listOrganizations( + config: Partial, + skipOrgNames?: Set + ): Promise { + return getGithubOrganizations({ octokit: this.octokit, config, skipOrgNames }); + } + + async getOrganization(name: string): Promise { + try { + const { data } = await this.octokit.orgs.get({ org: name }); + return { + name: data.login, + avatarUrl: data.avatar_url, + membershipRole: "member", + isIncluded: false, + status: "imported", + repositoryCount: data.public_repos + (data.total_private_repos ?? 0), + createdAt: toDate(data.created_at), + updatedAt: toDate(data.updated_at), + }; + } catch (error) { + if (statusOf(error) === 404) return null; + throw error; + } + } + + async listOrganizationRepositories(name: string): Promise { + // Public, private and member listings overlap; dedupe by id so a repo + // the token can reach through several paths is only imported once. + const seen = new Set(); + const collected: GithubRestRepository[] = []; + for (const type of ["public", "private", "member"] as const) { + const page = await this.octokit.paginate(this.octokit.repos.listForOrg, { + org: name, + type, + per_page: 100, + }); + for (const repo of page) { + if (seen.has(repo.id)) continue; + seen.add(repo.id); + collected.push(repo as GithubRestRepository); + } + } + return collected + .filter((repo) => !repo.disabled) + .map((repo) => mapGithubRestRepository(repo, this.connection)); + } + + async getRepository(owner: string, name: string): Promise { + try { + const { data } = await this.octokit.rest.repos.get({ owner, repo: name }); + return mapGithubRestRepository(data as GithubRestRepository, this.connection); + } catch (error) { + if (statusOf(error) === 404) return null; + throw error; + } + } + + async isRepositoryStarred(owner: string, name: string): Promise { + try { + await this.octokit.rest.activity.checkRepoIsStarredByAuthenticatedUser({ + owner, + repo: name, + }); + return true; + } catch (error) { + if (statusOf(error) === 404) return false; + throw error; + } + } + + async testConnection(): Promise { + const { data } = await this.octokit.users.getAuthenticated(); + return { login: data.login, name: data.name, avatarUrl: data.avatar_url }; + } + + resolveRepositoryPath(segments: string[]): SourceRepositoryPath | null { + return resolveFlatRepositoryPath(segments); + } +} diff --git a/src/lib/source-providers/gitlab.test.ts b/src/lib/source-providers/gitlab.test.ts new file mode 100644 index 0000000..7da3ef0 --- /dev/null +++ b/src/lib/source-providers/gitlab.test.ts @@ -0,0 +1,241 @@ +import { afterEach, beforeEach, describe, expect, mock, test } from "bun:test"; +import { + GitLabSourceProvider, + mapGitLabProject, + resolveGitLabRepositoryPath, + type GitLabProject, +} from "./gitlab"; +import { SourceApiError } from "./http"; + +type Call = { url: string; headers: Record }; + +function json(body: unknown, init: { status?: number; headers?: Record } = {}) { + return new Response(body === undefined ? null : JSON.stringify(body), { + status: init.status ?? 200, + headers: { "content-type": "application/json", ...(init.headers ?? {}) }, + }); +} + +function project(overrides: Partial = {}): GitLabProject { + return { + id: 1, + name: "Widget", + path: "widget", + path_with_namespace: "me/widget", + web_url: "https://gitlab.example.com/me/widget", + http_url_to_repo: "https://gitlab.example.com/me/widget.git", + namespace: { kind: "user", path: "me", full_path: "me" }, + visibility: "public", + default_branch: "main", + created_at: "2024-01-01T00:00:00Z", + last_activity_at: "2024-06-01T00:00:00Z", + ...overrides, + }; +} + +const connection = { + provider: "gitlab" as const, + url: "https://gitlab.example.com", + username: "me", + token: "glpat-secret", +}; + +let originalFetch: typeof globalThis.fetch; +let calls: Call[]; + +function installFetch(route: (url: string) => Response | Promise) { + globalThis.fetch = mock(async (input: RequestInfo | URL, init?: RequestInit) => { + calls.push({ url: String(input), headers: (init?.headers as Record) ?? {} }); + return route(String(input)); + }) as unknown as typeof globalThis.fetch; +} + +beforeEach(() => { + originalFetch = globalThis.fetch; + calls = []; +}); + +afterEach(() => { + globalThis.fetch = originalFetch; +}); + +describe("mapGitLabProject", () => { + test("flattens a subgroup project onto its top level group", () => { + const repo = mapGitLabProject( + project({ + path_with_namespace: "acme/tools/widget", + namespace: { kind: "group", path: "tools", full_path: "acme/tools" }, + web_url: "https://gitlab.example.com/acme/tools/widget", + http_url_to_repo: "https://gitlab.example.com/acme/tools/widget.git", + }), + connection + ); + + expect(repo.fullName).toBe("acme/tools/widget"); + expect(repo.name).toBe("widget"); + expect(repo.owner).toBe("acme"); + expect(repo.organization).toBe("acme"); + expect(repo.cloneUrl).toBe("https://gitlab.example.com/acme/tools/widget.git"); + expect(repo.sourceProvider).toBe("gitlab"); + expect(repo.sourceUrl).toBe("https://gitlab.example.com"); + }); + + test("uses the URL safe path as the name and leaves personal projects without an organization", () => { + const repo = mapGitLabProject(project({ name: "My Widget", path: "my-widget" }), connection); + expect(repo.name).toBe("my-widget"); + expect(repo.owner).toBe("me"); + expect(repo.organization).toBeUndefined(); + expect(repo.isStarred).toBe(false); + expect(repo.isDisabled).toBe(false); + }); + + test("internal projects are private as far as a backup is concerned", () => { + const repo = mapGitLabProject(project({ visibility: "internal" }), connection); + expect(repo.visibility).toBe("internal"); + expect(repo.isPrivate).toBe(true); + expect(mapGitLabProject(project({ visibility: "public" }), connection).isPrivate).toBe(false); + }); + + test("maps forks, archived state, disabled issues and dates", () => { + const repo = mapGitLabProject( + project({ + forked_from_project: { path_with_namespace: "upstream/widget" }, + archived: true, + issues_access_level: "disabled", + }), + connection + ); + expect(repo.isForked).toBe(true); + expect(repo.forkedFrom).toBe("upstream/widget"); + expect(repo.isArchived).toBe(true); + expect(repo.hasIssues).toBe(false); + expect(repo.createdAt.toISOString()).toBe("2024-01-01T00:00:00.000Z"); + expect(repo.updatedAt.toISOString()).toBe("2024-06-01T00:00:00.000Z"); + }); +}); + +describe("resolveGitLabRepositoryPath", () => { + test("keeps nested groups as the owner and cuts deep links at the dash", () => { + expect(resolveGitLabRepositoryPath(["acme", "tools", "widget", "-", "tree", "main"])).toEqual({ + owner: "acme/tools", + repo: "widget", + }); + expect(resolveGitLabRepositoryPath(["acme", "widget.git"])).toEqual({ + owner: "acme", + repo: "widget", + }); + expect(resolveGitLabRepositoryPath(["acme"])).toBeNull(); + }); +}); + +describe("GitLabSourceProvider", () => { + test("lists membership projects with the token, follows x-next-page and applies the config filters", async () => { + const page1 = [ + project({ id: 1 }), + project({ + id: 2, + path: "tool", + path_with_namespace: "acme/tools/tool", + namespace: { kind: "group", path: "tools", full_path: "acme/tools" }, + }), + ]; + const page2 = [ + project({ + id: 3, + path: "forked", + path_with_namespace: "me/forked", + forked_from_project: { path_with_namespace: "upstream/forked" }, + }), + ]; + installFetch((url) => { + // "per_page=100" also contains "page=1", so match the page parameter itself. + if (url.endsWith("&page=1")) return json(page1, { headers: { "x-next-page": "2" } }); + if (url.endsWith("&page=2")) return json(page2); + return json({ message: "unexpected" }, { status: 500 }); + }); + + const provider = new GitLabSourceProvider(connection); + const repos = await provider.listRepositories({ githubConfig: { skipForks: true } as any }); + + expect(calls[0].url).toBe( + "https://gitlab.example.com/api/v4/projects?membership=true&order_by=id&sort=asc&per_page=100&page=1" + ); + expect(calls[0].headers["PRIVATE-TOKEN"]).toBe("glpat-secret"); + expect(calls).toHaveLength(2); + expect(repos.map((r) => r.fullName)).toEqual(["me/widget", "acme/tools/tool"]); + expect(repos[1].organization).toBe("acme"); + }); + + test("without a token it lists the user's public projects and sends no token header", async () => { + installFetch(() => json([project()])); + + const provider = new GitLabSourceProvider({ ...connection, token: "" }); + const repos = await provider.listRepositories({}); + + expect(calls[0].url.startsWith("https://gitlab.example.com/api/v4/users/me/projects?")).toBe(true); + expect(calls[0].headers["PRIVATE-TOKEN"]).toBeUndefined(); + expect(repos).toHaveLength(1); + }); + + test("starred projects are flagged as starred", async () => { + installFetch(() => json([project()])); + const repos = await new GitLabSourceProvider(connection).listStarredRepositories({}); + expect(calls[0].url).toContain("/api/v4/projects?starred=true"); + expect(repos[0].isStarred).toBe(true); + }); + + test("getRepository encodes the project path, returns null on 404 and throws otherwise", async () => { + installFetch((url) => { + if (url.endsWith("/api/v4/projects/acme%2Ftools%2Fwidget")) return json(project()); + if (url.endsWith("/api/v4/projects/acme%2Fmissing")) return json({ message: "404 Not Found" }, { status: 404 }); + return json({ message: "boom" }, { status: 500 }); + }); + const provider = new GitLabSourceProvider(connection); + + expect((await provider.getRepository("acme/tools", "widget"))?.fullName).toBe("me/widget"); + expect(await provider.getRepository("acme", "missing")).toBeNull(); + await expect(provider.getRepository("acme", "broken")).rejects.toBeInstanceOf(SourceApiError); + }); + + test("lists top level groups and counts their projects from x-total", async () => { + installFetch((url) => { + if (url.includes("/api/v4/groups?")) { + return json([{ id: 7, full_path: "acme", avatar_url: null }, { id: 8, full_path: "ignored" }]); + } + if (url.includes("/api/v4/groups/7/projects?")) return json([{}], { headers: { "x-total": "12" } }); + return json({ message: "unexpected" }, { status: 500 }); + }); + const { organizations, failedOrgs } = await new GitLabSourceProvider(connection).listOrganizations( + {}, + new Set(["ignored"]) + ); + + expect(calls[0].url).toContain("top_level_only=true"); + expect(organizations).toHaveLength(1); + expect(organizations[0].name).toBe("acme"); + expect(organizations[0].repositoryCount).toBe(12); + expect(failedOrgs).toEqual([]); + }); + + test("isRepositoryStarred matches the full path among starred search results", async () => { + installFetch(() => + json([project({ path_with_namespace: "other/widget" }), project({ path_with_namespace: "Acme/Widget" })]) + ); + const provider = new GitLabSourceProvider(connection); + expect(await provider.isRepositoryStarred("acme", "widget")).toBe(true); + expect(await provider.isRepositoryStarred("acme", "gadget")).toBe(false); + expect(calls[0].url).toContain("starred=true"); + expect(calls[0].url).toContain("search=widget"); + }); + + test("testConnection reads /user and requires a token", async () => { + installFetch(() => json({ username: "me", name: "Me", avatar_url: "https://a/b.png" })); + const account = await new GitLabSourceProvider(connection).testConnection(); + expect(calls[0].url).toBe("https://gitlab.example.com/api/v4/user"); + expect(account).toEqual({ login: "me", name: "Me", avatarUrl: "https://a/b.png" }); + + await expect(new GitLabSourceProvider({ ...connection, token: "" }).testConnection()).rejects.toThrow( + /token is required/ + ); + }); +}); diff --git a/src/lib/source-providers/gitlab.ts b/src/lib/source-providers/gitlab.ts new file mode 100644 index 0000000..57b6278 --- /dev/null +++ b/src/lib/source-providers/gitlab.ts @@ -0,0 +1,385 @@ +/** + * GitLab source adapter (gitlab.com and self hosted). + * + * Talks to the v4 REST API with a personal access token in the PRIVATE-TOKEN + * header. Without a token only the configured user's public projects are + * visible. Nested groups are flattened onto their top level group, because a + * Gitea organization name cannot contain a slash. + */ +import type { Config } from "@/lib/db/schema"; +import type { GitRepo, RepositoryVisibility } from "@/types/Repository"; +import type { GitOrg } from "@/types/organizations"; +import { filterSourceRepositories } from "./filters"; +import { + isSourceNotFound, + sourceFetch, + stripGitSuffix, + toDate, + withQuery, +} from "./http"; +import type { + ListRepositoriesOptions, + SourceAccount, + SourceConnection, + SourceFailedOrganization, + SourceOrganizationResult, + SourceProvider, + SourceRepositoryPath, +} from "./types"; + +const PAGE_SIZE = 100; +const MAX_PAGES = 500; + +export interface GitLabNamespace { + id?: number; + kind?: string; + path?: string; + full_path?: string; + name?: string; +} + +export interface GitLabProject { + id?: number; + name?: string; + path?: string; + path_with_namespace?: string; + web_url?: string; + http_url_to_repo?: string; + namespace?: GitLabNamespace; + visibility?: string; + archived?: boolean; + forked_from_project?: { path_with_namespace?: string } | null; + issues_enabled?: boolean; + issues_access_level?: string; + description?: string | null; + default_branch?: string | null; + created_at?: string; + updated_at?: string; + last_activity_at?: string; + statistics?: { repository_size?: number }; +} + +export interface GitLabGroup { + id?: number; + name?: string; + path?: string; + full_path?: string; + avatar_url?: string | null; + created_at?: string; +} + +interface GitLabUser { + id?: number; + username: string; + name?: string | null; + avatar_url?: string | null; +} + +function normalizeVisibility(value: unknown): RepositoryVisibility { + return value === "private" || value === "internal" ? value : "public"; +} + +/** The group a project sits under, flattened to the top level group. */ +export function gitLabTopLevelNamespace(project: GitLabProject): string { + const fullPath = project.namespace?.full_path || ""; + return fullPath.split("/")[0] || project.namespace?.path || ""; +} + +export function mapGitLabProject( + project: GitLabProject, + connection: Pick, + options: { isStarred?: boolean } = {} +): GitRepo { + const namespacePath = project.namespace?.full_path || ""; + const fullName = + project.path_with_namespace || + (namespacePath && project.path ? `${namespacePath}/${project.path}` : project.path || ""); + const name = project.path || fullName.split("/").at(-1) || project.name || "repository"; + const owner = gitLabTopLevelNamespace(project) || fullName.split("/")[0] || "unknown"; + const isGroup = project.namespace?.kind === "group"; + const visibility = normalizeVisibility(project.visibility); + const repositorySizeBytes = project.statistics?.repository_size; + + return { + name, + fullName, + url: project.web_url || `${connection.url}/${fullName}`, + cloneUrl: project.http_url_to_repo || `${connection.url}/${fullName}.git`, + + owner, + organization: isGroup ? owner : undefined, + mirroredLocation: "", + destinationOrg: null, + + // Internal projects are only visible to signed in users, so they are + // private as far as a backup is concerned. + isPrivate: visibility !== "public", + isForked: Boolean(project.forked_from_project), + forkedFrom: project.forked_from_project?.path_with_namespace ?? undefined, + + hasIssues: + project.issues_enabled !== false && project.issues_access_level !== "disabled", + isStarred: options.isStarred ?? false, + isArchived: Boolean(project.archived), + + size: repositorySizeBytes ? Math.round(repositorySizeBytes / 1024) : 0, + hasLFS: false, + hasSubmodules: false, + + language: null, + description: project.description ?? null, + defaultBranch: project.default_branch || "main", + visibility, + + status: "imported", + isDisabled: false, + + importedAt: new Date(), + createdAt: toDate(project.created_at), + updatedAt: toDate(project.last_activity_at ?? project.updated_at), + + sourceProvider: "gitlab", + sourceUrl: connection.url, + }; +} + +export function mapGitLabGroup(group: GitLabGroup, repositoryCount: number): GitOrg { + return { + name: group.full_path || group.path || group.name || "", + avatarUrl: group.avatar_url || "", + membershipRole: "member", + isIncluded: false, + status: "imported", + repositoryCount, + createdAt: toDate(group.created_at), + updatedAt: new Date(), + }; +} + +/** + * GitLab deep links put a "-" segment between the project path and the rest + * (issues, tree, blob). Everything before it is the project; the last segment + * is the project and the rest is the (possibly nested) group. + */ +export function resolveGitLabRepositoryPath(segments: string[]): SourceRepositoryPath | null { + const marker = segments.indexOf("-"); + const path = (marker === -1 ? segments : segments.slice(0, marker)) + .map((segment) => segment.trim()) + .filter(Boolean); + if (path.length < 2) return null; + + const repo = stripGitSuffix(path[path.length - 1]); + const owner = path.slice(0, -1).join("/"); + if (!repo || !owner) return null; + + return { owner, repo }; +} + +export class GitLabSourceProvider implements SourceProvider { + readonly kind = "gitlab" as const; + private readonly apiRoot: string; + + constructor(readonly connection: SourceConnection) { + this.apiRoot = `${connection.url}/api/v4`; + } + + private get hasToken(): boolean { + return this.connection.token.trim().length > 0; + } + + private headers(): Record { + return this.hasToken ? { "PRIVATE-TOKEN": this.connection.token } : {}; + } + + private encodedUser(): string { + return encodeURIComponent(this.connection.username); + } + + private async paginate( + path: string, + params: Record = {} + ): Promise { + const items: T[] = []; + for (let page = 1; page <= MAX_PAGES; page += 1) { + const url = withQuery(`${this.apiRoot}${path}`, { + ...params, + per_page: PAGE_SIZE, + page, + }); + const { data, response } = await sourceFetch(url, { headers: this.headers() }); + if (!Array.isArray(data) || data.length === 0) break; + items.push(...data); + if (!response.headers.get("x-next-page")) break; + } + return items; + } + + /** Total item count for a list endpoint, from the x-total header. */ + private async total( + path: string, + params: Record = {} + ): Promise { + const url = withQuery(`${this.apiRoot}${path}`, { ...params, per_page: 1, page: 1 }); + const { data, response } = await sourceFetch(url, { headers: this.headers() }); + const header = response.headers.get("x-total"); + const parsed = header === null ? Number.NaN : Number(header); + if (Number.isFinite(parsed)) return parsed; + return Array.isArray(data) ? data.length : 0; + } + + async listRepositories( + config: Partial, + options: ListRepositoriesOptions = {} + ): Promise { + const projects = this.hasToken + ? await this.paginate("/projects", { + membership: true, + order_by: "id", + sort: "asc", + }) + : await this.paginate(`/users/${this.encodedUser()}/projects`, { + order_by: "id", + sort: "asc", + }); + + const repos = projects.map((project) => mapGitLabProject(project, this.connection)); + return filterSourceRepositories(repos, { + config, + options, + username: this.connection.username, + }); + } + + async listStarredRepositories(_config: Partial): Promise { + const projects = this.hasToken + ? await this.paginate("/projects", { + starred: true, + order_by: "id", + sort: "asc", + }) + : await this.paginate( + `/users/${this.encodedUser()}/starred_projects`, + { order_by: "id", sort: "asc" } + ); + + return projects.map((project) => + mapGitLabProject(project, this.connection, { isStarred: true }) + ); + } + + async listOrganizations( + _config: Partial, + skipOrgNames?: Set + ): Promise { + if (!this.hasToken) { + return { organizations: [], failedOrgs: [] }; + } + + const groups = await this.paginate("/groups", { + membership: true, + top_level_only: true, + order_by: "id", + sort: "asc", + }); + + const organizations: GitOrg[] = []; + const failedOrgs: SourceFailedOrganization[] = []; + + for (const group of groups) { + const name = group.full_path || group.path || group.name || ""; + if (!name) continue; + if (skipOrgNames?.has(name.toLowerCase())) { + console.log(`Skipping group ${name} - ignored by user`); + continue; + } + + try { + const count = await this.total(`/groups/${group.id}/projects`, { + include_subgroups: true, + }); + organizations.push(mapGitLabGroup(group, count)); + } catch (error) { + const reason = error instanceof Error ? error.message : String(error); + console.warn(`Failed to import group ${name} - ${reason}`); + failedOrgs.push({ name, avatarUrl: group.avatar_url || "", reason }); + } + } + + return { organizations, failedOrgs }; + } + + async getOrganization(name: string): Promise { + try { + const { data } = await sourceFetch( + `${this.apiRoot}/groups/${encodeURIComponent(name)}`, + { headers: this.headers() } + ); + let count = 0; + try { + count = await this.total(`/groups/${data.id}/projects`, { include_subgroups: true }); + } catch { + // The count is informational only. + } + return mapGitLabGroup(data, count); + } catch (error) { + if (isSourceNotFound(error)) return null; + throw error; + } + } + + async listOrganizationRepositories(name: string): Promise { + const projects = await this.paginate( + `/groups/${encodeURIComponent(name)}/projects`, + { include_subgroups: true, order_by: "id", sort: "asc" } + ); + return projects.map((project) => mapGitLabProject(project, this.connection)); + } + + async getRepository(owner: string, name: string): Promise { + try { + const { data } = await sourceFetch( + `${this.apiRoot}/projects/${encodeURIComponent(`${owner}/${name}`)}`, + { headers: this.headers() } + ); + return mapGitLabProject(data, this.connection); + } catch (error) { + if (isSourceNotFound(error)) return null; + throw error; + } + } + + async isRepositoryStarred(owner: string, name: string): Promise { + const target = `${owner}/${name}`.toLowerCase(); + const projects = this.hasToken + ? await this.paginate("/projects", { + starred: true, + search: name, + simple: true, + }) + : await this.paginate( + `/users/${this.encodedUser()}/starred_projects`, + { search: name, simple: true } + ); + return projects.some( + (project) => (project.path_with_namespace || "").toLowerCase() === target + ); + } + + async testConnection(): Promise { + if (!this.hasToken) { + throw new Error("A GitLab personal access token is required."); + } + const { data } = await sourceFetch(`${this.apiRoot}/user`, { + headers: this.headers(), + }); + return { + login: data.username, + name: data.name ?? null, + avatarUrl: data.avatar_url ?? null, + }; + } + + resolveRepositoryPath(segments: string[]): SourceRepositoryPath | null { + return resolveGitLabRepositoryPath(segments); + } +} diff --git a/src/lib/source-providers/http.ts b/src/lib/source-providers/http.ts new file mode 100644 index 0000000..1a711b6 --- /dev/null +++ b/src/lib/source-providers/http.ts @@ -0,0 +1,113 @@ +/** + * Small fetch wrapper shared by the GitLab and Gitea adapters. + * + * Deliberately built on the global fetch so tests can swap it out, and free + * of database imports so the adapters stay pure. + */ + +export const DEFAULT_SOURCE_TIMEOUT_MS = 30_000; + +export class SourceApiError extends Error { + constructor( + message: string, + public readonly status: number, + public readonly url: string, + public readonly body?: string + ) { + super(message); + this.name = "SourceApiError"; + } +} + +export function isSourceNotFound(error: unknown): boolean { + return error instanceof SourceApiError && error.status === 404; +} + +export interface SourceFetchInit { + method?: string; + headers?: Record; + timeoutMs?: number; +} + +export interface SourceFetchResult { + data: T; + response: Response; +} + +/** GET JSON from a source host, throwing SourceApiError on a non-2xx status. */ +export async function sourceFetch( + url: string, + init: SourceFetchInit = {} +): Promise> { + const { method = "GET", headers = {}, timeoutMs = DEFAULT_SOURCE_TIMEOUT_MS } = init; + + const response = await fetch(url, { + method, + headers: { Accept: "application/json", ...headers }, + signal: AbortSignal.timeout(timeoutMs), + }); + + if (!response.ok) { + let body = ""; + try { + body = await response.text(); + } catch { + // The status is what matters; the body is only for diagnostics. + } + const detail = summarizeErrorBody(body); + throw new SourceApiError( + `Request to ${url} failed with status ${response.status}${detail ? `: ${detail}` : ""}`, + response.status, + url, + body + ); + } + + if (response.status === 204) { + return { data: undefined as T, response }; + } + + const data = (await response.json()) as T; + return { data, response }; +} + +function summarizeErrorBody(body: string): string { + if (!body) return ""; + try { + const parsed = JSON.parse(body) as { message?: unknown; error?: unknown }; + const message = parsed.message ?? parsed.error; + if (typeof message === "string") return message.slice(0, 200); + } catch { + // Not JSON. + } + return body.slice(0, 200); +} + +/** Append query parameters, skipping undefined values. */ +export function withQuery( + url: string, + params: Record +): string { + const entries = Object.entries(params).filter( + (entry): entry is [string, string | number | boolean] => entry[1] !== undefined + ); + if (entries.length === 0) return url; + const query = entries + .map(([key, value]) => `${encodeURIComponent(key)}=${encodeURIComponent(String(value))}`) + .join("&"); + return `${url}${url.includes("?") ? "&" : "?"}${query}`; +} + +/** Coerce an API timestamp to a Date, falling back to now. */ +export function toDate(value: unknown): Date { + if (typeof value === "string" || typeof value === "number") { + const date = new Date(value); + if (!Number.isNaN(date.getTime())) return date; + } + return new Date(); +} + +/** Strip a trailing ".git" from a repository name segment. */ +export function stripGitSuffix(segment: string): string { + return segment.replace(/\.git$/i, ""); +} diff --git a/src/lib/source-providers/index.ts b/src/lib/source-providers/index.ts new file mode 100644 index 0000000..b25b72d --- /dev/null +++ b/src/lib/source-providers/index.ts @@ -0,0 +1,95 @@ +/** + * Source providers: where repositories come from. + * + * The configuration card picks one provider per user. Everything that talks + * to the source host (discovery, existence checks, clone credentials) goes + * through the SourceProvider built here, so the mirror pipeline does not need + * to know which host it is dealing with. + */ +import type { Config, Repository } from "@/lib/db/schema"; +import { getDecryptedGitHubToken } from "@/lib/utils/config-encryption"; +import { GiteaSourceProvider } from "./gitea-source"; +import { GitHubSourceProvider } from "./github-source"; +import { GitLabSourceProvider } from "./gitlab"; +import { + describeSource, + getRepositorySource, + isRepositoryFromConfiguredSource, + normalizeSourceProviderKind, + normalizeSourceUrl, + type RepositorySourceFields, + type SourceProviderKind, +} from "./kinds"; +import type { SourceConnection, SourceProvider } from "./types"; + +export * from "./kinds"; +export type * from "./types"; +export { SourceApiError, isSourceNotFound } from "./http"; + +/** The provider kind a config selects, defaulting to GitHub. */ +export function resolveSourceProviderKind( + config: Partial | null | undefined +): SourceProviderKind { + return normalizeSourceProviderKind(config?.githubConfig?.provider); +} + +/** Build the connection details for a config. The token is passed in decrypted. */ +export function resolveSourceConnection( + config: Partial, + { token = "", userId }: { token?: string; userId?: string } = {} +): SourceConnection { + const provider = resolveSourceProviderKind(config); + return { + provider, + url: normalizeSourceUrl(config.githubConfig?.url, provider), + username: config.githubConfig?.owner ?? "", + token, + userId: userId ?? config.userId, + }; +} + +export function createSourceProvider(connection: SourceConnection): SourceProvider { + switch (connection.provider) { + case "gitlab": + return new GitLabSourceProvider(connection); + case "gitea": + return new GiteaSourceProvider(connection); + case "github": + default: + return new GitHubSourceProvider(connection); + } +} + +/** Build the provider for a stored config, decrypting the token. */ +export function createSourceProviderFromConfig( + config: Partial, + { userId }: { userId?: string } = {} +): SourceProvider { + const token = config.githubConfig?.token ? getDecryptedGitHubToken(config as Config) : ""; + return createSourceProvider(resolveSourceConnection(config, { token, userId })); +} + +/** + * Refuse to mirror a repository with credentials for a different host. + * + * A repository imported from GitHub must not be sent to Gitea with the GitLab + * token that is configured now: the token would leak to the wrong host and + * the fetch would fail anyway. + */ +export function assertRepositoryMatchesConfiguredSource({ + repository, + config, +}: { + repository: Pick & RepositorySourceFields; + config: Partial; +}): void { + const connection = resolveSourceConnection(config); + if (isRepositoryFromConfiguredSource(repository, connection)) return; + + throw new Error( + `Repository ${repository.fullName} was imported from ${describeSource( + getRepositorySource(repository) + )} but the configured source is ${describeSource(connection)}. ` + + "Switch the source back, or remove the repository and add it again from the current source." + ); +} diff --git a/src/lib/source-providers/kinds.test.ts b/src/lib/source-providers/kinds.test.ts new file mode 100644 index 0000000..aa7ec41 --- /dev/null +++ b/src/lib/source-providers/kinds.test.ts @@ -0,0 +1,100 @@ +import { describe, expect, test } from "bun:test"; +import { + SOURCE_PROVIDER_DEFAULT_URLS, + getRepositorySource, + isRepositoryFromConfiguredSource, + isSourceProviderKind, + isValidSourceUrl, + normalizeSourceProviderKind, + normalizeSourceUrl, + sourceHostOf, +} from "./kinds"; + +describe("normalizeSourceProviderKind", () => { + test("accepts the three kinds and defaults everything else to GitHub", () => { + expect(normalizeSourceProviderKind("github")).toBe("github"); + expect(normalizeSourceProviderKind("gitlab")).toBe("gitlab"); + expect(normalizeSourceProviderKind("gitea")).toBe("gitea"); + expect(normalizeSourceProviderKind(undefined)).toBe("github"); + expect(normalizeSourceProviderKind(null)).toBe("github"); + expect(normalizeSourceProviderKind("bitbucket")).toBe("github"); + expect(normalizeSourceProviderKind(42)).toBe("github"); + }); + + test("treats forgejo and codeberg as the Gitea kind", () => { + expect(normalizeSourceProviderKind("forgejo")).toBe("gitea"); + expect(normalizeSourceProviderKind("codeberg")).toBe("gitea"); + expect(isSourceProviderKind("forgejo")).toBe(false); + }); +}); + +describe("normalizeSourceUrl", () => { + test("falls back to the provider default when empty", () => { + expect(normalizeSourceUrl("", "gitlab")).toBe("https://gitlab.com"); + expect(normalizeSourceUrl(undefined, "gitea")).toBe("https://codeberg.org"); + expect(normalizeSourceUrl(null, "github")).toBe(SOURCE_PROVIDER_DEFAULT_URLS.github); + }); + + test("adds https, lowercases the host and strips trailing slashes", () => { + expect(normalizeSourceUrl("gitlab.example.com/", "gitlab")).toBe("https://gitlab.example.com"); + expect(normalizeSourceUrl("HTTPS://Codeberg.org///", "gitea")).toBe("https://codeberg.org"); + expect(normalizeSourceUrl(" http://gitea.local:3000/gitea/ ", "gitea")).toBe( + "http://gitea.local:3000/gitea" + ); + }); + + test("rejects non http schemes and garbage", () => { + expect(normalizeSourceUrl("ftp://gitlab.example.com", "gitlab")).toBe("https://gitlab.com"); + expect(normalizeSourceUrl("not a url", "gitlab")).toBe("https://gitlab.com"); + }); +}); + +describe("isValidSourceUrl", () => { + test("accepts http(s) URLs with or without a scheme", () => { + expect(isValidSourceUrl("https://gitlab.example.com")).toBe(true); + expect(isValidSourceUrl("gitea.local:3000")).toBe(true); + }); + + test("rejects empty, non http and unparseable values", () => { + expect(isValidSourceUrl("")).toBe(false); + expect(isValidSourceUrl("ftp://x")).toBe(false); + expect(isValidSourceUrl("not a url")).toBe(false); + }); +}); + +describe("getRepositorySource / isRepositoryFromConfiguredSource", () => { + test("rows from before the source columns count as GitHub", () => { + expect(getRepositorySource({})).toEqual({ provider: "github", url: "https://github.com" }); + expect( + isRepositoryFromConfiguredSource({}, { provider: "github", url: "https://github.com" }) + ).toBe(true); + expect( + isRepositoryFromConfiguredSource({}, { provider: "gitlab", url: "https://gitlab.com" }) + ).toBe(false); + }); + + test("compares normalized URLs so a trailing slash does not split a source", () => { + const row = { sourceProvider: "gitlab", sourceUrl: "https://GitLab.example.com/" }; + expect(getRepositorySource(row)).toEqual({ + provider: "gitlab", + url: "https://gitlab.example.com", + }); + expect( + isRepositoryFromConfiguredSource(row, { + provider: "gitlab", + url: "https://gitlab.example.com", + }) + ).toBe(true); + expect( + isRepositoryFromConfiguredSource(row, { provider: "gitlab", url: "https://gitlab.com" }) + ).toBe(false); + }); +}); + +describe("sourceHostOf", () => { + test("returns the lowercase host including a port", () => { + expect(sourceHostOf("https://Codeberg.org")).toBe("codeberg.org"); + expect(sourceHostOf("http://gitea.local:3000/gitea")).toBe("gitea.local:3000"); + expect(sourceHostOf("nonsense")).toBe(""); + }); +}); diff --git a/src/lib/source-providers/kinds.ts b/src/lib/source-providers/kinds.ts new file mode 100644 index 0000000..95e6bbe --- /dev/null +++ b/src/lib/source-providers/kinds.ts @@ -0,0 +1,134 @@ +/** + * The source hosts Gitea Mirror can pull repositories from. + * + * Kept free of imports so both server code and client components can use it. + * "gitea" covers Gitea and Forgejo (and therefore Codeberg): they share the + * same API surface. + */ +export const SOURCE_PROVIDER_KINDS = ["github", "gitlab", "gitea"] as const; + +export type SourceProviderKind = (typeof SOURCE_PROVIDER_KINDS)[number]; + +export const DEFAULT_SOURCE_PROVIDER: SourceProviderKind = "github"; + +/** Base URL used when the config has no instance URL for the provider. */ +export const SOURCE_PROVIDER_DEFAULT_URLS: Record = { + github: "https://github.com", + gitlab: "https://gitlab.com", + gitea: "https://codeberg.org", +}; + +export const SOURCE_PROVIDER_LABELS: Record = { + github: "GitHub", + gitlab: "GitLab", + gitea: "Gitea / Forgejo", +}; + +/** What the host calls a group of repositories owned by a team. */ +export const SOURCE_PROVIDER_ORG_NOUNS: Record = { + github: "organization", + gitlab: "group", + gitea: "organization", +}; + +export function isSourceProviderKind(value: unknown): value is SourceProviderKind { + return ( + typeof value === "string" && + (SOURCE_PROVIDER_KINDS as readonly string[]).includes(value) + ); +} + +/** + * Coerce an untrusted value to a provider kind, defaulting to GitHub. + * "forgejo" and "codeberg" are accepted as spellings of the Gitea kind. + */ +export function normalizeSourceProviderKind(value: unknown): SourceProviderKind { + if (isSourceProviderKind(value)) return value; + if (value === "forgejo" || value === "codeberg") return "gitea"; + return DEFAULT_SOURCE_PROVIDER; +} + +/** + * Normalize an instance URL: trim, add https:// when the scheme is missing, + * lowercase the host, drop trailing slashes. Falls back to the provider's + * default when the value is empty or unparseable. + */ +export function normalizeSourceUrl( + raw: string | null | undefined, + provider: SourceProviderKind +): string { + const fallback = SOURCE_PROVIDER_DEFAULT_URLS[provider]; + const trimmed = typeof raw === "string" ? raw.trim() : ""; + if (!trimmed) return fallback; + + const withScheme = /^[a-z][a-z0-9+.-]*:\/\//i.test(trimmed) + ? trimmed + : `https://${trimmed}`; + + let url: URL; + try { + url = new URL(withScheme); + } catch { + return fallback; + } + + if (url.protocol !== "http:" && url.protocol !== "https:") { + return fallback; + } + + const path = url.pathname.replace(/\/+$/, ""); + return `${url.protocol}//${url.host}${path}`; +} + +/** True when the URL parses as an http(s) URL. Used by config validation. */ +export function isValidSourceUrl(raw: string): boolean { + const trimmed = raw.trim(); + if (!trimmed) return false; + const withScheme = /^[a-z][a-z0-9+.-]*:\/\//i.test(trimmed) + ? trimmed + : `https://${trimmed}`; + try { + const url = new URL(withScheme); + return url.protocol === "http:" || url.protocol === "https:"; + } catch { + return false; + } +} + +/** Hostname of a normalized source URL, for comparing pasted repository URLs. */ +export function sourceHostOf(url: string): string { + try { + return new URL(url).host.toLowerCase(); + } catch { + return ""; + } +} + +/** The two columns a stored repository carries about its origin. */ +export interface RepositorySourceFields { + sourceProvider?: string | null; + sourceUrl?: string | null; +} + +export interface RepositorySource { + provider: SourceProviderKind; + url: string; +} + +/** Where a stored repository came from. Rows from before this column default to GitHub. */ +export function getRepositorySource(repo: RepositorySourceFields): RepositorySource { + const provider = normalizeSourceProviderKind(repo.sourceProvider); + return { provider, url: normalizeSourceUrl(repo.sourceUrl, provider) }; +} + +export function isRepositoryFromConfiguredSource( + repo: RepositorySourceFields, + connection: RepositorySource +): boolean { + const source = getRepositorySource(repo); + return source.provider === connection.provider && source.url === connection.url; +} + +export function describeSource(source: RepositorySource): string { + return `${SOURCE_PROVIDER_LABELS[source.provider]} (${source.url})`; +} diff --git a/src/lib/source-providers/types.ts b/src/lib/source-providers/types.ts new file mode 100644 index 0000000..86a0acb --- /dev/null +++ b/src/lib/source-providers/types.ts @@ -0,0 +1,98 @@ +import type { Config } from "@/lib/db/schema"; +import type { GitRepo } from "@/types/Repository"; +import type { GitOrg } from "@/types/organizations"; +import type { SourceProviderKind } from "./kinds"; + +/** Everything an adapter needs to talk to one source host. */ +export interface SourceConnection { + provider: SourceProviderKind; + /** Normalized base URL of the instance, without a trailing slash. */ + url: string; + /** The account the token belongs to. */ + username: string; + /** Decrypted token. Empty means unauthenticated, public-only access. */ + token: string; + /** Owning user, used for rate limit tracking on GitHub. */ + userId?: string; +} + +export interface ListRepositoriesOptions { + /** Include collaborator repositories regardless of the config filter. */ + includeCollaboratorReposOverride?: boolean; + /** Ignore the organization allowlist and return every org repository. */ + includeAllOrgsOverride?: boolean; +} + +export interface SourceFailedOrganization { + name: string; + avatarUrl: string; + reason: string; +} + +export interface SourceOrganizationResult { + organizations: GitOrg[]; + failedOrgs: SourceFailedOrganization[]; +} + +export interface SourceAccount { + login: string; + name?: string | null; + avatarUrl?: string | null; +} + +export interface SourceRepositoryPath { + owner: string; + repo: string; +} + +/** + * The operations the mirror pipeline needs from a source host. + * + * Every method returns the shared GitRepo / GitOrg shapes with + * `sourceProvider` and `sourceUrl` already stamped, so callers can insert the + * result without knowing which host it came from. + */ +export interface SourceProvider { + readonly kind: SourceProviderKind; + readonly connection: SourceConnection; + + /** Repositories the account owns, collaborates on, or reaches through orgs. */ + listRepositories( + config: Partial, + options?: ListRepositoriesOptions + ): Promise; + + /** Repositories the account has starred. */ + listStarredRepositories(config: Partial): Promise; + + /** Organizations (GitLab: top level groups) the account belongs to. */ + listOrganizations( + config: Partial, + skipOrgNames?: Set + ): Promise; + + /** One organization by name, or null when it does not exist. */ + getOrganization(name: string): Promise; + + /** Every repository in an organization the token can see. */ + listOrganizationRepositories(name: string): Promise; + + /** + * One repository, or null on a clean 404. Any other failure throws, so + * callers that treat "gone" as destructive (cleanup) can fail safe. + */ + getRepository(owner: string, name: string): Promise; + + /** Whether the account currently stars the repository. Throws on errors other than 404. */ + isRepositoryStarred(owner: string, name: string): Promise; + + /** Verify the token and return the account it belongs to. */ + testConnection(): Promise; + + /** + * Turn the path segments of a pasted URL into owner and repository names + * the host understands. GitLab nests groups, so "group/sub/project" keeps + * "group/sub" as the owner. Returns null when the segments name no repo. + */ + resolveRepositoryPath(segments: string[]): SourceRepositoryPath | null; +} diff --git a/src/lib/utils/config-defaults.ts b/src/lib/utils/config-defaults.ts index c5f0220..b9f5f64 100644 --- a/src/lib/utils/config-defaults.ts +++ b/src/lib/utils/config-defaults.ts @@ -78,6 +78,7 @@ export async function createDefaultConfig({ userId, envOverrides = {} }: Default githubConfig: { owner: githubUsername, type: "personal", + provider: "github", token: githubToken ? encrypt(githubToken) : "", includeStarred: false, includeForks: true, @@ -93,6 +94,7 @@ export async function createDefaultConfig({ userId, envOverrides = {} }: Default }, giteaConfig: { url: giteaUrl, + provider: "gitea", externalUrl: giteaExternalUrl || undefined, token: giteaToken ? encrypt(giteaToken) : "", defaultOwner: giteaUsername, diff --git a/src/lib/utils/config-mapper.test.ts b/src/lib/utils/config-mapper.test.ts index ec1af3c..659fe25 100644 --- a/src/lib/utils/config-mapper.test.ts +++ b/src/lib/utils/config-mapper.test.ts @@ -257,3 +257,77 @@ test("mapUiToDbConfig resets a stale skip forkStrategy to reference when skipFor }); expect(db.giteaConfig.forkStrategy).toBe("reference"); }); + +import { describe } from "bun:test"; +import { + mapDbToUiConfig as mapDbToUiConfigForSource, + mapUiToDbConfig as mapUiToDbConfigForSource, +} from "./config-mapper"; + +describe("source provider round trip", () => { + const giteaConfig = { + url: "https://gitea.example.com", + username: "me", + token: "t", + organization: "mirrors", + visibility: "public", + starredReposOrg: "starred", + preserveOrgStructure: false, + } as any; + const mirrorOptions = { + mirrorReleases: false, + mirrorLFS: false, + mirrorMetadata: false, + metadataComponents: { issues: false, pullRequests: false, labels: false, milestones: false, wiki: false }, + } as any; + const advancedOptions = { skipForks: false, starredCodeOnly: false } as any; + + test("stores a normalized instance URL for GitLab and Gitea sources", () => { + const { githubConfig } = mapUiToDbConfigForSource( + { provider: "gitlab", url: "GitLab.example.com/", username: "me", token: "t", privateRepositories: false, mirrorStarred: false }, + giteaConfig, + mirrorOptions, + advancedOptions + ); + expect(githubConfig.provider).toBe("gitlab"); + expect(githubConfig.url).toBe("https://gitlab.example.com"); + }); + + test("GitHub stores no URL and an empty URL means the provider default", () => { + const github = mapUiToDbConfigForSource( + { provider: "github", url: "https://github.com", username: "me", token: "t", privateRepositories: false, mirrorStarred: false }, + giteaConfig, + mirrorOptions, + advancedOptions + ).githubConfig; + expect(github.provider).toBe("github"); + expect(github.url).toBeUndefined(); + + const gitea = mapUiToDbConfigForSource( + { provider: "gitea", url: " ", username: "me", token: "t", privateRepositories: false, mirrorStarred: false }, + giteaConfig, + mirrorOptions, + advancedOptions + ).githubConfig; + expect(gitea.provider).toBe("gitea"); + expect(gitea.url).toBeUndefined(); + }); + + test("a saved provider survives a UI payload that does not mention it", () => { + const { githubConfig } = mapUiToDbConfigForSource( + { username: "me", token: "t", privateRepositories: false, mirrorStarred: false }, + giteaConfig, + mirrorOptions, + advancedOptions, + { githubConfig: { provider: "gitea", url: "https://codeberg.org" } as any } + ); + expect(githubConfig.provider).toBe("gitea"); + expect(githubConfig.url).toBe("https://codeberg.org"); + }); + + test("a legacy row maps to GitHub with an empty URL in the UI", () => { + const { githubConfig } = mapDbToUiConfigForSource({ githubConfig: { owner: "octo", token: "" } }); + expect(githubConfig.provider).toBe("github"); + expect(githubConfig.url).toBe(""); + }); +}); diff --git a/src/lib/utils/config-mapper.ts b/src/lib/utils/config-mapper.ts index 63cdb88..8339f84 100644 --- a/src/lib/utils/config-mapper.ts +++ b/src/lib/utils/config-mapper.ts @@ -9,6 +9,11 @@ import type { AdvancedOptions, SaveConfigApiRequest } from "@/types/config"; +import { + normalizeSourceProviderKind, + normalizeSourceUrl, +} from "@/lib/source-providers/kinds"; +import { normalizeDestinationProviderKind } from "@/lib/destination-kinds"; import { z } from "zod"; import { githubConfigSchema, giteaConfigSchema, scheduleConfigSchema, cleanupConfigSchema } from "@/lib/db/schema"; import { parseInterval } from "@/lib/utils/duration-parser"; @@ -67,10 +72,23 @@ export function mapUiToDbConfig( giteaConfig?: Partial; } ): { githubConfig: DbGitHubConfig; giteaConfig: DbGiteaConfig } { + // The source host. GitHub keeps no per-config URL: GHES routing comes + // from GH_API_URL. Other hosts store their normalized instance URL. + const sourceProvider = normalizeSourceProviderKind( + githubConfig.provider ?? existing?.githubConfig?.provider + ); + const rawSourceUrl = githubConfig.url ?? existing?.githubConfig?.url ?? ""; + const sourceUrl = + sourceProvider === "github" || !rawSourceUrl.trim() + ? undefined + : normalizeSourceUrl(rawSourceUrl, sourceProvider); + // Map GitHub config to match database schema fields const dbGithubConfig: DbGitHubConfig = { // Map username to owner field owner: githubConfig.username, + provider: sourceProvider, + url: sourceUrl, type: existing?.githubConfig?.type || "personal", // Not in UI; preserve stored value token: githubConfig.token || "", @@ -105,6 +123,9 @@ export function mapUiToDbConfig( // Map Gitea config to match database schema const dbGiteaConfig: DbGiteaConfig = { url: giteaConfig.url, + provider: normalizeDestinationProviderKind( + giteaConfig.provider ?? existing?.giteaConfig?.provider + ), externalUrl: giteaConfig.externalUrl?.trim() || undefined, token: giteaConfig.token, defaultOwner: giteaConfig.username, // Map username to defaultOwner @@ -176,6 +197,8 @@ export function mapDbToUiConfig(dbConfig: any): { } { // Map from database GitHub config to UI fields const githubConfig: GitHubConfig = { + provider: normalizeSourceProviderKind(dbConfig.githubConfig?.provider), + url: dbConfig.githubConfig?.url || "", username: dbConfig.githubConfig?.owner || "", // Map owner to username token: dbConfig.githubConfig?.token || "", privateRepositories: dbConfig.githubConfig?.includePrivate || false, // Map includePrivate to privateRepositories @@ -188,6 +211,7 @@ export function mapDbToUiConfig(dbConfig: any): { // Map from database Gitea config to UI fields const giteaConfig: GiteaConfig = { + provider: normalizeDestinationProviderKind(dbConfig.giteaConfig?.provider), url: dbConfig.giteaConfig?.url || "", externalUrl: dbConfig.giteaConfig?.externalUrl || "", username: dbConfig.giteaConfig?.defaultOwner || "", // Map defaultOwner to username diff --git a/src/lib/utils/github-url.test.ts b/src/lib/utils/github-url.test.ts index f907d4f..098b3a5 100644 --- a/src/lib/utils/github-url.test.ts +++ b/src/lib/utils/github-url.test.ts @@ -99,3 +99,47 @@ describe("looksLikeUrl", () => { expect(looksLikeUrl("microsoft")).toBe(false); }); }); + +import { parseRepoReferenceParts } from "./github-url"; + +describe("parseRepoReferenceParts", () => { + it("keeps the host and every path segment for the server to resolve", () => { + expect(parseRepoReferenceParts("https://GitLab.com/acme/tools/widget/-/tree/main")).toEqual({ + host: "gitlab.com", + segments: ["acme", "tools", "widget", "-", "tree", "main"], + }); + expect(parseRepoReferenceParts("git@codeberg.org:forgejo/forgejo.git")).toEqual({ + host: "codeberg.org", + segments: ["forgejo", "forgejo.git"], + }); + expect(parseRepoReferenceParts("vercel/next.js")).toEqual({ + host: null, + segments: ["vercel", "next.js"], + }); + }); +}); + +describe("parseGitHubRepoReference on other hosts", () => { + it("keeps nested GitLab groups as the owner and cuts at the dash", () => { + expect( + parseGitHubRepoReference("https://gitlab.com/acme/tools/widget/-/issues/4") + ).toEqual({ owner: "acme/tools", repo: "widget" }); + expect(parseGitHubRepoReference("https://gitlab.com/acme/widget.git")).toEqual({ + owner: "acme", + repo: "widget", + }); + }); + + it("stops at Gitea deep link markers on flat hosts", () => { + expect( + parseGitHubRepoReference("https://codeberg.org/forgejo/forgejo/src/branch/forgejo/README.md") + ).toEqual({ owner: "forgejo", repo: "forgejo" }); + }); + + it("still parses github.com the strict way", () => { + expect(parseGitHubRepoReference("https://github.com/vercel/next.js/tree/canary")).toEqual({ + owner: "vercel", + repo: "next.js", + }); + }); +}); diff --git a/src/lib/utils/github-url.ts b/src/lib/utils/github-url.ts index 76d5d2a..7fb8307 100644 --- a/src/lib/utils/github-url.ts +++ b/src/lib/utils/github-url.ts @@ -45,18 +45,26 @@ function stripWrapping(input: string): string { .trim(); } +export interface RepoReferenceParts { + /** Lowercased host of the pasted URL, or null for the owner/repo shorthand. */ + host: string | null; + segments: string[]; +} + /** - * Reduce any accepted form to its path segments. - * Returns null when the input is not a GitHub-shaped reference at all. + * Reduce any accepted form to its host and path segments. + * Returns null when the input is not a repository-shaped reference at all. */ -function toPathSegments(input: string): string[] | null { +export function parseRepoReferenceParts(input: string): RepoReferenceParts | null { let value = stripWrapping(input); if (!value) return null; + let host: string | null = null; // git@github.com:owner/repo.git -> owner/repo.git - const sshMatch = value.match(/^(?:ssh:\/\/)?git@[^:/]+[:/](.+)$/i); + const sshMatch = value.match(/^(?:ssh:\/\/)?git@([^:/]+)[:/](.+)$/i); if (sshMatch) { - value = sshMatch[1]; + host = sshMatch[1].toLowerCase(); + value = sshMatch[2]; } else if (/^[a-z][a-z0-9+.-]*:\/\//i.test(value) || /^[^/\s]+\.[^/\s]+\//.test(value)) { // A full URL, or a bare host like "github.com/owner/repo". const withScheme = /^[a-z][a-z0-9+.-]*:\/\//i.test(value) @@ -68,6 +76,7 @@ function toPathSegments(input: string): string[] | null { } catch { return null; } + host = url.host.toLowerCase(); value = url.pathname; } @@ -76,7 +85,52 @@ function toPathSegments(input: string): string[] | null { .map((segment) => segment.trim()) .filter(Boolean); - return segments.length > 0 ? segments : null; + return segments.length > 0 ? { host, segments } : null; +} + +function toPathSegments(input: string): string[] | null { + return parseRepoReferenceParts(input)?.segments ?? null; +} + +/** Segments that start the "rest of the page" after owner/repo on a flat host. */ +const FLAT_HOST_DEEP_LINK_MARKERS = new Set([ + "-", + "actions", + "blob", + "branches", + "commit", + "commits", + "compare", + "issues", + "projects", + "pull", + "pulls", + "releases", + "settings", + "src", + "tags", + "tree", + "wiki", +]); + +/** + * Best effort owner/repo split for a host other than github.com. GitLab + * nests groups and marks deep links with a "-" segment; Gitea keeps a flat + * owner/repo layout. The server re-resolves from the raw segments through + * the configured source, so this only has to be good enough for the form. + */ +function splitNonGithubPath(segments: string[]): { owner: string; repo: string } | null { + const marker = segments.indexOf("-"); + let path = marker === -1 ? segments : segments.slice(0, marker); + if (marker === -1 && path.length > 2 && FLAT_HOST_DEEP_LINK_MARKERS.has(path[2].toLowerCase())) { + path = path.slice(0, 2); + } + if (path.length < 2) return null; + + const repo = path[path.length - 1].replace(GIT_SUFFIX, ""); + const owner = path.slice(0, -1).join("/"); + if (!isUsableRepoName(repo) || !owner) return null; + return { owner, repo }; } function isUsableAccount(segment: string | undefined): segment is string { @@ -98,12 +152,19 @@ function isUsableRepoName(segment: string | undefined): segment is string { export function parseGitHubRepoReference( input: string ): { owner: string; repo: string } | null { - const segments = toPathSegments(input); - if (!segments) return null; + const parts = parseRepoReferenceParts(input); + if (!parts) return null; + const { host, segments } = parts; // /orgs//... never names a repository. if (segments[0]?.toLowerCase() === "orgs") return null; + // Other hosts may nest the owner (GitLab groups) or use different reserved + // paths, so they get the looser split. + if (host && host !== "github.com" && !host.endsWith(".github.com")) { + return splitNonGithubPath(segments); + } + const [owner, rawRepo] = segments; if (!isUsableAccount(owner)) return null; diff --git a/src/lib/utils/mirror-overrides.test.ts b/src/lib/utils/mirror-overrides.test.ts index 3e0012f..87c212c 100644 --- a/src/lib/utils/mirror-overrides.test.ts +++ b/src/lib/utils/mirror-overrides.test.ts @@ -811,3 +811,57 @@ describe("getMirrorOverrideGating - release limit follows releases", () => { expect(gating.releaseLimit).toBe(MIRROR_GATING_REASONS.releasesOff); }); }); + +import { + GITHUB_ONLY_METADATA_KEYS, + resolveMirrorOptions as resolveOptionsForSource, +} from "./mirror-overrides"; + +describe("resolveMirrorOptions for non-GitHub sources", () => { + const config = { + giteaConfig: { + lfs: true, + wiki: true, + mirrorReleases: true, + mirrorMetadata: true, + mirrorIssues: true, + mirrorPullRequests: true, + mirrorLabels: true, + mirrorMilestones: true, + releaseLimit: 5, + }, + } as any; + + test("a GitLab repository keeps code, wiki and LFS but no GitHub API metadata", () => { + const resolved = resolveOptionsForSource({ + config, + repository: { isStarred: false, mirrorOverrides: null, sourceProvider: "gitlab" } as any, + }); + expect(resolved.lfs).toBe(true); + expect(resolved.wiki).toBe(true); + for (const key of GITHUB_ONLY_METADATA_KEYS) { + expect(resolved[key]).toBe(false); + } + }); + + test("the clamp outranks a per repository override", () => { + const resolved = resolveOptionsForSource({ + config, + repository: { + isStarred: false, + mirrorOverrides: { mirrorIssues: true }, + sourceProvider: "gitea", + } as any, + }); + expect(resolved.mirrorIssues).toBe(false); + }); + + test("rows without a source provider are GitHub rows and stay untouched", () => { + const resolved = resolveOptionsForSource({ + config, + repository: { isStarred: false, mirrorOverrides: null } as any, + }); + expect(resolved.mirrorIssues).toBe(true); + expect(resolved.mirrorReleases).toBe(true); + }); +}); diff --git a/src/lib/utils/mirror-overrides.ts b/src/lib/utils/mirror-overrides.ts index 9890d91..5f55e61 100644 --- a/src/lib/utils/mirror-overrides.ts +++ b/src/lib/utils/mirror-overrides.ts @@ -1,6 +1,7 @@ import type { Config } from "@/types/config"; import type { MirrorOverrides, Repository } from "@/lib/db/schema"; import { mirrorOverridesSchema } from "@/lib/db/schema"; +import { normalizeSourceProviderKind } from "@/lib/source-providers/kinds"; /** * The mirror option flags that can be overridden per organization and per @@ -76,6 +77,23 @@ export const STARRED_CLAMPED_KEYS = [ "mirrorMilestones", ] as const satisfies readonly MirrorOverrideKey[]; +/** + * Flags that only work for GitHub sources. + * + * Issue, pull request, release, label and milestone mirroring read the + * GitHub API. A repository from GitLab or Gitea gets code, wiki and LFS + * through the Gitea pull mirror and nothing else, so the resolver forces + * these off for it regardless of what any tier asked for. + */ +export const GITHUB_ONLY_METADATA_KEYS = [ + "mirrorReleases", + "mirrorMetadata", + "mirrorIssues", + "mirrorPullRequests", + "mirrorLabels", + "mirrorMilestones", +] as const satisfies readonly MirrorOverrideKey[]; + /** * Flags surfaced in the override UI. * @@ -342,7 +360,8 @@ export function resolveMirrorOptions({ orgOverrides, }: { config: Partial; - repository: Pick; + repository: Pick & + Partial>; orgOverrides?: unknown; }): ResolvedMirrorOptions { const globalConfig = config.giteaConfig; @@ -388,6 +407,14 @@ export function resolveMirrorOptions({ } } + // Metadata mirroring needs the GitHub API. Rows without a source + // provider predate the column and came from GitHub. + if (normalizeSourceProviderKind(repository.sourceProvider) !== "github") { + for (const key of GITHUB_ONLY_METADATA_KEYS) { + resolved[key] = false; + } + } + return resolved; } diff --git a/src/lib/utils/mirror-source-auth.test.ts b/src/lib/utils/mirror-source-auth.test.ts index 910c3f0..53c9177 100644 --- a/src/lib/utils/mirror-source-auth.test.ts +++ b/src/lib/utils/mirror-source-auth.test.ts @@ -61,3 +61,47 @@ describe("buildGithubSourceAuthPayload", () => { expect(result).toEqual({}); }); }); + +import { buildSourceAuthPayload } from "./mirror-source-auth"; + +describe("buildSourceAuthPayload", () => { + test("GitHub keeps the account name plus token as password", () => { + expect( + buildSourceAuthPayload({ + provider: "github", + token: "ghp_x", + username: "octo", + repositoryOwner: "someone", + }) + ).toEqual({ auth_username: "octo", auth_password: "ghp_x", auth_token: "ghp_x" }); + }); + + test("GitLab always uses the oauth2 username with a personal access token", () => { + expect( + buildSourceAuthPayload({ + provider: "gitlab", + token: " glpat-x ", + username: "me", + repositoryOwner: "acme", + }) + ).toEqual({ auth_username: "oauth2", auth_password: "glpat-x", auth_token: "glpat-x" }); + }); + + test("Gitea uses the configured username, then the repository owner, then a placeholder", () => { + expect( + buildSourceAuthPayload({ provider: "gitea", token: "t", username: "me", repositoryOwner: "acme" }) + .auth_username + ).toBe("me"); + expect( + buildSourceAuthPayload({ provider: "gitea", token: "t", username: "", repositoryOwner: "acme" }) + .auth_username + ).toBe("acme"); + expect(buildSourceAuthPayload({ provider: "gitea", token: "t" }).auth_username).toBe("token"); + }); + + test("returns an empty object without a token for every provider", () => { + for (const provider of ["github", "gitlab", "gitea"] as const) { + expect(buildSourceAuthPayload({ provider, token: " ", username: "me" })).toEqual({}); + } + }); +}); diff --git a/src/lib/utils/mirror-source-auth.ts b/src/lib/utils/mirror-source-auth.ts index 94b4deb..763e27d 100644 --- a/src/lib/utils/mirror-source-auth.ts +++ b/src/lib/utils/mirror-source-auth.ts @@ -1,3 +1,5 @@ +import type { SourceProviderKind } from "@/lib/source-providers/kinds"; + interface BuildGithubSourceAuthPayloadParams { token?: string | null; githubOwner?: string | null; @@ -47,3 +49,59 @@ export function buildGithubSourceAuthPayload({ auth_token: normalizedToken, }; } + +interface BuildSourceAuthPayloadParams { + provider: SourceProviderKind; + token?: string | null; + /** The account the token belongs to. */ + username?: string | null; + repositoryOwner?: string | null; +} + +const GITLAB_AUTH_USERNAME = "oauth2"; +const DEFAULT_GITEA_AUTH_USERNAME = "token"; + +/** + * Build the clone credentials Gitea stores for a pull mirror, per source host. + * + * - GitHub: account name plus the token as password. + * - GitLab: the documented "oauth2" username plus a personal access token. + * - Gitea and Forgejo: any username plus the token as password. + * + * Returns an empty object when no token is available so callers can + * Object.assign the result unconditionally. + */ +export function buildSourceAuthPayload({ + provider, + token, + username, + repositoryOwner, +}: BuildSourceAuthPayloadParams): GithubSourceAuthPayloadOrEmpty { + const normalizedToken = normalize(token); + if (!normalizedToken) { + return {}; + } + + switch (provider) { + case "gitlab": + return { + auth_username: GITLAB_AUTH_USERNAME, + auth_password: normalizedToken, + auth_token: normalizedToken, + }; + case "gitea": + return { + auth_username: + normalize(username) || normalize(repositoryOwner) || DEFAULT_GITEA_AUTH_USERNAME, + auth_password: normalizedToken, + auth_token: normalizedToken, + }; + case "github": + default: + return buildGithubSourceAuthPayload({ + token: normalizedToken, + githubOwner: username, + repositoryOwner, + }); + } +} diff --git a/src/pages/api/config/index.ts b/src/pages/api/config/index.ts index b6661b4..89e7f60 100644 --- a/src/pages/api/config/index.ts +++ b/src/pages/api/config/index.ts @@ -1,4 +1,11 @@ import type { APIRoute } from "astro"; +import { + SOURCE_PROVIDER_LABELS, + isSourceProviderKind, + isValidSourceUrl, + normalizeSourceProviderKind, +} from "@/lib/source-providers/kinds"; +import { evaluateConfigChange, loadConfigLocks } from "@/lib/config-locks"; import { db, configs, users } from "@/lib/db"; import { v4 as uuidv4 } from "uuid"; import { eq, sql } from "drizzle-orm"; @@ -31,6 +38,8 @@ export const POST: APIRoute = async ({ request, locals }) => { mirrorOptions, advancedOptions, notificationConfig, + confirmSourceChange, + confirmDestinationChange, } = body; if (!githubConfig || !giteaConfig || !scheduleConfig || !cleanupConfig || !mirrorOptions || !advancedOptions) { @@ -83,6 +92,27 @@ export const POST: APIRoute = async ({ request, locals }) => { } } + // Validate the source provider and, for GitLab and Gitea sources, the instance URL + if (githubConfig.provider !== undefined && !isSourceProviderKind(githubConfig.provider)) { + return new Response( + JSON.stringify({ success: false, message: "Unknown source provider." }), + { status: 400, headers: { "Content-Type": "application/json" } } + ); + } + const sourceProvider = normalizeSourceProviderKind(githubConfig.provider); + if (sourceProvider !== "github") { + const rawSourceUrl = typeof githubConfig.url === "string" ? githubConfig.url.trim() : ""; + if (rawSourceUrl && !isValidSourceUrl(rawSourceUrl)) { + return new Response( + JSON.stringify({ + success: false, + message: `${SOURCE_PROVIDER_LABELS[sourceProvider]} URL must be a valid http or https URL.`, + }), + { status: 400, headers: { "Content-Type": "application/json" } } + ); + } + } + // Fetch existing config — prefer the active config; fall back to most-recently-updated // so a stale inactive stub never wins over a populated active row (see issue #271). const existingConfigResult = await db @@ -115,6 +145,26 @@ export const POST: APIRoute = async ({ request, locals }) => { } } + // A locked source or destination only changes with explicit confirmation. + if (existingConfig) { + const locks = await loadConfigLocks(userId); + const verdict = evaluateConfigChange({ + locks, + existingSource: existingGithub, + incomingSource: githubConfig, + existingDestinationUrl: existingGitea?.url, + incomingDestinationUrl: giteaConfig.url, + confirmSourceChange: confirmSourceChange === true, + confirmDestinationChange: confirmDestinationChange === true, + }); + if (!verdict.ok) { + return new Response( + JSON.stringify({ success: false, message: verdict.message, lock: verdict.lock }), + { status: 409, headers: { "Content-Type": "application/json" } } + ); + } + } + // Map UI structure to database schema structure first const { githubConfig: mappedGithubConfig, giteaConfig: mappedGiteaConfig } = mapUiToDbConfig( githubConfig, @@ -291,6 +341,8 @@ export const GET: APIRoute = async ({ request, locals }) => { .orderBy(sql`${configs.isActive} DESC`, sql`${configs.updatedAt} DESC`) .limit(1); + const locks = await loadConfigLocks(userId); + if (config.length === 0) { // Create default configuration for the user const defaultConfig = await createDefaultConfig({ userId }); @@ -306,6 +358,7 @@ export const GET: APIRoute = async ({ request, locals }) => { ...uiConfig, scheduleConfig: uiScheduleConfig, cleanupConfig: uiCleanupConfig, + locks, }), { status: 200, @@ -398,6 +451,7 @@ export const GET: APIRoute = async ({ request, locals }) => { nextRun: dbConfig.cleanupConfig.nextRun, }, notificationConfig, + locks, }), { status: 200, headers: { "Content-Type": "application/json" }, @@ -423,6 +477,7 @@ export const GET: APIRoute = async ({ request, locals }) => { nextRun: dbConfig.cleanupConfig.nextRun, }, notificationConfig: dbConfig.notificationConfig, + locks, }), { status: 200, headers: { "Content-Type": "application/json" }, diff --git a/src/pages/api/config/source-url.test.ts b/src/pages/api/config/source-url.test.ts new file mode 100644 index 0000000..2403675 --- /dev/null +++ b/src/pages/api/config/source-url.test.ts @@ -0,0 +1,47 @@ +import { describe, expect, test } from "bun:test"; +import { POST } from "./index"; + +function request(githubConfig: Record) { + return new Request("http://localhost/api/config", { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ + userId: "user-1", + githubConfig, + giteaConfig: { url: "https://gitea.example.com", username: "me", token: "t" }, + scheduleConfig: { enabled: false }, + cleanupConfig: { enabled: false }, + mirrorOptions: { mirrorReleases: false }, + advancedOptions: { skipForks: false }, + }), + }); +} + +async function post(githubConfig: Record) { + return POST({ + request: request(githubConfig), + locals: { session: { userId: "user-1" } }, + } as any); +} + +describe("POST /api/config source validation", () => { + test("rejects an instance URL that is not http(s) for a GitLab source", async () => { + const response = await post({ + username: "me", + token: "glpat-x", + provider: "gitlab", + url: "ftp://gitlab.example.com", + }); + expect(response.status).toBe(400); + const data = await response.json(); + expect(data.success).toBe(false); + expect(data.message).toContain("GitLab URL"); + }); + + test("rejects an unknown source provider", async () => { + const response = await post({ username: "me", token: "t", provider: "bitbucket" }); + expect(response.status).toBe(400); + const data = await response.json(); + expect(data.message).toContain("Unknown source provider"); + }); +}); diff --git a/src/pages/api/github/starred-lists.ts b/src/pages/api/github/starred-lists.ts index 00e5b70..02622fe 100644 --- a/src/pages/api/github/starred-lists.ts +++ b/src/pages/api/github/starred-lists.ts @@ -1,4 +1,5 @@ import type { APIRoute } from "astro"; +import { resolveSourceProviderKind } from "@/lib/source-providers"; import { db, configs } from "@/lib/db"; import { eq, sql } from "drizzle-orm"; import { @@ -38,6 +39,11 @@ export const GET: APIRoute = async ({ request, locals }) => { }); } + // Star lists only exist on GitHub. Other sources simply have none. + if (resolveSourceProviderKind(config) !== "github") { + return jsonResponse({ data: { success: true, lists: [] }, status: 200 }); + } + const token = getDecryptedGitHubToken(config); const githubUsername = config.githubConfig?.owner || undefined; const octokit = createGitHubClient(token, userId, githubUsername); diff --git a/src/pages/api/github/test-connection.ts b/src/pages/api/github/test-connection.ts index b1f273b..b9da9e3 100644 --- a/src/pages/api/github/test-connection.ts +++ b/src/pages/api/github/test-connection.ts @@ -1,89 +1,85 @@ import type { APIRoute } from "astro"; -import { createGitHubClient } from "@/lib/github"; import { createSecureErrorResponse } from "@/lib/utils"; +import { + SOURCE_PROVIDER_LABELS, + createSourceProvider, + isSourceNotFound, + normalizeSourceProviderKind, + normalizeSourceUrl, +} from "@/lib/source-providers"; +function json(body: unknown, status: number): Response { + return new Response(JSON.stringify(body), { + status, + headers: { "Content-Type": "application/json" }, + }); +} + +/** + * Test a source connection. The route keeps its /github/ path for + * compatibility, but serves every source provider: the body carries the + * provider and, for GitLab and Gitea, the instance URL. + */ export const POST: APIRoute = async ({ request }) => { - try { - const body = await request.json(); - const { token, username } = body; + const body = await request.json().catch(() => ({})); + const { token, username, provider: rawProvider, url } = body ?? {}; + const provider = normalizeSourceProviderKind(rawProvider); + const label = SOURCE_PROVIDER_LABELS[provider]; + try { if (!token) { - return new Response( - JSON.stringify({ - success: false, - message: "GitHub token is required", - }), - { - status: 400, - headers: { - "Content-Type": "application/json", - }, - } - ); + return json({ success: false, message: `${label} token is required` }, 400); } - // Create an Octokit instance with the provided token. - // Uses createGitHubClient so GH_API_URL / GITHUB_API_URL routes the call - // to the correct endpoint for GHES / GHEC with data residency. - const octokit = createGitHubClient(token); + // GitHub honors GH_API_URL / GITHUB_API_URL for GHES / GHEC inside the adapter. + const sourceProvider = createSourceProvider({ + provider, + url: normalizeSourceUrl(url, provider), + username: typeof username === "string" ? username : "", + token, + }); - // Test the connection by fetching the authenticated user - const { data } = await octokit.users.getAuthenticated(); + const account = await sourceProvider.testConnection(); // Verify that the authenticated user matches the provided username (if provided) - if (username && data.login !== username) { - return new Response( - JSON.stringify({ - success: false, - message: `Token belongs to ${data.login}, not ${username}`, - }), - { - status: 400, - headers: { - "Content-Type": "application/json", - }, - } + if (username && account.login !== username) { + return json( + { success: false, message: `Token belongs to ${account.login}, not ${username}` }, + 400 ); } - // Return success response with user data - return new Response( - JSON.stringify({ - success: true, - message: `Successfully connected to GitHub as ${data.login}`, - user: { - login: data.login, - name: data.name, - avatar_url: data.avatar_url, - }, - }), + return json( { - status: 200, - headers: { - "Content-Type": "application/json", + success: true, + message: `Successfully connected to ${label} as ${account.login}`, + user: { + login: account.login, + name: account.name ?? null, + avatar_url: account.avatarUrl ?? null, }, - } + }, + 200 ); } catch (error) { - console.error("GitHub connection test failed:", error); + console.error(`${label} connection test failed:`, error); - // Handle specific error types - if (error instanceof Error && (error as any).status === 401) { - return new Response( - JSON.stringify({ - success: false, - message: "Invalid GitHub token", - }), - { - status: 401, - headers: { - "Content-Type": "application/json", - }, - } + const status = + error && typeof error === "object" && "status" in error + ? (error as { status?: unknown }).status + : undefined; + + if (status === 401 || status === 403) { + return json({ success: false, message: `Invalid ${label} token` }, 401); + } + + if (isSourceNotFound(error)) { + return json( + { success: false, message: `${label} API endpoint not found. Please check the instance URL.` }, + 404 ); } - // Generic error response - return createSecureErrorResponse(error, "GitHub connection test", 500); + return createSecureErrorResponse(error, `${label} connection test`, 500); } }; diff --git a/src/pages/api/job/mirror-org.ts b/src/pages/api/job/mirror-org.ts index 44c49c6..6c50a6d 100644 --- a/src/pages/api/job/mirror-org.ts +++ b/src/pages/api/job/mirror-org.ts @@ -1,4 +1,6 @@ import type { APIRoute } from "astro"; +import type { Octokit } from "@octokit/rest"; +import { resolveSourceProviderKind } from "@/lib/source-providers"; import type { MirrorOrgRequest, MirrorOrgResponse } from "@/types/mirror"; import { db, configs, organizations } from "@/lib/db"; import { and, eq, inArray, sql } from "drizzle-orm"; @@ -83,10 +85,14 @@ export const POST: APIRoute = async ({ request, locals }) => { throw new Error("GitHub token is missing in config."); } - // Create a single Octokit instance to be reused with rate limit tracking - const decryptedToken = getDecryptedGitHubToken(config); - const githubUsername = config.githubConfig?.owner || undefined; - const octokit = createGitHubClient(decryptedToken, userId, githubUsername); + // Only GitHub sources need an API client while mirroring (metadata). + // Other hosts get code-only mirrors through Gitea. + let octokit: Octokit | null = null; + if (resolveSourceProviderKind(config) === "github") { + const decryptedToken = getDecryptedGitHubToken(config); + const githubUsername = config.githubConfig?.owner || undefined; + octokit = createGitHubClient(decryptedToken, userId, githubUsername); + } // Define the concurrency limit - adjust based on API rate limits // Using a lower concurrency for organizations since each org might contain many repos diff --git a/src/pages/api/job/mirror-repo.ts b/src/pages/api/job/mirror-repo.ts index 1cb39ae..2d052c2 100644 --- a/src/pages/api/job/mirror-repo.ts +++ b/src/pages/api/job/mirror-repo.ts @@ -1,4 +1,6 @@ import type { APIRoute } from "astro"; +import type { Octokit } from "@octokit/rest"; +import { resolveSourceProviderKind } from "@/lib/source-providers"; import type { MirrorRepoRequest, MirrorRepoResponse } from "@/types/mirror"; import { db, configs, repositories } from "@/lib/db"; import { and, eq, inArray, sql } from "drizzle-orm"; @@ -85,10 +87,14 @@ export const POST: APIRoute = async ({ request, locals }) => { throw new Error("GitHub token is missing."); } - // Create a single Octokit instance to be reused with rate limit tracking - const decryptedToken = getDecryptedGitHubToken(config); - const githubUsername = config.githubConfig?.owner || undefined; - const octokit = createGitHubClient(decryptedToken, userId, githubUsername); + // Only GitHub sources need an API client while mirroring (metadata). + // Other hosts get code-only mirrors through Gitea. + let octokit: Octokit | null = null; + if (resolveSourceProviderKind(config) === "github") { + const decryptedToken = getDecryptedGitHubToken(config); + const githubUsername = config.githubConfig?.owner || undefined; + octokit = createGitHubClient(decryptedToken, userId, githubUsername); + } // Define the concurrency limit - adjust based on API rate limits const CONCURRENCY_LIMIT = 3; diff --git a/src/pages/api/job/retry-repo.ts b/src/pages/api/job/retry-repo.ts index f8bbe39..e14b59b 100644 --- a/src/pages/api/job/retry-repo.ts +++ b/src/pages/api/job/retry-repo.ts @@ -1,4 +1,5 @@ import type { APIRoute } from "astro"; +import { resolveSourceProviderKind } from "@/lib/source-providers"; import { db, configs, repositories } from "@/lib/db"; import { and, eq, inArray, sql } from "drizzle-orm"; import { getGiteaRepoOwnerAsync, isRepoPresentInGitea } from "@/lib/gitea"; @@ -88,9 +89,11 @@ export const POST: APIRoute = async ({ request, locals }) => { ? getDecryptedGitHubToken(config) : null; const githubUsername = config.githubConfig?.owner || undefined; - const octokit = decryptedToken - ? createGitHubClient(decryptedToken, userId, githubUsername) - : null; + // Only GitHub sources need an API client while mirroring (metadata). + const octokit = + decryptedToken && resolveSourceProviderKind(config) === "github" + ? createGitHubClient(decryptedToken, userId, githubUsername) + : null; // Define the concurrency limit - adjust based on API rate limits const CONCURRENCY_LIMIT = 3; diff --git a/src/pages/api/rate-limit/index.ts b/src/pages/api/rate-limit/index.ts index f7043e6..862c635 100644 --- a/src/pages/api/rate-limit/index.ts +++ b/src/pages/api/rate-limit/index.ts @@ -1,4 +1,5 @@ import type { APIRoute } from "astro"; +import { resolveSourceProviderKind } from "@/lib/source-providers"; import { db, rateLimits } from "@/lib/db"; import { eq, and, desc, sql } from "drizzle-orm"; import { jsonResponse, createSecureErrorResponse } from "@/lib/utils"; @@ -28,7 +29,7 @@ export const GET: APIRoute = async ({ request, locals }) => { .orderBy(sql`${configs.isActive} DESC`, sql`${configs.updatedAt} DESC`) .limit(1); - if (config && config.githubConfig?.token) { + if (config && config.githubConfig?.token && resolveSourceProviderKind(config) === "github") { const decryptedToken = getDecryptedGitHubToken(config); const githubUsername = config.githubConfig?.owner || undefined; const octokit = createGitHubClient(decryptedToken, userId, githubUsername); diff --git a/src/pages/api/sync/index.ts b/src/pages/api/sync/index.ts index 81c4985..de28e62 100644 --- a/src/pages/api/sync/index.ts +++ b/src/pages/api/sync/index.ts @@ -3,15 +3,13 @@ import { db, organizations, repositories, configs } from "@/lib/db"; import { eq, and, sql } from "drizzle-orm"; import { v4 as uuidv4 } from "uuid"; import { createMirrorJob } from "@/lib/helpers"; -import { - createGitHubClient, - getGithubOrganizations, - getGithubRepositories, - getGithubStarredRepositories, -} from "@/lib/github"; +import { createSourceProviderFromConfig, SOURCE_PROVIDER_LABELS } from "@/lib/source-providers"; import { jsonResponse, createSecureErrorResponse } from "@/lib/utils"; -import { mergeGitReposPreferStarred, calcBatchSizeForInsert } from "@/lib/repo-utils"; -import { getDecryptedGitHubToken } from "@/lib/utils/config-encryption"; +import { + mergeGitReposPreferStarred, + normalizeGitRepoToInsert, + calcBatchSizeForInsert, +} from "@/lib/repo-utils"; import { requireAuthenticatedUserId } from "@/lib/auth-guards"; import { isMirrorableGitHubRepo } from "@/lib/repo-eligibility"; @@ -39,15 +37,14 @@ export const POST: APIRoute = async ({ request, locals }) => { if (!config.githubConfig?.token) { return jsonResponse({ - data: { error: "GitHub token is missing in config" }, + data: { error: "Source token is missing in config" }, status: 400, }); } - // Decrypt the GitHub token before using it - const decryptedToken = getDecryptedGitHubToken(config); - const githubUsername = config.githubConfig?.owner || undefined; - const octokit = createGitHubClient(decryptedToken, userId, githubUsername); + // The configured source host, with its token decrypted + const sourceProvider = createSourceProviderFromConfig(config, { userId }); + const sourceLabel = SOURCE_PROVIDER_LABELS[sourceProvider.kind]; // Load ignored orgs from the DB so we can skip them during import const ignoredOrgRows = await db @@ -56,13 +53,13 @@ export const POST: APIRoute = async ({ request, locals }) => { .where(and(eq(organizations.userId, userId), eq(organizations.status, "ignored"))); const ignoredOrgNames = new Set(ignoredOrgRows.map((o) => o.normalizedName)); - // Fetch GitHub data in parallel + // Fetch source data in parallel const [basicAndForkedRepos, starredRepos, orgResult] = await Promise.all([ - getGithubRepositories({ octokit, config }), + sourceProvider.listRepositories(config), config.githubConfig?.includeStarred - ? getGithubStarredRepositories({ octokit, config }) + ? sourceProvider.listStarredRepositories(config) : Promise.resolve([]), - getGithubOrganizations({ octokit, config, skipOrgNames: ignoredOrgNames }), + sourceProvider.listOrganizations(config, ignoredOrgNames), ]); const { organizations: gitOrgs, failedOrgs } = orgResult; @@ -70,40 +67,11 @@ export const POST: APIRoute = async ({ request, locals }) => { const allGithubRepos = mergeGitReposPreferStarred(basicAndForkedRepos, starredRepos); const mirrorableGithubRepos = allGithubRepos.filter(isMirrorableGitHubRepo); - // Prepare full list of repos and orgs - const newRepos = mirrorableGithubRepos.map((repo) => ({ - id: uuidv4(), - userId, - configId: config.id, - name: repo.name, - fullName: repo.fullName, - normalizedFullName: repo.fullName.toLowerCase(), - url: repo.url, - cloneUrl: repo.cloneUrl, - owner: repo.owner, - organization: repo.organization ?? null, - mirroredLocation: repo.mirroredLocation || "", - destinationOrg: repo.destinationOrg || null, - isPrivate: repo.isPrivate, - isForked: repo.isForked, - forkedFrom: repo.forkedFrom ?? null, - hasIssues: repo.hasIssues, - isStarred: repo.isStarred, - isArchived: repo.isArchived, - size: repo.size, - hasLFS: repo.hasLFS, - hasSubmodules: repo.hasSubmodules, - language: repo.language ?? null, - description: repo.description ?? null, - defaultBranch: repo.defaultBranch, - visibility: repo.visibility, - status: repo.status, - lastMirrored: repo.lastMirrored ?? null, - errorMessage: repo.errorMessage ?? null, - importedAt: repo.importedAt, - createdAt: repo.createdAt, - updatedAt: repo.updatedAt, - })); + // Prepare full list of repos and orgs. The normalizer stamps the source + // provider and URL on every row. + const newRepos = mirrorableGithubRepos.map((repo) => + normalizeGitRepoToInsert(repo, { userId, configId: config.id }) + ); const newOrgs = gitOrgs.map((org) => ({ id: uuidv4(), @@ -243,7 +211,7 @@ export const POST: APIRoute = async ({ request, locals }) => { repositoryName: repo.name, status: "imported", message: `Repository ${repo.name} fetched successfully`, - details: `Repository ${repo.name} was fetched from GitHub`, + details: `Repository ${repo.name} was fetched from ${sourceLabel}`, }) ), ...insertedOrgs.map((org) => @@ -253,7 +221,7 @@ export const POST: APIRoute = async ({ request, locals }) => { organizationName: org.name, status: "imported", message: `Organization ${org.name} fetched successfully`, - details: `Organization ${org.name} was fetched from GitHub`, + details: `Organization ${org.name} was fetched from ${sourceLabel}`, }) ), ]; @@ -272,6 +240,6 @@ export const POST: APIRoute = async ({ request, locals }) => { }, }); } catch (error) { - return createSecureErrorResponse(error, "GitHub data sync", 500); + return createSecureErrorResponse(error, "source data sync", 500); } }; diff --git a/src/pages/api/sync/organization.ts b/src/pages/api/sync/organization.ts index f3ec40a..297a56d 100644 --- a/src/pages/api/sync/organization.ts +++ b/src/pages/api/sync/organization.ts @@ -6,10 +6,10 @@ import type { AddOrganizationApiRequest, AddOrganizationApiResponse, } from "@/types/organizations"; -import type { RepositoryVisibility, RepoStatus } from "@/types/Repository"; +import type { RepoStatus } from "@/types/Repository"; import { v4 as uuidv4 } from "uuid"; -import { decryptConfigTokens } from "@/lib/utils/config-encryption"; -import { createGitHubClient } from "@/lib/github"; +import { createSourceProviderFromConfig } from "@/lib/source-providers"; +import { normalizeGitRepoToInsert, calcBatchSizeForInsert } from "@/lib/repo-utils"; import { requireAuthenticatedUserId } from "@/lib/auth-guards"; export const POST: APIRoute = async ({ request, locals }) => { @@ -98,106 +98,46 @@ export const POST: APIRoute = async ({ request, locals }) => { } const configId = config.id; - - // Decrypt the config to get tokens - const decryptedConfig = decryptConfigTokens(config); - - // Check if we have a GitHub token - if (!decryptedConfig.githubConfig?.token) { + + if (!config.githubConfig?.token) { return jsonResponse({ - data: { error: "GitHub token not configured" }, + data: { error: "Source token not configured" }, status: 401, }); } - - // Create authenticated Octokit instance with rate limit tracking - const githubUsername = decryptedConfig.githubConfig?.owner || undefined; - const octokit = createGitHubClient( - decryptedConfig.githubConfig.token, - userId, - githubUsername - ); + + // The configured source host, with its token decrypted + const sourceProvider = createSourceProviderFromConfig(config, { userId }); // Fetch org metadata - const { data: orgData } = await octokit.orgs.get({ org: trimmedOrg }); + const orgData = await sourceProvider.getOrganization(trimmedOrg); + if (!orgData) { + return jsonResponse({ + data: { + success: false, + error: `Organization ${trimmedOrg} was not found on the configured source`, + }, + status: 404, + }); + } - // Fetch repos based on config settings - const allRepos = []; - - // Fetch all repos (public, private, and member) to show in UI - const publicRepos = await octokit.paginate(octokit.repos.listForOrg, { - org: trimmedOrg, - type: "public", - per_page: 100, - }); - allRepos.push(...publicRepos); - - // Always fetch private repos to show them in the UI - const privateRepos = await octokit.paginate(octokit.repos.listForOrg, { - org: trimmedOrg, - type: "private", - per_page: 100, - }); - allRepos.push(...privateRepos); - - // Also fetch member repos (includes private repos the user has access to) - const memberRepos = await octokit.paginate(octokit.repos.listForOrg, { - org: trimmedOrg, - type: "member", - per_page: 100, - }); - // Filter out duplicates - const existingIds = new Set(allRepos.map(r => r.id)); - const uniqueMemberRepos = memberRepos.filter(r => !existingIds.has(r.id)); - allRepos.push(...uniqueMemberRepos); - const mirrorableRepos = allRepos.filter((repo) => !repo.disabled); + // Fetch every repository the token can see in the organization + const orgRepos = await sourceProvider.listOrganizationRepositories(trimmedOrg); + const mirrorableRepos = orgRepos.filter((repo) => repo.isDisabled !== true); - // Insert repositories - const repoRecords = mirrorableRepos.map((repo) => { - const normalizedOwner = repo.owner.login.trim().toLowerCase(); - const normalizedRepoName = repo.name.trim().toLowerCase(); - - return { - id: uuidv4(), - userId, - configId, - name: repo.name, - fullName: repo.full_name, - normalizedFullName: `${normalizedOwner}/${normalizedRepoName}`, - url: repo.html_url, - cloneUrl: repo.clone_url ?? "", - owner: repo.owner.login, - organization: - repo.owner.type === "Organization" ? repo.owner.login : null, - mirroredLocation: "", - destinationOrg: null, - isPrivate: repo.private, - isForked: repo.fork, - forkedFrom: null, - hasIssues: repo.has_issues, - isStarred: false, - isArchived: repo.archived, - size: repo.size, - hasLFS: false, - hasSubmodules: false, - language: repo.language ?? null, - description: repo.description ?? null, - defaultBranch: repo.default_branch ?? "main", - visibility: (repo.visibility ?? "public") as RepositoryVisibility, - status: "imported" as RepoStatus, - lastMirrored: null, - errorMessage: null, - importedAt: new Date(), - createdAt: repo.created_at ? new Date(repo.created_at) : new Date(), - updatedAt: repo.updated_at ? new Date(repo.updated_at) : new Date(), - }; - }); + // Insert repositories. The normalizer stamps the source provider and URL. + const repoRecords = mirrorableRepos.map((repo) => + normalizeGitRepoToInsert( + { ...repo, organization: repo.organization ?? orgData.name }, + { userId, configId } + ) + ); // Batch insert repositories to avoid SQLite parameter limit // Compute batch size based on column count const sample = repoRecords[0]; const columnCount = Object.keys(sample ?? {}).length || 1; - const BATCH_SIZE = Math.max(1, Math.floor(999 / columnCount)); + const BATCH_SIZE = calcBatchSizeForInsert(columnCount); for (let i = 0; i < repoRecords.length; i += BATCH_SIZE) { const batch = repoRecords.slice(i, i + BATCH_SIZE); await db @@ -211,15 +151,15 @@ export const POST: APIRoute = async ({ request, locals }) => { id: uuidv4(), userId, configId, - name: orgData.login, + name: orgData.name, normalizedName: normalizedOrg, - avatarUrl: orgData.avatar_url, + avatarUrl: orgData.avatarUrl, membershipRole: role, isIncluded: false, status: "imported" as RepoStatus, - repositoryCount: allRepos.length, - createdAt: orgData.created_at ? new Date(orgData.created_at) : new Date(), - updatedAt: orgData.updated_at ? new Date(orgData.updated_at) : new Date(), + repositoryCount: orgRepos.length, + createdAt: orgData.createdAt, + updatedAt: orgData.updatedAt, }; await db.insert(organizations).values(organizationRecord); diff --git a/src/pages/api/sync/repository.ts b/src/pages/api/sync/repository.ts index cd0afa6..b6b2473 100644 --- a/src/pages/api/sync/repository.ts +++ b/src/pages/api/sync/repository.ts @@ -1,5 +1,4 @@ import type { APIRoute } from "astro"; -import { Octokit } from "@octokit/rest"; import { configs, db, repositories } from "@/lib/db"; import { v4 as uuidv4 } from "uuid"; import { and, eq, sql } from "drizzle-orm"; @@ -8,10 +7,16 @@ import { jsonResponse, createSecureErrorResponse } from "@/lib/utils"; import type { AddRepositoriesApiRequest, AddRepositoriesApiResponse, - RepositoryVisibility, } from "@/types/Repository"; import { createMirrorJob } from "@/lib/helpers"; import { requireAuthenticatedUserId } from "@/lib/auth-guards"; +import { + SOURCE_PROVIDER_LABELS, + createSourceProviderFromConfig, + describeSource, + sourceHostOf, +} from "@/lib/source-providers"; +import { repositorySourceColumns } from "@/lib/repo-utils"; export const POST: APIRoute = async ({ request, locals }) => { try { @@ -20,7 +25,7 @@ export const POST: APIRoute = async ({ request, locals }) => { const userId = authResult.userId; const body: AddRepositoriesApiRequest = await request.json(); - const { owner, repo, force = false, destinationOrg } = body; + const { owner, repo, force = false, destinationOrg, host, path } = body; if (!owner || !repo) { return new Response( @@ -45,9 +50,50 @@ export const POST: APIRoute = async ({ request, locals }) => { }); } - const normalizedOwner = trimmedOwner.toLowerCase(); - const normalizedRepo = trimmedRepo.toLowerCase(); - const normalizedFullName = `${normalizedOwner}/${normalizedRepo}`; + // Get user's active config — prefer active and most-recently-updated to avoid + // picking a stale inactive stub when multiple rows exist (see issue #271). + const [config] = await db + .select() + .from(configs) + .where(eq(configs.userId, userId)) + .orderBy(sql`${configs.isActive} DESC`, sql`${configs.updatedAt} DESC`) + .limit(1); + + if (!config) { + return jsonResponse({ + data: { error: "No configuration found for this user" }, + status: 404, + }); + } + + const configId = config.id; + + // The configured source host. Without a token GitHub still allows public + // lookups, so a missing token is not an error here. + const sourceProvider = createSourceProviderFromConfig(config, { userId }); + const sourceConnection = sourceProvider.connection; + const sourceLabel = SOURCE_PROVIDER_LABELS[sourceProvider.kind]; + + // A pasted URL that names a different host cannot be served by this source. + const configuredHost = sourceHostOf(sourceConnection.url); + if (host && configuredHost && host.trim().toLowerCase() !== configuredHost) { + return jsonResponse({ + data: { + success: false, + error: `That URL points at ${host}, but the configured source is ${describeSource(sourceConnection)}.`, + }, + status: 400, + }); + } + + // Prefer the pasted path segments: the source host knows how to split + // them (GitLab nests groups, GitHub and Gitea do not). + const resolvedPath = + (Array.isArray(path) && path.length > 0 + ? sourceProvider.resolveRepositoryPath(path) + : null) ?? { owner: trimmedOwner, repo: trimmedRepo }; + + const normalizedFullName = `${resolvedPath.owner}/${resolvedPath.repo}`.toLowerCase(); // Check if repository with the same owner, name, and userId already exists const [existingRepo] = await db @@ -72,71 +118,46 @@ export const POST: APIRoute = async ({ request, locals }) => { }); } - // Get user's active config — prefer active and most-recently-updated to avoid - // picking a stale inactive stub when multiple rows exist (see issue #271). - const [config] = await db - .select() - .from(configs) - .where(eq(configs.userId, userId)) - .orderBy(sql`${configs.isActive} DESC`, sql`${configs.updatedAt} DESC`) - .limit(1); - - if (!config) { + const repoData = await sourceProvider.getRepository(resolvedPath.owner, resolvedPath.repo); + if (!repoData) { return jsonResponse({ - data: { error: "No configuration found for this user" }, + data: { + success: false, + error: `Repository ${resolvedPath.owner}/${resolvedPath.repo} was not found on ${sourceLabel}`, + }, status: 404, }); } - const configId = config.id; - - // Unauthenticated one-shot lookup for public repos. - // Uses bare Octokit (not createGitHubClient) to preserve fast-fail on the - // 60 req/hr public rate limit — this endpoint is user-facing, we don't - // want the throttling plugin to wait multiple retry-after windows. - // Still respects GH_API_URL / GITHUB_API_URL for GHES / GHEC data residency. - const baseUrl = - process.env.GH_API_URL || - process.env.GITHUB_API_URL || - "https://api.github.com"; - const octokit = new Octokit({ baseUrl }); - - const { data: repoData } = await octokit.rest.repos.get({ - owner: trimmedOwner, - repo: trimmedRepo, - }); - const baseMetadata = { userId, configId, name: repoData.name, - fullName: repoData.full_name, - normalizedFullName, - url: repoData.html_url, - cloneUrl: repoData.clone_url, - owner: repoData.owner.login, - organization: - repoData.owner.type === "Organization" ? repoData.owner.login : null, - isPrivate: repoData.private, - isForked: repoData.fork, - forkedFrom: null, - hasIssues: repoData.has_issues, + fullName: repoData.fullName, + normalizedFullName: repoData.fullName.toLowerCase(), + url: repoData.url, + cloneUrl: repoData.cloneUrl, + owner: repoData.owner, + organization: repoData.organization ?? null, + ...repositorySourceColumns(repoData), + isPrivate: repoData.isPrivate, + isForked: repoData.isForked, + forkedFrom: repoData.forkedFrom ?? null, + hasIssues: repoData.hasIssues, isStarred: false, - isArchived: repoData.archived, + isArchived: repoData.isArchived, size: repoData.size, - hasLFS: false, - hasSubmodules: false, + hasLFS: repoData.hasLFS, + hasSubmodules: repoData.hasSubmodules, language: repoData.language ?? null, description: repoData.description ?? null, - defaultBranch: repoData.default_branch, - visibility: (repoData.visibility ?? "public") as RepositoryVisibility, + defaultBranch: repoData.defaultBranch, + visibility: repoData.visibility, lastMirrored: existingRepo?.lastMirrored ?? null, errorMessage: existingRepo?.errorMessage ?? null, mirroredLocation: existingRepo?.mirroredLocation ?? "", destinationOrg: destinationOrg?.trim() || existingRepo?.destinationOrg || null, - updatedAt: repoData.updated_at - ? new Date(repoData.updated_at) - : new Date(), + updatedAt: repoData.updatedAt, }; if (existingRepo && force) { @@ -144,7 +165,6 @@ export const POST: APIRoute = async ({ request, locals }) => { .update(repositories) .set({ ...baseMetadata, - normalizedFullName, configId, }) .where(eq(repositories.id, existingRepo.id)) @@ -167,9 +187,7 @@ export const POST: APIRoute = async ({ request, locals }) => { mirroredLocation: "", destinationOrg: null, importedAt: new Date(), - createdAt: repoData.created_at - ? new Date(repoData.created_at) - : new Date(), + createdAt: repoData.createdAt, ...baseMetadata, } satisfies Repository; @@ -186,7 +204,7 @@ export const POST: APIRoute = async ({ request, locals }) => { repositoryName: metadata.name, status: "imported", message: `Repository ${metadata.name} fetched successfully`, - details: `Repository ${metadata.name} was fetched from GitHub`, + details: `Repository ${metadata.name} was fetched from ${sourceLabel}`, }); const resPayload: AddRepositoriesApiResponse = { diff --git a/src/types/Repository.ts b/src/types/Repository.ts index 5f54789..ac1061f 100644 --- a/src/types/Repository.ts +++ b/src/types/Repository.ts @@ -1,4 +1,5 @@ import type { Repository } from "@/lib/db/schema"; +import type { SourceProviderKind } from "@/lib/source-providers/kinds"; import { z } from "zod"; export const repoStatusEnum = z.enum([ @@ -51,6 +52,9 @@ export interface GitRepo { owner: string; organization?: string; mirroredLocation?: string; + /** Host the repository came from. Absent means GitHub. */ + sourceProvider?: SourceProviderKind; + sourceUrl?: string; destinationOrg?: string | null; isPrivate: boolean; @@ -86,6 +90,10 @@ export interface AddRepositoriesApiRequest { owner: string; force?: boolean; destinationOrg?: string; + /** Host of the pasted URL, when one was pasted. Must match the configured source. */ + host?: string; + /** Path segments of the pasted URL, resolved per source on the server. */ + path?: string[]; } export interface AddRepositoriesApiResponse { diff --git a/src/types/config.ts b/src/types/config.ts index 7dafac3..b5c19ad 100644 --- a/src/types/config.ts +++ b/src/types/config.ts @@ -5,8 +5,14 @@ export type MirrorStrategy = "preserve" | "single-org" | "flat-user" | "mixed"; export type StarredReposMode = "dedicated-org" | "preserve-owner"; export type BackupStrategy = "disabled" | "always" | "on-force-push" | "block-on-force-push"; export type ScheduleMode = "interval" | "clock"; +/** Which host repositories are pulled from. "gitea" also covers Forgejo. */ +export type SourceProvider = "github" | "gitlab" | "gitea"; +/** Which host repositories are mirrored into. Same API, different label. */ +export type DestinationProvider = "gitea" | "forgejo"; export interface GiteaConfig { + /** Defaults to "gitea" when absent. */ + provider?: DestinationProvider; url: string; externalUrl?: string; username: string; @@ -58,6 +64,10 @@ export interface DatabaseCleanupConfig { export type DuplicateNameStrategy = "suffix" | "prefix" | "owner-org"; export interface GitHubConfig { + /** Defaults to "github" when absent. */ + provider?: SourceProvider; + /** Instance base URL for GitLab and Gitea sources. */ + url?: string; username: string; token: string; privateRepositories: boolean; @@ -90,6 +100,17 @@ export interface AdvancedOptions { skipPersonalRepos?: boolean; } +/** + * Whether the source and destination may still be changed freely. Once + * repositories have been imported the source is locked, and once anything + * has been mirrored the destination is locked; a change then needs the + * matching confirm flag on the save request. + */ +export interface ConfigLockState { + source: { locked: boolean; repositoryCount: number }; + destination: { locked: boolean; mirroredCount: number }; +} + export interface SaveConfigApiRequest { userId: string; githubConfig: GitHubConfig; @@ -99,6 +120,10 @@ export interface SaveConfigApiRequest { notificationConfig?: NotificationConfig; mirrorOptions?: MirrorOptions; advancedOptions?: AdvancedOptions; + /** Required to change a locked source (provider or instance URL). */ + confirmSourceChange?: boolean; + /** Required to change a locked destination (Gitea server URL). */ + confirmDestinationChange?: boolean; } export interface SaveConfigApiResponse { @@ -164,5 +189,6 @@ export interface ConfigApiResponse { exclude: string[]; createdAt: Date; updatedAt: Date; + locks?: ConfigLockState; error?: string; }