mirror of
https://github.com/RayLabsHQ/gitea-mirror.git
synced 2026-11-03 07:42:19 +01:00
Compare commits
44
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
d45ef33c1c | ||
|
|
4f96e7dfbf | ||
|
|
41799154fc | ||
|
|
8de3c0c091 | ||
|
|
89babdf461 | ||
|
|
44d927e768 | ||
|
|
0f55d29298 | ||
|
|
ba50792b0c | ||
|
|
24fd26656e | ||
|
|
34abda94fb | ||
|
|
1c1ff0b2fa | ||
|
|
e7e07d08f8 | ||
|
|
fdb394942b | ||
|
|
1d0afe2d2e | ||
|
|
aaef5d6e5c | ||
|
|
604ff3b6b3 | ||
|
|
935bc89e52 | ||
|
|
133268e63a | ||
|
|
8ae778d708 | ||
|
|
32e40c9262 | ||
|
|
c2bbcdf6ab | ||
|
|
c6157cdaad | ||
|
|
610bd6040d | ||
|
|
7dd14dbb2e | ||
|
|
6197352a8e | ||
|
|
a3ccc4ea26 | ||
|
|
f9f483ad91 | ||
|
|
261a9729e2 | ||
|
|
23431266cf | ||
|
|
f94752aa71 | ||
|
|
f581292912 | ||
|
|
8f60ccf6dc | ||
|
|
c8618743c8 | ||
|
|
2ff2e71d18 | ||
|
|
2d1795051d | ||
|
|
9c11d398e9 | ||
|
|
3df3fa577e | ||
|
|
0759db80d3 | ||
|
|
0299be92b1 | ||
|
|
22a0aafcc1 | ||
|
|
3b78f0a8d3 | ||
|
|
48e0c5c346 | ||
|
|
01f926d214 | ||
|
|
1cc7a9a05c |
No files matched your search
@@ -214,6 +214,10 @@ DOCKER_TAG=latest
|
||||
# AUTHENTICATION CONFIGURATION
|
||||
# ===========================================
|
||||
|
||||
# Login page: tab that opens first when both email and SSO are available
|
||||
# AUTH_DEFAULT_METHOD=email # email or sso
|
||||
# AUTH_ALLOW_SIGNUP=false # true keeps email sign-up open after the first account
|
||||
|
||||
# Header Authentication (for Reverse Proxy SSO)
|
||||
# Enable automatic authentication via reverse proxy headers
|
||||
# HEADER_AUTH_ENABLED=false
|
||||
|
||||
@@ -28,6 +28,13 @@ on:
|
||||
schedule:
|
||||
- cron: '0 0 * * 0' # Weekly security scan on Sunday at midnight
|
||||
|
||||
# One build per ref at a time. A merge and the release bump that follows it
|
||||
# land on main seconds apart and used to race each other for the latest tag
|
||||
# (#425). Branch and PR builds cancel the older run; tag builds always finish.
|
||||
concurrency:
|
||||
group: docker-build-${{ github.ref }}
|
||||
cancel-in-progress: ${{ !startsWith(github.ref, 'refs/tags/') }}
|
||||
|
||||
env:
|
||||
REGISTRY: ghcr.io
|
||||
IMAGE: ${{ github.repository }}
|
||||
@@ -90,9 +97,9 @@ jobs:
|
||||
echo "APP_VERSION=${APP_VERSION}" >> $GITHUB_OUTPUT
|
||||
echo "Using version tag: ${TAG_VERSION}"
|
||||
else
|
||||
echo "VERSION=latest" >> $GITHUB_OUTPUT
|
||||
echo "VERSION=edge" >> $GITHUB_OUTPUT
|
||||
echo "APP_VERSION=dev" >> $GITHUB_OUTPUT
|
||||
echo "No version tag, using 'latest'"
|
||||
echo "No version tag, using 'edge'"
|
||||
fi
|
||||
|
||||
# Keep version files aligned automatically for tag-based releases
|
||||
@@ -116,12 +123,15 @@ jobs:
|
||||
images: ${{ env.REGISTRY }}/${{ env.IMAGE }}
|
||||
labels: |
|
||||
org.opencontainers.image.revision=${{ env.SHA }}
|
||||
# latest follows the release tag, not main: a main build can carry
|
||||
# an unreleased package.json and must never overwrite a release.
|
||||
# Pre-release tags (v1.2.3-rc.1) keep latest on the last stable one.
|
||||
tags: |
|
||||
type=edge,branch=$repo.default_branch
|
||||
type=edge,branch=main
|
||||
type=semver,pattern=v{{version}}
|
||||
type=sha,prefix=,suffix=,format=short
|
||||
type=raw,value=latest,enable={{is_default_branch}}
|
||||
type=raw,value=${{ steps.tag_version.outputs.VERSION }}
|
||||
type=raw,value=latest,enable=${{ startsWith(github.ref, 'refs/tags/v') && !contains(github.ref, '-') }}
|
||||
type=raw,value=${{ steps.tag_version.outputs.VERSION }},enable=${{ startsWith(github.ref, 'refs/tags/v') }}
|
||||
type=ref,event=pr,prefix=pr-
|
||||
|
||||
# Build and push Docker image
|
||||
@@ -223,7 +233,10 @@ jobs:
|
||||
if: github.event_name != 'pull_request'
|
||||
with:
|
||||
command: cves,recommendations
|
||||
image: ${{ env.REGISTRY }}/${{ env.IMAGE }}:latest
|
||||
# Scan the image this run pushed: the release tag on a tag build,
|
||||
# edge on a main or scheduled build. The action rejects digest
|
||||
# references ("missing or wrong image input").
|
||||
image: ${{ env.REGISTRY }}/${{ env.IMAGE }}:${{ steps.tag_version.outputs.VERSION }}
|
||||
sarif-file: scout-results.sarif
|
||||
summary: true
|
||||
exit-code: false
|
||||
|
||||
@@ -7,10 +7,49 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
|
||||
|
||||
## [Unreleased]
|
||||
|
||||
### Fixed
|
||||
- The Activity Log summary no longer shows a repository as syncing after a quick sync finished (#454). Events are stored to the second, and a sync that starts and finishes in the same second could come back in the wrong order, so the start event was counted as the current state. Activities now come back newest first within the same second too, and the summary counts a finish over a start from the same second
|
||||
- Issue and pull request passes over large repositories finish across rate limit resets (#449 follow-up)
|
||||
- A pass stopped by the rate limit keeps the numbers it finished, and the next run repeats the same pass and skips them unless they changed since it started; before, every run started over from the first item, so a repository too large for one hour of budget never completed a pass
|
||||
- The weekly full pass no longer fetches comments for issues, or the detail, commits and files of pull requests, that are already in the destination and have not changed since the last completed pass; it lists everything and corrects title, body, state and labels from the listing
|
||||
- A sync paused by the rate limit logs one line instead of the error object with every partial result, and its activity entry says it was paused
|
||||
- No request leaves the process while GitHub is rate limited (#437, second report)
|
||||
- The v3.37.0 change stopped the scheduler run from holding its lock through a rate limit, but every other request already queued in the run still went to GitHub and came back 403, and the retry helper sent each of them three more times; GitHub counts requests made while limited toward abuse detection and one account was suspended that way
|
||||
- Every GitHub client now holds its requests while the source is paused: a limit that resets within two minutes is waited out inside the request, a longer one fails the request at once without contacting GitHub, and the rate limit probe endpoint stays available so the pause can end early
|
||||
- The retry helper no longer retries a rate limit refusal and stops starting the remaining items of the batch
|
||||
- A rate limit refusal inside the issues, pull requests, releases, labels or milestones pass now fails the repository (which the scheduler already puts back to its previous status and retries after the reset) instead of moving on to the next component, and a pull request whose detail call was refused is no longer written as a stripped down issue
|
||||
- A secondary rate limit pauses the whole client for its retry-after window, not only the request that was told to slow down
|
||||
|
||||
### Security
|
||||
- Source URLs go through the outbound guard too (GHSA-p7w3-46pg-mv6h): `POST /api/sources`, `PUT /api/sources/:id` and the configuration save refuse link local and metadata addresses for source and destination URLs, the source fetch helper pins and never follows redirects like the other user supplied URLs, source errors no longer carry the upstream body in their message, and errors raised by a request to another host are never forwarded to API clients whatever words they contain
|
||||
- Fixed four privately reported vulnerabilities (GHSA-9m33-xfrc-5jxw, GHSA-6m23-28hh-gjh2, GHSA-2hpx-83vg-gm45, GHSA-5pp8-r7f5-6q8p)
|
||||
- `POST /api/gitea/test-connection` and `POST /api/github/test-connection` now require a signed-in user; they were the only non-public API routes without a guard and made a server side request to any URL in the body
|
||||
- The email sign-up endpoint is closed server side once an account exists (`AUTH_ALLOW_SIGNUP=true` reopens it); only the signup page redirected before, so anyone reaching the instance could create an account through the API
|
||||
- SSO providers are scoped to the user who created them for listing, updating and deleting, and the OIDC client secret is no longer returned by the API (the form keeps the stored secret when the field is left blank)
|
||||
- HTTP errors from Gitea and other hosts no longer carry the upstream response body in the message, so an error can no longer echo the content of a host the caller chose
|
||||
- Requests to user supplied URLs (connection tests, OIDC discovery, ntfy, Gotify, Apprise and webhook notifications) refuse link local and cloud metadata addresses and no longer follow redirects; private networks stay allowed because mirroring to a LAN Gitea is the normal deployment
|
||||
- Raised the devalue floor to 5.9.2 in the application and the documentation site (GHSA-9rgm-9g3h-6x36, denial of service through malformed input); both lockfiles resolve 5.9.4
|
||||
- Raised the dependency floors for the advisories published on 2026-09-08: Astro 7.2.8 (remote code execution through AVIF image optimization, and an authorization bypass when stripping the configured base), @xmldom/xmldom 0.8.15 (eight parser and serializer issues), sharp 0.35.4 (libheif), svgo 4.1.0 (removeScripts sanitization) and js-yaml 4.3.2 (merge-key CPU use). Applied to both the application and the documentation site.
|
||||
|
||||
### Added
|
||||
- List view for the Organizations page (#428)
|
||||
- A cards/list switch in the toolbar; the list shows one row per organization with its role, destination, repository counts, last mirrored time, status, the same actions as the cards and the source and destination links, so a few dozen organizations fit on one screen
|
||||
- The choice is remembered per browser, next to the theme and time format preferences
|
||||
- Incremental issue and pull request sync (#449)
|
||||
- After a complete pass, the issues and pull request passes ask GitHub only for items updated since the last one (with a 10 minute margin), so comments, pull request details, commits and files are fetched only for what changed instead of for every item on every sync
|
||||
- The watermark is stored per repository in the metadata state and only moves when a pass finishes with no failed item
|
||||
- A full pass still runs when the last one is older than 7 days, when the destination has no mirrored issues or pull requests, and after Reset metadata
|
||||
- Releases mirror from Gitea and Forgejo sources, Codeberg included (#440)
|
||||
- The release mirror lists releases through the repository's own source instead of always through the GitHub API, and keeps the same release limit, asset limit, per-destination lock, tag check and retention pruning
|
||||
- Release assets are downloaded with the credentials of the host they live on, so a Gitea source uses its own token and a public one needs none
|
||||
- The release switch and its two limits are enabled for Gitea and Forgejo sources; GitLab stays code only
|
||||
- Sync an organization that is already mirrored (#429)
|
||||
- The organization card menu gains Sync Organization for mirrored and failed organizations; the Mirror button only ever covered the first run
|
||||
- `POST /api/job/sync-org` re-discovers the organization's repositories from its source, mirrors the imported ones and syncs the mirrored, synced and failed ones, leaving rows another run owns alone; the organization is claimed before the response so two clicks cannot both start a run
|
||||
- CSV export of repositories and organizations (#428)
|
||||
- `GET /api/repositories/export` and `GET /api/organizations/export` return the tables without the internal fields (ids, sync metadata, mirror option overrides); an Export CSV button sits in both toolbars and API keys work on both routes
|
||||
- The login page can open on SSO (#438)
|
||||
- `AUTH_DEFAULT_METHOD=sso` sets the instance default, and each browser remembers the method it last signed in with; the new public `GET /api/auth/methods` tells the login page which methods exist and which to open on
|
||||
- Public organizations without a source connection (#409)
|
||||
- The Add Organization dialog offers a Public only mode, the default when no source is connected: pick GitHub, GitLab or Gitea/Forgejo, optionally an instance URL, and the organization is imported anonymously
|
||||
- A tokenless source row is found or created for that provider and host and the organization is pinned to it, so attribution, locks, the scheduler and cleanup keep working per source
|
||||
@@ -41,12 +80,46 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
|
||||
- Proper documentation for multi-URL access patterns
|
||||
- Comprehensive fix report documentation
|
||||
|
||||
### Changed
|
||||
- A new configuration starts with scheduling off unless `SCHEDULE_ENABLED=true`, `SCHEDULE_INTERVAL` or `GITEA_MIRROR_INTERVAL` turns it on, the same rule the environment loader uses; the built-in default schedule is the daily 22:00 clock schedule the automation card shows, so enabling from the card and the scheduler agree from the first save (#427)
|
||||
|
||||
### Fixed
|
||||
- A GitHub rate limit no longer holds the scheduler lock until the reset (#437)
|
||||
- Every rate-limited request used to sleep for the full reset window inside the scheduler run, up to three times per request, so a large sync could keep the lock for hours while every tick logged "Scheduler is already running"; restarting was the only way out and the repositories that failed on the limit needed a manual retry
|
||||
- Waits of up to two minutes are still slept off in the request; a longer reset pauses the source instead, the run stops at the next batch or repository, the lock is released and the next run is moved to just after the reset
|
||||
- A repository whose attempt failed only because of the rate limit goes back to the status it had before, so the run after the reset picks it up on its own
|
||||
- The `latest` Docker image tag no longer lags behind a release (#425)
|
||||
- A merge and the version bump that follows it land on main seconds apart; both built the image and both pushed `latest`, and on v3.36.1 the older build finished last, so `latest` carried 3.36.0 until the weekly rebuild replaced it
|
||||
- The workflow runs one build per ref at a time, `latest` is pushed only by a stable release tag build, main builds push `edge` and the short commit sha, and the security scan looks at the image the run just pushed
|
||||
- The automation card shows the schedule that actually runs (#427)
|
||||
- A fresh install started with scheduling on and a plain 24 hour interval counted from first login, and the card, which only knows clock schedules, showed the 22:00 placeholder as if it were saved; the same misreport hit Docker installs with `GITEA_MIRROR_INTERVAL=8h`
|
||||
- When a plain interval is stored (from `SCHEDULE_INTERVAL`, `GITEA_MIRROR_INTERVAL` or an older version) the card says what runs and where it came from, and leaves frequency and start time unset until one is picked
|
||||
- The timezone chip shows the stored timezone instead of the browser's, and becomes a one-click switch to the browser timezone when the two differ
|
||||
- Ignoring an organization stops its repositories from syncing (#429)
|
||||
- Ignore Organization only changed the organization row; its mirrored repositories kept syncing and rediscovery kept importing new ones for it
|
||||
- The organization's idle repositories are ignored with it (rows in flight or being deleted finish on their own) and restored when it is included again, mirrored ones as mirrored and the rest as imported; rediscovery skips ignored organizations
|
||||
- Organization cards name the configured destination instead of always saying Gitea (#430)
|
||||
- The Nix snippets in the README, NIX.md and the deployment guide import the module before enabling `services.gitea-mirror`, and put the flake input in `flake.nix` where it belongs (#426)
|
||||
- Crash recovery no longer resumes a job that is still running
|
||||
- A job older than two hours was treated as interrupted even while it checkpointed every two minutes, so recovery started a second pass over the same repositories alongside the original; the age rule is gone and only a missing or stale checkpoint marks a job interrupted
|
||||
- Every completed item is now recorded in the job's checkpoint instead of one in every N, so a real resume skips exactly what was done; progress events are still throttled
|
||||
- Concurrent items completing at the same time could overwrite each other's checkpoint; job progress writes are serialized per job
|
||||
- A resumed job records a fresh start time
|
||||
- The same upstream repository is no longer imported twice under two sources that point at the same host
|
||||
- Discovery decided what was new by (source, full name), so an organization pinned to a public-only github.com source and a personal github.com token source each inserted their own row for one repository, and both rows mirrored to the same destination in the same scheduler batch
|
||||
- Auto import, organization re-discovery and the two import endpoints now identify a repository by its host and full name whichever source lists it; the same name on a different host is still a separate repository
|
||||
- A scheduler pass mirrors or syncs one row per upstream repository and logs the duplicate rows it skips, so existing duplicates never run side by side
|
||||
- A per-organization Mirror Destination equal to the organization's own name is kept instead of being dropped (#416)
|
||||
- The editor treated a typed value matching the organization name as "reset to default" and saved no override, which only holds under the preserve strategy; under single-org the default is the destination organization, so the repositories went there
|
||||
- The organization card now shows the default for the configured strategy in the preview, the placeholder, the helper text and the reset button, and marks any stored destination as custom
|
||||
- The repository destination column shows the organization's override as a repository's default, and pinning a repository to the value the strategy already produces is stored rather than discarded
|
||||
- Reset to Default on an organization card clears the override instead of saving the current value again
|
||||
- Release assets are never duplicated on the destination (#417)
|
||||
- Gitea and Forgejo accept any number of attachments with the same name, so two mirror passes that overlapped on one repository both saw an asset as missing and both uploaded it, leaving two, three or five copies
|
||||
- Release reconciliation now runs one pass at a time per destination repository, keeps a single copy of each asset, and deletes surplus and stale copies before uploading anything
|
||||
- A sync no longer starts on a repository that is already being mirrored or synced, and the mirror path claims the repository in one statement instead of checking and then writing
|
||||
- Nothing is uploaded when the destination cannot say which assets a release already has, or when a stale copy could not be removed first
|
||||
- The next sync of an affected repository removes the surplus copies for every release that still gets its assets; releases that have dropped out of the newest-N window keep their duplicates until they are removed by hand
|
||||
- Fixed metadata mirroring authentication errors (#68)
|
||||
- Changed field checking from `username` to `defaultOwner` in metadata functions
|
||||
- Added proper field validation for all metadata operations
|
||||
|
||||
@@ -76,6 +76,9 @@ COPY --from=builder /app/docker-entrypoint.sh ./docker-entrypoint.sh
|
||||
COPY --from=builder /app/scripts/runtime-server.ts ./scripts/runtime-server.ts
|
||||
COPY --from=builder /app/drizzle ./drizzle
|
||||
|
||||
# Database commands must use the bundled script: sources are not shipped.
|
||||
RUN sed -i 's#bun scripts/manage-db.ts#bun dist/scripts/manage-db.js#g' package.json
|
||||
|
||||
# Remove build-only packages that are not needed at runtime
|
||||
# (esbuild, vite, rollup, tailwind, svgo — all only used during `astro build`)
|
||||
RUN rm -rf node_modules/esbuild node_modules/@esbuild \
|
||||
|
||||
@@ -46,9 +46,13 @@ nix run --extra-experimental-features 'nix-command flakes' .#gitea-mirror
|
||||
|
||||
### 4. NixOS System Service
|
||||
```nix
|
||||
# configuration.nix
|
||||
# flake.nix
|
||||
inputs.gitea-mirror.url = "github:RayLabsHQ/gitea-mirror";
|
||||
|
||||
# configuration.nix (needs `inputs` passed through specialArgs)
|
||||
{ inputs, ... }:
|
||||
{
|
||||
inputs.gitea-mirror.url = "github:RayLabsHQ/gitea-mirror";
|
||||
imports = [ inputs.gitea-mirror.nixosModules.default ];
|
||||
|
||||
services.gitea-mirror = {
|
||||
enable = true;
|
||||
|
||||
@@ -31,7 +31,7 @@ First user signup becomes admin. Configure GitHub and Gitea/Forgejo through the
|
||||
## ✨ Features
|
||||
|
||||
- 🔁 Mirror public, private, and starred GitHub repos to Gitea/Forgejo
|
||||
- 🦊 **Multiple sources (GitLab and Gitea/Forgejo beta)** - Connect several sources at once: GitHub and GitHub Enterprise, GitLab, Gitea/Forgejo, and each repository keeps its source. gitlab.com, Codeberg and self hosted instances mirror code, tags, wiki and LFS (issues, pull requests and releases need a GitHub source). GitHub to Gitea is the supported default; every other source or destination is beta. See [docs/SOURCE_PROVIDERS.md](docs/SOURCE_PROVIDERS.md)
|
||||
- 🦊 **Multiple sources (GitLab and Gitea/Forgejo beta)** - Connect several sources at once: GitHub and GitHub Enterprise, GitLab, Gitea/Forgejo, and each repository keeps its source. gitlab.com, Codeberg and self hosted instances mirror code, tags, wiki and LFS; Gitea and Forgejo sources mirror releases with their assets too (issues and pull requests need a GitHub source). GitHub to Gitea is the supported default; every other source or destination is beta. See [docs/SOURCE_PROVIDERS.md](docs/SOURCE_PROVIDERS.md)
|
||||
- 🏛️ **GitHub Enterprise support** - Works with GHES and GHEC with data residency via `GH_API_URL`
|
||||
- 🚀 **Forgejo, GitHub and GitLab destinations (beta)** - Forgejo works like Gitea. Pick GitHub or GitLab and the app keeps a bare clone of each repository and pushes its branches and tags there. Code only, and the target is overwritten on every sync. See [docs/PUSH_TARGETS.md](docs/PUSH_TARGETS.md)
|
||||
- 🏢 Mirror entire organizations with flexible strategies, including public organizations from GitHub, GitLab or Gitea/Forgejo with no source account configured
|
||||
@@ -174,10 +174,15 @@ nix profile install --extra-experimental-features 'nix-command flakes' github:Ra
|
||||
gitea-mirror
|
||||
```
|
||||
|
||||
**NixOS users** - add to your configuration:
|
||||
**NixOS users** - add the flake input, import the module, then enable the service:
|
||||
```nix
|
||||
# flake.nix
|
||||
inputs.gitea-mirror.url = "github:RayLabsHQ/gitea-mirror";
|
||||
|
||||
# configuration.nix (needs `inputs` passed through specialArgs)
|
||||
{ inputs, ... }:
|
||||
{
|
||||
inputs.gitea-mirror.url = "github:RayLabsHQ/gitea-mirror";
|
||||
imports = [ inputs.gitea-mirror.nixosModules.default ];
|
||||
|
||||
services.gitea-mirror = {
|
||||
enable = true;
|
||||
|
||||
@@ -6,7 +6,7 @@
|
||||
"name": "gitea-mirror",
|
||||
"dependencies": {
|
||||
"@astrojs/mdx": "^7.0.3",
|
||||
"@astrojs/node": "^11.0.2",
|
||||
"@astrojs/node": "^11.1.5",
|
||||
"@astrojs/react": "^6.0.1",
|
||||
"@better-auth/api-key": "1.7.2",
|
||||
"@better-auth/oauth-provider": "1.7.2",
|
||||
@@ -86,7 +86,7 @@
|
||||
"@esbuild-kit/esm-loader": "npm:tsx@^4.21.0",
|
||||
"@xmldom/xmldom": "^0.8.15",
|
||||
"browserslist": "^4.28.7",
|
||||
"devalue": "^5.8.1",
|
||||
"devalue": "^5.9.2",
|
||||
"fast-uri": "^3.1.5",
|
||||
"fast-xml-builder": "^1.1.7",
|
||||
"js-yaml": "^4.3.2",
|
||||
@@ -94,6 +94,7 @@
|
||||
"lodash": "^4.18.1",
|
||||
"picomatch": "^4.0.4",
|
||||
"sharp": "^0.35.4",
|
||||
"smol-toml": "^1.7.1",
|
||||
"svgo": "^4.1.0",
|
||||
},
|
||||
"packages": {
|
||||
@@ -143,7 +144,7 @@
|
||||
|
||||
"@astrojs/mdx": ["@astrojs/mdx@7.0.3", "", { "dependencies": { "@astrojs/internal-helpers": "0.10.1", "@astrojs/markdown-remark": "7.2.1", "@mdx-js/mdx": "^3.1.1", "acorn": "^8.16.0", "es-module-lexer": "^2.0.0", "estree-util-visit": "^2.0.0", "hast-util-to-html": "^9.0.5", "piccolore": "^0.1.3", "rehype-raw": "^7.0.0", "remark-gfm": "^4.0.1", "remark-smartypants": "^3.0.2", "source-map": "^0.7.6", "unist-util-visit": "^5.1.0", "vfile": "^6.0.3" }, "peerDependencies": { "@astrojs/markdown-satteri": "^0.3.1", "astro": "^7.0.0" }, "optionalPeers": ["@astrojs/markdown-satteri"] }, "sha512-RxyIwU0uFam5ftwqKOjpIdhnFxZ/kEikeimLyQy3eGXbHT8WgRGzzesOIHVU8+m9TY8ag5WVOyvV24/GyqPdPQ=="],
|
||||
|
||||
"@astrojs/node": ["@astrojs/node@11.0.2", "", { "dependencies": { "@astrojs/internal-helpers": "0.10.1", "send": "^1.2.1", "server-destroy": "^1.0.1" }, "peerDependencies": { "astro": "^7.0.0" } }, "sha512-/ijULxT+A5Cm8wSwWZ2vgqfim1b05D6B8n/a9l6MMA4FCotIH73g7fL7y76XojKXpTe75FVvQH92OxsMqea9kQ=="],
|
||||
"@astrojs/node": ["@astrojs/node@11.1.5", "", { "dependencies": { "@astrojs/internal-helpers": "0.11.0", "send": "^1.2.1", "server-destroy": "^1.0.1" }, "peerDependencies": { "astro": "^7.2.1" } }, "sha512-CgA+LmG4UWqHxg/Vdc5MdxjjJNZSjKYQQx4uEUwnbj+1ZD2CYumJI3OWg2twsVdj2FlYsB5yvmCC54Re+OK+UQ=="],
|
||||
|
||||
"@astrojs/prism": ["@astrojs/prism@4.0.2", "", { "dependencies": { "prismjs": "^1.30.0" } }, "sha512-KTivpmnz6lDsC6o9H4+DNm2SrE/GHzw8cNAvEJwAvUT+eoaEnn/4NtbDNfRRaxaJHdp15gf+tfHAWiXR4wB3BA=="],
|
||||
|
||||
@@ -909,7 +910,7 @@
|
||||
|
||||
"detect-node-es": ["detect-node-es@1.1.0", "", {}, "sha512-ypdmJU/TbBby2Dxibuv7ZLW3Bs1QEmM7nHjEANfohJLvE0XVujisn1qPJcZxg+qDucsr+bP6fLD1rPS3AhJ7EQ=="],
|
||||
|
||||
"devalue": ["devalue@5.8.1", "", {}, "sha512-4CXDYRBGqN+57wVJkuXBYmpAVUSg3L6JAQa/DFqm238G73E1wuyc/JhGQJzN7vUf/CMphYau2zXbfWzDR5aTEw=="],
|
||||
"devalue": ["devalue@5.9.4", "", {}, "sha512-sPAT4pztbu6586/hrhOnMKS17IJrvg12mXiSPSS3W5qDeN2RGgvZ0diZCm31dBbnevfVmujNO3IM2wrS4Y2Rhg=="],
|
||||
|
||||
"devlop": ["devlop@1.1.0", "", { "dependencies": { "dequal": "^2.0.0" } }, "sha512-RWmIqhcFf1lRYBvNmr7qTNuyCt/7/ns2jbpp1+PalgE/rDQcBT0fioSMUpJ93irlUhC5hrg4cYqe6U+0ImW0rA=="],
|
||||
|
||||
@@ -1473,7 +1474,7 @@
|
||||
|
||||
"sisteransi": ["sisteransi@1.0.5", "", {}, "sha512-bLGGlR1QxBcynn2d5YmDX4MGjlZvy2MRBDRNHLJ8VI6l6+9FUiyTFNJ0IveOSP0bcXgVDPRcfGqA0pjaqUpfVg=="],
|
||||
|
||||
"smol-toml": ["smol-toml@1.6.0", "", {}, "sha512-4zemZi0HvTnYwLfrpk/CF9LOd9Lt87kAt50GnqhMpyF9U3poDAP2+iukq2bZsO/ufegbYehBkqINbsWxj4l4cw=="],
|
||||
"smol-toml": ["smol-toml@1.8.0", "", {}, "sha512-kCZr2V3ch9i00x8zXRhjUNVcjG9ijES5dDudkXvUVCT5QlJNQWElSJdZqyPemffHoLNUYwOcou0Fy+ojN0uHSQ=="],
|
||||
|
||||
"sonner": ["sonner@2.0.7", "", { "peerDependencies": { "react": "^18.0.0 || ^19.0.0 || ^19.0.0-rc", "react-dom": "^18.0.0 || ^19.0.0 || ^19.0.0-rc" } }, "sha512-W6ZN4p58k8aDKA4XPcx2hpIQXBRAgyiWVkYhT7CvK6D3iAu7xjvVyhQHg2/iaKJZ1XVJ4r7XuwGL+WGEK37i9w=="],
|
||||
|
||||
@@ -1695,6 +1696,8 @@
|
||||
|
||||
"@astrojs/compiler-binding-wasm32-wasi/@napi-rs/wasm-runtime": ["@napi-rs/wasm-runtime@1.2.3", "", { "dependencies": { "@tybys/wasm-util": "^0.10.3" }, "peerDependencies": { "@emnapi/core": "^1.7.1 || ^2.0.0-alpha.4", "@emnapi/runtime": "^1.7.1 || ^2.0.0-alpha.4" } }, "sha512-UMduMbqO5s5zF2NkNacMT/yK5Y5QiKvWr2+50bzIIxFDwVJ2h49b+oyjaCGPhJxd2/gC2x39EHv/gHVuu36x2Q=="],
|
||||
|
||||
"@astrojs/node/@astrojs/internal-helpers": ["@astrojs/internal-helpers@0.11.0", "", { "dependencies": { "@types/hast": "^3.0.4", "@types/mdast": "^4.0.4", "js-yaml": "^4.3.0", "picomatch": "^4.0.4", "retext-smartypants": "^6.2.0", "shiki": "^4.0.2", "smol-toml": "^1.6.0", "unified": "^11.0.5" } }, "sha512-3rzxJ+xbo0+8YyqOzLziIN32wmsHdCjEVz2sGOpRxJ+Ben/KiLph4ItxBy1abEL+E8fkRzqjg0rfXmaHJGw9JA=="],
|
||||
|
||||
"@astrojs/react/@vitejs/plugin-react": ["@vitejs/plugin-react@5.2.0", "", { "dependencies": { "@babel/core": "^7.29.0", "@babel/plugin-transform-react-jsx-self": "^7.27.1", "@babel/plugin-transform-react-jsx-source": "^7.27.1", "@rolldown/pluginutils": "1.0.0-rc.3", "@types/babel__core": "^7.20.5", "react-refresh": "^0.18.0" }, "peerDependencies": { "vite": "^4.2.0 || ^5.0.0 || ^6.0.0 || ^7.0.0 || ^8.0.0" } }, "sha512-YmKkfhOAi3wsB1PhJq5Scj3GXMn3WvtQ/JC0xoopuHoXSdmtdStOpFrYaT1kie2YgFBcIe64ROzMYRjCrYOdYw=="],
|
||||
|
||||
"@astrojs/react/vite": ["vite@8.1.5", "", { "dependencies": { "lightningcss": "^1.32.0", "picomatch": "^4.0.5", "postcss": "^8.5.17", "rolldown": "~1.1.5", "tinyglobby": "^0.2.17" }, "optionalDependencies": { "fsevents": "~2.3.3" }, "peerDependencies": { "@types/node": "^20.19.0 || >=22.12.0", "@vitejs/devtools": "^0.3.0", "esbuild": "^0.27.0 || ^0.28.0", "jiti": ">=1.21.0", "less": "^4.0.0", "sass": "^1.70.0", "sass-embedded": "^1.70.0", "stylus": ">=0.54.8", "sugarss": "^5.0.0", "terser": "^5.16.0", "tsx": "^4.8.1", "yaml": "^2.4.2" }, "optionalPeers": ["@types/node", "@vitejs/devtools", "esbuild", "jiti", "less", "sass", "sass-embedded", "stylus", "sugarss", "terser", "tsx", "yaml"], "bin": { "vite": "bin/vite.js" } }, "sha512-7ULLwsCdYx/nRyrpiEwvqb5TFHrMVZyBt+rg/OAXT7rgj/z+DtTDyKFeLAdDkubDVDKD8jOsndmy7m55XcfUsw=="],
|
||||
|
||||
@@ -113,9 +113,9 @@
|
||||
url = "https://registry.npmjs.org/@astrojs/mdx/-/mdx-7.0.3.tgz";
|
||||
hash = "sha512-RxyIwU0uFam5ftwqKOjpIdhnFxZ/kEikeimLyQy3eGXbHT8WgRGzzesOIHVU8+m9TY8ag5WVOyvV24/GyqPdPQ==";
|
||||
};
|
||||
"@astrojs/node@11.0.2" = fetchurl {
|
||||
url = "https://registry.npmjs.org/@astrojs/node/-/node-11.0.2.tgz";
|
||||
hash = "sha512-/ijULxT+A5Cm8wSwWZ2vgqfim1b05D6B8n/a9l6MMA4FCotIH73g7fL7y76XojKXpTe75FVvQH92OxsMqea9kQ==";
|
||||
"@astrojs/node@11.1.5" = fetchurl {
|
||||
url = "https://registry.npmjs.org/@astrojs/node/-/node-11.1.5.tgz";
|
||||
hash = "sha512-CgA+LmG4UWqHxg/Vdc5MdxjjJNZSjKYQQx4uEUwnbj+1ZD2CYumJI3OWg2twsVdj2FlYsB5yvmCC54Re+OK+UQ==";
|
||||
};
|
||||
"@astrojs/prism@4.0.2" = fetchurl {
|
||||
url = "https://registry.npmjs.org/@astrojs/prism/-/prism-4.0.2.tgz";
|
||||
@@ -2001,9 +2001,9 @@
|
||||
url = "https://registry.npmjs.org/detect-node-es/-/detect-node-es-1.1.0.tgz";
|
||||
hash = "sha512-ypdmJU/TbBby2Dxibuv7ZLW3Bs1QEmM7nHjEANfohJLvE0XVujisn1qPJcZxg+qDucsr+bP6fLD1rPS3AhJ7EQ==";
|
||||
};
|
||||
"devalue@5.8.1" = fetchurl {
|
||||
url = "https://registry.npmjs.org/devalue/-/devalue-5.8.1.tgz";
|
||||
hash = "sha512-4CXDYRBGqN+57wVJkuXBYmpAVUSg3L6JAQa/DFqm238G73E1wuyc/JhGQJzN7vUf/CMphYau2zXbfWzDR5aTEw==";
|
||||
"devalue@5.9.4" = fetchurl {
|
||||
url = "https://registry.npmjs.org/devalue/-/devalue-5.9.4.tgz";
|
||||
hash = "sha512-sPAT4pztbu6586/hrhOnMKS17IJrvg12mXiSPSS3W5qDeN2RGgvZ0diZCm31dBbnevfVmujNO3IM2wrS4Y2Rhg==";
|
||||
};
|
||||
"devlop@1.1.0" = fetchurl {
|
||||
url = "https://registry.npmjs.org/devlop/-/devlop-1.1.0.tgz";
|
||||
@@ -3205,9 +3205,9 @@
|
||||
url = "https://registry.npmjs.org/sisteransi/-/sisteransi-1.0.5.tgz";
|
||||
hash = "sha512-bLGGlR1QxBcynn2d5YmDX4MGjlZvy2MRBDRNHLJ8VI6l6+9FUiyTFNJ0IveOSP0bcXgVDPRcfGqA0pjaqUpfVg==";
|
||||
};
|
||||
"smol-toml@1.6.0" = fetchurl {
|
||||
url = "https://registry.npmjs.org/smol-toml/-/smol-toml-1.6.0.tgz";
|
||||
hash = "sha512-4zemZi0HvTnYwLfrpk/CF9LOd9Lt87kAt50GnqhMpyF9U3poDAP2+iukq2bZsO/ufegbYehBkqINbsWxj4l4cw==";
|
||||
"smol-toml@1.8.0" = fetchurl {
|
||||
url = "https://registry.npmjs.org/smol-toml/-/smol-toml-1.8.0.tgz";
|
||||
hash = "sha512-kCZr2V3ch9i00x8zXRhjUNVcjG9ijES5dDudkXvUVCT5QlJNQWElSJdZqyPemffHoLNUYwOcou0Fy+ojN0uHSQ==";
|
||||
};
|
||||
"sonner@2.0.7" = fetchurl {
|
||||
url = "https://registry.npmjs.org/sonner/-/sonner-2.0.7.tgz";
|
||||
|
||||
@@ -79,6 +79,9 @@ services:
|
||||
- CLEANUP_DRY_RUN=${CLEANUP_DRY_RUN:-true}
|
||||
# Optional: Skip TLS verification (insecure, use only for testing)
|
||||
# - GITEA_SKIP_TLS_VERIFY=${GITEA_SKIP_TLS_VERIFY:-false}
|
||||
# Login page: which tab opens first when both methods are available
|
||||
- AUTH_DEFAULT_METHOD=${AUTH_DEFAULT_METHOD:-email}
|
||||
- AUTH_ALLOW_SIGNUP=${AUTH_ALLOW_SIGNUP:-false}
|
||||
# Header Authentication (for Reverse Proxy SSO)
|
||||
- HEADER_AUTH_ENABLED=${HEADER_AUTH_ENABLED:-false}
|
||||
- HEADER_AUTH_USER_HEADER=${HEADER_AUTH_USER_HEADER:-X-Authentik-Username}
|
||||
|
||||
+18
@@ -125,6 +125,24 @@ When several sources are connected, an optional `sourceId` picks which one the o
|
||||
|
||||
`PATCH /api/organizations/:id` with `{ "sourceId": "…" }` re-pins an existing organization to another source, and `{ "sourceId": null }` clears the pin so the organization follows every source's repositories of the same name again. A pin whose source was removed falls back the same way.
|
||||
|
||||
### Export as CSV
|
||||
|
||||
`GET /api/repositories/export` and `GET /api/organizations/export` return everything the account tracks as a CSV file, so the list can be shared with people who do not have a login here (issue #428).
|
||||
|
||||
Both answer `text/csv; charset=utf-8` with a download name such as `gitea-mirror-repositories-2026-09-22.csv`, start with a header row, use CRLF line endings and RFC 4180 quoting, and write dates as ISO strings and booleans as `true` or `false`. Repositories are ordered by full name, organizations by name. The export has no filters: it is every row of the account. Internal fields are left out, that is the ids, the metadata sync state and the mirror option overrides.
|
||||
|
||||
Repository columns: `name`, `fullName`, `url`, `cloneUrl`, `owner`, `organization`, `sourceProvider`, `sourceUrl`, `destinationProvider`, `destinationUrl`, `destinationOrg`, `mirroredLocation`, `visibility`, `isPrivate`, `isForked`, `forkedFrom`, `isStarred`, `isArchived`, `hasLFS`, `hasSubmodules`, `hasIssues`, `language`, `description`, `defaultBranch`, `size`, `status`, `lastMirrored`, `errorMessage`, `importedAt`, `createdAt`, `updatedAt`.
|
||||
|
||||
Organization columns: `name`, `membershipRole`, `isIncluded`, `destinationOrg`, `status`, `repositoryCount`, `publicRepositoryCount`, `privateRepositoryCount`, `forkRepositoryCount`, `lastMirrored`, `errorMessage`, `createdAt`, `updatedAt`.
|
||||
|
||||
```bash
|
||||
curl -sS "https://mirror.example.com/api/repositories/export" \
|
||||
-H "x-api-key: gm_..." \
|
||||
-o repositories.csv
|
||||
```
|
||||
|
||||
The **Export CSV** button on the Repositories and Organizations pages downloads the same files.
|
||||
|
||||
### Reconcile the destination
|
||||
|
||||
`POST /api/cleanup/reconcile`
|
||||
|
||||
@@ -97,7 +97,7 @@ GITHUB_USERNAME=my-codeberg-user
|
||||
GITHUB_TOKEN=...
|
||||
```
|
||||
|
||||
Code, tags, wiki and LFS are mirrored from every source. Issues, pull requests, releases, labels, milestones and star lists need a GitHub source and are skipped for the others.
|
||||
Code, tags, wiki and LFS are mirrored from every source, and releases with their assets from GitHub and Gitea/Forgejo sources. Issues, pull requests, labels, milestones and star lists need a GitHub source and are skipped for the others.
|
||||
|
||||
Once repositories have been imported the source is locked, and once anything has been mirrored the Gitea server URL is locked. A `SOURCE_PROVIDER`, `SOURCE_URL` or `GITEA_URL` value that disagrees with a locked host is ignored on boot with a warning; change the host on the Configuration page, where the change asks for confirmation.
|
||||
|
||||
@@ -342,6 +342,16 @@ Configure automatic cleanup of old events and data.
|
||||
|
||||
Configure authentication methods and SSO.
|
||||
|
||||
### Login Page
|
||||
|
||||
| Variable | Description | Default | Options |
|
||||
|----------|-------------|---------|---------|
|
||||
| `AUTH_DEFAULT_METHOD` | Tab the login page opens on when both email and SSO are available. Any other value falls back to `email`. | `email` | `email`, `sso` |
|
||||
| `AUTH_ALLOW_SIGNUP` | Allow email and password sign-up after the first account exists. Off, the sign-up endpoint answers 403 once a user exists; the first account can always be created. | `false` | `true`, `false` |
|
||||
|
||||
The login page also remembers the method last used in that browser, which wins
|
||||
over this default. Anything unrecognised falls back to `email`.
|
||||
|
||||
### Header Authentication (Reverse Proxy SSO)
|
||||
|
||||
| Variable | Description | Default | Options |
|
||||
|
||||
@@ -115,10 +115,15 @@ Access at http://localhost:4321, sign up (first user is admin), and configure ev
|
||||
|
||||
### Production Configuration
|
||||
|
||||
For production with custom domain and firewall:
|
||||
For production with custom domain and firewall. Like every snippet on this
|
||||
page, it needs the module imported first; `services.gitea-mirror` does not
|
||||
exist until `gitea-mirror.nixosModules.default` is in your modules list.
|
||||
|
||||
```nix
|
||||
{ inputs, ... }:
|
||||
{
|
||||
imports = [ inputs.gitea-mirror.nixosModules.default ];
|
||||
|
||||
services.gitea-mirror = {
|
||||
enable = true;
|
||||
host = "0.0.0.0";
|
||||
|
||||
@@ -39,11 +39,11 @@ The mirror itself is done by Gitea's pull mirror, which treats every source as a
|
||||
| Entire organizations (GitLab: top level groups) | yes | yes | yes |
|
||||
| Add a single repository by URL | yes | yes | yes |
|
||||
| Issues, pull requests, labels, milestones | yes | no | no |
|
||||
| Releases with assets | yes | no | no |
|
||||
| Releases with assets | yes | no | yes |
|
||||
| Star lists | yes | no | no |
|
||||
| Force push detection | yes | no | no |
|
||||
|
||||
The GitHub only rows read the GitHub API. For other sources the corresponding switches are disabled on the Configuration page, and the mirror step skips them.
|
||||
The GitHub only rows read the GitHub API. For other sources the corresponding switches are disabled on the Configuration page, and the mirror step skips them. Releases are the exception: Gitea and Forgejo expose them at `GET /repos/{owner}/{repo}/releases`, so a Gitea, Forgejo or Codeberg source mirrors releases and their assets like GitHub does, with that instance's own token (public repositories need none).
|
||||
|
||||
Entire organizations also mirror without a token on every source; the next section covers public mode.
|
||||
|
||||
@@ -56,7 +56,7 @@ On the Organizations page, **Add organization** has a public mode, which is the
|
||||
Public mode degrades in known ways:
|
||||
|
||||
- **GitHub metadata is best effort.** Anonymous requests run under GitHub's 60 requests per hour limit, so a large organization can end up with partial metadata. Releases, description and topics mirror anonymously with no account at all. Issues, pull requests, labels and milestones are also fetched anonymously once any source in the account carries a token; with no token anywhere they are skipped, and the mirror itself is unaffected either way.
|
||||
- **GitLab and Gitea/Forgejo sources mirror code only.** Metadata remains GitHub only, as in the table above, and issues and merge requests from those sources stay a Not yet item with or without a token.
|
||||
- **GitLab sources mirror code only; Gitea and Forgejo sources add releases.** The rest of the metadata remains GitHub only, as in the table above, and issues and merge requests from those sources stay a Not yet item with or without a token.
|
||||
- **Personal auto discovery and starred repositories need a token.** Neither runs for a public only source.
|
||||
- **Cleanup does not run.** Repositories deleted upstream are not detected for public only sources.
|
||||
|
||||
|
||||
@@ -303,7 +303,7 @@ EOF
|
||||
|
||||
package = mkOption {
|
||||
type = types.package;
|
||||
default = self.packages.${pkgs.system}.default;
|
||||
default = self.packages.${pkgs.stdenv.hostPlatform.system}.default;
|
||||
description = "The Gitea Mirror package to use";
|
||||
};
|
||||
|
||||
|
||||
@@ -2,8 +2,8 @@ apiVersion: v2
|
||||
name: gitea-mirror
|
||||
description: Kubernetes helm chart for gitea-mirror
|
||||
type: application
|
||||
version: 3.35.1
|
||||
appVersion: "3.35.1"
|
||||
version: 3.40.5
|
||||
appVersion: "3.40.5"
|
||||
icon: https://github.com/RayLabsHQ/gitea-mirror/blob/main/.github/assets/logo.png
|
||||
keywords:
|
||||
- git
|
||||
|
||||
+4
-3
@@ -1,7 +1,7 @@
|
||||
{
|
||||
"name": "gitea-mirror",
|
||||
"type": "module",
|
||||
"version": "3.35.1",
|
||||
"version": "3.40.5",
|
||||
"engines": {
|
||||
"bun": ">=1.2.9"
|
||||
},
|
||||
@@ -47,10 +47,11 @@
|
||||
"@esbuild-kit/esm-loader": "npm:tsx@^4.21.0",
|
||||
"@xmldom/xmldom": "^0.8.15",
|
||||
"browserslist": "^4.28.7",
|
||||
"devalue": "^5.8.1",
|
||||
"devalue": "^5.9.2",
|
||||
"fast-uri": "^3.1.5",
|
||||
"fast-xml-builder": "^1.1.7",
|
||||
"js-yaml": "^4.3.2",
|
||||
"smol-toml": "^1.7.1",
|
||||
"kysely": "^0.28.17",
|
||||
"lodash": "^4.18.1",
|
||||
"picomatch": "^4.0.4",
|
||||
@@ -59,7 +60,7 @@
|
||||
},
|
||||
"dependencies": {
|
||||
"@astrojs/mdx": "^7.0.3",
|
||||
"@astrojs/node": "^11.0.2",
|
||||
"@astrojs/node": "^11.1.5",
|
||||
"@astrojs/react": "^6.0.1",
|
||||
"@better-auth/api-key": "1.7.2",
|
||||
"@better-auth/oauth-provider": "1.7.2",
|
||||
|
||||
@@ -221,7 +221,7 @@ async function resetPassword() {
|
||||
}
|
||||
|
||||
const sqlite = new Database(dbPath);
|
||||
const db = drizzle({ client: sqlite });
|
||||
const db = drizzle({ client: sqlite, schema: { users, accounts, sessions } });
|
||||
|
||||
try {
|
||||
const user = await db.query.users.findFirst({
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import { countLatestStatuses } from '@/lib/utils/activity-status';
|
||||
import { useCallback, useEffect, useState, useRef, useMemo } from 'react';
|
||||
import Fuse from 'fuse.js';
|
||||
import { Button } from '@/components/ui/button';
|
||||
@@ -407,28 +408,9 @@ export function ActivityLog() {
|
||||
|
||||
// Check if any filters are active
|
||||
// Counts the current state of each repository or organization, not every
|
||||
// event: a repo that failed and then synced is only "synced". Activities
|
||||
// arrive newest first, so the first entry seen for a subject is its state.
|
||||
// Taken from the unfiltered set so the chips stay put when one is clicked.
|
||||
const statusCounts = useMemo(() => {
|
||||
const latestBySubject = new Map<string, string>();
|
||||
activities.forEach((a) => {
|
||||
const subject =
|
||||
a.repositoryId ||
|
||||
a.organizationId ||
|
||||
a.repositoryName ||
|
||||
a.organizationName;
|
||||
// Events with no subject have no state of their own to report.
|
||||
if (!subject || latestBySubject.has(subject)) return;
|
||||
latestBySubject.set(subject, a.status);
|
||||
});
|
||||
|
||||
const counts = new Map<string, number>();
|
||||
latestBySubject.forEach((status) => {
|
||||
counts.set(status, (counts.get(status) ?? 0) + 1);
|
||||
});
|
||||
return counts;
|
||||
}, [activities]);
|
||||
// event: a repo that failed and then synced is only "synced". Taken from
|
||||
// the unfiltered set so the chips stay put when one is clicked.
|
||||
const statusCounts = useMemo(() => countLatestStatuses(activities), [activities]);
|
||||
|
||||
// Most actionable first, and only statuses that actually occurred.
|
||||
const statChips = ['failed', 'syncing', 'mirroring', 'imported', 'synced', 'mirrored']
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
'use client';
|
||||
|
||||
import * as React from 'react';
|
||||
import { useState } from 'react';
|
||||
import { useEffect, useRef, useState } from 'react';
|
||||
import { Button } from '@/components/ui/button';
|
||||
import { Card, CardContent, CardDescription, CardFooter, CardHeader, CardTitle } from '@/components/ui/card';
|
||||
import { useAuth } from '@/hooks/useAuth';
|
||||
@@ -13,6 +13,13 @@ import { toast, Toaster } from 'sonner';
|
||||
import { showErrorToast } from '@/lib/utils';
|
||||
import { Loader2, Mail, Globe, Eye, EyeOff } from 'lucide-react';
|
||||
import { withBase } from '@/lib/base-path';
|
||||
import {
|
||||
DEFAULT_AUTH_METHOD,
|
||||
getLastAuthMethod,
|
||||
resolveInitialAuthMethod,
|
||||
setLastAuthMethod,
|
||||
type AuthMethod,
|
||||
} from '@/lib/utils/auth-method';
|
||||
|
||||
|
||||
export function LoginForm() {
|
||||
@@ -21,13 +28,27 @@ export function LoginForm() {
|
||||
const [ssoEmail, setSsoEmail] = useState('');
|
||||
const { login } = useAuth();
|
||||
const { authMethods, isLoading: isLoadingMethods } = useAuthMethods();
|
||||
const [activeTab, setActiveTab] = useState<AuthMethod>(DEFAULT_AUTH_METHOD);
|
||||
// The tab is only resolved once, so a manual switch is not overridden.
|
||||
const hasResolvedTab = useRef(false);
|
||||
|
||||
// Determine which tab to show by default
|
||||
const getDefaultTab = () => {
|
||||
if (authMethods.emailPassword) return 'email';
|
||||
if (authMethods.sso.enabled) return 'sso';
|
||||
return 'email'; // fallback
|
||||
};
|
||||
// Which tab to open on: the method this browser last used, else the server
|
||||
// default (AUTH_DEFAULT_METHOD), else whichever method is available. This
|
||||
// runs once the available methods are known, before the tabs are rendered.
|
||||
useEffect(() => {
|
||||
if (isLoadingMethods || hasResolvedTab.current) return;
|
||||
hasResolvedTab.current = true;
|
||||
setActiveTab(
|
||||
resolveInitialAuthMethod({
|
||||
remembered: getLastAuthMethod(),
|
||||
serverDefault: authMethods.defaultMethod,
|
||||
available: {
|
||||
email: authMethods.emailPassword,
|
||||
sso: authMethods.sso.enabled,
|
||||
},
|
||||
})
|
||||
);
|
||||
}, [isLoadingMethods, authMethods]);
|
||||
|
||||
async function handleLogin(e: React.FormEvent<HTMLFormElement>) {
|
||||
e.preventDefault();
|
||||
@@ -45,6 +66,7 @@ export function LoginForm() {
|
||||
|
||||
try {
|
||||
await login(email, password);
|
||||
setLastAuthMethod('email');
|
||||
toast.success('Login successful!');
|
||||
// Small delay before redirecting to see the success message
|
||||
setTimeout(() => {
|
||||
@@ -65,6 +87,8 @@ export function LoginForm() {
|
||||
return;
|
||||
}
|
||||
|
||||
setLastAuthMethod('sso');
|
||||
|
||||
const callbackURL =
|
||||
typeof window !== 'undefined'
|
||||
? new URL(withBase('/'), window.location.origin).toString()
|
||||
@@ -119,7 +143,11 @@ export function LoginForm() {
|
||||
<>
|
||||
{/* Show tabs only if multiple auth methods are available */}
|
||||
{authMethods.sso.enabled && authMethods.emailPassword ? (
|
||||
<Tabs defaultValue={getDefaultTab()} className="w-full">
|
||||
<Tabs
|
||||
value={activeTab}
|
||||
onValueChange={(value) => setActiveTab(value as AuthMethod)}
|
||||
className="w-full"
|
||||
>
|
||||
<TabsList className="grid w-full grid-cols-2 mx-6" style={{ width: 'calc(100% - 3rem)' }}>
|
||||
<TabsTrigger value="email">
|
||||
<Mail className="h-4 w-4 mr-2" />
|
||||
|
||||
@@ -68,6 +68,24 @@ const retentionPeriods = [
|
||||
{ label: "3 months", value: 7776000 },
|
||||
];
|
||||
|
||||
/** "every 8 hours", "every 30 minutes", "every 2 days" for a plain interval. */
|
||||
function formatIntervalSeconds(interval: number | string | undefined): string {
|
||||
const seconds = typeof interval === "number" ? interval : Number(interval);
|
||||
if (!Number.isFinite(seconds) || seconds <= 0) return "on a fixed interval";
|
||||
const units: [number, string][] = [
|
||||
[86400, "day"],
|
||||
[3600, "hour"],
|
||||
[60, "minute"],
|
||||
];
|
||||
for (const [size, name] of units) {
|
||||
if (seconds % size === 0 || seconds >= size * 2) {
|
||||
const count = Math.round(seconds / size);
|
||||
return `every ${count === 1 ? name : `${count} ${name}s`}`;
|
||||
}
|
||||
}
|
||||
return `every ${seconds} seconds`;
|
||||
}
|
||||
|
||||
function getCleanupInterval(retentionSeconds: number): number {
|
||||
const days = retentionSeconds / 86400;
|
||||
if (days <= 1) return 21600; // 6 hours
|
||||
@@ -126,16 +144,22 @@ export function AutomationSettings({
|
||||
? Intl.DateTimeFormat().resolvedOptions().timeZone || "UTC"
|
||||
: "UTC";
|
||||
|
||||
// Use saved timezone, but treat "UTC" as unset for users who never chose
|
||||
// it: older versions stored UTC as a default without asking. Anyone truly
|
||||
// in UTC gets the same result via their browser timezone.
|
||||
const effectiveTimezone =
|
||||
scheduleConfig.timezone && scheduleConfig.timezone !== "UTC"
|
||||
? scheduleConfig.timezone
|
||||
: browserTimezone;
|
||||
// The timezone the scheduler really uses. It used to be shown as the
|
||||
// browser timezone whenever UTC was stored, which made the card promise a
|
||||
// local 22:00 while the sync ran at 22:00 UTC (#427). The chip below offers
|
||||
// a one-click switch to the browser timezone when the two differ.
|
||||
const effectiveTimezone = scheduleConfig.timezone || browserTimezone;
|
||||
const timezoneDiffers = effectiveTimezone !== browserTimezone;
|
||||
|
||||
// The stored schedule is a plain interval (seconds or a duration such as
|
||||
// 8h) rather than a clock schedule. That is what SCHEDULE_INTERVAL and
|
||||
// GITEA_MIRROR_INTERVAL produce, and what older versions stored. The card
|
||||
// only edits clock schedules, so say what actually runs instead of showing
|
||||
// the 22:00 placeholder as if it were saved (#427).
|
||||
const isIntervalSchedule = scheduleConfig.scheduleMode === "interval";
|
||||
|
||||
const nextScheduledRun = useMemo(() => {
|
||||
if (!scheduleConfig.enabled) return null;
|
||||
if (!scheduleConfig.enabled || isIntervalSchedule) return null;
|
||||
const startTime = scheduleConfig.startTime || "22:00";
|
||||
const frequencyHours = scheduleConfig.clockFrequencyHours || 24;
|
||||
const cronExpression = buildClockCronExpression(startTime, frequencyHours);
|
||||
@@ -145,7 +169,16 @@ export function AutomationSettings({
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}, [scheduleConfig.enabled, scheduleConfig.startTime, scheduleConfig.clockFrequencyHours, effectiveTimezone]);
|
||||
}, [scheduleConfig.enabled, isIntervalSchedule, scheduleConfig.startTime, scheduleConfig.clockFrequencyHours, effectiveTimezone]);
|
||||
|
||||
const switchToBrowserTimezone = () =>
|
||||
onScheduleChange({
|
||||
...scheduleConfig,
|
||||
scheduleMode: "clock",
|
||||
startTime: scheduleConfig.startTime || "22:00",
|
||||
clockFrequencyHours: scheduleConfig.clockFrequencyHours || 24,
|
||||
timezone: browserTimezone,
|
||||
});
|
||||
|
||||
// Update nextRun for cleanup when settings change
|
||||
useEffect(() => {
|
||||
@@ -217,15 +250,39 @@ export function AutomationSettings({
|
||||
<CardSection>
|
||||
<SectionTitle
|
||||
action={
|
||||
<span className="inline-flex items-center gap-1.5 rounded-full bg-muted px-2.5 py-1 text-[11px] text-muted-foreground">
|
||||
<Globe className="h-3 w-3" />
|
||||
{effectiveTimezone}
|
||||
</span>
|
||||
timezoneDiffers ? (
|
||||
<button
|
||||
type="button"
|
||||
onClick={switchToBrowserTimezone}
|
||||
title={`Switch the schedule to ${browserTimezone}`}
|
||||
className="inline-flex items-center gap-1.5 rounded-full bg-muted px-2.5 py-1 text-[11px] text-muted-foreground hover:bg-muted/70 hover:text-foreground"
|
||||
>
|
||||
<Globe className="h-3 w-3" />
|
||||
{effectiveTimezone}
|
||||
<span className="text-muted-foreground/60">· use {browserTimezone}</span>
|
||||
</button>
|
||||
) : (
|
||||
<span className="inline-flex items-center gap-1.5 rounded-full bg-muted px-2.5 py-1 text-[11px] text-muted-foreground">
|
||||
<Globe className="h-3 w-3" />
|
||||
{effectiveTimezone}
|
||||
</span>
|
||||
)
|
||||
}
|
||||
>
|
||||
Schedule
|
||||
</SectionTitle>
|
||||
|
||||
{isIntervalSchedule && (
|
||||
<p className="flex items-start gap-2 rounded-md border border-dashed border-border px-3 py-2 text-xs text-muted-foreground">
|
||||
<Info className="mt-0.5 h-3.5 w-3.5 shrink-0" />
|
||||
<span>
|
||||
Syncing {formatIntervalSeconds(scheduleConfig.interval)}, counted from the last run.
|
||||
{" "}This interval comes from SCHEDULE_INTERVAL or GITEA_MIRROR_INTERVAL, which is applied again on every restart while it is set, or from an older version.
|
||||
{" "}Pick a frequency or start time to switch to a clock schedule.
|
||||
</span>
|
||||
</p>
|
||||
)}
|
||||
|
||||
<div className="grid gap-3 sm:grid-cols-2">
|
||||
<div className="space-y-1.5">
|
||||
<Label
|
||||
@@ -235,7 +292,7 @@ export function AutomationSettings({
|
||||
Frequency
|
||||
</Label>
|
||||
<Select
|
||||
value={String(scheduleConfig.clockFrequencyHours || 24)}
|
||||
value={isIntervalSchedule ? "" : String(scheduleConfig.clockFrequencyHours || 24)}
|
||||
onValueChange={(value) =>
|
||||
onScheduleChange({
|
||||
...scheduleConfig,
|
||||
@@ -247,7 +304,7 @@ export function AutomationSettings({
|
||||
}
|
||||
>
|
||||
<SelectTrigger id="clock-frequency" className="w-full">
|
||||
<SelectValue />
|
||||
<SelectValue placeholder="Choose a frequency" />
|
||||
</SelectTrigger>
|
||||
<SelectContent>
|
||||
{clockFrequencies.map((option) => (
|
||||
@@ -276,7 +333,7 @@ export function AutomationSettings({
|
||||
<Input
|
||||
id="clock-start-time"
|
||||
type="time"
|
||||
value={scheduleConfig.startTime || "22:00"}
|
||||
value={isIntervalSchedule ? "" : scheduleConfig.startTime || "22:00"}
|
||||
onChange={(event) =>
|
||||
onScheduleChange({
|
||||
...scheduleConfig,
|
||||
|
||||
@@ -326,7 +326,7 @@ export function GitHubConfigForm({
|
||||
? "Where your repositories are pulled from"
|
||||
: provider === "gitlab"
|
||||
? "Beta. Code, tags, wiki and LFS are mirrored. Issues, merge requests and releases need a GitHub source."
|
||||
: "Beta. Code, tags, wiki and LFS are mirrored. Issues, pull requests and releases need a GitHub source."}
|
||||
: "Beta. Code, tags, releases, wiki and LFS are mirrored. Issues and pull requests need a GitHub source."}
|
||||
</p>
|
||||
{sourceLock?.locked && (
|
||||
<HostLockNotice
|
||||
|
||||
@@ -59,6 +59,7 @@ import {
|
||||
import { cn } from "@/lib/utils";
|
||||
import { githubApi } from "@/lib/api";
|
||||
import {
|
||||
sourceKindSupportsReleases,
|
||||
SOURCE_PROVIDER_LABELS,
|
||||
SOURCE_PROVIDER_ORG_NOUNS,
|
||||
} from "@/lib/source-providers/kinds";
|
||||
@@ -106,14 +107,21 @@ export function GitHubMirrorSettings({
|
||||
const isPushTarget = isPushDestinationKind(destinationProvider ?? "gitea");
|
||||
const destinationLabel = DESTINATION_PROVIDER_LABELS[destinationProvider ?? "gitea"];
|
||||
const pushTargetReason = `Not available for ${destinationLabel} destinations: only branches and tags are pushed`;
|
||||
// Star lists, issues, pull requests, releases, labels and milestones all
|
||||
// read the GitHub API, so they are only offered for GitHub sources.
|
||||
// Star lists, issues, pull requests, labels and milestones all read the
|
||||
// GitHub API, so they are only offered for GitHub sources.
|
||||
const sourceProvider = githubConfig.provider ?? "github";
|
||||
const hasGithubSource =
|
||||
sources && sources.length > 0
|
||||
? sources.some((source) => source.provider === "github")
|
||||
: sourceProvider === "github";
|
||||
const isGithubSource = hasGithubSource && !isPushTarget;
|
||||
// Releases are listed through the source API, which Gitea and Forgejo
|
||||
// expose as well (#440), so they follow their own capability check.
|
||||
const hasReleaseCapableSource =
|
||||
sources && sources.length > 0
|
||||
? sources.some((source) => sourceKindSupportsReleases(source.provider))
|
||||
: sourceKindSupportsReleases(sourceProvider);
|
||||
const isReleaseCapableSource = hasReleaseCapableSource && !isPushTarget;
|
||||
const sourceLabel = SOURCE_PROVIDER_LABELS[sourceProvider];
|
||||
const orgNoun = SOURCE_PROVIDER_ORG_NOUNS[sourceProvider];
|
||||
const [starListsOpen, setStarListsOpen] = React.useState(false);
|
||||
@@ -881,7 +889,9 @@ export function GitHubMirrorSettings({
|
||||
? `${destinationLabel} destinations receive branches and tags only`
|
||||
: isGithubSource
|
||||
? "Pull requests are mirrored as issues due to Gitea API limits"
|
||||
: "Issues, pull requests and releases are mirrored for GitHub sources only"
|
||||
: isReleaseCapableSource
|
||||
? "Releases are mirrored. Issues, pull requests, labels and milestones need a GitHub source"
|
||||
: "Issues, pull requests and releases are mirrored for GitHub sources only"
|
||||
}
|
||||
/>
|
||||
}
|
||||
@@ -906,13 +916,13 @@ export function GitHubMirrorSettings({
|
||||
description={
|
||||
isPushTarget
|
||||
? pushTargetReason
|
||||
: !isGithubSource
|
||||
? "Release assets need a GitHub source. Tags always come with the code"
|
||||
: !isReleaseCapableSource
|
||||
? "Release assets need a GitHub or Gitea/Forgejo source. Tags always come with the code"
|
||||
: mirrorOptions.mirrorReleases
|
||||
? "Includes tags. Leave the assets box empty to upload assets for every mirrored release, or set it to 0 for release notes only"
|
||||
: "Includes release assets and tags"
|
||||
}
|
||||
disabled={!isGithubSource}
|
||||
disabled={!isReleaseCapableSource}
|
||||
checked={mirrorOptions.mirrorReleases}
|
||||
onCheckedChange={(checked) => handleMirrorChange('mirrorReleases', checked)}
|
||||
extra={
|
||||
|
||||
@@ -454,6 +454,7 @@ export function SSOSettings() {
|
||||
<Input
|
||||
id="clientSecret"
|
||||
type="password"
|
||||
placeholder={editingProvider ? "Leave blank to keep the current secret" : ""}
|
||||
value={providerForm.clientSecret}
|
||||
onChange={e => setProviderForm(prev => ({ ...prev, clientSecret: e.target.value }))}
|
||||
/>
|
||||
|
||||
@@ -119,7 +119,7 @@ function providerHelp(provider: SourceProvider): string {
|
||||
if (provider === "github") return "Where your repositories are pulled from";
|
||||
if (provider === "gitlab")
|
||||
return "Beta. Code, tags, wiki and LFS are mirrored. Issues, merge requests and releases need a GitHub source.";
|
||||
return "Beta. Code, tags, wiki and LFS are mirrored. Issues, pull requests and releases need a GitHub source.";
|
||||
return "Beta. Code, tags, releases, wiki and LFS are mirrored. Issues and pull requests need a GitHub source.";
|
||||
}
|
||||
|
||||
function sourceHost(source: SourceApiRecord): string {
|
||||
|
||||
@@ -44,6 +44,11 @@ interface MirrorDestinationEditorProps {
|
||||
destinationLabel?: string;
|
||||
isUpdating?: boolean;
|
||||
className?: string;
|
||||
/**
|
||||
* Show only the destination and the edit button. The list view (#428)
|
||||
* already names the organization in its first column.
|
||||
*/
|
||||
compact?: boolean;
|
||||
}
|
||||
|
||||
const MAX_LISTED = 8;
|
||||
@@ -88,6 +93,7 @@ export function MirrorDestinationEditor({
|
||||
destinationLabel = "Gitea",
|
||||
isUpdating = false,
|
||||
className,
|
||||
compact = false,
|
||||
}: MirrorDestinationEditorProps) {
|
||||
const [isOpen, setIsOpen] = useState(false);
|
||||
const [editValue, setEditValue] = useState(currentDestination || "");
|
||||
@@ -185,14 +191,25 @@ export function MirrorDestinationEditor({
|
||||
|
||||
return (
|
||||
<div className={cn("flex items-center gap-2 w-full", className)}>
|
||||
<div className="flex items-center gap-1.5 text-xs text-muted-foreground min-w-0 flex-1">
|
||||
<Building2 className="h-3 w-3 flex-shrink-0" />
|
||||
<span className="font-medium truncate">{organizationName}</span>
|
||||
<ArrowRight className="h-3 w-3 flex-shrink-0" />
|
||||
<span className={cn(
|
||||
"font-medium truncate",
|
||||
hasOverride && "text-orange-600 dark:text-orange-400"
|
||||
)}>
|
||||
<div className={cn(
|
||||
"flex items-center gap-1.5 text-muted-foreground min-w-0 flex-1",
|
||||
compact ? "text-sm" : "text-xs"
|
||||
)}>
|
||||
{!compact && (
|
||||
<>
|
||||
<Building2 className="h-3 w-3 flex-shrink-0" />
|
||||
<span className="font-medium truncate">{organizationName}</span>
|
||||
<ArrowRight className="h-3 w-3 flex-shrink-0" />
|
||||
</>
|
||||
)}
|
||||
<span
|
||||
className={cn(
|
||||
"font-medium truncate",
|
||||
compact && "text-foreground",
|
||||
hasOverride && "text-orange-600 dark:text-orange-400"
|
||||
)}
|
||||
title={effectiveDestination}
|
||||
>
|
||||
{effectiveDestination}
|
||||
</span>
|
||||
{hasOverride && (
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
import { useCallback, useEffect, useState } from "react";
|
||||
import { Button } from "@/components/ui/button";
|
||||
import { Search, RefreshCw, FlipHorizontal, Filter, LoaderCircle, Trash2 } from "lucide-react";
|
||||
import { Search, RefreshCw, FlipHorizontal, Filter, LoaderCircle, Trash2, Download, LayoutGrid, List } from "lucide-react";
|
||||
import type { MirrorJob, Organization } from "@/lib/db/schema";
|
||||
import { OrganizationList } from "./OrganizationsList";
|
||||
import AddOrganizationDialog from "./AddOrganizationDialog";
|
||||
@@ -21,6 +21,7 @@ import {
|
||||
SelectValue,
|
||||
} from "../ui/select";
|
||||
import type { MirrorOrgRequest, MirrorOrgResponse } from "@/types/mirror";
|
||||
import type { SyncOrgRequest, SyncOrgResponse } from "@/types/sync";
|
||||
import type { SourceProviderKind } from "@/lib/source-providers/kinds";
|
||||
import { useSSE } from "@/hooks/useSEE";
|
||||
import { useFilterParams } from "@/hooks/useFilterParams";
|
||||
@@ -28,6 +29,10 @@ import { toast } from "sonner";
|
||||
import { invalidateConfigCache, useConfigStatus } from "@/hooks/useConfigStatus";
|
||||
import { useNavigation } from "@/components/layout/MainLayout";
|
||||
import { useLiveRefresh } from "@/hooks/useLiveRefresh";
|
||||
import { withBase } from "@/lib/base-path";
|
||||
import { useOrganizationsView } from "@/hooks/useOrganizationsView";
|
||||
import type { OrganizationsViewMode } from "@/lib/utils/organizations-view";
|
||||
import { cn } from "@/lib/utils";
|
||||
import {
|
||||
Drawer,
|
||||
DrawerClose,
|
||||
@@ -51,6 +56,8 @@ export function Organization() {
|
||||
const [organizations, setOrganizations] = useState<Organization[]>([]);
|
||||
const [isLoading, setIsLoading] = useState<boolean>(true);
|
||||
const [isDialogOpen, setIsDialogOpen] = useState<boolean>(false);
|
||||
// Cards or the compact list (#428); remembered per browser.
|
||||
const { view, setView } = useOrganizationsView();
|
||||
const { user } = useAuth();
|
||||
const { sourceProvider, sourceUrl, sources } = useConfigStatus();
|
||||
const { navigationKey } = useNavigation();
|
||||
@@ -204,6 +211,62 @@ export function Organization() {
|
||||
}
|
||||
};
|
||||
|
||||
// Re-sync an organization that is already mirrored: the Mirror button only
|
||||
// covers the first run, so this is the by-hand refresh (#429).
|
||||
const handleSyncOrg = async ({ orgId }: { orgId: string }) => {
|
||||
try {
|
||||
if (!user || !user.id) {
|
||||
return;
|
||||
}
|
||||
|
||||
setLoadingOrgIds((prev) => new Set(prev).add(orgId));
|
||||
|
||||
const reqPayload: SyncOrgRequest = { orgId };
|
||||
|
||||
const response = await apiRequest<SyncOrgResponse>("/job/sync-org", {
|
||||
method: "POST",
|
||||
data: reqPayload,
|
||||
});
|
||||
|
||||
if (response.success) {
|
||||
const queued = response.queued;
|
||||
const parts: string[] = [];
|
||||
if (queued?.mirror) parts.push(`${queued.mirror} to mirror`);
|
||||
if (queued?.sync) parts.push(`${queued.sync} to sync`);
|
||||
toast.success(
|
||||
parts.length > 0
|
||||
? `Sync started: ${parts.join(", ")}`
|
||||
: "Sync started: checking for new repositories"
|
||||
);
|
||||
|
||||
// Show the spinner right away; the card settles once the run reports back.
|
||||
setOrganizations((prevOrgs) =>
|
||||
prevOrgs.map((org) =>
|
||||
org.id === orgId ? { ...org, status: "mirroring" } : org
|
||||
)
|
||||
);
|
||||
|
||||
// Refresh organization data to get updated repository breakdown
|
||||
// Use a small delay to allow the backend to process the sync request
|
||||
setTimeout(() => {
|
||||
fetchOrganizations(true);
|
||||
}, 1000);
|
||||
} else {
|
||||
toast.error(response.error || "Error starting sync job");
|
||||
}
|
||||
} catch (error) {
|
||||
toast.error(
|
||||
error instanceof Error ? error.message : "Error starting sync job"
|
||||
);
|
||||
} finally {
|
||||
setLoadingOrgIds((prev) => {
|
||||
const newSet = new Set(prev);
|
||||
newSet.delete(orgId);
|
||||
return newSet;
|
||||
});
|
||||
}
|
||||
};
|
||||
|
||||
const handleIgnoreOrg = async ({ orgId, ignore }: { orgId: string; ignore: boolean }) => {
|
||||
try {
|
||||
if (!user || !user.id) {
|
||||
@@ -222,21 +285,33 @@ export function Organization() {
|
||||
|
||||
const newStatus = ignore ? "ignored" : "imported";
|
||||
|
||||
const response = await apiRequest<{ success: boolean; organization?: Organization; error?: string }>(
|
||||
// cascade: the organization's repositories follow it, so ignoring an
|
||||
// organization actually stops its mirrors from syncing (#429).
|
||||
const response = await apiRequest<{
|
||||
success: boolean;
|
||||
organization?: Organization;
|
||||
repositoriesChanged?: number;
|
||||
error?: string;
|
||||
}>(
|
||||
`/organizations/${orgId}/status`,
|
||||
{
|
||||
method: "PATCH",
|
||||
data: {
|
||||
status: newStatus,
|
||||
userId: user.id
|
||||
userId: user.id,
|
||||
cascade: true,
|
||||
},
|
||||
}
|
||||
);
|
||||
|
||||
if (response.success) {
|
||||
const changed = response.repositoriesChanged ?? 0;
|
||||
const repoNote = changed > 0
|
||||
? ` (${changed} ${changed === 1 ? "repository" : "repositories"} ${ignore ? "ignored too" : "restored"})`
|
||||
: "";
|
||||
toast.success(ignore
|
||||
? `Organization will be ignored in future operations`
|
||||
: `Organization included for mirroring`
|
||||
? `Organization will be ignored in future operations${repoNote}`
|
||||
: `Organization included for mirroring${repoNote}`
|
||||
);
|
||||
|
||||
// Update local state
|
||||
@@ -499,7 +574,7 @@ export function Organization() {
|
||||
<div className="flex flex-col lg:flex-row items-start lg:items-center gap-2 lg:gap-4 w-full">
|
||||
{/* Mobile: Search bar with filter button */}
|
||||
<div className="flex items-center gap-2 w-full lg:hidden">
|
||||
<div className="relative flex-grow">
|
||||
<div className="relative flex-grow min-w-0">
|
||||
<Search className="absolute left-3 top-1/2 -translate-y-1/2 h-4 w-4 text-muted-foreground" />
|
||||
<input
|
||||
type="text"
|
||||
@@ -708,6 +783,8 @@ export function Organization() {
|
||||
</DrawerContent>
|
||||
</Drawer>
|
||||
|
||||
<OrganizationsViewToggle view={view} onChange={setView} compact />
|
||||
|
||||
<Button
|
||||
variant="outline"
|
||||
size="icon"
|
||||
@@ -717,7 +794,19 @@ export function Organization() {
|
||||
>
|
||||
<RefreshCw className="h-4 w-4" />
|
||||
</Button>
|
||||
|
||||
|
||||
<Button
|
||||
asChild
|
||||
variant="outline"
|
||||
size="icon"
|
||||
title="Export organizations as CSV"
|
||||
className="h-10 w-10 shrink-0"
|
||||
>
|
||||
<a href={withBase("/api/organizations/export")} download>
|
||||
<Download className="h-4 w-4" />
|
||||
</a>
|
||||
</Button>
|
||||
|
||||
<Button
|
||||
variant="default"
|
||||
size="icon"
|
||||
@@ -856,6 +945,8 @@ export function Organization() {
|
||||
|
||||
{/* Action buttons */}
|
||||
<div className="flex items-center gap-2 ml-auto">
|
||||
<OrganizationsViewToggle view={view} onChange={setView} />
|
||||
|
||||
<Button
|
||||
variant="outline"
|
||||
size="icon"
|
||||
@@ -866,6 +957,19 @@ export function Organization() {
|
||||
<RefreshCw className="h-4 w-4" />
|
||||
</Button>
|
||||
|
||||
{/* CSV export of every tracked organization */}
|
||||
<Button
|
||||
asChild
|
||||
variant="outline"
|
||||
title="Export organizations as CSV"
|
||||
className="h-10 whitespace-nowrap"
|
||||
>
|
||||
<a href={withBase("/api/organizations/export")} download>
|
||||
<Download className="h-4 w-4 xl:mr-2" />
|
||||
<span className="hidden xl:inline">Export CSV</span>
|
||||
</a>
|
||||
</Button>
|
||||
|
||||
<Button
|
||||
variant="default"
|
||||
onClick={handleMirrorAllOrgs}
|
||||
@@ -886,6 +990,7 @@ export function Organization() {
|
||||
setFilter={setFilter}
|
||||
loadingOrgIds={loadingOrgIds}
|
||||
onMirror={handleMirrorOrg}
|
||||
onSync={handleSyncOrg}
|
||||
onIgnore={handleIgnoreOrg}
|
||||
onAddOrganization={() => setIsDialogOpen(true)}
|
||||
onDelete={handleRequestDeleteOrganization}
|
||||
@@ -895,6 +1000,7 @@ export function Organization() {
|
||||
sourceProvider={sourceProvider}
|
||||
sourceUrl={sourceUrl}
|
||||
sources={sources}
|
||||
view={view}
|
||||
/>
|
||||
|
||||
<AddOrganizationDialog
|
||||
@@ -970,3 +1076,73 @@ export function Organization() {
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Cards or list switch for the Organizations page (#428). A segmented
|
||||
* pair of icon buttons so it sits next to the other toolbar icons. The
|
||||
* compact form is one button that flips to the other view, for the
|
||||
* narrow toolbar where every icon costs search box width.
|
||||
*/
|
||||
function OrganizationsViewToggle({
|
||||
view,
|
||||
onChange,
|
||||
compact = false,
|
||||
}: {
|
||||
view: OrganizationsViewMode;
|
||||
onChange: (next: OrganizationsViewMode) => void;
|
||||
compact?: boolean;
|
||||
}) {
|
||||
const options: { value: OrganizationsViewMode; label: string; Icon: typeof LayoutGrid }[] = [
|
||||
{ value: "cards", label: "Cards", Icon: LayoutGrid },
|
||||
{ value: "list", label: "List", Icon: List },
|
||||
];
|
||||
if (compact) {
|
||||
const next = options.find((option) => option.value !== view) ?? options[0];
|
||||
return (
|
||||
<Button
|
||||
type="button"
|
||||
variant="outline"
|
||||
size="icon"
|
||||
onClick={() => onChange(next.value)}
|
||||
title={`Switch to ${next.label.toLowerCase()} view`}
|
||||
aria-label={`Switch to ${next.label.toLowerCase()} view`}
|
||||
className="h-10 w-10 shrink-0"
|
||||
data-testid="organizations-view-toggle"
|
||||
>
|
||||
<next.Icon className="h-4 w-4" />
|
||||
</Button>
|
||||
);
|
||||
}
|
||||
return (
|
||||
<div
|
||||
role="radiogroup"
|
||||
aria-label="Organizations view"
|
||||
className="flex items-center h-10 rounded-md border border-input bg-background shrink-0"
|
||||
data-testid="organizations-view-toggle"
|
||||
>
|
||||
{options.map(({ value, label, Icon }, index) => (
|
||||
<Button
|
||||
key={value}
|
||||
type="button"
|
||||
variant="ghost"
|
||||
size="icon"
|
||||
role="radio"
|
||||
aria-checked={view === value}
|
||||
aria-label={`${label} view`}
|
||||
title={`${label} view`}
|
||||
data-state={view === value ? "on" : "off"}
|
||||
onClick={() => onChange(value)}
|
||||
className={cn(
|
||||
"h-full w-10 rounded-none",
|
||||
index === 0 ? "rounded-l-md" : "rounded-r-md border-l border-input",
|
||||
view === value
|
||||
? "bg-muted text-foreground hover:bg-muted"
|
||||
: "text-muted-foreground hover:text-foreground"
|
||||
)}
|
||||
>
|
||||
<Icon className="h-4 w-4" />
|
||||
</Button>
|
||||
))}
|
||||
</div>
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,90 @@
|
||||
import { Button } from "@/components/ui/button";
|
||||
import {
|
||||
DropdownMenu,
|
||||
DropdownMenuContent,
|
||||
DropdownMenuItem,
|
||||
DropdownMenuSeparator,
|
||||
DropdownMenuTrigger,
|
||||
} from "@/components/ui/dropdown-menu";
|
||||
import { Ban, MoreVertical, RefreshCw, SlidersHorizontal, Trash2 } from "lucide-react";
|
||||
import type { Organization } from "@/lib/db/schema";
|
||||
import { cn } from "@/lib/utils";
|
||||
|
||||
export interface OrganizationActionsMenuProps {
|
||||
org: Organization;
|
||||
disabled?: boolean;
|
||||
onSync?: ({ orgId }: { orgId: string }) => Promise<void>;
|
||||
onIgnore?: ({ orgId, ignore }: { orgId: string; ignore: boolean }) => Promise<void>;
|
||||
onDelete?: (orgId: string) => void;
|
||||
onEditMirrorOptions: (org: Organization) => void;
|
||||
triggerClassName?: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* The secondary actions of an organization, shared by the cards and the
|
||||
* list (#428). Hidden while the organization is being mirrored, like the
|
||||
* primary action.
|
||||
*/
|
||||
export function OrganizationActionsMenu({
|
||||
org,
|
||||
disabled = false,
|
||||
onSync,
|
||||
onIgnore,
|
||||
onDelete,
|
||||
onEditMirrorOptions,
|
||||
triggerClassName,
|
||||
}: OrganizationActionsMenuProps) {
|
||||
if (org.status === "mirroring") return null;
|
||||
|
||||
return (
|
||||
<DropdownMenu>
|
||||
<DropdownMenuTrigger asChild>
|
||||
<Button
|
||||
variant="ghost"
|
||||
size="icon"
|
||||
disabled={disabled}
|
||||
className={cn(triggerClassName)}
|
||||
title="More actions"
|
||||
aria-label={`More actions for ${org.name}`}
|
||||
>
|
||||
<MoreVertical className="h-4 w-4" />
|
||||
</Button>
|
||||
</DropdownMenuTrigger>
|
||||
<DropdownMenuContent align="end">
|
||||
{onSync && (org.status === "mirrored" || org.status === "failed") && (
|
||||
<>
|
||||
<DropdownMenuItem onClick={() => org.id && onSync({ orgId: org.id })}>
|
||||
<RefreshCw className="h-4 w-4 mr-2" />
|
||||
Sync Organization
|
||||
</DropdownMenuItem>
|
||||
<DropdownMenuSeparator />
|
||||
</>
|
||||
)}
|
||||
<DropdownMenuItem onClick={() => onEditMirrorOptions(org)}>
|
||||
<SlidersHorizontal className="h-4 w-4 mr-2" />
|
||||
Mirror Options
|
||||
</DropdownMenuItem>
|
||||
{org.status !== "ignored" && (
|
||||
<DropdownMenuItem
|
||||
onClick={() => org.id && onIgnore && onIgnore({ orgId: org.id, ignore: true })}
|
||||
>
|
||||
<Ban className="h-4 w-4 mr-2" />
|
||||
Ignore Organization
|
||||
</DropdownMenuItem>
|
||||
)}
|
||||
{onDelete && (
|
||||
<>
|
||||
{org.status !== "ignored" && <DropdownMenuSeparator />}
|
||||
<DropdownMenuItem
|
||||
className="text-destructive focus:text-destructive"
|
||||
onClick={() => org.id && onDelete(org.id)}
|
||||
>
|
||||
<Trash2 className="h-4 w-4 mr-2" />
|
||||
Delete from Mirror
|
||||
</DropdownMenuItem>
|
||||
</>
|
||||
)}
|
||||
</DropdownMenuContent>
|
||||
</DropdownMenu>
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,88 @@
|
||||
import { useState } from "react";
|
||||
import { toast } from "sonner";
|
||||
import { Layers } from "lucide-react";
|
||||
import type { SourceApiRecord } from "@/types/config";
|
||||
import { SOURCE_PROVIDER_ICONS } from "@/lib/source-providers/icons";
|
||||
import { cn } from "@/lib/utils";
|
||||
import {
|
||||
Select,
|
||||
SelectContent,
|
||||
SelectItem,
|
||||
SelectTrigger,
|
||||
SelectValue,
|
||||
} from "@/components/ui/select";
|
||||
|
||||
// Radix reserves the empty string as a Select value, so unpinned
|
||||
// organizations use this sentinel instead.
|
||||
const EVERY_SOURCE_VALUE = "every-source";
|
||||
|
||||
export interface OrganizationSourceEditorProps {
|
||||
sources: SourceApiRecord[];
|
||||
value?: string | null;
|
||||
disabled?: boolean;
|
||||
onUpdateSource: (sourceId: string | null) => Promise<void>;
|
||||
/** Drop the "Source" label; the list view puts it in the column header. */
|
||||
compact?: boolean;
|
||||
className?: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* Pin an organization to one connected source, or let it follow every
|
||||
* source. Shown by the cards and the list only when more than one source
|
||||
* is connected.
|
||||
*/
|
||||
export function OrganizationSourceEditor({
|
||||
sources,
|
||||
value,
|
||||
disabled = false,
|
||||
onUpdateSource,
|
||||
compact = false,
|
||||
className,
|
||||
}: OrganizationSourceEditorProps) {
|
||||
const [isUpdating, setIsUpdating] = useState(false);
|
||||
|
||||
const handleValueChange = async (next: string) => {
|
||||
setIsUpdating(true);
|
||||
try {
|
||||
await onUpdateSource(next === EVERY_SOURCE_VALUE ? null : next);
|
||||
} catch (error) {
|
||||
toast.error(error instanceof Error ? error.message : "Failed to update source");
|
||||
} finally {
|
||||
setIsUpdating(false);
|
||||
}
|
||||
};
|
||||
|
||||
return (
|
||||
<div className={cn("flex items-center gap-2 w-full", className)}>
|
||||
{!compact && (
|
||||
<span className="flex items-center gap-1.5 text-xs text-muted-foreground shrink-0">
|
||||
<Layers className="h-3 w-3" />
|
||||
Source
|
||||
</span>
|
||||
)}
|
||||
<Select
|
||||
value={value ?? EVERY_SOURCE_VALUE}
|
||||
onValueChange={(next) => void handleValueChange(next)}
|
||||
disabled={disabled || isUpdating}
|
||||
>
|
||||
<SelectTrigger className="h-8 min-w-0 flex-1 text-xs" aria-label="Source">
|
||||
<SelectValue placeholder="Source" />
|
||||
</SelectTrigger>
|
||||
<SelectContent>
|
||||
<SelectItem value={EVERY_SOURCE_VALUE}>Every source</SelectItem>
|
||||
{sources.map((source) => {
|
||||
const SourceIcon = SOURCE_PROVIDER_ICONS[source.provider];
|
||||
return (
|
||||
<SelectItem key={source.id} value={source.id}>
|
||||
<span className="flex items-center gap-2">
|
||||
<SourceIcon className="h-3.5 w-3.5" />
|
||||
{source.name}
|
||||
</span>
|
||||
</SelectItem>
|
||||
);
|
||||
})}
|
||||
</SelectContent>
|
||||
</Select>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
@@ -2,9 +2,7 @@ import { useMemo, useState } from "react";
|
||||
import { Card } from "@/components/ui/card";
|
||||
import { Button } from "@/components/ui/button";
|
||||
import { Badge } from "@/components/ui/badge";
|
||||
import { Plus, RefreshCw, Building2, Check, AlertCircle, Clock, MoreVertical, Ban, SlidersHorizontal, Trash2, Layers } from "lucide-react";
|
||||
import { SiGitea } from "react-icons/si";
|
||||
import { toast } from "sonner";
|
||||
import { Plus, RefreshCw, Building2, Check, AlertCircle, Clock, Ban, Layers } from "lucide-react";
|
||||
import type { MirrorOverrides, Organization } from "@/lib/db/schema";
|
||||
import type { FilterParams } from "@/types/filter";
|
||||
import type { SourceApiRecord } from "@/types/config";
|
||||
@@ -13,7 +11,7 @@ import { Skeleton } from "@/components/ui/skeleton";
|
||||
import { cn } from "@/lib/utils";
|
||||
import { buildGiteaWebUrl } from "@/lib/gitea-url";
|
||||
import { MirrorDestinationEditor } from "./MirrorDestinationEditor";
|
||||
import { destinationInfo } from "@/components/destination/DestinationIcon";
|
||||
import { DestinationIcon, destinationInfo } from "@/components/destination/DestinationIcon";
|
||||
import type { OrganizationMoveResult } from "@/lib/destination-transfer";
|
||||
import { MirrorOverridesDialog } from "@/components/config/MirrorOverridesDialog";
|
||||
import { hasMirrorOverrides, mirrorOptionsToFlags } from "@/lib/utils/mirror-overrides";
|
||||
@@ -27,21 +25,11 @@ import {
|
||||
type SourceProviderKind,
|
||||
} from "@/lib/source-providers/kinds";
|
||||
import { SOURCE_PROVIDER_ICONS } from "@/lib/source-providers/icons";
|
||||
import {
|
||||
Select,
|
||||
SelectContent,
|
||||
SelectItem,
|
||||
SelectTrigger,
|
||||
SelectValue,
|
||||
} from "@/components/ui/select";
|
||||
import {
|
||||
DropdownMenu,
|
||||
DropdownMenuContent,
|
||||
DropdownMenuItem,
|
||||
DropdownMenuLabel,
|
||||
DropdownMenuSeparator,
|
||||
DropdownMenuTrigger,
|
||||
} from "@/components/ui/dropdown-menu";
|
||||
import type { OrganizationsViewMode } from "@/lib/utils/organizations-view";
|
||||
import { OrganizationSourceEditor } from "./OrganizationSourceEditor";
|
||||
import { OrganizationActionsMenu } from "./OrganizationActionsMenu";
|
||||
import { OrganizationsTable } from "./OrganizationsTable";
|
||||
import { getOrganizationStatusBadge, resolveOrganizationSource } from "./organization-presentation";
|
||||
|
||||
interface OrganizationListProps {
|
||||
organizations: Organization[];
|
||||
@@ -49,6 +37,12 @@ interface OrganizationListProps {
|
||||
filter: FilterParams;
|
||||
setFilter: (filter: FilterParams) => void;
|
||||
onMirror: ({ orgId }: { orgId: string }) => Promise<void>;
|
||||
/**
|
||||
* Re-sync an organization that has already been mirrored. The Mirror
|
||||
* button only covers the first run, so without this an organization could
|
||||
* never be refreshed by hand (#429).
|
||||
*/
|
||||
onSync?: ({ orgId }: { orgId: string }) => Promise<void>;
|
||||
onIgnore?: ({ orgId, ignore }: { orgId: string; ignore: boolean }) => Promise<void>;
|
||||
loadingOrgIds: Set<string>;
|
||||
onAddOrganization?: () => void;
|
||||
@@ -59,93 +53,25 @@ interface OrganizationListProps {
|
||||
sourceUrl?: string;
|
||||
/** Connected sources; with more than one, each card shows and edits its own source. */
|
||||
sources?: SourceApiRecord[];
|
||||
/** Cards (default) or the compact list (#428). */
|
||||
view?: OrganizationsViewMode;
|
||||
}
|
||||
|
||||
const STATUS_ICONS = { check: Check, alert: AlertCircle, clock: Clock, ban: Ban } as const;
|
||||
|
||||
// Helper function to get status badge variant and icon
|
||||
const getStatusBadge = (status: string | null) => {
|
||||
switch (status) {
|
||||
case "imported":
|
||||
return { variant: "secondary" as const, label: "Not Mirrored", icon: null };
|
||||
case "mirroring":
|
||||
return { variant: "outline" as const, label: "Mirroring", icon: Clock };
|
||||
case "mirrored":
|
||||
return { variant: "default" as const, label: "Mirrored", icon: Check };
|
||||
case "failed":
|
||||
return { variant: "destructive" as const, label: "Failed", icon: AlertCircle };
|
||||
case "ignored":
|
||||
return { variant: "outline" as const, label: "Ignored", icon: Ban };
|
||||
default:
|
||||
return { variant: "secondary" as const, label: "Unknown", icon: null };
|
||||
}
|
||||
const badge = getOrganizationStatusBadge(status);
|
||||
return { ...badge, icon: badge.icon ? STATUS_ICONS[badge.icon] : null };
|
||||
};
|
||||
|
||||
// Radix reserves the empty string as a Select value, so unpinned
|
||||
// organizations use this sentinel instead.
|
||||
const EVERY_SOURCE_VALUE = "every-source";
|
||||
|
||||
function OrganizationSourceEditor({
|
||||
sources,
|
||||
value,
|
||||
disabled = false,
|
||||
onUpdateSource,
|
||||
}: {
|
||||
sources: SourceApiRecord[];
|
||||
value?: string | null;
|
||||
disabled?: boolean;
|
||||
onUpdateSource: (sourceId: string | null) => Promise<void>;
|
||||
}) {
|
||||
const [isUpdating, setIsUpdating] = useState(false);
|
||||
|
||||
const handleValueChange = async (next: string) => {
|
||||
setIsUpdating(true);
|
||||
try {
|
||||
await onUpdateSource(next === EVERY_SOURCE_VALUE ? null : next);
|
||||
} catch (error) {
|
||||
toast.error(error instanceof Error ? error.message : "Failed to update source");
|
||||
} finally {
|
||||
setIsUpdating(false);
|
||||
}
|
||||
};
|
||||
|
||||
return (
|
||||
<div className="flex items-center gap-2 w-full">
|
||||
<span className="flex items-center gap-1.5 text-xs text-muted-foreground shrink-0">
|
||||
<Layers className="h-3 w-3" />
|
||||
Source
|
||||
</span>
|
||||
<Select
|
||||
value={value ?? EVERY_SOURCE_VALUE}
|
||||
onValueChange={(next) => void handleValueChange(next)}
|
||||
disabled={disabled || isUpdating}
|
||||
>
|
||||
<SelectTrigger className="h-8 min-w-0 flex-1 text-xs">
|
||||
<SelectValue placeholder="Source" />
|
||||
</SelectTrigger>
|
||||
<SelectContent>
|
||||
<SelectItem value={EVERY_SOURCE_VALUE}>Every source</SelectItem>
|
||||
{sources.map((source) => {
|
||||
const SourceIcon = SOURCE_PROVIDER_ICONS[source.provider];
|
||||
return (
|
||||
<SelectItem key={source.id} value={source.id}>
|
||||
<span className="flex items-center gap-2">
|
||||
<SourceIcon className="h-3.5 w-3.5" />
|
||||
{source.name}
|
||||
</span>
|
||||
</SelectItem>
|
||||
);
|
||||
})}
|
||||
</SelectContent>
|
||||
</Select>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
export function OrganizationList({
|
||||
organizations,
|
||||
isLoading,
|
||||
filter,
|
||||
setFilter,
|
||||
onMirror,
|
||||
onSync,
|
||||
onIgnore,
|
||||
loadingOrgIds,
|
||||
onAddOrganization,
|
||||
@@ -154,6 +80,7 @@ export function OrganizationList({
|
||||
sourceProvider = "github",
|
||||
sourceUrl: sourceUrlProp,
|
||||
sources,
|
||||
view = "cards",
|
||||
}: OrganizationListProps) {
|
||||
const { giteaConfig, mirrorOptions, advancedOptions } = useGiteaConfig();
|
||||
const [overridesTarget, setOverridesTarget] = useState<Organization | null>(null);
|
||||
@@ -272,6 +199,17 @@ export function OrganizationList({
|
||||
// GitHub and GitLab destinations are pushed to; only Gitea and Forgejo can transfer a repository.
|
||||
const destination = destinationInfo(giteaConfig);
|
||||
|
||||
// Tooltip for the destination link on a card. Named after the configured
|
||||
// destination, so a Forgejo user is not told to view the org on Gitea (#430).
|
||||
const destinationLinkTooltip = (org: Organization, orgUrl: string | null): string => {
|
||||
if (!giteaConfig?.url) return `${destination.label} not configured`;
|
||||
if (org.status === "imported") return `Organization not yet mirrored to ${destination.label}`;
|
||||
if (org.status === "failed") return "Organization mirroring failed";
|
||||
if (org.status === "mirroring") return `Organization is being mirrored to ${destination.label}`;
|
||||
if (orgUrl) return `View on ${destination.label}`;
|
||||
return `${destination.label} organization not available`;
|
||||
};
|
||||
|
||||
// Where an organization's repositories land with no override, which is what
|
||||
// the mirror strategy produces for organization repositories (see
|
||||
// getGiteaRepoOwner in src/lib/gitea.ts). Only "preserve" and "mixed" make
|
||||
@@ -322,12 +260,43 @@ export function OrganizationList({
|
||||
return result;
|
||||
}, [organizations, filter]);
|
||||
|
||||
const overridesDialog = (
|
||||
<MirrorOverridesDialog
|
||||
open={!!overridesTarget}
|
||||
onOpenChange={(open) => {
|
||||
if (!open) setOverridesTarget(null);
|
||||
}}
|
||||
targetKind="organization"
|
||||
targetName={overridesTarget?.name ?? ""}
|
||||
value={overridesTarget?.mirrorOverrides ?? null}
|
||||
destinationProvider={giteaConfig?.provider}
|
||||
inheritedFrom={{
|
||||
...mirrorOptionsToFlags(mirrorOptions),
|
||||
skipForks: !!advancedOptions?.skipForks,
|
||||
}}
|
||||
inheritedLabel="global settings"
|
||||
onSave={async (overrides) => {
|
||||
if (overridesTarget?.id) {
|
||||
await handleUpdateMirrorOverrides(overridesTarget.id, overrides);
|
||||
}
|
||||
}}
|
||||
/>
|
||||
);
|
||||
|
||||
return isLoading ? (
|
||||
<div className="grid grid-cols-1 sm:grid-cols-[repeat(auto-fill,minmax(27rem,1fr))] gap-4">
|
||||
{Array.from({ length: 5 }).map((_, i) => (
|
||||
<Skeleton key={i} className="h-[11.25rem] w-full" />
|
||||
))}
|
||||
</div>
|
||||
view === "list" ? (
|
||||
<div className="flex flex-col gap-2">
|
||||
{Array.from({ length: 6 }).map((_, i) => (
|
||||
<Skeleton key={i} className="h-14 w-full" />
|
||||
))}
|
||||
</div>
|
||||
) : (
|
||||
<div className="grid grid-cols-1 sm:grid-cols-[repeat(auto-fill,minmax(27rem,1fr))] gap-4">
|
||||
{Array.from({ length: 5 }).map((_, i) => (
|
||||
<Skeleton key={i} className="h-[11.25rem] w-full" />
|
||||
))}
|
||||
</div>
|
||||
)
|
||||
) : filteredOrganizations.length === 0 ? (
|
||||
<div className="flex flex-col items-center justify-center py-12 text-center">
|
||||
<Building2 className="h-12 w-12 text-muted-foreground mb-4" />
|
||||
@@ -356,6 +325,29 @@ export function OrganizationList({
|
||||
</Button>
|
||||
)}
|
||||
</div>
|
||||
) : view === "list" ? (
|
||||
<>
|
||||
<OrganizationsTable
|
||||
organizations={filteredOrganizations}
|
||||
loadingOrgIds={loadingOrgIds}
|
||||
sources={sources}
|
||||
sourceProvider={sourceProvider}
|
||||
sourceUrl={sourceUrl}
|
||||
destination={destination}
|
||||
getDestinationUrl={getGiteaOrgUrl}
|
||||
destinationLinkTooltip={destinationLinkTooltip}
|
||||
defaultDestinationFor={defaultDestinationFor}
|
||||
onUpdateDestination={handleUpdateDestination}
|
||||
onMoveMirrors={destination.isPushTarget ? undefined : handleMoveMirrors}
|
||||
onUpdateSource={handleUpdateSource}
|
||||
onMirror={onMirror}
|
||||
onSync={onSync}
|
||||
onIgnore={onIgnore}
|
||||
onDelete={onDelete}
|
||||
onEditMirrorOptions={setOverridesTarget}
|
||||
/>
|
||||
{overridesDialog}
|
||||
</>
|
||||
) : (
|
||||
<div className="grid grid-cols-1 sm:grid-cols-[repeat(auto-fill,minmax(27rem,1fr))] gap-4 pb-20 sm:pb-0">
|
||||
{filteredOrganizations.map((org, index) => {
|
||||
@@ -363,16 +355,12 @@ export function OrganizationList({
|
||||
const statusBadge = getStatusBadge(org.status);
|
||||
const StatusIcon = statusBadge.icon;
|
||||
const orgSources = sources ?? [];
|
||||
const pinnedSource =
|
||||
orgSources.find((source) => source.id === org.sourceId) ?? null;
|
||||
const isPublicOnlySource = pinnedSource?.token === "";
|
||||
const orgProvider: SourceProviderKind = pinnedSource?.provider ?? sourceProvider;
|
||||
const orgSourceUrl = pinnedSource
|
||||
? normalizeSourceUrl(pinnedSource.url, pinnedSource.provider)
|
||||
: sourceUrl;
|
||||
const orgSourceLabel = SOURCE_PROVIDER_LABELS[orgProvider];
|
||||
const orgSourceShortLabel = orgProvider === "gitea" ? "Gitea" : orgSourceLabel;
|
||||
const OrgSourceIcon = SOURCE_PROVIDER_ICONS[orgProvider];
|
||||
const orgSource = resolveOrganizationSource(org, sources, sourceProvider, sourceUrl);
|
||||
const { pinnedSource, isPublicOnlySource } = orgSource;
|
||||
const orgSourceUrl = orgSource.sourceUrl;
|
||||
const orgSourceLabel = orgSource.label;
|
||||
const orgSourceShortLabel = orgSource.shortLabel;
|
||||
const OrgSourceIcon = SOURCE_PROVIDER_ICONS[orgSource.provider];
|
||||
|
||||
return (
|
||||
<Card
|
||||
@@ -707,63 +695,22 @@ export function OrganizationList({
|
||||
</>
|
||||
)}
|
||||
|
||||
{/* Dropdown menu for additional actions */}
|
||||
{org.status !== "mirroring" && (
|
||||
<DropdownMenu>
|
||||
<DropdownMenuTrigger asChild>
|
||||
<Button variant="ghost" size="icon" disabled={isLoading} className="h-10 w-10">
|
||||
<MoreVertical className="h-4 w-4" />
|
||||
</Button>
|
||||
</DropdownMenuTrigger>
|
||||
<DropdownMenuContent align="end">
|
||||
<DropdownMenuItem onClick={() => setOverridesTarget(org)}>
|
||||
<SlidersHorizontal className="h-4 w-4 mr-2" />
|
||||
Mirror Options
|
||||
</DropdownMenuItem>
|
||||
{org.status !== "ignored" && (
|
||||
<DropdownMenuItem
|
||||
onClick={() => org.id && onIgnore && onIgnore({ orgId: org.id, ignore: true })}
|
||||
>
|
||||
<Ban className="h-4 w-4 mr-2" />
|
||||
Ignore Organization
|
||||
</DropdownMenuItem>
|
||||
)}
|
||||
{onDelete && (
|
||||
<>
|
||||
{org.status !== "ignored" && <DropdownMenuSeparator />}
|
||||
<DropdownMenuItem
|
||||
className="text-destructive focus:text-destructive"
|
||||
onClick={() => org.id && onDelete(org.id)}
|
||||
>
|
||||
<Trash2 className="h-4 w-4 mr-2" />
|
||||
Delete from Mirror
|
||||
</DropdownMenuItem>
|
||||
</>
|
||||
)}
|
||||
</DropdownMenuContent>
|
||||
</DropdownMenu>
|
||||
)}
|
||||
<OrganizationActionsMenu
|
||||
org={org}
|
||||
disabled={isLoading}
|
||||
onSync={onSync}
|
||||
onIgnore={onIgnore}
|
||||
onDelete={onDelete}
|
||||
onEditMirrorOptions={setOverridesTarget}
|
||||
triggerClassName="h-10 w-10"
|
||||
/>
|
||||
</div>
|
||||
|
||||
<div className="flex items-center gap-2 justify-center">
|
||||
{(() => {
|
||||
const giteaUrl = getGiteaOrgUrl(org);
|
||||
|
||||
// Determine tooltip based on status and configuration
|
||||
let tooltip: string;
|
||||
if (!giteaConfig?.url) {
|
||||
tooltip = "Gitea not configured";
|
||||
} else if (org.status === 'imported') {
|
||||
tooltip = "Organization not yet mirrored to Gitea";
|
||||
} else if (org.status === 'failed') {
|
||||
tooltip = "Organization mirroring failed";
|
||||
} else if (org.status === 'mirroring') {
|
||||
tooltip = "Organization is being mirrored to Gitea";
|
||||
} else if (giteaUrl) {
|
||||
tooltip = "View on Gitea";
|
||||
} else {
|
||||
tooltip = "Gitea organization not available";
|
||||
}
|
||||
const tooltip = destinationLinkTooltip(org, giteaUrl);
|
||||
|
||||
return giteaUrl ? (
|
||||
<Button variant="outline" size="default" asChild className="flex-1 h-10 min-w-0">
|
||||
@@ -774,14 +721,14 @@ export function OrganizationList({
|
||||
title={tooltip}
|
||||
className="flex items-center justify-center gap-2"
|
||||
>
|
||||
<SiGitea className="h-4 w-4 flex-shrink-0" />
|
||||
<span className="text-xs">Gitea</span>
|
||||
<DestinationIcon provider={destination.provider} className="h-4 w-4 flex-shrink-0" />
|
||||
<span className="text-xs">{destination.label}</span>
|
||||
</a>
|
||||
</Button>
|
||||
) : (
|
||||
<Button variant="outline" size="default" disabled title={tooltip} className="flex-1 h-10">
|
||||
<SiGitea className="h-4 w-4" />
|
||||
<span className="text-xs ml-2">Gitea</span>
|
||||
<DestinationIcon provider={destination.provider} className="h-4 w-4" />
|
||||
<span className="text-xs ml-2">{destination.label}</span>
|
||||
</Button>
|
||||
);
|
||||
})()}
|
||||
@@ -872,63 +819,21 @@ export function OrganizationList({
|
||||
</>
|
||||
)}
|
||||
|
||||
{/* Dropdown menu for additional actions */}
|
||||
{org.status !== "mirroring" && (
|
||||
<DropdownMenu>
|
||||
<DropdownMenuTrigger asChild>
|
||||
<Button variant="ghost" size="icon" disabled={isLoading}>
|
||||
<MoreVertical className="h-4 w-4" />
|
||||
</Button>
|
||||
</DropdownMenuTrigger>
|
||||
<DropdownMenuContent align="end">
|
||||
<DropdownMenuItem onClick={() => setOverridesTarget(org)}>
|
||||
<SlidersHorizontal className="h-4 w-4 mr-2" />
|
||||
Mirror Options
|
||||
</DropdownMenuItem>
|
||||
{org.status !== "ignored" && (
|
||||
<DropdownMenuItem
|
||||
onClick={() => org.id && onIgnore && onIgnore({ orgId: org.id, ignore: true })}
|
||||
>
|
||||
<Ban className="h-4 w-4 mr-2" />
|
||||
Ignore Organization
|
||||
</DropdownMenuItem>
|
||||
)}
|
||||
{onDelete && (
|
||||
<>
|
||||
<DropdownMenuSeparator />
|
||||
<DropdownMenuItem
|
||||
className="text-destructive focus:text-destructive"
|
||||
onClick={() => org.id && onDelete(org.id)}
|
||||
>
|
||||
<Trash2 className="h-4 w-4 mr-2" />
|
||||
Delete from Mirror
|
||||
</DropdownMenuItem>
|
||||
</>
|
||||
)}
|
||||
</DropdownMenuContent>
|
||||
</DropdownMenu>
|
||||
)}
|
||||
<OrganizationActionsMenu
|
||||
org={org}
|
||||
disabled={isLoading}
|
||||
onSync={onSync}
|
||||
onIgnore={onIgnore}
|
||||
onDelete={onDelete}
|
||||
onEditMirrorOptions={setOverridesTarget}
|
||||
/>
|
||||
</div>
|
||||
|
||||
<div className="flex items-center gap-2">
|
||||
{(() => {
|
||||
const giteaUrl = getGiteaOrgUrl(org);
|
||||
|
||||
// Determine tooltip based on status and configuration
|
||||
let tooltip: string;
|
||||
if (!giteaConfig?.url) {
|
||||
tooltip = "Gitea not configured";
|
||||
} else if (org.status === 'imported') {
|
||||
tooltip = "Organization not yet mirrored to Gitea";
|
||||
} else if (org.status === 'failed') {
|
||||
tooltip = "Organization mirroring failed";
|
||||
} else if (org.status === 'mirroring') {
|
||||
tooltip = "Organization is being mirrored to Gitea";
|
||||
} else if (giteaUrl) {
|
||||
tooltip = "View on Gitea";
|
||||
} else {
|
||||
tooltip = "Gitea organization not available";
|
||||
}
|
||||
const tooltip = destinationLinkTooltip(org, giteaUrl);
|
||||
|
||||
return (
|
||||
<div className="flex items-center border rounded-md">
|
||||
@@ -946,13 +851,13 @@ export function OrganizationList({
|
||||
target="_blank"
|
||||
rel="noopener noreferrer"
|
||||
>
|
||||
<SiGitea className="h-4 w-4 mr-2" />
|
||||
Gitea
|
||||
<DestinationIcon provider={destination.provider} className="h-4 w-4 mr-2" />
|
||||
{destination.label}
|
||||
</a>
|
||||
) : (
|
||||
<>
|
||||
<SiGitea className="h-4 w-4 mr-2" />
|
||||
Gitea
|
||||
<DestinationIcon provider={destination.provider} className="h-4 w-4 mr-2" />
|
||||
{destination.label}
|
||||
</>
|
||||
)}
|
||||
</Button>
|
||||
@@ -981,26 +886,7 @@ export function OrganizationList({
|
||||
);
|
||||
})}
|
||||
|
||||
<MirrorOverridesDialog
|
||||
open={!!overridesTarget}
|
||||
onOpenChange={(open) => {
|
||||
if (!open) setOverridesTarget(null);
|
||||
}}
|
||||
targetKind="organization"
|
||||
targetName={overridesTarget?.name ?? ""}
|
||||
value={overridesTarget?.mirrorOverrides ?? null}
|
||||
destinationProvider={giteaConfig?.provider}
|
||||
inheritedFrom={{
|
||||
...mirrorOptionsToFlags(mirrorOptions),
|
||||
skipForks: !!advancedOptions?.skipForks,
|
||||
}}
|
||||
inheritedLabel="global settings"
|
||||
onSave={async (overrides) => {
|
||||
if (overridesTarget?.id) {
|
||||
await handleUpdateMirrorOverrides(overridesTarget.id, overrides);
|
||||
}
|
||||
}}
|
||||
/>
|
||||
{overridesDialog}
|
||||
</div>
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,348 @@
|
||||
import { AlertCircle, Ban, Check, Clock, RefreshCw } from "lucide-react";
|
||||
import { Badge } from "@/components/ui/badge";
|
||||
import { Button } from "@/components/ui/button";
|
||||
import {
|
||||
Tooltip,
|
||||
TooltipContent,
|
||||
TooltipProvider,
|
||||
TooltipTrigger,
|
||||
} from "@/components/ui/tooltip";
|
||||
import type { Organization } from "@/lib/db/schema";
|
||||
import type { SourceApiRecord } from "@/types/config";
|
||||
import type { OrganizationMoveResult } from "@/lib/destination-transfer";
|
||||
import type { SourceProviderKind } from "@/lib/source-providers/kinds";
|
||||
import { SOURCE_PROVIDER_ICONS } from "@/lib/source-providers/icons";
|
||||
import { DestinationIcon, type destinationInfo } from "@/components/destination/DestinationIcon";
|
||||
import { cn, formatLastSyncTime } from "@/lib/utils";
|
||||
import { hasMirrorOverrides } from "@/lib/utils/mirror-overrides";
|
||||
import { withBase } from "@/lib/base-path";
|
||||
import { MirrorDestinationEditor } from "./MirrorDestinationEditor";
|
||||
import { OrganizationSourceEditor } from "./OrganizationSourceEditor";
|
||||
import { OrganizationActionsMenu } from "./OrganizationActionsMenu";
|
||||
import {
|
||||
describeRepositoryBreakdown,
|
||||
getOrganizationStatusBadge,
|
||||
resolveOrganizationSource,
|
||||
} from "./organization-presentation";
|
||||
|
||||
const STATUS_ICONS = {
|
||||
check: Check,
|
||||
alert: AlertCircle,
|
||||
clock: Clock,
|
||||
ban: Ban,
|
||||
} as const;
|
||||
|
||||
export interface OrganizationsTableProps {
|
||||
organizations: Organization[];
|
||||
loadingOrgIds: Set<string>;
|
||||
sources?: SourceApiRecord[];
|
||||
sourceProvider: SourceProviderKind;
|
||||
sourceUrl: string;
|
||||
destination: ReturnType<typeof destinationInfo>;
|
||||
getDestinationUrl: (org: Organization) => string | null;
|
||||
destinationLinkTooltip: (org: Organization, url: string | null) => string;
|
||||
defaultDestinationFor: (orgName: string) => string;
|
||||
onUpdateDestination: (orgId: string, destination: string | null) => Promise<void>;
|
||||
onMoveMirrors?: (
|
||||
orgId: string,
|
||||
destination: string | null,
|
||||
dryRun: boolean
|
||||
) => Promise<OrganizationMoveResult>;
|
||||
onUpdateSource: (orgId: string, sourceId: string | null) => Promise<void>;
|
||||
onMirror: ({ orgId }: { orgId: string }) => Promise<void>;
|
||||
onSync?: ({ orgId }: { orgId: string }) => Promise<void>;
|
||||
onIgnore?: ({ orgId, ignore }: { orgId: string; ignore: boolean }) => Promise<void>;
|
||||
onDelete?: (orgId: string) => void;
|
||||
onEditMirrorOptions: (org: Organization) => void;
|
||||
}
|
||||
|
||||
/**
|
||||
* The compact list view of the Organizations page (#428): one row per
|
||||
* organization so a few dozen fit on one screen. Same actions as the cards,
|
||||
* with the secondary ones behind the menu.
|
||||
*/
|
||||
export function OrganizationsTable({
|
||||
organizations,
|
||||
loadingOrgIds,
|
||||
sources,
|
||||
sourceProvider,
|
||||
sourceUrl,
|
||||
destination,
|
||||
getDestinationUrl,
|
||||
destinationLinkTooltip,
|
||||
defaultDestinationFor,
|
||||
onUpdateDestination,
|
||||
onMoveMirrors,
|
||||
onUpdateSource,
|
||||
onMirror,
|
||||
onSync,
|
||||
onIgnore,
|
||||
onDelete,
|
||||
onEditMirrorOptions,
|
||||
}: OrganizationsTableProps) {
|
||||
const hasMultipleSources = !!sources && sources.length > 1;
|
||||
|
||||
return (
|
||||
<div className="overflow-x-auto pb-20 sm:pb-0" data-testid="organizations-table">
|
||||
<div className="min-w-[64rem] border rounded-md">
|
||||
{/* Header */}
|
||||
<div className="h-[45px] flex items-center border-b bg-muted/50 text-sm font-medium">
|
||||
<div className="h-full p-3 flex items-center flex-[2.2] sticky left-0 z-10 bg-background before:absolute before:inset-0 before:-z-10 before:bg-muted/50">
|
||||
Organization
|
||||
</div>
|
||||
{hasMultipleSources && (
|
||||
<div className="h-full p-3 flex items-center flex-[1.1]">Source</div>
|
||||
)}
|
||||
<div className="h-full p-3 flex items-center flex-[1.6]">Destination</div>
|
||||
<div className="h-full p-3 flex items-center flex-[1]">Repositories</div>
|
||||
<div className="h-full p-3 flex items-center flex-[1]">Last Mirrored</div>
|
||||
<div className="h-full p-3 flex items-center flex-[0.9]">Status</div>
|
||||
<div className="h-full p-3 flex items-center flex-[1.3]">Actions</div>
|
||||
<div className="h-full p-3 flex items-center justify-center flex-[0.6]">Links</div>
|
||||
</div>
|
||||
|
||||
{organizations.map((org) => {
|
||||
const orgId = org.id ?? "";
|
||||
const isLoading = loadingOrgIds.has(orgId);
|
||||
const statusBadge = getOrganizationStatusBadge(org.status);
|
||||
const StatusIcon = statusBadge.icon ? STATUS_ICONS[statusBadge.icon] : null;
|
||||
const source = resolveOrganizationSource(org, sources, sourceProvider, sourceUrl);
|
||||
const SourceIcon = SOURCE_PROVIDER_ICONS[source.provider];
|
||||
const breakdown = describeRepositoryBreakdown(org, "long");
|
||||
const destinationUrl = getDestinationUrl(org);
|
||||
const destinationTooltip = destinationLinkTooltip(org, destinationUrl);
|
||||
|
||||
return (
|
||||
<div
|
||||
key={org.id ?? org.name}
|
||||
data-testid="organization-row"
|
||||
className={cn(
|
||||
"group min-h-[48px] flex items-center border-b last:border-b-0 hover:bg-muted/50",
|
||||
isLoading && "opacity-75"
|
||||
)}
|
||||
>
|
||||
{/* Organization. Pinned to the left edge so the row stays
|
||||
identifiable while a narrow screen scrolls the rest. */}
|
||||
<div className="h-full px-3 py-2 flex items-center gap-2 flex-[2.2] min-w-0 sticky left-0 z-10 bg-background before:absolute before:inset-0 before:-z-10 group-hover:before:bg-muted/50">
|
||||
<div className="min-w-0 flex-1">
|
||||
<div className="flex items-center gap-2 min-w-0">
|
||||
<a
|
||||
href={`${withBase("/repositories")}?organization=${encodeURIComponent(org.name || "")}`}
|
||||
className="font-medium hover:underline truncate"
|
||||
title={`Show the repositories of ${org.name}`}
|
||||
>
|
||||
{org.name}
|
||||
</a>
|
||||
<Badge
|
||||
variant={org.membershipRole === "member" ? "secondary" : "default"}
|
||||
className="capitalize shrink-0 px-1.5 text-[11px] font-normal"
|
||||
>
|
||||
{org.membershipRole.replace(/_/g, " ")}
|
||||
</Badge>
|
||||
{source.isPublicOnlySource && (
|
||||
<Badge
|
||||
variant="secondary"
|
||||
className="shrink-0 px-1.5 font-normal text-[11px]"
|
||||
title="Mirrors public repositories only; its source has no account"
|
||||
>
|
||||
Public only
|
||||
</Badge>
|
||||
)}
|
||||
{hasMirrorOverrides(org.mirrorOverrides) && (
|
||||
<Badge
|
||||
variant="outline"
|
||||
className="shrink-0 px-1.5 text-[11px] font-normal text-amber-600 dark:text-amber-400 border-amber-500/40"
|
||||
title="This organization overrides the global mirror options"
|
||||
>
|
||||
Custom
|
||||
</Badge>
|
||||
)}
|
||||
</div>
|
||||
{org.status === "failed" && org.errorMessage && (
|
||||
<p
|
||||
className="text-xs text-destructive truncate"
|
||||
title={org.errorMessage}
|
||||
>
|
||||
{org.errorMessage}
|
||||
</p>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{/* Source (multi-source only) */}
|
||||
{hasMultipleSources && (
|
||||
<div className="h-full px-3 py-2 flex items-center flex-[1.1] min-w-0">
|
||||
<OrganizationSourceEditor
|
||||
sources={sources ?? []}
|
||||
value={source.pinnedSource?.id ?? null}
|
||||
disabled={isLoading}
|
||||
onUpdateSource={(sourceId) => onUpdateSource(orgId, sourceId)}
|
||||
compact
|
||||
/>
|
||||
</div>
|
||||
)}
|
||||
|
||||
{/* Destination */}
|
||||
<div className="h-full px-3 py-2 flex items-center flex-[1.6] min-w-0">
|
||||
<MirrorDestinationEditor
|
||||
organizationId={orgId}
|
||||
organizationName={org.name!}
|
||||
currentDestination={org.destinationOrg ?? undefined}
|
||||
defaultDestination={defaultDestinationFor(org.name!)}
|
||||
onUpdate={(next) => onUpdateDestination(orgId, next)}
|
||||
onMoveMirrors={
|
||||
onMoveMirrors
|
||||
? (next, dryRun) => onMoveMirrors(orgId, next, dryRun)
|
||||
: undefined
|
||||
}
|
||||
destinationLabel={destination.label}
|
||||
isUpdating={isLoading}
|
||||
compact
|
||||
/>
|
||||
</div>
|
||||
|
||||
{/* Repositories */}
|
||||
<div className="h-full px-3 py-2 flex items-center flex-[1] min-w-0">
|
||||
<div className="min-w-0">
|
||||
<p className="text-sm">
|
||||
<span className="font-semibold tabular-nums">{org.repositoryCount}</span>
|
||||
<span className="text-muted-foreground ml-1">
|
||||
{org.repositoryCount === 1 ? "repository" : "repositories"}
|
||||
</span>
|
||||
</p>
|
||||
{breakdown.length > 0 && (
|
||||
<p className="text-xs text-muted-foreground truncate" title={breakdown.join(", ")}>
|
||||
{breakdown.join(" | ")}
|
||||
</p>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{/* Last Mirrored */}
|
||||
<div className="h-full px-3 py-2 flex items-center flex-[1]">
|
||||
<p className="text-sm">{formatLastSyncTime(org.lastMirrored ?? null)}</p>
|
||||
</div>
|
||||
|
||||
{/* Status */}
|
||||
<div className="h-full px-3 py-2 flex items-center flex-[0.9]">
|
||||
{org.status === "failed" && org.errorMessage ? (
|
||||
<TooltipProvider>
|
||||
<Tooltip>
|
||||
<TooltipTrigger asChild>
|
||||
<Badge variant="destructive" className="cursor-help flex items-center gap-1">
|
||||
<AlertCircle className="h-3 w-3" />
|
||||
{statusBadge.label}
|
||||
</Badge>
|
||||
</TooltipTrigger>
|
||||
<TooltipContent className="max-w-xs">
|
||||
<p className="text-sm">{org.errorMessage}</p>
|
||||
</TooltipContent>
|
||||
</Tooltip>
|
||||
</TooltipProvider>
|
||||
) : (
|
||||
<Badge variant={statusBadge.variant} className="flex items-center gap-1">
|
||||
{StatusIcon && (
|
||||
<StatusIcon
|
||||
className={cn("h-3 w-3", org.status === "mirroring" && "animate-pulse")}
|
||||
/>
|
||||
)}
|
||||
{statusBadge.label}
|
||||
</Badge>
|
||||
)}
|
||||
</div>
|
||||
|
||||
{/* Actions */}
|
||||
<div className="h-full px-3 py-2 flex items-center gap-1 flex-[1.3]">
|
||||
{org.status === "ignored" ? (
|
||||
<Button
|
||||
size="sm"
|
||||
variant="outline"
|
||||
onClick={() => orgId && onIgnore && onIgnore({ orgId, ignore: false })}
|
||||
disabled={isLoading}
|
||||
>
|
||||
<Check className="h-4 w-4 mr-1.5" />
|
||||
Include
|
||||
</Button>
|
||||
) : org.status === "imported" ? (
|
||||
<Button size="sm" onClick={() => orgId && onMirror({ orgId })} disabled={isLoading}>
|
||||
<RefreshCw className={cn("h-4 w-4 mr-1.5", isLoading && "animate-spin")} />
|
||||
{isLoading ? "Starting" : "Mirror"}
|
||||
</Button>
|
||||
) : org.status === "mirroring" ? (
|
||||
<Button size="sm" variant="outline" disabled>
|
||||
<RefreshCw className="h-4 w-4 mr-1.5 animate-spin" />
|
||||
Mirroring
|
||||
</Button>
|
||||
) : org.status === "failed" ? (
|
||||
<Button
|
||||
size="sm"
|
||||
variant="destructive"
|
||||
onClick={() => orgId && onMirror({ orgId })}
|
||||
disabled={isLoading}
|
||||
>
|
||||
{isLoading ? (
|
||||
<RefreshCw className="h-4 w-4 mr-1.5 animate-spin" />
|
||||
) : (
|
||||
<AlertCircle className="h-4 w-4 mr-1.5" />
|
||||
)}
|
||||
{isLoading ? "Retrying" : "Retry"}
|
||||
</Button>
|
||||
) : org.status === "mirrored" && onSync ? (
|
||||
<Button
|
||||
size="sm"
|
||||
variant="outline"
|
||||
onClick={() => orgId && onSync({ orgId })}
|
||||
disabled={isLoading}
|
||||
title="Re-sync this organization now"
|
||||
>
|
||||
<RefreshCw className={cn("h-4 w-4 mr-1.5", isLoading && "animate-spin")} />
|
||||
Sync
|
||||
</Button>
|
||||
) : null}
|
||||
<OrganizationActionsMenu
|
||||
org={org}
|
||||
disabled={isLoading}
|
||||
onSync={onSync}
|
||||
onIgnore={onIgnore}
|
||||
onDelete={onDelete}
|
||||
onEditMirrorOptions={onEditMirrorOptions}
|
||||
triggerClassName="h-8 w-8"
|
||||
/>
|
||||
</div>
|
||||
|
||||
{/* Links */}
|
||||
<div className="h-full px-3 py-2 flex items-center justify-center gap-x-1 flex-[0.6]">
|
||||
{destinationUrl ? (
|
||||
<Button variant="ghost" size="icon" asChild className="h-8 w-8">
|
||||
<a
|
||||
href={destinationUrl}
|
||||
target="_blank"
|
||||
rel="noopener noreferrer"
|
||||
title={destinationTooltip}
|
||||
>
|
||||
<DestinationIcon provider={destination.provider} className="h-4 w-4" />
|
||||
</a>
|
||||
</Button>
|
||||
) : (
|
||||
<Button variant="ghost" size="icon" disabled title={destinationTooltip} className="h-8 w-8">
|
||||
<DestinationIcon provider={destination.provider} className="h-4 w-4" />
|
||||
</Button>
|
||||
)}
|
||||
<Button variant="ghost" size="icon" asChild className="h-8 w-8">
|
||||
<a
|
||||
href={`${source.sourceUrl}/${org.name}`}
|
||||
target="_blank"
|
||||
rel="noopener noreferrer"
|
||||
title={`View on ${source.label}`}
|
||||
>
|
||||
<SourceIcon className="h-4 w-4" />
|
||||
</a>
|
||||
</Button>
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
})}
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,89 @@
|
||||
import type { Organization } from "@/lib/db/schema";
|
||||
import type { SourceApiRecord } from "@/types/config";
|
||||
import {
|
||||
SOURCE_PROVIDER_LABELS,
|
||||
normalizeSourceUrl,
|
||||
type SourceProviderKind,
|
||||
} from "@/lib/source-providers/kinds";
|
||||
|
||||
export interface OrganizationSourceInfo {
|
||||
/** The source this organization is pinned to, if any. */
|
||||
pinnedSource: SourceApiRecord | null;
|
||||
/** A pinned source with no account mirrors public repositories only. */
|
||||
isPublicOnlySource: boolean;
|
||||
provider: SourceProviderKind;
|
||||
/** Normalized instance URL of the source the organization lives on. */
|
||||
sourceUrl: string;
|
||||
/** Full provider label, for tooltips. */
|
||||
label: string;
|
||||
/** Shorter label for buttons: Gitea and Forgejo sources both read "Gitea". */
|
||||
shortLabel: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* Which source an organization belongs to, for the cards and the list.
|
||||
* Unpinned organizations follow the configured (or first) source.
|
||||
*/
|
||||
export function resolveOrganizationSource(
|
||||
org: Pick<Organization, "sourceId">,
|
||||
sources: SourceApiRecord[] | undefined,
|
||||
fallbackProvider: SourceProviderKind,
|
||||
fallbackUrl: string
|
||||
): OrganizationSourceInfo {
|
||||
const pinnedSource = (sources ?? []).find((source) => source.id === org.sourceId) ?? null;
|
||||
const provider: SourceProviderKind = pinnedSource?.provider ?? fallbackProvider;
|
||||
const label = SOURCE_PROVIDER_LABELS[provider];
|
||||
return {
|
||||
pinnedSource,
|
||||
isPublicOnlySource: pinnedSource?.token === "",
|
||||
provider,
|
||||
sourceUrl: pinnedSource
|
||||
? normalizeSourceUrl(pinnedSource.url, pinnedSource.provider)
|
||||
: fallbackUrl,
|
||||
label,
|
||||
shortLabel: provider === "gitea" ? "Gitea" : label,
|
||||
};
|
||||
}
|
||||
|
||||
export type OrganizationStatusBadge = {
|
||||
variant: "default" | "secondary" | "outline" | "destructive";
|
||||
label: string;
|
||||
icon: "check" | "alert" | "clock" | "ban" | null;
|
||||
};
|
||||
|
||||
/** Badge variant, label and icon for an organization status. */
|
||||
export function getOrganizationStatusBadge(status: string | null | undefined): OrganizationStatusBadge {
|
||||
switch (status) {
|
||||
case "imported":
|
||||
return { variant: "secondary", label: "Not Mirrored", icon: null };
|
||||
case "mirroring":
|
||||
return { variant: "outline", label: "Mirroring", icon: "clock" };
|
||||
case "mirrored":
|
||||
return { variant: "default", label: "Mirrored", icon: "check" };
|
||||
case "failed":
|
||||
return { variant: "destructive", label: "Failed", icon: "alert" };
|
||||
case "ignored":
|
||||
return { variant: "outline", label: "Ignored", icon: "ban" };
|
||||
default:
|
||||
return { variant: "secondary", label: "Unknown", icon: null };
|
||||
}
|
||||
}
|
||||
|
||||
/** "3 public | 2 private | 1 fork" style breakdown, only the non-zero parts. */
|
||||
export function describeRepositoryBreakdown(
|
||||
org: Pick<Organization, "publicRepositoryCount" | "privateRepositoryCount" | "forkRepositoryCount">,
|
||||
style: "long" | "short" = "long"
|
||||
): string[] {
|
||||
const parts: string[] = [];
|
||||
const pub = org.publicRepositoryCount ?? 0;
|
||||
const priv = org.privateRepositoryCount ?? 0;
|
||||
const forks = org.forkRepositoryCount ?? 0;
|
||||
if (pub > 0) parts.push(style === "long" ? `${pub} public` : `${pub} pub`);
|
||||
if (priv > 0) parts.push(style === "long" ? `${priv} private` : `${priv} priv`);
|
||||
if (forks > 0) {
|
||||
parts.push(
|
||||
style === "long" ? `${forks} ${forks === 1 ? "fork" : "forks"}` : `${forks} fork`
|
||||
);
|
||||
}
|
||||
return parts;
|
||||
}
|
||||
@@ -18,7 +18,7 @@ import {
|
||||
SelectValue,
|
||||
} from "../ui/select";
|
||||
import { Button } from "@/components/ui/button";
|
||||
import { Search, RefreshCw, FlipHorizontal, RotateCcw, X, Filter, Ban, Check, LoaderCircle, Trash2 } from "lucide-react";
|
||||
import { Search, RefreshCw, FlipHorizontal, RotateCcw, X, Filter, Ban, Check, LoaderCircle, Trash2, Download } from "lucide-react";
|
||||
import type { MirrorRepoRequest, MirrorRepoResponse } from "@/types/mirror";
|
||||
import {
|
||||
Drawer,
|
||||
@@ -1075,7 +1075,7 @@ export default function Repository() {
|
||||
<div className="flex flex-col sm:flex-row items-start sm:items-center gap-2 sm:gap-4 w-full">
|
||||
{/* Mobile: Search bar with filter button */}
|
||||
<div className="flex items-center gap-2 w-full lg:hidden">
|
||||
<div className="relative flex-grow">
|
||||
<div className="relative flex-grow min-w-0">
|
||||
<Search className="absolute left-3 top-1/2 -translate-y-1/2 h-4 w-4 text-muted-foreground" />
|
||||
<input
|
||||
type="text"
|
||||
@@ -1292,7 +1292,19 @@ export default function Repository() {
|
||||
>
|
||||
<RefreshCw className="h-4 w-4" />
|
||||
</Button>
|
||||
|
||||
|
||||
<Button
|
||||
asChild
|
||||
variant="outline"
|
||||
size="icon"
|
||||
title="Export repositories as CSV"
|
||||
className="h-10 w-10 shrink-0"
|
||||
>
|
||||
<a href={withBase("/api/repositories/export")} download>
|
||||
<Download className="h-4 w-4" />
|
||||
</a>
|
||||
</Button>
|
||||
|
||||
<Button
|
||||
variant="default"
|
||||
size="icon"
|
||||
@@ -1348,6 +1360,19 @@ export default function Repository() {
|
||||
<RefreshCw className="h-4 w-4" />
|
||||
</Button>
|
||||
|
||||
{/* CSV export of every tracked repository */}
|
||||
<Button
|
||||
asChild
|
||||
variant="outline"
|
||||
title="Export repositories as CSV"
|
||||
className="h-10 shrink-0 whitespace-nowrap"
|
||||
>
|
||||
<a href={withBase("/api/repositories/export")} download>
|
||||
<Download className="h-4 w-4 xl:mr-2" />
|
||||
<span className="hidden xl:inline">Export CSV</span>
|
||||
</a>
|
||||
</Button>
|
||||
|
||||
{/* Mirror All action */}
|
||||
<div className="flex items-center gap-2 border-l pl-4">
|
||||
<Button
|
||||
|
||||
@@ -1,5 +1,10 @@
|
||||
import { useState, useEffect } from 'react';
|
||||
import { apiRequest } from '@/lib/utils';
|
||||
import {
|
||||
DEFAULT_AUTH_METHOD,
|
||||
parseDefaultAuthMethod,
|
||||
type AuthMethod,
|
||||
} from '@/lib/utils/auth-method';
|
||||
|
||||
interface AuthMethods {
|
||||
emailPassword: boolean;
|
||||
@@ -14,6 +19,8 @@ interface AuthMethods {
|
||||
oidc: {
|
||||
enabled: boolean;
|
||||
};
|
||||
/** Instance default from AUTH_DEFAULT_METHOD, used when nothing is remembered. */
|
||||
defaultMethod: AuthMethod;
|
||||
}
|
||||
|
||||
export function useAuthMethods() {
|
||||
@@ -26,6 +33,7 @@ export function useAuthMethods() {
|
||||
oidc: {
|
||||
enabled: false,
|
||||
},
|
||||
defaultMethod: DEFAULT_AUTH_METHOD,
|
||||
});
|
||||
const [isLoading, setIsLoading] = useState(true);
|
||||
|
||||
@@ -35,9 +43,9 @@ export function useAuthMethods() {
|
||||
|
||||
const loadAuthMethods = async () => {
|
||||
try {
|
||||
// Check SSO providers - use public endpoint since this is used on login page
|
||||
const providers = await apiRequest<any[]>('/sso/providers/public').catch(() => []);
|
||||
const applications = await apiRequest<any[]>('/sso/applications').catch(() => []);
|
||||
// Public endpoint: the login page calls it before anyone is signed in.
|
||||
const methods = await apiRequest<Partial<AuthMethods>>('/auth/methods');
|
||||
const providers = methods.sso?.providers ?? [];
|
||||
|
||||
setAuthMethods({
|
||||
emailPassword: true, // Always enabled
|
||||
@@ -50,8 +58,9 @@ export function useAuthMethods() {
|
||||
})),
|
||||
},
|
||||
oidc: {
|
||||
enabled: applications.length > 0,
|
||||
enabled: methods.oidc?.enabled === true,
|
||||
},
|
||||
defaultMethod: parseDefaultAuthMethod(methods.defaultMethod),
|
||||
});
|
||||
} catch (error) {
|
||||
// If we can't load auth methods, default to email/password only
|
||||
@@ -62,4 +71,4 @@ export function useAuthMethods() {
|
||||
};
|
||||
|
||||
return { authMethods, isLoading };
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,28 @@
|
||||
import { useCallback, useSyncExternalStore } from "react";
|
||||
import {
|
||||
DEFAULT_ORGANIZATIONS_VIEW,
|
||||
getOrganizationsView,
|
||||
setOrganizationsView,
|
||||
subscribeToOrganizationsViewChange,
|
||||
type OrganizationsViewMode,
|
||||
} from "@/lib/utils/organizations-view";
|
||||
|
||||
const getServerSnapshot = (): OrganizationsViewMode => DEFAULT_ORGANIZATIONS_VIEW;
|
||||
|
||||
/**
|
||||
* Cards or list on the Organizations page (#428). The choice is kept in
|
||||
* localStorage and shared across tabs.
|
||||
*/
|
||||
export function useOrganizationsView() {
|
||||
const view = useSyncExternalStore(
|
||||
subscribeToOrganizationsViewChange,
|
||||
getOrganizationsView,
|
||||
getServerSnapshot
|
||||
);
|
||||
|
||||
const setView = useCallback((next: OrganizationsViewMode) => {
|
||||
setOrganizationsView(next);
|
||||
}, []);
|
||||
|
||||
return { view, setView };
|
||||
}
|
||||
@@ -0,0 +1,65 @@
|
||||
import { describe, expect, it } from "bun:test";
|
||||
import {
|
||||
isOpenRegistration,
|
||||
isSignupClosed,
|
||||
isSignupRequest,
|
||||
signupClosedError,
|
||||
signupGuardPlugin,
|
||||
} from "./auth-signup-guard";
|
||||
|
||||
describe("signup guard", () => {
|
||||
it("only matches the email sign-up endpoint", () => {
|
||||
expect(isSignupRequest("/sign-up/email")).toBe(true);
|
||||
expect(isSignupRequest("/sign-in/email")).toBe(false);
|
||||
expect(isSignupRequest("/sign-up/email/extra")).toBe(false);
|
||||
expect(isSignupRequest(undefined)).toBe(false);
|
||||
});
|
||||
|
||||
it("opens registration only for the literal true", () => {
|
||||
expect(isOpenRegistration({ AUTH_ALLOW_SIGNUP: "true" } as NodeJS.ProcessEnv)).toBe(true);
|
||||
expect(isOpenRegistration({ AUTH_ALLOW_SIGNUP: "TRUE" } as NodeJS.ProcessEnv)).toBe(false);
|
||||
expect(isOpenRegistration({ AUTH_ALLOW_SIGNUP: "1" } as NodeJS.ProcessEnv)).toBe(false);
|
||||
expect(isOpenRegistration({} as NodeJS.ProcessEnv)).toBe(false);
|
||||
});
|
||||
|
||||
it("closes sign-up once a user exists unless registration is open", () => {
|
||||
expect(isSignupClosed({ userCount: 0, allowSignup: false })).toBe(false);
|
||||
expect(isSignupClosed({ userCount: 1, allowSignup: false })).toBe(true);
|
||||
expect(isSignupClosed({ userCount: 5, allowSignup: true })).toBe(false);
|
||||
});
|
||||
|
||||
it("answers 403 with a stable code", () => {
|
||||
const error = signupClosedError();
|
||||
expect(error.status).toBe("FORBIDDEN");
|
||||
expect(error.body?.code).toBe("SIGNUP_CLOSED");
|
||||
});
|
||||
|
||||
it("registers a before hook on the sign-up path that counts users", async () => {
|
||||
let counted = 0;
|
||||
const plugin = signupGuardPlugin({
|
||||
countUsers: async () => {
|
||||
counted += 1;
|
||||
return 1;
|
||||
},
|
||||
});
|
||||
const hook = plugin.hooks.before[0];
|
||||
expect(hook.matcher({ path: "/sign-up/email" } as any)).toBe(true);
|
||||
expect(hook.matcher({ path: "/sign-in/email" } as any)).toBe(false);
|
||||
|
||||
const previous = process.env.AUTH_ALLOW_SIGNUP;
|
||||
delete process.env.AUTH_ALLOW_SIGNUP;
|
||||
try {
|
||||
await expect(hook.handler({ path: "/sign-up/email", context: {} } as any)).rejects.toMatchObject({
|
||||
body: { code: "SIGNUP_CLOSED" },
|
||||
});
|
||||
expect(counted).toBe(1);
|
||||
|
||||
process.env.AUTH_ALLOW_SIGNUP = "true";
|
||||
await hook.handler({ path: "/sign-up/email", context: {} } as any);
|
||||
expect(counted).toBe(1);
|
||||
} finally {
|
||||
if (previous === undefined) delete process.env.AUTH_ALLOW_SIGNUP;
|
||||
else process.env.AUTH_ALLOW_SIGNUP = previous;
|
||||
}
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,68 @@
|
||||
import type { BetterAuthPlugin } from "better-auth";
|
||||
import { APIError, createAuthMiddleware } from "better-auth/api";
|
||||
|
||||
/**
|
||||
* Closes email and password sign-up once the first account exists.
|
||||
*
|
||||
* The signup page already redirects to the login page when a user exists,
|
||||
* but that check lived only in the page. The Better Auth endpoint behind
|
||||
* it, POST /api/auth/sign-up/email, accepted new accounts from anyone who
|
||||
* could reach the instance, which turned every "signed in users only" API
|
||||
* route into a public one. This plugin refuses the endpoint server side
|
||||
* once a user exists, unless the operator opts into open registration with
|
||||
* AUTH_ALLOW_SIGNUP=true.
|
||||
*
|
||||
* SSO sign-ins through a registered identity provider and header auth
|
||||
* provisioning (HEADER_AUTH_AUTO_PROVISION) are separate flows and are not
|
||||
* affected: the operator chose those providers.
|
||||
*/
|
||||
|
||||
/** Endpoint path (without the auth base path) of email sign-up. */
|
||||
export const SIGNUP_PATH = "/sign-up/email";
|
||||
|
||||
export function isSignupRequest(path: string | undefined): boolean {
|
||||
return path === SIGNUP_PATH;
|
||||
}
|
||||
|
||||
/** Reads the opt-in from the environment: only the literal "true" opens sign-up. */
|
||||
export function isOpenRegistration(env: NodeJS.ProcessEnv = process.env): boolean {
|
||||
return env.AUTH_ALLOW_SIGNUP === "true";
|
||||
}
|
||||
|
||||
/** True when a sign-up must be refused: an account exists and registration is closed. */
|
||||
export function isSignupClosed({
|
||||
userCount,
|
||||
allowSignup,
|
||||
}: {
|
||||
userCount: number;
|
||||
allowSignup: boolean;
|
||||
}): boolean {
|
||||
return userCount > 0 && !allowSignup;
|
||||
}
|
||||
|
||||
export function signupClosedError(): APIError {
|
||||
return new APIError("FORBIDDEN", {
|
||||
message:
|
||||
"Sign-up is closed: an account already exists on this instance. Set AUTH_ALLOW_SIGNUP=true to allow more accounts.",
|
||||
code: "SIGNUP_CLOSED",
|
||||
});
|
||||
}
|
||||
|
||||
export const signupGuardPlugin = (options: { countUsers: () => Promise<number> }) =>
|
||||
({
|
||||
id: "signup-guard",
|
||||
hooks: {
|
||||
before: [
|
||||
{
|
||||
matcher: (ctx) => isSignupRequest(ctx.path),
|
||||
handler: createAuthMiddleware(async () => {
|
||||
if (isOpenRegistration()) return;
|
||||
const userCount = await options.countUsers();
|
||||
if (isSignupClosed({ userCount, allowSignup: false })) {
|
||||
throw signupClosedError();
|
||||
}
|
||||
}),
|
||||
},
|
||||
],
|
||||
},
|
||||
}) satisfies BetterAuthPlugin;
|
||||
@@ -11,6 +11,7 @@ import { eq } from "drizzle-orm";
|
||||
import { withBase } from "./base-path";
|
||||
import { headerAuthPlugin } from "./auth-header-plugin";
|
||||
import { apiKeyGuardPlugin } from "./auth-api-key-guard";
|
||||
import { signupGuardPlugin } from "./auth-signup-guard";
|
||||
|
||||
/**
|
||||
* Extracts the origins implied by registered SSO identity providers.
|
||||
@@ -402,6 +403,18 @@ export const auth = betterAuth({
|
||||
// can never create, list or revoke keys. See auth-api-key-guard.ts.
|
||||
apiKeyGuardPlugin(),
|
||||
|
||||
// Refuses email sign-up once an account exists unless
|
||||
// AUTH_ALLOW_SIGNUP=true. The signup page only redirected; the endpoint
|
||||
// itself was open. See auth-signup-guard.ts.
|
||||
signupGuardPlugin({
|
||||
countUsers: async () => {
|
||||
const { db, users } = await import("./db");
|
||||
const { sql } = await import("drizzle-orm");
|
||||
const [row] = await db.select({ count: sql<number>`count(*)` }).from(users);
|
||||
return Number(row?.count ?? 0);
|
||||
},
|
||||
}),
|
||||
|
||||
// Header / forward authentication bridge. Exposes
|
||||
// POST /api/auth/sign-in/header so the middleware can mint a real
|
||||
// Better Auth session from trusted upstream headers (Authentik /
|
||||
|
||||
@@ -0,0 +1,106 @@
|
||||
/**
|
||||
* Regression test for the password reset CLI (#461).
|
||||
*
|
||||
* `manage-db reset-password` used a Drizzle client without a schema, so
|
||||
* `db.query.users` was undefined and every reset failed. Each case builds a
|
||||
* real database with `manage-db init` (the migrations in drizzle/), seeds it,
|
||||
* runs the reset through the source script and through the bundle the Docker
|
||||
* image ships, and checks the result with Better Auth.
|
||||
*
|
||||
* Other test files replace bun:sqlite with mock.module, which is process wide
|
||||
* in bun and depends on file order, so the suite runs in an isolated child
|
||||
* process (same harness as gitea-incremental-metadata.test.ts).
|
||||
*/
|
||||
|
||||
import { afterAll, beforeAll, describe, expect, test } from "bun:test";
|
||||
import { mkdtempSync, rmSync, symlinkSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join, resolve } from "node:path";
|
||||
|
||||
const CHILD_FLAG = "GM_RESET_PASSWORD_CLI_ISOLATED";
|
||||
const isChild = !!process.env[CHILD_FLAG];
|
||||
|
||||
if (!isChild) {
|
||||
test("reset-password CLI - isolated child suite", () => {
|
||||
const res = Bun.spawnSync({
|
||||
cmd: [process.execPath, "test", import.meta.path],
|
||||
env: { ...process.env, [CHILD_FLAG]: "1" },
|
||||
stdout: "pipe",
|
||||
stderr: "pipe",
|
||||
});
|
||||
if (res.exitCode !== 0) {
|
||||
console.error(res.stdout.toString());
|
||||
console.error(res.stderr.toString());
|
||||
}
|
||||
expect(res.exitCode).toBe(0);
|
||||
}, 60_000);
|
||||
}
|
||||
|
||||
const root = resolve(import.meta.dir, "../../..");
|
||||
let temporaryRoot = "";
|
||||
let bundle = "";
|
||||
|
||||
function run(cmd: string[], cwd: string) {
|
||||
const result = Bun.spawnSync({ cmd, cwd, stdout: "pipe", stderr: "pipe" });
|
||||
expect(result.exitCode, Buffer.from(result.stderr).toString()).toBe(0);
|
||||
return Buffer.from(result.stdout).toString();
|
||||
}
|
||||
|
||||
describe.skipIf(!isChild)("manage-db reset-password", () => {
|
||||
beforeAll(() => {
|
||||
temporaryRoot = mkdtempSync(join(tmpdir(), "gitea-mirror-reset-password-"));
|
||||
bundle = join(temporaryRoot, "manage-db.js");
|
||||
run([process.execPath, "build", "scripts/manage-db.ts", "--target=bun", `--outfile=${bundle}`], root);
|
||||
});
|
||||
afterAll(() => {
|
||||
if (temporaryRoot) rmSync(temporaryRoot, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
for (const bundled of [false, true]) {
|
||||
for (const existingCredential of [false, true]) {
|
||||
test(`${bundled ? "bundled" : "source"} script, ${existingCredential ? "existing" : "new"} credential`, async () => {
|
||||
const { Database } = await import("bun:sqlite");
|
||||
const { verifyPassword } = await import("better-auth/crypto");
|
||||
const script = bundled ? bundle : join(root, "scripts/manage-db.ts");
|
||||
|
||||
// A real schema: init runs the migrations from ./drizzle.
|
||||
const cwd = mkdtempSync(join(temporaryRoot, "case-"));
|
||||
symlinkSync(join(root, "drizzle"), join(cwd, "drizzle"));
|
||||
run([process.execPath, script, "init"], cwd);
|
||||
|
||||
const db = new Database(join(cwd, "data/gitea-mirror.db"));
|
||||
try {
|
||||
db.exec(`
|
||||
INSERT INTO users (id, name, email) VALUES ('target', 'Target', 'admin@example.com'), ('other', 'Other', 'other@example.com');
|
||||
INSERT INTO accounts (id, account_id, user_id, provider_id, password)
|
||||
VALUES ('oauth', 'remote', 'target', 'github', NULL), ('other-account', 'other', 'other', 'credential', 'unchanged');
|
||||
INSERT INTO sessions (id, token, user_id, expires_at) VALUES ('one', 'one', 'target', 0), ('two', 'two', 'target', 0), ('other', 'other', 'other', 0);
|
||||
`);
|
||||
if (existingCredential) {
|
||||
db.exec("INSERT INTO accounts (id, account_id, user_id, provider_id, password) VALUES ('credential', 'target', 'target', 'credential', 'old')");
|
||||
}
|
||||
|
||||
const output = run(
|
||||
[process.execPath, script, "reset-password", "--email=admin@example.com", "--new-password=new-secure-password"],
|
||||
cwd
|
||||
);
|
||||
expect(output).toContain("Password reset for admin@example.com");
|
||||
expect(output).toContain("Cleared 2 active session(s)");
|
||||
|
||||
const credentials = db
|
||||
.query("SELECT * FROM accounts WHERE user_id = 'target' AND provider_id = 'credential'")
|
||||
.all() as Array<{ password: string; issuer: string }>;
|
||||
expect(credentials).toHaveLength(1);
|
||||
expect(await verifyPassword({ hash: credentials[0].password, password: "new-secure-password" })).toBe(true);
|
||||
if (!existingCredential) expect(credentials[0].issuer).toBe("local:credential");
|
||||
expect(db.query("SELECT id FROM sessions").all()).toEqual([{ id: "other" }]);
|
||||
expect(db.query("SELECT password FROM accounts WHERE id = 'other-account'").get()).toEqual({ password: "unchanged" });
|
||||
expect(db.query("SELECT password FROM accounts WHERE id = 'oauth'").get()).toEqual({ password: null });
|
||||
expect(db.query("SELECT id FROM users").all()).toHaveLength(2);
|
||||
} finally {
|
||||
db.close();
|
||||
}
|
||||
}, 15_000);
|
||||
}
|
||||
}
|
||||
});
|
||||
@@ -9,10 +9,11 @@ mock.module("@/lib/helpers", () => ({
|
||||
}));
|
||||
|
||||
const mockMirrorGitHubReleasesToGitea = mock(() => Promise.resolve());
|
||||
const mockMirrorGitRepoIssuesToGitea = mock(() => Promise.resolve());
|
||||
const mockMirrorGitRepoPullRequestsToGitea = mock(() => Promise.resolve());
|
||||
const mockMirrorGitRepoIssuesToGitea = mock(() => Promise.resolve(undefined));
|
||||
const mockMirrorGitRepoPullRequestsToGitea = mock(() => Promise.resolve(undefined));
|
||||
const mockMirrorGitRepoLabelsToGitea = mock(() => Promise.resolve());
|
||||
const mockMirrorGitRepoMilestonesToGitea = mock(() => Promise.resolve());
|
||||
const mockMilestoneIds = new Map<string, number>([["v1.0", 7]]);
|
||||
const mockMirrorGitRepoMilestonesToGitea = mock(() => Promise.resolve(mockMilestoneIds));
|
||||
const mockSyncRepositoryMetadataToGitea = mock(() => Promise.resolve());
|
||||
const mockGetGiteaRepoOwnerAsync = mock(() => Promise.resolve("starred"));
|
||||
const mockCreatePreSyncBundleBackup = mock(() =>
|
||||
@@ -25,6 +26,9 @@ let mockShouldBlockSyncOnBackupFailure = true;
|
||||
// dbUpdateSetCalls so tests can assert on what got written (e.g. archived
|
||||
// repos keeping status "archived" after a Manual Sync).
|
||||
const dbUpdateSetCalls: any[] = [];
|
||||
// Rows the sync's atomic status claim gets back (#417): one row means this run
|
||||
// claimed the repository, an empty array means another mirror or sync owns it.
|
||||
let mockClaimRows: Array<{ id: string }> = [{ id: "repo-1" }];
|
||||
const mockDb = {
|
||||
insert: mock((table: any) => ({
|
||||
values: mock((data: any) => Promise.resolve({ insertedId: "mock-id" }))
|
||||
@@ -32,7 +36,13 @@ const mockDb = {
|
||||
update: mock(() => ({
|
||||
set: mock((data: any) => {
|
||||
dbUpdateSetCalls.push(data);
|
||||
return { where: mock(() => Promise.resolve()) };
|
||||
return {
|
||||
where: mock(() => {
|
||||
const result: any = Promise.resolve();
|
||||
result.returning = mock(() => Promise.resolve(mockClaimRows));
|
||||
return result;
|
||||
})
|
||||
};
|
||||
})
|
||||
}))
|
||||
};
|
||||
@@ -453,6 +463,7 @@ describe("Enhanced Gitea Operations", () => {
|
||||
mockHttpDelete.mockClear();
|
||||
mockHttpPatch.mockClear();
|
||||
dbUpdateSetCalls.length = 0;
|
||||
mockClaimRows = [{ id: "repo-1" }];
|
||||
mockCreatePreSyncBundleBackup.mockClear();
|
||||
mockCreatePreSyncBundleBackup.mockImplementation(() =>
|
||||
Promise.resolve({ bundlePath: "/tmp/mock.bundle" })
|
||||
@@ -671,6 +682,118 @@ describe("Enhanced Gitea Operations", () => {
|
||||
expect(mockMirrorGitHubReleasesToGitea).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
test("refuses to start when another run already owns the repository (#417)", async () => {
|
||||
// Cast: the fixture only sets the fields this path reads.
|
||||
const config = {
|
||||
userId: "user123",
|
||||
githubConfig: {
|
||||
username: "testuser",
|
||||
token: "github-token",
|
||||
privateRepositories: false,
|
||||
mirrorStarred: true,
|
||||
},
|
||||
giteaConfig: {
|
||||
url: "https://gitea.example.com",
|
||||
token: "encrypted-token",
|
||||
defaultOwner: "testuser",
|
||||
mirrorReleases: true,
|
||||
},
|
||||
} as unknown as Partial<Config>;
|
||||
|
||||
const repository = {
|
||||
id: "repo456",
|
||||
name: "mirror-repo",
|
||||
fullName: "user/mirror-repo",
|
||||
owner: "user",
|
||||
cloneUrl: "https://github.com/user/mirror-repo.git",
|
||||
isPrivate: false,
|
||||
isStarred: true,
|
||||
// The row is already "syncing" from another run; the claim loses.
|
||||
status: repoStatusEnum.parse("syncing"),
|
||||
visibility: "public",
|
||||
userId: "user123",
|
||||
createdAt: new Date(),
|
||||
updatedAt: new Date(),
|
||||
} as unknown as Repository;
|
||||
|
||||
// The conditional UPDATE matches no row: another mirror or sync owns it.
|
||||
mockClaimRows = [];
|
||||
|
||||
const result = await syncGiteaRepoEnhanced(
|
||||
{ config, repository },
|
||||
{
|
||||
getGiteaRepoOwnerAsync: mockGetGiteaRepoOwnerAsync,
|
||||
mirrorGitHubReleasesToGitea: mockMirrorGitHubReleasesToGitea,
|
||||
mirrorGitRepoIssuesToGitea: mockMirrorGitRepoIssuesToGitea,
|
||||
mirrorGitRepoPullRequestsToGitea: mockMirrorGitRepoPullRequestsToGitea,
|
||||
mirrorGitRepoLabelsToGitea: mockMirrorGitRepoLabelsToGitea,
|
||||
mirrorGitRepoMilestonesToGitea: mockMirrorGitRepoMilestonesToGitea,
|
||||
syncRepositoryMetadataToGitea: mockSyncRepositoryMetadataToGitea,
|
||||
}
|
||||
);
|
||||
|
||||
expect(result).toEqual({ skipped: true, reason: "already-in-progress" });
|
||||
// Nothing ran: no release or metadata mirroring, and the row was not
|
||||
// marked failed (it belongs to the run that owns it).
|
||||
expect(mockMirrorGitHubReleasesToGitea).not.toHaveBeenCalled();
|
||||
expect(mockSyncRepositoryMetadataToGitea).not.toHaveBeenCalled();
|
||||
expect(dbUpdateSetCalls.some((call) => call.status === "failed")).toBe(false);
|
||||
});
|
||||
|
||||
test("syncs when the caller already claimed the row (approve-sync)", async () => {
|
||||
// Cast: the fixture only sets the fields this path reads.
|
||||
const config = {
|
||||
userId: "user123",
|
||||
githubConfig: {
|
||||
username: "testuser",
|
||||
token: "github-token",
|
||||
privateRepositories: false,
|
||||
mirrorStarred: true,
|
||||
},
|
||||
giteaConfig: {
|
||||
url: "https://gitea.example.com",
|
||||
token: "encrypted-token",
|
||||
defaultOwner: "testuser",
|
||||
mirrorReleases: true,
|
||||
},
|
||||
} as unknown as Partial<Config>;
|
||||
|
||||
const repository = {
|
||||
id: "repo456",
|
||||
name: "mirror-repo",
|
||||
fullName: "user/mirror-repo",
|
||||
owner: "user",
|
||||
cloneUrl: "https://github.com/user/mirror-repo.git",
|
||||
isPrivate: false,
|
||||
isStarred: true,
|
||||
// approve-sync has already moved the row to "syncing" itself.
|
||||
status: repoStatusEnum.parse("syncing"),
|
||||
visibility: "public",
|
||||
userId: "user123",
|
||||
createdAt: new Date(),
|
||||
updatedAt: new Date(),
|
||||
} as unknown as Repository;
|
||||
|
||||
// Would refuse the claim, but alreadyClaimed skips it entirely.
|
||||
mockClaimRows = [];
|
||||
|
||||
const result = await syncGiteaRepoEnhanced(
|
||||
{ config, repository, alreadyClaimed: true },
|
||||
{
|
||||
getGiteaRepoOwnerAsync: mockGetGiteaRepoOwnerAsync,
|
||||
mirrorGitHubReleasesToGitea: mockMirrorGitHubReleasesToGitea,
|
||||
mirrorGitRepoIssuesToGitea: mockMirrorGitRepoIssuesToGitea,
|
||||
mirrorGitRepoPullRequestsToGitea: mockMirrorGitRepoPullRequestsToGitea,
|
||||
mirrorGitRepoLabelsToGitea: mockMirrorGitRepoLabelsToGitea,
|
||||
mirrorGitRepoMilestonesToGitea: mockMirrorGitRepoMilestonesToGitea,
|
||||
syncRepositoryMetadataToGitea: mockSyncRepositoryMetadataToGitea,
|
||||
}
|
||||
);
|
||||
|
||||
expect(result).toEqual({ success: true });
|
||||
expect(mockMirrorGitHubReleasesToGitea).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
test("should successfully sync a mirror repository", async () => {
|
||||
const config: Partial<Config> = {
|
||||
userId: "user123",
|
||||
@@ -1454,6 +1577,12 @@ describe("Enhanced Gitea Operations", () => {
|
||||
expect(mockMirrorGitRepoMilestonesToGitea).toHaveBeenCalledTimes(1);
|
||||
// Labels should be skipped because issues already import them
|
||||
expect(mockMirrorGitRepoLabelsToGitea).not.toHaveBeenCalled();
|
||||
// Milestones run first and their ids reach the issues and pull
|
||||
// requests passes, which assign each item to its milestone (#460).
|
||||
const issuesArgs = (mockMirrorGitRepoIssuesToGitea.mock.calls[0] as any[])[0];
|
||||
const pullsArgs = (mockMirrorGitRepoPullRequestsToGitea.mock.calls[0] as any[])[0];
|
||||
expect(issuesArgs.milestoneIds).toBe(mockMilestoneIds);
|
||||
expect(pullsArgs.milestoneIds).toBe(mockMilestoneIds);
|
||||
});
|
||||
|
||||
test("skips issues and PRs when metadata shows they were already synced", async () => {
|
||||
|
||||
+181
-61
@@ -7,7 +7,13 @@
|
||||
*/
|
||||
|
||||
import type { Config } from "@/types/config";
|
||||
import { getRepositorySource } from "@/lib/source-providers/kinds";
|
||||
import { isRateLimitError } from "@/lib/rate-limit-gate";
|
||||
import { persistMetadataPassProgress } from "./metadata-progress-store";
|
||||
import {
|
||||
getRepositorySource,
|
||||
SOURCE_PROVIDER_LABELS,
|
||||
} from "@/lib/source-providers/kinds";
|
||||
import { sourceConnectionFromSource } from "@/lib/source-providers";
|
||||
import {
|
||||
decryptSourceToken,
|
||||
findSourceForRepository,
|
||||
@@ -24,6 +30,7 @@ import { httpPost, httpGet, httpPatch, HttpError } from "./http-client";
|
||||
import { db, repositories } from "./db";
|
||||
import { eq } from "drizzle-orm";
|
||||
import { repoStatusEnum } from "@/types/Repository";
|
||||
import { claimRepositoryForInFlightWork } from "./utils/repo-status-claim";
|
||||
import { resolveMirrorOptionsForRepository } from "./utils/mirror-overrides";
|
||||
import {
|
||||
createPreSyncBundleBackup,
|
||||
@@ -35,7 +42,7 @@ import {
|
||||
strategyNeedsDetection,
|
||||
} from "./repo-backup";
|
||||
import { detectForcePush } from "./utils/force-push-detection";
|
||||
import { sanitizeRepoNameAlphaDashDot } from "./gitea";
|
||||
import { sanitizeRepoNameAlphaDashDot, type GiteaMilestoneIds } from "./gitea";
|
||||
import {
|
||||
parseRepositoryMetadataState,
|
||||
serializeRepositoryMetadataState,
|
||||
@@ -45,6 +52,12 @@ type SyncDependencies = {
|
||||
getGiteaRepoOwnerAsync: typeof import("./gitea")["getGiteaRepoOwnerAsync"];
|
||||
syncRepositoryMetadataToGitea: typeof import("./gitea")["syncRepositoryMetadataToGitea"];
|
||||
mirrorGitHubReleasesToGitea: typeof import("./gitea")["mirrorGitHubReleasesToGitea"];
|
||||
/**
|
||||
* Optional so the many existing test doubles of this dependency bag keep
|
||||
* type checking; the real module always carries it and the sync path falls
|
||||
* back to importing it when a double leaves it out.
|
||||
*/
|
||||
mirrorGiteaSourceReleasesToGitea?: typeof import("./gitea")["mirrorGiteaSourceReleasesToGitea"];
|
||||
mirrorGitRepoIssuesToGitea: typeof import("./gitea")["mirrorGitRepoIssuesToGitea"];
|
||||
mirrorGitRepoPullRequestsToGitea: typeof import("./gitea")["mirrorGitRepoPullRequestsToGitea"];
|
||||
mirrorGitRepoLabelsToGitea: typeof import("./gitea")["mirrorGitRepoLabelsToGitea"];
|
||||
@@ -444,11 +457,19 @@ export async function syncGiteaRepoEnhanced({
|
||||
config,
|
||||
repository,
|
||||
skipForcePushDetection,
|
||||
alreadyClaimed,
|
||||
}: {
|
||||
config: Partial<Config>;
|
||||
repository: Repository;
|
||||
/** When true, skip force-push detection and blocking (used by approve-sync). */
|
||||
skipForcePushDetection?: boolean;
|
||||
/**
|
||||
* Set by a caller that already claimed this row for the sync with its own
|
||||
* conditional UPDATE (approve-sync does, so the UI sees "syncing" right
|
||||
* away). Skips the claim below instead of refusing to run on the caller's
|
||||
* own "syncing" status.
|
||||
*/
|
||||
alreadyClaimed?: boolean;
|
||||
}, deps?: SyncDependencies): Promise<any> {
|
||||
try {
|
||||
if (!config.userId || !config.giteaConfig?.url || !config.giteaConfig?.token) {
|
||||
@@ -463,7 +484,8 @@ export async function syncGiteaRepoEnhanced({
|
||||
const sources = config.userId ? await listSources(config.userId) : [];
|
||||
const repoSource = findSourceForRepository(repository, sources);
|
||||
const repoSourceToken = repoSource ? decryptSourceToken(repoSource.token) : "";
|
||||
const repoIsGitHub = getRepositorySource(repository).provider === "github";
|
||||
const repoSourceKind = getRepositorySource(repository).provider;
|
||||
const repoIsGitHub = repoSourceKind === "github";
|
||||
|
||||
if (usesPushEngine(config)) {
|
||||
throw new Error("The configured destination is a push target; syncing goes through the push engine.");
|
||||
@@ -472,14 +494,30 @@ export async function syncGiteaRepoEnhanced({
|
||||
|
||||
console.log(`[Sync] Starting sync for repository ${repository.name}`);
|
||||
|
||||
// Mark repo as "syncing" in DB
|
||||
await db
|
||||
.update(repositories)
|
||||
.set({
|
||||
status: repoStatusEnum.parse("syncing"),
|
||||
updatedAt: new Date(),
|
||||
})
|
||||
.where(eq(repositories.id, repository.id!));
|
||||
// Mark repo as "syncing" in DB, but only while no other run owns the row.
|
||||
// This used to be an unconditional write: a sync would start on a
|
||||
// repository that was already mirroring or syncing, and the two passes
|
||||
// reconciled the same releases against the same destination at the same
|
||||
// time, which duplicated release assets (#417). A row left in an in-flight
|
||||
// status by a crash is taken over once it goes stale (two hours), and
|
||||
// resetStuckMirrorStatuses resets those rows to "failed" as well, so
|
||||
// nothing is locked out permanently.
|
||||
if (!alreadyClaimed) {
|
||||
const claimed = await claimRepositoryForInFlightWork({
|
||||
repositoryId: repository.id!,
|
||||
set: {
|
||||
status: repoStatusEnum.parse("syncing"),
|
||||
updatedAt: new Date(),
|
||||
},
|
||||
});
|
||||
|
||||
if (!claimed) {
|
||||
console.log(
|
||||
`[Sync] Skipping ${repository.name}: it is already being mirrored or synced by another run.`
|
||||
);
|
||||
return { skipped: true, reason: "already-in-progress" };
|
||||
}
|
||||
}
|
||||
|
||||
// Resolve sync target in a backward-compatible order:
|
||||
// 1) recorded mirroredLocation (actual historical mirror location)
|
||||
@@ -977,33 +1015,110 @@ export async function syncGiteaRepoEnhanced({
|
||||
const shouldMirrorMilestones = mirrorOptions.mirrorMilestones;
|
||||
|
||||
if (shouldMirrorReleases) {
|
||||
const octokit = ensureOctokit();
|
||||
if (!octokit) {
|
||||
console.warn(
|
||||
`[Sync] Skipping release mirroring for ${repository.name}: Missing GitHub token`
|
||||
);
|
||||
} else {
|
||||
try {
|
||||
await dependencies.mirrorGitHubReleasesToGitea({
|
||||
// GitHub reads releases through Octokit; Gitea and Forgejo sources
|
||||
// read them from their own instance with their own token (#440).
|
||||
// Any other source has no release listing and is skipped.
|
||||
try {
|
||||
let mirrored = false;
|
||||
|
||||
if (repoIsGitHub) {
|
||||
const octokit = ensureOctokit();
|
||||
if (!octokit) {
|
||||
console.warn(
|
||||
`[Sync] Skipping release mirroring for ${repository.name}: Missing GitHub token`
|
||||
);
|
||||
} else {
|
||||
await dependencies.mirrorGitHubReleasesToGitea({
|
||||
config,
|
||||
octokit,
|
||||
repository,
|
||||
sourceToken: repoSourceToken,
|
||||
giteaOwner: repoOwner,
|
||||
giteaRepoName: repoName,
|
||||
releaseLimit: mirrorOptions.releaseLimit,
|
||||
releaseAssetLimit: mirrorOptions.releaseAssetLimit,
|
||||
});
|
||||
mirrored = true;
|
||||
}
|
||||
} else if (repoSourceKind === "gitea" && repoSource) {
|
||||
const mirrorGiteaSourceReleases =
|
||||
dependencies.mirrorGiteaSourceReleasesToGitea ??
|
||||
(await import("./gitea")).mirrorGiteaSourceReleasesToGitea;
|
||||
await mirrorGiteaSourceReleases({
|
||||
config,
|
||||
octokit,
|
||||
repository,
|
||||
// Carries that instance's own decrypted token, so the release
|
||||
// listing and the asset downloads authenticate against it.
|
||||
connection: sourceConnectionFromSource(repoSource, {
|
||||
userId: config.userId,
|
||||
}),
|
||||
giteaOwner: repoOwner,
|
||||
giteaRepoName: repoName,
|
||||
releaseLimit: mirrorOptions.releaseLimit,
|
||||
releaseAssetLimit: mirrorOptions.releaseAssetLimit,
|
||||
});
|
||||
mirrored = true;
|
||||
} else {
|
||||
console.warn(
|
||||
`[Sync] Skipping release mirroring for ${repository.name}: ${
|
||||
SOURCE_PROVIDER_LABELS[repoSourceKind]
|
||||
} sources cannot list releases`
|
||||
);
|
||||
}
|
||||
|
||||
if (mirrored) {
|
||||
metadataState.components.releases = true;
|
||||
metadataUpdated = true;
|
||||
console.log(
|
||||
`[Sync] Mirrored releases for ${repository.name} after sync`
|
||||
);
|
||||
} catch (releaseError) {
|
||||
}
|
||||
} catch (releaseError) {
|
||||
// Rate limited: fail the sync so the scheduler restores the row and
|
||||
// retries after the reset instead of running the next component (#437).
|
||||
if (isRateLimitError(releaseError)) throw releaseError;
|
||||
console.error(
|
||||
`[Sync] Failed to mirror releases for ${repository.name}: ${
|
||||
releaseError instanceof Error
|
||||
? releaseError.message
|
||||
: String(releaseError)
|
||||
}`
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
// Milestones go first so the issues and pull requests passes below
|
||||
// can assign each item to its milestone (#460).
|
||||
let milestoneIds: GiteaMilestoneIds | undefined;
|
||||
if (shouldMirrorMilestones) {
|
||||
const octokit = ensureOctokit();
|
||||
if (!octokit) {
|
||||
console.warn(
|
||||
`[Sync] Skipping milestone mirroring for ${repository.name}: Missing GitHub token`
|
||||
);
|
||||
} else {
|
||||
try {
|
||||
milestoneIds = await dependencies.mirrorGitRepoMilestonesToGitea({
|
||||
config,
|
||||
octokit,
|
||||
repository,
|
||||
giteaOwner: repoOwner,
|
||||
giteaRepoName: repoName,
|
||||
});
|
||||
metadataState.components.milestones = true;
|
||||
metadataUpdated = true;
|
||||
console.log(
|
||||
`[Sync] Mirrored milestones for ${repository.name} after sync`
|
||||
);
|
||||
} catch (milestoneError) {
|
||||
// Rate limited: fail the sync so the scheduler restores the row and
|
||||
// retries after the reset instead of running the next component (#437).
|
||||
if (isRateLimitError(milestoneError)) throw milestoneError;
|
||||
console.error(
|
||||
`[Sync] Failed to mirror releases for ${repository.name}: ${
|
||||
releaseError instanceof Error
|
||||
? releaseError.message
|
||||
: String(releaseError)
|
||||
`[Sync] Failed to mirror milestones for ${repository.name}: ${
|
||||
milestoneError instanceof Error
|
||||
? milestoneError.message
|
||||
: String(milestoneError)
|
||||
}`
|
||||
);
|
||||
}
|
||||
@@ -1018,13 +1133,21 @@ export async function syncGiteaRepoEnhanced({
|
||||
);
|
||||
} else {
|
||||
try {
|
||||
await dependencies.mirrorGitRepoIssuesToGitea({
|
||||
const issuesCursor = await dependencies.mirrorGitRepoIssuesToGitea({
|
||||
config,
|
||||
octokit,
|
||||
milestoneIds,
|
||||
repository,
|
||||
giteaOwner: repoOwner,
|
||||
giteaRepoName: repoName,
|
||||
syncCursor: metadataState.syncCursors.issues,
|
||||
passProgress: metadataState.passProgress.issues,
|
||||
onPassProgress: async (progress) => {
|
||||
metadataUpdated = true;
|
||||
await persistMetadataPassProgress(repository.id, metadataState, "issues", progress);
|
||||
},
|
||||
});
|
||||
if (issuesCursor) metadataState.syncCursors.issues = issuesCursor;
|
||||
metadataState.components.issues = true;
|
||||
metadataState.components.labels = true;
|
||||
metadataUpdated = true;
|
||||
@@ -1032,6 +1155,9 @@ export async function syncGiteaRepoEnhanced({
|
||||
`[Sync] Mirrored issues for ${repository.name} after sync`
|
||||
);
|
||||
} catch (issueError) {
|
||||
// Rate limited: fail the sync so the scheduler restores the row and
|
||||
// retries after the reset instead of running the next component (#437).
|
||||
if (isRateLimitError(issueError)) throw issueError;
|
||||
console.error(
|
||||
`[Sync] Failed to mirror issues for ${repository.name}: ${
|
||||
issueError instanceof Error
|
||||
@@ -1051,19 +1177,30 @@ export async function syncGiteaRepoEnhanced({
|
||||
);
|
||||
} else {
|
||||
try {
|
||||
await dependencies.mirrorGitRepoPullRequestsToGitea({
|
||||
const pullRequestsCursor = await dependencies.mirrorGitRepoPullRequestsToGitea({
|
||||
config,
|
||||
octokit,
|
||||
milestoneIds,
|
||||
repository,
|
||||
giteaOwner: repoOwner,
|
||||
giteaRepoName: repoName,
|
||||
syncCursor: metadataState.syncCursors.pullRequests,
|
||||
passProgress: metadataState.passProgress.pullRequests,
|
||||
onPassProgress: async (progress) => {
|
||||
metadataUpdated = true;
|
||||
await persistMetadataPassProgress(repository.id, metadataState, "pullRequests", progress);
|
||||
},
|
||||
});
|
||||
if (pullRequestsCursor) metadataState.syncCursors.pullRequests = pullRequestsCursor;
|
||||
metadataState.components.pullRequests = true;
|
||||
metadataUpdated = true;
|
||||
console.log(
|
||||
`[Sync] Mirrored pull requests for ${repository.name} after sync`
|
||||
);
|
||||
} catch (prError) {
|
||||
// Rate limited: fail the sync so the scheduler restores the row and
|
||||
// retries after the reset instead of running the next component (#437).
|
||||
if (isRateLimitError(prError)) throw prError;
|
||||
console.error(
|
||||
`[Sync] Failed to mirror pull requests for ${repository.name}: ${
|
||||
prError instanceof Error ? prError.message : String(prError)
|
||||
@@ -1094,6 +1231,9 @@ export async function syncGiteaRepoEnhanced({
|
||||
`[Sync] Mirrored labels for ${repository.name} after sync`
|
||||
);
|
||||
} catch (labelError) {
|
||||
// Rate limited: fail the sync so the scheduler restores the row and
|
||||
// retries after the reset instead of running the next component (#437).
|
||||
if (isRateLimitError(labelError)) throw labelError;
|
||||
console.error(
|
||||
`[Sync] Failed to mirror labels for ${repository.name}: ${
|
||||
labelError instanceof Error
|
||||
@@ -1105,38 +1245,6 @@ export async function syncGiteaRepoEnhanced({
|
||||
}
|
||||
}
|
||||
|
||||
if (shouldMirrorMilestones) {
|
||||
const octokit = ensureOctokit();
|
||||
if (!octokit) {
|
||||
console.warn(
|
||||
`[Sync] Skipping milestone mirroring for ${repository.name}: Missing GitHub token`
|
||||
);
|
||||
} else {
|
||||
try {
|
||||
await dependencies.mirrorGitRepoMilestonesToGitea({
|
||||
config,
|
||||
octokit,
|
||||
repository,
|
||||
giteaOwner: repoOwner,
|
||||
giteaRepoName: repoName,
|
||||
});
|
||||
metadataState.components.milestones = true;
|
||||
metadataUpdated = true;
|
||||
console.log(
|
||||
`[Sync] Mirrored milestones for ${repository.name} after sync`
|
||||
);
|
||||
} catch (milestoneError) {
|
||||
console.error(
|
||||
`[Sync] Failed to mirror milestones for ${repository.name}: ${
|
||||
milestoneError instanceof Error
|
||||
? milestoneError.message
|
||||
: String(milestoneError)
|
||||
}`
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (metadataUpdated) {
|
||||
metadataState.lastSyncedAt = new Date().toISOString();
|
||||
}
|
||||
@@ -1229,7 +1337,17 @@ export async function syncGiteaRepoEnhanced({
|
||||
throw syncError;
|
||||
}
|
||||
} catch (error) {
|
||||
console.error(`[Sync] Error while syncing repository ${repository.name}:`, error);
|
||||
if (isRateLimitError(error)) {
|
||||
// Expected and handled: the scheduler puts the row back and retries
|
||||
// after the reset. One line instead of the whole error object.
|
||||
console.warn(
|
||||
`[Sync] Sync of ${repository.name} paused by the source rate limit: ${
|
||||
error instanceof Error ? error.message : String(error)
|
||||
}`
|
||||
);
|
||||
} else {
|
||||
console.error(`[Sync] Error while syncing repository ${repository.name}:`, error);
|
||||
}
|
||||
|
||||
// Update repo with error status
|
||||
await db
|
||||
@@ -1246,7 +1364,9 @@ export async function syncGiteaRepoEnhanced({
|
||||
userId: config.userId,
|
||||
repositoryId: repository.id,
|
||||
repositoryName: repository.name,
|
||||
message: `Failed to sync repository: ${repository.name}`,
|
||||
message: isRateLimitError(error)
|
||||
? `Sync paused by the source rate limit: ${repository.name}`
|
||||
: `Failed to sync repository: ${repository.name}`,
|
||||
details: error instanceof Error ? error.message : "Unknown error",
|
||||
status: "failed",
|
||||
});
|
||||
|
||||
@@ -0,0 +1,985 @@
|
||||
/**
|
||||
* Behavioural tests for incremental issue and pull request sync (#449).
|
||||
*
|
||||
* With a watermark from a completed pass, the GitHub listing is asked only
|
||||
* for what changed (`since`), and the per-issue comment listing and per-PR
|
||||
* detail, commits and files calls run only for the items that came back.
|
||||
* Without one, the pass is full and unchanged from before.
|
||||
*
|
||||
* The destination is a fake global fetch and GitHub is a fake Octokit. Other
|
||||
* test files replace @/lib/http-client and friends with mock.module, which is
|
||||
* process wide in bun and depends on file order, so the suites run in an
|
||||
* isolated child process (same harness as gitea-source-releases.test.ts).
|
||||
*/
|
||||
|
||||
import { afterEach, describe, expect, it, test } from "bun:test";
|
||||
import type { Config } from "@/types/config";
|
||||
import type { Repository } from "@/lib/db/schema";
|
||||
import type { MetadataPassProgress, MetadataSyncCursor } from "@/lib/metadata-state";
|
||||
|
||||
const CHILD_FLAG = "GM_INCREMENTAL_METADATA_ISOLATED";
|
||||
const isChild = !!process.env[CHILD_FLAG];
|
||||
|
||||
if (!isChild) {
|
||||
test("incremental metadata sync - isolated child suite", () => {
|
||||
const res = Bun.spawnSync({
|
||||
cmd: [process.execPath, "test", import.meta.path],
|
||||
env: { ...process.env, [CHILD_FLAG]: "1" },
|
||||
stdout: "pipe",
|
||||
stderr: "pipe",
|
||||
});
|
||||
if (res.exitCode !== 0) {
|
||||
console.error(res.stdout.toString());
|
||||
console.error(res.stderr.toString());
|
||||
}
|
||||
expect(res.exitCode).toBe(0);
|
||||
}, 60_000);
|
||||
}
|
||||
|
||||
const { mirrorGitRepoIssuesToGitea, mirrorGitRepoPullRequestsToGitea } = isChild
|
||||
? await import("@/lib/gitea")
|
||||
: ({} as typeof import("@/lib/gitea"));
|
||||
const { INCREMENTAL_SYNC_SAFETY_MARGIN_MS } = isChild
|
||||
? await import("@/lib/metadata-state")
|
||||
: ({} as typeof import("@/lib/metadata-state"));
|
||||
|
||||
const GITEA_URL = "https://gitea.example.com";
|
||||
const DEST_OWNER = "mirror-owner";
|
||||
const DEST_REPO = "demo";
|
||||
const DEST_API = `${GITEA_URL}/api/v1/repos/${DEST_OWNER}/${DEST_REPO}`;
|
||||
|
||||
const realFetch = globalThis.fetch;
|
||||
|
||||
afterEach(() => {
|
||||
globalThis.fetch = realFetch;
|
||||
});
|
||||
|
||||
const config = {
|
||||
userId: "user-1",
|
||||
githubConfig: { owner: "acme", token: "github-token" },
|
||||
giteaConfig: {
|
||||
url: GITEA_URL,
|
||||
token: "gitea-token",
|
||||
defaultOwner: DEST_OWNER,
|
||||
issueConcurrency: 1,
|
||||
pullRequestConcurrency: 1,
|
||||
},
|
||||
} as unknown as Partial<Config>;
|
||||
|
||||
const repository = {
|
||||
id: "repo-1",
|
||||
name: DEST_REPO,
|
||||
fullName: "acme/demo",
|
||||
owner: "acme",
|
||||
isStarred: false,
|
||||
mirrorOverrides: null,
|
||||
organization: null,
|
||||
} as unknown as Repository;
|
||||
|
||||
interface Recorded {
|
||||
method: string;
|
||||
url: string;
|
||||
body?: string;
|
||||
}
|
||||
|
||||
/** Destination Gitea with a fixed set of issues already present. */
|
||||
function fakeGitea(
|
||||
existingIssues: Array<{ number: number; title: string; milestone?: { id: number } | null }>
|
||||
) {
|
||||
const requests: Recorded[] = [];
|
||||
let nextNumber = 1000;
|
||||
|
||||
globalThis.fetch = (async (input: any, init?: any) => {
|
||||
const url = typeof input === "string" ? input : String(input);
|
||||
const method = String(init?.method ?? "GET").toUpperCase();
|
||||
const body = typeof init?.body === "string" ? init.body : undefined;
|
||||
requests.push({ method, url, body });
|
||||
|
||||
const json = (payload: unknown, status = 200) =>
|
||||
new Response(JSON.stringify(payload), {
|
||||
status,
|
||||
headers: { "content-type": "application/json" },
|
||||
});
|
||||
|
||||
if (method === "GET" && url === DEST_API) return json({ id: 1, name: DEST_REPO });
|
||||
if (method === "GET" && url === `${DEST_API}/labels`) return json([]);
|
||||
if (method === "POST" && url === `${DEST_API}/labels`) return json({ id: 7 }, 201);
|
||||
if (method === "GET" && url.startsWith(`${DEST_API}/issues?`)) {
|
||||
// The defensive recheck searches by marker; nothing to recover.
|
||||
if (url.includes("&q=")) return json([]);
|
||||
const page = Number(new URL(url).searchParams.get("page") ?? "1");
|
||||
return json(
|
||||
page === 1
|
||||
? existingIssues.map((issue) => ({ ...issue, body: "", state: "open" }))
|
||||
: []
|
||||
);
|
||||
}
|
||||
if (method === "POST" && url === `${DEST_API}/issues`) {
|
||||
nextNumber += 1;
|
||||
return json({ number: nextNumber, state: "open" }, 201);
|
||||
}
|
||||
if (method === "PATCH" && /\/issues\/\d+$/.test(url)) return json({});
|
||||
if (method === "PUT" && /\/issues\/\d+\/labels$/.test(url)) return json([]);
|
||||
if (method === "GET" && /\/issues\/\d+\/comments\?/.test(url)) return json([]);
|
||||
if (method === "POST" && /\/issues\/\d+\/comments$/.test(url)) return json({ id: 1 }, 201);
|
||||
|
||||
throw new Error(`unexpected request: ${method} ${url}`);
|
||||
}) as unknown as typeof fetch;
|
||||
|
||||
return {
|
||||
requests,
|
||||
createdTitles: () =>
|
||||
requests
|
||||
.filter((r) => r.method === "POST" && r.url === `${DEST_API}/issues`)
|
||||
.map((r) => JSON.parse(r.body!).title as string),
|
||||
createdBodies: () =>
|
||||
requests
|
||||
.filter((r) => r.method === "POST" && r.url === `${DEST_API}/issues`)
|
||||
.map((r) => JSON.parse(r.body!).body as string),
|
||||
};
|
||||
}
|
||||
|
||||
function ghIssue(number: number, extra: Record<string, unknown> = {}) {
|
||||
return {
|
||||
number,
|
||||
title: `Issue ${number}`,
|
||||
body: `body ${number}`,
|
||||
state: "open",
|
||||
html_url: `https://github.com/acme/demo/issues/${number}`,
|
||||
user: { login: "alice", html_url: "https://github.com/alice" },
|
||||
labels: [],
|
||||
assignees: [],
|
||||
created_at: `2026-01-0${number % 9 || 1}T00:00:00Z`,
|
||||
updated_at: "2026-09-23T00:00:00Z",
|
||||
...extra,
|
||||
};
|
||||
}
|
||||
|
||||
function ghPull(number: number, extra: Record<string, unknown> = {}) {
|
||||
return {
|
||||
number,
|
||||
title: `PR ${number}`,
|
||||
body: `pr body ${number}`,
|
||||
state: "open",
|
||||
html_url: `https://github.com/acme/demo/pull/${number}`,
|
||||
user: { login: "bob", html_url: "https://github.com/bob" },
|
||||
created_at: "2026-01-01T00:00:00Z",
|
||||
merged_at: null,
|
||||
base: { ref: "main" },
|
||||
head: { ref: `feature-${number}` },
|
||||
...extra,
|
||||
};
|
||||
}
|
||||
|
||||
interface GitHubData {
|
||||
/** What issues.listForRepo returns (issues and PR entries). */
|
||||
issueListing: any[];
|
||||
/** What pulls.list returns. */
|
||||
pullListing?: any[];
|
||||
/** When set, pulls.get throws for these numbers. */
|
||||
failDetailFor?: number[];
|
||||
/** GitHub refuses the comment listing of these issues with a rate limit. */
|
||||
rateLimitCommentsFor?: number[];
|
||||
/** GitHub refuses pulls.get for these numbers with a rate limit. */
|
||||
rateLimitDetailFor?: number[];
|
||||
}
|
||||
|
||||
function rateLimitRefusal() {
|
||||
return Object.assign(new Error("API rate limit exceeded for user ID 1."), { status: 403 });
|
||||
}
|
||||
|
||||
/** Fake Octokit that records every call made through it. */
|
||||
function fakeOctokit(data: GitHubData) {
|
||||
const calls: Array<{ endpoint: string; params: any }> = [];
|
||||
|
||||
const issues = {
|
||||
listForRepo: async (params: any) => {
|
||||
calls.push({ endpoint: "issues.listForRepo", params });
|
||||
return { data: data.issueListing };
|
||||
},
|
||||
listComments: async (params: any) => {
|
||||
calls.push({ endpoint: "issues.listComments", params });
|
||||
if (data.rateLimitCommentsFor?.includes(params.issue_number)) throw rateLimitRefusal();
|
||||
return { data: [] };
|
||||
},
|
||||
};
|
||||
const pulls = {
|
||||
list: async (params: any) => {
|
||||
calls.push({ endpoint: "pulls.list", params });
|
||||
return { data: data.pullListing ?? [] };
|
||||
},
|
||||
get: async (params: any) => {
|
||||
calls.push({ endpoint: "pulls.get", params });
|
||||
if (data.rateLimitDetailFor?.includes(params.pull_number)) throw rateLimitRefusal();
|
||||
if (data.failDetailFor?.includes(params.pull_number)) {
|
||||
throw new Error("detail unavailable");
|
||||
}
|
||||
return {
|
||||
data: {
|
||||
...ghPull(params.pull_number),
|
||||
additions: 1,
|
||||
deletions: 0,
|
||||
changed_files: 1,
|
||||
merged_by: null,
|
||||
},
|
||||
};
|
||||
},
|
||||
listCommits: async (params: any) => {
|
||||
calls.push({ endpoint: "pulls.listCommits", params });
|
||||
return { data: [] };
|
||||
},
|
||||
listFiles: async (params: any) => {
|
||||
calls.push({ endpoint: "pulls.listFiles", params });
|
||||
return { data: [] };
|
||||
},
|
||||
};
|
||||
|
||||
const octokit = {
|
||||
rest: { issues, pulls },
|
||||
paginate: async (fn: any, params: any, map?: (res: any) => any) => {
|
||||
const res = await fn(params);
|
||||
return map ? map(res) : res.data;
|
||||
},
|
||||
};
|
||||
|
||||
return {
|
||||
octokit: octokit as any,
|
||||
calls,
|
||||
callsTo: (endpoint: string) => calls.filter((c) => c.endpoint === endpoint),
|
||||
};
|
||||
}
|
||||
|
||||
function recentCursor(): MetadataSyncCursor {
|
||||
return {
|
||||
lastPassStartedAt: new Date(Date.now() - 60 * 60 * 1000).toISOString(),
|
||||
lastFullPassStartedAt: new Date(Date.now() - 2 * 24 * 60 * 60 * 1000).toISOString(),
|
||||
};
|
||||
}
|
||||
|
||||
function expectedSince(cursor: MetadataSyncCursor) {
|
||||
return new Date(
|
||||
Date.parse(cursor.lastPassStartedAt) - INCREMENTAL_SYNC_SAFETY_MARGIN_MS
|
||||
).toISOString();
|
||||
}
|
||||
|
||||
describe.skipIf(!isChild)("mirrorGitRepoIssuesToGitea incremental sync", () => {
|
||||
it("runs a full pass without a watermark and records one", async () => {
|
||||
fakeGitea([]);
|
||||
const github = fakeOctokit({
|
||||
issueListing: [ghIssue(1), ghIssue(2), ghPull(3, { pull_request: {} })],
|
||||
});
|
||||
|
||||
const before = Date.now();
|
||||
const cursor = await mirrorGitRepoIssuesToGitea({
|
||||
config,
|
||||
octokit: github.octokit,
|
||||
repository,
|
||||
giteaOwner: DEST_OWNER,
|
||||
giteaRepoName: DEST_REPO,
|
||||
});
|
||||
const after = Date.now();
|
||||
|
||||
const [listing] = github.callsTo("issues.listForRepo");
|
||||
expect(listing.params.since).toBeUndefined();
|
||||
expect(listing.params.sort).toBe("created");
|
||||
expect(listing.params.direction).toBe("asc");
|
||||
expect(github.callsTo("issues.listComments").map((c) => c.params.issue_number)).toEqual([1, 2]);
|
||||
|
||||
expect(cursor).toBeDefined();
|
||||
const started = Date.parse(cursor!.lastPassStartedAt);
|
||||
expect(started).toBeGreaterThanOrEqual(before);
|
||||
expect(started).toBeLessThanOrEqual(after);
|
||||
expect(cursor!.lastFullPassStartedAt).toBe(cursor!.lastPassStartedAt);
|
||||
});
|
||||
|
||||
it("lists with since and only reconciles the returned issues", async () => {
|
||||
fakeGitea([
|
||||
{ number: 11, title: "[GH-ISSUE #1] Issue 1" },
|
||||
{ number: 12, title: "[GH-ISSUE #2] Issue 2" },
|
||||
{ number: 13, title: "[GH-ISSUE #3] Issue 3" },
|
||||
]);
|
||||
const github = fakeOctokit({ issueListing: [ghIssue(2)] });
|
||||
const previous = recentCursor();
|
||||
|
||||
const cursor = await mirrorGitRepoIssuesToGitea({
|
||||
config,
|
||||
octokit: github.octokit,
|
||||
repository,
|
||||
giteaOwner: DEST_OWNER,
|
||||
giteaRepoName: DEST_REPO,
|
||||
syncCursor: previous,
|
||||
});
|
||||
|
||||
const listings = github.callsTo("issues.listForRepo");
|
||||
expect(listings).toHaveLength(1);
|
||||
expect(listings[0].params.since).toBe(expectedSince(previous));
|
||||
expect(listings[0].params.sort).toBe("updated");
|
||||
expect(github.callsTo("issues.listComments").map((c) => c.params.issue_number)).toEqual([2]);
|
||||
|
||||
// The watermark moves to this listing; the last full pass is kept.
|
||||
expect(cursor).toBeDefined();
|
||||
expect(Date.parse(cursor!.lastPassStartedAt)).toBeGreaterThan(
|
||||
Date.parse(previous.lastPassStartedAt)
|
||||
);
|
||||
expect(cursor!.lastFullPassStartedAt).toBe(previous.lastFullPassStartedAt);
|
||||
});
|
||||
|
||||
it("creates new issues from an incremental listing in creation order", async () => {
|
||||
const gitea = fakeGitea([{ number: 11, title: "[GH-ISSUE #1] Issue 1" }]);
|
||||
// GitHub returns them by updated_at: #5 changed before #4.
|
||||
const github = fakeOctokit({
|
||||
issueListing: [
|
||||
ghIssue(5, { created_at: "2026-09-22T10:00:00Z", updated_at: "2026-09-23T01:00:00Z" }),
|
||||
ghIssue(4, { created_at: "2026-09-22T09:00:00Z", updated_at: "2026-09-23T02:00:00Z" }),
|
||||
],
|
||||
});
|
||||
|
||||
await mirrorGitRepoIssuesToGitea({
|
||||
config,
|
||||
octokit: github.octokit,
|
||||
repository,
|
||||
giteaOwner: DEST_OWNER,
|
||||
giteaRepoName: DEST_REPO,
|
||||
syncCursor: recentCursor(),
|
||||
});
|
||||
|
||||
expect(gitea.createdTitles()).toEqual(["[GH-ISSUE #4] Issue 4", "[GH-ISSUE #5] Issue 5"]);
|
||||
});
|
||||
|
||||
it("falls back to a full pass when the destination has no mirrored issues", async () => {
|
||||
fakeGitea([]);
|
||||
const github = fakeOctokit({ issueListing: [ghIssue(1)] });
|
||||
const previous = recentCursor();
|
||||
|
||||
const cursor = await mirrorGitRepoIssuesToGitea({
|
||||
config,
|
||||
octokit: github.octokit,
|
||||
repository,
|
||||
giteaOwner: DEST_OWNER,
|
||||
giteaRepoName: DEST_REPO,
|
||||
syncCursor: previous,
|
||||
});
|
||||
|
||||
const listings = github.callsTo("issues.listForRepo");
|
||||
expect(listings).toHaveLength(2);
|
||||
expect(listings[0].params.since).toBe(expectedSince(previous));
|
||||
expect(listings[1].params.since).toBeUndefined();
|
||||
expect(cursor!.lastFullPassStartedAt).toBe(cursor!.lastPassStartedAt);
|
||||
});
|
||||
|
||||
it("runs a full pass when the last full pass is older than 7 days", async () => {
|
||||
fakeGitea([{ number: 11, title: "[GH-ISSUE #1] Issue 1" }]);
|
||||
const github = fakeOctokit({ issueListing: [ghIssue(1), ghIssue(2)] });
|
||||
|
||||
await mirrorGitRepoIssuesToGitea({
|
||||
config,
|
||||
octokit: github.octokit,
|
||||
repository,
|
||||
giteaOwner: DEST_OWNER,
|
||||
giteaRepoName: DEST_REPO,
|
||||
syncCursor: {
|
||||
lastPassStartedAt: new Date(Date.now() - 60 * 60 * 1000).toISOString(),
|
||||
lastFullPassStartedAt: new Date(Date.now() - 8 * 24 * 60 * 60 * 1000).toISOString(),
|
||||
},
|
||||
});
|
||||
|
||||
const listings = github.callsTo("issues.listForRepo");
|
||||
expect(listings).toHaveLength(1);
|
||||
expect(listings[0].params.since).toBeUndefined();
|
||||
// #1 is in Gitea and has not changed since the last pass, so the full
|
||||
// pass does not list its comments again; #2 is new.
|
||||
expect(github.callsTo("issues.listComments").map((c) => c.params.issue_number)).toEqual([2]);
|
||||
});
|
||||
});
|
||||
|
||||
describe.skipIf(!isChild)("full passes that revisit unchanged items (#449 follow-up)", () => {
|
||||
const weekOldFullPass = (): MetadataSyncCursor => ({
|
||||
lastPassStartedAt: new Date(Date.now() - 60 * 60 * 1000).toISOString(),
|
||||
lastFullPassStartedAt: new Date(Date.now() - 8 * 24 * 60 * 60 * 1000).toISOString(),
|
||||
});
|
||||
const recently = () => new Date(Date.now() - 5 * 60 * 1000).toISOString();
|
||||
|
||||
it("only lists comments for issues that changed or are missing", async () => {
|
||||
const gitea = fakeGitea([
|
||||
{ number: 11, title: "[GH-ISSUE #1] Issue 1" },
|
||||
{ number: 12, title: "[GH-ISSUE #2] Issue 2" },
|
||||
]);
|
||||
const github = fakeOctokit({
|
||||
issueListing: [ghIssue(1), ghIssue(2, { updated_at: recently() }), ghIssue(3)],
|
||||
});
|
||||
|
||||
const cursor = await mirrorGitRepoIssuesToGitea({
|
||||
config,
|
||||
octokit: github.octokit,
|
||||
repository,
|
||||
giteaOwner: DEST_OWNER,
|
||||
giteaRepoName: DEST_REPO,
|
||||
syncCursor: weekOldFullPass(),
|
||||
});
|
||||
|
||||
expect(github.callsTo("issues.listComments").map((c) => c.params.issue_number)).toEqual([2, 3]);
|
||||
// The unchanged issue is still brought in line from the listing.
|
||||
expect(
|
||||
gitea.requests.some((r) => r.method === "PATCH" && r.url === `${DEST_API}/issues/11`)
|
||||
).toBe(true);
|
||||
expect(gitea.createdTitles()).toEqual(["[GH-ISSUE #3] Issue 3"]);
|
||||
expect(cursor!.lastFullPassStartedAt).toBe(cursor!.lastPassStartedAt);
|
||||
});
|
||||
|
||||
it("leaves an unchanged issue alone when the Gitea copy already matches", async () => {
|
||||
const gitea = fakeGitea([]);
|
||||
const issue = ghIssue(1);
|
||||
const matching = {
|
||||
number: 11,
|
||||
title: "[GH-ISSUE #1] Issue 1",
|
||||
body:
|
||||
`Originally created by @alice on GitHub (${new Intl.DateTimeFormat("en-US", {
|
||||
year: "numeric",
|
||||
month: "short",
|
||||
day: "numeric",
|
||||
}).format(new Date(issue.created_at))}).\n` +
|
||||
`Original GitHub issue: ${issue.html_url}\n\n${issue.body}`,
|
||||
state: "open",
|
||||
labels: [],
|
||||
};
|
||||
// Serve an exact copy by overriding the listing response.
|
||||
globalThis.fetch = ((original) =>
|
||||
(async (input: any, init?: any) => {
|
||||
const url = typeof input === "string" ? input : String(input);
|
||||
const method = String(init?.method ?? "GET").toUpperCase();
|
||||
if (method === "GET" && url.startsWith(`${DEST_API}/issues?`) && !url.includes("&q=")) {
|
||||
const page = Number(new URL(url).searchParams.get("page") ?? "1");
|
||||
return new Response(JSON.stringify(page === 1 ? [matching] : []), {
|
||||
status: 200,
|
||||
headers: { "content-type": "application/json" },
|
||||
});
|
||||
}
|
||||
return original(input, init);
|
||||
}) as unknown as typeof fetch)(globalThis.fetch);
|
||||
const github = fakeOctokit({ issueListing: [issue] });
|
||||
|
||||
await mirrorGitRepoIssuesToGitea({
|
||||
config,
|
||||
octokit: github.octokit,
|
||||
repository,
|
||||
giteaOwner: DEST_OWNER,
|
||||
giteaRepoName: DEST_REPO,
|
||||
syncCursor: weekOldFullPass(),
|
||||
});
|
||||
|
||||
expect(github.callsTo("issues.listComments")).toHaveLength(0);
|
||||
// Nothing to write either: no edit, no label reset, no create.
|
||||
expect(gitea.requests.filter((r) => r.method !== "GET")).toEqual([]);
|
||||
});
|
||||
|
||||
it("skips the detail, commits and files calls for unchanged pull requests", async () => {
|
||||
fakeGitea([
|
||||
{ number: 21, title: "[PR #1] PR 1" },
|
||||
{ number: 22, title: "[PR #2] PR 2" },
|
||||
]);
|
||||
const github = fakeOctokit({
|
||||
issueListing: [],
|
||||
pullListing: [
|
||||
ghPull(1, { updated_at: "2026-09-01T00:00:00Z" }),
|
||||
ghPull(2, { updated_at: recently() }),
|
||||
ghPull(3, { updated_at: "2026-09-01T00:00:00Z" }),
|
||||
],
|
||||
});
|
||||
|
||||
const cursor = await mirrorGitRepoPullRequestsToGitea({
|
||||
config,
|
||||
octokit: github.octokit,
|
||||
repository,
|
||||
giteaOwner: DEST_OWNER,
|
||||
giteaRepoName: DEST_REPO,
|
||||
syncCursor: weekOldFullPass(),
|
||||
});
|
||||
|
||||
// #1 is unchanged and mirrored; #2 changed; #3 is missing in Gitea.
|
||||
expect(github.callsTo("pulls.get").map((c) => c.params.pull_number)).toEqual([2, 3]);
|
||||
expect(github.callsTo("pulls.listCommits")).toHaveLength(2);
|
||||
expect(github.callsTo("pulls.listFiles")).toHaveLength(2);
|
||||
expect(cursor!.lastFullPassStartedAt).toBe(cursor!.lastPassStartedAt);
|
||||
});
|
||||
|
||||
it("does not treat anything as unchanged on the first pass, when there is no watermark", async () => {
|
||||
fakeGitea([{ number: 21, title: "[PR #1] PR 1" }]);
|
||||
const github = fakeOctokit({
|
||||
issueListing: [],
|
||||
pullListing: [ghPull(1, { updated_at: "2026-09-01T00:00:00Z" })],
|
||||
});
|
||||
|
||||
await mirrorGitRepoPullRequestsToGitea({
|
||||
config,
|
||||
octokit: github.octokit,
|
||||
repository,
|
||||
giteaOwner: DEST_OWNER,
|
||||
giteaRepoName: DEST_REPO,
|
||||
});
|
||||
|
||||
expect(github.callsTo("pulls.get").map((c) => c.params.pull_number)).toEqual([1]);
|
||||
});
|
||||
});
|
||||
|
||||
describe.skipIf(!isChild)("passes stopped by the rate limit resume (#449 follow-up)", () => {
|
||||
it("keeps the finished issues and continues from there on the next run", async () => {
|
||||
fakeGitea([]);
|
||||
const listing = [ghIssue(1), ghIssue(2), ghIssue(3), ghIssue(4)];
|
||||
const saved: Array<MetadataPassProgress | undefined> = [];
|
||||
|
||||
const first = fakeOctokit({ issueListing: listing, rateLimitCommentsFor: [3] });
|
||||
let caught: unknown;
|
||||
try {
|
||||
await mirrorGitRepoIssuesToGitea({
|
||||
config,
|
||||
octokit: first.octokit,
|
||||
repository,
|
||||
giteaOwner: DEST_OWNER,
|
||||
giteaRepoName: DEST_REPO,
|
||||
onPassProgress: (progress) => void saved.push(progress),
|
||||
});
|
||||
} catch (error) {
|
||||
caught = error;
|
||||
}
|
||||
expect(String(caught)).toContain("rate limit");
|
||||
// #3 was refused and #4 never started.
|
||||
expect(first.callsTo("issues.listComments").map((c) => c.params.issue_number)).toEqual([1, 2, 3]);
|
||||
expect(saved).toHaveLength(1);
|
||||
const progress = saved[0]!;
|
||||
expect(progress.mode).toBe("full");
|
||||
expect(progress.done).toEqual([[1, 2]]);
|
||||
|
||||
// Next run, budget back: #1 and #2 are not fetched again.
|
||||
fakeGitea([
|
||||
{ number: 11, title: "[GH-ISSUE #1] Issue 1" },
|
||||
{ number: 12, title: "[GH-ISSUE #2] Issue 2" },
|
||||
{ number: 13, title: "[GH-ISSUE #3] Issue 3" },
|
||||
]);
|
||||
const second = fakeOctokit({ issueListing: listing });
|
||||
const cursor = await mirrorGitRepoIssuesToGitea({
|
||||
config,
|
||||
octokit: second.octokit,
|
||||
repository,
|
||||
giteaOwner: DEST_OWNER,
|
||||
giteaRepoName: DEST_REPO,
|
||||
passProgress: progress,
|
||||
onPassProgress: (next) => void saved.push(next),
|
||||
});
|
||||
|
||||
const [relisting] = second.callsTo("issues.listForRepo");
|
||||
expect(relisting.params.since).toBeUndefined();
|
||||
expect(second.callsTo("issues.listComments").map((c) => c.params.issue_number)).toEqual([3, 4]);
|
||||
// Completed: the progress is cleared and the watermark is the start of
|
||||
// the first attempt, so nothing changed in between is missed.
|
||||
expect(saved[saved.length - 1]).toBeUndefined();
|
||||
expect(cursor!.lastPassStartedAt).toBe(progress.startedAt);
|
||||
expect(cursor!.lastFullPassStartedAt).toBe(progress.startedAt);
|
||||
});
|
||||
|
||||
it("redoes a finished issue that changed after the interrupted pass started", async () => {
|
||||
fakeGitea([
|
||||
{ number: 11, title: "[GH-ISSUE #1] Issue 1" },
|
||||
{ number: 12, title: "[GH-ISSUE #2] Issue 2" },
|
||||
]);
|
||||
const startedAt = new Date(Date.now() - 2 * 60 * 60 * 1000).toISOString();
|
||||
const github = fakeOctokit({
|
||||
issueListing: [ghIssue(1), ghIssue(2, { updated_at: new Date().toISOString() })],
|
||||
});
|
||||
|
||||
await mirrorGitRepoIssuesToGitea({
|
||||
config,
|
||||
octokit: github.octokit,
|
||||
repository,
|
||||
giteaOwner: DEST_OWNER,
|
||||
giteaRepoName: DEST_REPO,
|
||||
passProgress: { mode: "full", startedAt, done: [[1, 2]] },
|
||||
});
|
||||
|
||||
expect(github.callsTo("issues.listComments").map((c) => c.params.issue_number)).toEqual([2]);
|
||||
});
|
||||
|
||||
it("resumes an incremental pass with the same since", async () => {
|
||||
fakeGitea([{ number: 11, title: "[GH-ISSUE #1] Issue 1" }]);
|
||||
const since = new Date(Date.now() - 3 * 60 * 60 * 1000).toISOString();
|
||||
const startedAt = new Date(Date.now() - 2 * 60 * 60 * 1000).toISOString();
|
||||
const github = fakeOctokit({ issueListing: [ghIssue(1), ghIssue(2)] });
|
||||
|
||||
const cursor = await mirrorGitRepoIssuesToGitea({
|
||||
config,
|
||||
octokit: github.octokit,
|
||||
repository,
|
||||
giteaOwner: DEST_OWNER,
|
||||
giteaRepoName: DEST_REPO,
|
||||
syncCursor: recentCursor(),
|
||||
passProgress: { mode: "incremental", since, startedAt, done: [[1, 1]] },
|
||||
});
|
||||
|
||||
const [listing] = github.callsTo("issues.listForRepo");
|
||||
expect(listing.params.since).toBe(since);
|
||||
expect(github.callsTo("issues.listComments").map((c) => c.params.issue_number)).toEqual([2]);
|
||||
expect(cursor!.lastPassStartedAt).toBe(startedAt);
|
||||
});
|
||||
|
||||
it("keeps the finished pull requests when the detail call is refused", async () => {
|
||||
fakeGitea([]);
|
||||
const pullListing = [ghPull(1), ghPull(2), ghPull(3)].map((pr) => ({
|
||||
...pr,
|
||||
updated_at: "2026-09-01T00:00:00Z",
|
||||
}));
|
||||
const saved: Array<MetadataPassProgress | undefined> = [];
|
||||
const first = fakeOctokit({ issueListing: [], pullListing, rateLimitDetailFor: [2] });
|
||||
|
||||
await expect(
|
||||
mirrorGitRepoPullRequestsToGitea({
|
||||
config,
|
||||
octokit: first.octokit,
|
||||
repository,
|
||||
giteaOwner: DEST_OWNER,
|
||||
giteaRepoName: DEST_REPO,
|
||||
onPassProgress: (progress) => void saved.push(progress),
|
||||
})
|
||||
).rejects.toThrow(/rate limit/i);
|
||||
expect(saved[0]!.done).toEqual([[1, 1]]);
|
||||
|
||||
fakeGitea([{ number: 21, title: "[PR #1] PR 1" }]);
|
||||
const second = fakeOctokit({ issueListing: [], pullListing });
|
||||
await mirrorGitRepoPullRequestsToGitea({
|
||||
config,
|
||||
octokit: second.octokit,
|
||||
repository,
|
||||
giteaOwner: DEST_OWNER,
|
||||
giteaRepoName: DEST_REPO,
|
||||
passProgress: saved[0],
|
||||
});
|
||||
expect(second.callsTo("pulls.get").map((c) => c.params.pull_number)).toEqual([2, 3]);
|
||||
});
|
||||
});
|
||||
|
||||
describe.skipIf(!isChild)("mirrorGitRepoPullRequestsToGitea incremental sync", () => {
|
||||
it("runs a full pass with pulls.list without a watermark", async () => {
|
||||
fakeGitea([]);
|
||||
const github = fakeOctokit({
|
||||
issueListing: [],
|
||||
pullListing: [ghPull(1), ghPull(2)],
|
||||
});
|
||||
|
||||
const cursor = await mirrorGitRepoPullRequestsToGitea({
|
||||
config,
|
||||
octokit: github.octokit,
|
||||
repository,
|
||||
giteaOwner: DEST_OWNER,
|
||||
giteaRepoName: DEST_REPO,
|
||||
});
|
||||
|
||||
expect(github.callsTo("pulls.list")).toHaveLength(1);
|
||||
expect(github.callsTo("issues.listForRepo")).toHaveLength(0);
|
||||
expect(github.callsTo("pulls.get").map((c) => c.params.pull_number)).toEqual([1, 2]);
|
||||
expect(github.callsTo("pulls.listCommits")).toHaveLength(2);
|
||||
expect(github.callsTo("pulls.listFiles")).toHaveLength(2);
|
||||
expect(cursor).toBeDefined();
|
||||
expect(cursor!.lastFullPassStartedAt).toBe(cursor!.lastPassStartedAt);
|
||||
});
|
||||
|
||||
it("lists changed pull requests through the issues endpoint with since", async () => {
|
||||
const gitea = fakeGitea([
|
||||
{ number: 21, title: "[PR #1] PR 1" },
|
||||
{ number: 22, title: "[PR #2] PR 2" },
|
||||
{ number: 23, title: "[PR #3] PR 3" },
|
||||
]);
|
||||
const github = fakeOctokit({
|
||||
// Issue #9 changed too; it is not a pull request and is skipped here.
|
||||
issueListing: [
|
||||
ghIssue(9),
|
||||
{ ...ghIssue(3), title: "PR 3", pull_request: { merged_at: null } },
|
||||
{ ...ghIssue(4), title: "PR 4", pull_request: { merged_at: null } },
|
||||
],
|
||||
});
|
||||
const previous = recentCursor();
|
||||
|
||||
const cursor = await mirrorGitRepoPullRequestsToGitea({
|
||||
config,
|
||||
octokit: github.octokit,
|
||||
repository,
|
||||
giteaOwner: DEST_OWNER,
|
||||
giteaRepoName: DEST_REPO,
|
||||
syncCursor: previous,
|
||||
});
|
||||
|
||||
expect(github.callsTo("pulls.list")).toHaveLength(0);
|
||||
const [listing] = github.callsTo("issues.listForRepo");
|
||||
expect(listing.params.since).toBe(expectedSince(previous));
|
||||
expect(listing.params.sort).toBe("updated");
|
||||
|
||||
expect(github.callsTo("pulls.get").map((c) => c.params.pull_number).sort()).toEqual([3, 4]);
|
||||
expect(github.callsTo("pulls.listCommits")).toHaveLength(2);
|
||||
expect(github.callsTo("pulls.listFiles")).toHaveLength(2);
|
||||
|
||||
// The new PR is built from the detail response, which has the refs.
|
||||
const [created] = gitea.createdBodies();
|
||||
expect(created).toContain("feature-4");
|
||||
expect(gitea.createdTitles()).toEqual(["[PR #4] PR 4"]);
|
||||
|
||||
expect(cursor!.lastFullPassStartedAt).toBe(previous.lastFullPassStartedAt);
|
||||
});
|
||||
|
||||
it("keeps the previous watermark when a pull request fails", async () => {
|
||||
fakeGitea([{ number: 21, title: "[PR #1] PR 1" }]);
|
||||
// Detail fails and the basic fallback create also fails.
|
||||
globalThis.fetch = ((original) =>
|
||||
(async (input: any, init?: any) => {
|
||||
const url = typeof input === "string" ? input : String(input);
|
||||
if (String(init?.method ?? "GET").toUpperCase() === "POST" && url === `${DEST_API}/issues`) {
|
||||
return new Response("boom", { status: 500 });
|
||||
}
|
||||
return original(input, init);
|
||||
}) as unknown as typeof fetch)(globalThis.fetch);
|
||||
const github = fakeOctokit({
|
||||
issueListing: [{ ...ghIssue(5), pull_request: { merged_at: null } }],
|
||||
failDetailFor: [5],
|
||||
});
|
||||
|
||||
const cursor = await mirrorGitRepoPullRequestsToGitea({
|
||||
config,
|
||||
octokit: github.octokit,
|
||||
repository,
|
||||
giteaOwner: DEST_OWNER,
|
||||
giteaRepoName: DEST_REPO,
|
||||
syncCursor: recentCursor(),
|
||||
});
|
||||
|
||||
expect(cursor).toBeUndefined();
|
||||
expect(github.callsTo("pulls.get").map((c) => c.params.pull_number)).toEqual([5]);
|
||||
});
|
||||
});
|
||||
|
||||
describe.skipIf(!isChild)("milestone assignment (#460)", () => {
|
||||
const milestoneIds = new Map<string, number>([
|
||||
["v1.0", 7],
|
||||
["v2.0", 8],
|
||||
]);
|
||||
const weekOldFullPass = (): MetadataSyncCursor => ({
|
||||
lastPassStartedAt: new Date(Date.now() - 60 * 60 * 1000).toISOString(),
|
||||
lastFullPassStartedAt: new Date(Date.now() - 8 * 24 * 60 * 60 * 1000).toISOString(),
|
||||
});
|
||||
const patchesTo = (gitea: ReturnType<typeof fakeGitea>, number: number) =>
|
||||
gitea.requests
|
||||
.filter((r) => r.method === "PATCH" && r.url === `${DEST_API}/issues/${number}`)
|
||||
.map((r) => JSON.parse(r.body!));
|
||||
const created = (gitea: ReturnType<typeof fakeGitea>) =>
|
||||
gitea.requests
|
||||
.filter((r) => r.method === "POST" && r.url === `${DEST_API}/issues`)
|
||||
.map((r) => JSON.parse(r.body!));
|
||||
|
||||
it("creates issues with their milestone", async () => {
|
||||
const gitea = fakeGitea([]);
|
||||
const github = fakeOctokit({
|
||||
issueListing: [ghIssue(1, { milestone: { title: "v1.0" } }), ghIssue(2, { milestone: null })],
|
||||
});
|
||||
|
||||
await mirrorGitRepoIssuesToGitea({
|
||||
config,
|
||||
octokit: github.octokit,
|
||||
repository,
|
||||
giteaOwner: DEST_OWNER,
|
||||
giteaRepoName: DEST_REPO,
|
||||
milestoneIds,
|
||||
});
|
||||
|
||||
const [first, second] = created(gitea);
|
||||
expect(first.milestone).toBe(7);
|
||||
expect(second.milestone).toBeUndefined();
|
||||
});
|
||||
|
||||
it("assigns a milestone set on GitHub after the issue was mirrored", async () => {
|
||||
const gitea = fakeGitea([{ number: 11, title: "[GH-ISSUE #1] Issue 1", milestone: null }]);
|
||||
const github = fakeOctokit({ issueListing: [ghIssue(1, { milestone: { title: "v2.0" } })] });
|
||||
|
||||
await mirrorGitRepoIssuesToGitea({
|
||||
config,
|
||||
octokit: github.octokit,
|
||||
repository,
|
||||
giteaOwner: DEST_OWNER,
|
||||
giteaRepoName: DEST_REPO,
|
||||
syncCursor: recentCursor(),
|
||||
milestoneIds,
|
||||
});
|
||||
|
||||
expect(patchesTo(gitea, 11).map((p) => p.milestone)).toEqual([8]);
|
||||
});
|
||||
|
||||
it("fixes and clears milestones on unchanged issues during a full pass", async () => {
|
||||
const gitea = fakeGitea([
|
||||
// Mirrored before #460: no milestone in Gitea yet.
|
||||
{ number: 11, title: "[GH-ISSUE #1] Issue 1", milestone: null },
|
||||
// Milestone removed on GitHub.
|
||||
{ number: 12, title: "[GH-ISSUE #2] Issue 2", milestone: { id: 7 } },
|
||||
// Already right: the edit leaves it alone.
|
||||
{ number: 13, title: "[GH-ISSUE #3] Issue 3", milestone: { id: 8 } },
|
||||
// On a milestone made in Gitea alone: kept.
|
||||
{ number: 14, title: "[GH-ISSUE #4] Issue 4", milestone: { id: 99 } },
|
||||
]);
|
||||
const github = fakeOctokit({
|
||||
issueListing: [
|
||||
ghIssue(1, { milestone: { title: "v1.0" } }),
|
||||
ghIssue(2, { milestone: null }),
|
||||
ghIssue(3, { milestone: { title: "v2.0" } }),
|
||||
ghIssue(4, { milestone: null }),
|
||||
],
|
||||
});
|
||||
|
||||
await mirrorGitRepoIssuesToGitea({
|
||||
config,
|
||||
octokit: github.octokit,
|
||||
repository,
|
||||
giteaOwner: DEST_OWNER,
|
||||
giteaRepoName: DEST_REPO,
|
||||
syncCursor: weekOldFullPass(),
|
||||
milestoneIds,
|
||||
});
|
||||
|
||||
// All three are unchanged, so no comment listing runs.
|
||||
expect(github.callsTo("issues.listComments")).toHaveLength(0);
|
||||
expect(patchesTo(gitea, 11).map((p) => p.milestone)).toEqual([7]);
|
||||
expect(patchesTo(gitea, 12).map((p) => p.milestone)).toEqual([0]);
|
||||
expect(patchesTo(gitea, 13).every((p) => !("milestone" in p))).toBe(true);
|
||||
expect(patchesTo(gitea, 14).every((p) => !("milestone" in p))).toBe(true);
|
||||
});
|
||||
|
||||
it("leaves milestones alone when they are not mirrored or not found", async () => {
|
||||
const gitea = fakeGitea([{ number: 11, title: "[GH-ISSUE #1] Issue 1", milestone: { id: 7 } }]);
|
||||
const github = fakeOctokit({
|
||||
issueListing: [
|
||||
ghIssue(1, { milestone: null }),
|
||||
ghIssue(2, { milestone: { title: "v1.0" } }),
|
||||
],
|
||||
});
|
||||
|
||||
await mirrorGitRepoIssuesToGitea({
|
||||
config,
|
||||
octokit: github.octokit,
|
||||
repository,
|
||||
giteaOwner: DEST_OWNER,
|
||||
giteaRepoName: DEST_REPO,
|
||||
syncCursor: recentCursor(),
|
||||
});
|
||||
|
||||
expect(patchesTo(gitea, 11).every((p) => !("milestone" in p))).toBe(true);
|
||||
expect(created(gitea)[0].milestone).toBeUndefined();
|
||||
|
||||
// A milestone Gitea does not have is not cleared from the issue either.
|
||||
const gitea2 = fakeGitea([{ number: 11, title: "[GH-ISSUE #1] Issue 1", milestone: { id: 7 } }]);
|
||||
await mirrorGitRepoIssuesToGitea({
|
||||
config,
|
||||
octokit: fakeOctokit({ issueListing: [ghIssue(1, { milestone: { title: "v9" } })] }).octokit,
|
||||
repository,
|
||||
giteaOwner: DEST_OWNER,
|
||||
giteaRepoName: DEST_REPO,
|
||||
syncCursor: recentCursor(),
|
||||
milestoneIds,
|
||||
});
|
||||
expect(patchesTo(gitea2, 11).every((p) => !("milestone" in p))).toBe(true);
|
||||
});
|
||||
|
||||
it("assigns milestones to pull requests, new and unchanged", async () => {
|
||||
const gitea = fakeGitea([{ number: 21, title: "[PR #1] PR 1", milestone: null }]);
|
||||
const github = fakeOctokit({
|
||||
issueListing: [],
|
||||
pullListing: [
|
||||
ghPull(1, { updated_at: "2026-01-01T00:00:00Z", milestone: { title: "v1.0" } }),
|
||||
ghPull(2, { milestone: { title: "v2.0" } }),
|
||||
],
|
||||
});
|
||||
|
||||
await mirrorGitRepoPullRequestsToGitea({
|
||||
config,
|
||||
octokit: github.octokit,
|
||||
repository,
|
||||
giteaOwner: DEST_OWNER,
|
||||
giteaRepoName: DEST_REPO,
|
||||
syncCursor: weekOldFullPass(),
|
||||
milestoneIds,
|
||||
});
|
||||
|
||||
// #1 is unchanged: no detail calls, only the milestone is patched.
|
||||
expect(github.callsTo("pulls.get").map((c) => c.params.pull_number)).toEqual([2]);
|
||||
expect(patchesTo(gitea, 21)).toEqual([{ milestone: 7 }]);
|
||||
expect(created(gitea).map((c) => c.milestone)).toEqual([8]);
|
||||
});
|
||||
|
||||
it("carries the milestone through an incremental pull request listing", async () => {
|
||||
const gitea = fakeGitea([{ number: 21, title: "[PR #1] PR 1", milestone: null }]);
|
||||
const github = fakeOctokit({
|
||||
issueListing: [
|
||||
{ ...ghIssue(1), title: "PR 1", pull_request: { merged_at: null }, milestone: { title: "v2.0" } },
|
||||
],
|
||||
});
|
||||
|
||||
await mirrorGitRepoPullRequestsToGitea({
|
||||
config,
|
||||
octokit: github.octokit,
|
||||
repository,
|
||||
giteaOwner: DEST_OWNER,
|
||||
giteaRepoName: DEST_REPO,
|
||||
syncCursor: recentCursor(),
|
||||
milestoneIds,
|
||||
});
|
||||
|
||||
expect(patchesTo(gitea, 21).map((p) => p.milestone)).toEqual([8]);
|
||||
});
|
||||
|
||||
it("does not take a mirrored issue that mentions [PR #N] for that pull request", async () => {
|
||||
const gitea = fakeGitea([
|
||||
{ number: 21, title: "[PR #1] PR 1", milestone: null },
|
||||
{ number: 40, title: "[GH-ISSUE #40] Revert [PR #1]", milestone: null },
|
||||
]);
|
||||
const github = fakeOctokit({
|
||||
issueListing: [],
|
||||
pullListing: [ghPull(1, { updated_at: "2026-01-01T00:00:00Z", milestone: { title: "v1.0" } })],
|
||||
});
|
||||
|
||||
await mirrorGitRepoPullRequestsToGitea({
|
||||
config,
|
||||
octokit: github.octokit,
|
||||
repository,
|
||||
giteaOwner: DEST_OWNER,
|
||||
giteaRepoName: DEST_REPO,
|
||||
syncCursor: weekOldFullPass(),
|
||||
milestoneIds,
|
||||
});
|
||||
|
||||
expect(patchesTo(gitea, 21)).toEqual([{ milestone: 7 }]);
|
||||
expect(patchesTo(gitea, 40)).toEqual([]);
|
||||
});
|
||||
|
||||
it("finishes the pass when the milestone edit of an unchanged pull request fails", async () => {
|
||||
fakeGitea([{ number: 21, title: "[PR #1] PR 1", milestone: null }]);
|
||||
let milestoneEdits = 0;
|
||||
globalThis.fetch = ((original) =>
|
||||
(async (input: any, init?: any) => {
|
||||
const url = typeof input === "string" ? input : String(input);
|
||||
if (String(init?.method ?? "GET").toUpperCase() === "PATCH" && url === `${DEST_API}/issues/21`) {
|
||||
milestoneEdits += 1;
|
||||
return new Response("boom", { status: 500 });
|
||||
}
|
||||
return original(input, init);
|
||||
}) as unknown as typeof fetch)(globalThis.fetch);
|
||||
const github = fakeOctokit({
|
||||
issueListing: [],
|
||||
pullListing: [ghPull(1, { updated_at: "2026-01-01T00:00:00Z", milestone: { title: "v1.0" } })],
|
||||
});
|
||||
|
||||
const cursor = await mirrorGitRepoPullRequestsToGitea({
|
||||
config,
|
||||
octokit: github.octokit,
|
||||
repository,
|
||||
giteaOwner: DEST_OWNER,
|
||||
giteaRepoName: DEST_REPO,
|
||||
syncCursor: weekOldFullPass(),
|
||||
milestoneIds,
|
||||
});
|
||||
|
||||
// Tried once, not retried, and the pass still completes.
|
||||
expect(milestoneEdits).toBe(1);
|
||||
expect(cursor).toBeDefined();
|
||||
});
|
||||
});
|
||||
@@ -14,7 +14,13 @@
|
||||
*/
|
||||
|
||||
import { describe, test, expect } from "bun:test";
|
||||
import { buildGiteaIssueEditPayload, buildGiteaIssueLabelsPayload } from "@/lib/gitea";
|
||||
import {
|
||||
buildGiteaIssueEditPayload,
|
||||
buildGiteaIssueLabelsPayload,
|
||||
buildGiteaMilestoneEditPayload,
|
||||
giteaMilestoneEdit,
|
||||
resolveGiteaMilestoneId,
|
||||
} from "@/lib/gitea";
|
||||
|
||||
describe("buildGiteaIssueEditPayload (#334 sibling)", () => {
|
||||
test("edit body never carries `labels` (Gitea's EditIssueOption ignores it)", () => {
|
||||
@@ -37,6 +43,90 @@ describe("buildGiteaIssueEditPayload (#334 sibling)", () => {
|
||||
});
|
||||
});
|
||||
|
||||
describe("issue milestones (#460)", () => {
|
||||
const ids = new Map([["v1.0", 7]]);
|
||||
|
||||
test("the edit body carries a milestone only when one is given", () => {
|
||||
expect(buildGiteaIssueEditPayload({ title: "t", body: "b", closed: false })).not.toHaveProperty(
|
||||
"milestone"
|
||||
);
|
||||
expect(
|
||||
buildGiteaIssueEditPayload({ title: "t", body: "b", closed: false, milestone: 7 }).milestone
|
||||
).toBe(7);
|
||||
// 0 is how Gitea clears a milestone, so it has to survive.
|
||||
expect(
|
||||
buildGiteaIssueEditPayload({ title: "t", body: "b", closed: false, milestone: 0 }).milestone
|
||||
).toBe(0);
|
||||
});
|
||||
|
||||
test("resolves the Gitea id by title, 0 for none, undefined when unknown", () => {
|
||||
expect(resolveGiteaMilestoneId(ids, { title: "v1.0" })).toBe(7);
|
||||
expect(resolveGiteaMilestoneId(ids, null)).toBe(0);
|
||||
expect(resolveGiteaMilestoneId(ids, { title: "missing" })).toBeUndefined();
|
||||
expect(resolveGiteaMilestoneId(undefined, { title: "v1.0" })).toBeUndefined();
|
||||
expect(resolveGiteaMilestoneId(undefined, null)).toBeUndefined();
|
||||
});
|
||||
|
||||
test("only edits when the issue carries a different milestone", () => {
|
||||
expect(giteaMilestoneEdit({ milestone: null }, ids, { title: "v1.0" })).toBe(7);
|
||||
expect(giteaMilestoneEdit({ milestone: { id: 7 } }, ids, { title: "v1.0" })).toBeUndefined();
|
||||
expect(giteaMilestoneEdit({ milestone: { id: 7 } }, ids, null)).toBe(0);
|
||||
expect(giteaMilestoneEdit({}, ids, null)).toBeUndefined();
|
||||
expect(giteaMilestoneEdit({ milestone: { id: 7 } }, undefined, null)).toBeUndefined();
|
||||
});
|
||||
|
||||
test("never clears a milestone made in Gitea alone", () => {
|
||||
// 42 is not in the map: no GitHub milestone has its title.
|
||||
expect(giteaMilestoneEdit({ milestone: { id: 42 } }, ids, null)).toBeUndefined();
|
||||
// GitHub does name a milestone, so it wins over the Gitea-only one.
|
||||
expect(giteaMilestoneEdit({ milestone: { id: 42 } }, ids, { title: "v1.0" })).toBe(7);
|
||||
});
|
||||
});
|
||||
|
||||
describe("buildGiteaMilestoneEditPayload (#460 follow-up)", () => {
|
||||
test("no edit when state, due day and description match", () => {
|
||||
expect(
|
||||
buildGiteaMilestoneEditPayload(
|
||||
// Created from GitHub: Gitea keeps the instant, shown in its own zone.
|
||||
{ state: "open", due_on: "2026-11-01T00:00:00-07:00", description: "d" },
|
||||
{ state: "open", due_on: "2026-11-01T07:00:00Z", description: "d" }
|
||||
)
|
||||
).toBeNull();
|
||||
expect(
|
||||
buildGiteaMilestoneEditPayload(
|
||||
// Edited through the API: moved to the end of the day.
|
||||
{ state: "closed", due_on: "2026-12-15T23:59:59+09:00", description: null },
|
||||
{ state: "closed", due_on: "2026-12-15T08:00:00Z", description: null }
|
||||
)
|
||||
).toBeNull();
|
||||
// Year 9999 is how Gitea stores no deadline.
|
||||
expect(
|
||||
buildGiteaMilestoneEditPayload(
|
||||
{ state: "open", due_on: "9999-12-31T23:59:59-08:00", description: "" },
|
||||
{ state: "open", due_on: null, description: "" }
|
||||
)
|
||||
).toBeNull();
|
||||
});
|
||||
|
||||
test("carries over a close, a due date and a description", () => {
|
||||
expect(
|
||||
buildGiteaMilestoneEditPayload(
|
||||
{ state: "open", due_on: null, description: "old" },
|
||||
{ state: "closed", due_on: "2026-12-15T08:00:00Z", description: "new" }
|
||||
)
|
||||
).toEqual({ state: "closed", due_on: "2026-12-15T12:00:00Z", description: "new" });
|
||||
});
|
||||
|
||||
test("clears a removed due date with Gitea's no-deadline value", () => {
|
||||
expect(
|
||||
buildGiteaMilestoneEditPayload(
|
||||
{ state: "open", due_on: "2026-11-01T00:00:00-07:00", description: "" },
|
||||
{ state: "open", due_on: null, description: "" }
|
||||
)
|
||||
).toEqual({ due_on: "9999-12-31T00:00:00Z" });
|
||||
});
|
||||
});
|
||||
|
||||
describe("buildGiteaIssueLabelsPayload (#334 sibling)", () => {
|
||||
test("replaces the full label set with the resolved Gitea label ids", () => {
|
||||
expect(buildGiteaIssueLabelsPayload([7, 9])).toEqual({ labels: [7, 9] });
|
||||
|
||||
@@ -146,7 +146,8 @@ describe("milestone dedup on sync", () => {
|
||||
// Defensive: if `milestones` ever contains a same-named entry
|
||||
// twice (unlikely but cheap to guard), we shouldn't POST it twice.
|
||||
expect(
|
||||
/existingMilestones\.add\(\s*milestone\.title\s*\)/.test(body),
|
||||
// A map since #460 (title to id, for issue assignment).
|
||||
/existingMilestones\.set\(\s*milestone\.title\b/.test(body),
|
||||
"after a successful create, the new milestone title must be added to existingMilestones"
|
||||
).toBe(true);
|
||||
});
|
||||
|
||||
@@ -113,7 +113,16 @@ mock.module("@/lib/db", () => {
|
||||
}),
|
||||
}),
|
||||
update: (_table: any) => ({
|
||||
set: (_data: any) => ({ where: (_cond: any) => Promise.resolve() }),
|
||||
set: (_data: any) => ({
|
||||
where: (_cond: any) => {
|
||||
// The mirror path claims the row with a conditional UPDATE and reads
|
||||
// the returned rows to find out whether it won (#417); one row means
|
||||
// claimed.
|
||||
const result: any = Promise.resolve();
|
||||
result.returning = () => Promise.resolve([{ id: "repo-1" }]);
|
||||
return result;
|
||||
},
|
||||
}),
|
||||
}),
|
||||
insert: (_table: any) => ({ values: (_data: any) => Promise.resolve() }),
|
||||
delete: (_table: any) => ({ where: (_cond: any) => Promise.resolve() }),
|
||||
|
||||
@@ -0,0 +1,322 @@
|
||||
/**
|
||||
* Behavioral tests for release asset reconciliation, regression for #417.
|
||||
*
|
||||
* Reported symptom: release assets mirrored to Gitea/Forgejo appear twice (some
|
||||
* three times, one five times), with identical sizes and identical creation
|
||||
* times, so the copies came from passes that overlapped inside one run.
|
||||
*
|
||||
* Cause: reconcileReleaseAssets listed the attachments a release already had,
|
||||
* then downloaded and uploaded everything it judged missing. The destination
|
||||
* has no name-collision check on attachment upload (Gitea's and Forgejo's
|
||||
* CreateReleaseAttachment appends a row for every POST), so two overlapping
|
||||
* passes both read "missing" and both uploaded. A failed listing was read as
|
||||
* "the release has no assets", and a failed delete was followed by the upload
|
||||
* anyway; both added copies too.
|
||||
*
|
||||
* The fake destination below models the real behavior that makes duplicates
|
||||
* possible: every POST appends another attachment with the same name.
|
||||
*
|
||||
* No module mocks here on purpose (bun's mock.module is process-wide and leaks
|
||||
* into other test files): reconcileReleaseAssets takes the fetch it should use.
|
||||
*/
|
||||
|
||||
import { describe, expect, it } from "bun:test";
|
||||
import { reconcileReleaseAssets, buildReleaseTargetLockKey } from "@/lib/gitea";
|
||||
import { withKeyedLock } from "@/lib/utils/keyed-mutex";
|
||||
import type { Config } from "@/types/config";
|
||||
|
||||
const GITEA_URL = "https://gitea.example.com";
|
||||
const OWNER = "mirror-owner";
|
||||
const REPO = "demo";
|
||||
const RELEASE_ID = 77;
|
||||
const ASSETS_URL = `${GITEA_URL}/api/v1/repos/${OWNER}/${REPO}/releases/${RELEASE_ID}/assets`;
|
||||
|
||||
const config = {
|
||||
giteaConfig: { url: GITEA_URL, token: "gitea-token", defaultOwner: OWNER },
|
||||
} as unknown as Partial<Config>;
|
||||
|
||||
const decryptedConfig = {
|
||||
giteaConfig: { url: GITEA_URL, token: "gitea-token", defaultOwner: OWNER },
|
||||
githubConfig: { token: "github-token", username: "upstream" },
|
||||
} as unknown as Config;
|
||||
|
||||
type Attachment = { id: number; name: string; size: number };
|
||||
|
||||
interface FakeDestinationOptions {
|
||||
/** Attachments the release already carries. */
|
||||
existing?: Attachment[];
|
||||
/** Status for the "list attachments" GET. */
|
||||
listStatus?: number;
|
||||
/** Status for attachment deletes. */
|
||||
deleteStatus?: number;
|
||||
/** Status for attachment uploads. */
|
||||
uploadStatus?: number;
|
||||
}
|
||||
|
||||
/** A GitHub asset plus the destination-side size the fake records on upload. */
|
||||
function githubAsset(name: string, size: number) {
|
||||
return {
|
||||
name,
|
||||
size,
|
||||
browser_download_url: `https://github.com/upstream/demo/releases/download/v1.0.0/${name}`,
|
||||
};
|
||||
}
|
||||
|
||||
function createFakeDestination(options: FakeDestinationOptions = {}) {
|
||||
const attachments: Attachment[] = [...(options.existing ?? [])];
|
||||
const requests: Array<{ method: string; url: string }> = [];
|
||||
let nextId = 500;
|
||||
|
||||
const fetchImpl = (async (input: any, init?: any) => {
|
||||
const url = typeof input === "string" ? input : String(input);
|
||||
const method = String(init?.method ?? "GET").toUpperCase();
|
||||
requests.push({ method, url });
|
||||
|
||||
// Every request yields the event loop, so a second pass running at the
|
||||
// same time gets to interleave, which is what created the duplicates.
|
||||
await new Promise((resolve) => setTimeout(resolve, 0));
|
||||
|
||||
// Downloading the asset from the source host.
|
||||
if (url.startsWith("https://github.com/")) {
|
||||
return new Response(new Uint8Array([1, 2, 3, 4]), { status: 200 });
|
||||
}
|
||||
|
||||
if (method === "GET" && url === ASSETS_URL) {
|
||||
const status = options.listStatus ?? 200;
|
||||
if (status !== 200) {
|
||||
return new Response("boom", { status, statusText: "Internal Server Error" });
|
||||
}
|
||||
return new Response(JSON.stringify(attachments), {
|
||||
status: 200,
|
||||
headers: { "content-type": "application/json" },
|
||||
});
|
||||
}
|
||||
|
||||
if (method === "DELETE" && url.startsWith(`${ASSETS_URL}/`)) {
|
||||
const status = options.deleteStatus ?? 204;
|
||||
if (status >= 400) {
|
||||
return new Response("nope", { status, statusText: "Internal Server Error" });
|
||||
}
|
||||
const id = Number(url.slice(`${ASSETS_URL}/`.length));
|
||||
const index = attachments.findIndex((a) => a.id === id);
|
||||
if (index >= 0) attachments.splice(index, 1);
|
||||
return new Response(null, { status });
|
||||
}
|
||||
|
||||
if (method === "POST" && url.startsWith(`${ASSETS_URL}?`)) {
|
||||
const status = options.uploadStatus ?? 201;
|
||||
if (status >= 400) {
|
||||
return new Response("rejected", { status, statusText: "Bad Request" });
|
||||
}
|
||||
const name = decodeURIComponent(new URL(url).searchParams.get("name") ?? "");
|
||||
// The real destination does no name-collision check: every POST appends
|
||||
// another attachment, even when one with that name is already there.
|
||||
attachments.push({ id: nextId++, name, size: uploadSizes.get(name) ?? 4 });
|
||||
return new Response(JSON.stringify({ id: nextId }), {
|
||||
status,
|
||||
headers: { "content-type": "application/json" },
|
||||
});
|
||||
}
|
||||
|
||||
throw new Error(`unexpected request: ${method} ${url}`);
|
||||
}) as unknown as typeof fetch;
|
||||
|
||||
const uploadSizes = new Map<string, number>();
|
||||
|
||||
return {
|
||||
fetchImpl,
|
||||
attachments,
|
||||
requests,
|
||||
/** Size the destination records for an uploaded asset (defaults to 4). */
|
||||
setUploadedSize(name: string, size: number) {
|
||||
uploadSizes.set(name, size);
|
||||
},
|
||||
posts: () => requests.filter((r) => r.method === "POST"),
|
||||
deletes: () => requests.filter((r) => r.method === "DELETE"),
|
||||
};
|
||||
}
|
||||
|
||||
function reconcile(
|
||||
fetchImpl: typeof fetch,
|
||||
sourceAssets: Array<{ name: string; size: number; browser_download_url: string }>
|
||||
) {
|
||||
return reconcileReleaseAssets({
|
||||
config,
|
||||
decryptedConfig,
|
||||
repoOwner: OWNER,
|
||||
repoName: REPO,
|
||||
giteaReleaseId: RELEASE_ID,
|
||||
sourceAssets,
|
||||
// The GitHub path's credential, which is what these cases model.
|
||||
downloadAuthorization: `token ${decryptedConfig.githubConfig!.token}`,
|
||||
tagName: "v1.0.0",
|
||||
fetchImpl,
|
||||
});
|
||||
}
|
||||
|
||||
describe("reconcileReleaseAssets", () => {
|
||||
it("uploads every asset once on a release that has none", async () => {
|
||||
const destination = createFakeDestination();
|
||||
destination.setUploadedSize("base.zip", 4);
|
||||
destination.setUploadedSize("extras.zip", 4);
|
||||
|
||||
const result = await reconcile(destination.fetchImpl, [
|
||||
githubAsset("base.zip", 4),
|
||||
githubAsset("extras.zip", 4),
|
||||
]);
|
||||
|
||||
expect(result).toEqual({ uploaded: 2, failed: 0, skipped: 0 });
|
||||
expect(destination.posts()).toHaveLength(2);
|
||||
expect(destination.attachments.map((a) => a.name).sort()).toEqual([
|
||||
"base.zip",
|
||||
"extras.zip",
|
||||
]);
|
||||
});
|
||||
|
||||
it("two overlapping passes under the destination lock upload each asset once (#417)", async () => {
|
||||
const destination = createFakeDestination();
|
||||
destination.setUploadedSize("base.zip", 4);
|
||||
const assets = [githubAsset("base.zip", 4)];
|
||||
const key = buildReleaseTargetLockKey(GITEA_URL, OWNER, REPO);
|
||||
|
||||
// Two runs of the same repository starting at the same time: the scheduler
|
||||
// and a manual sync, or two repository rows pointing at one destination.
|
||||
const [first, second] = await Promise.all([
|
||||
withKeyedLock(key, () => reconcile(destination.fetchImpl, assets)),
|
||||
withKeyedLock(key, () => reconcile(destination.fetchImpl, assets)),
|
||||
]);
|
||||
|
||||
expect(destination.posts()).toHaveLength(1);
|
||||
expect(destination.attachments).toHaveLength(1);
|
||||
expect(first).toEqual({ uploaded: 1, failed: 0, skipped: 0 });
|
||||
// The second pass finds the asset already there and leaves it alone.
|
||||
expect(second).toEqual({ uploaded: 0, failed: 0, skipped: 1 });
|
||||
});
|
||||
|
||||
it("without the lock, two overlapping passes are what duplicated the asset", async () => {
|
||||
const destination = createFakeDestination();
|
||||
destination.setUploadedSize("base.zip", 4);
|
||||
const assets = [githubAsset("base.zip", 4)];
|
||||
|
||||
await Promise.all([
|
||||
reconcile(destination.fetchImpl, assets),
|
||||
reconcile(destination.fetchImpl, assets),
|
||||
]);
|
||||
|
||||
// The fake reproduces the reported failure: identical name, identical size,
|
||||
// two attachments. This is the state the lock prevents.
|
||||
expect(destination.posts()).toHaveLength(2);
|
||||
expect(destination.attachments).toHaveLength(2);
|
||||
});
|
||||
|
||||
it("uploads nothing when the existing attachments cannot be listed", async () => {
|
||||
const destination = createFakeDestination({ listStatus: 500 });
|
||||
|
||||
const result = await reconcile(destination.fetchImpl, [
|
||||
githubAsset("base.zip", 4),
|
||||
githubAsset("extras.zip", 4),
|
||||
]);
|
||||
|
||||
// Fail closed: a failed listing used to read as "the release has no
|
||||
// assets", so the pass re-uploaded everything the release already had.
|
||||
expect(destination.posts()).toHaveLength(0);
|
||||
expect(destination.deletes()).toHaveLength(0);
|
||||
expect(result).toEqual({ uploaded: 0, failed: 2, skipped: 0 });
|
||||
});
|
||||
|
||||
it("does not upload an asset whose stale copy could not be deleted", async () => {
|
||||
const destination = createFakeDestination({
|
||||
existing: [{ id: 11, name: "firmware.bin", size: 1024 }],
|
||||
deleteStatus: 500,
|
||||
});
|
||||
|
||||
const result = await reconcile(destination.fetchImpl, [
|
||||
githubAsset("firmware.bin", 2048),
|
||||
]);
|
||||
|
||||
expect(destination.deletes()).toHaveLength(1);
|
||||
// Uploading after the failed delete would leave two copies.
|
||||
expect(destination.posts()).toHaveLength(0);
|
||||
expect(destination.attachments).toHaveLength(1);
|
||||
expect(result).toEqual({ uploaded: 0, failed: 1, skipped: 0 });
|
||||
});
|
||||
|
||||
it("cleans up an existing duplicate without uploading anything", async () => {
|
||||
const destination = createFakeDestination({
|
||||
existing: [
|
||||
{ id: 11, name: "base.zip", size: 4 },
|
||||
{ id: 12, name: "base.zip", size: 4 },
|
||||
],
|
||||
});
|
||||
|
||||
const result = await reconcile(destination.fetchImpl, [githubAsset("base.zip", 4)]);
|
||||
|
||||
expect(destination.deletes()).toHaveLength(1);
|
||||
expect(destination.deletes()[0].url).toBe(`${ASSETS_URL}/12`);
|
||||
expect(destination.posts()).toHaveLength(0);
|
||||
expect(destination.attachments).toEqual([{ id: 11, name: "base.zip", size: 4 }]);
|
||||
expect(result).toEqual({ uploaded: 0, failed: 0, skipped: 1 });
|
||||
});
|
||||
|
||||
it("replaces stale copies: deletes both, then uploads once", async () => {
|
||||
const destination = createFakeDestination({
|
||||
existing: [
|
||||
{ id: 11, name: "firmware.bin", size: 1024 },
|
||||
{ id: 12, name: "firmware.bin", size: 999 },
|
||||
],
|
||||
});
|
||||
destination.setUploadedSize("firmware.bin", 2048);
|
||||
|
||||
const result = await reconcile(destination.fetchImpl, [
|
||||
githubAsset("firmware.bin", 2048),
|
||||
]);
|
||||
|
||||
expect(destination.deletes().map((r) => r.url)).toEqual([
|
||||
`${ASSETS_URL}/11`,
|
||||
`${ASSETS_URL}/12`,
|
||||
]);
|
||||
expect(destination.posts()).toHaveLength(1);
|
||||
expect(destination.attachments).toHaveLength(1);
|
||||
expect(destination.attachments[0].size).toBe(2048);
|
||||
expect(result).toEqual({ uploaded: 1, failed: 0, skipped: 0 });
|
||||
// Deletes come before the upload, never after it.
|
||||
const kinds = destination.requests.map((r) => r.method);
|
||||
expect(kinds.indexOf("POST")).toBeGreaterThan(kinds.lastIndexOf("DELETE"));
|
||||
});
|
||||
|
||||
it("leaves attachments alone that the release does not have upstream", async () => {
|
||||
const destination = createFakeDestination({
|
||||
existing: [{ id: 11, name: "from-an-older-sync.zip", size: 4 }],
|
||||
});
|
||||
destination.setUploadedSize("base.zip", 4);
|
||||
|
||||
const result = await reconcile(destination.fetchImpl, [githubAsset("base.zip", 4)]);
|
||||
|
||||
expect(destination.deletes()).toHaveLength(0);
|
||||
expect(destination.posts()).toHaveLength(1);
|
||||
expect(result).toEqual({ uploaded: 1, failed: 0, skipped: 0 });
|
||||
});
|
||||
|
||||
it("does nothing at all for a release with no assets", async () => {
|
||||
const destination = createFakeDestination();
|
||||
|
||||
const result = await reconcile(destination.fetchImpl, []);
|
||||
|
||||
expect(destination.requests).toHaveLength(0);
|
||||
expect(result).toEqual({ uploaded: 0, failed: 0, skipped: 0 });
|
||||
});
|
||||
});
|
||||
|
||||
describe("buildReleaseTargetLockKey", () => {
|
||||
it("keys on destination, owner and repository, case-insensitively", () => {
|
||||
expect(buildReleaseTargetLockKey("https://gitea.example.com/", "Owner", "Repo")).toBe(
|
||||
buildReleaseTargetLockKey("https://GITEA.example.com", "owner", "repo")
|
||||
);
|
||||
});
|
||||
|
||||
it("separates different repositories on the same destination", () => {
|
||||
expect(buildReleaseTargetLockKey(GITEA_URL, OWNER, "one")).not.toBe(
|
||||
buildReleaseTargetLockKey(GITEA_URL, OWNER, "two")
|
||||
);
|
||||
});
|
||||
});
|
||||
@@ -164,6 +164,11 @@ describe("classifyReleasesForReconciliation", () => {
|
||||
* existing release" and left it at 0.
|
||||
*
|
||||
* Fix: reconcile assets idempotently on both paths via classifyAssetsForReconciliation.
|
||||
*
|
||||
* Extended for #417: the destination allows several attachments with the same
|
||||
* name (no name-collision check in CreateReleaseAttachment), so the classifier
|
||||
* groups existing attachments by name, keeps one copy and reports the surplus
|
||||
* ones in toDelete.
|
||||
*/
|
||||
describe("classifyAssetsForReconciliation", () => {
|
||||
it("uploads all assets when the Gitea release has none (the #331 broken state)", () => {
|
||||
@@ -173,13 +178,11 @@ describe("classifyAssetsForReconciliation", () => {
|
||||
];
|
||||
const gitea: Array<{ id: number; name: string; size: number }> = [];
|
||||
|
||||
const { toUpload, toSkip } = classifyAssetsForReconciliation(github, gitea);
|
||||
const { toUpload, toSkip, toDelete } = classifyAssetsForReconciliation(github, gitea);
|
||||
|
||||
expect(toSkip).toEqual([]);
|
||||
expect(toUpload).toEqual([
|
||||
{ name: "base.zip", replaceAssetId: null },
|
||||
{ name: "extras.zip", replaceAssetId: null },
|
||||
]);
|
||||
expect(toDelete).toEqual([]);
|
||||
expect(toUpload).toEqual(["base.zip", "extras.zip"]);
|
||||
});
|
||||
|
||||
it("backfills only the missing asset when one already exists", () => {
|
||||
@@ -189,10 +192,11 @@ describe("classifyAssetsForReconciliation", () => {
|
||||
];
|
||||
const gitea = [{ id: 9, name: "base.zip", size: 40_264_954 }];
|
||||
|
||||
const { toUpload, toSkip } = classifyAssetsForReconciliation(github, gitea);
|
||||
const { toUpload, toSkip, toDelete } = classifyAssetsForReconciliation(github, gitea);
|
||||
|
||||
expect(toSkip).toEqual(["base.zip"]);
|
||||
expect(toUpload).toEqual([{ name: "extras.zip", replaceAssetId: null }]);
|
||||
expect(toUpload).toEqual(["extras.zip"]);
|
||||
expect(toDelete).toEqual([]);
|
||||
});
|
||||
|
||||
it("is idempotent — skips everything when all assets already match by name+size", () => {
|
||||
@@ -205,28 +209,111 @@ describe("classifyAssetsForReconciliation", () => {
|
||||
{ id: 10, name: "extras.zip", size: 37_098_528 },
|
||||
];
|
||||
|
||||
const { toUpload, toSkip } = classifyAssetsForReconciliation(github, gitea);
|
||||
const { toUpload, toSkip, toDelete } = classifyAssetsForReconciliation(github, gitea);
|
||||
|
||||
expect(toUpload).toEqual([]);
|
||||
expect(toSkip).toEqual(["base.zip", "extras.zip"]);
|
||||
expect(toDelete).toEqual([]);
|
||||
});
|
||||
|
||||
it("replaces an asset whose size changed upstream (re-upload over the stale copy)", () => {
|
||||
it("replaces an asset whose size changed upstream (delete the stale copy, upload once)", () => {
|
||||
const github = [{ name: "firmware.bin", size: 2048 }];
|
||||
const gitea = [{ id: 42, name: "firmware.bin", size: 1024 }]; // truncated/stale
|
||||
|
||||
const { toUpload, toSkip } = classifyAssetsForReconciliation(github, gitea);
|
||||
const { toUpload, toSkip, toDelete } = classifyAssetsForReconciliation(github, gitea);
|
||||
|
||||
expect(toSkip).toEqual([]);
|
||||
expect(toUpload).toEqual([{ name: "firmware.bin", replaceAssetId: 42 }]);
|
||||
expect(toUpload).toEqual(["firmware.bin"]);
|
||||
expect(toDelete).toEqual([42]);
|
||||
});
|
||||
|
||||
it("handles a release with no GitHub assets", () => {
|
||||
const { toUpload, toSkip } = classifyAssetsForReconciliation(
|
||||
const { toUpload, toSkip, toDelete } = classifyAssetsForReconciliation(
|
||||
[],
|
||||
[{ id: 1, name: "leftover.zip", size: 10 }]
|
||||
);
|
||||
expect(toUpload).toEqual([]);
|
||||
expect(toSkip).toEqual([]);
|
||||
expect(toDelete).toEqual([]);
|
||||
});
|
||||
|
||||
// --- #417: duplicate attachments on the destination ---
|
||||
|
||||
it("keeps one copy and deletes the extra when two copies both match (#417)", () => {
|
||||
const github = [{ name: "base.zip", size: 40_264_954 }];
|
||||
const gitea = [
|
||||
{ id: 9, name: "base.zip", size: 40_264_954 },
|
||||
{ id: 14, name: "base.zip", size: 40_264_954 },
|
||||
];
|
||||
|
||||
const { toUpload, toSkip, toDelete } = classifyAssetsForReconciliation(github, gitea);
|
||||
|
||||
expect(toUpload).toEqual([]);
|
||||
expect(toSkip).toEqual(["base.zip"]);
|
||||
expect(toDelete).toEqual([14]);
|
||||
});
|
||||
|
||||
it("deletes all three copies and uploads once when none matches the size", () => {
|
||||
const github = [{ name: "firmware.bin", size: 2048 }];
|
||||
const gitea = [
|
||||
{ id: 7, name: "firmware.bin", size: 1024 },
|
||||
{ id: 8, name: "firmware.bin", size: 1024 },
|
||||
{ id: 9, name: "firmware.bin", size: 99 },
|
||||
];
|
||||
|
||||
const { toUpload, toSkip, toDelete } = classifyAssetsForReconciliation(github, gitea);
|
||||
|
||||
expect(toUpload).toEqual(["firmware.bin"]);
|
||||
expect(toSkip).toEqual([]);
|
||||
expect(toDelete).toEqual([7, 8, 9]);
|
||||
});
|
||||
|
||||
it("keeps the matching copy and deletes the two that do not match", () => {
|
||||
const github = [{ name: "installer.dmg", size: 5_000 }];
|
||||
const gitea = [
|
||||
{ id: 3, name: "installer.dmg", size: 1_000 },
|
||||
{ id: 4, name: "installer.dmg", size: 5_000 },
|
||||
{ id: 5, name: "installer.dmg", size: 2_000 },
|
||||
];
|
||||
|
||||
const { toUpload, toSkip, toDelete } = classifyAssetsForReconciliation(github, gitea);
|
||||
|
||||
expect(toUpload).toEqual([]);
|
||||
expect(toSkip).toEqual(["installer.dmg"]);
|
||||
expect(toDelete).toEqual([3, 5]);
|
||||
});
|
||||
|
||||
it("never deletes a destination attachment whose name is not in the GitHub list", () => {
|
||||
const github = [{ name: "base.zip", size: 10 }];
|
||||
const gitea = [
|
||||
{ id: 1, name: "base.zip", size: 10 },
|
||||
{ id: 2, name: "from-an-older-release.zip", size: 10 },
|
||||
{ id: 3, name: "from-an-older-release.zip", size: 99 },
|
||||
];
|
||||
|
||||
const { toUpload, toSkip, toDelete } = classifyAssetsForReconciliation(github, gitea);
|
||||
|
||||
expect(toUpload).toEqual([]);
|
||||
expect(toSkip).toEqual(["base.zip"]);
|
||||
expect(toDelete).toEqual([]);
|
||||
});
|
||||
|
||||
it("keeps the lowest id whatever order the destination listed the copies in", () => {
|
||||
const github = [{ name: "base.zip", size: 10 }];
|
||||
const listedOneWay = [
|
||||
{ id: 31, name: "base.zip", size: 10 },
|
||||
{ id: 12, name: "base.zip", size: 10 },
|
||||
{ id: 25, name: "base.zip", size: 10 },
|
||||
];
|
||||
const listedTheOtherWay = [...listedOneWay].reverse();
|
||||
|
||||
const first = classifyAssetsForReconciliation(github, listedOneWay);
|
||||
const second = classifyAssetsForReconciliation(github, listedTheOtherWay);
|
||||
|
||||
// id 12 survives in both, and the deletes come back in the same order.
|
||||
expect(first.toDelete).toEqual([25, 31]);
|
||||
expect(second.toDelete).toEqual([25, 31]);
|
||||
expect(first.toSkip).toEqual(["base.zip"]);
|
||||
expect(second.toSkip).toEqual(["base.zip"]);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,462 @@
|
||||
/**
|
||||
* Behavioural tests for source aware release mirroring (#440).
|
||||
*
|
||||
* Release mirroring used to call the GitHub API unconditionally and download
|
||||
* every asset with the primary config's GitHub token. A Gitea or Forgejo
|
||||
* source (Codeberg included) now lists its own releases and downloads its own
|
||||
* assets with its own credentials, while the GitHub path keeps its exact
|
||||
* previous behaviour, including the "Originally published on GitHub:" header
|
||||
* line, which must stay byte for byte or every mirrored release body would
|
||||
* differ and be PATCHed on the next sync.
|
||||
*
|
||||
* Both entry points are driven through a fake global fetch, which is what the
|
||||
* destination client, the source adapter and the asset reconciliation all use.
|
||||
* Other test files replace @/lib/http-client and friends with mock.module,
|
||||
* which is process-wide in bun and depends on file order, so the suites here
|
||||
* run in an isolated child process (same harness as
|
||||
* gitea-org-mirror-destination.test.ts) where the real modules are loaded.
|
||||
*/
|
||||
|
||||
import { afterEach, describe, expect, it, test } from "bun:test";
|
||||
import type { Config } from "@/types/config";
|
||||
import type { Repository } from "@/lib/db/schema";
|
||||
import type { SourceConnection } from "@/lib/source-providers/types";
|
||||
|
||||
const CHILD_FLAG = "GM_SOURCE_RELEASES_ISOLATED";
|
||||
const isChild = !!process.env[CHILD_FLAG];
|
||||
|
||||
if (!isChild) {
|
||||
test("source aware release mirroring - isolated child suite", () => {
|
||||
const res = Bun.spawnSync({
|
||||
cmd: [process.execPath, "test", import.meta.path],
|
||||
env: { ...process.env, [CHILD_FLAG]: "1" },
|
||||
stdout: "pipe",
|
||||
stderr: "pipe",
|
||||
});
|
||||
if (res.exitCode !== 0) {
|
||||
console.error(res.stdout.toString());
|
||||
console.error(res.stderr.toString());
|
||||
}
|
||||
expect(res.exitCode).toBe(0);
|
||||
}, 60_000);
|
||||
}
|
||||
|
||||
const { mirrorGiteaSourceReleasesToGitea, mirrorGitHubReleasesToGitea } = isChild
|
||||
? await import("@/lib/gitea")
|
||||
: ({} as typeof import("@/lib/gitea"));
|
||||
const GITEA_URL = "https://gitea.example.com";
|
||||
const SOURCE_URL = "https://codeberg.org";
|
||||
const DEST_OWNER = "mirror-owner";
|
||||
const DEST_REPO = "demo";
|
||||
const DEST_API = `${GITEA_URL}/api/v1/repos/${DEST_OWNER}/${DEST_REPO}`;
|
||||
|
||||
const realFetch = globalThis.fetch;
|
||||
|
||||
afterEach(() => {
|
||||
globalThis.fetch = realFetch;
|
||||
});
|
||||
|
||||
const config = {
|
||||
userId: "user-1",
|
||||
githubConfig: { owner: "acme", token: "github-token" },
|
||||
giteaConfig: { url: GITEA_URL, token: "gitea-token", defaultOwner: DEST_OWNER },
|
||||
} as unknown as Partial<Config>;
|
||||
|
||||
const repository = {
|
||||
id: "repo-1",
|
||||
name: DEST_REPO,
|
||||
fullName: "acme/demo",
|
||||
owner: "acme",
|
||||
isStarred: false,
|
||||
mirrorOverrides: null,
|
||||
organization: null,
|
||||
} as unknown as Repository;
|
||||
|
||||
function giteaConnection(token: string): SourceConnection {
|
||||
return { provider: "gitea", url: SOURCE_URL, username: "acme", token };
|
||||
}
|
||||
|
||||
interface Recorded {
|
||||
method: string;
|
||||
url: string;
|
||||
authorization: string | null;
|
||||
body?: string;
|
||||
}
|
||||
|
||||
interface DestinationOptions {
|
||||
/** Releases the source reports, newest first. */
|
||||
sourceReleases: Array<Record<string, unknown>>;
|
||||
/** Release already on the destination, matched by tag. */
|
||||
existingRelease?: { id: number; name: string; body: string; tag_name: string };
|
||||
/** Tags the destination already has. Defaults to every source tag. */
|
||||
destinationTags?: string[];
|
||||
}
|
||||
|
||||
function fakeWorld(options: DestinationOptions) {
|
||||
const requests: Recorded[] = [];
|
||||
const tags =
|
||||
options.destinationTags ??
|
||||
options.sourceReleases.map((release) => String(release.tag_name));
|
||||
|
||||
globalThis.fetch = (async (input: any, init?: any) => {
|
||||
const url = typeof input === "string" ? input : String(input);
|
||||
const method = String(init?.method ?? "GET").toUpperCase();
|
||||
const headers = new Headers(init?.headers ?? {});
|
||||
const body = typeof init?.body === "string" ? init.body : undefined;
|
||||
requests.push({ method, url, authorization: headers.get("authorization"), body });
|
||||
|
||||
const json = (payload: unknown, status = 200) =>
|
||||
new Response(JSON.stringify(payload), {
|
||||
status,
|
||||
headers: { "content-type": "application/json" },
|
||||
});
|
||||
|
||||
// The source lists its releases.
|
||||
if (url.startsWith(`${SOURCE_URL}/api/v1/`)) {
|
||||
return json(options.sourceReleases);
|
||||
}
|
||||
|
||||
// Downloading an asset from the source host.
|
||||
if (url.startsWith(`${SOURCE_URL}/attachments/`) || url.startsWith("https://github.com/")) {
|
||||
return new Response(new Uint8Array([1, 2, 3, 4]), { status: 200 });
|
||||
}
|
||||
|
||||
// The destination repository exists.
|
||||
if (method === "GET" && url === DEST_API) {
|
||||
return json({ id: 1, name: DEST_REPO });
|
||||
}
|
||||
|
||||
// Does the release already exist for this tag?
|
||||
if (method === "GET" && url.startsWith(`${DEST_API}/releases/tags/`)) {
|
||||
return options.existingRelease
|
||||
? json(options.existingRelease)
|
||||
: new Response("not found", { status: 404 });
|
||||
}
|
||||
|
||||
// Is the git tag synced yet?
|
||||
if (method === "GET" && url.startsWith(`${DEST_API}/tags/`)) {
|
||||
const tag = decodeURIComponent(url.slice(`${DEST_API}/tags/`.length));
|
||||
return tags.includes(tag)
|
||||
? json({ name: tag })
|
||||
: new Response("not found", { status: 404 });
|
||||
}
|
||||
|
||||
// Existing attachments of a release.
|
||||
if (method === "GET" && /\/releases\/\d+\/assets$/.test(url)) {
|
||||
return json([]);
|
||||
}
|
||||
|
||||
// Uploading an attachment.
|
||||
if (method === "POST" && url.includes("/assets?name=")) {
|
||||
return json({ id: 900 }, 201);
|
||||
}
|
||||
|
||||
// Creating a release.
|
||||
if (method === "POST" && url === `${DEST_API}/releases`) {
|
||||
return json({ id: 77 }, 201);
|
||||
}
|
||||
|
||||
// Updating a release.
|
||||
if (method === "PATCH" && /\/releases\/\d+$/.test(url)) {
|
||||
return json({ id: 77 });
|
||||
}
|
||||
|
||||
// The retention pass lists everything the destination has.
|
||||
if (method === "GET" && url.startsWith(`${DEST_API}/releases?`)) {
|
||||
return json([]);
|
||||
}
|
||||
|
||||
throw new Error(`unexpected request: ${method} ${url}`);
|
||||
}) as unknown as typeof fetch;
|
||||
|
||||
return {
|
||||
requests,
|
||||
creates: () => requests.filter((r) => r.method === "POST" && r.url === `${DEST_API}/releases`),
|
||||
patches: () => requests.filter((r) => r.method === "PATCH"),
|
||||
uploads: () => requests.filter((r) => r.method === "POST" && r.url.includes("/assets?name=")),
|
||||
downloads: () =>
|
||||
requests.filter(
|
||||
(r) => r.url.startsWith(`${SOURCE_URL}/attachments/`) || r.url.startsWith("https://github.com/")
|
||||
),
|
||||
};
|
||||
}
|
||||
|
||||
function giteaRelease(tag: string, extra: Record<string, unknown> = {}) {
|
||||
return {
|
||||
id: 5,
|
||||
tag_name: tag,
|
||||
name: `Release ${tag}`,
|
||||
body: `notes for ${tag}`,
|
||||
draft: false,
|
||||
prerelease: false,
|
||||
created_at: "2026-01-01T00:00:00Z",
|
||||
published_at: "2026-01-01T00:00:00Z",
|
||||
assets: [],
|
||||
...extra,
|
||||
};
|
||||
}
|
||||
|
||||
function mirrorFromGitea(connection: SourceConnection) {
|
||||
return mirrorGiteaSourceReleasesToGitea({
|
||||
config,
|
||||
repository,
|
||||
connection,
|
||||
giteaOwner: DEST_OWNER,
|
||||
giteaRepoName: DEST_REPO,
|
||||
releaseLimit: 10,
|
||||
releaseAssetLimit: null,
|
||||
});
|
||||
}
|
||||
|
||||
describe.skipIf(!isChild)("mirrorGiteaSourceReleasesToGitea", () => {
|
||||
it("creates a missing release with the Gitea/Forgejo header", async () => {
|
||||
const world = fakeWorld({ sourceReleases: [giteaRelease("v1.0.0")] });
|
||||
|
||||
await mirrorFromGitea(giteaConnection("source-token"));
|
||||
|
||||
const [create] = world.creates();
|
||||
expect(create).toBeDefined();
|
||||
const payload = JSON.parse(create.body!);
|
||||
expect(payload.tag_name).toBe("v1.0.0");
|
||||
expect(payload.name).toBe("Release v1.0.0");
|
||||
expect(payload.body).toBe(
|
||||
"> 📅 **Originally published on Gitea / Forgejo:** Thu, 01 Jan 2026 00:00:00 GMT\n\nnotes for v1.0.0"
|
||||
);
|
||||
// The destination is always addressed with the destination token.
|
||||
expect(create.authorization).toBe("token gitea-token");
|
||||
});
|
||||
|
||||
it("updates an existing release whose body drifted", async () => {
|
||||
const world = fakeWorld({
|
||||
sourceReleases: [giteaRelease("v1.0.0")],
|
||||
existingRelease: {
|
||||
id: 77,
|
||||
tag_name: "v1.0.0",
|
||||
name: "Release v1.0.0",
|
||||
body: "stale body",
|
||||
},
|
||||
});
|
||||
|
||||
await mirrorFromGitea(giteaConnection("source-token"));
|
||||
|
||||
expect(world.creates()).toHaveLength(0);
|
||||
const [patch] = world.patches();
|
||||
expect(patch.url).toBe(`${DEST_API}/releases/77`);
|
||||
expect(JSON.parse(patch.body!).body).toContain(
|
||||
"**Originally published on Gitea / Forgejo:**"
|
||||
);
|
||||
});
|
||||
|
||||
it("leaves a release alone when the body and title already match", async () => {
|
||||
const world = fakeWorld({
|
||||
sourceReleases: [giteaRelease("v1.0.0")],
|
||||
existingRelease: {
|
||||
id: 77,
|
||||
tag_name: "v1.0.0",
|
||||
name: "Release v1.0.0",
|
||||
body:
|
||||
"> 📅 **Originally published on Gitea / Forgejo:** Thu, 01 Jan 2026 00:00:00 GMT\n\nnotes for v1.0.0",
|
||||
},
|
||||
});
|
||||
|
||||
await mirrorFromGitea(giteaConnection("source-token"));
|
||||
|
||||
expect(world.creates()).toHaveLength(0);
|
||||
expect(world.patches()).toHaveLength(0);
|
||||
});
|
||||
|
||||
it("skips a release whose git tag has not reached the destination yet", async () => {
|
||||
const world = fakeWorld({
|
||||
sourceReleases: [giteaRelease("v1.0.0")],
|
||||
destinationTags: [],
|
||||
});
|
||||
|
||||
await mirrorFromGitea(giteaConnection("source-token"));
|
||||
|
||||
expect(world.creates()).toHaveLength(0);
|
||||
});
|
||||
|
||||
it("downloads assets with the source instance's token, never the GitHub one", async () => {
|
||||
const world = fakeWorld({
|
||||
sourceReleases: [
|
||||
giteaRelease("v1.0.0", {
|
||||
assets: [
|
||||
{
|
||||
id: 1,
|
||||
name: "firmware.bin",
|
||||
size: 4,
|
||||
browser_download_url: `${SOURCE_URL}/attachments/abc`,
|
||||
},
|
||||
],
|
||||
}),
|
||||
],
|
||||
});
|
||||
|
||||
await mirrorFromGitea(giteaConnection("source-token"));
|
||||
|
||||
const [download] = world.downloads();
|
||||
expect(download.url).toBe(`${SOURCE_URL}/attachments/abc`);
|
||||
expect(download.authorization).toBe("token source-token");
|
||||
expect(world.uploads()).toHaveLength(1);
|
||||
// No request anywhere carried the GitHub token.
|
||||
expect(
|
||||
world.requests.some((r) => r.authorization === "token github-token")
|
||||
).toBe(false);
|
||||
});
|
||||
|
||||
it("downloads assets with no Authorization header from a tokenless public source", async () => {
|
||||
const world = fakeWorld({
|
||||
sourceReleases: [
|
||||
giteaRelease("v1.0.0", {
|
||||
assets: [
|
||||
{
|
||||
id: 1,
|
||||
name: "firmware.bin",
|
||||
size: 4,
|
||||
browser_download_url: `${SOURCE_URL}/attachments/abc`,
|
||||
},
|
||||
],
|
||||
}),
|
||||
],
|
||||
});
|
||||
|
||||
await mirrorFromGitea(giteaConnection(""));
|
||||
|
||||
const [download] = world.downloads();
|
||||
expect(download.authorization).toBeNull();
|
||||
// The source listing is anonymous too.
|
||||
const listing = world.requests.find((r) => r.url.startsWith(`${SOURCE_URL}/api/v1/`));
|
||||
expect(listing?.authorization).toBeNull();
|
||||
});
|
||||
});
|
||||
|
||||
describe.skipIf(!isChild)("mirrorGitHubReleasesToGitea keeps its own header and credentials", () => {
|
||||
function githubOctokit(releases: Array<Record<string, unknown>>) {
|
||||
return {
|
||||
rest: {
|
||||
repos: {
|
||||
listReleases: async ({ page }: { page: number }) => ({
|
||||
data: page === 1 ? releases : [],
|
||||
}),
|
||||
},
|
||||
},
|
||||
} as any;
|
||||
}
|
||||
|
||||
it("writes the GitHub header byte for byte and downloads with the GitHub token", async () => {
|
||||
const world = fakeWorld({
|
||||
sourceReleases: [],
|
||||
destinationTags: ["v1.0.0"],
|
||||
});
|
||||
|
||||
await mirrorGitHubReleasesToGitea({
|
||||
config,
|
||||
octokit: githubOctokit([
|
||||
{
|
||||
tag_name: "v1.0.0",
|
||||
name: "Release v1.0.0",
|
||||
body: "notes for v1.0.0",
|
||||
draft: false,
|
||||
prerelease: false,
|
||||
created_at: "2026-01-01T00:00:00Z",
|
||||
published_at: "2026-01-01T00:00:00Z",
|
||||
assets: [
|
||||
{
|
||||
name: "firmware.bin",
|
||||
size: 4,
|
||||
browser_download_url:
|
||||
"https://github.com/acme/demo/releases/download/v1.0.0/firmware.bin",
|
||||
},
|
||||
],
|
||||
},
|
||||
]),
|
||||
repository,
|
||||
giteaOwner: DEST_OWNER,
|
||||
giteaRepoName: DEST_REPO,
|
||||
releaseLimit: 10,
|
||||
releaseAssetLimit: null,
|
||||
});
|
||||
|
||||
const [create] = world.creates();
|
||||
expect(JSON.parse(create.body!).body).toBe(
|
||||
"> 📅 **Originally published on GitHub:** Thu, 01 Jan 2026 00:00:00 GMT\n\nnotes for v1.0.0"
|
||||
);
|
||||
|
||||
const [download] = world.downloads();
|
||||
expect(download.authorization).toBe("token github-token");
|
||||
});
|
||||
|
||||
it("sends no Authorization header when the GitHub source has no token", async () => {
|
||||
const world = fakeWorld({ sourceReleases: [], destinationTags: ["v1.0.0"] });
|
||||
|
||||
await mirrorGitHubReleasesToGitea({
|
||||
config: {
|
||||
...config,
|
||||
githubConfig: { ...(config.githubConfig as any), token: "" },
|
||||
} as Partial<Config>,
|
||||
octokit: githubOctokit([
|
||||
{
|
||||
tag_name: "v1.0.0",
|
||||
name: "Release v1.0.0",
|
||||
body: "",
|
||||
draft: false,
|
||||
prerelease: false,
|
||||
created_at: "2026-01-01T00:00:00Z",
|
||||
published_at: "2026-01-01T00:00:00Z",
|
||||
assets: [
|
||||
{
|
||||
name: "firmware.bin",
|
||||
size: 4,
|
||||
browser_download_url:
|
||||
"https://github.com/acme/demo/releases/download/v1.0.0/firmware.bin",
|
||||
},
|
||||
],
|
||||
},
|
||||
]),
|
||||
repository,
|
||||
giteaOwner: DEST_OWNER,
|
||||
giteaRepoName: DEST_REPO,
|
||||
releaseLimit: 10,
|
||||
releaseAssetLimit: null,
|
||||
});
|
||||
|
||||
const [download] = world.downloads();
|
||||
expect(download.authorization).toBeNull();
|
||||
});
|
||||
|
||||
it("prefers the repository's own GitHub source token over the primary config token", async () => {
|
||||
const world = fakeWorld({ sourceReleases: [], destinationTags: ["v1.0.0"] });
|
||||
|
||||
await mirrorGitHubReleasesToGitea({
|
||||
config,
|
||||
octokit: githubOctokit([
|
||||
{
|
||||
tag_name: "v1.0.0",
|
||||
name: "Release v1.0.0",
|
||||
body: "",
|
||||
draft: false,
|
||||
prerelease: false,
|
||||
created_at: "2026-01-01T00:00:00Z",
|
||||
published_at: "2026-01-01T00:00:00Z",
|
||||
assets: [
|
||||
{
|
||||
name: "firmware.bin",
|
||||
size: 4,
|
||||
browser_download_url:
|
||||
"https://github.com/acme/demo/releases/download/v1.0.0/firmware.bin",
|
||||
},
|
||||
],
|
||||
},
|
||||
]),
|
||||
repository,
|
||||
sourceToken: "ghes-token",
|
||||
giteaOwner: DEST_OWNER,
|
||||
giteaRepoName: DEST_REPO,
|
||||
releaseLimit: 10,
|
||||
releaseAssetLimit: null,
|
||||
});
|
||||
|
||||
const [download] = world.downloads();
|
||||
expect(download.authorization).toBe("token ghes-token");
|
||||
});
|
||||
});
|
||||
+1324
-293
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,141 @@
|
||||
/**
|
||||
* The GitHub clients built by github.ts hold their requests while the gate
|
||||
* is closed, and the secondary limit handler closes the gate for everyone
|
||||
* (issue #437, second report). Uses a fake global fetch so nothing leaves
|
||||
* the process; the point of these tests is that nothing does.
|
||||
*/
|
||||
|
||||
import { afterEach, beforeEach, describe, expect, test } from "bun:test";
|
||||
import {
|
||||
createGitHubClient,
|
||||
createPublicGitHubClient,
|
||||
githubThrottleOptions,
|
||||
} from "@/lib/github";
|
||||
import {
|
||||
RateLimitPausedError,
|
||||
clearAllRateLimits,
|
||||
isRateLimitError,
|
||||
markRateLimited,
|
||||
rateLimitGateKey,
|
||||
rateLimitedUntil,
|
||||
} from "@/lib/rate-limit-gate";
|
||||
|
||||
const realFetch = globalThis.fetch;
|
||||
|
||||
describe("GitHub clients while the gate is closed", () => {
|
||||
let sent: string[] = [];
|
||||
|
||||
beforeEach(() => {
|
||||
clearAllRateLimits();
|
||||
sent = [];
|
||||
globalThis.fetch = (async (input: any) => {
|
||||
const url = typeof input === "string" ? input : input.url;
|
||||
sent.push(url);
|
||||
return new Response(JSON.stringify({ resources: {}, rate: { remaining: 1 } }), {
|
||||
status: 200,
|
||||
headers: {
|
||||
"content-type": "application/json",
|
||||
"x-ratelimit-remaining": "4999",
|
||||
"x-ratelimit-reset": String(Math.floor(Date.now() / 1000) + 3600),
|
||||
},
|
||||
});
|
||||
}) as unknown as typeof fetch;
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
globalThis.fetch = realFetch;
|
||||
clearAllRateLimits();
|
||||
});
|
||||
|
||||
test("an authenticated client sends nothing during a long pause and fails with the paused error", async () => {
|
||||
const octokit = createGitHubClient("ghp_test", "user-1");
|
||||
markRateLimited(rateLimitGateKey("user-1"), Date.now() + 3_263_000);
|
||||
|
||||
let caught: unknown;
|
||||
try {
|
||||
await octokit.rest.repos.get({ owner: "a", repo: "b" });
|
||||
} catch (error) {
|
||||
caught = error;
|
||||
}
|
||||
expect(caught).toBeInstanceOf(RateLimitPausedError);
|
||||
expect(isRateLimitError(caught)).toBe(true);
|
||||
expect(sent).toEqual([]);
|
||||
});
|
||||
|
||||
test("paginate stops at the gate too, before the first page", async () => {
|
||||
const octokit = createGitHubClient("ghp_test", "user-2");
|
||||
markRateLimited(rateLimitGateKey("user-2"), Date.now() + 3_600_000);
|
||||
|
||||
await expect(
|
||||
octokit.paginate(octokit.rest.issues.listForRepo, { owner: "a", repo: "b", per_page: 100 })
|
||||
).rejects.toBeInstanceOf(RateLimitPausedError);
|
||||
expect(sent).toEqual([]);
|
||||
});
|
||||
|
||||
test("the rate limit probe still goes out, so the pause can be confirmed or lifted", async () => {
|
||||
const octokit = createGitHubClient("ghp_test", "user-3");
|
||||
markRateLimited(rateLimitGateKey("user-3"), Date.now() + 3_600_000);
|
||||
|
||||
await octokit.rest.rateLimit.get();
|
||||
expect(sent).toHaveLength(1);
|
||||
expect(sent[0]).toContain("/rate_limit");
|
||||
// The probe answered with budget left, which opens the gate again.
|
||||
expect(rateLimitedUntil(rateLimitGateKey("user-3"))).toBeNull();
|
||||
await octokit.rest.repos.get({ owner: "a", repo: "b" });
|
||||
expect(sent).toHaveLength(2);
|
||||
});
|
||||
|
||||
test("another user's pause does not hold this user's client", async () => {
|
||||
const octokit = createGitHubClient("ghp_test", "user-4");
|
||||
markRateLimited(rateLimitGateKey("user-5"), Date.now() + 3_600_000);
|
||||
|
||||
await octokit.rest.repos.get({ owner: "a", repo: "b" });
|
||||
expect(sent).toHaveLength(1);
|
||||
});
|
||||
|
||||
test("tokenless clients share the anonymous gate", async () => {
|
||||
const octokit = createPublicGitHubClient();
|
||||
markRateLimited(rateLimitGateKey(undefined), Date.now() + 3_600_000);
|
||||
|
||||
await expect(octokit.rest.repos.get({ owner: "a", repo: "b" })).rejects.toBeInstanceOf(
|
||||
RateLimitPausedError
|
||||
);
|
||||
expect(sent).toEqual([]);
|
||||
});
|
||||
});
|
||||
|
||||
describe("secondary rate limit handler", () => {
|
||||
beforeEach(() => {
|
||||
clearAllRateLimits();
|
||||
});
|
||||
|
||||
test("closes the gate for the retry-after window and still retries the request itself", async () => {
|
||||
const before = Date.now();
|
||||
const options = githubThrottleOptions("user-secondary");
|
||||
const retry = await options.onSecondaryRateLimit(
|
||||
60,
|
||||
{ method: "GET", url: "/repos/a/b/issues" },
|
||||
{},
|
||||
0
|
||||
);
|
||||
expect(retry).toBe(true);
|
||||
|
||||
const until = rateLimitedUntil(rateLimitGateKey("user-secondary"));
|
||||
expect(until).not.toBeNull();
|
||||
const pausedFor = until!.getTime() - before;
|
||||
expect(pausedFor).toBeGreaterThanOrEqual(59_000);
|
||||
expect(pausedFor).toBeLessThanOrEqual(61_000);
|
||||
});
|
||||
|
||||
test("gives up after two retries but the gate stays closed for the window", async () => {
|
||||
const options = githubThrottleOptions("user-secondary-2");
|
||||
expect(await options.onSecondaryRateLimit(30, { method: "GET", url: "/x" }, {}, 2)).toBe(false);
|
||||
expect(rateLimitedUntil(rateLimitGateKey("user-secondary-2"))).not.toBeNull();
|
||||
});
|
||||
|
||||
test("a tokenless client records the pause under the anonymous key", async () => {
|
||||
const options = githubThrottleOptions();
|
||||
await options.onSecondaryRateLimit(10, { method: "GET", url: "/x" }, {}, 0);
|
||||
expect(rateLimitedUntil(rateLimitGateKey(undefined))).not.toBeNull();
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,62 @@
|
||||
import { describe, expect, test } from "bun:test";
|
||||
import {
|
||||
RATE_LIMIT_FAST_FAIL_THRESHOLD_SECONDS,
|
||||
decideRateLimitRetry,
|
||||
} from "@/lib/github";
|
||||
|
||||
// The decision the Octokit throttling plugin's onRateLimit handler makes,
|
||||
// pulled out of the handler so the policy can be checked without a client.
|
||||
describe("decideRateLimitRetry", () => {
|
||||
const maxRetries = 3;
|
||||
|
||||
test("short waits are still retried inside the request", () => {
|
||||
expect(decideRateLimitRetry({ retryAfter: 5, retryCount: 0, maxRetries })).toBe("retry");
|
||||
expect(decideRateLimitRetry({ retryAfter: 60, retryCount: 2, maxRetries })).toBe("retry");
|
||||
expect(
|
||||
decideRateLimitRetry({
|
||||
retryAfter: RATE_LIMIT_FAST_FAIL_THRESHOLD_SECONDS,
|
||||
retryCount: 0,
|
||||
maxRetries,
|
||||
})
|
||||
).toBe("retry");
|
||||
});
|
||||
|
||||
test("short waits stop once the retries are used up", () => {
|
||||
expect(decideRateLimitRetry({ retryAfter: 30, retryCount: 3, maxRetries })).toBe("give-up");
|
||||
expect(decideRateLimitRetry({ retryAfter: 30, retryCount: 9, maxRetries })).toBe("give-up");
|
||||
});
|
||||
|
||||
test("a wait past the threshold fails fast on the first hit, whatever the retry count", () => {
|
||||
// The reporter's log showed a 3263s wait, which used to be slept off
|
||||
// inside the scheduler lock three times over (issue #437).
|
||||
expect(decideRateLimitRetry({ retryAfter: 3263, retryCount: 0, maxRetries })).toBe("fail-fast");
|
||||
expect(decideRateLimitRetry({ retryAfter: 3263, retryCount: 2, maxRetries })).toBe("fail-fast");
|
||||
expect(
|
||||
decideRateLimitRetry({
|
||||
retryAfter: RATE_LIMIT_FAST_FAIL_THRESHOLD_SECONDS + 1,
|
||||
retryCount: 0,
|
||||
maxRetries,
|
||||
})
|
||||
).toBe("fail-fast");
|
||||
});
|
||||
|
||||
test("search endpoints get their extra retries for short waits only", () => {
|
||||
const searchRetries = 5;
|
||||
expect(decideRateLimitRetry({ retryAfter: 10, retryCount: 4, maxRetries: searchRetries })).toBe("retry");
|
||||
expect(decideRateLimitRetry({ retryAfter: 10, retryCount: 5, maxRetries: searchRetries })).toBe("give-up");
|
||||
expect(decideRateLimitRetry({ retryAfter: 900, retryCount: 0, maxRetries: searchRetries })).toBe("fail-fast");
|
||||
});
|
||||
|
||||
test("the threshold is configurable for callers that want a different budget", () => {
|
||||
expect(
|
||||
decideRateLimitRetry({ retryAfter: 45, retryCount: 0, maxRetries, thresholdSeconds: 30 })
|
||||
).toBe("fail-fast");
|
||||
expect(
|
||||
decideRateLimitRetry({ retryAfter: 45, retryCount: 0, maxRetries, thresholdSeconds: 600 })
|
||||
).toBe("retry");
|
||||
});
|
||||
|
||||
test("the default threshold keeps waits to two minutes", () => {
|
||||
expect(RATE_LIMIT_FAST_FAIL_THRESHOLD_SECONDS).toBe(120);
|
||||
});
|
||||
});
|
||||
+112
-12
@@ -8,6 +8,12 @@ import {
|
||||
applyConditionalRequests,
|
||||
conditionalRequestTokenScope,
|
||||
} from "@/lib/github-conditional-requests";
|
||||
import {
|
||||
clearRateLimit,
|
||||
installRateLimitGate,
|
||||
markRateLimited,
|
||||
rateLimitGateKey,
|
||||
} from "@/lib/rate-limit-gate";
|
||||
// Conditionally import rate limit manager (not available in test environment)
|
||||
let RateLimitManager: any = null;
|
||||
let publishEvent: any = null;
|
||||
@@ -29,13 +35,53 @@ const MyOctokit: any = (Octokit as any)?.plugin?.call
|
||||
? (Octokit as any).plugin(throttling)
|
||||
: (Octokit as any);
|
||||
|
||||
/**
|
||||
* Longest wait a request may sleep through inside the throttling plugin.
|
||||
* A primary rate limit resets at a fixed time that is often tens of minutes
|
||||
* away; sleeping that long inside the request holds the scheduler lock for the
|
||||
* whole run (issue #437), so anything longer is handed to the rate limit gate
|
||||
* and the request fails fast instead.
|
||||
*/
|
||||
export const RATE_LIMIT_FAST_FAIL_THRESHOLD_SECONDS = 120;
|
||||
|
||||
export type RateLimitRetryAction = "retry" | "fail-fast" | "give-up";
|
||||
|
||||
/**
|
||||
* What the throttle handler should do with a primary rate limit. Pure, so the
|
||||
* policy can be tested without standing up Octokit.
|
||||
*
|
||||
* - "retry": short wait, sleep it off inside the request as before.
|
||||
* - "fail-fast": the reset is too far out to wait for; pause the source and
|
||||
* let the caller fail so the scheduler can end its run and come back.
|
||||
* - "give-up": short waits were retried to the limit and got nowhere.
|
||||
*/
|
||||
export function decideRateLimitRetry({
|
||||
retryAfter,
|
||||
retryCount,
|
||||
maxRetries,
|
||||
thresholdSeconds = RATE_LIMIT_FAST_FAIL_THRESHOLD_SECONDS,
|
||||
}: {
|
||||
retryAfter: number;
|
||||
retryCount: number;
|
||||
maxRetries: number;
|
||||
thresholdSeconds?: number;
|
||||
}): RateLimitRetryAction {
|
||||
if (retryAfter > thresholdSeconds) {
|
||||
return "fail-fast";
|
||||
}
|
||||
if (retryCount < maxRetries) {
|
||||
return "retry";
|
||||
}
|
||||
return "give-up";
|
||||
}
|
||||
|
||||
/**
|
||||
* Throttling and rate-limit handling shared by every Octokit this module
|
||||
* builds, authenticated or anonymous. `userId` is optional: without it the
|
||||
* handlers only log and schedule the retry — there is no per-user
|
||||
* rate-limit state or UI event to update for a tokenless source.
|
||||
*/
|
||||
function githubThrottleOptions(userId?: string) {
|
||||
export function githubThrottleOptions(userId?: string) {
|
||||
return {
|
||||
onRateLimit: async (
|
||||
retryAfter: number,
|
||||
@@ -45,10 +91,23 @@ function githubThrottleOptions(userId?: string) {
|
||||
) => {
|
||||
const isSearch = options.url.includes("/search/");
|
||||
const maxRetries = isSearch ? 5 : 3; // Search endpoints get more retries
|
||||
const action = decideRateLimitRetry({ retryAfter, retryCount, maxRetries });
|
||||
const resetAt = new Date(Date.now() + retryAfter * 1000);
|
||||
|
||||
console.warn(
|
||||
`[GitHub] Rate limit hit for ${options.method} ${options.url}. Retry ${retryCount + 1}/${maxRetries}`,
|
||||
);
|
||||
if (action === "fail-fast") {
|
||||
// Record the reset for the whole source, not just this request: every
|
||||
// other call on the same token would run into the same wall.
|
||||
markRateLimited(rateLimitGateKey(userId), resetAt);
|
||||
console.warn(
|
||||
`[GitHub] Rate limit hit for ${options.method} ${options.url}. ` +
|
||||
`The limit resets in ${retryAfter}s, over the ${RATE_LIMIT_FAST_FAIL_THRESHOLD_SECONDS}s a request may wait, ` +
|
||||
`so this request fails now and GitHub work pauses until ${resetAt.toISOString()}.`,
|
||||
);
|
||||
} else {
|
||||
console.warn(
|
||||
`[GitHub] Rate limit hit for ${options.method} ${options.url}. Retry ${retryCount + 1}/${maxRetries}`,
|
||||
);
|
||||
}
|
||||
|
||||
// Update rate limit status and notify UI (if available)
|
||||
if (userId && RateLimitManager) {
|
||||
@@ -69,21 +128,28 @@ function githubThrottleOptions(userId?: string) {
|
||||
retryAfter,
|
||||
retryCount,
|
||||
endpoint: options.url,
|
||||
message: `Rate limit hit. Waiting ${retryAfter}s before retry ${retryCount + 1}/${maxRetries}...`,
|
||||
resetAt: resetAt.toISOString(),
|
||||
willRetry: action === "retry",
|
||||
message:
|
||||
action === "fail-fast"
|
||||
? `Rate limit exceeded. GitHub work is paused until ${resetAt.toISOString()} and resumes automatically.`
|
||||
: `Rate limit hit. Waiting ${retryAfter}s before retry ${retryCount + 1}/${maxRetries}...`,
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
// Retry with exponential backoff
|
||||
if (retryCount < maxRetries) {
|
||||
// Short waits are still slept off inside the request.
|
||||
if (action === "retry") {
|
||||
console.log(`[GitHub] Waiting ${retryAfter}s before retry...`);
|
||||
return true;
|
||||
}
|
||||
|
||||
// Max retries reached
|
||||
console.error(
|
||||
`[GitHub] Max retries (${maxRetries}) reached for ${options.url}`,
|
||||
);
|
||||
if (action === "give-up") {
|
||||
console.error(
|
||||
`[GitHub] Max retries (${maxRetries}) reached for ${options.url}`,
|
||||
);
|
||||
}
|
||||
|
||||
return false;
|
||||
},
|
||||
onSecondaryRateLimit: async (
|
||||
@@ -96,6 +162,11 @@ function githubThrottleOptions(userId?: string) {
|
||||
`[GitHub] Secondary rate limit hit for ${options.method} ${options.url}`,
|
||||
);
|
||||
|
||||
// A secondary limit is GitHub asking the whole client to slow down,
|
||||
// not only this request. Close the gate for the retry-after window so
|
||||
// the other queued calls wait instead of piling on (issue #437).
|
||||
markRateLimited(rateLimitGateKey(userId), Date.now() + retryAfter * 1000);
|
||||
|
||||
// Update status and notify UI (if available)
|
||||
if (userId && publishEvent) {
|
||||
await publishEvent({
|
||||
@@ -170,6 +241,19 @@ export function createGitHubClient(
|
||||
throttle: githubThrottleOptions(userId),
|
||||
});
|
||||
|
||||
// A response that still has budget means the limit is over, whatever reset
|
||||
// time was recorded earlier. Open the gate so the scheduler resumes at once
|
||||
// instead of sitting out the rest of a stale window. Some tests stub Octokit
|
||||
// without the hook system; skip wiring in that case.
|
||||
if (typeof (octokit as any)?.hook?.after === "function") {
|
||||
octokit.hook.after("request", async (response: any) => {
|
||||
const remaining = Number(response?.headers?.["x-ratelimit-remaining"]);
|
||||
if (Number.isFinite(remaining) && remaining > 0) {
|
||||
clearRateLimit(rateLimitGateKey(userId));
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
// Add rate limit tracking hooks if userId is provided and RateLimitManager is available
|
||||
if (userId && RateLimitManager) {
|
||||
octokit.hook.after("request", async (response: any, _options: any) => {
|
||||
@@ -228,6 +312,13 @@ export function createGitHubClient(
|
||||
scope: userId ?? username ?? conditionalRequestTokenScope(token),
|
||||
});
|
||||
|
||||
// Registered last so it runs first: a closed gate is answered before the
|
||||
// conditional request cache and before anything reaches the network.
|
||||
installRateLimitGate(octokit, {
|
||||
key: rateLimitGateKey(userId),
|
||||
maxWaitMs: RATE_LIMIT_FAST_FAIL_THRESHOLD_SECONDS * 1000,
|
||||
});
|
||||
|
||||
return octokit;
|
||||
}
|
||||
|
||||
@@ -250,7 +341,7 @@ export function createPublicGitHubClient(apiBaseUrl?: string): Octokit {
|
||||
// with GitHub Actions, which sets it to https://api.github.com by default).
|
||||
const baseUrl = apiBaseUrl || process.env.GH_API_URL || process.env.GITHUB_API_URL || "https://api.github.com";
|
||||
|
||||
return new MyOctokit({
|
||||
const octokit = new MyOctokit({
|
||||
userAgent: "gitea-mirror/3.5.4",
|
||||
baseUrl, // Configurable for E2E testing
|
||||
log: {
|
||||
@@ -267,6 +358,15 @@ export function createPublicGitHubClient(apiBaseUrl?: string): Octokit {
|
||||
},
|
||||
throttle: githubThrottleOptions(),
|
||||
});
|
||||
|
||||
// Anonymous clients share one gate key, so one exhausted 60 req/hr budget
|
||||
// pauses every tokenless source together.
|
||||
installRateLimitGate(octokit, {
|
||||
key: rateLimitGateKey(undefined),
|
||||
maxWaitMs: RATE_LIMIT_FAST_FAIL_THRESHOLD_SECONDS * 1000,
|
||||
});
|
||||
|
||||
return octokit;
|
||||
}
|
||||
|
||||
/**
|
||||
|
||||
+42
-11
@@ -4,6 +4,7 @@ import { eq, and } from "drizzle-orm";
|
||||
import { buildInterruptedJobsCondition } from "./interrupted-job-detection";
|
||||
import { v4 as uuidv4 } from "uuid";
|
||||
import { publishEvent } from "./events";
|
||||
import { withKeyedLock } from "./utils/keyed-mutex";
|
||||
import { triggerJobNotification } from "./notification-service";
|
||||
|
||||
export async function createMirrorJob({
|
||||
@@ -113,17 +114,18 @@ export async function createMirrorJob({
|
||||
}
|
||||
|
||||
/**
|
||||
* Updates the progress of a mirror job
|
||||
* Updates the progress of a mirror job.
|
||||
*
|
||||
* Progress is a read-modify-write of the completedItemIds list, and items
|
||||
* finish concurrently, so the update runs under a per-job lock: two items
|
||||
* completing in the same tick used to read the same list and one of them
|
||||
* silently overwrote the other. A resume then re-processed the lost item.
|
||||
*
|
||||
* `silent` records the progress without publishing an event. Every completed
|
||||
* item is checkpointed so a resume knows exactly what is done; the events the
|
||||
* dashboard listens to are throttled by the caller instead.
|
||||
*/
|
||||
export async function updateMirrorJobProgress({
|
||||
jobId,
|
||||
completedItemId,
|
||||
status,
|
||||
message,
|
||||
details,
|
||||
inProgress,
|
||||
isCompleted,
|
||||
}: {
|
||||
export async function updateMirrorJobProgress(params: {
|
||||
jobId: string;
|
||||
completedItemId?: string;
|
||||
status?: RepoStatus;
|
||||
@@ -131,7 +133,25 @@ export async function updateMirrorJobProgress({
|
||||
details?: string;
|
||||
inProgress?: boolean;
|
||||
isCompleted?: boolean;
|
||||
/** Record a new start time, as when recovery resumes the job. */
|
||||
startedAt?: Date;
|
||||
/** Write the progress without publishing a mirror-status event. */
|
||||
silent?: boolean;
|
||||
}) {
|
||||
return withKeyedLock(`mirror-job:${params.jobId}`, () => updateMirrorJobProgressLocked(params));
|
||||
}
|
||||
|
||||
async function updateMirrorJobProgressLocked({
|
||||
jobId,
|
||||
completedItemId,
|
||||
status,
|
||||
message,
|
||||
details,
|
||||
inProgress,
|
||||
isCompleted,
|
||||
startedAt,
|
||||
silent = false,
|
||||
}: Parameters<typeof updateMirrorJobProgress>[0]) {
|
||||
try {
|
||||
// Get the current job
|
||||
const [job] = await db
|
||||
@@ -157,6 +177,10 @@ export async function updateMirrorJobProgress({
|
||||
}
|
||||
}
|
||||
|
||||
if (startedAt) {
|
||||
updates.startedAt = startedAt;
|
||||
}
|
||||
|
||||
// Update status if provided
|
||||
if (status) {
|
||||
updates.status = status;
|
||||
@@ -195,6 +219,10 @@ export async function updateMirrorJobProgress({
|
||||
...updates,
|
||||
};
|
||||
|
||||
if (silent) {
|
||||
return updatedJob;
|
||||
}
|
||||
|
||||
// Create deduplication key for progress updates
|
||||
let deduplicationKey: string | undefined;
|
||||
if (completedItemId) {
|
||||
@@ -324,12 +352,15 @@ export async function resumeInterruptedJob(job: any) {
|
||||
return null;
|
||||
}
|
||||
|
||||
// Update the job to show it's being resumed
|
||||
// Update the job to show it's being resumed. The start time moves to
|
||||
// now so the row records this pass; the fresh checkpoint written here is
|
||||
// what keeps a second recovery sweep from resuming the same job again.
|
||||
await updateMirrorJobProgress({
|
||||
jobId: job.id,
|
||||
message: `Resuming job with ${remainingItemIds.length} remaining items`,
|
||||
details: `Job was interrupted and is being resumed. ${job.completedItemIds.length} of ${job.itemIds.length} items were already processed.`,
|
||||
inProgress: true,
|
||||
startedAt: new Date(),
|
||||
});
|
||||
|
||||
return {
|
||||
|
||||
+8
-16
@@ -41,25 +41,17 @@ export async function httpRequest<T = any>(
|
||||
const responseClone = response.clone();
|
||||
|
||||
if (!response.ok) {
|
||||
let errorMessage = `HTTP ${response.status}: ${response.statusText}`;
|
||||
// The message carries the status only. The upstream body is kept on
|
||||
// `HttpError.response` for logs and for callers that inspect it, but
|
||||
// it never goes into the message: messages can reach API clients
|
||||
// through createSecureErrorResponse, and a URL under the user's
|
||||
// control would otherwise turn any error into a readable proxy of
|
||||
// whatever host it pointed at.
|
||||
const errorMessage = `HTTP ${response.status}: ${response.statusText}`;
|
||||
let responseText = '';
|
||||
|
||||
try {
|
||||
responseText = await responseClone.text();
|
||||
if (responseText) {
|
||||
// Try to parse as JSON for better error messages
|
||||
try {
|
||||
const errorData = JSON.parse(responseText);
|
||||
if (errorData.message) {
|
||||
errorMessage = `HTTP ${response.status}: ${errorData.message}`;
|
||||
} else {
|
||||
errorMessage += ` - ${responseText}`;
|
||||
}
|
||||
} catch {
|
||||
// Not JSON, use as-is
|
||||
errorMessage += ` - ${responseText}`;
|
||||
}
|
||||
}
|
||||
} catch {
|
||||
// Ignore text parsing errors
|
||||
}
|
||||
@@ -104,7 +96,7 @@ export async function httpRequest<T = any>(
|
||||
}
|
||||
|
||||
throw new HttpError(
|
||||
`Failed to parse JSON response from ${url}: ${jsonError instanceof Error ? jsonError.message : String(jsonError)}. Response: ${responseText.substring(0, 200)}${responseText.length > 200 ? '...' : ''}`,
|
||||
`Failed to parse JSON response from ${url}: ${jsonError instanceof Error ? jsonError.message : String(jsonError)}`,
|
||||
response.status,
|
||||
response.statusText,
|
||||
responseText
|
||||
|
||||
@@ -18,7 +18,6 @@ import { drizzle } from "drizzle-orm/bun-sqlite";
|
||||
import { mirrorJobs } from "@/lib/db/schema";
|
||||
import {
|
||||
INTERRUPTED_CHECKPOINT_AGE_MS,
|
||||
STALE_JOB_AGE_MS,
|
||||
JOB_HEARTBEAT_INTERVAL_MS,
|
||||
computeInterruptedJobCutoffs,
|
||||
isJobInterrupted,
|
||||
@@ -32,14 +31,15 @@ const NOW = new Date("2026-09-02T12:00:00Z");
|
||||
const ago = (ms: number) => new Date(NOW.getTime() - ms);
|
||||
|
||||
describe("computeInterruptedJobCutoffs", () => {
|
||||
test("keeps the 10 minute checkpoint window and 2 hour stale window", () => {
|
||||
test("keeps the 10 minute checkpoint window and has no stale age cutoff", () => {
|
||||
expect(INTERRUPTED_CHECKPOINT_AGE_MS).toBe(10 * MINUTE);
|
||||
expect(STALE_JOB_AGE_MS).toBe(2 * HOUR);
|
||||
|
||||
const cutoffs = computeInterruptedJobCutoffs(NOW);
|
||||
|
||||
expect(cutoffs.checkpointCutoff.getTime()).toBe(NOW.getTime() - 10 * MINUTE);
|
||||
expect(cutoffs.staleCutoff.getTime()).toBe(NOW.getTime() - 2 * HOUR);
|
||||
// The two hour "stale regardless of checkpoint" rule resumed live jobs
|
||||
// and is gone; a checkpointing job is alive however old it is.
|
||||
expect(Object.keys(cutoffs)).toEqual(["checkpointCutoff"]);
|
||||
});
|
||||
|
||||
test("heartbeat interval leaves room for a missed beat inside the checkpoint window", () => {
|
||||
@@ -78,9 +78,18 @@ describe("isJobInterrupted", () => {
|
||||
).toBe(true);
|
||||
});
|
||||
|
||||
test("a job running past the stale window is interrupted even with a fresh checkpoint", () => {
|
||||
test("a job that keeps checkpointing stays live however long ago it started", () => {
|
||||
expect(
|
||||
isJobInterrupted({ inProgress: true, startedAt: ago(3 * HOUR), lastCheckpoint: ago(MINUTE) }, NOW)
|
||||
).toBe(false);
|
||||
expect(
|
||||
isJobInterrupted({ inProgress: true, startedAt: ago(30 * HOUR), lastCheckpoint: ago(2 * MINUTE) }, NOW)
|
||||
).toBe(false);
|
||||
});
|
||||
|
||||
test("an old job whose checkpoint went stale is still interrupted", () => {
|
||||
expect(
|
||||
isJobInterrupted({ inProgress: true, startedAt: ago(3 * HOUR), lastCheckpoint: ago(11 * MINUTE) }, NOW)
|
||||
).toBe(true);
|
||||
});
|
||||
|
||||
@@ -235,6 +244,22 @@ describe("wiring", () => {
|
||||
).toBe(true);
|
||||
});
|
||||
|
||||
test("resumeInterruptedJob records a fresh start time for the resumed pass", () => {
|
||||
const resumeBody = HELPERS_SRC.slice(HELPERS_SRC.indexOf("export async function resumeInterruptedJob("));
|
||||
expect(/inProgress:\s*true,\s*startedAt:\s*new Date\(\)/.test(resumeBody)).toBe(true);
|
||||
});
|
||||
|
||||
test("updateMirrorJobProgress serializes writes per job so concurrent items cannot drop each other", () => {
|
||||
expect(
|
||||
/withKeyedLock\(`mirror-job:\$\{params\.jobId\}`,\s*\(\)\s*=>\s*updateMirrorJobProgressLocked\(params\)\)/.test(HELPERS_SRC)
|
||||
).toBe(true);
|
||||
});
|
||||
|
||||
test("processWithRetry checkpoints every completed item instead of the one that trips the interval", () => {
|
||||
expect(/await onCheckpoint\(jobId,\s*itemId,\s*publish\)/.test(CONCURRENCY_SRC)).toBe(true);
|
||||
expect(CONCURRENCY_SRC.includes("onCheckpoint(jobId, 'final')")).toBe(false);
|
||||
});
|
||||
|
||||
test("middleware keeps the recovery latch until the recovery promise settles", () => {
|
||||
expect(
|
||||
/recoveryPromise\s*\.catch\(\(\)\s*=>\s*false\)\s*\.finally\(\(\)\s*=>\s*\{\s*recoveryInFlight\s*=\s*false;\s*\}\)/.test(
|
||||
|
||||
@@ -2,11 +2,20 @@
|
||||
* Decides which in-progress mirror jobs count as interrupted and should be
|
||||
* resumed by recovery (issue #372).
|
||||
*
|
||||
* A job is interrupted when any of these hold:
|
||||
* A job is interrupted when either of these holds:
|
||||
* - it has a checkpoint older than the checkpoint window, or
|
||||
* - it has no checkpoint and is older than the checkpoint window (or has
|
||||
* no recorded start at all, as legacy rows may), or
|
||||
* - it started longer ago than the stale window, whatever its checkpoint.
|
||||
* no recorded start at all, as legacy rows may).
|
||||
*
|
||||
* A job that keeps checkpointing is alive, however long ago it started.
|
||||
* Running jobs refresh their checkpoint on a heartbeat between items, so a
|
||||
* missing or old checkpoint is the only evidence that the process behind a
|
||||
* job is gone. There used to be a third rule, "started more than two hours
|
||||
* ago, whatever the checkpoint says", which resumed large jobs while they
|
||||
* were still running: the resumed pass then re-processed the same
|
||||
* repositories alongside the original, which is one way release assets got
|
||||
* duplicated (#417). A job that hangs without ever finishing an item is a
|
||||
* different problem and is not what recovery is for.
|
||||
*
|
||||
* The grace period for jobs without a checkpoint is the point of this
|
||||
* module. Checkpoints are written when an item completes, so a job that has
|
||||
@@ -24,9 +33,6 @@ import type { mirrorJobs } from "@/lib/db/schema";
|
||||
/** A job with no checkpoint newer than this is considered inactive. */
|
||||
export const INTERRUPTED_CHECKPOINT_AGE_MS = 10 * 60 * 1000;
|
||||
|
||||
/** A job that started longer ago than this is considered stale regardless. */
|
||||
export const STALE_JOB_AGE_MS = 2 * 60 * 60 * 1000;
|
||||
|
||||
/**
|
||||
* How often a running job refreshes its checkpoint between item
|
||||
* completions. Kept well inside the checkpoint window so one missed beat
|
||||
@@ -37,14 +43,11 @@ export const JOB_HEARTBEAT_INTERVAL_MS = 2 * 60 * 1000;
|
||||
export interface InterruptedJobCutoffs {
|
||||
/** Checkpoints (or starts, for jobs without one) before this are too old. */
|
||||
checkpointCutoff: Date;
|
||||
/** Starts before this mark the job stale no matter what. */
|
||||
staleCutoff: Date;
|
||||
}
|
||||
|
||||
export function computeInterruptedJobCutoffs(now: Date = new Date()): InterruptedJobCutoffs {
|
||||
return {
|
||||
checkpointCutoff: new Date(now.getTime() - INTERRUPTED_CHECKPOINT_AGE_MS),
|
||||
staleCutoff: new Date(now.getTime() - STALE_JOB_AGE_MS),
|
||||
};
|
||||
}
|
||||
|
||||
@@ -61,10 +64,9 @@ export interface JobLivenessFields {
|
||||
export function isJobInterrupted(job: JobLivenessFields, now: Date = new Date()): boolean {
|
||||
if (!job.inProgress) return false;
|
||||
|
||||
const { checkpointCutoff, staleCutoff } = computeInterruptedJobCutoffs(now);
|
||||
|
||||
if (job.startedAt && job.startedAt < staleCutoff) return true;
|
||||
const { checkpointCutoff } = computeInterruptedJobCutoffs(now);
|
||||
|
||||
// A checkpoint is the job's own proof of life; its age decides on its own.
|
||||
if (job.lastCheckpoint) return job.lastCheckpoint < checkpointCutoff;
|
||||
|
||||
// No checkpoint yet: give the job the same window it would get after a
|
||||
@@ -82,7 +84,7 @@ export function buildInterruptedJobsCondition(
|
||||
table: typeof mirrorJobs,
|
||||
now: Date = new Date(),
|
||||
): SQL {
|
||||
const { checkpointCutoff, staleCutoff } = computeInterruptedJobCutoffs(now);
|
||||
const { checkpointCutoff } = computeInterruptedJobCutoffs(now);
|
||||
|
||||
return and(
|
||||
eq(table.inProgress, true),
|
||||
@@ -95,8 +97,6 @@ export function buildInterruptedJobsCondition(
|
||||
),
|
||||
// Checkpoint present but too old.
|
||||
lt(table.lastCheckpoint, checkpointCutoff),
|
||||
// Running far too long, whatever the checkpoint says.
|
||||
lt(table.startedAt, staleCutoff),
|
||||
),
|
||||
)!;
|
||||
}
|
||||
@@ -0,0 +1,44 @@
|
||||
/**
|
||||
* Write the progress of an unfinished issues or pull request pass as soon
|
||||
* as it is known (#449 follow-up). A rate limit ends the sync with a throw,
|
||||
* before the usual metadata write at the end of it, so without this the
|
||||
* next run would start the pass over from the first item.
|
||||
*/
|
||||
|
||||
import { eq } from "drizzle-orm";
|
||||
import { db, repositories } from "./db";
|
||||
import {
|
||||
serializeRepositoryMetadataState,
|
||||
type MetadataPassProgress,
|
||||
type MetadataPassProgressByKind,
|
||||
type RepositoryMetadataState,
|
||||
} from "./metadata-state";
|
||||
|
||||
export async function persistMetadataPassProgress(
|
||||
repositoryId: string | null | undefined,
|
||||
metadataState: RepositoryMetadataState,
|
||||
kind: keyof MetadataPassProgressByKind,
|
||||
progress: MetadataPassProgress | undefined
|
||||
): Promise<void> {
|
||||
if (progress) {
|
||||
metadataState.passProgress[kind] = progress;
|
||||
} else {
|
||||
delete metadataState.passProgress[kind];
|
||||
}
|
||||
if (!repositoryId) return;
|
||||
|
||||
try {
|
||||
await db
|
||||
.update(repositories)
|
||||
.set({ metadata: serializeRepositoryMetadataState(metadataState) })
|
||||
.where(eq(repositories.id, repositoryId));
|
||||
} catch (error) {
|
||||
// The in-memory state still carries it for the write at the end of a
|
||||
// sync that does not throw; only a rate limited run loses it.
|
||||
console.warn(
|
||||
`[Metadata] Could not save ${kind} pass progress: ${
|
||||
error instanceof Error ? error.message : String(error)
|
||||
}`
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,265 @@
|
||||
/**
|
||||
* Incremental metadata sync watermarks (#449): parsing, serialisation and
|
||||
* the full vs incremental decision.
|
||||
*/
|
||||
import { describe, expect, test } from "bun:test";
|
||||
import {
|
||||
FULL_METADATA_PASS_INTERVAL_MS,
|
||||
INCREMENTAL_SYNC_SAFETY_MARGIN_MS,
|
||||
advanceMetadataSyncCursor,
|
||||
buildPassProgress,
|
||||
classifyPassItem,
|
||||
createDefaultMetadataState,
|
||||
numberInRanges,
|
||||
parseRepositoryMetadataState,
|
||||
planMetadataPass,
|
||||
planMetadataPassWithProgress,
|
||||
serializeRepositoryMetadataState,
|
||||
toNumberRanges,
|
||||
} from "./metadata-state";
|
||||
|
||||
const HOUR = 60 * 60 * 1000;
|
||||
const DAY = 24 * HOUR;
|
||||
|
||||
describe("sync cursor parsing", () => {
|
||||
test("a row written before #449 loads with no cursors", () => {
|
||||
const legacy = JSON.stringify({
|
||||
components: { releases: true, issues: true, pullRequests: true, labels: true, milestones: true },
|
||||
lastSyncedAt: "2026-09-01T00:00:00.000Z",
|
||||
});
|
||||
const state = parseRepositoryMetadataState(legacy);
|
||||
expect(state.syncCursors).toEqual({});
|
||||
expect(state.components.issues).toBe(true);
|
||||
expect(state.lastSyncedAt).toBe("2026-09-01T00:00:00.000Z");
|
||||
});
|
||||
|
||||
test("the default state has no cursors", () => {
|
||||
expect(createDefaultMetadataState().syncCursors).toEqual({});
|
||||
expect(parseRepositoryMetadataState(null).syncCursors).toEqual({});
|
||||
expect(parseRepositoryMetadataState("not json").syncCursors).toEqual({});
|
||||
});
|
||||
|
||||
test("cursors survive a round trip", () => {
|
||||
const state = createDefaultMetadataState();
|
||||
state.syncCursors.issues = {
|
||||
lastPassStartedAt: "2026-09-20T10:00:00.000Z",
|
||||
lastFullPassStartedAt: "2026-09-18T10:00:00.000Z",
|
||||
};
|
||||
state.syncCursors.pullRequests = {
|
||||
lastPassStartedAt: "2026-09-20T11:00:00.000Z",
|
||||
lastFullPassStartedAt: "2026-09-20T11:00:00.000Z",
|
||||
};
|
||||
const reparsed = parseRepositoryMetadataState(serializeRepositoryMetadataState(state));
|
||||
expect(reparsed.syncCursors).toEqual(state.syncCursors);
|
||||
});
|
||||
|
||||
test("malformed cursors are dropped instead of failing the row", () => {
|
||||
const raw = {
|
||||
components: {},
|
||||
syncCursors: {
|
||||
issues: { lastPassStartedAt: "yesterday", lastFullPassStartedAt: "2026-09-18T10:00:00.000Z" },
|
||||
pullRequests: { lastPassStartedAt: "2026-09-20T11:00:00.000Z" },
|
||||
},
|
||||
};
|
||||
expect(parseRepositoryMetadataState(raw).syncCursors).toEqual({});
|
||||
expect(parseRepositoryMetadataState({ syncCursors: "nope" }).syncCursors).toEqual({});
|
||||
expect(parseRepositoryMetadataState({ syncCursors: { issues: 42 } }).syncCursors).toEqual({});
|
||||
});
|
||||
});
|
||||
|
||||
describe("planMetadataPass", () => {
|
||||
const now = new Date("2026-09-23T12:00:00.000Z");
|
||||
|
||||
test("no cursor means a full pass", () => {
|
||||
const plan = planMetadataPass(undefined, now);
|
||||
expect(plan.mode).toBe("full");
|
||||
});
|
||||
|
||||
test("a recent cursor gives an incremental pass with the safety margin", () => {
|
||||
const lastPass = new Date(now.getTime() - 2 * HOUR);
|
||||
const plan = planMetadataPass(
|
||||
{
|
||||
lastPassStartedAt: lastPass.toISOString(),
|
||||
lastFullPassStartedAt: new Date(now.getTime() - 3 * DAY).toISOString(),
|
||||
},
|
||||
now
|
||||
);
|
||||
expect(plan).toEqual({
|
||||
mode: "incremental",
|
||||
since: new Date(lastPass.getTime() - INCREMENTAL_SYNC_SAFETY_MARGIN_MS).toISOString(),
|
||||
});
|
||||
expect(INCREMENTAL_SYNC_SAFETY_MARGIN_MS).toBeGreaterThanOrEqual(5 * 60 * 1000);
|
||||
});
|
||||
|
||||
test("a full pass older than the interval forces a full pass", () => {
|
||||
const plan = planMetadataPass(
|
||||
{
|
||||
lastPassStartedAt: new Date(now.getTime() - HOUR).toISOString(),
|
||||
lastFullPassStartedAt: new Date(now.getTime() - FULL_METADATA_PASS_INTERVAL_MS).toISOString(),
|
||||
},
|
||||
now
|
||||
);
|
||||
expect(plan.mode).toBe("full");
|
||||
expect(FULL_METADATA_PASS_INTERVAL_MS).toBe(7 * DAY);
|
||||
});
|
||||
|
||||
test("a full pass just inside the interval stays incremental", () => {
|
||||
const plan = planMetadataPass(
|
||||
{
|
||||
lastPassStartedAt: new Date(now.getTime() - HOUR).toISOString(),
|
||||
lastFullPassStartedAt: new Date(now.getTime() - FULL_METADATA_PASS_INTERVAL_MS + HOUR).toISOString(),
|
||||
},
|
||||
now
|
||||
);
|
||||
expect(plan.mode).toBe("incremental");
|
||||
});
|
||||
|
||||
test("a watermark in the future is not trusted", () => {
|
||||
const plan = planMetadataPass(
|
||||
{
|
||||
lastPassStartedAt: new Date(now.getTime() + DAY).toISOString(),
|
||||
lastFullPassStartedAt: now.toISOString(),
|
||||
},
|
||||
now
|
||||
);
|
||||
expect(plan.mode).toBe("full");
|
||||
});
|
||||
});
|
||||
|
||||
describe("advanceMetadataSyncCursor", () => {
|
||||
const started = new Date("2026-09-23T12:00:00.000Z");
|
||||
|
||||
test("a full pass moves both timestamps", () => {
|
||||
expect(advanceMetadataSyncCursor(undefined, "full", started)).toEqual({
|
||||
lastPassStartedAt: started.toISOString(),
|
||||
lastFullPassStartedAt: started.toISOString(),
|
||||
});
|
||||
});
|
||||
|
||||
test("an incremental pass keeps the last full pass time", () => {
|
||||
const previous = {
|
||||
lastPassStartedAt: "2026-09-23T06:00:00.000Z",
|
||||
lastFullPassStartedAt: "2026-09-20T06:00:00.000Z",
|
||||
};
|
||||
expect(advanceMetadataSyncCursor(previous, "incremental", started)).toEqual({
|
||||
lastPassStartedAt: started.toISOString(),
|
||||
lastFullPassStartedAt: "2026-09-20T06:00:00.000Z",
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe("unfinished pass progress (#449 follow-up)", () => {
|
||||
const now = new Date("2026-09-25T12:00:00.000Z");
|
||||
|
||||
test("numbers collapse into ranges and can be looked up", () => {
|
||||
const ranges = toNumberRanges([5, 1, 2, 3, 3, 9, 10, 7]);
|
||||
expect(ranges).toEqual([[1, 3], [5, 5], [7, 7], [9, 10]]);
|
||||
expect(numberInRanges(2, ranges)).toBe(true);
|
||||
expect(numberInRanges(8, ranges)).toBe(false);
|
||||
expect(toNumberRanges([])).toEqual([]);
|
||||
});
|
||||
|
||||
test("progress survives a round trip and bad entries are dropped", () => {
|
||||
const state = createDefaultMetadataState();
|
||||
state.passProgress.issues = {
|
||||
mode: "full",
|
||||
startedAt: "2026-09-25T10:00:00.000Z",
|
||||
done: [[1, 164]],
|
||||
};
|
||||
const reparsed = parseRepositoryMetadataState(serializeRepositoryMetadataState(state));
|
||||
expect(reparsed.passProgress).toEqual(state.passProgress);
|
||||
|
||||
const junk = parseRepositoryMetadataState({
|
||||
passProgress: {
|
||||
issues: { mode: "sideways", startedAt: "2026-09-25T10:00:00.000Z", done: [] },
|
||||
pullRequests: { mode: "incremental", startedAt: "2026-09-25T10:00:00.000Z", done: [] },
|
||||
},
|
||||
});
|
||||
// Unknown mode, and an incremental pass without its since.
|
||||
expect(junk.passProgress).toEqual({});
|
||||
expect(parseRepositoryMetadataState(null).passProgress).toEqual({});
|
||||
});
|
||||
|
||||
test("a stored pass is resumed with its mode, since and start", () => {
|
||||
const cursor = {
|
||||
lastPassStartedAt: "2026-09-25T09:00:00.000Z",
|
||||
lastFullPassStartedAt: "2026-09-20T09:00:00.000Z",
|
||||
};
|
||||
const incremental = planMetadataPassWithProgress(
|
||||
cursor,
|
||||
{ mode: "incremental", since: "2026-09-25T08:50:00.000Z", startedAt: "2026-09-25T10:00:00.000Z", done: [] },
|
||||
now
|
||||
);
|
||||
expect(incremental.plan).toEqual({ mode: "incremental", since: "2026-09-25T08:50:00.000Z" });
|
||||
expect(incremental.startedAt.toISOString()).toBe("2026-09-25T10:00:00.000Z");
|
||||
expect(incremental.resume).toBeDefined();
|
||||
|
||||
const full = planMetadataPassWithProgress(
|
||||
undefined,
|
||||
{ mode: "full", startedAt: "2026-09-25T10:00:00.000Z", done: [[1, 10]] },
|
||||
now
|
||||
);
|
||||
expect(full.plan.mode).toBe("full");
|
||||
|
||||
const none = planMetadataPassWithProgress(cursor, undefined, now);
|
||||
expect(none.plan.mode).toBe("incremental");
|
||||
expect(none.resume).toBeUndefined();
|
||||
expect(none.startedAt).toEqual(now);
|
||||
});
|
||||
|
||||
test("progress stamped in the future is ignored", () => {
|
||||
const planned = planMetadataPassWithProgress(
|
||||
undefined,
|
||||
{ mode: "full", startedAt: "2026-10-25T10:00:00.000Z", done: [[1, 10]] },
|
||||
now
|
||||
);
|
||||
expect(planned.resume).toBeUndefined();
|
||||
});
|
||||
|
||||
test("items are classified by what the pass already knows about them", () => {
|
||||
const cursor = {
|
||||
lastPassStartedAt: "2026-09-25T09:00:00.000Z",
|
||||
lastFullPassStartedAt: "2026-09-17T09:00:00.000Z",
|
||||
};
|
||||
const fullPlan = { mode: "full" as const, reason: "test" };
|
||||
const resume = { mode: "full" as const, startedAt: "2026-09-25T10:00:00.000Z", done: [[1, 5]] as Array<[number, number]> };
|
||||
const base = { plan: fullPlan, cursor, resume, existsInDestination: true };
|
||||
|
||||
expect(classifyPassItem({ ...base, number: 3, updatedAt: "2026-09-24T00:00:00Z" })).toBe("skip");
|
||||
// Changed after the interrupted pass started: done does not count.
|
||||
expect(classifyPassItem({ ...base, number: 3, updatedAt: "2026-09-25T11:00:00Z" })).toBe("sync");
|
||||
// Not done yet, but unchanged since the last completed pass.
|
||||
expect(classifyPassItem({ ...base, number: 8, updatedAt: "2026-09-24T00:00:00Z" })).toBe("unchanged");
|
||||
// Inside the safety margin of the last pass.
|
||||
expect(
|
||||
classifyPassItem({ ...base, number: 8, updatedAt: new Date(Date.parse(cursor.lastPassStartedAt) - INCREMENTAL_SYNC_SAFETY_MARGIN_MS + 1000).toISOString() })
|
||||
).toBe("sync");
|
||||
// Missing in the destination, no watermark, or no updated_at: full treatment.
|
||||
expect(classifyPassItem({ ...base, number: 8, updatedAt: "2026-09-24T00:00:00Z", existsInDestination: false })).toBe("sync");
|
||||
expect(classifyPassItem({ ...base, number: 8, updatedAt: "2026-09-24T00:00:00Z", cursor: undefined })).toBe("sync");
|
||||
expect(classifyPassItem({ ...base, number: 8, updatedAt: null })).toBe("sync");
|
||||
// An incremental pass never skips on the watermark alone.
|
||||
expect(
|
||||
classifyPassItem({ ...base, plan: { mode: "incremental", since: "x" }, resume: undefined, number: 8, updatedAt: "2026-09-24T00:00:00Z" })
|
||||
).toBe("sync");
|
||||
});
|
||||
|
||||
test("progress keeps what an earlier attempt of the same pass finished", () => {
|
||||
const earlier = { mode: "full" as const, startedAt: "2026-09-25T10:00:00.000Z", done: [[1, 3]] as Array<[number, number]> };
|
||||
const progress = buildPassProgress(
|
||||
{ mode: "full", reason: "resume" },
|
||||
new Date(earlier.startedAt),
|
||||
[4, 5, 7],
|
||||
earlier
|
||||
);
|
||||
expect(progress).toEqual({ mode: "full", startedAt: earlier.startedAt, done: [[1, 5], [7, 7]] });
|
||||
|
||||
const incremental = buildPassProgress(
|
||||
{ mode: "incremental", since: "2026-09-25T08:50:00.000Z" },
|
||||
new Date("2026-09-25T10:00:00.000Z"),
|
||||
[2],
|
||||
undefined
|
||||
);
|
||||
expect(incremental.since).toBe("2026-09-25T08:50:00.000Z");
|
||||
});
|
||||
});
|
||||
@@ -18,12 +18,80 @@ export interface AcknowledgedDeletion {
|
||||
giteaSha: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* Watermark for one metadata kind that can be fetched incrementally
|
||||
* from GitHub (#449). Both values are the time the GitHub listing
|
||||
* started, not when the pass finished, so anything that changed while
|
||||
* a long pass was running is picked up by the next one.
|
||||
*/
|
||||
export interface MetadataSyncCursor {
|
||||
/** Start of the last pass (full or incremental) that completed cleanly. */
|
||||
lastPassStartedAt: string;
|
||||
/** Start of the last full pass that completed cleanly. */
|
||||
lastFullPassStartedAt: string;
|
||||
}
|
||||
|
||||
export interface MetadataSyncCursors {
|
||||
issues?: MetadataSyncCursor;
|
||||
pullRequests?: MetadataSyncCursor;
|
||||
}
|
||||
|
||||
/**
|
||||
* Progress of an issues or pull request pass that did not finish, most
|
||||
* often because the source ran out of rate limit budget (#449 follow-up).
|
||||
* The next run repeats the same pass and skips the items listed in
|
||||
* `done`, as long as they have not changed since `startedAt`, so a pass
|
||||
* over a large repository gets further every run instead of starting
|
||||
* over each time.
|
||||
*/
|
||||
export interface MetadataPassProgress {
|
||||
mode: MetadataPassPlan["mode"];
|
||||
/** Listing start for an incremental pass; repeated as is on resume. */
|
||||
since?: string;
|
||||
/**
|
||||
* When the listing of the first attempt started. Used as the watermark
|
||||
* once the pass completes, so nothing that changed while it was
|
||||
* interrupted is missed.
|
||||
*/
|
||||
startedAt: string;
|
||||
/** GitHub numbers finished by this pass, as inclusive ranges. */
|
||||
done: Array<[number, number]>;
|
||||
}
|
||||
|
||||
export interface MetadataPassProgressByKind {
|
||||
issues?: MetadataPassProgress;
|
||||
pullRequests?: MetadataPassProgress;
|
||||
}
|
||||
|
||||
export interface RepositoryMetadataState {
|
||||
components: MetadataComponentsState;
|
||||
lastSyncedAt?: string;
|
||||
acknowledgedDeletions: AcknowledgedDeletion[];
|
||||
syncCursors: MetadataSyncCursors;
|
||||
passProgress: MetadataPassProgressByKind;
|
||||
}
|
||||
|
||||
/**
|
||||
* How far before the stored watermark an incremental listing starts.
|
||||
* Covers clock skew between this host and GitHub and items whose
|
||||
* updated_at was written just before the previous listing began.
|
||||
* Reprocessing a few items twice is harmless: every write is matched
|
||||
* by its [GH-ISSUE #N] / [PR #N] marker and comments by their id.
|
||||
*/
|
||||
export const INCREMENTAL_SYNC_SAFETY_MARGIN_MS = 10 * 60 * 1000;
|
||||
|
||||
/**
|
||||
* A full pass still runs when the last one is older than this. It
|
||||
* catches what `since` cannot see, such as a label renamed on GitHub
|
||||
* (which does not touch the issues that carry it) or an issue removed
|
||||
* on the Gitea side by hand.
|
||||
*/
|
||||
export const FULL_METADATA_PASS_INTERVAL_MS = 7 * 24 * 60 * 60 * 1000;
|
||||
|
||||
export type MetadataPassPlan =
|
||||
| { mode: "full"; reason: string }
|
||||
| { mode: "incremental"; since: string };
|
||||
|
||||
const defaultComponents: MetadataComponentsState = {
|
||||
releases: false,
|
||||
issues: false,
|
||||
@@ -36,6 +104,212 @@ export function createDefaultMetadataState(): RepositoryMetadataState {
|
||||
return {
|
||||
components: { ...defaultComponents },
|
||||
acknowledgedDeletions: [],
|
||||
syncCursors: {},
|
||||
passProgress: {},
|
||||
};
|
||||
}
|
||||
|
||||
function isValidTimestamp(value: unknown): value is string {
|
||||
return typeof value === "string" && Number.isFinite(Date.parse(value));
|
||||
}
|
||||
|
||||
function parseSyncCursor(raw: unknown): MetadataSyncCursor | undefined {
|
||||
if (!raw || typeof raw !== "object") return undefined;
|
||||
const { lastPassStartedAt, lastFullPassStartedAt } = raw as {
|
||||
lastPassStartedAt?: unknown;
|
||||
lastFullPassStartedAt?: unknown;
|
||||
};
|
||||
if (!isValidTimestamp(lastPassStartedAt) || !isValidTimestamp(lastFullPassStartedAt)) {
|
||||
return undefined;
|
||||
}
|
||||
return { lastPassStartedAt, lastFullPassStartedAt };
|
||||
}
|
||||
|
||||
/**
|
||||
* Decide whether the next issues or pull request pass can ask GitHub
|
||||
* only for what changed. Without a usable watermark, or when the last
|
||||
* full pass is too old, the pass is full.
|
||||
*/
|
||||
export function planMetadataPass(
|
||||
cursor: MetadataSyncCursor | undefined,
|
||||
now: Date = new Date()
|
||||
): MetadataPassPlan {
|
||||
if (!cursor) {
|
||||
return { mode: "full", reason: "no previous completed pass" };
|
||||
}
|
||||
const nowMs = now.getTime();
|
||||
const lastPassMs = Date.parse(cursor.lastPassStartedAt);
|
||||
const lastFullMs = Date.parse(cursor.lastFullPassStartedAt);
|
||||
if (!Number.isFinite(lastPassMs) || !Number.isFinite(lastFullMs)) {
|
||||
return { mode: "full", reason: "stored watermark is not a valid date" };
|
||||
}
|
||||
if (lastPassMs > nowMs + INCREMENTAL_SYNC_SAFETY_MARGIN_MS) {
|
||||
return { mode: "full", reason: "stored watermark is in the future" };
|
||||
}
|
||||
if (nowMs - lastFullMs >= FULL_METADATA_PASS_INTERVAL_MS) {
|
||||
return { mode: "full", reason: "last full pass is older than 7 days" };
|
||||
}
|
||||
return {
|
||||
mode: "incremental",
|
||||
since: new Date(lastPassMs - INCREMENTAL_SYNC_SAFETY_MARGIN_MS).toISOString(),
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Collapse item numbers into sorted inclusive ranges, so a pass over
|
||||
* thousands of issues stores a handful of pairs.
|
||||
*/
|
||||
export function toNumberRanges(numbers: Iterable<number>): Array<[number, number]> {
|
||||
const sorted = [...new Set(numbers)]
|
||||
.filter((n) => Number.isInteger(n))
|
||||
.sort((a, b) => a - b);
|
||||
const ranges: Array<[number, number]> = [];
|
||||
for (const n of sorted) {
|
||||
const last = ranges[ranges.length - 1];
|
||||
if (last && n === last[1] + 1) {
|
||||
last[1] = n;
|
||||
} else {
|
||||
ranges.push([n, n]);
|
||||
}
|
||||
}
|
||||
return ranges;
|
||||
}
|
||||
|
||||
export function numberInRanges(n: number, ranges: Array<[number, number]>): boolean {
|
||||
return ranges.some(([from, to]) => n >= from && n <= to);
|
||||
}
|
||||
|
||||
function parsePassProgress(raw: unknown): MetadataPassProgress | undefined {
|
||||
if (!raw || typeof raw !== "object") return undefined;
|
||||
const { mode, since, startedAt, done } = raw as {
|
||||
mode?: unknown;
|
||||
since?: unknown;
|
||||
startedAt?: unknown;
|
||||
done?: unknown;
|
||||
};
|
||||
if (mode !== "full" && mode !== "incremental") return undefined;
|
||||
if (!isValidTimestamp(startedAt)) return undefined;
|
||||
if (mode === "incremental" && !isValidTimestamp(since)) return undefined;
|
||||
if (!Array.isArray(done)) return undefined;
|
||||
const ranges = done.filter(
|
||||
(entry): entry is [number, number] =>
|
||||
Array.isArray(entry) &&
|
||||
entry.length === 2 &&
|
||||
Number.isInteger(entry[0]) &&
|
||||
Number.isInteger(entry[1]) &&
|
||||
entry[0] <= entry[1]
|
||||
);
|
||||
return mode === "incremental"
|
||||
? { mode, since: since as string, startedAt, done: ranges }
|
||||
: { mode, startedAt, done: ranges };
|
||||
}
|
||||
|
||||
/**
|
||||
* Plan a pass, picking up an unfinished one when there is one. A resumed
|
||||
* pass keeps its mode and `since`, and reports its original start so the
|
||||
* watermark stored at the end covers the whole interrupted window.
|
||||
*/
|
||||
export function planMetadataPassWithProgress(
|
||||
cursor: MetadataSyncCursor | undefined,
|
||||
progress: MetadataPassProgress | undefined,
|
||||
now: Date = new Date()
|
||||
): { plan: MetadataPassPlan; startedAt: Date; resume?: MetadataPassProgress } {
|
||||
const startedAtMs = progress ? Date.parse(progress.startedAt) : NaN;
|
||||
if (progress && Number.isFinite(startedAtMs) && startedAtMs <= now.getTime()) {
|
||||
const plan: MetadataPassPlan =
|
||||
progress.mode === "incremental" && progress.since
|
||||
? { mode: "incremental", since: progress.since }
|
||||
: { mode: "full", reason: "resuming a full pass that did not finish" };
|
||||
return { plan, startedAt: new Date(startedAtMs), resume: progress };
|
||||
}
|
||||
return { plan: planMetadataPass(cursor, now), startedAt: now };
|
||||
}
|
||||
|
||||
/**
|
||||
* Decides how much work an item needs in the current pass.
|
||||
*
|
||||
* - "skip": a resumed pass already finished it and it has not changed
|
||||
* since that pass started.
|
||||
* - "unchanged": a full pass is revisiting an item that is already in
|
||||
* Gitea and has not changed since the last completed pass. Its
|
||||
* comments, commits and files cannot have changed either (each of
|
||||
* those bumps updated_at), so no per-item source call is needed.
|
||||
* - "sync": everything else gets the full treatment.
|
||||
*/
|
||||
export function classifyPassItem({
|
||||
number,
|
||||
updatedAt,
|
||||
plan,
|
||||
cursor,
|
||||
resume,
|
||||
existsInDestination,
|
||||
}: {
|
||||
number: number;
|
||||
updatedAt: string | null | undefined;
|
||||
plan: MetadataPassPlan;
|
||||
cursor: MetadataSyncCursor | undefined;
|
||||
resume: MetadataPassProgress | undefined;
|
||||
existsInDestination: boolean;
|
||||
}): "skip" | "unchanged" | "sync" {
|
||||
const updatedMs = updatedAt ? Date.parse(updatedAt) : NaN;
|
||||
if (!Number.isFinite(updatedMs)) return "sync";
|
||||
|
||||
if (resume && numberInRanges(number, resume.done)) {
|
||||
const resumeStartMs = Date.parse(resume.startedAt);
|
||||
if (updatedMs < resumeStartMs - INCREMENTAL_SYNC_SAFETY_MARGIN_MS) {
|
||||
return "skip";
|
||||
}
|
||||
}
|
||||
|
||||
if (plan.mode === "full" && cursor && existsInDestination) {
|
||||
const lastPassMs = Date.parse(cursor.lastPassStartedAt);
|
||||
if (
|
||||
Number.isFinite(lastPassMs) &&
|
||||
updatedMs < lastPassMs - INCREMENTAL_SYNC_SAFETY_MARGIN_MS
|
||||
) {
|
||||
return "unchanged";
|
||||
}
|
||||
}
|
||||
|
||||
return "sync";
|
||||
}
|
||||
|
||||
/**
|
||||
* Progress to store for a pass that did not finish. Numbers finished by
|
||||
* an earlier attempt of the same pass are kept.
|
||||
*/
|
||||
export function buildPassProgress(
|
||||
plan: MetadataPassPlan,
|
||||
startedAt: Date,
|
||||
completed: Iterable<number>,
|
||||
resume: MetadataPassProgress | undefined
|
||||
): MetadataPassProgress {
|
||||
const numbers = new Set<number>(completed);
|
||||
for (const [from, to] of resume?.done ?? []) {
|
||||
for (let n = from; n <= to; n++) numbers.add(n);
|
||||
}
|
||||
return {
|
||||
mode: plan.mode,
|
||||
...(plan.mode === "incremental" ? { since: plan.since } : {}),
|
||||
startedAt: startedAt.toISOString(),
|
||||
done: toNumberRanges(numbers),
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* The watermark to store after a pass that completed without failures.
|
||||
* `startedAt` is the time the GitHub listing started.
|
||||
*/
|
||||
export function advanceMetadataSyncCursor(
|
||||
previous: MetadataSyncCursor | undefined,
|
||||
mode: MetadataPassPlan["mode"],
|
||||
startedAt: Date
|
||||
): MetadataSyncCursor {
|
||||
const started = startedAt.toISOString();
|
||||
return {
|
||||
lastPassStartedAt: started,
|
||||
lastFullPassStartedAt:
|
||||
mode === "full" || !previous ? started : previous.lastFullPassStartedAt,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -93,6 +367,20 @@ export function parseRepositoryMetadataState(
|
||||
);
|
||||
}
|
||||
|
||||
if (parsed.syncCursors && typeof parsed.syncCursors === "object") {
|
||||
const issues = parseSyncCursor(parsed.syncCursors.issues);
|
||||
const pullRequests = parseSyncCursor(parsed.syncCursors.pullRequests);
|
||||
if (issues) base.syncCursors.issues = issues;
|
||||
if (pullRequests) base.syncCursors.pullRequests = pullRequests;
|
||||
}
|
||||
|
||||
if (parsed.passProgress && typeof parsed.passProgress === "object") {
|
||||
const issues = parsePassProgress(parsed.passProgress.issues);
|
||||
const pullRequests = parsePassProgress(parsed.passProgress.pullRequests);
|
||||
if (issues) base.passProgress.issues = issues;
|
||||
if (pullRequests) base.passProgress.pullRequests = pullRequests;
|
||||
}
|
||||
|
||||
return base;
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,171 @@
|
||||
/**
|
||||
* Organization repository discovery.
|
||||
*
|
||||
* An organization pinned to a source can gain newly published upstream
|
||||
* repositories at any time, and a public-only (tokenless) source never runs
|
||||
* the personal listings that would otherwise surface them. The scheduler
|
||||
* re-lists every included organization on each tick and the manual
|
||||
* organization sync re-lists a single one, so both go through the function
|
||||
* below and the identity rules (what already counts as tracked) live in one
|
||||
* place.
|
||||
*
|
||||
* Repositories missing from the listing are deliberately NOT removed: a
|
||||
* tokenless listing only shows public repositories, so cleanup stays
|
||||
* membership-based (repository-cleanup-service).
|
||||
*/
|
||||
|
||||
import { eq } from 'drizzle-orm';
|
||||
import { db, organizations, repositories } from '@/lib/db';
|
||||
import type { Config } from '@/lib/db/schema';
|
||||
import {
|
||||
calcBatchSizeForInsert,
|
||||
normalizeGitRepoToInsert,
|
||||
repositoryIdentityKeys,
|
||||
selectNewRepositoriesByIdentity,
|
||||
} from '@/lib/repo-utils';
|
||||
import { resolveOrganizationSkipForks } from '@/lib/utils/mirror-overrides';
|
||||
import type { SourceRecord } from '@/lib/sources';
|
||||
|
||||
/** The organization fields re-discovery reads. */
|
||||
export interface DiscoverableOrganization {
|
||||
id: string;
|
||||
name: string;
|
||||
sourceId?: string | null;
|
||||
status?: string | null;
|
||||
isIncluded?: boolean | null;
|
||||
mirrorOverrides?: unknown;
|
||||
}
|
||||
|
||||
export interface OrganizationRediscoveryResult {
|
||||
/**
|
||||
* False when nothing ran: the organization is ignored or excluded, or its
|
||||
* pin resolves to no enabled source.
|
||||
*/
|
||||
ran: boolean;
|
||||
/** Repositories the source listed for the organization. */
|
||||
listed: number;
|
||||
/** New repository rows inserted by this pass. */
|
||||
imported: number;
|
||||
/** Listed repositories that another row already tracks on the same host. */
|
||||
alreadyTracked: number;
|
||||
}
|
||||
|
||||
const NOTHING_DISCOVERED: OrganizationRediscoveryResult = {
|
||||
ran: false,
|
||||
listed: 0,
|
||||
imported: 0,
|
||||
alreadyTracked: 0,
|
||||
};
|
||||
|
||||
/**
|
||||
* Re-list one organization on the source it is pinned to and insert the
|
||||
* repositories that are not tracked yet (onConflictDoNothing keeps repeats
|
||||
* idempotent). Unpinned organizations and pins whose source is gone keep the
|
||||
* legacy DB-only behavior, so nothing changes for them.
|
||||
*/
|
||||
export async function rediscoverRepositoriesForOrganization({
|
||||
config,
|
||||
userId,
|
||||
organization,
|
||||
sources,
|
||||
existingRepoKeys,
|
||||
phase,
|
||||
logPrefix = '[Scheduler]',
|
||||
}: {
|
||||
config: Partial<Config> & { id: string };
|
||||
userId: string;
|
||||
organization: DiscoverableOrganization;
|
||||
/** Enabled sources, when the caller already loaded them. */
|
||||
sources?: SourceRecord[];
|
||||
/**
|
||||
* Identity keys of the repositories already tracked, when the caller
|
||||
* already built them. Claimed keys are added to the set, so a caller that
|
||||
* loops over organizations passes one set and never inserts the same
|
||||
* upstream repository twice.
|
||||
*/
|
||||
existingRepoKeys?: Set<string>;
|
||||
/** Named in the log lines, e.g. "scheduled sync". */
|
||||
phase: string;
|
||||
logPrefix?: string;
|
||||
}): Promise<OrganizationRediscoveryResult> {
|
||||
// An ignored organization is left alone entirely: rediscovering it would
|
||||
// insert its new repositories as imported and auto-mirror could pick them
|
||||
// up, which is what the user just asked to stop (#429).
|
||||
if (organization.status === 'ignored' || organization.isIncluded === false) {
|
||||
return NOTHING_DISCOVERED;
|
||||
}
|
||||
|
||||
const { listSources, findSourceForOrganization } = await import('@/lib/sources');
|
||||
const { createSourceProviderFromSource } = await import('@/lib/source-providers');
|
||||
|
||||
const sourceList = sources ?? (await listSources(userId)).filter(source => source.enabled);
|
||||
const source = findSourceForOrganization(organization, sourceList);
|
||||
if (!source) return NOTHING_DISCOVERED;
|
||||
|
||||
// Same identity rule as personal discovery: an organization pinned to a
|
||||
// public-only source must not re-insert repositories the personal source
|
||||
// already tracks on the same host (or the other way round).
|
||||
const knownKeys = existingRepoKeys ?? repositoryIdentityKeys(
|
||||
await db
|
||||
.select({
|
||||
normalizedFullName: repositories.normalizedFullName,
|
||||
sourceProvider: repositories.sourceProvider,
|
||||
sourceUrl: repositories.sourceUrl,
|
||||
})
|
||||
.from(repositories)
|
||||
.where(eq(repositories.userId, userId))
|
||||
);
|
||||
|
||||
const sourceProvider = createSourceProviderFromSource(source, { userId });
|
||||
const orgRepos = await sourceProvider.listOrganizationRepositories(organization.name);
|
||||
|
||||
// The organization's fork policy (override -> global skipForks), the same
|
||||
// resolution the organization mirror path applies.
|
||||
const skipOrgForks = resolveOrganizationSkipForks({
|
||||
orgOverrides: organization.mirrorOverrides,
|
||||
config,
|
||||
});
|
||||
const mirrorableRepos = orgRepos.filter(
|
||||
repo => repo.isDisabled !== true && !(skipOrgForks && repo.isForked)
|
||||
);
|
||||
|
||||
const { fresh: repoRecords, alreadyTracked } = selectNewRepositoriesByIdentity(
|
||||
mirrorableRepos.map(repo =>
|
||||
normalizeGitRepoToInsert(
|
||||
{ ...repo, organization: repo.organization ?? organization.name },
|
||||
{ userId, configId: config.id, sourceId: source.id }
|
||||
)
|
||||
),
|
||||
knownKeys
|
||||
);
|
||||
|
||||
if (repoRecords.length > 0) {
|
||||
// Batch insert to avoid SQLite parameter limit
|
||||
const sample = repoRecords[0];
|
||||
const columnCount = Object.keys(sample ?? {}).length || 1;
|
||||
const BATCH_SIZE = calcBatchSizeForInsert(columnCount);
|
||||
for (let i = 0; i < repoRecords.length; i += BATCH_SIZE) {
|
||||
const batch = repoRecords.slice(i, i + BATCH_SIZE);
|
||||
await db
|
||||
.insert(repositories)
|
||||
.values(batch)
|
||||
.onConflictDoNothing({ target: [repositories.userId, repositories.sourceId, repositories.normalizedFullName] });
|
||||
}
|
||||
console.log(`${logPrefix} Re-discovered ${repoRecords.length} new repositories for organization ${organization.name} on source ${source.name} for user ${userId} during ${phase}`);
|
||||
}
|
||||
if (alreadyTracked.length > 0) {
|
||||
console.log(`${logPrefix} Organization ${organization.name}: ${alreadyTracked.length} repositories are already tracked for user ${userId} on the same host, not adding them again under source ${source.name}`);
|
||||
}
|
||||
|
||||
await db
|
||||
.update(organizations)
|
||||
.set({ repositoryCount: orgRepos.length, updatedAt: new Date() })
|
||||
.where(eq(organizations.id, organization.id));
|
||||
|
||||
return {
|
||||
ran: true,
|
||||
listed: orgRepos.length,
|
||||
imported: repoRecords.length,
|
||||
alreadyTracked: alreadyTracked.length,
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,479 @@
|
||||
/**
|
||||
* Manual organization sync (issue #429).
|
||||
*
|
||||
* The organization card offers "Mirror Organization" only while an
|
||||
* organization has never been mirrored. Once it reads "Successfully
|
||||
* mirrored" there was no way to refresh it by hand: the user had to wait for
|
||||
* the scheduler or sync every repository one at a time. This module is the
|
||||
* by-hand version of what a scheduled run does for a single organization.
|
||||
* It re-discovers the organization's repositories from its source, mirrors
|
||||
* the ones that were never mirrored and syncs the ones that were, leaving
|
||||
* rows another run already owns alone.
|
||||
*/
|
||||
|
||||
import type { Octokit } from "@octokit/rest";
|
||||
import { and, eq } from "drizzle-orm";
|
||||
import { db, organizations, repositories } from "@/lib/db";
|
||||
import type { Config, Repository } from "@/lib/db/schema";
|
||||
import { usesPushEngine } from "@/lib/destination-connection";
|
||||
import { createGitHubClient, createPublicGitHubClient } from "@/lib/github";
|
||||
import { createMirrorJob } from "@/lib/helpers";
|
||||
import {
|
||||
decryptSourceToken,
|
||||
findSourceForOrganization,
|
||||
findSourceForRepository,
|
||||
listSources,
|
||||
resolveGitHubApiBaseUrl,
|
||||
type SourceRecord,
|
||||
} from "@/lib/sources";
|
||||
import { mirrorRepositoryToDestination, syncRepositoryOnDestination } from "@/lib/mirror-dispatch";
|
||||
import { rediscoverRepositoriesForOrganization, type DiscoverableOrganization } from "@/lib/organization-discovery";
|
||||
import { repoStatusEnum, repositoryVisibilityEnum } from "@/types/Repository";
|
||||
import { processWithResilience } from "@/lib/utils/concurrency";
|
||||
import { resolveOrganizationSkipForks } from "@/lib/utils/mirror-overrides";
|
||||
|
||||
const LOG_PREFIX = "[SyncOrg]";
|
||||
|
||||
/** Repository rows are processed a few at a time, as the other job routes do. */
|
||||
const MIRROR_CONCURRENCY_LIMIT = 3;
|
||||
const SYNC_CONCURRENCY_LIMIT = 5;
|
||||
|
||||
/** Statuses a manual organization sync mirrors for the first time. */
|
||||
export const ORG_SYNC_MIRROR_STATUSES = ["imported"] as const;
|
||||
|
||||
/**
|
||||
* Statuses a manual organization sync refreshes on the destination. A failed
|
||||
* row is synced rather than re-created: its mirror usually exists and only
|
||||
* the metadata pass failed. Every other status is left alone, which is also
|
||||
* how rows another run owns ("mirroring", "syncing") are skipped.
|
||||
*/
|
||||
export const ORG_SYNC_SYNC_STATUSES = ["mirrored", "synced", "failed"] as const;
|
||||
|
||||
/** The organization fields a manual sync reads. */
|
||||
export interface SyncableOrganization extends DiscoverableOrganization {
|
||||
id: string;
|
||||
name: string;
|
||||
}
|
||||
|
||||
type RepositoryRow = typeof repositories.$inferSelect;
|
||||
|
||||
export interface OrganizationSyncPlan {
|
||||
/** Never mirrored, so they get an initial mirror. */
|
||||
toMirror: RepositoryRow[];
|
||||
/** Already on the destination, so they get a refresh. */
|
||||
toSync: RepositoryRow[];
|
||||
/** In flight, ignored, being deleted or otherwise not ours to touch. */
|
||||
skipped: RepositoryRow[];
|
||||
}
|
||||
|
||||
export interface OrganizationSyncResult {
|
||||
/** New repository rows the re-discovery pass imported. */
|
||||
discovered: number;
|
||||
/** Repositories mirrored for the first time by this run. */
|
||||
mirrored: number;
|
||||
/** Repositories refreshed on the destination by this run. */
|
||||
synced: number;
|
||||
/** Repositories another run had already claimed when the sync reached them. */
|
||||
skippedInFlight: number;
|
||||
/** Repositories that failed after their retries. */
|
||||
failed: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* Split an organization's repository rows into what a manual sync mirrors,
|
||||
* refreshes and leaves alone.
|
||||
*
|
||||
* Forks are held back from the mirror set when the organization skips forks,
|
||||
* the same rule mirrorGitHubOrgToGitea applies, so a run never creates a
|
||||
* mirror the organization opted out of. A fork that was mirrored before the
|
||||
* opt-out is still refreshed, so an existing mirror does not silently go
|
||||
* stale.
|
||||
*/
|
||||
export function partitionOrganizationRepositories(
|
||||
rows: RepositoryRow[],
|
||||
{ skipForks }: { skipForks: boolean }
|
||||
): OrganizationSyncPlan {
|
||||
const plan: OrganizationSyncPlan = { toMirror: [], toSync: [], skipped: [] };
|
||||
|
||||
for (const row of rows) {
|
||||
const status = row.status as (typeof ORG_SYNC_MIRROR_STATUSES)[number] | string;
|
||||
|
||||
if ((ORG_SYNC_MIRROR_STATUSES as readonly string[]).includes(status)) {
|
||||
if (skipForks && row.isForked) {
|
||||
plan.skipped.push(row);
|
||||
} else {
|
||||
plan.toMirror.push(row);
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
if ((ORG_SYNC_SYNC_STATUSES as readonly string[]).includes(status)) {
|
||||
plan.toSync.push(row);
|
||||
continue;
|
||||
}
|
||||
|
||||
plan.skipped.push(row);
|
||||
}
|
||||
|
||||
return plan;
|
||||
}
|
||||
|
||||
/**
|
||||
* The organization's repository rows. An organization pinned to one source
|
||||
* only covers that source's repositories of the name, the same rule
|
||||
* mirrorGitHubOrgToGitea applies; unpinned organizations follow every
|
||||
* connected source.
|
||||
*/
|
||||
export async function selectOrganizationRepositoryRows({
|
||||
userId,
|
||||
organization,
|
||||
sources,
|
||||
}: {
|
||||
userId: string;
|
||||
organization: SyncableOrganization;
|
||||
sources: SourceRecord[];
|
||||
}): Promise<RepositoryRow[]> {
|
||||
const pinnedSource = findSourceForOrganization(organization, sources);
|
||||
|
||||
const conditions = [
|
||||
eq(repositories.userId, userId),
|
||||
eq(repositories.organization, organization.name),
|
||||
];
|
||||
if (pinnedSource) {
|
||||
conditions.push(eq(repositories.sourceId, pinnedSource.id));
|
||||
}
|
||||
|
||||
return db
|
||||
.select()
|
||||
.from(repositories)
|
||||
.where(and(...conditions));
|
||||
}
|
||||
|
||||
/** The row as the mirror and sync entry points expect it. */
|
||||
function toRepository(row: RepositoryRow): Repository {
|
||||
return {
|
||||
...row,
|
||||
status: repoStatusEnum.parse(row.status),
|
||||
visibility: repositoryVisibilityEnum.parse(row.visibility),
|
||||
organization: row.organization ?? undefined,
|
||||
lastMirrored: row.lastMirrored ?? undefined,
|
||||
errorMessage: row.errorMessage ?? undefined,
|
||||
forkedFrom: row.forkedFrom ?? undefined,
|
||||
mirroredLocation: row.mirroredLocation || "",
|
||||
} as Repository;
|
||||
}
|
||||
|
||||
/**
|
||||
* A sync that found another run already owning the row returns a skip marker
|
||||
* instead of throwing, so the counts can tell it apart from real work.
|
||||
*/
|
||||
function wasSkippedAsInFlight(result: unknown): boolean {
|
||||
return (
|
||||
!!result &&
|
||||
typeof result === "object" &&
|
||||
(result as { skipped?: unknown }).skipped === true
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Only GitHub sources need an API client while mirroring (metadata); other
|
||||
* hosts get code-only mirrors through the destination. One organization can
|
||||
* hold repositories from several hosts, so each row is handed a client built
|
||||
* from its own source.
|
||||
*/
|
||||
function createRepositoryOctokitResolver(
|
||||
userId: string,
|
||||
sources: SourceRecord[]
|
||||
): (repository: Repository) => Octokit | null {
|
||||
return (repository) => {
|
||||
const source = findSourceForRepository(repository, sources);
|
||||
if (!source || source.provider !== "github") return null;
|
||||
|
||||
const token = decryptSourceToken(source.token);
|
||||
// A tokenless GitHub source still gets an anonymous public client
|
||||
// (60 req/hr) so public-repo metadata mirrors; an empty token must never
|
||||
// reach createGitHubClient, which would send `auth: ""`.
|
||||
return token
|
||||
? createGitHubClient(
|
||||
token,
|
||||
userId,
|
||||
source.username || undefined,
|
||||
resolveGitHubApiBaseUrl(source.url)
|
||||
)
|
||||
: createPublicGitHubClient(resolveGitHubApiBaseUrl(source.url));
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Mirror one never-mirrored repository, the per-repository path retry-repo
|
||||
* uses: a push destination creates the target itself, a pull destination
|
||||
* needs the owning organization, which the org mirror entry point creates on
|
||||
* demand.
|
||||
*/
|
||||
async function mirrorOneRepository({
|
||||
config,
|
||||
repository,
|
||||
octokit,
|
||||
sourceUsername,
|
||||
}: {
|
||||
config: Partial<Config>;
|
||||
repository: Repository;
|
||||
octokit: Octokit | null;
|
||||
sourceUsername?: string;
|
||||
}): Promise<void> {
|
||||
if (usesPushEngine(config)) {
|
||||
await mirrorRepositoryToDestination({ config, octokit, repository });
|
||||
return;
|
||||
}
|
||||
|
||||
const { getGiteaRepoOwnerAsync } = await import("@/lib/gitea");
|
||||
const owner = await getGiteaRepoOwnerAsync({ config, repository, sourceUsername });
|
||||
|
||||
const mirrorStrategy =
|
||||
config.githubConfig?.mirrorStrategy ||
|
||||
(config.giteaConfig?.preserveOrgStructure ? "preserve" : "flat-user");
|
||||
const shouldUseOrgMirror =
|
||||
owner !== config.giteaConfig?.defaultOwner || mirrorStrategy === "single-org";
|
||||
|
||||
await mirrorRepositoryToDestination({
|
||||
config,
|
||||
octokit,
|
||||
repository,
|
||||
orgName: shouldUseOrgMirror ? owner : undefined,
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Run a manual sync for one organization: re-discover, mirror what was never
|
||||
* mirrored, refresh what was. The organization sits at "mirroring" for the
|
||||
* duration and settles on "mirrored" or "failed".
|
||||
*/
|
||||
export async function runOrganizationSync({
|
||||
config,
|
||||
userId,
|
||||
organization,
|
||||
}: {
|
||||
config: Partial<Config> & { id: string };
|
||||
userId: string;
|
||||
organization: SyncableOrganization;
|
||||
}): Promise<OrganizationSyncResult> {
|
||||
const startedAt = new Date();
|
||||
|
||||
await db
|
||||
.update(organizations)
|
||||
.set({ status: repoStatusEnum.parse("mirroring"), updatedAt: startedAt })
|
||||
.where(eq(organizations.id, organization.id));
|
||||
|
||||
await createMirrorJob({
|
||||
userId,
|
||||
organizationId: organization.id,
|
||||
organizationName: organization.name,
|
||||
message: `Started sync for organization: ${organization.name}`,
|
||||
details: `Organization ${organization.name} is being re-discovered, mirrored and synced.`,
|
||||
status: repoStatusEnum.parse("mirroring"),
|
||||
jobType: "sync",
|
||||
});
|
||||
|
||||
const result: OrganizationSyncResult = {
|
||||
discovered: 0,
|
||||
mirrored: 0,
|
||||
synced: 0,
|
||||
skippedInFlight: 0,
|
||||
failed: 0,
|
||||
};
|
||||
const notes: string[] = [];
|
||||
|
||||
try {
|
||||
// 1. New upstream repositories. A source that cannot be listed must not
|
||||
// stop the refresh of everything already mirrored, so the failure is
|
||||
// reported in the job details instead of ending the run.
|
||||
try {
|
||||
const discovery = await rediscoverRepositoriesForOrganization({
|
||||
config,
|
||||
userId,
|
||||
organization,
|
||||
phase: "manual organization sync",
|
||||
logPrefix: LOG_PREFIX,
|
||||
});
|
||||
result.discovered = discovery.imported;
|
||||
} catch (error) {
|
||||
const message = error instanceof Error ? error.message : String(error);
|
||||
console.error(
|
||||
`${LOG_PREFIX} Failed to re-discover repositories for organization ${organization.name}: ${message}`
|
||||
);
|
||||
notes.push(`Re-discovery failed: ${message}`);
|
||||
}
|
||||
|
||||
// 2. Plan over the rows as they stand now, the newly imported included.
|
||||
const sources = await listSources(userId);
|
||||
const rows = await selectOrganizationRepositoryRows({ userId, organization, sources });
|
||||
const skipForks = resolveOrganizationSkipForks({
|
||||
orgOverrides: organization.mirrorOverrides,
|
||||
config,
|
||||
});
|
||||
const { toMirror, toSync, skipped } = partitionOrganizationRepositories(rows, { skipForks });
|
||||
|
||||
console.log(
|
||||
`${LOG_PREFIX} Organization ${organization.name}: mirroring ${toMirror.length}, syncing ${toSync.length}, skipping ${skipped.length} repositories`
|
||||
);
|
||||
|
||||
const resolveRepositoryOctokit = createRepositoryOctokitResolver(userId, sources);
|
||||
|
||||
// 3. Initial mirrors.
|
||||
if (toMirror.length > 0) {
|
||||
const mirrored = await processWithResilience(
|
||||
toMirror,
|
||||
async (row) => {
|
||||
const repository = toRepository(row);
|
||||
const source = findSourceForRepository(repository, sources);
|
||||
console.log(`${LOG_PREFIX} Starting mirror for repository: ${row.name}`);
|
||||
await mirrorOneRepository({
|
||||
config,
|
||||
repository,
|
||||
octokit: resolveRepositoryOctokit(repository),
|
||||
sourceUsername: source?.username,
|
||||
});
|
||||
return row;
|
||||
},
|
||||
{
|
||||
userId,
|
||||
jobType: "mirror",
|
||||
getItemId: (row) => row.id,
|
||||
getItemName: (row) => row.name,
|
||||
concurrencyLimit: MIRROR_CONCURRENCY_LIMIT,
|
||||
maxRetries: 2,
|
||||
retryDelay: 2000,
|
||||
checkpointInterval: 1,
|
||||
onProgress: (completed, total) => {
|
||||
const percentComplete = Math.round((completed / total) * 100);
|
||||
console.log(
|
||||
`${LOG_PREFIX} Mirror progress for ${organization.name}: ${percentComplete}% (${completed}/${total})`
|
||||
);
|
||||
},
|
||||
onRetry: (row, error, attempt) => {
|
||||
console.log(
|
||||
`${LOG_PREFIX} Retrying mirror for repository ${row.name} (attempt ${attempt}): ${error.message}`
|
||||
);
|
||||
},
|
||||
}
|
||||
);
|
||||
|
||||
result.mirrored = mirrored.length;
|
||||
result.failed += toMirror.length - mirrored.length;
|
||||
}
|
||||
|
||||
// 4. Refresh of the existing mirrors.
|
||||
if (toSync.length > 0) {
|
||||
const synced = await processWithResilience(
|
||||
toSync,
|
||||
async (row) => {
|
||||
const repository = toRepository(row);
|
||||
console.log(`${LOG_PREFIX} Starting sync for repository: ${row.name}`);
|
||||
const syncResult = await syncRepositoryOnDestination({ config, repository });
|
||||
return { row, skipped: wasSkippedAsInFlight(syncResult) };
|
||||
},
|
||||
{
|
||||
userId,
|
||||
jobType: "sync",
|
||||
getItemId: (row) => row.id,
|
||||
getItemName: (row) => row.name,
|
||||
concurrencyLimit: SYNC_CONCURRENCY_LIMIT,
|
||||
maxRetries: 2,
|
||||
retryDelay: 2000,
|
||||
checkpointInterval: 1,
|
||||
onProgress: (completed, total) => {
|
||||
const percentComplete = Math.round((completed / total) * 100);
|
||||
console.log(
|
||||
`${LOG_PREFIX} Sync progress for ${organization.name}: ${percentComplete}% (${completed}/${total})`
|
||||
);
|
||||
},
|
||||
onRetry: (row, error, attempt) => {
|
||||
console.log(
|
||||
`${LOG_PREFIX} Retrying sync for repository ${row.name} (attempt ${attempt}): ${error.message}`
|
||||
);
|
||||
},
|
||||
}
|
||||
);
|
||||
|
||||
result.skippedInFlight = synced.filter((entry) => entry.skipped).length;
|
||||
result.synced = synced.length - result.skippedInFlight;
|
||||
result.failed += toSync.length - synced.length;
|
||||
}
|
||||
|
||||
const summary =
|
||||
`${result.mirrored} mirrored, ${result.synced} synced, ${result.discovered} newly discovered, ` +
|
||||
`${result.skippedInFlight} already in progress, ${result.failed} failed.`;
|
||||
const details = [`Organization ${organization.name}: ${summary}`, ...notes].join(" ");
|
||||
|
||||
if (result.failed > 0) {
|
||||
await db
|
||||
.update(organizations)
|
||||
.set({
|
||||
status: repoStatusEnum.parse("failed"),
|
||||
updatedAt: new Date(),
|
||||
errorMessage: `${result.failed} ${result.failed === 1 ? "repository" : "repositories"} failed to sync.`,
|
||||
})
|
||||
.where(eq(organizations.id, organization.id));
|
||||
|
||||
await createMirrorJob({
|
||||
userId,
|
||||
organizationId: organization.id,
|
||||
organizationName: organization.name,
|
||||
message: `Failed to sync organization: ${organization.name}`,
|
||||
details,
|
||||
status: repoStatusEnum.parse("failed"),
|
||||
jobType: "sync",
|
||||
});
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
await db
|
||||
.update(organizations)
|
||||
.set({
|
||||
status: repoStatusEnum.parse("mirrored"),
|
||||
updatedAt: new Date(),
|
||||
lastMirrored: new Date(),
|
||||
errorMessage: null,
|
||||
})
|
||||
.where(eq(organizations.id, organization.id));
|
||||
|
||||
await createMirrorJob({
|
||||
userId,
|
||||
organizationId: organization.id,
|
||||
organizationName: organization.name,
|
||||
message: `Successfully synced organization: ${organization.name}`,
|
||||
details,
|
||||
status: repoStatusEnum.parse("mirrored"),
|
||||
jobType: "sync",
|
||||
});
|
||||
|
||||
return result;
|
||||
} catch (error) {
|
||||
const message = error instanceof Error ? error.message : "Unknown error";
|
||||
console.error(
|
||||
`${LOG_PREFIX} Error while syncing organization ${organization.name}: ${message}`
|
||||
);
|
||||
|
||||
await db
|
||||
.update(organizations)
|
||||
.set({
|
||||
status: repoStatusEnum.parse("failed"),
|
||||
updatedAt: new Date(),
|
||||
errorMessage: message,
|
||||
})
|
||||
.where(eq(organizations.id, organization.id));
|
||||
|
||||
await createMirrorJob({
|
||||
userId,
|
||||
organizationId: organization.id,
|
||||
organizationName: organization.name,
|
||||
message: `Failed to sync organization: ${organization.name}`,
|
||||
details: `Organization ${organization.name} failed to sync. Error: ${message}`,
|
||||
status: repoStatusEnum.parse("failed"),
|
||||
jobType: "sync",
|
||||
});
|
||||
|
||||
return result;
|
||||
}
|
||||
}
|
||||
@@ -1,4 +1,5 @@
|
||||
import type { AppriseConfig } from "@/types/config";
|
||||
import { safeFetch } from "@/lib/utils/outbound-url";
|
||||
import type { NotificationEvent } from "./ntfy";
|
||||
|
||||
export async function sendAppriseNotification(config: AppriseConfig, event: NotificationEvent): Promise<void> {
|
||||
@@ -10,6 +11,6 @@ export async function sendAppriseNotification(config: AppriseConfig, event: Noti
|
||||
type: event.type === "sync_error" ? "failure" : "success",
|
||||
tag: config.tag || undefined,
|
||||
});
|
||||
const resp = await fetch(url, { method: "POST", body, headers });
|
||||
if (!resp.ok) throw new Error(`Apprise error: ${resp.status} ${await resp.text()}`);
|
||||
const resp = await safeFetch(url, { method: "POST", body, headers });
|
||||
if (!resp.ok) throw new Error(`Apprise error: ${resp.status}`);
|
||||
}
|
||||
@@ -1,4 +1,5 @@
|
||||
import type { GotifyConfig } from "@/types/config";
|
||||
import { safeFetch } from "@/lib/utils/outbound-url";
|
||||
import type { NotificationEvent } from "./ntfy";
|
||||
|
||||
export async function sendGotifyNotification(config: GotifyConfig, event: NotificationEvent): Promise<void> {
|
||||
@@ -12,6 +13,6 @@ export async function sendGotifyNotification(config: GotifyConfig, event: Notifi
|
||||
message: event.message,
|
||||
priority: event.type === "sync_error" ? 8 : (config.priority ?? 5),
|
||||
});
|
||||
const resp = await fetch(url, { method: "POST", body, headers });
|
||||
if (!resp.ok) throw new Error(`Gotify error: ${resp.status} ${await resp.text()}`);
|
||||
const resp = await safeFetch(url, { method: "POST", body, headers });
|
||||
if (!resp.ok) throw new Error(`Gotify error: ${resp.status}`);
|
||||
}
|
||||
@@ -1,4 +1,5 @@
|
||||
import type { NtfyConfig } from "@/types/config";
|
||||
import { safeFetch } from "@/lib/utils/outbound-url";
|
||||
|
||||
export interface NotificationEvent {
|
||||
title: string;
|
||||
@@ -16,6 +17,6 @@ export async function sendNtfyNotification(config: NtfyConfig, event: Notificati
|
||||
if (config.token) {
|
||||
headers["Authorization"] = `Bearer ${config.token}`;
|
||||
}
|
||||
const resp = await fetch(url, { method: "POST", body: event.message, headers });
|
||||
if (!resp.ok) throw new Error(`Ntfy error: ${resp.status} ${await resp.text()}`);
|
||||
const resp = await safeFetch(url, { method: "POST", body: event.message, headers });
|
||||
if (!resp.ok) throw new Error(`Ntfy error: ${resp.status}`);
|
||||
}
|
||||
@@ -1,4 +1,5 @@
|
||||
import { createHmac } from "node:crypto";
|
||||
import { safeFetch } from "@/lib/utils/outbound-url";
|
||||
import type { WebhookConfig } from "@/types/config";
|
||||
import type { NotificationEvent } from "./ntfy";
|
||||
|
||||
@@ -16,6 +17,6 @@ export async function sendWebhookNotification(config: WebhookConfig, event: Noti
|
||||
const signature = createHmac("sha256", config.secret).update(body).digest("hex");
|
||||
headers["X-Webhook-Signature"] = `sha256=${signature}`;
|
||||
}
|
||||
const resp = await fetch(config.url, { method: "POST", body, headers });
|
||||
if (!resp.ok) throw new Error(`Webhook error: ${resp.status} ${await resp.text()}`);
|
||||
const resp = await safeFetch(config.url, { method: "POST", body, headers });
|
||||
if (!resp.ok) throw new Error(`Webhook error: ${resp.status}`);
|
||||
}
|
||||
@@ -0,0 +1,125 @@
|
||||
/**
|
||||
* The request wrap that holds every GitHub call while the gate is closed
|
||||
* (issue #437, second report). Tested against a hand made hook so the
|
||||
* clock and the sleep are under control.
|
||||
*/
|
||||
|
||||
import { beforeEach, describe, expect, test } from "bun:test";
|
||||
import {
|
||||
RateLimitPausedError,
|
||||
clearAllRateLimits,
|
||||
installRateLimitGate,
|
||||
isRateLimitError,
|
||||
markRateLimited,
|
||||
} from "@/lib/rate-limit-gate";
|
||||
|
||||
type Wrapper = (request: (o: any) => Promise<any>, options: any) => Promise<any>;
|
||||
|
||||
function fakeOctokit() {
|
||||
const holder: { wrapper?: Wrapper } = {};
|
||||
return {
|
||||
holder,
|
||||
octokit: {
|
||||
hook: {
|
||||
wrap: (_name: string, wrapper: Wrapper) => {
|
||||
holder.wrapper = wrapper;
|
||||
},
|
||||
},
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
describe("installRateLimitGate", () => {
|
||||
const key = "user-gate";
|
||||
const base = new Date("2026-09-24T10:00:00.000Z");
|
||||
|
||||
beforeEach(() => {
|
||||
clearAllRateLimits();
|
||||
});
|
||||
|
||||
test("an open gate passes the request straight through", async () => {
|
||||
const { octokit, holder } = fakeOctokit();
|
||||
const sleeps: number[] = [];
|
||||
installRateLimitGate(octokit, { key, maxWaitMs: 120_000, sleep: async (ms) => void sleeps.push(ms), now: () => base });
|
||||
|
||||
const sent: any[] = [];
|
||||
const result = await holder.wrapper!(async (o) => { sent.push(o); return "ok"; }, { method: "GET", url: "/repos/a/b" });
|
||||
expect(result).toBe("ok");
|
||||
expect(sent).toHaveLength(1);
|
||||
expect(sleeps).toEqual([]);
|
||||
});
|
||||
|
||||
test("a long pause fails the request without calling the inner request", async () => {
|
||||
const { octokit, holder } = fakeOctokit();
|
||||
installRateLimitGate(octokit, { key, maxWaitMs: 120_000, sleep: async () => {}, now: () => base });
|
||||
markRateLimited(key, base.getTime() + 3_263_000);
|
||||
|
||||
let inner = 0;
|
||||
let caught: unknown;
|
||||
try {
|
||||
await holder.wrapper!(async () => { inner += 1; return "no"; }, { method: "GET", url: "/repos/a/b/issues" });
|
||||
} catch (error) {
|
||||
caught = error;
|
||||
}
|
||||
expect(inner).toBe(0);
|
||||
expect(caught).toBeInstanceOf(RateLimitPausedError);
|
||||
expect((caught as RateLimitPausedError).status).toBe(403);
|
||||
expect((caught as RateLimitPausedError).message).toContain("paused until 2026-09-24T10:54:23.000Z");
|
||||
expect((caught as RateLimitPausedError).message).toContain("GET /repos/a/b/issues");
|
||||
// The rest of the code base classifies it like GitHub's own refusal.
|
||||
expect(isRateLimitError(caught)).toBe(true);
|
||||
});
|
||||
|
||||
test("a short pause is slept off with a one second cushion, then the request goes out", async () => {
|
||||
const { octokit, holder } = fakeOctokit();
|
||||
const sleeps: number[] = [];
|
||||
installRateLimitGate(octokit, { key, maxWaitMs: 120_000, sleep: async (ms) => void sleeps.push(ms), now: () => base });
|
||||
markRateLimited(key, base.getTime() + 45_000);
|
||||
|
||||
let inner = 0;
|
||||
const result = await holder.wrapper!(async () => { inner += 1; return "sent"; }, { method: "GET", url: "/repos/a/b" });
|
||||
expect(result).toBe("sent");
|
||||
expect(inner).toBe(1);
|
||||
expect(sleeps).toEqual([46_000]);
|
||||
});
|
||||
|
||||
test("exactly the threshold still waits; one millisecond more fails", async () => {
|
||||
const { octokit, holder } = fakeOctokit();
|
||||
const sleeps: number[] = [];
|
||||
installRateLimitGate(octokit, { key, maxWaitMs: 120_000, sleep: async (ms) => void sleeps.push(ms), now: () => base });
|
||||
|
||||
markRateLimited(key, base.getTime() + 120_000);
|
||||
await holder.wrapper!(async () => "a", { method: "GET", url: "/x" });
|
||||
expect(sleeps).toEqual([121_000]);
|
||||
|
||||
clearAllRateLimits();
|
||||
markRateLimited(key, base.getTime() + 120_001);
|
||||
await expect(holder.wrapper!(async () => "b", { method: "GET", url: "/x" })).rejects.toBeInstanceOf(RateLimitPausedError);
|
||||
});
|
||||
|
||||
test("the rate limit probe is exempt so the pause can be checked", async () => {
|
||||
const { octokit, holder } = fakeOctokit();
|
||||
installRateLimitGate(octokit, { key, maxWaitMs: 120_000, sleep: async () => {}, now: () => base });
|
||||
markRateLimited(key, base.getTime() + 3_600_000);
|
||||
|
||||
let inner = 0;
|
||||
await holder.wrapper!(async () => { inner += 1; return "probe"; }, { method: "GET", url: "/rate_limit" });
|
||||
expect(inner).toBe(1);
|
||||
await expect(holder.wrapper!(async () => "no", { method: "GET", url: "/rate_limited_looking/but/not" })).rejects.toBeInstanceOf(RateLimitPausedError);
|
||||
});
|
||||
|
||||
test("keys are independent: another user's pause does not hold this client", async () => {
|
||||
const { octokit, holder } = fakeOctokit();
|
||||
installRateLimitGate(octokit, { key, maxWaitMs: 120_000, sleep: async () => {}, now: () => base });
|
||||
markRateLimited("someone-else", base.getTime() + 3_600_000);
|
||||
|
||||
let inner = 0;
|
||||
await holder.wrapper!(async () => { inner += 1; return "ok"; }, { method: "GET", url: "/repos/a/b" });
|
||||
expect(inner).toBe(1);
|
||||
});
|
||||
|
||||
test("a client without a hook system is left alone", () => {
|
||||
expect(() => installRateLimitGate({}, { key, maxWaitMs: 1 })).not.toThrow();
|
||||
expect(() => installRateLimitGate(undefined, { key, maxWaitMs: 1 })).not.toThrow();
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,156 @@
|
||||
import { describe, expect, test, beforeEach } from "bun:test";
|
||||
import {
|
||||
TOKENLESS_RATE_LIMIT_KEY,
|
||||
clearAllRateLimits,
|
||||
clearRateLimit,
|
||||
isRateLimitError,
|
||||
markRateLimited,
|
||||
rateLimitGateKey,
|
||||
rateLimitedUntil,
|
||||
} from "@/lib/rate-limit-gate";
|
||||
|
||||
describe("rate limit gate keys", () => {
|
||||
beforeEach(() => {
|
||||
clearAllRateLimits();
|
||||
});
|
||||
|
||||
test("a user id is its own key and an empty one falls back to the tokenless key", () => {
|
||||
expect(rateLimitGateKey("user-1")).toBe("user-1");
|
||||
expect(rateLimitGateKey(" user-2 ")).toBe("user-2");
|
||||
expect(rateLimitGateKey("")).toBe(TOKENLESS_RATE_LIMIT_KEY);
|
||||
expect(rateLimitGateKey(" ")).toBe(TOKENLESS_RATE_LIMIT_KEY);
|
||||
expect(rateLimitGateKey(undefined)).toBe(TOKENLESS_RATE_LIMIT_KEY);
|
||||
expect(rateLimitGateKey(null)).toBe(TOKENLESS_RATE_LIMIT_KEY);
|
||||
});
|
||||
|
||||
test("keys do not interfere with each other", () => {
|
||||
const resetAt = new Date(Date.now() + 600_000);
|
||||
markRateLimited("user-1", resetAt);
|
||||
|
||||
expect(rateLimitedUntil("user-1")?.getTime()).toBe(resetAt.getTime());
|
||||
expect(rateLimitedUntil("user-2")).toBeNull();
|
||||
expect(rateLimitedUntil(TOKENLESS_RATE_LIMIT_KEY)).toBeNull();
|
||||
});
|
||||
});
|
||||
|
||||
describe("rate limit gate state", () => {
|
||||
beforeEach(() => {
|
||||
clearAllRateLimits();
|
||||
});
|
||||
|
||||
test("an unknown key is not limited", () => {
|
||||
expect(rateLimitedUntil("user-1")).toBeNull();
|
||||
});
|
||||
|
||||
test("a recorded reset is reported until it passes, then forgotten", () => {
|
||||
const now = new Date("2026-01-01T09:00:00.000Z");
|
||||
const resetAt = new Date("2026-01-01T09:30:00.000Z");
|
||||
markRateLimited("user-1", resetAt);
|
||||
|
||||
expect(rateLimitedUntil("user-1", now)?.toISOString()).toBe(resetAt.toISOString());
|
||||
|
||||
const afterReset = new Date("2026-01-01T09:30:00.001Z");
|
||||
expect(rateLimitedUntil("user-1", afterReset)).toBeNull();
|
||||
// Reading past the reset drops the entry, so the gate opens on its own.
|
||||
expect(rateLimitedUntil("user-1", now)).toBeNull();
|
||||
});
|
||||
|
||||
test("a reset exactly at the current time is already over", () => {
|
||||
const now = new Date("2026-01-01T09:00:00.000Z");
|
||||
markRateLimited("user-1", now);
|
||||
|
||||
expect(rateLimitedUntil("user-1", now)).toBeNull();
|
||||
});
|
||||
|
||||
test("the furthest reset wins", () => {
|
||||
const now = new Date("2026-01-01T09:00:00.000Z");
|
||||
const nearer = new Date("2026-01-01T09:10:00.000Z");
|
||||
const further = new Date("2026-01-01T09:45:00.000Z");
|
||||
|
||||
markRateLimited("user-1", nearer);
|
||||
markRateLimited("user-1", further);
|
||||
expect(rateLimitedUntil("user-1", now)?.toISOString()).toBe(further.toISOString());
|
||||
|
||||
// A later call reporting a nearer reset must not shorten the pause.
|
||||
markRateLimited("user-1", nearer);
|
||||
expect(rateLimitedUntil("user-1", now)?.toISOString()).toBe(further.toISOString());
|
||||
});
|
||||
|
||||
test("epoch milliseconds are accepted and nonsense is ignored", () => {
|
||||
const now = new Date("2026-01-01T09:00:00.000Z");
|
||||
const resetAt = new Date("2026-01-01T09:20:00.000Z");
|
||||
|
||||
markRateLimited("user-1", resetAt.getTime());
|
||||
expect(rateLimitedUntil("user-1", now)?.toISOString()).toBe(resetAt.toISOString());
|
||||
|
||||
markRateLimited("user-2", Number.NaN);
|
||||
markRateLimited("user-3", new Date("not a date"));
|
||||
expect(rateLimitedUntil("user-2", now)).toBeNull();
|
||||
expect(rateLimitedUntil("user-3", now)).toBeNull();
|
||||
});
|
||||
|
||||
test("clearRateLimit opens one key and clearAllRateLimits opens every key", () => {
|
||||
const resetAt = new Date(Date.now() + 600_000);
|
||||
markRateLimited("user-1", resetAt);
|
||||
markRateLimited("user-2", resetAt);
|
||||
|
||||
clearRateLimit("user-1");
|
||||
expect(rateLimitedUntil("user-1")).toBeNull();
|
||||
expect(rateLimitedUntil("user-2")).not.toBeNull();
|
||||
|
||||
clearAllRateLimits();
|
||||
expect(rateLimitedUntil("user-2")).toBeNull();
|
||||
});
|
||||
});
|
||||
|
||||
describe("isRateLimitError", () => {
|
||||
test("recognises Octokit's primary rate limit refusal", () => {
|
||||
const error = Object.assign(
|
||||
new Error("API rate limit exceeded for user ID 1234."),
|
||||
{ status: 403 }
|
||||
);
|
||||
expect(isRateLimitError(error)).toBe(true);
|
||||
});
|
||||
|
||||
test("recognises the secondary rate limit and 429", () => {
|
||||
const secondary = Object.assign(
|
||||
new Error("You have exceeded a secondary rate limit."),
|
||||
{ status: 403 }
|
||||
);
|
||||
expect(isRateLimitError(secondary)).toBe(true);
|
||||
|
||||
const tooManyRequests = Object.assign(new Error("Too Many Requests"), { status: 429 });
|
||||
expect(isRateLimitError(tooManyRequests)).toBe(true);
|
||||
});
|
||||
|
||||
test("reads the status off a nested response when the error does not carry one", () => {
|
||||
const error = Object.assign(new Error("Too Many Requests"), {
|
||||
response: { status: 429 },
|
||||
});
|
||||
expect(isRateLimitError(error)).toBe(true);
|
||||
});
|
||||
|
||||
test("recognises a rate limit that was re-thrown with only its message", () => {
|
||||
// gitea.ts wraps mirror failures in a plain Error, so no status survives
|
||||
// to the scheduler.
|
||||
const wrapped = new Error(
|
||||
"Failed to mirror repository: API rate limit exceeded for user ID 1234."
|
||||
);
|
||||
expect(isRateLimitError(wrapped)).toBe(true);
|
||||
});
|
||||
|
||||
test("ignores everything else", () => {
|
||||
expect(isRateLimitError(null)).toBe(false);
|
||||
expect(isRateLimitError(undefined)).toBe(false);
|
||||
expect(isRateLimitError(new Error("Repository not found in Gitea"))).toBe(false);
|
||||
expect(
|
||||
isRateLimitError(Object.assign(new Error("Bad credentials"), { status: 401 }))
|
||||
).toBe(false);
|
||||
// A 404 body that happens to mention the phrase is not a refusal to serve.
|
||||
expect(
|
||||
isRateLimitError(
|
||||
Object.assign(new Error("see the rate limit documentation"), { status: 404 })
|
||||
)
|
||||
).toBe(false);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,195 @@
|
||||
/**
|
||||
* In-memory gate for source rate limits (issue #437).
|
||||
*
|
||||
* A GitHub primary rate limit resets at a fixed time that can be close to an
|
||||
* hour away. Sleeping through that window inside an Octokit request holds the
|
||||
* scheduler lock for the whole run, and every following tick logs "Scheduler
|
||||
* is already running, skipping this cycle" until the process restarts.
|
||||
*
|
||||
* The throttle handler records the reset here and fails the request instead of
|
||||
* sleeping. The scheduler reads the gate between batches and between
|
||||
* repositories, ends the run early and schedules the next one just after the
|
||||
* reset, so nothing is stuck and the lock is released as usual.
|
||||
*
|
||||
* The state is process local on purpose: it is a coordination hint for the
|
||||
* work running right now, not durable state. RateLimitManager
|
||||
* (src/lib/rate-limit-manager.ts) still persists the per-user numbers.
|
||||
*/
|
||||
|
||||
/** Gate key for clients built without a user, such as tokenless public sources. */
|
||||
export const TOKENLESS_RATE_LIMIT_KEY = "anonymous";
|
||||
|
||||
/** Reset time (epoch ms) per key, for the keys that are currently limited. */
|
||||
const resetTimesByKey = new Map<string, number>();
|
||||
|
||||
/**
|
||||
* The gate key for a user. Clients created without a userId share one key, so
|
||||
* a tokenless source that runs into the anonymous limit still pauses.
|
||||
*/
|
||||
export function rateLimitGateKey(userId?: string | null): string {
|
||||
const trimmed = typeof userId === "string" ? userId.trim() : "";
|
||||
return trimmed === "" ? TOKENLESS_RATE_LIMIT_KEY : trimmed;
|
||||
}
|
||||
|
||||
/**
|
||||
* Record that this key is rate limited until `resetAt`. A reset that is
|
||||
* further out than the one already recorded wins: two endpoints can report
|
||||
* different windows and the work has to wait for the later one.
|
||||
*/
|
||||
export function markRateLimited(key: string, resetAt: Date | number): void {
|
||||
const resetMs = resetAt instanceof Date ? resetAt.getTime() : resetAt;
|
||||
if (!Number.isFinite(resetMs)) {
|
||||
return;
|
||||
}
|
||||
|
||||
const current = resetTimesByKey.get(key);
|
||||
if (current !== undefined && current >= resetMs) {
|
||||
return;
|
||||
}
|
||||
|
||||
resetTimesByKey.set(key, resetMs);
|
||||
}
|
||||
|
||||
/**
|
||||
* The reset time while the key is still rate limited, otherwise null. A reset
|
||||
* that has passed is forgotten on read, so the gate opens on its own.
|
||||
*/
|
||||
export function rateLimitedUntil(key: string, now: Date = new Date()): Date | null {
|
||||
const resetMs = resetTimesByKey.get(key);
|
||||
if (resetMs === undefined) {
|
||||
return null;
|
||||
}
|
||||
|
||||
if (resetMs <= now.getTime()) {
|
||||
resetTimesByKey.delete(key);
|
||||
return null;
|
||||
}
|
||||
|
||||
return new Date(resetMs);
|
||||
}
|
||||
|
||||
/** Open the gate for one key, for example after a request came back with budget left. */
|
||||
export function clearRateLimit(key: string): void {
|
||||
resetTimesByKey.delete(key);
|
||||
}
|
||||
|
||||
/** Open the gate for every key. Used by tests and by a full restart of the scheduler. */
|
||||
export function clearAllRateLimits(): void {
|
||||
resetTimesByKey.clear();
|
||||
}
|
||||
|
||||
/**
|
||||
* Whether an error is a rate limit refusal from the source.
|
||||
*
|
||||
* Octokit raises 403 with "rate limit" in the message for the primary and
|
||||
* secondary limits and 429 for too many requests. Callers in between re-throw
|
||||
* some of those with only the message (gitea.ts wraps mirror failures in a
|
||||
* plain Error), so the message alone has to be enough when no status survived.
|
||||
*/
|
||||
export function isRateLimitError(error: unknown): boolean {
|
||||
if (!error) {
|
||||
return false;
|
||||
}
|
||||
|
||||
const candidate = error as { status?: unknown; response?: { status?: unknown }; message?: unknown };
|
||||
const rawStatus = candidate.status ?? candidate.response?.status;
|
||||
const status = typeof rawStatus === "number" ? rawStatus : undefined;
|
||||
|
||||
if (status === 429) {
|
||||
return true;
|
||||
}
|
||||
|
||||
const message =
|
||||
typeof candidate.message === "string" ? candidate.message : String(error);
|
||||
const mentionsRateLimit = /rate limit/i.test(message);
|
||||
|
||||
if (!mentionsRateLimit) {
|
||||
return false;
|
||||
}
|
||||
|
||||
// With a status attached, only the two GitHub uses of the phrase count.
|
||||
// Without one the error has been re-thrown along the way, and the message is
|
||||
// all that is left to go on.
|
||||
return status === undefined || status === 403;
|
||||
}
|
||||
|
||||
/**
|
||||
* The error a gated request fails with before anything is sent. Shaped like
|
||||
* Octokit's own primary rate limit refusal (status 403, "rate limit" in the
|
||||
* message) so every existing `isRateLimitError` check treats it the same.
|
||||
*/
|
||||
export class RateLimitPausedError extends Error {
|
||||
readonly status = 403;
|
||||
readonly resetAt: Date;
|
||||
|
||||
constructor(resetAt: Date, endpoint?: string) {
|
||||
super(
|
||||
`GitHub API rate limit exceeded; requests are paused until ${resetAt.toISOString()}` +
|
||||
(endpoint ? ` (not sent: ${endpoint})` : "")
|
||||
);
|
||||
this.name = "RateLimitPausedError";
|
||||
this.resetAt = resetAt;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Endpoints that never count against the limit and are needed to learn when
|
||||
* it opens again. Everything else is held while the gate is closed.
|
||||
*/
|
||||
const GATE_EXEMPT_ROUTES = [/\/rate_limit(\?|$)/];
|
||||
|
||||
export interface RateLimitGateOptions {
|
||||
/** Gate key of this client, from rateLimitGateKey(). */
|
||||
key: string;
|
||||
/**
|
||||
* A closed gate that opens again within this many milliseconds is waited
|
||||
* out inside the request instead of failing it, so a short primary or
|
||||
* secondary limit stays invisible to the caller. Longer waits fail at once.
|
||||
*/
|
||||
maxWaitMs: number;
|
||||
/** Injectable for tests. */
|
||||
sleep?: (ms: number) => Promise<void>;
|
||||
now?: () => Date;
|
||||
}
|
||||
|
||||
/**
|
||||
* Hold every request of an Octokit client while its gate is closed
|
||||
* (issue #437, second report).
|
||||
*
|
||||
* The throttle handler records a reset time when a request is refused, but
|
||||
* on its own that only affects the one request: every other call queued in
|
||||
* the same run still went to GitHub and came back 403, and the retry helper
|
||||
* sent each of them three more times. GitHub counts requests made while
|
||||
* limited toward abuse detection, and one reporter's account was suspended
|
||||
* that way. This wrap runs before the network: a closed gate either waits
|
||||
* (short reset) or throws (long reset), and nothing leaves the process.
|
||||
*/
|
||||
export function installRateLimitGate(octokit: unknown, options: RateLimitGateOptions): void {
|
||||
const hook = (octokit as { hook?: { wrap?: unknown } })?.hook;
|
||||
if (typeof hook?.wrap !== "function") return;
|
||||
|
||||
const sleep = options.sleep ?? ((ms: number) => new Promise<void>((resolve) => setTimeout(resolve, ms)));
|
||||
const now = options.now ?? (() => new Date());
|
||||
|
||||
(hook.wrap as (name: string, wrapper: (request: any, requestOptions: any) => Promise<any>) => void)(
|
||||
"request",
|
||||
async (request, requestOptions) => {
|
||||
const url = String(requestOptions?.url ?? "");
|
||||
if (GATE_EXEMPT_ROUTES.some((route) => route.test(url))) {
|
||||
return request(requestOptions);
|
||||
}
|
||||
|
||||
const until = rateLimitedUntil(options.key, now());
|
||||
if (until) {
|
||||
const waitMs = until.getTime() - now().getTime();
|
||||
if (waitMs > options.maxWaitMs) {
|
||||
throw new RateLimitPausedError(until, `${requestOptions?.method ?? "GET"} ${url}`);
|
||||
}
|
||||
// Small cushion: GitHub's reset is second precision.
|
||||
await sleep(waitMs + 1000);
|
||||
}
|
||||
|
||||
return request(requestOptions);
|
||||
}
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,101 @@
|
||||
import { describe, test, expect } from 'bun:test';
|
||||
import {
|
||||
repositoryIdentityKey,
|
||||
repositoryIdentityKeys,
|
||||
selectNewRepositoriesByIdentity,
|
||||
dedupeRepositoriesByIdentity,
|
||||
} from './repo-utils';
|
||||
|
||||
describe('repositoryIdentityKey', () => {
|
||||
test('is the host plus the full name, independent of the source row', () => {
|
||||
const a = repositoryIdentityKey({ sourceProvider: 'github', sourceUrl: 'https://github.com', normalizedFullName: 'acme/tools' });
|
||||
const b = repositoryIdentityKey({ sourceProvider: 'github', sourceUrl: 'https://github.com/', normalizedFullName: 'ACME/Tools' });
|
||||
expect(a).toBe(b);
|
||||
});
|
||||
|
||||
test('treats missing provider and url as github.com, like legacy rows', () => {
|
||||
expect(repositoryIdentityKey({ normalizedFullName: 'acme/tools' })).toBe(
|
||||
repositoryIdentityKey({ sourceProvider: 'github', sourceUrl: 'https://github.com', normalizedFullName: 'acme/tools' })
|
||||
);
|
||||
});
|
||||
|
||||
test('keeps the same name on two hosts apart', () => {
|
||||
const ghe = repositoryIdentityKey({ sourceProvider: 'github', sourceUrl: 'https://ghe.example.com', normalizedFullName: 'acme/tools' });
|
||||
const com = repositoryIdentityKey({ sourceProvider: 'github', sourceUrl: 'https://github.com', normalizedFullName: 'acme/tools' });
|
||||
expect(ghe).not.toBe(com);
|
||||
});
|
||||
});
|
||||
|
||||
describe('selectNewRepositoriesByIdentity', () => {
|
||||
test('skips a repository already tracked on the same host under another source', () => {
|
||||
const existing = repositoryIdentityKeys([
|
||||
{ sourceProvider: 'github', sourceUrl: 'https://github.com', normalizedFullName: 'acme/tools' },
|
||||
]);
|
||||
const { fresh, alreadyTracked } = selectNewRepositoriesByIdentity(
|
||||
[
|
||||
{ sourceProvider: 'github', sourceUrl: 'https://github.com', normalizedFullName: 'acme/tools', sourceId: 'public-only' },
|
||||
{ sourceProvider: 'github', sourceUrl: 'https://github.com', normalizedFullName: 'acme/new', sourceId: 'public-only' },
|
||||
],
|
||||
existing
|
||||
);
|
||||
expect(fresh.map((r) => r.normalizedFullName)).toEqual(['acme/new']);
|
||||
expect(alreadyTracked.map((r) => r.normalizedFullName)).toEqual(['acme/tools']);
|
||||
});
|
||||
|
||||
test('extends the existing set so a later source does not re-add the same repository', () => {
|
||||
const existing = new Set<string>();
|
||||
const first = selectNewRepositoriesByIdentity(
|
||||
[{ sourceProvider: 'github', sourceUrl: 'https://github.com', normalizedFullName: 'acme/tools' }],
|
||||
existing
|
||||
);
|
||||
const second = selectNewRepositoriesByIdentity(
|
||||
[{ sourceProvider: 'github', sourceUrl: 'https://github.com', normalizedFullName: 'acme/tools' }],
|
||||
existing
|
||||
);
|
||||
expect(first.fresh).toHaveLength(1);
|
||||
expect(second.fresh).toHaveLength(0);
|
||||
expect(second.alreadyTracked).toHaveLength(1);
|
||||
});
|
||||
|
||||
test('still allows the same name from a different host', () => {
|
||||
const existing = repositoryIdentityKeys([
|
||||
{ sourceProvider: 'github', sourceUrl: 'https://github.com', normalizedFullName: 'acme/tools' },
|
||||
]);
|
||||
const { fresh } = selectNewRepositoriesByIdentity(
|
||||
[{ sourceProvider: 'gitlab', sourceUrl: 'https://gitlab.com', normalizedFullName: 'acme/tools' }],
|
||||
existing
|
||||
);
|
||||
expect(fresh).toHaveLength(1);
|
||||
});
|
||||
});
|
||||
|
||||
describe('dedupeRepositoriesByIdentity', () => {
|
||||
test('keeps one row per upstream repository in a batch', () => {
|
||||
const { kept, dropped } = dedupeRepositoriesByIdentity([
|
||||
{ id: 'a', sourceId: 's1', sourceProvider: 'github', sourceUrl: 'https://github.com', normalizedFullName: 'acme/tools' },
|
||||
{ id: 'b', sourceId: 's2', sourceProvider: 'github', sourceUrl: 'https://github.com', normalizedFullName: 'acme/tools' },
|
||||
{ id: 'c', sourceId: 's1', sourceProvider: 'github', sourceUrl: 'https://github.com', normalizedFullName: 'acme/other' },
|
||||
]);
|
||||
expect(kept.map((r) => r.id)).toEqual(['a', 'c']);
|
||||
expect(dropped.map((r) => r.id)).toEqual(['b']);
|
||||
});
|
||||
|
||||
test('prefers a row linked to a source over a legacy row without one', () => {
|
||||
const { kept, dropped } = dedupeRepositoriesByIdentity([
|
||||
{ id: 'legacy', sourceId: null, normalizedFullName: 'acme/tools' },
|
||||
{ id: 'linked', sourceId: 's1', sourceProvider: 'github', sourceUrl: 'https://github.com', normalizedFullName: 'acme/tools' },
|
||||
]);
|
||||
expect(kept.map((r) => r.id)).toEqual(['linked']);
|
||||
expect(dropped.map((r) => r.id)).toEqual(['legacy']);
|
||||
});
|
||||
|
||||
test('leaves a batch without duplicates alone', () => {
|
||||
const rows = [
|
||||
{ id: 'a', sourceId: 's1', normalizedFullName: 'acme/a' },
|
||||
{ id: 'b', sourceId: 's1', normalizedFullName: 'acme/b' },
|
||||
];
|
||||
const { kept, dropped } = dedupeRepositoriesByIdentity(rows);
|
||||
expect(kept).toEqual(rows);
|
||||
expect(dropped).toEqual([]);
|
||||
});
|
||||
});
|
||||
@@ -93,3 +93,82 @@ export function repositoryDestinationColumns(
|
||||
): Pick<RepoInsert, 'destinationProvider' | 'destinationUrl'> {
|
||||
return { destinationProvider: destination.provider, destinationUrl: destination.url };
|
||||
}
|
||||
|
||||
/**
|
||||
* Upstream identity of a repository row: which host it lives on and its full
|
||||
* name there. The repositories table is unique on (userId, sourceId,
|
||||
* normalizedFullName) so that the same owner/name can be tracked on two
|
||||
* hosts, but two source rows can point at the same host (a personal token
|
||||
* source and a public-only source, both github.com), and a legacy row with a
|
||||
* NULL sourceId is distinct from a re-imported one. Discovery used the unique
|
||||
* index alone to decide what was new, so one upstream repository could end up
|
||||
* as two rows that both mirror to the same destination repository and get
|
||||
* processed together in one batch (one of the overlaps behind #417).
|
||||
*/
|
||||
export interface RepositoryIdentityFields {
|
||||
sourceProvider?: string | null;
|
||||
sourceUrl?: string | null;
|
||||
normalizedFullName: string;
|
||||
}
|
||||
|
||||
export function repositoryIdentityKey(row: RepositoryIdentityFields): string {
|
||||
const provider = (row.sourceProvider || 'github').trim().toLowerCase();
|
||||
const url = (row.sourceUrl || 'https://github.com').trim().toLowerCase().replace(/\/+$/, '');
|
||||
return `${provider}|${url}|${row.normalizedFullName.toLowerCase()}`;
|
||||
}
|
||||
|
||||
/**
|
||||
* Split discovered rows into the ones not yet tracked for this user on the
|
||||
* same host under any source, and the ones that are. Also drops repeats
|
||||
* within `candidates` themselves. `existing` is extended with every fresh
|
||||
* row, so a caller looping over several sources can pass the same set.
|
||||
*/
|
||||
export function selectNewRepositoriesByIdentity<T extends RepositoryIdentityFields>(
|
||||
candidates: T[],
|
||||
existing: Set<string>
|
||||
): { fresh: T[]; alreadyTracked: T[] } {
|
||||
const fresh: T[] = [];
|
||||
const alreadyTracked: T[] = [];
|
||||
for (const candidate of candidates) {
|
||||
const key = repositoryIdentityKey(candidate);
|
||||
if (existing.has(key)) {
|
||||
alreadyTracked.push(candidate);
|
||||
} else {
|
||||
existing.add(key);
|
||||
fresh.push(candidate);
|
||||
}
|
||||
}
|
||||
return { fresh, alreadyTracked };
|
||||
}
|
||||
|
||||
export function repositoryIdentityKeys(rows: Iterable<RepositoryIdentityFields>): Set<string> {
|
||||
const keys = new Set<string>();
|
||||
for (const row of rows) keys.add(repositoryIdentityKey(row));
|
||||
return keys;
|
||||
}
|
||||
|
||||
/**
|
||||
* Keep one row per upstream identity in a processing batch, so two rows that
|
||||
* resolve to the same destination repository are never mirrored or synced in
|
||||
* the same pass. A row linked to a source wins over a legacy row without one;
|
||||
* otherwise the first row in the batch is kept.
|
||||
*/
|
||||
export function dedupeRepositoriesByIdentity<
|
||||
T extends RepositoryIdentityFields & { sourceId?: string | null },
|
||||
>(rows: T[]): { kept: T[]; dropped: T[] } {
|
||||
const byKey = new Map<string, T>();
|
||||
const dropped: T[] = [];
|
||||
for (const row of rows) {
|
||||
const key = repositoryIdentityKey(row);
|
||||
const current = byKey.get(key);
|
||||
if (!current) {
|
||||
byKey.set(key, row);
|
||||
} else if (!current.sourceId && row.sourceId) {
|
||||
dropped.push(current);
|
||||
byKey.set(key, row);
|
||||
} else {
|
||||
dropped.push(row);
|
||||
}
|
||||
}
|
||||
return { kept: Array.from(byKey.values()), dropped };
|
||||
}
|
||||
@@ -1,5 +1,10 @@
|
||||
import { describe, test, expect, mock, beforeEach } from "bun:test";
|
||||
import { repoStatusEnum } from "@/types/Repository";
|
||||
import {
|
||||
clearAllRateLimits,
|
||||
markRateLimited,
|
||||
rateLimitGateKey,
|
||||
} from "@/lib/rate-limit-gate";
|
||||
import type { Repository } from "./db/schema";
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
@@ -188,7 +193,11 @@ const configsTable = { __table: "configs" } as any;
|
||||
let configRows: any[] = [];
|
||||
let orgRows: any[] = [];
|
||||
let repoRows: any[] = [];
|
||||
// Rows the unprojected repository selects return: the auto-mirror and sync
|
||||
// pools. Empty for the discovery scenarios, filled by the rate limit ones.
|
||||
let poolRows: any[] = [];
|
||||
let orgUpdates: Array<Record<string, any>> = [];
|
||||
let configUpdates: Array<Record<string, any>> = [];
|
||||
let repoInsertAttempts = 0;
|
||||
|
||||
const publicOnlySource = {
|
||||
@@ -212,6 +221,8 @@ const orgRepoListings = new Map<string, any[]>();
|
||||
const mockListOrganizationRepositories = mock(async (name: string) =>
|
||||
orgRepoListings.get(name) ?? []
|
||||
);
|
||||
const mockMirrorRepositoryToDestination = mock(async () => {});
|
||||
const mockSyncRepositoryOnDestination = mock(async () => {});
|
||||
const mockCreateSourceProviderFromSource = mock(() => ({
|
||||
listRepositories: mockListRepositories,
|
||||
listStarredRepositories: mockListStarredRepositories,
|
||||
@@ -230,7 +241,7 @@ if (isChild) {
|
||||
if (table === configsTable) return Promise.resolve(configRows);
|
||||
if (table === organizationsTable) return Promise.resolve(orgRows);
|
||||
if (table === repositoriesTable) {
|
||||
return Promise.resolve(fields ? repoRows : []);
|
||||
return Promise.resolve(fields ? repoRows : poolRows);
|
||||
}
|
||||
return Promise.resolve([]);
|
||||
},
|
||||
@@ -262,6 +273,7 @@ if (isChild) {
|
||||
set: (data: any) => ({
|
||||
where: (_cond: any) => {
|
||||
if (table === organizationsTable) orgUpdates.push(data);
|
||||
if (table === configsTable) configUpdates.push(data);
|
||||
return Promise.resolve();
|
||||
},
|
||||
}),
|
||||
@@ -305,8 +317,8 @@ if (isChild) {
|
||||
}));
|
||||
|
||||
mock.module("@/lib/mirror-dispatch", () => ({
|
||||
mirrorRepositoryToDestination: mock(async () => {}),
|
||||
syncRepositoryOnDestination: mock(async () => {}),
|
||||
mirrorRepositoryToDestination: mockMirrorRepositoryToDestination,
|
||||
syncRepositoryOnDestination: mockSyncRepositoryOnDestination,
|
||||
}));
|
||||
}
|
||||
|
||||
@@ -400,8 +412,11 @@ describe.skipIf(!isChild)("Scheduler public-only sources (WP4)", () => {
|
||||
}),
|
||||
];
|
||||
repoRows = [];
|
||||
poolRows = [];
|
||||
orgUpdates = [];
|
||||
configUpdates = [];
|
||||
repoInsertAttempts = 0;
|
||||
clearAllRateLimits();
|
||||
orgRepoListings.clear();
|
||||
mockListRepositories.mockClear();
|
||||
mockListStarredRepositories.mockClear();
|
||||
@@ -465,6 +480,169 @@ describe.skipIf(!isChild)("Scheduler public-only sources (WP4)", () => {
|
||||
.filter((row) => row.normalizedFullName?.startsWith("pinned-org/"))
|
||||
.map((row) => row.name);
|
||||
expect(orgRepoNamesAfterSecondRun).toEqual(["existing", "newly-published"]);
|
||||
expect(repoInsertAttempts).toBeGreaterThan(attemptsAfterFirstRun);
|
||||
// Nothing new on the second run, so discovery does not even attempt an
|
||||
// insert: already-tracked repositories are filtered out by identity
|
||||
// before the write, not dropped by the unique index afterwards.
|
||||
expect(repoInsertAttempts).toBe(attemptsAfterFirstRun);
|
||||
});
|
||||
});
|
||||
|
||||
test("re-discovery does not add a second row for a repository the user already tracks under another source on the same host", async () => {
|
||||
// Imported earlier through a personal token source on github.com, so it
|
||||
// has a different sourceId than the pinned public-only source.
|
||||
repoRows.push({
|
||||
userId: "user-1",
|
||||
sourceId: "source-personal",
|
||||
name: "shared",
|
||||
fullName: "pinned-org/shared",
|
||||
normalizedFullName: "pinned-org/shared",
|
||||
organization: "pinned-org",
|
||||
sourceProvider: "github",
|
||||
sourceUrl: "https://github.com",
|
||||
status: "mirrored",
|
||||
});
|
||||
orgRepoListings.set("pinned-org", [makeGitRepo("shared"), makeGitRepo("fresh")]);
|
||||
|
||||
await schedulerLoop();
|
||||
|
||||
const sharedRows = repoRows.filter((row) => row.normalizedFullName === "pinned-org/shared");
|
||||
expect(sharedRows).toHaveLength(1);
|
||||
expect(sharedRows[0].sourceId).toBe("source-personal");
|
||||
expect(repoRows.some((row) => row.normalizedFullName === "pinned-org/fresh")).toBe(true);
|
||||
expect(consoleLines.some((line) => line.includes("already tracked") && line.includes("pinned-org"))).toBe(true);
|
||||
});
|
||||
});
|
||||
// ---------------------------------------------------------------------------
|
||||
// Rate limit gate (issue #437).
|
||||
//
|
||||
// A scheduled run used to sit inside Octokit until the GitHub limit reset,
|
||||
// which held the scheduler lock for the whole run: every later tick logged
|
||||
// "Scheduler is already running, skipping this cycle" and nothing synced until
|
||||
// the container was restarted. The throttle handler now records the reset on
|
||||
// the gate and fails the request, and the run below has to notice that and
|
||||
// stop by itself.
|
||||
// ---------------------------------------------------------------------------
|
||||
function makeSyncRow(name: string, overrides: Record<string, any> = {}): any {
|
||||
return {
|
||||
id: `repo-${name}`,
|
||||
userId: "user-1",
|
||||
sourceId: "source-public",
|
||||
name,
|
||||
fullName: `pinned-org/${name}`,
|
||||
normalizedFullName: `pinned-org/${name}`,
|
||||
owner: "pinned-org",
|
||||
organization: "pinned-org",
|
||||
sourceProvider: "github",
|
||||
sourceUrl: "https://github.com",
|
||||
status: "mirrored",
|
||||
isStarred: false,
|
||||
isForked: false,
|
||||
isPrivate: false,
|
||||
visibility: "public",
|
||||
mirroredLocation: `gitea-org/${name}`,
|
||||
lastMirrored: null,
|
||||
errorMessage: null,
|
||||
forkedFrom: null,
|
||||
createdAt: new Date(0),
|
||||
updatedAt: new Date(0),
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
describe.skipIf(!isChild)("Scheduler rate limit gate (#437)", () => {
|
||||
let consoleLines: string[];
|
||||
|
||||
beforeEach(() => {
|
||||
consoleLines = [];
|
||||
const capture = mock((...args: unknown[]) => {
|
||||
consoleLines.push(args.map((arg) => (typeof arg === "string" ? arg : String(arg))).join(" "));
|
||||
}) as any;
|
||||
console.log = capture;
|
||||
console.warn = capture;
|
||||
console.error = capture;
|
||||
|
||||
configRows = [makeSchedulerConfig()];
|
||||
orgRows = [];
|
||||
repoRows = [];
|
||||
poolRows = [];
|
||||
orgUpdates = [];
|
||||
configUpdates = [];
|
||||
repoInsertAttempts = 0;
|
||||
orgRepoListings.clear();
|
||||
mockListOrganizationRepositories.mockClear();
|
||||
mockMirrorRepositoryToDestination.mockClear();
|
||||
mockSyncRepositoryOnDestination.mockClear();
|
||||
clearAllRateLimits();
|
||||
});
|
||||
|
||||
test("a run that starts rate limited is skipped and comes back just after the reset", async () => {
|
||||
const resetAt = new Date(Date.now() + 45 * 60 * 1000);
|
||||
markRateLimited(rateLimitGateKey("user-1"), resetAt);
|
||||
poolRows = [makeSyncRow("alpha"), makeSyncRow("beta")];
|
||||
|
||||
await schedulerLoop();
|
||||
|
||||
expect(mockSyncRepositoryOnDestination).not.toHaveBeenCalled();
|
||||
expect(
|
||||
consoleLines.some(
|
||||
(line) => line.includes("rate limited until") && line.includes("skipping this run")
|
||||
)
|
||||
).toBe(true);
|
||||
|
||||
const nextRun = configUpdates.at(-1)?.scheduleConfig?.nextRun as Date;
|
||||
expect(nextRun).toBeInstanceOf(Date);
|
||||
expect(nextRun.getTime()).toBe(resetAt.getTime() + 60_000);
|
||||
});
|
||||
|
||||
test("a rate limit hit mid-run stops the sync loop and leaves the rest for the next run", async () => {
|
||||
const resetAt = new Date(Date.now() + 45 * 60 * 1000);
|
||||
orgRows = [makeOrgRow()];
|
||||
orgRepoListings.set("pinned-org", [makeGitRepo("existing")]);
|
||||
poolRows = [makeSyncRow("alpha"), makeSyncRow("beta"), makeSyncRow("gamma")];
|
||||
|
||||
// Stands in for the throttle handler: discovery runs into the limit and
|
||||
// records the reset instead of sleeping through it.
|
||||
mockListOrganizationRepositories.mockImplementationOnce(async () => {
|
||||
markRateLimited(rateLimitGateKey("user-1"), resetAt);
|
||||
throw Object.assign(new Error("API rate limit exceeded for user ID 1."), { status: 403 });
|
||||
});
|
||||
|
||||
await schedulerLoop();
|
||||
|
||||
// The run reached the sync phase and then stopped without syncing anything.
|
||||
expect(consoleLines.some((line) => line.includes("Syncing 3 repositories for user user-1"))).toBe(true);
|
||||
expect(mockSyncRepositoryOnDestination).not.toHaveBeenCalled();
|
||||
expect(
|
||||
consoleLines.some(
|
||||
(line) =>
|
||||
line.includes("rate limited until") &&
|
||||
line.includes("leaving 3 repositories to sync on the next run")
|
||||
)
|
||||
).toBe(true);
|
||||
|
||||
// The lock is released, so the next tick is free to run.
|
||||
expect(consoleLines.some((line) => line.includes("already running"))).toBe(false);
|
||||
|
||||
const nextRun = configUpdates.at(-1)?.scheduleConfig?.nextRun as Date;
|
||||
expect(nextRun.getTime()).toBe(resetAt.getTime() + 60_000);
|
||||
});
|
||||
|
||||
test("the gate only logs once per run", async () => {
|
||||
markRateLimited(rateLimitGateKey("user-1"), new Date(Date.now() + 45 * 60 * 1000));
|
||||
poolRows = [makeSyncRow("alpha")];
|
||||
|
||||
await schedulerLoop();
|
||||
|
||||
const gateLines = consoleLines.filter((line) => line.includes("rate limited until"));
|
||||
expect(gateLines).toHaveLength(1);
|
||||
});
|
||||
|
||||
test("a run that is not rate limited syncs every repository as before", async () => {
|
||||
poolRows = [makeSyncRow("alpha"), makeSyncRow("beta")];
|
||||
|
||||
await schedulerLoop();
|
||||
|
||||
expect(mockSyncRepositoryOnDestination).toHaveBeenCalledTimes(2);
|
||||
expect(consoleLines.some((line) => line.includes("rate limited until"))).toBe(false);
|
||||
expect(consoleLines.some((line) => line.includes("Completed scheduled sync for user user-1"))).toBe(true);
|
||||
});
|
||||
});
|
||||
+273
-74
@@ -5,7 +5,7 @@
|
||||
*/
|
||||
|
||||
import { db, configs, repositories, organizations } from '@/lib/db';
|
||||
import { eq, and, or } from 'drizzle-orm';
|
||||
import { eq, and, or, ne } from 'drizzle-orm';
|
||||
import { mirrorRepositoryToDestination, syncRepositoryOnDestination } from '@/lib/mirror-dispatch';
|
||||
import { usesPushEngine } from '@/lib/destination-connection';
|
||||
import { getDecryptedGitHubToken } from '@/lib/utils/config-encryption';
|
||||
@@ -13,12 +13,21 @@ import { formatDuration } from '@/lib/utils/duration-parser';
|
||||
import type { Repository } from '@/lib/db/schema';
|
||||
import type { Octokit } from '@octokit/rest';
|
||||
import { repoStatusEnum, repositoryVisibilityEnum } from '@/types/Repository';
|
||||
import { mergeGitReposPreferStarred, normalizeGitRepoToInsert, calcBatchSizeForInsert } from '@/lib/repo-utils';
|
||||
import {
|
||||
mergeGitReposPreferStarred,
|
||||
normalizeGitRepoToInsert,
|
||||
calcBatchSizeForInsert,
|
||||
repositoryIdentityKeys,
|
||||
selectNewRepositoriesByIdentity,
|
||||
dedupeRepositoriesByIdentity,
|
||||
} from '@/lib/repo-utils';
|
||||
import { isMirrorableGitHubRepo } from '@/lib/repo-eligibility';
|
||||
import { loadOrganizationForkPolicies, orgForkSkipDecision, resolveOrganizationSkipForks } from '@/lib/utils/mirror-overrides';
|
||||
import { loadOrganizationForkPolicies, orgForkSkipDecision } from '@/lib/utils/mirror-overrides';
|
||||
import { createMirrorJob } from '@/lib/helpers';
|
||||
import { rediscoverRepositoriesForOrganization } from '@/lib/organization-discovery';
|
||||
import { getNextScheduledRun, isCronExpression, normalizeTimezone } from '@/lib/utils/schedule-utils';
|
||||
import { resetStuckMirrorStatuses } from '@/lib/stuck-status-recovery';
|
||||
import { isRateLimitError, rateLimitGateKey, rateLimitedUntil } from '@/lib/rate-limit-gate';
|
||||
import type { SourceRecord } from '@/lib/sources';
|
||||
|
||||
let schedulerInterval: NodeJS.Timeout | null = null;
|
||||
@@ -79,6 +88,103 @@ async function persistScheduleRunState(config: any, currentTime: Date, forceEnab
|
||||
return nextRun;
|
||||
}
|
||||
|
||||
/** Head start the next run gets after a rate limit reset, so the window is really over. */
|
||||
const RATE_LIMIT_RESUME_MARGIN_MS = 60 * 1000;
|
||||
|
||||
/**
|
||||
* Watches the rate limit gate for one run.
|
||||
*
|
||||
* A run used to sit inside Octokit until the limit reset, which held the
|
||||
* scheduler lock for up to an hour per repository and left every later tick
|
||||
* logging "Scheduler is already running" (issue #437). The throttle handler
|
||||
* now records the reset instead of sleeping, and the loops below ask this
|
||||
* watcher between batches and between repositories whether to stop. Only the
|
||||
* first stop is logged, so a long batch loop does not repeat itself.
|
||||
*/
|
||||
function createRateLimitWatcher(userId: string) {
|
||||
const key = rateLimitGateKey(userId);
|
||||
let logged = false;
|
||||
|
||||
return {
|
||||
/** The reset time when the source is paused, otherwise null. */
|
||||
check(describeEffect: () => string): Date | null {
|
||||
const until = rateLimitedUntil(key);
|
||||
if (!until) {
|
||||
return null;
|
||||
}
|
||||
|
||||
if (!logged) {
|
||||
logged = true;
|
||||
console.warn(
|
||||
`[Scheduler] Source is rate limited until ${until.toISOString()} for user ${userId}, ${describeEffect()}`
|
||||
);
|
||||
}
|
||||
|
||||
return until;
|
||||
},
|
||||
/** The reset time without logging, for the next run calculation. */
|
||||
current(): Date | null {
|
||||
return rateLimitedUntil(key);
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Bring the next run forward to just after the rate limit reset.
|
||||
*
|
||||
* Without this the run that stopped early would keep the schedule's own
|
||||
* nextRun, so a daily cron would leave the remaining repositories untouched
|
||||
* for a day even though the limit is back minutes later.
|
||||
*/
|
||||
async function persistRateLimitedNextRun(config: any, currentTime: Date, resetAt: Date): Promise<Date> {
|
||||
const scheduleConfig = config.scheduleConfig || {};
|
||||
const nextRun = new Date(resetAt.getTime() + RATE_LIMIT_RESUME_MARGIN_MS);
|
||||
|
||||
await db.update(configs).set({
|
||||
scheduleConfig: {
|
||||
...scheduleConfig,
|
||||
lastRun: currentTime,
|
||||
nextRun,
|
||||
},
|
||||
updatedAt: new Date(),
|
||||
}).where(eq(configs.id, config.id));
|
||||
|
||||
console.log(
|
||||
`[Scheduler] Next sync for user ${config.userId} moved to ${nextRun.toISOString()} to resume after the rate limit reset`
|
||||
);
|
||||
|
||||
return nextRun;
|
||||
}
|
||||
|
||||
/**
|
||||
* Put a repository back to the status it had before an attempt that only
|
||||
* failed because of a rate limit.
|
||||
*
|
||||
* The mirror and sync paths write "failed" on any error, which leaves the row
|
||||
* waiting for a manual retry. A rate limit is not a repository failure, and
|
||||
* the scheduler already re-syncs "failed", "mirrored", "synced" and "pending"
|
||||
* rows, so restoring the previous status is enough for the run after the reset
|
||||
* to pick the repository up on its own.
|
||||
*/
|
||||
async function restoreRepositoryAfterRateLimit(
|
||||
repo: { id: string; fullName: string; status: string },
|
||||
phase: string
|
||||
): Promise<void> {
|
||||
try {
|
||||
await db.update(repositories).set({
|
||||
status: repo.status,
|
||||
updatedAt: new Date(),
|
||||
}).where(eq(repositories.id, repo.id));
|
||||
|
||||
console.warn(
|
||||
`[Scheduler] ${repo.fullName} hit the source rate limit during ${phase}; ` +
|
||||
`left it as "${repo.status}" so it is retried after the reset instead of needing a manual retry`
|
||||
);
|
||||
} catch (error) {
|
||||
console.error(`[Scheduler] Failed to restore the status of ${repo.fullName} after a rate limit:`, error);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Auto-discovery: import new repositories from every enabled source. Shared
|
||||
* by the scheduled-sync phase and the boot auto-start phase. A failing
|
||||
@@ -102,14 +208,19 @@ async function importRepositoriesFromSources(
|
||||
const orgForkOverrides = await loadOrganizationForkPolicies({ userId });
|
||||
|
||||
const existingRepos = await db
|
||||
.select({ normalizedFullName: repositories.normalizedFullName, sourceId: repositories.sourceId })
|
||||
.select({
|
||||
normalizedFullName: repositories.normalizedFullName,
|
||||
sourceProvider: repositories.sourceProvider,
|
||||
sourceUrl: repositories.sourceUrl,
|
||||
})
|
||||
.from(repositories)
|
||||
.where(eq(repositories.userId, userId));
|
||||
// Repositories are unique per source: the same full name under two sources
|
||||
// counts as two repositories.
|
||||
const existingRepoKeys = new Set(
|
||||
existingRepos.map(r => `${r.sourceId ?? ''}|${r.normalizedFullName}`)
|
||||
);
|
||||
// A repository is identified by its host and full name, not by the source
|
||||
// row that found it. Two sources can point at the same host (a personal
|
||||
// token and a public-only source, both github.com), and a repository the
|
||||
// user already tracks must not be inserted a second time under the other
|
||||
// source: both rows would mirror to the same destination repository.
|
||||
const existingRepoKeys = repositoryIdentityKeys(existingRepos);
|
||||
|
||||
for (const source of sources) {
|
||||
try {
|
||||
@@ -132,17 +243,21 @@ async function importRepositoriesFromSources(
|
||||
const allGithubRepos = mergeGitReposPreferStarred(basicAndForkedRepos, starredRepos);
|
||||
const mirrorableGithubRepos = allGithubRepos.filter(isMirrorableGitHubRepo);
|
||||
|
||||
const newRepos = mirrorableGithubRepos.filter(
|
||||
r => !existingRepoKeys.has(`${source.id}|${r.fullName.toLowerCase()}`)
|
||||
const candidates = mirrorableGithubRepos.map(repo =>
|
||||
normalizeGitRepoToInsert(repo, { userId, configId: config.id, sourceId: source.id })
|
||||
);
|
||||
const { fresh: reposToInsert, alreadyTracked } = selectNewRepositoriesByIdentity(
|
||||
candidates,
|
||||
existingRepoKeys
|
||||
);
|
||||
const newRepos = reposToInsert;
|
||||
if (alreadyTracked.length > 0) {
|
||||
console.log(`[Scheduler] ${alreadyTracked.length} repositories listed by source ${source.name} are already tracked for user ${userId} on the same host; not adding them again`);
|
||||
}
|
||||
|
||||
if (newRepos.length > 0) {
|
||||
console.log(`[Scheduler] Found ${newRepos.length} new repositories for user ${userId} on source ${source.name}`);
|
||||
|
||||
const reposToInsert = newRepos.map(repo =>
|
||||
normalizeGitRepoToInsert(repo, { userId, configId: config.id, sourceId: source.id })
|
||||
);
|
||||
|
||||
// Batch insert to avoid SQLite parameter limit
|
||||
const sample = reposToInsert[0];
|
||||
const columnCount = Object.keys(sample ?? {}).length || 1;
|
||||
@@ -181,79 +296,59 @@ async function importRepositoriesFromSources(
|
||||
}
|
||||
|
||||
/**
|
||||
* Re-discover organization repositories: an organization pinned to a source
|
||||
* can gain newly published upstream repositories at any time, and a
|
||||
* public-only (tokenless) source never runs the personal listings that
|
||||
* would otherwise surface them. Each tick re-lists every included
|
||||
* Re-discover organization repositories: each tick re-lists every included
|
||||
* organization whose pin resolves to an existing enabled source and inserts
|
||||
* the repositories it does not have yet (onConflictDoNothing keeps repeats
|
||||
* idempotent). Unpinned organizations and pins whose source is gone keep
|
||||
* the legacy DB-only behavior — no behavior change for existing users.
|
||||
*
|
||||
* Repositories missing from the listing are deliberately NOT removed:
|
||||
* a tokenless listing only shows public repositories, so cleanup stays
|
||||
* membership-based (repository-cleanup-service).
|
||||
* the repositories it does not have yet. The per-organization work lives in
|
||||
* rediscoverRepositoriesForOrganization, which the manual organization sync
|
||||
* route shares.
|
||||
*/
|
||||
async function rediscoverOrganizationRepositories(
|
||||
config: any,
|
||||
userId: string,
|
||||
phase: 'scheduled sync' | 'auto-start'
|
||||
): Promise<void> {
|
||||
const { listSources, findSourceForOrganization } = await import('@/lib/sources');
|
||||
const { createSourceProviderFromSource } = await import('@/lib/source-providers');
|
||||
const { listSources } = await import('@/lib/sources');
|
||||
const sources = (await listSources(userId)).filter(source => source.enabled);
|
||||
|
||||
// An ignored organization is left alone entirely: rediscovering it would
|
||||
// insert its new repositories as imported and auto-mirror could pick them
|
||||
// up, which is what the user just asked to stop (#429).
|
||||
const includedOrgs = await db
|
||||
.select()
|
||||
.from(organizations)
|
||||
.where(
|
||||
and(
|
||||
eq(organizations.userId, userId),
|
||||
eq(organizations.isIncluded, true)
|
||||
eq(organizations.isIncluded, true),
|
||||
ne(organizations.status, 'ignored')
|
||||
)
|
||||
);
|
||||
|
||||
// Same identity rule as personal discovery: an organization pinned to a
|
||||
// public-only source must not re-insert repositories the personal source
|
||||
// already tracks on the same host (or the other way round). One set is
|
||||
// shared by every organization in the pass, so it also keeps two
|
||||
// organizations from inserting the same repository twice.
|
||||
const existingRepos = await db
|
||||
.select({
|
||||
normalizedFullName: repositories.normalizedFullName,
|
||||
sourceProvider: repositories.sourceProvider,
|
||||
sourceUrl: repositories.sourceUrl,
|
||||
})
|
||||
.from(repositories)
|
||||
.where(eq(repositories.userId, userId));
|
||||
const existingRepoKeys = repositoryIdentityKeys(existingRepos);
|
||||
|
||||
for (const org of includedOrgs) {
|
||||
try {
|
||||
const source = findSourceForOrganization(org, sources);
|
||||
if (!source) continue;
|
||||
|
||||
const sourceProvider = createSourceProviderFromSource(source, { userId });
|
||||
const orgRepos = await sourceProvider.listOrganizationRepositories(org.name);
|
||||
|
||||
// The organization's fork policy (override -> global skipForks), the
|
||||
// same resolution the organization mirror path applies.
|
||||
const skipOrgForks = resolveOrganizationSkipForks({ orgOverrides: org.mirrorOverrides, config });
|
||||
const mirrorableRepos = orgRepos.filter(
|
||||
repo => repo.isDisabled !== true && !(skipOrgForks && repo.isForked)
|
||||
);
|
||||
|
||||
if (mirrorableRepos.length > 0) {
|
||||
const repoRecords = mirrorableRepos.map(repo =>
|
||||
normalizeGitRepoToInsert(
|
||||
{ ...repo, organization: repo.organization ?? org.name },
|
||||
{ userId, configId: config.id, sourceId: source.id }
|
||||
)
|
||||
);
|
||||
|
||||
// Batch insert to avoid SQLite parameter limit
|
||||
const sample = repoRecords[0];
|
||||
const columnCount = Object.keys(sample ?? {}).length || 1;
|
||||
const BATCH_SIZE = calcBatchSizeForInsert(columnCount);
|
||||
for (let i = 0; i < repoRecords.length; i += BATCH_SIZE) {
|
||||
const batch = repoRecords.slice(i, i + BATCH_SIZE);
|
||||
await db
|
||||
.insert(repositories)
|
||||
.values(batch)
|
||||
.onConflictDoNothing({ target: [repositories.userId, repositories.sourceId, repositories.normalizedFullName] });
|
||||
}
|
||||
console.log(`[Scheduler] Re-discovered ${mirrorableRepos.length} repositories for organization ${org.name} on source ${source.name} for user ${userId} during ${phase}`);
|
||||
}
|
||||
|
||||
await db
|
||||
.update(organizations)
|
||||
.set({ repositoryCount: orgRepos.length, updatedAt: new Date() })
|
||||
.where(eq(organizations.id, org.id));
|
||||
await rediscoverRepositoriesForOrganization({
|
||||
config,
|
||||
userId,
|
||||
organization: org,
|
||||
sources,
|
||||
existingRepoKeys,
|
||||
phase,
|
||||
});
|
||||
} catch (orgError) {
|
||||
console.error(`[Scheduler] Failed to re-discover repositories for organization ${org.name} for user ${userId} during ${phase}:`, orgError);
|
||||
}
|
||||
@@ -331,8 +426,27 @@ function sourceUsernamesBySourceId(sources: SourceRecord[]): Map<string, string>
|
||||
/**
|
||||
* Run scheduled mirror sync for a single user configuration
|
||||
*/
|
||||
/**
|
||||
* Keep one row per upstream repository in a scheduler batch. Duplicate rows
|
||||
* (the same repository imported under two sources on the same host) resolve
|
||||
* to one destination repository, and processing both in one Promise.all ran
|
||||
* every step twice against it. The dropped rows are logged so the duplicate
|
||||
* can be cleaned up; discovery no longer creates new ones.
|
||||
*/
|
||||
function dropDuplicateRowsForPass<T extends { fullName: string; sourceId?: string | null; sourceProvider?: string | null; sourceUrl?: string | null; normalizedFullName: string }>(
|
||||
rows: T[],
|
||||
pass: string
|
||||
): T[] {
|
||||
const { kept, dropped } = dedupeRepositoriesByIdentity(rows);
|
||||
for (const row of dropped) {
|
||||
console.warn(`[Scheduler] ${row.fullName} is tracked more than once (another row on the same host covers it); skipping the duplicate row in this ${pass} pass`);
|
||||
}
|
||||
return kept;
|
||||
}
|
||||
|
||||
async function runScheduledSync(config: any): Promise<void> {
|
||||
const userId = config.userId;
|
||||
const rateLimit = createRateLimitWatcher(userId);
|
||||
console.log(`[Scheduler] Running scheduled sync for user ${userId}`);
|
||||
|
||||
try {
|
||||
@@ -351,6 +465,15 @@ async function runScheduledSync(config: any): Promise<void> {
|
||||
console.log(`[Scheduler] Using schedule source for user ${userId}: ${String(source)} (timezone=${timezone})`);
|
||||
await persistScheduleRunState(config, currentTime);
|
||||
|
||||
// A rate limit left open by an earlier run (or by the boot auto-start)
|
||||
// would fail every source call here. Skip the whole run and come back
|
||||
// just after the reset instead of burning a tick on failures.
|
||||
const pausedAtStart = rateLimit.check(() => 'skipping this run');
|
||||
if (pausedAtStart) {
|
||||
await persistRateLimitedNextRun(config, currentTime, pausedAtStart);
|
||||
return;
|
||||
}
|
||||
|
||||
// Auto-discovery: Check for new repositories on every enabled source
|
||||
if (scheduleConfig.autoImport !== false) {
|
||||
console.log(`[Scheduler] Checking for new source repositories for user ${userId}...`);
|
||||
@@ -457,6 +580,11 @@ async function runScheduledSync(config: any): Promise<void> {
|
||||
console.log(`[Scheduler] Skipped ${forkSkippedCount - reposNeedingMirror.length} forked repositories from auto-mirror (organization skip forks)`);
|
||||
}
|
||||
|
||||
// Two rows for one upstream repository (imported under two sources
|
||||
// that point at the same host) resolve to one destination
|
||||
// repository; mirror it once per pass, not twice at the same time.
|
||||
reposNeedingMirror = dropDuplicateRowsForPass(reposNeedingMirror, 'auto-mirror');
|
||||
|
||||
if (reposNeedingMirror.length > 0) {
|
||||
console.log(`[Scheduler] Found ${reposNeedingMirror.length} repositories that need initial mirroring`);
|
||||
|
||||
@@ -467,11 +595,20 @@ async function runScheduledSync(config: any): Promise<void> {
|
||||
const batchSize = scheduleConfig.batchSize || 10;
|
||||
const pauseBetweenBatches = scheduleConfig.pauseBetweenBatches || 2000;
|
||||
for (let i = 0; i < reposNeedingMirror.length; i += batchSize) {
|
||||
const remaining = reposNeedingMirror.length - i;
|
||||
if (rateLimit.check(() => `leaving ${remaining} repositories to auto-mirror on the next run`)) {
|
||||
break;
|
||||
}
|
||||
|
||||
const batch = reposNeedingMirror.slice(i, Math.min(i + batchSize, reposNeedingMirror.length));
|
||||
console.log(`[Scheduler] Auto-mirror batch ${Math.floor(i / batchSize) + 1} of ${Math.ceil(reposNeedingMirror.length / batchSize)} (${batch.length} repos)`);
|
||||
|
||||
await Promise.all(
|
||||
batch.map(async (repo) => {
|
||||
if (rateLimit.check(() => `leaving ${remaining} repositories to auto-mirror on the next run`)) {
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
const repository: Repository = {
|
||||
...repo,
|
||||
@@ -508,6 +645,10 @@ async function runScheduledSync(config: any): Promise<void> {
|
||||
await mirrorRepositoryToDestination({ octokit: await resolveOctokit(repo), repository, config });
|
||||
console.log(`[Scheduler] Auto-mirrored repository: ${repo.fullName}`);
|
||||
} catch (error) {
|
||||
if (isRateLimitError(error)) {
|
||||
await restoreRepositoryAfterRateLimit(repo, 'auto-mirror');
|
||||
return;
|
||||
}
|
||||
console.error(`[Scheduler] Failed to auto-mirror repository ${repo.fullName}:`, error);
|
||||
}
|
||||
})
|
||||
@@ -584,8 +725,14 @@ async function runScheduledSync(config: any): Promise<void> {
|
||||
// Non-critical logging, ignore errors
|
||||
}
|
||||
|
||||
reposToSync = dropDuplicateRowsForPass(reposToSync, 'sync');
|
||||
|
||||
if (reposToSync.length === 0) {
|
||||
console.log(`[Scheduler] No repositories to sync for user ${userId}`);
|
||||
const pausedAfterMirror = rateLimit.current();
|
||||
if (pausedAfterMirror) {
|
||||
await persistRateLimitedNextRun(config, currentTime, pausedAfterMirror);
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
@@ -596,7 +743,11 @@ async function runScheduledSync(config: any): Promise<void> {
|
||||
const pauseBetweenBatches = scheduleConfig.pauseBetweenBatches || 5000;
|
||||
const concurrent = scheduleConfig.concurrent ?? false;
|
||||
|
||||
for (let i = 0; i < reposToSync.length; i += batchSize) {
|
||||
syncBatches: for (let i = 0; i < reposToSync.length; i += batchSize) {
|
||||
if (rateLimit.check(() => `leaving ${reposToSync.length - i} repositories to sync on the next run`)) {
|
||||
break;
|
||||
}
|
||||
|
||||
const batch = reposToSync.slice(i, i + batchSize);
|
||||
|
||||
if (concurrent) {
|
||||
@@ -606,8 +757,11 @@ async function runScheduledSync(config: any): Promise<void> {
|
||||
);
|
||||
} else {
|
||||
// Process batch sequentially
|
||||
for (const repo of batch) {
|
||||
await syncSingleRepository(config, repo);
|
||||
for (let j = 0; j < batch.length; j++) {
|
||||
if (rateLimit.check(() => `leaving ${reposToSync.length - i - j} repositories to sync on the next run`)) {
|
||||
break syncBatches;
|
||||
}
|
||||
await syncSingleRepository(config, batch[j]);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -616,7 +770,16 @@ async function runScheduledSync(config: any): Promise<void> {
|
||||
await new Promise(resolve => setTimeout(resolve, pauseBetweenBatches));
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
// A run that stopped on the rate limit comes back just after the reset,
|
||||
// not at the schedule's own next slot, so the repositories it left behind
|
||||
// are not held for a whole cron period.
|
||||
const pausedUntil = rateLimit.current();
|
||||
if (pausedUntil) {
|
||||
await persistRateLimitedNextRun(config, currentTime, pausedUntil);
|
||||
return;
|
||||
}
|
||||
|
||||
console.log(`[Scheduler] Completed scheduled sync for user ${userId}`);
|
||||
} catch (error) {
|
||||
console.error(`[Scheduler] Error during scheduled sync for user ${userId}:`, error);
|
||||
@@ -642,6 +805,14 @@ async function syncSingleRepository(config: any, repo: any): Promise<void> {
|
||||
await syncRepositoryOnDestination({ config, repository });
|
||||
console.log(`[Scheduler] Successfully synced repository ${repo.fullName}`);
|
||||
} catch (error) {
|
||||
// The sync path claimed the row as "syncing" and the failure handler below
|
||||
// it wrote "failed". A rate limit is not the repository's fault, so put the
|
||||
// row back where it was and let the run after the reset retry it.
|
||||
if (isRateLimitError(error)) {
|
||||
await restoreRepositoryAfterRateLimit(repo, 'sync');
|
||||
return;
|
||||
}
|
||||
|
||||
console.error(`[Scheduler] Failed to sync repository ${repo.fullName}:`, error);
|
||||
|
||||
// Update repository status to failed
|
||||
@@ -726,6 +897,12 @@ async function performInitialAutoStart(): Promise<void> {
|
||||
|
||||
console.log(`[Scheduler] Auto-starting for user ${config.userId}...`);
|
||||
|
||||
// Auto-start runs outside the scheduler lock, but it mirrors the same
|
||||
// repositories and must stop on a rate limit too: otherwise it keeps
|
||||
// failing every repository in turn and hands the scheduler a pile of
|
||||
// rows marked "failed" (issue #437).
|
||||
const rateLimit = createRateLimitWatcher(config.userId);
|
||||
|
||||
try {
|
||||
// Step 1: Import repositories from every enabled source
|
||||
console.log(`[Scheduler] Step 1: Importing repositories from the configured sources for user ${config.userId}...`);
|
||||
@@ -808,6 +985,7 @@ async function performInitialAutoStart(): Promise<void> {
|
||||
if (skippedCount > 0) {
|
||||
console.log(`[Scheduler] Skipped ${skippedCount} repositories from initial auto-mirror (autoMirror=${autoMirrorOwned}, autoMirrorStarred=${autoMirrorStarred})`);
|
||||
}
|
||||
reposNeedingMirror = dropDuplicateRowsForPass(reposNeedingMirror, 'initial auto-mirror');
|
||||
|
||||
if (reposNeedingMirror.length > 0) {
|
||||
console.log(`[Scheduler] Found ${reposNeedingMirror.length} repositories that need mirroring`);
|
||||
@@ -818,11 +996,20 @@ async function performInitialAutoStart(): Promise<void> {
|
||||
// Process repositories in batches
|
||||
const batchSize = config.scheduleConfig?.batchSize || 5;
|
||||
for (let i = 0; i < reposNeedingMirror.length; i += batchSize) {
|
||||
const remaining = reposNeedingMirror.length - i;
|
||||
if (rateLimit.check(() => `leaving ${remaining} repositories for the first scheduled run`)) {
|
||||
break;
|
||||
}
|
||||
|
||||
const batch = reposNeedingMirror.slice(i, Math.min(i + batchSize, reposNeedingMirror.length));
|
||||
console.log(`[Scheduler] Processing batch ${Math.floor(i / batchSize) + 1} of ${Math.ceil(reposNeedingMirror.length / batchSize)} (${batch.length} repos)`);
|
||||
|
||||
await Promise.all(
|
||||
batch.map(async (repo) => {
|
||||
if (rateLimit.check(() => `leaving ${remaining} repositories for the first scheduled run`)) {
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
const repository: Repository = {
|
||||
...repo,
|
||||
@@ -842,6 +1029,10 @@ async function performInitialAutoStart(): Promise<void> {
|
||||
});
|
||||
console.log(`[Scheduler] Successfully mirrored repository: ${repo.fullName}`);
|
||||
} catch (error) {
|
||||
if (isRateLimitError(error)) {
|
||||
await restoreRepositoryAfterRateLimit(repo, 'auto-start mirror');
|
||||
return;
|
||||
}
|
||||
console.error(`[Scheduler] Failed to mirror repository ${repo.fullName}:`, error);
|
||||
}
|
||||
})
|
||||
@@ -862,8 +1053,16 @@ async function performInitialAutoStart(): Promise<void> {
|
||||
|
||||
// Update the schedule config to indicate we've run
|
||||
const currentTime = new Date();
|
||||
const nextRun = await persistScheduleRunState(config, currentTime, true);
|
||||
|
||||
let nextRun = await persistScheduleRunState(config, currentTime, true);
|
||||
|
||||
// A rate limit that stopped the initial mirror also moves the first
|
||||
// scheduled run to just after the reset, so the repositories left
|
||||
// behind are not held for a whole schedule period.
|
||||
const pausedUntil = rateLimit.current();
|
||||
if (pausedUntil) {
|
||||
nextRun = await persistRateLimitedNextRun(config, currentTime, pausedUntil);
|
||||
}
|
||||
|
||||
console.log(`[Scheduler] Auto-start completed for user ${config.userId}, next sync at ${nextRun.toISOString()}`);
|
||||
|
||||
} catch (error) {
|
||||
|
||||
@@ -0,0 +1,15 @@
|
||||
import { describe, expect, test } from "bun:test";
|
||||
import { shutdownExitCode } from "./shutdown-manager";
|
||||
|
||||
describe("shutdownExitCode (#465)", () => {
|
||||
test("a crash-driven shutdown exits non-zero so Restart=on-failure restarts the service", () => {
|
||||
expect(shutdownExitCode("UNCAUGHT_EXCEPTION")).toBe(1);
|
||||
expect(shutdownExitCode("UNHANDLED_REJECTION")).toBe(1);
|
||||
});
|
||||
|
||||
test("a signal-driven stop is a clean exit", () => {
|
||||
expect(shutdownExitCode("SIGTERM")).toBe(0);
|
||||
expect(shutdownExitCode("SIGINT")).toBe(0);
|
||||
expect(shutdownExitCode("SIGHUP")).toBe(0);
|
||||
});
|
||||
});
|
||||
@@ -147,6 +147,11 @@ async function executeShutdownCallbacks(): Promise<void> {
|
||||
/**
|
||||
* Perform graceful shutdown of the application
|
||||
*/
|
||||
/** Exit status for a completed shutdown: 1 after a crash, 0 after a signal. */
|
||||
export function shutdownExitCode(signal: string): number {
|
||||
return signal === 'UNCAUGHT_EXCEPTION' || signal === 'UNHANDLED_REJECTION' ? 1 : 0;
|
||||
}
|
||||
|
||||
export async function gracefulShutdown(signal: string = 'UNKNOWN'): Promise<void> {
|
||||
if (shutdownInProgress) {
|
||||
console.log('⚠️ Shutdown already in progress, ignoring additional signal');
|
||||
@@ -187,8 +192,10 @@ export async function gracefulShutdown(signal: string = 'UNKNOWN'): Promise<void
|
||||
shutdownTimeout = null;
|
||||
}
|
||||
|
||||
// Exit with success code
|
||||
process.exit(0);
|
||||
// A shutdown forced by a crash exits non-zero so a supervisor with
|
||||
// Restart=on-failure (the Proxmox script's unit) starts the service
|
||||
// again; a signal-driven stop is a clean exit (#465).
|
||||
process.exit(shutdownExitCode(signal));
|
||||
|
||||
} catch (error) {
|
||||
console.error('\n❌ Error during graceful shutdown:', error);
|
||||
|
||||
@@ -44,7 +44,17 @@ describe("gitea.ts migrate sites", () => {
|
||||
|
||||
test("the mirror entry points accept a null GitHub client and skip metadata without one", () => {
|
||||
expect(count(source, "octokit: Octokit | null;")).toBeGreaterThanOrEqual(4);
|
||||
expect(count(source, "&& octokit !== null;")).toBe(10);
|
||||
// Releases no longer gate on a GitHub client here: both mirror paths hand
|
||||
// the decision to mirrorRepositoryReleases, which picks the source's own
|
||||
// release path (#440). Everything else still needs Octokit.
|
||||
expect(count(source, "&& octokit !== null;")).toBe(8);
|
||||
});
|
||||
|
||||
test("both mirror paths route releases through the repository's own source", () => {
|
||||
expect(count(source, "await mirrorRepositoryReleases({")).toBe(2);
|
||||
expect(count(source, "sourceToken: repoSourceToken,")).toBe(2);
|
||||
expect(source).toContain('if (sourceConnection.provider === "gitea") {');
|
||||
expect(source).toContain("await mirrorGiteaSourceReleasesToGitea({");
|
||||
});
|
||||
});
|
||||
|
||||
@@ -77,14 +87,25 @@ describe("discovery and housekeeping go through the source provider", () => {
|
||||
expect(recovery).toContain("findSourceForRepository(repo, sources)");
|
||||
expect(recovery).toContain("repoSource?.provider === 'github'");
|
||||
const enhanced = read("gitea-enhanced.ts");
|
||||
expect(enhanced).toContain("getRepositorySource(repository).provider === \"github\"");
|
||||
expect(enhanced).toContain("const repoSourceKind = getRepositorySource(repository).provider;");
|
||||
expect(enhanced).toContain('const repoIsGitHub = repoSourceKind === "github";');
|
||||
expect(enhanced).toContain("findSourceForRepository(repository, sources)");
|
||||
// The sync path mirrors releases from a Gitea/Forgejo source too (#440).
|
||||
expect(enhanced).toContain('repoSourceKind === "gitea" && repoSource');
|
||||
expect(enhanced).toContain("mirrorGiteaSourceReleasesToGitea");
|
||||
});
|
||||
|
||||
test("the metadata resolver clamps GitHub only options for other sources", () => {
|
||||
const source = read("utils", "mirror-overrides.ts");
|
||||
expect(source).toContain("GITHUB_ONLY_METADATA_KEYS");
|
||||
expect(source).toContain('normalizeSourceProviderKind(repository.sourceProvider) !== "github"');
|
||||
expect(source).toContain("const sourceKind = normalizeSourceProviderKind(repository.sourceProvider);");
|
||||
expect(source).toContain('if (sourceKind !== "github") {');
|
||||
// Releases have their own gate: GitHub and Gitea/Forgejo can list them,
|
||||
// GitLab cannot, so they left the GitHub only set (#440).
|
||||
expect(source).toContain(
|
||||
'export const GITHUB_ONLY_METADATA_KEYS = [\n "mirrorMetadata",'
|
||||
);
|
||||
expect(source).toContain("if (!sourceKindSupportsReleases(sourceKind)) {");
|
||||
});
|
||||
|
||||
test("the discovery routes use the provider, match the pasted host against connected sources and stamp the source on inserted rows", () => {
|
||||
|
||||
@@ -0,0 +1,203 @@
|
||||
/**
|
||||
* Unit tests for the Gitea/Forgejo source adapter's release listing (#440).
|
||||
*
|
||||
* Release mirroring used to be GitHub only. Gitea and Forgejo expose
|
||||
* `GET /repos/{owner}/{repo}/releases` with the same limit/page pagination and
|
||||
* x-total-count header as their other list endpoints, which is what lets a
|
||||
* Codeberg or self hosted repository mirror its releases.
|
||||
*
|
||||
* The global fetch is swapped rather than the module, because bun's
|
||||
* mock.module is process-wide and leaks into other test files.
|
||||
*/
|
||||
|
||||
import { afterEach, describe, expect, it } from "bun:test";
|
||||
import { GiteaSourceProvider } from "./gitea-source";
|
||||
import type { SourceConnection } from "./types";
|
||||
|
||||
const SOURCE_URL = "https://codeberg.org";
|
||||
const RELEASES_PATH = `${SOURCE_URL}/api/v1/repos/acme/demo/releases`;
|
||||
|
||||
const realFetch = globalThis.fetch;
|
||||
|
||||
afterEach(() => {
|
||||
globalThis.fetch = realFetch;
|
||||
});
|
||||
|
||||
function connection(token = "source-token"): SourceConnection {
|
||||
return {
|
||||
provider: "gitea",
|
||||
url: SOURCE_URL,
|
||||
username: "acme",
|
||||
token,
|
||||
};
|
||||
}
|
||||
|
||||
function release(tag: string, extra: Record<string, unknown> = {}) {
|
||||
return {
|
||||
id: Number(tag.replace(/\D/g, "")) || 1,
|
||||
tag_name: tag,
|
||||
name: `Release ${tag}`,
|
||||
body: `notes for ${tag}`,
|
||||
draft: false,
|
||||
prerelease: false,
|
||||
created_at: "2026-01-01T00:00:00Z",
|
||||
published_at: "2026-01-02T00:00:00Z",
|
||||
assets: [],
|
||||
...extra,
|
||||
};
|
||||
}
|
||||
|
||||
interface FakeOptions {
|
||||
/** Pages returned in order, newest first overall. */
|
||||
pages: Array<Array<Record<string, unknown>>>;
|
||||
/** Value for the x-total-count header, or null to omit it. */
|
||||
totalCount?: number | null;
|
||||
}
|
||||
|
||||
function fakeSource(options: FakeOptions) {
|
||||
const requests: Array<{ url: string; authorization: string | null }> = [];
|
||||
|
||||
globalThis.fetch = (async (input: any, init?: any) => {
|
||||
const url = typeof input === "string" ? input : String(input);
|
||||
const headers = new Headers(init?.headers ?? {});
|
||||
requests.push({ url, authorization: headers.get("authorization") });
|
||||
|
||||
const page = Number(new URL(url).searchParams.get("page") ?? "1");
|
||||
const body = options.pages[page - 1] ?? [];
|
||||
const responseHeaders: Record<string, string> = {
|
||||
"content-type": "application/json",
|
||||
};
|
||||
if (options.totalCount !== null && options.totalCount !== undefined) {
|
||||
responseHeaders["x-total-count"] = String(options.totalCount);
|
||||
}
|
||||
return new Response(JSON.stringify(body), {
|
||||
status: 200,
|
||||
headers: responseHeaders,
|
||||
});
|
||||
}) as unknown as typeof fetch;
|
||||
|
||||
return requests;
|
||||
}
|
||||
|
||||
describe("GiteaSourceProvider.listReleases", () => {
|
||||
it("asks for the newest releases with limit and page", async () => {
|
||||
const requests = fakeSource({ pages: [[release("v1.0.0"), release("v0.9.0")]], totalCount: 2 });
|
||||
|
||||
const releases = await new GiteaSourceProvider(connection()).listReleases("acme", "demo", 10);
|
||||
|
||||
expect(releases.map((r) => r.tag_name)).toEqual(["v1.0.0", "v0.9.0"]);
|
||||
expect(requests).toHaveLength(1);
|
||||
expect(requests[0].url).toBe(`${RELEASES_PATH}?limit=10&page=1`);
|
||||
});
|
||||
|
||||
it("caps the page size at the requested limit and never returns more", async () => {
|
||||
const requests = fakeSource({
|
||||
pages: [[release("v3"), release("v2"), release("v1")]],
|
||||
totalCount: 99,
|
||||
});
|
||||
|
||||
const releases = await new GiteaSourceProvider(connection()).listReleases("acme", "demo", 2);
|
||||
|
||||
expect(requests[0].url).toBe(`${RELEASES_PATH}?limit=2&page=1`);
|
||||
// The instance may over-serve; the limit still wins.
|
||||
expect(releases.map((r) => r.tag_name)).toEqual(["v3", "v2"]);
|
||||
expect(requests).toHaveLength(1);
|
||||
});
|
||||
|
||||
it("pages until the limit is reached, capping the page size at 50", async () => {
|
||||
const firstPage = Array.from({ length: 50 }, (_, i) => release(`v${100 - i}`));
|
||||
const secondPage = [release("v50"), release("v49")];
|
||||
const requests = fakeSource({ pages: [firstPage, secondPage], totalCount: 120 });
|
||||
|
||||
const releases = await new GiteaSourceProvider(connection()).listReleases("acme", "demo", 52);
|
||||
|
||||
expect(releases).toHaveLength(52);
|
||||
expect(requests.map((r) => r.url)).toEqual([
|
||||
`${RELEASES_PATH}?limit=50&page=1`,
|
||||
`${RELEASES_PATH}?limit=2&page=2`,
|
||||
]);
|
||||
});
|
||||
|
||||
it("stops at x-total-count instead of asking for a page that cannot exist", async () => {
|
||||
const firstPage = Array.from({ length: 50 }, (_, i) => release(`v${100 - i}`));
|
||||
const requests = fakeSource({ pages: [firstPage, []], totalCount: 50 });
|
||||
|
||||
const releases = await new GiteaSourceProvider(connection()).listReleases("acme", "demo", 100);
|
||||
|
||||
expect(releases).toHaveLength(50);
|
||||
expect(requests).toHaveLength(1);
|
||||
});
|
||||
|
||||
it("falls back to a short page when the instance omits x-total-count", async () => {
|
||||
const firstPage = Array.from({ length: 50 }, (_, i) => release(`v${100 - i}`));
|
||||
const requests = fakeSource({ pages: [firstPage, [release("v50")]], totalCount: null });
|
||||
|
||||
const releases = await new GiteaSourceProvider(connection()).listReleases("acme", "demo", 100);
|
||||
|
||||
expect(releases).toHaveLength(51);
|
||||
expect(requests).toHaveLength(2);
|
||||
});
|
||||
|
||||
it("sends the source token when the connection has one", async () => {
|
||||
const requests = fakeSource({ pages: [[release("v1")]], totalCount: 1 });
|
||||
|
||||
await new GiteaSourceProvider(connection("source-token")).listReleases("acme", "demo", 5);
|
||||
|
||||
expect(requests[0].authorization).toBe("token source-token");
|
||||
});
|
||||
|
||||
it("sends no Authorization header for a tokenless public source", async () => {
|
||||
const requests = fakeSource({ pages: [[release("v1")]], totalCount: 1 });
|
||||
|
||||
await new GiteaSourceProvider(connection("")).listReleases("acme", "demo", 5);
|
||||
|
||||
expect(requests[0].authorization).toBeNull();
|
||||
});
|
||||
|
||||
it("normalizes assets and drops entries with nothing to download", async () => {
|
||||
fakeSource({
|
||||
pages: [
|
||||
[
|
||||
release("v1.0.0", {
|
||||
assets: [
|
||||
{
|
||||
id: 1,
|
||||
name: "firmware.bin",
|
||||
size: 2048,
|
||||
browser_download_url: `${SOURCE_URL}/attachments/abc`,
|
||||
},
|
||||
{ id: 2, name: "no-url.bin", size: 10 },
|
||||
{ id: 3, size: 10, browser_download_url: `${SOURCE_URL}/attachments/def` },
|
||||
],
|
||||
}),
|
||||
],
|
||||
],
|
||||
totalCount: 1,
|
||||
});
|
||||
|
||||
const [mirrored] = await new GiteaSourceProvider(connection()).listReleases(
|
||||
"acme",
|
||||
"demo",
|
||||
5
|
||||
);
|
||||
|
||||
expect(mirrored.assets).toEqual([
|
||||
{
|
||||
name: "firmware.bin",
|
||||
size: 2048,
|
||||
browser_download_url: `${SOURCE_URL}/attachments/abc`,
|
||||
},
|
||||
]);
|
||||
expect(mirrored.name).toBe("Release v1.0.0");
|
||||
expect(mirrored.body).toBe("notes for v1.0.0");
|
||||
expect(mirrored.published_at).toBe("2026-01-02T00:00:00Z");
|
||||
});
|
||||
|
||||
it("drops releases without a tag, which cannot be mirrored", async () => {
|
||||
fakeSource({ pages: [[release("v1"), { ...release("x"), tag_name: "" }]], totalCount: 2 });
|
||||
|
||||
const releases = await new GiteaSourceProvider(connection()).listReleases("acme", "demo", 5);
|
||||
|
||||
expect(releases.map((r) => r.tag_name)).toEqual(["v1"]);
|
||||
});
|
||||
});
|
||||
@@ -24,6 +24,7 @@ import type {
|
||||
SourceFailedOrganization,
|
||||
SourceOrganizationResult,
|
||||
SourceProvider,
|
||||
SourceRelease,
|
||||
SourceRepositoryPath,
|
||||
} from "./types";
|
||||
|
||||
@@ -60,6 +61,50 @@ export interface GiteaRepository {
|
||||
updated_at?: string;
|
||||
}
|
||||
|
||||
export interface GiteaReleaseAsset {
|
||||
id?: number;
|
||||
name?: string;
|
||||
size?: number;
|
||||
browser_download_url?: string;
|
||||
}
|
||||
|
||||
export interface GiteaRelease {
|
||||
id?: number;
|
||||
tag_name?: string;
|
||||
name?: string | null;
|
||||
body?: string | null;
|
||||
draft?: boolean;
|
||||
prerelease?: boolean;
|
||||
created_at?: string;
|
||||
published_at?: string | null;
|
||||
assets?: GiteaReleaseAsset[] | null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Normalize one Gitea/Forgejo release into the shape the release mirror works
|
||||
* on. Assets without a name or a download URL are dropped: there is nothing to
|
||||
* fetch or to name the attachment after.
|
||||
*/
|
||||
export function mapGiteaRelease(release: GiteaRelease): SourceRelease {
|
||||
const createdAt = release.created_at || release.published_at || "";
|
||||
return {
|
||||
tag_name: release.tag_name || "",
|
||||
name: release.name ?? null,
|
||||
body: release.body ?? "",
|
||||
draft: Boolean(release.draft),
|
||||
prerelease: Boolean(release.prerelease),
|
||||
created_at: createdAt,
|
||||
published_at: release.published_at ?? null,
|
||||
assets: (release.assets ?? [])
|
||||
.filter((asset) => !!asset?.name && !!asset?.browser_download_url)
|
||||
.map((asset) => ({
|
||||
name: asset.name!,
|
||||
size: Number(asset.size) || 0,
|
||||
browser_download_url: asset.browser_download_url!,
|
||||
})),
|
||||
};
|
||||
}
|
||||
|
||||
export interface GiteaOrganization {
|
||||
id?: number;
|
||||
username?: string;
|
||||
@@ -167,24 +212,31 @@ export class GiteaSourceProvider implements SourceProvider {
|
||||
|
||||
private async paginate<T>(
|
||||
path: string,
|
||||
params: Record<string, string | number | boolean | undefined> = {}
|
||||
params: Record<string, string | number | boolean | undefined> = {},
|
||||
/** Stop once this many items are collected. Absent means every page. */
|
||||
maxItems?: number
|
||||
): Promise<T[]> {
|
||||
const items: T[] = [];
|
||||
for (let page = 1; page <= MAX_PAGES; page += 1) {
|
||||
const pageSize =
|
||||
maxItems === undefined ? PAGE_SIZE : Math.min(PAGE_SIZE, maxItems - items.length);
|
||||
if (pageSize <= 0) break;
|
||||
|
||||
const url = withQuery(`${this.apiRoot}${path}`, {
|
||||
...params,
|
||||
limit: PAGE_SIZE,
|
||||
limit: pageSize,
|
||||
page,
|
||||
});
|
||||
const { data, response } = await sourceFetch<T[]>(url, { headers: this.headers() });
|
||||
if (!Array.isArray(data) || data.length === 0) break;
|
||||
items.push(...data);
|
||||
if (maxItems !== undefined && items.length >= maxItems) break;
|
||||
|
||||
const header = response.headers.get("x-total-count");
|
||||
const total = header === null ? Number.NaN : Number(header);
|
||||
if (Number.isFinite(total)) {
|
||||
if (items.length >= total) break;
|
||||
} else if (data.length < PAGE_SIZE) {
|
||||
} else if (data.length < pageSize) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
@@ -364,6 +416,26 @@ export class GiteaSourceProvider implements SourceProvider {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* The newest `limit` releases of a repository (#440).
|
||||
*
|
||||
* Gitea and Forgejo return releases newest first and accept the same
|
||||
* limit/page pagination as every other list endpoint. Public repositories
|
||||
* answer without a token, so a tokenless source can mirror releases too.
|
||||
*/
|
||||
async listReleases(owner: string, repo: string, limit: number): Promise<SourceRelease[]> {
|
||||
const max = Math.max(1, Math.floor(limit) || 1);
|
||||
const releases = await this.paginate<GiteaRelease>(
|
||||
`/repos/${encodeURIComponent(owner)}/${encodeURIComponent(repo)}/releases`,
|
||||
{},
|
||||
max
|
||||
);
|
||||
return releases
|
||||
.slice(0, max)
|
||||
.map(mapGiteaRelease)
|
||||
.filter((release) => release.tag_name.length > 0);
|
||||
}
|
||||
|
||||
async testConnection(): Promise<SourceAccount> {
|
||||
if (!this.hasToken) {
|
||||
throw new Error("An access token is required.");
|
||||
|
||||
@@ -0,0 +1,68 @@
|
||||
/**
|
||||
* sourceFetch goes through the outbound guard (GHSA-p7w3-46pg-mv6h): the
|
||||
* source URL is stored by a signed in user, so it gets the same treatment
|
||||
* as every other user supplied URL, and the upstream body never lands in
|
||||
* the error message.
|
||||
*/
|
||||
|
||||
import { afterEach, describe, expect, it } from "bun:test";
|
||||
import { OutboundUrlError } from "@/lib/utils/outbound-url";
|
||||
import { sourceFetch, SourceApiError } from "./http";
|
||||
|
||||
const realFetch = globalThis.fetch;
|
||||
|
||||
afterEach(() => {
|
||||
globalThis.fetch = realFetch;
|
||||
});
|
||||
|
||||
describe("sourceFetch", () => {
|
||||
it("refuses a link local source before any request leaves", async () => {
|
||||
let called = false;
|
||||
globalThis.fetch = (async () => {
|
||||
called = true;
|
||||
return new Response("{}", { status: 200 });
|
||||
}) as unknown as typeof fetch;
|
||||
|
||||
await expect(
|
||||
sourceFetch("http://169.254.169.254/api/v1/orgs/x")
|
||||
).rejects.toBeInstanceOf(OutboundUrlError);
|
||||
expect(called).toBe(false);
|
||||
});
|
||||
|
||||
it("never follows redirects", async () => {
|
||||
let redirect: RequestRedirect | undefined;
|
||||
globalThis.fetch = (async (_input: any, init?: RequestInit) => {
|
||||
redirect = init?.redirect;
|
||||
return new Response("[]", {
|
||||
status: 200,
|
||||
headers: { "content-type": "application/json" },
|
||||
});
|
||||
}) as typeof fetch;
|
||||
|
||||
const { data } = await sourceFetch<unknown[]>("http://192.168.1.10:3000/api/v1/user/repos");
|
||||
expect(data).toEqual([]);
|
||||
expect(redirect).toBe("manual");
|
||||
});
|
||||
|
||||
it("keeps the upstream body out of the error message", async () => {
|
||||
globalThis.fetch = (async () =>
|
||||
new Response(JSON.stringify({ message: "Forbidden: token lacks scope. cluster=prod" }), {
|
||||
status: 403,
|
||||
headers: { "content-type": "application/json" },
|
||||
})) as unknown as typeof fetch;
|
||||
|
||||
let caught: unknown;
|
||||
try {
|
||||
await sourceFetch("http://192.168.1.10:3000/api/v1/orgs/any/repos");
|
||||
} catch (error) {
|
||||
caught = error;
|
||||
}
|
||||
expect(caught).toBeInstanceOf(SourceApiError);
|
||||
const error = caught as SourceApiError;
|
||||
expect(error.status).toBe(403);
|
||||
expect(error.message).toBe(
|
||||
"Request to http://192.168.1.10:3000/api/v1/orgs/any/repos failed with status 403"
|
||||
);
|
||||
expect(error.body).toContain("token lacks scope");
|
||||
});
|
||||
});
|
||||
@@ -5,6 +5,8 @@
|
||||
* of database imports so the adapters stay pure.
|
||||
*/
|
||||
|
||||
import { safeFetch } from "@/lib/utils/outbound-url";
|
||||
|
||||
export const DEFAULT_SOURCE_TIMEOUT_MS = 30_000;
|
||||
|
||||
export class SourceApiError extends Error {
|
||||
@@ -41,7 +43,11 @@ export async function sourceFetch<T>(
|
||||
): Promise<SourceFetchResult<T>> {
|
||||
const { method = "GET", headers = {}, timeoutMs = DEFAULT_SOURCE_TIMEOUT_MS } = init;
|
||||
|
||||
const response = await fetch(url, {
|
||||
// Source URLs are stored by signed in users, so they go through the same
|
||||
// outbound guard as every other user supplied URL: link local and
|
||||
// metadata addresses are refused, plain http is pinned to the checked
|
||||
// address and redirects are not followed (GHSA-p7w3-46pg-mv6h).
|
||||
const response = await safeFetch(url, {
|
||||
method,
|
||||
headers: { Accept: "application/json", ...headers },
|
||||
signal: AbortSignal.timeout(timeoutMs),
|
||||
@@ -54,9 +60,14 @@ export async function sourceFetch<T>(
|
||||
} catch {
|
||||
// The status is what matters; the body is only for diagnostics.
|
||||
}
|
||||
// The body stays on the error for logs and callers that inspect it and
|
||||
// never goes into the message, which can reach API clients.
|
||||
const detail = summarizeErrorBody(body);
|
||||
if (detail) {
|
||||
console.warn(`[Source] ${method} ${url} failed with status ${response.status}: ${detail}`);
|
||||
}
|
||||
throw new SourceApiError(
|
||||
`Request to ${url} failed with status ${response.status}${detail ? `: ${detail}` : ""}`,
|
||||
`Request to ${url} failed with status ${response.status}`,
|
||||
response.status,
|
||||
url,
|
||||
body
|
||||
|
||||
@@ -16,6 +16,26 @@ export function isBetaSourceProvider(kind: SourceProviderKind): boolean {
|
||||
return kind !== DEFAULT_SOURCE_PROVIDER;
|
||||
}
|
||||
|
||||
/**
|
||||
* Sources whose API can list releases with their assets.
|
||||
*
|
||||
* GitHub has always been able to. Gitea and Forgejo expose the same shape at
|
||||
* `GET /repos/{owner}/{repo}/releases`, so Codeberg and self hosted instances
|
||||
* can mirror releases too (#440). GitLab is code only: its release API is not
|
||||
* wired up, so the resolver still forces releases off for it.
|
||||
*/
|
||||
export const RELEASE_CAPABLE_SOURCE_KINDS = [
|
||||
"github",
|
||||
"gitea",
|
||||
] as const satisfies readonly SourceProviderKind[];
|
||||
|
||||
/** True when a source kind's API can list releases. Unknown values are GitHub rows. */
|
||||
export function sourceKindSupportsReleases(kind: unknown): boolean {
|
||||
return (RELEASE_CAPABLE_SOURCE_KINDS as readonly string[]).includes(
|
||||
normalizeSourceProviderKind(kind)
|
||||
);
|
||||
}
|
||||
|
||||
/** Base URL used when the config has no instance URL for the provider. */
|
||||
export const SOURCE_PROVIDER_DEFAULT_URLS: Record<SourceProviderKind, string> = {
|
||||
github: "https://github.com",
|
||||
|
||||
@@ -51,6 +51,32 @@ export interface SourceRepositoryPath {
|
||||
repo: string;
|
||||
}
|
||||
|
||||
/** One file attached to a release on a source host. */
|
||||
export interface SourceReleaseAsset {
|
||||
name: string;
|
||||
size: number;
|
||||
/** Direct download URL on the source host. */
|
||||
browser_download_url: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* One release, in the shape the release mirror works on.
|
||||
*
|
||||
* Field names follow the GitHub and Gitea payloads (both use snake_case and
|
||||
* agree on every field here), so the GitHub path needed no mapping when the
|
||||
* mirror was made source agnostic (#440).
|
||||
*/
|
||||
export interface SourceRelease {
|
||||
tag_name: string;
|
||||
name?: string | null;
|
||||
body?: string | null;
|
||||
draft?: boolean;
|
||||
prerelease?: boolean;
|
||||
created_at: string;
|
||||
published_at?: string | null;
|
||||
assets: SourceReleaseAsset[];
|
||||
}
|
||||
|
||||
/**
|
||||
* The operations the mirror pipeline needs from a source host.
|
||||
*
|
||||
@@ -92,6 +118,15 @@ export interface SourceProvider {
|
||||
/** Whether the account currently stars the repository. Throws on errors other than 404. */
|
||||
isRepositoryStarred(owner: string, name: string): Promise<boolean>;
|
||||
|
||||
/**
|
||||
* The newest `limit` releases of a repository, newest first.
|
||||
*
|
||||
* Optional: only hosts whose API exposes releases implement it (GitHub and
|
||||
* Gitea/Forgejo). GitLab is code only, so it leaves this out and
|
||||
* `sourceKindSupportsReleases` keeps the release switch off for it.
|
||||
*/
|
||||
listReleases?(owner: string, repo: string, limit: number): Promise<SourceRelease[]>;
|
||||
|
||||
/** Verify the token and return the account it belongs to. */
|
||||
testConnection(): Promise<SourceAccount>;
|
||||
|
||||
|
||||
@@ -198,3 +198,29 @@ describe("showErrorToast", () => {
|
||||
expect(calls[0][1].description).toContain("http://10.10.20.45:4321");
|
||||
});
|
||||
});
|
||||
|
||||
describe("createSecureErrorResponse and errors from other hosts", () => {
|
||||
test("never forwards the message of an error raised by a request to another host", async () => {
|
||||
const { createSecureErrorResponse } = await import("./utils");
|
||||
const { HttpError } = await import("./http-client");
|
||||
const { SourceApiError } = await import("./source-providers/http");
|
||||
|
||||
const remote = new SourceApiError(
|
||||
"Request to http://10.0.0.9/api/v1/orgs/x failed with status 403: Forbidden",
|
||||
403,
|
||||
"http://10.0.0.9/api/v1/orgs/x"
|
||||
);
|
||||
const destination = new HttpError("HTTP 404: Not Found", 404, "Not Found");
|
||||
const local = new Error("Organization not found");
|
||||
|
||||
for (const [error, forwarded] of [
|
||||
[remote, false],
|
||||
[destination, false],
|
||||
[local, true],
|
||||
] as const) {
|
||||
const response = createSecureErrorResponse(error, "test");
|
||||
const body = (await response.json()) as { error: string };
|
||||
expect(body.error === error.message).toBe(forwarded);
|
||||
}
|
||||
});
|
||||
});
|
||||
+9
-3
@@ -351,9 +351,15 @@ export function createSecureErrorResponse(
|
||||
/use post method/i,
|
||||
];
|
||||
|
||||
const isSafeError = safeErrorPatterns.some(pattern =>
|
||||
pattern.test(error.message)
|
||||
);
|
||||
// An error raised by a request to another host carries that host's URL
|
||||
// and status, and the words above are exactly what such hosts put in
|
||||
// their bodies. Those errors are matched by type, never by text.
|
||||
const describesRemoteHost =
|
||||
error.name === "SourceApiError" || error.name === "HttpError";
|
||||
|
||||
const isSafeError =
|
||||
!describesRemoteHost &&
|
||||
safeErrorPatterns.some(pattern => pattern.test(error.message));
|
||||
|
||||
if (isSafeError) {
|
||||
clientMessage = error.message;
|
||||
|
||||
@@ -0,0 +1,40 @@
|
||||
import { describe, expect, test } from "bun:test";
|
||||
import { countLatestStatuses, latestStatusBySubject } from "./activity-status";
|
||||
|
||||
const at = (iso: string) => new Date(iso);
|
||||
|
||||
describe("Activity Log status chips (#454)", () => {
|
||||
test("a start and finish in the same second count as finished, in either order", () => {
|
||||
const start = { repositoryId: "r1", status: "syncing", timestamp: at("2026-09-26T00:29:48.000Z") };
|
||||
const finish = { repositoryId: "r1", status: "synced", timestamp: at("2026-09-26T00:29:48.000Z") };
|
||||
const aggregate = { status: "mirrored", timestamp: at("2026-09-26T00:29:48.000Z") };
|
||||
|
||||
// The order the reporter's export came back in.
|
||||
expect(countLatestStatuses([aggregate, start, finish])).toEqual(new Map([["synced", 1]]));
|
||||
expect(countLatestStatuses([finish, start])).toEqual(new Map([["synced", 1]]));
|
||||
});
|
||||
|
||||
test("a sync that is really still running stays syncing", () => {
|
||||
const earlier = { repositoryId: "r1", status: "synced", timestamp: at("2026-09-26T00:00:00.000Z") };
|
||||
const running = { repositoryId: "r1", status: "syncing", timestamp: at("2026-09-26T01:00:00.000Z") };
|
||||
expect(latestStatusBySubject([running, earlier]).get("r1")).toBe("syncing");
|
||||
});
|
||||
|
||||
test("the newest event decides otherwise, and events without a subject are ignored", () => {
|
||||
const counts = countLatestStatuses([
|
||||
{ repositoryId: "r1", status: "synced", timestamp: "2026-09-26T02:00:00Z" },
|
||||
{ repositoryId: "r1", status: "failed", timestamp: "2026-09-26T01:00:00Z" },
|
||||
{ organizationId: "o1", status: "failed", timestamp: "2026-09-26T01:00:00Z" },
|
||||
{ status: "mirrored", timestamp: "2026-09-26T01:00:00Z" },
|
||||
]);
|
||||
expect(counts).toEqual(new Map([["synced", 1], ["failed", 1]]));
|
||||
});
|
||||
|
||||
test("a newer start never loses to an older finish from another second", () => {
|
||||
const counts = countLatestStatuses([
|
||||
{ repositoryName: "a", status: "mirroring", timestamp: "2026-09-26T00:00:02Z" },
|
||||
{ repositoryName: "a", status: "mirrored", timestamp: "2026-09-26T00:00:01Z" },
|
||||
]);
|
||||
expect(counts).toEqual(new Map([["mirroring", 1]]));
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,62 @@
|
||||
/**
|
||||
* Current state per repository or organization for the Activity Log
|
||||
* summary chips. Activities arrive newest first, so the first event seen
|
||||
* for a subject is its state.
|
||||
*
|
||||
* Timestamps are stored to the second, and a quick sync writes its
|
||||
* "syncing" and "synced" events in the same second (#454). When the newest
|
||||
* event for a subject says work started, an event from the same second
|
||||
* that says it finished wins, whatever order the tie came back in.
|
||||
*/
|
||||
|
||||
const IN_PROGRESS_STATUSES = new Set(["syncing", "mirroring", "deleting"]);
|
||||
|
||||
interface ActivityLike {
|
||||
status: string;
|
||||
timestamp: Date | string | number;
|
||||
repositoryId?: string | null;
|
||||
organizationId?: string | null;
|
||||
repositoryName?: string | null;
|
||||
organizationName?: string | null;
|
||||
}
|
||||
|
||||
function toSecond(timestamp: ActivityLike["timestamp"]): number {
|
||||
return Math.floor(new Date(timestamp).getTime() / 1000);
|
||||
}
|
||||
|
||||
export function latestStatusBySubject(activities: ActivityLike[]): Map<string, string> {
|
||||
const latest = new Map<string, { status: string; second: number }>();
|
||||
for (const activity of activities) {
|
||||
const subject =
|
||||
activity.repositoryId ||
|
||||
activity.organizationId ||
|
||||
activity.repositoryName ||
|
||||
activity.organizationName;
|
||||
// Events with no subject have no state of their own to report.
|
||||
if (!subject) continue;
|
||||
|
||||
const second = toSecond(activity.timestamp);
|
||||
const seen = latest.get(subject);
|
||||
if (!seen) {
|
||||
latest.set(subject, { status: activity.status, second });
|
||||
continue;
|
||||
}
|
||||
if (
|
||||
IN_PROGRESS_STATUSES.has(seen.status) &&
|
||||
!IN_PROGRESS_STATUSES.has(activity.status) &&
|
||||
seen.second === second
|
||||
) {
|
||||
latest.set(subject, { status: activity.status, second });
|
||||
}
|
||||
}
|
||||
|
||||
return new Map([...latest].map(([subject, { status }]) => [subject, status]));
|
||||
}
|
||||
|
||||
export function countLatestStatuses(activities: ActivityLike[]): Map<string, number> {
|
||||
const counts = new Map<string, number>();
|
||||
latestStatusBySubject(activities).forEach((status) => {
|
||||
counts.set(status, (counts.get(status) ?? 0) + 1);
|
||||
});
|
||||
return counts;
|
||||
}
|
||||
@@ -0,0 +1,176 @@
|
||||
import { describe, test, expect, beforeAll, beforeEach, afterAll } from "bun:test";
|
||||
import {
|
||||
LAST_AUTH_METHOD_STORAGE_KEY,
|
||||
isAuthMethod,
|
||||
parseDefaultAuthMethod,
|
||||
resolveInitialAuthMethod,
|
||||
getLastAuthMethod,
|
||||
setLastAuthMethod,
|
||||
} from "./auth-method";
|
||||
|
||||
const BOTH = { email: true, sso: true };
|
||||
const EMAIL_ONLY = { email: true, sso: false };
|
||||
const SSO_ONLY = { email: false, sso: true };
|
||||
|
||||
describe("parseDefaultAuthMethod", () => {
|
||||
test("accepts the two known values", () => {
|
||||
expect(parseDefaultAuthMethod("email")).toBe("email");
|
||||
expect(parseDefaultAuthMethod("sso")).toBe("sso");
|
||||
});
|
||||
|
||||
test("ignores case and surrounding whitespace", () => {
|
||||
expect(parseDefaultAuthMethod(" SSO ")).toBe("sso");
|
||||
expect(parseDefaultAuthMethod("Email")).toBe("email");
|
||||
});
|
||||
|
||||
test("falls back to email for anything else", () => {
|
||||
expect(parseDefaultAuthMethod(undefined)).toBe("email");
|
||||
expect(parseDefaultAuthMethod(null)).toBe("email");
|
||||
expect(parseDefaultAuthMethod("")).toBe("email");
|
||||
expect(parseDefaultAuthMethod("oidc")).toBe("email");
|
||||
expect(parseDefaultAuthMethod(42)).toBe("email");
|
||||
});
|
||||
});
|
||||
|
||||
describe("isAuthMethod", () => {
|
||||
test("recognises only the two methods", () => {
|
||||
expect(isAuthMethod("email")).toBe(true);
|
||||
expect(isAuthMethod("sso")).toBe(true);
|
||||
expect(isAuthMethod("saml")).toBe(false);
|
||||
expect(isAuthMethod(null)).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
describe("resolveInitialAuthMethod", () => {
|
||||
test("uses the remembered method when it is available", () => {
|
||||
expect(
|
||||
resolveInitialAuthMethod({
|
||||
remembered: "sso",
|
||||
serverDefault: "email",
|
||||
available: BOTH,
|
||||
})
|
||||
).toBe("sso");
|
||||
expect(
|
||||
resolveInitialAuthMethod({
|
||||
remembered: "email",
|
||||
serverDefault: "sso",
|
||||
available: BOTH,
|
||||
})
|
||||
).toBe("email");
|
||||
});
|
||||
|
||||
test("ignores a remembered method that is no longer available", () => {
|
||||
expect(
|
||||
resolveInitialAuthMethod({
|
||||
remembered: "sso",
|
||||
serverDefault: "email",
|
||||
available: EMAIL_ONLY,
|
||||
})
|
||||
).toBe("email");
|
||||
});
|
||||
|
||||
test("uses the server default when nothing is remembered", () => {
|
||||
expect(
|
||||
resolveInitialAuthMethod({ serverDefault: "sso", available: BOTH })
|
||||
).toBe("sso");
|
||||
expect(
|
||||
resolveInitialAuthMethod({
|
||||
remembered: null,
|
||||
serverDefault: "sso",
|
||||
available: BOTH,
|
||||
})
|
||||
).toBe("sso");
|
||||
});
|
||||
|
||||
test("falls back to email when the server default is unavailable", () => {
|
||||
expect(
|
||||
resolveInitialAuthMethod({ serverDefault: "sso", available: EMAIL_ONLY })
|
||||
).toBe("email");
|
||||
});
|
||||
|
||||
test("falls back to sso when email is unavailable", () => {
|
||||
expect(
|
||||
resolveInitialAuthMethod({ serverDefault: "email", available: SSO_ONLY })
|
||||
).toBe("sso");
|
||||
expect(
|
||||
resolveInitialAuthMethod({
|
||||
remembered: "email",
|
||||
serverDefault: "email",
|
||||
available: SSO_ONLY,
|
||||
})
|
||||
).toBe("sso");
|
||||
});
|
||||
|
||||
test("defaults to email when no server default is given", () => {
|
||||
expect(resolveInitialAuthMethod({ available: BOTH })).toBe("email");
|
||||
});
|
||||
});
|
||||
|
||||
// The Bun test runtime has no DOM. Install a minimal localStorage shim for the
|
||||
// storage tests and remove it afterwards.
|
||||
const g = globalThis as any;
|
||||
const createdLocalStorage = typeof g.localStorage === "undefined";
|
||||
|
||||
beforeAll(() => {
|
||||
if (createdLocalStorage) {
|
||||
const store = new Map<string, string>();
|
||||
g.localStorage = {
|
||||
getItem: (key: string) => (store.has(key) ? store.get(key)! : null),
|
||||
setItem: (key: string, value: string) => {
|
||||
store.set(key, String(value));
|
||||
},
|
||||
removeItem: (key: string) => {
|
||||
store.delete(key);
|
||||
},
|
||||
clear: () => {
|
||||
store.clear();
|
||||
},
|
||||
};
|
||||
}
|
||||
});
|
||||
|
||||
afterAll(() => {
|
||||
if (createdLocalStorage) {
|
||||
delete g.localStorage;
|
||||
}
|
||||
});
|
||||
|
||||
describe("last auth method storage", () => {
|
||||
beforeEach(() => {
|
||||
g.localStorage.clear();
|
||||
});
|
||||
|
||||
test("round trips a stored method", () => {
|
||||
setLastAuthMethod("sso");
|
||||
expect(getLastAuthMethod()).toBe("sso");
|
||||
setLastAuthMethod("email");
|
||||
expect(getLastAuthMethod()).toBe("email");
|
||||
});
|
||||
|
||||
test("returns null when nothing is stored", () => {
|
||||
expect(getLastAuthMethod()).toBeNull();
|
||||
});
|
||||
|
||||
test("ignores a bogus stored value", () => {
|
||||
g.localStorage.setItem(LAST_AUTH_METHOD_STORAGE_KEY, "carrier-pigeon");
|
||||
expect(getLastAuthMethod()).toBeNull();
|
||||
});
|
||||
|
||||
test("swallows storage errors", () => {
|
||||
const original = g.localStorage;
|
||||
g.localStorage = {
|
||||
getItem: () => {
|
||||
throw new Error("storage disabled");
|
||||
},
|
||||
setItem: () => {
|
||||
throw new Error("storage disabled");
|
||||
},
|
||||
};
|
||||
try {
|
||||
expect(getLastAuthMethod()).toBeNull();
|
||||
expect(() => setLastAuthMethod("sso")).not.toThrow();
|
||||
} finally {
|
||||
g.localStorage = original;
|
||||
}
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,85 @@
|
||||
/**
|
||||
* Login method selection: which tab the login page opens on.
|
||||
*
|
||||
* Three inputs decide it:
|
||||
* - The method this browser last signed in with (localStorage, per browser).
|
||||
* - The server default from the AUTH_DEFAULT_METHOD environment variable,
|
||||
* served by GET /api/auth/methods.
|
||||
* - Which methods the instance actually offers right now.
|
||||
*
|
||||
* The parsing and resolution helpers are pure so they can run on the server
|
||||
* (the API route) and in tests; only the storage helpers touch the browser.
|
||||
*/
|
||||
|
||||
export type AuthMethod = "email" | "sso";
|
||||
|
||||
/** Used whenever nothing else applies, and for an unrecognised env value. */
|
||||
export const DEFAULT_AUTH_METHOD: AuthMethod = "email";
|
||||
|
||||
export const LAST_AUTH_METHOD_STORAGE_KEY = "lastAuthMethod";
|
||||
|
||||
export function isAuthMethod(value: unknown): value is AuthMethod {
|
||||
return value === "email" || value === "sso";
|
||||
}
|
||||
|
||||
/**
|
||||
* Read AUTH_DEFAULT_METHOD (or any other source). Anything that is not
|
||||
* "email" or "sso", case and whitespace insensitive, falls back to "email".
|
||||
*/
|
||||
export function parseDefaultAuthMethod(value: unknown): AuthMethod {
|
||||
if (typeof value !== "string") return DEFAULT_AUTH_METHOD;
|
||||
const normalized = value.trim().toLowerCase();
|
||||
return isAuthMethod(normalized) ? normalized : DEFAULT_AUTH_METHOD;
|
||||
}
|
||||
|
||||
export interface AvailableAuthMethods {
|
||||
email: boolean;
|
||||
sso: boolean;
|
||||
}
|
||||
|
||||
export interface ResolveInitialAuthMethodInput {
|
||||
/** What this browser used last, or null when nothing is stored. */
|
||||
remembered?: AuthMethod | null;
|
||||
/** The instance default from the server. */
|
||||
serverDefault?: AuthMethod;
|
||||
available: AvailableAuthMethods;
|
||||
}
|
||||
|
||||
/**
|
||||
* Pick the method the login page opens on: the remembered one if it is still
|
||||
* available, else the server default if it is available, else email, else SSO.
|
||||
*/
|
||||
export function resolveInitialAuthMethod({
|
||||
remembered,
|
||||
serverDefault = DEFAULT_AUTH_METHOD,
|
||||
available,
|
||||
}: ResolveInitialAuthMethodInput): AuthMethod {
|
||||
if (remembered && available[remembered]) return remembered;
|
||||
if (available[serverDefault]) return serverDefault;
|
||||
if (available.email) return "email";
|
||||
return "sso";
|
||||
}
|
||||
|
||||
/**
|
||||
* Read the method this browser last signed in with. Safe during SSR and when
|
||||
* storage is unavailable (private window, blocked site data): returns null.
|
||||
*/
|
||||
export function getLastAuthMethod(): AuthMethod | null {
|
||||
if (typeof localStorage === "undefined") return null;
|
||||
try {
|
||||
const stored = localStorage.getItem(LAST_AUTH_METHOD_STORAGE_KEY);
|
||||
return isAuthMethod(stored) ? stored : null;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/** Remember the method that was just used. Best effort, never throws. */
|
||||
export function setLastAuthMethod(method: AuthMethod): void {
|
||||
if (typeof localStorage === "undefined") return;
|
||||
try {
|
||||
localStorage.setItem(LAST_AUTH_METHOD_STORAGE_KEY, method);
|
||||
} catch {
|
||||
// Storage disabled or full; the login page just falls back to the default.
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,91 @@
|
||||
/**
|
||||
* processWithRetry and a rate limited source (issue #437, second report).
|
||||
* A refusal used to be retried three times with backoff like any other
|
||||
* error, and the items behind it were started anyway, so one exhausted
|
||||
* budget turned into thousands of 403 responses.
|
||||
*/
|
||||
|
||||
import { describe, expect, test } from "bun:test";
|
||||
import { RateLimitedBatchError, processWithRetry } from "./concurrency";
|
||||
import { isRateLimitError } from "@/lib/rate-limit-gate";
|
||||
|
||||
function rateLimitRefusal() {
|
||||
const error = new Error("API rate limit exceeded for user ID 1.") as Error & { status: number };
|
||||
error.status = 403;
|
||||
return error;
|
||||
}
|
||||
|
||||
describe("processWithRetry under a rate limit", () => {
|
||||
test("a rate limit refusal is not retried and the remaining items are not started", async () => {
|
||||
const attempts: number[] = [];
|
||||
let caught: unknown;
|
||||
try {
|
||||
await processWithRetry(
|
||||
[1, 2, 3, 4, 5, 6],
|
||||
async (n) => {
|
||||
attempts.push(n);
|
||||
if (n === 2) throw rateLimitRefusal();
|
||||
return n * 10;
|
||||
},
|
||||
{ concurrencyLimit: 2, maxRetries: 3, retryDelay: 1 }
|
||||
);
|
||||
} catch (error) {
|
||||
caught = error;
|
||||
}
|
||||
|
||||
expect(caught).toBeInstanceOf(RateLimitedBatchError);
|
||||
const failure = caught as RateLimitedBatchError<number>;
|
||||
// Item 2 was attempted exactly once; items 3 to 6 never started. The two
|
||||
// items of the first batch run concurrently, so their order is not fixed.
|
||||
expect([...attempts].sort()).toEqual([1, 2]);
|
||||
expect(failure.results).toEqual([10]);
|
||||
expect(failure.message).toContain("5 item(s) left");
|
||||
expect(failure.status).toBe(403);
|
||||
// Callers up the stack classify it like the refusal that caused it.
|
||||
expect(isRateLimitError(failure)).toBe(true);
|
||||
expect(isRateLimitError(failure.cause)).toBe(true);
|
||||
});
|
||||
|
||||
test("a 429 from a paginated call counts as a rate limit too", async () => {
|
||||
const attempts: number[] = [];
|
||||
const tooMany = Object.assign(new Error("Too many requests"), { status: 429 });
|
||||
await expect(
|
||||
processWithRetry(
|
||||
[1, 2, 3],
|
||||
async (n) => {
|
||||
attempts.push(n);
|
||||
if (n === 1) throw tooMany;
|
||||
return n;
|
||||
},
|
||||
{ concurrencyLimit: 1, maxRetries: 3, retryDelay: 1 }
|
||||
)
|
||||
).rejects.toBeInstanceOf(RateLimitedBatchError);
|
||||
expect(attempts).toEqual([1]);
|
||||
});
|
||||
|
||||
test("other errors keep their retries and do not stop the batch", async () => {
|
||||
const attempts: number[] = [];
|
||||
const results = await processWithRetry(
|
||||
[1, 2, 3],
|
||||
async (n) => {
|
||||
attempts.push(n);
|
||||
if (n === 2 && attempts.filter((a) => a === 2).length < 3) {
|
||||
throw new Error("Gitea hiccup");
|
||||
}
|
||||
return n;
|
||||
},
|
||||
{ concurrencyLimit: 1, maxRetries: 3, retryDelay: 1 }
|
||||
);
|
||||
expect(results).toEqual([1, 2, 3]);
|
||||
expect(attempts).toEqual([1, 2, 2, 2, 3]);
|
||||
});
|
||||
|
||||
test("a batch that finishes without a refusal returns every result as before", async () => {
|
||||
const results = await processWithRetry([1, 2, 3], async (n) => n + 1, {
|
||||
concurrencyLimit: 3,
|
||||
maxRetries: 0,
|
||||
retryDelay: 1,
|
||||
});
|
||||
expect(results).toEqual([2, 3, 4]);
|
||||
});
|
||||
});
|
||||
@@ -165,3 +165,46 @@ describe("processWithRetry", () => {
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
describe("processWithRetry checkpointing", () => {
|
||||
test("records every completed item and publishes on the interval", async () => {
|
||||
const items = ["a", "b", "c", "d", "e"];
|
||||
const calls: Array<[string, string, boolean]> = [];
|
||||
|
||||
await processWithRetry(items, async (item) => item.toUpperCase(), {
|
||||
concurrencyLimit: 1,
|
||||
jobId: "job-1",
|
||||
getItemId: (item) => item,
|
||||
checkpointInterval: 2,
|
||||
onCheckpoint: async (jobId, itemId, publish) => {
|
||||
calls.push([jobId, itemId, publish]);
|
||||
},
|
||||
});
|
||||
|
||||
// One checkpoint per item, in completion order, and no "final" placeholder.
|
||||
expect(calls.map(([, itemId]) => itemId)).toEqual(items);
|
||||
expect(calls.every(([jobId]) => jobId === "job-1")).toBe(true);
|
||||
// Progress events only every second item.
|
||||
expect(calls.map(([, , publish]) => publish)).toEqual([false, true, false, true, false]);
|
||||
});
|
||||
|
||||
test("does not checkpoint an item that failed for good", async () => {
|
||||
const calls: string[] = [];
|
||||
|
||||
// processInParallel logs a failed item and carries on, so the call resolves.
|
||||
await processWithRetry(["ok", "bad"], async (item) => {
|
||||
if (item === "bad") throw new Error("nope");
|
||||
return item;
|
||||
}, {
|
||||
concurrencyLimit: 1,
|
||||
maxRetries: 0,
|
||||
jobId: "job-2",
|
||||
getItemId: (item) => item,
|
||||
onCheckpoint: async (_jobId, itemId) => {
|
||||
calls.push(itemId);
|
||||
},
|
||||
});
|
||||
|
||||
expect(calls).toEqual(["ok"]);
|
||||
});
|
||||
});
|
||||
@@ -1,4 +1,28 @@
|
||||
import { JOB_HEARTBEAT_INTERVAL_MS } from "@/lib/interrupted-job-detection";
|
||||
import { isRateLimitError } from "@/lib/rate-limit-gate";
|
||||
|
||||
/**
|
||||
* Thrown by processWithRetry when a batch stopped because the source is rate
|
||||
* limited. Carries the results gathered so far so a caller can still count
|
||||
* what was done. Shaped like a GitHub refusal so isRateLimitError matches it.
|
||||
*/
|
||||
export class RateLimitedBatchError<R = unknown> extends Error {
|
||||
readonly status = 403;
|
||||
declare readonly results: R[];
|
||||
declare readonly cause: unknown;
|
||||
|
||||
constructor(cause: unknown, results: R[], remaining: number) {
|
||||
super(
|
||||
`Rate limit hit while processing items; stopped with ${remaining} item(s) left ` +
|
||||
`(${cause instanceof Error ? cause.message : String(cause)})`
|
||||
);
|
||||
this.name = "RateLimitedBatchError";
|
||||
// Not enumerable, so logging the error does not print every partial
|
||||
// result and the whole source response it carries.
|
||||
Object.defineProperty(this, "cause", { value: cause, enumerable: false });
|
||||
Object.defineProperty(this, "results", { value: results, enumerable: false });
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Utility for processing items in parallel with concurrency control
|
||||
@@ -13,7 +37,12 @@ export async function processInParallel<T, R>(
|
||||
items: T[],
|
||||
processItem: (item: T) => Promise<R>,
|
||||
concurrencyLimit: number = 5, // Safe default for GitHub API (max 100 concurrent, but 5-10 recommended)
|
||||
onProgress?: (completed: number, total: number, result?: R) => void
|
||||
onProgress?: (completed: number, total: number, result?: R) => void,
|
||||
/**
|
||||
* Asked before each batch. Returning true ends the loop with the results
|
||||
* gathered so far; the remaining items are not started.
|
||||
*/
|
||||
shouldStop?: () => boolean
|
||||
): Promise<R[]> {
|
||||
const results: R[] = [];
|
||||
let completed = 0;
|
||||
@@ -21,6 +50,11 @@ export async function processInParallel<T, R>(
|
||||
|
||||
// Process items in batches to control concurrency
|
||||
for (let i = 0; i < total; i += concurrencyLimit) {
|
||||
if (shouldStop?.()) {
|
||||
console.warn(`[Batch] Stopping before item ${i} of ${total}: source is rate limited`);
|
||||
break;
|
||||
}
|
||||
|
||||
const batch = items.slice(i, i + concurrencyLimit);
|
||||
|
||||
const batchPromises = batch.map(async (item) => {
|
||||
@@ -93,8 +127,15 @@ export async function processWithRetry<T, R>(
|
||||
onRetry?: (item: T, error: Error, attempt: number) => void;
|
||||
jobId?: string; // Optional job ID for checkpointing
|
||||
getItemId?: (item: T) => string; // Function to get a unique ID for each item
|
||||
onCheckpoint?: (jobId: string, completedItemId: string) => Promise<void>; // Callback for checkpointing
|
||||
checkpointInterval?: number; // How many items to process before checkpointing
|
||||
/**
|
||||
* Called once for EVERY completed item, so a resume knows exactly what is
|
||||
* done. `publish` is true every `checkpointInterval` items: the callback
|
||||
* uses it to throttle progress events, not the checkpoint itself. It used
|
||||
* to be called only for the item that tripped the interval, and a resume
|
||||
* re-processed everything finished since the previous checkpoint.
|
||||
*/
|
||||
onCheckpoint?: (jobId: string, completedItemId: string, publish: boolean) => Promise<void>;
|
||||
checkpointInterval?: number; // How many completed items between published progress events
|
||||
} = {}
|
||||
): Promise<R[]> {
|
||||
const {
|
||||
@@ -112,6 +153,11 @@ export async function processWithRetry<T, R>(
|
||||
// Track checkpoint counter
|
||||
let itemsProcessedSinceLastCheckpoint = 0;
|
||||
|
||||
// The first rate limit refusal ends the run: retrying it, or starting the
|
||||
// items behind it, would only send more requests into the same wall
|
||||
// (issue #437). The caller gets a RateLimitedBatchError with what finished.
|
||||
let rateLimitError: unknown = null;
|
||||
|
||||
// Wrap the process function with retry logic
|
||||
const processWithRetryLogic = async (item: T): Promise<R> => {
|
||||
let lastError: Error | null = null;
|
||||
@@ -130,22 +176,32 @@ export async function processWithRetry<T, R>(
|
||||
|
||||
const result = await processItem(item);
|
||||
|
||||
// Handle checkpointing if enabled
|
||||
// Record every completed item; publish progress on the interval.
|
||||
if (jobId && getItemId && onCheckpoint) {
|
||||
const itemId = getItemId(item);
|
||||
itemsProcessedSinceLastCheckpoint++;
|
||||
|
||||
// Checkpoint based on the interval
|
||||
if (itemsProcessedSinceLastCheckpoint >= checkpointInterval) {
|
||||
await onCheckpoint(jobId, itemId);
|
||||
const publish = itemsProcessedSinceLastCheckpoint >= checkpointInterval;
|
||||
if (publish) {
|
||||
itemsProcessedSinceLastCheckpoint = 0;
|
||||
}
|
||||
await onCheckpoint(jobId, itemId, publish);
|
||||
}
|
||||
|
||||
return result;
|
||||
} catch (error) {
|
||||
lastError = error instanceof Error ? error : new Error(String(error));
|
||||
|
||||
if (isRateLimitError(error)) {
|
||||
if (!rateLimitError) {
|
||||
rateLimitError = error;
|
||||
console.warn(
|
||||
`[Batch] Rate limit hit on ${getItemId ? getItemId(item) : "an item"}; not retrying and not starting the remaining items`
|
||||
);
|
||||
}
|
||||
throw lastError;
|
||||
}
|
||||
|
||||
if (attempt <= maxRetries) {
|
||||
if (onRetry) {
|
||||
onRetry(item, lastError, attempt);
|
||||
@@ -183,13 +239,12 @@ export async function processWithRetry<T, R>(
|
||||
items,
|
||||
processWithRetryLogic,
|
||||
concurrencyLimit,
|
||||
onProgress
|
||||
onProgress,
|
||||
() => rateLimitError !== null
|
||||
);
|
||||
|
||||
// Final checkpoint if there are remaining items since the last checkpoint
|
||||
if (jobId && getItemId && onCheckpoint && itemsProcessedSinceLastCheckpoint > 0) {
|
||||
// We don't have a specific item ID for the final checkpoint, so we'll use a placeholder
|
||||
await onCheckpoint(jobId, 'final');
|
||||
if (rateLimitError) {
|
||||
throw new RateLimitedBatchError(rateLimitError, results, items.length - results.length);
|
||||
}
|
||||
|
||||
return results;
|
||||
@@ -309,8 +364,10 @@ export async function processWithResilience<T, R>(
|
||||
});
|
||||
}, JOB_HEARTBEAT_INTERVAL_MS);
|
||||
|
||||
// Define the checkpoint function
|
||||
const onCheckpoint = async (jobId: string, completedItemId: string) => {
|
||||
// Define the checkpoint function. Every completed item is written to the
|
||||
// job row so a resume skips exactly what is done; only every
|
||||
// checkpointInterval-th write also publishes a progress event.
|
||||
const onCheckpoint = async (jobId: string, completedItemId: string, publish: boolean = true) => {
|
||||
const itemName = items.find(item => getItemId(item) === completedItemId)
|
||||
? getItemName(items.find(item => getItemId(item) === completedItemId)!)
|
||||
: 'unknown';
|
||||
@@ -319,6 +376,7 @@ export async function processWithResilience<T, R>(
|
||||
jobId,
|
||||
completedItemId,
|
||||
message: `Processed item: ${itemName}`,
|
||||
silent: !publish,
|
||||
});
|
||||
};
|
||||
|
||||
|
||||
@@ -47,11 +47,17 @@ export async function createDefaultConfig({ userId, envOverrides = {} }: Default
|
||||
const giteaToken = envOverrides.giteaToken || process.env.GITEA_TOKEN || "";
|
||||
const giteaUsername = envOverrides.giteaUsername || process.env.GITEA_USERNAME || "";
|
||||
|
||||
// Schedule config from env - default to ENABLED
|
||||
const scheduleEnabled = envOverrides.scheduleEnabled ??
|
||||
(process.env.SCHEDULE_ENABLED === "false" ? false : true); // Default: ENABLED
|
||||
// Schedule config from env. Off unless the environment turns it on, with
|
||||
// the same rule as env-config-loader: SCHEDULE_ENABLED=true or any interval
|
||||
// variable. A fresh install used to start enabled on a plain 24h interval
|
||||
// counted from first login, which the settings card (clock mode only)
|
||||
// showed as "22:00 daily" (#427). Enabling from the card stores the clock
|
||||
// schedule and the browser timezone, so the card and the scheduler agree.
|
||||
const envInterval = process.env.SCHEDULE_INTERVAL || process.env.GITEA_MIRROR_INTERVAL;
|
||||
const scheduleEnabled = envOverrides.scheduleEnabled ??
|
||||
(process.env.SCHEDULE_ENABLED === "true" || !!envInterval);
|
||||
const scheduleInterval = envOverrides.scheduleInterval ??
|
||||
(process.env.SCHEDULE_INTERVAL || 86400); // Default: daily
|
||||
(envInterval || "0 22 * * *"); // Default: daily at 22:00 in scheduleTimezone
|
||||
const scheduleTimezone = normalizeTimezone(process.env.SCHEDULE_TIMEZONE || "UTC");
|
||||
let scheduleNextRun: Date | null = null;
|
||||
if (scheduleEnabled) {
|
||||
|
||||
@@ -82,6 +82,23 @@ test("mapDbScheduleToUi - infers clock mode for generated cron", () => {
|
||||
expect(mapped.timezone).toBe("Asia/Kolkata");
|
||||
});
|
||||
|
||||
test("mapDbScheduleToUi - reports interval mode for a plain interval instead of a clock schedule", () => {
|
||||
// What SCHEDULE_INTERVAL / GITEA_MIRROR_INTERVAL and older versions store.
|
||||
// The settings card must not present the 22:00 placeholder as saved (#427).
|
||||
const mapped = mapDbScheduleToUi(
|
||||
scheduleConfigSchema.parse({
|
||||
enabled: true,
|
||||
interval: "8h",
|
||||
timezone: "UTC",
|
||||
})
|
||||
);
|
||||
|
||||
expect(mapped.scheduleMode).toBe("interval");
|
||||
expect(mapped.interval).toBe(8 * 3600);
|
||||
expect(mapped.intervalExpression).toBe("8h");
|
||||
expect(mapped.timezone).toBe("UTC");
|
||||
});
|
||||
|
||||
test("includeCollaboratorRepos round-trips through UI -> DB -> UI when true", () => {
|
||||
const ui = buildMinimalUiConfigs({ includeCollaboratorRepos: true });
|
||||
const db = mapUiToDbConfig(
|
||||
|
||||
@@ -0,0 +1,176 @@
|
||||
import { describe, test, expect } from "bun:test";
|
||||
import { escapeCsvValue, formatCsvValue, toCsv } from "./csv";
|
||||
|
||||
describe("formatCsvValue", () => {
|
||||
test("empty for null and undefined", () => {
|
||||
expect(formatCsvValue(null)).toBe("");
|
||||
expect(formatCsvValue(undefined)).toBe("");
|
||||
});
|
||||
|
||||
test("dates become ISO strings", () => {
|
||||
expect(formatCsvValue(new Date("2026-09-22T08:30:00.000Z"))).toBe(
|
||||
"2026-09-22T08:30:00.000Z"
|
||||
);
|
||||
});
|
||||
|
||||
test("an invalid date is empty rather than 'Invalid Date'", () => {
|
||||
expect(formatCsvValue(new Date("nope"))).toBe("");
|
||||
});
|
||||
|
||||
test("booleans become true and false", () => {
|
||||
expect(formatCsvValue(true)).toBe("true");
|
||||
expect(formatCsvValue(false)).toBe("false");
|
||||
});
|
||||
|
||||
test("numbers keep their text form, including zero", () => {
|
||||
expect(formatCsvValue(0)).toBe("0");
|
||||
expect(formatCsvValue(42)).toBe("42");
|
||||
});
|
||||
|
||||
test("objects are stringified as JSON", () => {
|
||||
expect(formatCsvValue({ a: 1 })).toBe('{"a":1}');
|
||||
});
|
||||
});
|
||||
|
||||
describe("escapeCsvValue", () => {
|
||||
test("leaves a plain value alone", () => {
|
||||
expect(escapeCsvValue("hello-world")).toBe("hello-world");
|
||||
});
|
||||
|
||||
test("quotes a value with a comma", () => {
|
||||
expect(escapeCsvValue("one, two")).toBe('"one, two"');
|
||||
});
|
||||
|
||||
test("quotes a value with a quote and doubles the quote", () => {
|
||||
expect(escapeCsvValue('say "hi"')).toBe('"say ""hi"""');
|
||||
});
|
||||
|
||||
test("a value that is only quotes doubles every one of them", () => {
|
||||
expect(escapeCsvValue('"""')).toBe('""""""""');
|
||||
});
|
||||
|
||||
test("quotes a value with a line feed", () => {
|
||||
expect(escapeCsvValue("first\nsecond")).toBe('"first\nsecond"');
|
||||
});
|
||||
|
||||
test("quotes a value with a carriage return", () => {
|
||||
expect(escapeCsvValue("first\r\nsecond")).toBe('"first\r\nsecond"');
|
||||
});
|
||||
|
||||
test("keeps leading and trailing spaces without quoting", () => {
|
||||
expect(escapeCsvValue(" padded ")).toBe(" padded ");
|
||||
});
|
||||
});
|
||||
|
||||
describe("toCsv", () => {
|
||||
type Row = {
|
||||
name: string;
|
||||
description: string | null;
|
||||
isPrivate: boolean;
|
||||
size: number;
|
||||
lastMirrored: Date | null;
|
||||
};
|
||||
|
||||
const columns = [
|
||||
"name",
|
||||
"description",
|
||||
"isPrivate",
|
||||
"size",
|
||||
"lastMirrored",
|
||||
] as const;
|
||||
|
||||
test("writes a header row from the column keys", () => {
|
||||
const csv = toCsv<Row>([], columns);
|
||||
expect(csv).toBe("name,description,isPrivate,size,lastMirrored\r\n");
|
||||
});
|
||||
|
||||
test("writes one CRLF terminated line per row", () => {
|
||||
const csv = toCsv<Row>(
|
||||
[
|
||||
{
|
||||
name: "hello-world",
|
||||
description: "A greeting",
|
||||
isPrivate: false,
|
||||
size: 12,
|
||||
lastMirrored: new Date("2026-09-22T08:30:00.000Z"),
|
||||
},
|
||||
{
|
||||
name: "second",
|
||||
description: null,
|
||||
isPrivate: true,
|
||||
size: 0,
|
||||
lastMirrored: null,
|
||||
},
|
||||
],
|
||||
columns
|
||||
);
|
||||
|
||||
expect(csv).toBe(
|
||||
"name,description,isPrivate,size,lastMirrored\r\n" +
|
||||
"hello-world,A greeting,false,12,2026-09-22T08:30:00.000Z\r\n" +
|
||||
"second,,true,0,\r\n"
|
||||
);
|
||||
});
|
||||
|
||||
test("escapes values that hold commas, quotes and newlines", () => {
|
||||
const csv = toCsv<Row>(
|
||||
[
|
||||
{
|
||||
name: "tricky",
|
||||
description: 'Uses "quotes", commas\nand a newline',
|
||||
isPrivate: false,
|
||||
size: 1,
|
||||
lastMirrored: null,
|
||||
},
|
||||
],
|
||||
columns
|
||||
);
|
||||
|
||||
expect(csv.split("\r\n")[1]).toBe(
|
||||
'tricky,"Uses ""quotes"", commas\nand a newline",false,1,'
|
||||
);
|
||||
});
|
||||
|
||||
test("escapes a header that needs quoting too", () => {
|
||||
const csv = toCsv<{ "size, bytes": number }>([{ "size, bytes": 3 }], [
|
||||
"size, bytes",
|
||||
]);
|
||||
expect(csv).toBe('"size, bytes"\r\n3\r\n');
|
||||
});
|
||||
|
||||
test("a missing key on a row is an empty field", () => {
|
||||
const rows = [{ name: "partial" }] as unknown as Row[];
|
||||
const csv = toCsv<Row>(rows, columns);
|
||||
expect(csv.split("\r\n")[1]).toBe("partial,,,,");
|
||||
});
|
||||
|
||||
test("only the listed columns are written, in the listed order", () => {
|
||||
const rows = [
|
||||
{ name: "a", secret: "token", description: "b" },
|
||||
] as unknown as Row[];
|
||||
const csv = toCsv<Row>(rows, ["description", "name"]);
|
||||
expect(csv).toBe("description,name\r\nb,a\r\n");
|
||||
expect(csv).not.toContain("token");
|
||||
});
|
||||
});
|
||||
|
||||
describe("formula injection guard", () => {
|
||||
test("prefixes strings a spreadsheet would run as a formula", () => {
|
||||
expect(formatCsvValue("=1+1")).toBe("'=1+1");
|
||||
expect(formatCsvValue("+SUM(A1)")).toBe("'+SUM(A1)");
|
||||
expect(formatCsvValue("-2+3")).toBe("'-2+3");
|
||||
expect(formatCsvValue("@cmd")).toBe("'@cmd");
|
||||
expect(formatCsvValue("\t=1")).toBe("'\t=1");
|
||||
});
|
||||
|
||||
test("leaves numbers, booleans and ordinary text alone", () => {
|
||||
expect(formatCsvValue(-5)).toBe("-5");
|
||||
expect(formatCsvValue(true)).toBe("true");
|
||||
expect(formatCsvValue("a - b")).toBe("a - b");
|
||||
expect(formatCsvValue("main")).toBe("main");
|
||||
});
|
||||
|
||||
test("still quotes a guarded value that needs quoting", () => {
|
||||
expect(escapeCsvValue('=HYPERLINK("x")')).toBe('"\'=HYPERLINK(""x"")"');
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,76 @@
|
||||
/**
|
||||
* Minimal CSV builder used by the repository and organization exports.
|
||||
*
|
||||
* Follows RFC 4180: a header row, CRLF line endings, and fields quoted when
|
||||
* they hold a comma, a double quote, a carriage return or a line feed, with
|
||||
* inner quotes doubled.
|
||||
*/
|
||||
|
||||
/** Fields that need quoting per RFC 4180. */
|
||||
const NEEDS_QUOTING = /[",\r\n]/;
|
||||
|
||||
/**
|
||||
* Text that a spreadsheet would run as a formula. Repository descriptions
|
||||
* and error messages come from third parties, so a value starting with one
|
||||
* of these is prefixed with a single quote, which Excel and Sheets treat as
|
||||
* "this is text". Only strings are affected; numbers stay as they are.
|
||||
*/
|
||||
const FORMULA_PREFIX = /^[=+\-@\t\r]/;
|
||||
|
||||
/**
|
||||
* Turn one value into its CSV text, before any quoting:
|
||||
* null and undefined become empty, dates become ISO strings, booleans become
|
||||
* "true" or "false", and anything else is stringified.
|
||||
*/
|
||||
export function formatCsvValue(value: unknown): string {
|
||||
if (value === null || value === undefined) {
|
||||
return "";
|
||||
}
|
||||
|
||||
if (value instanceof Date) {
|
||||
return Number.isNaN(value.getTime()) ? "" : value.toISOString();
|
||||
}
|
||||
|
||||
if (typeof value === "boolean") {
|
||||
return value ? "true" : "false";
|
||||
}
|
||||
|
||||
if (typeof value === "object") {
|
||||
return JSON.stringify(value) ?? "";
|
||||
}
|
||||
|
||||
if (typeof value === "string" && FORMULA_PREFIX.test(value)) {
|
||||
return `'${value}`;
|
||||
}
|
||||
|
||||
return String(value);
|
||||
}
|
||||
|
||||
/** Format a value and quote it when the content requires it. */
|
||||
export function escapeCsvValue(value: unknown): string {
|
||||
const text = formatCsvValue(value);
|
||||
|
||||
if (!NEEDS_QUOTING.test(text)) {
|
||||
return text;
|
||||
}
|
||||
|
||||
return `"${text.replace(/"/g, '""')}"`;
|
||||
}
|
||||
|
||||
/**
|
||||
* Build a CSV document from rows and the column keys to export. The keys
|
||||
* double as the header row, so the caller controls both the selection and the
|
||||
* column order.
|
||||
*/
|
||||
export function toCsv<Row>(
|
||||
rows: readonly Row[],
|
||||
columns: readonly (keyof Row & string)[]
|
||||
): string {
|
||||
const lines: string[] = [columns.map(escapeCsvValue).join(",")];
|
||||
|
||||
for (const row of rows) {
|
||||
lines.push(columns.map((column) => escapeCsvValue(row[column])).join(","));
|
||||
}
|
||||
|
||||
return `${lines.join("\r\n")}\r\n`;
|
||||
}
|
||||
@@ -0,0 +1,115 @@
|
||||
/**
|
||||
* Unit tests for the per-key async mutex used to serialize release
|
||||
* reconciliation per destination repository (issue #417).
|
||||
*/
|
||||
|
||||
import { describe, expect, it } from "bun:test";
|
||||
import { keyedLockCount, withKeyedLock } from "@/lib/utils/keyed-mutex";
|
||||
|
||||
const tick = (ms = 0) => new Promise((resolve) => setTimeout(resolve, ms));
|
||||
|
||||
describe("withKeyedLock", () => {
|
||||
it("runs two callers on the same key strictly one after the other", async () => {
|
||||
const events: string[] = [];
|
||||
|
||||
const first = withKeyedLock("repo-a", async () => {
|
||||
events.push("a:start");
|
||||
await tick(10);
|
||||
events.push("a:end");
|
||||
});
|
||||
const second = withKeyedLock("repo-a", async () => {
|
||||
events.push("b:start");
|
||||
await tick(1);
|
||||
events.push("b:end");
|
||||
});
|
||||
|
||||
await Promise.all([first, second]);
|
||||
|
||||
expect(events).toEqual(["a:start", "a:end", "b:start", "b:end"]);
|
||||
});
|
||||
|
||||
it("runs callers on different keys concurrently", async () => {
|
||||
const events: string[] = [];
|
||||
|
||||
await Promise.all([
|
||||
withKeyedLock("repo-a", async () => {
|
||||
events.push("a:start");
|
||||
await tick(10);
|
||||
events.push("a:end");
|
||||
}),
|
||||
withKeyedLock("repo-b", async () => {
|
||||
events.push("b:start");
|
||||
await tick(10);
|
||||
events.push("b:end");
|
||||
}),
|
||||
]);
|
||||
|
||||
// Both started before either finished, so no serialization across keys.
|
||||
expect(events.slice(0, 2).sort()).toEqual(["a:start", "b:start"]);
|
||||
});
|
||||
|
||||
it("releases the key when the callback throws, and surfaces the error", async () => {
|
||||
const events: string[] = [];
|
||||
|
||||
const failing = withKeyedLock("repo-c", async () => {
|
||||
events.push("a:start");
|
||||
await tick(1);
|
||||
throw new Error("upload exploded");
|
||||
});
|
||||
const following = withKeyedLock("repo-c", async () => {
|
||||
events.push("b:start");
|
||||
return "ran anyway";
|
||||
});
|
||||
|
||||
await expect(failing).rejects.toThrow("upload exploded");
|
||||
expect(await following).toBe("ran anyway");
|
||||
expect(events).toEqual(["a:start", "b:start"]);
|
||||
});
|
||||
|
||||
it("keeps FIFO order for several waiters on one key", async () => {
|
||||
const order: number[] = [];
|
||||
|
||||
await Promise.all(
|
||||
[1, 2, 3, 4].map((n) =>
|
||||
withKeyedLock("repo-d", async () => {
|
||||
order.push(n);
|
||||
// Later callers ask for a shorter delay: without the lock they would
|
||||
// finish out of order.
|
||||
await tick(5 - n);
|
||||
})
|
||||
)
|
||||
);
|
||||
|
||||
expect(order).toEqual([1, 2, 3, 4]);
|
||||
});
|
||||
|
||||
it("returns the callback's value and forgets the key once nobody is queued", async () => {
|
||||
const before = keyedLockCount();
|
||||
|
||||
const value = await withKeyedLock("repo-e", async () => 42);
|
||||
expect(value).toBe(42);
|
||||
|
||||
// The cleanup runs in a microtask after the tail settles.
|
||||
await tick(0);
|
||||
expect(keyedLockCount()).toBe(before);
|
||||
});
|
||||
|
||||
it("serializes a second caller that arrives while the first is running", async () => {
|
||||
let holderRunning = false;
|
||||
let sawOverlap = false;
|
||||
|
||||
const holder = withKeyedLock("repo-f", async () => {
|
||||
holderRunning = true;
|
||||
await tick(15);
|
||||
holderRunning = false;
|
||||
});
|
||||
|
||||
await tick(1);
|
||||
const waiter = withKeyedLock("repo-f", async () => {
|
||||
if (holderRunning) sawOverlap = true;
|
||||
});
|
||||
|
||||
await Promise.all([holder, waiter]);
|
||||
expect(sawOverlap).toBe(false);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,63 @@
|
||||
/**
|
||||
* Per-key async mutex (issue #417).
|
||||
*
|
||||
* Release asset reconciliation is a read-then-write against the destination:
|
||||
* list the attachments a release already has, then upload whatever is missing.
|
||||
* The destination has no name-collision check on attachment upload (Gitea's and
|
||||
* Forgejo's CreateReleaseAttachment appends a row for every POST), so two
|
||||
* passes that overlap on the same release both see an asset as absent and both
|
||||
* upload it. N overlapping passes leave N copies of every asset.
|
||||
*
|
||||
* A per-row status guard cannot close that window on its own: two repository
|
||||
* rows (the same upstream repository imported under two sources, say) resolve
|
||||
* to the same destination repository, so the exclusion has to be keyed on the
|
||||
* destination rather than on the database row.
|
||||
*
|
||||
* This lock is process-local, which is where the overlapping passes seen in
|
||||
* #417 come from: the scheduler racing a manual run, recovery resuming a job
|
||||
* that is still running, or duplicate repository rows processed together by one
|
||||
* Promise.all.
|
||||
*/
|
||||
|
||||
/**
|
||||
* Tail of the queue for each key. It settles (and never rejects) once the last
|
||||
* queued caller is done, so the next caller only has to wait on it.
|
||||
*/
|
||||
const queues = new Map<string, Promise<void>>();
|
||||
|
||||
/**
|
||||
* Run `fn` with exclusive access to `key`. Callers on the same key run one at a
|
||||
* time in the order they arrived; callers on different keys are unaffected.
|
||||
* The lock is released whether `fn` resolves or throws, and the caller still
|
||||
* sees `fn`'s own result or error.
|
||||
*/
|
||||
export function withKeyedLock<T>(key: string, fn: () => Promise<T>): Promise<T> {
|
||||
const previous = queues.get(key) ?? Promise.resolve();
|
||||
|
||||
// `previous` never rejects, so a caller that threw still hands the key on.
|
||||
const result = previous.then(() => fn());
|
||||
|
||||
// The tail swallows the outcome: the next caller only needs to know the
|
||||
// previous one finished, and an unobserved rejection here would surface as
|
||||
// an unhandled rejection.
|
||||
const tail = result.then(
|
||||
() => undefined,
|
||||
() => undefined
|
||||
);
|
||||
queues.set(key, tail);
|
||||
|
||||
// Drop the key once nobody is queued behind this caller, so the map does not
|
||||
// grow one permanent entry per repository.
|
||||
void tail.then(() => {
|
||||
if (queues.get(key) === tail) {
|
||||
queues.delete(key);
|
||||
}
|
||||
});
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
/** Number of keys currently held or queued. For tests and diagnostics. */
|
||||
export function keyedLockCount(): number {
|
||||
return queues.size;
|
||||
}
|
||||
@@ -822,6 +822,7 @@ import {
|
||||
GITHUB_ONLY_METADATA_KEYS,
|
||||
resolveMirrorOptions as resolveOptionsForSource,
|
||||
} from "./mirror-overrides";
|
||||
import { sourceKindSupportsReleases } from "@/lib/source-providers/kinds";
|
||||
|
||||
describe("resolveMirrorOptions for non-GitHub sources", () => {
|
||||
const config = {
|
||||
@@ -870,6 +871,87 @@ describe("resolveMirrorOptions for non-GitHub sources", () => {
|
||||
expect(resolved.mirrorIssues).toBe(true);
|
||||
expect(resolved.mirrorReleases).toBe(true);
|
||||
});
|
||||
|
||||
// #440: Gitea and Forgejo expose releases through their own API, so the
|
||||
// release switch is no longer part of the GitHub only clamp.
|
||||
test("a Gitea repository keeps releases while the GitHub only metadata is clamped", () => {
|
||||
const resolved = resolveOptionsForSource({
|
||||
config,
|
||||
repository: { isStarred: false, mirrorOverrides: null, sourceProvider: "gitea" } as any,
|
||||
});
|
||||
expect(resolved.mirrorReleases).toBe(true);
|
||||
expect(resolved.releaseLimit).toBe(5);
|
||||
expect(resolved.mirrorIssues).toBe(false);
|
||||
expect(resolved.mirrorPullRequests).toBe(false);
|
||||
expect(resolved.mirrorLabels).toBe(false);
|
||||
expect(resolved.mirrorMilestones).toBe(false);
|
||||
expect(resolved.mirrorMetadata).toBe(false);
|
||||
});
|
||||
|
||||
test("forgejo and codeberg spellings resolve to the Gitea kind and keep releases", () => {
|
||||
for (const sourceProvider of ["forgejo", "codeberg"]) {
|
||||
const resolved = resolveOptionsForSource({
|
||||
config,
|
||||
repository: { isStarred: false, mirrorOverrides: null, sourceProvider } as any,
|
||||
});
|
||||
expect(resolved.mirrorReleases).toBe(true);
|
||||
expect(resolved.mirrorIssues).toBe(false);
|
||||
}
|
||||
});
|
||||
|
||||
test("a GitLab repository still has releases and every other metadata flag clamped", () => {
|
||||
const resolved = resolveOptionsForSource({
|
||||
config,
|
||||
repository: {
|
||||
isStarred: false,
|
||||
mirrorOverrides: { mirrorReleases: true },
|
||||
sourceProvider: "gitlab",
|
||||
} as any,
|
||||
});
|
||||
expect(resolved.mirrorReleases).toBe(false);
|
||||
for (const key of GITHUB_ONLY_METADATA_KEYS) {
|
||||
expect(resolved[key]).toBe(false);
|
||||
}
|
||||
});
|
||||
|
||||
test("releases left the GitHub only set, so the constant no longer lists them", () => {
|
||||
expect(GITHUB_ONLY_METADATA_KEYS as readonly string[]).not.toContain("mirrorReleases");
|
||||
});
|
||||
|
||||
test("a Gitea repository still loses releases on a push destination", () => {
|
||||
const resolved = resolveOptionsForSource({
|
||||
config: {
|
||||
...config,
|
||||
giteaConfig: { ...config.giteaConfig, provider: "github" },
|
||||
} as any,
|
||||
repository: { isStarred: false, mirrorOverrides: null, sourceProvider: "gitea" } as any,
|
||||
});
|
||||
expect(resolved.mirrorReleases).toBe(false);
|
||||
});
|
||||
|
||||
test("starredCodeOnly still outranks the Gitea release capability", () => {
|
||||
const resolved = resolveOptionsForSource({
|
||||
config: { ...config, githubConfig: { starredCodeOnly: true } } as any,
|
||||
repository: { isStarred: true, mirrorOverrides: null, sourceProvider: "gitea" } as any,
|
||||
});
|
||||
expect(resolved.mirrorReleases).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
describe("sourceKindSupportsReleases", () => {
|
||||
test("GitHub and Gitea can list releases, GitLab cannot", () => {
|
||||
expect(sourceKindSupportsReleases("github")).toBe(true);
|
||||
expect(sourceKindSupportsReleases("gitea")).toBe(true);
|
||||
expect(sourceKindSupportsReleases("forgejo")).toBe(true);
|
||||
expect(sourceKindSupportsReleases("codeberg")).toBe(true);
|
||||
expect(sourceKindSupportsReleases("gitlab")).toBe(false);
|
||||
});
|
||||
|
||||
test("an unknown value is a GitHub row, as everywhere else", () => {
|
||||
expect(sourceKindSupportsReleases(undefined)).toBe(true);
|
||||
expect(sourceKindSupportsReleases(null)).toBe(true);
|
||||
expect(sourceKindSupportsReleases("bitbucket")).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
describe("release asset limit (#311)", () => {
|
||||
|
||||
@@ -1,7 +1,10 @@
|
||||
import type { Config } from "@/types/config";
|
||||
import type { MirrorOverrides, Repository } from "@/lib/db/schema";
|
||||
import { mirrorOverridesSchema } from "@/lib/db/schema";
|
||||
import { normalizeSourceProviderKind } from "@/lib/source-providers/kinds";
|
||||
import {
|
||||
normalizeSourceProviderKind,
|
||||
sourceKindSupportsReleases,
|
||||
} from "@/lib/source-providers/kinds";
|
||||
import { isPushDestinationKind, normalizeDestinationProviderKind } from "@/lib/destination-kinds";
|
||||
|
||||
/**
|
||||
@@ -139,13 +142,16 @@ export const STARRED_CLAMPED_KEYS = [
|
||||
/**
|
||||
* Flags that only work for GitHub sources.
|
||||
*
|
||||
* Issue, pull request, release, label and milestone mirroring read the
|
||||
* GitHub API. A repository from GitLab or Gitea gets code, wiki and LFS
|
||||
* through the Gitea pull mirror and nothing else, so the resolver forces
|
||||
* these off for it regardless of what any tier asked for.
|
||||
* Issue, pull request, label and milestone mirroring read the GitHub API. A
|
||||
* repository from GitLab or Gitea gets code, wiki and LFS through the Gitea
|
||||
* pull mirror, so the resolver forces these off for it regardless of what any
|
||||
* tier asked for.
|
||||
*
|
||||
* `mirrorReleases` is deliberately not in this list since #440: releases are
|
||||
* gated on `sourceKindSupportsReleases` instead, which also allows Gitea and
|
||||
* Forgejo sources. GitLab still ends up with releases off, through that gate.
|
||||
*/
|
||||
export const GITHUB_ONLY_METADATA_KEYS = [
|
||||
"mirrorReleases",
|
||||
"mirrorMetadata",
|
||||
"mirrorIssues",
|
||||
"mirrorPullRequests",
|
||||
@@ -531,12 +537,19 @@ export function resolveMirrorOptions({
|
||||
|
||||
// Metadata mirroring needs the GitHub API. Rows without a source
|
||||
// provider predate the column and came from GitHub.
|
||||
if (normalizeSourceProviderKind(repository.sourceProvider) !== "github") {
|
||||
const sourceKind = normalizeSourceProviderKind(repository.sourceProvider);
|
||||
if (sourceKind !== "github") {
|
||||
for (const key of GITHUB_ONLY_METADATA_KEYS) {
|
||||
resolved[key] = false;
|
||||
}
|
||||
}
|
||||
|
||||
// Releases are listed through the source API, which GitHub and
|
||||
// Gitea/Forgejo both expose (#440). GitLab is code only, so it lands here.
|
||||
if (!sourceKindSupportsReleases(sourceKind)) {
|
||||
resolved.mirrorReleases = false;
|
||||
}
|
||||
|
||||
// A push target receives branches and tags only.
|
||||
if (isPushDestinationConfig(config)) {
|
||||
for (const key of PUSH_TARGET_CLAMPED_KEYS) {
|
||||
|
||||
@@ -0,0 +1,127 @@
|
||||
import { afterEach, beforeEach, describe, expect, it } from "bun:test";
|
||||
import {
|
||||
DEFAULT_ORGANIZATIONS_VIEW,
|
||||
ORGANIZATIONS_VIEW_CHANGE_EVENT,
|
||||
ORGANIZATIONS_VIEW_STORAGE_KEY,
|
||||
getOrganizationsView,
|
||||
isOrganizationsViewMode,
|
||||
setOrganizationsView,
|
||||
subscribeToOrganizationsViewChange,
|
||||
} from "./organizations-view";
|
||||
|
||||
// A minimal window and localStorage so the helper can be exercised
|
||||
// without a DOM library. Both are removed again afterwards.
|
||||
function installBrowserGlobals() {
|
||||
const store = new Map<string, string>();
|
||||
const listeners = new Map<string, Set<(event: any) => void>>();
|
||||
(globalThis as any).localStorage = {
|
||||
getItem: (key: string) => store.get(key) ?? null,
|
||||
setItem: (key: string, value: string) => void store.set(key, value),
|
||||
removeItem: (key: string) => void store.delete(key),
|
||||
};
|
||||
(globalThis as any).window = {
|
||||
addEventListener: (type: string, cb: (event: any) => void) => {
|
||||
if (!listeners.has(type)) listeners.set(type, new Set());
|
||||
listeners.get(type)!.add(cb);
|
||||
},
|
||||
removeEventListener: (type: string, cb: (event: any) => void) => {
|
||||
listeners.get(type)?.delete(cb);
|
||||
},
|
||||
dispatchEvent: (event: any) => {
|
||||
listeners.get(event.type)?.forEach((cb) => cb(event));
|
||||
return true;
|
||||
},
|
||||
};
|
||||
if (typeof (globalThis as any).CustomEvent === "undefined") {
|
||||
(globalThis as any).CustomEvent = class CustomEvent {
|
||||
type: string;
|
||||
detail: unknown;
|
||||
constructor(type: string, init?: { detail?: unknown }) {
|
||||
this.type = type;
|
||||
this.detail = init?.detail;
|
||||
}
|
||||
};
|
||||
}
|
||||
return { store, listeners };
|
||||
}
|
||||
|
||||
describe("organizations view preference", () => {
|
||||
const hadWindow = typeof (globalThis as any).window !== "undefined";
|
||||
const hadStorage = typeof (globalThis as any).localStorage !== "undefined";
|
||||
const savedWindow = (globalThis as any).window;
|
||||
const savedStorage = (globalThis as any).localStorage;
|
||||
|
||||
beforeEach(() => {
|
||||
delete (globalThis as any).window;
|
||||
delete (globalThis as any).localStorage;
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
if (hadWindow) (globalThis as any).window = savedWindow;
|
||||
else delete (globalThis as any).window;
|
||||
if (hadStorage) (globalThis as any).localStorage = savedStorage;
|
||||
else delete (globalThis as any).localStorage;
|
||||
});
|
||||
|
||||
it("falls back to cards without a browser", () => {
|
||||
expect(getOrganizationsView()).toBe(DEFAULT_ORGANIZATIONS_VIEW);
|
||||
expect(DEFAULT_ORGANIZATIONS_VIEW).toBe("cards");
|
||||
// Setting without a window must not throw either.
|
||||
expect(() => setOrganizationsView("list")).not.toThrow();
|
||||
});
|
||||
|
||||
it("only accepts the two known modes", () => {
|
||||
expect(isOrganizationsViewMode("cards")).toBe(true);
|
||||
expect(isOrganizationsViewMode("list")).toBe(true);
|
||||
expect(isOrganizationsViewMode("table")).toBe(false);
|
||||
expect(isOrganizationsViewMode(null)).toBe(false);
|
||||
});
|
||||
|
||||
it("persists the choice and ignores garbage in storage", () => {
|
||||
const { store } = installBrowserGlobals();
|
||||
expect(getOrganizationsView()).toBe("cards");
|
||||
|
||||
setOrganizationsView("list");
|
||||
expect(store.get(ORGANIZATIONS_VIEW_STORAGE_KEY)).toBe("list");
|
||||
expect(getOrganizationsView()).toBe("list");
|
||||
|
||||
store.set(ORGANIZATIONS_VIEW_STORAGE_KEY, "sideways");
|
||||
expect(getOrganizationsView()).toBe("cards");
|
||||
});
|
||||
|
||||
it("notifies subscribers in this tab and from the storage event", () => {
|
||||
const { listeners } = installBrowserGlobals();
|
||||
let calls = 0;
|
||||
const unsubscribe = subscribeToOrganizationsViewChange(() => {
|
||||
calls += 1;
|
||||
});
|
||||
|
||||
setOrganizationsView("list");
|
||||
expect(calls).toBe(1);
|
||||
|
||||
// Another tab changed it.
|
||||
listeners.get("storage")?.forEach((cb) => cb({ key: ORGANIZATIONS_VIEW_STORAGE_KEY }));
|
||||
expect(calls).toBe(2);
|
||||
// An unrelated key does nothing.
|
||||
listeners.get("storage")?.forEach((cb) => cb({ key: "timeFormat" }));
|
||||
expect(calls).toBe(2);
|
||||
|
||||
unsubscribe();
|
||||
(globalThis as any).window.dispatchEvent(new CustomEvent(ORGANIZATIONS_VIEW_CHANGE_EVENT));
|
||||
expect(calls).toBe(2);
|
||||
});
|
||||
|
||||
it("swallows a storage that throws", () => {
|
||||
installBrowserGlobals();
|
||||
(globalThis as any).localStorage = {
|
||||
getItem: () => {
|
||||
throw new Error("blocked");
|
||||
},
|
||||
setItem: () => {
|
||||
throw new Error("blocked");
|
||||
},
|
||||
};
|
||||
expect(getOrganizationsView()).toBe("cards");
|
||||
expect(() => setOrganizationsView("list")).not.toThrow();
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,55 @@
|
||||
/**
|
||||
* Cards or list on the Organizations page (#428).
|
||||
*
|
||||
* A pure display preference, so it lives in localStorage next to the theme
|
||||
* and time format preferences rather than in the database. Reading is safe
|
||||
* during SSR and in tests, where localStorage does not exist.
|
||||
*/
|
||||
|
||||
export type OrganizationsViewMode = "cards" | "list";
|
||||
|
||||
export const ORGANIZATIONS_VIEW_STORAGE_KEY = "organizationsView";
|
||||
export const ORGANIZATIONS_VIEW_CHANGE_EVENT = "gitea-mirror:organizations-view-change";
|
||||
|
||||
export const DEFAULT_ORGANIZATIONS_VIEW: OrganizationsViewMode = "cards";
|
||||
|
||||
export function isOrganizationsViewMode(value: unknown): value is OrganizationsViewMode {
|
||||
return value === "cards" || value === "list";
|
||||
}
|
||||
|
||||
export function getOrganizationsView(): OrganizationsViewMode {
|
||||
if (typeof localStorage === "undefined") return DEFAULT_ORGANIZATIONS_VIEW;
|
||||
try {
|
||||
const stored = localStorage.getItem(ORGANIZATIONS_VIEW_STORAGE_KEY);
|
||||
return isOrganizationsViewMode(stored) ? stored : DEFAULT_ORGANIZATIONS_VIEW;
|
||||
} catch {
|
||||
return DEFAULT_ORGANIZATIONS_VIEW;
|
||||
}
|
||||
}
|
||||
|
||||
export function setOrganizationsView(mode: OrganizationsViewMode): void {
|
||||
try {
|
||||
if (typeof localStorage !== "undefined") {
|
||||
localStorage.setItem(ORGANIZATIONS_VIEW_STORAGE_KEY, mode);
|
||||
}
|
||||
} catch {
|
||||
// Persisting is best effort; listeners are still told.
|
||||
}
|
||||
if (typeof window !== "undefined") {
|
||||
window.dispatchEvent(new CustomEvent(ORGANIZATIONS_VIEW_CHANGE_EVENT, { detail: mode }));
|
||||
}
|
||||
}
|
||||
|
||||
/** Subscribe to changes from this tab and from other tabs. */
|
||||
export function subscribeToOrganizationsViewChange(callback: () => void): () => void {
|
||||
if (typeof window === "undefined") return () => {};
|
||||
const onStorage = (event: StorageEvent) => {
|
||||
if (event.key === null || event.key === ORGANIZATIONS_VIEW_STORAGE_KEY) callback();
|
||||
};
|
||||
window.addEventListener(ORGANIZATIONS_VIEW_CHANGE_EVENT, callback);
|
||||
window.addEventListener("storage", onStorage);
|
||||
return () => {
|
||||
window.removeEventListener(ORGANIZATIONS_VIEW_CHANGE_EVENT, callback);
|
||||
window.removeEventListener("storage", onStorage);
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,145 @@
|
||||
import { afterEach, describe, expect, it } from "bun:test";
|
||||
import {
|
||||
assertSafeOutboundUrl,
|
||||
isLinkLocalAddress,
|
||||
OutboundUrlError,
|
||||
resolveOutboundTarget,
|
||||
safeFetch,
|
||||
} from "./outbound-url";
|
||||
|
||||
const noResolve = async () => [];
|
||||
|
||||
describe("isLinkLocalAddress", () => {
|
||||
it("matches the IPv4 link local range", () => {
|
||||
expect(isLinkLocalAddress("169.254.169.254")).toBe(true);
|
||||
expect(isLinkLocalAddress("169.254.0.1")).toBe(true);
|
||||
expect(isLinkLocalAddress("169.253.1.1")).toBe(false);
|
||||
expect(isLinkLocalAddress("10.0.0.1")).toBe(false);
|
||||
expect(isLinkLocalAddress("192.168.1.10")).toBe(false);
|
||||
expect(isLinkLocalAddress("127.0.0.1")).toBe(false);
|
||||
});
|
||||
|
||||
it("matches IPv6 link local and the mapped IPv4 forms", () => {
|
||||
expect(isLinkLocalAddress("fe80::1")).toBe(true);
|
||||
expect(isLinkLocalAddress("FE9A::1")).toBe(true);
|
||||
expect(isLinkLocalAddress("febf::1")).toBe(true);
|
||||
expect(isLinkLocalAddress("fec0::1")).toBe(false);
|
||||
expect(isLinkLocalAddress("::ffff:169.254.169.254")).toBe(true);
|
||||
expect(isLinkLocalAddress("::ffff:a9fe:a9fe")).toBe(true);
|
||||
expect(isLinkLocalAddress("::ffff:192.168.1.1")).toBe(false);
|
||||
expect(isLinkLocalAddress("::1")).toBe(false);
|
||||
expect(isLinkLocalAddress("not-an-ip")).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
describe("assertSafeOutboundUrl", () => {
|
||||
it("allows private networks, localhost and Docker service names", async () => {
|
||||
for (const url of [
|
||||
"http://192.168.1.10:3000",
|
||||
"http://10.0.0.5",
|
||||
"http://172.16.4.4:8080/api",
|
||||
"http://localhost:3000",
|
||||
"http://127.0.0.1:3000",
|
||||
"http://gitea:3000",
|
||||
"https://codeberg.org",
|
||||
]) {
|
||||
await expect(assertSafeOutboundUrl(url, noResolve)).resolves.toBeInstanceOf(URL);
|
||||
}
|
||||
});
|
||||
|
||||
it("refuses the metadata address in every spelling", async () => {
|
||||
for (const url of [
|
||||
"http://169.254.169.254/latest/meta-data/",
|
||||
"http://[fe80::1]/",
|
||||
"http://[::ffff:169.254.169.254]/",
|
||||
"http://metadata.google.internal/computeMetadata/v1/",
|
||||
"http://METADATA/",
|
||||
]) {
|
||||
await expect(assertSafeOutboundUrl(url, noResolve)).rejects.toBeInstanceOf(OutboundUrlError);
|
||||
}
|
||||
});
|
||||
|
||||
it("refuses a host name that resolves to a link local address", async () => {
|
||||
const resolver = async () => ["93.184.216.34", "169.254.169.254"];
|
||||
await expect(assertSafeOutboundUrl("http://evil.example/", resolver)).rejects.toThrow(
|
||||
/link local/
|
||||
);
|
||||
});
|
||||
|
||||
it("leaves unresolvable names to fetch", async () => {
|
||||
await expect(assertSafeOutboundUrl("http://nope.invalid/", noResolve)).resolves.toBeInstanceOf(
|
||||
URL
|
||||
);
|
||||
});
|
||||
|
||||
it("refuses other schemes and malformed URLs", async () => {
|
||||
await expect(assertSafeOutboundUrl("file:///etc/passwd", noResolve)).rejects.toThrow(
|
||||
/http and https/
|
||||
);
|
||||
await expect(assertSafeOutboundUrl("gopher://x/", noResolve)).rejects.toThrow(/http and https/);
|
||||
await expect(assertSafeOutboundUrl("not a url", noResolve)).rejects.toThrow(/Invalid URL/);
|
||||
});
|
||||
});
|
||||
|
||||
describe("resolveOutboundTarget", () => {
|
||||
it("pins plain http to the checked address and keeps the name in Host", async () => {
|
||||
const resolver = async () => ["10.0.0.5", "10.0.0.6"];
|
||||
const target = await resolveOutboundTarget("http://ntfy.local:8080/topic?x=1", resolver);
|
||||
expect(target.requestUrl).toBe("http://10.0.0.5:8080/topic?x=1");
|
||||
expect(target.headers).toEqual({ Host: "ntfy.local:8080" });
|
||||
});
|
||||
|
||||
it("brackets a pinned IPv6 address", async () => {
|
||||
const resolver = async () => ["fd00::10"];
|
||||
const target = await resolveOutboundTarget("http://gotify.local/message", resolver);
|
||||
expect(target.requestUrl).toBe("http://[fd00::10]/message");
|
||||
expect(target.headers).toEqual({ Host: "gotify.local" });
|
||||
});
|
||||
|
||||
it("does not pin https, IP literals or unresolvable names", async () => {
|
||||
const resolver = async () => ["10.0.0.5"];
|
||||
expect((await resolveOutboundTarget("https://idp.example.com/x", resolver)).headers).toEqual({});
|
||||
expect((await resolveOutboundTarget("http://192.168.1.10:3000/", resolver)).requestUrl).toBe(
|
||||
"http://192.168.1.10:3000/"
|
||||
);
|
||||
const unresolved = await resolveOutboundTarget("http://nope.invalid/", noResolve);
|
||||
expect(unresolved.requestUrl).toBe("http://nope.invalid/");
|
||||
expect(unresolved.headers).toEqual({});
|
||||
});
|
||||
});
|
||||
|
||||
describe("safeFetch", () => {
|
||||
const realFetch = globalThis.fetch;
|
||||
afterEach(() => {
|
||||
globalThis.fetch = realFetch;
|
||||
});
|
||||
|
||||
it("never follows redirects, pins the address and refuses blocked hosts before fetching", async () => {
|
||||
let seen: any = null;
|
||||
globalThis.fetch = (async (input: any, init?: RequestInit) => {
|
||||
const headers = new Headers(init?.headers);
|
||||
seen = { url: String(input), redirect: init?.redirect, host: headers.get("host"), auth: headers.get("authorization") };
|
||||
return new Response("", { status: 302, headers: { location: "http://169.254.169.254/" } });
|
||||
}) as typeof fetch;
|
||||
|
||||
const resolver = async () => ["10.0.0.5"];
|
||||
const response = await safeFetch(
|
||||
"http://ntfy.local/topic",
|
||||
{ method: "POST", headers: { Authorization: "Bearer t" } },
|
||||
resolver
|
||||
);
|
||||
expect(response.status).toBe(302);
|
||||
expect(seen).toEqual({
|
||||
url: "http://10.0.0.5/topic",
|
||||
redirect: "manual",
|
||||
host: "ntfy.local",
|
||||
auth: "Bearer t",
|
||||
});
|
||||
|
||||
seen = null;
|
||||
await expect(safeFetch("http://169.254.169.254/", {}, noResolve)).rejects.toBeInstanceOf(
|
||||
OutboundUrlError
|
||||
);
|
||||
expect(seen).toBeNull();
|
||||
});
|
||||
});
|
||||
Loaded 100 of 143 files, more files were not shown because too many files have changed in this diff.
Show more
Reference in new issue
Block a user