mirror of
https://github.com/RayLabsHQ/gitea-mirror.git
synced 2026-11-03 07:42:19 +01:00
Compare commits
14
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
f581292912 | ||
|
|
8f60ccf6dc | ||
|
|
c8618743c8 | ||
|
|
2ff2e71d18 | ||
|
|
2d1795051d | ||
|
|
9c11d398e9 | ||
|
|
3df3fa577e | ||
|
|
0759db80d3 | ||
|
|
0299be92b1 | ||
|
|
22a0aafcc1 | ||
|
|
3b78f0a8d3 | ||
|
|
48e0c5c346 | ||
|
|
01f926d214 | ||
|
|
1cc7a9a05c |
No files matched your search
@@ -28,6 +28,13 @@ on:
|
||||
schedule:
|
||||
- cron: '0 0 * * 0' # Weekly security scan on Sunday at midnight
|
||||
|
||||
# One build per ref at a time. A merge and the release bump that follows it
|
||||
# land on main seconds apart and used to race each other for the latest tag
|
||||
# (#425). Branch and PR builds cancel the older run; tag builds always finish.
|
||||
concurrency:
|
||||
group: docker-build-${{ github.ref }}
|
||||
cancel-in-progress: ${{ !startsWith(github.ref, 'refs/tags/') }}
|
||||
|
||||
env:
|
||||
REGISTRY: ghcr.io
|
||||
IMAGE: ${{ github.repository }}
|
||||
@@ -90,9 +97,9 @@ jobs:
|
||||
echo "APP_VERSION=${APP_VERSION}" >> $GITHUB_OUTPUT
|
||||
echo "Using version tag: ${TAG_VERSION}"
|
||||
else
|
||||
echo "VERSION=latest" >> $GITHUB_OUTPUT
|
||||
echo "VERSION=edge" >> $GITHUB_OUTPUT
|
||||
echo "APP_VERSION=dev" >> $GITHUB_OUTPUT
|
||||
echo "No version tag, using 'latest'"
|
||||
echo "No version tag, using 'edge'"
|
||||
fi
|
||||
|
||||
# Keep version files aligned automatically for tag-based releases
|
||||
@@ -116,12 +123,15 @@ jobs:
|
||||
images: ${{ env.REGISTRY }}/${{ env.IMAGE }}
|
||||
labels: |
|
||||
org.opencontainers.image.revision=${{ env.SHA }}
|
||||
# latest follows the release tag, not main: a main build can carry
|
||||
# an unreleased package.json and must never overwrite a release.
|
||||
# Pre-release tags (v1.2.3-rc.1) keep latest on the last stable one.
|
||||
tags: |
|
||||
type=edge,branch=$repo.default_branch
|
||||
type=edge,branch=main
|
||||
type=semver,pattern=v{{version}}
|
||||
type=sha,prefix=,suffix=,format=short
|
||||
type=raw,value=latest,enable={{is_default_branch}}
|
||||
type=raw,value=${{ steps.tag_version.outputs.VERSION }}
|
||||
type=raw,value=latest,enable=${{ startsWith(github.ref, 'refs/tags/v') && !contains(github.ref, '-') }}
|
||||
type=raw,value=${{ steps.tag_version.outputs.VERSION }},enable=${{ startsWith(github.ref, 'refs/tags/v') }}
|
||||
type=ref,event=pr,prefix=pr-
|
||||
|
||||
# Build and push Docker image
|
||||
@@ -223,7 +233,10 @@ jobs:
|
||||
if: github.event_name != 'pull_request'
|
||||
with:
|
||||
command: cves,recommendations
|
||||
image: ${{ env.REGISTRY }}/${{ env.IMAGE }}:latest
|
||||
# Scan the image this run pushed: the release tag on a tag build,
|
||||
# edge on a main or scheduled build. The action rejects digest
|
||||
# references ("missing or wrong image input").
|
||||
image: ${{ env.REGISTRY }}/${{ env.IMAGE }}:${{ steps.tag_version.outputs.VERSION }}
|
||||
sarif-file: scout-results.sarif
|
||||
summary: true
|
||||
exit-code: false
|
||||
|
||||
@@ -41,12 +41,42 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
|
||||
- Proper documentation for multi-URL access patterns
|
||||
- Comprehensive fix report documentation
|
||||
|
||||
### Changed
|
||||
- A new configuration starts with scheduling off unless `SCHEDULE_ENABLED=true`, `SCHEDULE_INTERVAL` or `GITEA_MIRROR_INTERVAL` turns it on, the same rule the environment loader uses; the built-in default schedule is the daily 22:00 clock schedule the automation card shows, so enabling from the card and the scheduler agree from the first save (#427)
|
||||
|
||||
### Fixed
|
||||
- The `latest` Docker image tag no longer lags behind a release (#425)
|
||||
- A merge and the version bump that follows it land on main seconds apart; both built the image and both pushed `latest`, and on v3.36.1 the older build finished last, so `latest` carried 3.36.0 until the weekly rebuild replaced it
|
||||
- The workflow runs one build per ref at a time, `latest` is pushed only by a stable release tag build, main builds push `edge` and the short commit sha, and the security scan looks at the image the run just pushed
|
||||
- The automation card shows the schedule that actually runs (#427)
|
||||
- A fresh install started with scheduling on and a plain 24 hour interval counted from first login, and the card, which only knows clock schedules, showed the 22:00 placeholder as if it were saved; the same misreport hit Docker installs with `GITEA_MIRROR_INTERVAL=8h`
|
||||
- When a plain interval is stored (from `SCHEDULE_INTERVAL`, `GITEA_MIRROR_INTERVAL` or an older version) the card says what runs and where it came from, and leaves frequency and start time unset until one is picked
|
||||
- The timezone chip shows the stored timezone instead of the browser's, and becomes a one-click switch to the browser timezone when the two differ
|
||||
- Ignoring an organization stops its repositories from syncing (#429)
|
||||
- Ignore Organization only changed the organization row; its mirrored repositories kept syncing and rediscovery kept importing new ones for it
|
||||
- The organization's idle repositories are ignored with it (rows in flight or being deleted finish on their own) and restored when it is included again, mirrored ones as mirrored and the rest as imported; rediscovery skips ignored organizations
|
||||
- Organization cards name the configured destination instead of always saying Gitea (#430)
|
||||
- The Nix snippets in the README, NIX.md and the deployment guide import the module before enabling `services.gitea-mirror`, and put the flake input in `flake.nix` where it belongs (#426)
|
||||
- Crash recovery no longer resumes a job that is still running
|
||||
- A job older than two hours was treated as interrupted even while it checkpointed every two minutes, so recovery started a second pass over the same repositories alongside the original; the age rule is gone and only a missing or stale checkpoint marks a job interrupted
|
||||
- Every completed item is now recorded in the job's checkpoint instead of one in every N, so a real resume skips exactly what was done; progress events are still throttled
|
||||
- Concurrent items completing at the same time could overwrite each other's checkpoint; job progress writes are serialized per job
|
||||
- A resumed job records a fresh start time
|
||||
- The same upstream repository is no longer imported twice under two sources that point at the same host
|
||||
- Discovery decided what was new by (source, full name), so an organization pinned to a public-only github.com source and a personal github.com token source each inserted their own row for one repository, and both rows mirrored to the same destination in the same scheduler batch
|
||||
- Auto import, organization re-discovery and the two import endpoints now identify a repository by its host and full name whichever source lists it; the same name on a different host is still a separate repository
|
||||
- A scheduler pass mirrors or syncs one row per upstream repository and logs the duplicate rows it skips, so existing duplicates never run side by side
|
||||
- A per-organization Mirror Destination equal to the organization's own name is kept instead of being dropped (#416)
|
||||
- The editor treated a typed value matching the organization name as "reset to default" and saved no override, which only holds under the preserve strategy; under single-org the default is the destination organization, so the repositories went there
|
||||
- The organization card now shows the default for the configured strategy in the preview, the placeholder, the helper text and the reset button, and marks any stored destination as custom
|
||||
- The repository destination column shows the organization's override as a repository's default, and pinning a repository to the value the strategy already produces is stored rather than discarded
|
||||
- Reset to Default on an organization card clears the override instead of saving the current value again
|
||||
- Release assets are never duplicated on the destination (#417)
|
||||
- Gitea and Forgejo accept any number of attachments with the same name, so two mirror passes that overlapped on one repository both saw an asset as missing and both uploaded it, leaving two, three or five copies
|
||||
- Release reconciliation now runs one pass at a time per destination repository, keeps a single copy of each asset, and deletes surplus and stale copies before uploading anything
|
||||
- A sync no longer starts on a repository that is already being mirrored or synced, and the mirror path claims the repository in one statement instead of checking and then writing
|
||||
- Nothing is uploaded when the destination cannot say which assets a release already has, or when a stale copy could not be removed first
|
||||
- The next sync of an affected repository removes the surplus copies for every release that still gets its assets; releases that have dropped out of the newest-N window keep their duplicates until they are removed by hand
|
||||
- Fixed metadata mirroring authentication errors (#68)
|
||||
- Changed field checking from `username` to `defaultOwner` in metadata functions
|
||||
- Added proper field validation for all metadata operations
|
||||
|
||||
@@ -46,9 +46,13 @@ nix run --extra-experimental-features 'nix-command flakes' .#gitea-mirror
|
||||
|
||||
### 4. NixOS System Service
|
||||
```nix
|
||||
# configuration.nix
|
||||
# flake.nix
|
||||
inputs.gitea-mirror.url = "github:RayLabsHQ/gitea-mirror";
|
||||
|
||||
# configuration.nix (needs `inputs` passed through specialArgs)
|
||||
{ inputs, ... }:
|
||||
{
|
||||
inputs.gitea-mirror.url = "github:RayLabsHQ/gitea-mirror";
|
||||
imports = [ inputs.gitea-mirror.nixosModules.default ];
|
||||
|
||||
services.gitea-mirror = {
|
||||
enable = true;
|
||||
|
||||
@@ -174,10 +174,15 @@ nix profile install --extra-experimental-features 'nix-command flakes' github:Ra
|
||||
gitea-mirror
|
||||
```
|
||||
|
||||
**NixOS users** - add to your configuration:
|
||||
**NixOS users** - add the flake input, import the module, then enable the service:
|
||||
```nix
|
||||
# flake.nix
|
||||
inputs.gitea-mirror.url = "github:RayLabsHQ/gitea-mirror";
|
||||
|
||||
# configuration.nix (needs `inputs` passed through specialArgs)
|
||||
{ inputs, ... }:
|
||||
{
|
||||
inputs.gitea-mirror.url = "github:RayLabsHQ/gitea-mirror";
|
||||
imports = [ inputs.gitea-mirror.nixosModules.default ];
|
||||
|
||||
services.gitea-mirror = {
|
||||
enable = true;
|
||||
|
||||
@@ -6,7 +6,7 @@
|
||||
"name": "gitea-mirror",
|
||||
"dependencies": {
|
||||
"@astrojs/mdx": "^7.0.3",
|
||||
"@astrojs/node": "^11.0.2",
|
||||
"@astrojs/node": "^11.1.5",
|
||||
"@astrojs/react": "^6.0.1",
|
||||
"@better-auth/api-key": "1.7.2",
|
||||
"@better-auth/oauth-provider": "1.7.2",
|
||||
@@ -94,6 +94,7 @@
|
||||
"lodash": "^4.18.1",
|
||||
"picomatch": "^4.0.4",
|
||||
"sharp": "^0.35.4",
|
||||
"smol-toml": "^1.7.1",
|
||||
"svgo": "^4.1.0",
|
||||
},
|
||||
"packages": {
|
||||
@@ -143,7 +144,7 @@
|
||||
|
||||
"@astrojs/mdx": ["@astrojs/mdx@7.0.3", "", { "dependencies": { "@astrojs/internal-helpers": "0.10.1", "@astrojs/markdown-remark": "7.2.1", "@mdx-js/mdx": "^3.1.1", "acorn": "^8.16.0", "es-module-lexer": "^2.0.0", "estree-util-visit": "^2.0.0", "hast-util-to-html": "^9.0.5", "piccolore": "^0.1.3", "rehype-raw": "^7.0.0", "remark-gfm": "^4.0.1", "remark-smartypants": "^3.0.2", "source-map": "^0.7.6", "unist-util-visit": "^5.1.0", "vfile": "^6.0.3" }, "peerDependencies": { "@astrojs/markdown-satteri": "^0.3.1", "astro": "^7.0.0" }, "optionalPeers": ["@astrojs/markdown-satteri"] }, "sha512-RxyIwU0uFam5ftwqKOjpIdhnFxZ/kEikeimLyQy3eGXbHT8WgRGzzesOIHVU8+m9TY8ag5WVOyvV24/GyqPdPQ=="],
|
||||
|
||||
"@astrojs/node": ["@astrojs/node@11.0.2", "", { "dependencies": { "@astrojs/internal-helpers": "0.10.1", "send": "^1.2.1", "server-destroy": "^1.0.1" }, "peerDependencies": { "astro": "^7.0.0" } }, "sha512-/ijULxT+A5Cm8wSwWZ2vgqfim1b05D6B8n/a9l6MMA4FCotIH73g7fL7y76XojKXpTe75FVvQH92OxsMqea9kQ=="],
|
||||
"@astrojs/node": ["@astrojs/node@11.1.5", "", { "dependencies": { "@astrojs/internal-helpers": "0.11.0", "send": "^1.2.1", "server-destroy": "^1.0.1" }, "peerDependencies": { "astro": "^7.2.1" } }, "sha512-CgA+LmG4UWqHxg/Vdc5MdxjjJNZSjKYQQx4uEUwnbj+1ZD2CYumJI3OWg2twsVdj2FlYsB5yvmCC54Re+OK+UQ=="],
|
||||
|
||||
"@astrojs/prism": ["@astrojs/prism@4.0.2", "", { "dependencies": { "prismjs": "^1.30.0" } }, "sha512-KTivpmnz6lDsC6o9H4+DNm2SrE/GHzw8cNAvEJwAvUT+eoaEnn/4NtbDNfRRaxaJHdp15gf+tfHAWiXR4wB3BA=="],
|
||||
|
||||
@@ -1473,7 +1474,7 @@
|
||||
|
||||
"sisteransi": ["sisteransi@1.0.5", "", {}, "sha512-bLGGlR1QxBcynn2d5YmDX4MGjlZvy2MRBDRNHLJ8VI6l6+9FUiyTFNJ0IveOSP0bcXgVDPRcfGqA0pjaqUpfVg=="],
|
||||
|
||||
"smol-toml": ["smol-toml@1.6.0", "", {}, "sha512-4zemZi0HvTnYwLfrpk/CF9LOd9Lt87kAt50GnqhMpyF9U3poDAP2+iukq2bZsO/ufegbYehBkqINbsWxj4l4cw=="],
|
||||
"smol-toml": ["smol-toml@1.8.0", "", {}, "sha512-kCZr2V3ch9i00x8zXRhjUNVcjG9ijES5dDudkXvUVCT5QlJNQWElSJdZqyPemffHoLNUYwOcou0Fy+ojN0uHSQ=="],
|
||||
|
||||
"sonner": ["sonner@2.0.7", "", { "peerDependencies": { "react": "^18.0.0 || ^19.0.0 || ^19.0.0-rc", "react-dom": "^18.0.0 || ^19.0.0 || ^19.0.0-rc" } }, "sha512-W6ZN4p58k8aDKA4XPcx2hpIQXBRAgyiWVkYhT7CvK6D3iAu7xjvVyhQHg2/iaKJZ1XVJ4r7XuwGL+WGEK37i9w=="],
|
||||
|
||||
@@ -1695,6 +1696,8 @@
|
||||
|
||||
"@astrojs/compiler-binding-wasm32-wasi/@napi-rs/wasm-runtime": ["@napi-rs/wasm-runtime@1.2.3", "", { "dependencies": { "@tybys/wasm-util": "^0.10.3" }, "peerDependencies": { "@emnapi/core": "^1.7.1 || ^2.0.0-alpha.4", "@emnapi/runtime": "^1.7.1 || ^2.0.0-alpha.4" } }, "sha512-UMduMbqO5s5zF2NkNacMT/yK5Y5QiKvWr2+50bzIIxFDwVJ2h49b+oyjaCGPhJxd2/gC2x39EHv/gHVuu36x2Q=="],
|
||||
|
||||
"@astrojs/node/@astrojs/internal-helpers": ["@astrojs/internal-helpers@0.11.0", "", { "dependencies": { "@types/hast": "^3.0.4", "@types/mdast": "^4.0.4", "js-yaml": "^4.3.0", "picomatch": "^4.0.4", "retext-smartypants": "^6.2.0", "shiki": "^4.0.2", "smol-toml": "^1.6.0", "unified": "^11.0.5" } }, "sha512-3rzxJ+xbo0+8YyqOzLziIN32wmsHdCjEVz2sGOpRxJ+Ben/KiLph4ItxBy1abEL+E8fkRzqjg0rfXmaHJGw9JA=="],
|
||||
|
||||
"@astrojs/react/@vitejs/plugin-react": ["@vitejs/plugin-react@5.2.0", "", { "dependencies": { "@babel/core": "^7.29.0", "@babel/plugin-transform-react-jsx-self": "^7.27.1", "@babel/plugin-transform-react-jsx-source": "^7.27.1", "@rolldown/pluginutils": "1.0.0-rc.3", "@types/babel__core": "^7.20.5", "react-refresh": "^0.18.0" }, "peerDependencies": { "vite": "^4.2.0 || ^5.0.0 || ^6.0.0 || ^7.0.0 || ^8.0.0" } }, "sha512-YmKkfhOAi3wsB1PhJq5Scj3GXMn3WvtQ/JC0xoopuHoXSdmtdStOpFrYaT1kie2YgFBcIe64ROzMYRjCrYOdYw=="],
|
||||
|
||||
"@astrojs/react/vite": ["vite@8.1.5", "", { "dependencies": { "lightningcss": "^1.32.0", "picomatch": "^4.0.5", "postcss": "^8.5.17", "rolldown": "~1.1.5", "tinyglobby": "^0.2.17" }, "optionalDependencies": { "fsevents": "~2.3.3" }, "peerDependencies": { "@types/node": "^20.19.0 || >=22.12.0", "@vitejs/devtools": "^0.3.0", "esbuild": "^0.27.0 || ^0.28.0", "jiti": ">=1.21.0", "less": "^4.0.0", "sass": "^1.70.0", "sass-embedded": "^1.70.0", "stylus": ">=0.54.8", "sugarss": "^5.0.0", "terser": "^5.16.0", "tsx": "^4.8.1", "yaml": "^2.4.2" }, "optionalPeers": ["@types/node", "@vitejs/devtools", "esbuild", "jiti", "less", "sass", "sass-embedded", "stylus", "sugarss", "terser", "tsx", "yaml"], "bin": { "vite": "bin/vite.js" } }, "sha512-7ULLwsCdYx/nRyrpiEwvqb5TFHrMVZyBt+rg/OAXT7rgj/z+DtTDyKFeLAdDkubDVDKD8jOsndmy7m55XcfUsw=="],
|
||||
|
||||
@@ -113,9 +113,9 @@
|
||||
url = "https://registry.npmjs.org/@astrojs/mdx/-/mdx-7.0.3.tgz";
|
||||
hash = "sha512-RxyIwU0uFam5ftwqKOjpIdhnFxZ/kEikeimLyQy3eGXbHT8WgRGzzesOIHVU8+m9TY8ag5WVOyvV24/GyqPdPQ==";
|
||||
};
|
||||
"@astrojs/node@11.0.2" = fetchurl {
|
||||
url = "https://registry.npmjs.org/@astrojs/node/-/node-11.0.2.tgz";
|
||||
hash = "sha512-/ijULxT+A5Cm8wSwWZ2vgqfim1b05D6B8n/a9l6MMA4FCotIH73g7fL7y76XojKXpTe75FVvQH92OxsMqea9kQ==";
|
||||
"@astrojs/node@11.1.5" = fetchurl {
|
||||
url = "https://registry.npmjs.org/@astrojs/node/-/node-11.1.5.tgz";
|
||||
hash = "sha512-CgA+LmG4UWqHxg/Vdc5MdxjjJNZSjKYQQx4uEUwnbj+1ZD2CYumJI3OWg2twsVdj2FlYsB5yvmCC54Re+OK+UQ==";
|
||||
};
|
||||
"@astrojs/prism@4.0.2" = fetchurl {
|
||||
url = "https://registry.npmjs.org/@astrojs/prism/-/prism-4.0.2.tgz";
|
||||
@@ -3205,9 +3205,9 @@
|
||||
url = "https://registry.npmjs.org/sisteransi/-/sisteransi-1.0.5.tgz";
|
||||
hash = "sha512-bLGGlR1QxBcynn2d5YmDX4MGjlZvy2MRBDRNHLJ8VI6l6+9FUiyTFNJ0IveOSP0bcXgVDPRcfGqA0pjaqUpfVg==";
|
||||
};
|
||||
"smol-toml@1.6.0" = fetchurl {
|
||||
url = "https://registry.npmjs.org/smol-toml/-/smol-toml-1.6.0.tgz";
|
||||
hash = "sha512-4zemZi0HvTnYwLfrpk/CF9LOd9Lt87kAt50GnqhMpyF9U3poDAP2+iukq2bZsO/ufegbYehBkqINbsWxj4l4cw==";
|
||||
"smol-toml@1.8.0" = fetchurl {
|
||||
url = "https://registry.npmjs.org/smol-toml/-/smol-toml-1.8.0.tgz";
|
||||
hash = "sha512-kCZr2V3ch9i00x8zXRhjUNVcjG9ijES5dDudkXvUVCT5QlJNQWElSJdZqyPemffHoLNUYwOcou0Fy+ojN0uHSQ==";
|
||||
};
|
||||
"sonner@2.0.7" = fetchurl {
|
||||
url = "https://registry.npmjs.org/sonner/-/sonner-2.0.7.tgz";
|
||||
|
||||
@@ -115,10 +115,15 @@ Access at http://localhost:4321, sign up (first user is admin), and configure ev
|
||||
|
||||
### Production Configuration
|
||||
|
||||
For production with custom domain and firewall:
|
||||
For production with custom domain and firewall. Like every snippet on this
|
||||
page, it needs the module imported first; `services.gitea-mirror` does not
|
||||
exist until `gitea-mirror.nixosModules.default` is in your modules list.
|
||||
|
||||
```nix
|
||||
{ inputs, ... }:
|
||||
{
|
||||
imports = [ inputs.gitea-mirror.nixosModules.default ];
|
||||
|
||||
services.gitea-mirror = {
|
||||
enable = true;
|
||||
host = "0.0.0.0";
|
||||
|
||||
@@ -2,8 +2,8 @@ apiVersion: v2
|
||||
name: gitea-mirror
|
||||
description: Kubernetes helm chart for gitea-mirror
|
||||
type: application
|
||||
version: 3.35.1
|
||||
appVersion: "3.35.1"
|
||||
version: 3.36.2
|
||||
appVersion: "3.36.2"
|
||||
icon: https://github.com/RayLabsHQ/gitea-mirror/blob/main/.github/assets/logo.png
|
||||
keywords:
|
||||
- git
|
||||
|
||||
+3
-2
@@ -1,7 +1,7 @@
|
||||
{
|
||||
"name": "gitea-mirror",
|
||||
"type": "module",
|
||||
"version": "3.35.1",
|
||||
"version": "3.36.2",
|
||||
"engines": {
|
||||
"bun": ">=1.2.9"
|
||||
},
|
||||
@@ -51,6 +51,7 @@
|
||||
"fast-uri": "^3.1.5",
|
||||
"fast-xml-builder": "^1.1.7",
|
||||
"js-yaml": "^4.3.2",
|
||||
"smol-toml": "^1.7.1",
|
||||
"kysely": "^0.28.17",
|
||||
"lodash": "^4.18.1",
|
||||
"picomatch": "^4.0.4",
|
||||
@@ -59,7 +60,7 @@
|
||||
},
|
||||
"dependencies": {
|
||||
"@astrojs/mdx": "^7.0.3",
|
||||
"@astrojs/node": "^11.0.2",
|
||||
"@astrojs/node": "^11.1.5",
|
||||
"@astrojs/react": "^6.0.1",
|
||||
"@better-auth/api-key": "1.7.2",
|
||||
"@better-auth/oauth-provider": "1.7.2",
|
||||
|
||||
@@ -68,6 +68,24 @@ const retentionPeriods = [
|
||||
{ label: "3 months", value: 7776000 },
|
||||
];
|
||||
|
||||
/** "every 8 hours", "every 30 minutes", "every 2 days" for a plain interval. */
|
||||
function formatIntervalSeconds(interval: number | string | undefined): string {
|
||||
const seconds = typeof interval === "number" ? interval : Number(interval);
|
||||
if (!Number.isFinite(seconds) || seconds <= 0) return "on a fixed interval";
|
||||
const units: [number, string][] = [
|
||||
[86400, "day"],
|
||||
[3600, "hour"],
|
||||
[60, "minute"],
|
||||
];
|
||||
for (const [size, name] of units) {
|
||||
if (seconds % size === 0 || seconds >= size * 2) {
|
||||
const count = Math.round(seconds / size);
|
||||
return `every ${count === 1 ? name : `${count} ${name}s`}`;
|
||||
}
|
||||
}
|
||||
return `every ${seconds} seconds`;
|
||||
}
|
||||
|
||||
function getCleanupInterval(retentionSeconds: number): number {
|
||||
const days = retentionSeconds / 86400;
|
||||
if (days <= 1) return 21600; // 6 hours
|
||||
@@ -126,16 +144,22 @@ export function AutomationSettings({
|
||||
? Intl.DateTimeFormat().resolvedOptions().timeZone || "UTC"
|
||||
: "UTC";
|
||||
|
||||
// Use saved timezone, but treat "UTC" as unset for users who never chose
|
||||
// it: older versions stored UTC as a default without asking. Anyone truly
|
||||
// in UTC gets the same result via their browser timezone.
|
||||
const effectiveTimezone =
|
||||
scheduleConfig.timezone && scheduleConfig.timezone !== "UTC"
|
||||
? scheduleConfig.timezone
|
||||
: browserTimezone;
|
||||
// The timezone the scheduler really uses. It used to be shown as the
|
||||
// browser timezone whenever UTC was stored, which made the card promise a
|
||||
// local 22:00 while the sync ran at 22:00 UTC (#427). The chip below offers
|
||||
// a one-click switch to the browser timezone when the two differ.
|
||||
const effectiveTimezone = scheduleConfig.timezone || browserTimezone;
|
||||
const timezoneDiffers = effectiveTimezone !== browserTimezone;
|
||||
|
||||
// The stored schedule is a plain interval (seconds or a duration such as
|
||||
// 8h) rather than a clock schedule. That is what SCHEDULE_INTERVAL and
|
||||
// GITEA_MIRROR_INTERVAL produce, and what older versions stored. The card
|
||||
// only edits clock schedules, so say what actually runs instead of showing
|
||||
// the 22:00 placeholder as if it were saved (#427).
|
||||
const isIntervalSchedule = scheduleConfig.scheduleMode === "interval";
|
||||
|
||||
const nextScheduledRun = useMemo(() => {
|
||||
if (!scheduleConfig.enabled) return null;
|
||||
if (!scheduleConfig.enabled || isIntervalSchedule) return null;
|
||||
const startTime = scheduleConfig.startTime || "22:00";
|
||||
const frequencyHours = scheduleConfig.clockFrequencyHours || 24;
|
||||
const cronExpression = buildClockCronExpression(startTime, frequencyHours);
|
||||
@@ -145,7 +169,16 @@ export function AutomationSettings({
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}, [scheduleConfig.enabled, scheduleConfig.startTime, scheduleConfig.clockFrequencyHours, effectiveTimezone]);
|
||||
}, [scheduleConfig.enabled, isIntervalSchedule, scheduleConfig.startTime, scheduleConfig.clockFrequencyHours, effectiveTimezone]);
|
||||
|
||||
const switchToBrowserTimezone = () =>
|
||||
onScheduleChange({
|
||||
...scheduleConfig,
|
||||
scheduleMode: "clock",
|
||||
startTime: scheduleConfig.startTime || "22:00",
|
||||
clockFrequencyHours: scheduleConfig.clockFrequencyHours || 24,
|
||||
timezone: browserTimezone,
|
||||
});
|
||||
|
||||
// Update nextRun for cleanup when settings change
|
||||
useEffect(() => {
|
||||
@@ -217,15 +250,39 @@ export function AutomationSettings({
|
||||
<CardSection>
|
||||
<SectionTitle
|
||||
action={
|
||||
<span className="inline-flex items-center gap-1.5 rounded-full bg-muted px-2.5 py-1 text-[11px] text-muted-foreground">
|
||||
<Globe className="h-3 w-3" />
|
||||
{effectiveTimezone}
|
||||
</span>
|
||||
timezoneDiffers ? (
|
||||
<button
|
||||
type="button"
|
||||
onClick={switchToBrowserTimezone}
|
||||
title={`Switch the schedule to ${browserTimezone}`}
|
||||
className="inline-flex items-center gap-1.5 rounded-full bg-muted px-2.5 py-1 text-[11px] text-muted-foreground hover:bg-muted/70 hover:text-foreground"
|
||||
>
|
||||
<Globe className="h-3 w-3" />
|
||||
{effectiveTimezone}
|
||||
<span className="text-muted-foreground/60">· use {browserTimezone}</span>
|
||||
</button>
|
||||
) : (
|
||||
<span className="inline-flex items-center gap-1.5 rounded-full bg-muted px-2.5 py-1 text-[11px] text-muted-foreground">
|
||||
<Globe className="h-3 w-3" />
|
||||
{effectiveTimezone}
|
||||
</span>
|
||||
)
|
||||
}
|
||||
>
|
||||
Schedule
|
||||
</SectionTitle>
|
||||
|
||||
{isIntervalSchedule && (
|
||||
<p className="flex items-start gap-2 rounded-md border border-dashed border-border px-3 py-2 text-xs text-muted-foreground">
|
||||
<Info className="mt-0.5 h-3.5 w-3.5 shrink-0" />
|
||||
<span>
|
||||
Syncing {formatIntervalSeconds(scheduleConfig.interval)}, counted from the last run.
|
||||
{" "}This interval comes from SCHEDULE_INTERVAL or GITEA_MIRROR_INTERVAL, which is applied again on every restart while it is set, or from an older version.
|
||||
{" "}Pick a frequency or start time to switch to a clock schedule.
|
||||
</span>
|
||||
</p>
|
||||
)}
|
||||
|
||||
<div className="grid gap-3 sm:grid-cols-2">
|
||||
<div className="space-y-1.5">
|
||||
<Label
|
||||
@@ -235,7 +292,7 @@ export function AutomationSettings({
|
||||
Frequency
|
||||
</Label>
|
||||
<Select
|
||||
value={String(scheduleConfig.clockFrequencyHours || 24)}
|
||||
value={isIntervalSchedule ? "" : String(scheduleConfig.clockFrequencyHours || 24)}
|
||||
onValueChange={(value) =>
|
||||
onScheduleChange({
|
||||
...scheduleConfig,
|
||||
@@ -247,7 +304,7 @@ export function AutomationSettings({
|
||||
}
|
||||
>
|
||||
<SelectTrigger id="clock-frequency" className="w-full">
|
||||
<SelectValue />
|
||||
<SelectValue placeholder="Choose a frequency" />
|
||||
</SelectTrigger>
|
||||
<SelectContent>
|
||||
{clockFrequencies.map((option) => (
|
||||
@@ -276,7 +333,7 @@ export function AutomationSettings({
|
||||
<Input
|
||||
id="clock-start-time"
|
||||
type="time"
|
||||
value={scheduleConfig.startTime || "22:00"}
|
||||
value={isIntervalSchedule ? "" : scheduleConfig.startTime || "22:00"}
|
||||
onChange={(event) =>
|
||||
onScheduleChange({
|
||||
...scheduleConfig,
|
||||
|
||||
@@ -222,21 +222,33 @@ export function Organization() {
|
||||
|
||||
const newStatus = ignore ? "ignored" : "imported";
|
||||
|
||||
const response = await apiRequest<{ success: boolean; organization?: Organization; error?: string }>(
|
||||
// cascade: the organization's repositories follow it, so ignoring an
|
||||
// organization actually stops its mirrors from syncing (#429).
|
||||
const response = await apiRequest<{
|
||||
success: boolean;
|
||||
organization?: Organization;
|
||||
repositoriesChanged?: number;
|
||||
error?: string;
|
||||
}>(
|
||||
`/organizations/${orgId}/status`,
|
||||
{
|
||||
method: "PATCH",
|
||||
data: {
|
||||
status: newStatus,
|
||||
userId: user.id
|
||||
userId: user.id,
|
||||
cascade: true,
|
||||
},
|
||||
}
|
||||
);
|
||||
|
||||
if (response.success) {
|
||||
const changed = response.repositoriesChanged ?? 0;
|
||||
const repoNote = changed > 0
|
||||
? ` (${changed} ${changed === 1 ? "repository" : "repositories"} ${ignore ? "ignored too" : "restored"})`
|
||||
: "";
|
||||
toast.success(ignore
|
||||
? `Organization will be ignored in future operations`
|
||||
: `Organization included for mirroring`
|
||||
? `Organization will be ignored in future operations${repoNote}`
|
||||
: `Organization included for mirroring${repoNote}`
|
||||
);
|
||||
|
||||
// Update local state
|
||||
|
||||
@@ -3,7 +3,6 @@ import { Card } from "@/components/ui/card";
|
||||
import { Button } from "@/components/ui/button";
|
||||
import { Badge } from "@/components/ui/badge";
|
||||
import { Plus, RefreshCw, Building2, Check, AlertCircle, Clock, MoreVertical, Ban, SlidersHorizontal, Trash2, Layers } from "lucide-react";
|
||||
import { SiGitea } from "react-icons/si";
|
||||
import { toast } from "sonner";
|
||||
import type { MirrorOverrides, Organization } from "@/lib/db/schema";
|
||||
import type { FilterParams } from "@/types/filter";
|
||||
@@ -13,7 +12,7 @@ import { Skeleton } from "@/components/ui/skeleton";
|
||||
import { cn } from "@/lib/utils";
|
||||
import { buildGiteaWebUrl } from "@/lib/gitea-url";
|
||||
import { MirrorDestinationEditor } from "./MirrorDestinationEditor";
|
||||
import { destinationInfo } from "@/components/destination/DestinationIcon";
|
||||
import { DestinationIcon, destinationInfo } from "@/components/destination/DestinationIcon";
|
||||
import type { OrganizationMoveResult } from "@/lib/destination-transfer";
|
||||
import { MirrorOverridesDialog } from "@/components/config/MirrorOverridesDialog";
|
||||
import { hasMirrorOverrides, mirrorOptionsToFlags } from "@/lib/utils/mirror-overrides";
|
||||
@@ -272,6 +271,17 @@ export function OrganizationList({
|
||||
// GitHub and GitLab destinations are pushed to; only Gitea and Forgejo can transfer a repository.
|
||||
const destination = destinationInfo(giteaConfig);
|
||||
|
||||
// Tooltip for the destination link on a card. Named after the configured
|
||||
// destination, so a Forgejo user is not told to view the org on Gitea (#430).
|
||||
const destinationLinkTooltip = (org: Organization, orgUrl: string | null): string => {
|
||||
if (!giteaConfig?.url) return `${destination.label} not configured`;
|
||||
if (org.status === "imported") return `Organization not yet mirrored to ${destination.label}`;
|
||||
if (org.status === "failed") return "Organization mirroring failed";
|
||||
if (org.status === "mirroring") return `Organization is being mirrored to ${destination.label}`;
|
||||
if (orgUrl) return `View on ${destination.label}`;
|
||||
return `${destination.label} organization not available`;
|
||||
};
|
||||
|
||||
// Where an organization's repositories land with no override, which is what
|
||||
// the mirror strategy produces for organization repositories (see
|
||||
// getGiteaRepoOwner in src/lib/gitea.ts). Only "preserve" and "mixed" make
|
||||
@@ -749,21 +759,7 @@ export function OrganizationList({
|
||||
{(() => {
|
||||
const giteaUrl = getGiteaOrgUrl(org);
|
||||
|
||||
// Determine tooltip based on status and configuration
|
||||
let tooltip: string;
|
||||
if (!giteaConfig?.url) {
|
||||
tooltip = "Gitea not configured";
|
||||
} else if (org.status === 'imported') {
|
||||
tooltip = "Organization not yet mirrored to Gitea";
|
||||
} else if (org.status === 'failed') {
|
||||
tooltip = "Organization mirroring failed";
|
||||
} else if (org.status === 'mirroring') {
|
||||
tooltip = "Organization is being mirrored to Gitea";
|
||||
} else if (giteaUrl) {
|
||||
tooltip = "View on Gitea";
|
||||
} else {
|
||||
tooltip = "Gitea organization not available";
|
||||
}
|
||||
const tooltip = destinationLinkTooltip(org, giteaUrl);
|
||||
|
||||
return giteaUrl ? (
|
||||
<Button variant="outline" size="default" asChild className="flex-1 h-10 min-w-0">
|
||||
@@ -774,14 +770,14 @@ export function OrganizationList({
|
||||
title={tooltip}
|
||||
className="flex items-center justify-center gap-2"
|
||||
>
|
||||
<SiGitea className="h-4 w-4 flex-shrink-0" />
|
||||
<span className="text-xs">Gitea</span>
|
||||
<DestinationIcon provider={destination.provider} className="h-4 w-4 flex-shrink-0" />
|
||||
<span className="text-xs">{destination.label}</span>
|
||||
</a>
|
||||
</Button>
|
||||
) : (
|
||||
<Button variant="outline" size="default" disabled title={tooltip} className="flex-1 h-10">
|
||||
<SiGitea className="h-4 w-4" />
|
||||
<span className="text-xs ml-2">Gitea</span>
|
||||
<DestinationIcon provider={destination.provider} className="h-4 w-4" />
|
||||
<span className="text-xs ml-2">{destination.label}</span>
|
||||
</Button>
|
||||
);
|
||||
})()}
|
||||
@@ -914,21 +910,7 @@ export function OrganizationList({
|
||||
{(() => {
|
||||
const giteaUrl = getGiteaOrgUrl(org);
|
||||
|
||||
// Determine tooltip based on status and configuration
|
||||
let tooltip: string;
|
||||
if (!giteaConfig?.url) {
|
||||
tooltip = "Gitea not configured";
|
||||
} else if (org.status === 'imported') {
|
||||
tooltip = "Organization not yet mirrored to Gitea";
|
||||
} else if (org.status === 'failed') {
|
||||
tooltip = "Organization mirroring failed";
|
||||
} else if (org.status === 'mirroring') {
|
||||
tooltip = "Organization is being mirrored to Gitea";
|
||||
} else if (giteaUrl) {
|
||||
tooltip = "View on Gitea";
|
||||
} else {
|
||||
tooltip = "Gitea organization not available";
|
||||
}
|
||||
const tooltip = destinationLinkTooltip(org, giteaUrl);
|
||||
|
||||
return (
|
||||
<div className="flex items-center border rounded-md">
|
||||
@@ -946,13 +928,13 @@ export function OrganizationList({
|
||||
target="_blank"
|
||||
rel="noopener noreferrer"
|
||||
>
|
||||
<SiGitea className="h-4 w-4 mr-2" />
|
||||
Gitea
|
||||
<DestinationIcon provider={destination.provider} className="h-4 w-4 mr-2" />
|
||||
{destination.label}
|
||||
</a>
|
||||
) : (
|
||||
<>
|
||||
<SiGitea className="h-4 w-4 mr-2" />
|
||||
Gitea
|
||||
<DestinationIcon provider={destination.provider} className="h-4 w-4 mr-2" />
|
||||
{destination.label}
|
||||
</>
|
||||
)}
|
||||
</Button>
|
||||
|
||||
@@ -25,6 +25,9 @@ let mockShouldBlockSyncOnBackupFailure = true;
|
||||
// dbUpdateSetCalls so tests can assert on what got written (e.g. archived
|
||||
// repos keeping status "archived" after a Manual Sync).
|
||||
const dbUpdateSetCalls: any[] = [];
|
||||
// Rows the sync's atomic status claim gets back (#417): one row means this run
|
||||
// claimed the repository, an empty array means another mirror or sync owns it.
|
||||
let mockClaimRows: Array<{ id: string }> = [{ id: "repo-1" }];
|
||||
const mockDb = {
|
||||
insert: mock((table: any) => ({
|
||||
values: mock((data: any) => Promise.resolve({ insertedId: "mock-id" }))
|
||||
@@ -32,7 +35,13 @@ const mockDb = {
|
||||
update: mock(() => ({
|
||||
set: mock((data: any) => {
|
||||
dbUpdateSetCalls.push(data);
|
||||
return { where: mock(() => Promise.resolve()) };
|
||||
return {
|
||||
where: mock(() => {
|
||||
const result: any = Promise.resolve();
|
||||
result.returning = mock(() => Promise.resolve(mockClaimRows));
|
||||
return result;
|
||||
})
|
||||
};
|
||||
})
|
||||
}))
|
||||
};
|
||||
@@ -453,6 +462,7 @@ describe("Enhanced Gitea Operations", () => {
|
||||
mockHttpDelete.mockClear();
|
||||
mockHttpPatch.mockClear();
|
||||
dbUpdateSetCalls.length = 0;
|
||||
mockClaimRows = [{ id: "repo-1" }];
|
||||
mockCreatePreSyncBundleBackup.mockClear();
|
||||
mockCreatePreSyncBundleBackup.mockImplementation(() =>
|
||||
Promise.resolve({ bundlePath: "/tmp/mock.bundle" })
|
||||
@@ -671,6 +681,118 @@ describe("Enhanced Gitea Operations", () => {
|
||||
expect(mockMirrorGitHubReleasesToGitea).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
test("refuses to start when another run already owns the repository (#417)", async () => {
|
||||
// Cast: the fixture only sets the fields this path reads.
|
||||
const config = {
|
||||
userId: "user123",
|
||||
githubConfig: {
|
||||
username: "testuser",
|
||||
token: "github-token",
|
||||
privateRepositories: false,
|
||||
mirrorStarred: true,
|
||||
},
|
||||
giteaConfig: {
|
||||
url: "https://gitea.example.com",
|
||||
token: "encrypted-token",
|
||||
defaultOwner: "testuser",
|
||||
mirrorReleases: true,
|
||||
},
|
||||
} as unknown as Partial<Config>;
|
||||
|
||||
const repository = {
|
||||
id: "repo456",
|
||||
name: "mirror-repo",
|
||||
fullName: "user/mirror-repo",
|
||||
owner: "user",
|
||||
cloneUrl: "https://github.com/user/mirror-repo.git",
|
||||
isPrivate: false,
|
||||
isStarred: true,
|
||||
// The row is already "syncing" from another run; the claim loses.
|
||||
status: repoStatusEnum.parse("syncing"),
|
||||
visibility: "public",
|
||||
userId: "user123",
|
||||
createdAt: new Date(),
|
||||
updatedAt: new Date(),
|
||||
} as unknown as Repository;
|
||||
|
||||
// The conditional UPDATE matches no row: another mirror or sync owns it.
|
||||
mockClaimRows = [];
|
||||
|
||||
const result = await syncGiteaRepoEnhanced(
|
||||
{ config, repository },
|
||||
{
|
||||
getGiteaRepoOwnerAsync: mockGetGiteaRepoOwnerAsync,
|
||||
mirrorGitHubReleasesToGitea: mockMirrorGitHubReleasesToGitea,
|
||||
mirrorGitRepoIssuesToGitea: mockMirrorGitRepoIssuesToGitea,
|
||||
mirrorGitRepoPullRequestsToGitea: mockMirrorGitRepoPullRequestsToGitea,
|
||||
mirrorGitRepoLabelsToGitea: mockMirrorGitRepoLabelsToGitea,
|
||||
mirrorGitRepoMilestonesToGitea: mockMirrorGitRepoMilestonesToGitea,
|
||||
syncRepositoryMetadataToGitea: mockSyncRepositoryMetadataToGitea,
|
||||
}
|
||||
);
|
||||
|
||||
expect(result).toEqual({ skipped: true, reason: "already-in-progress" });
|
||||
// Nothing ran: no release or metadata mirroring, and the row was not
|
||||
// marked failed (it belongs to the run that owns it).
|
||||
expect(mockMirrorGitHubReleasesToGitea).not.toHaveBeenCalled();
|
||||
expect(mockSyncRepositoryMetadataToGitea).not.toHaveBeenCalled();
|
||||
expect(dbUpdateSetCalls.some((call) => call.status === "failed")).toBe(false);
|
||||
});
|
||||
|
||||
test("syncs when the caller already claimed the row (approve-sync)", async () => {
|
||||
// Cast: the fixture only sets the fields this path reads.
|
||||
const config = {
|
||||
userId: "user123",
|
||||
githubConfig: {
|
||||
username: "testuser",
|
||||
token: "github-token",
|
||||
privateRepositories: false,
|
||||
mirrorStarred: true,
|
||||
},
|
||||
giteaConfig: {
|
||||
url: "https://gitea.example.com",
|
||||
token: "encrypted-token",
|
||||
defaultOwner: "testuser",
|
||||
mirrorReleases: true,
|
||||
},
|
||||
} as unknown as Partial<Config>;
|
||||
|
||||
const repository = {
|
||||
id: "repo456",
|
||||
name: "mirror-repo",
|
||||
fullName: "user/mirror-repo",
|
||||
owner: "user",
|
||||
cloneUrl: "https://github.com/user/mirror-repo.git",
|
||||
isPrivate: false,
|
||||
isStarred: true,
|
||||
// approve-sync has already moved the row to "syncing" itself.
|
||||
status: repoStatusEnum.parse("syncing"),
|
||||
visibility: "public",
|
||||
userId: "user123",
|
||||
createdAt: new Date(),
|
||||
updatedAt: new Date(),
|
||||
} as unknown as Repository;
|
||||
|
||||
// Would refuse the claim, but alreadyClaimed skips it entirely.
|
||||
mockClaimRows = [];
|
||||
|
||||
const result = await syncGiteaRepoEnhanced(
|
||||
{ config, repository, alreadyClaimed: true },
|
||||
{
|
||||
getGiteaRepoOwnerAsync: mockGetGiteaRepoOwnerAsync,
|
||||
mirrorGitHubReleasesToGitea: mockMirrorGitHubReleasesToGitea,
|
||||
mirrorGitRepoIssuesToGitea: mockMirrorGitRepoIssuesToGitea,
|
||||
mirrorGitRepoPullRequestsToGitea: mockMirrorGitRepoPullRequestsToGitea,
|
||||
mirrorGitRepoLabelsToGitea: mockMirrorGitRepoLabelsToGitea,
|
||||
mirrorGitRepoMilestonesToGitea: mockMirrorGitRepoMilestonesToGitea,
|
||||
syncRepositoryMetadataToGitea: mockSyncRepositoryMetadataToGitea,
|
||||
}
|
||||
);
|
||||
|
||||
expect(result).toEqual({ success: true });
|
||||
expect(mockMirrorGitHubReleasesToGitea).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
test("should successfully sync a mirror repository", async () => {
|
||||
const config: Partial<Config> = {
|
||||
userId: "user123",
|
||||
|
||||
@@ -24,6 +24,7 @@ import { httpPost, httpGet, httpPatch, HttpError } from "./http-client";
|
||||
import { db, repositories } from "./db";
|
||||
import { eq } from "drizzle-orm";
|
||||
import { repoStatusEnum } from "@/types/Repository";
|
||||
import { claimRepositoryForInFlightWork } from "./utils/repo-status-claim";
|
||||
import { resolveMirrorOptionsForRepository } from "./utils/mirror-overrides";
|
||||
import {
|
||||
createPreSyncBundleBackup,
|
||||
@@ -444,11 +445,19 @@ export async function syncGiteaRepoEnhanced({
|
||||
config,
|
||||
repository,
|
||||
skipForcePushDetection,
|
||||
alreadyClaimed,
|
||||
}: {
|
||||
config: Partial<Config>;
|
||||
repository: Repository;
|
||||
/** When true, skip force-push detection and blocking (used by approve-sync). */
|
||||
skipForcePushDetection?: boolean;
|
||||
/**
|
||||
* Set by a caller that already claimed this row for the sync with its own
|
||||
* conditional UPDATE (approve-sync does, so the UI sees "syncing" right
|
||||
* away). Skips the claim below instead of refusing to run on the caller's
|
||||
* own "syncing" status.
|
||||
*/
|
||||
alreadyClaimed?: boolean;
|
||||
}, deps?: SyncDependencies): Promise<any> {
|
||||
try {
|
||||
if (!config.userId || !config.giteaConfig?.url || !config.giteaConfig?.token) {
|
||||
@@ -472,14 +481,30 @@ export async function syncGiteaRepoEnhanced({
|
||||
|
||||
console.log(`[Sync] Starting sync for repository ${repository.name}`);
|
||||
|
||||
// Mark repo as "syncing" in DB
|
||||
await db
|
||||
.update(repositories)
|
||||
.set({
|
||||
status: repoStatusEnum.parse("syncing"),
|
||||
updatedAt: new Date(),
|
||||
})
|
||||
.where(eq(repositories.id, repository.id!));
|
||||
// Mark repo as "syncing" in DB, but only while no other run owns the row.
|
||||
// This used to be an unconditional write: a sync would start on a
|
||||
// repository that was already mirroring or syncing, and the two passes
|
||||
// reconciled the same releases against the same destination at the same
|
||||
// time, which duplicated release assets (#417). A row left in an in-flight
|
||||
// status by a crash is taken over once it goes stale (two hours), and
|
||||
// resetStuckMirrorStatuses resets those rows to "failed" as well, so
|
||||
// nothing is locked out permanently.
|
||||
if (!alreadyClaimed) {
|
||||
const claimed = await claimRepositoryForInFlightWork({
|
||||
repositoryId: repository.id!,
|
||||
set: {
|
||||
status: repoStatusEnum.parse("syncing"),
|
||||
updatedAt: new Date(),
|
||||
},
|
||||
});
|
||||
|
||||
if (!claimed) {
|
||||
console.log(
|
||||
`[Sync] Skipping ${repository.name}: it is already being mirrored or synced by another run.`
|
||||
);
|
||||
return { skipped: true, reason: "already-in-progress" };
|
||||
}
|
||||
}
|
||||
|
||||
// Resolve sync target in a backward-compatible order:
|
||||
// 1) recorded mirroredLocation (actual historical mirror location)
|
||||
|
||||
@@ -113,7 +113,16 @@ mock.module("@/lib/db", () => {
|
||||
}),
|
||||
}),
|
||||
update: (_table: any) => ({
|
||||
set: (_data: any) => ({ where: (_cond: any) => Promise.resolve() }),
|
||||
set: (_data: any) => ({
|
||||
where: (_cond: any) => {
|
||||
// The mirror path claims the row with a conditional UPDATE and reads
|
||||
// the returned rows to find out whether it won (#417); one row means
|
||||
// claimed.
|
||||
const result: any = Promise.resolve();
|
||||
result.returning = () => Promise.resolve([{ id: "repo-1" }]);
|
||||
return result;
|
||||
},
|
||||
}),
|
||||
}),
|
||||
insert: (_table: any) => ({ values: (_data: any) => Promise.resolve() }),
|
||||
delete: (_table: any) => ({ where: (_cond: any) => Promise.resolve() }),
|
||||
|
||||
@@ -0,0 +1,320 @@
|
||||
/**
|
||||
* Behavioral tests for release asset reconciliation, regression for #417.
|
||||
*
|
||||
* Reported symptom: release assets mirrored to Gitea/Forgejo appear twice (some
|
||||
* three times, one five times), with identical sizes and identical creation
|
||||
* times, so the copies came from passes that overlapped inside one run.
|
||||
*
|
||||
* Cause: reconcileReleaseAssets listed the attachments a release already had,
|
||||
* then downloaded and uploaded everything it judged missing. The destination
|
||||
* has no name-collision check on attachment upload (Gitea's and Forgejo's
|
||||
* CreateReleaseAttachment appends a row for every POST), so two overlapping
|
||||
* passes both read "missing" and both uploaded. A failed listing was read as
|
||||
* "the release has no assets", and a failed delete was followed by the upload
|
||||
* anyway; both added copies too.
|
||||
*
|
||||
* The fake destination below models the real behavior that makes duplicates
|
||||
* possible: every POST appends another attachment with the same name.
|
||||
*
|
||||
* No module mocks here on purpose (bun's mock.module is process-wide and leaks
|
||||
* into other test files): reconcileReleaseAssets takes the fetch it should use.
|
||||
*/
|
||||
|
||||
import { describe, expect, it } from "bun:test";
|
||||
import { reconcileReleaseAssets, buildReleaseTargetLockKey } from "@/lib/gitea";
|
||||
import { withKeyedLock } from "@/lib/utils/keyed-mutex";
|
||||
import type { Config } from "@/types/config";
|
||||
|
||||
const GITEA_URL = "https://gitea.example.com";
|
||||
const OWNER = "mirror-owner";
|
||||
const REPO = "demo";
|
||||
const RELEASE_ID = 77;
|
||||
const ASSETS_URL = `${GITEA_URL}/api/v1/repos/${OWNER}/${REPO}/releases/${RELEASE_ID}/assets`;
|
||||
|
||||
const config = {
|
||||
giteaConfig: { url: GITEA_URL, token: "gitea-token", defaultOwner: OWNER },
|
||||
} as unknown as Partial<Config>;
|
||||
|
||||
const decryptedConfig = {
|
||||
giteaConfig: { url: GITEA_URL, token: "gitea-token", defaultOwner: OWNER },
|
||||
githubConfig: { token: "github-token", username: "upstream" },
|
||||
} as unknown as Config;
|
||||
|
||||
type Attachment = { id: number; name: string; size: number };
|
||||
|
||||
interface FakeDestinationOptions {
|
||||
/** Attachments the release already carries. */
|
||||
existing?: Attachment[];
|
||||
/** Status for the "list attachments" GET. */
|
||||
listStatus?: number;
|
||||
/** Status for attachment deletes. */
|
||||
deleteStatus?: number;
|
||||
/** Status for attachment uploads. */
|
||||
uploadStatus?: number;
|
||||
}
|
||||
|
||||
/** A GitHub asset plus the destination-side size the fake records on upload. */
|
||||
function githubAsset(name: string, size: number) {
|
||||
return {
|
||||
name,
|
||||
size,
|
||||
browser_download_url: `https://github.com/upstream/demo/releases/download/v1.0.0/${name}`,
|
||||
};
|
||||
}
|
||||
|
||||
function createFakeDestination(options: FakeDestinationOptions = {}) {
|
||||
const attachments: Attachment[] = [...(options.existing ?? [])];
|
||||
const requests: Array<{ method: string; url: string }> = [];
|
||||
let nextId = 500;
|
||||
|
||||
const fetchImpl = (async (input: any, init?: any) => {
|
||||
const url = typeof input === "string" ? input : String(input);
|
||||
const method = String(init?.method ?? "GET").toUpperCase();
|
||||
requests.push({ method, url });
|
||||
|
||||
// Every request yields the event loop, so a second pass running at the
|
||||
// same time gets to interleave, which is what created the duplicates.
|
||||
await new Promise((resolve) => setTimeout(resolve, 0));
|
||||
|
||||
// Downloading the asset from the source host.
|
||||
if (url.startsWith("https://github.com/")) {
|
||||
return new Response(new Uint8Array([1, 2, 3, 4]), { status: 200 });
|
||||
}
|
||||
|
||||
if (method === "GET" && url === ASSETS_URL) {
|
||||
const status = options.listStatus ?? 200;
|
||||
if (status !== 200) {
|
||||
return new Response("boom", { status, statusText: "Internal Server Error" });
|
||||
}
|
||||
return new Response(JSON.stringify(attachments), {
|
||||
status: 200,
|
||||
headers: { "content-type": "application/json" },
|
||||
});
|
||||
}
|
||||
|
||||
if (method === "DELETE" && url.startsWith(`${ASSETS_URL}/`)) {
|
||||
const status = options.deleteStatus ?? 204;
|
||||
if (status >= 400) {
|
||||
return new Response("nope", { status, statusText: "Internal Server Error" });
|
||||
}
|
||||
const id = Number(url.slice(`${ASSETS_URL}/`.length));
|
||||
const index = attachments.findIndex((a) => a.id === id);
|
||||
if (index >= 0) attachments.splice(index, 1);
|
||||
return new Response(null, { status });
|
||||
}
|
||||
|
||||
if (method === "POST" && url.startsWith(`${ASSETS_URL}?`)) {
|
||||
const status = options.uploadStatus ?? 201;
|
||||
if (status >= 400) {
|
||||
return new Response("rejected", { status, statusText: "Bad Request" });
|
||||
}
|
||||
const name = decodeURIComponent(new URL(url).searchParams.get("name") ?? "");
|
||||
// The real destination does no name-collision check: every POST appends
|
||||
// another attachment, even when one with that name is already there.
|
||||
attachments.push({ id: nextId++, name, size: uploadSizes.get(name) ?? 4 });
|
||||
return new Response(JSON.stringify({ id: nextId }), {
|
||||
status,
|
||||
headers: { "content-type": "application/json" },
|
||||
});
|
||||
}
|
||||
|
||||
throw new Error(`unexpected request: ${method} ${url}`);
|
||||
}) as unknown as typeof fetch;
|
||||
|
||||
const uploadSizes = new Map<string, number>();
|
||||
|
||||
return {
|
||||
fetchImpl,
|
||||
attachments,
|
||||
requests,
|
||||
/** Size the destination records for an uploaded asset (defaults to 4). */
|
||||
setUploadedSize(name: string, size: number) {
|
||||
uploadSizes.set(name, size);
|
||||
},
|
||||
posts: () => requests.filter((r) => r.method === "POST"),
|
||||
deletes: () => requests.filter((r) => r.method === "DELETE"),
|
||||
};
|
||||
}
|
||||
|
||||
function reconcile(
|
||||
fetchImpl: typeof fetch,
|
||||
githubAssets: Array<{ name: string; size: number; browser_download_url: string }>
|
||||
) {
|
||||
return reconcileReleaseAssets({
|
||||
config,
|
||||
decryptedConfig,
|
||||
repoOwner: OWNER,
|
||||
repoName: REPO,
|
||||
giteaReleaseId: RELEASE_ID,
|
||||
githubAssets,
|
||||
tagName: "v1.0.0",
|
||||
fetchImpl,
|
||||
});
|
||||
}
|
||||
|
||||
describe("reconcileReleaseAssets", () => {
|
||||
it("uploads every asset once on a release that has none", async () => {
|
||||
const destination = createFakeDestination();
|
||||
destination.setUploadedSize("base.zip", 4);
|
||||
destination.setUploadedSize("extras.zip", 4);
|
||||
|
||||
const result = await reconcile(destination.fetchImpl, [
|
||||
githubAsset("base.zip", 4),
|
||||
githubAsset("extras.zip", 4),
|
||||
]);
|
||||
|
||||
expect(result).toEqual({ uploaded: 2, failed: 0, skipped: 0 });
|
||||
expect(destination.posts()).toHaveLength(2);
|
||||
expect(destination.attachments.map((a) => a.name).sort()).toEqual([
|
||||
"base.zip",
|
||||
"extras.zip",
|
||||
]);
|
||||
});
|
||||
|
||||
it("two overlapping passes under the destination lock upload each asset once (#417)", async () => {
|
||||
const destination = createFakeDestination();
|
||||
destination.setUploadedSize("base.zip", 4);
|
||||
const assets = [githubAsset("base.zip", 4)];
|
||||
const key = buildReleaseTargetLockKey(GITEA_URL, OWNER, REPO);
|
||||
|
||||
// Two runs of the same repository starting at the same time: the scheduler
|
||||
// and a manual sync, or two repository rows pointing at one destination.
|
||||
const [first, second] = await Promise.all([
|
||||
withKeyedLock(key, () => reconcile(destination.fetchImpl, assets)),
|
||||
withKeyedLock(key, () => reconcile(destination.fetchImpl, assets)),
|
||||
]);
|
||||
|
||||
expect(destination.posts()).toHaveLength(1);
|
||||
expect(destination.attachments).toHaveLength(1);
|
||||
expect(first).toEqual({ uploaded: 1, failed: 0, skipped: 0 });
|
||||
// The second pass finds the asset already there and leaves it alone.
|
||||
expect(second).toEqual({ uploaded: 0, failed: 0, skipped: 1 });
|
||||
});
|
||||
|
||||
it("without the lock, two overlapping passes are what duplicated the asset", async () => {
|
||||
const destination = createFakeDestination();
|
||||
destination.setUploadedSize("base.zip", 4);
|
||||
const assets = [githubAsset("base.zip", 4)];
|
||||
|
||||
await Promise.all([
|
||||
reconcile(destination.fetchImpl, assets),
|
||||
reconcile(destination.fetchImpl, assets),
|
||||
]);
|
||||
|
||||
// The fake reproduces the reported failure: identical name, identical size,
|
||||
// two attachments. This is the state the lock prevents.
|
||||
expect(destination.posts()).toHaveLength(2);
|
||||
expect(destination.attachments).toHaveLength(2);
|
||||
});
|
||||
|
||||
it("uploads nothing when the existing attachments cannot be listed", async () => {
|
||||
const destination = createFakeDestination({ listStatus: 500 });
|
||||
|
||||
const result = await reconcile(destination.fetchImpl, [
|
||||
githubAsset("base.zip", 4),
|
||||
githubAsset("extras.zip", 4),
|
||||
]);
|
||||
|
||||
// Fail closed: a failed listing used to read as "the release has no
|
||||
// assets", so the pass re-uploaded everything the release already had.
|
||||
expect(destination.posts()).toHaveLength(0);
|
||||
expect(destination.deletes()).toHaveLength(0);
|
||||
expect(result).toEqual({ uploaded: 0, failed: 2, skipped: 0 });
|
||||
});
|
||||
|
||||
it("does not upload an asset whose stale copy could not be deleted", async () => {
|
||||
const destination = createFakeDestination({
|
||||
existing: [{ id: 11, name: "firmware.bin", size: 1024 }],
|
||||
deleteStatus: 500,
|
||||
});
|
||||
|
||||
const result = await reconcile(destination.fetchImpl, [
|
||||
githubAsset("firmware.bin", 2048),
|
||||
]);
|
||||
|
||||
expect(destination.deletes()).toHaveLength(1);
|
||||
// Uploading after the failed delete would leave two copies.
|
||||
expect(destination.posts()).toHaveLength(0);
|
||||
expect(destination.attachments).toHaveLength(1);
|
||||
expect(result).toEqual({ uploaded: 0, failed: 1, skipped: 0 });
|
||||
});
|
||||
|
||||
it("cleans up an existing duplicate without uploading anything", async () => {
|
||||
const destination = createFakeDestination({
|
||||
existing: [
|
||||
{ id: 11, name: "base.zip", size: 4 },
|
||||
{ id: 12, name: "base.zip", size: 4 },
|
||||
],
|
||||
});
|
||||
|
||||
const result = await reconcile(destination.fetchImpl, [githubAsset("base.zip", 4)]);
|
||||
|
||||
expect(destination.deletes()).toHaveLength(1);
|
||||
expect(destination.deletes()[0].url).toBe(`${ASSETS_URL}/12`);
|
||||
expect(destination.posts()).toHaveLength(0);
|
||||
expect(destination.attachments).toEqual([{ id: 11, name: "base.zip", size: 4 }]);
|
||||
expect(result).toEqual({ uploaded: 0, failed: 0, skipped: 1 });
|
||||
});
|
||||
|
||||
it("replaces stale copies: deletes both, then uploads once", async () => {
|
||||
const destination = createFakeDestination({
|
||||
existing: [
|
||||
{ id: 11, name: "firmware.bin", size: 1024 },
|
||||
{ id: 12, name: "firmware.bin", size: 999 },
|
||||
],
|
||||
});
|
||||
destination.setUploadedSize("firmware.bin", 2048);
|
||||
|
||||
const result = await reconcile(destination.fetchImpl, [
|
||||
githubAsset("firmware.bin", 2048),
|
||||
]);
|
||||
|
||||
expect(destination.deletes().map((r) => r.url)).toEqual([
|
||||
`${ASSETS_URL}/11`,
|
||||
`${ASSETS_URL}/12`,
|
||||
]);
|
||||
expect(destination.posts()).toHaveLength(1);
|
||||
expect(destination.attachments).toHaveLength(1);
|
||||
expect(destination.attachments[0].size).toBe(2048);
|
||||
expect(result).toEqual({ uploaded: 1, failed: 0, skipped: 0 });
|
||||
// Deletes come before the upload, never after it.
|
||||
const kinds = destination.requests.map((r) => r.method);
|
||||
expect(kinds.indexOf("POST")).toBeGreaterThan(kinds.lastIndexOf("DELETE"));
|
||||
});
|
||||
|
||||
it("leaves attachments alone that the release does not have upstream", async () => {
|
||||
const destination = createFakeDestination({
|
||||
existing: [{ id: 11, name: "from-an-older-sync.zip", size: 4 }],
|
||||
});
|
||||
destination.setUploadedSize("base.zip", 4);
|
||||
|
||||
const result = await reconcile(destination.fetchImpl, [githubAsset("base.zip", 4)]);
|
||||
|
||||
expect(destination.deletes()).toHaveLength(0);
|
||||
expect(destination.posts()).toHaveLength(1);
|
||||
expect(result).toEqual({ uploaded: 1, failed: 0, skipped: 0 });
|
||||
});
|
||||
|
||||
it("does nothing at all for a release with no assets", async () => {
|
||||
const destination = createFakeDestination();
|
||||
|
||||
const result = await reconcile(destination.fetchImpl, []);
|
||||
|
||||
expect(destination.requests).toHaveLength(0);
|
||||
expect(result).toEqual({ uploaded: 0, failed: 0, skipped: 0 });
|
||||
});
|
||||
});
|
||||
|
||||
describe("buildReleaseTargetLockKey", () => {
|
||||
it("keys on destination, owner and repository, case-insensitively", () => {
|
||||
expect(buildReleaseTargetLockKey("https://gitea.example.com/", "Owner", "Repo")).toBe(
|
||||
buildReleaseTargetLockKey("https://GITEA.example.com", "owner", "repo")
|
||||
);
|
||||
});
|
||||
|
||||
it("separates different repositories on the same destination", () => {
|
||||
expect(buildReleaseTargetLockKey(GITEA_URL, OWNER, "one")).not.toBe(
|
||||
buildReleaseTargetLockKey(GITEA_URL, OWNER, "two")
|
||||
);
|
||||
});
|
||||
});
|
||||
@@ -164,6 +164,11 @@ describe("classifyReleasesForReconciliation", () => {
|
||||
* existing release" and left it at 0.
|
||||
*
|
||||
* Fix: reconcile assets idempotently on both paths via classifyAssetsForReconciliation.
|
||||
*
|
||||
* Extended for #417: the destination allows several attachments with the same
|
||||
* name (no name-collision check in CreateReleaseAttachment), so the classifier
|
||||
* groups existing attachments by name, keeps one copy and reports the surplus
|
||||
* ones in toDelete.
|
||||
*/
|
||||
describe("classifyAssetsForReconciliation", () => {
|
||||
it("uploads all assets when the Gitea release has none (the #331 broken state)", () => {
|
||||
@@ -173,13 +178,11 @@ describe("classifyAssetsForReconciliation", () => {
|
||||
];
|
||||
const gitea: Array<{ id: number; name: string; size: number }> = [];
|
||||
|
||||
const { toUpload, toSkip } = classifyAssetsForReconciliation(github, gitea);
|
||||
const { toUpload, toSkip, toDelete } = classifyAssetsForReconciliation(github, gitea);
|
||||
|
||||
expect(toSkip).toEqual([]);
|
||||
expect(toUpload).toEqual([
|
||||
{ name: "base.zip", replaceAssetId: null },
|
||||
{ name: "extras.zip", replaceAssetId: null },
|
||||
]);
|
||||
expect(toDelete).toEqual([]);
|
||||
expect(toUpload).toEqual(["base.zip", "extras.zip"]);
|
||||
});
|
||||
|
||||
it("backfills only the missing asset when one already exists", () => {
|
||||
@@ -189,10 +192,11 @@ describe("classifyAssetsForReconciliation", () => {
|
||||
];
|
||||
const gitea = [{ id: 9, name: "base.zip", size: 40_264_954 }];
|
||||
|
||||
const { toUpload, toSkip } = classifyAssetsForReconciliation(github, gitea);
|
||||
const { toUpload, toSkip, toDelete } = classifyAssetsForReconciliation(github, gitea);
|
||||
|
||||
expect(toSkip).toEqual(["base.zip"]);
|
||||
expect(toUpload).toEqual([{ name: "extras.zip", replaceAssetId: null }]);
|
||||
expect(toUpload).toEqual(["extras.zip"]);
|
||||
expect(toDelete).toEqual([]);
|
||||
});
|
||||
|
||||
it("is idempotent — skips everything when all assets already match by name+size", () => {
|
||||
@@ -205,28 +209,111 @@ describe("classifyAssetsForReconciliation", () => {
|
||||
{ id: 10, name: "extras.zip", size: 37_098_528 },
|
||||
];
|
||||
|
||||
const { toUpload, toSkip } = classifyAssetsForReconciliation(github, gitea);
|
||||
const { toUpload, toSkip, toDelete } = classifyAssetsForReconciliation(github, gitea);
|
||||
|
||||
expect(toUpload).toEqual([]);
|
||||
expect(toSkip).toEqual(["base.zip", "extras.zip"]);
|
||||
expect(toDelete).toEqual([]);
|
||||
});
|
||||
|
||||
it("replaces an asset whose size changed upstream (re-upload over the stale copy)", () => {
|
||||
it("replaces an asset whose size changed upstream (delete the stale copy, upload once)", () => {
|
||||
const github = [{ name: "firmware.bin", size: 2048 }];
|
||||
const gitea = [{ id: 42, name: "firmware.bin", size: 1024 }]; // truncated/stale
|
||||
|
||||
const { toUpload, toSkip } = classifyAssetsForReconciliation(github, gitea);
|
||||
const { toUpload, toSkip, toDelete } = classifyAssetsForReconciliation(github, gitea);
|
||||
|
||||
expect(toSkip).toEqual([]);
|
||||
expect(toUpload).toEqual([{ name: "firmware.bin", replaceAssetId: 42 }]);
|
||||
expect(toUpload).toEqual(["firmware.bin"]);
|
||||
expect(toDelete).toEqual([42]);
|
||||
});
|
||||
|
||||
it("handles a release with no GitHub assets", () => {
|
||||
const { toUpload, toSkip } = classifyAssetsForReconciliation(
|
||||
const { toUpload, toSkip, toDelete } = classifyAssetsForReconciliation(
|
||||
[],
|
||||
[{ id: 1, name: "leftover.zip", size: 10 }]
|
||||
);
|
||||
expect(toUpload).toEqual([]);
|
||||
expect(toSkip).toEqual([]);
|
||||
expect(toDelete).toEqual([]);
|
||||
});
|
||||
|
||||
// --- #417: duplicate attachments on the destination ---
|
||||
|
||||
it("keeps one copy and deletes the extra when two copies both match (#417)", () => {
|
||||
const github = [{ name: "base.zip", size: 40_264_954 }];
|
||||
const gitea = [
|
||||
{ id: 9, name: "base.zip", size: 40_264_954 },
|
||||
{ id: 14, name: "base.zip", size: 40_264_954 },
|
||||
];
|
||||
|
||||
const { toUpload, toSkip, toDelete } = classifyAssetsForReconciliation(github, gitea);
|
||||
|
||||
expect(toUpload).toEqual([]);
|
||||
expect(toSkip).toEqual(["base.zip"]);
|
||||
expect(toDelete).toEqual([14]);
|
||||
});
|
||||
|
||||
it("deletes all three copies and uploads once when none matches the size", () => {
|
||||
const github = [{ name: "firmware.bin", size: 2048 }];
|
||||
const gitea = [
|
||||
{ id: 7, name: "firmware.bin", size: 1024 },
|
||||
{ id: 8, name: "firmware.bin", size: 1024 },
|
||||
{ id: 9, name: "firmware.bin", size: 99 },
|
||||
];
|
||||
|
||||
const { toUpload, toSkip, toDelete } = classifyAssetsForReconciliation(github, gitea);
|
||||
|
||||
expect(toUpload).toEqual(["firmware.bin"]);
|
||||
expect(toSkip).toEqual([]);
|
||||
expect(toDelete).toEqual([7, 8, 9]);
|
||||
});
|
||||
|
||||
it("keeps the matching copy and deletes the two that do not match", () => {
|
||||
const github = [{ name: "installer.dmg", size: 5_000 }];
|
||||
const gitea = [
|
||||
{ id: 3, name: "installer.dmg", size: 1_000 },
|
||||
{ id: 4, name: "installer.dmg", size: 5_000 },
|
||||
{ id: 5, name: "installer.dmg", size: 2_000 },
|
||||
];
|
||||
|
||||
const { toUpload, toSkip, toDelete } = classifyAssetsForReconciliation(github, gitea);
|
||||
|
||||
expect(toUpload).toEqual([]);
|
||||
expect(toSkip).toEqual(["installer.dmg"]);
|
||||
expect(toDelete).toEqual([3, 5]);
|
||||
});
|
||||
|
||||
it("never deletes a destination attachment whose name is not in the GitHub list", () => {
|
||||
const github = [{ name: "base.zip", size: 10 }];
|
||||
const gitea = [
|
||||
{ id: 1, name: "base.zip", size: 10 },
|
||||
{ id: 2, name: "from-an-older-release.zip", size: 10 },
|
||||
{ id: 3, name: "from-an-older-release.zip", size: 99 },
|
||||
];
|
||||
|
||||
const { toUpload, toSkip, toDelete } = classifyAssetsForReconciliation(github, gitea);
|
||||
|
||||
expect(toUpload).toEqual([]);
|
||||
expect(toSkip).toEqual(["base.zip"]);
|
||||
expect(toDelete).toEqual([]);
|
||||
});
|
||||
|
||||
it("keeps the lowest id whatever order the destination listed the copies in", () => {
|
||||
const github = [{ name: "base.zip", size: 10 }];
|
||||
const listedOneWay = [
|
||||
{ id: 31, name: "base.zip", size: 10 },
|
||||
{ id: 12, name: "base.zip", size: 10 },
|
||||
{ id: 25, name: "base.zip", size: 10 },
|
||||
];
|
||||
const listedTheOtherWay = [...listedOneWay].reverse();
|
||||
|
||||
const first = classifyAssetsForReconciliation(github, listedOneWay);
|
||||
const second = classifyAssetsForReconciliation(github, listedTheOtherWay);
|
||||
|
||||
// id 12 survives in both, and the deletes come back in the same order.
|
||||
expect(first.toDelete).toEqual([25, 31]);
|
||||
expect(second.toDelete).toEqual([25, 31]);
|
||||
expect(first.toSkip).toEqual(["base.zip"]);
|
||||
expect(second.toSkip).toEqual(["base.zip"]);
|
||||
});
|
||||
});
|
||||
+258
-92
@@ -40,6 +40,8 @@ import {
|
||||
} from "./utils/mirror-overrides";
|
||||
import { repositoryDestinationColumns } from "./repo-utils";
|
||||
import { resolveDestinationIdentity } from "./destination-connection";
|
||||
import { withKeyedLock } from "./utils/keyed-mutex";
|
||||
import { claimRepositoryForInFlightWork } from "./utils/repo-status-claim";
|
||||
|
||||
/**
|
||||
* Helper function to get organization configuration including destination override
|
||||
@@ -875,34 +877,30 @@ export const mirrorGithubRepoToGitea = async ({
|
||||
|
||||
console.log(`Mirroring repository ${repository.name}`);
|
||||
|
||||
// DOUBLE-CHECK: Final idempotency check right before updating status
|
||||
// This catches race conditions in the small window between first check and status update
|
||||
const finalCheck = await isRepoCurrentlyMirroring({
|
||||
config,
|
||||
repoName: targetRepoName,
|
||||
expectedLocation: targetLocation,
|
||||
});
|
||||
|
||||
if (finalCheck) {
|
||||
console.log(
|
||||
`[Idempotency] Race condition detected - ${repository.fullName} is now being mirrored by another process. Skipping.`
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
// Mark repos as "mirroring" in DB
|
||||
// CLAIM THE ROW: mark it "mirroring" only while no other run owns it. This
|
||||
// used to be a second isRepoCurrentlyMirroring read followed by an
|
||||
// unconditional write, so two workers starting in the same tick both read
|
||||
// "free" and both mirrored the repository (#417). One conditional UPDATE
|
||||
// closes that window.
|
||||
// CRITICAL: Set mirroredLocation NOW (not after success) so idempotency checks work
|
||||
// This becomes the "target location" - where we intend to mirror to
|
||||
// Without this, the idempotency check can't detect concurrent operations on first mirror
|
||||
await db
|
||||
.update(repositories)
|
||||
.set({
|
||||
const claimed = await claimRepositoryForInFlightWork({
|
||||
repositoryId: repository.id!,
|
||||
set: {
|
||||
status: repoStatusEnum.parse("mirroring"),
|
||||
mirroredLocation: targetLocation,
|
||||
...repositoryDestinationColumns(resolveDestinationIdentity(config)),
|
||||
updatedAt: new Date(),
|
||||
})
|
||||
.where(eq(repositories.id, repository.id!));
|
||||
},
|
||||
});
|
||||
|
||||
if (!claimed) {
|
||||
console.log(
|
||||
`[Idempotency] Race condition detected - ${repository.fullName} is now being mirrored or synced by another process. Skipping.`
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
// Append log for "mirroring" status
|
||||
await createMirrorJob({
|
||||
@@ -1727,34 +1725,28 @@ export async function mirrorGitHubRepoToGiteaOrg({
|
||||
// Use clean clone URL without embedded credentials (Forgejo 12+ security requirement)
|
||||
const cloneAddress = repository.cloneUrl;
|
||||
|
||||
// DOUBLE-CHECK: Final idempotency check right before updating status
|
||||
// This catches race conditions in the small window between first check and status update
|
||||
const finalCheck = await isRepoCurrentlyMirroring({
|
||||
config,
|
||||
repoName: targetRepoName,
|
||||
expectedLocation: targetLocation,
|
||||
});
|
||||
|
||||
if (finalCheck) {
|
||||
console.log(
|
||||
`[Idempotency] Race condition detected - ${repository.fullName} is now being mirrored by another process. Skipping.`
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
// Mark repos as "mirroring" in DB
|
||||
// CLAIM THE ROW: see the same claim on the user-owner path above. One
|
||||
// conditional UPDATE replaces the old read-then-write, so two workers
|
||||
// starting in the same tick cannot both mirror this repository (#417).
|
||||
// CRITICAL: Set mirroredLocation NOW (not after success) so idempotency checks work
|
||||
// This becomes the "target location" - where we intend to mirror to
|
||||
// Without this, the idempotency check can't detect concurrent operations on first mirror
|
||||
await db
|
||||
.update(repositories)
|
||||
.set({
|
||||
const claimed = await claimRepositoryForInFlightWork({
|
||||
repositoryId: repository.id!,
|
||||
set: {
|
||||
status: repoStatusEnum.parse("mirroring"),
|
||||
mirroredLocation: targetLocation,
|
||||
...repositoryDestinationColumns(resolveDestinationIdentity(config)),
|
||||
updatedAt: new Date(),
|
||||
})
|
||||
.where(eq(repositories.id, repository.id!));
|
||||
},
|
||||
});
|
||||
|
||||
if (!claimed) {
|
||||
console.log(
|
||||
`[Idempotency] Race condition detected - ${repository.fullName} is now being mirrored or synced by another process. Skipping.`
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
// Note: "mirroring" status events are handled by the concurrency system
|
||||
// to avoid duplicate events during batch operations
|
||||
@@ -3037,37 +3029,90 @@ export function classifyReleasesForReconciliation(
|
||||
}
|
||||
|
||||
/**
|
||||
* Decide which of a GitHub release's assets need (re)uploading to Gitea.
|
||||
* Decide what to upload, keep and delete for one release's assets.
|
||||
*
|
||||
* Compared by name:
|
||||
* - present in Gitea with a matching size -> skip (already mirrored)
|
||||
* - present with a different size -> upload, replacing the stale copy
|
||||
* - absent -> upload (fresh)
|
||||
* Destination attachments are grouped by name because the destination happily
|
||||
* holds several attachments with the same name: Gitea's and Forgejo's
|
||||
* CreateReleaseAttachment has no name-collision check, so every POST appends a
|
||||
* new attachment. Two overlapping mirror passes each saw an asset as missing
|
||||
* and each uploaded it, which is how releases ended up with two, three or five
|
||||
* copies of every asset (#417). Grouping means a pass that finds duplicates
|
||||
* cleans them up instead of adding to them.
|
||||
*
|
||||
* Pure function so the create/update reconciliation decision is unit-testable
|
||||
* without hitting the network (regression guard for #331).
|
||||
* For each GitHub asset, matched by name:
|
||||
* - one or more copies match the size -> keep the lowest id, skip the
|
||||
* upload, delete every other copy of that name
|
||||
* - copies exist but none matches the size -> delete all of them, upload once
|
||||
* - no copy exists -> upload
|
||||
*
|
||||
* A destination attachment whose name is not in the GitHub list is never
|
||||
* deleted: the asset limit (#311), an older upstream release and manual
|
||||
* uploads all leave attachments this reconciliation has no claim over.
|
||||
*
|
||||
* Pure function so the reconciliation decision is unit-testable without
|
||||
* hitting the network (regression guard for #331 and #417).
|
||||
*/
|
||||
export function classifyAssetsForReconciliation(
|
||||
githubAssets: Array<{ name: string; size: number }>,
|
||||
giteaAssets: Array<{ id: number; name: string; size: number }>
|
||||
): {
|
||||
toUpload: Array<{ name: string; replaceAssetId: number | null }>;
|
||||
toUpload: string[];
|
||||
toSkip: string[];
|
||||
toDelete: number[];
|
||||
} {
|
||||
const existingByName = new Map(giteaAssets.map((a) => [a.name, a]));
|
||||
const toUpload: Array<{ name: string; replaceAssetId: number | null }> = [];
|
||||
const existingByName = new Map<string, Array<{ id: number; name: string; size: number }>>();
|
||||
for (const existing of giteaAssets) {
|
||||
const copies = existingByName.get(existing.name);
|
||||
if (copies) {
|
||||
copies.push(existing);
|
||||
} else {
|
||||
existingByName.set(existing.name, [existing]);
|
||||
}
|
||||
}
|
||||
// Lowest id first, so "keep one copy" and the delete order are deterministic
|
||||
// whatever order the destination listed the attachments in.
|
||||
for (const copies of existingByName.values()) {
|
||||
copies.sort((a, b) => a.id - b.id);
|
||||
}
|
||||
|
||||
const toUpload: string[] = [];
|
||||
const toSkip: string[] = [];
|
||||
const toDelete: number[] = [];
|
||||
const seen = new Set<string>();
|
||||
|
||||
for (const asset of githubAssets) {
|
||||
const existing = existingByName.get(asset.name);
|
||||
if (existing && existing.size === asset.size) {
|
||||
if (seen.has(asset.name)) {
|
||||
continue;
|
||||
}
|
||||
seen.add(asset.name);
|
||||
|
||||
const copies = existingByName.get(asset.name) ?? [];
|
||||
if (copies.length === 0) {
|
||||
toUpload.push(asset.name);
|
||||
continue;
|
||||
}
|
||||
|
||||
const keep = copies.find((copy) => copy.size === asset.size);
|
||||
if (keep) {
|
||||
// Already mirrored: keep the first (lowest id) matching copy and drop
|
||||
// every other attachment carrying that name.
|
||||
toSkip.push(asset.name);
|
||||
for (const copy of copies) {
|
||||
if (copy.id !== keep.id) {
|
||||
toDelete.push(copy.id);
|
||||
}
|
||||
}
|
||||
} else {
|
||||
toUpload.push({ name: asset.name, replaceAssetId: existing ? existing.id : null });
|
||||
// Every copy is stale (truncated, or changed upstream): remove them all
|
||||
// and upload a single fresh one.
|
||||
for (const copy of copies) {
|
||||
toDelete.push(copy.id);
|
||||
}
|
||||
toUpload.push(asset.name);
|
||||
}
|
||||
}
|
||||
|
||||
return { toUpload, toSkip };
|
||||
return { toUpload, toSkip, toDelete };
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -3080,11 +3125,20 @@ export function classifyAssetsForReconciliation(
|
||||
* failed or were interrupted on first creation therefore stayed permanently
|
||||
* asset-less, and re-syncing could never heal it (#331).
|
||||
*
|
||||
* Strategy: compare by asset name. Skip assets already present with a matching size;
|
||||
* (re)upload anything missing, and replace an existing asset whose size differs
|
||||
* (truncated/changed upstream). Returns per-release counts so the caller can report.
|
||||
* Never adds a copy of an asset that is already there (#417):
|
||||
* - the existing attachments are listed first, and if that listing fails the
|
||||
* whole release is skipped ("fail closed") instead of being uploaded blind;
|
||||
* - surplus and stale attachments are deleted BEFORE anything is uploaded;
|
||||
* - an asset whose stale copy could not be deleted is not uploaded either, so
|
||||
* a failed delete never turns into one more copy.
|
||||
*
|
||||
* The caller is expected to hold the per-destination lock (see
|
||||
* mirrorGitHubReleasesToGitea), which keeps two passes over the same
|
||||
* destination repository from interleaving their list and upload steps.
|
||||
*
|
||||
* Returns per-release counts so the caller can report.
|
||||
*/
|
||||
async function reconcileReleaseAssets({
|
||||
export async function reconcileReleaseAssets({
|
||||
config,
|
||||
decryptedConfig,
|
||||
repoOwner,
|
||||
@@ -3092,6 +3146,7 @@ async function reconcileReleaseAssets({
|
||||
giteaReleaseId,
|
||||
githubAssets,
|
||||
tagName,
|
||||
fetchImpl,
|
||||
}: {
|
||||
config: Partial<Config>;
|
||||
decryptedConfig: Config;
|
||||
@@ -3100,6 +3155,8 @@ async function reconcileReleaseAssets({
|
||||
giteaReleaseId: number;
|
||||
githubAssets: Array<{ name: string; size: number; browser_download_url: string }>;
|
||||
tagName: string;
|
||||
/** Injected in tests; production uses the global fetch. */
|
||||
fetchImpl?: typeof fetch;
|
||||
}): Promise<{ uploaded: number; failed: number; skipped: number }> {
|
||||
let uploaded = 0;
|
||||
let failed = 0;
|
||||
@@ -3109,27 +3166,87 @@ async function reconcileReleaseAssets({
|
||||
return { uploaded, failed, skipped };
|
||||
}
|
||||
|
||||
const doFetch: typeof fetch = fetchImpl ?? fetch;
|
||||
const giteaBaseUrl = config.giteaConfig!.url;
|
||||
const giteaAuth = { Authorization: `token ${decryptedConfig.giteaConfig!.token}` };
|
||||
const assetsUrl = `${giteaBaseUrl}/api/v1/repos/${repoOwner}/${repoName}/releases/${giteaReleaseId}/assets`;
|
||||
|
||||
// Fetch existing attachments so we only transfer what's missing or changed.
|
||||
const existingAssets: Array<{ id: number; name: string; size: number }> = await httpGet(
|
||||
`${giteaBaseUrl}/api/v1/repos/${repoOwner}/${repoName}/releases/${giteaReleaseId}/assets`,
|
||||
giteaAuth
|
||||
)
|
||||
.then((r) => (Array.isArray(r?.data) ? r.data : []))
|
||||
.catch(() => []);
|
||||
// FAIL CLOSED: the list of what the release already has is the only thing
|
||||
// standing between a sync and a second copy of every asset. If it cannot be
|
||||
// read, upload nothing and let the next sync try again. Treating a failed
|
||||
// GET as "the release has no assets" is exactly how duplicates appeared.
|
||||
let existingAssets: Array<{ id: number; name: string; size: number }>;
|
||||
try {
|
||||
const listResponse = await doFetch(assetsUrl, { headers: giteaAuth });
|
||||
if (!listResponse.ok) {
|
||||
throw new Error(`HTTP ${listResponse.status} ${listResponse.statusText}`);
|
||||
}
|
||||
const body = await listResponse.json();
|
||||
if (!Array.isArray(body)) {
|
||||
throw new Error("the assets endpoint did not return a list");
|
||||
}
|
||||
existingAssets = body;
|
||||
} catch (listError) {
|
||||
console.error(
|
||||
`[Releases] Could not list the existing assets of ${tagName}: ${
|
||||
listError instanceof Error ? listError.message : String(listError)
|
||||
}. Skipping all ${githubAssets.length} asset(s) for this release so no duplicate is created; the next sync retries.`
|
||||
);
|
||||
return { uploaded, failed: githubAssets.length, skipped };
|
||||
}
|
||||
|
||||
const { toUpload, toSkip } = classifyAssetsForReconciliation(
|
||||
const { toUpload, toSkip, toDelete } = classifyAssetsForReconciliation(
|
||||
githubAssets,
|
||||
existingAssets
|
||||
);
|
||||
skipped = toSkip.length;
|
||||
|
||||
const nameById = new Map(existingAssets.map((a) => [a.id, a.name]));
|
||||
// Names that still have a surplus or stale attachment on the destination:
|
||||
// uploading now would leave one copy too many, so the upload waits.
|
||||
const undeletedNames = new Set<string>();
|
||||
|
||||
// Deletes run before any upload, so a release that already has duplicates is
|
||||
// cleaned up by the very sync that notices them.
|
||||
for (const assetId of toDelete) {
|
||||
const name = nameById.get(assetId);
|
||||
try {
|
||||
const deleteResponse = await doFetch(`${assetsUrl}/${assetId}`, {
|
||||
method: "DELETE",
|
||||
headers: giteaAuth,
|
||||
});
|
||||
// 404 means it is already gone, which is the state we wanted.
|
||||
if (!deleteResponse.ok && deleteResponse.status !== 404) {
|
||||
throw new Error(`HTTP ${deleteResponse.status} ${deleteResponse.statusText}`);
|
||||
}
|
||||
console.log(
|
||||
`[Releases] Removed surplus/stale attachment ${name ?? assetId} (id ${assetId}) from ${tagName}`
|
||||
);
|
||||
} catch (deleteError) {
|
||||
if (name) {
|
||||
undeletedNames.add(name);
|
||||
}
|
||||
console.error(
|
||||
`[Releases] Failed to delete attachment ${name ?? assetId} (id ${assetId}) from ${tagName}: ${
|
||||
deleteError instanceof Error ? deleteError.message : String(deleteError)
|
||||
}`
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
const githubByName = new Map(githubAssets.map((a) => [a.name, a]));
|
||||
|
||||
for (const { name, replaceAssetId } of toUpload) {
|
||||
for (const name of toUpload) {
|
||||
const asset = githubByName.get(name)!;
|
||||
|
||||
if (undeletedNames.has(name)) {
|
||||
console.warn(
|
||||
`[Releases] Not uploading ${name} for ${tagName}: its stale copy is still on the destination and uploading would leave two copies. Retrying on the next sync.`
|
||||
);
|
||||
failed++;
|
||||
continue;
|
||||
}
|
||||
|
||||
try {
|
||||
// Download from GitHub. fetch strips the Authorization header on the
|
||||
// cross-host redirect to GitHub's object storage, so this works for both
|
||||
@@ -3137,7 +3254,7 @@ async function reconcileReleaseAssets({
|
||||
console.log(
|
||||
`[Releases] Downloading asset: ${asset.name} (${asset.size} bytes) for ${tagName}`
|
||||
);
|
||||
const assetResponse = await fetch(asset.browser_download_url, {
|
||||
const assetResponse = await doFetch(asset.browser_download_url, {
|
||||
headers: {
|
||||
Accept: "application/octet-stream",
|
||||
Authorization: `token ${decryptedConfig.githubConfig!.token}`,
|
||||
@@ -3154,23 +3271,14 @@ async function reconcileReleaseAssets({
|
||||
|
||||
const assetData = await assetResponse.arrayBuffer();
|
||||
|
||||
// Gitea rejects a duplicate attachment name, so drop a stale/mismatched
|
||||
// copy before re-uploading.
|
||||
if (replaceAssetId !== null) {
|
||||
await httpDelete(
|
||||
`${giteaBaseUrl}/api/v1/repos/${repoOwner}/${repoName}/releases/${giteaReleaseId}/assets/${replaceAssetId}`,
|
||||
giteaAuth
|
||||
).catch(() => null);
|
||||
}
|
||||
|
||||
const formData = new FormData();
|
||||
formData.append("attachment", new Blob([assetData]), asset.name);
|
||||
|
||||
const uploadResponse = await fetch(
|
||||
`${giteaBaseUrl}/api/v1/repos/${repoOwner}/${repoName}/releases/${giteaReleaseId}/assets?name=${encodeURIComponent(asset.name)}`,
|
||||
const uploadResponse = await doFetch(
|
||||
`${assetsUrl}?name=${encodeURIComponent(asset.name)}`,
|
||||
{
|
||||
method: "POST",
|
||||
headers: { Authorization: `token ${decryptedConfig.giteaConfig!.token}` },
|
||||
headers: giteaAuth,
|
||||
body: formData,
|
||||
}
|
||||
);
|
||||
@@ -3198,15 +3306,7 @@ async function reconcileReleaseAssets({
|
||||
return { uploaded, failed, skipped };
|
||||
}
|
||||
|
||||
export async function mirrorGitHubReleasesToGitea({
|
||||
octokit,
|
||||
repository,
|
||||
config,
|
||||
giteaOwner,
|
||||
giteaRepoName,
|
||||
releaseLimit: releaseLimitOverride,
|
||||
releaseAssetLimit: releaseAssetLimitOverride,
|
||||
}: {
|
||||
type MirrorReleasesParams = {
|
||||
octokit: Octokit;
|
||||
repository: Repository;
|
||||
config: Partial<Config>;
|
||||
@@ -3224,7 +3324,73 @@ export async function mirrorGitHubReleasesToGitea({
|
||||
* none, `undefined` means resolve it here like the release limit.
|
||||
*/
|
||||
releaseAssetLimit?: number | null;
|
||||
}) {
|
||||
};
|
||||
|
||||
/**
|
||||
* Lock key for one destination repository's releases: the destination base URL
|
||||
* plus the owner and repository name it is mirrored as, lowercased.
|
||||
*
|
||||
* Keyed on the destination and not on the repository row on purpose. Two rows
|
||||
* (the same upstream repository imported under two sources) resolve to the same
|
||||
* destination repository, and a per-row status guard cannot see that (#417).
|
||||
*/
|
||||
export function buildReleaseTargetLockKey(
|
||||
giteaBaseUrl: string,
|
||||
owner: string,
|
||||
repoName: string
|
||||
): string {
|
||||
return `${giteaBaseUrl.replace(/\/+$/, "")}|${owner}|${repoName}`.toLowerCase();
|
||||
}
|
||||
|
||||
/**
|
||||
* Mirror a repository's releases, and the assets of the newest ones, onto the
|
||||
* destination.
|
||||
*
|
||||
* One pass per destination repository at a time (#417): release reconciliation
|
||||
* reads what the destination already has and then writes what is missing, and
|
||||
* the destination appends a new attachment for every upload POST without any
|
||||
* name-collision check. Two passes that overlap on one repository (the
|
||||
* scheduler racing a manual sync, recovery resuming a live job, or two
|
||||
* repository rows pointing at the same destination) both read "missing" and
|
||||
* both upload, leaving duplicates. The lock makes the read and the write one
|
||||
* unit per destination repository.
|
||||
*/
|
||||
export async function mirrorGitHubReleasesToGitea(params: MirrorReleasesParams) {
|
||||
const { config, repository, giteaOwner, giteaRepoName } = params;
|
||||
|
||||
if (
|
||||
!config.giteaConfig?.defaultOwner ||
|
||||
!config.giteaConfig?.token ||
|
||||
!config.giteaConfig?.url
|
||||
) {
|
||||
throw new Error("Gitea config is incomplete for mirroring releases.");
|
||||
}
|
||||
|
||||
// Resolved here so the lock key names the destination repository, then passed
|
||||
// down so the locked body does not resolve it a second time.
|
||||
const repoOwner = giteaOwner || (await getGiteaRepoOwnerAsync({ config, repository }));
|
||||
const repoName = giteaRepoName || repository.name;
|
||||
|
||||
return withKeyedLock(
|
||||
buildReleaseTargetLockKey(config.giteaConfig.url, repoOwner, repoName),
|
||||
() =>
|
||||
mirrorGitHubReleasesToGiteaLocked({
|
||||
...params,
|
||||
giteaOwner: repoOwner,
|
||||
giteaRepoName: repoName,
|
||||
})
|
||||
);
|
||||
}
|
||||
|
||||
async function mirrorGitHubReleasesToGiteaLocked({
|
||||
octokit,
|
||||
repository,
|
||||
config,
|
||||
giteaOwner,
|
||||
giteaRepoName,
|
||||
releaseLimit: releaseLimitOverride,
|
||||
releaseAssetLimit: releaseAssetLimitOverride,
|
||||
}: MirrorReleasesParams) {
|
||||
if (
|
||||
!config.giteaConfig?.defaultOwner ||
|
||||
!config.giteaConfig?.token ||
|
||||
|
||||
+42
-11
@@ -4,6 +4,7 @@ import { eq, and } from "drizzle-orm";
|
||||
import { buildInterruptedJobsCondition } from "./interrupted-job-detection";
|
||||
import { v4 as uuidv4 } from "uuid";
|
||||
import { publishEvent } from "./events";
|
||||
import { withKeyedLock } from "./utils/keyed-mutex";
|
||||
import { triggerJobNotification } from "./notification-service";
|
||||
|
||||
export async function createMirrorJob({
|
||||
@@ -113,17 +114,18 @@ export async function createMirrorJob({
|
||||
}
|
||||
|
||||
/**
|
||||
* Updates the progress of a mirror job
|
||||
* Updates the progress of a mirror job.
|
||||
*
|
||||
* Progress is a read-modify-write of the completedItemIds list, and items
|
||||
* finish concurrently, so the update runs under a per-job lock: two items
|
||||
* completing in the same tick used to read the same list and one of them
|
||||
* silently overwrote the other. A resume then re-processed the lost item.
|
||||
*
|
||||
* `silent` records the progress without publishing an event. Every completed
|
||||
* item is checkpointed so a resume knows exactly what is done; the events the
|
||||
* dashboard listens to are throttled by the caller instead.
|
||||
*/
|
||||
export async function updateMirrorJobProgress({
|
||||
jobId,
|
||||
completedItemId,
|
||||
status,
|
||||
message,
|
||||
details,
|
||||
inProgress,
|
||||
isCompleted,
|
||||
}: {
|
||||
export async function updateMirrorJobProgress(params: {
|
||||
jobId: string;
|
||||
completedItemId?: string;
|
||||
status?: RepoStatus;
|
||||
@@ -131,7 +133,25 @@ export async function updateMirrorJobProgress({
|
||||
details?: string;
|
||||
inProgress?: boolean;
|
||||
isCompleted?: boolean;
|
||||
/** Record a new start time, as when recovery resumes the job. */
|
||||
startedAt?: Date;
|
||||
/** Write the progress without publishing a mirror-status event. */
|
||||
silent?: boolean;
|
||||
}) {
|
||||
return withKeyedLock(`mirror-job:${params.jobId}`, () => updateMirrorJobProgressLocked(params));
|
||||
}
|
||||
|
||||
async function updateMirrorJobProgressLocked({
|
||||
jobId,
|
||||
completedItemId,
|
||||
status,
|
||||
message,
|
||||
details,
|
||||
inProgress,
|
||||
isCompleted,
|
||||
startedAt,
|
||||
silent = false,
|
||||
}: Parameters<typeof updateMirrorJobProgress>[0]) {
|
||||
try {
|
||||
// Get the current job
|
||||
const [job] = await db
|
||||
@@ -157,6 +177,10 @@ export async function updateMirrorJobProgress({
|
||||
}
|
||||
}
|
||||
|
||||
if (startedAt) {
|
||||
updates.startedAt = startedAt;
|
||||
}
|
||||
|
||||
// Update status if provided
|
||||
if (status) {
|
||||
updates.status = status;
|
||||
@@ -195,6 +219,10 @@ export async function updateMirrorJobProgress({
|
||||
...updates,
|
||||
};
|
||||
|
||||
if (silent) {
|
||||
return updatedJob;
|
||||
}
|
||||
|
||||
// Create deduplication key for progress updates
|
||||
let deduplicationKey: string | undefined;
|
||||
if (completedItemId) {
|
||||
@@ -324,12 +352,15 @@ export async function resumeInterruptedJob(job: any) {
|
||||
return null;
|
||||
}
|
||||
|
||||
// Update the job to show it's being resumed
|
||||
// Update the job to show it's being resumed. The start time moves to
|
||||
// now so the row records this pass; the fresh checkpoint written here is
|
||||
// what keeps a second recovery sweep from resuming the same job again.
|
||||
await updateMirrorJobProgress({
|
||||
jobId: job.id,
|
||||
message: `Resuming job with ${remainingItemIds.length} remaining items`,
|
||||
details: `Job was interrupted and is being resumed. ${job.completedItemIds.length} of ${job.itemIds.length} items were already processed.`,
|
||||
inProgress: true,
|
||||
startedAt: new Date(),
|
||||
});
|
||||
|
||||
return {
|
||||
|
||||
@@ -18,7 +18,6 @@ import { drizzle } from "drizzle-orm/bun-sqlite";
|
||||
import { mirrorJobs } from "@/lib/db/schema";
|
||||
import {
|
||||
INTERRUPTED_CHECKPOINT_AGE_MS,
|
||||
STALE_JOB_AGE_MS,
|
||||
JOB_HEARTBEAT_INTERVAL_MS,
|
||||
computeInterruptedJobCutoffs,
|
||||
isJobInterrupted,
|
||||
@@ -32,14 +31,15 @@ const NOW = new Date("2026-09-02T12:00:00Z");
|
||||
const ago = (ms: number) => new Date(NOW.getTime() - ms);
|
||||
|
||||
describe("computeInterruptedJobCutoffs", () => {
|
||||
test("keeps the 10 minute checkpoint window and 2 hour stale window", () => {
|
||||
test("keeps the 10 minute checkpoint window and has no stale age cutoff", () => {
|
||||
expect(INTERRUPTED_CHECKPOINT_AGE_MS).toBe(10 * MINUTE);
|
||||
expect(STALE_JOB_AGE_MS).toBe(2 * HOUR);
|
||||
|
||||
const cutoffs = computeInterruptedJobCutoffs(NOW);
|
||||
|
||||
expect(cutoffs.checkpointCutoff.getTime()).toBe(NOW.getTime() - 10 * MINUTE);
|
||||
expect(cutoffs.staleCutoff.getTime()).toBe(NOW.getTime() - 2 * HOUR);
|
||||
// The two hour "stale regardless of checkpoint" rule resumed live jobs
|
||||
// and is gone; a checkpointing job is alive however old it is.
|
||||
expect(Object.keys(cutoffs)).toEqual(["checkpointCutoff"]);
|
||||
});
|
||||
|
||||
test("heartbeat interval leaves room for a missed beat inside the checkpoint window", () => {
|
||||
@@ -78,9 +78,18 @@ describe("isJobInterrupted", () => {
|
||||
).toBe(true);
|
||||
});
|
||||
|
||||
test("a job running past the stale window is interrupted even with a fresh checkpoint", () => {
|
||||
test("a job that keeps checkpointing stays live however long ago it started", () => {
|
||||
expect(
|
||||
isJobInterrupted({ inProgress: true, startedAt: ago(3 * HOUR), lastCheckpoint: ago(MINUTE) }, NOW)
|
||||
).toBe(false);
|
||||
expect(
|
||||
isJobInterrupted({ inProgress: true, startedAt: ago(30 * HOUR), lastCheckpoint: ago(2 * MINUTE) }, NOW)
|
||||
).toBe(false);
|
||||
});
|
||||
|
||||
test("an old job whose checkpoint went stale is still interrupted", () => {
|
||||
expect(
|
||||
isJobInterrupted({ inProgress: true, startedAt: ago(3 * HOUR), lastCheckpoint: ago(11 * MINUTE) }, NOW)
|
||||
).toBe(true);
|
||||
});
|
||||
|
||||
@@ -235,6 +244,22 @@ describe("wiring", () => {
|
||||
).toBe(true);
|
||||
});
|
||||
|
||||
test("resumeInterruptedJob records a fresh start time for the resumed pass", () => {
|
||||
const resumeBody = HELPERS_SRC.slice(HELPERS_SRC.indexOf("export async function resumeInterruptedJob("));
|
||||
expect(/inProgress:\s*true,\s*startedAt:\s*new Date\(\)/.test(resumeBody)).toBe(true);
|
||||
});
|
||||
|
||||
test("updateMirrorJobProgress serializes writes per job so concurrent items cannot drop each other", () => {
|
||||
expect(
|
||||
/withKeyedLock\(`mirror-job:\$\{params\.jobId\}`,\s*\(\)\s*=>\s*updateMirrorJobProgressLocked\(params\)\)/.test(HELPERS_SRC)
|
||||
).toBe(true);
|
||||
});
|
||||
|
||||
test("processWithRetry checkpoints every completed item instead of the one that trips the interval", () => {
|
||||
expect(/await onCheckpoint\(jobId,\s*itemId,\s*publish\)/.test(CONCURRENCY_SRC)).toBe(true);
|
||||
expect(CONCURRENCY_SRC.includes("onCheckpoint(jobId, 'final')")).toBe(false);
|
||||
});
|
||||
|
||||
test("middleware keeps the recovery latch until the recovery promise settles", () => {
|
||||
expect(
|
||||
/recoveryPromise\s*\.catch\(\(\)\s*=>\s*false\)\s*\.finally\(\(\)\s*=>\s*\{\s*recoveryInFlight\s*=\s*false;\s*\}\)/.test(
|
||||
|
||||
@@ -2,11 +2,20 @@
|
||||
* Decides which in-progress mirror jobs count as interrupted and should be
|
||||
* resumed by recovery (issue #372).
|
||||
*
|
||||
* A job is interrupted when any of these hold:
|
||||
* A job is interrupted when either of these holds:
|
||||
* - it has a checkpoint older than the checkpoint window, or
|
||||
* - it has no checkpoint and is older than the checkpoint window (or has
|
||||
* no recorded start at all, as legacy rows may), or
|
||||
* - it started longer ago than the stale window, whatever its checkpoint.
|
||||
* no recorded start at all, as legacy rows may).
|
||||
*
|
||||
* A job that keeps checkpointing is alive, however long ago it started.
|
||||
* Running jobs refresh their checkpoint on a heartbeat between items, so a
|
||||
* missing or old checkpoint is the only evidence that the process behind a
|
||||
* job is gone. There used to be a third rule, "started more than two hours
|
||||
* ago, whatever the checkpoint says", which resumed large jobs while they
|
||||
* were still running: the resumed pass then re-processed the same
|
||||
* repositories alongside the original, which is one way release assets got
|
||||
* duplicated (#417). A job that hangs without ever finishing an item is a
|
||||
* different problem and is not what recovery is for.
|
||||
*
|
||||
* The grace period for jobs without a checkpoint is the point of this
|
||||
* module. Checkpoints are written when an item completes, so a job that has
|
||||
@@ -24,9 +33,6 @@ import type { mirrorJobs } from "@/lib/db/schema";
|
||||
/** A job with no checkpoint newer than this is considered inactive. */
|
||||
export const INTERRUPTED_CHECKPOINT_AGE_MS = 10 * 60 * 1000;
|
||||
|
||||
/** A job that started longer ago than this is considered stale regardless. */
|
||||
export const STALE_JOB_AGE_MS = 2 * 60 * 60 * 1000;
|
||||
|
||||
/**
|
||||
* How often a running job refreshes its checkpoint between item
|
||||
* completions. Kept well inside the checkpoint window so one missed beat
|
||||
@@ -37,14 +43,11 @@ export const JOB_HEARTBEAT_INTERVAL_MS = 2 * 60 * 1000;
|
||||
export interface InterruptedJobCutoffs {
|
||||
/** Checkpoints (or starts, for jobs without one) before this are too old. */
|
||||
checkpointCutoff: Date;
|
||||
/** Starts before this mark the job stale no matter what. */
|
||||
staleCutoff: Date;
|
||||
}
|
||||
|
||||
export function computeInterruptedJobCutoffs(now: Date = new Date()): InterruptedJobCutoffs {
|
||||
return {
|
||||
checkpointCutoff: new Date(now.getTime() - INTERRUPTED_CHECKPOINT_AGE_MS),
|
||||
staleCutoff: new Date(now.getTime() - STALE_JOB_AGE_MS),
|
||||
};
|
||||
}
|
||||
|
||||
@@ -61,10 +64,9 @@ export interface JobLivenessFields {
|
||||
export function isJobInterrupted(job: JobLivenessFields, now: Date = new Date()): boolean {
|
||||
if (!job.inProgress) return false;
|
||||
|
||||
const { checkpointCutoff, staleCutoff } = computeInterruptedJobCutoffs(now);
|
||||
|
||||
if (job.startedAt && job.startedAt < staleCutoff) return true;
|
||||
const { checkpointCutoff } = computeInterruptedJobCutoffs(now);
|
||||
|
||||
// A checkpoint is the job's own proof of life; its age decides on its own.
|
||||
if (job.lastCheckpoint) return job.lastCheckpoint < checkpointCutoff;
|
||||
|
||||
// No checkpoint yet: give the job the same window it would get after a
|
||||
@@ -82,7 +84,7 @@ export function buildInterruptedJobsCondition(
|
||||
table: typeof mirrorJobs,
|
||||
now: Date = new Date(),
|
||||
): SQL {
|
||||
const { checkpointCutoff, staleCutoff } = computeInterruptedJobCutoffs(now);
|
||||
const { checkpointCutoff } = computeInterruptedJobCutoffs(now);
|
||||
|
||||
return and(
|
||||
eq(table.inProgress, true),
|
||||
@@ -95,8 +97,6 @@ export function buildInterruptedJobsCondition(
|
||||
),
|
||||
// Checkpoint present but too old.
|
||||
lt(table.lastCheckpoint, checkpointCutoff),
|
||||
// Running far too long, whatever the checkpoint says.
|
||||
lt(table.startedAt, staleCutoff),
|
||||
),
|
||||
)!;
|
||||
}
|
||||
@@ -0,0 +1,101 @@
|
||||
import { describe, test, expect } from 'bun:test';
|
||||
import {
|
||||
repositoryIdentityKey,
|
||||
repositoryIdentityKeys,
|
||||
selectNewRepositoriesByIdentity,
|
||||
dedupeRepositoriesByIdentity,
|
||||
} from './repo-utils';
|
||||
|
||||
describe('repositoryIdentityKey', () => {
|
||||
test('is the host plus the full name, independent of the source row', () => {
|
||||
const a = repositoryIdentityKey({ sourceProvider: 'github', sourceUrl: 'https://github.com', normalizedFullName: 'acme/tools' });
|
||||
const b = repositoryIdentityKey({ sourceProvider: 'github', sourceUrl: 'https://github.com/', normalizedFullName: 'ACME/Tools' });
|
||||
expect(a).toBe(b);
|
||||
});
|
||||
|
||||
test('treats missing provider and url as github.com, like legacy rows', () => {
|
||||
expect(repositoryIdentityKey({ normalizedFullName: 'acme/tools' })).toBe(
|
||||
repositoryIdentityKey({ sourceProvider: 'github', sourceUrl: 'https://github.com', normalizedFullName: 'acme/tools' })
|
||||
);
|
||||
});
|
||||
|
||||
test('keeps the same name on two hosts apart', () => {
|
||||
const ghe = repositoryIdentityKey({ sourceProvider: 'github', sourceUrl: 'https://ghe.example.com', normalizedFullName: 'acme/tools' });
|
||||
const com = repositoryIdentityKey({ sourceProvider: 'github', sourceUrl: 'https://github.com', normalizedFullName: 'acme/tools' });
|
||||
expect(ghe).not.toBe(com);
|
||||
});
|
||||
});
|
||||
|
||||
describe('selectNewRepositoriesByIdentity', () => {
|
||||
test('skips a repository already tracked on the same host under another source', () => {
|
||||
const existing = repositoryIdentityKeys([
|
||||
{ sourceProvider: 'github', sourceUrl: 'https://github.com', normalizedFullName: 'acme/tools' },
|
||||
]);
|
||||
const { fresh, alreadyTracked } = selectNewRepositoriesByIdentity(
|
||||
[
|
||||
{ sourceProvider: 'github', sourceUrl: 'https://github.com', normalizedFullName: 'acme/tools', sourceId: 'public-only' },
|
||||
{ sourceProvider: 'github', sourceUrl: 'https://github.com', normalizedFullName: 'acme/new', sourceId: 'public-only' },
|
||||
],
|
||||
existing
|
||||
);
|
||||
expect(fresh.map((r) => r.normalizedFullName)).toEqual(['acme/new']);
|
||||
expect(alreadyTracked.map((r) => r.normalizedFullName)).toEqual(['acme/tools']);
|
||||
});
|
||||
|
||||
test('extends the existing set so a later source does not re-add the same repository', () => {
|
||||
const existing = new Set<string>();
|
||||
const first = selectNewRepositoriesByIdentity(
|
||||
[{ sourceProvider: 'github', sourceUrl: 'https://github.com', normalizedFullName: 'acme/tools' }],
|
||||
existing
|
||||
);
|
||||
const second = selectNewRepositoriesByIdentity(
|
||||
[{ sourceProvider: 'github', sourceUrl: 'https://github.com', normalizedFullName: 'acme/tools' }],
|
||||
existing
|
||||
);
|
||||
expect(first.fresh).toHaveLength(1);
|
||||
expect(second.fresh).toHaveLength(0);
|
||||
expect(second.alreadyTracked).toHaveLength(1);
|
||||
});
|
||||
|
||||
test('still allows the same name from a different host', () => {
|
||||
const existing = repositoryIdentityKeys([
|
||||
{ sourceProvider: 'github', sourceUrl: 'https://github.com', normalizedFullName: 'acme/tools' },
|
||||
]);
|
||||
const { fresh } = selectNewRepositoriesByIdentity(
|
||||
[{ sourceProvider: 'gitlab', sourceUrl: 'https://gitlab.com', normalizedFullName: 'acme/tools' }],
|
||||
existing
|
||||
);
|
||||
expect(fresh).toHaveLength(1);
|
||||
});
|
||||
});
|
||||
|
||||
describe('dedupeRepositoriesByIdentity', () => {
|
||||
test('keeps one row per upstream repository in a batch', () => {
|
||||
const { kept, dropped } = dedupeRepositoriesByIdentity([
|
||||
{ id: 'a', sourceId: 's1', sourceProvider: 'github', sourceUrl: 'https://github.com', normalizedFullName: 'acme/tools' },
|
||||
{ id: 'b', sourceId: 's2', sourceProvider: 'github', sourceUrl: 'https://github.com', normalizedFullName: 'acme/tools' },
|
||||
{ id: 'c', sourceId: 's1', sourceProvider: 'github', sourceUrl: 'https://github.com', normalizedFullName: 'acme/other' },
|
||||
]);
|
||||
expect(kept.map((r) => r.id)).toEqual(['a', 'c']);
|
||||
expect(dropped.map((r) => r.id)).toEqual(['b']);
|
||||
});
|
||||
|
||||
test('prefers a row linked to a source over a legacy row without one', () => {
|
||||
const { kept, dropped } = dedupeRepositoriesByIdentity([
|
||||
{ id: 'legacy', sourceId: null, normalizedFullName: 'acme/tools' },
|
||||
{ id: 'linked', sourceId: 's1', sourceProvider: 'github', sourceUrl: 'https://github.com', normalizedFullName: 'acme/tools' },
|
||||
]);
|
||||
expect(kept.map((r) => r.id)).toEqual(['linked']);
|
||||
expect(dropped.map((r) => r.id)).toEqual(['legacy']);
|
||||
});
|
||||
|
||||
test('leaves a batch without duplicates alone', () => {
|
||||
const rows = [
|
||||
{ id: 'a', sourceId: 's1', normalizedFullName: 'acme/a' },
|
||||
{ id: 'b', sourceId: 's1', normalizedFullName: 'acme/b' },
|
||||
];
|
||||
const { kept, dropped } = dedupeRepositoriesByIdentity(rows);
|
||||
expect(kept).toEqual(rows);
|
||||
expect(dropped).toEqual([]);
|
||||
});
|
||||
});
|
||||
@@ -93,3 +93,82 @@ export function repositoryDestinationColumns(
|
||||
): Pick<RepoInsert, 'destinationProvider' | 'destinationUrl'> {
|
||||
return { destinationProvider: destination.provider, destinationUrl: destination.url };
|
||||
}
|
||||
|
||||
/**
|
||||
* Upstream identity of a repository row: which host it lives on and its full
|
||||
* name there. The repositories table is unique on (userId, sourceId,
|
||||
* normalizedFullName) so that the same owner/name can be tracked on two
|
||||
* hosts, but two source rows can point at the same host (a personal token
|
||||
* source and a public-only source, both github.com), and a legacy row with a
|
||||
* NULL sourceId is distinct from a re-imported one. Discovery used the unique
|
||||
* index alone to decide what was new, so one upstream repository could end up
|
||||
* as two rows that both mirror to the same destination repository and get
|
||||
* processed together in one batch (one of the overlaps behind #417).
|
||||
*/
|
||||
export interface RepositoryIdentityFields {
|
||||
sourceProvider?: string | null;
|
||||
sourceUrl?: string | null;
|
||||
normalizedFullName: string;
|
||||
}
|
||||
|
||||
export function repositoryIdentityKey(row: RepositoryIdentityFields): string {
|
||||
const provider = (row.sourceProvider || 'github').trim().toLowerCase();
|
||||
const url = (row.sourceUrl || 'https://github.com').trim().toLowerCase().replace(/\/+$/, '');
|
||||
return `${provider}|${url}|${row.normalizedFullName.toLowerCase()}`;
|
||||
}
|
||||
|
||||
/**
|
||||
* Split discovered rows into the ones not yet tracked for this user on the
|
||||
* same host under any source, and the ones that are. Also drops repeats
|
||||
* within `candidates` themselves. `existing` is extended with every fresh
|
||||
* row, so a caller looping over several sources can pass the same set.
|
||||
*/
|
||||
export function selectNewRepositoriesByIdentity<T extends RepositoryIdentityFields>(
|
||||
candidates: T[],
|
||||
existing: Set<string>
|
||||
): { fresh: T[]; alreadyTracked: T[] } {
|
||||
const fresh: T[] = [];
|
||||
const alreadyTracked: T[] = [];
|
||||
for (const candidate of candidates) {
|
||||
const key = repositoryIdentityKey(candidate);
|
||||
if (existing.has(key)) {
|
||||
alreadyTracked.push(candidate);
|
||||
} else {
|
||||
existing.add(key);
|
||||
fresh.push(candidate);
|
||||
}
|
||||
}
|
||||
return { fresh, alreadyTracked };
|
||||
}
|
||||
|
||||
export function repositoryIdentityKeys(rows: Iterable<RepositoryIdentityFields>): Set<string> {
|
||||
const keys = new Set<string>();
|
||||
for (const row of rows) keys.add(repositoryIdentityKey(row));
|
||||
return keys;
|
||||
}
|
||||
|
||||
/**
|
||||
* Keep one row per upstream identity in a processing batch, so two rows that
|
||||
* resolve to the same destination repository are never mirrored or synced in
|
||||
* the same pass. A row linked to a source wins over a legacy row without one;
|
||||
* otherwise the first row in the batch is kept.
|
||||
*/
|
||||
export function dedupeRepositoriesByIdentity<
|
||||
T extends RepositoryIdentityFields & { sourceId?: string | null },
|
||||
>(rows: T[]): { kept: T[]; dropped: T[] } {
|
||||
const byKey = new Map<string, T>();
|
||||
const dropped: T[] = [];
|
||||
for (const row of rows) {
|
||||
const key = repositoryIdentityKey(row);
|
||||
const current = byKey.get(key);
|
||||
if (!current) {
|
||||
byKey.set(key, row);
|
||||
} else if (!current.sourceId && row.sourceId) {
|
||||
dropped.push(current);
|
||||
byKey.set(key, row);
|
||||
} else {
|
||||
dropped.push(row);
|
||||
}
|
||||
}
|
||||
return { kept: Array.from(byKey.values()), dropped };
|
||||
}
|
||||
@@ -465,6 +465,34 @@ describe.skipIf(!isChild)("Scheduler public-only sources (WP4)", () => {
|
||||
.filter((row) => row.normalizedFullName?.startsWith("pinned-org/"))
|
||||
.map((row) => row.name);
|
||||
expect(orgRepoNamesAfterSecondRun).toEqual(["existing", "newly-published"]);
|
||||
expect(repoInsertAttempts).toBeGreaterThan(attemptsAfterFirstRun);
|
||||
// Nothing new on the second run, so discovery does not even attempt an
|
||||
// insert: already-tracked repositories are filtered out by identity
|
||||
// before the write, not dropped by the unique index afterwards.
|
||||
expect(repoInsertAttempts).toBe(attemptsAfterFirstRun);
|
||||
});
|
||||
|
||||
test("re-discovery does not add a second row for a repository the user already tracks under another source on the same host", async () => {
|
||||
// Imported earlier through a personal token source on github.com, so it
|
||||
// has a different sourceId than the pinned public-only source.
|
||||
repoRows.push({
|
||||
userId: "user-1",
|
||||
sourceId: "source-personal",
|
||||
name: "shared",
|
||||
fullName: "pinned-org/shared",
|
||||
normalizedFullName: "pinned-org/shared",
|
||||
organization: "pinned-org",
|
||||
sourceProvider: "github",
|
||||
sourceUrl: "https://github.com",
|
||||
status: "mirrored",
|
||||
});
|
||||
orgRepoListings.set("pinned-org", [makeGitRepo("shared"), makeGitRepo("fresh")]);
|
||||
|
||||
await schedulerLoop();
|
||||
|
||||
const sharedRows = repoRows.filter((row) => row.normalizedFullName === "pinned-org/shared");
|
||||
expect(sharedRows).toHaveLength(1);
|
||||
expect(sharedRows[0].sourceId).toBe("source-personal");
|
||||
expect(repoRows.some((row) => row.normalizedFullName === "pinned-org/fresh")).toBe(true);
|
||||
expect(consoleLines.some((line) => line.includes("already tracked") && line.includes("pinned-org"))).toBe(true);
|
||||
});
|
||||
});
|
||||
@@ -5,7 +5,7 @@
|
||||
*/
|
||||
|
||||
import { db, configs, repositories, organizations } from '@/lib/db';
|
||||
import { eq, and, or } from 'drizzle-orm';
|
||||
import { eq, and, or, ne } from 'drizzle-orm';
|
||||
import { mirrorRepositoryToDestination, syncRepositoryOnDestination } from '@/lib/mirror-dispatch';
|
||||
import { usesPushEngine } from '@/lib/destination-connection';
|
||||
import { getDecryptedGitHubToken } from '@/lib/utils/config-encryption';
|
||||
@@ -13,7 +13,14 @@ import { formatDuration } from '@/lib/utils/duration-parser';
|
||||
import type { Repository } from '@/lib/db/schema';
|
||||
import type { Octokit } from '@octokit/rest';
|
||||
import { repoStatusEnum, repositoryVisibilityEnum } from '@/types/Repository';
|
||||
import { mergeGitReposPreferStarred, normalizeGitRepoToInsert, calcBatchSizeForInsert } from '@/lib/repo-utils';
|
||||
import {
|
||||
mergeGitReposPreferStarred,
|
||||
normalizeGitRepoToInsert,
|
||||
calcBatchSizeForInsert,
|
||||
repositoryIdentityKeys,
|
||||
selectNewRepositoriesByIdentity,
|
||||
dedupeRepositoriesByIdentity,
|
||||
} from '@/lib/repo-utils';
|
||||
import { isMirrorableGitHubRepo } from '@/lib/repo-eligibility';
|
||||
import { loadOrganizationForkPolicies, orgForkSkipDecision, resolveOrganizationSkipForks } from '@/lib/utils/mirror-overrides';
|
||||
import { createMirrorJob } from '@/lib/helpers';
|
||||
@@ -102,14 +109,19 @@ async function importRepositoriesFromSources(
|
||||
const orgForkOverrides = await loadOrganizationForkPolicies({ userId });
|
||||
|
||||
const existingRepos = await db
|
||||
.select({ normalizedFullName: repositories.normalizedFullName, sourceId: repositories.sourceId })
|
||||
.select({
|
||||
normalizedFullName: repositories.normalizedFullName,
|
||||
sourceProvider: repositories.sourceProvider,
|
||||
sourceUrl: repositories.sourceUrl,
|
||||
})
|
||||
.from(repositories)
|
||||
.where(eq(repositories.userId, userId));
|
||||
// Repositories are unique per source: the same full name under two sources
|
||||
// counts as two repositories.
|
||||
const existingRepoKeys = new Set(
|
||||
existingRepos.map(r => `${r.sourceId ?? ''}|${r.normalizedFullName}`)
|
||||
);
|
||||
// A repository is identified by its host and full name, not by the source
|
||||
// row that found it. Two sources can point at the same host (a personal
|
||||
// token and a public-only source, both github.com), and a repository the
|
||||
// user already tracks must not be inserted a second time under the other
|
||||
// source: both rows would mirror to the same destination repository.
|
||||
const existingRepoKeys = repositoryIdentityKeys(existingRepos);
|
||||
|
||||
for (const source of sources) {
|
||||
try {
|
||||
@@ -132,17 +144,21 @@ async function importRepositoriesFromSources(
|
||||
const allGithubRepos = mergeGitReposPreferStarred(basicAndForkedRepos, starredRepos);
|
||||
const mirrorableGithubRepos = allGithubRepos.filter(isMirrorableGitHubRepo);
|
||||
|
||||
const newRepos = mirrorableGithubRepos.filter(
|
||||
r => !existingRepoKeys.has(`${source.id}|${r.fullName.toLowerCase()}`)
|
||||
const candidates = mirrorableGithubRepos.map(repo =>
|
||||
normalizeGitRepoToInsert(repo, { userId, configId: config.id, sourceId: source.id })
|
||||
);
|
||||
const { fresh: reposToInsert, alreadyTracked } = selectNewRepositoriesByIdentity(
|
||||
candidates,
|
||||
existingRepoKeys
|
||||
);
|
||||
const newRepos = reposToInsert;
|
||||
if (alreadyTracked.length > 0) {
|
||||
console.log(`[Scheduler] ${alreadyTracked.length} repositories listed by source ${source.name} are already tracked for user ${userId} on the same host; not adding them again`);
|
||||
}
|
||||
|
||||
if (newRepos.length > 0) {
|
||||
console.log(`[Scheduler] Found ${newRepos.length} new repositories for user ${userId} on source ${source.name}`);
|
||||
|
||||
const reposToInsert = newRepos.map(repo =>
|
||||
normalizeGitRepoToInsert(repo, { userId, configId: config.id, sourceId: source.id })
|
||||
);
|
||||
|
||||
// Batch insert to avoid SQLite parameter limit
|
||||
const sample = reposToInsert[0];
|
||||
const columnCount = Object.keys(sample ?? {}).length || 1;
|
||||
@@ -203,16 +219,33 @@ async function rediscoverOrganizationRepositories(
|
||||
const { createSourceProviderFromSource } = await import('@/lib/source-providers');
|
||||
const sources = (await listSources(userId)).filter(source => source.enabled);
|
||||
|
||||
// An ignored organization is left alone entirely: rediscovering it would
|
||||
// insert its new repositories as imported and auto-mirror could pick them
|
||||
// up, which is what the user just asked to stop (#429).
|
||||
const includedOrgs = await db
|
||||
.select()
|
||||
.from(organizations)
|
||||
.where(
|
||||
and(
|
||||
eq(organizations.userId, userId),
|
||||
eq(organizations.isIncluded, true)
|
||||
eq(organizations.isIncluded, true),
|
||||
ne(organizations.status, 'ignored')
|
||||
)
|
||||
);
|
||||
|
||||
// Same identity rule as personal discovery: an organization pinned to a
|
||||
// public-only source must not re-insert repositories the personal source
|
||||
// already tracks on the same host (or the other way round).
|
||||
const existingRepos = await db
|
||||
.select({
|
||||
normalizedFullName: repositories.normalizedFullName,
|
||||
sourceProvider: repositories.sourceProvider,
|
||||
sourceUrl: repositories.sourceUrl,
|
||||
})
|
||||
.from(repositories)
|
||||
.where(eq(repositories.userId, userId));
|
||||
const existingRepoKeys = repositoryIdentityKeys(existingRepos);
|
||||
|
||||
for (const org of includedOrgs) {
|
||||
try {
|
||||
const source = findSourceForOrganization(org, sources);
|
||||
@@ -228,14 +261,17 @@ async function rediscoverOrganizationRepositories(
|
||||
repo => repo.isDisabled !== true && !(skipOrgForks && repo.isForked)
|
||||
);
|
||||
|
||||
if (mirrorableRepos.length > 0) {
|
||||
const repoRecords = mirrorableRepos.map(repo =>
|
||||
const { fresh: repoRecords, alreadyTracked } = selectNewRepositoriesByIdentity(
|
||||
mirrorableRepos.map(repo =>
|
||||
normalizeGitRepoToInsert(
|
||||
{ ...repo, organization: repo.organization ?? org.name },
|
||||
{ userId, configId: config.id, sourceId: source.id }
|
||||
)
|
||||
);
|
||||
),
|
||||
existingRepoKeys
|
||||
);
|
||||
|
||||
if (repoRecords.length > 0) {
|
||||
// Batch insert to avoid SQLite parameter limit
|
||||
const sample = repoRecords[0];
|
||||
const columnCount = Object.keys(sample ?? {}).length || 1;
|
||||
@@ -247,7 +283,10 @@ async function rediscoverOrganizationRepositories(
|
||||
.values(batch)
|
||||
.onConflictDoNothing({ target: [repositories.userId, repositories.sourceId, repositories.normalizedFullName] });
|
||||
}
|
||||
console.log(`[Scheduler] Re-discovered ${mirrorableRepos.length} repositories for organization ${org.name} on source ${source.name} for user ${userId} during ${phase}`);
|
||||
console.log(`[Scheduler] Re-discovered ${repoRecords.length} new repositories for organization ${org.name} on source ${source.name} for user ${userId} during ${phase}`);
|
||||
}
|
||||
if (alreadyTracked.length > 0) {
|
||||
console.log(`[Scheduler] Organization ${org.name}: ${alreadyTracked.length} repositories are already tracked for user ${userId} on the same host, not adding them again under source ${source.name}`);
|
||||
}
|
||||
|
||||
await db
|
||||
@@ -331,6 +370,24 @@ function sourceUsernamesBySourceId(sources: SourceRecord[]): Map<string, string>
|
||||
/**
|
||||
* Run scheduled mirror sync for a single user configuration
|
||||
*/
|
||||
/**
|
||||
* Keep one row per upstream repository in a scheduler batch. Duplicate rows
|
||||
* (the same repository imported under two sources on the same host) resolve
|
||||
* to one destination repository, and processing both in one Promise.all ran
|
||||
* every step twice against it. The dropped rows are logged so the duplicate
|
||||
* can be cleaned up; discovery no longer creates new ones.
|
||||
*/
|
||||
function dropDuplicateRowsForPass<T extends { fullName: string; sourceId?: string | null; sourceProvider?: string | null; sourceUrl?: string | null; normalizedFullName: string }>(
|
||||
rows: T[],
|
||||
pass: string
|
||||
): T[] {
|
||||
const { kept, dropped } = dedupeRepositoriesByIdentity(rows);
|
||||
for (const row of dropped) {
|
||||
console.warn(`[Scheduler] ${row.fullName} is tracked more than once (another row on the same host covers it); skipping the duplicate row in this ${pass} pass`);
|
||||
}
|
||||
return kept;
|
||||
}
|
||||
|
||||
async function runScheduledSync(config: any): Promise<void> {
|
||||
const userId = config.userId;
|
||||
console.log(`[Scheduler] Running scheduled sync for user ${userId}`);
|
||||
@@ -457,6 +514,11 @@ async function runScheduledSync(config: any): Promise<void> {
|
||||
console.log(`[Scheduler] Skipped ${forkSkippedCount - reposNeedingMirror.length} forked repositories from auto-mirror (organization skip forks)`);
|
||||
}
|
||||
|
||||
// Two rows for one upstream repository (imported under two sources
|
||||
// that point at the same host) resolve to one destination
|
||||
// repository; mirror it once per pass, not twice at the same time.
|
||||
reposNeedingMirror = dropDuplicateRowsForPass(reposNeedingMirror, 'auto-mirror');
|
||||
|
||||
if (reposNeedingMirror.length > 0) {
|
||||
console.log(`[Scheduler] Found ${reposNeedingMirror.length} repositories that need initial mirroring`);
|
||||
|
||||
@@ -584,6 +646,8 @@ async function runScheduledSync(config: any): Promise<void> {
|
||||
// Non-critical logging, ignore errors
|
||||
}
|
||||
|
||||
reposToSync = dropDuplicateRowsForPass(reposToSync, 'sync');
|
||||
|
||||
if (reposToSync.length === 0) {
|
||||
console.log(`[Scheduler] No repositories to sync for user ${userId}`);
|
||||
return;
|
||||
@@ -808,6 +872,7 @@ async function performInitialAutoStart(): Promise<void> {
|
||||
if (skippedCount > 0) {
|
||||
console.log(`[Scheduler] Skipped ${skippedCount} repositories from initial auto-mirror (autoMirror=${autoMirrorOwned}, autoMirrorStarred=${autoMirrorStarred})`);
|
||||
}
|
||||
reposNeedingMirror = dropDuplicateRowsForPass(reposNeedingMirror, 'initial auto-mirror');
|
||||
|
||||
if (reposNeedingMirror.length > 0) {
|
||||
console.log(`[Scheduler] Found ${reposNeedingMirror.length} repositories that need mirroring`);
|
||||
|
||||
@@ -165,3 +165,46 @@ describe("processWithRetry", () => {
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
describe("processWithRetry checkpointing", () => {
|
||||
test("records every completed item and publishes on the interval", async () => {
|
||||
const items = ["a", "b", "c", "d", "e"];
|
||||
const calls: Array<[string, string, boolean]> = [];
|
||||
|
||||
await processWithRetry(items, async (item) => item.toUpperCase(), {
|
||||
concurrencyLimit: 1,
|
||||
jobId: "job-1",
|
||||
getItemId: (item) => item,
|
||||
checkpointInterval: 2,
|
||||
onCheckpoint: async (jobId, itemId, publish) => {
|
||||
calls.push([jobId, itemId, publish]);
|
||||
},
|
||||
});
|
||||
|
||||
// One checkpoint per item, in completion order, and no "final" placeholder.
|
||||
expect(calls.map(([, itemId]) => itemId)).toEqual(items);
|
||||
expect(calls.every(([jobId]) => jobId === "job-1")).toBe(true);
|
||||
// Progress events only every second item.
|
||||
expect(calls.map(([, , publish]) => publish)).toEqual([false, true, false, true, false]);
|
||||
});
|
||||
|
||||
test("does not checkpoint an item that failed for good", async () => {
|
||||
const calls: string[] = [];
|
||||
|
||||
// processInParallel logs a failed item and carries on, so the call resolves.
|
||||
await processWithRetry(["ok", "bad"], async (item) => {
|
||||
if (item === "bad") throw new Error("nope");
|
||||
return item;
|
||||
}, {
|
||||
concurrencyLimit: 1,
|
||||
maxRetries: 0,
|
||||
jobId: "job-2",
|
||||
getItemId: (item) => item,
|
||||
onCheckpoint: async (_jobId, itemId) => {
|
||||
calls.push(itemId);
|
||||
},
|
||||
});
|
||||
|
||||
expect(calls).toEqual(["ok"]);
|
||||
});
|
||||
});
|
||||
@@ -93,8 +93,15 @@ export async function processWithRetry<T, R>(
|
||||
onRetry?: (item: T, error: Error, attempt: number) => void;
|
||||
jobId?: string; // Optional job ID for checkpointing
|
||||
getItemId?: (item: T) => string; // Function to get a unique ID for each item
|
||||
onCheckpoint?: (jobId: string, completedItemId: string) => Promise<void>; // Callback for checkpointing
|
||||
checkpointInterval?: number; // How many items to process before checkpointing
|
||||
/**
|
||||
* Called once for EVERY completed item, so a resume knows exactly what is
|
||||
* done. `publish` is true every `checkpointInterval` items: the callback
|
||||
* uses it to throttle progress events, not the checkpoint itself. It used
|
||||
* to be called only for the item that tripped the interval, and a resume
|
||||
* re-processed everything finished since the previous checkpoint.
|
||||
*/
|
||||
onCheckpoint?: (jobId: string, completedItemId: string, publish: boolean) => Promise<void>;
|
||||
checkpointInterval?: number; // How many completed items between published progress events
|
||||
} = {}
|
||||
): Promise<R[]> {
|
||||
const {
|
||||
@@ -130,16 +137,16 @@ export async function processWithRetry<T, R>(
|
||||
|
||||
const result = await processItem(item);
|
||||
|
||||
// Handle checkpointing if enabled
|
||||
// Record every completed item; publish progress on the interval.
|
||||
if (jobId && getItemId && onCheckpoint) {
|
||||
const itemId = getItemId(item);
|
||||
itemsProcessedSinceLastCheckpoint++;
|
||||
|
||||
// Checkpoint based on the interval
|
||||
if (itemsProcessedSinceLastCheckpoint >= checkpointInterval) {
|
||||
await onCheckpoint(jobId, itemId);
|
||||
const publish = itemsProcessedSinceLastCheckpoint >= checkpointInterval;
|
||||
if (publish) {
|
||||
itemsProcessedSinceLastCheckpoint = 0;
|
||||
}
|
||||
await onCheckpoint(jobId, itemId, publish);
|
||||
}
|
||||
|
||||
return result;
|
||||
@@ -186,12 +193,6 @@ export async function processWithRetry<T, R>(
|
||||
onProgress
|
||||
);
|
||||
|
||||
// Final checkpoint if there are remaining items since the last checkpoint
|
||||
if (jobId && getItemId && onCheckpoint && itemsProcessedSinceLastCheckpoint > 0) {
|
||||
// We don't have a specific item ID for the final checkpoint, so we'll use a placeholder
|
||||
await onCheckpoint(jobId, 'final');
|
||||
}
|
||||
|
||||
return results;
|
||||
}
|
||||
|
||||
@@ -309,8 +310,10 @@ export async function processWithResilience<T, R>(
|
||||
});
|
||||
}, JOB_HEARTBEAT_INTERVAL_MS);
|
||||
|
||||
// Define the checkpoint function
|
||||
const onCheckpoint = async (jobId: string, completedItemId: string) => {
|
||||
// Define the checkpoint function. Every completed item is written to the
|
||||
// job row so a resume skips exactly what is done; only every
|
||||
// checkpointInterval-th write also publishes a progress event.
|
||||
const onCheckpoint = async (jobId: string, completedItemId: string, publish: boolean = true) => {
|
||||
const itemName = items.find(item => getItemId(item) === completedItemId)
|
||||
? getItemName(items.find(item => getItemId(item) === completedItemId)!)
|
||||
: 'unknown';
|
||||
@@ -319,6 +322,7 @@ export async function processWithResilience<T, R>(
|
||||
jobId,
|
||||
completedItemId,
|
||||
message: `Processed item: ${itemName}`,
|
||||
silent: !publish,
|
||||
});
|
||||
};
|
||||
|
||||
|
||||
@@ -47,11 +47,17 @@ export async function createDefaultConfig({ userId, envOverrides = {} }: Default
|
||||
const giteaToken = envOverrides.giteaToken || process.env.GITEA_TOKEN || "";
|
||||
const giteaUsername = envOverrides.giteaUsername || process.env.GITEA_USERNAME || "";
|
||||
|
||||
// Schedule config from env - default to ENABLED
|
||||
const scheduleEnabled = envOverrides.scheduleEnabled ??
|
||||
(process.env.SCHEDULE_ENABLED === "false" ? false : true); // Default: ENABLED
|
||||
// Schedule config from env. Off unless the environment turns it on, with
|
||||
// the same rule as env-config-loader: SCHEDULE_ENABLED=true or any interval
|
||||
// variable. A fresh install used to start enabled on a plain 24h interval
|
||||
// counted from first login, which the settings card (clock mode only)
|
||||
// showed as "22:00 daily" (#427). Enabling from the card stores the clock
|
||||
// schedule and the browser timezone, so the card and the scheduler agree.
|
||||
const envInterval = process.env.SCHEDULE_INTERVAL || process.env.GITEA_MIRROR_INTERVAL;
|
||||
const scheduleEnabled = envOverrides.scheduleEnabled ??
|
||||
(process.env.SCHEDULE_ENABLED === "true" || !!envInterval);
|
||||
const scheduleInterval = envOverrides.scheduleInterval ??
|
||||
(process.env.SCHEDULE_INTERVAL || 86400); // Default: daily
|
||||
(envInterval || "0 22 * * *"); // Default: daily at 22:00 in scheduleTimezone
|
||||
const scheduleTimezone = normalizeTimezone(process.env.SCHEDULE_TIMEZONE || "UTC");
|
||||
let scheduleNextRun: Date | null = null;
|
||||
if (scheduleEnabled) {
|
||||
|
||||
@@ -82,6 +82,23 @@ test("mapDbScheduleToUi - infers clock mode for generated cron", () => {
|
||||
expect(mapped.timezone).toBe("Asia/Kolkata");
|
||||
});
|
||||
|
||||
test("mapDbScheduleToUi - reports interval mode for a plain interval instead of a clock schedule", () => {
|
||||
// What SCHEDULE_INTERVAL / GITEA_MIRROR_INTERVAL and older versions store.
|
||||
// The settings card must not present the 22:00 placeholder as saved (#427).
|
||||
const mapped = mapDbScheduleToUi(
|
||||
scheduleConfigSchema.parse({
|
||||
enabled: true,
|
||||
interval: "8h",
|
||||
timezone: "UTC",
|
||||
})
|
||||
);
|
||||
|
||||
expect(mapped.scheduleMode).toBe("interval");
|
||||
expect(mapped.interval).toBe(8 * 3600);
|
||||
expect(mapped.intervalExpression).toBe("8h");
|
||||
expect(mapped.timezone).toBe("UTC");
|
||||
});
|
||||
|
||||
test("includeCollaboratorRepos round-trips through UI -> DB -> UI when true", () => {
|
||||
const ui = buildMinimalUiConfigs({ includeCollaboratorRepos: true });
|
||||
const db = mapUiToDbConfig(
|
||||
|
||||
@@ -0,0 +1,115 @@
|
||||
/**
|
||||
* Unit tests for the per-key async mutex used to serialize release
|
||||
* reconciliation per destination repository (issue #417).
|
||||
*/
|
||||
|
||||
import { describe, expect, it } from "bun:test";
|
||||
import { keyedLockCount, withKeyedLock } from "@/lib/utils/keyed-mutex";
|
||||
|
||||
const tick = (ms = 0) => new Promise((resolve) => setTimeout(resolve, ms));
|
||||
|
||||
describe("withKeyedLock", () => {
|
||||
it("runs two callers on the same key strictly one after the other", async () => {
|
||||
const events: string[] = [];
|
||||
|
||||
const first = withKeyedLock("repo-a", async () => {
|
||||
events.push("a:start");
|
||||
await tick(10);
|
||||
events.push("a:end");
|
||||
});
|
||||
const second = withKeyedLock("repo-a", async () => {
|
||||
events.push("b:start");
|
||||
await tick(1);
|
||||
events.push("b:end");
|
||||
});
|
||||
|
||||
await Promise.all([first, second]);
|
||||
|
||||
expect(events).toEqual(["a:start", "a:end", "b:start", "b:end"]);
|
||||
});
|
||||
|
||||
it("runs callers on different keys concurrently", async () => {
|
||||
const events: string[] = [];
|
||||
|
||||
await Promise.all([
|
||||
withKeyedLock("repo-a", async () => {
|
||||
events.push("a:start");
|
||||
await tick(10);
|
||||
events.push("a:end");
|
||||
}),
|
||||
withKeyedLock("repo-b", async () => {
|
||||
events.push("b:start");
|
||||
await tick(10);
|
||||
events.push("b:end");
|
||||
}),
|
||||
]);
|
||||
|
||||
// Both started before either finished, so no serialization across keys.
|
||||
expect(events.slice(0, 2).sort()).toEqual(["a:start", "b:start"]);
|
||||
});
|
||||
|
||||
it("releases the key when the callback throws, and surfaces the error", async () => {
|
||||
const events: string[] = [];
|
||||
|
||||
const failing = withKeyedLock("repo-c", async () => {
|
||||
events.push("a:start");
|
||||
await tick(1);
|
||||
throw new Error("upload exploded");
|
||||
});
|
||||
const following = withKeyedLock("repo-c", async () => {
|
||||
events.push("b:start");
|
||||
return "ran anyway";
|
||||
});
|
||||
|
||||
await expect(failing).rejects.toThrow("upload exploded");
|
||||
expect(await following).toBe("ran anyway");
|
||||
expect(events).toEqual(["a:start", "b:start"]);
|
||||
});
|
||||
|
||||
it("keeps FIFO order for several waiters on one key", async () => {
|
||||
const order: number[] = [];
|
||||
|
||||
await Promise.all(
|
||||
[1, 2, 3, 4].map((n) =>
|
||||
withKeyedLock("repo-d", async () => {
|
||||
order.push(n);
|
||||
// Later callers ask for a shorter delay: without the lock they would
|
||||
// finish out of order.
|
||||
await tick(5 - n);
|
||||
})
|
||||
)
|
||||
);
|
||||
|
||||
expect(order).toEqual([1, 2, 3, 4]);
|
||||
});
|
||||
|
||||
it("returns the callback's value and forgets the key once nobody is queued", async () => {
|
||||
const before = keyedLockCount();
|
||||
|
||||
const value = await withKeyedLock("repo-e", async () => 42);
|
||||
expect(value).toBe(42);
|
||||
|
||||
// The cleanup runs in a microtask after the tail settles.
|
||||
await tick(0);
|
||||
expect(keyedLockCount()).toBe(before);
|
||||
});
|
||||
|
||||
it("serializes a second caller that arrives while the first is running", async () => {
|
||||
let holderRunning = false;
|
||||
let sawOverlap = false;
|
||||
|
||||
const holder = withKeyedLock("repo-f", async () => {
|
||||
holderRunning = true;
|
||||
await tick(15);
|
||||
holderRunning = false;
|
||||
});
|
||||
|
||||
await tick(1);
|
||||
const waiter = withKeyedLock("repo-f", async () => {
|
||||
if (holderRunning) sawOverlap = true;
|
||||
});
|
||||
|
||||
await Promise.all([holder, waiter]);
|
||||
expect(sawOverlap).toBe(false);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,63 @@
|
||||
/**
|
||||
* Per-key async mutex (issue #417).
|
||||
*
|
||||
* Release asset reconciliation is a read-then-write against the destination:
|
||||
* list the attachments a release already has, then upload whatever is missing.
|
||||
* The destination has no name-collision check on attachment upload (Gitea's and
|
||||
* Forgejo's CreateReleaseAttachment appends a row for every POST), so two
|
||||
* passes that overlap on the same release both see an asset as absent and both
|
||||
* upload it. N overlapping passes leave N copies of every asset.
|
||||
*
|
||||
* A per-row status guard cannot close that window on its own: two repository
|
||||
* rows (the same upstream repository imported under two sources, say) resolve
|
||||
* to the same destination repository, so the exclusion has to be keyed on the
|
||||
* destination rather than on the database row.
|
||||
*
|
||||
* This lock is process-local, which is where the overlapping passes seen in
|
||||
* #417 come from: the scheduler racing a manual run, recovery resuming a job
|
||||
* that is still running, or duplicate repository rows processed together by one
|
||||
* Promise.all.
|
||||
*/
|
||||
|
||||
/**
|
||||
* Tail of the queue for each key. It settles (and never rejects) once the last
|
||||
* queued caller is done, so the next caller only has to wait on it.
|
||||
*/
|
||||
const queues = new Map<string, Promise<void>>();
|
||||
|
||||
/**
|
||||
* Run `fn` with exclusive access to `key`. Callers on the same key run one at a
|
||||
* time in the order they arrived; callers on different keys are unaffected.
|
||||
* The lock is released whether `fn` resolves or throws, and the caller still
|
||||
* sees `fn`'s own result or error.
|
||||
*/
|
||||
export function withKeyedLock<T>(key: string, fn: () => Promise<T>): Promise<T> {
|
||||
const previous = queues.get(key) ?? Promise.resolve();
|
||||
|
||||
// `previous` never rejects, so a caller that threw still hands the key on.
|
||||
const result = previous.then(() => fn());
|
||||
|
||||
// The tail swallows the outcome: the next caller only needs to know the
|
||||
// previous one finished, and an unobserved rejection here would surface as
|
||||
// an unhandled rejection.
|
||||
const tail = result.then(
|
||||
() => undefined,
|
||||
() => undefined
|
||||
);
|
||||
queues.set(key, tail);
|
||||
|
||||
// Drop the key once nobody is queued behind this caller, so the map does not
|
||||
// grow one permanent entry per repository.
|
||||
void tail.then(() => {
|
||||
if (queues.get(key) === tail) {
|
||||
queues.delete(key);
|
||||
}
|
||||
});
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
/** Number of keys currently held or queued. For tests and diagnostics. */
|
||||
export function keyedLockCount(): number {
|
||||
return queues.size;
|
||||
}
|
||||
@@ -0,0 +1,68 @@
|
||||
/**
|
||||
* Atomic claim of a repository row for an in-flight operation (issue #417).
|
||||
*
|
||||
* Mirroring and syncing both used to read the row's status, decide the
|
||||
* repository was free, and then write the in-flight status in a second
|
||||
* statement. Two workers that started in the same tick both read "free" and
|
||||
* both went on to mirror the same repository, which is one of the ways release
|
||||
* assets ended up duplicated on the destination.
|
||||
*
|
||||
* The claim below is a single conditional UPDATE: the row moves into the
|
||||
* in-flight status only if it is not already "mirroring" or "syncing", and the
|
||||
* caller learns from the returned rows whether it won. A row whose in-flight
|
||||
* status has gone stale (older than the same two-hour window
|
||||
* isRepoCurrentlyMirroring has always used) can be taken over, so a process
|
||||
* that died mid-operation never locks a repository out permanently. Rows like
|
||||
* that are also reset to "failed" by resetStuckMirrorStatuses
|
||||
* (src/lib/stuck-status-recovery.ts), which runs on startup, in recovery and in
|
||||
* the scheduler loop.
|
||||
*/
|
||||
|
||||
import { and, eq, lt, notInArray, or, type SQL } from "drizzle-orm";
|
||||
import { db, repositories } from "@/lib/db";
|
||||
import {
|
||||
IN_FLIGHT_REPO_STATUSES,
|
||||
STUCK_IN_FLIGHT_THRESHOLD_MS,
|
||||
} from "@/lib/stuck-status-recovery";
|
||||
|
||||
/**
|
||||
* WHERE clause that matches one repository row only while no other run owns it.
|
||||
*/
|
||||
export function repositoryClaimCondition(
|
||||
repositoryId: string,
|
||||
now: Date = new Date()
|
||||
): SQL | undefined {
|
||||
const staleCutoff = new Date(now.getTime() - STUCK_IN_FLIGHT_THRESHOLD_MS);
|
||||
|
||||
return and(
|
||||
eq(repositories.id, repositoryId),
|
||||
or(
|
||||
notInArray(repositories.status, [...IN_FLIGHT_REPO_STATUSES]),
|
||||
lt(repositories.updatedAt, staleCutoff)
|
||||
)
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Move one repository row into an in-flight status, but only if no other run
|
||||
* already owns it. Returns true when this caller claimed the row and false when
|
||||
* another mirror or sync is in flight for it.
|
||||
*/
|
||||
export async function claimRepositoryForInFlightWork({
|
||||
repositoryId,
|
||||
set,
|
||||
now,
|
||||
}: {
|
||||
repositoryId: string;
|
||||
/** The full update payload, including the in-flight status itself. */
|
||||
set: Partial<typeof repositories.$inferInsert>;
|
||||
now?: Date;
|
||||
}): Promise<boolean> {
|
||||
const claimed = await db
|
||||
.update(repositories)
|
||||
.set(set)
|
||||
.where(repositoryClaimCondition(repositoryId, now))
|
||||
.returning({ id: repositories.id });
|
||||
|
||||
return claimed.length > 0;
|
||||
}
|
||||
@@ -113,9 +113,49 @@ export const POST: APIRoute = async ({ request, locals }) => {
|
||||
// action === "approve": create backup first (safety), then trigger sync
|
||||
const decryptedConfig = decryptConfigTokens(config as unknown as Config);
|
||||
|
||||
// Claim the rows before the background work starts: one conditional UPDATE
|
||||
// per repository, so the status flips to "syncing" immediately (the UI asks
|
||||
// for that) without stealing a row that a mirror or sync already owns
|
||||
// (#417). The sync itself is then told the row is claimed.
|
||||
const claimedRepos: typeof repos = [];
|
||||
for (const repo of repos) {
|
||||
const claimed = await db
|
||||
.update(repositories)
|
||||
.set({
|
||||
status: "syncing",
|
||||
errorMessage: null,
|
||||
updatedAt: new Date(),
|
||||
})
|
||||
.where(
|
||||
and(
|
||||
eq(repositories.id, repo.id),
|
||||
eq(repositories.status, "pending-approval"),
|
||||
),
|
||||
)
|
||||
.returning({ id: repositories.id });
|
||||
|
||||
if (claimed.length > 0) {
|
||||
claimedRepos.push(repo);
|
||||
} else {
|
||||
console.log(
|
||||
`[ApproveSync] Skipping ${repo.name}: it is no longer waiting for approval (another run may have picked it up).`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
if (claimedRepos.length === 0) {
|
||||
return new Response(
|
||||
JSON.stringify({
|
||||
success: false,
|
||||
message: "No pending-approval repositories were available to sync.",
|
||||
}),
|
||||
{ status: 409, headers: { "Content-Type": "application/json" } },
|
||||
);
|
||||
}
|
||||
|
||||
// Process in background
|
||||
setTimeout(async () => {
|
||||
for (const repo of repos) {
|
||||
for (const repo of claimedRepos) {
|
||||
try {
|
||||
const { getGiteaRepoOwnerAsync } = await import("@/lib/gitea");
|
||||
const repoOwner = await getGiteaRepoOwnerAsync({ config, repository: repo });
|
||||
@@ -163,6 +203,8 @@ export const POST: APIRoute = async ({ request, locals }) => {
|
||||
config,
|
||||
repository: repoData,
|
||||
skipForcePushDetection: true,
|
||||
// The row was claimed above, before this background task started.
|
||||
alreadyClaimed: true,
|
||||
});
|
||||
console.log(`[ApproveSync] Sync completed for approved repository: ${repo.name}`);
|
||||
} catch (error) {
|
||||
@@ -174,23 +216,11 @@ export const POST: APIRoute = async ({ request, locals }) => {
|
||||
}
|
||||
}, 0);
|
||||
|
||||
// Immediately update status to syncing for responsiveness
|
||||
for (const repo of repos) {
|
||||
await db
|
||||
.update(repositories)
|
||||
.set({
|
||||
status: "syncing",
|
||||
errorMessage: null,
|
||||
updatedAt: new Date(),
|
||||
})
|
||||
.where(eq(repositories.id, repo.id));
|
||||
}
|
||||
|
||||
return new Response(
|
||||
JSON.stringify({
|
||||
success: true,
|
||||
message: `Approved sync for ${repos.length} repository(ies). Backup + sync started.`,
|
||||
repositories: repos.map((repo) => ({
|
||||
message: `Approved sync for ${claimedRepos.length} repository(ies). Backup + sync started.`,
|
||||
repositories: claimedRepos.map((repo) => ({
|
||||
...repo,
|
||||
status: "syncing",
|
||||
errorMessage: null,
|
||||
|
||||
@@ -0,0 +1,223 @@
|
||||
/**
|
||||
* Route tests for PATCH /api/organizations/:id/status with cascade: ignoring
|
||||
* an organization ignores its repositories so the scheduler leaves them
|
||||
* alone, and including it again restores them (#429).
|
||||
*
|
||||
* Runs against the real schema in an in-memory SQLite database. The route
|
||||
* imports @/lib/db, which is replaced with that database through
|
||||
* mock.module; bun's mock.module is process-wide and leaks into other test
|
||||
* files, so this file registers nothing in the shared process and re-runs
|
||||
* itself in an isolated child (same harness as ../[id].test.ts).
|
||||
*/
|
||||
import { describe, test, expect, mock, beforeEach } from "bun:test";
|
||||
import { Database } from "bun:sqlite";
|
||||
import { drizzle } from "drizzle-orm/bun-sqlite";
|
||||
import { readFileSync } from "node:fs";
|
||||
import { join } from "node:path";
|
||||
import * as schema from "@/lib/db/schema";
|
||||
|
||||
const CHILD_FLAG = "GM_ORG_STATUS_ROUTE_ISOLATED";
|
||||
const isChild = !!process.env[CHILD_FLAG];
|
||||
|
||||
if (!isChild) {
|
||||
test("organization status route (cascade) - isolated child suite", () => {
|
||||
const res = Bun.spawnSync({
|
||||
cmd: [process.execPath, "test", import.meta.path],
|
||||
env: { ...process.env, [CHILD_FLAG]: "1" },
|
||||
stdout: "pipe",
|
||||
stderr: "pipe",
|
||||
});
|
||||
if (res.exitCode !== 0) {
|
||||
console.error(res.stdout.toString());
|
||||
console.error(res.stderr.toString());
|
||||
}
|
||||
expect(res.exitCode).toBe(0);
|
||||
}, 60_000);
|
||||
}
|
||||
|
||||
const sqlite = new Database(":memory:");
|
||||
const db = drizzle({ client: sqlite });
|
||||
|
||||
/**
|
||||
* Apply every migration in drizzle/ in journal order. The shared test setup
|
||||
* stubs drizzle's migrator, so the SQL is run here the way the migrator
|
||||
* would: one file per journal entry, statements split on its breakpoint.
|
||||
*/
|
||||
function applyMigrations(): void {
|
||||
const folder = join(process.cwd(), "drizzle");
|
||||
const journal = JSON.parse(readFileSync(join(folder, "meta", "_journal.json"), "utf8")) as {
|
||||
entries: { tag: string }[];
|
||||
};
|
||||
for (const entry of journal.entries) {
|
||||
const sql = readFileSync(join(folder, `${entry.tag}.sql`), "utf8");
|
||||
for (const statement of sql.split("--> statement-breakpoint")) {
|
||||
if (statement.trim()) sqlite.run(statement);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (isChild) {
|
||||
applyMigrations();
|
||||
// The seed below only needs the two tables the route touches.
|
||||
sqlite.run("PRAGMA foreign_keys = OFF");
|
||||
|
||||
mock.module("@/lib/auth-guards", () => ({
|
||||
requireAuthenticatedUserId: mock(async () => ({ userId: "user-1" })),
|
||||
}));
|
||||
|
||||
mock.module("@/lib/db", () => ({ ...schema, db }));
|
||||
}
|
||||
|
||||
const { PATCH } = isChild
|
||||
? await import("./status")
|
||||
: { PATCH: undefined as any };
|
||||
|
||||
function patch(orgId: string, body: Record<string, unknown>) {
|
||||
return PATCH({
|
||||
params: { id: orgId },
|
||||
request: new Request(`http://localhost/api/organizations/${orgId}/status`, {
|
||||
method: "PATCH",
|
||||
headers: { "Content-Type": "application/json" },
|
||||
body: JSON.stringify(body),
|
||||
}),
|
||||
locals: { session: { userId: "user-1" } },
|
||||
} as any);
|
||||
}
|
||||
|
||||
type RepoSeed = {
|
||||
id: string;
|
||||
userId?: string;
|
||||
organization: string | null;
|
||||
status: string;
|
||||
lastMirrored?: number | null;
|
||||
};
|
||||
|
||||
function seedRepo({ id, userId = "user-1", organization, status, lastMirrored = null }: RepoSeed) {
|
||||
sqlite.run(
|
||||
`INSERT INTO repositories (id, user_id, config_id, name, full_name, normalized_full_name, url, clone_url, owner, organization, default_branch, status, last_mirrored)
|
||||
VALUES (?, ?, 'config-1', ?, ?, ?, ?, ?, ?, ?, 'main', ?, ?)`,
|
||||
[
|
||||
id,
|
||||
userId,
|
||||
id,
|
||||
`${organization ?? "someone"}/${id}`,
|
||||
`${organization ?? "someone"}/${id}`.toLowerCase(),
|
||||
`https://github.com/${organization ?? "someone"}/${id}`,
|
||||
`https://github.com/${organization ?? "someone"}/${id}.git`,
|
||||
organization ?? "someone",
|
||||
organization,
|
||||
status,
|
||||
lastMirrored,
|
||||
]
|
||||
);
|
||||
}
|
||||
|
||||
function repoStatuses(): Record<string, string> {
|
||||
const rows = sqlite.query("SELECT id, status FROM repositories ORDER BY id").all() as { id: string; status: string }[];
|
||||
return Object.fromEntries(rows.map((row) => [row.id, row.status]));
|
||||
}
|
||||
|
||||
function orgStatus(id: string): string | undefined {
|
||||
const row = sqlite.query("SELECT status FROM organizations WHERE id = ?").get(id) as { status: string } | null;
|
||||
return row?.status;
|
||||
}
|
||||
|
||||
beforeEach(() => {
|
||||
if (!isChild) return;
|
||||
sqlite.run("DELETE FROM repositories");
|
||||
sqlite.run("DELETE FROM organizations");
|
||||
sqlite.run(
|
||||
`INSERT INTO organizations (id, user_id, config_id, name, normalized_name, avatar_url, status)
|
||||
VALUES ('org-acme', 'user-1', 'config-1', 'acme', 'acme', '', 'mirrored'),
|
||||
('org-beta', 'user-1', 'config-1', 'beta', 'beta', '', 'mirrored'),
|
||||
('org-theirs', 'user-2', 'config-2', 'acme', 'acme', '', 'mirrored')`
|
||||
);
|
||||
const mirroredAt = 1_700_000_000;
|
||||
seedRepo({ id: "acme-mirrored", organization: "acme", status: "mirrored", lastMirrored: mirroredAt });
|
||||
seedRepo({ id: "acme-synced", organization: "acme", status: "synced", lastMirrored: mirroredAt });
|
||||
seedRepo({ id: "acme-imported", organization: "acme", status: "imported" });
|
||||
seedRepo({ id: "acme-failed", organization: "acme", status: "failed", lastMirrored: mirroredAt });
|
||||
seedRepo({ id: "acme-syncing", organization: "acme", status: "syncing", lastMirrored: mirroredAt });
|
||||
seedRepo({ id: "acme-mirroring", organization: "acme", status: "mirroring" });
|
||||
seedRepo({ id: "acme-deleting", organization: "acme", status: "deleting", lastMirrored: mirroredAt });
|
||||
seedRepo({ id: "acme-ignored-before", organization: "acme", status: "ignored" });
|
||||
seedRepo({ id: "beta-mirrored", organization: "beta", status: "mirrored", lastMirrored: mirroredAt });
|
||||
seedRepo({ id: "personal-mirrored", organization: null, status: "mirrored", lastMirrored: mirroredAt });
|
||||
seedRepo({ id: "theirs-acme-mirrored", userId: "user-2", organization: "acme", status: "mirrored", lastMirrored: mirroredAt });
|
||||
});
|
||||
|
||||
describe.skipIf(!isChild)("PATCH /api/organizations/:id/status", () => {
|
||||
test("ignoring with cascade ignores the organization's idle repositories only", async () => {
|
||||
const response = await patch("org-acme", { status: "ignored", cascade: true });
|
||||
expect(response.status).toBe(200);
|
||||
const data = await response.json();
|
||||
expect(data.success).toBe(true);
|
||||
expect(data.repositoriesChanged).toBe(4);
|
||||
expect(orgStatus("org-acme")).toBe("ignored");
|
||||
|
||||
expect(repoStatuses()).toEqual({
|
||||
"acme-mirrored": "ignored",
|
||||
"acme-synced": "ignored",
|
||||
"acme-imported": "ignored",
|
||||
"acme-failed": "ignored",
|
||||
// In flight and on the way out: left to finish on their own.
|
||||
"acme-syncing": "syncing",
|
||||
"acme-mirroring": "mirroring",
|
||||
"acme-deleting": "deleting",
|
||||
"acme-ignored-before": "ignored",
|
||||
// Other organization, personal repository, other user: untouched.
|
||||
"beta-mirrored": "mirrored",
|
||||
"personal-mirrored": "mirrored",
|
||||
"theirs-acme-mirrored": "mirrored",
|
||||
});
|
||||
});
|
||||
|
||||
test("including with cascade restores mirrored repositories as mirrored and the rest as imported", async () => {
|
||||
await patch("org-acme", { status: "ignored", cascade: true });
|
||||
const response = await patch("org-acme", { status: "imported", cascade: true });
|
||||
expect(response.status).toBe(200);
|
||||
const data = await response.json();
|
||||
// The four the ignore touched plus the one ignored beforehand.
|
||||
expect(data.repositoriesChanged).toBe(5);
|
||||
expect(orgStatus("org-acme")).toBe("imported");
|
||||
|
||||
const statuses = repoStatuses();
|
||||
expect(statuses["acme-mirrored"]).toBe("mirrored");
|
||||
expect(statuses["acme-synced"]).toBe("mirrored");
|
||||
expect(statuses["acme-failed"]).toBe("mirrored");
|
||||
expect(statuses["acme-imported"]).toBe("imported");
|
||||
expect(statuses["acme-ignored-before"]).toBe("imported");
|
||||
expect(statuses["beta-mirrored"]).toBe("mirrored");
|
||||
expect(statuses["theirs-acme-mirrored"]).toBe("mirrored");
|
||||
});
|
||||
|
||||
test("without cascade only the organization changes", async () => {
|
||||
const response = await patch("org-acme", { status: "ignored" });
|
||||
expect(response.status).toBe(200);
|
||||
const data = await response.json();
|
||||
expect(data.repositoriesChanged).toBe(0);
|
||||
expect(orgStatus("org-acme")).toBe("ignored");
|
||||
expect(repoStatuses()["acme-mirrored"]).toBe("mirrored");
|
||||
});
|
||||
|
||||
test("a status other than ignored or imported never touches repositories", async () => {
|
||||
const response = await patch("org-acme", { status: "failed", cascade: true });
|
||||
expect(response.status).toBe(200);
|
||||
const data = await response.json();
|
||||
expect(data.repositoriesChanged).toBe(0);
|
||||
expect(repoStatuses()["acme-mirrored"]).toBe("mirrored");
|
||||
});
|
||||
|
||||
test("answers 404 for another user's organization and changes nothing", async () => {
|
||||
const response = await patch("org-theirs", { status: "ignored", cascade: true });
|
||||
expect(response.status).toBe(404);
|
||||
expect(orgStatus("org-theirs")).toBe("mirrored");
|
||||
expect(repoStatuses()["theirs-acme-mirrored"]).toBe("mirrored");
|
||||
});
|
||||
|
||||
test("rejects an unknown status", async () => {
|
||||
const response = await patch("org-acme", { status: "paused", cascade: true });
|
||||
expect(response.status).toBe(400);
|
||||
expect(orgStatus("org-acme")).toBe("mirrored");
|
||||
});
|
||||
});
|
||||
@@ -1,8 +1,68 @@
|
||||
import type { APIContext } from "astro";
|
||||
import { db, organizations } from "@/lib/db";
|
||||
import { eq, and } from "drizzle-orm";
|
||||
import { db, organizations, repositories } from "@/lib/db";
|
||||
import { eq, and, notInArray, sql } from "drizzle-orm";
|
||||
import { createSecureErrorResponse } from "@/lib/utils";
|
||||
import { requireAuthenticatedUserId } from "@/lib/auth-guards";
|
||||
import { IN_FLIGHT_REPO_STATUSES } from "@/lib/stuck-status-recovery";
|
||||
|
||||
const validStatuses = ["imported", "mirroring", "mirrored", "failed", "ignored"] as const;
|
||||
type OrganizationStatus = (typeof validStatuses)[number];
|
||||
|
||||
/**
|
||||
* Repository statuses an organization-level ignore leaves alone. Rows in
|
||||
* flight are locked by their status (see repo-status-claim.ts) and settle on
|
||||
* their own when the job finishes, a deletion is already on its way out, and
|
||||
* rows already ignored stay that way.
|
||||
*/
|
||||
const IGNORE_CASCADE_EXCLUDED_STATUSES = [...IN_FLIGHT_REPO_STATUSES, "deleting", "ignored"];
|
||||
|
||||
/**
|
||||
* Ignoring an organization also ignores its repositories, so the scheduler
|
||||
* stops syncing them (#429). Including it again restores the ones the ignore
|
||||
* touched: a repository that has been mirrored goes back to mirrored, one
|
||||
* that never was goes back to imported.
|
||||
*/
|
||||
async function cascadeOrganizationStatus(
|
||||
userId: string,
|
||||
organizationName: string,
|
||||
status: OrganizationStatus,
|
||||
now: Date
|
||||
): Promise<number> {
|
||||
if (status === "ignored") {
|
||||
const changed = await db
|
||||
.update(repositories)
|
||||
.set({ status: "ignored", updatedAt: now })
|
||||
.where(
|
||||
and(
|
||||
eq(repositories.userId, userId),
|
||||
eq(repositories.organization, organizationName),
|
||||
notInArray(repositories.status, IGNORE_CASCADE_EXCLUDED_STATUSES)
|
||||
)
|
||||
)
|
||||
.returning({ id: repositories.id });
|
||||
return changed.length;
|
||||
}
|
||||
|
||||
if (status === "imported") {
|
||||
const changed = await db
|
||||
.update(repositories)
|
||||
.set({
|
||||
status: sql`CASE WHEN ${repositories.lastMirrored} IS NOT NULL THEN 'mirrored' ELSE 'imported' END`,
|
||||
updatedAt: now,
|
||||
})
|
||||
.where(
|
||||
and(
|
||||
eq(repositories.userId, userId),
|
||||
eq(repositories.organization, organizationName),
|
||||
eq(repositories.status, "ignored")
|
||||
)
|
||||
)
|
||||
.returning({ id: repositories.id });
|
||||
return changed.length;
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
export async function PATCH({ params, request, locals }: APIContext) {
|
||||
try {
|
||||
@@ -13,6 +73,7 @@ export async function PATCH({ params, request, locals }: APIContext) {
|
||||
const { id } = params;
|
||||
const body = await request.json();
|
||||
const { status } = body;
|
||||
const cascade = body.cascade === true;
|
||||
|
||||
if (!id) {
|
||||
return new Response(
|
||||
@@ -28,7 +89,6 @@ export async function PATCH({ params, request, locals }: APIContext) {
|
||||
}
|
||||
|
||||
// Validate the status
|
||||
const validStatuses = ["imported", "mirroring", "mirrored", "failed", "ignored"];
|
||||
if (!validStatuses.includes(status)) {
|
||||
return new Response(
|
||||
JSON.stringify({
|
||||
@@ -42,12 +102,14 @@ export async function PATCH({ params, request, locals }: APIContext) {
|
||||
);
|
||||
}
|
||||
|
||||
const now = new Date();
|
||||
|
||||
// Update the organization status
|
||||
const [updatedOrg] = await db
|
||||
.update(organizations)
|
||||
.set({
|
||||
.set({
|
||||
status,
|
||||
updatedAt: new Date()
|
||||
updatedAt: now,
|
||||
})
|
||||
.where(
|
||||
and(
|
||||
@@ -70,10 +132,15 @@ export async function PATCH({ params, request, locals }: APIContext) {
|
||||
);
|
||||
}
|
||||
|
||||
const repositoriesChanged = cascade
|
||||
? await cascadeOrganizationStatus(userId, updatedOrg.name, status, now)
|
||||
: 0;
|
||||
|
||||
return new Response(
|
||||
JSON.stringify({
|
||||
success: true,
|
||||
organization: updatedOrg,
|
||||
repositoriesChanged,
|
||||
}),
|
||||
{
|
||||
status: 200,
|
||||
|
||||
+16
-10
@@ -9,6 +9,8 @@ import {
|
||||
mergeGitReposPreferStarred,
|
||||
normalizeGitRepoToInsert,
|
||||
calcBatchSizeForInsert,
|
||||
repositoryIdentityKeys,
|
||||
selectNewRepositoriesByIdentity,
|
||||
} from "@/lib/repo-utils";
|
||||
import { loadOrganizationForkPolicies } from "@/lib/utils/mirror-overrides";
|
||||
import { requireAuthenticatedUserId } from "@/lib/auth-guards";
|
||||
@@ -165,7 +167,8 @@ export const POST: APIRoute = async ({ request, locals }) => {
|
||||
tx
|
||||
.select({
|
||||
normalizedFullName: repositories.normalizedFullName,
|
||||
sourceId: repositories.sourceId,
|
||||
sourceProvider: repositories.sourceProvider,
|
||||
sourceUrl: repositories.sourceUrl,
|
||||
})
|
||||
.from(repositories)
|
||||
.where(eq(repositories.userId, userId)),
|
||||
@@ -175,17 +178,20 @@ export const POST: APIRoute = async ({ request, locals }) => {
|
||||
.where(eq(organizations.userId, userId)),
|
||||
]);
|
||||
|
||||
// The same full name under two sources is two different repositories.
|
||||
const existingRepoKeys = new Set(
|
||||
existingRepos.map((r) => `${r.sourceId ?? ""}|${r.normalizedFullName}`)
|
||||
);
|
||||
// A repository is identified by its host and full name, whichever
|
||||
// source row lists it. The same name on two hosts is two
|
||||
// repositories; the same repository seen by two sources on one host
|
||||
// (a personal token and a public-only source) is one, and must not
|
||||
// become two rows that mirror to the same destination.
|
||||
const existingRepoKeys = repositoryIdentityKeys(existingRepos);
|
||||
const existingOrgMap = new Map(existingOrgs.map((o) => [o.normalizedName, o.status]));
|
||||
|
||||
insertedRepos = newRepos.filter(
|
||||
(r) =>
|
||||
!existingRepoKeys.has(`${source.id}|${r.normalizedFullName}`) &&
|
||||
(!r.organization || !ignoredOrgNames.has(r.organization.toLowerCase()))
|
||||
);
|
||||
insertedRepos = selectNewRepositoriesByIdentity(
|
||||
newRepos.filter(
|
||||
(r) => !r.organization || !ignoredOrgNames.has(r.organization.toLowerCase())
|
||||
),
|
||||
existingRepoKeys
|
||||
).fresh;
|
||||
insertedOrgs = newOrgs.filter((o) => !existingOrgMap.has(o.normalizedName));
|
||||
|
||||
// Update previously failed orgs that now succeeded. The pin must
|
||||
|
||||
@@ -148,6 +148,17 @@ if (isChild) {
|
||||
select: () => ({
|
||||
from: (table: unknown) => ({
|
||||
where: (cond: unknown) => ({
|
||||
// Awaited directly by the existing-repository lookup that keeps
|
||||
// one row per upstream repository; the inserted rows stand in
|
||||
// for the table.
|
||||
then: (
|
||||
onfulfilled: (rows: OrgRow[]) => unknown,
|
||||
onrejected?: (reason: unknown) => unknown
|
||||
) =>
|
||||
Promise.resolve(table === repositoriesTable ? [...insertedRepoRows] : []).then(
|
||||
onfulfilled,
|
||||
onrejected
|
||||
),
|
||||
limit: async (): Promise<OrgRow[]> => {
|
||||
const leaves = conditionLeafValues(cond);
|
||||
if (table === organizationsTable) {
|
||||
|
||||
@@ -15,7 +15,12 @@ import {
|
||||
normalizeSourceUrl,
|
||||
} from "@/lib/source-providers/kinds";
|
||||
import type { SourceRecord } from "@/lib/sources";
|
||||
import { normalizeGitRepoToInsert, calcBatchSizeForInsert } from "@/lib/repo-utils";
|
||||
import {
|
||||
normalizeGitRepoToInsert,
|
||||
calcBatchSizeForInsert,
|
||||
repositoryIdentityKeys,
|
||||
selectNewRepositoriesByIdentity,
|
||||
} from "@/lib/repo-utils";
|
||||
import { resolveOrganizationSkipForks } from "@/lib/utils/mirror-overrides";
|
||||
import { requireAuthenticatedUserId } from "@/lib/auth-guards";
|
||||
|
||||
@@ -259,12 +264,31 @@ export const POST: APIRoute = async ({ request, locals }) => {
|
||||
);
|
||||
|
||||
// Insert repositories. The normalizer stamps the source provider and URL.
|
||||
const repoRecords = mirrorableRepos.map((repo) =>
|
||||
normalizeGitRepoToInsert(
|
||||
{ ...repo, organization: repo.organization ?? orgData.name },
|
||||
{ userId, configId, sourceId: source.id }
|
||||
)
|
||||
// Repositories the user already tracks on this host under another source
|
||||
// (the personal source, say) are not inserted a second time: two rows for
|
||||
// one upstream repository would both mirror to the same destination.
|
||||
const existingRepos = await db
|
||||
.select({
|
||||
normalizedFullName: repositories.normalizedFullName,
|
||||
sourceProvider: repositories.sourceProvider,
|
||||
sourceUrl: repositories.sourceUrl,
|
||||
})
|
||||
.from(repositories)
|
||||
.where(eq(repositories.userId, userId));
|
||||
const { fresh: repoRecords, alreadyTracked } = selectNewRepositoriesByIdentity(
|
||||
mirrorableRepos.map((repo) =>
|
||||
normalizeGitRepoToInsert(
|
||||
{ ...repo, organization: repo.organization ?? orgData.name },
|
||||
{ userId, configId, sourceId: source.id }
|
||||
)
|
||||
),
|
||||
repositoryIdentityKeys(existingRepos)
|
||||
);
|
||||
if (alreadyTracked.length > 0) {
|
||||
console.log(
|
||||
`[Organization import] ${alreadyTracked.length} repositories of ${orgData.name} are already tracked for user ${userId} on this host; not adding them again under source ${source.name}`
|
||||
);
|
||||
}
|
||||
|
||||
// Batch insert repositories to avoid SQLite parameter limit
|
||||
// Compute batch size based on column count
|
||||
|
||||
+2
-1
@@ -38,7 +38,8 @@
|
||||
"esbuild@>=0.27.3 <0.28.1": ">=0.28.1",
|
||||
"sharp": "^0.35.4",
|
||||
"svgo": "^4.1.0",
|
||||
"js-yaml": "^4.3.2"
|
||||
"js-yaml": "^4.3.2",
|
||||
"smol-toml": "^1.7.1"
|
||||
}
|
||||
}
|
||||
}
|
||||
Generated
+8
-7
@@ -9,6 +9,7 @@ overrides:
|
||||
sharp: ^0.35.4
|
||||
svgo: ^4.1.0
|
||||
js-yaml: ^4.3.2
|
||||
smol-toml: ^1.7.1
|
||||
|
||||
importers:
|
||||
|
||||
@@ -2022,8 +2023,8 @@ packages:
|
||||
engines: {node: '>=20.19.5', npm: '>=10.8.2'}
|
||||
hasBin: true
|
||||
|
||||
smol-toml@1.7.0:
|
||||
resolution: {integrity: sha512-aqVvWoyO21L23mb+drl4RmMXbf6N7FdHjAhTRA9ZBL7apWBgfWC16KjrASI+1p9GAroljyMHj6fK67i0UiTNvQ==}
|
||||
smol-toml@1.8.0:
|
||||
resolution: {integrity: sha512-kCZr2V3ch9i00x8zXRhjUNVcjG9ijES5dDudkXvUVCT5QlJNQWElSJdZqyPemffHoLNUYwOcou0Fy+ojN0uHSQ==}
|
||||
engines: {node: '>= 18'}
|
||||
|
||||
source-map-js@1.2.1:
|
||||
@@ -2364,7 +2365,7 @@ snapshots:
|
||||
picomatch: 4.0.5
|
||||
retext-smartypants: 6.2.0
|
||||
shiki: 4.3.1
|
||||
smol-toml: 1.7.0
|
||||
smol-toml: 1.8.0
|
||||
unified: 11.0.5
|
||||
|
||||
'@astrojs/internal-helpers@0.11.0':
|
||||
@@ -2375,7 +2376,7 @@ snapshots:
|
||||
picomatch: 4.0.5
|
||||
retext-smartypants: 6.2.0
|
||||
shiki: 4.3.1
|
||||
smol-toml: 1.7.0
|
||||
smol-toml: 1.8.0
|
||||
unified: 11.0.5
|
||||
|
||||
'@astrojs/markdown-remark@7.2.1':
|
||||
@@ -2640,7 +2641,7 @@ snapshots:
|
||||
dependencies:
|
||||
'@emnapi/core': 1.11.1
|
||||
'@emnapi/runtime': 1.11.1
|
||||
'@napi-rs/wasm-runtime': 1.1.6(@emnapi/core@1.11.1)(@emnapi/runtime@1.11.1)
|
||||
'@napi-rs/wasm-runtime': 1.2.3(@emnapi/core@1.11.1)(@emnapi/runtime@1.11.1)
|
||||
optional: true
|
||||
|
||||
'@bruits/satteri-win32-arm64-msvc@0.10.5':
|
||||
@@ -3295,7 +3296,7 @@ snapshots:
|
||||
picomatch: 4.0.5
|
||||
semver: 7.8.5
|
||||
shiki: 4.3.1
|
||||
smol-toml: 1.7.0
|
||||
smol-toml: 1.8.0
|
||||
svgo: 4.1.0
|
||||
tinyclip: 0.1.15
|
||||
tinyexec: 1.2.4
|
||||
@@ -4680,7 +4681,7 @@ snapshots:
|
||||
arg: 5.0.2
|
||||
sax: 1.6.0
|
||||
|
||||
smol-toml@1.7.0: {}
|
||||
smol-toml@1.8.0: {}
|
||||
|
||||
source-map-js@1.2.1: {}
|
||||
|
||||
|
||||
Reference in new issue
Block a user