mirror of
https://github.com/RayLabsHQ/gitea-mirror.git
synced 2026-11-04 00:02:23 +01:00
Compare commits
1
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
 Arunavo Ray
|
7e95dc640b |
No files matched your search
@@ -28,13 +28,6 @@ on:
|
||||
schedule:
|
||||
- cron: '0 0 * * 0' # Weekly security scan on Sunday at midnight
|
||||
|
||||
# One build per ref at a time. A merge and the release bump that follows it
|
||||
# land on main seconds apart and used to race each other for the latest tag
|
||||
# (#425). Branch and PR builds cancel the older run; tag builds always finish.
|
||||
concurrency:
|
||||
group: docker-build-${{ github.ref }}
|
||||
cancel-in-progress: ${{ !startsWith(github.ref, 'refs/tags/') }}
|
||||
|
||||
env:
|
||||
REGISTRY: ghcr.io
|
||||
IMAGE: ${{ github.repository }}
|
||||
@@ -97,9 +90,9 @@ jobs:
|
||||
echo "APP_VERSION=${APP_VERSION}" >> $GITHUB_OUTPUT
|
||||
echo "Using version tag: ${TAG_VERSION}"
|
||||
else
|
||||
echo "VERSION=edge" >> $GITHUB_OUTPUT
|
||||
echo "VERSION=latest" >> $GITHUB_OUTPUT
|
||||
echo "APP_VERSION=dev" >> $GITHUB_OUTPUT
|
||||
echo "No version tag, using 'edge'"
|
||||
echo "No version tag, using 'latest'"
|
||||
fi
|
||||
|
||||
# Keep version files aligned automatically for tag-based releases
|
||||
@@ -123,15 +116,12 @@ jobs:
|
||||
images: ${{ env.REGISTRY }}/${{ env.IMAGE }}
|
||||
labels: |
|
||||
org.opencontainers.image.revision=${{ env.SHA }}
|
||||
# latest follows the release tag, not main: a main build can carry
|
||||
# an unreleased package.json and must never overwrite a release.
|
||||
# Pre-release tags (v1.2.3-rc.1) keep latest on the last stable one.
|
||||
tags: |
|
||||
type=edge,branch=main
|
||||
type=edge,branch=$repo.default_branch
|
||||
type=semver,pattern=v{{version}}
|
||||
type=sha,prefix=,suffix=,format=short
|
||||
type=raw,value=latest,enable=${{ startsWith(github.ref, 'refs/tags/v') && !contains(github.ref, '-') }}
|
||||
type=raw,value=${{ steps.tag_version.outputs.VERSION }},enable=${{ startsWith(github.ref, 'refs/tags/v') }}
|
||||
type=raw,value=latest,enable={{is_default_branch}}
|
||||
type=raw,value=${{ steps.tag_version.outputs.VERSION }}
|
||||
type=ref,event=pr,prefix=pr-
|
||||
|
||||
# Build and push Docker image
|
||||
@@ -233,10 +223,7 @@ jobs:
|
||||
if: github.event_name != 'pull_request'
|
||||
with:
|
||||
command: cves,recommendations
|
||||
# Scan the image this run pushed: the release tag on a tag build,
|
||||
# edge on a main or scheduled build. The action rejects digest
|
||||
# references ("missing or wrong image input").
|
||||
image: ${{ env.REGISTRY }}/${{ env.IMAGE }}:${{ steps.tag_version.outputs.VERSION }}
|
||||
image: ${{ env.REGISTRY }}/${{ env.IMAGE }}:latest
|
||||
sarif-file: scout-results.sarif
|
||||
summary: true
|
||||
exit-code: false
|
||||
|
||||
@@ -41,31 +41,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
|
||||
- Proper documentation for multi-URL access patterns
|
||||
- Comprehensive fix report documentation
|
||||
|
||||
### Changed
|
||||
- A new configuration starts with scheduling off unless `SCHEDULE_ENABLED=true`, `SCHEDULE_INTERVAL` or `GITEA_MIRROR_INTERVAL` turns it on, the same rule the environment loader uses; the built-in default schedule is the daily 22:00 clock schedule the automation card shows, so enabling from the card and the scheduler agree from the first save (#427)
|
||||
|
||||
### Fixed
|
||||
- The `latest` Docker image tag no longer lags behind a release (#425)
|
||||
- A merge and the version bump that follows it land on main seconds apart; both built the image and both pushed `latest`, and on v3.36.1 the older build finished last, so `latest` carried 3.36.0 until the weekly rebuild replaced it
|
||||
- The workflow runs one build per ref at a time, `latest` is pushed only by a stable release tag build, main builds push `edge` and the short commit sha, and the security scan looks at the image the run just pushed
|
||||
- The automation card shows the schedule that actually runs (#427)
|
||||
- A fresh install started with scheduling on and a plain 24 hour interval counted from first login, and the card, which only knows clock schedules, showed the 22:00 placeholder as if it were saved; the same misreport hit Docker installs with `GITEA_MIRROR_INTERVAL=8h`
|
||||
- When a plain interval is stored (from `SCHEDULE_INTERVAL`, `GITEA_MIRROR_INTERVAL` or an older version) the card says what runs and where it came from, and leaves frequency and start time unset until one is picked
|
||||
- The timezone chip shows the stored timezone instead of the browser's, and becomes a one-click switch to the browser timezone when the two differ
|
||||
- Ignoring an organization stops its repositories from syncing (#429)
|
||||
- Ignore Organization only changed the organization row; its mirrored repositories kept syncing and rediscovery kept importing new ones for it
|
||||
- The organization's idle repositories are ignored with it (rows in flight or being deleted finish on their own) and restored when it is included again, mirrored ones as mirrored and the rest as imported; rediscovery skips ignored organizations
|
||||
- Organization cards name the configured destination instead of always saying Gitea (#430)
|
||||
- The Nix snippets in the README, NIX.md and the deployment guide import the module before enabling `services.gitea-mirror`, and put the flake input in `flake.nix` where it belongs (#426)
|
||||
- Crash recovery no longer resumes a job that is still running
|
||||
- A job older than two hours was treated as interrupted even while it checkpointed every two minutes, so recovery started a second pass over the same repositories alongside the original; the age rule is gone and only a missing or stale checkpoint marks a job interrupted
|
||||
- Every completed item is now recorded in the job's checkpoint instead of one in every N, so a real resume skips exactly what was done; progress events are still throttled
|
||||
- Concurrent items completing at the same time could overwrite each other's checkpoint; job progress writes are serialized per job
|
||||
- A resumed job records a fresh start time
|
||||
- The same upstream repository is no longer imported twice under two sources that point at the same host
|
||||
- Discovery decided what was new by (source, full name), so an organization pinned to a public-only github.com source and a personal github.com token source each inserted their own row for one repository, and both rows mirrored to the same destination in the same scheduler batch
|
||||
- Auto import, organization re-discovery and the two import endpoints now identify a repository by its host and full name whichever source lists it; the same name on a different host is still a separate repository
|
||||
- A scheduler pass mirrors or syncs one row per upstream repository and logs the duplicate rows it skips, so existing duplicates never run side by side
|
||||
- A per-organization Mirror Destination equal to the organization's own name is kept instead of being dropped (#416)
|
||||
- The editor treated a typed value matching the organization name as "reset to default" and saved no override, which only holds under the preserve strategy; under single-org the default is the destination organization, so the repositories went there
|
||||
- The organization card now shows the default for the configured strategy in the preview, the placeholder, the helper text and the reset button, and marks any stored destination as custom
|
||||
|
||||
@@ -46,13 +46,9 @@ nix run --extra-experimental-features 'nix-command flakes' .#gitea-mirror
|
||||
|
||||
### 4. NixOS System Service
|
||||
```nix
|
||||
# flake.nix
|
||||
inputs.gitea-mirror.url = "github:RayLabsHQ/gitea-mirror";
|
||||
|
||||
# configuration.nix (needs `inputs` passed through specialArgs)
|
||||
{ inputs, ... }:
|
||||
# configuration.nix
|
||||
{
|
||||
imports = [ inputs.gitea-mirror.nixosModules.default ];
|
||||
inputs.gitea-mirror.url = "github:RayLabsHQ/gitea-mirror";
|
||||
|
||||
services.gitea-mirror = {
|
||||
enable = true;
|
||||
|
||||
@@ -174,15 +174,10 @@ nix profile install --extra-experimental-features 'nix-command flakes' github:Ra
|
||||
gitea-mirror
|
||||
```
|
||||
|
||||
**NixOS users** - add the flake input, import the module, then enable the service:
|
||||
**NixOS users** - add to your configuration:
|
||||
```nix
|
||||
# flake.nix
|
||||
inputs.gitea-mirror.url = "github:RayLabsHQ/gitea-mirror";
|
||||
|
||||
# configuration.nix (needs `inputs` passed through specialArgs)
|
||||
{ inputs, ... }:
|
||||
{
|
||||
imports = [ inputs.gitea-mirror.nixosModules.default ];
|
||||
inputs.gitea-mirror.url = "github:RayLabsHQ/gitea-mirror";
|
||||
|
||||
services.gitea-mirror = {
|
||||
enable = true;
|
||||
|
||||
@@ -6,7 +6,7 @@
|
||||
"name": "gitea-mirror",
|
||||
"dependencies": {
|
||||
"@astrojs/mdx": "^7.0.3",
|
||||
"@astrojs/node": "^11.1.5",
|
||||
"@astrojs/node": "^11.0.2",
|
||||
"@astrojs/react": "^6.0.1",
|
||||
"@better-auth/api-key": "1.7.2",
|
||||
"@better-auth/oauth-provider": "1.7.2",
|
||||
@@ -94,7 +94,6 @@
|
||||
"lodash": "^4.18.1",
|
||||
"picomatch": "^4.0.4",
|
||||
"sharp": "^0.35.4",
|
||||
"smol-toml": "^1.7.1",
|
||||
"svgo": "^4.1.0",
|
||||
},
|
||||
"packages": {
|
||||
@@ -144,7 +143,7 @@
|
||||
|
||||
"@astrojs/mdx": ["@astrojs/mdx@7.0.3", "", { "dependencies": { "@astrojs/internal-helpers": "0.10.1", "@astrojs/markdown-remark": "7.2.1", "@mdx-js/mdx": "^3.1.1", "acorn": "^8.16.0", "es-module-lexer": "^2.0.0", "estree-util-visit": "^2.0.0", "hast-util-to-html": "^9.0.5", "piccolore": "^0.1.3", "rehype-raw": "^7.0.0", "remark-gfm": "^4.0.1", "remark-smartypants": "^3.0.2", "source-map": "^0.7.6", "unist-util-visit": "^5.1.0", "vfile": "^6.0.3" }, "peerDependencies": { "@astrojs/markdown-satteri": "^0.3.1", "astro": "^7.0.0" }, "optionalPeers": ["@astrojs/markdown-satteri"] }, "sha512-RxyIwU0uFam5ftwqKOjpIdhnFxZ/kEikeimLyQy3eGXbHT8WgRGzzesOIHVU8+m9TY8ag5WVOyvV24/GyqPdPQ=="],
|
||||
|
||||
"@astrojs/node": ["@astrojs/node@11.1.5", "", { "dependencies": { "@astrojs/internal-helpers": "0.11.0", "send": "^1.2.1", "server-destroy": "^1.0.1" }, "peerDependencies": { "astro": "^7.2.1" } }, "sha512-CgA+LmG4UWqHxg/Vdc5MdxjjJNZSjKYQQx4uEUwnbj+1ZD2CYumJI3OWg2twsVdj2FlYsB5yvmCC54Re+OK+UQ=="],
|
||||
"@astrojs/node": ["@astrojs/node@11.0.2", "", { "dependencies": { "@astrojs/internal-helpers": "0.10.1", "send": "^1.2.1", "server-destroy": "^1.0.1" }, "peerDependencies": { "astro": "^7.0.0" } }, "sha512-/ijULxT+A5Cm8wSwWZ2vgqfim1b05D6B8n/a9l6MMA4FCotIH73g7fL7y76XojKXpTe75FVvQH92OxsMqea9kQ=="],
|
||||
|
||||
"@astrojs/prism": ["@astrojs/prism@4.0.2", "", { "dependencies": { "prismjs": "^1.30.0" } }, "sha512-KTivpmnz6lDsC6o9H4+DNm2SrE/GHzw8cNAvEJwAvUT+eoaEnn/4NtbDNfRRaxaJHdp15gf+tfHAWiXR4wB3BA=="],
|
||||
|
||||
@@ -1474,7 +1473,7 @@
|
||||
|
||||
"sisteransi": ["sisteransi@1.0.5", "", {}, "sha512-bLGGlR1QxBcynn2d5YmDX4MGjlZvy2MRBDRNHLJ8VI6l6+9FUiyTFNJ0IveOSP0bcXgVDPRcfGqA0pjaqUpfVg=="],
|
||||
|
||||
"smol-toml": ["smol-toml@1.8.0", "", {}, "sha512-kCZr2V3ch9i00x8zXRhjUNVcjG9ijES5dDudkXvUVCT5QlJNQWElSJdZqyPemffHoLNUYwOcou0Fy+ojN0uHSQ=="],
|
||||
"smol-toml": ["smol-toml@1.6.0", "", {}, "sha512-4zemZi0HvTnYwLfrpk/CF9LOd9Lt87kAt50GnqhMpyF9U3poDAP2+iukq2bZsO/ufegbYehBkqINbsWxj4l4cw=="],
|
||||
|
||||
"sonner": ["sonner@2.0.7", "", { "peerDependencies": { "react": "^18.0.0 || ^19.0.0 || ^19.0.0-rc", "react-dom": "^18.0.0 || ^19.0.0 || ^19.0.0-rc" } }, "sha512-W6ZN4p58k8aDKA4XPcx2hpIQXBRAgyiWVkYhT7CvK6D3iAu7xjvVyhQHg2/iaKJZ1XVJ4r7XuwGL+WGEK37i9w=="],
|
||||
|
||||
@@ -1696,8 +1695,6 @@
|
||||
|
||||
"@astrojs/compiler-binding-wasm32-wasi/@napi-rs/wasm-runtime": ["@napi-rs/wasm-runtime@1.2.3", "", { "dependencies": { "@tybys/wasm-util": "^0.10.3" }, "peerDependencies": { "@emnapi/core": "^1.7.1 || ^2.0.0-alpha.4", "@emnapi/runtime": "^1.7.1 || ^2.0.0-alpha.4" } }, "sha512-UMduMbqO5s5zF2NkNacMT/yK5Y5QiKvWr2+50bzIIxFDwVJ2h49b+oyjaCGPhJxd2/gC2x39EHv/gHVuu36x2Q=="],
|
||||
|
||||
"@astrojs/node/@astrojs/internal-helpers": ["@astrojs/internal-helpers@0.11.0", "", { "dependencies": { "@types/hast": "^3.0.4", "@types/mdast": "^4.0.4", "js-yaml": "^4.3.0", "picomatch": "^4.0.4", "retext-smartypants": "^6.2.0", "shiki": "^4.0.2", "smol-toml": "^1.6.0", "unified": "^11.0.5" } }, "sha512-3rzxJ+xbo0+8YyqOzLziIN32wmsHdCjEVz2sGOpRxJ+Ben/KiLph4ItxBy1abEL+E8fkRzqjg0rfXmaHJGw9JA=="],
|
||||
|
||||
"@astrojs/react/@vitejs/plugin-react": ["@vitejs/plugin-react@5.2.0", "", { "dependencies": { "@babel/core": "^7.29.0", "@babel/plugin-transform-react-jsx-self": "^7.27.1", "@babel/plugin-transform-react-jsx-source": "^7.27.1", "@rolldown/pluginutils": "1.0.0-rc.3", "@types/babel__core": "^7.20.5", "react-refresh": "^0.18.0" }, "peerDependencies": { "vite": "^4.2.0 || ^5.0.0 || ^6.0.0 || ^7.0.0 || ^8.0.0" } }, "sha512-YmKkfhOAi3wsB1PhJq5Scj3GXMn3WvtQ/JC0xoopuHoXSdmtdStOpFrYaT1kie2YgFBcIe64ROzMYRjCrYOdYw=="],
|
||||
|
||||
"@astrojs/react/vite": ["vite@8.1.5", "", { "dependencies": { "lightningcss": "^1.32.0", "picomatch": "^4.0.5", "postcss": "^8.5.17", "rolldown": "~1.1.5", "tinyglobby": "^0.2.17" }, "optionalDependencies": { "fsevents": "~2.3.3" }, "peerDependencies": { "@types/node": "^20.19.0 || >=22.12.0", "@vitejs/devtools": "^0.3.0", "esbuild": "^0.27.0 || ^0.28.0", "jiti": ">=1.21.0", "less": "^4.0.0", "sass": "^1.70.0", "sass-embedded": "^1.70.0", "stylus": ">=0.54.8", "sugarss": "^5.0.0", "terser": "^5.16.0", "tsx": "^4.8.1", "yaml": "^2.4.2" }, "optionalPeers": ["@types/node", "@vitejs/devtools", "esbuild", "jiti", "less", "sass", "sass-embedded", "stylus", "sugarss", "terser", "tsx", "yaml"], "bin": { "vite": "bin/vite.js" } }, "sha512-7ULLwsCdYx/nRyrpiEwvqb5TFHrMVZyBt+rg/OAXT7rgj/z+DtTDyKFeLAdDkubDVDKD8jOsndmy7m55XcfUsw=="],
|
||||
|
||||
@@ -113,9 +113,9 @@
|
||||
url = "https://registry.npmjs.org/@astrojs/mdx/-/mdx-7.0.3.tgz";
|
||||
hash = "sha512-RxyIwU0uFam5ftwqKOjpIdhnFxZ/kEikeimLyQy3eGXbHT8WgRGzzesOIHVU8+m9TY8ag5WVOyvV24/GyqPdPQ==";
|
||||
};
|
||||
"@astrojs/node@11.1.5" = fetchurl {
|
||||
url = "https://registry.npmjs.org/@astrojs/node/-/node-11.1.5.tgz";
|
||||
hash = "sha512-CgA+LmG4UWqHxg/Vdc5MdxjjJNZSjKYQQx4uEUwnbj+1ZD2CYumJI3OWg2twsVdj2FlYsB5yvmCC54Re+OK+UQ==";
|
||||
"@astrojs/node@11.0.2" = fetchurl {
|
||||
url = "https://registry.npmjs.org/@astrojs/node/-/node-11.0.2.tgz";
|
||||
hash = "sha512-/ijULxT+A5Cm8wSwWZ2vgqfim1b05D6B8n/a9l6MMA4FCotIH73g7fL7y76XojKXpTe75FVvQH92OxsMqea9kQ==";
|
||||
};
|
||||
"@astrojs/prism@4.0.2" = fetchurl {
|
||||
url = "https://registry.npmjs.org/@astrojs/prism/-/prism-4.0.2.tgz";
|
||||
@@ -3205,9 +3205,9 @@
|
||||
url = "https://registry.npmjs.org/sisteransi/-/sisteransi-1.0.5.tgz";
|
||||
hash = "sha512-bLGGlR1QxBcynn2d5YmDX4MGjlZvy2MRBDRNHLJ8VI6l6+9FUiyTFNJ0IveOSP0bcXgVDPRcfGqA0pjaqUpfVg==";
|
||||
};
|
||||
"smol-toml@1.8.0" = fetchurl {
|
||||
url = "https://registry.npmjs.org/smol-toml/-/smol-toml-1.8.0.tgz";
|
||||
hash = "sha512-kCZr2V3ch9i00x8zXRhjUNVcjG9ijES5dDudkXvUVCT5QlJNQWElSJdZqyPemffHoLNUYwOcou0Fy+ojN0uHSQ==";
|
||||
"smol-toml@1.6.0" = fetchurl {
|
||||
url = "https://registry.npmjs.org/smol-toml/-/smol-toml-1.6.0.tgz";
|
||||
hash = "sha512-4zemZi0HvTnYwLfrpk/CF9LOd9Lt87kAt50GnqhMpyF9U3poDAP2+iukq2bZsO/ufegbYehBkqINbsWxj4l4cw==";
|
||||
};
|
||||
"sonner@2.0.7" = fetchurl {
|
||||
url = "https://registry.npmjs.org/sonner/-/sonner-2.0.7.tgz";
|
||||
|
||||
@@ -115,15 +115,10 @@ Access at http://localhost:4321, sign up (first user is admin), and configure ev
|
||||
|
||||
### Production Configuration
|
||||
|
||||
For production with custom domain and firewall. Like every snippet on this
|
||||
page, it needs the module imported first; `services.gitea-mirror` does not
|
||||
exist until `gitea-mirror.nixosModules.default` is in your modules list.
|
||||
For production with custom domain and firewall:
|
||||
|
||||
```nix
|
||||
{ inputs, ... }:
|
||||
{
|
||||
imports = [ inputs.gitea-mirror.nixosModules.default ];
|
||||
|
||||
services.gitea-mirror = {
|
||||
enable = true;
|
||||
host = "0.0.0.0";
|
||||
|
||||
@@ -2,8 +2,8 @@ apiVersion: v2
|
||||
name: gitea-mirror
|
||||
description: Kubernetes helm chart for gitea-mirror
|
||||
type: application
|
||||
version: 3.36.2
|
||||
appVersion: "3.36.2"
|
||||
version: 3.35.1
|
||||
appVersion: "3.35.1"
|
||||
icon: https://github.com/RayLabsHQ/gitea-mirror/blob/main/.github/assets/logo.png
|
||||
keywords:
|
||||
- git
|
||||
|
||||
+2
-3
@@ -1,7 +1,7 @@
|
||||
{
|
||||
"name": "gitea-mirror",
|
||||
"type": "module",
|
||||
"version": "3.36.2",
|
||||
"version": "3.35.1",
|
||||
"engines": {
|
||||
"bun": ">=1.2.9"
|
||||
},
|
||||
@@ -51,7 +51,6 @@
|
||||
"fast-uri": "^3.1.5",
|
||||
"fast-xml-builder": "^1.1.7",
|
||||
"js-yaml": "^4.3.2",
|
||||
"smol-toml": "^1.7.1",
|
||||
"kysely": "^0.28.17",
|
||||
"lodash": "^4.18.1",
|
||||
"picomatch": "^4.0.4",
|
||||
@@ -60,7 +59,7 @@
|
||||
},
|
||||
"dependencies": {
|
||||
"@astrojs/mdx": "^7.0.3",
|
||||
"@astrojs/node": "^11.1.5",
|
||||
"@astrojs/node": "^11.0.2",
|
||||
"@astrojs/react": "^6.0.1",
|
||||
"@better-auth/api-key": "1.7.2",
|
||||
"@better-auth/oauth-provider": "1.7.2",
|
||||
|
||||
@@ -68,24 +68,6 @@ const retentionPeriods = [
|
||||
{ label: "3 months", value: 7776000 },
|
||||
];
|
||||
|
||||
/** "every 8 hours", "every 30 minutes", "every 2 days" for a plain interval. */
|
||||
function formatIntervalSeconds(interval: number | string | undefined): string {
|
||||
const seconds = typeof interval === "number" ? interval : Number(interval);
|
||||
if (!Number.isFinite(seconds) || seconds <= 0) return "on a fixed interval";
|
||||
const units: [number, string][] = [
|
||||
[86400, "day"],
|
||||
[3600, "hour"],
|
||||
[60, "minute"],
|
||||
];
|
||||
for (const [size, name] of units) {
|
||||
if (seconds % size === 0 || seconds >= size * 2) {
|
||||
const count = Math.round(seconds / size);
|
||||
return `every ${count === 1 ? name : `${count} ${name}s`}`;
|
||||
}
|
||||
}
|
||||
return `every ${seconds} seconds`;
|
||||
}
|
||||
|
||||
function getCleanupInterval(retentionSeconds: number): number {
|
||||
const days = retentionSeconds / 86400;
|
||||
if (days <= 1) return 21600; // 6 hours
|
||||
@@ -144,22 +126,16 @@ export function AutomationSettings({
|
||||
? Intl.DateTimeFormat().resolvedOptions().timeZone || "UTC"
|
||||
: "UTC";
|
||||
|
||||
// The timezone the scheduler really uses. It used to be shown as the
|
||||
// browser timezone whenever UTC was stored, which made the card promise a
|
||||
// local 22:00 while the sync ran at 22:00 UTC (#427). The chip below offers
|
||||
// a one-click switch to the browser timezone when the two differ.
|
||||
const effectiveTimezone = scheduleConfig.timezone || browserTimezone;
|
||||
const timezoneDiffers = effectiveTimezone !== browserTimezone;
|
||||
|
||||
// The stored schedule is a plain interval (seconds or a duration such as
|
||||
// 8h) rather than a clock schedule. That is what SCHEDULE_INTERVAL and
|
||||
// GITEA_MIRROR_INTERVAL produce, and what older versions stored. The card
|
||||
// only edits clock schedules, so say what actually runs instead of showing
|
||||
// the 22:00 placeholder as if it were saved (#427).
|
||||
const isIntervalSchedule = scheduleConfig.scheduleMode === "interval";
|
||||
// Use saved timezone, but treat "UTC" as unset for users who never chose
|
||||
// it: older versions stored UTC as a default without asking. Anyone truly
|
||||
// in UTC gets the same result via their browser timezone.
|
||||
const effectiveTimezone =
|
||||
scheduleConfig.timezone && scheduleConfig.timezone !== "UTC"
|
||||
? scheduleConfig.timezone
|
||||
: browserTimezone;
|
||||
|
||||
const nextScheduledRun = useMemo(() => {
|
||||
if (!scheduleConfig.enabled || isIntervalSchedule) return null;
|
||||
if (!scheduleConfig.enabled) return null;
|
||||
const startTime = scheduleConfig.startTime || "22:00";
|
||||
const frequencyHours = scheduleConfig.clockFrequencyHours || 24;
|
||||
const cronExpression = buildClockCronExpression(startTime, frequencyHours);
|
||||
@@ -169,16 +145,7 @@ export function AutomationSettings({
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}, [scheduleConfig.enabled, isIntervalSchedule, scheduleConfig.startTime, scheduleConfig.clockFrequencyHours, effectiveTimezone]);
|
||||
|
||||
const switchToBrowserTimezone = () =>
|
||||
onScheduleChange({
|
||||
...scheduleConfig,
|
||||
scheduleMode: "clock",
|
||||
startTime: scheduleConfig.startTime || "22:00",
|
||||
clockFrequencyHours: scheduleConfig.clockFrequencyHours || 24,
|
||||
timezone: browserTimezone,
|
||||
});
|
||||
}, [scheduleConfig.enabled, scheduleConfig.startTime, scheduleConfig.clockFrequencyHours, effectiveTimezone]);
|
||||
|
||||
// Update nextRun for cleanup when settings change
|
||||
useEffect(() => {
|
||||
@@ -250,39 +217,15 @@ export function AutomationSettings({
|
||||
<CardSection>
|
||||
<SectionTitle
|
||||
action={
|
||||
timezoneDiffers ? (
|
||||
<button
|
||||
type="button"
|
||||
onClick={switchToBrowserTimezone}
|
||||
title={`Switch the schedule to ${browserTimezone}`}
|
||||
className="inline-flex items-center gap-1.5 rounded-full bg-muted px-2.5 py-1 text-[11px] text-muted-foreground hover:bg-muted/70 hover:text-foreground"
|
||||
>
|
||||
<Globe className="h-3 w-3" />
|
||||
{effectiveTimezone}
|
||||
<span className="text-muted-foreground/60">· use {browserTimezone}</span>
|
||||
</button>
|
||||
) : (
|
||||
<span className="inline-flex items-center gap-1.5 rounded-full bg-muted px-2.5 py-1 text-[11px] text-muted-foreground">
|
||||
<Globe className="h-3 w-3" />
|
||||
{effectiveTimezone}
|
||||
</span>
|
||||
)
|
||||
<span className="inline-flex items-center gap-1.5 rounded-full bg-muted px-2.5 py-1 text-[11px] text-muted-foreground">
|
||||
<Globe className="h-3 w-3" />
|
||||
{effectiveTimezone}
|
||||
</span>
|
||||
}
|
||||
>
|
||||
Schedule
|
||||
</SectionTitle>
|
||||
|
||||
{isIntervalSchedule && (
|
||||
<p className="flex items-start gap-2 rounded-md border border-dashed border-border px-3 py-2 text-xs text-muted-foreground">
|
||||
<Info className="mt-0.5 h-3.5 w-3.5 shrink-0" />
|
||||
<span>
|
||||
Syncing {formatIntervalSeconds(scheduleConfig.interval)}, counted from the last run.
|
||||
{" "}This interval comes from SCHEDULE_INTERVAL or GITEA_MIRROR_INTERVAL, which is applied again on every restart while it is set, or from an older version.
|
||||
{" "}Pick a frequency or start time to switch to a clock schedule.
|
||||
</span>
|
||||
</p>
|
||||
)}
|
||||
|
||||
<div className="grid gap-3 sm:grid-cols-2">
|
||||
<div className="space-y-1.5">
|
||||
<Label
|
||||
@@ -292,7 +235,7 @@ export function AutomationSettings({
|
||||
Frequency
|
||||
</Label>
|
||||
<Select
|
||||
value={isIntervalSchedule ? "" : String(scheduleConfig.clockFrequencyHours || 24)}
|
||||
value={String(scheduleConfig.clockFrequencyHours || 24)}
|
||||
onValueChange={(value) =>
|
||||
onScheduleChange({
|
||||
...scheduleConfig,
|
||||
@@ -304,7 +247,7 @@ export function AutomationSettings({
|
||||
}
|
||||
>
|
||||
<SelectTrigger id="clock-frequency" className="w-full">
|
||||
<SelectValue placeholder="Choose a frequency" />
|
||||
<SelectValue />
|
||||
</SelectTrigger>
|
||||
<SelectContent>
|
||||
{clockFrequencies.map((option) => (
|
||||
@@ -333,7 +276,7 @@ export function AutomationSettings({
|
||||
<Input
|
||||
id="clock-start-time"
|
||||
type="time"
|
||||
value={isIntervalSchedule ? "" : scheduleConfig.startTime || "22:00"}
|
||||
value={scheduleConfig.startTime || "22:00"}
|
||||
onChange={(event) =>
|
||||
onScheduleChange({
|
||||
...scheduleConfig,
|
||||
|
||||
@@ -222,33 +222,21 @@ export function Organization() {
|
||||
|
||||
const newStatus = ignore ? "ignored" : "imported";
|
||||
|
||||
// cascade: the organization's repositories follow it, so ignoring an
|
||||
// organization actually stops its mirrors from syncing (#429).
|
||||
const response = await apiRequest<{
|
||||
success: boolean;
|
||||
organization?: Organization;
|
||||
repositoriesChanged?: number;
|
||||
error?: string;
|
||||
}>(
|
||||
const response = await apiRequest<{ success: boolean; organization?: Organization; error?: string }>(
|
||||
`/organizations/${orgId}/status`,
|
||||
{
|
||||
method: "PATCH",
|
||||
data: {
|
||||
status: newStatus,
|
||||
userId: user.id,
|
||||
cascade: true,
|
||||
userId: user.id
|
||||
},
|
||||
}
|
||||
);
|
||||
|
||||
if (response.success) {
|
||||
const changed = response.repositoriesChanged ?? 0;
|
||||
const repoNote = changed > 0
|
||||
? ` (${changed} ${changed === 1 ? "repository" : "repositories"} ${ignore ? "ignored too" : "restored"})`
|
||||
: "";
|
||||
toast.success(ignore
|
||||
? `Organization will be ignored in future operations${repoNote}`
|
||||
: `Organization included for mirroring${repoNote}`
|
||||
? `Organization will be ignored in future operations`
|
||||
: `Organization included for mirroring`
|
||||
);
|
||||
|
||||
// Update local state
|
||||
|
||||
@@ -3,6 +3,7 @@ import { Card } from "@/components/ui/card";
|
||||
import { Button } from "@/components/ui/button";
|
||||
import { Badge } from "@/components/ui/badge";
|
||||
import { Plus, RefreshCw, Building2, Check, AlertCircle, Clock, MoreVertical, Ban, SlidersHorizontal, Trash2, Layers } from "lucide-react";
|
||||
import { SiGitea } from "react-icons/si";
|
||||
import { toast } from "sonner";
|
||||
import type { MirrorOverrides, Organization } from "@/lib/db/schema";
|
||||
import type { FilterParams } from "@/types/filter";
|
||||
@@ -12,7 +13,7 @@ import { Skeleton } from "@/components/ui/skeleton";
|
||||
import { cn } from "@/lib/utils";
|
||||
import { buildGiteaWebUrl } from "@/lib/gitea-url";
|
||||
import { MirrorDestinationEditor } from "./MirrorDestinationEditor";
|
||||
import { DestinationIcon, destinationInfo } from "@/components/destination/DestinationIcon";
|
||||
import { destinationInfo } from "@/components/destination/DestinationIcon";
|
||||
import type { OrganizationMoveResult } from "@/lib/destination-transfer";
|
||||
import { MirrorOverridesDialog } from "@/components/config/MirrorOverridesDialog";
|
||||
import { hasMirrorOverrides, mirrorOptionsToFlags } from "@/lib/utils/mirror-overrides";
|
||||
@@ -271,17 +272,6 @@ export function OrganizationList({
|
||||
// GitHub and GitLab destinations are pushed to; only Gitea and Forgejo can transfer a repository.
|
||||
const destination = destinationInfo(giteaConfig);
|
||||
|
||||
// Tooltip for the destination link on a card. Named after the configured
|
||||
// destination, so a Forgejo user is not told to view the org on Gitea (#430).
|
||||
const destinationLinkTooltip = (org: Organization, orgUrl: string | null): string => {
|
||||
if (!giteaConfig?.url) return `${destination.label} not configured`;
|
||||
if (org.status === "imported") return `Organization not yet mirrored to ${destination.label}`;
|
||||
if (org.status === "failed") return "Organization mirroring failed";
|
||||
if (org.status === "mirroring") return `Organization is being mirrored to ${destination.label}`;
|
||||
if (orgUrl) return `View on ${destination.label}`;
|
||||
return `${destination.label} organization not available`;
|
||||
};
|
||||
|
||||
// Where an organization's repositories land with no override, which is what
|
||||
// the mirror strategy produces for organization repositories (see
|
||||
// getGiteaRepoOwner in src/lib/gitea.ts). Only "preserve" and "mixed" make
|
||||
@@ -759,7 +749,21 @@ export function OrganizationList({
|
||||
{(() => {
|
||||
const giteaUrl = getGiteaOrgUrl(org);
|
||||
|
||||
const tooltip = destinationLinkTooltip(org, giteaUrl);
|
||||
// Determine tooltip based on status and configuration
|
||||
let tooltip: string;
|
||||
if (!giteaConfig?.url) {
|
||||
tooltip = "Gitea not configured";
|
||||
} else if (org.status === 'imported') {
|
||||
tooltip = "Organization not yet mirrored to Gitea";
|
||||
} else if (org.status === 'failed') {
|
||||
tooltip = "Organization mirroring failed";
|
||||
} else if (org.status === 'mirroring') {
|
||||
tooltip = "Organization is being mirrored to Gitea";
|
||||
} else if (giteaUrl) {
|
||||
tooltip = "View on Gitea";
|
||||
} else {
|
||||
tooltip = "Gitea organization not available";
|
||||
}
|
||||
|
||||
return giteaUrl ? (
|
||||
<Button variant="outline" size="default" asChild className="flex-1 h-10 min-w-0">
|
||||
@@ -770,14 +774,14 @@ export function OrganizationList({
|
||||
title={tooltip}
|
||||
className="flex items-center justify-center gap-2"
|
||||
>
|
||||
<DestinationIcon provider={destination.provider} className="h-4 w-4 flex-shrink-0" />
|
||||
<span className="text-xs">{destination.label}</span>
|
||||
<SiGitea className="h-4 w-4 flex-shrink-0" />
|
||||
<span className="text-xs">Gitea</span>
|
||||
</a>
|
||||
</Button>
|
||||
) : (
|
||||
<Button variant="outline" size="default" disabled title={tooltip} className="flex-1 h-10">
|
||||
<DestinationIcon provider={destination.provider} className="h-4 w-4" />
|
||||
<span className="text-xs ml-2">{destination.label}</span>
|
||||
<SiGitea className="h-4 w-4" />
|
||||
<span className="text-xs ml-2">Gitea</span>
|
||||
</Button>
|
||||
);
|
||||
})()}
|
||||
@@ -910,7 +914,21 @@ export function OrganizationList({
|
||||
{(() => {
|
||||
const giteaUrl = getGiteaOrgUrl(org);
|
||||
|
||||
const tooltip = destinationLinkTooltip(org, giteaUrl);
|
||||
// Determine tooltip based on status and configuration
|
||||
let tooltip: string;
|
||||
if (!giteaConfig?.url) {
|
||||
tooltip = "Gitea not configured";
|
||||
} else if (org.status === 'imported') {
|
||||
tooltip = "Organization not yet mirrored to Gitea";
|
||||
} else if (org.status === 'failed') {
|
||||
tooltip = "Organization mirroring failed";
|
||||
} else if (org.status === 'mirroring') {
|
||||
tooltip = "Organization is being mirrored to Gitea";
|
||||
} else if (giteaUrl) {
|
||||
tooltip = "View on Gitea";
|
||||
} else {
|
||||
tooltip = "Gitea organization not available";
|
||||
}
|
||||
|
||||
return (
|
||||
<div className="flex items-center border rounded-md">
|
||||
@@ -928,13 +946,13 @@ export function OrganizationList({
|
||||
target="_blank"
|
||||
rel="noopener noreferrer"
|
||||
>
|
||||
<DestinationIcon provider={destination.provider} className="h-4 w-4 mr-2" />
|
||||
{destination.label}
|
||||
<SiGitea className="h-4 w-4 mr-2" />
|
||||
Gitea
|
||||
</a>
|
||||
) : (
|
||||
<>
|
||||
<DestinationIcon provider={destination.provider} className="h-4 w-4 mr-2" />
|
||||
{destination.label}
|
||||
<SiGitea className="h-4 w-4 mr-2" />
|
||||
Gitea
|
||||
</>
|
||||
)}
|
||||
</Button>
|
||||
|
||||
+12
-43
@@ -4,7 +4,6 @@ import { eq, and } from "drizzle-orm";
|
||||
import { buildInterruptedJobsCondition } from "./interrupted-job-detection";
|
||||
import { v4 as uuidv4 } from "uuid";
|
||||
import { publishEvent } from "./events";
|
||||
import { withKeyedLock } from "./utils/keyed-mutex";
|
||||
import { triggerJobNotification } from "./notification-service";
|
||||
|
||||
export async function createMirrorJob({
|
||||
@@ -114,34 +113,9 @@ export async function createMirrorJob({
|
||||
}
|
||||
|
||||
/**
|
||||
* Updates the progress of a mirror job.
|
||||
*
|
||||
* Progress is a read-modify-write of the completedItemIds list, and items
|
||||
* finish concurrently, so the update runs under a per-job lock: two items
|
||||
* completing in the same tick used to read the same list and one of them
|
||||
* silently overwrote the other. A resume then re-processed the lost item.
|
||||
*
|
||||
* `silent` records the progress without publishing an event. Every completed
|
||||
* item is checkpointed so a resume knows exactly what is done; the events the
|
||||
* dashboard listens to are throttled by the caller instead.
|
||||
* Updates the progress of a mirror job
|
||||
*/
|
||||
export async function updateMirrorJobProgress(params: {
|
||||
jobId: string;
|
||||
completedItemId?: string;
|
||||
status?: RepoStatus;
|
||||
message?: string;
|
||||
details?: string;
|
||||
inProgress?: boolean;
|
||||
isCompleted?: boolean;
|
||||
/** Record a new start time, as when recovery resumes the job. */
|
||||
startedAt?: Date;
|
||||
/** Write the progress without publishing a mirror-status event. */
|
||||
silent?: boolean;
|
||||
}) {
|
||||
return withKeyedLock(`mirror-job:${params.jobId}`, () => updateMirrorJobProgressLocked(params));
|
||||
}
|
||||
|
||||
async function updateMirrorJobProgressLocked({
|
||||
export async function updateMirrorJobProgress({
|
||||
jobId,
|
||||
completedItemId,
|
||||
status,
|
||||
@@ -149,9 +123,15 @@ async function updateMirrorJobProgressLocked({
|
||||
details,
|
||||
inProgress,
|
||||
isCompleted,
|
||||
startedAt,
|
||||
silent = false,
|
||||
}: Parameters<typeof updateMirrorJobProgress>[0]) {
|
||||
}: {
|
||||
jobId: string;
|
||||
completedItemId?: string;
|
||||
status?: RepoStatus;
|
||||
message?: string;
|
||||
details?: string;
|
||||
inProgress?: boolean;
|
||||
isCompleted?: boolean;
|
||||
}) {
|
||||
try {
|
||||
// Get the current job
|
||||
const [job] = await db
|
||||
@@ -177,10 +157,6 @@ async function updateMirrorJobProgressLocked({
|
||||
}
|
||||
}
|
||||
|
||||
if (startedAt) {
|
||||
updates.startedAt = startedAt;
|
||||
}
|
||||
|
||||
// Update status if provided
|
||||
if (status) {
|
||||
updates.status = status;
|
||||
@@ -219,10 +195,6 @@ async function updateMirrorJobProgressLocked({
|
||||
...updates,
|
||||
};
|
||||
|
||||
if (silent) {
|
||||
return updatedJob;
|
||||
}
|
||||
|
||||
// Create deduplication key for progress updates
|
||||
let deduplicationKey: string | undefined;
|
||||
if (completedItemId) {
|
||||
@@ -352,15 +324,12 @@ export async function resumeInterruptedJob(job: any) {
|
||||
return null;
|
||||
}
|
||||
|
||||
// Update the job to show it's being resumed. The start time moves to
|
||||
// now so the row records this pass; the fresh checkpoint written here is
|
||||
// what keeps a second recovery sweep from resuming the same job again.
|
||||
// Update the job to show it's being resumed
|
||||
await updateMirrorJobProgress({
|
||||
jobId: job.id,
|
||||
message: `Resuming job with ${remainingItemIds.length} remaining items`,
|
||||
details: `Job was interrupted and is being resumed. ${job.completedItemIds.length} of ${job.itemIds.length} items were already processed.`,
|
||||
inProgress: true,
|
||||
startedAt: new Date(),
|
||||
});
|
||||
|
||||
return {
|
||||
|
||||
@@ -18,6 +18,7 @@ import { drizzle } from "drizzle-orm/bun-sqlite";
|
||||
import { mirrorJobs } from "@/lib/db/schema";
|
||||
import {
|
||||
INTERRUPTED_CHECKPOINT_AGE_MS,
|
||||
STALE_JOB_AGE_MS,
|
||||
JOB_HEARTBEAT_INTERVAL_MS,
|
||||
computeInterruptedJobCutoffs,
|
||||
isJobInterrupted,
|
||||
@@ -31,15 +32,14 @@ const NOW = new Date("2026-09-02T12:00:00Z");
|
||||
const ago = (ms: number) => new Date(NOW.getTime() - ms);
|
||||
|
||||
describe("computeInterruptedJobCutoffs", () => {
|
||||
test("keeps the 10 minute checkpoint window and has no stale age cutoff", () => {
|
||||
test("keeps the 10 minute checkpoint window and 2 hour stale window", () => {
|
||||
expect(INTERRUPTED_CHECKPOINT_AGE_MS).toBe(10 * MINUTE);
|
||||
expect(STALE_JOB_AGE_MS).toBe(2 * HOUR);
|
||||
|
||||
const cutoffs = computeInterruptedJobCutoffs(NOW);
|
||||
|
||||
expect(cutoffs.checkpointCutoff.getTime()).toBe(NOW.getTime() - 10 * MINUTE);
|
||||
// The two hour "stale regardless of checkpoint" rule resumed live jobs
|
||||
// and is gone; a checkpointing job is alive however old it is.
|
||||
expect(Object.keys(cutoffs)).toEqual(["checkpointCutoff"]);
|
||||
expect(cutoffs.staleCutoff.getTime()).toBe(NOW.getTime() - 2 * HOUR);
|
||||
});
|
||||
|
||||
test("heartbeat interval leaves room for a missed beat inside the checkpoint window", () => {
|
||||
@@ -78,18 +78,9 @@ describe("isJobInterrupted", () => {
|
||||
).toBe(true);
|
||||
});
|
||||
|
||||
test("a job that keeps checkpointing stays live however long ago it started", () => {
|
||||
test("a job running past the stale window is interrupted even with a fresh checkpoint", () => {
|
||||
expect(
|
||||
isJobInterrupted({ inProgress: true, startedAt: ago(3 * HOUR), lastCheckpoint: ago(MINUTE) }, NOW)
|
||||
).toBe(false);
|
||||
expect(
|
||||
isJobInterrupted({ inProgress: true, startedAt: ago(30 * HOUR), lastCheckpoint: ago(2 * MINUTE) }, NOW)
|
||||
).toBe(false);
|
||||
});
|
||||
|
||||
test("an old job whose checkpoint went stale is still interrupted", () => {
|
||||
expect(
|
||||
isJobInterrupted({ inProgress: true, startedAt: ago(3 * HOUR), lastCheckpoint: ago(11 * MINUTE) }, NOW)
|
||||
).toBe(true);
|
||||
});
|
||||
|
||||
@@ -244,22 +235,6 @@ describe("wiring", () => {
|
||||
).toBe(true);
|
||||
});
|
||||
|
||||
test("resumeInterruptedJob records a fresh start time for the resumed pass", () => {
|
||||
const resumeBody = HELPERS_SRC.slice(HELPERS_SRC.indexOf("export async function resumeInterruptedJob("));
|
||||
expect(/inProgress:\s*true,\s*startedAt:\s*new Date\(\)/.test(resumeBody)).toBe(true);
|
||||
});
|
||||
|
||||
test("updateMirrorJobProgress serializes writes per job so concurrent items cannot drop each other", () => {
|
||||
expect(
|
||||
/withKeyedLock\(`mirror-job:\$\{params\.jobId\}`,\s*\(\)\s*=>\s*updateMirrorJobProgressLocked\(params\)\)/.test(HELPERS_SRC)
|
||||
).toBe(true);
|
||||
});
|
||||
|
||||
test("processWithRetry checkpoints every completed item instead of the one that trips the interval", () => {
|
||||
expect(/await onCheckpoint\(jobId,\s*itemId,\s*publish\)/.test(CONCURRENCY_SRC)).toBe(true);
|
||||
expect(CONCURRENCY_SRC.includes("onCheckpoint(jobId, 'final')")).toBe(false);
|
||||
});
|
||||
|
||||
test("middleware keeps the recovery latch until the recovery promise settles", () => {
|
||||
expect(
|
||||
/recoveryPromise\s*\.catch\(\(\)\s*=>\s*false\)\s*\.finally\(\(\)\s*=>\s*\{\s*recoveryInFlight\s*=\s*false;\s*\}\)/.test(
|
||||
|
||||
@@ -2,20 +2,11 @@
|
||||
* Decides which in-progress mirror jobs count as interrupted and should be
|
||||
* resumed by recovery (issue #372).
|
||||
*
|
||||
* A job is interrupted when either of these holds:
|
||||
* A job is interrupted when any of these hold:
|
||||
* - it has a checkpoint older than the checkpoint window, or
|
||||
* - it has no checkpoint and is older than the checkpoint window (or has
|
||||
* no recorded start at all, as legacy rows may).
|
||||
*
|
||||
* A job that keeps checkpointing is alive, however long ago it started.
|
||||
* Running jobs refresh their checkpoint on a heartbeat between items, so a
|
||||
* missing or old checkpoint is the only evidence that the process behind a
|
||||
* job is gone. There used to be a third rule, "started more than two hours
|
||||
* ago, whatever the checkpoint says", which resumed large jobs while they
|
||||
* were still running: the resumed pass then re-processed the same
|
||||
* repositories alongside the original, which is one way release assets got
|
||||
* duplicated (#417). A job that hangs without ever finishing an item is a
|
||||
* different problem and is not what recovery is for.
|
||||
* no recorded start at all, as legacy rows may), or
|
||||
* - it started longer ago than the stale window, whatever its checkpoint.
|
||||
*
|
||||
* The grace period for jobs without a checkpoint is the point of this
|
||||
* module. Checkpoints are written when an item completes, so a job that has
|
||||
@@ -33,6 +24,9 @@ import type { mirrorJobs } from "@/lib/db/schema";
|
||||
/** A job with no checkpoint newer than this is considered inactive. */
|
||||
export const INTERRUPTED_CHECKPOINT_AGE_MS = 10 * 60 * 1000;
|
||||
|
||||
/** A job that started longer ago than this is considered stale regardless. */
|
||||
export const STALE_JOB_AGE_MS = 2 * 60 * 60 * 1000;
|
||||
|
||||
/**
|
||||
* How often a running job refreshes its checkpoint between item
|
||||
* completions. Kept well inside the checkpoint window so one missed beat
|
||||
@@ -43,11 +37,14 @@ export const JOB_HEARTBEAT_INTERVAL_MS = 2 * 60 * 1000;
|
||||
export interface InterruptedJobCutoffs {
|
||||
/** Checkpoints (or starts, for jobs without one) before this are too old. */
|
||||
checkpointCutoff: Date;
|
||||
/** Starts before this mark the job stale no matter what. */
|
||||
staleCutoff: Date;
|
||||
}
|
||||
|
||||
export function computeInterruptedJobCutoffs(now: Date = new Date()): InterruptedJobCutoffs {
|
||||
return {
|
||||
checkpointCutoff: new Date(now.getTime() - INTERRUPTED_CHECKPOINT_AGE_MS),
|
||||
staleCutoff: new Date(now.getTime() - STALE_JOB_AGE_MS),
|
||||
};
|
||||
}
|
||||
|
||||
@@ -64,9 +61,10 @@ export interface JobLivenessFields {
|
||||
export function isJobInterrupted(job: JobLivenessFields, now: Date = new Date()): boolean {
|
||||
if (!job.inProgress) return false;
|
||||
|
||||
const { checkpointCutoff } = computeInterruptedJobCutoffs(now);
|
||||
const { checkpointCutoff, staleCutoff } = computeInterruptedJobCutoffs(now);
|
||||
|
||||
if (job.startedAt && job.startedAt < staleCutoff) return true;
|
||||
|
||||
// A checkpoint is the job's own proof of life; its age decides on its own.
|
||||
if (job.lastCheckpoint) return job.lastCheckpoint < checkpointCutoff;
|
||||
|
||||
// No checkpoint yet: give the job the same window it would get after a
|
||||
@@ -84,7 +82,7 @@ export function buildInterruptedJobsCondition(
|
||||
table: typeof mirrorJobs,
|
||||
now: Date = new Date(),
|
||||
): SQL {
|
||||
const { checkpointCutoff } = computeInterruptedJobCutoffs(now);
|
||||
const { checkpointCutoff, staleCutoff } = computeInterruptedJobCutoffs(now);
|
||||
|
||||
return and(
|
||||
eq(table.inProgress, true),
|
||||
@@ -97,6 +95,8 @@ export function buildInterruptedJobsCondition(
|
||||
),
|
||||
// Checkpoint present but too old.
|
||||
lt(table.lastCheckpoint, checkpointCutoff),
|
||||
// Running far too long, whatever the checkpoint says.
|
||||
lt(table.startedAt, staleCutoff),
|
||||
),
|
||||
)!;
|
||||
}
|
||||
@@ -1,101 +0,0 @@
|
||||
import { describe, test, expect } from 'bun:test';
|
||||
import {
|
||||
repositoryIdentityKey,
|
||||
repositoryIdentityKeys,
|
||||
selectNewRepositoriesByIdentity,
|
||||
dedupeRepositoriesByIdentity,
|
||||
} from './repo-utils';
|
||||
|
||||
describe('repositoryIdentityKey', () => {
|
||||
test('is the host plus the full name, independent of the source row', () => {
|
||||
const a = repositoryIdentityKey({ sourceProvider: 'github', sourceUrl: 'https://github.com', normalizedFullName: 'acme/tools' });
|
||||
const b = repositoryIdentityKey({ sourceProvider: 'github', sourceUrl: 'https://github.com/', normalizedFullName: 'ACME/Tools' });
|
||||
expect(a).toBe(b);
|
||||
});
|
||||
|
||||
test('treats missing provider and url as github.com, like legacy rows', () => {
|
||||
expect(repositoryIdentityKey({ normalizedFullName: 'acme/tools' })).toBe(
|
||||
repositoryIdentityKey({ sourceProvider: 'github', sourceUrl: 'https://github.com', normalizedFullName: 'acme/tools' })
|
||||
);
|
||||
});
|
||||
|
||||
test('keeps the same name on two hosts apart', () => {
|
||||
const ghe = repositoryIdentityKey({ sourceProvider: 'github', sourceUrl: 'https://ghe.example.com', normalizedFullName: 'acme/tools' });
|
||||
const com = repositoryIdentityKey({ sourceProvider: 'github', sourceUrl: 'https://github.com', normalizedFullName: 'acme/tools' });
|
||||
expect(ghe).not.toBe(com);
|
||||
});
|
||||
});
|
||||
|
||||
describe('selectNewRepositoriesByIdentity', () => {
|
||||
test('skips a repository already tracked on the same host under another source', () => {
|
||||
const existing = repositoryIdentityKeys([
|
||||
{ sourceProvider: 'github', sourceUrl: 'https://github.com', normalizedFullName: 'acme/tools' },
|
||||
]);
|
||||
const { fresh, alreadyTracked } = selectNewRepositoriesByIdentity(
|
||||
[
|
||||
{ sourceProvider: 'github', sourceUrl: 'https://github.com', normalizedFullName: 'acme/tools', sourceId: 'public-only' },
|
||||
{ sourceProvider: 'github', sourceUrl: 'https://github.com', normalizedFullName: 'acme/new', sourceId: 'public-only' },
|
||||
],
|
||||
existing
|
||||
);
|
||||
expect(fresh.map((r) => r.normalizedFullName)).toEqual(['acme/new']);
|
||||
expect(alreadyTracked.map((r) => r.normalizedFullName)).toEqual(['acme/tools']);
|
||||
});
|
||||
|
||||
test('extends the existing set so a later source does not re-add the same repository', () => {
|
||||
const existing = new Set<string>();
|
||||
const first = selectNewRepositoriesByIdentity(
|
||||
[{ sourceProvider: 'github', sourceUrl: 'https://github.com', normalizedFullName: 'acme/tools' }],
|
||||
existing
|
||||
);
|
||||
const second = selectNewRepositoriesByIdentity(
|
||||
[{ sourceProvider: 'github', sourceUrl: 'https://github.com', normalizedFullName: 'acme/tools' }],
|
||||
existing
|
||||
);
|
||||
expect(first.fresh).toHaveLength(1);
|
||||
expect(second.fresh).toHaveLength(0);
|
||||
expect(second.alreadyTracked).toHaveLength(1);
|
||||
});
|
||||
|
||||
test('still allows the same name from a different host', () => {
|
||||
const existing = repositoryIdentityKeys([
|
||||
{ sourceProvider: 'github', sourceUrl: 'https://github.com', normalizedFullName: 'acme/tools' },
|
||||
]);
|
||||
const { fresh } = selectNewRepositoriesByIdentity(
|
||||
[{ sourceProvider: 'gitlab', sourceUrl: 'https://gitlab.com', normalizedFullName: 'acme/tools' }],
|
||||
existing
|
||||
);
|
||||
expect(fresh).toHaveLength(1);
|
||||
});
|
||||
});
|
||||
|
||||
describe('dedupeRepositoriesByIdentity', () => {
|
||||
test('keeps one row per upstream repository in a batch', () => {
|
||||
const { kept, dropped } = dedupeRepositoriesByIdentity([
|
||||
{ id: 'a', sourceId: 's1', sourceProvider: 'github', sourceUrl: 'https://github.com', normalizedFullName: 'acme/tools' },
|
||||
{ id: 'b', sourceId: 's2', sourceProvider: 'github', sourceUrl: 'https://github.com', normalizedFullName: 'acme/tools' },
|
||||
{ id: 'c', sourceId: 's1', sourceProvider: 'github', sourceUrl: 'https://github.com', normalizedFullName: 'acme/other' },
|
||||
]);
|
||||
expect(kept.map((r) => r.id)).toEqual(['a', 'c']);
|
||||
expect(dropped.map((r) => r.id)).toEqual(['b']);
|
||||
});
|
||||
|
||||
test('prefers a row linked to a source over a legacy row without one', () => {
|
||||
const { kept, dropped } = dedupeRepositoriesByIdentity([
|
||||
{ id: 'legacy', sourceId: null, normalizedFullName: 'acme/tools' },
|
||||
{ id: 'linked', sourceId: 's1', sourceProvider: 'github', sourceUrl: 'https://github.com', normalizedFullName: 'acme/tools' },
|
||||
]);
|
||||
expect(kept.map((r) => r.id)).toEqual(['linked']);
|
||||
expect(dropped.map((r) => r.id)).toEqual(['legacy']);
|
||||
});
|
||||
|
||||
test('leaves a batch without duplicates alone', () => {
|
||||
const rows = [
|
||||
{ id: 'a', sourceId: 's1', normalizedFullName: 'acme/a' },
|
||||
{ id: 'b', sourceId: 's1', normalizedFullName: 'acme/b' },
|
||||
];
|
||||
const { kept, dropped } = dedupeRepositoriesByIdentity(rows);
|
||||
expect(kept).toEqual(rows);
|
||||
expect(dropped).toEqual([]);
|
||||
});
|
||||
});
|
||||
@@ -93,82 +93,3 @@ export function repositoryDestinationColumns(
|
||||
): Pick<RepoInsert, 'destinationProvider' | 'destinationUrl'> {
|
||||
return { destinationProvider: destination.provider, destinationUrl: destination.url };
|
||||
}
|
||||
|
||||
/**
|
||||
* Upstream identity of a repository row: which host it lives on and its full
|
||||
* name there. The repositories table is unique on (userId, sourceId,
|
||||
* normalizedFullName) so that the same owner/name can be tracked on two
|
||||
* hosts, but two source rows can point at the same host (a personal token
|
||||
* source and a public-only source, both github.com), and a legacy row with a
|
||||
* NULL sourceId is distinct from a re-imported one. Discovery used the unique
|
||||
* index alone to decide what was new, so one upstream repository could end up
|
||||
* as two rows that both mirror to the same destination repository and get
|
||||
* processed together in one batch (one of the overlaps behind #417).
|
||||
*/
|
||||
export interface RepositoryIdentityFields {
|
||||
sourceProvider?: string | null;
|
||||
sourceUrl?: string | null;
|
||||
normalizedFullName: string;
|
||||
}
|
||||
|
||||
export function repositoryIdentityKey(row: RepositoryIdentityFields): string {
|
||||
const provider = (row.sourceProvider || 'github').trim().toLowerCase();
|
||||
const url = (row.sourceUrl || 'https://github.com').trim().toLowerCase().replace(/\/+$/, '');
|
||||
return `${provider}|${url}|${row.normalizedFullName.toLowerCase()}`;
|
||||
}
|
||||
|
||||
/**
|
||||
* Split discovered rows into the ones not yet tracked for this user on the
|
||||
* same host under any source, and the ones that are. Also drops repeats
|
||||
* within `candidates` themselves. `existing` is extended with every fresh
|
||||
* row, so a caller looping over several sources can pass the same set.
|
||||
*/
|
||||
export function selectNewRepositoriesByIdentity<T extends RepositoryIdentityFields>(
|
||||
candidates: T[],
|
||||
existing: Set<string>
|
||||
): { fresh: T[]; alreadyTracked: T[] } {
|
||||
const fresh: T[] = [];
|
||||
const alreadyTracked: T[] = [];
|
||||
for (const candidate of candidates) {
|
||||
const key = repositoryIdentityKey(candidate);
|
||||
if (existing.has(key)) {
|
||||
alreadyTracked.push(candidate);
|
||||
} else {
|
||||
existing.add(key);
|
||||
fresh.push(candidate);
|
||||
}
|
||||
}
|
||||
return { fresh, alreadyTracked };
|
||||
}
|
||||
|
||||
export function repositoryIdentityKeys(rows: Iterable<RepositoryIdentityFields>): Set<string> {
|
||||
const keys = new Set<string>();
|
||||
for (const row of rows) keys.add(repositoryIdentityKey(row));
|
||||
return keys;
|
||||
}
|
||||
|
||||
/**
|
||||
* Keep one row per upstream identity in a processing batch, so two rows that
|
||||
* resolve to the same destination repository are never mirrored or synced in
|
||||
* the same pass. A row linked to a source wins over a legacy row without one;
|
||||
* otherwise the first row in the batch is kept.
|
||||
*/
|
||||
export function dedupeRepositoriesByIdentity<
|
||||
T extends RepositoryIdentityFields & { sourceId?: string | null },
|
||||
>(rows: T[]): { kept: T[]; dropped: T[] } {
|
||||
const byKey = new Map<string, T>();
|
||||
const dropped: T[] = [];
|
||||
for (const row of rows) {
|
||||
const key = repositoryIdentityKey(row);
|
||||
const current = byKey.get(key);
|
||||
if (!current) {
|
||||
byKey.set(key, row);
|
||||
} else if (!current.sourceId && row.sourceId) {
|
||||
dropped.push(current);
|
||||
byKey.set(key, row);
|
||||
} else {
|
||||
dropped.push(row);
|
||||
}
|
||||
}
|
||||
return { kept: Array.from(byKey.values()), dropped };
|
||||
}
|
||||
@@ -465,34 +465,6 @@ describe.skipIf(!isChild)("Scheduler public-only sources (WP4)", () => {
|
||||
.filter((row) => row.normalizedFullName?.startsWith("pinned-org/"))
|
||||
.map((row) => row.name);
|
||||
expect(orgRepoNamesAfterSecondRun).toEqual(["existing", "newly-published"]);
|
||||
// Nothing new on the second run, so discovery does not even attempt an
|
||||
// insert: already-tracked repositories are filtered out by identity
|
||||
// before the write, not dropped by the unique index afterwards.
|
||||
expect(repoInsertAttempts).toBe(attemptsAfterFirstRun);
|
||||
});
|
||||
|
||||
test("re-discovery does not add a second row for a repository the user already tracks under another source on the same host", async () => {
|
||||
// Imported earlier through a personal token source on github.com, so it
|
||||
// has a different sourceId than the pinned public-only source.
|
||||
repoRows.push({
|
||||
userId: "user-1",
|
||||
sourceId: "source-personal",
|
||||
name: "shared",
|
||||
fullName: "pinned-org/shared",
|
||||
normalizedFullName: "pinned-org/shared",
|
||||
organization: "pinned-org",
|
||||
sourceProvider: "github",
|
||||
sourceUrl: "https://github.com",
|
||||
status: "mirrored",
|
||||
});
|
||||
orgRepoListings.set("pinned-org", [makeGitRepo("shared"), makeGitRepo("fresh")]);
|
||||
|
||||
await schedulerLoop();
|
||||
|
||||
const sharedRows = repoRows.filter((row) => row.normalizedFullName === "pinned-org/shared");
|
||||
expect(sharedRows).toHaveLength(1);
|
||||
expect(sharedRows[0].sourceId).toBe("source-personal");
|
||||
expect(repoRows.some((row) => row.normalizedFullName === "pinned-org/fresh")).toBe(true);
|
||||
expect(consoleLines.some((line) => line.includes("already tracked") && line.includes("pinned-org"))).toBe(true);
|
||||
expect(repoInsertAttempts).toBeGreaterThan(attemptsAfterFirstRun);
|
||||
});
|
||||
});
|
||||
@@ -5,7 +5,7 @@
|
||||
*/
|
||||
|
||||
import { db, configs, repositories, organizations } from '@/lib/db';
|
||||
import { eq, and, or, ne } from 'drizzle-orm';
|
||||
import { eq, and, or } from 'drizzle-orm';
|
||||
import { mirrorRepositoryToDestination, syncRepositoryOnDestination } from '@/lib/mirror-dispatch';
|
||||
import { usesPushEngine } from '@/lib/destination-connection';
|
||||
import { getDecryptedGitHubToken } from '@/lib/utils/config-encryption';
|
||||
@@ -13,14 +13,7 @@ import { formatDuration } from '@/lib/utils/duration-parser';
|
||||
import type { Repository } from '@/lib/db/schema';
|
||||
import type { Octokit } from '@octokit/rest';
|
||||
import { repoStatusEnum, repositoryVisibilityEnum } from '@/types/Repository';
|
||||
import {
|
||||
mergeGitReposPreferStarred,
|
||||
normalizeGitRepoToInsert,
|
||||
calcBatchSizeForInsert,
|
||||
repositoryIdentityKeys,
|
||||
selectNewRepositoriesByIdentity,
|
||||
dedupeRepositoriesByIdentity,
|
||||
} from '@/lib/repo-utils';
|
||||
import { mergeGitReposPreferStarred, normalizeGitRepoToInsert, calcBatchSizeForInsert } from '@/lib/repo-utils';
|
||||
import { isMirrorableGitHubRepo } from '@/lib/repo-eligibility';
|
||||
import { loadOrganizationForkPolicies, orgForkSkipDecision, resolveOrganizationSkipForks } from '@/lib/utils/mirror-overrides';
|
||||
import { createMirrorJob } from '@/lib/helpers';
|
||||
@@ -109,19 +102,14 @@ async function importRepositoriesFromSources(
|
||||
const orgForkOverrides = await loadOrganizationForkPolicies({ userId });
|
||||
|
||||
const existingRepos = await db
|
||||
.select({
|
||||
normalizedFullName: repositories.normalizedFullName,
|
||||
sourceProvider: repositories.sourceProvider,
|
||||
sourceUrl: repositories.sourceUrl,
|
||||
})
|
||||
.select({ normalizedFullName: repositories.normalizedFullName, sourceId: repositories.sourceId })
|
||||
.from(repositories)
|
||||
.where(eq(repositories.userId, userId));
|
||||
// A repository is identified by its host and full name, not by the source
|
||||
// row that found it. Two sources can point at the same host (a personal
|
||||
// token and a public-only source, both github.com), and a repository the
|
||||
// user already tracks must not be inserted a second time under the other
|
||||
// source: both rows would mirror to the same destination repository.
|
||||
const existingRepoKeys = repositoryIdentityKeys(existingRepos);
|
||||
// Repositories are unique per source: the same full name under two sources
|
||||
// counts as two repositories.
|
||||
const existingRepoKeys = new Set(
|
||||
existingRepos.map(r => `${r.sourceId ?? ''}|${r.normalizedFullName}`)
|
||||
);
|
||||
|
||||
for (const source of sources) {
|
||||
try {
|
||||
@@ -144,21 +132,17 @@ async function importRepositoriesFromSources(
|
||||
const allGithubRepos = mergeGitReposPreferStarred(basicAndForkedRepos, starredRepos);
|
||||
const mirrorableGithubRepos = allGithubRepos.filter(isMirrorableGitHubRepo);
|
||||
|
||||
const candidates = mirrorableGithubRepos.map(repo =>
|
||||
normalizeGitRepoToInsert(repo, { userId, configId: config.id, sourceId: source.id })
|
||||
const newRepos = mirrorableGithubRepos.filter(
|
||||
r => !existingRepoKeys.has(`${source.id}|${r.fullName.toLowerCase()}`)
|
||||
);
|
||||
const { fresh: reposToInsert, alreadyTracked } = selectNewRepositoriesByIdentity(
|
||||
candidates,
|
||||
existingRepoKeys
|
||||
);
|
||||
const newRepos = reposToInsert;
|
||||
if (alreadyTracked.length > 0) {
|
||||
console.log(`[Scheduler] ${alreadyTracked.length} repositories listed by source ${source.name} are already tracked for user ${userId} on the same host; not adding them again`);
|
||||
}
|
||||
|
||||
if (newRepos.length > 0) {
|
||||
console.log(`[Scheduler] Found ${newRepos.length} new repositories for user ${userId} on source ${source.name}`);
|
||||
|
||||
const reposToInsert = newRepos.map(repo =>
|
||||
normalizeGitRepoToInsert(repo, { userId, configId: config.id, sourceId: source.id })
|
||||
);
|
||||
|
||||
// Batch insert to avoid SQLite parameter limit
|
||||
const sample = reposToInsert[0];
|
||||
const columnCount = Object.keys(sample ?? {}).length || 1;
|
||||
@@ -219,33 +203,16 @@ async function rediscoverOrganizationRepositories(
|
||||
const { createSourceProviderFromSource } = await import('@/lib/source-providers');
|
||||
const sources = (await listSources(userId)).filter(source => source.enabled);
|
||||
|
||||
// An ignored organization is left alone entirely: rediscovering it would
|
||||
// insert its new repositories as imported and auto-mirror could pick them
|
||||
// up, which is what the user just asked to stop (#429).
|
||||
const includedOrgs = await db
|
||||
.select()
|
||||
.from(organizations)
|
||||
.where(
|
||||
and(
|
||||
eq(organizations.userId, userId),
|
||||
eq(organizations.isIncluded, true),
|
||||
ne(organizations.status, 'ignored')
|
||||
eq(organizations.isIncluded, true)
|
||||
)
|
||||
);
|
||||
|
||||
// Same identity rule as personal discovery: an organization pinned to a
|
||||
// public-only source must not re-insert repositories the personal source
|
||||
// already tracks on the same host (or the other way round).
|
||||
const existingRepos = await db
|
||||
.select({
|
||||
normalizedFullName: repositories.normalizedFullName,
|
||||
sourceProvider: repositories.sourceProvider,
|
||||
sourceUrl: repositories.sourceUrl,
|
||||
})
|
||||
.from(repositories)
|
||||
.where(eq(repositories.userId, userId));
|
||||
const existingRepoKeys = repositoryIdentityKeys(existingRepos);
|
||||
|
||||
for (const org of includedOrgs) {
|
||||
try {
|
||||
const source = findSourceForOrganization(org, sources);
|
||||
@@ -261,17 +228,14 @@ async function rediscoverOrganizationRepositories(
|
||||
repo => repo.isDisabled !== true && !(skipOrgForks && repo.isForked)
|
||||
);
|
||||
|
||||
const { fresh: repoRecords, alreadyTracked } = selectNewRepositoriesByIdentity(
|
||||
mirrorableRepos.map(repo =>
|
||||
if (mirrorableRepos.length > 0) {
|
||||
const repoRecords = mirrorableRepos.map(repo =>
|
||||
normalizeGitRepoToInsert(
|
||||
{ ...repo, organization: repo.organization ?? org.name },
|
||||
{ userId, configId: config.id, sourceId: source.id }
|
||||
)
|
||||
),
|
||||
existingRepoKeys
|
||||
);
|
||||
);
|
||||
|
||||
if (repoRecords.length > 0) {
|
||||
// Batch insert to avoid SQLite parameter limit
|
||||
const sample = repoRecords[0];
|
||||
const columnCount = Object.keys(sample ?? {}).length || 1;
|
||||
@@ -283,10 +247,7 @@ async function rediscoverOrganizationRepositories(
|
||||
.values(batch)
|
||||
.onConflictDoNothing({ target: [repositories.userId, repositories.sourceId, repositories.normalizedFullName] });
|
||||
}
|
||||
console.log(`[Scheduler] Re-discovered ${repoRecords.length} new repositories for organization ${org.name} on source ${source.name} for user ${userId} during ${phase}`);
|
||||
}
|
||||
if (alreadyTracked.length > 0) {
|
||||
console.log(`[Scheduler] Organization ${org.name}: ${alreadyTracked.length} repositories are already tracked for user ${userId} on the same host, not adding them again under source ${source.name}`);
|
||||
console.log(`[Scheduler] Re-discovered ${mirrorableRepos.length} repositories for organization ${org.name} on source ${source.name} for user ${userId} during ${phase}`);
|
||||
}
|
||||
|
||||
await db
|
||||
@@ -370,24 +331,6 @@ function sourceUsernamesBySourceId(sources: SourceRecord[]): Map<string, string>
|
||||
/**
|
||||
* Run scheduled mirror sync for a single user configuration
|
||||
*/
|
||||
/**
|
||||
* Keep one row per upstream repository in a scheduler batch. Duplicate rows
|
||||
* (the same repository imported under two sources on the same host) resolve
|
||||
* to one destination repository, and processing both in one Promise.all ran
|
||||
* every step twice against it. The dropped rows are logged so the duplicate
|
||||
* can be cleaned up; discovery no longer creates new ones.
|
||||
*/
|
||||
function dropDuplicateRowsForPass<T extends { fullName: string; sourceId?: string | null; sourceProvider?: string | null; sourceUrl?: string | null; normalizedFullName: string }>(
|
||||
rows: T[],
|
||||
pass: string
|
||||
): T[] {
|
||||
const { kept, dropped } = dedupeRepositoriesByIdentity(rows);
|
||||
for (const row of dropped) {
|
||||
console.warn(`[Scheduler] ${row.fullName} is tracked more than once (another row on the same host covers it); skipping the duplicate row in this ${pass} pass`);
|
||||
}
|
||||
return kept;
|
||||
}
|
||||
|
||||
async function runScheduledSync(config: any): Promise<void> {
|
||||
const userId = config.userId;
|
||||
console.log(`[Scheduler] Running scheduled sync for user ${userId}`);
|
||||
@@ -514,11 +457,6 @@ async function runScheduledSync(config: any): Promise<void> {
|
||||
console.log(`[Scheduler] Skipped ${forkSkippedCount - reposNeedingMirror.length} forked repositories from auto-mirror (organization skip forks)`);
|
||||
}
|
||||
|
||||
// Two rows for one upstream repository (imported under two sources
|
||||
// that point at the same host) resolve to one destination
|
||||
// repository; mirror it once per pass, not twice at the same time.
|
||||
reposNeedingMirror = dropDuplicateRowsForPass(reposNeedingMirror, 'auto-mirror');
|
||||
|
||||
if (reposNeedingMirror.length > 0) {
|
||||
console.log(`[Scheduler] Found ${reposNeedingMirror.length} repositories that need initial mirroring`);
|
||||
|
||||
@@ -646,8 +584,6 @@ async function runScheduledSync(config: any): Promise<void> {
|
||||
// Non-critical logging, ignore errors
|
||||
}
|
||||
|
||||
reposToSync = dropDuplicateRowsForPass(reposToSync, 'sync');
|
||||
|
||||
if (reposToSync.length === 0) {
|
||||
console.log(`[Scheduler] No repositories to sync for user ${userId}`);
|
||||
return;
|
||||
@@ -872,7 +808,6 @@ async function performInitialAutoStart(): Promise<void> {
|
||||
if (skippedCount > 0) {
|
||||
console.log(`[Scheduler] Skipped ${skippedCount} repositories from initial auto-mirror (autoMirror=${autoMirrorOwned}, autoMirrorStarred=${autoMirrorStarred})`);
|
||||
}
|
||||
reposNeedingMirror = dropDuplicateRowsForPass(reposNeedingMirror, 'initial auto-mirror');
|
||||
|
||||
if (reposNeedingMirror.length > 0) {
|
||||
console.log(`[Scheduler] Found ${reposNeedingMirror.length} repositories that need mirroring`);
|
||||
|
||||
@@ -165,46 +165,3 @@ describe("processWithRetry", () => {
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
describe("processWithRetry checkpointing", () => {
|
||||
test("records every completed item and publishes on the interval", async () => {
|
||||
const items = ["a", "b", "c", "d", "e"];
|
||||
const calls: Array<[string, string, boolean]> = [];
|
||||
|
||||
await processWithRetry(items, async (item) => item.toUpperCase(), {
|
||||
concurrencyLimit: 1,
|
||||
jobId: "job-1",
|
||||
getItemId: (item) => item,
|
||||
checkpointInterval: 2,
|
||||
onCheckpoint: async (jobId, itemId, publish) => {
|
||||
calls.push([jobId, itemId, publish]);
|
||||
},
|
||||
});
|
||||
|
||||
// One checkpoint per item, in completion order, and no "final" placeholder.
|
||||
expect(calls.map(([, itemId]) => itemId)).toEqual(items);
|
||||
expect(calls.every(([jobId]) => jobId === "job-1")).toBe(true);
|
||||
// Progress events only every second item.
|
||||
expect(calls.map(([, , publish]) => publish)).toEqual([false, true, false, true, false]);
|
||||
});
|
||||
|
||||
test("does not checkpoint an item that failed for good", async () => {
|
||||
const calls: string[] = [];
|
||||
|
||||
// processInParallel logs a failed item and carries on, so the call resolves.
|
||||
await processWithRetry(["ok", "bad"], async (item) => {
|
||||
if (item === "bad") throw new Error("nope");
|
||||
return item;
|
||||
}, {
|
||||
concurrencyLimit: 1,
|
||||
maxRetries: 0,
|
||||
jobId: "job-2",
|
||||
getItemId: (item) => item,
|
||||
onCheckpoint: async (_jobId, itemId) => {
|
||||
calls.push(itemId);
|
||||
},
|
||||
});
|
||||
|
||||
expect(calls).toEqual(["ok"]);
|
||||
});
|
||||
});
|
||||
@@ -93,15 +93,8 @@ export async function processWithRetry<T, R>(
|
||||
onRetry?: (item: T, error: Error, attempt: number) => void;
|
||||
jobId?: string; // Optional job ID for checkpointing
|
||||
getItemId?: (item: T) => string; // Function to get a unique ID for each item
|
||||
/**
|
||||
* Called once for EVERY completed item, so a resume knows exactly what is
|
||||
* done. `publish` is true every `checkpointInterval` items: the callback
|
||||
* uses it to throttle progress events, not the checkpoint itself. It used
|
||||
* to be called only for the item that tripped the interval, and a resume
|
||||
* re-processed everything finished since the previous checkpoint.
|
||||
*/
|
||||
onCheckpoint?: (jobId: string, completedItemId: string, publish: boolean) => Promise<void>;
|
||||
checkpointInterval?: number; // How many completed items between published progress events
|
||||
onCheckpoint?: (jobId: string, completedItemId: string) => Promise<void>; // Callback for checkpointing
|
||||
checkpointInterval?: number; // How many items to process before checkpointing
|
||||
} = {}
|
||||
): Promise<R[]> {
|
||||
const {
|
||||
@@ -137,16 +130,16 @@ export async function processWithRetry<T, R>(
|
||||
|
||||
const result = await processItem(item);
|
||||
|
||||
// Record every completed item; publish progress on the interval.
|
||||
// Handle checkpointing if enabled
|
||||
if (jobId && getItemId && onCheckpoint) {
|
||||
const itemId = getItemId(item);
|
||||
itemsProcessedSinceLastCheckpoint++;
|
||||
|
||||
const publish = itemsProcessedSinceLastCheckpoint >= checkpointInterval;
|
||||
if (publish) {
|
||||
// Checkpoint based on the interval
|
||||
if (itemsProcessedSinceLastCheckpoint >= checkpointInterval) {
|
||||
await onCheckpoint(jobId, itemId);
|
||||
itemsProcessedSinceLastCheckpoint = 0;
|
||||
}
|
||||
await onCheckpoint(jobId, itemId, publish);
|
||||
}
|
||||
|
||||
return result;
|
||||
@@ -193,6 +186,12 @@ export async function processWithRetry<T, R>(
|
||||
onProgress
|
||||
);
|
||||
|
||||
// Final checkpoint if there are remaining items since the last checkpoint
|
||||
if (jobId && getItemId && onCheckpoint && itemsProcessedSinceLastCheckpoint > 0) {
|
||||
// We don't have a specific item ID for the final checkpoint, so we'll use a placeholder
|
||||
await onCheckpoint(jobId, 'final');
|
||||
}
|
||||
|
||||
return results;
|
||||
}
|
||||
|
||||
@@ -310,10 +309,8 @@ export async function processWithResilience<T, R>(
|
||||
});
|
||||
}, JOB_HEARTBEAT_INTERVAL_MS);
|
||||
|
||||
// Define the checkpoint function. Every completed item is written to the
|
||||
// job row so a resume skips exactly what is done; only every
|
||||
// checkpointInterval-th write also publishes a progress event.
|
||||
const onCheckpoint = async (jobId: string, completedItemId: string, publish: boolean = true) => {
|
||||
// Define the checkpoint function
|
||||
const onCheckpoint = async (jobId: string, completedItemId: string) => {
|
||||
const itemName = items.find(item => getItemId(item) === completedItemId)
|
||||
? getItemName(items.find(item => getItemId(item) === completedItemId)!)
|
||||
: 'unknown';
|
||||
@@ -322,7 +319,6 @@ export async function processWithResilience<T, R>(
|
||||
jobId,
|
||||
completedItemId,
|
||||
message: `Processed item: ${itemName}`,
|
||||
silent: !publish,
|
||||
});
|
||||
};
|
||||
|
||||
|
||||
@@ -47,17 +47,11 @@ export async function createDefaultConfig({ userId, envOverrides = {} }: Default
|
||||
const giteaToken = envOverrides.giteaToken || process.env.GITEA_TOKEN || "";
|
||||
const giteaUsername = envOverrides.giteaUsername || process.env.GITEA_USERNAME || "";
|
||||
|
||||
// Schedule config from env. Off unless the environment turns it on, with
|
||||
// the same rule as env-config-loader: SCHEDULE_ENABLED=true or any interval
|
||||
// variable. A fresh install used to start enabled on a plain 24h interval
|
||||
// counted from first login, which the settings card (clock mode only)
|
||||
// showed as "22:00 daily" (#427). Enabling from the card stores the clock
|
||||
// schedule and the browser timezone, so the card and the scheduler agree.
|
||||
const envInterval = process.env.SCHEDULE_INTERVAL || process.env.GITEA_MIRROR_INTERVAL;
|
||||
const scheduleEnabled = envOverrides.scheduleEnabled ??
|
||||
(process.env.SCHEDULE_ENABLED === "true" || !!envInterval);
|
||||
// Schedule config from env - default to ENABLED
|
||||
const scheduleEnabled = envOverrides.scheduleEnabled ??
|
||||
(process.env.SCHEDULE_ENABLED === "false" ? false : true); // Default: ENABLED
|
||||
const scheduleInterval = envOverrides.scheduleInterval ??
|
||||
(envInterval || "0 22 * * *"); // Default: daily at 22:00 in scheduleTimezone
|
||||
(process.env.SCHEDULE_INTERVAL || 86400); // Default: daily
|
||||
const scheduleTimezone = normalizeTimezone(process.env.SCHEDULE_TIMEZONE || "UTC");
|
||||
let scheduleNextRun: Date | null = null;
|
||||
if (scheduleEnabled) {
|
||||
|
||||
@@ -82,23 +82,6 @@ test("mapDbScheduleToUi - infers clock mode for generated cron", () => {
|
||||
expect(mapped.timezone).toBe("Asia/Kolkata");
|
||||
});
|
||||
|
||||
test("mapDbScheduleToUi - reports interval mode for a plain interval instead of a clock schedule", () => {
|
||||
// What SCHEDULE_INTERVAL / GITEA_MIRROR_INTERVAL and older versions store.
|
||||
// The settings card must not present the 22:00 placeholder as saved (#427).
|
||||
const mapped = mapDbScheduleToUi(
|
||||
scheduleConfigSchema.parse({
|
||||
enabled: true,
|
||||
interval: "8h",
|
||||
timezone: "UTC",
|
||||
})
|
||||
);
|
||||
|
||||
expect(mapped.scheduleMode).toBe("interval");
|
||||
expect(mapped.interval).toBe(8 * 3600);
|
||||
expect(mapped.intervalExpression).toBe("8h");
|
||||
expect(mapped.timezone).toBe("UTC");
|
||||
});
|
||||
|
||||
test("includeCollaboratorRepos round-trips through UI -> DB -> UI when true", () => {
|
||||
const ui = buildMinimalUiConfigs({ includeCollaboratorRepos: true });
|
||||
const db = mapUiToDbConfig(
|
||||
|
||||
@@ -1,223 +0,0 @@
|
||||
/**
|
||||
* Route tests for PATCH /api/organizations/:id/status with cascade: ignoring
|
||||
* an organization ignores its repositories so the scheduler leaves them
|
||||
* alone, and including it again restores them (#429).
|
||||
*
|
||||
* Runs against the real schema in an in-memory SQLite database. The route
|
||||
* imports @/lib/db, which is replaced with that database through
|
||||
* mock.module; bun's mock.module is process-wide and leaks into other test
|
||||
* files, so this file registers nothing in the shared process and re-runs
|
||||
* itself in an isolated child (same harness as ../[id].test.ts).
|
||||
*/
|
||||
import { describe, test, expect, mock, beforeEach } from "bun:test";
|
||||
import { Database } from "bun:sqlite";
|
||||
import { drizzle } from "drizzle-orm/bun-sqlite";
|
||||
import { readFileSync } from "node:fs";
|
||||
import { join } from "node:path";
|
||||
import * as schema from "@/lib/db/schema";
|
||||
|
||||
const CHILD_FLAG = "GM_ORG_STATUS_ROUTE_ISOLATED";
|
||||
const isChild = !!process.env[CHILD_FLAG];
|
||||
|
||||
if (!isChild) {
|
||||
test("organization status route (cascade) - isolated child suite", () => {
|
||||
const res = Bun.spawnSync({
|
||||
cmd: [process.execPath, "test", import.meta.path],
|
||||
env: { ...process.env, [CHILD_FLAG]: "1" },
|
||||
stdout: "pipe",
|
||||
stderr: "pipe",
|
||||
});
|
||||
if (res.exitCode !== 0) {
|
||||
console.error(res.stdout.toString());
|
||||
console.error(res.stderr.toString());
|
||||
}
|
||||
expect(res.exitCode).toBe(0);
|
||||
}, 60_000);
|
||||
}
|
||||
|
||||
const sqlite = new Database(":memory:");
|
||||
const db = drizzle({ client: sqlite });
|
||||
|
||||
/**
|
||||
* Apply every migration in drizzle/ in journal order. The shared test setup
|
||||
* stubs drizzle's migrator, so the SQL is run here the way the migrator
|
||||
* would: one file per journal entry, statements split on its breakpoint.
|
||||
*/
|
||||
function applyMigrations(): void {
|
||||
const folder = join(process.cwd(), "drizzle");
|
||||
const journal = JSON.parse(readFileSync(join(folder, "meta", "_journal.json"), "utf8")) as {
|
||||
entries: { tag: string }[];
|
||||
};
|
||||
for (const entry of journal.entries) {
|
||||
const sql = readFileSync(join(folder, `${entry.tag}.sql`), "utf8");
|
||||
for (const statement of sql.split("--> statement-breakpoint")) {
|
||||
if (statement.trim()) sqlite.run(statement);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (isChild) {
|
||||
applyMigrations();
|
||||
// The seed below only needs the two tables the route touches.
|
||||
sqlite.run("PRAGMA foreign_keys = OFF");
|
||||
|
||||
mock.module("@/lib/auth-guards", () => ({
|
||||
requireAuthenticatedUserId: mock(async () => ({ userId: "user-1" })),
|
||||
}));
|
||||
|
||||
mock.module("@/lib/db", () => ({ ...schema, db }));
|
||||
}
|
||||
|
||||
const { PATCH } = isChild
|
||||
? await import("./status")
|
||||
: { PATCH: undefined as any };
|
||||
|
||||
function patch(orgId: string, body: Record<string, unknown>) {
|
||||
return PATCH({
|
||||
params: { id: orgId },
|
||||
request: new Request(`http://localhost/api/organizations/${orgId}/status`, {
|
||||
method: "PATCH",
|
||||
headers: { "Content-Type": "application/json" },
|
||||
body: JSON.stringify(body),
|
||||
}),
|
||||
locals: { session: { userId: "user-1" } },
|
||||
} as any);
|
||||
}
|
||||
|
||||
type RepoSeed = {
|
||||
id: string;
|
||||
userId?: string;
|
||||
organization: string | null;
|
||||
status: string;
|
||||
lastMirrored?: number | null;
|
||||
};
|
||||
|
||||
function seedRepo({ id, userId = "user-1", organization, status, lastMirrored = null }: RepoSeed) {
|
||||
sqlite.run(
|
||||
`INSERT INTO repositories (id, user_id, config_id, name, full_name, normalized_full_name, url, clone_url, owner, organization, default_branch, status, last_mirrored)
|
||||
VALUES (?, ?, 'config-1', ?, ?, ?, ?, ?, ?, ?, 'main', ?, ?)`,
|
||||
[
|
||||
id,
|
||||
userId,
|
||||
id,
|
||||
`${organization ?? "someone"}/${id}`,
|
||||
`${organization ?? "someone"}/${id}`.toLowerCase(),
|
||||
`https://github.com/${organization ?? "someone"}/${id}`,
|
||||
`https://github.com/${organization ?? "someone"}/${id}.git`,
|
||||
organization ?? "someone",
|
||||
organization,
|
||||
status,
|
||||
lastMirrored,
|
||||
]
|
||||
);
|
||||
}
|
||||
|
||||
function repoStatuses(): Record<string, string> {
|
||||
const rows = sqlite.query("SELECT id, status FROM repositories ORDER BY id").all() as { id: string; status: string }[];
|
||||
return Object.fromEntries(rows.map((row) => [row.id, row.status]));
|
||||
}
|
||||
|
||||
function orgStatus(id: string): string | undefined {
|
||||
const row = sqlite.query("SELECT status FROM organizations WHERE id = ?").get(id) as { status: string } | null;
|
||||
return row?.status;
|
||||
}
|
||||
|
||||
beforeEach(() => {
|
||||
if (!isChild) return;
|
||||
sqlite.run("DELETE FROM repositories");
|
||||
sqlite.run("DELETE FROM organizations");
|
||||
sqlite.run(
|
||||
`INSERT INTO organizations (id, user_id, config_id, name, normalized_name, avatar_url, status)
|
||||
VALUES ('org-acme', 'user-1', 'config-1', 'acme', 'acme', '', 'mirrored'),
|
||||
('org-beta', 'user-1', 'config-1', 'beta', 'beta', '', 'mirrored'),
|
||||
('org-theirs', 'user-2', 'config-2', 'acme', 'acme', '', 'mirrored')`
|
||||
);
|
||||
const mirroredAt = 1_700_000_000;
|
||||
seedRepo({ id: "acme-mirrored", organization: "acme", status: "mirrored", lastMirrored: mirroredAt });
|
||||
seedRepo({ id: "acme-synced", organization: "acme", status: "synced", lastMirrored: mirroredAt });
|
||||
seedRepo({ id: "acme-imported", organization: "acme", status: "imported" });
|
||||
seedRepo({ id: "acme-failed", organization: "acme", status: "failed", lastMirrored: mirroredAt });
|
||||
seedRepo({ id: "acme-syncing", organization: "acme", status: "syncing", lastMirrored: mirroredAt });
|
||||
seedRepo({ id: "acme-mirroring", organization: "acme", status: "mirroring" });
|
||||
seedRepo({ id: "acme-deleting", organization: "acme", status: "deleting", lastMirrored: mirroredAt });
|
||||
seedRepo({ id: "acme-ignored-before", organization: "acme", status: "ignored" });
|
||||
seedRepo({ id: "beta-mirrored", organization: "beta", status: "mirrored", lastMirrored: mirroredAt });
|
||||
seedRepo({ id: "personal-mirrored", organization: null, status: "mirrored", lastMirrored: mirroredAt });
|
||||
seedRepo({ id: "theirs-acme-mirrored", userId: "user-2", organization: "acme", status: "mirrored", lastMirrored: mirroredAt });
|
||||
});
|
||||
|
||||
describe.skipIf(!isChild)("PATCH /api/organizations/:id/status", () => {
|
||||
test("ignoring with cascade ignores the organization's idle repositories only", async () => {
|
||||
const response = await patch("org-acme", { status: "ignored", cascade: true });
|
||||
expect(response.status).toBe(200);
|
||||
const data = await response.json();
|
||||
expect(data.success).toBe(true);
|
||||
expect(data.repositoriesChanged).toBe(4);
|
||||
expect(orgStatus("org-acme")).toBe("ignored");
|
||||
|
||||
expect(repoStatuses()).toEqual({
|
||||
"acme-mirrored": "ignored",
|
||||
"acme-synced": "ignored",
|
||||
"acme-imported": "ignored",
|
||||
"acme-failed": "ignored",
|
||||
// In flight and on the way out: left to finish on their own.
|
||||
"acme-syncing": "syncing",
|
||||
"acme-mirroring": "mirroring",
|
||||
"acme-deleting": "deleting",
|
||||
"acme-ignored-before": "ignored",
|
||||
// Other organization, personal repository, other user: untouched.
|
||||
"beta-mirrored": "mirrored",
|
||||
"personal-mirrored": "mirrored",
|
||||
"theirs-acme-mirrored": "mirrored",
|
||||
});
|
||||
});
|
||||
|
||||
test("including with cascade restores mirrored repositories as mirrored and the rest as imported", async () => {
|
||||
await patch("org-acme", { status: "ignored", cascade: true });
|
||||
const response = await patch("org-acme", { status: "imported", cascade: true });
|
||||
expect(response.status).toBe(200);
|
||||
const data = await response.json();
|
||||
// The four the ignore touched plus the one ignored beforehand.
|
||||
expect(data.repositoriesChanged).toBe(5);
|
||||
expect(orgStatus("org-acme")).toBe("imported");
|
||||
|
||||
const statuses = repoStatuses();
|
||||
expect(statuses["acme-mirrored"]).toBe("mirrored");
|
||||
expect(statuses["acme-synced"]).toBe("mirrored");
|
||||
expect(statuses["acme-failed"]).toBe("mirrored");
|
||||
expect(statuses["acme-imported"]).toBe("imported");
|
||||
expect(statuses["acme-ignored-before"]).toBe("imported");
|
||||
expect(statuses["beta-mirrored"]).toBe("mirrored");
|
||||
expect(statuses["theirs-acme-mirrored"]).toBe("mirrored");
|
||||
});
|
||||
|
||||
test("without cascade only the organization changes", async () => {
|
||||
const response = await patch("org-acme", { status: "ignored" });
|
||||
expect(response.status).toBe(200);
|
||||
const data = await response.json();
|
||||
expect(data.repositoriesChanged).toBe(0);
|
||||
expect(orgStatus("org-acme")).toBe("ignored");
|
||||
expect(repoStatuses()["acme-mirrored"]).toBe("mirrored");
|
||||
});
|
||||
|
||||
test("a status other than ignored or imported never touches repositories", async () => {
|
||||
const response = await patch("org-acme", { status: "failed", cascade: true });
|
||||
expect(response.status).toBe(200);
|
||||
const data = await response.json();
|
||||
expect(data.repositoriesChanged).toBe(0);
|
||||
expect(repoStatuses()["acme-mirrored"]).toBe("mirrored");
|
||||
});
|
||||
|
||||
test("answers 404 for another user's organization and changes nothing", async () => {
|
||||
const response = await patch("org-theirs", { status: "ignored", cascade: true });
|
||||
expect(response.status).toBe(404);
|
||||
expect(orgStatus("org-theirs")).toBe("mirrored");
|
||||
expect(repoStatuses()["theirs-acme-mirrored"]).toBe("mirrored");
|
||||
});
|
||||
|
||||
test("rejects an unknown status", async () => {
|
||||
const response = await patch("org-acme", { status: "paused", cascade: true });
|
||||
expect(response.status).toBe(400);
|
||||
expect(orgStatus("org-acme")).toBe("mirrored");
|
||||
});
|
||||
});
|
||||
@@ -1,68 +1,8 @@
|
||||
import type { APIContext } from "astro";
|
||||
import { db, organizations, repositories } from "@/lib/db";
|
||||
import { eq, and, notInArray, sql } from "drizzle-orm";
|
||||
import { db, organizations } from "@/lib/db";
|
||||
import { eq, and } from "drizzle-orm";
|
||||
import { createSecureErrorResponse } from "@/lib/utils";
|
||||
import { requireAuthenticatedUserId } from "@/lib/auth-guards";
|
||||
import { IN_FLIGHT_REPO_STATUSES } from "@/lib/stuck-status-recovery";
|
||||
|
||||
const validStatuses = ["imported", "mirroring", "mirrored", "failed", "ignored"] as const;
|
||||
type OrganizationStatus = (typeof validStatuses)[number];
|
||||
|
||||
/**
|
||||
* Repository statuses an organization-level ignore leaves alone. Rows in
|
||||
* flight are locked by their status (see repo-status-claim.ts) and settle on
|
||||
* their own when the job finishes, a deletion is already on its way out, and
|
||||
* rows already ignored stay that way.
|
||||
*/
|
||||
const IGNORE_CASCADE_EXCLUDED_STATUSES = [...IN_FLIGHT_REPO_STATUSES, "deleting", "ignored"];
|
||||
|
||||
/**
|
||||
* Ignoring an organization also ignores its repositories, so the scheduler
|
||||
* stops syncing them (#429). Including it again restores the ones the ignore
|
||||
* touched: a repository that has been mirrored goes back to mirrored, one
|
||||
* that never was goes back to imported.
|
||||
*/
|
||||
async function cascadeOrganizationStatus(
|
||||
userId: string,
|
||||
organizationName: string,
|
||||
status: OrganizationStatus,
|
||||
now: Date
|
||||
): Promise<number> {
|
||||
if (status === "ignored") {
|
||||
const changed = await db
|
||||
.update(repositories)
|
||||
.set({ status: "ignored", updatedAt: now })
|
||||
.where(
|
||||
and(
|
||||
eq(repositories.userId, userId),
|
||||
eq(repositories.organization, organizationName),
|
||||
notInArray(repositories.status, IGNORE_CASCADE_EXCLUDED_STATUSES)
|
||||
)
|
||||
)
|
||||
.returning({ id: repositories.id });
|
||||
return changed.length;
|
||||
}
|
||||
|
||||
if (status === "imported") {
|
||||
const changed = await db
|
||||
.update(repositories)
|
||||
.set({
|
||||
status: sql`CASE WHEN ${repositories.lastMirrored} IS NOT NULL THEN 'mirrored' ELSE 'imported' END`,
|
||||
updatedAt: now,
|
||||
})
|
||||
.where(
|
||||
and(
|
||||
eq(repositories.userId, userId),
|
||||
eq(repositories.organization, organizationName),
|
||||
eq(repositories.status, "ignored")
|
||||
)
|
||||
)
|
||||
.returning({ id: repositories.id });
|
||||
return changed.length;
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
export async function PATCH({ params, request, locals }: APIContext) {
|
||||
try {
|
||||
@@ -73,7 +13,6 @@ export async function PATCH({ params, request, locals }: APIContext) {
|
||||
const { id } = params;
|
||||
const body = await request.json();
|
||||
const { status } = body;
|
||||
const cascade = body.cascade === true;
|
||||
|
||||
if (!id) {
|
||||
return new Response(
|
||||
@@ -89,6 +28,7 @@ export async function PATCH({ params, request, locals }: APIContext) {
|
||||
}
|
||||
|
||||
// Validate the status
|
||||
const validStatuses = ["imported", "mirroring", "mirrored", "failed", "ignored"];
|
||||
if (!validStatuses.includes(status)) {
|
||||
return new Response(
|
||||
JSON.stringify({
|
||||
@@ -102,14 +42,12 @@ export async function PATCH({ params, request, locals }: APIContext) {
|
||||
);
|
||||
}
|
||||
|
||||
const now = new Date();
|
||||
|
||||
// Update the organization status
|
||||
const [updatedOrg] = await db
|
||||
.update(organizations)
|
||||
.set({
|
||||
.set({
|
||||
status,
|
||||
updatedAt: now,
|
||||
updatedAt: new Date()
|
||||
})
|
||||
.where(
|
||||
and(
|
||||
@@ -132,15 +70,10 @@ export async function PATCH({ params, request, locals }: APIContext) {
|
||||
);
|
||||
}
|
||||
|
||||
const repositoriesChanged = cascade
|
||||
? await cascadeOrganizationStatus(userId, updatedOrg.name, status, now)
|
||||
: 0;
|
||||
|
||||
return new Response(
|
||||
JSON.stringify({
|
||||
success: true,
|
||||
organization: updatedOrg,
|
||||
repositoriesChanged,
|
||||
}),
|
||||
{
|
||||
status: 200,
|
||||
|
||||
+10
-16
@@ -9,8 +9,6 @@ import {
|
||||
mergeGitReposPreferStarred,
|
||||
normalizeGitRepoToInsert,
|
||||
calcBatchSizeForInsert,
|
||||
repositoryIdentityKeys,
|
||||
selectNewRepositoriesByIdentity,
|
||||
} from "@/lib/repo-utils";
|
||||
import { loadOrganizationForkPolicies } from "@/lib/utils/mirror-overrides";
|
||||
import { requireAuthenticatedUserId } from "@/lib/auth-guards";
|
||||
@@ -167,8 +165,7 @@ export const POST: APIRoute = async ({ request, locals }) => {
|
||||
tx
|
||||
.select({
|
||||
normalizedFullName: repositories.normalizedFullName,
|
||||
sourceProvider: repositories.sourceProvider,
|
||||
sourceUrl: repositories.sourceUrl,
|
||||
sourceId: repositories.sourceId,
|
||||
})
|
||||
.from(repositories)
|
||||
.where(eq(repositories.userId, userId)),
|
||||
@@ -178,20 +175,17 @@ export const POST: APIRoute = async ({ request, locals }) => {
|
||||
.where(eq(organizations.userId, userId)),
|
||||
]);
|
||||
|
||||
// A repository is identified by its host and full name, whichever
|
||||
// source row lists it. The same name on two hosts is two
|
||||
// repositories; the same repository seen by two sources on one host
|
||||
// (a personal token and a public-only source) is one, and must not
|
||||
// become two rows that mirror to the same destination.
|
||||
const existingRepoKeys = repositoryIdentityKeys(existingRepos);
|
||||
// The same full name under two sources is two different repositories.
|
||||
const existingRepoKeys = new Set(
|
||||
existingRepos.map((r) => `${r.sourceId ?? ""}|${r.normalizedFullName}`)
|
||||
);
|
||||
const existingOrgMap = new Map(existingOrgs.map((o) => [o.normalizedName, o.status]));
|
||||
|
||||
insertedRepos = selectNewRepositoriesByIdentity(
|
||||
newRepos.filter(
|
||||
(r) => !r.organization || !ignoredOrgNames.has(r.organization.toLowerCase())
|
||||
),
|
||||
existingRepoKeys
|
||||
).fresh;
|
||||
insertedRepos = newRepos.filter(
|
||||
(r) =>
|
||||
!existingRepoKeys.has(`${source.id}|${r.normalizedFullName}`) &&
|
||||
(!r.organization || !ignoredOrgNames.has(r.organization.toLowerCase()))
|
||||
);
|
||||
insertedOrgs = newOrgs.filter((o) => !existingOrgMap.has(o.normalizedName));
|
||||
|
||||
// Update previously failed orgs that now succeeded. The pin must
|
||||
|
||||
@@ -148,17 +148,6 @@ if (isChild) {
|
||||
select: () => ({
|
||||
from: (table: unknown) => ({
|
||||
where: (cond: unknown) => ({
|
||||
// Awaited directly by the existing-repository lookup that keeps
|
||||
// one row per upstream repository; the inserted rows stand in
|
||||
// for the table.
|
||||
then: (
|
||||
onfulfilled: (rows: OrgRow[]) => unknown,
|
||||
onrejected?: (reason: unknown) => unknown
|
||||
) =>
|
||||
Promise.resolve(table === repositoriesTable ? [...insertedRepoRows] : []).then(
|
||||
onfulfilled,
|
||||
onrejected
|
||||
),
|
||||
limit: async (): Promise<OrgRow[]> => {
|
||||
const leaves = conditionLeafValues(cond);
|
||||
if (table === organizationsTable) {
|
||||
|
||||
@@ -15,12 +15,7 @@ import {
|
||||
normalizeSourceUrl,
|
||||
} from "@/lib/source-providers/kinds";
|
||||
import type { SourceRecord } from "@/lib/sources";
|
||||
import {
|
||||
normalizeGitRepoToInsert,
|
||||
calcBatchSizeForInsert,
|
||||
repositoryIdentityKeys,
|
||||
selectNewRepositoriesByIdentity,
|
||||
} from "@/lib/repo-utils";
|
||||
import { normalizeGitRepoToInsert, calcBatchSizeForInsert } from "@/lib/repo-utils";
|
||||
import { resolveOrganizationSkipForks } from "@/lib/utils/mirror-overrides";
|
||||
import { requireAuthenticatedUserId } from "@/lib/auth-guards";
|
||||
|
||||
@@ -264,31 +259,12 @@ export const POST: APIRoute = async ({ request, locals }) => {
|
||||
);
|
||||
|
||||
// Insert repositories. The normalizer stamps the source provider and URL.
|
||||
// Repositories the user already tracks on this host under another source
|
||||
// (the personal source, say) are not inserted a second time: two rows for
|
||||
// one upstream repository would both mirror to the same destination.
|
||||
const existingRepos = await db
|
||||
.select({
|
||||
normalizedFullName: repositories.normalizedFullName,
|
||||
sourceProvider: repositories.sourceProvider,
|
||||
sourceUrl: repositories.sourceUrl,
|
||||
})
|
||||
.from(repositories)
|
||||
.where(eq(repositories.userId, userId));
|
||||
const { fresh: repoRecords, alreadyTracked } = selectNewRepositoriesByIdentity(
|
||||
mirrorableRepos.map((repo) =>
|
||||
normalizeGitRepoToInsert(
|
||||
{ ...repo, organization: repo.organization ?? orgData.name },
|
||||
{ userId, configId, sourceId: source.id }
|
||||
)
|
||||
),
|
||||
repositoryIdentityKeys(existingRepos)
|
||||
const repoRecords = mirrorableRepos.map((repo) =>
|
||||
normalizeGitRepoToInsert(
|
||||
{ ...repo, organization: repo.organization ?? orgData.name },
|
||||
{ userId, configId, sourceId: source.id }
|
||||
)
|
||||
);
|
||||
if (alreadyTracked.length > 0) {
|
||||
console.log(
|
||||
`[Organization import] ${alreadyTracked.length} repositories of ${orgData.name} are already tracked for user ${userId} on this host; not adding them again under source ${source.name}`
|
||||
);
|
||||
}
|
||||
|
||||
// Batch insert repositories to avoid SQLite parameter limit
|
||||
// Compute batch size based on column count
|
||||
|
||||
+1
-2
@@ -38,8 +38,7 @@
|
||||
"esbuild@>=0.27.3 <0.28.1": ">=0.28.1",
|
||||
"sharp": "^0.35.4",
|
||||
"svgo": "^4.1.0",
|
||||
"js-yaml": "^4.3.2",
|
||||
"smol-toml": "^1.7.1"
|
||||
"js-yaml": "^4.3.2"
|
||||
}
|
||||
}
|
||||
}
|
||||
Generated
+7
-8
@@ -9,7 +9,6 @@ overrides:
|
||||
sharp: ^0.35.4
|
||||
svgo: ^4.1.0
|
||||
js-yaml: ^4.3.2
|
||||
smol-toml: ^1.7.1
|
||||
|
||||
importers:
|
||||
|
||||
@@ -2023,8 +2022,8 @@ packages:
|
||||
engines: {node: '>=20.19.5', npm: '>=10.8.2'}
|
||||
hasBin: true
|
||||
|
||||
smol-toml@1.8.0:
|
||||
resolution: {integrity: sha512-kCZr2V3ch9i00x8zXRhjUNVcjG9ijES5dDudkXvUVCT5QlJNQWElSJdZqyPemffHoLNUYwOcou0Fy+ojN0uHSQ==}
|
||||
smol-toml@1.7.0:
|
||||
resolution: {integrity: sha512-aqVvWoyO21L23mb+drl4RmMXbf6N7FdHjAhTRA9ZBL7apWBgfWC16KjrASI+1p9GAroljyMHj6fK67i0UiTNvQ==}
|
||||
engines: {node: '>= 18'}
|
||||
|
||||
source-map-js@1.2.1:
|
||||
@@ -2365,7 +2364,7 @@ snapshots:
|
||||
picomatch: 4.0.5
|
||||
retext-smartypants: 6.2.0
|
||||
shiki: 4.3.1
|
||||
smol-toml: 1.8.0
|
||||
smol-toml: 1.7.0
|
||||
unified: 11.0.5
|
||||
|
||||
'@astrojs/internal-helpers@0.11.0':
|
||||
@@ -2376,7 +2375,7 @@ snapshots:
|
||||
picomatch: 4.0.5
|
||||
retext-smartypants: 6.2.0
|
||||
shiki: 4.3.1
|
||||
smol-toml: 1.8.0
|
||||
smol-toml: 1.7.0
|
||||
unified: 11.0.5
|
||||
|
||||
'@astrojs/markdown-remark@7.2.1':
|
||||
@@ -2641,7 +2640,7 @@ snapshots:
|
||||
dependencies:
|
||||
'@emnapi/core': 1.11.1
|
||||
'@emnapi/runtime': 1.11.1
|
||||
'@napi-rs/wasm-runtime': 1.2.3(@emnapi/core@1.11.1)(@emnapi/runtime@1.11.1)
|
||||
'@napi-rs/wasm-runtime': 1.1.6(@emnapi/core@1.11.1)(@emnapi/runtime@1.11.1)
|
||||
optional: true
|
||||
|
||||
'@bruits/satteri-win32-arm64-msvc@0.10.5':
|
||||
@@ -3296,7 +3295,7 @@ snapshots:
|
||||
picomatch: 4.0.5
|
||||
semver: 7.8.5
|
||||
shiki: 4.3.1
|
||||
smol-toml: 1.8.0
|
||||
smol-toml: 1.7.0
|
||||
svgo: 4.1.0
|
||||
tinyclip: 0.1.15
|
||||
tinyexec: 1.2.4
|
||||
@@ -4681,7 +4680,7 @@ snapshots:
|
||||
arg: 5.0.2
|
||||
sax: 1.6.0
|
||||
|
||||
smol-toml@1.8.0: {}
|
||||
smol-toml@1.7.0: {}
|
||||
|
||||
source-map-js@1.2.1: {}
|
||||
|
||||
|
||||
Reference in new issue
Block a user