From 10b34804f211126789c632b82830bfc328e2da20 Mon Sep 17 00:00:00 2001 From: omen <14905147+adamzend@users.noreply.github.com> Date: Tue, 27 Oct 2026 17:55:53 +0100 Subject: [PATCH] Add progress logging and reliable GitHub mirror enrichment --- CHANGELOG.md | 38 ++++++ README.md | 36 ++++- README.txt | 34 +++++ gitea-github-sync.conf.example | 3 + gitea-github-sync.sh | 87 ++++++++++-- tests/test_error_status.py | 30 +++++ tests/test_progress.py | 20 +++ tests/test_sync.py | 239 +++++++++++++++++++++++++++++++++ 8 files changed, 475 insertions(+), 12 deletions(-) create mode 100644 tests/test_error_status.py create mode 100644 tests/test_progress.py create mode 100644 tests/test_sync.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 90222ad..490ad9d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,39 @@ # Changelog +## v1.0.3 - 2026-10-27 + +### Progress and diagnostics + +- Added configurable progress logs (PROGRESS_INTERVAL, default 60 seconds) with + repository, stage, pages, processed counters and elapsed time. Quiet mode keeps + all messages in its log. Slow operations may delay entries until a work boundary. +- Replaced the expected nonzero worker exit for unavailable GitHub sources with + an explicit result marker. This removes the misleading ERR-trap Failure message + without suppressing genuine exit-code-4 errors such as invalid JSON. +- The October 27 production candidate completed in 7h 49m: 59 repositories found, + 56 successful, one unavailable source, two filtered out, zero failed. This run + validated the comment/redirect fixes; progress and diagnostic changes were + subsequently validated with mock APIs before deployment. + +### Fixed + +- Read Gitea per-issue comments once as a complete list. This endpoint does not + support page/limit; requesting further pages returned the same comments and + incorrectly stopped synchronization. Paginated repository, issue and GitHub + comment lists still use all pages and retain repeated-page protection. +- Follow GitHub GET redirects (301, 302, 303, 307, 308), including root-relative + locations, up to five hops. Only the configured API origin is allowed, so + credentials cannot be forwarded to another host or a downgraded protocol. +- Keep mutation requests non-retrying and non-redirecting. A missing GitHub + source (404) is counted as unavailable, preserves the mirror and does not fail + the run. A 404 can also mean a private source or insufficient token access. + +### Validation + +Mock API regression tests cover discovery pagination, unpaginated Gitea comments, +repeat-run idempotency, redirects, unsafe targets, loops, rate limits, isolation +and quiet background execution. Tests use local mock APIs, not production writes. + ## v1.0.2 - 2026-10-19 ### Changes @@ -44,3 +78,7 @@ packages the scheduling setup rather than claiming a new quiet-mode code change. with metadata, topics, issues and comments. - Included dry-run, foreground apply and quiet background apply modes, rate-limit handling, locking and English/YunoHost documentation. + +GitHub sources returning HTTP 404 are counted as `unavailable`, separately from +`failed` and discovery filters (`skipped`). Existing mirrors are preserved; the +run can succeed with unavailable sources. Check the source or access if unexpected. diff --git a/README.md b/README.md index 2e5633c..c22ad30 100644 --- a/README.md +++ b/README.md @@ -148,9 +148,43 @@ mock API tests, including multiple mirrors, idempotency, throttling and quiet mo Download the script or the complete bundle from [GitHub Releases](https://github.com/adamzend/gitea-github-sync/releases). -The latest release is **v1.0.2**, with weekly root cron setup and deployment notes. +The latest release is **v1.0.3**, adding progress logs and fixing comment retrieval, redirects and unavailable-source diagnostics. ## License This project is licensed under the GNU General Public License v3.0 (GPL-3.0-only). See [LICENSE](LICENSE) for the full license text. + +## Comment pagination and moved GitHub repositories (v1.0.3) + +Gitea's per-issue comments endpoint returns the complete comment list without +pagination. The script reads it once, including lists longer than 50 comments. +Other paginated lists are read until an empty page, with repeated-page detection. +GitHub GET redirects are followed for up to five hops within the configured API +origin only. External origins, protocol downgrades and redirect loops fail safely. +HTTP 404 still requires checking the source URL and token access. + +After upgrading, keep the existing configuration and tokens. A failed run may +have completed other repositories; synchronization markers make reruns idempotent. +Quiet mode secures its log with chmod 600 on each launch, which can mask an ACL +previously granted to a monitoring account. An administrator can restore that +account's read ACL after launch without making the log world-readable. + +GitHub sources returning HTTP 404 are counted as `unavailable`, separately from +`failed` and discovery filters (`skipped`). Existing mirrors are preserved; the +run can succeed with unavailable sources. Check the source or access if unexpected. + +## Progress logging + +Progress is logged at the start of each stage and approximately every 60 seconds +while processing pages, indexing issues, reading Gitea comments, mapping GitHub +comments and comparing/applying updates. Entries include the repository, stage, +page or processed counts and elapsed seconds since script startup. Page counts +are completed pages; processed counts include the current item, not successful +writes. The final summaries report completed operations. +Set PROGRESS_INTERVAL to a positive integer in seconds (default 60, max 99999). +Progress is checked at work boundaries: a slow API request, retry or JSON operation +can delay the next entry. These messages are not a separate liveness watchdog. +In --apply-quiet mode all progress goes to QUIET_LOG_FILE, not the terminal. +An unavailable GitHub source is preserved and counted without a Failure diagnostic. +Other failures, including JSON validation errors, still fail the repository. diff --git a/README.txt b/README.txt index bd55eb0..c6998ff 100644 --- a/README.txt +++ b/README.txt @@ -305,3 +305,37 @@ foreground synchronization engine in a detached process. API reference: https://docs.gitea.com/api/1.23/operations/user-list-repos/ + +## Comment pagination and moved GitHub repositories (v1.0.3) + +Gitea's per-issue comments endpoint returns the complete comment list without +pagination. The script reads it once, including lists longer than 50 comments. +Other paginated lists are read until an empty page, with repeated-page detection. +GitHub GET redirects are followed for up to five hops within the configured API +origin only. External origins, protocol downgrades and redirect loops fail safely. +HTTP 404 still requires checking the source URL and token access. + +After upgrading, keep the existing configuration and tokens. A failed run may +have completed other repositories; synchronization markers make reruns idempotent. +Quiet mode secures its log with chmod 600 on each launch, which can mask an ACL +previously granted to a monitoring account. An administrator can restore that +account's read ACL after launch without making the log world-readable. + +GitHub sources returning HTTP 404 are counted as `unavailable`, separately from +`failed` and discovery filters (`skipped`). Existing mirrors are preserved; the +run can succeed with unavailable sources. Check the source or access if unexpected. + +## Progress logging + +Progress is logged at the start of each stage and approximately every 60 seconds +while processing pages, indexing issues, reading Gitea comments, mapping GitHub +comments and comparing/applying updates. Entries include the repository, stage, +page or processed counts and elapsed seconds since script startup. Page counts +are completed pages; processed counts include the current item, not successful +writes. The final summaries report completed operations. +Set PROGRESS_INTERVAL to a positive integer in seconds (default 60, max 99999). +Progress is checked at work boundaries: a slow API request, retry or JSON operation +can delay the next entry. These messages are not a separate liveness watchdog. +In --apply-quiet mode all progress goes to QUIET_LOG_FILE, not the terminal. +An unavailable GitHub source is preserved and counted without a Failure diagnostic. +Other failures, including JSON validation errors, still fail the repository. diff --git a/gitea-github-sync.conf.example b/gitea-github-sync.conf.example index f21373b..ed79c00 100644 --- a/gitea-github-sync.conf.example +++ b/gitea-github-sync.conf.example @@ -19,3 +19,6 @@ GITHUB_RATE_LIMIT_RETRIES=2 LOCK_FILE='/var/run/gitea-github-sync.lock' # Used only by --apply-quiet. Parent directory must already exist and be writable. QUIET_LOG_FILE='/var/log/gitea-github-sync.log' + +# Progress interval in seconds; slow requests may delay the next entry. +PROGRESS_INTERVAL=60 diff --git a/gitea-github-sync.sh b/gitea-github-sync.sh index a7408e1..241de5b 100755 --- a/gitea-github-sync.sh +++ b/gitea-github-sync.sh @@ -29,6 +29,17 @@ USAGE } log() { printf '[%s] %s%s\n' "$(date '+%F %T')" "${REPO_LABEL:+[$REPO_LABEL] }" "$*" >&2; } die() { log "ERROR: $*"; exit 1; } +# Throttled progress at work boundaries; no extra process or credential output. +PROGRESS_LAST=-1 +PROGRESS_STAGE= +progress() { + local stage=$1; shift + if [[ "$stage" != "$PROGRESS_STAGE" ]] || ((PROGRESS_LAST < 0 || SECONDS - PROGRESS_LAST >= PROGRESS_INTERVAL)); then + log "Progress: $stage; $*; elapsed=${SECONDS}s" + PROGRESS_LAST=$SECONDS + PROGRESS_STAGE=$stage + fi +} on_error() { local rc=$1 line=$2; log "Failure at line $line (exit $rc); synchronization incomplete."; exit "$rc"; } trap 'on_error "$?" "$LINENO"' ERR while (($#)); do @@ -78,6 +89,8 @@ fi : "${GITHUB_API_VERSION:=2026-03-10}" : "${LOCK_FILE:=/var/run/gitea-github-sync.lock}" : "${MAX_PAGES:=10000}" +: "${PROGRESS_INTERVAL:=60}" +[[ "$PROGRESS_INTERVAL" =~ ^[1-9][0-9]{0,4}$ ]] || die "PROGRESS_INTERVAL must be a positive integer (seconds)" : "${GITHUB_RATE_LIMIT_MAX_WAIT:=60}" : "${GITHUB_RATE_LIMIT_RETRIES:=2}" [[ -n "${GITEA_TOKEN:-}" ]] || die "GITEA_TOKEN is missing from the configuration" @@ -142,7 +155,9 @@ response_header() { request() { local service=$1 method=$2 url=$3 output=$4 payload=${5:-} allow_missing=${6:-0} status local headers="$WORK_DIR/response.headers" message remaining reset retry_after wait_seconds - local attempt=0 waited=0 now reset_wait + local attempt=0 waited=0 now reset_wait redirects=0 location api_origin + api_origin=${GITHUB_API#*://} + api_origin="${GITHUB_API%%://*}://${api_origin%%/*}" local -a args=(-q --silent --show-error --connect-timeout 15 --max-time 120 --user-agent 'gitea-github-sync/3.1' -D "$headers" -H "@$HEADER_DIR/$service.headers" -X "$method" -o "$output" -w '%{http_code}') @@ -157,6 +172,18 @@ request() { while :; do status=$(curl "${args[@]}" "$url") || die "Transport error: $method $url; run dry-run before retrying APPLY" if [[ "$status" == 2[0-9][0-9] ]]; then return 0; fi + if [[ "$service" == gh && "$method" == GET && "$status" =~ ^(301|302|303|307|308)$ ]]; then + location=$(response_header "$headers" location) + # Resolve root-relative redirects; never send credentials to another origin. + if [[ "$location" == /* && "$location" != //* ]]; then location="$api_origin$location"; fi + [[ "$location" == "$api_origin/"* && "$location" != *[[:space:]]* && "$location" != *\\* ]] || + die "Rejected GitHub redirect outside the configured API origin" + ((redirects < 5)) || die "GitHub redirect limit exceeded" + redirects=$((redirects + 1)) + log "Following GitHub HTTP $status redirect ($redirects/5)" + url=$location + continue + fi if [[ "$status" == 404 && "$allow_missing" == 1 ]]; then return 4; fi message=$(jq -r 'if type=="object" and (.message|type)=="string" then .message | gsub("[[:cntrl:]]"; " ") | .[:1000] @@ -204,13 +231,18 @@ request() { } validate() { jq -e "$2" "$1" >/dev/null || die "Invalid JSON data: ${1##*/}"; } paginate() { - local service=$1 url=$2 output=$3 page count + local service=$1 url=$2 output=$3 page count total=0 : > "$output" for ((page=1; page<=MAX_PAGES; page++)); do + progress "Reading $service ${output##*/}" "page=$page, records_read=$total" request "$service" GET "$url&page=$page" "$WORK_DIR/page.json" validate "$WORK_DIR/page.json" 'type == "array" and all(.[]; type == "object")' count=$(jq length "$WORK_DIR/page.json") - ((count > 0)) || return 0 + if ((count == 0)); then + log "Read complete: $service ${output##*/}; pages=$((page - 1)), records=$total" + return 0 + fi + total=$((total + count)) if ((page > 1)) && cmp -s "$WORK_DIR/page.json" "$WORK_DIR/previous-page.json"; then die "API returned a repeated page: $url" fi @@ -238,6 +270,9 @@ comment_schema='def pos: type == "number" and . > 0 and floor == .; sync_repository() ( set -Eeuo pipefail local entry=$1 repo_id source current_source +local indexed=0 read_issues=0 read_comments=0 mapped_comments=0 processed_issues=0 processed_comments=0 +PROGRESS_LAST=-1 +PROGRESS_STAGE= local GITEA_OWNER GITEA_REPO GITHUB_OWNER GITHUB_REPO WORK_DIR repo_id=$(jq -r .id <<< "$entry") GITEA_OWNER=$(jq -r .owner.login <<< "$entry") @@ -255,7 +290,17 @@ GT="$GITEA_URL/api/v1/repos/$GITEA_OWNER/$GITEA_REPO" GH="$GITHUB_API/repos/$GITHUB_OWNER/$GITHUB_REPO" log "GitHub $GITHUB_OWNER/$GITHUB_REPO -> Gitea $GITEA_OWNER/$GITEA_REPO; mode $MODE" request gt GET "$GT" "$WORK_DIR/gt-repo.json" -request gh GET "$GH" "$WORK_DIR/gh-repo.json" +if request gh GET "$GH" "$WORK_DIR/gh-repo.json" "" 1; then + : +else + rc=$? + if ((rc == 4)); then + log "Source unavailable (GitHub HTTP 404): removed, private or inaccessible; preserving the mirror" + : > "$RUN_DIR/unavailable-$repo_id" + exit 0 + fi + exit "$rc" +fi for service in gt gh; do validate "$WORK_DIR/$service-repo.json" 'type=="object" and (.id|type=="number") and (.default_branch|type=="string") and (.description==null or (.description|type=="string")) and @@ -296,6 +341,8 @@ done declare -A gh_numbers=() gt_numbers=() gt_comment_ids=() seen_gh=() while IFS= read -r record; do printf '%s\n' "$record" > "$WORK_DIR/record.json" + indexed=$((indexed + 1)) + progress "Indexing GitHub issues" "processed=$indexed" id=$(jq -r .id "$WORK_DIR/record.json") number=$(jq -r .number "$WORK_DIR/record.json") [[ -z "${seen_gh[$id]+x}" && -z "${gh_numbers[$number]+x}" ]] || die "Duplicate GitHub issue $id" @@ -304,17 +351,23 @@ while IFS= read -r record; do done < "$WORK_DIR/gh-issues.jsonl" while IFS= read -r record; do printf '%s\n' "$record" > "$WORK_DIR/record.json" + read_issues=$((read_issues + 1)) + progress "Reading Gitea issues and comments" "issues=$read_issues, comments=$read_comments" id=$(marker "$WORK_DIR/record.json" github-sync-id) [[ -n "$id" ]] || continue [[ -z "${gt_numbers[$id]+x}" ]] || die "Duplicate github-sync-id:$id in Gitea" number=$(jq -r .number "$WORK_DIR/record.json") gt_numbers[$id]=$number printf '%s\n' "$record" > "$WORK_DIR/gt-issue-$id.json" - paginate gt "$GT/issues/$number/comments?limit=50" "$WORK_DIR/comments.jsonl" - jq -s '.' "$WORK_DIR/comments.jsonl" > "$WORK_DIR/comments.json" - validate "$WORK_DIR/comments.json" "$comment_schema" + # Gitea's per-issue comments endpoint returns the complete list and ignores + # page/limit. Only list endpoints that support pagination use paginate(). + request gt GET "$GT/issues/$number/comments" "$WORK_DIR/comments.json" + validate "$WORK_DIR/comments.json" "type == \"array\" and $comment_schema" + jq -c '.[]' "$WORK_DIR/comments.json" > "$WORK_DIR/comments.jsonl" while IFS= read -r comment; do printf '%s\n' "$comment" > "$WORK_DIR/record.json" + read_comments=$((read_comments + 1)) + progress "Reading Gitea issues and comments" "issues=$read_issues, current_issue=#$number, comments=$read_comments" cid=$(marker "$WORK_DIR/record.json" github-comment-id) [[ -n "$cid" ]] || continue key="$number:$cid" @@ -333,6 +386,8 @@ validate "$WORK_DIR/gh-comments.json" "$comment_schema and all(.[]; (.issue_url| declare -A seen_comments=() while IFS= read -r record; do printf '%s\n' "$record" > "$WORK_DIR/record.json" + mapped_comments=$((mapped_comments + 1)) + progress "Mapping GitHub comments" "processed=$mapped_comments" number=$(jq -r '.issue_url | split("/") | last' "$WORK_DIR/record.json") [[ "$number" =~ ^[1-9][0-9]*$ ]] || die "Invalid issue number in GitHub comment" [[ -n "${gh_numbers[$number]+x}" ]] || continue @@ -381,6 +436,8 @@ if ! cmp -s "$WORK_DIR/topics-old.json" "$WORK_DIR/topics-payload.json"; then fi while IFS= read -r record; do printf '%s\n' "$record" > "$WORK_DIR/source.json" + processed_issues=$((processed_issues + 1)) + progress "Comparing and synchronizing issues/comments" "issues=$processed_issues, comments=$processed_comments, mode=$MODE" id=$(jq -r .id "$WORK_DIR/source.json") number=$(jq -r .number "$WORK_DIR/source.json") target=${gt_numbers[$id]:-} @@ -409,6 +466,8 @@ while IFS= read -r record; do [[ -f "$WORK_DIR/gh-comments-$number.jsonl" ]] || continue while IFS= read -r comment; do printf '%s\n' "$comment" > "$WORK_DIR/source.json" + processed_comments=$((processed_comments + 1)) + progress "Comparing and synchronizing issues/comments" "issues=$processed_issues, current_issue=#$number, comments=$processed_comments, mode=$MODE" cid=$(jq -r .id "$WORK_DIR/source.json") jq "$body_filter"'{body:(.body | marked("github-comment-id"; $id))}' \ --argjson id "$cid" "$WORK_DIR/source.json" > "$WORK_DIR/comment-payload.json" @@ -479,7 +538,7 @@ validate "$WORK_DIR/repositories.json" 'all(.[]; (.owner.login|type=="string" and test("^[A-Za-z0-9_.-]+$") and .!="." and .!="..") and (.mirror|type=="boolean") and (.original_url==null or (.original_url|type=="string")))' declare -A discovered_ids=() -discovered=0 selected=0 skipped=0 succeeded=0 failed=0 +discovered=0 selected=0 skipped=0 succeeded=0 failed=0 unavailable=0 while IFS= read -r entry; do repo_id=$(jq -r .id <<< "$entry") [[ -z "${discovered_ids[$repo_id]+x}" ]] || continue @@ -507,9 +566,15 @@ while IFS= read -r entry; do sync_repository "$entry" & worker_pid=$! if wait "$worker_pid"; then - succeeded=$((succeeded + 1)) - log "OK $full_name" + if [[ -f "$RUN_DIR/unavailable-$repo_id" ]]; then + unavailable=$((unavailable + 1)) + log "UNAVAILABLE $full_name: mirror preserved" + else + succeeded=$((succeeded + 1)) + log "OK $full_name" + fi else + rc=$? failed=$((failed + 1)) log "FAILED $full_name" fi @@ -519,6 +584,6 @@ while IFS= read -r entry; do break fi done < "$WORK_DIR/repositories.jsonl" -log "Repositories: discovered=$discovered, selected=$selected, succeeded=$succeeded, failed=$failed, skipped=$skipped" +log "Repositories: discovered=$discovered, selected=$selected, succeeded=$succeeded, failed=$failed, skipped=$skipped, unavailable=$unavailable" ((failed == 0)) || exit 1 log "Account synchronization completed ($MODE)" diff --git a/tests/test_error_status.py b/tests/test_error_status.py new file mode 100644 index 0000000..54c5853 --- /dev/null +++ b/tests/test_error_status.py @@ -0,0 +1,30 @@ +import subprocess +import tempfile +import threading +from pathlib import Path +from http.server import ThreadingHTTPServer + +# Reuse the mock API definitions without running its full regression suite. +ns = {'__file__': str(Path(__file__).with_name('test_sync.py'))} +exec(Path(ns['__file__']).read_text().split('server=ThreadingHTTPServer')[0], ns) +class InvalidJSON(ns['Handler']): + def do_GET(self): + if self.path == '/gh/repos/first/alpha': + self.send_response(200) + self.end_headers() + self.wfile.write(b'not-json') + else: + super().do_GET() +ns['reset']() +ns['repos'] = ns['repos'][:1] +server = ThreadingHTTPServer(('127.0.0.1', 0), InvalidJSON) +threading.Thread(target=server.serve_forever, daemon=True).start() +with tempfile.TemporaryDirectory() as tmp: + config = Path(tmp) / 'config' + config.write_text(f"GITEA_TOKEN=test\nGITEA_URL=http://127.0.0.1:{server.server_port}\nGITHUB_API=http://127.0.0.1:{server.server_port}/gh\nLOCK_FILE={tmp}/lock\n") + result = subprocess.run(['bash', ns['SCRIPT'], '--config', str(config), '--apply', '--no-git'], capture_output=True, text=True) + assert result.returncode != 0, result.stderr + assert 'failed=1' in result.stderr and 'unavailable=0' in result.stderr, result.stderr + assert 'Invalid JSON' in result.stderr and not ns['writes'], result.stderr +server.shutdown() +print('PASS invalid JSON remains a genuine failure, not an unavailable source') diff --git a/tests/test_progress.py b/tests/test_progress.py new file mode 100644 index 0000000..c18949c --- /dev/null +++ b/tests/test_progress.py @@ -0,0 +1,20 @@ +import subprocess +from pathlib import Path + +source = (Path(__file__).resolve().parents[1] / 'gitea-github-sync.sh').read_text() +progress = source[source.index('PROGRESS_LAST=-1'):source.index('on_error() {')] +program = "set -Eeuo pipefail\nlog() { printf '%s\\n' \"$*\"; }\nPROGRESS_INTERVAL=60\n" + progress + """ +SECONDS=0 +progress "Reading comments" "processed=0" +SECONDS=59 +progress "Reading comments" "processed=59" +SECONDS=60 +progress "Reading comments" "processed=60" +progress "Applying issues" "processed=0" +""" +result = subprocess.run(['bash', '-c', program], capture_output=True, text=True, check=True) +lines = result.stdout.splitlines() +assert len(lines) == 3, result.stdout +assert 'processed=59' not in result.stdout +assert 'processed=60' in lines[1] and 'Applying issues' in lines[2] +print('PASS progress interval, counters and immediate stage transitions') diff --git a/tests/test_sync.py b/tests/test_sync.py new file mode 100644 index 0000000..e8549c4 --- /dev/null +++ b/tests/test_sync.py @@ -0,0 +1,239 @@ +import copy +import json +import subprocess +import tempfile +import threading +import time +from pathlib import Path +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from urllib.parse import urlsplit, parse_qs + +SCRIPT = str(Path(__file__).resolve().parents[1] / 'gitea-github-sync.sh') +repos = [] +reads = [] +writes = [] +fail_repo = None +bad_discovery = False +failure_mode = 'permission' +request_gate = None +redirect_mode = None + +def make(n, source, **kwargs): + r = dict(id=n, name=f'local-{n}', owner={'login':'adam'}, mirror=True, + original_url=source, archived=False, has_issues=True, + default_branch='main', description='old', website='', topics=[], issues=[], comments=[]) + r.update(kwargs) + return r + +def reset(): + global repos, fail_repo, bad_discovery + fail_repo = None + bad_discovery = False + reads.clear() + writes.clear() + repos = [make(1, 'https://github.com/first/alpha.git'), + make(2, 'git@github.com:second/beta.git', private=True), + make(3, 'ssh://git@github.com:22/third/gamma.git/', has_issues=False), + make(4, 'https://gitlab.com/other/project.git'), + make(5, 'https://github.com/first/alpha.git', mirror=False), + make(6, 'https://github.com/first/alpha.git', owner={'login':'someone-else'}), + make(7, 'https://github.com.evil.invalid/a/b.git'), + make(8, 'https://user:password@GitHub.com/Fourth/delta.git/')] + +class Handler(BaseHTTPRequestHandler): + def log_message(self, *args): pass + def answer(self, code, obj=None, headers=None): + self.send_response(code) + self.send_header('Content-Type', 'application/json') + for k,v in (headers or {}).items(): self.send_header(k,str(v)) + self.end_headers() + if obj is not None: self.wfile.write(json.dumps(obj).encode()) + def do_GET(self): + global fail_repo + if request_gate is not None: request_gate.wait(timeout=10) + reads.append(self.path) + u = urlsplit(self.path) + page = int(parse_qs(u.query).get('page', ['1'])[0]) + if u.path == '/api/v1/user': return self.answer(200, {'login':'adam'}) + if u.path == '/api/v1/users/adam/repos': + return self.answer(200, {'error':'invalid'} if bad_discovery else repos[page-1:page]) + if u.path.startswith('/gh/') and redirect_mode: + if redirect_mode == 'external': + return self.answer(301, {}, {'Location':'https://example.invalid/steal'}) + if redirect_mode == 'loop': + return self.answer(301, {}, {'Location':self.path}) + if not u.path.startswith('/gh/moved/'): + return self.answer(301, {}, {'Location':self.path.replace('/gh/', '/gh/moved/', 1)}) + gh = u.path.startswith('/gh/') + parts = u.path.split('/repos/')[-1].split('/') + owner, name = parts[:2] + tail = '/' + '/'.join(parts[2:]) if len(parts)>2 else '' + mapping = {'alpha':1, 'beta':2, 'gamma':3, 'delta':8} + if gh: + if name not in mapping: return self.answer(404, {}) + n = mapping[name] + expected = {1:'first',2:'second',3:'third',8:'Fourth'}[n] + assert owner == expected + else: + assert owner == 'adam' + n = int(name.removeprefix('local-')) + r = next(x for x in repos if x['id']==n) + if gh and n == fail_repo: + if failure_mode == 'primary': + return self.answer(403, {'message':'API rate limit exceeded'}, {'X-RateLimit-Remaining':0,'X-RateLimit-Reset':int(time.time())+3600}) + if failure_mode == 'secondary': + return self.answer(403, {'message':'You have exceeded a secondary rate limit'}) + if failure_mode == 'retry': + fail_repo = None + return self.answer(429, {'message':'Too many requests'}, {'Retry-After':1}) + if failure_mode == 'missing': return self.answer(404, {'message':'Not Found'}) + if failure_mode == 'auth': return self.answer(401, {'message':'Bad credentials'}) + return self.answer(403, {'message':'denied'}) + if tail == '': return self.answer(200, dict(id=n, default_branch='main', description=f'source-{n}', homepage=f'https://example.com/{n}') if gh else r) + if tail == '/topics': return self.answer(200, {'names' if gh else 'topics':[f'topic-{n}'] if gh else r['topics']}) + if tail == '/issues': + assert r['has_issues'], 'must not request disabled issues' + data = [dict(id=100+n, number=1, title=f'Issue-{n}', body=f'Body-{n}', state='closed')] if gh else r['issues'] + elif tail == '/issues/comments': + assert r['has_issues'] + data = [dict(id=200+n, body=f'Comment-{n}', issue_url=f'https://api.github.com/repos/{owner}/{name}/issues/1')] if gh else r['comments'] + elif tail.endswith('/comments'): return self.answer(200, r['comments']) + else: return self.answer(404, {}) + self.answer(200, data[page-1:page]) + def write(self): + data = json.loads(self.rfile.read(int(self.headers.get('Content-Length',0))) or '{}') + parts = self.path.split('/repos/adam/local-')[1].split('/') + n = int(parts[0]); tail = '/'+'/'.join(parts[1:]) if len(parts)>1 else '' + writes.append((n,self.command,tail,data)) + r = next(x for x in repos if x['id']==n) + if tail == '': r.update(data); return self.answer(200,r) + if tail == '/topics': r['topics']=data['topics']; return self.answer(204) + if tail == '/mirror-sync': return self.answer(200) + if tail == '/issues': + item = dict(data,id=1000+n,number=10,state='closed' if data['closed'] else 'open') + r['issues'].append(item); return self.answer(201,item) + if tail.endswith('/comments'): + item = dict(data,id=2000+n); r['comments'].append(item); return self.answer(201,item) + self.answer(404,{}) + do_POST=write + do_PATCH=write + do_PUT=write + +server=ThreadingHTTPServer(('127.0.0.1',0),Handler) +threading.Thread(target=server.serve_forever,daemon=True).start() +with tempfile.TemporaryDirectory() as tmp: + conf=Path(tmp)/'config' + conf.write_text(f"GITEA_TOKEN='test'\nGITEA_URL='http://127.0.0.1:{server.server_port}'\nGITHUB_API='http://127.0.0.1:{server.server_port}/gh'\nLOCK_FILE='{tmp}/lock'\nGITEA_REPO='obsolete-name'\nGITHUB_RATE_LIMIT_MAX_WAIT=2\n") + def run(mode='--apply', success=True): + p=subprocess.run(['bash',SCRIPT,'--config',str(conf),mode,'--no-git'],capture_output=True,text=True) + assert (p.returncode==0)==success,p.stderr + assert 'password' not in p.stderr + return p.stderr + reset() + out=run('--dry-run') + assert not writes and 'selected=4, succeeded=4, failed=0, skipped=4' in out,out + assert any('page=9' in p for p in reads) + assert 'Progress: Reading gt repositories.jsonl; page=1' in out,out + assert 'Progress: Comparing and synchronizing issues/comments' in out,out + print('PASS account discovery, full pagination, private mirror and source URL filters') + run() + assert {n for n,_,_,_ in writes}=={1,2,3,8} + for r in repos: + if r['id'] in {1,2,8}: + assert len(r['issues'])==len(r['comments'])==1 + assert r['issues'][0]['title']==f"Issue-{r['id']}" + assert r['comments'][0]['body'].startswith(f"Comment-{r['id']}") + if r['id']==3: assert not r['issues'] and r['description']=='source-3' + writes.clear() + run() + assert not writes + print('PASS isolated state, renamed destinations, disabled issues, multi-repository idempotency') + repos[0]['comments'].extend(dict(id=9000+i,body=f'Local comment {i}') for i in range(60)) + reads.clear(); run(); assert not writes + assert len([p for p in reads if p == '/api/v1/repos/adam/local-1/issues/10/comments']) == 1 + assert len(repos[0]['comments']) == 61 + print('PASS complete unpaginated list over 50 comments read once without duplicate writes') + reset(); fail_repo=1 + out=run(success=False) + assert {n for n,_,_,_ in writes}=={2,3,8} + assert 'selected=4, succeeded=3, failed=1' in out,out + print('PASS one repository failure does not stop remaining repositories') + reset(); fail_repo=1; failure_mode='missing' + out=run() + assert 'unavailable=1' in out and 'failed=0' in out,out + assert 'Failure at line' not in out and 'FAILED ' not in out,out + assert {n for n,_,_,_ in writes}=={2,3,8} + failure_mode='permission' + print('PASS missing GitHub source preserved and counted separately without failing the run') + reset(); repos[0]['archived']=True + out=run(success=False) + assert {n for n,_,_,_ in writes}=={2,3,8} + print('PASS archived mirrors reported without blocking other repositories') + reset(); repos=[] + out=run() + assert not writes and 'discovered=0' in out + print('PASS empty account') + reset(); bad_discovery=True + run(success=False) + assert not writes + print('PASS discovery failure prevents writes') + for failure_mode in ['primary','secondary','auth']: + reset(); fail_repo=1 + out=run(success=False) + assert not writes and 'Account scan aborted' in out,out + assert len([p for p in reads if p.startswith('/gh/')])==1,reads + if failure_mode=='primary': assert 'quota resets at' in out + if failure_mode=='auth': assert 'Bad credentials' in out + print(f'PASS {failure_mode} error is diagnosed and stops further GitHub requests') + failure_mode='retry' + reset(); fail_repo=1 + out=run() + assert 'Waiting 1 seconds' in out and 'succeeded=4' in out,out + assert {n for n,_,_,_ in writes}=={1,2,3,8} + print('PASS Retry-After respected and synchronization resumes') + reset(); redirect_mode='same' + out=run() + assert 'Following GitHub HTTP 301' in out and 'succeeded=4' in out,out + writes.clear(); run(); assert not writes + print('PASS same-origin redirects and idempotent unpaginated Gitea comments') + for redirect_mode in ['external','loop']: + reset(); out=run(success=False) + assert not writes + assert ('Rejected GitHub redirect' if redirect_mode=='external' else 'redirect limit exceeded') in out,out + redirect_mode=None + print('PASS cross-origin redirects and redirect loops rejected before writes') + reset() + quiet_log=Path(tmp)/'quiet.log' + with conf.open('a') as f: f.write(f"QUIET_LOG_FILE='{quiet_log}'\n") + request_gate=threading.Event() + p=subprocess.run(['bash',SCRIPT,'--config',str(conf),'--apply-quiet','--no-git'],capture_output=True,text=True,timeout=5) + assert p.returncode==0 and not p.stdout and not p.stderr,p + assert not writes, 'launcher must return before blocked background work completes' + request_gate.set() + request_gate=None + deadline=time.monotonic()+60 + while time.monotonic()