Release downloads are hard links into the store, which name no tree the way a site's symlink does, so forge gc keeps every store tree a refs/releases/* ref names, frees the trees of deleted releases, and removes the downloads of a release whose ref was deleted outside the hooks (their space only comes back with them). A backup needs nothing new, since releases are refs; restore_finish links a public repo's downloads again from them. git archive stamps the files with the release commit's time, so the restored files have the mtimes they had, and the mirror sees nothing to upload.
- commit
- bccd786d4bf2f62cc7f639c6091ca447af604c2e
- parent
- 1616038df12dd64efb5a03f499359406e76d5904
- author
- cat <admin@dev.site>
- kind
- fix risk score 1.4 of 3, Jev's estimate
| M | bin/ | 31 | +++++++++++++++++---- |
| A | test/ | 43 | ++++++++++++++++++++++++++++++ |
| A | test/ | 34 | ++++++++++++++++++++++++ |
@@ -27,7 +27,7 @@
2727 # forge build [owner/repo]2828 # forge jev [owner/repo] ask Jev about new commits and reviews (needs TYPESAFE_API_KEY)2929 # forge metrics [n] [owner/repo] recent builds and where their time went30-# forge gc [--dry-run] [--age minutes] remove unused site files and hostnames; repack, commit-graph30+# forge gc [--dry-run] [--age minutes] remove unused site and release files, hostnames; repack, commit-graph3131 # forge backup [--dry-run] [--force] snapshot the forge with restic, then check it and test-restore it3232 # forge backups [--status] the last backup's health, then the snapshots3333 # forge restore [<snapshot>|latest] [--to <dir>] [--from <dir>] bring the forge back from a backup@@ -2275,8 +2275,9 @@ was_review() { grep -qxF "branch $2" "$1"/forge/reviews/* 2>/dev/null; }
22752275 22762276 # Clean up after deploys; safe to run from cron. Removes previews of review22772277 # branches that are gone (deleted outside the hook, or before it removed2278-# previews), hostnames whose deploy is gone, and store trees that no site or2279-# preview links to (hostnames point at those links, never at the store).2278+# previews), hostnames whose deploy is gone, downloads of deleted releases,2279+# and store trees that no site or preview links to (hostnames point at those2280+# links, never at the store) and no release ref names.22802281 # A deploy extracts its tree before linking it, so entries changed in the22812282 # last --age minutes stay. Then repacks every repo (repack_repo).22822283 cmd_gc() {@@ -2304,6 +2305,18 @@ cmd_gc() {
23042305 [ "$dry" = 1 ] || drop_preview "$p" || { say "forge: could not remove $p"; continue; }23052306 echo "$did preview $OWNER/$NAME ${p#refs/site-previews/}"23062307 done2308+ # Downloads of releases whose ref went outside the hooks (hard2309+ # links, so their store tree's space comes back only with them).2310+ gone=02311+ for w in "$FORGE_WWW/$OWNER/$NAME/releases/download"/*/; do2312+ [ -d "$w" ] || continue2313+ t=$(basename "$w" | tr '~' /)2314+ [ "$t" != latest ] && ! git rev-parse -q --verify "refs/releases/$t" >/dev/null || continue2315+ [ "$dry" = 1 ] || rm -rf "$w"2316+ echo "$did release $OWNER/$NAME $t"2317+ gone=12318+ done2319+ if [ "$gone" = 1 ] && [ "$dry" = 0 ]; then release_latest "$FORGE_WWW"; fi23072320 unset GIT_DIR23082321 done23092322 if [ -d "$FORGE_WWW/hosts" ]; then@@ -2316,7 +2329,9 @@ cmd_gc() {
23162329 set --23172330 [ "$age" = 0 ] || set -- -mmin +"$age"23182331 old=$(find "$FORGE_STATE/store" -mindepth 1 -maxdepth 1 "$@")2319- used=$(for l in "$FORGE_WWW"/sites/*/*/* "$FORGE_WWW"/previews/*/*/*/*; do if [ -L "$l" ]; then basename "$(readlink "$l")"; fi; done)2332+ used=$(for l in "$FORGE_WWW"/sites/*/*/* "$FORGE_WWW"/previews/*/*/*/*; do if [ -L "$l" ]; then basename "$(readlink "$l")"; fi; done2333+ # Release files are hard links, which don't name their tree: the refs do.2334+ for d in $repos; do git --git-dir="$FORGE_ROOT/$d" for-each-ref --format='%(tree)' refs/releases/; done)23202335 for s in $old; do23212336 printf '%s\n' "$used" | grep -qxF "${s##*/}" && continue23222337 k=$(du -sk "$s" | cut -f1)@@ -2735,8 +2750,9 @@ cmd_backups() {
27352750 27362751 # What a backup leaves out, rebuilt for a forge whose files were just27372752 # restored: what server init and keys sync write, hooks pointing at this2738-# host's forge, pinned review revisions, extracted sites, every page. Which2739-# repos are private came back with their files, and nothing here changes it.2753+# host's forge, pinned review revisions, extracted sites and release files,2754+# every page. Which repos are private came back with their files, and2755+# nothing here changes it.27402756 restore_finish() {27412757 load_config27422758 cmd_server_init@@ -2752,6 +2768,9 @@ restore_finish() {
27522768 awk '$2 == "commit" { print "update refs/forge/keep/" $1 " " $1 }' | git update-ref --stdin27532769 # A private repo's sites stay unserved (sites_public), as before the backup.27542770 if sites_public "$GIT_DIR"; then redeploy_sites; fi2771+ # Releases are refs too; a private repo's are served nowhere. The files2772+ # get their commit's time again, so the mirror sees nothing new.2773+ if [ -f "$GIT_DIR/git-daemon-export-ok" ]; then releases_into "$FORGE_WWW"; fi27552774 git update-server-info27562775 unset GIT_DIR27572776 done@@ -0,0 +1,43 @@
1+# forge gc keeps the store trees that release refs name (their files are2+# hard links, which point at no tree), and frees those of deleted3+# releases, with the downloads of a release whose ref went behind the4+# hooks' back.5+. "$(dirname "$0")/../lib.sh"6+user dami7+repo dami/greet8+clone dami dami/greet "$T/w/d"9+mkdir "$T/f"10+export FORGE_USER=dami11+g=$FORGE_ROOT/dami/greet.git12+for v in 1 2 3; do13+ tick 360014+ at "$T/w/d" commit "v$v"15+ at "$T/w/d" git tag "v$v"16+ ok push_as dami "$T/w/d" origin main17+ printf 'greet %s\n' "$v" >"$T/f/greet"18+ ok at "$T/w/d" "$FORGE" release publish "v$v" "$T/f/greet"19+ eval "tree$v=$(git --git-dir="$g" rev-parse "refs/releases/v$v^{tree}")"20+done21+store=$FORGE_ROOT/.forge/store22+dl=$FORGE_WWW/dami/greet/releases/download23+ok at "$T/w/d" "$FORGE" release rm v324+git --git-dir="$g" update-ref -d refs/releases/v225+exists "$dl/v2/greet"26+27+ok forge gc --age 0 --dry-run28+has "$T_LOG" '^would remove release dami/greet v2$'29+has "$T_LOG" "^would remove store $tree3 "30+exists "$dl/v2/greet"31+ok forge gc --age 032+has "$T_LOG" '^removed release dami/greet v2$'33+has "$T_LOG" "^removed store $tree2 "34+has "$T_LOG" "^removed store $tree3 "35+lacks "$T_LOG" "$tree1"36+missing "$dl/v2"37+missing "$store/$tree2"38+missing "$store/$tree3"39+exists "$store/$tree1/greet"40+has "$dl/v1/greet" 'greet 1'41+has "$dl/latest/greet" 'greet 1'42+ok forge gc --age 043+lacks "$T_LOG" 'removed (release|store)'@@ -0,0 +1,34 @@
1+# Releases are refs, so a backup holds them with nothing more, and a2+# restore links a public repo's downloads again, the same bytes with the3+# same mtimes (so the mirror uploads nothing); a private repo's stay unserved.4+. "$(dirname "$0")/../lib.sh"5+user dami6+repo dami/open7+repo --private dami/secret8+mkdir "$T/f"9+printf 'tool\000\n' >"$T/f/tool"10+export FORGE_USER=dami11+for r in open secret; do12+ clone dami "dami/$r" "$T/w/$r"13+ at "$T/w/$r" commit "first"14+ at "$T/w/$r" git tag v115+ ok push_as dami "$T/w/$r" origin main16+ ok at "$T/w/$r" "$FORGE" release publish v1 "$T/f/tool"17+done18+unset FORGE_USER19+dl=releases/download/v120+mtime() { node -p 'Math.floor(require("fs").statSync(process.argv[1]).mtimeMs)' "$1"; }21+was=$(mtime "$FORGE_WWW/dami/open/$dl/tool")22+23+ok forge backup --dry-run24+st=$FORGE_ROOT/.forge/backup/stage25+missing "$st/.forge/store"26+r=$T/restored27+ok env FORGE_ROOT="$r" FORGE_WWW="$r/.forge/www" FORGE_CACHE="$T/cache2" "$FORGE" restore --from "$st"28+ok cmp "$T/f/tool" "$r/.forge/www/dami/open/$dl/tool"29+ok cmp "$FORGE_WWW/dami/open/$dl/SHA256SUMS.sig" "$r/.forge/www/dami/open/$dl/SHA256SUMS.sig"30+exists "$r/.forge/www/dami/open/releases/download/latest/tool"31+exists "$r/.forge/www/dami/open/releases/index.html"32+eq "$(mtime "$r/.forge/www/dami/open/$dl/tool")" "$was"33+missing "$r/.forge/www/dami/secret"34+eq "$(git --git-dir="$r/dami/secret.git" rev-parse refs/releases/v1)" "$(git --git-dir="$FORGE_ROOT/dami/secret.git" rev-parse refs/releases/v1)"