forge server caddyfile leaves /<owner>/<repo>/releases/download/ out of gzip, gives it no-cache like pages, since a tag's files can be published again, and sends Content-Disposition: attachment with nosniff, since the files are anyone's and sit on the forge's own origin. demo/serve.ts does the same. forge mirror syncs the pages by checksum as before, without the downloads, then the downloads alone by size and mtime, rclone's default: their mtime is their release commit's, so an unchanged binary isn't read again on every sync. Each pass's filters keep it off the other's files in the bucket. The edge served dot-less names like greet-linux or SHA256SUMS as directories (a redirect to .../index.html); downloads are files now, with the same headers as from the forge.
- commit
- 56edc8a4a7c8b3469bcf58f7d70da14860bc6a7b
- parent
- bccd786d4bf2f62cc7f639c6091ca447af604c2e
- author
- cat <admin@dev.site>
- kind
- feat risk score 1.9 of 3, Jev's estimate
| M | bin/ | 35 | ++++++++++++++++++++++-------- |
| M | demo/ | 3 | +++ |
| M | deploy/ | 10 | ++++++++- |
| A | test/ | 18 | +++++++++++++++ |
| A | test/ | 36 | ++++++++++++++++++++++++++++++ |
@@ -966,9 +966,11 @@ cmd_server_caddyfile() {
966966 [ -n "$gohost" ] || gohost={host}967967 cat <<EOF968968 $addr {969- # Compress pages and assets, never the event stream (it would buffer) or970- # code.db, which code search reads in byte ranges of the file as stored.971- encode {969+ # Compress pages and assets, never the event stream (it would buffer),970+ # code.db, which code search reads in byte ranges of the file as stored,971+ # or release downloads, which are mostly compressed already.972+ @compress not path_regexp ^/[^/]+/[^/]+/releases/download/973+ encode @compress {972974 gzip973975 match {974976 header Content-Type text/html*@@ -1028,7 +1030,9 @@ EOF
10281030 # Assets have content-hashed names (lib/assets.ts), so they never change.10291031 # Pages and the JSON and text indexes they fetch revalidate on every use:10301032 # Caddy sends ETag and Last-Modified, so an unchanged one costs a 304.1031- # Sites served from here keep their own caching.1033+ # Sites served from here keep their own caching. Release downloads can be1034+ # published again under their tag, so they revalidate like pages, and1035+ # they download rather than open: they are anyone's files, on this origin.10321036 hashed='^/assets/([^/]+\.)?[0-9A-Za-z_-]{10}\.[a-z]+$'10331037 cat <<EOF10341038 handle {@@ -1040,6 +1044,9 @@ EOF
10401044 not path /sites/* /previews/*10411045 }10421046 header @fresh Cache-Control no-cache1047+ @download path_regexp ^/[^/]+/[^/]+/releases/download/.1048+ header @download Content-Disposition attachment1049+ header @download X-Content-Type-Options nosniff10431050 file_server10441051 }10451052 }@@ -2881,12 +2888,22 @@ cmd_mirror() {
28812888 trap 'rm -rf "$FORGE_STATE/mirror.lock"' EXIT28822889 trim_log "$FORGE_STATE/mirror.log"28832890 t0=$(date +%s)2884- RCLONE_S3_PROVIDER=${FORGE_MIRROR_PROVIDER:-Cloudflare} RCLONE_S3_ENDPOINT=${FORGE_MIRROR_S3%%://*}://${u%%/*} \2885- RCLONE_S3_ACCESS_KEY_ID=${FORGE_MIRROR_KEY_ID:-} RCLONE_S3_SECRET_ACCESS_KEY=${FORGE_MIRROR_SECRET:-} \2886- RCLONE_S3_REGION=${FORGE_MIRROR_REGION:-auto} RCLONE_S3_NO_CHECK_BUCKET=true \2887- rclone sync "$FORGE_WWW" ":s3:${u#*/}" --config "" --checksum --fast-list --transfers 16 --checkers 32 $dry \2888- --exclude '/sites/**' --exclude '/previews/**' --exclude '/hosts/**' --exclude '*.old/**' ||2891+ export RCLONE_S3_PROVIDER="${FORGE_MIRROR_PROVIDER:-Cloudflare}" RCLONE_S3_ENDPOINT="${FORGE_MIRROR_S3%%://*}://${u%%/*}" \2892+ RCLONE_S3_ACCESS_KEY_ID="${FORGE_MIRROR_KEY_ID:-}" RCLONE_S3_SECRET_ACCESS_KEY="${FORGE_MIRROR_SECRET:-}" \2893+ RCLONE_S3_REGION="${FORGE_MIRROR_REGION:-auto}" RCLONE_S3_NO_CHECK_BUCKET=true2894+ # Pages by checksum: a rebuilt page that came out the same has a new2895+ # mtime, and isn't uploaded again. That reads every file, so release2896+ # downloads, which can be big and never change in place, are left to a2897+ # second pass by size and mtime (rclone's default): theirs is their2898+ # release commit's time (store_tree), so unchanged ones aren't read.2899+ # Each pass leaves what the other's filters cover alone in the bucket.2900+ rclone sync "$FORGE_WWW" ":s3:${u#*/}" --config "" --checksum --fast-list --transfers 16 --checkers 32 $dry \2901+ --exclude '/sites/**' --exclude '/previews/**' --exclude '/hosts/**' --exclude '*.old/**' \2902+ --exclude '/*/*/releases/download/**' ||28892903 die "mirror: rclone sync failed"2904+ rclone sync "$FORGE_WWW" ":s3:${u#*/}" --config "" --fast-list --transfers 4 --checkers 32 $dry \2905+ --filter '- *.old/**' --filter '- *.tmp.*/**' --filter '+ /*/*/releases/download/**' --filter '- **' ||2906+ die "mirror: rclone sync of the release downloads failed"28902907 say "$(date -u +%Y-%m-%dT%H:%M:%SZ) mirror: synced $FORGE_WWW to ${u#*/} in $(($(date +%s) - t0))s"28912908 }28922909 @@ -34,6 +34,7 @@ function byteRange(h: string, size: number): [number, number] | null | undefined
3434 // Hashed assets never change; pages and the indexes they fetch revalidate,3535 // and get a 304 when unchanged. Sites keep their own caching.3636 const HASHED = /^\/assets\/([^/]+\.)?[0-9A-Za-z_-]{10}\.[a-z]+$/;37+const DOWNLOAD = /^\/[^/]+\/[^/]+\/releases\/download\/./;3738 const cacheControl = (p: string): string | null =>3839 HASHED.test(p) ? 'public, max-age=31536000, immutable' : /^\/(sites|previews)\//.test(p) ? null : 'no-cache';3940 @@ -64,6 +65,8 @@ http.createServer((req, res) => {
6465 const headers: http.OutgoingHttpHeaders = { 'Content-Type': TYPES[path.extname(file)] ?? 'application/octet-stream', ETag: etag, 'Last-Modified': st.mtime.toUTCString(), 'Accept-Ranges': 'bytes' };6566 const cc = cacheControl(url.pathname);6667 if (cc) headers['Cache-Control'] = cc;68+ // Release downloads are saved, not opened, on the forge's origin.69+ if (DOWNLOAD.test(url.pathname)) Object.assign(headers, { 'Content-Disposition': 'attachment', 'X-Content-Type-Options': 'nosniff' });6770 const since = req.headers['if-modified-since'];6871 const match = req.headers['if-none-match'];6972 if (match ? match === etag : since && Date.parse(since) >= Math.floor(st.mtimeMs / 1000) * 1000) {@@ -41,8 +41,16 @@
4141 header X-Forge-Offline "mirror"4242 header Cache-Control "no-store"4343 # Like the forge's file server: /a/b -> /a/b/, /a/b/ -> /a/b/index.html.44- @bare path_regexp ^(.*/)?[^./]+$44+ # Release downloads are files whatever their names (greet-linux,45+ # SHA256SUMS), and download rather than open, as from the forge.46+ @bare {47+ path_regexp ^(.*/)?[^./]+$48+ not path_regexp ^/[^/]+/[^/]+/releases/download/.49+ }4550 redir @bare {path}/ 30851+ @download path_regexp ^/[^/]+/[^/]+/releases/download/.52+ header @download Content-Disposition attachment53+ header @download X-Content-Type-Options nosniff4654 # No query strings: on a bucket some (?acl, ?list-type) mean something.4755 @dir path */4856 rewrite @dir {path}index.html?@@ -0,0 +1,18 @@
1+# Caddy serves release downloads uncompressed, revalidated like pages2+# (they can be published again under their tag), and as attachments, since3+# anyone's files are on the forge's own origin. So does the edge from the4+# mirror, where a download is a file whatever its name.5+. "$(dirname "$0")/../lib.sh"6+forge server caddyfile git.test >"$T/Caddyfile" 2>"$T_LOG" || fail "caddyfile"7+has "$T/Caddyfile" '@compress not path_regexp \^/\[\^/\]\+/\[\^/\]\+/releases/download/$'8+has "$T/Caddyfile" 'encode @compress \{'9+has "$T/Caddyfile" 'header @download Content-Disposition attachment'10+has "$T/Caddyfile" 'header @fresh Cache-Control no-cache'11+lacks "$T/Caddyfile" 'releases.*immutable'12+e=$T_HOME/deploy/edge/Caddyfile13+has "$e" 'not path_regexp \^/\[\^/\]\+/\[\^/\]\+/releases/download/\.$'14+has "$e" 'header @download Content-Disposition attachment'15+if command -v caddy >/dev/null 2>&1; then16+ ok caddy adapt --config "$T/Caddyfile" --adapter caddyfile17+ FORGE_DOMAIN=src.test FORGE_MIRROR_URL=https://mirror.test ok caddy adapt --config "$e" --adapter caddyfile18+fi@@ -0,0 +1,36 @@
1+# forge mirror syncs the pages by checksum without the release downloads,2+# then the downloads alone by size and mtime, so unchanged binaries aren't3+# read; each pass leaves the other's files in the bucket alone. rclone here4+# is a stand-in that writes down how it was called.5+. "$(dirname "$0")/../lib.sh"6+user dami7+repo dami/greet8+clone dami dami/greet "$T/w/d"9+at "$T/w/d" commit "first"10+at "$T/w/d" git tag v111+ok push_as dami "$T/w/d" origin main12+mkdir "$T/f"13+printf 'greet\n' >"$T/f/greet"14+FORGE_USER=dami ok at "$T/w/d" "$FORGE" release publish v1 "$T/f/greet"15+touch "$FORGE_WWW/index.html"16+17+mkdir "$T/bin"18+cat >"$T/bin/rclone" <<EOF19+#!/bin/sh20+printf '%s\n' "\$*" >>"$T/rclone.log"21+EOF22+chmod +x "$T/bin/rclone"23+export PATH="$T/bin:$PATH" FORGE_MIRROR_S3=https://r2.test/pages24+ok forge mirror25+eq "$(wc -l <"$T/rclone.log" | tr -d ' ')" 226+sed -n 1p "$T/rclone.log" >"$T/pass1"27+sed -n 2p "$T/rclone.log" >"$T/pass2"28+has "$T/pass1" "^sync $FORGE_WWW :s3:pages .*--checksum"29+has "$T/pass1" "--exclude /sites/\*\* --exclude /previews/\*\* --exclude /hosts/\*\* --exclude \*\.old/\*\* --exclude /\*/\*/releases/download/\*\*$"30+has "$T/pass2" "^sync $FORGE_WWW :s3:pages "31+lacks "$T/pass2" 'checksum'32+has "$T/pass2" "--filter - \*\.old/\*\* --filter - \*\.tmp\.\*/\*\* --filter \+ /\*/\*/releases/download/\*\* --filter - \*\*$"33+# --dry-run goes to both.34+: >"$T/rclone.log"35+ok forge mirror --dry-run36+eq "$(grep -c -- '--dry-run' "$T/rclone.log")" 2