lib/private-server.ts serves the private root to a repo's owner, collaborators and readers (lib/access.ts, the rule can_read applies) who show a token, as Authorization: Bearer or as basic auth with the token for password or user name, and are still users of the forge. A token for one repo reads that repo only. Pages get the headers Caddy gives pages (ETag, 304s, a byte range for code.db, through lib/static.ts) with Cache-Control "private, no-cache", which keeps them out of shared caches. Caddy serves a file of www when there is one and hands any other path to it, so it needs no list of private repos, and every missing page, public or not, gets the same 404 from one place: no token, a bad, expired or revoked one, a token of a user whose keys are gone, someone else's private repo and a page that isn't there all look alike, byte for byte. start.sh and provision.sh run it as git. demo/serve.ts hands it its misses the same way, and the code search bench can send a token, so private-web-range reads a private code.db in ranges.
- commit
- dcbb5255e4f0f8bc79fcbf7975dcb7184e22106b
- parent
- 244787abfb892215ef648f0a06415435f0657488
- author
- cat <admin@dev.site>
- kind
- feat risk score 2.1 of 3, Jev's estimate
| M | bin/ | 14 | +++- |
| M | demo/ | 86 | ++++------------- |
| A | lib/ | 148 | ++++++++++++++++++++++++++++++ |
| A | lib/ | 91 | ++++++++++++++++++ |
| M | server/ | 5 | + |
| M | server/ | 5 | +- |
| M | test/ | 16 | ++- |
| M | test/ | 35 | +++++++ |
| A | test/ | 68 | ++++++++++++++ |
| A | test/ | 24 | +++++ |
| A | test/ | 90 | ++++++++++++++++++ |
@@ -1078,9 +1078,18 @@ EOF
10781078 # Sites served from here keep their own caching. Release downloads can be10791079 # published again under their tag, so they revalidate like pages, and10801080 # they download rather than open: they are anyone's files, on this origin.1081+ #1082+ # A path with no file (or directory) in www goes to the private server:1083+ # private repos' pages for their readers, which live outside www, and1084+ # the 404 every other missing page gets (lib/private-server.ts). So1085+ # Caddy needs no list of private repos.10811086 hashed='^/assets/([^/]+\.)?[0-9A-Za-z_-]{10}\.[a-z]+$'10821087 cat <<EOF1083- handle {1088+ @public file {1089+ root $FORGE_WWW1090+ try_files {path} {path}/1091+ }1092+ handle @public {10841093 root * $FORGE_WWW10851094 @hashed path_regexp $hashed10861095 header @hashed Cache-Control "public, max-age=31536000, immutable"@@ -1094,6 +1103,9 @@ EOF
10941103 header @download X-Content-Type-Options nosniff10951104 file_server10961105 }1106+ handle {1107+ reverse_proxy 127.0.0.1:${FORGE_PRIVATE_PORT:-8093}1108+ }10971109 }10981110 EOF10991111 [ -n "$pages" ] || return 0@@ -1,13 +1,17 @@
11 // Local preview of the demo forge: static pages from demo/root/.forge/www22 // plus live events, the way Caddy and lib/events-server.ts serve them, with33 // the same cache headers (bin/forge server caddyfile) and, like Caddy, byte4-// ranges, which code search reads code.db with.4+// ranges, which code search reads code.db with (lib/static.ts). A path with5+// no file in www goes to the private server's handler, as Caddy sends it6+// there: private repos' pages for their readers, and the 404. Sign-in uses7+// FORGE_BASE_URL, http://localhost:<port> unless set.58 // node demo/serve.ts [port]69 import http from 'node:http';7-import fs from 'node:fs';810 import path from 'node:path';911 import { fileURLToPath } from 'node:url';1012 import { serveEvents } from '../lib/events-server.ts';13+import { privateHandler } from '../lib/private-server.ts';14+import { HASHED, resolve, sendFile } from '../lib/static.ts';1115 1216 const here = path.dirname(fileURLToPath(import.meta.url));1317 const state = process.env.FORGE_STATE || path.join(here, 'root/.forge');@@ -15,33 +19,24 @@ const www = path.join(state, 'www');
1519 const port = Number(process.argv[2] || 4817);1620 const eventsPort = port + 1;1721 serveEvents(state, eventsPort);18-19-const TYPES: Record<string, string> = {20- '.html': 'text/html; charset=utf-8', '.css': 'text/css', '.js': 'text/javascript', '.svg': 'image/svg+xml',21- '.png': 'image/png', '.gif': 'image/gif', '.txt': 'text/plain; charset=utf-8', '.xml': 'application/xml', '.json': 'application/json',22- '.wasm': 'application/wasm', '.db': 'application/vnd.sqlite3', '.parquet': 'application/vnd.apache.parquet',23-};24-25-// One byte range, as code search reads code.db: [start, end] inclusive, or26-// null for a malformed or unsatisfiable one. Several ranges get the whole file.27-function byteRange(h: string, size: number): [number, number] | null | undefined {28- const m = /^bytes=(\d*)-(\d*)$/.exec(h.trim());29- if (!m || (!m[1] && !m[2])) return h.includes(',') ? undefined : null;30- const [start, end] = m[1] ? [Number(m[1]), m[2] ? Math.min(Number(m[2]), size - 1) : size - 1] : [Math.max(0, size - Number(m[2])), size - 1];31- return start <= end && start < size ? [start, end] : null;32-}22+const misses = privateHandler({23+ root: path.dirname(state), state, priv: process.env.FORGE_PRIVATE_WWW || path.join(state, 'private', 'www'),24+ baseUrl: (process.env.FORGE_BASE_URL || `http://localhost:${port}`).replace(/\/$/, ''),25+});3326 3427 // Hashed assets never change; pages and the indexes they fetch revalidate,3528 // and get a 304 when unchanged. Sites keep their own caching.36-const HASHED = /^\/assets\/([^/]+\.)?[0-9A-Za-z_-]{10}\.[a-z]+$/;29+const cacheControl = (p: string): Record<string, string> =>30+ HASHED.test(p) ? { 'Cache-Control': 'public, max-age=31536000, immutable' } : /^\/(sites|previews)\//.test(p) ? {} : { 'Cache-Control': 'no-cache' };31+// Release downloads are saved, not opened, on the forge's origin.3732 const DOWNLOAD = /^\/[^/]+\/[^/]+\/releases\/download\/./;38-const cacheControl = (p: string): string | null =>39- HASHED.test(p) ? 'public, max-age=31536000, immutable' : /^\/(sites|previews)\//.test(p) ? null : 'no-cache';33+const headersFor = (p: string): Record<string, string> =>34+ DOWNLOAD.test(p) ? { ...cacheControl(p), 'Content-Disposition': 'attachment', 'X-Content-Type-Options': 'nosniff' } : cacheControl(p);4035 4136 http.createServer((req, res) => {4237 const url = new URL(req.url ?? '/', 'http://x');4338 if (url.pathname.startsWith('/events/')) {44- const up = http.request({ host: '127.0.0.1', port: eventsPort, path: url.pathname, method: 'GET' }, (r) => {39+ const up = http.request({ host: '127.0.0.1', port: eventsPort, path: url.pathname, method: 'GET', headers: { cookie: req.headers.cookie ?? '' } }, (r) => {4540 res.writeHead(r.statusCode ?? 502, r.headers);4641 r.pipe(res);4742 });@@ -53,47 +48,8 @@ http.createServer((req, res) => {
5348 up.end();5449 return;5550 }56- let file = path.join(www, decodeURIComponent(url.pathname));57- if (!file.startsWith(www)) {58- res.writeHead(403).end();59- return;60- }61- try {62- let st = fs.statSync(file);63- if (st.isDirectory()) st = fs.statSync((file = path.join(file, 'index.html')));64- const etag = `"${Math.floor(st.mtimeMs).toString(36)}-${st.size.toString(36)}"`;65- const headers: http.OutgoingHttpHeaders = { 'Content-Type': TYPES[path.extname(file)] ?? 'application/octet-stream', ETag: etag, 'Last-Modified': st.mtime.toUTCString(), 'Accept-Ranges': 'bytes' };66- const cc = cacheControl(url.pathname);67- if (cc) headers['Cache-Control'] = cc;68- // Release downloads are saved, not opened, on the forge's origin.69- if (DOWNLOAD.test(url.pathname)) Object.assign(headers, { 'Content-Disposition': 'attachment', 'X-Content-Type-Options': 'nosniff' });70- const since = req.headers['if-modified-since'];71- const match = req.headers['if-none-match'];72- if (match ? match === etag : since && Date.parse(since) >= Math.floor(st.mtimeMs / 1000) * 1000) {73- res.writeHead(304, headers).end();74- return;75- }76- // A range of the file, unless If-Range names another version of it.77- const ifRange = req.headers['if-range'];78- const r = req.headers.range && (!ifRange || ifRange === etag || ifRange === headers['Last-Modified']) ? byteRange(req.headers.range, st.size) : undefined;79- if (r === null) {80- res.writeHead(416, { ...headers, 'Content-Range': `bytes */${st.size}` }).end();81- return;82- }83- if (r) {84- const [start, end] = r;85- const buf = Buffer.alloc(end - start + 1);86- const fd = fs.openSync(file, 'r');87- try {88- fs.readSync(fd, buf, 0, buf.length, start);89- } finally {90- fs.closeSync(fd);91- }92- res.writeHead(206, { ...headers, 'Content-Range': `bytes ${start}-${end}/${st.size}`, 'Content-Length': buf.length }).end(buf);93- return;94- }95- res.writeHead(200, headers).end(fs.readFileSync(file));96- } catch {97- res.writeHead(404).end('not found');98- }99-}).listen(port, '127.0.0.1', () => console.log(`demo forge on http://127.0.0.1:${port}/`));51+ const file = resolve(www, url.pathname);52+ if (file === 'redirect') res.writeHead(308, { Location: url.pathname + '/' + url.search }).end();53+ else if (file && (req.method === 'GET' || req.method === 'HEAD')) sendFile(req, res, file, headersFor(url.pathname));54+ else misses(req, res);55+}).listen(port, '127.0.0.1', () => console.log(`demo forge on http://localhost:${port}/`));@@ -0,0 +1,148 @@
1+// Private repos' pages, for the people who may read them. Caddy serves www2+// as plain files and hands this server every request for a file that isn't3+// there (bin/forge server caddyfile), so it needs no list of private repos.4+// It serves a private repo's pages from the private root (bin/forge5+// build_private), with the same layout and the same headers as Caddy gives6+// pages (ETag, 304s, byte ranges for code.db) except Cache-Control, which7+// is "private, no-cache": the same revalidation, kept out of shared caches.8+//9+// A reader is the repo's owner, a collaborator or a name in its readers10+// (lib/access.ts canRead, the rule bin/forge applies), who is still a user11+// of the forge and shows a token: Authorization: Bearer <token>, or basic12+// auth with the token as the password (or the user name), for curl and git.13+// Everyone else, a page that isn't there, and a repo that isn't, get the14+// same 404, byte for byte; so does any missing page of www, since Caddy15+// sends those here too.16+//17+// node lib/private-server.ts (FORGE_ROOT, FORGE_PRIVATE_PORT, default 8093)18+import http from 'node:http';19+import fs from 'node:fs';20+import path from 'node:path';21+import { fileURLToPath } from 'node:url';22+import { Identity, TOKEN_PREFIX, type Token } from './identity.ts';23+import { canRead, isPublic, repoDir, users } from './access.ts';24+import { resolve, sendFile } from './static.ts';25+26+export interface PrivateConfig {27+ root: string;28+ state: string;29+ // The private root, and www (for nothing but the missing pages it sends).30+ priv: string;31+ baseUrl: string;32+}33+34+export function privateConfig(env: NodeJS.ProcessEnv = process.env): PrivateConfig {35+ const root = path.resolve(env.FORGE_ROOT || 'demo/root');36+ const state = path.join(root, '.forge');37+ return { root, state, priv: env.FORGE_PRIVATE_WWW || path.join(state, 'private', 'www'), baseUrl: (env.FORGE_BASE_URL || '').replace(/\/$/, '') };38+}39+40+// What every missing or unreadable page gets.41+export const NOT_FOUND = Buffer.from(42+ '<!DOCTYPE html>\n<html lang="en">\n<head><meta charset="utf-8"><meta name="viewport" content="width=device-width, initial-scale=1"><title>Not found</title></head>\n' +43+ '<body><h1>Not found</h1><p>There is no page here. Private repos show their pages only to their readers, signed in.</p><p><a href="/">All repositories</a></p></body>\n</html>\n',44+);45+46+export function notFound(res: http.ServerResponse, head = false): void {47+ res.writeHead(404, { 'Content-Type': 'text/html; charset=utf-8', 'Cache-Control': 'no-cache', 'Content-Length': NOT_FOUND.length });48+ res.end(head ? undefined : NOT_FOUND);49+}50+51+// Who is asking.52+export interface Who {53+ user: string;54+ token: Token | null;55+}56+57+// A token from Authorization: Bearer, or Basic with the token as the58+// password or the user name (git sends user:password from the URL).59+export function presented(req: http.IncomingMessage): string | null {60+ const h = req.headers.authorization ?? '';61+ const bearer = /^Bearer +(\S+)$/i.exec(h);62+ if (bearer) return bearer[1];63+ const basic = /^Basic +([A-Za-z0-9+/=]+)$/i.exec(h);64+ if (!basic) return null;65+ const text = Buffer.from(basic[1], 'base64').toString('utf8');66+ const i = text.indexOf(':');67+ const [name, pass] = i < 0 ? [text, ''] : [text.slice(0, i), text.slice(i + 1)];68+ return pass.startsWith(TOKEN_PREFIX) ? pass : name.startsWith(TOKEN_PREFIX) ? name : null;69+}70+71+export class Gate {72+ cfg: PrivateConfig;73+ private db: Identity | null = null;74+ constructor(cfg: PrivateConfig) {75+ this.cfg = cfg;76+ }77+78+ get id(): Identity {79+ return (this.db ??= Identity.open(this.cfg.state)!);80+ }81+82+ isUser(name: string): boolean {83+ return users(this.cfg.state).has(name);84+ }85+86+ // The token's holder, if it is a live token of a user who still has keys.87+ byToken(value: string | null): Who | null {88+ if (!value) return null;89+ const t = this.id.token(value);90+ if (!t || !this.isUser(t.user)) return null;91+ this.id.usedToken(t);92+ return { user: t.user, token: t };93+ }94+95+ // May who read the private repo slug (at dir)? A token for one repo reads that repo only.96+ reads(who: Who | null, slug: string, dir: string): boolean {97+ if (!who || !canRead(dir, who.user)) return false;98+ return !who.token?.repo || who.token.repo === slug;99+ }100+}101+102+// The server's requests, for servePrivate and demo/serve.ts.103+export function privateHandler(cfg: PrivateConfig): http.RequestListener {104+ const gate = new Gate(cfg);105+106+ function page(req: http.IncomingMessage, res: http.ServerResponse, url: URL): void {107+ const head = req.method === 'HEAD';108+ const m = /^\/([^/]+)\/([^/]+)(?:\/|$)/.exec(url.pathname);109+ if (!m || (req.method !== 'GET' && !head)) return notFound(res, head);110+ let slug: string;111+ try {112+ slug = `${decodeURIComponent(m[1])}/${decodeURIComponent(m[2])}`;113+ } catch {114+ return notFound(res, head);115+ }116+ const dir = repoDir(cfg.root, slug);117+ if (!dir || isPublic(dir)) return notFound(res, head);118+ const who = gate.byToken(presented(req));119+ if (!gate.reads(who, slug, dir)) return notFound(res, head);120+ const file = resolve(cfg.priv, url.pathname);121+ if (!file) return notFound(res, head);122+ if (file === 'redirect') {123+ res.writeHead(308, { Location: url.pathname + '/' + url.search }).end();124+ return;125+ }126+ sendFile(req, res, file, { 'Cache-Control': 'private, no-cache' });127+ }128+129+ return (req, res) => {130+ try {131+ const url = new URL(req.url ?? '/', 'http://x');132+ page(req, res, url);133+ } catch (e) {134+ console.error(e);135+ if (!res.headersSent) res.writeHead(500).end();136+ else res.destroy();137+ }138+ };139+}140+141+export function servePrivate(cfg: PrivateConfig, port: number): http.Server {142+ return http.createServer(privateHandler(cfg)).listen(port, '127.0.0.1', () => console.log(`private pages on 127.0.0.1:${port}, from ${cfg.priv}`));143+}144+145+// Run directly: node lib/private-server.ts146+if (process.argv[1] && fs.realpathSync(process.argv[1]) === fileURLToPath(import.meta.url)) {147+ servePrivate(privateConfig(), Number(process.env.FORGE_PRIVATE_PORT || 8093));148+}@@ -0,0 +1,91 @@
1+// Static files the way Caddy's file_server sends them, for the servers that2+// serve built pages themselves (lib/private-server.ts, demo/serve.ts): the3+// content type by extension, ETag and Last-Modified with 304s, and one4+// byte range (206), which code search reads code.db with, unless If-Range5+// names another version of the file. Cache-Control is the caller's.6+import fs from 'node:fs';7+import path from 'node:path';8+import type http from 'node:http';9+10+export const TYPES: Record<string, string> = {11+ '.html': 'text/html; charset=utf-8', '.css': 'text/css; charset=utf-8', '.js': 'text/javascript; charset=utf-8', '.svg': 'image/svg+xml',12+ '.png': 'image/png', '.jpg': 'image/jpeg', '.jpeg': 'image/jpeg', '.gif': 'image/gif', '.webp': 'image/webp', '.avif': 'image/avif', '.ico': 'image/x-icon',13+ '.txt': 'text/plain; charset=utf-8', '.xml': 'application/xml', '.json': 'application/json', '.wasm': 'application/wasm', '.db': 'application/vnd.sqlite3', '.parquet': 'application/vnd.apache.parquet',14+};15+16+// Hashed assets never change; pages and the indexes they fetch revalidate.17+export const HASHED = /^\/assets\/([^/]+\.)?[0-9A-Za-z_-]{10}\.[a-z]+$/;18+19+// One byte range, as code search reads code.db: [start, end] inclusive, or20+// null for a malformed or unsatisfiable one. Several ranges get the whole file.21+export function byteRange(h: string, size: number): [number, number] | null | undefined {22+ const m = /^bytes=(\d*)-(\d*)$/.exec(h.trim());23+ if (!m || (!m[1] && !m[2])) return h.includes(',') ? undefined : null;24+ const [start, end] = m[1] ? [Number(m[1]), m[2] ? Math.min(Number(m[2]), size - 1) : size - 1] : [Math.max(0, size - Number(m[2])), size - 1];25+ return start <= end && start < size ? [start, end] : null;26+}27+28+// The file a URL path names under root: itself, or a directory's29+// index.html. "redirect" for a directory asked for without its slash, null30+// for nothing (or anything that leaves root, through .. or a symlink).31+export function resolve(root: string, pathname: string): string | 'redirect' | null {32+ let rel: string;33+ try {34+ rel = decodeURIComponent(pathname);35+ } catch {36+ return null;37+ }38+ if (rel.includes('\0') || rel.split('/').includes('..')) return null;39+ let real: string;40+ let top: string;41+ try {42+ top = fs.realpathSync(root);43+ real = fs.realpathSync(path.join(root, rel));44+ } catch {45+ return null;46+ }47+ if (real !== top && !real.startsWith(top + path.sep)) return null;48+ const st = fs.statSync(real);49+ if (st.isFile()) return rel.endsWith('/') ? null : real;50+ if (!st.isDirectory()) return null;51+ const index = path.join(real, 'index.html');52+ if (!fs.existsSync(index)) return null;53+ return rel.endsWith('/') ? index : 'redirect';54+}55+56+export function sendFile(req: http.IncomingMessage, res: http.ServerResponse, file: string, extra: http.OutgoingHttpHeaders = {}): void {57+ const st = fs.statSync(file);58+ const etag = `"${Math.floor(st.mtimeMs).toString(36)}-${st.size.toString(36)}"`;59+ const headers: http.OutgoingHttpHeaders = {60+ 'Content-Type': TYPES[path.extname(file)] ?? 'application/octet-stream', ETag: etag, 'Last-Modified': st.mtime.toUTCString(), 'Accept-Ranges': 'bytes', ...extra,61+ };62+ const since = req.headers['if-modified-since'];63+ const match = req.headers['if-none-match'];64+ if (match ? match === etag : since && Date.parse(since) >= Math.floor(st.mtimeMs / 1000) * 1000) {65+ res.writeHead(304, headers).end();66+ return;67+ }68+ // A range of the file, unless If-Range names another version of it.69+ const ifRange = req.headers['if-range'];70+ const r = req.headers.range && (!ifRange || ifRange === etag || ifRange === headers['Last-Modified']) ? byteRange(req.headers.range, st.size) : undefined;71+ if (r === null) {72+ res.writeHead(416, { ...headers, 'Content-Range': `bytes */${st.size}` }).end();73+ return;74+ }75+ if (r) {76+ const [start, end] = r;77+ const buf = Buffer.alloc(end - start + 1);78+ const fd = fs.openSync(file, 'r');79+ try {80+ fs.readSync(fd, buf, 0, buf.length, start);81+ } finally {82+ fs.closeSync(fd);83+ }84+ res.writeHead(206, { ...headers, 'Content-Range': `bytes ${start}-${end}/${st.size}`, 'Content-Length': buf.length });85+ res.end(req.method === 'HEAD' ? undefined : buf);86+ return;87+ }88+ res.writeHead(200, { ...headers, 'Content-Length': st.size });89+ if (req.method === 'HEAD') res.end();90+ else fs.createReadStream(file).pipe(res);91+}@@ -122,6 +122,11 @@ touch $root/.forge/events.jsonl
122122 chown git:git $root/.forge/events.jsonl123123 runuser -u git -- env FORGE_STATE=$root/.forge node /opt/forge/lib/events-server.ts >>$root/.forge/events.log 2>&1 &124124 125+# Private repos' pages for their readers, and the 404 any other missing page126+# gets: Caddy hands it every path www has no file for. Runs as git, which127+# owns the private pages and identity.db.128+runuser -u git -- env FORGE_ROOT=$root sh -c "set -a; . $root/.forge/config; exec node /opt/forge/lib/private-server.ts" >>$root/.forge/private.log 2>&1 &129+125130 # Repo sites, each on its own hostname under the pages domain, behind Caddy.126131 # Runs as caddy: it only reads deploys, which are untrusted content.127132 if [ -n "$pages" ]; then@@ -91,7 +91,10 @@ EOF
9191 if command -v systemctl >/dev/null 2>&1 && [ -d /run/systemd/system ]; then9292 # Live page updates, proxied at /events/.9393 unit forge-events git lib/events-server.ts "forge live page updates"94- units=forge-events94+ # Private repos' pages, and the 404 of every missing page: Caddy sends it95+ # whatever www has no file for.96+ unit forge-private git lib/private-server.ts "forge private pages"97+ units="forge-events forge-private"9598 # Sites on the pages domain. Runs as caddy: deploys are untrusted content.9699 if [ -n "$pages" ]; then97100 unit forge-pages caddy lib/pages-server.ts "forge repo sites"@@ -27,12 +27,12 @@ const [mode, ...args] = process.argv.slice(2);
2727 const MAX = 4 << 20;2828 const FETCHER = `2929 const { parentPort } = require('node:worker_threads');30-parentPort.on('message', async ({ url, from, to, sab }) => {30+parentPort.on('message', async ({ url, from, to, sab, auth }) => {3131 const ctl = new Int32Array(sab, 0, 4);3232 const body = new Uint8Array(sab, 1024);3333 const meta = new Uint8Array(sab, 16, 1008);3434 try {35- const r = await fetch(url, { headers: { Range: from < 0 ? 'bytes=' + from : 'bytes=' + from + '-' + (to - 1) } });35+ const r = await fetch(url, { headers: { Range: from < 0 ? 'bytes=' + from : 'bytes=' + from + '-' + (to - 1), ...auth } });3636 const data = new Uint8Array(await r.arrayBuffer());3737 if (data.length > body.length) throw new Error('response too large: ' + data.length);3838 body.set(data);@@ -57,7 +57,7 @@ function syncRange(): { get: GetRange; stop: () => void } {
5757 const ctl = new Int32Array(sab, 0, 4);5858 const get: GetRange = (url, from, to) => {5959 Atomics.store(ctl, 0, 0);60- w.postMessage({ url, from, to, sab });60+ w.postMessage({ url, from, to, sab, auth });6161 Atomics.wait(ctl, 0, 0);6262 const info = JSON.parse(new TextDecoder().decode(new Uint8Array(sab, 16, ctl[2]).slice()));6363 if (ctl[1] < 0) throw new Error(info.error);@@ -71,6 +71,13 @@ function syncRange(): { get: GetRange; stop: () => void } {
7171 return { get, stop: () => void w.terminate() };7272 }7373 74+// FORGE_BENCH_TOKEN: a token for every request, to read a private repo's75+// code.db through the private server (which demo/serve.ts hands misses to).76+const auth: Record<string, string> = process.env.FORGE_BENCH_TOKEN ? { Authorization: `Bearer ${process.env.FORGE_BENCH_TOKEN}` } : {};77+// The worker's prefetches fetch, as a browser's would with its cookie.78+const plainFetch = globalThis.fetch;79+globalThis.fetch = (url, init) => plainFetch(url, { ...init, headers: { ...(init?.headers as Record<string, string>), ...auth } });80+7481 // demo/serve.ts on a free port (it also takes the next one, for events).7582 async function serve(state: string): Promise<{ base: string; stop: () => void }> {7683 const free = (): Promise<number> => new Promise((ok, no) => {@@ -127,8 +134,9 @@ if (mode === 'sql') {
127134 const r = await ix.search(q, Number(limit || MORE));128135 for (const h of r.hits) console.log(`${h.path}:${h.line}: ${h.text}`);129136 console.log(`requests ${r.requests} rounds ${r.rounds} bytes ${r.bytes}`);130- const head = await fetch(`${srv.base}/${slug}/code.db`, { headers: { Range: 'bytes=0-99' } });137+ const head = await fetch(`${srv.base}/${slug}/code.db`, { headers: { Range: 'bytes=0-99', ...auth } });131138 console.log(`range ${head.status} ${head.headers.get('content-range')} type ${head.headers.get('content-type')} encoding ${head.headers.get('content-encoding') ?? 'none'} length ${(await head.arrayBuffer()).byteLength}`);139+ console.log(`cache ${head.headers.get('cache-control')}`);132140 // What the browser instantiates sqlite-wasm from.133141 const wasmFile = fs.readdirSync(path.join(state, 'www', 'assets')).find((a) => a.endsWith('.wasm'));134142 const w = await fetch(`${srv.base}/assets/${wasmFile}`, { method: 'HEAD' });@@ -19,6 +19,8 @@
1919 # aside dami/secret ... back dami/secret move a repo out of the forge and back2020 # sync_builds hooks and ssh commands build pages before returning2121 # lq "<sql>" forge lake query: the rows as CSV, no header22+# private_server lib/private-server.ts for this forge, at $PRIV23+# response url [curl args] a whole HTTP response less its Date, to compare2224 #2325 # Assertions print what failed and stop the scenario:2426 # ok cmd... cmd succeeds fails cmd... cmd fails@@ -224,3 +226,36 @@ lq() {
224226 "$FORGE" lake query "$1" --csv >"$T/lq" 2>"$T_LOG" || fail "forge lake query failed: $1"225227 sed 1d "$T/lq"226228 }229+230+# free_port: a TCP port nothing listens on just now.231+free_port() { node -e 'const s = require("net").createServer().listen(0, "127.0.0.1", () => { console.log(s.address().port); s.close(); })'; }232+233+# wait_http url: until something answers there (5s at most).234+wait_http() {235+ n=0236+ until curl -s -o /dev/null "$1" || [ "$n" -ge 50 ]; do sleep 0.1; n=$((n + 1)); done237+}238+239+# private_server: lib/private-server.ts on a free port, in the background,240+# for this forge (with FORGE_BASE_URL as it is now). Sets PRIV, its URL.241+private_server() {242+ PRIV_PORT=$(free_port)243+ FORGE_PRIVATE_PORT=$PRIV_PORT node "$T_HOME/lib/private-server.ts" >>"$T/private.log" 2>&1 &244+ T_PIDS="$T_PIDS $!"245+ PRIV=http://127.0.0.1:$PRIV_PORT246+ wait_http "$PRIV/"247+}248+249+# stop_servers: stop what the scenario started in the background so far.250+stop_servers() {251+ [ -z "$T_PIDS" ] || { kill $T_PIDS 2>/dev/null || true; wait $T_PIDS 2>/dev/null || true; }252+ T_PIDS=253+}254+255+# response url [curl args...]: the whole HTTP response (status line,256+# headers, body) less its Date, to compare byte for byte.257+response() {258+ u=$1259+ shift260+ curl -s -i "$@" "$u" | grep -av '^Date: '261+}@@ -0,0 +1,68 @@
1+# Caddy serves a file of www when there is one and hands every other path2+# to the private server, so it needs no list of private repos. With caddy3+# installed, the real thing: public pages come from www with Caddy's cache4+# headers, a private page goes to its reader with a token, and anyone else5+# gets the same response as for a page that isn't there.6+. "$(dirname "$0")/../lib.sh"7+user dami8+user cat9+repo --private dami/secret "hidden thing"10+repo dami/open11+for r in secret open; do12+ clone dami "dami/$r" "$T/w/$r"13+ at "$T/w/$r" commit "first"14+ ok push_as dami "$T/w/$r" origin main15+done16+build17+18+ok forge server caddyfile git.test19+cp "$T_LOG" "$T/Caddyfile"20+has "$T/Caddyfile" '^ @public file \{$'21+has "$T/Caddyfile" '^ try_files \{path\} \{path\}/$'22+has "$T/Caddyfile" '^ reverse_proxy 127\.0\.0\.1:8093$'23+lacks "$T/Caddyfile" 'private/www'24+# The private server is the last handler: only what nothing else took.25+eq "$(grep -n 'reverse_proxy 127.0.0.1:8093' "$T/Caddyfile" | cut -d: -f1)" "$(($(grep -n '^}$' "$T/Caddyfile" | head -n 1 | cut -d: -f1) - 2))"26+command -v caddy >/dev/null 2>&1 || { note "caddy is not installed: skipped running it"; exit 0; }27+ok caddy adapt --config "$T/Caddyfile" --adapter caddyfile28+29+cport=$(free_port)30+private_server31+{32+ printf '{\n\tadmin off\n\tauto_https off\n\tpersist_config off\n}\n\n'33+ FORGE_PRIVATE_PORT=$PRIV_PORT "$FORGE" server caddyfile "http://127.0.0.1:$cport"34+} >"$T/Caddyfile.run"35+XDG_DATA_HOME=$T/xdg XDG_CONFIG_HOME=$T/xdg caddy run --config "$T/Caddyfile.run" --adapter caddyfile >"$T/caddy.log" 2>&1 &36+T_PIDS="$T_PIDS $!"37+c=http://127.0.0.1:$cport38+wait_http "$c/"39+dami=$(ssh_as dami token 2>/dev/null)40+cat=$(ssh_as cat token 2>/dev/null)41+42+response "$c/dami/open/" >"$T/open"43+has "$T/open" '^HTTP/1.1 200 OK'44+has "$T/open" '^Cache-Control: no-cache'45+style=$(grep -o '/assets/style\.[^"]*\.css' "$T/open" | head -n 1)46+response "$c$style" -I >"$T/style"47+has "$T/style" '^Cache-Control: public, max-age=31536000, immutable'48+49+response "$c/dami/secret/" -H "Authorization: Bearer $dami" >"$T/secret"50+has "$T/secret" '^HTTP/1.1 200 OK'51+has "$T/secret" 'hidden thing'52+eq "$(grep -c '^Cache-Control:' "$T/secret")" 153+has "$T/secret" '^Cache-Control: private, no-cache'54+response "$c/dami/secret/code.db" -H "Authorization: Bearer $dami" -H 'Range: bytes=0-99' -H 'Accept-Encoding: gzip' -o /dev/null -D - >"$T/db"55+has "$T/db" '^HTTP/1.1 206 Partial Content'56+has "$T/db" '^Content-Range: bytes 0-99/'57+lacks "$T/db" '^Content-Encoding'58+59+response "$c/dami/nope/" >"$T/missing"60+has "$T/missing" '^HTTP/1.1 404 Not Found'61+same() {62+ response "$@" >"$T/got"63+ cmp -s "$T/missing" "$T/got" || { diff "$T/missing" "$T/got" >&2 || true; fail "through Caddy, $* is not the missing page's 404"; }64+}65+same "$c/dami/secret/"66+same "$c/dami/secret/" -H "Authorization: Bearer $cat"67+same "$c/dami/open/nope/"68+same "$c/nobody/x/y/"@@ -0,0 +1,24 @@
1+# Code search on a private repo: its code.db is read in byte ranges through2+# the private server, which demo/serve.ts hands every path www doesn't have3+# (as Caddy does), with the reader's token: 206s of the file as stored,4+# uncompressed, "private, no-cache". Without a token there is nothing to read.5+. "$(dirname "$0")/../lib.sh"6+user dami7+repo --private dami/secret8+clone dami dami/secret "$T/w/dami"9+i=110+while [ $i -le 30 ]; do printf 'filler line %s\n' "$i"; i=$((i + 1)); done >"$T/w/dami/greet.sh"11+printf 'echo "Hello, $1"\n' >>"$T/w/dami/greet.sh"12+at "$T/w/dami" commit "first"13+push_as dami "$T/w/dami" origin main14+build15+missing "$FORGE_WWW/dami/secret/code.db"16+exists "$FORGE_ROOT/.forge/private/www/dami/secret/code.db"17+tok=$(ssh_as dami token --repo dami/secret 2>/dev/null)18+ok env FORGE_BENCH_TOKEN="$tok" node "$T_HOME/test/code-search-bench.ts" http "$FORGE_ROOT/.forge" dami/secret 'hello, $1'19+has "$T_LOG" '^greet\.sh:31: echo "Hello, \$1"$'20+has "$T_LOG" '^requests [1-9][0-9]* rounds [1-9][0-9]* bytes [1-9][0-9]*$'21+has "$T_LOG" '^range 206 bytes 0-99/[0-9]+ type application/vnd\.sqlite3 encoding none length 100$'22+has "$T_LOG" '^cache private, no-cache$'23+has "$T_LOG" '^wasm 200 type application/wasm cache public, max-age=31536000, immutable$'24+fails_with 'HTTP 404' node "$T_HOME/test/code-search-bench.ts" http "$FORGE_ROOT/.forge" dami/secret 'hello, $1'@@ -0,0 +1,90 @@
1+# The private server serves a private repo's pages to its readers holding a2+# token (Bearer, or basic auth with the token as password or user name),3+# with Caddy's headers for pages but "private, no-cache". Anyone else gets4+# the 404 a missing page gets, byte for byte: no token, a bad, expired or5+# revoked one, a reader's token for another repo, a user whose keys are6+# gone, a page or repo that isn't there.7+. "$(dirname "$0")/../lib.sh"8+for u in dami ana bob cat; do user "$u"; done9+repo --private dami/secret "hidden thing"10+repo --private dami/other11+repo dami/open12+for r in secret open; do13+ clone dami "dami/$r" "$T/w/$r"14+ at "$T/w/$r" commit "first"15+ ok push_as dami "$T/w/$r" origin main16+done17+ok ssh_as dami readers dami/secret add ana18+ok ssh_as dami readers dami/secret add bob19+build20+tok() { ssh_as "$@" 2>/dev/null; }21+dami=$(tok dami token)22+ana=$(tok ana token --repo dami/secret)23+ana_other=$(tok ana token --repo dami/open)24+bob=$(tok bob token)25+cat=$(tok cat token)26+short=$(tok dami token --ttl 1h)27+gone=$(ssh_as dami token 2>"$T/err")28+ok ssh_as dami tokens rm "$(sed -n 's/^token \([0-9a-f]*\) .*/\1/p' "$T/err")"29+ok forge user rm bob30+private_server31+u=$PRIV/dami/secret/32+33+# Readers.34+for t in "$dami" "$ana"; do35+ eq "$(curl -s -o /dev/null -w '%{http_code}' -H "Authorization: Bearer $t" "$u")" 20036+done37+eq "$(curl -s -o /dev/null -w '%{http_code}' -u "ana:$ana" "$u")" 20038+eq "$(curl -s -o /dev/null -w '%{http_code}' -u "$ana:" "$u")" 20039+response "$u" -H "Authorization: Bearer $dami" >"$T/page"40+has "$T/page" '^HTTP/1.1 200 OK'41+has "$T/page" '^Cache-Control: private, no-cache'42+has "$T/page" '^Content-Type: text/html; charset=utf-8'43+has "$T/page" '^ETag: "'44+has "$T/page" '^Last-Modified: '45+has "$T/page" 'hidden thing'46+etag=$(sed -n 's/^ETag: //p' "$T/page" | tr -d '\r')47+eq "$(curl -s -o /dev/null -w '%{http_code}' -H "Authorization: Bearer $dami" -H "If-None-Match: $etag" "$u")" 30448+eq "$(curl -s -o /dev/null -w '%{http_code} %{redirect_url}' -H "Authorization: Bearer $dami" "$PRIV/dami/secret")" "308 $PRIV/dami/secret/"49+eq "$(curl -s -o /dev/null -w '%{http_code}' -H "Authorization: Bearer $dami" "$PRIV/dami/secret/search.json")" 20050+51+# Everyone and everything else: the same 404.52+response "$PRIV/dami/nope/" >"$T/missing"53+has "$T/missing" '^HTTP/1.1 404 Not Found'54+has "$T/missing" '^Cache-Control: no-cache'55+same() {56+ response "$@" >"$T/got"57+ cmp -s "$T/missing" "$T/got" || { diff "$T/missing" "$T/got" >&2 || true; fail "not the missing page's 404: $*"; }58+}59+same "$u"60+same "$PRIV/dami/secret"61+same "$u" -H "Authorization: Bearer $cat"62+same "$u" -u "cat:$cat"63+same "$u" -H "Authorization: Bearer $ana_other"64+same "$u" -H "Authorization: Bearer $gone"65+same "$u" -H "Authorization: Bearer $bob"66+same "$u" -H "Authorization: Bearer forge_nope"67+same "$u" -H "Authorization: Basic !!!"68+same "$PRIV/dami/other/" -H "Authorization: Bearer $ana"69+same "$PRIV/dami/secret/nope/" -H "Authorization: Bearer $dami"70+same "$PRIV/dami/secret/..%2F..%2Fdami/open/" -H "Authorization: Bearer $dami"71+same "$PRIV/dami/open/nope/"72+same "$PRIV/dami/open/" -H "Authorization: Bearer $dami"73+same "$PRIV/login.php"74+same "$PRIV/dami/secret/" -X POST -H "Authorization: Bearer $dami"75+# HEAD: the same headers, no body.76+response "$PRIV/dami/nope/" -I >"$T/missing"77+same "$u" -I78+79+# An hour and a bit later, the 1h token has expired; the 30-day ones80+# haven't, and say when they were last used.81+ok ssh_as dami tokens82+eq "$(grep -c ' token$' "$T_LOG")" 283+stop_servers84+export FORGE_NOW=$((T_NOW + 3700))85+private_server86+same "$PRIV/dami/secret/" -H "Authorization: Bearer $short" -I87+eq "$(curl -s -o /dev/null -w '%{http_code}' -H "Authorization: Bearer $dami" "$PRIV/dami/secret/")" 20088+ok ssh_as dami tokens89+eq "$(grep -c ' token$' "$T_LOG")" 190+has "$T_LOG" ' 2026-09-01 01:01 +token$'