12// Benchmark committed repository copies. Source checkouts are never changed.3import { execFile } from "node:child_process";4import { appendFile, mkdtemp, mkdir, readdir, rm, writeFile, rename } from "node:fs/promises";5import { tmpdir } from "node:os";6import { resolve, join, basename } from "node:path";7import { promisify } from "node:util";8import { randomUUID } from "node:crypto";9import WebSocket from "ws";10import { finalizeEvent, generateSecretKey, getPublicKey } from "nostr-tools/pure";11import { npubEncode } from "nostr-tools/nip19";12import { startGitSigningProxy } from "./git-signing-proxy.mjs";1314const exec = promisify(execFile);15const args = process.argv.slice(2);16const option = (key, fallback) => args.includes(key) ? args[args.indexOf(key) + 1] : fallback;17const relay = option("--relay", "ws://127.0.0.1:17447");18const reposDir = option("--repos-dir");19const outDir = resolve(option("--out", "artifacts/git-performance/run"));20const baseline = option("--baseline", "").replace(/\/$/, "");21const only = option("--only", "").split(",").filter(Boolean);22const privateRepos = args.includes("--private");23const privateRunID = randomUUID().slice(0, 8);24const privateOwnerSecretHex = option("--private-owner-secret", process.env.TINY_BENCHMARK_PRIVATE_OWNER_SECRET || "fc1d06a0fd5e622dcf448d0b3c2fccc891a5ecf74546a7675460d92441fecfdd");25if (privateRepos && !/^[0-9a-fA-F]{64}$/.test(privateOwnerSecretHex)) throw Error("--private-owner-secret must be 64 hex characters");26const privateOwnerSecret = privateRepos ? Uint8Array.from(Buffer.from(privateOwnerSecretHex, "hex")) : null;27const repeats = Math.max(1, Number(option("--repeats", "3")));28if (!reposDir) throw Error("--repos-dir is required");29const httpURL = relay.replace(/^ws/, "http").replace(/\/$/, "");30const sleep = ms => new Promise(resolve => setTimeout(resolve, ms));31const report = {schema: 3, started: new Date().toISOString(), relay, baseline, repeats, privateRepos, results: [], failures: []};32await mkdir(outDir, {recursive: true});33const progressPath = join(outDir, "progress.jsonl");34await writeFile(progressPath, "");35const progress = async row => {36 const value = {at: new Date().toISOString(), ...row};37 console.log(JSON.stringify(value));38 await appendFile(progressPath, JSON.stringify(value) + "\n");39};40const save = async () => {41 const path = join(outDir, "git-performance.json");42 await writeFile(path + ".tmp", JSON.stringify(report, null, 2) + "\n");43 await rename(path + ".tmp", path);44};45const git = async (cwd, ...args) => (await exec("git", ["-C", cwd, ...args], {46 encoding: "utf8", maxBuffer: 16 * 1024 * 1024,47 env: {...process.env, GIT_TERMINAL_PROMPT: "0", GIT_AUTHOR_NAME: "tiny benchmark", GIT_AUTHOR_EMAIL: "benchmark@example.invalid", GIT_COMMITTER_NAME: "tiny benchmark", GIT_COMMITTER_EMAIL: "benchmark@example.invalid"},48})).stdout.trim();4950class Client {51 constructor(url) {52 this.url = url;53 this.challenge = new Promise(resolve => { this.setChallenge = resolve; });54 this.ws = new WebSocket(url);55 this.waiters = new Map();56 this.connected = new Promise((resolve, reject) => {57 this.ws.once("open", resolve);58 this.ws.once("error", reject);59 });60 this.ws.on("message", raw => {61 let message;62 try { message = JSON.parse(raw); } catch { return; }63 if (message[0] === "AUTH") { this.setChallenge(message[1]); return; }64 const waiter = this.waiters.get(message[1]);65 if (!waiter) return;66 if (message[0] === "EVENT") waiter.events.push(message[2]);67 else if (message[0] === "EOSE") waiter.finish(null, waiter.events);68 else if (message[0] === "OK") waiter.finish(message[2] ? null : Error(message[3]), message);69 else if (message[0] === "CLOSED") waiter.finish(Error(message[2]));70 });71 this.ws.on("error", error => { for (const waiter of [...this.waiters.values()]) waiter.finish(error); });72 this.ws.on("close", () => { for (const waiter of [...this.waiters.values()]) waiter.finish(Error("websocket closed")); });73 }74 async request(message, key) {75 await this.connected;76 if (this.ws.readyState !== WebSocket.OPEN) throw Error("websocket is not open");77 return new Promise((resolve, reject) => {78 const timer = setTimeout(() => finish(Error("Nostr request timed out after 30 seconds")), 30000);79 const finish = (error, value) => {80 clearTimeout(timer);81 this.waiters.delete(key);82 if (message[0] === "REQ" && this.ws.readyState === WebSocket.OPEN) this.ws.send(JSON.stringify(["CLOSE", key]));83 if (error) reject(error); else resolve(value);84 };85 this.waiters.set(key, {events: [], finish});86 this.ws.send(JSON.stringify(message));87 });88 }89 publish(event) { return this.request(["EVENT", event], event.id); }90 async authenticate(secret) {91 const challenge = await this.challenge;92 const proof = finalizeEvent({kind: 22242, created_at: Math.floor(Date.now()/1000), content: "",93 tags: [["relay", this.url], ["challenge", challenge]]}, secret);94 return this.request(["AUTH", proof], proof.id);95 }96 query(filter) {97 const id = Math.random().toString(36).slice(2);98 return this.request(["REQ", id, filter], id);99 }100 close() { this.ws.close(); }101}102103const control = new Client(relay);104const probes = new Client(relay);105await Promise.all([control.connected, probes.connected]);106const probeSecret = generateSecretKey();107let sequence = 0;108async function probeSample(label) {109 const row = {at: new Date().toISOString(), phase: label};110 const event = finalizeEvent({kind: 1, created_at: Math.floor(Date.now()/1000), tags: [], content: `tiny benchmark ${sequence++}`}, probeSecret);111 for (const [operation, run] of [["publish", () => probes.publish(event)], ["query", async () => {112 const found = await probes.query({ids: [event.id], limit: 1});113 if (found.length !== 1 || found[0].id !== event.id) throw Error("probe event was not returned by query");114 }]]) {115 const start = performance.now();116 try { await run(); row[operation + "Ms"] = performance.now() - start; }117 catch (error) { row[operation + "Error"] = error.message; row[operation + "Ms"] = performance.now() - start; }118 }119 return row;120}121122const refsFor = async directory => (await git(directory, "for-each-ref", "--format=%(refname) %(objectname)", "refs/heads", "refs/tags"))123 .split("\n").filter(Boolean).map(line => line.split(" "));124async function verifyRemote(url, refs, head) {125 const output = (await exec("git", ["ls-remote", url], {encoding: "utf8", maxBuffer: 16*1024*1024})).stdout;126 const actual = new Map(output.trim().split("\n").filter(Boolean).map(line => { const [oid, ref] = line.split(/\s+/); return [ref, oid]; }));127 for (const [ref, oid] of refs) if (actual.get(ref) !== oid) throw Error(`remote ref mismatch: ${ref}`);128 const expectedHead = refs.find(([ref]) => ref === head)?.[1];129 if (!expectedHead || actual.get("HEAD") !== expectedHead) throw Error(`remote HEAD does not resolve to ${head}`);130}131132async function benchmarkFixture(fixture, index) {133 const name = basename(fixture);134 const result = {fixture: name, phases: [], probes: [], status: "running"};135 report.results.push(result);136 const temporaries = [];137 const temp = async () => { const path = await mkdtemp(join(tmpdir(), "tiny-git-benchmark-")); temporaries.push(path); return path; };138 let active = "idle", stopProbes = false, loop, signingProxy;139 const phase = async (name, fn) => {140 active = name;141 const started = new Date().toISOString(), start = performance.now();142 await progress({fixture: result.fixture, phase: name, event: "start"});143 let error;144 try { return await fn(); } catch (failure) { error = failure; throw failure; }145 finally {146 const row = {name, started, finished: new Date().toISOString(), ms: performance.now() - start, ok: !error};147 if (error) row.error = error.message.slice(0, 2000);148 result.phases.push(row);149 await progress({fixture: result.fixture, phase: name, event: "end", ms: row.ms, ok: row.ok});150 await save();151 active = "between_phases";152 }153 };154 try {155 await progress({fixture: name, event: "fixture_start"});156 for (let i=0; i<50; i++) { result.probes.push(await probeSample("idle")); await sleep(20); }157 loop = (async () => { while (!stopProbes) { result.probes.push(await probeSample(active)); await sleep(50); } })();158 const head = await git(fixture, "symbolic-ref", "HEAD");159 const refs = await refsFor(fixture);160 result.head = head;161 result.refs = refs.length;162 const sk = privateRepos ? privateOwnerSecret : generateSecretKey(), pubkey = getPublicKey(sk), id = `${name.replace(/\.git$/, "")}-${index}${privateRepos ? "-" + privateRunID : ""}`;163 const repositoryPath = `/${npubEncode(pubkey)}/${encodeURIComponent(id)}.git`;164 const canonicalURL = httpURL + repositoryPath;165 let url = canonicalURL;166 result.url = canonicalURL;167 if (privateRepos) {168 await control.authenticate(sk);169 signingProxy = await startGitSigningProxy(httpURL, sk, {grasp08: true});170 url = signingProxy.url + repositoryPath;171 result.transport = "local GRASP-08 signing proxy; request body spooled on client";172 }173 const announceAt = Math.floor(Date.now()/1000);174 const announcement = finalizeEvent({kind: 30617, created_at: announceAt, content: "", tags: [["d", id], ["clone", canonicalURL], ["relays", relay], ["maintainers", pubkey], ...(privateRepos ? [["private", "true"]] : [])]}, sk);175 const makeState = (refs, created_at) => finalizeEvent({kind: 30618, created_at, content: "", tags: [["d", id], ["HEAD", `ref: ${head}`], ...refs]}, sk);176 await phase("announcement_ack", () => control.publish(announcement));177 const state = makeState(refs, announceAt+1);178 await phase("state_ack", () => control.publish(state));179 // --shared only uses immutable fixture objects; updates live in this copy.180 const source = await temp();181 await git(source, "clone", "--bare", "--shared", fixture, ".");182 const refspecs = ["refs/heads/*:refs/heads/*", "refs/tags/*:refs/tags/*"];183 const waitVisible = async event => {184 const until = Date.now()+30000;185 while (Date.now()<until) {186 if ((await control.query({ids:[event.id],limit:1})).length) return;187 await sleep(25);188 }189 throw Error("signed Git state did not become visible after transfer");190 };191 await phase("initial_push", () => git(source, "push", url, ...refspecs));192 await phase("promotion_wait", () => waitVisible(state));193 const baseURL = baseline ? `${baseline}/${encodeURIComponent(name.endsWith(".git") ? name : name+".git")}` : "";194 if (baseURL) await phase("baseline_initial_push", () => git(source, "push", baseURL, ...refspecs));195 const clones = new Map();196 for (const [prefix, target] of [["", url], ["baseline_", baseURL]]) {197 if (!target) continue;198 for (let i=0; i<repeats; i++) {199 const destination = await temp();200 await phase(`${prefix}fresh_clone_${i+1}`, () => git(destination, "clone", target, "."));201 const actualHead = await git(destination, "rev-parse", "HEAD");202 if (actualHead !== refs.find(([ref])=>ref===head)[1]) throw Error(`${prefix}clone checked out the wrong HEAD`);203 if (i===0) clones.set(prefix, destination);204 else await rm(destination, {recursive: true, force: true});205 }206 await phase(prefix+"verify", async () => { await verifyRemote(target, refs, head); await git(clones.get(prefix), "fsck", "--full"); });207 for (let i=0; i<repeats; i++) await phase(`${prefix}noop_fetch_${i+1}`, () => git(clones.get(prefix), "fetch", "origin"));208 }209 const parent = await git(source, "rev-parse", head);210 const tree = await git(source, "rev-parse", `${head}^{tree}`);211 const next = await git(source, "commit-tree", tree, "-p", parent, "-m", "tiny performance incremental commit");212 await git(source, "update-ref", head, next);213 const nextRefs = refs.map(([ref, oid]) => [ref, ref===head ? next : oid]);214 const nextState = makeState(nextRefs, Math.max(announceAt+2, Math.floor(Date.now()/1000)));215 await phase("incremental_state_ack", () => control.publish(nextState));216 await phase("incremental_push", () => git(source, "push", url, `${head}:${head}`));217 await phase("incremental_promotion_wait", () => waitVisible(nextState));218 if (baseURL) await phase("baseline_incremental_push", () => git(source, "push", baseURL, `${head}:${head}`));219 for (const [prefix, target] of [["",url], ["baseline_",baseURL]]) {220 if (!target) continue;221 await phase(prefix+"incremental_fetch", () => git(clones.get(prefix), "fetch", "origin", head));222 if (await git(clones.get(prefix), "rev-parse", "FETCH_HEAD") !== next) throw Error("incremental fetch returned wrong commit");223 for (const count of [2,4]) {224 const destinations = await Promise.all(Array.from({length:count},()=>temp()));225 await phase(`${prefix}concurrent_clone_${count}`, async () => {226 const started = performance.now();227 const runs = await Promise.allSettled(destinations.map(async destination => {228 await git(destination,"clone",target,".");229 return performance.now()-started;230 }));231 const failure = runs.find(run=>run.status==="rejected");232 if (failure) throw failure.reason;233 result.phases.push({name:`${prefix}concurrent_clone_${count}_individual`, durationsMs:runs.map(run=>run.value), ok:true});234 });235 await Promise.all(destinations.map(path=>rm(path,{recursive:true,force:true})));236 }237 }238 result.status="passed";239 } catch (error) {240 result.status="failed";241 result.error=error.message.slice(0,4000);242 report.failures.push({fixture:name,error:result.error});243 } finally {244 stopProbes=true;245 if (loop) await loop;246 if (signingProxy) await signingProxy.close();247 await Promise.all(temporaries.map(path=>rm(path,{recursive:true,force:true})));248 await progress({fixture:name,event:"fixture_end",status:result.status});249 await save();250 }251}252253try {254 const fixtures = (await readdir(reposDir,{withFileTypes:true})).filter(entry=>entry.isDirectory() && (!only.length || only.includes(entry.name.replace(/\.git$/,""))))255 .map(entry=>join(resolve(reposDir),entry.name));256 fixtures.sort((a,b)=>basename(a).localeCompare(basename(b)));257 for (let i=0;i<fixtures.length;i++) await benchmarkFixture(fixtures[i],i);258} finally {259 control.close(); probes.close();260 report.finished=new Date().toISOString();261 await save();262}263console.log(JSON.stringify({results:report.results.map(result=>({fixture:result.fixture,status:result.status})),failures:report.failures.length}));264process.exitCode=report.failures.length?1:0;