1// Load one bare repository into the model the templates render.2//3// Loading is split in two. loadRepo reads everything needed to decide which4// pages changed: refs, commit metadata and line counts (cached by sha), trees5// with their object ids, CI notes, reviews, sites. The heavy parts, diffs and6// file contents, are read later by loadDiffs/loadBlobs, and only for pages7// that will actually be rendered.8import fs from 'node:fs';9import path from 'node:path';10import { randomBytes } from 'node:crypto';11import { run, tryRun, revList, catFileBatch, isBinary } from './git.ts';12import { parseDiff } from './diff.ts';13import { parseNote, ciStatus, Verifier } from './attest.ts';14import { approverMap, check, needText } from './rules.ts';15import { latestFor, loadAgentReviews, REVIEW_NAMESPACE } from './critique.ts';16import { readReviewLogs, buildReviews } from './reviews.ts';17import { settle } from './discussion.ts';18import { loadSites } from './sites.ts';19import { loadReleases } from './releases.ts';20import { repoUrls } from './urls.ts';21import { renderMarkdown, resolvePath, type LinkResolver } from './markdown.ts';22import { Spans } from './metrics.ts';23import type { RepoCache } from './cache.ts';24import type { Attestation, Branch, Commit, Config, DiffFile, Readme, Ref, Repo, RepoSummary, Stats, Tag, Tree, TreeEntry, WikiPage } from './types.ts';2526export const IMAGE = /\.(png|jpe?g|gif|webp|avif|ico|svg)$/i;27const README = /^readme(\.(md|markdown|txt))?$/i;28const EMPTY_TREE = '4b825dc642cb6eb9a060e54bf8d69288fbee4904';2930function readFile(p: string): string {31 try {32 return fs.readFileSync(p, 'utf8').trim();33 } catch {34 return '';35 }36}3738export function summarize(cfg: Config, owner: string, name: string): RepoSummary {39 const dir = path.join(cfg.root, owner, `${name}.git`);40 let description = readFile(path.join(dir, 'description'));41 if (description.startsWith('Unnamed repository')) description = '';42 const head = tryRun(dir, ['symbolic-ref', '--short', 'HEAD'])?.trim() || 'main';43 const last = tryRun(dir, ['for-each-ref', '--sort=-committerdate', '--count=1', '--format=%(committerdate:iso-strict)', 'refs/heads'])?.trim() || '';44 const isPrivate = !fs.existsSync(path.join(dir, 'git-daemon-export-ok'));45 const category = readFile(path.join(dir, 'category')).toLowerCase();46 return { owner, name, dir, description, category, defaultBranch: head, lastCommit: last, slug: `${owner}/${name}`, urls: repoUrls(cfg, owner, name), private: isPrivate };47}4849// Record markers carry a per-build nonce: patches can contain any byte,50// including the control characters a fixed separator would use.51const nonce = randomBytes(8).toString('hex');52const START = `<<forge-commit-${nonce}>>`;53const FIELD = `<<forge-field-${nonce}>>`;54const END = `<<forge-end-${nonce}>>`;55const META_FMT = START + ['%H', '%T', '%P', '%an', '%ae', '%aI', '%cn', '%ce', '%cI', '%B'].join(FIELD) + END;56const SHA_FMT = START + '%H' + END;5758function chunks<T>(list: T[], n: number): T[][] {59 const out: T[][] = [];60 for (let i = 0; i < list.length; i += n) out.push(list.slice(i, i + n));61 return out;62}6364// Sum `--numstat` lines ("add<TAB>del<TAB>path", "-" for binary).65function numstat(text: string): Stats {66 const s = { files: 0, add: 0, del: 0 };67 for (const line of text.split('\n')) {68 const m = /^(\d+|-)\t(\d+|-)\t/.exec(line);69 if (!m) continue;70 s.files++;71 if (m[1] !== '-') s.add += Number(m[1]);72 if (m[2] !== '-') s.del += Number(m[2]);73 }74 return s;75}7677// Commit metadata and line counts, from --numstat (no patch text). Cached by78// sha, since a commit never changes.79function loadCommitMeta(dir: string, shas: string[], cache: RepoCache): Map<string, Commit> {80 const missing = shas.filter((s) => !cache.commits[s]);81 for (const batch of chunks(missing, 2000)) {82 const out = run(dir, ['log', '--no-walk=unsorted', '--stdin', '--no-color', '-M', '--diff-merges=first-parent', '--numstat', `--format=${META_FMT}`], { input: batch.join('\n') + '\n' });83 for (const chunk of out.split(START)) {84 const end = chunk.indexOf(END);85 if (end < 0) continue;86 const [sha, tree, parents, an, ae, ad, cn, ce, cd, msg = ''] = chunk.slice(0, end).split(FIELD);87 const message = msg.replace(/\n+$/, '');88 const nl = message.indexOf('\n');89 cache.commits[sha] = {90 tree,91 parents: parents ? parents.split(' ') : [],92 author: { name: an, email: ae, date: ad },93 committer: { name: cn, email: ce, date: cd },94 subject: nl < 0 ? message : message.slice(0, nl),95 body: nl < 0 ? '' : message.slice(nl + 1).replace(/^\n+/, ''),96 stats: numstat(chunk.slice(end + END.length)),97 };98 }99 }100 const commits = new Map<string, Commit>();101 for (const sha of shas) {102 const m = cache.commits[sha];103 if (m) commits.set(sha, { sha, short: sha.slice(0, 7), ...m, ci: null, reviews: [] });104 }105 return commits;106}107108// Patches for the given commits. A commit whose patch would exceed maxLines109// changed lines gets its file list and counts only, so a vendored or110// generated data commit can't blow up memory or the page. Diffs carry full111// blob ids (--full-index), which key Jev's answers about files.112function loadDiffs(dir: string, commits: Commit[], maxLines: number): void {113 const todo = commits.filter((c) => !c.files);114 const small = todo.filter((c) => c.stats.add + c.stats.del <= maxLines);115 const large = todo.filter((c) => c.stats.add + c.stats.del > maxLines);116 for (const batch of chunks(small, 200)) {117 const out = run(dir, ['log', '--no-walk=unsorted', '--stdin', '--no-color', '-M', '--full-index', '--diff-merges=first-parent', '-p', `--format=${SHA_FMT}`], { input: batch.map((c) => c.sha).join('\n') + '\n' });118 const bySha = new Map(batch.map((c) => [c.sha, c]));119 for (const chunk of out.split(START)) {120 const end = chunk.indexOf(END);121 if (end < 0) continue;122 const c = bySha.get(chunk.slice(0, end));123 if (c) c.files = parseDiff(chunk.slice(end + END.length));124 }125 }126 for (const c of large) {127 const out = tryRun(dir, ['show', '--no-color', '-M', '--diff-merges=first-parent', '--format=', '--numstat', c.sha]) ?? '';128 c.files = out.split('\n').filter(Boolean).map((line): DiffFile => {129 const [a, d, ...p] = line.split('\t');130 const file = p.join('\t');131 return { status: 'M', path: file, oldPath: file, newPath: file, binary: a === '-', hunks: [], add: a === '-' ? 0 : Number(a), del: d === '-' ? 0 : Number(d), omitted: true };132 });133 c.omitted = true;134 }135 for (const c of commits) c.files ??= [];136}137138function loadTree(dir: string, sha: string, maxFiles: number): Tree {139 const out = run(dir, ['ls-tree', '-r', '-t', '-l', '-z', '--full-tree', sha]);140 const dirs = new Map<string, TreeEntry[]>([['', []]]);141 const dirOids = new Map<string, string>();142 const blobs: TreeEntry[] = [];143 for (const rec of out.split('\0')) {144 if (!rec) continue;145 const tab = rec.indexOf('\t');146 const [mode, type, oid, size] = rec.slice(0, tab).split(/ +/);147 const p = rec.slice(tab + 1);148 const slash = p.lastIndexOf('/');149 const parent = slash < 0 ? '' : p.slice(0, slash);150 const entry: TreeEntry = { name: p.slice(slash + 1), path: p, mode, type, oid, size: size === '-' ? null : Number(size) };151 if (type === 'tree') {152 dirs.set(p, []);153 dirOids.set(p, oid);154 }155 if (!dirs.has(parent)) dirs.set(parent, []);156 dirs.get(parent)!.push(entry);157 if (type === 'blob' && blobs.length < maxFiles) blobs.push(entry);158 }159 for (const list of dirs.values()) {160 list.sort((a, b) => ((a.type === 'tree') === (b.type === 'tree') ? a.name.localeCompare(b.name) : a.type === 'tree' ? -1 : 1));161 }162 return { dirs, dirOids, blobs };163}164165function loadCi(dir: string, verifier: Verifier): Map<string, Attestation[]> {166 const list = tryRun(dir, ['notes', '--ref=ci', 'list']);167 const byTree = new Map<string, Attestation[]>();168 if (!list) return byTree;169 const pairs = list.split('\n').filter(Boolean).map((l) => l.split(' '));170 const blobs = catFileBatch(dir, pairs.map(([note]) => note));171 for (const [note, object] of pairs) {172 const text = blobs.get(note)?.data.toString('utf8');173 if (!text) continue;174 for (const att of parseNote(text)) {175 if (!att.tree) continue;176 att.object = object;177 att.verification = verifier.verify(att);178 if (!byTree.has(att.tree)) byTree.set(att.tree, []);179 byTree.get(att.tree)!.push(att);180 }181 }182 return byTree;183}184185// [ahead, behind] the default branch's tip for every branch. One for-each-ref186// walks them all at once (git 2.41+); older git gets a rev-list per branch.187function aheadBehind(dir: string, base: string | undefined, heads: Map<string, string>): Map<string, [number, number]> {188 const out = new Map<string, [number, number]>();189 if (!base) return out;190 const all = tryRun(dir, ['for-each-ref', `--format=%(ahead-behind:${base}) %(refname:lstrip=2)`, 'refs/heads/']);191 if (all !== null) {192 for (const l of all.split('\n')) {193 const m = /^(\d+) (\d+) (.+)$/.exec(l);194 if (m) out.set(m[3], [Number(m[1]), Number(m[2])]);195 }196 return out;197 }198 for (const [b, sha] of heads) {199 const c = tryRun(dir, ['rev-list', '--left-right', '--count', `${base}...${sha}`])?.trim().split(/\s+/).map(Number);200 if (c) out.set(b, [c[1], c[0]]);201 }202 return out;203}204205export function loadRepo(cfg: Config, summary: RepoSummary, verifier: Verifier, spans: Spans, cache: RepoCache): Repo {206 const { dir, urls } = summary;207 const refs = spans.time('refs', (): Ref[] => (tryRun(dir, ['for-each-ref', '--format=%(refname)%00%(objectname)%00%(*objectname)%00%(objecttype)%00%(*objecttype)%00%(creatordate:iso-strict)%00%(subject)%00%(authorname)']) ?? '')208 .split('\n').filter(Boolean).map((l) => {209 const [name, sha, peeled, type, peeledType, date, subject, author] = l.split('\0');210 return { name, sha: peeled || sha, type: peeledType || type, date, subject, author };211 }));212213 const branchMap = new Map<string, string>();214 for (const r of refs) if (r.name.startsWith('refs/heads/')) branchMap.set(r.name.slice(11), r.sha);215 let defaultBranch = summary.defaultBranch;216 if (!branchMap.has(defaultBranch) && branchMap.size) defaultBranch = [...branchMap.keys()][0];217 const defaultTip = branchMap.get(defaultBranch);218219 const branches = spans.time('branches', (): Branch[] => {220 const counts = aheadBehind(dir, defaultTip, branchMap);221 return refs.filter((r) => r.name.startsWith('refs/heads/')).map((r) => {222 const b = r.name.slice(11);223 const [ahead, behind] = b !== defaultBranch ? counts.get(b) ?? [0, 0] : [0, 0];224 return { name: b, sha: r.sha, date: r.date, subject: r.subject, author: r.author, behind, ahead, isDefault: b === defaultBranch };225 }).sort((a, b) => Number(b.isDefault) - Number(a.isDefault) || String(b.date).localeCompare(String(a.date)));226 });227 const tags: Tag[] = refs.filter((r) => r.name.startsWith('refs/tags/')).map((r) => ({ name: r.name.slice(10), sha: r.sha, type: r.type, date: r.date, subject: r.subject }))228 .sort((a, b) => String(b.date).localeCompare(String(a.date)));229230 const reviews = spans.time('reviews', () => buildReviews(dir, { logs: readReviewLogs(dir), branches: branchMap, defaultBranch, ranges: cache.ranges }));231 const reviewByBranch = new Map(reviews.filter((r) => r.live).map((r) => [r.branch, r]));232 for (const b of branches) b.review = reviewByBranch.get(b.name) ?? null;233234 // Every commit that gets its own page.235 const logs = spans.time('revlist', () => new Map(branches.map((b) => [b.name, revList(dir, [`-n${cfg.maxCommits}`, b.sha])])));236 const want = new Set([...logs.values()].flat());237 for (const t of tags) if (t.type === 'commit') want.add(t.sha);238 for (const r of reviews) for (const c of r.commits) want.add(c);239 const commits = spans.time('commits', () => loadCommitMeta(dir, [...want], cache));240 spans.count('commits', commits.size);241 spans.count('branches', branches.length);242243 const ciByTree = spans.time('ci', () => loadCi(dir, verifier));244 for (const c of commits.values()) {245 const s = ciStatus(ciByTree.get(c.tree));246 if (s) s.latest = { ...s.latest, via: s.latest.object !== c.sha ? s.latest.object ?? null : null };247 c.ci = s;248 }249250 for (const r of reviews) for (const sha of r.commits) commits.get(sha)?.reviews.push(r);251252 for (const r of reviews) {253 r.layer = r.commits.map((sha) => commits.get(sha)).filter((c): c is Commit => !!c);254 const states = r.layer.map((c) => c.ci?.state ?? 'none');255 r.ci = !states.length || states.every((s) => s === 'none') ? null : {256 pass: states.filter((s) => s === 'pass').length,257 total: states.length,258 state: states.every((s) => s === 'pass') ? 'pass' : states.some((s) => s === 'fail') ? 'fail' : 'partial',259 };260 r.title = r.opts.title || r.layer[0]?.subject || r.branch;261 // The combined diff is read only when its page renders; counts come from numstat.262 r.combinedBase = r.layer[0] ? r.layer[0].parents[0] ?? EMPTY_TREE : null;263 if (r.combinedBase) {264 const key = `numstat ${r.combinedBase} ${r.tip}`;265 r.combinedStats = (cache.ranges[key] ??= numstat(tryRun(dir, ['diff', '--numstat', '-M', r.combinedBase, r.tip]) ?? '')) as Stats;266 }267 }268 // Comments and approvals against the tip: outdated, stale.269 for (const r of reviews) settle(r, commits.get(r.tip)?.tree ?? tryRun(dir, ['rev-parse', `${r.tip}^{tree}`])?.trim() ?? null);270271 // What landing each live review needs, by the approval rules on its272 // target (lib/rules.ts), and its latest agent review (lib/critique.ts).273 spans.time('gates', () => {274 const live = reviews.filter((r) => r.live && (r.status === 'open' || r.status === 'draft'));275 if (!live.length) return;276 const required = tryRun(dir, ['config', '--bool', 'forge.requireapproval'])?.trim() === 'true';277 const agents = loadAgentReviews(dir, new Verifier({ allowedSigners: cfg.allowedSigners, cacheFile: null, namespace: REVIEW_NAMESPACE }));278 const approvers = approverMap(dir);279 for (const r of live) {280 const tree = commits.get(r.tip)?.tree ?? '';281 const last = latestFor(agents.values(), `${summary.slug}#${r.id}`);282 r.agent = last ? {283 sha: last.commit, model: last.model, signer: last.verification.signer, started: last.started, scope: last.scope,284 findings: last.findings, open: last.open, current: last.tree === tree, resolved: last.resolved.length,285 } : null;286 const target = branchMap.get(r.target);287 if (!target) continue;288 // From where the review left the target: a review behind it doesn't289 // change what the target gained since.290 const base = tryRun(dir, ['merge-base', target, r.tip])?.trim() || target;291 const v = check({ gitDir: dir, base, tip: r.tip, rulesAt: target, approvers: (t) => approvers.get(t) ?? [], agent: (t) => agents.get(t) ?? [] });292 r.gate = {293 required, ok: v.ok, rules: v.found, errors: v.errors, free: v.files.length > 0 && !v.needs.length,294 needs: v.needs.map((n) => ({ line: n.rule.line, text: needText(n.rule), paths: n.paths, have: n.have, agent: n.rule.agent, agentOk: n.agentOk, ok: n.ok })),295 };296 }297 });298299 const trees = new Map<string, Tree>();300 spans.time('trees', () => {301 for (const b of branches) trees.set(b.name, loadTree(dir, b.sha, cfg.maxFiles));302 });303 for (const b of branches) trees.get(b.name)!.dirOids.set('', commits.get(b.sha)?.tree ?? b.sha);304305 const blobPaths = (b: string): Set<string> => new Set(trees.get(b)?.blobs.map((e) => e.path));306 const wikiSlug = (p: string): string => p.slice(5, -3).replace(/(^|\/)(home|index|readme)$/i, '$1').replace(/\/$/, '');307 const isWiki = (p: string): boolean => p.startsWith('wiki/') && /\.md$/i.test(p);308 // Resolve relative links in markdown from `from` on `branch`: wiki pages to309 // the rendered wiki, other files and dirs to their pages, images to raw copies.310 const linkInto = (branch: string, from: string): LinkResolver => {311 const files = blobPaths(branch);312 const dirs = trees.get(branch)?.dirs;313 return {314 link: (u) => {315 const { path: p, frag } = resolvePath(from, u);316 if (files.has(p) && branch === defaultBranch && isWiki(p)) return urls.wiki(wikiSlug(p)).href + frag;317 if (files.has(p)) return urls.blob(branch, p).href + frag;318 if (dirs?.has(p)) return urls.tree(branch, p).href + frag;319 return u;320 },321 image: (u) => {322 const { path: p } = resolvePath(from, u);323 return files.has(p) && IMAGE.test(p) ? urls.raw(branch, p).href : u;324 },325 };326 };327328 // README and wiki come from the default branch.329 let readme: Readme | null = null;330 const wiki: WikiPage[] = [];331 const root = trees.get(defaultBranch);332 if (root) spans.time('markdown', () => {333 const entry = root.dirs.get('')?.find((e) => e.type === 'blob' && README.test(e.name));334 const wikiEntries = root.blobs.filter((e) => isWiki(e.path) && (e.size ?? 0) <= cfg.maxBlob);335 const data = catFileBatch(dir, [entry?.oid, ...wikiEntries.map((e) => e.oid)].filter((x): x is string => !!x));336 const buf = entry && data.get(entry.oid)?.data;337 if (entry && buf && !isBinary(buf)) {338 const s = buf.toString('utf8');339 readme = { path: entry.path, oid: entry.oid, html: /\.(md|markdown)$/i.test(entry.name) ? renderMarkdown(s, linkInto(defaultBranch, entry.path)) : null, text: s };340 }341 for (const e of wikiEntries) {342 const b = data.get(e.oid)?.data;343 if (!b) continue;344 const s = b.toString('utf8');345 const slug = wikiSlug(e.path);346 wiki.push({ path: e.path, oid: e.oid, slug, title: /^#\s+(.+)$/m.exec(s)?.[1] ?? (slug || 'Home'), html: renderMarkdown(s, linkInto(defaultBranch, e.path)) });347 }348 wiki.sort((a, b) => (a.slug === '' ? -1 : b.slug === '' ? 1 : a.slug.localeCompare(b.slug)));349 });350351 const { sites, previews } = spans.time('sites', () => loadSites(dir, refs, { urls, allowedSigners: cfg.allowedSigners, cache }));352 for (const r of reviews) r.previews = previews.filter((p) => p.branch === r.branch);353 const releaseFiles = spans.time('releases', () => loadReleases(dir, refs, { slug: summary.slug, urls, allowedSigners: cfg.allowedSigners, cache }));354355 return {356 ...summary, defaultBranch, branches, tags, commits, logs, trees,357 blobData: new Map(),358 reviews, sites, previews, releaseFiles, readme, wiki,359 // Heavy reads, for pages that will render.360 loadDiffs: (list) => spans.time('diffs', () => loadDiffs(dir, list, cfg.maxDiffLines)),361 loadBlobs: (oids) => spans.time('blobs', () => catFileBatch(dir, oids)),362 combinedDiff: (r) => spans.time('diffs', () => (r.combinedBase ? parseDiff(tryRun(dir, ['diff', '--no-color', '-M', '--full-index', r.combinedBase, r.tip]) ?? '') : [])),363 diff: (from, to) => spans.time('diffs', () => parseDiff(tryRun(dir, ['diff', '--no-color', '-M', '--full-index', from, to]) ?? '')),364 markdown: (branch, file, text) => renderMarkdown(text, linkInto(branch, file)),365 };366}