1// Discover repos under FORGE_ROOT/<owner>/<repo>.git and flatten everything2// into page lists for 11ty pagination. Heavy HTML (diffs, blobs) is rendered3// lazily by filters so a build never holds every page's markup in memory.4import fs from 'node:fs';5import path from 'node:path';6import { createHash } from 'node:crypto';7import { summarize, loadRepo, IMAGE } from './repo.ts';89const README = /^readme(\.(md|markdown|txt))?$/i;10const MARKDOWN = /\.(md|markdown|mdown|mkd)$/i;11import { Verifier } from './attest.ts';12import { stackOf, nest } from './reviews.ts';13import { inline, markInDiff } from './discussion.ts';14import { reviewList } from './review-list.ts';15import { isBinary } from './git.ts';16import * as html from './render.ts';17import { Spans, build, codeVersion } from './metrics.ts';18import { RepoCache } from './cache.ts';19import { repoIndex, reposIndex } from './search.ts';20import { codeIndex } from './code-index.ts';21import { blames } from './blame.ts';22import { labelCommits, labelKey, releases, repoTopic } from './jev-commits.ts';23import { annotateReviews, answersStamp, isMechanical, pairChanges } from './jev-reviews.ts';24import { HighlightCache } from './highlight.ts';25import { assets } from './assets.ts';26import { recent, type RecentItem } from './recent.ts';27import { motd } from './motd.ts';28import { avatar } from './avatar.ts';29import { readProfile, linkText } from './profile.ts';30import { decodeMinutes, grid, readExport, type Grid, type PersonExport, type RepoActivityExport } from './activity.ts';31import { extraTopics, languages, license } from './front.ts';32import { catFileBatch } from './git.ts';33import type { CiStatus, Commit, Config, DiffFile, Repo, RepoCounts, RepoSummary, RepoUrls, Review, ReviewComment, SiteLink } from './types.ts';3435export function config(env: NodeJS.ProcessEnv = process.env): Config {36 const root = path.resolve(env.FORGE_ROOT || 'demo/root');37 const state = path.join(root, '.forge');38 return {39 name: env.FORGE_NAME || 'forge',40 root,41 state,42 allowedSigners: env.FORGE_ALLOWED_SIGNERS || path.join(state, 'allowed_signers'),43 cacheDir: env.FORGE_CACHE || path.join(state, 'cache'),44 baseUrl: (env.FORGE_BASE_URL || '').replace(/\/$/, ''),45 ssh: env.FORGE_SSH || 'git@localhost',46 sitesUrl: env.FORGE_SITES_URL || '/sites/{owner}/{repo}/{site}/',47 previewsUrl: env.FORGE_PREVIEWS_URL || '/previews/{owner}/{repo}/{site}/{branch}/',48 only: env.FORGE_ONLY || null,49 maxCommits: Number(env.FORGE_MAX_COMMITS || 1000),50 maxFiles: Number(env.FORGE_MAX_FILES || 5000),51 maxBlob: Number(env.FORGE_MAX_BLOB || 1 << 20),52 // Commits changing more lines than this show their file list, not the patch.53 maxDiffLines: Number(env.FORGE_MAX_DIFF_LINES || 20000),54 // Images (for READMEs and file pages) are copied as raw files up to this size.55 maxRaw: Number(env.FORGE_MAX_RAW || 20 << 20),56 };57}5859// Repo and owner names sort ignoring case, so "Zed" doesn't come before "api".60const caseless = (a: string, b: string): number => a.localeCompare(b, 'en', { sensitivity: 'base' }) || (a < b ? -1 : a > b ? 1 : 0);6162// Top-level names used by the forge itself; never owners.63const RESERVED = new Set(['assets', 'sites', 'previews', 'hosts', 'insights', 'topics', 'index.html', 'git-shell-commands', 'lost+found', 'login', 'private']);6465// Every repo, public or not (RepoSummary.private); builds filter.66export function discover(cfg: Config): { repos: RepoSummary[]; owners: string[] } {67 if (!fs.existsSync(cfg.root)) return { repos: [], owners: [] };68 const repos: RepoSummary[] = [];69 const owners: string[] = [];70 for (const owner of fs.readdirSync(cfg.root).sort(caseless)) {71 if (owner.startsWith('.') || RESERVED.has(owner)) continue;72 const odir = path.join(cfg.root, owner);73 let entries: string[] | null;74 // Skip anything that isn't a readable directory (e.g. lost+found on a fresh volume).75 try {76 entries = fs.statSync(odir).isDirectory() ? fs.readdirSync(odir) : null;77 } catch {78 entries = null;79 }80 if (!entries) continue;81 owners.push(owner);82 for (const d of entries.sort(caseless)) {83 if (!d.endsWith('.git') || !fs.existsSync(path.join(odir, d, 'HEAD'))) continue;84 repos.push(summarize(cfg, owner, d.slice(0, -4)));85 }86 }87 return { repos, owners };88}8990// Loaded repos by slug, for the lazy-rendering filters.91const registry = new Map<string, Repo>();92const highlights = new Map<string, HighlightCache>();9394// What every page of a repo shows in its header.95export interface RepoHead {96 owner: string;97 name: string;98 slug: string;99 description: string;100 defaultBranch: string;101 urls: RepoUrls;102 counts: RepoCounts;103 // Built into the private root, for its readers only.104 private: boolean;105}106107// One generated file: a page for pagination, or raw bytes.108export interface Page {109 permalink: string;110 title?: string;111 nav?: string;112 repo: RepoHead;113 [key: string]: unknown;114}115116export interface RawFile {117 path: string;118 oid?: string;119 data?: Buffer;120 // A file to link instead (a code index in the cache).121 file?: string;122}123124export interface Forge {125 config: Config;126 repos: RepoSummary[];127 owners: Owner[];128 // The index (site/index.njk): its line under the header (null: none),129 // the public owners, the shelves across them, and what happened lately.130 motd: string | null;131 profiles: Profile[];132 topics: Topic[];133 recent: RecentItem[];134 pages: Record<string, Page[]>;135 raw: RawFile[];136 // Incremental builds: the live site, and the pages taken from it unchanged.137 live: string | null;138 code: string;139 skipped: string[];140 pageCaches: { cache: RepoCache; fps: Record<string, string> }[];141}142143export interface Profile {144 owner: string;145 repos: number;146 // When they last did something public (lib/recent.ts), ISO.147 active: string;148}149150// An owner's page: their public repos, their front matter (what they set151// with ssh profile, lib/profile.ts, and facts) and their activity grid.152export interface Owner {153 owner: string;154 repos: RepoSummary[];155 person: Person;156 // data: each repo's commit times, as forge lake exports them (ui.ts puts157 // them on the reader's days); the grid without JS has UTC days.158 activity: { grid: Grid; data: { repos: Record<string, number[]> } } | null;159}160161export interface Person {162 bio: string;163 link: string;164 linkText: string;165 sites: number;166 // The month they joined, "Sep 2026", or ''.167 joined: string;168 // What they pinned (still public), or their most recently active repos.169 pins: RepoSummary[];170 pinned: boolean;171}172173export interface Topic {174 id: string;175 title: string;176 repos: number;177 // One owner's: that shelf on their page; several owners': on /topics/.178 href: string;179}180181const MONTHS = ['Jan', 'Feb', 'Mar', 'Apr', 'May', 'Jun', 'Jul', 'Aug', 'Sep', 'Oct', 'Nov', 'Dec'];182const month = (ms: number): string => {183 const t = new Date(ms);184 return `${MONTHS[t.getUTCMonth()]} ${t.getUTCFullYear()}`;185};186187let last: Forge | null = null;188export const current = (): Forge | null => last;189190// Path breadcrumbs: directories link to /tree/, a final file to /blob/.191const crumbs = (urls: RepoUrls, branch: string, p: string, isFile = false): { name: string; href: string }[] => {192 const parts = p ? p.split('/') : [];193 return parts.map((name, i) => {194 const sub = parts.slice(0, i + 1).join('/');195 return { name, href: (isFile && i === parts.length - 1 ? urls.blob(branch, sub) : urls.tree(branch, sub)).href };196 });197};198199const fingerprint = (...parts: unknown[]): string =>200 createHash('sha1').update(parts.map((x) => (typeof x === 'string' ? x : JSON.stringify(x))).join('\0')).digest('hex').slice(0, 16);201const ciKey = (ci: CiStatus | null): string => (ci ? `${ci.state}:${ci.latest.started}:${ci.latest.verification.signer}:${ci.latest.via ?? ''}` : '');202203export function loadForge(env: NodeJS.ProcessEnv = process.env): Forge {204 build.dataStart = performance.now();205 const cfg = config(env);206 const { repos: all, owners: ownerNames } = discover(cfg);207 // Public and private repos are built apart. A public build leaves private208 // repos out: no pages, and no entry on the index, owner pages or209 // repos.json. FORGE_PRIVATE=1 builds private ones, whose pages bin/forge210 // moves into the private root (build_private); its index and owner pages211 // aren't used.212 const priv = env.FORGE_PRIVATE === '1';213 const summaries = all.filter((s) => s.private === priv);214 const verifier = new Verifier({ allowedSigners: cfg.allowedSigners, cacheFile: path.join(cfg.cacheDir, 'verify.json') });215 // Index and owner pages list every repo, but a single-repo build only loads216 // one. Each repo's counts and sites are cached when it is built.217 const summaryCache = path.join(cfg.cacheDir, priv ? 'summaries-private.json' : 'summaries.json');218 let cached: Record<string, { counts: RepoCounts; sites: SiteLink[]; topic?: string | null; release?: { tag: string; date: string; href: string } | null }> = {};219 try {220 cached = JSON.parse(fs.readFileSync(summaryCache, 'utf8'));221 } catch {}222 const pages: Record<string, Page[]> = { homes: [], logs: [], trees: [], blobs: [], commits: [], refs: [], reviewLists: [], reviews: [], reviewDiffs: [], revisions: [], sites: [], wiki: [], atom: [], branchMenus: [], releases: [], insights: [] };223 const raw: RawFile[] = [];224 // Incremental builds: when building next to a live site, a page whose225 // fingerprint matches the last build (and whose file still exists) isn't226 // rendered; its file is hard-linked from the live site instead.227 const live = env.FORGE_LIVE_WWW || null;228 // Pages link assets by hashed name, so a new name (sqlite-wasm or Node229 // updated, say) must re-render them too.230 const code = fingerprint(codeVersion(), Object.values(assets()).map((a) => a.file));231 const skipped: string[] = [];232 const pageCaches: Forge['pageCaches'] = [];233234 for (const s of summaries) {235 if (cfg.only && cfg.only !== s.slug) continue;236 const spans = new Spans();237 const before: Record<string, number> = Object.fromEntries(Object.entries(pages).map(([k, v]) => [k, v.length]));238 const t0 = performance.now();239 const cache = RepoCache.load(cfg, s.slug);240 const repo = loadRepo(cfg, s, verifier, spans, cache);241 registry.set(s.slug, repo);242 // Jev's stored answers (lib/jev-commits.ts): commit labels, the repo's topic.243 labelCommits(repo.dir, repo.commits.values(), cache.ranges);244 const topic = repoTopic(repo);245 const hl = new HighlightCache(path.join(cfg.cacheDir, 'highlight', s.slug));246 hl.sweep(new Set([...repo.trees.values()].flatMap((t) => t.blobs.map((e) => e.oid))));247 highlights.set(s.slug, hl);248 const u = repo.urls;249 const head: RepoHead = {250 owner: repo.owner, name: repo.name, slug: repo.slug, description: repo.description, defaultBranch: repo.defaultBranch, urls: u,251 counts: {252 reviews: repo.reviews.filter((r) => r.status === 'open' || r.status === 'draft').length,253 sites: repo.sites.length,254 wiki: repo.wiki.length,255 releases: repo.tags.length,256 },257 private: s.private,258 };259 cached[s.slug] = { counts: head.counts, sites: repo.sites.map((x) => ({ name: x.name, url: x.url, date: x.history[0]?.date ?? null })), topic };260 // Everything every page of this repo shows in its header.261 const headKey = fingerprint(code, cfg.name, cfg.baseUrl, cfg.ssh, head.description, head.defaultBranch, (head.counts.wiki ?? 0) > 0, (head.counts.sites ?? 0) > 0, head.private);262 const prev = cache.pages?.fps ?? {};263 const next: Record<string, string> = {};264 let skippedHere = 0;265 const want = (out: string, key: string): boolean => {266 const k = fingerprint(headKey, key);267 next[out] = k;268 if (live && prev[out] === k && fs.existsSync(path.join(live, out))) {269 skipped.push(out);270 skippedHere++;271 return false;272 }273 return true;274 };275 const page = (perma: string, title: string, nav: string, extra: Record<string, unknown>): Page =>276 ({ permalink: perma + (perma.endsWith('/') ? 'index.html' : ''), title, nav, repo: head, ...extra });277 // Pages without a key are cheap and always rendered.278 const add = (list: Page[], p: Page, key?: string): void => {279 if (key == null || want(p.permalink, key)) list.push(p);280 };281282 const defaultLog = (repo.logs.get(repo.defaultBranch) ?? []).map((sha) => repo.commits.get(sha)).filter((c): c is Commit => !!c);283 const rootTree = repo.trees.get(repo.defaultBranch);284 const entryLinks = (b: string, entries: { type: string; path: string }[]) => entries.map((e) => ({ ...e, href: e.type === 'commit' ? null : e.type === 'blob' ? u.blob(b, e.path).href : u.tree(b, e.path).href }));285 // Each tag's notes, its files if it has a release, and its commit's CI.286 const notes = releases(repo, cache.ranges).map((r) => ({ ...r, files: repo.releaseFiles.get(r.tag.name) ?? null, ci: repo.commits.get(r.tag.sha)?.ci ?? null }));287 // The newest release with files, linked from the home page.288 const shipped = notes.find((r) => r.files);289 const release = shipped ? { tag: shipped.tag.name, files: shipped.files?.files.length ?? 0, href: `${u.releases.href}#${shipped.tag.name}`, date: shipped.tag.date } : null;290 cached[s.slug].release = release && { tag: release.tag, date: release.date, href: release.href };291 // The front matter's facts (lib/front.ts); its topics need every repo's292 // shelf, so they are filled in once all are loaded.293 const rootEntries = rootTree?.dirs.get('') ?? [];294 const topicsFile = rootTree?.blobs.find((e) => e.path === '.forge/topics' && (e.size ?? 0) <= 4096);295 const tip = defaultLog[0];296 const lic = license(repo.dir, rootEntries);297 const langs = rootTree ? languages(repo.dir, rootTree.blobs) : [];298 const front = {299 shelf: s.category || (topic && SHELVES[topic]) || '',300 extras: topicsFile ? extraTopics(catFileBatch(repo.dir, [topicsFile.oid]).get(topicsFile.oid)?.data.toString('utf8') ?? '') : [],301 topics: [] as { title: string; href: string | null }[],302 sites: repo.sites.length === 1 ? { name: repo.sites[0].name, url: repo.sites[0].url } : repo.sites.length ? { n: repo.sites.length, href: u.sites.href } : null,303 release,304 ci: tip?.ci ? { state: tip.ci.state, short: tip.short, href: u.commit(tip.sha).href } : defaultLog.some((c) => c.ci) && tip ? { state: 'none', short: tip.short, href: u.commit(tip.sha).href } : null,305 reviews: repo.reviews.filter((r) => r.status === 'open').length,306 license: lic && { id: lic.id, path: lic.path, href: u.blob(repo.defaultBranch, lic.path).href },307 languages: langs.slice(0, 3),308 rest: Math.round(langs.slice(3).reduce((a, l) => a + l.pct, 0) * 10) / 10,309 };310 add(pages.homes, page(u.home.path, `${repo.slug}`, 'home', {311 readme: repo.readme, latest: defaultLog[0] ?? null, commitCount: defaultLog.length, branchCount: repo.branches.length,312 entries: rootTree ? entryLinks(repo.defaultBranch, rootTree.dirs.get('') ?? []) : [],313 filesHref: rootTree ? u.files(repo.defaultBranch).href : null,314 reviews: firstStacks(nest(repo.reviews.filter((r) => r.status === 'open')), 5), topic, release, front,315 }));316 pages.atom.push({ permalink: u.atom.path, repo: head, commits: defaultLog.slice(0, 30) });317 for (const kind of ['tree', 'log']) pages.branchMenus.push({ permalink: u.branchMenu(kind).path, repo: head, kind, branches: repo.branches });318319 const wantBlobs = new Set<string>();320 // Blame and history of the default branch's files (lib/blame.ts), a321 // blame.json beside each file page, which the page fetches.322 const blame = rootTree ? spans.time('blame', () => blames(cfg, repo, rootTree.blobs)) : null;323 if (blame) spans.count('blamed', blame.computed);324 for (const b of repo.branches) {325 const commits = (repo.logs.get(b.name) ?? []).map((sha) => repo.commits.get(sha)).filter((c): c is Commit => !!c);326 add(pages.logs, page(u.log(b.name).path, `Log - ${b.name} - ${repo.slug}`, 'log', { branch: b, commits }),327 `log ${b.name} ${b.sha} ${fingerprint(commits.map((c) => c.sha + ciKey(c.ci) + labelKey(c)))}`);328329 const tree = repo.trees.get(b.name)!;330 for (const [dirPath, entries] of tree.dirs) {331 const readme = entries.find((e) => e.type === 'blob' && README.test(e.name) && (e.size ?? 0) <= cfg.maxBlob);332 const p = page(u.tree(b.name, dirPath).path, `${dirPath || 'Files'} - ${b.name} - ${repo.slug}`, 'files', {333 branch: b, path: dirPath, crumbs: crumbs(u, b.name, dirPath), entries: entryLinks(b.name, entries),334 parentHref: dirPath ? u.tree(b.name, dirPath.includes('/') ? dirPath.slice(0, dirPath.lastIndexOf('/')) : '').href : null,335 readme: readme ? { oid: readme.oid, path: readme.path, name: readme.name } : null,336 filesHref: u.files(b.name).href,337 });338 if (want(p.permalink, `tree ${b.name} ${dirPath} ${tree.dirOids.get(dirPath) ?? ''}`)) {339 pages.trees.push(p);340 if (readme) wantBlobs.add(readme.oid);341 }342 }343 for (const e of tree.blobs) {344 const small = e.size != null && e.size <= cfg.maxBlob;345 const image = IMAGE.test(e.path) && e.size != null && e.size <= cfg.maxRaw;346 const markdown = MARKDOWN.test(e.path) && small;347 const info = {348 branch: b, path: e.path, crumbs: crumbs(u, b.name, e.path, true), oid: e.oid, size: e.size, mode: e.mode,349 parentHref: u.tree(b.name, e.path.includes('/') ? e.path.slice(0, e.path.lastIndexOf('/')) : '').href,350 image: image ? u.raw(b.name, e.path).href : null,351 filesHref: u.files(b.name).href,352 sourceHref: markdown ? u.blobSource(b.name, e.path).href : null,353 renderedHref: markdown ? u.blob(b.name, e.path).href : null,354 blameHref: null as string | null,355 commitBase: u.home.href + 'commit/',356 reviewBase: u.reviews.href,357 };358 const bl = b.isDefault ? blame?.files.get(e.path) : undefined;359 if (bl) {360 const at = `${u.blob(b.name, e.path).path}blame.json`;361 info.blameHref = `${u.blob(b.name, e.path).href}blame.json`;362 if (want(at, `blame ${fingerprint(bl.toString('utf8'))}`)) raw.push({ path: at, data: bl });363 }364 const key = `blob ${b.name} ${e.path} ${e.oid} ${e.mode}${bl ? ' blame.v1' : ''}`;365 // Markdown files render as documents; their source is a page of its own.366 const p = page(u.blob(b.name, e.path).path, `${e.path} - ${b.name} - ${repo.slug}`, 'files', { ...info, markdown });367 if (want(p.permalink, key)) {368 pages.blobs.push(p);369 if (small) wantBlobs.add(e.oid);370 }371 if (markdown) {372 const src = page(u.blobSource(b.name, e.path).path, `${e.path} (source) - ${b.name} - ${repo.slug}`, 'files', { ...info, markdown: false, isSource: true });373 if (want(src.permalink, `source ${key}`)) {374 pages.blobs.push(src);375 wantBlobs.add(e.oid);376 }377 }378 if (image && want(u.raw(b.name, e.path).path, `raw ${e.oid}`)) {379 raw.push({ path: u.raw(b.name, e.path).path, oid: e.oid });380 wantBlobs.add(e.oid);381 }382 }383 const list = tree.blobs.map((e) => e.path).join('\n') + '\n';384 if (want(u.files(b.name).path, `files ${fingerprint(list)}`)) raw.push({ path: u.files(b.name).path, data: Buffer.from(list) });385 }386387 const diffFor = new Set<Commit>();388 for (const c of repo.commits.values()) {389 const p = page(u.commit(c.sha).path, `${c.subject} - ${repo.slug}`, 'log', { commit: c });390 if (want(p.permalink, `commit ${c.sha} ${ciKey(c.ci)} ${labelKey(c)} ${c.reviews.map((r) => `${r.id}:${r.title}`).join(',')}`)) {391 pages.commits.push(p);392 diffFor.add(c);393 }394 }395 add(pages.refs, page(u.refs.path, `Refs - ${repo.slug}`, 'refs', { branches: repo.branches, tags: repo.tags }));396 add(pages.releases, page(u.releases.path, `Releases - ${repo.slug}`, 'releases', { releases: notes }));397398 const groups = (['open', 'draft', 'merged', 'closed'] as const).map((status) => ({ status, items: nest(repo.reviews.filter((r) => r.status === status)) })).filter((g) => g.items.length);399 add(pages.reviewLists, page(u.reviews.path, `Reviews - ${repo.slug}`, 'reviews', { groups, total: repo.reviews.length }));400 // Jev's stored answers on review pages (lib/jev-reviews.ts). Live reviews401 // get answers after each push, so their pages also key on the store.402 annotateReviews(repo.dir, repo.reviews);403 const answers = answersStamp(repo.dir);404 for (const r of repo.reviews) {405 const jev = r.status === 'open' || r.status === 'draft' ? answers : '';406 const stack = stackOf(r);407 const key = fingerprint({408 id: r.id, status: r.status, title: r.title, branch: r.branch, target: r.target, author: r.author, updated: r.updated,409 revs: r.revisions.map((v) => [v.n, v.sha, v.ts, v.user, v.view]),410 layer: r.layer.map((c) => c.sha + ciKey(c.ci)),411 previews: r.previews.map((v) => [v.name, v.url, v.history[0]?.source]),412 stack: stack.map((x) => [x.review.id, x.review.status, x.review.title, x.review.branch, x.review.layer.length, x.review.ci, x.depth]),413 live: r.live,414 discussion: [r.opts.description, r.comments.map((c) => [c.n, c.rev, c.outdated]), r.approvals.map((a) => [a.user, a.rev, a.current])],415 // The to land panel: the rules' verdict and the latest agent review.416 gate: r.gate ?? null, agent: r.agent ?? null, ci: r.ci,417 jev,418 });419 const p = page(u.review(r.id).path, `#${r.id} ${r.title} - ${repo.slug}`, 'reviews', { review: r, stack, stackAt: stack.findIndex((x) => x.review === r) });420 if (want(p.permalink, `review ${key}`)) {421 pages.reviews.push(p);422 for (const c of r.layer) diffFor.add(c);423 // Line comments link to the combined diff when their line is in it.424 if (r.comments.some((c) => c.path && !c.outdated)) markInDiff(r, (r.combined ??= repo.combinedDiff(r)));425 }426 const d = page(u.reviewDiff(r.id).path, `#${r.id} combined diff - ${repo.slug}`, 'reviews', { review: r });427 const inlined = r.comments.filter((c) => c.path && !c.outdated).map((c) => c.n).join(',');428 if (want(d.permalink, `combined ${r.combinedBase} ${r.tip} ${r.live ? r.branch : ''} ${inlined} ${jev}`)) {429 r.combined ??= repo.combinedDiff(r);430 pages.reviewDiffs.push(d);431 }432 for (const rev of r.revisions) {433 const from = rev.prev ? r.revisions[rev.prev - 1] : null;434 if (!rev.view || !from) continue;435 const v = page(u.revision(r.id, rev.n).path, `#${r.id} r${rev.prev} vs r${rev.n} - ${repo.slug}`, 'reviews', { review: r, rev, from });436 if (want(v.permalink, `revision ${r.id} ${rev.n} ${rev.sha} ${rev.view} ${fingerprint(rev.range ?? from.sha)} ${r.title} ${r.parent?.id} ${jev}`)) {437 if (rev.view === 'diff') v.files = repo.diff(from.sha, rev.sha);438 pages.revisions.push(v);439 }440 }441 }442 // For `forge reviews`, `checkout` and `land`, which read it over ssh.443 const list = JSON.stringify(reviewList(repo.reviews, (id) => cfg.baseUrl + u.review(id).href), null, 1) + '\n';444 if (want(`${u.home.path}reviews.json`, `reviews.json ${fingerprint(list)}`)) raw.push({ path: `${u.home.path}reviews.json`, data: Buffer.from(list) });445 add(pages.sites, page(u.sites.path, `Sites - ${repo.slug}`, 'sites', { sites: repo.sites, previews: repo.previews }));446 // The page is the same until the code changes; its charts fetch the447 // lake's exports, which go next to it (and which forge lake replaces in448 // the live pages after each run).449 // The activity grid comes from the lake's last export (lib/insights.ts);450 // with JS the page fetches the export forge lake keeps fresh.451 const act = readExport<RepoActivityExport>(cfg.state, repo.owner, repo.name, 'insights', 'activity.json');452 const activity = act ? { grid: grid(act.days, 'event') } : null;453 add(pages.insights, page(u.insights.path, `Insights - ${repo.slug}`, 'insights', { activity }), `insights ${fingerprint(activity)}`);454 raw.push(...insightFiles(cfg, path.join(repo.owner, repo.name, 'insights'), u.insights.path));455 for (const w of repo.wiki) {456 add(pages.wiki, page(u.wiki(w.slug).path, `${w.title} - wiki - ${repo.slug}`, 'wiki', { wiki: w, pages: repo.wiki.map((x) => ({ title: x.title, slug: x.slug, href: u.wiki(x.slug).href })) }));457 }458 // The default branch's code index (lib/code-index.ts), kept by tree id.459 const codeDb = spans.time('code', () => codeIndex(cfg, repo));460 if (codeDb) raw.push({ path: u.code.path, file: codeDb.file });461 if (codeDb?.stats) spans.count('code indexed', codeDb.stats.files - codeDb.stats.reused);462 const index = repoIndex(repo, codeDb ? u.code.href : null);463 if (want(u.search.path, `search ${fingerprint(index)}`)) raw.push({ path: u.search.path, data: Buffer.from(index) });464465 // The heavy reads, for exactly the pages that will render.466 repo.loadDiffs([...diffFor]);467 repo.blobData = repo.loadBlobs([...wantBlobs]);468 for (const r of raw) if (r.oid && !r.data) r.data = repo.blobData.get(r.oid)?.data;469 cache.save();470 if (live) pageCaches.push({ cache, fps: next });471472 const added: Record<string, number> = Object.fromEntries(Object.entries(pages).map(([k, v]): [string, number] => [k, v.length - before[k]]).filter(([, n]) => n));473 const rendered = Object.values(added).reduce((a, b) => a + b, 0);474 spans.count('rendered', rendered);475 spans.count('skipped', skippedHere);476 build.repos.push({ slug: s.slug, ms: { ...spans.ms, load: Math.round(performance.now() - t0) }, counts: { ...spans.counts, pages: rendered }, pages: added });477 }478 verifier.save();479 build.dataEnd = performance.now();480 for (const s of summaries) Object.assign(s, { counts: {}, sites: [] }, cached[s.slug]);481 fs.mkdirSync(cfg.cacheDir, { recursive: true });482 fs.writeFileSync(summaryCache, JSON.stringify(Object.fromEntries(summaries.map((s) => [s.slug, cached[s.slug] ?? { counts: {}, sites: [] }]))));483484 // The index's panels; a private build has no index.485 const lately = priv ? { items: [], active: {} } : recent(cfg, summaries);486 // Only owners with a public repo have a page: a user's name alone (forge487 // user add makes their directory) is nobody's business.488 const owners = ownerNames489 .filter((o) => (!cfg.only?.startsWith('@') || cfg.only === `@${o}`) && summaries.some((s) => s.owner === o))490 .map((owner) => ownerPage(cfg, owner, summaries.filter((s) => s.owner === owner)));491 for (const o of owners) {492 if (!priv && o.activity) raw.push({ path: `/${o.owner}/activity.json`, data: Buffer.from(JSON.stringify(o.activity.data) + '\n') });493 }494 const profiles = ownerNames.map((owner) => ({ owner, repos: summaries.filter((s) => s.owner === owner).length, active: lately.active[owner] ?? '' })).filter((p) => p.repos);495 const topics = filters.shelves(summaries).map((sh) => {496 const who = [...new Set(sh.repos.map((r) => r.owner))];497 return { id: sh.id, title: sh.title, repos: sh.repos.length, href: who.length === 1 ? `/${who[0]}/#${sh.id}` : `/topics/#${sh.id}` };498 });499 // A repo's topics: its shelf, then its .forge/topics; each links where500 // the home page's topics do, if any shelf has that name.501 const topicHref = new Map(topics.map((t) => [t.id, t.href]));502 for (const p of pages.homes) {503 const f = p.front as { shelf: string; extras: string[]; topics: { title: string; href: string | null }[] };504 for (const title of [f.shelf, ...f.extras]) {505 if (!title || f.topics.some((t) => t.title === title)) continue;506 f.topics.push({ title, href: topicHref.get(title.replace(/[^a-z0-9]+/g, '-')) ?? null });507 }508 }509 // Repo search on the index and owner pages.510 raw.push({ path: '/repos.json', data: Buffer.from(reposIndex(summaries)) });511 if (!cfg.only || cfg.only === '-') raw.push(...insightFiles(cfg, 'insights', '/insights/'));512 last = { config: cfg, repos: summaries, owners, motd: motd(cfg.state), profiles, topics, recent: lately.items, pages, raw: raw.filter((r) => r.data || r.file), live, code, skipped, pageCaches };513 return last;514}515516// An owner's page: front matter and activity. Pins are checked again here,517// so a pinned repo that went private or away just drops out.518function ownerPage(cfg: Config, owner: string, repos: RepoSummary[]): Owner {519 const prof = readProfile(cfg.state, owner);520 const byName = new Map(repos.map((r) => [r.name, r]));521 const pinned = prof.pins.map((n) => byName.get(n)).filter((r): r is RepoSummary => !!r);522 const pins = pinned.length ? pinned : [...repos].sort((a, b) => b.lastCommit.localeCompare(a.lastCommit)).slice(0, 3);523 const exp = readExport<PersonExport>(cfg.state, 'people', `${owner}.json`);524 const joinedAt = [exp?.joined, prof.joined ? prof.joined * 1000 : null].filter((t): t is number => !!t);525 // Only the repos on the page count: one gone private since the export isn't.526 const kept: Record<string, number[]> = {};527 const totals: Record<string, number> = {};528 for (const [name, enc] of Object.entries(exp?.repos ?? {})) {529 if (!byName.has(name) || !Array.isArray(enc)) continue;530 kept[name] = enc;531 for (const m of decodeMinutes(enc)) {532 const d = new Date(m * 60000).toISOString().slice(0, 10);533 totals[d] = (totals[d] ?? 0) + 1;534 }535 }536 return {537 owner,538 repos,539 person: {540 bio: prof.bio, link: prof.link, linkText: prof.link ? linkText(prof.link) : '',541 sites: repos.reduce((a, r) => a + (r.counts?.sites ?? 0), 0),542 joined: joinedAt.length ? month(Math.min(...joinedAt)) : '',543 pins, pinned: pinned.length > 0,544 },545 activity: repos.length ? { grid: grid(totals, 'commit'), data: { repos: kept } } : null,546 };547}548549// The lake's insights exports for a scope (lib/insights.ts), read now: forge550// lake may swap the directory while the build runs.551function insightFiles(cfg: Config, dir: string, at: string): RawFile[] {552 const from = path.join(cfg.state, 'lake', 'export', dir);553 const out: RawFile[] = [];554 let names: string[] = [];555 try {556 names = fs.readdirSync(from).filter((f) => /^[a-z]+\.(json|parquet)$/.test(f));557 } catch {558 return out;559 }560 for (const f of names) {561 try {562 out.push({ path: at + f, data: fs.readFileSync(path.join(from, f)) });563 } catch {}564 }565 return out;566}567568interface BlobPage {569 repo: RepoHead;570 oid: string;571 size: number | null;572 path: string;573 image: string | null;574 isSource?: boolean;575 branch?: { name: string };576}577578// The first stacks of a nested list, whole, up to about `n` rows.579function firstStacks<T extends { depth: number }>(list: T[], n: number): T[] {580 const end = list.findIndex((e, i) => i >= n && e.depth === 0);581 return end < 0 ? list : list.slice(0, end);582}583584// A comment as it shows inline in the combined diff.585function commentHtml(c: ReviewComment, repo: RepoHead, review: Review): string {586 return `<div class="cmt" id="comment-${c.n}"><div class="cmt-head"><b>${html.esc(c.user)}</b> <a href="${repo.urls.review(review.id).href}#comment-${c.n}">${filters.ago(c.ts)}</a></div>` +587 `<div class="markdown">${filters.reviewMarkdown(c.body, repo, review)}</div></div>`;588}589590// Link a commit if it has a page, else show the short sha.591function commitLink(sha: string, repo: RepoHead, label?: string): string {592 const text = html.esc(label ?? String(sha).slice(0, 7));593 return registry.get(repo.slug)?.commits.has(sha) ? `<a href="${repo.urls.commit(sha).href}">${text}</a>` : text;594}595596// The entities rendered markdown uses (sanitize-html writes these).597const ENTITIES: Record<string, string> = { amp: '&', lt: '<', gt: '>', quot: '"', apos: "'", nbsp: ' ' };598const decode = (s: string): string =>599 s.replace(/&(#x?)?(\w+);/gi, (m, num: string | undefined, e: string) => {600 if (!num) return ENTITIES[e.toLowerCase()] ?? m;601 const c = parseInt(e, num.length > 1 ? 16 : 10);602 return c > 0 && c <= 0x10ffff ? String.fromCodePoint(c) : m;603 });604605const MODES: Record<string, string> = { '040000': 'drwxr-xr-x', '100644': '-rw-r--r--', '100755': '-rwxr-xr-x', '120000': 'lrwxrwxrwx', '160000': 'm---------' };606607// Shelves for Jev's topics (repoTopic in lib/jev-commits.ts), for repos608// whose owner hasn't given a category.609const SHELVES: Record<string, string> = {610 'web app': 'web apps',611 cli: 'terminal',612 service: 'services',613 library: 'libraries',614 game: 'games',615 'static site': 'sites',616 docs: 'writing',617 experiment: 'experiments',618 config: 'config',619};620621// Filters that return markup; setup() in render-page.ts marks their output safe.622export const markupFilters = new Set(['ago', 'date', 'avatar', 'json']);623624// Lazy renderers, registered as Nunjucks filters (setup() in render-page.ts).625export const filters = {626 // `branch` links each changed file to its page on that branch, if present.627 // With `review`, its current line comments show under their lines, and628 // files carry their paths for new ones (site/assets/discuss.ts). With629 // `mechanical` (review pages), files Jev judged mechanical are marked.630 diffHtml(files: DiffFile[] | undefined, repo: RepoHead, prefix = '', branch: string | null = null, review: Review | null = null, mechanical = false): string {631 const paths = branch ? new Set(registry.get(repo.slug)?.trees.get(branch)?.blobs.map((e) => e.path)) : null;632 const blobHref = paths && branch ? (p: string) => (paths.has(p) ? repo.urls.blob(branch, p).href : null) : null;633 const annotate = review ? inline(review, (c) => commentHtml(c, repo, review)) : null;634 const dir = registry.get(repo.slug)?.dir;635 return html.diff(files ?? [], { prefix, blobHref, annotate, mechanical: mechanical && dir ? isMechanical(dir) : null, paths: !!review });636 },637 // Review descriptions (one line in the log, "\n" for line breaks) and638 // comments are markdown, sanitized like READMEs; links resolve on the branch.639 reviewMarkdown(text: string | null | undefined, repo: RepoHead, review: Review, escaped = false): string {640 const r = registry.get(repo.slug);641 if (!r || !text) return '';642 return r.markdown(review.live ? review.branch : r.defaultBranch, '', escaped ? text.replaceAll('\\n', '\n') : text);643 },644 diffstatHtml(files: DiffFile[] | undefined, prefix = '', tools = ''): string {645 return html.diffstat(files ?? [], prefix, String(tools));646 },647 blobHtml(p: BlobPage): string {648 if (p.image && !p.isSource) return `<div class="image-view"><img src="${html.esc(p.image)}" alt="${html.esc(p.path)}"></div>`;649 const data = registry.get(p.repo.slug)?.blobData.get(p.oid)?.data;650 if (!data) return `<p class="empty">This file is too large to show here (${filters.size(p.size)}).</p>`;651 if (isBinary(data)) return `<p class="empty">Binary file, ${filters.size(p.size)}.</p>`;652 const hl = highlights.get(p.repo.slug);653 return html.blob(data.toString('utf8'), p.path, hl && ((code, lang) => hl.highlight(p.oid, code, lang)));654 },655 // A markdown file rendered as a document.656 markdownHtml(p: BlobPage): string {657 const repo = registry.get(p.repo.slug);658 const data = repo?.blobData.get(p.oid)?.data;659 if (!repo || !data || !p.branch) return '';660 return repo.markdown(p.branch.name, p.path, data.toString('utf8'));661 },662 // A directory's README, under its listing.663 dirReadmeHtml(readme: { oid: string; path: string; name: string } | null, repo: RepoHead, branch: string): string {664 const r = registry.get(repo.slug);665 const data = readme && r?.blobData.get(readme.oid)?.data;666 if (!r || !readme || !data || isBinary(data)) return '';667 const text = data.toString('utf8');668 return MARKDOWN.test(readme.name) ? r.markdown(branch, readme.path, text) : `<pre>${html.esc(text)}</pre>`;669 },670 // Rewritten pairs carry Jev's label for what the rewrite changed.671 rangeDiffHtml(text: string, repo: RepoHead): string {672 const r = registry.get(repo.slug);673 const shas = [...(r?.commits.keys() ?? [])];674 return html.rangeDiff(text, {675 commitHref: (short) => {676 const full = shas.find((s) => s.startsWith(short));677 return full ? repo.urls.commit(full).href : null;678 },679 change: r ? pairChanges(r.dir, text) : null,680 });681 },682 ciHtml(status: CiStatus | null, long = false): string {683 return html.ci(status, { long });684 },685 date: html.date,686 // Owner pages, /topics/ and the home page's topics: repos on shelves, newest first. A shelf is687 // the category the repo's owner gave it, else Jev's topic, else "other".688 // Fuller shelves come first, "other" last.689 shelves(repos: RepoSummary[]): { id: string; title: string; repos: RepoSummary[] }[] {690 const by = new Map<string, RepoSummary[]>();691 for (const r of [...repos].sort((a, b) => b.lastCommit.localeCompare(a.lastCommit))) {692 const k = r.category || (r.topic && SHELVES[r.topic]) || 'other';693 by.set(k, [...(by.get(k) ?? []), r]);694 }695 return [...by.keys()]696 .sort((a, b) => Number(a === 'other') - Number(b === 'other') || by.get(b)!.length - by.get(a)!.length || a.localeCompare(b))697 .map((k) => ({ id: k.replace(/[^a-z0-9]+/g, '-'), title: k === 'other' && by.size === 1 ? 'repositories' : k, repos: by.get(k)! }));698 },699 // An attestation as a page shows it: the runner's value left out, since a700 // machine name can say who someone is; the note in refs/notes/ci has the701 // statement as signed, for checking the signature.702 statement: (payload: string): string => payload.replace(/^runner .*$/m, 'runner (not shown here; refs/notes/ci has it)'),703 // A score with one decimal, 1.0 rather than 1.704 tenths: (n: number): string => Number(n).toFixed(1),705 short: (sha: string | null | undefined): string => String(sha ?? '').slice(0, 7),706 // A time in a sentence ("updated 5m ago"); date is one in a column ("5m").707 ago: html.ago,708 // A page's description, for link previews: one line of plain text, cut at709 // a word near n characters. With html, the text of rendered markdown710 // (markdown-it puts a newline between blocks, so tags can just go), less711 // its diagrams, whose labels are no sentence.712 excerpt(s: string | null | undefined, html = false, n = 200): string {713 let text = String(s ?? '');714 if (html) text = decode(text.replace(/<figure class="diagram">.*?<\/figure>/gs, '').replace(/<[^>]*>/g, ''));715 text = text.replace(/\s+/g, ' ').trim();716 if (text.length <= n) return text;717 const cut = text.lastIndexOf(' ', n);718 return text.slice(0, cut > n / 2 ? cut : n) + '…';719 },720 plural: (n: number, word: string, many?: string): string => `${n} ${n === 1 ? word : many ?? word + 's'}`,721 // 1,284.722 thousands: (n: number): string => Number(n).toLocaleString('en-US'),723 // A made-up year for the style guide's activity grid: busier on724 // weekdays and lately, the same every build.725 demoActivity(noun: string): Grid {726 const counts: Record<string, number> = {};727 const end = Number(process.env.FORGE_NOW) * 1000 || Date.now();728 let h = 7;729 for (let i = 0; i < 371; i++) {730 h = (h * 1103515245 + 12345) >>> 0;731 const t = new Date(end - i * 86400000);732 const weekday = t.getUTCDay() % 6 ? 1 : 0.3;733 const n = Math.floor(((h >>> 16) % 100) / 100 * 9 * weekday * (1 - i / 500)) - 2;734 if (n > 0) counts[t.toISOString().slice(0, 10)] = n;735 }736 return grid(counts, noun);737 },738 // A person's avatar (lib/avatar.ts), size pixels square.739 avatar: (name: string, size = 36): string => avatar(name, Number(size)),740 // Data for a page's script, in a <script type="application/json">.741 json: (v: unknown): string => JSON.stringify(v ?? null).replace(/</g, '\\u003c'),742 indent: (depth: number): string => ' '.repeat(depth),743 mode: (m: string): string => MODES[m] ?? m,744 size(n: number | null): string {745 if (n == null) return '';746 if (n < 1024) return `${n}B`;747 if (n < 1 << 20) return `${(n / 1024).toFixed(1)}K`;748 return `${(n / (1 << 20)).toFixed(1)}M`;749 },750 // The header block of a commit page.751 commitHeader(c: Commit, repo: RepoHead): string {752 const e = html.esc;753 const lines = [`<b>commit</b> ${commitLink(c.sha, repo, c.sha)}`];754 for (const p of c.parents) lines.push(`<b>parent</b> ${commitLink(p, repo, p)}`);755 lines.push(`<b>Author:</b> ${e(c.author.name)} <<a href="mailto:${e(c.author.email)}">${e(c.author.email)}</a>>`);756 lines.push(`<b>Date:</b> ${e(c.author.date)}`);757 if (c.ci) {758 const via = c.ci.latest.via ? ` <span class="desc">(same tree as ${commitLink(c.ci.latest.via, repo)})</span>` : '';759 lines.push(`<b>CI:</b> ${html.ci(c.ci, { long: true })}${via}`);760 }761 for (const r of c.reviews) lines.push(`<b>Review:</b> <a href="${repo.urls.review(r.id).href}">#${r.id} ${e(r.title)}</a>`);762 lines.push('', e(c.subject));763 if (c.body) lines.push('', e(c.body));764 return `<pre class="meta">${lines.join('\n')}</pre>`;765 },766 commitLink,767};