1// Jev on commits and repos: what kind of change each commit is, release2// notes between tags, and topics for the owner page.3//4// fill() runs after a push (lib/jev-fill.ts) and asks; the rest reads stored5// answers for the build and never calls the API.6//7// commit.v1/<patch id>-<message hash>: one request per commit, for its kind,8// risk, whether its message describes its diff, its release-notes section9// and whether users would notice it. A rebase keeps the patch id and so10// the answer; a reworded message asks again.11// repo.v1/<README blob>-<hash of top-level names and description>: what12// the repo is. The key covers exactly what the request shows, so a push13// that leaves those alone keeps the answer instead of asking again.14import fs from 'node:fs';15import path from 'node:path';16import { createHash, randomBytes } from 'node:crypto';17import { spawnSync } from 'node:child_process';18import type { SystemOneResult } from '@typesafe-ai/sdk';19import { run, tryRun, revList, catFileBatch, isBinary } from './git.ts';20import { parseDiff } from './diff.ts';21import { readReviewLogs, buildReviews } from './reviews.ts';22import { summarize } from './repo.ts';23import { RepoCache } from './cache.ts';24import { JevStore, remember, pool, diffState, noul, choice, score } from './jev.ts';25import type { FillContext } from './jev-fill.ts';26import type { CommitLabel, DiffFile, Repo, Tag } from './types.ts';2728const COMMIT = 'commit.v1';29const REPO = 'repo.v1';3031// What fill() asks about: the newest commits on the default branch, live32// reviews' commits and the newest tags' commits, at most ASK in all, so a big33// import can't ask thousands.34const RECENT = 50;35const ASK = 300;36// The releases page: the newest tags, each with its commits up to a cap.37const RELEASES = 20;38const PER_RELEASE = 200;39// Past this many changed lines a commit is sent as its file list alone.40const HUGE = 5000;41// Below this, the message may not describe the diff.42const MISMATCH = 0.3;43// Below this, the topic is too unsure to show.44const TOPIC_MIN = 0.4;45const README = /^readme(\.(md|markdown|txt))?$/i;46const README_CHARS = 4000;4748const KINDS = {49 feat: 'adds something new: a feature, option or capability',50 fix: 'corrects behavior that was wrong',51 refactor: 'restructures code without changing what it does',52 docs: 'documentation, comments or help text only',53 test: 'tests only',54 build: 'build, CI, packaging or developer tooling',55 deps: 'adds, upgrades or removes dependencies or vendored code',56 chore: 'other upkeep: formatting, renames, config, cleanup',57};58const RISK = [59 'none: cannot break anything, such as docs, comments or tests',60 'low: small and contained',61 'moderate: changes behavior that some uses may rely on',62 'high: broad or subtle, likely to break something',63] as const;64const SECTIONS = {65 added: 'something new users can do or see',66 changed: 'something users already had works differently',67 fixed: 'a bug users could hit is fixed',68 removed: 'something users had is gone',69 internal: 'nothing users would notice: refactors, tests, tooling, developer docs',70};71const TOPICS = {72 cli: 'a command-line tool or scripts people run in a terminal',73 library: 'code other programs import: a package, module or SDK',74 'web-app': 'an interactive application that runs in the browser',75 'static-site': 'a website, blog or docs site built to static pages',76 service: 'a server, API, daemon or bot that keeps running',77 game: 'a game',78 config: 'configuration or dotfiles: shell, editor or machine setup',79 docs: 'documentation, notes or writing rather than code',80 experiment: 'a sketch, prototype, exercise or learning project',81 other: 'none of these',82};83const TOPIC_NAMES: Record<string, string> = { 'web-app': 'web app', 'static-site': 'static site' };8485// The subject goes in each question, so Jev knows which change is meant.86const commitQuestions = (subject: string) => {87 const it = `the commit "${subject}"`;88 return {89 kind: choice(`What kind of change is ${it}?`, KINDS),90 risk: score(`How likely is ${it} to break something that worked before?`, RISK),91 matches: noul(`Does the message of ${it} describe what its diff does?`),92 section: choice(`Under which heading would ${it} go in release notes?`, SECTIONS),93 // "Notice" alone counts tests and CI as user-facing.94 user_facing: noul(`Would someone who uses this project, rather than someone who works on its code, notice ${it}?`),95 };96};97const topicQuestions = (slug: string) => ({ topic: choice(`What is the repository ${slug}?`, TOPICS) });98type CommitAnswers = SystemOneResult<ReturnType<typeof commitQuestions>>['answers'];99type TopicAnswers = SystemOneResult<ReturnType<typeof topicQuestions>>['answers'];100101const hash = (s: string, n: number): string => createHash('sha1').update(s).digest('hex').slice(0, n);102103interface Message {104 subject: string;105 body: string;106 lines: number;107}108109// Split as lib/repo.ts does, so message hashes agree with the build's.110function splitMessage(msg: string): { subject: string; body: string } {111 const message = msg.replace(/\n+$/, '');112 const nl = message.indexOf('\n');113 return { subject: nl < 0 ? message : message.slice(0, nl), body: nl < 0 ? '' : message.slice(nl + 1).replace(/^\n+/, '') };114}115116const nonce = randomBytes(8).toString('hex');117const START = `<<jev-commit-${nonce}>>`;118const FIELD = `<<jev-field-${nonce}>>`;119const END = `<<jev-end-${nonce}>>`;120121// Subject, body and changed lines of each commit, in one git call.122function messages(gitDir: string, shas: string[]): Map<string, Message> {123 const out = new Map<string, Message>();124 if (!shas.length) return out;125 const text = run(gitDir, ['log', '--no-walk=unsorted', '--stdin', '--no-color', '-M', '--diff-merges=first-parent', '--numstat', `--format=${START}%H${FIELD}%B${END}`], { input: shas.join('\n') + '\n' });126 for (const chunk of text.split(START)) {127 const end = chunk.indexOf(END);128 if (end < 0) continue;129 const [sha, msg = ''] = chunk.slice(0, end).split(FIELD);130 let lines = 0;131 for (const m of chunk.slice(end + END.length).matchAll(/^(\d+)\t(\d+)\t/gm)) lines += Number(m[1]) + Number(m[2]);132 out.set(sha, { ...splitMessage(msg), lines });133 }134 return out;135}136137// `git patch-id --stable` of each commit's diff against its first parent, as138// rebase and `git cherry` recognize a change. The patch streams from one git139// to the other without passing through here. A commit that changes nothing140// gets no id. Null when git failed.141export function patchIds(gitDir: string, shas: string[]): Map<string, string> | null {142 const ids = new Map<string, string>();143 if (!shas.length) return ids;144 const log = 'git -c core.quotepath=false --git-dir="$1" log --no-walk=unsorted --stdin --no-color --no-ext-diff --no-renames --diff-merges=first-parent --root -p --format="commit %H"';145 const r = spawnSync('sh', ['-c', `${log} | git patch-id --stable`, 'sh', gitDir], { input: shas.join('\n') + '\n', encoding: 'utf8', maxBuffer: 1 << 26 });146 if (r.status !== 0 || r.stderr) return null;147 for (const line of r.stdout.split('\n')) {148 const [id, sha] = line.split(' ');149 if (id && sha) ids.set(sha, id);150 }151 return ids;152}153154// A commit's key; "empty" stands in for the patch id of a commit that changes155// no lines. Null when it changes lines but has no patch id.156export function commitKey(id: string | undefined, subject: string, body: string, lines: number): string | null {157 const pid = id || (lines ? null : 'empty');158 return pid && `${pid}-${hash(`${subject}\n\n${body}`, 10)}`;159}160161// What a commit is sent as: its diff against its first parent, or for a huge162// one (vendored code, generated data) only the paths it touches.163function commitDiff(gitDir: string, sha: string, lines: number): DiffFile[] {164 if (lines <= HUGE) return parseDiff(tryRun(gitDir, ['show', '--no-color', '--no-ext-diff', '-M', '--diff-merges=first-parent', '--format=', sha]) ?? '');165 return (tryRun(gitDir, ['show', '--no-color', '-M', '--diff-merges=first-parent', '--format=', '--name-status', sha]) ?? '').split('\n').filter(Boolean).map((l): DiffFile => {166 const [st, ...paths] = l.split('\t');167 const p = paths.at(-1) ?? '';168 return { status: (st[0] as DiffFile['status']) ?? 'M', path: p, oldPath: paths[0] ?? p, newPath: p, binary: false, hunks: [], add: 0, del: 0, omitted: true };169 });170}171172interface TagRange {173 prev: string | null;174 shas: string[];175 more: boolean;176}177178// Each of the newest tags' commits since the tag before it (the nearest one179// it contains), or since the root. Merges are left out.180function tagRanges(gitDir: string, tags: { sha: string }[]): TagRange[] {181 return tags.slice(0, RELEASES).map((t) => {182 const prev = tryRun(gitDir, ['describe', '--tags', '--abbrev=0', `${t.sha}^`])?.trim() || null;183 const shas = revList(gitDir, ['--no-merges', `-n${PER_RELEASE + 1}`, t.sha, ...(prev ? ['--not', `refs/tags/${prev}`] : [])]);184 return { prev, shas: shas.slice(0, PER_RELEASE), more: shas.length > PER_RELEASE };185 });186}187188// Tags on commits, newest first, ordered as lib/repo.ts orders them.189function commitTags(gitDir: string): { sha: string; date: string }[] {190 return (tryRun(gitDir, ['for-each-ref', '--format=%(objectname)%00%(*objectname)%00%(objecttype)%00%(*objecttype)%00%(creatordate:iso-strict)', 'refs/tags']) ?? '')191 .split('\n').filter(Boolean).map((l) => l.split('\0'))192 .filter(([, , type, peeledType]) => (peeledType || type) === 'commit')193 .map(([sha, peeled, , , date]) => ({ sha: peeled || sha, date }))194 .sort((a, b) => b.date.localeCompare(a.date));195}196197// The commits of reviews whose branches are still pushed, layer by layer.198function reviewCommits(ctx: FillContext): string[] {199 const branches = new Map<string, string>();200 for (const l of (tryRun(ctx.gitDir, ['for-each-ref', '--format=%(refname) %(objectname)', 'refs/heads']) ?? '').split('\n')) {201 const sp = l.lastIndexOf(' ');202 if (sp > 0) branches.set(l.slice('refs/heads/'.length, sp), l.slice(sp + 1));203 }204 // Read-only: range-diffs the build already made come from its cache.205 const ranges = RepoCache.load(ctx.cfg, ctx.slug).ranges;206 const reviews = buildReviews(ctx.gitDir, { logs: readReviewLogs(ctx.gitDir), branches, defaultBranch: ctx.defaultBranch, ranges });207 return reviews.filter((r) => r.live).flatMap((r) => r.commits.slice(-RECENT));208}209210interface Entry {211 name: string;212 type: string;213 oid: string;214}215216const listing = (entries: Entry[]): string[] => entries.map((e) => (e.type === 'tree' ? `${e.name}/` : e.name)).sort();217const topicKey = (entries: Entry[], description: string): string =>218 `${entries.find((e) => e.type === 'blob' && README.test(e.name))?.oid ?? 'none'}-${hash(listing(entries).join('\n') + '\0' + description, 12)}`;219220async function fillTopic(ctx: FillContext): Promise<void> {221 const out = tryRun(ctx.gitDir, ['ls-tree', '-z', `refs/heads/${ctx.defaultBranch}`]);222 if (!out) return;223 const entries = out.split('\0').filter(Boolean).map((rec): Entry => {224 const tab = rec.indexOf('\t');225 const [, type, oid] = rec.slice(0, tab).split(' ');226 return { name: rec.slice(tab + 1), type, oid };227 });228 const { description } = summarize(ctx.cfg, ctx.owner, ctx.name);229 const key = topicKey(entries, description);230 if (ctx.store.has(REPO, key)) return;231 const readme = entries.find((e) => e.type === 'blob' && README.test(e.name));232 const data = readme && catFileBatch(ctx.gitDir, [readme.oid]).get(readme.oid)?.data;233 const text = data && !isBinary(data) ? data.toString('utf8').trim().slice(0, README_CHARS).trim() : '';234 await remember(ctx.store, REPO, key, { description, files: listing(entries), readme: text }, topicQuestions(ctx.slug));235}236237async function fillCommits(ctx: FillContext): Promise<void> {238 const { gitDir, store } = ctx;239 const picked = new Set<string>();240 const add = (shas: string[]): void => {241 for (const s of shas) if (picked.size < ASK) picked.add(s);242 };243 add(revList(gitDir, [`-n${RECENT}`, `refs/heads/${ctx.defaultBranch}`]));244 add(reviewCommits(ctx));245 for (const r of tagRanges(gitDir, commitTags(gitDir))) add(r.shas);246 const shas = [...picked];247 const meta = messages(gitDir, shas);248 const ids = patchIds(gitDir, shas);249 if (!ids) return;250 // One request per key: a cherry-pick or an unchanged rebase shares its answer.251 const todo = new Map<string, string>();252 for (const sha of shas) {253 const m = meta.get(sha);254 const key = m && commitKey(ids.get(sha), m.subject, m.body, m.lines);255 if (key && !todo.has(key) && !store.has(COMMIT, key)) todo.set(key, sha);256 }257 await pool([...todo], 4, async ([key, sha]) => {258 const m = meta.get(sha)!;259 const state = { subject: m.subject, body: m.body, files: diffState(commitDiff(gitDir, sha, m.lines)) };260 await remember(store, COMMIT, key, state, commitQuestions(m.subject));261 });262}263264export async function fill(ctx: FillContext): Promise<void> {265 await fillTopic(ctx);266 await fillCommits(ctx);267}268269// The build's side: stored answers only.270271function label(a: CommitAnswers): CommitLabel | null {272 const kind = a.kind?.choice;273 if (!kind || !Object.hasOwn(KINDS, kind)) return null;274 // The expected level on RISK's 0 to 3 scale; pages show moderate and up.275 const risk = a.risk?.score ?? 0;276 const section = a.section?.choice;277 return {278 kind,279 risk: risk >= 2.5 ? 'high' : risk >= 1.5 ? 'moderate' : null,280 riskScore: a.risk?.score == null ? null : Math.round(a.risk.score * 10) / 10,281 mismatch: (a.matches?.noul ?? 1) < MISMATCH,282 section: section && Object.hasOwn(SECTIONS, section) ? section : null,283 userFacing: (a.user_facing?.noul ?? 0) >= 0.5,284 };285}286287interface Labelled {288 sha: string;289 subject: string;290 body: string;291 stats: { add: number; del: number };292 jev?: CommitLabel | null;293}294295// Set each commit's label from the stored answers. Only commits whose message296// hash matches a stored key need a patch id, and those are cached by sha.297export function labelCommits(gitDir: string, commits: Iterable<Labelled>, cache: Record<string, unknown>): void {298 const store = new JevStore(gitDir);299 let keys: string[];300 try {301 keys = fs.readdirSync(path.join(store.dir, COMMIT));302 } catch {303 return;304 }305 const hashes = new Set(keys.map((k) => k.slice(k.indexOf('-') + 1)));306 const maybe = [...commits].filter((c) => hashes.has(hash(`${c.subject}\n\n${c.body}`, 10)));307 const todo = maybe.filter((c) => typeof cache[`patchid ${c.sha}`] !== 'string');308 const ids = patchIds(gitDir, todo.map((c) => c.sha));309 if (ids) for (const c of todo) cache[`patchid ${c.sha}`] = ids.get(c.sha) ?? '';310 for (const c of maybe) {311 const id = cache[`patchid ${c.sha}`];312 const key = typeof id === 'string' ? commitKey(id, c.subject, c.body, c.stats.add + c.stats.del) : null;313 const stored = key ? store.get<CommitAnswers>(COMMIT, key) : null;314 c.jev = stored ? label(stored.answers) : null;315 }316}317318// What the pages show of a label, for page fingerprints.319export const labelKey = (c: { jev?: CommitLabel | null }): string => (c.jev ? `${c.jev.kind}:${c.jev.riskScore ?? ''}:${c.jev.mismatch ? 1 : 0}` : '');320321// What the repo is, if Jev was sure enough and it is one of the named kinds.322export function repoTopic(repo: Repo): string | null {323 const entries = repo.trees.get(repo.defaultBranch)?.dirs.get('');324 if (!entries) return null;325 const a = new JevStore(repo.dir).get<TopicAnswers>(REPO, topicKey(entries, repo.description))?.answers.topic;326 if (!a || a.choice === 'other' || !Object.hasOwn(TOPICS, a.choice) || (a.probabilities[a.choice] ?? 0) < TOPIC_MIN) return null;327 return TOPIC_NAMES[a.choice] ?? a.choice;328}329330export interface ReleaseItem {331 sha: string;332 subject: string;333 userFacing: boolean;334}335336export interface ReleaseGroup {337 section: string;338 title: string;339 folded: boolean;340 items: ReleaseItem[];341}342343export interface Release {344 tag: Tag;345 prev: string | null;346 count: number;347 more: boolean;348 groups: ReleaseGroup[];349}350351// Release notes: the order sections show in. Commits without an answer go352// under "other", so the page works without Jev; internal ones fold.353const GROUPS = [['added', 'added'], ['changed', 'changed'], ['fixed', 'fixed'], ['removed', 'removed'], ['other', 'other'], ['internal', 'internal']] as const;354355// Each tag's commits since the tag before it, sorted into release-note356// sections by the stored answers. The wording is the commit subjects.357export function releases(repo: Repo, cache: Record<string, unknown>): Release[] {358 const tags = repo.tags.filter((t) => t.type === 'commit');359 const ranges = tagRanges(repo.dir, tags);360 // Commits too old for a page of their own still need a subject.361 const missing = [...new Set(ranges.flatMap((r) => r.shas))].filter((s) => !repo.commits.has(s));362 const extra: Labelled[] = [...messages(repo.dir, missing)].map(([sha, m]) => ({ sha, subject: m.subject, body: m.body, stats: { add: m.lines, del: 0 } }));363 labelCommits(repo.dir, extra, cache);364 const bySha = new Map<string, Labelled>([...repo.commits, ...extra.map((c): [string, Labelled] => [c.sha, c])]);365 return ranges.map((r, i): Release => {366 const groups: ReleaseGroup[] = GROUPS.map(([section, title]) => ({ section, title, folded: section === 'internal', items: [] }));367 for (const sha of r.shas) {368 const c = bySha.get(sha);369 if (c) groups.find((g) => g.section === (c.jev?.section ?? 'other'))!.items.push({ sha, subject: c.subject, userFacing: !!c.jev?.userFacing });370 }371 // User-facing changes first; otherwise newest first, as git lists them.372 for (const g of groups) g.items.sort((a, b) => Number(b.userFacing) - Number(a.userFacing));373 return { tag: tags[i], prev: r.prev, count: r.shas.length, more: r.more, groups: groups.filter((g) => g.items.length) };374 });375}