metrics.jsonl isn't backed up, so a restored forge starts its build history over. With a lake, forge metrics also reads the lake's copy of each build line and merges it with the file by the line's hash: while the file has every line the output is the same as before, and once it has lost some, the older builds come from the lake, with a line saying how many. Without a lake, or if it can't be read, it reads the file as it did.
- commit
- 597e742510cfea75fe38d75fe81501e0dfd07223
- parent
- dc590460b92a9f89fe5b7a25eb51219ed8e87653
- author
- cat <admin@dev.site>
- kind
- feat risk score 1.6 of 3, Jev's estimate
| M | bin/ | 2 | ++ |
| M | lib/ | 2 | +- |
| M | lib/ | 27 | ++++++++++++++++++++++++--- |
| A | test/ | 28 | ++++++++++++++++++++++++++++ |
@@ -413,7 +413,7 @@ export async function buildLines(cfg: LakeConfig): Promise<{ id: string; line: s
413413 if (!fs.existsSync(cfg.catalog)) return null;414414 const lake = await Lake.open(cfg, { write: false });415415 try {416- return (await lake.rows(`SELECT id, line FROM lake.builds ORDER BY at, id`)).map(([id, line]) => ({ id: String(id), line: String(line) }));416+ return (await lake.rows(`SELECT id, line FROM lake.builds ORDER BY "at", id`)).map(([id, line]) => ({ id: String(id), line: String(line) }));417417 } finally {418418 lake.close();419419 }@@ -1,5 +1,8 @@
11 // `forge metrics [n] [repo]`: the last n builds, and how each code version2-// performs, from $FORGE_STATE/metrics.jsonl.2+// performs, from $FORGE_STATE/metrics.jsonl. With a lake (lib/lake.ts), its3+// copy of every build line is read too, so the history goes back past what4+// metrics.jsonl still has (backups leave that file out); lines it hasn't5+// taken in yet come from the file.36 import fs from 'node:fs';47 import path from 'node:path';58 import type { MetricsLine } from './metrics.ts';@@ -9,12 +12,30 @@ const file = path.join(state, 'metrics.jsonl');
912 const n = Number(process.argv[2]) || 15;1013 const only = process.argv[3] || null;1114 12-if (!fs.existsSync(file)) {15+const lines = fs.existsSync(file) ? fs.readFileSync(file, 'utf8').split('\n').filter(Boolean) : [];16+let older = 0;17+if (fs.existsSync(path.join(state, 'lake', 'catalog.sqlite'))) {18+ try {19+ // Loaded only with a lake: DuckDB takes a moment to start.20+ const { buildLines, lakeConfig } = await import('./lake.ts');21+ const { lineHash } = await import('./lake-sources.ts');22+ const have = new Set(lines.map(lineHash));23+ const lake = (await buildLines(lakeConfig({ ...process.env, FORGE_ROOT: path.dirname(state) }))) ?? [];24+ const extra = lake.filter((l) => !have.has(l.id)).map((l) => l.line);25+ older = extra.length;26+ lines.unshift(...extra);27+ } catch (e) {28+ console.error(`forge metrics: the lake can't be read, so this is metrics.jsonl only (${String((e as Error).message ?? e).split('\n')[0]})`);29+ }30+}31+if (!lines.length) {1332 console.log(`no builds recorded yet (${file})`);1433 process.exit(0);1534 }16-let rows = fs.readFileSync(file, 'utf8').split('\n').filter(Boolean).map((l) => JSON.parse(l) as MetricsLine);35+let rows = lines.map((l) => JSON.parse(l) as MetricsLine);36+if (older) rows.sort((a, b) => a.at.localeCompare(b.at));1737 if (only) rows = rows.filter((r) => r.only === only || r.repos.some((x) => x.slug === only));38+if (older) console.log(`${older} older ${older === 1 ? 'build' : 'builds'} from the lake, ${rows.length} in all\n`);1839 1940 const pad = (s: unknown, w: number): string => String(s).padEnd(w);2041 const num = (s: unknown, w: number): string => String(s).padStart(w);@@ -0,0 +1,28 @@
1+# forge metrics reads the lake's copy of every build too: the same output2+# while metrics.jsonl has it all, and the older builds once the file has3+# lost them (backups leave it out). Without a lake it reads the file alone.4+. "$(dirname "$0")/../lib.sh"5+user dami6+repo dami/greet7+clone dami dami/greet "$T/w/d"8+at "$T/w/d" commit "first"9+push_as dami "$T/w/d" origin main10+build dami/greet11+build dami/greet12+ok forge metrics13+cp "$T_LOG" "$T/before"14+has "$T/before" '^20[0-9-]+ [0-9:]+ +dami/greet +manual'15+eq "$(grep -c ' dami/greet *manual' "$T/before")" 216+ok forge lake17+ok forge metrics18+cp "$T_LOG" "$T/after"19+ok cmp "$T/before" "$T/after"20+# The file is gone (a restored forge starts without it); a new build21+# writes a new one, which the lake hasn't read yet.22+rm "$FORGE_ROOT/.forge/metrics.jsonl"23+build dami/greet24+ok forge metrics25+has "$T_LOG" '^2 older builds from the lake, 3 in all$'26+eq "$(grep -c ' dami/greet *manual' "$T_LOG")" 327+ok forge metrics 15 dami/greet28+eq "$(grep -c ' dami/greet *manual' "$T_LOG")" 3