After each run, forge lake exports small aggregates for the forge and for every repo whose facts changed: insights.json for the charts, and the rows behind them as Parquet (events, builds, reviews, CI). They go to .forge/lake/export and straight into the live pages, so the pages need no rebuild; a build copies a repo's exports into that repo's page directory, which keeps them through the swap and would take them along to wherever private pages end up being served. /insights/ and /<owner>/<repo>/insights/ (an Insights tab, key i) are static pages whose script fetches insights.json and draws stat tiles and six SVG charts: build time, pushes, time to land, revisions per review, CI results and CI duration. Colors are the data-viz palette's, checked for color-blind readers on both themes (passed blue, failed red, not green and red); each chart has a tooltip on hover and keyboard focus and a table of the same numbers. The script loads on insights pages only. DuckDB-Wasm is left out: its build is 34 MiB (7.7 MiB gzipped) for exports of a few kilobytes, so the page shows the DuckDB command that reads the Parquet files over HTTP instead. The forge's exports cover public repos only. Going private removes the forge's from the pages at once and the next run writes them without the repo; its own exports stay in the lake's state. "insights" is reserved as an owner name, and full builds keep www/insights. The demo runs forge lake last, and CI checks its exports.
- commit
- ee68744ca6d346e384a1427a2ceadaae8abaa74e
- parent
- 76f9dccc207a5bc577a98a4e633c811925b5d2a3
- author
- cat <admin@dev.site>
- kind
- feat risk score 1.8 of 3, Jev's estimate
| M | .forge/ | 6 | + |
| M | bin/ | 23 | +- |
| M | demo/ | 5 | + |
| M | demo/ | 2 | +- |
| M | lib/ | 29 | ++- |
| A | lib/ | 184 | ++++++++++++ |
| M | lib/ | 9 | +- |
| M | lib/ | 5 | +- |
| M | lib/ | 2 | + |
| M | lib/ | 1 | + |
| M | site/ | 3 | +- |
| A | site/ | 95 | ++++++ |
| A | site/ | 476 | ++++++++++++++++++++++++++++++ |
| M | site/ | 4 | +- |
| M | site/ | 1 | + |
| A | site/ | 15 | + |
| A | site/ | 11 | + |
| A | test/ | 50 | +++ |
| A | test/ | 51 | +++ |
@@ -18,6 +18,12 @@ grep -q '<code class="language-go">resp, err := http.Get(<span class="hljs-strin
1818 grep -q '"reviews":\[\[7,"Regional notes"' $www/dami/greet/search.json1919 grep -q '"dami/greet","say hello, with flags"' $www/repos.json2020 grep -q 'data-search="/dami/greet/search.json"' $www/dami/greet/index.html21+# Insights: the demo runs forge lake last, which puts its exports next to22+# the insights pages; the forge's leave out nothing public.23+grep -q '"scope":"dami/greet".*"landed":1,"closed":1,' $www/dami/greet/insights/insights.json24+grep -q '"ci":17,"ci_pass":16,' $www/dami/greet/insights/insights.json25+grep -q '"repos":\[\["ana/dotfiles",' $www/insights/insights.json26+for f in events builds reviews ci; do test "$(head -c 4 $www/insights/$f.parquet)" = PAR1; done2127 # Templates render the same without 11ty (lib/render-page.ts), with the2228 # demo build's settings; every asset a page links exists under its hashed2329 # name; link previews.@@ -27,7 +27,7 @@
2727 # forge build [owner/repo]2828 # forge jev [owner/repo] ask Jev about new commits and reviews (needs TYPESAFE_API_KEY)2929 # forge metrics [n] [owner/repo] recent builds and where their time went30-# forge lake [--dry-run] add new events, builds, reviews and CI results to the lake30+# forge lake [--dry-run] add new events, builds, reviews and CI results to the lake; export insights3131 # forge lake query "<sql>" [--json|--csv] read-only SQL over the lake3232 # forge gc [--dry-run] [--age minutes] remove unused site and release files, hostnames; repack, commit-graph3333 # forge backup [--dry-run] [--force] snapshot the forge with restic, then check it and test-restore it@@ -962,7 +962,7 @@ load_config() {
962962 963963 valid_name() {964964 case $1 in965- '' | .* | *[!A-Za-z0-9._-]* | *.git | assets | sites | previews | hosts | git-shell-commands) return 1 ;;965+ '' | .* | *[!A-Za-z0-9._-]* | *.git | assets | sites | previews | hosts | insights | git-shell-commands) return 1 ;;966966 esac967967 }968968 @@ -1631,6 +1631,9 @@ sc_private() {
16311631 done16321632 rm -rf "${FORGE_WWW:?}/sites/$1" "${FORGE_WWW:?}/previews/$1"16331633 fi1634+ # The forge's insights count public repos only; they are written again1635+ # without this one after the build.1636+ rm -f "$FORGE_WWW"/insights/*.json "$FORGE_WWW"/insights/*.parquet16341637 rebuild - private16351638 mirror_kick16361639 echo "$1 is private: only ${1%%/*}, its collaborators and its readers (readers $1 add <user>) can clone it."@@ -2238,7 +2241,7 @@ build_all() {
22382241 [ -d "$w" ] || continue22392242 slug=${w#"$FORGE_WWW"/}22402243 slug=${slug%/}2241- case $slug in assets/* | sites/* | previews/* | hosts/*) continue ;; esac2244+ case $slug in assets/* | sites/* | previews/* | hosts/* | insights/*) continue ;; esac22422245 [ -f "$FORGE_ROOT/$slug.git/HEAD" ] && [ -f "$FORGE_ROOT/$slug.git/git-daemon-export-ok" ] || rm -rf "$w"22432246 done22442247 for w in "$FORGE_WWW"/sites/*/*/ "$FORGE_WWW"/previews/*/*/; do@@ -2249,7 +2252,7 @@ build_all() {
22492252 done22502253 for w in "$FORGE_WWW"/*/; do22512254 b=$(basename "$w")2252- case $b in assets | sites | previews | hosts) continue ;; esac2255+ case $b in assets | sites | previews | hosts | insights) continue ;; esac22532256 [ -d "$FORGE_ROOT/$b" ] || rm -rf "$w"22542257 done22552258 if [ -d "$FORGE_WWW/hosts" ]; then node "$FORGE_HOME/lib/hosts.ts" prune; fi@@ -2327,11 +2330,11 @@ cmd_metrics() {
23272330 # ---------------------------------------------------------------- the lake23282331 #23292332 # forge lake adds the forge's facts (events, builds, review logs, CI notes)2330-# to a DuckLake in $FORGE_STATE/lake (lib/lake.ts). One writer at a time,2331-# under lake.lock; a second run waits for it. Builds start it the way they2332-# start the mirror: each stamps lake.want and waits FORGE_LAKE_DELAY seconds2333-# (10), and only the newest stamp runs, so a burst of pushes makes one run.2334-# FORGE_LAKE=0 turns that off.2333+# to a DuckLake in $FORGE_STATE/lake and writes the insights exports2334+# (lib/lake.ts). One writer at a time, under lake.lock; a second run waits2335+# for it. Builds start it the way they start the mirror: each stamps2336+# lake.want and waits FORGE_LAKE_DELAY seconds (10), and only the newest2337+# stamp runs, so a burst of pushes makes one run. FORGE_LAKE=0 turns that off.23352338 23362339 lake_kick() {23372340 [ "${FORGE_LAKE:-1}" != 0 ] || return 0@@ -2371,6 +2374,8 @@ cmd_lake() {
23712374 trap 'rm -rf "$FORGE_STATE/lake.lock"' EXIT23722375 trim_log "$FORGE_STATE/lake.log"23732376 node "$FORGE_HOME/lib/lake.ts" ingest $dry2377+ # The exports are in the pages now; the mirror picks them up.2378+ [ -n "$dry" ] || mirror_kick23742379 }23752380 23762381 # Was branch $2 ever a review in repo $1? Then its previews are leftovers.@@ -13,6 +13,8 @@ export FORGE_ROOT=$here/root
1313 export FORGE_WWW=$FORGE_ROOT/.forge/www1414 export FORGE_SSH=git@forge.local1515 export FORGE_NO_BUILD=116+# The lake runs once, at the end, not in the background after each build.17+export FORGE_LAKE=01618 export GIT_CONFIG_GLOBAL=$work/gitconfig1719 export GIT_CONFIG_NOSYSTEM=11820 @@ -515,4 +517,7 @@ git push -q origin main
515517 step "build"516518 unset FORGE_USER517519 "$forge" build520+521+step "lake: the facts behind the insights pages"522+"$forge" lake518523 printf '\nserve it: python3 -m http.server 8080 -d %s\n' "$FORGE_WWW"@@ -19,7 +19,7 @@ serveEvents(state, eventsPort);
1919 const TYPES: Record<string, string> = {2020 '.html': 'text/html; charset=utf-8', '.css': 'text/css', '.js': 'text/javascript', '.svg': 'image/svg+xml',2121 '.png': 'image/png', '.gif': 'image/gif', '.txt': 'text/plain; charset=utf-8', '.xml': 'application/xml', '.json': 'application/json',22- '.wasm': 'application/wasm', '.db': 'application/vnd.sqlite3',22+ '.wasm': 'application/wasm', '.db': 'application/vnd.sqlite3', '.parquet': 'application/vnd.apache.parquet',2323 };2424 2525 // One byte range, as code search reads code.db: [start, end] inclusive, or@@ -52,7 +52,7 @@ export function config(env: NodeJS.ProcessEnv = process.env): Config {
5252 const caseless = (a: string, b: string): number => a.localeCompare(b, 'en', { sensitivity: 'base' }) || (a < b ? -1 : a > b ? 1 : 0);5353 5454 // Top-level names used by the forge itself; never owners.55-const RESERVED = new Set(['assets', 'sites', 'previews', 'hosts', 'index.html', 'git-shell-commands', 'lost+found']);55+const RESERVED = new Set(['assets', 'sites', 'previews', 'hosts', 'insights', 'index.html', 'git-shell-commands', 'lost+found']);5656 5757 // Every repo, public or not (RepoSummary.private); builds filter.5858 export function discover(cfg: Config): { repos: RepoSummary[]; owners: string[] } {@@ -155,7 +155,7 @@ export function loadForge(env: NodeJS.ProcessEnv = process.env): Forge {
155155 try {156156 cached = JSON.parse(fs.readFileSync(summaryCache, 'utf8'));157157 } catch {}158- const pages: Record<string, Page[]> = { homes: [], logs: [], trees: [], blobs: [], commits: [], refs: [], reviewLists: [], reviews: [], reviewDiffs: [], revisions: [], sites: [], wiki: [], atom: [], branchMenus: [], releases: [] };158+ const pages: Record<string, Page[]> = { homes: [], logs: [], trees: [], blobs: [], commits: [], refs: [], reviewLists: [], reviews: [], reviewDiffs: [], revisions: [], sites: [], wiki: [], atom: [], branchMenus: [], releases: [], insights: [] };159159 const raw: RawFile[] = [];160160 // Incremental builds: when building next to a live site, a page whose161161 // fingerprint matches the last build (and whose file still exists) isn't@@ -339,6 +339,11 @@ export function loadForge(env: NodeJS.ProcessEnv = process.env): Forge {
339339 const list = JSON.stringify(reviewList(repo.reviews, (id) => cfg.baseUrl + u.review(id).href), null, 1) + '\n';340340 if (want(`${u.home.path}reviews.json`, `reviews.json ${fingerprint(list)}`)) raw.push({ path: `${u.home.path}reviews.json`, data: Buffer.from(list) });341341 add(pages.sites, page(u.sites.path, `Sites - ${repo.slug}`, 'sites', { sites: repo.sites, previews: repo.previews }));342+ // The page is the same until the code changes; its charts fetch the343+ // lake's exports, which go next to it (and which forge lake replaces in344+ // the live pages after each run).345+ add(pages.insights, page(u.insights.path, `Insights - ${repo.slug}`, 'insights', {}), 'insights');346+ raw.push(...insightFiles(cfg, path.join(repo.owner, repo.name, 'insights'), u.insights.path));342347 for (const w of repo.wiki) {343348 add(pages.wiki, page(u.wiki(w.slug).path, `${w.title} - wiki - ${repo.slug}`, 'wiki', { wiki: w, pages: repo.wiki.map((x) => ({ title: x.title, slug: x.slug, href: u.wiki(x.slug).href })) }));344349 }@@ -371,10 +376,30 @@ export function loadForge(env: NodeJS.ProcessEnv = process.env): Forge {
371376 const owners = ownerNames.map((owner) => ({ owner, repos: summaries.filter((s) => s.owner === owner) }));372377 // Repo search on the index and owner pages.373378 raw.push({ path: '/repos.json', data: Buffer.from(reposIndex(summaries)) });379+ if (!cfg.only || cfg.only === '-') raw.push(...insightFiles(cfg, 'insights', '/insights/'));374380 last = { config: cfg, repos: summaries, owners, pages, raw: raw.filter((r) => r.data || r.file), live, code, skipped, pageCaches };375381 return last;376382 }377383 384+// The lake's insights exports for a scope (lib/insights.ts), read now: forge385+// lake may swap the directory while the build runs.386+function insightFiles(cfg: Config, dir: string, at: string): RawFile[] {387+ const from = path.join(cfg.state, 'lake', 'export', dir);388+ const out: RawFile[] = [];389+ let names: string[] = [];390+ try {391+ names = fs.readdirSync(from).filter((f) => /^[a-z]+\.(json|parquet)$/.test(f));392+ } catch {393+ return out;394+ }395+ for (const f of names) {396+ try {397+ out.push({ path: at + f, data: fs.readFileSync(path.join(from, f)) });398+ } catch {}399+ }400+ return out;401+}402+378403 interface BlobPage {379404 repo: RepoHead;380405 oid: string;@@ -0,0 +1,184 @@
1+// Insights exports: small aggregates of the lake (lib/lake.ts) for the2+// insights pages, as JSON the charts draw from (site/assets/insights.ts),3+// plus the rows behind them as Parquet to explore with DuckDB.4+//5+// Every repo's go to $FORGE_STATE/lake/export/<owner>/<repo>/insights/, the6+// forge's to .../export/insights/. The build copies a repo's into that repo's7+// pages (lib/forge.ts), so they follow its page directory wherever its pages8+// are served; publishInsights also puts them straight into the live pages9+// after each run, since the pages fetch them and need no rebuild. The forge's10+// cover public repos only: a private repo's facts are in no public export.11+import fs from 'node:fs';12+import path from 'node:path';13+import { listRepos } from './lake-sources.ts';14+import { lit, type Lake, type LakeConfig } from './lake.ts';15+16+export const FILES = ['insights.json', 'events.parquet', 'builds.parquet', 'reviews.parquet', 'ci.parquet'];17+// The most points a chart gets; older ones are only in the Parquet files.18+const CAP = 1000;19+20+const inList = (col: string, xs: string[]): string => (xs.length ? `${col} IN (${xs.map(lit).join(', ')})` : 'false');21+const n = (v: unknown): number | null => (v == null ? null : Number(v));22+23+interface Scope {24+ // A repo's slug, or null for the whole forge.25+ slug: string | null;26+ dir: string;27+ // Rows of events, reviews and CI: on repo.28+ where: string;29+ // Rows of builds: on scope.30+ builds: string;31+}32+33+async function exportScope(lake: Lake, s: Scope, publicRepos: string[]): Promise<void> {34+ const tmp = `${s.dir}.tmp`;35+ fs.rmSync(tmp, { recursive: true, force: true });36+ fs.mkdirSync(tmp, { recursive: true });37+ const rows = (sql: string) => lake.rows(sql);38+ const one = async (sql: string): Promise<unknown[]> => (await rows(sql))[0] ?? [];39+40+ const builds = (await rows(`SELECT epoch_ms("at"), ms_total FROM lake.builds WHERE ${s.builds} AND "at" IS NOT NULL AND ms_total IS NOT NULL ORDER BY "at" DESC LIMIT ${CAP}`)).reverse();41+ const pushes = await rows(`SELECT strftime(t, '%Y-%m-%d'), count(*) FROM lake.events WHERE type = 'push' AND t IS NOT NULL AND ${s.where} GROUP BY 1 ORDER BY 1`);42+ const reviews = (await rows(`43+ WITH r AS (44+ SELECT repo, review, min(ts) FILTER (kind = 'created') AS created, count(*) FILTER (kind = 'rev') AS revisions,45+ count(DISTINCT "user") FILTER (kind = 'approve') AS approvals46+ FROM lake.reviews WHERE ${s.where} GROUP BY repo, review)47+ SELECT r.repo, r.review, epoch_ms(r.created), epoch_ms(e.ts), e.kind, r.revisions, r.approvals48+ FROM r LEFT JOIN lake.review_ends e USING (repo, review)49+ ORDER BY coalesce(e.ts, r.created) DESC NULLS LAST LIMIT ${CAP}`)).reverse();50+ const ci = (await rows(`SELECT epoch_ms(started), duration_s, result = 'pass' FROM lake.ci WHERE ${s.where} AND started IS NOT NULL ORDER BY started DESC LIMIT ${CAP}`)).reverse();51+ const [pushN, firstT, lastT] = await one(`SELECT count(*) FILTER (type = 'push'), epoch_ms(min(t)), epoch_ms(max(t)) FROM lake.events WHERE ${s.where}`);52+ const [buildN, buildMed] = await one(`SELECT count(*), median(ms_total) FROM lake.builds WHERE ${s.builds}`);53+ const [landed, closed, landH, revMed] = await one(`54+ WITH r AS (SELECT repo, review, min(ts) FILTER (kind = 'created') AS created, count(*) FILTER (kind = 'rev') AS revisions FROM lake.reviews WHERE ${s.where} GROUP BY ALL)55+ SELECT count(*) FILTER (e.kind = 'landed'), count(*) FILTER (e.kind = 'closed'),56+ median(epoch(e.ts) - epoch(r.created)) FILTER (e.kind = 'landed') / 3600, median(r.revisions)57+ FROM r LEFT JOIN lake.review_ends e USING (repo, review)`);58+ const [reviewN] = await one(`SELECT count(DISTINCT (repo, review)) FROM lake.reviews WHERE ${s.where}`);59+ const [ciN, ciPass, ciMed, ciVerified] = await one(`SELECT count(*), count(*) FILTER (result = 'pass'), median(duration_s), count(*) FILTER (verified) FROM lake.ci WHERE ${s.where}`);60+61+ const out: Record<string, unknown> = {62+ scope: s.slug ?? 'forge',63+ generated: new Date().toISOString(),64+ from: n(firstT), to: n(lastT),65+ totals: {66+ pushes: n(pushN), builds: n(buildN), build_ms_median: n(buildMed), reviews: n(reviewN), landed: n(landed), closed: n(closed),67+ land_hours_median: n(landH), revisions_median: n(revMed), ci: n(ciN), ci_pass: n(ciPass), ci_verified: n(ciVerified), ci_seconds_median: n(ciMed),68+ },69+ // [time (ms), total (ms)] for each build; [day, pushes]; [repo, id,70+ // created, ended, how, revisions, approvals]; [started, seconds, passed].71+ builds: builds.map((r) => r.map(n)),72+ pushes: pushes.map(([d, c]) => [d, n(c)]),73+ reviews: reviews.map(([repo, id, c, e, how, revs, appr]) => [repo, n(id), n(c), n(e), how ?? null, n(revs), n(appr)]),74+ ci: ci.map(([t, d, p]) => [n(t), n(d), p ? 1 : 0]),75+ };76+ if (!s.slug) {77+ const per = await rows(`78+ WITH p AS (SELECT repo, count(*) FILTER (type = 'push') AS pushes, max(t) FILTER (type = 'push') AS last FROM lake.events WHERE ${s.where} GROUP BY repo),79+ b AS (SELECT scope AS repo, count(*) AS builds, median(ms_total) AS build_ms FROM lake.builds WHERE ${inList('scope', publicRepos)} GROUP BY scope),80+ e AS (SELECT repo, count(*) FILTER (kind = 'landed') AS landed FROM lake.review_ends WHERE ${s.where} GROUP BY repo),81+ c AS (SELECT repo, count(*) AS ci, count(*) FILTER (result = 'pass') AS pass FROM lake.ci WHERE ${s.where} GROUP BY repo)82+ SELECT repo, coalesce(p.pushes, 0), coalesce(b.builds, 0), b.build_ms, coalesce(e.landed, 0), coalesce(c.ci, 0), coalesce(c.pass, 0), epoch_ms(p.last)83+ FROM (SELECT unnest([${publicRepos.map(lit).join(', ')}]::VARCHAR[]) AS repo) LEFT JOIN p USING (repo) LEFT JOIN b USING (repo) LEFT JOIN e USING (repo) LEFT JOIN c USING (repo)84+ ORDER BY lower(repo)`);85+ // [repo, pushes, builds, median build (ms), landed, CI runs, passed, last event]86+ out.repos = per.map(([r, ...v]) => [r, ...v.map(n)]);87+ }88+89+ const copy = async (file: string, sql: string): Promise<void> => {90+ await lake.run(`COPY (${sql}) TO ${lit(path.join(tmp, file))} (FORMAT parquet, COMPRESSION zstd)`);91+ };92+ await copy('events.parquet', `SELECT t, repo, type, "user", review, refs, ms, rendered FROM lake.events WHERE ${s.where} ORDER BY t`);93+ await copy('builds.parquet', `SELECT id, "at", code, reason, scope, ms_total, ms_data, ms_render, rss_mb, written, linked FROM lake.builds WHERE ${s.builds} ORDER BY "at"`);94+ await copy('reviews.parquet', `95+ SELECT repo, review, kind, ts, "user", rev, sha, base, tree, branch, title, NULL::BOOLEAN AS observed FROM lake.reviews WHERE ${s.where}96+ UNION ALL SELECT repo, review, kind, ts, "user", NULL, sha, NULL, NULL, NULL, NULL, observed FROM lake.review_ends WHERE ${s.where}97+ ORDER BY repo, review, ts`);98+ await copy('ci.parquet', `SELECT repo, "commit", tree, result, exit, started, duration_s, runner, system, env, signer, verified FROM lake.ci WHERE ${s.where} ORDER BY started`);99+ out.files = FILES.slice(1).map((f) => [f, fs.statSync(path.join(tmp, f)).size]);100+ fs.writeFileSync(path.join(tmp, 'insights.json'), JSON.stringify(out) + '\n');101+102+ // Swapped in whole; a build reading it meanwhile (lib/forge.ts) skips what103+ // it doesn't find, and the next run puts it back.104+ const old = `${s.dir}.old`;105+ fs.rmSync(old, { recursive: true, force: true });106+ if (fs.existsSync(s.dir)) fs.renameSync(s.dir, old);107+ fs.renameSync(tmp, s.dir);108+ fs.rmSync(old, { recursive: true, force: true });109+}110+111+const exportFingerprint = (repos: string[]): string => repos.join('\n');112+113+// Export the forge's insights, and those of each repo in `touched` (and of114+// any repo without an export yet). Returns how many scopes it wrote.115+export async function exportInsights(lake: Lake, touched: Set<string>): Promise<number> {116+ const cfg = lake.cfg;117+ const repos = listRepos(cfg.root);118+ const pub = repos.filter((r) => r.public).map((r) => r.slug);119+ const dir = (slug: string | null): string => path.join(cfg.exportDir, ...(slug ? [...slug.split('/'), 'insights'] : ['insights']));120+ // Which repos are public is part of the forge's export: a repo that went121+ // private leaves it at the next run, even if nothing else changed.122+ const mark = path.join(cfg.exportDir, 'public');123+ const was = fs.existsSync(mark) ? fs.readFileSync(mark, 'utf8') : null;124+ let count = 0;125+ fs.mkdirSync(cfg.exportDir, { recursive: true });126+ for (const r of repos) {127+ if (!touched.has(r.slug) && fs.existsSync(dir(r.slug))) continue;128+ const w = `repo = ${lit(r.slug)}`;129+ await exportScope(lake, { slug: r.slug, dir: dir(r.slug), where: w, builds: `scope = ${lit(r.slug)}` }, pub);130+ count++;131+ }132+ if (touched.size || was !== exportFingerprint(pub) || !fs.existsSync(dir(null))) {133+ await exportScope(lake, { slug: null, dir: dir(null), where: inList('repo', pub), builds: `(${inList('scope', pub)} OR scope IN ('-', 'all'))` }, pub);134+ fs.writeFileSync(mark, exportFingerprint(pub));135+ count++;136+ }137+ // Repos that are gone.138+ const live = new Set(repos.map((r) => r.slug));139+ for (const o of fs.readdirSync(cfg.exportDir, { withFileTypes: true })) {140+ if (!o.isDirectory() || o.name === 'insights' || o.name.endsWith('.tmp') || o.name.endsWith('.old')) continue;141+ for (const r of fs.readdirSync(path.join(cfg.exportDir, o.name))) {142+ if (!live.has(`${o.name}/${r}`)) fs.rmSync(path.join(cfg.exportDir, o.name, r), { recursive: true, force: true });143+ }144+ }145+ return count;146+}147+148+// Put each export into the live pages: the forge's into www/insights/, a149+// public repo's into its page directory, if it has one (pages of a repo150+// that just went private are gone, and stay gone). A file at a time, each151+// replaced by a rename, so a page never fetches half of one.152+export function publishInsights(cfg: LakeConfig): number {153+ let count = 0;154+ const put = (from: string, to: string): void => {155+ fs.mkdirSync(to, { recursive: true });156+ for (const f of FILES) {157+ const src = path.join(from, f);158+ if (!fs.existsSync(src)) continue;159+ const tmp = path.join(to, `.${f}.${process.pid}`);160+ fs.rmSync(tmp, { force: true });161+ try {162+ fs.linkSync(src, tmp);163+ } catch {164+ fs.copyFileSync(src, tmp);165+ }166+ fs.renameSync(tmp, path.join(to, f));167+ }168+ count++;169+ };170+ if (!fs.existsSync(cfg.www)) return 0;171+ const forge = path.join(cfg.exportDir, 'insights');172+ if (fs.existsSync(forge)) put(forge, path.join(cfg.www, 'insights'));173+ for (const r of listRepos(cfg.root)) {174+ const from = path.join(cfg.exportDir, r.slug, 'insights');175+ const pages = path.join(cfg.www, r.slug);176+ if (!r.public || !fs.existsSync(from) || !fs.existsSync(pages)) continue;177+ put(from, path.join(pages, 'insights'));178+ // Made private meanwhile: take them out again.179+ if (!fs.existsSync(path.join(r.dir, 'git-daemon-export-ok'))) {180+ for (const f of FILES) fs.rmSync(path.join(pages, 'insights', f), { force: true });181+ }182+ }183+ return count;184+}@@ -5,7 +5,7 @@
55 // (review logs, refs/notes/ci). Each run reads what is new since the last66 // one (lib/lake-sources.ts) and adds it in one transaction, together with77 // where it got to, so a run that dies adds nothing and the next one starts8-// from the same place.8+// from the same place. Then it writes the insights exports (lib/insights.ts).99 //1010 // One writer: bin/forge runs this under $FORGE_STATE/lake.lock. DuckDB1111 // can't share a database file between processes, and DuckLake over SQLite@@ -13,7 +13,7 @@
1313 // under contention. Readers (forge lake query, forge metrics) attach it1414 // read-only and can run alongside.1515 //16-// node lib/lake.ts ingest [--dry-run] add what is new16+// node lib/lake.ts ingest [--dry-run] add what is new, then export insights1717 // node lib/lake.ts query <sql> [--json|--csv] read-only, no file access1818 // node lib/lake.ts install install the DuckDB extensions (the Dockerfile runs it)1919 // node lib/lake.ts stage <dir> a consistent copy of the lake for a backup@@ -29,6 +29,7 @@ import {
2929 buildRows, ciRows, eventRow, listRepos, notesTip, readNew, reviewEnd, reviewLogs, reviewRows,3030 type BuildRepoRow, type BuildRow, type CiRow, type EventRow, type Offset, type Read, type ReviewEnd, type ReviewRow,3131 } from './lake-sources.ts';32+import { exportInsights, publishInsights } from './insights.ts';3233 3334 const home = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..');3435 @@ -522,8 +523,10 @@ async function main(argv: string[]): Promise<void> {
522523 return;523524 }524525 const ck = await checkpoint(lake, now);526+ const ex = await exportInsights(lake, s.touched);527+ const pub = publishInsights(cfg);525528 const size = du(cfg.dir);526- console.log(`lake: added ${what}${ck ? '; checkpointed' : ''}; ${((performance.now() - t0) / 1000).toFixed(2)}s, ${(size / 1024).toFixed(0)}K on disk`);529+ console.log(`lake: added ${what}${ck ? '; checkpointed' : ''}; exported ${plural(ex, 'scope')}, published ${plural(pub, 'scope')}; ${((performance.now() - t0) / 1000).toFixed(2)}s, ${(size / 1024).toFixed(0)}K on disk`);527530 } finally {528531 lake.close();529532 if (scratch) fs.rmSync(scratch, { recursive: true, force: true });@@ -158,10 +158,11 @@ if (process.argv[1] && fs.realpathSync(process.argv[1]) === fileURLToPath(import
158158 const forge = loadForge();159159 const render = pageRenderer();160160 const templates = listTemplates();161- // Every page, unless lists are named: the index, owners, and forge.pages.<list>.161+ // Every page, unless lists are named: the index, owners, the forge's insights, and forge.pages.<list>.162162 const jobs: { file: string; template: string; data: Record<string, unknown> }[] = [];163- for (const list of lists.length ? lists : ['index', 'owners', ...templates.keys()]) {163+ for (const list of lists.length ? lists : ['index', 'owners', 'forgeInsights', ...templates.keys()]) {164164 if (list === 'index') jobs.push({ file: '/index.html', template: 'index.njk', data: { forge } });165+ else if (list === 'forgeInsights') jobs.push({ file: '/insights/index.html', template: 'insights.njk', data: { forge } });165166 else if (list === 'owners') for (const o of forge.owners) jobs.push({ file: `/${o.owner}/index.html`, template: 'owner.njk', data: { forge, o } });166167 else {167168 const template = templates.get(list);@@ -330,6 +330,8 @@ export interface RepoUrls {
330330 revision: (id: number, n: number) => Both;331331 sites: Both;332332 download: (tag: string, file: string) => Both;333+ // Charts from the lake (lib/insights.ts), and the files they draw from.334+ insights: Both;333335 wiki: (path?: string) => Both;334336 atom: Both;335337 site: (site: string) => string;@@ -58,6 +58,7 @@ export function repoUrls(cfg: Config, owner: string, name: string): RepoUrls {
5858 sites: both(`${base}sites/`),5959 // A release's file, hard-linked there by the hooks and the build.6060 download: (tag, file) => both(`${base}releases/download/${encRef(tag)}/${file}`),61+ insights: both(`${base}insights/`),6162 wiki: (p = '') => both(`${base}wiki/${p ? p + '/' : ''}`),6263 atom: both(`${base}atom.xml`),6364 site: (site) => hostUrl(owner, name, site) ?? fill(cfg.sitesUrl, { owner, repo: name, site }),@@ -9,7 +9,7 @@
99 <head>1010 <meta charset="utf-8">1111 <meta name="viewport" content="width=device-width, initial-scale=1">12-<title>{{ p.title if p and p.title else forge.config.name }}</title>12+<title>{% block title %}{{ p.title if p and p.title else forge.config.name }}{% endblock %}</title>1313 <link rel="icon" href="{{ assets.logo }}" type="image/svg+xml">1414 <link rel="stylesheet" href="{{ assets.style }}">1515 {# The CSS that this page's templates and includes carry ({% css %}). -#}@@ -59,6 +59,7 @@
5959 <a href="{{ r.urls.reviews.href }}" data-key="r"{% if p.nav == 'reviews' %} class="cur"{% endif %}>Reviews</a>6060 {% if r.counts.wiki %}<a href="{{ r.urls.wiki().href }}" data-key="w"{% if p.nav == 'wiki' %} class="cur"{% endif %}>Wiki</a>{% endif %}6161 {% if r.counts.sites %}<a href="{{ r.urls.sites.href }}" data-key="s"{% if p.nav == 'sites' %} class="cur"{% endif %}>Sites</a>{% endif %}62+<a href="{{ r.urls.insights.href }}" data-key="i"{% if p.nav == 'insights' %} class="cur"{% endif %}>Insights</a>6263 </nav>6364 {% endif %}6465 </div>@@ -0,0 +1,95 @@
1+{#- The insights page's body, for the forge (insights.njk) and for a repo2+ (repo/insights.njk): stat tiles and charts that insights.ts draws from3+ insights.json, each chart with its table, and the Parquet files behind4+ them. A repo's page has `p`; the exports are in the page's directory. -#}5+{% set forgeWide = not p %}6+{% set base = '/insights/' if forgeWide else p.repo.urls.insights.href %}7+{% css %}8+/* insights (insights.ts): the chart colors are the data-viz palette's,9+ checked for color-blind readers against this page's light and dark10+ backgrounds; text keeps the page's own colors. */11+.insights {12+ --viz-1: #2a78d6;13+ --viz-fail: #d03b3b;14+ --viz-grid: #eaeef2;15+ --viz-axis: #c3c2b7;16+}17+@media (prefers-color-scheme: dark) {18+ .insights {19+ --viz-1: #3987e5;20+ --viz-fail: #d03b3b;21+ --viz-grid: #21262d;22+ --viz-axis: #3d444d;23+ }24+}25+.insights .ins-meta { margin: 0.4em 0 1em; }26+.insights .kpis { display: grid; grid-template-columns: repeat(auto-fit, minmax(150px, 1fr)); gap: 12px; margin: 0 0 8px; }27+.insights .kpi { border: 1px solid var(--border); border-radius: var(--radius); padding: 10px 14px; }28+.insights .kpi .label { color: var(--muted); font-size: 0.85em; }29+.insights .kpi .value { font-size: 1.6rem; font-weight: 600; line-height: 1.2; margin-top: 2px; }30+.insights .kpi .sub { color: var(--muted); font-size: 0.85em; }31+.insights .charts { display: grid; grid-template-columns: repeat(auto-fit, minmax(min(100%, 440px), 1fr)); gap: 12px 20px; }32+.insights figure { margin: 0; min-width: 0; position: relative; }33+.insights figcaption h3 { margin: 1.2em 0 0; }34+.insights figcaption .desc { font-size: 0.85em; }35+.insights .plot { position: relative; margin-top: 6px; }36+.insights .plot:focus-visible { outline: 2px solid var(--accent); outline-offset: 2px; border-radius: 3px; }37+.insights svg { display: block; width: 100%; overflow: visible; font: 11px var(--sans); font-variant-numeric: tabular-nums; }38+.insights svg .grid { stroke: var(--viz-grid); stroke-width: 1; shape-rendering: crispEdges; }39+.insights svg .base { stroke: var(--viz-axis); stroke-width: 1; shape-rendering: crispEdges; }40+.insights svg .tick { fill: var(--muted); }41+.insights svg .s1 { fill: var(--viz-1); }42+.insights svg .fail { fill: var(--viz-fail); }43+.insights svg .dot { stroke: var(--bg); stroke-width: 2; }44+.insights svg .hair { stroke: var(--muted); stroke-width: 1; shape-rendering: crispEdges; }45+.insights svg .on { filter: brightness(1.15); }46+.insights svg .mark-label { fill: var(--fg); paint-order: stroke; stroke: var(--bg); stroke-width: 3px; stroke-linejoin: round; }47+.insights .empty-chart { color: var(--muted); font-size: 0.9em; padding: 28px 0; text-align: center; border: 1px solid var(--border); border-radius: var(--radius); }48+.insights .legend { display: flex; gap: 14px; font-size: 0.85em; color: var(--muted); margin-top: 4px; }49+.insights .legend i { display: inline-block; width: 10px; height: 10px; border-radius: 2px; margin-right: 5px; vertical-align: -1px; }50+.insights .tip { position: absolute; z-index: 2; pointer-events: none; background: var(--bg); border: 1px solid var(--border); border-radius: 6px; box-shadow: 0 4px 16px rgba(0, 0, 0, 0.12); padding: 6px 9px; font-size: 0.85em; white-space: nowrap; }51+.insights .tip b { font-size: 1.05em; }52+.insights .tip .k { display: inline-block; width: 10px; height: 2px; margin-right: 6px; vertical-align: middle; }53+.insights details.table { margin-top: 4px; font-size: 0.85em; }54+.insights details.table summary { color: var(--muted); cursor: pointer; }55+.insights details.table table { border-collapse: collapse; margin-top: 4px; font-variant-numeric: tabular-nums; }56+.insights details.table td, .insights details.table th { padding: 2px 10px 2px 0; text-align: left; border-bottom: 1px solid var(--border); }57+.insights details.table .scroll { max-height: 240px; overflow: auto; }58+.insights .files td { vertical-align: top; padding: 1px 12px 1px 0; }59+.insights .files td:first-child { white-space: nowrap; }60+.insights pre.cmd { overflow-x: auto; background: var(--subtle); border: 1px solid var(--border); border-radius: var(--radius); padding: 8px 12px; font-size: 0.85em; }61+@media (forced-colors: active) {62+ .insights svg .s1, .insights svg .fail { fill: CanvasText; }63+}64+{% endcss %}65+<div class="insights" data-insights="{{ base }}insights.json"{% if forgeWide %} data-forge="1"{% endif %}>66+<p class="desc ins-meta" id="ins-meta">From the forge's lake (<code>forge lake</code>).<noscript> The charts need JavaScript; the numbers are in the files below.</noscript></p>67+<div class="kpis" id="ins-kpis"></div>68+<div class="charts">69+<figure data-chart="builds"><figcaption><h3>Build time</h3><span class="desc">seconds per build of {{ 'the pages' if forgeWide else "this repo's pages" }}</span></figcaption><div class="plot"></div><details class="table"><summary>Table</summary></details></figure>70+<figure data-chart="pushes"><figcaption><h3>Pushes</h3><span class="desc">pushes a <span class="ins-bucket">day</span></span></figcaption><div class="plot"></div><details class="table"><summary>Table</summary></details></figure>71+<figure data-chart="land"><figcaption><h3>Time to land</h3><span class="desc">hours from a review's first push to landing, by when it landed</span></figcaption><div class="plot"></div><details class="table"><summary>Table</summary></details></figure>72+<figure data-chart="revisions"><figcaption><h3>Revisions per review</h3><span class="desc">reviews by how many revisions they were pushed in</span></figcaption><div class="plot"></div><details class="table"><summary>Table</summary></details></figure>73+<figure data-chart="ci"><figcaption><h3>CI results</h3><span class="desc">attestations a <span class="ins-bucket">day</span>, passed and failed</span></figcaption><div class="plot"></div><details class="table"><summary>Table</summary></details></figure>74+<figure data-chart="ciTime"><figcaption><h3>CI duration</h3><span class="desc">seconds per run of <code>.forge/ci</code></span></figcaption><div class="plot"></div><details class="table"><summary>Table</summary></details></figure>75+</div>76+{% if forgeWide %}77+<h3>By repo</h3>78+<div class="box">79+<table class="list nav" id="ins-repos">80+<thead><tr><th>Repository</th><th class="num">Pushes</th><th class="num">Builds</th><th class="num">Median build</th><th class="num">Landed</th><th class="num">CI passed</th><th>Last push</th></tr></thead>81+<tbody></tbody>82+</table>83+</div>84+{% endif %}85+<h3>Data</h3>86+<p class="desc">The rows behind the charts{{ ', for public repos only' if forgeWide else '' }}, as Parquet. DuckDB reads them over HTTP in byte ranges, so a query fetches only what it needs:</p>87+<table class="files desc">88+<tr><td><a href="{{ base }}insights.json">insights.json</a></td><td>what the charts draw</td></tr>89+<tr><td><a href="{{ base }}events.parquet">events.parquet</a></td><td>pushes, comments, approvals and builds as they happened</td></tr>90+<tr><td><a href="{{ base }}builds.parquet">builds.parquet</a></td><td>each page build: time spent, pages written</td></tr>91+<tr><td><a href="{{ base }}reviews.parquet">reviews.parquet</a></td><td>each review's lifecycle: created, revisions, approvals, landed or closed</td></tr>92+<tr><td><a href="{{ base }}ci.parquet">ci.parquet</a></td><td>each CI attestation: commit, tree, result, duration, runner, signer</td></tr>93+</table>94+<pre class="cmd">duckdb -c "SELECT result, count(*), median(duration_s) FROM '{{ forge.config.baseUrl or 'https://<this forge>' }}{{ base }}ci.parquet' GROUP BY result"</pre>95+</div>@@ -0,0 +1,476 @@
1+// The insights page's charts (site/_includes/insights.njk): stat tiles and2+// SVG charts drawn from insights.json (lib/insights.ts), which forge lake3+// writes next to the page. Each chart has a tooltip on hover and on keyboard4+// focus (arrow keys move it), and a table with the same numbers.5+(() => {6+ const found = document.querySelector<HTMLElement>('.insights[data-insights]');7+ if (!found) return;8+ const root: HTMLElement = found;9+ const forgeWide = root.dataset.forge === '1';10+ const NS = 'http://www.w3.org/2000/svg';11+ const DAY = 864e5;12+13+ // [repo, id, created, ended, how, revisions, approvals]14+ type ReviewRow = [string, number, number | null, number | null, string | null, number, number];15+ interface Data {16+ generated: string;17+ from: number | null;18+ totals: Record<string, number | null>;19+ builds: [number, number][];20+ pushes: [string, number][];21+ reviews: ReviewRow[];22+ ci: [number, number | null, number][];23+ repos?: [string, number, number, number | null, number, number, number, number | null][];24+ }25+26+ // ---------------------------------------------------------------- formats27+28+ const r1 = (n: number): string => (Math.round(n * 10) / 10).toString();29+ const secs = (s: number): string => (s < 10 ? `${Math.round(s * 100) / 100}s` : s < 60 ? `${r1(s)}s` : s < 3600 ? `${Math.floor(s / 60)}m ${Math.round(s % 60)}s` : `${r1(s / 3600)}h`);30+ const hours = (h: number): string => (h < 1 ? `${Math.round(h * 60)}m` : h < 48 ? `${r1(h)}h` : `${r1(h / 24)}d`);31+ const count = (n: number): string => (n >= 1e4 ? `${Math.round(n / 1e3)}K` : n >= 1e3 ? `${r1(n / 1e3)}K` : String(n));32+ const plural = (n: number, w: string): string => `${count(n)} ${w}${n === 1 ? '' : 's'}`;33+ const when = (ms: number): string => new Date(ms).toLocaleString(undefined, { month: 'short', day: 'numeric', hour: '2-digit', minute: '2-digit' });34+ const dayOf = (ms: number): string => new Date(ms).toLocaleDateString(undefined, { month: 'short', day: 'numeric', timeZone: 'UTC' });35+ const dayLong = (ms: number): string => new Date(ms).toLocaleDateString(undefined, { year: 'numeric', month: 'short', day: 'numeric', timeZone: 'UTC' });36+37+ // ---------------------------------------------------------------- dom38+39+ const el = <K extends keyof HTMLElementTagNameMap>(tag: K, cls?: string, text?: string): HTMLElementTagNameMap[K] => {40+ const e = document.createElement(tag);41+ if (cls) e.className = cls;42+ if (text != null) e.textContent = text;43+ return e;44+ };45+ const svgEl = (tag: string, attrs: Record<string, string | number>, parent?: Element): SVGElement => {46+ const e = document.createElementNS(NS, tag);47+ for (const [k, v] of Object.entries(attrs)) e.setAttribute(k, String(v));48+ parent?.appendChild(e);49+ return e;50+ };51+52+ // Round numbers from 0 to at least max, about n steps.53+ function ticks(max: number, n = 4): number[] {54+ if (!(max > 0)) return [0, 1];55+ const raw = max / n;56+ const mag = 10 ** Math.floor(Math.log10(raw));57+ const step = [1, 2, 2.5, 5, 10].map((m) => m * mag).find((s) => s >= raw) ?? 10 * mag;58+ const out: number[] = [];59+ for (let v = 0; v < max + step; v += step) {60+ out.push(Number(v.toPrecision(12)));61+ if (v >= max) break;62+ }63+ return out;64+ }65+66+ // Times for the x axis between a and b, about n of them, on day (or hour) boundaries.67+ function timeTicks(a: number, b: number, n: number): { at: number; label: string }[] {68+ const span = Math.max(b - a, 1);69+ const steps = [3600e3, 3 * 3600e3, 6 * 3600e3, 12 * 3600e3, DAY, 2 * DAY, 7 * DAY, 14 * DAY, 30 * DAY, 91 * DAY, 182 * DAY, 365 * DAY];70+ const step = steps.find((s) => span / s <= n) ?? 365 * DAY;71+ const out: { at: number; label: string }[] = [];72+ let t = Math.ceil(a / step) * step;73+ if (step >= DAY) t = Math.ceil(a / DAY) * DAY;74+ for (; t <= b; t += step) {75+ const d = new Date(t);76+ out.push({ at: t, label: step < DAY ? d.toLocaleTimeString(undefined, { hour: '2-digit', minute: '2-digit' }) : step >= 182 * DAY ? String(d.getUTCFullYear()) : dayOf(t) });77+ }78+ return out;79+ }80+81+ // ---------------------------------------------------------------- charts82+83+ interface Mark {84+ cx: number;85+ cy: number;86+ // Columns: the band the pointer selects them by.87+ x0?: number;88+ x1?: number;89+ els: SVGElement[];90+ head: string;91+ rows: [string, string, string?][];92+ }93+94+ const H = 190;95+ const PAD = { top: 14, right: 12, bottom: 24, left: 44 };96+97+ interface Frame {98+ svg: SVGElement;99+ w: number;100+ y: (v: number) => number;101+ plotW: number;102+ }103+104+ // An SVG with y gridlines and labels from 0 to the top tick; x is the caller's.105+ function frame(plot: HTMLElement, yMax: number, yFmt: (v: number) => string): Frame {106+ const w = Math.max(280, plot.clientWidth || 440);107+ const svg = svgEl('svg', { viewBox: `0 0 ${w} ${H}`, height: H, role: 'img' });108+ const ts = ticks(yMax);109+ const top = ts.at(-1)!;110+ const plotH = H - PAD.top - PAD.bottom;111+ const y = (v: number): number => PAD.top + plotH - (v / top) * plotH;112+ for (const t of ts) {113+ svgEl('line', { class: t === 0 ? 'base' : 'grid', x1: PAD.left, x2: w - PAD.right, y1: Math.round(y(t)) + 0.5, y2: Math.round(y(t)) + 0.5 }, svg);114+ const label = svgEl('text', { class: 'tick', x: PAD.left - 6, y: y(t) + 4, 'text-anchor': 'end' }, svg);115+ label.textContent = yFmt(t);116+ }117+ plot.replaceChildren(svg);118+ return { svg, w, y, plotW: w - PAD.left - PAD.right };119+ }120+121+ function xLabels(f: Frame, list: { x: number; label: string }[]): void {122+ let last = -Infinity;123+ for (const { x, label } of list) {124+ // Skip a label that would run into the one before it.125+ if (x - last < label.length * 6.5 + 8) continue;126+ last = x;127+ const t = svgEl('text', { class: 'tick', x, y: H - 7, 'text-anchor': 'middle' }, f.svg);128+ t.textContent = label;129+ }130+ }131+132+ // Hover and keyboard: the nearest mark (by its band, or by distance) gets133+ // a tooltip and lifts; dots get a vertical hairline.134+ const listening = new WeakMap<HTMLElement, AbortController>();135+ function interact(plot: HTMLElement, f: Frame, marks: Mark[], hair: boolean): void {136+ // A chart drawn again (a resize) drops the last drawing's listeners.137+ listening.get(plot)?.abort();138+ const ac = new AbortController();139+ listening.set(plot, ac);140+ const signal = ac.signal;141+ if (!marks.length) return;142+ plot.tabIndex = 0;143+ plot.setAttribute('aria-label', `${marks.length} values; arrow keys step through them`);144+ const tip = el('div', 'tip');145+ tip.hidden = true;146+ plot.appendChild(tip);147+ const line = hair ? svgEl('line', { class: 'hair', y1: PAD.top, y2: H - PAD.bottom, visibility: 'hidden' }, f.svg) : null;148+ let at = -1;149+ const show = (i: number): void => {150+ if (at >= 0) for (const e of marks[at].els) e.classList.remove('on');151+ at = i;152+ if (i < 0) {153+ tip.hidden = true;154+ line?.setAttribute('visibility', 'hidden');155+ return;156+ }157+ const m = marks[i];158+ for (const e of m.els) e.classList.add('on');159+ tip.replaceChildren();160+ tip.appendChild(el('div', 'desc', m.head));161+ // The value leads, its label follows; a short line keys the series.162+ for (const [value, label, cls] of m.rows) {163+ const row = el('div');164+ if (cls) {165+ const k = el('span', 'k');166+ k.style.background = cls === 'fail' ? 'var(--viz-fail)' : 'var(--viz-1)';167+ row.appendChild(k);168+ }169+ row.append(el('b', '', value), ` ${label}`);170+ tip.appendChild(row);171+ }172+ tip.hidden = false;173+ const scale = plot.clientWidth / f.w;174+ const px = m.cx * scale;175+ const tw = tip.offsetWidth;176+ tip.style.left = `${Math.min(Math.max(0, px - tw / 2), plot.clientWidth - tw)}px`;177+ tip.style.top = `${Math.max(0, m.cy * scale - tip.offsetHeight - 12)}px`;178+ if (line) {179+ line.setAttribute('x1', String(m.cx));180+ line.setAttribute('x2', String(m.cx));181+ line.setAttribute('visibility', 'visible');182+ }183+ };184+ plot.addEventListener('pointermove', (e) => {185+ const box = f.svg.getBoundingClientRect();186+ const x = ((e.clientX - box.left) / box.width) * f.w;187+ const y = ((e.clientY - box.top) / box.height) * H;188+ let best = -1;189+ let bd = Infinity;190+ marks.forEach((m, i) => {191+ const d = m.x0 != null ? (x >= m.x0 && x <= m.x1! ? 0 : Math.min(Math.abs(x - m.x0), Math.abs(x - m.x1!))) : Math.hypot(x - m.cx, (y - m.cy) / 3);192+ if (d < bd) [bd, best] = [d, i];193+ });194+ if (best !== at) show(best);195+ }, { signal });196+ plot.addEventListener('pointerleave', () => show(-1), { signal });197+ plot.addEventListener('blur', () => show(-1), { signal });198+ plot.addEventListener('focus', () => show(at >= 0 ? at : marks.length - 1), { signal });199+ plot.addEventListener('keydown', (e) => {200+ if (e.key === 'ArrowRight' || e.key === 'ArrowLeft') {201+ e.preventDefault();202+ show(Math.min(marks.length - 1, Math.max(0, (at < 0 ? marks.length : at) + (e.key === 'ArrowRight' ? 1 : -1))));203+ } else if (e.key === 'Escape') show(-1);204+ }, { signal });205+ }206+207+ interface Dot {208+ t: number;209+ v: number;210+ fail?: boolean;211+ head: string;212+ rows: [string, string, string?][];213+ }214+215+ // Dots over time, one per fact. The last one carries its value.216+ function dots(plot: HTMLElement, list: Dot[], yFmt: (v: number) => string): void {217+ const max = Math.max(...list.map((d) => d.v));218+ const f = frame(plot, max, yFmt);219+ const a = Math.min(...list.map((d) => d.t));220+ const b = Math.max(...list.map((d) => d.t));221+ const pad = (b - a) * 0.03 || 3600e3;222+ const x = (t: number): number => PAD.left + ((t - (a - pad)) / (b - a + 2 * pad)) * f.plotW;223+ xLabels(f, timeTicks(a - pad, b + pad, Math.max(2, Math.floor(f.plotW / 90))).map((k) => ({ x: x(k.at), label: k.label })));224+ const marks: Mark[] = list.map((d) => {225+ const c = svgEl('circle', { class: `dot ${d.fail ? 'fail' : 's1'}`, cx: x(d.t), cy: f.y(d.v), r: 4 }, f.svg);226+ return { cx: x(d.t), cy: f.y(d.v), els: [c], head: d.head, rows: d.rows };227+ });228+ const last = list.reduce((p, d) => (d.t >= p.t ? d : p), list[0]);229+ const lbl = svgEl('text', { class: 'mark-label', x: Math.min(x(last.t) + 7, f.w - 2), y: f.y(last.v) - 7, 'text-anchor': x(last.t) > f.w - 60 ? 'end' : 'start' }, f.svg);230+ lbl.textContent = yFmt(last.v);231+ interact(plot, f, marks, true);232+ }233+234+ interface Column {235+ label: string;236+ head: string;237+ segs: { v: number; cls: string; name: string }[];238+ }239+240+ // A column's path: square at the baseline, 4px round corners on top.241+ const colPath = (x: number, y: number, w: number, h: number, round: boolean): string => {242+ const r = round ? Math.min(4, w / 2, h) : 0;243+ return `M${x},${y + h}V${y + r}Q${x},${y} ${x + r},${y}H${x + w - r}Q${x + w},${y} ${x + w},${y + r}V${y + h}Z`;244+ };245+246+ // Columns, stacked when they have more than one segment, with a 2px gap247+ // between segments. The tallest carries its total.248+ function columns(plot: HTMLElement, cols: Column[], yFmt: (v: number) => string, every = 1): void {249+ const total = (c: Column): number => c.segs.reduce((s, x) => s + x.v, 0);250+ const max = Math.max(...cols.map(total));251+ const f = frame(plot, max, yFmt);252+ const band = f.plotW / cols.length;253+ const w = Math.max(1, Math.min(24, band * 0.7));254+ const marks: Mark[] = [];255+ const labels: { x: number; label: string }[] = [];256+ let tallest: { x: number; y: number; v: number } | null = null;257+ cols.forEach((c, i) => {258+ const cx = PAD.left + band * i + band / 2;259+ const els: SVGElement[] = [];260+ let base = f.y(0);261+ const shown = c.segs.filter((s) => s.v > 0);262+ shown.forEach((s, j) => {263+ const h = f.y(0) - f.y(s.v) - (j > 0 ? 2 : 0);264+ if (h <= 0) return;265+ const y = base - (j > 0 ? 2 : 0) - h;266+ els.push(svgEl('path', { class: s.cls, d: colPath(cx - w / 2, y, w, h, j === shown.length - 1) }, f.svg));267+ base = y;268+ });269+ if (i % every === 0) labels.push({ x: cx, label: c.label });270+ const t = total(c);271+ if (!tallest || t > tallest.v) tallest = { x: cx, y: base, v: t };272+ marks.push({ cx, cy: Math.min(base, f.y(0) - 4), x0: PAD.left + band * i, x1: PAD.left + band * (i + 1), els, head: c.head, rows: c.segs.map((s) => [count(s.v), s.name, c.segs.length > 1 ? s.cls : undefined]) });273+ });274+ xLabels(f, labels);275+ if (tallest && (tallest as { v: number }).v > 0) {276+ const { x, y, v } = tallest as { x: number; y: number; v: number };277+ const t = svgEl('text', { class: 'mark-label', x, y: y - 5, 'text-anchor': 'middle' }, f.svg);278+ t.textContent = yFmt(v);279+ }280+ interact(plot, f, marks, false);281+ }282+283+ // ---------------------------------------------------------------- buckets284+285+ // Days for a short span, weeks (from Monday) for up to two years, else months; UTC.286+ function buckets(times: number[]): { unit: 'day' | 'week' | 'month'; keys: number[]; key: (t: number) => number } {287+ const a = Math.min(...times);288+ const b = Math.max(...times);289+ const span = (b - a) / DAY;290+ const unit = span <= 62 ? 'day' : span <= 730 ? 'week' : 'month';291+ const key = (t: number): number => {292+ const d = new Date(Math.floor(t / DAY) * DAY);293+ if (unit === 'week') return d.getTime() - ((d.getUTCDay() + 6) % 7) * DAY;294+ if (unit === 'month') return Date.UTC(d.getUTCFullYear(), d.getUTCMonth(), 1);295+ return d.getTime();296+ };297+ const keys: number[] = [];298+ for (let k = key(a); k <= key(b); ) {299+ keys.push(k);300+ const d = new Date(k);301+ k = unit === 'day' ? k + DAY : unit === 'week' ? k + 7 * DAY : Date.UTC(d.getUTCFullYear(), d.getUTCMonth() + 1, 1);302+ }303+ return { unit, keys, key };304+ }305+306+ const bucketLabel = (unit: string, k: number): string => (unit === 'month' ? new Date(k).toLocaleDateString(undefined, { month: 'short', year: 'numeric', timeZone: 'UTC' }) : dayOf(k));307+308+ // ---------------------------------------------------------------- tables309+310+ function table(fig: HTMLElement, head: string[], rows: () => string[][]): void {311+ const d = fig.querySelector<HTMLDetailsElement>('details.table');312+ if (!d || d.dataset.ready) return;313+ d.dataset.ready = '1';314+ d.addEventListener('toggle', () => {315+ if (!d.open || d.querySelector('table')) return;316+ const t = el('table');317+ const tr = el('tr');318+ for (const h of head) tr.appendChild(el('th', '', h));319+ t.appendChild(el('thead')).appendChild(tr);320+ const body = t.appendChild(el('tbody'));321+ for (const r of rows()) {322+ const row = el('tr');323+ for (const c of r) row.appendChild(el('td', '', c));324+ body.appendChild(row);325+ }326+ d.appendChild(el('div', 'scroll')).appendChild(t);327+ });328+ }329+330+ function empty(fig: HTMLElement, text: string): void {331+ fig.querySelector('.plot')?.replaceChildren(el('div', 'empty-chart', text));332+ fig.querySelector('details.table')?.remove();333+ }334+335+ function legend(fig: HTMLElement, items: [string, string][]): void {336+ const l = el('div', 'legend');337+ for (const [cls, name] of items) {338+ const s = el('span');339+ const k = el('i');340+ k.style.background = cls === 'fail' ? 'var(--viz-fail)' : 'var(--viz-1)';341+ s.append(k, name);342+ l.appendChild(s);343+ }344+ fig.querySelector('.plot')?.after(l);345+ }346+347+ // ---------------------------------------------------------------- the page348+349+ const charts: (() => void)[] = [];350+ const figure = (name: string): HTMLElement | null => root.querySelector<HTMLElement>(`figure[data-chart="${name}"]`);351+ const chart = (name: string, draw: (fig: HTMLElement, plot: HTMLElement) => void): void => {352+ const fig = figure(name);353+ const plot = fig?.querySelector<HTMLElement>('.plot');354+ if (!fig || !plot) return;355+ const run = (): void => draw(fig, plot);356+ charts.push(run);357+ run();358+ };359+ const reviewName = (r: ReviewRow): string => (forgeWide ? `${r[0]} #${r[1]}` : `#${r[1]}`);360+361+ function tiles(d: Data): void {362+ const t = d.totals;363+ const box = root.querySelector('#ins-kpis');364+ if (!box) return;365+ const tile = (label: string, value: string, sub: string): void => {366+ const k = el('div', 'kpi');367+ k.append(el('div', 'label', label), el('div', 'value', value), el('div', 'sub', sub));368+ box.appendChild(k);369+ };370+ const n = (k: string): number => t[k] ?? 0;371+ tile('Pushes', count(n('pushes')), d.from ? `since ${dayLong(d.from)}` : 'none yet');372+ tile('Builds', count(n('builds')), t.build_ms_median != null ? `median ${secs(t.build_ms_median / 1000)}` : 'none yet');373+ tile('Reviews landed', `${count(n('landed'))} of ${count(n('reviews'))}`, t.land_hours_median != null ? `median ${hours(t.land_hours_median)} to land` : `${count(n('closed'))} closed`);374+ tile('Revisions', t.revisions_median != null ? r1(t.revisions_median) : '-', 'per review, median');375+ tile('CI passed', n('ci') ? `${Math.round((100 * n('ci_pass')) / n('ci'))}%` : '-', n('ci') ? `${n('ci_pass')} of ${plural(n('ci'), 'run')}${t.ci_seconds_median != null ? `, median ${secs(t.ci_seconds_median)}` : ''}` : 'no attestations yet');376+ }377+378+ function draw(d: Data): void {379+ const meta = root.querySelector('#ins-meta');380+ if (meta) meta.textContent = `From the forge's lake (forge lake), as of ${when(Date.parse(d.generated))}.`;381+ tiles(d);382+383+ chart('builds', (fig, plot) => {384+ if (!d.builds.length) return empty(fig, 'No builds yet.');385+ dots(plot, d.builds.map(([t, ms]) => ({ t, v: ms / 1000, head: when(t), rows: [[secs(ms / 1000), 'to build']] })), (v) => secs(v));386+ table(fig, ['Built', 'Took'], () => [...d.builds].reverse().map(([t, ms]) => [when(t), secs(ms / 1000)]));387+ });388+389+ chart('pushes', (fig, plot) => {390+ if (!d.pushes.length) return empty(fig, 'No pushes yet.');391+ const days = d.pushes.map(([day, n]) => [Date.parse(`${day}T00:00:00Z`), n] as [number, number]);392+ const bk = buckets(days.map(([t]) => t));393+ const sums = new Map<number, number>();394+ for (const [t, n] of days) sums.set(bk.key(t), (sums.get(bk.key(t)) ?? 0) + n);395+ for (const s of fig.querySelectorAll('.ins-bucket')) s.textContent = bk.unit;396+ const every = Math.ceil(bk.keys.length / Math.max(2, Math.floor((plot.clientWidth || 440) / 70)));397+ columns(plot, bk.keys.map((k) => ({ label: bucketLabel(bk.unit, k), head: `${bk.unit === 'week' ? 'week of ' : ''}${bk.unit === 'month' ? bucketLabel('month', k) : dayLong(k)}`, segs: [{ v: sums.get(k) ?? 0, cls: 's1', name: 'pushes' }] })), (v) => count(v), every);398+ table(fig, [bk.unit === 'week' ? 'Week of' : bk.unit === 'month' ? 'Month' : 'Day', 'Pushes'], () => bk.keys.filter((k) => sums.get(k)).reverse().map((k) => [dayLong(k), String(sums.get(k))]));399+ });400+401+ chart('land', (fig, plot) => {402+ const landed = d.reviews.filter((r) => r[4] === 'landed' && r[2] != null && r[3] != null);403+ if (!landed.length) return empty(fig, 'No review has landed yet.');404+ const h = (r: ReviewRow): number => Math.max(0, (r[3]! - r[2]!) / 3600e3);405+ dots(plot, landed.map((r) => ({ t: r[3]!, v: h(r), head: `${reviewName(r)}, landed ${when(r[3]!)}`, rows: [[hours(h(r)), 'to land'], [String(r[5]), r[5] === 1 ? 'revision' : 'revisions']] })), (v) => hours(v));406+ table(fig, ['Review', 'Landed', 'Took', 'Revisions'], () => [...landed].reverse().map((r) => [reviewName(r), when(r[3]!), hours(h(r)), String(r[5])]));407+ });408+409+ chart('revisions', (fig, plot) => {410+ const revs = d.reviews.filter((r) => r[5] > 0);411+ if (!revs.length) return empty(fig, 'No reviews yet.');412+ const top = Math.max(...revs.map((r) => r[5]));413+ const cap = Math.min(top, 10);414+ const n = new Array(cap + 1).fill(0);415+ for (const r of revs) n[Math.min(r[5], cap)]++;416+ const name = (i: number): string => (i === cap && top > cap ? `${i}+` : String(i));417+ columns(plot, n.slice(1).map((v, i) => ({ label: name(i + 1), head: `${name(i + 1)} ${i === 0 ? 'revision' : 'revisions'}`, segs: [{ v, cls: 's1', name: v === 1 ? 'review' : 'reviews' }] })), (v) => count(v));418+ table(fig, ['Review', 'Revisions', 'Approvals', 'Ended'], () => [...revs].reverse().map((r) => [reviewName(r), String(r[5]), String(r[6]), r[4] ?? 'open']));419+ });420+421+ chart('ci', (fig, plot) => {422+ if (!d.ci.length) return empty(fig, 'No CI attestations yet (forge attest).');423+ const bk = buckets(d.ci.map(([t]) => t));424+ const pass = new Map<number, number>();425+ const fail = new Map<number, number>();426+ for (const [t, , p] of d.ci) (p ? pass : fail).set(bk.key(t), ((p ? pass : fail).get(bk.key(t)) ?? 0) + 1);427+ for (const s of fig.querySelectorAll('.ins-bucket')) s.textContent = bk.unit;428+ const every = Math.ceil(bk.keys.length / Math.max(2, Math.floor((plot.clientWidth || 440) / 70)));429+ columns(plot, bk.keys.map((k) => ({ label: bucketLabel(bk.unit, k), head: `${bk.unit === 'week' ? 'week of ' : ''}${dayLong(k)}`, segs: [{ v: pass.get(k) ?? 0, cls: 's1', name: 'passed' }, { v: fail.get(k) ?? 0, cls: 'fail', name: 'failed' }] })), (v) => count(v), every);430+ if (!fig.querySelector('.legend')) legend(fig, [['s1', 'passed'], ['fail', 'failed']]);431+ table(fig, [bk.unit === 'week' ? 'Week of' : bk.unit === 'month' ? 'Month' : 'Day', 'Passed', 'Failed'], () => bk.keys.filter((k) => pass.get(k) || fail.get(k)).reverse().map((k) => [dayLong(k), String(pass.get(k) ?? 0), String(fail.get(k) ?? 0)]));432+ });433+434+ chart('ciTime', (fig, plot) => {435+ const runs = d.ci.filter(([, s]) => s != null);436+ if (!runs.length) return empty(fig, 'No CI attestations yet (forge attest).');437+ dots(plot, runs.map(([t, s, p]) => ({ t, v: s!, fail: !p, head: when(t), rows: [[secs(s!), p ? 'passed' : 'failed', p ? 's1' : 'fail']] })), (v) => secs(v));438+ if (!fig.querySelector('.legend')) legend(fig, [['s1', 'passed'], ['fail', 'failed']]);439+ table(fig, ['Started', 'Took', 'Result'], () => [...runs].reverse().map(([t, s, p]) => [when(t), secs(s!), p ? 'passed' : 'failed']));440+ });441+442+ const repos = root.querySelector('#ins-repos tbody');443+ if (repos && d.repos) {444+ for (const [slug, pushes, builds, buildMs, landed, ci, pass, last] of d.repos) {445+ const tr = el('tr');446+ const a = el('a');447+ a.href = `/${slug.split('/').map(encodeURIComponent).join('/')}/insights/`;448+ const [o, n] = slug.split('/');449+ a.append(el('span', 'desc', `${o} /`), ' ', el('b', '', n));450+ tr.appendChild(el('td', 'nowrap')).appendChild(a);451+ for (const v of [count(pushes), count(builds), buildMs != null ? secs(buildMs / 1000) : '', count(landed), ci ? `${pass} of ${ci}` : '']) tr.appendChild(el('td', 'num', v));452+ tr.appendChild(el('td', 'date', last ? dayLong(last) : ''));453+ repos.appendChild(tr);454+ }455+ }456+457+ // Charts are drawn at their width; draw again when it changes.458+ let frameId = 0;459+ let width = root.clientWidth;460+ new ResizeObserver(() => {461+ if (root.clientWidth === width) return;462+ width = root.clientWidth;463+ cancelAnimationFrame(frameId);464+ frameId = requestAnimationFrame(() => charts.forEach((c) => c()));465+ }).observe(root);466+ }467+468+ fetch(root.dataset.insights!)469+ .then((r) => (r.ok ? r.json() : Promise.reject(new Error(`${r.status}`))))470+ .then((d: Data) => draw(d))471+ .catch(() => {472+ const meta = root.querySelector('#ins-meta');473+ if (meta) meta.textContent = 'No insights yet: forge lake writes them after the next build, or run it on the server.';474+ for (const fig of root.querySelectorAll<HTMLElement>('figure')) empty(fig, 'Nothing yet.');475+ });476+})();@@ -3,7 +3,7 @@
33 // n/N next/previous commit or file in a diff44 // t or p find a file on this branch55 // ? the list of shortcuts below6-// links with data-key: header letters (l f b r w s h g), u goes up,6+// links with data-key: header letters (l f b r w s i h g), u goes up,77 // [ and ] for stack layers. diffs.ts adds x and v for diff files.88 (() => {99 // Every shortcut, for the help overlay.@@ -19,7 +19,7 @@
1919 [['v'], 'mark the file in view as viewed'],2020 [['h'], 'repository home'],2121 [['f', 'l', 'b', 'r'], 'code, commits, branches, reviews'],22- [['w', 's'], 'wiki, sites'],22+ [['w', 's', 'i'], 'wiki, sites, insights'],2323 [['u'], 'up a level'],2424 [['g'], 'all repositories'],2525 ];@@ -0,0 +1,15 @@
1+---2+permalink: /insights/index.html3+---4+{#- The forge's insights: every public repo's facts from the lake, the5+ same charts as a repo's insights page, and a table by repo. -#}6+{% extends "base.njk" %}7+{% import "macros.njk" as m with context %}8+{% block scripts %}<script src="{{ assets.insights }}" defer></script>{% endblock %}9+{% block title %}Insights - {{ forge.config.name }}{% endblock %}10+{% block head %}{{ m.meta('Insights - ' + forge.config.name, 'Builds, pushes, reviews and CI across ' + forge.config.name + '.', '/insights/') }}{% endblock %}11+{% block heading %}<a href="/">{{ forge.config.name }}</a> / <b>insights</b>12+{% endblock %}13+{% block content %}14+{% include "insights-body.njk" %}15+{% endblock %}@@ -0,0 +1,11 @@
1+---2+pagination:3+ data: forge.pages.insights4+---5+{% extends "base.njk" %}6+{% import "macros.njk" as m with context %}7+{% block scripts %}<script src="{{ assets.insights }}" defer></script>{% endblock %}8+{% block content %}9+<h2>Insights</h2>10+{% include "insights-body.njk" %}11+{% endblock %}@@ -0,0 +1,50 @@
1+# Insights pages: the forge's at /insights/ and each repo's at2+# /<owner>/<repo>/insights/, static pages whose charts fetch insights.json,3+# with the rows as Parquet next to it. forge lake writes the exports into4+# the live pages; builds keep them.5+. "$(dirname "$0")/../lib.sh"6+user dami7+user ana8+repo dami/greet9+clone dami dami/greet "$T/w/d"10+at "$T/w/d" commit "first"11+push_as dami "$T/w/d" origin main12+clone ana dami/greet "$T/w/a"13+at "$T/w/a" commit "idea"14+push_as ana "$T/w/a" origin HEAD:ana/idea15+build16+w=$FORGE_WWW17+has "$w/dami/greet/insights/index.html" 'data-insights="/dami/greet/insights/insights.json"'18+has "$w/dami/greet/insights/index.html" '<script src="/assets/insights\.[A-Za-z0-9_-]{10}\.js" defer>'19+has "$w/dami/greet/index.html" '<a href="/dami/greet/insights/" data-key="i">Insights</a>'20+has "$w/insights/index.html" 'data-insights="/insights/insights.json" data-forge="1"'21+has "$w/index.html" '<a href="/insights/">Insights</a>'22+# The charts' script is only on insights pages.23+lacks "$w/dami/greet/index.html" 'assets/insights\.'24+# No lake yet: no data, and the page says so when it can't fetch it.25+missing "$w/insights/insights.json"26+ok forge lake27+has "$T_LOG" 'exported 2 scopes, published 2 scopes'28+for d in "$w/insights" "$w/dami/greet/insights"; do29+ for f in events builds reviews ci; do30+ [ "$(head -c 4 "$d/$f.parquet")" = PAR1 ] || fail "$d/$f.parquet isn't Parquet"31+ done32+done33+has "$w/dami/greet/insights/insights.json" '"scope":"dami/greet"'34+has "$w/dami/greet/insights/insights.json" '"pushes":2,"builds":1,'35+has "$w/insights/insights.json" '"scope":"forge"'36+has "$w/insights/insights.json" '"repos":\[\["dami/greet",2,'37+# A push and a run of the lake update the live exports without a build.38+at "$T/w/d" commit "second"39+push_as dami "$T/w/d" origin main40+ok forge lake41+has "$w/dami/greet/insights/insights.json" '"pushes":3,'42+# A build of the repo swaps its pages, and keeps the exports; a full build43+# keeps the forge's.44+build dami/greet45+has "$w/dami/greet/insights/insights.json" '"pushes":3,'46+build47+has "$w/insights/insights.json" '"pushes":3,'48+exists "$w/dami/greet/insights/ci.parquet"49+# insights can't be an owner: the forge's page lives there.50+fails_with "invalid|not a valid|can't" forge repo new insights/x@@ -0,0 +1,51 @@
1+# Private repos are in no public insights: no insights page or export in2+# the pages, and nothing of theirs in the forge's (JSON or Parquet). Their3+# exports stay in the lake's state, beside where their pages would be. A4+# repo that goes private leaves the forge's insights at once; one that goes5+# public joins them at the next run.6+. "$(dirname "$0")/../lib.sh"7+user dami8+repo --private dami/secret "hidden thing"9+repo dami/open "open thing"10+for r in secret open; do11+ clone dami "dami/$r" "$T/w/$r"12+ at "$T/w/$r" commit "first"13+ ok push_as dami "$T/w/$r" origin main14+ at "$T/w/$r" commit "idea"15+ ok push_as dami "$T/w/$r" origin HEAD:dami/idea16+done17+build18+ok forge lake19+w=$FORGE_WWW20+exists "$w/dami/open/insights/insights.json"21+missing "$w/dami/secret"22+exists "$FORGE_ROOT/.forge/lake/export/dami/secret/insights/insights.json"23+has "$w/insights/insights.json" 'dami/open'24+lacks "$w/insights/insights.json" 'dami/secret'25+if grep -rlq 'secret' "$w/insights" "$w/dami/open"; then fail "a public export names the private repo"; fi26+# The Parquet files, read: rows of dami/open only.27+pq() { (cd "$T_HOME" && node --input-type=module -e "28+import { DuckDBInstance } from '@duckdb/node-api';29+const db = await DuckDBInstance.create(':memory:');30+const c = await db.connect();31+const r = await c.runAndReadAll(\"SELECT string_agg(DISTINCT repo, ' ' ORDER BY repo) FROM read_parquet('\" + process.argv[1] + \"')\");32+console.log(r.getRowsJson()[0][0] ?? '');33+" "$1"); }34+for f in events reviews; do eq "$(pq "$w/insights/$f.parquet")" "dami/open"; done35+eq "$(pq "$FORGE_ROOT/.forge/lake/export/dami/secret/insights/events.parquet")" "dami/secret"36+# Going private takes it out of the forge's insights right away, and the37+# next run doesn't put its pages' exports back.38+ok ssh_as dami private dami/open39+missing "$w/dami/open"40+missing "$w/insights/insights.json"41+ok forge lake42+exists "$w/insights/insights.json"43+lacks "$w/insights/insights.json" 'dami/open'44+missing "$w/dami/open"45+# Going public: in the forge's insights after the next run, and its own page.46+ok ssh_as dami public dami/secret47+build dami/secret48+ok forge lake49+has "$w/insights/insights.json" 'dami/secret'50+exists "$w/dami/secret/insights/insights.json"51+exists "$w/dami/secret/insights/index.html"