The lake can't be rebuilt: events.jsonl gets cut and metrics.jsonl isn't backed up. So staging copies it by itself instead of with the rest of .forge: it holds lake.lock so no forge lake writes meanwhile, copies the catalog with SQLite's VACUUM INTO and checks the copy, and hard-links the local data files, which are never changed once written. Data on R2 stays in its bucket. A restore brings the catalog and data back with the other files, and since each attach points the lake at this host's data directory, a lake restored under another root reads its own files.
- commit
- 76f9dccc207a5bc577a98a4e633c811925b5d2a3
- parent
- 597e742510cfea75fe38d75fe81501e0dfd07223
- author
- cat <admin@dev.site>
- kind
- feat risk score 1.4 of 3, Jev's estimate
| M | bin/ | 23 | ++++++++++- |
| M | lib/ | 62 | ++++++++++++++++++++++++++++++- |
| A | test/ | 46 | ++++++++++++++++++++++ |
@@ -2615,14 +2615,32 @@ stage_repo() {
26152615 26162616 # .forge entries a backup leaves out: rebuilt (www, store, cache,26172617 # allowed_signers), regenerated at boot (secrets), or logs, locks and scratch.2618+# The lake is staged by itself (stage_lake).26182619 derived_state() {26192620 case $1 in2620- www | store | cache | backup | secrets | allowed_signers | config.restored | restore.tmp | \2621- *.log | *.lock | *.tmp | backup.* | build.* | events.* | metrics.* | mirror.*) return 0 ;;2621+ www | store | cache | backup | secrets | allowed_signers | config.restored | restore.tmp | lake | \2622+ *.log | *.lock | *.tmp | backup.* | build.* | events.* | metrics.* | mirror.* | lake.*) return 0 ;;26222623 esac26232624 return 126242625 }26252626 2627+# stage_lake <dir>: the lake, into <dir>/lake. It can't be rebuilt (the2628+# files it read are cut or left out), and its catalog mustn't be copied2629+# while a run writes it: so no forge lake runs meanwhile (lake.lock), the2630+# catalog is copied with SQLite's VACUUM INTO and checked, and local data2631+# files are hard-linked (lib/lake.ts stage). Data on R2 stays there.2632+stage_lake() {2633+ [ -f "$FORGE_STATE/lake/catalog.sqlite" ] || return 02634+ take_lock "$FORGE_STATE/lake.lock" 600 || { berr="staging the lake: a forge lake has held $FORGE_STATE/lake.lock for 10 minutes"; return 1; }2635+ rc=02636+ (lake_env && node "$FORGE_HOME/lib/lake.ts" stage "$1") >"$BK_WORK/out" 2>&1 || rc=$?2637+ rm -rf "$FORGE_STATE/lake.lock"2638+ [ "$rc" = 0 ] && return 02639+ cat "$BK_WORK/out" >>"$BK_LOG"2640+ berr="staging the lake: $(tail -n 1 "$BK_WORK/out")"2641+ return 12642+}2643+26262644 # stage_all <dir>: copy everything a restore needs into dir. That is all of26272645 # FORGE_ROOT except caches, git-shell-commands (forge server init writes it)26282646 # and the derived parts of .forge, so state added later is kept by default.@@ -2645,6 +2663,7 @@ stage_all() {
26452663 mkdir -p "$st/.forge/www"26462664 cp -RPp "$p/www/hosts" "$st/.forge/www/" || return 126472665 fi2666+ stage_lake "$st/.forge" || return 126482667 ;;26492668 .cache | .npm | git-shell-commands | lost+found) ;;26502669 .*) cp -RPp "$p" "$st/" || return 1 ;;@@ -16,9 +16,11 @@
1616 // node lib/lake.ts ingest [--dry-run] add what is new1717 // node lib/lake.ts query <sql> [--json|--csv] read-only, no file access1818 // node lib/lake.ts install install the DuckDB extensions (the Dockerfile runs it)19+// node lib/lake.ts stage <dir> a consistent copy of the lake for a backup1920 import fs from 'node:fs';2021 import os from 'node:os';2122 import path from 'node:path';23+import { createRequire } from 'node:module';2224 import { fileURLToPath } from 'node:url';2325 import { DuckDBInstance, StatementType, type DuckDBConnection } from '@duckdb/node-api';2426 import { Verifier } from './attest.ts';@@ -419,7 +421,7 @@ export async function buildLines(cfg: LakeConfig): Promise<{ id: string; line: s
419421 }420422 }421423 422-// ---------------------------------------------------------------- install424+// ---------------------------------------------------------------- install, stage423425 424426 // Install the extensions into the extension directory and check their425427 // versions (the Dockerfile runs this, so the container never downloads one).@@ -441,6 +443,61 @@ async function install(cfg: LakeConfig): Promise<void> {
441443 if (bad) process.exit(1);442444 }443445 446+// A copy of the lake for a backup, into <dir>/lake: the catalog with447+// SQLite's VACUUM INTO, which reads one consistent state of it, never the448+// live file, then checked; and a local data directory as hard links449+// (Parquet files are written once and never changed). bin/forge holds450+// lake.lock meanwhile, so no file the copy names goes away.451+function stage(cfg: LakeConfig, dest: string): void {452+ if (!fs.existsSync(cfg.catalog)) return;453+ const out = path.join(dest, 'lake');454+ const copy = path.join(out, 'catalog.sqlite');455+ fs.mkdirSync(out, { recursive: true });456+ fs.rmSync(copy, { force: true });457+ const { DatabaseSync } = sqlite();458+ const db = new DatabaseSync(cfg.catalog, { readOnly: true });459+ try {460+ db.exec(`VACUUM INTO ${lit(copy)}`);461+ } finally {462+ db.close();463+ }464+ const check = new DatabaseSync(copy, { readOnly: true });465+ try {466+ const r = check.prepare('PRAGMA integrity_check').get() as Record<string, unknown> | undefined;467+ if (Object.values(r ?? {})[0] !== 'ok') throw new LakeError(`the copy of the catalog fails SQLite's integrity check: ${JSON.stringify(r)}`);468+ } finally {469+ check.close();470+ }471+ if (!cfg.local || !fs.existsSync(cfg.data)) return;472+ const walk = (from: string, to: string): void => {473+ fs.mkdirSync(to, { recursive: true });474+ for (const e of fs.readdirSync(from, { withFileTypes: true })) {475+ const a = path.join(from, e.name);476+ const b = path.join(to, e.name);477+ if (e.isDirectory()) walk(a, b);478+ else if (e.isFile()) {479+ try {480+ fs.linkSync(a, b);481+ } catch {482+ fs.copyFileSync(a, b);483+ }484+ }485+ }486+ };487+ walk(cfg.data, path.join(out, 'data'));488+}489+490+// node:sqlite, without its experimental warning.491+function sqlite(): typeof import('node:sqlite') {492+ const emit = process.emitWarning;493+ process.emitWarning = ((w: string | Error, ...rest: never[]) => (/SQLite/.test(String(w)) ? undefined : emit.call(process, w, ...rest))) as typeof emit;494+ try {495+ return createRequire(import.meta.url)('node:sqlite') as typeof import('node:sqlite');496+ } finally {497+ process.emitWarning = emit;498+ }499+}500+444501 // ---------------------------------------------------------------- main445502 446503 const plural = (n: number, w: string): string => `${n} ${w}${n === 1 ? '' : 's'}`;@@ -478,6 +535,9 @@ async function main(argv: string[]): Promise<void> {
478535 process.stdout.write(await query(cfg, sql, fmt));479536 } else if (cmd === 'install') {480537 await install(cfg);538+ } else if (cmd === 'stage') {539+ if (!rest[0]) throw new LakeError('usage: node lib/lake.ts stage <dir>');540+ stage(cfg, rest[0]);481541 } else {482542 throw new LakeError('usage: forge lake [--dry-run] | forge lake query "<sql>" [--json|--csv]');483543 }@@ -0,0 +1,46 @@
1+# A backup stages the lake, which nothing can rebuild: a checked copy of2+# its catalog (VACUUM INTO, never the live file) and its data files. A3+# restore brings it back whole, and the next run there adds nothing twice.4+. "$(dirname "$0")/../lib.sh"5+user dami6+user ana7+repo dami/greet8+clone dami dami/greet "$T/w/d"9+at "$T/w/d" commit "first"10+push_as dami "$T/w/d" origin main11+clone ana dami/greet "$T/w/a"12+at "$T/w/a" commit "idea"13+push_as ana "$T/w/a" origin HEAD:ana/idea14+build dami/greet15+# Enough events that some go to a Parquet file rather than the catalog.16+i=017+while [ "$i" -lt 30 ]; do18+ i=$((i + 1))19+ printf '{"t":%s000,"repo":"dami/greet","type":"push","user":"dami","refs":"refs/heads/n%s"}\n' "$((T_NOW + i))" "$i" >>"$FORGE_ROOT/.forge/events.jsonl"20+done21+ok forge lake22+facts="SELECT (SELECT count(*) FROM events) || ' ' || (SELECT count(*) FROM builds) || ' ' || (SELECT count(*) FROM reviews)"23+want=$(lq "$facts")24+eq "$want" "33 1 2"25+ls "$FORGE_ROOT"/.forge/lake/data/main/events/*.parquet >/dev/null 2>&1 || fail "no Parquet file in the lake's data"26+ok forge backup --dry-run27+st=$FORGE_ROOT/.forge/backup/stage/.forge28+exists "$st/lake/catalog.sqlite"29+ls "$st"/lake/data/main/events/*.parquet >/dev/null 2>&1 || fail "the lake's data files aren't staged"30+missing "$st/lake.lock"31+missing "$FORGE_ROOT/.forge/lake.lock"32+# A copy, not the live file.33+[ "$(ls -i "$st/lake/catalog.sqlite" | awk '{ print $1 }')" != "$(ls -i "$FORGE_ROOT/.forge/lake/catalog.sqlite" | awk '{ print $1 }')" ] || fail "the staged catalog is the live file"34+r=$T/restored35+ok env FORGE_ROOT="$r" FORGE_WWW="$r/.forge/www" "$FORGE" restore --from "$FORGE_ROOT/.forge/backup/stage"36+exists "$r/.forge/lake/catalog.sqlite"37+eq "$(FORGE_ROOT=$r FORGE_WWW=$r/.forge/www lq "$facts")" "$want"38+# The restored lake reads its own data, not the old forge's.39+rm -rf "$FORGE_ROOT/.forge/lake"40+eq "$(FORGE_ROOT=$r FORGE_WWW=$r/.forge/www lq "SELECT count(DISTINCT refs) FROM events WHERE refs LIKE 'refs/heads/n%'")" 3041+# The restore built every page, so a run there adds those builds and the42+# events they wrote, and none of the review lines or events it already had.43+ok env FORGE_ROOT="$r" FORGE_WWW="$r/.forge/www" "$FORGE" lake44+has "$T_LOG" ', 0 review log lines, 0 ended,'45+eq "$(FORGE_ROOT=$r FORGE_WWW=$r/.forge/www lq "SELECT count(*) FROM reviews")" 246+eq "$(FORGE_ROOT=$r FORGE_WWW=$r/.forge/www lq "SELECT count(*) FROM events WHERE type = 'push'")" 32