- commit
- 13cd5c22732a2c7c45791aee74586587ca489479
- parent
- 44f4aec53de4dd7a6572c0b4b2b16f37e5e7ae38
- author
- Dami <dami.etoile@gmail.com>
| A | test/ | 97 | ++++++++++++++++++++++++++++++ |
| M | test/ | 23 | +++++-- |
@@ -0,0 +1,97 @@
1+// Quiescent test fixtures reconcile the quota ledger against R2 and classify2+// current-format dependencies. This walk is not an online deletion policy.3+import { expect } from "vitest";4+import { npubEncode } from "nostr-tools/nip19";5+import { repositoryAddress, repositoryStoragePrefix, type WalRepository } from "ntig";6+import type { Relay } from "../../src/relay.ts";7+import type { RepositoryAnnouncement } from "../../src/grasp-policy.ts";8+9+const digest = async (bytes: Uint8Array) => Array.from(new Uint8Array(await crypto.subtle.digest("SHA-256", bytes)), (b) => b.toString(16).padStart(2, "0")).join("");10+11+// inventory reconciles an isolated repository after its writer has stopped.12+export async function inventory(relay: Relay, repo: RepositoryAnnouncement, wal: WalRepository) {13+ const prefix = `${relay.slug}/git/${await repositoryStoragePrefix(repositoryAddress(npubEncode(repo.owner), repo.identifier))}data/`;14+ const root = await relay.media.get(`${prefix}root.json`);15+ const rootText = root ? await root.text() : null;16+ const snapshot = await wal.load();17+ const physical = new Map<string, number>();18+ let cursor: string | undefined;19+ do {20+ const page = await relay.media.list({ prefix, cursor, limit: 100 });21+ for (const item of page.objects) physical.set(item.key, item.size);22+ cursor = page.truncated ? page.cursor : undefined;23+ } while (cursor);24+ const ledger = relay.sql.exec<{ key: string; size: number }>("SELECT key, size FROM grasp_objects WHERE owner=?", repo.owner).toArray().filter(({ key }) => key.startsWith(prefix));25+ expect(new Map(ledger.map(({ key, size }) => [key, size]))).toEqual(physical);26+ const live = new Set<string>();27+ const records = new Map<string, string>();28+ const sequences = new Set<number>();29+ const read = async (family: string, hash: string) => {30+ expect(hash).toMatch(/^[0-9a-f]{64}$/);31+ const key = `${prefix}${family}/${hash}`;32+ const item = await relay.media.get(key);33+ expect(item).not.toBeNull();34+ const bytes = new Uint8Array(await item!.arrayBuffer());35+ expect(bytes.length).toBe(physical.get(key));36+ expect(await digest(bytes)).toBe(hash);37+ live.add(key);38+ return JSON.parse(new TextDecoder().decode(bytes));39+ };40+ if (root) live.add(root.key);41+ if (snapshot.checkpoint) {42+ await read("manifests", snapshot.checkpoint.manifestHash);43+ for (const id of snapshot.checkpoint.packIds) live.add(`${prefix}packs/${id}`);44+ const walk = async (hash: string, route = ""): Promise<void> => {45+ expect(route.length).toBeLessThanOrEqual(64);46+ expect(live.has(`${prefix}receipt-index/${hash}`)).toBe(false);47+ const node = await read("receipt-index", hash);48+ expect(node.v).toBe(1);49+ if (node.t === "l") {50+ for (const [key, recordHash] of node.e as [string, string][]) {51+ expect(key.startsWith(route)).toBe(true);52+ const record = await read("records", recordHash);53+ expect(await digest(new TextEncoder().encode(record.id))).toBe(key);54+ expect(records.has(record.id)).toBe(false);55+ records.set(record.id, recordHash);56+ sequences.add(record.sequence);57+ expect(await wal.lookupRecord(record.id)).toEqual(record);58+ }59+ } else {60+ expect(node.t).toBe("b");61+ expect(node.c).toHaveLength(16);62+ for (const [i, child] of (node.c as (string | null)[]).entries()) {63+ if (child !== null) await walk(child, route + i.toString(16));64+ }65+ }66+ };67+ if (snapshot.checkpoint.receiptRoot) await walk(snapshot.checkpoint.receiptRoot);68+ expect(records.size).toBe(snapshot.sequence);69+ expect([...sequences].sort((a, b) => a - b)).toEqual(Array.from({ length: snapshot.sequence }, (_, i) => i + 1));70+ } else {71+ let tip = snapshot.tip;72+ while (tip) {73+ expect(live.has(`${prefix}records/${tip}`)).toBe(false);74+ const record = await read("records", tip);75+ if (record.pack) live.add(`${prefix}packs/${record.pack}`);76+ tip = record.parent;77+ }78+ }79+ for (const key of live) expect(physical.has(key)).toBe(true);80+ const categories: Record<string, { objects: number; bytes: number }> = {};81+ for (const [key, bytes] of physical) {82+ const family = key.slice(prefix.length).split("/")[0];83+ const category = `${live.has(key) ? "current" : "unreferenced"}/${family}`;84+ const row = categories[category] ??= { objects: 0, bytes: 0 };85+ row.objects++; row.bytes += bytes;86+ }87+ const after = await relay.media.get(`${prefix}root.json`);88+ expect(after?.etag).toBe(root?.etag);89+ expect(after ? await after.text() : null).toBe(rootText);90+ return {91+ sequence: snapshot.sequence, refs: Object.keys(snapshot.refs).length,92+ physicalBytes: [...physical.values()].reduce((a, b) => a + b, 0),93+ reservedBytes: ledger.reduce((n, row) => n + row.size, 0),94+ currentBytes: [...live].reduce((n, key) => n + physical.get(key)!, 0),95+ objects: physical.size, categories,96+ };97+}@@ -11,6 +11,7 @@ import { gitRepository } from "../../src/grasp.ts";
1111 import type { RepositoryAnnouncement } from "../../src/grasp-policy.ts";1212 import { ev, pk, rpc } from "../helpers/relay.ts";1313 import { WS } from "../helpers/ws.ts";14+import { inventory } from "../helpers/grasp-inventory.ts";1415 1516 const encoder = new TextEncoder();1617 const oid = async (type: string, data: Uint8Array) => {@@ -38,8 +39,8 @@ const announcement = (sk: Uint8Array, host: string, identifier: string) => ev(sk
3839 ]);3940 4041 describe("GRASP checkpoint integration", () => {41- it("migrates an isolated legacy repository and retains indexed receipts", async () => {42- const host = "grasp-checkpoint.bind.ws";42+ it.each(["growing", "fixed"])("migrates an isolated %s-ref repository and reconciles retained storage", async (workload) => {43+ const host = `grasp-checkpoint-${workload}.bind.ws`;4344 const owner = generateSecretKey();4445 const identifier = "checkpoint";4546 await rpc(host, owner, "claim");@@ -54,12 +55,14 @@ describe("GRASP checkpoint integration", () => {
5455 const repo = repository(relay, pk(owner), identifier) as RepositoryAnnouncement | null;5556 expect(repo).not.toBeNull();5657 const wal = await gitRepository(relay, repo!);58+ const empty = await inventory(relay, repo!, wal);5759 const first = await wal.commit({ id: "checkpoint-initial", updates: [{ name: "refs/heads/main", old: null, new: fixture.commitID }], pack: fixture.pack });60+ const oneCommit = await inventory(relay, repo!, wal);5861 const legacy: WalRecord[] = [];5962 legacy.push((await wal.lookupRecord!(first.id))!);6063 let prior = fixture.commitID;6164 for (let i = 1; i < 128; i++) {62- const receipt = await wal.commit({ id: `checkpoint-legacy-${i}`, updates: [{ name: `refs/tags/legacy-${i}`, old: null, new: prior }] });65+ const receipt = await wal.commit({ id: `checkpoint-legacy-${i}`, updates: [{ name: workload === "growing" ? `refs/tags/legacy-${i}` : "refs/heads/main", old: workload === "growing" ? null : prior, new: prior }] });6366 legacy.push((await wal.lookupRecord!(receipt.id))!);6467 }6568 expect((await wal.load()).sequence).toBe(128);@@ -77,14 +80,14 @@ describe("GRASP checkpoint integration", () => {
7780 // Format 2 allows more than the legacy 128-record ceiling. Ref-only7881 // commits exercise manifest/index growth without creating more packs.7982 for (let i = 128; i < 260; i++) {80- await wal.commit({ id: `checkpoint-v2-${i}`, updates: [{ name: `refs/tags/v2-${i}`, old: null, new: prior }] });83+ await wal.commit({ id: `checkpoint-v2-${i}`, updates: [{ name: workload === "growing" ? `refs/tags/v2-${i}` : "refs/heads/main", old: workload === "growing" ? null : prior, new: prior }] });8184 }8285 const replay = await wal.commit({ id: first.id, updates: [{ name: "refs/heads/main", old: null, new: fixture.commitID }], pack: fixture.pack });8386 expect(replay).toMatchObject({ id: first.id, sequence: 1, replayed: true });8487 const rebuilt = await gitRepository(relay, repo!);8588 const cold = await rebuilt.load();8689 expect(cold.sequence).toBe(260);87- expect(cold.refs["refs/tags/v2-259"]).toBe(fixture.commitID);90+ expect(cold.refs[workload === "growing" ? "refs/tags/v2-259" : "refs/heads/main"]).toBe(fixture.commitID);8891 const advertised = await rebuilt.loadRefs!();8992 expect(advertised.sequence).toBe(260);9093 expect(advertised.refs["refs/heads/main"]).toBe(fixture.commitID);@@ -98,10 +101,13 @@ describe("GRASP checkpoint integration", () => {
98101 counts[category] = (counts[category] ?? 0) + 1;99102 return counts;100103 }, {});101- return { before, after, keyCount: keys.length, categories, sequence: cold.sequence };104+ const retained = await inventory(relay, repo!, rebuilt);105+ return { before, after, keyCount: keys.length, categories, sequence: cold.sequence, empty, oneCommit, retained };102106 });103107 108+ console.info("checkpoint inventory", JSON.stringify({ workload, empty: measurements.empty, oneCommit: measurements.oneCommit, retained: measurements.retained }));104109 console.info("checkpoint storage measurement", {110+ workload,105111 beforeObjects: measurements.before.n,106112 afterObjects: measurements.after.n,107113 beforeBytes: measurements.before.bytes,@@ -117,6 +123,11 @@ describe("GRASP checkpoint integration", () => {
117123 expect(measurements.after.bytes).toBeGreaterThan(measurements.before.bytes);118124 expect(measurements.keyCount).toBeGreaterThan(128);119125 expect(measurements.sequence).toBe(260);126+ expect(measurements.empty.physicalBytes).toBe(0);127+ expect(measurements.empty.reservedBytes).toBe(0);128+ expect(measurements.retained.currentBytes).toBeLessThan(measurements.retained.physicalBytes);129+ expect(measurements.retained.categories["unreferenced/records"]).toBeUndefined();130+ expect(measurements.retained.categories["unreferenced/packs"]).toBeUndefined();120131 });121132 122133 it("retains a reservation when an immutable checkpoint write is ambiguous", async () => {