- commit
- 05ac41274d12b3aa2938badbd5f7aad730d9a848
- parent
- c2b58176f2346d8ea7517e85916579d16a51881e
- author
- Dami <dami.etoile@gmail.com>
| A | src/ | 49 | ++++++++ |
| M | src/ | 9 | + |
| M | test/ | 11 | ++ |
| A | test/ | 194 | ++++++++++++++++++++++++++++++ |
| M | test/ | 15 | ++- |
@@ -0,0 +1,49 @@
1+// Owner-requested Git storage inventory. Repository identity selects the R22+// namespace; server limits bound work and the report never permits deletion.3+import { boundedInventory, R2InventoryListing, R2ObjectStore, LimitError, type InventoryLimits } from "ntig";4+import { repository } from "./grasp-state.ts";5+import { gitStoragePrefix } from "./grasp.ts";6+import { featureOn } from "./settings.ts";7+import { now } from "./event.ts";8+import type { Relay } from "./relay.ts";9+10+export const GIT_INVENTORY_LIMITS: Readonly<InventoryLimits> = Object.freeze({11+ maxGets: 10_000, maxReadBytes: 16 * 1024 * 1024, maxObjectBytes: 4 * 1024 * 1024,12+ maxListedKeys: 10_000, maxPages: 100, maxIndexNodes: 10_000, maxReceipts: 10_000,13+ maxKeyBytes: 1024, maxCursorBytes: 8192,14+});15+const nextInventory = new WeakMap<Relay, number>();16+const failure = (error: string, status: number) => ({ body: { error }, status });17+18+// gitStorage compares one complete physical inventory with quota reservations.19+// The cooldown limits repeated manual scans in this instance, including failures.20+export async function gitStorage(relay: Relay, owner: string, identifier: string) {21+ return relay.repositoryAccess.run("control", async () => {22+ if (!featureOn(relay.settings.policy, "grasp")) return failure("not found", 404);23+ if (relay.settings.isUnclaimed() || relay.settings.leaseExpired(now())) return failure("restricted: relay is not active", 403);24+ const repo = repository(relay, owner, identifier);25+ if (!repo) return failure("not found", 404);26+ if (relay.fuelStatus().outOfFuel) return failure("restricted: relay storage or fuel limit reached", 403);27+ const retryAfter = (nextInventory.get(relay) ?? 0) - now();28+ if (retryAfter > 0) return { body: { error: "restricted: Git inventory cooldown; retry", retryAfter }, status: 429 };29+ nextInventory.set(relay, now() + 60);30+ const operations = { gets: 0, lists: 0 };31+ let completed = false;32+ try {33+ const prefix = await gitStoragePrefix(relay, repo);34+ const r2 = new R2ObjectStore(relay.media, { prefix, maxObjectBytes: GIT_INVENTORY_LIMITS.maxObjectBytes });35+ const listing = new R2InventoryListing({ list: (options) => { operations.lists++; return relay.media.list(options); } }, { prefix, pageSize: 1000 });36+ const inventory = await boundedInventory({ get: (key) => { operations.gets++; return r2.get(key); } }, listing, { prefix: "data/", limits: GIT_INVENTORY_LIMITS });37+ const reservations = relay.sql.exec<{ keys: number; bytes: number }>(`SELECT count(*) AS keys, coalesce(sum(size),0) AS bytes FROM grasp_objects WHERE substr(key,1,?)=?`, prefix.length, prefix).one();38+ completed = true;39+ return { body: { result: { repository: { owner, identifier, announcement: repo.id }, inventory, reservations,40+ reservationMinusListedBytes: reservations.bytes - inventory.listed.bytes, operations, limits: GIT_INVENTORY_LIMITS,41+ cooldownSeconds: 60, capturedAt: now() } }, status: 200 };42+ } catch (error) {43+ return error instanceof LimitError ? failure("restricted: Git inventory limit reached; no complete report", 413)44+ : failure("error: Git inventory unavailable; no complete report", 503);45+ } finally {46+ console.log(JSON.stringify({ msg: "git-inventory", relay: relay.slug, ...operations, completed }));47+ }48+ }, () => failure("restricted: relay operation in progress; retry", 429));49+}@@ -25,6 +25,7 @@ import { PRESETS, applyPreset, findPreset } from "./presets.ts";
2525 import { addDomain, checkDomain, listDomains, removeDomain, setDomainSite } from "./domains.ts";2626 import { verifyNIP98 } from "./auth.ts";2727 import { SITE_KINDS, checkSite, siteLabel, sitePaths } from "./sites.ts";28+import { gitStorage } from "./git-storage.ts";2829 2930 // A call: the relay and the request, who is calling and as what, the3031 // parameters with their readers, and how to answer.@@ -455,6 +456,14 @@ export const METHODS: Record<string, Method> = {
455456 return reply({ result: { kinds, events: kinds.reduce((a, k) => a + k.n, 0), eventBytes: kinds.reduce((a, k) => a + k.bytes, 0), databaseBytes: relay.eventBytes(), blobs: blobs.n ?? 0, mediaBytes: blobs.bytes ?? 0, dumps: listDumps(relay.sql).length, dumpBytes: dumpBytes(relay.sql), retention: s.listRetention() } });456457 },457458 },459+ gitstorage: {460+ action: "storage", reads: true,461+ run: async ({ relay, params, str, hex64, reply }) => {462+ if (params.length !== 2 || !hex64(str(0)) || typeof params[1] !== "string" || !str(1) || new TextEncoder().encode(str(1)).length > 256) return reply({ error: "invalid: expected repository owner hex pubkey and identifier" }, 400);463+ const result = await gitStorage(relay, str(0), str(1));464+ return reply(result.body, result.status);465+ },466+ },458467 setretention: {459468 action: "rules",460469 run: ({ s, params, num, reply }) => {@@ -81,6 +81,17 @@ function doors(f: Fixture): { path: string; host?: string; method?: string; gate
8181 const leaks = (text: string, secrets: string[]) => secrets.filter((s) => text.includes(s));8282 8383 describe("the read rule at every door", () => {84+ it("keeps Git storage inventory owner-only at the management door", async () => {85+ const host = "exposure-git-storage.bind.ws";86+ const owner = generateSecretKey();87+ const stranger = generateSecretKey();88+ await rpc(host, owner, "claim");89+ expect((await rpc(host, null, "gitstorage", pk(owner), "missing")).status).toBe(401);90+ expect((await rpc(host, stranger, "gitstorage", pk(owner), "missing")).status).toBe(403);91+ const methods = (await rpc(host, owner, "supportedmethods")).result as string[];92+ expect(methods).toContain("gitstorage");93+ });94+8495 it("shows a stranger nothing of a member, her note or her file", async () => {8596 const f = await seed("exposure-stranger.bind.ws");8697 for (const d of doors(f)) {@@ -0,0 +1,194 @@
1+// Owner-only inventory reports compare a bounded physical listing with the2+// repository root and the durable reservation ledger. They never mutate R2.3+import { env, runInDurableObject } from "cloudflare:test";4+import { describe, expect, it } from "vitest";5+import { generateSecretKey } from "nostr-tools/pure";6+import { npubEncode } from "nostr-tools/nip19";7+import { encodePack } from "ntig";8+import { KIND_REPO, KIND_REPO_STATE } from "../../src/kinds.ts";9+import { gitRepositoryPath } from "../../src/grasp-policy.ts";10+import { repository } from "../../src/grasp-state.ts";11+import { gitRepository } from "../../src/grasp.ts";12+import type { Relay } from "../../src/relay.ts";13+import { ev, nip98, pk, rpc } from "../helpers/relay.ts";14+import { WS } from "../helpers/ws.ts";15+16+const encoder = new TextEncoder();17+const objectID = async (type: string, bytes: Uint8Array) => {18+ const head = encoder.encode(`${type} ${bytes.length}\0`);19+ const input = new Uint8Array(head.length + bytes.length);20+ input.set(head); input.set(bytes, head.length);21+ return Array.from(new Uint8Array(await crypto.subtle.digest("SHA-1", input)), (b) => b.toString(16).padStart(2, "0")).join("");22+};23+async function tinyPack() {24+ const blob = encoder.encode("inventory\n");25+ const blobID = await objectID("blob", blob);26+ const tree = new Uint8Array([...encoder.encode("100644 README\0"), ...Uint8Array.from(blobID.match(/../g)!, (x) => parseInt(x, 16))]);27+ const treeID = await objectID("tree", tree);28+ const commit = encoder.encode(`tree ${treeID}\nauthor inventory <inventory@example.com> 1 +0000\ncommitter inventory <inventory@example.com> 1 +0000\n\ninitial\n`);29+ const commitID = await objectID("commit", commit);30+ return { commitID, pack: await encodePack([{ type: "blob", data: blob }, { type: "tree", data: tree }, { type: "commit", data: commit }]) };31+}32+async function seed(host: string, format = 1) {33+ const owner = generateSecretKey();34+ const stranger = generateSecretKey();35+ const moderator = generateSecretKey();36+ const identifier = "inventory";37+ await rpc(host, owner, "claim");38+ await rpc(host, owner, "applypreset", "grasp");39+ await rpc(host, owner, "setmember", pk(moderator), { role: "moderator" });40+ const path = gitRepositoryPath(npubEncode(pk(owner)), identifier);41+ const c = await WS.connect(host);42+ const announcement = ev(owner, KIND_REPO, "", [["d", identifier], ["clone", `https://${host}${path}`], ["relays", `wss://${host}`], ["maintainers", pk(owner)]]);43+ expect((await c.ok(announcement)).ok).toBe(true);44+ c.ws.close();45+ const pack = await tinyPack();46+ await runInDurableObject(env.RELAY.getByName(host.split(".")[0]), async (relay) => {47+ const repo = repository(relay, pk(owner), identifier)!;48+ await relay.repositoryAccess.run("control", async () => {49+ const wal = await gitRepository(relay, repo);50+ await wal.commit({ id: "inventory-initial", updates: [{ name: "refs/heads/main", old: null, new: pack.commitID }], pack: pack.pack });51+ if (format === 2) await wal.checkpoint();52+ }, () => Promise.reject(new Error("repository access unexpectedly refused during fixture setup")));53+ });54+ return { owner, stranger, moderator, identifier };55+}56+57+async function scan(relay: Relay, host: string, owner: Uint8Array, identifier: string) {58+ const url = `http://${host}/`;59+ const payload = { method: "gitstorage", params: [pk(owner), identifier] };60+ const authorization = await nip98(owner, url, "POST", payload);61+ return relay.fetch(new Request(url, { method: "POST", headers: { authorization, "content-type": "application/nostr+json+rpc" }, body: JSON.stringify(payload) }));62+}63+64+function proxyBucket(relay: Relay, overrides: Record<string, unknown>) {65+ const original = relay.media;66+ const calls: string[] = [];67+ const observed = new Proxy(original, { get(target, name) {68+ calls.push(String(name));69+ if (name in overrides) return overrides[name as string];70+ if (name !== "get" && name !== "list") throw new Error(`unexpected R2 method ${String(name)}`);71+ const value = Reflect.get(target, name, target);72+ return typeof value === "function" ? value.bind(target) : value;73+ } });74+ Object.defineProperty(relay, "media", { value: observed, configurable: true });75+ return { calls, restore: () => Object.defineProperty(relay, "media", { value: original, configurable: true }) };76+}77+78+describe("Git storage inventory management", () => {79+ it.each([1, 2])("returns a complete format-%i report only to the owner", async (format) => {80+ const host = `git-storage-owner-${format}.bind.ws`;81+ const f = await seed(host, format);82+ const report = await rpc(host, f.owner, "gitstorage", pk(f.owner), f.identifier);83+ expect(report.status, JSON.stringify(report)).toBe(200);84+ expect(report.result.inventory.format).toBe(format);85+ expect(report.result.inventory.sequence).toBe(1);86+ expect(report.result.inventory.listed.keys).toBeGreaterThanOrEqual(3);87+ expect(report.result.inventory.listed.bytes).toBeGreaterThan(0);88+ expect(report.result.inventory.live.keys).toBe(report.result.inventory.listed.keys);89+ expect(report.result.reservations.keys).toBe(report.result.inventory.listed.keys);90+ expect(report.result.reservationMinusListedBytes).toBe(0);91+ expect(report.result.operations.gets).toBeGreaterThan(0);92+ expect(report.result.operations.lists).toBeGreaterThan(0);93+94+ expect((await rpc(host, null, "gitstorage", pk(f.owner), f.identifier)).status).toBe(401);95+ expect((await rpc(host, f.stranger, "gitstorage", pk(f.owner), f.identifier)).status).toBe(403);96+ expect((await rpc(host, f.moderator, "gitstorage", pk(f.owner), f.identifier)).status).toBe(403);97+ expect((await rpc(host, f.owner, "gitstorage", "not-a-pubkey", f.identifier)).status).toBe(400);98+ expect((await rpc(host, f.owner, "gitstorage", pk(f.owner), "")).status).toBe(400);99+ expect((await rpc(host, f.owner, "gitstorage", pk(f.owner), "../data")).status).toBe(404);100+ expect((await rpc(host, f.owner, "gitstorage", pk(f.owner), f.identifier)).status).toBe(429);101+ });102+103+ it("uses only R2 get and list for a successful owner scan and releases its control scope", async () => {104+ const host = "git-storage-methods.bind.ws";105+ const f = await seed(host);106+ const result = await runInDurableObject(env.RELAY.getByName(host.split(".")[0]), async (relay) => {107+ const bucket = proxyBucket(relay, {});108+ try {109+ const response = await scan(relay, host, f.owner, f.identifier);110+ expect(response.status).toBe(200);111+ const report = await response.json<any>();112+ expect(report.result.reservations.bytes).toBe(report.result.inventory.listed.bytes);113+ expect(report.result.inventory.authority).toBe(false);114+ expect(report.result.inventory.gcCandidate).toBe(false);115+ const before = bucket.calls.length;116+ expect((await scan(relay, host, f.owner, f.identifier)).status).toBe(429);117+ expect(bucket.calls).toHaveLength(before);118+ return { calls: bucket.calls, busy: relay.repositoryAccess.busy };119+ } finally { bucket.restore(); }120+ });121+ expect(result.calls.filter((name: string) => name === "get" || name === "list").length).toBe(result.calls.length);122+ expect(result.calls).toContain("get");123+ expect(result.calls).toContain("list");124+ expect(result.busy).toBe(false);125+ });126+127+ it("returns 503 without a report when R2 fails and releases the control scope", async () => {128+ const host = "git-storage-failure.bind.ws";129+ const f = await seed(host);130+ await runInDurableObject(env.RELAY.getByName(host.split(".")[0]), async (relay) => {131+ const bucket = proxyBucket(relay, { get: async () => { throw new Error("provider failure"); } });132+ try {133+ const response = await scan(relay, host, f.owner, f.identifier);134+ expect(response.status).toBe(503);135+ expect(await response.json<any>()).toEqual({ error: "error: Git inventory unavailable; no complete report" });136+ expect(relay.repositoryAccess.busy).toBe(false);137+ } finally { bucket.restore(); }138+ });139+ });140+141+ it("returns 413 for an oversized listing page without leaving a lease", async () => {142+ const host = "git-storage-budget.bind.ws";143+ const f = await seed(host);144+ await runInDurableObject(env.RELAY.getByName(host.split(".")[0]), async (relay) => {145+ const bucket = proxyBucket(relay, { list: async () => ({ objects: Array.from({ length: 10001 }, (_, i) => ({ key: `${relay.slug}/git/data/${i}`, size: 1 })), truncated: false, cursor: undefined }) });146+ try {147+ const response = await scan(relay, host, f.owner, f.identifier);148+ expect(response.status).toBe(413);149+ expect(await response.json<any>()).toEqual({ error: "restricted: Git inventory limit reached; no complete report" });150+ expect(relay.repositoryAccess.busy).toBe(false);151+ } finally { bucket.restore(); }152+ });153+ });154+155+ it("holds control admission across the first R2 read and refuses Git, event and control work", async () => {156+ const host = "git-storage-coordination.bind.ws";157+ const f = await seed(host);158+ await runInDurableObject(env.RELAY.getByName(host.split(".")[0]), async (relay) => {159+ let entered!: () => void;160+ let release!: () => void;161+ const ready = new Promise<void>((resolve) => { entered = resolve; });162+ const blocked = new Promise<void>((resolve) => { release = resolve; });163+ const original = relay.media;164+ const bucket = proxyBucket(relay, { get: async (key: string, options?: R2GetOptions) => {165+ if (key.endsWith("/root.json")) { entered(); await blocked; }166+ return original.get(key, options);167+ } });168+ const active = scan(relay, host, f.owner, f.identifier);169+ try {170+ await ready;171+ expect(relay.repositoryAccess.busy).toBe(true);172+ expect(relay.repositoryAccess.kind).toBe("control");173+ const path = gitRepositoryPath(npubEncode(pk(f.owner)), f.identifier);174+ const git = await relay.fetch(new Request(`http://${host}${path}/info/refs?service=git-upload-pack`));175+ expect(git.status).toBe(429);176+ const event = ev(f.owner, KIND_REPO_STATE, "", [["d", f.identifier], ["refs/heads/main", "a".repeat(40)]]);177+ const admitted = await relay.acceptAny(event, relay.virtualConn(host, pk(f.owner)));178+ expect(admitted.ok).toBe(false);179+ expect(admitted.msg).toContain("relay operation in progress");180+ const control = await scan(relay, host, f.owner, f.identifier);181+ expect(control.status).toBe(429);182+ release();183+ const response = await active;184+ expect(response.status).toBe(200);185+ await response.arrayBuffer();186+ } finally {187+ release();188+ await active;189+ bucket.restore();190+ }191+ expect(relay.repositoryAccess.busy).toBe(false);192+ });193+ });194+});@@ -7,6 +7,7 @@ import { npubEncode } from "nostr-tools/nip19";
77 import { encodePack, type WalRecord } from "ntig";88 import { KIND_REPO } from "../../src/kinds.ts";99 import { repository } from "../../src/grasp-state.ts";10+import { gitStorage } from "../../src/git-storage.ts";1011 import { gitRepository } from "../../src/grasp.ts";1112 import type { RepositoryAnnouncement } from "../../src/grasp-policy.ts";1213 import { ev, pk, rpc } from "../helpers/relay.ts";@@ -51,7 +52,7 @@ describe("GRASP checkpoint integration", () => {
5152 connection.ws.close();5253 5354 const fixture = await fixturePack();54- const measurements = await runInDurableObject(env.RELAY.getByName(host.split(".")[0]), async (relay) => {55+ const measurements = await runInDurableObject(env.RELAY.getByName(host.split(".")[0]), async (relay) => relay.repositoryAccess.run("control", async () => {5556 const repo = repository(relay, pk(owner), identifier) as RepositoryAnnouncement | null;5657 expect(repo).not.toBeNull();5758 const wal = await gitRepository(relay, repo!);@@ -102,8 +103,18 @@ describe("GRASP checkpoint integration", () => {
102103 return counts;103104 }, {});104105 const retained = await inventory(relay, repo!, rebuilt);106+ const scan = await gitStorage(relay, pk(owner), identifier);107+ expect(scan.status).toBe(200);108+ if (!("result" in scan.body)) throw new Error("inventory did not complete");109+ const report = scan.body.result;110+ if (!report) throw new Error("inventory report missing");111+ expect(report.inventory.listed.bytes).toBe(retained.physicalBytes);112+ expect(report.inventory.live.bytes).toBe(retained.currentBytes);113+ expect(report.reservations.bytes).toBe(retained.reservedBytes);114+ expect(report.inventory.unknown.bytes).toBe(0);115+ console.info("runtime checkpoint inventory", JSON.stringify({ workload, physical: report.inventory.listed.bytes, live: report.inventory.live.bytes, ...report.operations, ...report.inventory.observed }));105116 return { before, after, keyCount: keys.length, categories, sequence: cold.sequence, empty, oneCommit, retained };106- });117+ }, () => { throw new Error("fixture admission refused"); }));107118 108119 console.info("checkpoint inventory", JSON.stringify({ workload, empty: measurements.empty, oneCommit: measurements.oneCommit, retained: measurements.retained }));109120 console.info("checkpoint storage measurement", {