- commit
- 5d1179c2a5dbdc3784e9dfb02173c0bd4ba79601
- parent
- 5754a9ca78f02fb3f20e4a69f8016598eb2c79d9
- author
- Dami <dami.etoile@gmail.com>
| A | src/ | 43 | +++++ |
| M | src/ | 39 | +--- |
| M | src/ | 28 | +-- |
| M | src/ | 221 | ++++++++++++++++----------- |
| M | src/ | 2 | +- |
| D | test/ | 97 | ------------ |
| M | test/ | 168 | ++++---------------- |
| D | test/ | 245 | ------------------------------ |
| M | test/ | 44 | ++---- |
| M | test/ | 4 | +- |
| M | test/ | 205 | +++++++------------------ |
@@ -0,0 +1,43 @@
1+// Git object reads share one bounded view. Indexed repositories fetch by ID;2+// legacy repositories decode their bounded retained pack set once per view.3+import { sha1 } from "@noble/hashes/legacy.js";4+import { bytesToHex } from "@noble/hashes/utils.js";5+import { readObjects, objectLinks, LimitError, IntegrityError, type GitRepository, type GitObject, type GitObjectInfo } from "ntig";6+7+export async function gitObjects(repository: GitRepository) {8+ const fallback = repository.getObject && repository.getObjectInfo ? null : await readObjects((await repository.load()).packs);9+ return {10+ get: async (oid: string): Promise<GitObject | null> => fallback ? fallback.get(oid) ?? null : repository.getObject!(oid),11+ info: async (oid: string): Promise<GitObjectInfo | null> => {12+ if (!fallback) return repository.getObjectInfo!(oid);13+ const object = fallback.get(oid);14+ return object ? { oid, type: object.type, size: object.data.length, links: objectLinks(object) } : null;15+ },16+ };17+}18+19+// gitGraph reads only the requested dependency graph, with the same decoded20+// byte budget as a received pack. It also serves local GRASP-06 reconciliation.21+export async function gitGraph(repository: GitRepository, roots: readonly string[]): Promise<GitObject[]> {22+ const reader = await gitObjects(repository);23+ const seen = new Map<string, GitObjectInfo["type"]>(), queue = roots.map(oid => ({ oid, type: undefined as GitObjectInfo["type"] | undefined })), result: GitObject[] = [];24+ let bytes = 0, edges = 0;25+ while (queue.length) {26+ const { oid, type } = queue.pop()!;27+ if (seen.has(oid)) { if (type && seen.get(oid) !== type) throw new IntegrityError("Git dependency type mismatch"); continue; }28+ if (seen.size >= 4096) throw new LimitError("Git graph object limit reached");29+ const info = await reader.info(oid);30+ if (!info || info.oid !== oid || (type && info.type !== type)) throw new IntegrityError("Git graph dependency is missing");31+ if (!Number.isSafeInteger(info.size) || info.size < 0 || info.size > 4 * 1024 * 1024 || bytes + info.size > 16 * 1024 * 1024) throw new LimitError("Git graph byte limit reached");32+ bytes += info.size;33+ edges += info.links.length;34+ if (edges > 65_536) throw new LimitError("Git graph edge limit reached");35+ const object = await reader.get(oid);36+ if (!object || object.oid !== oid || object.type !== info.type || object.data.length !== info.size) throw new IntegrityError("Git graph changed during read");37+ if (bytesToHex(sha1.create().update(new TextEncoder().encode(`${object.type} ${object.data.length}\0`)).update(object.data).digest()) !== oid) throw new IntegrityError("Git object hash mismatch");38+ seen.set(oid, info.type);39+ result.push(object);40+ for (const link of info.links) queue.push(link);41+ }42+ return result;43+}@@ -1,22 +1,16 @@
1-// Owner-requested Git storage inventory. Repository identity selects the R22-// namespace; server limits bound work and the report never permits deletion.3-import { boundedInventory, R2InventoryListing, R2ObjectStore, LimitError, type InventoryLimits } from "ntig";1+// Owner-requested Git inventory reads bounded SQLite metadata. The database's2+// physical byte meter already includes these objects, refs and retry receipts.43 import { repository } from "./grasp-state.ts";54 import { gitStoragePrefix } from "./grasp.ts";5+import { DEFAULT_GIT_SQLITE_LIMITS } from "./git-sqlite.ts";66 import { featureOn } from "./settings.ts";77 import { now } from "./event.ts";88 import type { Relay } from "./relay.ts";99 10-export const GIT_INVENTORY_LIMITS: Readonly<InventoryLimits> = Object.freeze({11- maxGets: 10_000, maxReadBytes: 16 * 1024 * 1024, maxObjectBytes: 4 * 1024 * 1024,12- maxListedKeys: 10_000, maxPages: 100, maxIndexNodes: 10_000, maxReceipts: 10_000,13- maxKeyBytes: 1024, maxCursorBytes: 8192,14-});1510 const nextInventory = new WeakMap<Relay, number>();1611 const failure = (error: string, status: number) => ({ body: { error }, status });1712 18-// gitStorage compares one complete physical inventory with quota reservations.19-// The cooldown limits repeated manual scans in this instance, including failures.13+// gitStorage reports one repository's retained payload and the shared database.2014 export async function gitStorage(relay: Relay, owner: string, identifier: string) {2115 return relay.repositoryAccess.run("control", async () => {2216 if (!featureOn(relay.settings.policy, "grasp")) return failure("not found", 404);@@ -27,23 +21,12 @@ export async function gitStorage(relay: Relay, owner: string, identifier: string
2721 const retryAfter = (nextInventory.get(relay) ?? 0) - now();2822 if (retryAfter > 0) return { body: { error: "restricted: Git inventory cooldown; retry", retryAfter }, status: 429 };2923 nextInventory.set(relay, now() + 60);30- const operations = { gets: 0, lists: 0 };31- let completed = false;32- try {33- const prefix = await gitStoragePrefix(relay, repo);34- const r2 = new R2ObjectStore(relay.media, { prefix, maxObjectBytes: GIT_INVENTORY_LIMITS.maxObjectBytes });35- const listing = new R2InventoryListing({ list: (options) => { operations.lists++; return relay.media.list(options); } }, { prefix, pageSize: 1000 });36- const inventory = await boundedInventory({ get: (key) => { operations.gets++; return r2.get(key); } }, listing, { prefix: "data/", limits: GIT_INVENTORY_LIMITS });37- const reservations = relay.sql.exec<{ keys: number; bytes: number }>(`SELECT count(*) AS keys, coalesce(sum(size),0) AS bytes FROM grasp_objects WHERE substr(key,1,?)=?`, prefix.length, prefix).one();38- completed = true;39- return { body: { result: { repository: { owner, identifier, announcement: repo.id }, inventory, reservations,40- reservationMinusListedBytes: reservations.bytes - inventory.listed.bytes, operations, limits: GIT_INVENTORY_LIMITS,41- cooldownSeconds: 60, capturedAt: now() } }, status: 200 };42- } catch (error) {43- return error instanceof LimitError ? failure("restricted: Git inventory limit reached; no complete report", 413)44- : failure("error: Git inventory unavailable; no complete report", 503);45- } finally {46- console.log(JSON.stringify({ msg: "git-inventory", relay: relay.slug, ...operations, completed }));47- }24+ const namespace = (await gitStoragePrefix(relay, repo)).split("/").filter(Boolean).at(-1)!;25+ const cursor = relay.sql.exec<{ count: number; rawBytes: number; compressedBytes: number; metadataBytes: number }>("SELECT object_count AS count, raw_bytes AS rawBytes, compressed_bytes AS compressedBytes, metadata_bytes AS metadataBytes FROM git_sqlite_meta WHERE repository=?", namespace);26+ const objects = cursor.toArray()[0] ?? { count: 0, rawBytes: 0, compressedBytes: 0, metadataBytes: 0 };27+ const counts = relay.sql.exec<{ refs: number; receipts: number }>("SELECT (SELECT count(*) FROM git_sqlite_refs WHERE repository=?) AS refs, (SELECT count(*) FROM git_sqlite_receipts WHERE repository=?) AS receipts", namespace, namespace);28+ const { refs, receipts } = counts.one();29+ relay.meterPush(cursor.rowsRead + counts.rowsRead, cursor.rowsWritten + counts.rowsWritten);30+ return { body: { result: { repository: { owner, identifier, announcement: repo.id }, backend: "sqlite", objects, refs, receipts, physicalDatabaseBytes: relay.eventBytes(), operations: { gets: 0, lists: 0 }, limits: DEFAULT_GIT_SQLITE_LIMITS, cooldownSeconds: 60, capturedAt: now() } }, status: 200 };4831 }, () => failure("restricted: relay operation in progress; retry", 429));4932 }@@ -1,6 +1,6 @@
11 // GRASP-02 Git reconciliation uses signed event targets and one bounded source22 // attempt per alarm. Durable due times survive restarts and rotate failed sources.3-import { fetchGitPack, readObjects, objectLinks, encodePack, sha256, classifyError } from "ntig";3+import { fetchGitPack, encodePack, sha256, classifyError } from "ntig";44 import { now, tag, type Event } from "./event.ts";55 import { featureOn } from "./settings.ts";66 import { KIND_GIT_PR, KIND_GIT_PR_UPDATE } from "./kinds.ts";@@ -9,6 +9,7 @@ import { authorizedRepository, alternativePrAddress, gitRepository, promote } fr
99 import { localName } from "./pull.ts";1010 import type { RepositoryAnnouncement } from "./grasp-policy.ts";1111 import type { Relay } from "./relay.ts";12+import { gitGraph, gitObjects } from "./git-objects.ts";1213 1314 interface Task { id: string; repo: RepositoryAnnouncement; eventId: string; event?: Event; refs: Record<string, string>; sources: string[]; }1415 const HOUR = 3600;@@ -66,9 +67,11 @@ async function reconcile(relay: Relay, task: Task, attempt: number): Promise<boo
6667 if (!task.event) for (const name of Object.keys(snapshot.refs)) if (!name.startsWith("refs/nostr/")) names.add(name);6768 const updates = [...names].filter(name => (snapshot.refs[name] ?? null) !== (task.refs[name] ?? null)).map(name => ({ name, old: snapshot.refs[name] ?? null, new: task.refs[name] ?? null }));6869 if (!updates.length) { await promote(relay, task.repo, wal); return true; }69- const current = await wal.load();70- const objects = await readObjects(current.packs);71- const missing = [...new Set(Object.values(task.refs))].filter(oid => !objects.has(oid));70+ const objects = await gitObjects(wal);71+ const missing: string[] = [];72+ for (const oid of new Set(Object.values(task.refs))) {73+ if (!(await objects.info(oid))) missing.push(oid);74+ }7275 let pack: Uint8Array | undefined;7376 if (missing.length && task.event) {7477 // A contributor may have uploaded here through GRASP-06. Copy its@@ -78,21 +81,8 @@ async function reconcile(relay: Relay, task: Task, attempt: number): Promise<boo
7881 const local = await gitRepository(relay, address, true);7982 const ref = `refs/nostr/${task.event.id}`;8083 if ((await local.loadRefs()).refs[ref] === tag(task.event, "c")) {81- const available = await readObjects((await local.load()).packs);82- const needed = new Set<string>(), queue = new Set(missing);83- while (queue.size) {84- const oid = queue.values().next().value!;85- queue.delete(oid);86- if (needed.has(oid)) continue;87- const object = available.get(oid);88- if (!object) throw new Error("Local PR object graph is incomplete");89- needed.add(oid);90- for (const link of objectLinks(object)) {91- if (!available.has(link.oid)) throw new Error("Local PR dependency is missing");92- if (!needed.has(link.oid)) queue.add(link.oid);93- }94- }95- pack = await encodePack([...needed].map(oid => available.get(oid)!));84+ const available = await gitGraph(local, missing);85+ pack = await encodePack(available);9686 }9787 }9888 }@@ -1,16 +1,20 @@
11 // GRASP-01 Smart HTTP on a relay origin. Nostr events authorize changes;2-// ntig owns the conditional R2 root, pack verification and Git protocol.3-import { createGitHandler, createAcceptedStateRepository, createPrRepository, WalRepository, NativeGitEngine, R2ObjectStore, parseRepositoryPath, repositoryAddress, repositoryStoragePrefix, readObjects, LimitError, type ObjectStore, type CommitRequest, type GitRepository } from "ntig";2+// Repositories publish compressed objects in SQLite. ntig supplies pack3+// verification and the Git protocol.4+import { createGitHandler, createAcceptedStateRepository, createPrRepository, parseRepositoryPath, repositoryAddress, repositoryStoragePrefix, LimitError, AuthorizationError, type CommitRequest, type GitRepository, type RefUpdate } from "ntig";45 import { npubEncode, naddrEncode } from "nostr-tools/nip19";56 import { now, tag, type Event } from "./event.ts";67 import { KIND_REPO, KIND_REPO_STATE, KIND_GIT_PR, KIND_GIT_PR_UPDATE } from "./kinds.ts";78 import { featureOn } from "./settings.ts";89 import { page, escapeHTML as esc } from "./ui.ts";910 import { relatedRepositoryCoordinates, repositoryCoordinate, type RepositoryAnnouncement } from "./grasp-policy.ts";10-import { hostedAnnouncements, repository, repositoryState, graspBytes, requiredObjects, eventRepository, PR_REF_SECONDS } from "./grasp-state.ts";11+import { hostedAnnouncements, repository, repositoryState, requiredObjects, eventRepository, PR_REF_SECONDS } from "./grasp-state.ts";1112 import type { Relay } from "./relay.ts";1213 import { syncGitTick } from "./grasp-git-sync.ts";1314 import { graspSyncTick } from "./grasp-sync.ts";15+import { GitSqliteRepository } from "./git-sqlite.ts";16+import { sqliteGitBytes, gitSql } from "./git-catalog.ts";17+import { gitObjects } from "./git-objects.ts";1418 1519 const CORS = { "access-control-allow-origin": "*", "access-control-allow-methods": "GET, POST", "access-control-allow-headers": "Content-Type, Authorization, Git-Protocol, X-Git-Request-Id" };1620 export function graspCORS(response: Response, preflight = false): Response {@@ -25,42 +29,43 @@ export const isGitPath = (url: URL) => /^\/(?:npub|prs\/)/.test(url.pathname) ||
2529 2630 export const gitStoragePrefix = async (relay: Relay, repo: RepositoryAnnouncement, alternativePRs = false) => `${relay.slug}/git/${await repositoryStoragePrefix(repositoryAddress(npubEncode(repo.owner), repo.identifier), alternativePRs)}`;2731 28-// gitRepository records each R2 key's retained size, including orphan writes.29-// Reservations precede PUT: an ambiguous response never makes bytes free.30-export async function gitRepository(relay: Relay, repo: RepositoryAnnouncement, alternativePRs = false): Promise<WalRepository> {32+export type GitBackend = GitSqliteRepository;33+34+// gitRepository binds one repository to the relay SQLite database.35+export async function gitRepository(relay: Relay, repo: RepositoryAnnouncement, alternativePRs = false): Promise<GitBackend> {3136 const prefix = await gitStoragePrefix(relay, repo, alternativePRs);32- const r2 = new R2ObjectStore(relay.media, { prefix, maxObjectBytes: 4 * 1024 * 1024 });33- const store: ObjectStore = {34- get: async (key) => r2.get(key),35- put: async (key, bytes, expected) => {36- const full = prefix + key;37- const old = relay.sql.exec<{ size: number }>(`SELECT size FROM grasp_objects WHERE key=?`, full).toArray()[0]?.size ?? 0;38- const increase = Math.max(0, bytes.length - old);39- if (increase && relay.fuelStatus().outOfFuel) throw new LimitError("Relay storage or fuel limit reached");40- const limit = relay.settings.limitsOf(repo.owner)?.cap ?? 0;41- const owned = relay.sql.exec<{ n: number }>(`SELECT coalesce(sum(size),0) AS n FROM grasp_objects WHERE owner=?`, repo.owner).one().n;42- if (limit && owned + relay.store.authorBytes(repo.owner) + increase > limit) throw new LimitError("Repository owner's storage cap reached");43- if (graspBytes(relay) + increase > 320 * 1024 * 1024) throw new LimitError("Relay Git storage quota reached");44- relay.sql.exec(`INSERT INTO grasp_objects(key,owner,size) VALUES(?,?,?) ON CONFLICT(key) DO UPDATE SET size=max(size,excluded.size)`, full, repo.owner, bytes.length);45- const stored = await r2.put(key, bytes, expected);46- if (stored) relay.sql.exec(`UPDATE grasp_objects SET size=? WHERE key=?`, bytes.length, full);47- else {48- const actual = await relay.media.head(full);49- if (actual) relay.sql.exec(`UPDATE grasp_objects SET size=? WHERE key=?`, actual.size, full);50- else relay.sql.exec(`DELETE FROM grasp_objects WHERE key=?`, full);51- }52- return stored;37+ const namespace = prefix.split("/").filter(Boolean).at(-1)!;38+ let released: Event[] = [];39+ const sql = new GitSqliteRepository({40+ sql: relay.sql, repository: namespace,41+ transactionSync: work => relay.storage.transactionSync(work),42+ meter: counts => relay.meterPush(counts.rowsRead, counts.rowsWritten),43+ quota: () => {44+ // This callback runs after staging SQL rows in the transaction. A quota45+ // failure rolls back publication and its receipt together.46+ const fuel = relay.fuel.status(now(), relay.eventBytes(), relay.mediaBytes());47+ if (fuel.outOfFuel) throw new LimitError("Relay storage or fuel limit reached");48+ const cap = relay.settings.limitsOf(repo.owner)?.cap ?? 0;49+ if (cap && sqliteGitBytes(relay, repo.owner, namespace) + relay.store.authorBytes(repo.owner) > cap) throw new LimitError("Repository owner's storage cap reached");50+ if (sqliteGitBytes(relay) > 320 * 1024 * 1024) throw new LimitError("Relay Git storage quota reached");5351 },54- };55- return new WalRepository(store, new NativeGitEngine(), { prefix: "data/" });52+ onCommit: change => {53+ gitSql(relay, "INSERT OR IGNORE INTO git_sqlite_catalog(repository,owner,identifier,alternative) VALUES(?,?,?,?)", namespace, repo.owner, repo.identifier, alternativePRs ? 1 : 0);54+ const updates = JSON.parse(gitSql<{ updates: string }>(relay, "SELECT updates FROM git_sqlite_receipts WHERE repository=? AND id=?", namespace, change.id).one().updates) as RefUpdate[];55+ trackPrRefs(relay, repo, updates, alternativePRs);56+ released = releasePending(relay, repo, alternativePRs, oid => sql.hasObjectSync(oid));57+ },58+ afterCommit: () => { for (const event of released) relay.broadcast(event); released = []; },59+ });60+ return sql;5661 }5762 5863 // prTip distinguishes unknown IDs from known events that cannot authorize5964 // this repository. Purgatory PRs may authorize their own arriving objects.6065 function prTip(relay: Relay, repo: RepositoryAnnouncement, id: string): string | null | false {61- const row = relay.sql.exec<{ raw: string }>(`SELECT raw FROM events WHERE id=?`, id).toArray()[0];66+ const row = gitSql<{ raw: string }>(relay, `SELECT raw FROM events WHERE id=?`, id).toArray()[0];6267 if (!row) {63- const timed = relay.sql.exec<{ until: number }>(`SELECT until FROM grasp_pr_refs WHERE repo=? AND ref=?`, repositoryCoordinate(repo), `refs/nostr/${id}`).toArray()[0];68+ const timed = gitSql<{ until: number }>(relay, `SELECT until FROM grasp_pr_refs WHERE repo=? AND ref=?`, repositoryCoordinate(repo), `refs/nostr/${id}`).toArray()[0];6469 return timed && timed.until <= now() ? false : null;6570 }6671 const e = JSON.parse(row.raw) as Event;@@ -92,39 +97,34 @@ export function alternativePrAddress(relay: Relay, e: Event, host = ""): Reposit
9297 9398 // alternativePrTip retains unknown upload deadlines independently of normal repos.9499 function alternativePrTip(relay: Relay, repo: RepositoryAnnouncement, id: string, host = ""): string | null | false {95- const row = relay.sql.exec<{ raw: string }>(`SELECT raw FROM events WHERE id=?`, id).toArray()[0];100+ const row = gitSql<{ raw: string }>(relay, `SELECT raw FROM events WHERE id=?`, id).toArray()[0];96101 if (!row) {97- const timed = relay.sql.exec<{ until: number }>(`SELECT until FROM grasp_pr_refs WHERE repo=? AND ref=?`, prCoordinate(repo), `refs/nostr/${id}`).toArray()[0];102+ const timed = gitSql<{ until: number }>(relay, `SELECT until FROM grasp_pr_refs WHERE repo=? AND ref=?`, prCoordinate(repo), `refs/nostr/${id}`).toArray()[0];98103 return timed && timed.until <= now() ? false : null;99104 }100105 const e = JSON.parse(row.raw) as Event;101106 if (![KIND_GIT_PR, KIND_GIT_PR_UPDATE].includes(e.kind) || e.pubkey !== repo.owner || relay.settings.isEventHidden(id) || relay.settings.isEventBanned(id) || relay.settings.isBanned(e.pubkey)) return false;102107 const expires = Number(tag(e, "expiration"));103- const pending = relay.sql.exec<{ until: number }>(`SELECT until FROM grasp_pending WHERE id=?`, id).toArray()[0];108+ const pending = gitSql<{ until: number }>(relay, `SELECT until FROM grasp_pending WHERE id=?`, id).toArray()[0];104109 if ((expires && expires <= now()) || (pending && pending.until <= now())) return false;105110 if (alternativePrAddress(relay, e, host)?.id !== repo.id) return false;106111 return /^(?!0{40}$)[0-9a-f]{40}$/.test(tag(e, "c")) ? tag(e, "c") : false;107112 }108113 109114 // alternativePrRepository keeps transport policy in ntig and visibility in the relay.110-function alternativePrRepository(relay: Relay, repo: RepositoryAnnouncement, wal: WalRepository, host: string): GitRepository {115+function alternativePrRepository(relay: Relay, repo: RepositoryAnnouncement, wal: GitBackend, host: string): GitRepository {111116 const tracked: GitRepository = {112117 load: () => wal.load(), loadRefs: () => wal.loadRefs(), lookupRecord: id => wal.lookupRecord(id),118+ ...objectMethods(wal),113119 commit: async request => {114- const coordinate = prCoordinate(repo);115- const existing = relay.sql.exec<{ ref: string }>(`SELECT ref FROM grasp_pr_refs WHERE repo=?`, coordinate).toArray();116- const namespaces = relay.sql.exec<{ n: number }>(`SELECT count(DISTINCT repo) AS n FROM grasp_pr_refs WHERE repo LIKE 'pr:%'`).one().n;117- if (!existing.length && namespaces >= 16) throw new LimitError("Alternative PR repository quota reached");118- if (new Set([...existing.map(row => row.ref), ...request.updates.map(update => update.name)]).size > 1024) throw new LimitError("Alternative PR ref quota reached");119- // Reserve deadlines before publication, including an ambiguous successful120- // root write. Failed uploads expire through the same bounded bookkeeping.121- for (const update of request.updates) relay.sql.exec(`INSERT INTO grasp_pr_refs(repo,ref,until) VALUES(?,?,?) ON CONFLICT(repo,ref) DO NOTHING`, coordinate, update.name, typeof alternativePrTip(relay, repo, update.name.slice(11), host) === "string" ? 0 : now() + PR_REF_SECONDS);122- return wal.commit(request);120+ return wal.commit(request, () => checkAuthority(relay, repo, request.updates, true, host));123121 },124122 };125123 const authorized = createPrRepository(tracked, { lookupTip: async id => alternativePrTip(relay, repo, id, host), allowUnknownPrRefs: true });126124 return {127125 load: () => authorized.load(),126+ ...objectMethods(authorized),127+ lookupRecord: id => authorized.lookupRecord!(id),128128 loadRefs: () => authorized.loadRefs!(),129129 commit: async request => {130130 const receipt = await authorized.commit(request);@@ -134,47 +134,92 @@ function alternativePrRepository(relay: Relay, repo: RepositoryAnnouncement, wal
134134 };135135 }136136 137-// promote checks pending work before loading Git packs, and rechecks authority138-// after the read. Replays still finish promotion left pending by an earlier write.139-export async function promote(relay: Relay, repo: RepositoryAnnouncement, wal: WalRepository, alternative = false) {140- const requiredFor = (e: Event) => {141- if (relay.settings.isEventHidden(e.id) || relay.settings.isEventBanned(e.id) || relay.settings.isBanned(e.pubkey)) return null;142- const relevant = alternative ? alternativePrTip(relay, repo, e.id) === tag(e, "c") : e.kind === KIND_REPO ? repository(relay, repo.owner, repo.identifier)?.id === e.id : e.kind === KIND_REPO_STATE ? tag(e, "d") === repo.identifier && repositoryState(relay, repo)?.id === e.id : eventRepository(relay, e)?.id === repo.id;143- return relevant ? requiredObjects(relay, e) : null;144- };145- const pending = () => relay.sql.exec<{ raw: string }>(`SELECT raw FROM events JOIN grasp_pending ON grasp_pending.id=events.id WHERE grasp_pending.until>? AND (events.expires=0 OR events.expires>?) LIMIT 1024`, now(), now()).toArray();146- if (!pending().some(({ raw }) => requiredFor(JSON.parse(raw) as Event) !== null)) return;147- const snapshot = await wal.load();148- if (!snapshot.sequence) return;149- const objects = await readObjects(snapshot.packs);150- for (const { raw } of pending()) {151- const e = JSON.parse(raw) as Event;152- const required = requiredFor(e);153- if (required && required.every((oid) => objects.has(oid))) {154- if (e.kind === KIND_GIT_PR || e.kind === KIND_GIT_PR_UPDATE) relay.sql.exec(`UPDATE grasp_pr_refs SET until=0 WHERE repo=? AND ref=?`, alternative ? prCoordinate(repo) : repositoryCoordinate(repo), `refs/nostr/${e.id}`);155- relay.sql.exec(`DELETE FROM grasp_pending WHERE id=?`, e.id);156- relay.broadcast(e);137+// objectMethods preserves the indexed path through policy wrappers.138+const objectMethods = (repo: GitRepository): Pick<GitRepository, "getObject" | "getObjectInfo"> => repo.getObject && repo.getObjectInfo ? { getObject: oid => repo.getObject!(oid), getObjectInfo: oid => repo.getObjectInfo!(oid) } : {};139+140+// checkAuthority runs again inside SQLite publication after asynchronous decoding.141+function checkAuthority(relay: Relay, repo: RepositoryAnnouncement, updates: readonly RefUpdate[], alternative = false, host = "") {142+ if (!featureOn(relay.settings.policy, "grasp") || (alternative && !featureOn(relay.settings.policy, "grasp06")) || relay.settings.policy.reads !== "open" || relay.settings.isUnclaimed() || relay.settings.leaseExpired(now()) || relay.settings.isBanned(repo.owner)) throw new AuthorizationError("Repository hosting is unavailable");143+ if (!alternative && repository(relay, repo.owner, repo.identifier)?.id !== repo.id) throw new AuthorizationError("Repository announcement changed");144+ const state = alternative ? null : repositoryState(relay, repo);145+ for (const update of updates) {146+ if (/^refs\/nostr\/[0-9a-f]{64}$/.test(update.name)) {147+ const tip = alternative ? alternativePrTip(relay, repo, update.name.slice(11), host) : prTip(relay, repo, update.name.slice(11));148+ if (update.new === null || tip === false || (tip !== null && tip !== update.new)) throw new AuthorizationError("Pull request authority changed");149+ } else if (alternative || !state || (state.refs[update.name] ?? null) !== update.new) throw new AuthorizationError("Repository state changed");150+ }151+}152+153+// trackPrRefs retains the deadline for an upload whose signed PR arrives later.154+function trackPrRefs(relay: Relay, repo: RepositoryAnnouncement, updates: readonly RefUpdate[], alternative = false, host = "") {155+ const coordinate = alternative ? prCoordinate(repo) : repositoryCoordinate(repo);156+ if (alternative) {157+ const existing = gitSql<{ ref: string }>(relay, "SELECT ref FROM grasp_pr_refs WHERE repo=?", coordinate).toArray();158+ const namespaces = gitSql<{ n: number }>(relay, "SELECT count(DISTINCT repo) AS n FROM grasp_pr_refs WHERE repo LIKE 'pr:%'").one().n;159+ if (!existing.length && namespaces >= 16) throw new LimitError("Alternative PR repository quota reached");160+ if (new Set([...existing.map(row => row.ref), ...updates.filter(u => u.new !== null).map(u => u.name)]).size > 1024) throw new LimitError("Alternative PR ref quota reached");161+ }162+ for (const update of updates) {163+ if (!/^refs\/nostr\/[0-9a-f]{64}$/.test(update.name)) continue;164+ if (update.new === null) gitSql(relay, "DELETE FROM grasp_pr_refs WHERE repo=? AND ref=?", coordinate, update.name);165+ else {166+ const tip = alternative ? alternativePrTip(relay, repo, update.name.slice(11), host) : prTip(relay, repo, update.name.slice(11));167+ gitSql(relay, "INSERT INTO grasp_pr_refs(repo,ref,until) VALUES(?,?,?) ON CONFLICT(repo,ref) DO NOTHING", coordinate, update.name, typeof tip === "string" ? 0 : now() + PR_REF_SECONDS);157168 }158169 }159170 }160171 161-// authorizedRepository adds event authority to the WAL without duplicating refs.162-export function authorizedRepository(relay: Relay, repo: RepositoryAnnouncement, wal: WalRepository): GitRepository {163- const authorized = createAcceptedStateRepository(wal, {172+const pendingEvents = (relay: Relay) => gitSql<{ raw: string }>(relay, "SELECT raw FROM events JOIN grasp_pending ON grasp_pending.id=events.id WHERE grasp_pending.until>? AND (events.expires=0 OR events.expires>?) LIMIT 1024", now(), now()).toArray().map(row => JSON.parse(row.raw) as Event);173+const pendingObjects = (relay: Relay, repo: RepositoryAnnouncement, e: Event, alternative: boolean) => {174+ if (relay.settings.isEventHidden(e.id) || relay.settings.isEventBanned(e.id) || relay.settings.isBanned(e.pubkey)) return null;175+ const relevant = alternative ? alternativePrTip(relay, repo, e.id) === tag(e, "c") : e.kind === KIND_REPO ? repository(relay, repo.owner, repo.identifier)?.id === e.id : e.kind === KIND_REPO_STATE ? tag(e, "d") === repo.identifier && repositoryState(relay, repo)?.id === e.id : eventRepository(relay, e)?.id === repo.id;176+ return relevant ? requiredObjects(relay, e) : null;177+};178+179+// releasePending changes event visibility in the same transaction as refs.180+function releasePending(relay: Relay, repo: RepositoryAnnouncement, alternative: boolean, hasObject: (oid: string) => boolean): Event[] {181+ const released: Event[] = [];182+ const present = new Map<string, boolean>();183+ for (const event of pendingEvents(relay)) {184+ const required = pendingObjects(relay, repo, event, alternative);185+ if (!required || !required.every(oid => { if (!present.has(oid)) present.set(oid, hasObject(oid)); return present.get(oid); })) continue;186+ if (event.kind === KIND_GIT_PR || event.kind === KIND_GIT_PR_UPDATE) gitSql(relay, "UPDATE grasp_pr_refs SET until=0 WHERE repo=? AND ref=?", alternative ? prCoordinate(repo) : repositoryCoordinate(repo), `refs/nostr/${event.id}`);187+ gitSql(relay, "DELETE FROM grasp_pending WHERE id=?", event.id);188+ released.push(event);189+ }190+ return released;191+}192+193+// promote finishes event-first arrivals and retries, rechecking authority after reads.194+export async function promote(relay: Relay, repo: RepositoryAnnouncement, wal: GitRepository, alternative = false) {195+ const pending = pendingEvents(relay).flatMap(e => pendingObjects(relay, repo, e, alternative) ?? []);196+ if (!pending.length || !(await (wal.loadRefs ? wal.loadRefs() : wal.load())).sequence) return;197+ const objects = await gitObjects(wal);198+ const available = new Set<string>();199+ for (const oid of new Set(pending)) if (await objects.info(oid)) available.add(oid);200+ const released = relay.storage.transactionSync(() => releasePending(relay, repo, alternative, oid => available.has(oid)));201+ for (const event of released) relay.broadcast(event);202+}203+204+// authorizedRepository applies signed event authority to stored Git refs.205+export function authorizedRepository(relay: Relay, repo: RepositoryAnnouncement, wal: GitBackend): GitRepository {206+ const checked: GitRepository = {207+ load: () => wal.load(), loadRefs: () => wal.loadRefs(), lookupRecord: id => wal.lookupRecord(id),208+ ...objectMethods(wal),209+ commit: request => wal.commit(request, () => checkAuthority(relay, repo, request.updates)),210+ };211+ const authorized = createAcceptedStateRepository(checked, {164212 lookupState: async () => { const state = repositoryState(relay, repo); return state ? { eventId: state.id, refs: state.refs, head: state.head } : null; },165213 lookupPrTip: async (id) => prTip(relay, repo, id),166214 allowUnknownPrRefs: true,167215 });168216 return {169217 load: () => authorized.load(),218+ ...objectMethods(authorized),219+ lookupRecord: id => authorized.lookupRecord!(id),170220 loadRefs: () => authorized.loadRefs ? authorized.loadRefs() : authorized.load(),171221 commit: async (request: CommitRequest) => {172222 const receipt = await authorized.commit(request);173- for (const update of request.updates) {174- if (!/^refs\/nostr\/[0-9a-f]{64}$/.test(update.name)) continue;175- if (update.new === null) relay.sql.exec(`DELETE FROM grasp_pr_refs WHERE repo=? AND ref=?`, repositoryCoordinate(repo), update.name);176- else relay.sql.exec(`INSERT INTO grasp_pr_refs(repo,ref,until) VALUES(?,?,?) ON CONFLICT(repo,ref) DO NOTHING`, repositoryCoordinate(repo), update.name, typeof prTip(relay, repo, update.name.slice(11)) === "string" ? 0 : now() + PR_REF_SECONDS);177- }178223 await promote(relay, repo, wal);179224 return receipt;180225 },@@ -222,34 +267,34 @@ export async function grasp(relay: Relay, req: Request, url: URL): Promise<Respo
222267 }223268 224269 // graspTick expires purgatory and cleans one repository per alarm. Event225-// deletion removes visibility; retained R2 objects remain charged until teardown.270+// deletion removes visibility; retained Git objects remain charged until teardown.226271 export async function graspTick(relay: Relay): Promise<number> {227- relay.sql.exec(`DELETE FROM events WHERE id IN (SELECT id FROM grasp_pending WHERE until<=?)`, now());272+ gitSql(relay, `DELETE FROM events WHERE id IN (SELECT id FROM grasp_pending WHERE until<=?)`, now());228273 const all = hostedAnnouncements(relay);229- const last = relay.sql.exec<{ coordinate: string }>(`SELECT coordinate FROM grasp_tick WHERE id=1`).toArray()[0]?.coordinate ?? "";274+ const last = gitSql<{ coordinate: string }>(relay, `SELECT coordinate FROM grasp_tick WHERE id=1`).toArray()[0]?.coordinate ?? "";230275 const ordered = all.sort((a, b) => repositoryCoordinate(a).localeCompare(repositoryCoordinate(b)));231276 const repo = ordered.find((r) => repositoryCoordinate(r) > last) ?? ordered[0];232- if (repo) relay.sql.exec(`INSERT OR REPLACE INTO grasp_tick(id,coordinate) VALUES(1,?)`, repositoryCoordinate(repo));233- for (const { repo: coordinate } of relay.sql.exec<{ repo: string }>(`SELECT DISTINCT repo FROM grasp_pr_refs`).toArray()) {234- if (!coordinate.startsWith("pr:") && !all.some((r) => repositoryCoordinate(r) === coordinate)) relay.sql.exec(`DELETE FROM grasp_pr_refs WHERE repo=?`, coordinate);277+ if (repo) gitSql(relay, `INSERT OR REPLACE INTO grasp_tick(id,coordinate) VALUES(1,?)`, repositoryCoordinate(repo));278+ for (const { repo: coordinate } of gitSql<{ repo: string }>(relay, `SELECT DISTINCT repo FROM grasp_pr_refs`).toArray()) {279+ if (!coordinate.startsWith("pr:") && !all.some((r) => repositoryCoordinate(r) === coordinate)) gitSql(relay, `DELETE FROM grasp_pr_refs WHERE repo=?`, coordinate);235280 }236281 if (repo && featureOn(relay.settings.policy, "grasp") && !relay.fuelStatus().outOfFuel) {237282 try {238283 const wal = await gitRepository(relay, repo);239- const snapshot = await wal.load();240- const refs = relay.sql.exec<{ ref: string }>(`SELECT ref FROM grasp_pr_refs WHERE repo=? AND until>0 AND until<=?`, repositoryCoordinate(repo), now()).toArray();284+ const snapshot = await wal.loadRefs();285+ const refs = gitSql<{ ref: string }>(relay, `SELECT ref FROM grasp_pr_refs WHERE repo=? AND until>0 AND until<=?`, repositoryCoordinate(repo), now()).toArray();241286 for (const { ref } of refs) {242287 const current = snapshot.refs[ref];243288 const tip = prTip(relay, repo, ref.slice("refs/nostr/".length));244289 if (current && tip !== current) await wal.commit({ id: `expire:${ref.slice(-64)}:${current}`, updates: [{ name: ref, old: current, new: null }] });245- if (tip === current && current) relay.sql.exec(`UPDATE grasp_pr_refs SET until=0 WHERE repo=? AND ref=?`, repositoryCoordinate(repo), ref);246- else relay.sql.exec(`DELETE FROM grasp_pr_refs WHERE repo=? AND ref=?`, repositoryCoordinate(repo), ref);290+ if (tip === current && current) gitSql(relay, `UPDATE grasp_pr_refs SET until=0 WHERE repo=? AND ref=?`, repositoryCoordinate(repo), ref);291+ else gitSql(relay, `DELETE FROM grasp_pr_refs WHERE repo=? AND ref=?`, repositoryCoordinate(repo), ref);247292 }248293 await promote(relay, repo, wal);249294 } catch { /* The next bounded alarm retries; retained bytes stay metered. */ }250295 }251296 if (featureOn(relay.settings.policy, "grasp06") && !relay.fuelStatus().outOfFuel) {252- const expired = relay.sql.exec<{ repo: string; ref: string }>(`SELECT repo,ref FROM grasp_pr_refs WHERE repo LIKE 'pr:%' AND until>0 AND until<=? ORDER BY until,repo,ref LIMIT 1`, now()).toArray()[0];297+ const expired = gitSql<{ repo: string; ref: string }>(relay, `SELECT repo,ref FROM grasp_pr_refs WHERE repo LIKE 'pr:%' AND until>0 AND until<=? ORDER BY until,repo,ref LIMIT 1`, now()).toArray()[0];253298 if (expired) {254299 const match = /^pr:([0-9a-f]{64}):(.+)$/u.exec(expired.repo);255300 if (match) {@@ -259,14 +304,14 @@ export async function graspTick(relay: Relay): Promise<number> {
259304 const current = (await wal.loadRefs()).refs[expired.ref];260305 const tip = alternativePrTip(relay, pr, expired.ref.slice(11));261306 if (current && tip !== current) await wal.commit({ id: `expire:${expired.ref.slice(-64)}:${current}`, updates: [{ name: expired.ref, old: current, new: null }] });262- if (current && tip === current) relay.sql.exec(`UPDATE grasp_pr_refs SET until=0 WHERE repo=? AND ref=?`, expired.repo, expired.ref);263- else relay.sql.exec(`DELETE FROM grasp_pr_refs WHERE repo=? AND ref=?`, expired.repo, expired.ref);307+ if (current && tip === current) gitSql(relay, `UPDATE grasp_pr_refs SET until=0 WHERE repo=? AND ref=?`, expired.repo, expired.ref);308+ else gitSql(relay, `DELETE FROM grasp_pr_refs WHERE repo=? AND ref=?`, expired.repo, expired.ref);264309 await promote(relay, pr, wal, true);265- } catch { /* Expired uploads stay hidden when the WAL cannot advance. */ }310+ } catch { /* Expired uploads stay hidden when cleanup cannot commit. */ }266311 }267312 }268313 // Event-first and Git-first pairs are promoted even without a later push.269- const pending = relay.sql.exec<{ raw: string }>(`SELECT raw FROM events JOIN grasp_pending ON grasp_pending.id=events.id WHERE kind IN (?,?) AND grasp_pending.until>? ORDER BY grasp_pending.until LIMIT 16`, KIND_GIT_PR, KIND_GIT_PR_UPDATE, now()).toArray();314+ const pending = gitSql<{ raw: string }>(relay, `SELECT raw FROM events JOIN grasp_pending ON grasp_pending.id=events.id WHERE kind IN (?,?) AND grasp_pending.until>? ORDER BY grasp_pending.until LIMIT 16`, KIND_GIT_PR, KIND_GIT_PR_UPDATE, now()).toArray();270315 for (const row of pending) {271316 const e = JSON.parse(row.raw) as Event;272317 const parsed = alternativePrAddress(relay, e);@@ -277,7 +322,7 @@ export async function graspTick(relay: Relay): Promise<number> {
277322 }278323 const syncAt = await syncGitTick(relay);279324 const eventsAt = await graspSyncTick(relay);280- const earliest = relay.sql.exec<{ n: number | null }>(`SELECT min(at) AS n FROM (SELECT until AS at FROM grasp_pending UNION ALL SELECT until AS at FROM grasp_pr_refs WHERE until>0)`).one().n;325+ const earliest = gitSql<{ n: number | null }>(relay, `SELECT min(at) AS n FROM (SELECT until AS at FROM grasp_pending UNION ALL SELECT until AS at FROM grasp_pr_refs WHERE until>0)`).one().n;281326 const next = Math.min(earliest === null ? Infinity : now() + 60, syncAt || Infinity, eventsAt || Infinity);282327 return Number.isFinite(next) ? next : 0;283328 }@@ -94,7 +94,7 @@ export function nip11(relay: Relay, host: string) {
9494 : access[p.writes];9595 const guests = p.writes !== "open" && p.openKinds.length ? ` Guest write exceptions apply to kinds ${p.openKinds.join(", ")}.` : "";9696 const listing = featureOn(p, "grasp05") ? "Archive preview accepts repository announcements without naming this service in clone and relays." : "Repository announcements name this service in clone and relays.";97- doc.repo_acceptance_criteria = `${eligibility} ${listing} State and collaboration events follow the relay write rule (${p.writes}) and kind rules; bans, proof of work and fuel apply.${guests} At most 16 repositories, 4 MiB per pack, 16 MiB packed history and 128 Git transactions per unmigrated format-1 repository. Explicitly migrated format-2 repositories retain at most 128 unique packs.`;97+ doc.repo_acceptance_criteria = `${eligibility} ${listing} State and collaboration events follow the relay write rule (${p.writes}) and kind rules; bans, proof of work and fuel apply.${guests} At most 16 repositories, 4 MiB per pack, 4 MiB per object, 16 MiB decoded per operation, and 16 MiB compressed objects plus 16 MiB metadata per repository. Git storage shares the relay SQLite allowance.`;9898 if (p.writes !== "open" || p.blockedWords.length) doc.curation = "The relay's configured write rule and blocked topics apply to Nostr repository and collaboration events.";9999 }100100 if (featureOn(p, "sites")) doc.nsites = {@@ -1,97 +0,0 @@
1-// Quiescent test fixtures reconcile the quota ledger against R2 and classify2-// current-format dependencies. This walk is not an online deletion policy.3-import { expect } from "vitest";4-import { npubEncode } from "nostr-tools/nip19";5-import { repositoryAddress, repositoryStoragePrefix, type WalRepository } from "ntig";6-import type { Relay } from "../../src/relay.ts";7-import type { RepositoryAnnouncement } from "../../src/grasp-policy.ts";8-9-const digest = async (bytes: Uint8Array) => Array.from(new Uint8Array(await crypto.subtle.digest("SHA-256", bytes)), (b) => b.toString(16).padStart(2, "0")).join("");10-11-// inventory reconciles an isolated repository after its writer has stopped.12-export async function inventory(relay: Relay, repo: RepositoryAnnouncement, wal: WalRepository) {13- const prefix = `${relay.slug}/git/${await repositoryStoragePrefix(repositoryAddress(npubEncode(repo.owner), repo.identifier))}data/`;14- const root = await relay.media.get(`${prefix}root.json`);15- const rootText = root ? await root.text() : null;16- const snapshot = await wal.load();17- const physical = new Map<string, number>();18- let cursor: string | undefined;19- do {20- const page = await relay.media.list({ prefix, cursor, limit: 100 });21- for (const item of page.objects) physical.set(item.key, item.size);22- cursor = page.truncated ? page.cursor : undefined;23- } while (cursor);24- const ledger = relay.sql.exec<{ key: string; size: number }>("SELECT key, size FROM grasp_objects WHERE owner=?", repo.owner).toArray().filter(({ key }) => key.startsWith(prefix));25- expect(new Map(ledger.map(({ key, size }) => [key, size]))).toEqual(physical);26- const live = new Set<string>();27- const records = new Map<string, string>();28- const sequences = new Set<number>();29- const read = async (family: string, hash: string) => {30- expect(hash).toMatch(/^[0-9a-f]{64}$/);31- const key = `${prefix}${family}/${hash}`;32- const item = await relay.media.get(key);33- expect(item).not.toBeNull();34- const bytes = new Uint8Array(await item!.arrayBuffer());35- expect(bytes.length).toBe(physical.get(key));36- expect(await digest(bytes)).toBe(hash);37- live.add(key);38- return JSON.parse(new TextDecoder().decode(bytes));39- };40- if (root) live.add(root.key);41- if (snapshot.checkpoint) {42- await read("manifests", snapshot.checkpoint.manifestHash);43- for (const id of snapshot.checkpoint.packIds) live.add(`${prefix}packs/${id}`);44- const walk = async (hash: string, route = ""): Promise<void> => {45- expect(route.length).toBeLessThanOrEqual(64);46- expect(live.has(`${prefix}receipt-index/${hash}`)).toBe(false);47- const node = await read("receipt-index", hash);48- expect(node.v).toBe(1);49- if (node.t === "l") {50- for (const [key, recordHash] of node.e as [string, string][]) {51- expect(key.startsWith(route)).toBe(true);52- const record = await read("records", recordHash);53- expect(await digest(new TextEncoder().encode(record.id))).toBe(key);54- expect(records.has(record.id)).toBe(false);55- records.set(record.id, recordHash);56- sequences.add(record.sequence);57- expect(await wal.lookupRecord(record.id)).toEqual(record);58- }59- } else {60- expect(node.t).toBe("b");61- expect(node.c).toHaveLength(16);62- for (const [i, child] of (node.c as (string | null)[]).entries()) {63- if (child !== null) await walk(child, route + i.toString(16));64- }65- }66- };67- if (snapshot.checkpoint.receiptRoot) await walk(snapshot.checkpoint.receiptRoot);68- expect(records.size).toBe(snapshot.sequence);69- expect([...sequences].sort((a, b) => a - b)).toEqual(Array.from({ length: snapshot.sequence }, (_, i) => i + 1));70- } else {71- let tip = snapshot.tip;72- while (tip) {73- expect(live.has(`${prefix}records/${tip}`)).toBe(false);74- const record = await read("records", tip);75- if (record.pack) live.add(`${prefix}packs/${record.pack}`);76- tip = record.parent;77- }78- }79- for (const key of live) expect(physical.has(key)).toBe(true);80- const categories: Record<string, { objects: number; bytes: number }> = {};81- for (const [key, bytes] of physical) {82- const family = key.slice(prefix.length).split("/")[0];83- const category = `${live.has(key) ? "current" : "unreferenced"}/${family}`;84- const row = categories[category] ??= { objects: 0, bytes: 0 };85- row.objects++; row.bytes += bytes;86- }87- const after = await relay.media.get(`${prefix}root.json`);88- expect(after?.etag).toBe(root?.etag);89- expect(after ? await after.text() : null).toBe(rootText);90- return {91- sequence: snapshot.sequence, refs: Object.keys(snapshot.refs).length,92- physicalBytes: [...physical.values()].reduce((a, b) => a + b, 0),93- reservedBytes: ledger.reduce((n, row) => n + row.size, 0),94- currentBytes: [...live].reduce((n, key) => n + physical.get(key)!, 0),95- objects: physical.size, categories,96- };97-}@@ -1,11 +1,11 @@
1-// Owner-only inventory reports compare a bounded physical listing with the2-// repository root and the durable reservation ledger. They never mutate R2.1+// Owner-only Git storage reports describe the SQLite repository without2+// opening the R2 bucket or granting a maintenance action.33 import { env, runInDurableObject } from "cloudflare:test";44 import { describe, expect, it } from "vitest";55 import { generateSecretKey } from "nostr-tools/pure";66 import { npubEncode } from "nostr-tools/nip19";77 import { encodePack } from "ntig";8-import { KIND_REPO, KIND_REPO_STATE } from "../../src/kinds.ts";8+import { KIND_REPO } from "../../src/kinds.ts";99 import { gitRepositoryPath } from "../../src/grasp-policy.ts";1010 import { repository } from "../../src/grasp-state.ts";1111 import { gitRepository } from "../../src/grasp.ts";@@ -18,79 +18,54 @@ const objectID = async (type: string, bytes: Uint8Array) => {
1818 const head = encoder.encode(`${type} ${bytes.length}\0`);1919 const input = new Uint8Array(head.length + bytes.length);2020 input.set(head); input.set(bytes, head.length);21- return Array.from(new Uint8Array(await crypto.subtle.digest("SHA-1", input)), (b) => b.toString(16).padStart(2, "0")).join("");21+ return Array.from(new Uint8Array(await crypto.subtle.digest("SHA-1", input)), (x) => x.toString(16).padStart(2, "0")).join("");2222 };2323 async function tinyPack() {24- const blob = encoder.encode("inventory\n");25- const blobID = await objectID("blob", blob);24+ const blob = encoder.encode("inventory\n"), blobID = await objectID("blob", blob);2625 const tree = new Uint8Array([...encoder.encode("100644 README\0"), ...Uint8Array.from(blobID.match(/../g)!, (x) => parseInt(x, 16))]);2726 const treeID = await objectID("tree", tree);2827 const commit = encoder.encode(`tree ${treeID}\nauthor inventory <inventory@example.com> 1 +0000\ncommitter inventory <inventory@example.com> 1 +0000\n\ninitial\n`);29- const commitID = await objectID("commit", commit);30- return { commitID, pack: await encodePack([{ type: "blob", data: blob }, { type: "tree", data: tree }, { type: "commit", data: commit }]) };28+ return { commitID: await objectID("commit", commit), pack: await encodePack([{ type: "blob", data: blob }, { type: "tree", data: tree }, { type: "commit", data: commit }]) };3129 }32-async function seed(host: string, format = 1) {33- const owner = generateSecretKey();34- const stranger = generateSecretKey();35- const moderator = generateSecretKey();36- const identifier = "inventory";30+async function seed(host: string) {31+ const owner = generateSecretKey(), stranger = generateSecretKey(), moderator = generateSecretKey(), identifier = "inventory";3732 await rpc(host, owner, "claim");3833 await rpc(host, owner, "applypreset", "grasp");3934 await rpc(host, owner, "setmember", pk(moderator), { role: "moderator" });40- const path = gitRepositoryPath(npubEncode(pk(owner)), identifier);41- const c = await WS.connect(host);35+ const path = gitRepositoryPath(npubEncode(pk(owner)), identifier), c = await WS.connect(host);4236 const announcement = ev(owner, KIND_REPO, "", [["d", identifier], ["clone", `https://${host}${path}`], ["relays", `wss://${host}`], ["maintainers", pk(owner)]]);43- expect((await c.ok(announcement)).ok).toBe(true);44- c.ws.close();37+ expect((await c.ok(announcement)).ok).toBe(true); c.ws.close();4538 const pack = await tinyPack();4639 await runInDurableObject(env.RELAY.getByName(host.split(".")[0]), async (relay) => {4740 const repo = repository(relay, pk(owner), identifier)!;4841 await relay.repositoryAccess.run("control", async () => {49- const wal = await gitRepository(relay, repo);50- await wal.commit({ id: "inventory-initial", updates: [{ name: "refs/heads/main", old: null, new: pack.commitID }], pack: pack.pack });51- if (format === 2) await wal.checkpoint();42+ const git = await gitRepository(relay, repo);43+ await git.commit({ id: "inventory-initial", updates: [{ name: "refs/heads/main", old: null, new: pack.commitID }], pack: pack.pack });5244 }, () => Promise.reject(new Error("repository access unexpectedly refused during fixture setup")));5345 });5446 return { owner, stranger, moderator, identifier };5547 }56-5748 async function scan(relay: Relay, host: string, owner: Uint8Array, identifier: string) {58- const url = `http://${host}/`;59- const payload = { method: "gitstorage", params: [pk(owner), identifier] };49+ const url = `http://${host}/`, payload = { method: "gitstorage", params: [pk(owner), identifier] };6050 const authorization = await nip98(owner, url, "POST", payload);6151 return relay.fetch(new Request(url, { method: "POST", headers: { authorization, "content-type": "application/nostr+json+rpc" }, body: JSON.stringify(payload) }));6252 }6353 64-function proxyBucket(relay: Relay, overrides: Record<string, unknown>) {65- const original = relay.media;66- const calls: string[] = [];67- const observed = new Proxy(original, { get(target, name) {68- calls.push(String(name));69- if (name in overrides) return overrides[name as string];70- if (name !== "get" && name !== "list") throw new Error(`unexpected R2 method ${String(name)}`);71- const value = Reflect.get(target, name, target);72- return typeof value === "function" ? value.bind(target) : value;73- } });74- Object.defineProperty(relay, "media", { value: observed, configurable: true });75- return { calls, restore: () => Object.defineProperty(relay, "media", { value: original, configurable: true }) };76-}77-7854 describe("Git storage inventory management", () => {79- it.each([1, 2])("returns a complete format-%i report only to the owner", async (format) => {80- const host = `git-storage-owner-${format}.bind.ws`;81- const f = await seed(host, format);55+ it("returns the SQLite object, ref and receipt totals only to the owner", async () => {56+ const host = "git-storage-sql-owner.bind.ws", f = await seed(host);8257 const report = await rpc(host, f.owner, "gitstorage", pk(f.owner), f.identifier);8358 expect(report.status, JSON.stringify(report)).toBe(200);84- expect(report.result.inventory.format).toBe(format);85- expect(report.result.inventory.sequence).toBe(1);86- expect(report.result.inventory.listed.keys).toBeGreaterThanOrEqual(3);87- expect(report.result.inventory.listed.bytes).toBeGreaterThan(0);88- expect(report.result.inventory.live.keys).toBe(report.result.inventory.listed.keys);89- expect(report.result.reservations.keys).toBe(report.result.inventory.listed.keys);90- expect(report.result.reservationMinusListedBytes).toBe(0);91- expect(report.result.operations.gets).toBeGreaterThan(0);92- expect(report.result.operations.lists).toBeGreaterThan(0);93-59+ expect(report.result.backend).toBe("sqlite");60+ expect(report.result.objects.count).toBeGreaterThanOrEqual(3);61+ expect(report.result.objects.rawBytes).toBeGreaterThan(0);62+ expect(report.result.objects.compressedBytes).toBeGreaterThan(0);63+ expect(report.result.objects.metadataBytes).toBeGreaterThan(0);64+ expect(report.result.refs).toBe(1);65+ expect(report.result.receipts).toBe(1);66+ expect(report.result.operations).toEqual({ gets: 0, lists: 0 });67+ expect(report.result.physicalDatabaseBytes).toBeGreaterThan(0);68+ expect(report.result.cooldownSeconds).toBe(60);9469 expect((await rpc(host, null, "gitstorage", pk(f.owner), f.identifier)).status).toBe(401);9570 expect((await rpc(host, f.stranger, "gitstorage", pk(f.owner), f.identifier)).status).toBe(403);9671 expect((await rpc(host, f.moderator, "gitstorage", pk(f.owner), f.identifier)).status).toBe(403);@@ -100,95 +75,20 @@ describe("Git storage inventory management", () => {
10075 expect((await rpc(host, f.owner, "gitstorage", pk(f.owner), f.identifier)).status).toBe(429);10176 });10277 103- it("uses only R2 get and list for a successful owner scan and releases its control scope", async () => {104- const host = "git-storage-methods.bind.ws";105- const f = await seed(host);78+ it("does not use R2 for a successful SQLite report and releases its control scope", async () => {79+ const host = "git-storage-sql-methods.bind.ws", f = await seed(host);10680 const result = await runInDurableObject(env.RELAY.getByName(host.split(".")[0]), async (relay) => {107- const bucket = proxyBucket(relay, {});81+ const original = relay.media;82+ Object.defineProperty(relay, "media", { value: new Proxy(original, { get() { throw new Error("SQLite inventory must not touch R2"); } }), configurable: true });10883 try {10984 const response = await scan(relay, host, f.owner, f.identifier);11085 expect(response.status).toBe(200);111- const report = await response.json<any>();112- expect(report.result.reservations.bytes).toBe(report.result.inventory.listed.bytes);113- expect(report.result.inventory.authority).toBe(false);114- expect(report.result.inventory.gcCandidate).toBe(false);115- const before = bucket.calls.length;116- expect((await scan(relay, host, f.owner, f.identifier)).status).toBe(429);117- expect(bucket.calls).toHaveLength(before);118- return { calls: bucket.calls, busy: relay.repositoryAccess.busy };119- } finally { bucket.restore(); }120- });121- expect(result.calls.filter((name: string) => name === "get" || name === "list").length).toBe(result.calls.length);122- expect(result.calls).toContain("get");123- expect(result.calls).toContain("list");124- expect(result.busy).toBe(false);125- });126-127- it("returns 503 without a report when R2 fails and releases the control scope", async () => {128- const host = "git-storage-failure.bind.ws";129- const f = await seed(host);130- await runInDurableObject(env.RELAY.getByName(host.split(".")[0]), async (relay) => {131- const bucket = proxyBucket(relay, { get: async () => { throw new Error("provider failure"); } });132- try {133- const response = await scan(relay, host, f.owner, f.identifier);134- expect(response.status).toBe(503);135- expect(await response.json<any>()).toEqual({ error: "error: Git inventory unavailable; no complete report" });136- expect(relay.repositoryAccess.busy).toBe(false);137- } finally { bucket.restore(); }138- });139- });140-141- it("returns 413 for an oversized listing page without leaving a lease", async () => {142- const host = "git-storage-budget.bind.ws";143- const f = await seed(host);144- await runInDurableObject(env.RELAY.getByName(host.split(".")[0]), async (relay) => {145- const bucket = proxyBucket(relay, { list: async () => ({ objects: Array.from({ length: 10001 }, (_, i) => ({ key: `${relay.slug}/git/data/${i}`, size: 1 })), truncated: false, cursor: undefined }) });146- try {147- const response = await scan(relay, host, f.owner, f.identifier);148- expect(response.status).toBe(413);149- expect(await response.json<any>()).toEqual({ error: "restricted: Git inventory limit reached; no complete report" });86+ const body = await response.json<any>();87+ expect(body.result.operations).toEqual({ gets: 0, lists: 0 });15088 expect(relay.repositoryAccess.busy).toBe(false);151- } finally { bucket.restore(); }152- });153- });154-155- it("holds control admission across the first R2 read and refuses Git, event and control work", async () => {156- const host = "git-storage-coordination.bind.ws";157- const f = await seed(host);158- await runInDurableObject(env.RELAY.getByName(host.split(".")[0]), async (relay) => {159- let entered!: () => void;160- let release!: () => void;161- const ready = new Promise<void>((resolve) => { entered = resolve; });162- const blocked = new Promise<void>((resolve) => { release = resolve; });163- const original = relay.media;164- const bucket = proxyBucket(relay, { get: async (key: string, options?: R2GetOptions) => {165- if (key.endsWith("/root.json")) { entered(); await blocked; }166- return original.get(key, options);167- } });168- const active = scan(relay, host, f.owner, f.identifier);169- try {170- await ready;171- expect(relay.repositoryAccess.busy).toBe(true);172- expect(relay.repositoryAccess.kind).toBe("control");173- const path = gitRepositoryPath(npubEncode(pk(f.owner)), f.identifier);174- const git = await relay.fetch(new Request(`http://${host}${path}/info/refs?service=git-upload-pack`));175- expect(git.status).toBe(429);176- const event = ev(f.owner, KIND_REPO_STATE, "", [["d", f.identifier], ["refs/heads/main", "a".repeat(40)]]);177- const admitted = await relay.acceptAny(event, relay.virtualConn(host, pk(f.owner)));178- expect(admitted.ok).toBe(false);179- expect(admitted.msg).toContain("relay operation in progress");180- const control = await scan(relay, host, f.owner, f.identifier);181- expect(control.status).toBe(429);182- release();183- const response = await active;184- expect(response.status).toBe(200);185- await response.arrayBuffer();186- } finally {187- release();188- await active;189- bucket.restore();190- }191- expect(relay.repositoryAccess.busy).toBe(false);89+ return body.result;90+ } finally { Object.defineProperty(relay, "media", { value: original, configurable: true }); }19291 });92+ expect(result.backend).toBe("sqlite");19393 });19494 });@@ -1,245 +0,0 @@
1-// Checkpoint migration is exercised through bindws' repository store. The2-// fixture is isolated to one Durable Object and never touches a production key.3-import { env, runInDurableObject } from "cloudflare:test";4-import { describe, expect, it } from "vitest";5-import { generateSecretKey } from "nostr-tools/pure";6-import { npubEncode } from "nostr-tools/nip19";7-import { encodePack, type WalRecord } from "ntig";8-import { KIND_REPO } from "../../src/kinds.ts";9-import { repository } from "../../src/grasp-state.ts";10-import { gitStorage } from "../../src/git-storage.ts";11-import { gitRepository } from "../../src/grasp.ts";12-import type { RepositoryAnnouncement } from "../../src/grasp-policy.ts";13-import { ev, pk, rpc } from "../helpers/relay.ts";14-import { WS } from "../helpers/ws.ts";15-import { inventory } from "../helpers/grasp-inventory.ts";16-17-const encoder = new TextEncoder();18-const oid = async (type: string, data: Uint8Array) => {19- const header = encoder.encode(`${type} ${data.length}\0`);20- const input = new Uint8Array(header.length + data.length);21- input.set(header); input.set(data, header.length);22- return Array.from(new Uint8Array(await crypto.subtle.digest("SHA-1", input)), (b) => b.toString(16).padStart(2, "0")).join("");23-};24-25-async function fixturePack() {26- const blob = encoder.encode("checkpoint fixture\n");27- const blobID = await oid("blob", blob);28- const tree = new Uint8Array([...encoder.encode("100644 README\0"), ...Uint8Array.from(blobID.match(/../g)!, (part) => parseInt(part, 16))]);29- const treeID = await oid("tree", tree);30- const commit = encoder.encode(`tree ${treeID}\nauthor ntig <ntig@example.com> 1 +0000\ncommitter ntig <ntig@example.com> 1 +0000\n\ncheckpoint\n`);31- const commitID = await oid("commit", commit);32- return { commitID, pack: await encodePack([{ type: "blob", data: blob }, { type: "tree", data: tree }, { type: "commit", data: commit }]) };33-}34-35-const announcement = (sk: Uint8Array, host: string, identifier: string) => ev(sk, KIND_REPO, "", [36- ["d", identifier],37- ["clone", `https://${host}/${npubEncode(pk(sk))}/${identifier}.git`],38- ["relays", `wss://${host}`],39- ["maintainers", pk(sk)],40-]);41-42-describe("GRASP checkpoint integration", () => {43- // Each migration performs 260 sequential commits and retained-object scans.44- // Hosted CI takes over 30 seconds for the growing-ref fixture.45- it.each(["growing", "fixed"])("migrates an isolated %s-ref repository and reconciles retained storage", async (workload) => {46- const host = `grasp-checkpoint-${workload}.bind.ws`;47- const owner = generateSecretKey();48- const identifier = "checkpoint";49- await rpc(host, owner, "claim");50- await rpc(host, owner, "setpolicy", { features: { grasp: true } });51- const connection = await WS.connect(host);52- const event = announcement(owner, host, identifier);53- expect((await connection.ok(event)).ok).toBe(true);54- connection.ws.close();55-56- const fixture = await fixturePack();57- const measurements = await runInDurableObject(env.RELAY.getByName(host.split(".")[0]), async (relay) => relay.repositoryAccess.run("control", async () => {58- const repo = repository(relay, pk(owner), identifier) as RepositoryAnnouncement | null;59- expect(repo).not.toBeNull();60- const wal = await gitRepository(relay, repo!);61- const empty = await inventory(relay, repo!, wal);62- const first = await wal.commit({ id: "checkpoint-initial", updates: [{ name: "refs/heads/main", old: null, new: fixture.commitID }], pack: fixture.pack });63- const oneCommit = await inventory(relay, repo!, wal);64- const legacy: WalRecord[] = [];65- legacy.push((await wal.lookupRecord!(first.id))!);66- let prior = fixture.commitID;67- for (let i = 1; i < 128; i++) {68- const receipt = await wal.commit({ id: `checkpoint-legacy-${i}`, updates: [{ name: workload === "growing" ? `refs/tags/legacy-${i}` : "refs/heads/main", old: workload === "growing" ? null : prior, new: prior }] });69- legacy.push((await wal.lookupRecord!(receipt.id))!);70- }71- expect((await wal.load()).sequence).toBe(128);72- const before = relay.sql.exec<{ n: number; bytes: number }>("SELECT count(*) AS n, coalesce(sum(size),0) AS bytes FROM grasp_objects WHERE owner=?", pk(owner)).one();73-74- const migration = await wal.checkpoint();75- expect(migration).toEqual({ sequence: 128, changed: true });76- expect(await wal.checkpoint()).toEqual({ sequence: 128, changed: false });77- const migrated = await wal.load();78- expect(migrated.checkpoint?.manifestHash).toMatch(/^[0-9a-f]{64}$/);79- expect(migrated.records).toHaveLength(1);80- expect(migrated.refs["refs/heads/main"]).toBe(fixture.commitID);81- expect(await wal.lookupRecord!(legacy[0]!.id)).toMatchObject({ id: legacy[0]!.id, sequence: 1 });82-83- // Format 2 allows more than the legacy 128-record ceiling. Ref-only84- // commits exercise manifest/index growth without creating more packs.85- for (let i = 128; i < 260; i++) {86- await wal.commit({ id: `checkpoint-v2-${i}`, updates: [{ name: workload === "growing" ? `refs/tags/v2-${i}` : "refs/heads/main", old: workload === "growing" ? null : prior, new: prior }] });87- }88- const replay = await wal.commit({ id: first.id, updates: [{ name: "refs/heads/main", old: null, new: fixture.commitID }], pack: fixture.pack });89- expect(replay).toMatchObject({ id: first.id, sequence: 1, replayed: true });90- const rebuilt = await gitRepository(relay, repo!);91- const cold = await rebuilt.load();92- expect(cold.sequence).toBe(260);93- expect(cold.refs[workload === "growing" ? "refs/tags/v2-259" : "refs/heads/main"]).toBe(fixture.commitID);94- const advertised = await rebuilt.loadRefs!();95- expect(advertised.sequence).toBe(260);96- expect(advertised.refs["refs/heads/main"]).toBe(fixture.commitID);97- expect(await rebuilt.lookupRecord!("checkpoint-legacy-127")).toMatchObject({ sequence: 128 });98- expect(await rebuilt.lookupRecord!("missing-checkpoint-receipt")).toBeNull();99-100- const after = relay.sql.exec<{ n: number; bytes: number }>("SELECT count(*) AS n, coalesce(sum(size),0) AS bytes FROM grasp_objects WHERE owner=?", pk(owner)).one();101- const keys = relay.sql.exec<{ key: string; size: number }>("SELECT key, size FROM grasp_objects WHERE owner=? ORDER BY key", pk(owner)).toArray();102- const categories = keys.reduce<Record<string, number>>((counts, { key }) => {103- const category = key.split("/").at(-2) ?? key;104- counts[category] = (counts[category] ?? 0) + 1;105- return counts;106- }, {});107- const retained = await inventory(relay, repo!, rebuilt);108- const scan = await gitStorage(relay, pk(owner), identifier);109- expect(scan.status).toBe(200);110- if (!("result" in scan.body)) throw new Error("inventory did not complete");111- const report = scan.body.result;112- if (!report) throw new Error("inventory report missing");113- expect(report.inventory.listed.bytes).toBe(retained.physicalBytes);114- expect(report.inventory.live.bytes).toBe(retained.currentBytes);115- expect(report.reservations.bytes).toBe(retained.reservedBytes);116- expect(report.inventory.unknown.bytes).toBe(0);117- console.info("runtime checkpoint inventory", JSON.stringify({ workload, physical: report.inventory.listed.bytes, live: report.inventory.live.bytes, ...report.operations, ...report.inventory.observed }));118- return { before, after, keyCount: keys.length, categories, sequence: cold.sequence, empty, oneCommit, retained };119- }, () => { throw new Error("fixture admission refused"); }));120-121- console.info("checkpoint inventory", JSON.stringify({ workload, empty: measurements.empty, oneCommit: measurements.oneCommit, retained: measurements.retained }));122- console.info("checkpoint storage measurement", {123- workload,124- beforeObjects: measurements.before.n,125- afterObjects: measurements.after.n,126- beforeBytes: measurements.before.bytes,127- afterBytes: measurements.after.bytes,128- retainedKeys: measurements.keyCount,129- categories: measurements.categories,130- sequence: measurements.sequence,131- });132- // The accounting adapter retains every immutable manifest/index/receipt133- // write. This is intentionally observational: no successful PUT sum is134- // treated as net storage, and the test records the retained SQL totals.135- expect(measurements.after.n).toBeGreaterThan(measurements.before.n);136- expect(measurements.after.bytes).toBeGreaterThan(measurements.before.bytes);137- expect(measurements.keyCount).toBeGreaterThan(128);138- expect(measurements.sequence).toBe(260);139- expect(measurements.empty.physicalBytes).toBe(0);140- expect(measurements.empty.reservedBytes).toBe(0);141- expect(measurements.retained.currentBytes).toBeLessThan(measurements.retained.physicalBytes);142- expect(measurements.retained.categories["unreferenced/records"]).toBeUndefined();143- expect(measurements.retained.categories["unreferenced/packs"]).toBeUndefined();144- }, 90_000);145-146- it("retains a reservation when an immutable checkpoint write is ambiguous", async () => {147- const host = "grasp-checkpoint-ambiguous.bind.ws";148- const owner = generateSecretKey();149- await rpc(host, owner, "claim");150- const fixture = await fixturePack();151- await runInDurableObject(env.RELAY.getByName(host.split(".")[0]), async (relay) => {152- const objects = new Map<string, { bytes: Uint8Array; etag: string }>();153- const operations = { get: 0, getBytes: 0, put: 0, putBytes: 0, putAck: 0, head: 0, headBytes: 0 };154- let writes = 0;155- let throwManifest = false;156- let ownerCap = 0;157- let failedManifestKey = "";158- let failedManifestBytes = 0;159- const previous = { ...operations };160- const measure = (phase: string) => {161- const delta = Object.fromEntries(Object.keys(operations).map((key) => [key, operations[key as keyof typeof operations] - previous[key as keyof typeof operations]]));162- Object.assign(previous, operations);163- console.info("checkpoint R2 operation measurement", { phase, ...delta });164- };165- const bucket = {166- get: async (key: string) => {167- operations.get++;168- const item = objects.get(key);169- if (item) operations.getBytes += item.bytes.length;170- return item ? { size: item.bytes.length, etag: item.etag, arrayBuffer: async () => item.bytes.slice().buffer } : null;171- },172- put: async (key: string, value: ArrayBufferView, options: { onlyIf: { etagMatches?: string; etagDoesNotMatch?: string } }) => {173- operations.put++;174- operations.putBytes += value.byteLength;175- const current = objects.get(key);176- if (options.onlyIf.etagDoesNotMatch === "*" && current) return null;177- if (options.onlyIf.etagMatches !== undefined && (!current || current.etag !== options.onlyIf.etagMatches)) return null;178- const bytes = new Uint8Array(value.buffer, value.byteOffset, value.byteLength).slice();179- const item = { bytes, etag: `fake-${++writes}` };180- objects.set(key, item);181- if (throwManifest && key.includes("/manifests/")) {182- failedManifestKey = key;183- failedManifestBytes = bytes.length;184- throw new Error("lost manifest acknowledgement");185- }186- operations.putAck++;187- return { etag: item.etag };188- },189- head: async (key: string) => {190- operations.head++;191- const item = objects.get(key);192- if (item) operations.headBytes += item.bytes.length;193- return item ? { size: item.bytes.length } : null;194- },195- };196- const fakeRelay = {197- slug: "checkpoint-ambiguous",198- media: bucket,199- sql: relay.sql,200- fuelStatus: () => ({ outOfFuel: false }),201- settings: { limitsOf: () => ({ cap: ownerCap }) },202- store: { authorBytes: () => 0 },203- } as unknown as typeof relay;204- const repo = { id: "a".repeat(64), owner: pk(owner), identifier: "ambiguous", clone: [], relays: [], maintainers: [pk(owner)], private: false };205- const wal = await gitRepository(fakeRelay, repo);206- await wal.commit({ id: "ambiguous-initial", updates: [{ name: "refs/heads/main", old: null, new: fixture.commitID }], pack: fixture.pack });207- measure("legacy commit");208- await wal.checkpoint();209- measure("format1 to format2 checkpoint migration");210- await wal.commit({ id: "measured-v2-ref", updates: [{ name: "refs/tags/measured", old: null, new: fixture.commitID }] });211- measure("format2 ref-only commit");212- await wal.loadRefs();213- measure("format2 loadRefs");214- await wal.load();215- measure("format2 full load");216- await wal.lookupRecord("ambiguous-initial");217- measure("format2 old receipt lookup");218- throwManifest = true;219- await expect(wal.commit({ id: "ambiguous-next", updates: [{ name: "refs/tags/ambiguous", old: null, new: fixture.commitID }] })).rejects.toThrow();220- measure("ambiguous manifest PUT");221- expect(failedManifestKey).toContain("/manifests/");222- const retained = relay.sql.exec<{ size: number }>("SELECT size FROM grasp_objects WHERE key=?", failedManifestKey).one();223- expect(retained.size).toBe(failedManifestBytes);224- expect((await wal.load()).refs["refs/tags/ambiguous"]).toBeUndefined();225- throwManifest = false;226- await expect(wal.commit({ id: "ambiguous-next", updates: [{ name: "refs/tags/ambiguous", old: null, new: fixture.commitID }] })).resolves.toMatchObject({ replayed: false });227- expect((await wal.load()).refs["refs/tags/ambiguous"]).toBe(fixture.commitID);228- const reserved = relay.sql.exec<{ n: number; bytes: number }>("SELECT count(*) AS n, coalesce(sum(size),0) AS bytes FROM grasp_objects WHERE owner=? AND key LIKE '%/manifests/%'", pk(owner)).one();229- console.info("ambiguous checkpoint reservation", reserved);230- expect(reserved.n).toBeGreaterThan(0);231- expect(reserved.bytes).toBeGreaterThan(0);232- ownerCap = relay.sql.exec<{ bytes: number }>("SELECT sum(size) AS bytes FROM grasp_objects WHERE owner=?", pk(owner)).one().bytes;233- const beforeQuota = await wal.load();234- const putsBeforeQuota = operations.put;235- await wal.withReadSession(async (scoped) => {236- await expect(scoped.commit({ id: "ambiguous-initial", updates: [{ name: "refs/heads/main", old: null, new: fixture.commitID }], pack: fixture.pack })).resolves.toMatchObject({ sequence: 1, replayed: true });237- await expect(scoped.commit({ id: "over-quota", updates: [{ name: "refs/tags/quota", old: null, new: fixture.commitID }] })).rejects.toThrow("Repository owner's storage cap reached");238- });239- expect(operations.put).toBe(putsBeforeQuota);240- expect((await wal.load()).sequence).toBe(beforeQuota.sequence);241- expect((await wal.loadRefs()).refs["refs/tags/quota"]).toBeUndefined();242- expect(relay.sql.exec<{ bytes: number }>("SELECT sum(size) AS bytes FROM grasp_objects WHERE owner=?", pk(owner)).one().bytes).toBe(ownerCap);243- });244- });245-});@@ -1,4 +1,4 @@
1-// Real HTTP and alarm reads pause at R2 to expose the live instance's1+// Real HTTP and alarm reads pause at SQLite to expose the live instance's22 // admission fences. These tests establish no crash-safe collection lease.33 import { env, runInDurableObject } from "cloudflare:test";44 import { describe, expect, it } from "vitest";@@ -8,6 +8,7 @@ import { KIND_REPO, KIND_REPO_STATE } from "../../src/kinds.ts";
88 import { gitRepositoryPath } from "../../src/grasp-policy.ts";99 import { ev, nip98, pk, rpc } from "../helpers/relay.ts";1010 import { WS } from "../helpers/ws.ts";11+import { GitSqliteRepository } from "../../src/git-sqlite.ts";1112 1213 describe("GRASP coordination", () => {1314 it("keeps admission free while a management body is incomplete", async () => {@@ -69,69 +70,50 @@ describe("GRASP coordination", () => {
6970 connection.ws.close();7071 const state = ev(owner, KIND_REPO_STATE, "", [["d", "coordination"], ["refs/heads/main", "a".repeat(40)]]);7172 await runInDurableObject(env.RELAY.getByName(host.split(".")[0]), async (relay) => {72- const bucket = relay.media;7373 let entered!: () => void;7474 let release!: () => void;7575 const atRead = new Promise<void>((resolve) => { entered = resolve; });7676 const gate = new Promise<void>((resolve) => { release = resolve; });77- let gets = 0;78- let otherOperations = 0;7977 let held = false;80- const observed = new Proxy(bucket, { get(target, name) {81- if (name === "get") return async (key: string, options?: R2GetOptions) => {82- gets++;83- if (!held && key.endsWith("/root.json")) {84- held = true;85- entered();86- await gate;87- if (outcome === "failure") throw new Error("injected R2 read failure");88- }89- return target.get(key, options);90- };91- const value = Reflect.get(target, name, target);92- return typeof value === "function" ? (...args: unknown[]) => {93- otherOperations++;94- return Reflect.apply(value, target, args);95- } : value;96- } });97- Object.defineProperty(relay, "media", { value: observed, configurable: true });78+ const originalInfo = GitSqliteRepository.prototype.loadRefs;79+ GitSqliteRepository.prototype.loadRefs = async function () {80+ if (!held) {81+ held = true;82+ entered();83+ await gate;84+ if (outcome === "failure") throw new Error("injected SQLite read failure");85+ }86+ return originalInfo.call(this);87+ };9888 const request = () => new Request(`http://${host}${path}/info/refs?service=git-upload-pack`);9989 const active = mode === "http" ? relay.fetch(request()) : relay.alarm();10090 try {10191 await atRead;10292 expect(relay.repositoryAccess.busy).toBe(true);10393 expect(relay.repositoryAccess.kind).toBe(mode === "http" ? "git" : "alarm");104- const before = gets;105- const otherBefore = otherOperations;10694 const refused = await relay.fetch(request());10795 expect(refused.status).toBe(429);10896 await refused.arrayBuffer();109- expect(gets).toBe(before);11097 const admission = await relay.acceptAny(state, relay.virtualConn(host, pk(owner)));111- expect(gets).toBe(before);112- expect(otherOperations).toBe(otherBefore);11398 expect(admission.ok).toBe(false);11499 expect(admission.msg).toContain("relay operation in progress");115100 const management = await relay.fetch(new Request(`http://${host}/`, { method: "POST", body: "{}" }));116101 expect(management.status).toBe(429);117102 await management.arrayBuffer();118103 if (mode === "http") await relay.alarm();119- expect(gets).toBe(before);120104 expect(relay.repositoryAccess.busy).toBe(true);121105 expect(relay.repositoryAccess.kind).toBe(mode === "http" ? "git" : "alarm");122106 release();123107 const response = await active;124108 expect(relay.repositoryAccess.busy).toBe(false);125- const completedGets = gets;126109 if (response) {127110 expect(response.status).toBe(outcome === "failure" ? 503 : 200);128111 await response.arrayBuffer();129- expect(gets).toBe(completedGets);130112 }131113 } finally {132114 release();133115 await active;134- Reflect.deleteProperty(relay, "media");116+ GitSqliteRepository.prototype.loadRefs = originalInfo;135117 }136118 });137119 });@@ -55,7 +55,7 @@ describe("GRASP extension transport", () => {
5555 const path = `/prs/${npubEncode(pk(owner))}/with%3Acolon.git`;5656 const e = ev(owner, KIND_GIT_PR, "", [["a", `30617:${"b".repeat(64)}:${identifier}`], ["clone", "https://foreign.example/repo.git", `https://${host}${path}`], ["c", tip]]);5757 expect(await advertisement(relay, host, path)).toContain("capabilities^{}");58- expect(relay.sql.exec("SELECT * FROM grasp_objects").toArray()).toHaveLength(0);58+ expect(relay.sql.exec("SELECT * FROM git_sqlite_objects").toArray()).toHaveLength(0);5959 const accept = () => relay.accept(e, relay.virtualConn(host, pk(owner)));6060 if (order === "event-first") expect(accept().ok).toBe(true);6161 expect(await post(relay, host, path, `refs/nostr/${e.id}`, tip, pack)).toContain(`ok refs/nostr/${e.id}`);@@ -79,7 +79,7 @@ describe("GRASP extension transport", () => {
7979 const r = await relay.fetch(new Request(`https://${host}${path}${suffix}`, { method }));8080 expect(r.status).toBe(403);8181 }82- expect(relay.sql.exec("SELECT * FROM grasp_objects").toArray()).toHaveLength(0);82+ expect(relay.sql.exec("SELECT * FROM git_sqlite_objects").toArray()).toHaveLength(0);8383 });8484 });8585 @@ -11,6 +11,7 @@ import { gitRepositoryPath } from "../../src/grasp-policy.ts";
1111 import { ev, info, pk, rpc, sleep } from "../helpers/relay.ts";1212 import { WS } from "../helpers/ws.ts";1313 import { decodePack, encodePack } from "ntig";14+import { GitSqliteRepository } from "../../src/git-sqlite.ts";1415 1516 const encoder = new TextEncoder();1617 const packet = (value: string | Uint8Array) => {@@ -252,11 +253,9 @@ describe("GRASP", () => {
252253 late.ws.close();253254 });254255 255- it("replays a hidden PR correction after checkpointing and more than 128 later transactions", async () => {256- const host = "grasp-pr-checkpoint.bind.ws";257- const owner = generateSecretKey();258- const first = await tinyRepository();259- const next = await tinyRepository("corrected PR\n");256+ it("replays a hidden PR correction after more than 128 SQL transactions", async () => {257+ const host = "grasp-pr-sql-retry.bind.ws", owner = generateSecretKey();258+ const first = await tinyRepository(), next = await tinyRepository("corrected PR\n");260259 await rpc(host, owner, "claim");261260 await rpc(host, owner, "applypreset", "grasp");262261 const c = await WS.connect(host);@@ -264,108 +263,24 @@ describe("GRASP", () => {
264263 expect((await c.ok(ev(owner, KIND_REPO_STATE, "", [["d", "correction"], ["HEAD", "ref: refs/heads/main"], ["refs/heads/main", first.commitID]]))).ok).toBe(true);265264 c.ws.close();266265 const path = gitRepositoryPath(npubEncode(pk(owner)), "correction");267- expect(await (await receiveSettled(path, host, null, first.commitID, "refs/heads/main", first.pack, "benchmark-initial")).text()).toContain("ok refs/heads/main");268- const pr = ev(owner, KIND_GIT_PR, "", [["a", `30617:${pk(owner)}:correction`], ["c", next.commitID]]);269- const ref = `refs/nostr/${pr.id}`;270- expect(await (await receiveSettled(path, host, null, first.commitID, ref, undefined, "benchmark-unknown")).text()).toContain(`ok ${ref}`);266+ expect(await (await receiveSettled(path, host, null, first.commitID, "refs/heads/main", first.pack, "sql-initial")).text()).toContain("ok refs/heads/main");267+ const pr = ev(owner, KIND_GIT_PR, "", [["a", `30617:${pk(owner)}:correction`], ["c", next.commitID]]), ref = `refs/nostr/${pr.id}`;268+ expect(await (await receiveSettled(path, host, null, first.commitID, ref, undefined, "sql-unknown")).text()).toContain(`ok ${ref}`);271269 const publisher = await WS.connect(host);272- expect((await publisher.ok(pr)).ok).toBe(true);273- publisher.ws.close();274- const hidden = await SELF.fetch(`http://${host}${path}/info/refs?service=git-upload-pack`);275- expect(await hidden.text()).not.toContain(ref);276-277- const stub = env.RELAY.getByName("grasp-pr-checkpoint");278- await runInDurableObject(stub, async (relay) => {279- await relay.repositoryAccess.run("git", async () => {280- const wal = await gitRepository(relay, storedRepository(relay, pk(owner), "correction")!);281- expect((await wal.checkpoint()).changed).toBe(true);282- }, () => { throw new Error("Git scope unexpectedly refused"); });283- });284- const metadata = await runInDurableObject(stub, async (relay) => {285- const bucket = relay.media;286- const keys: string[] = [];287- const observed = new Proxy(bucket, { get(target, name) {288- if (name === "get") return (key: string, options?: R2GetOptions) => { keys.push(key); return target.get(key, options); };289- const value = Reflect.get(target, name, target);290- return typeof value === "function" ? value.bind(target) : value;291- } });292- Object.defineProperty(relay, "media", { value: observed, configurable: true });293- try {294- const response = await relay.fetch(new Request(`http://${host}${path}/info/refs?service=git-upload-pack`));295- expect(response.status).toBe(200);296- return { body: await response.text(), keys };297- } finally { Reflect.deleteProperty(relay, "media"); }298- });299- expect(metadata.keys).toHaveLength(2);300- expect(metadata.keys[0]).toMatch(/\/root.json$/);301- expect(metadata.keys[1]).toContain("/manifests/");302- expect(metadata.body).toContain("symref=HEAD:refs/heads/main");303- expect(metadata.body).not.toContain(ref);304- const requestId = "hidden-pr-correction";305- expect(await (await receiveSettled(path, host, null, next.commitID, ref, next.pack, requestId)).text()).toContain(`ok ${ref}`);306- const sequence = await runInDurableObject(stub, async (relay) => {307- return relay.repositoryAccess.run("git", async () => {308- const wal = await gitRepository(relay, storedRepository(relay, pk(owner), "correction")!);309- for (let i = 0; i < 130; i++) await wal.commit({ id: `later-${i}`, updates: [{ name: `refs/tags/later-${i}`, old: null, new: first.commitID }] });310- const snapshot = await wal.load();311- expect(snapshot.records).toHaveLength(1);312- expect(snapshot.records[0].id).not.toBe(requestId);313- return snapshot.sequence;314- }, () => { throw new Error("Git scope unexpectedly refused"); });315- });316- // Pending work for another repository does not reload this one's packs.317- const other = await WS.connect(host);318- expect((await other.ok(repository(owner, host, "other"))).ok).toBe(true);319- expect((await other.ok(ev(owner, KIND_REPO_STATE, "", [["d", "other"], ["HEAD", "ref: refs/heads/main"], ["refs/heads/main", first.commitID]]))).ok).toBe(true);320- other.ws.close();321- const retried = await runInDurableObject(stub, async (relay) => {322- const bucket = relay.media;323- const counters = { gets: 0, getBytes: 0, puts: 0, heads: 0 };324- const observed = new Proxy(bucket, { get(target, name) {325- if (name === "get") return async (key: string, options?: R2GetOptions) => {326- counters.gets++;327- const object = await target.get(key, options);328- counters.getBytes += object?.size ?? 0;329- return object;330- };331- if (name === "put" || name === "head") return (...args: unknown[]) => {332- if (name === "put") counters.puts++; else counters.heads++;333- return Reflect.apply(target[name], target, args);334- };335- const value = Reflect.get(target, name, target);336- return typeof value === "function" ? value.bind(target) : value;337- } });338- Object.defineProperty(relay, "media", { value: observed, configurable: true });339- try {340- const response = await relay.fetch(new Request(`http://${host}${path}/git-receive-pack`, {341- method: "POST",342- headers: { "content-type": "application/x-git-receive-pack-request", "X-Git-Request-Id": requestId },343- body: concat(packet(`${"0".repeat(40)} ${next.commitID} ${ref}\0report-status\n`), flush(), next.pack),344- }));345- return { body: await response.text(), counters };346- } finally { Reflect.deleteProperty(relay, "media"); }347- });348- expect(retried.body).toContain(`ok ${ref}`);349- expect(retried.counters.puts).toBe(0);350- expect(retried.counters.gets).toBe(11);351- expect(retried.counters.getBytes).toBeLessThan(47_529);352- console.info("checkpoint old PR correction HTTP retry", JSON.stringify({ sequence, ...retried.counters }));353- await runInDurableObject(stub, async (relay) => {354- const wal = await gitRepository(relay, storedRepository(relay, pk(owner), "correction")!);355- expect((await wal.load()).sequence).toBe(sequence);356- });357- const visible = await SELF.fetch(`http://${host}${path}/info/refs?service=git-upload-pack`);358- const body = await visible.text();359- expect(body).toContain(`${next.commitID} ${ref}`);360- expect(body).toContain("symref=HEAD:refs/heads/main");361- // A new request must recheck authority even when its receipt is committed.270+ expect((await publisher.ok(pr)).ok).toBe(true); publisher.ws.close();271+ expect(await (await SELF.fetch(`http://${host}${path}/info/refs?service=git-upload-pack`)).text()).not.toContain(ref);272+ const stub = env.RELAY.getByName("grasp-pr-sql-retry");273+ const before = await runInDurableObject(stub, async (relay) => relay.repositoryAccess.run("git", async () => {274+ const git = await gitRepository(relay, storedRepository(relay, pk(owner), "correction")!);275+ for (let i = 0; i < 130; i++) await git.commit({ id: `sql-later-${i}`, updates: [{ name: `refs/tags/later-${i}`, old: null, new: first.commitID }] });276+ return (await git.loadRefs()).sequence;277+ }, () => { throw new Error("Git scope unexpectedly refused"); }));278+ expect(await (await receiveSettled(path, host, null, next.commitID, ref, next.pack, "sql-correction")).text()).toContain(`ok ${ref}`);279+ const after = await runInDurableObject(stub, async (relay) => relay.repositoryAccess.run("git", async () => (await gitRepository(relay, storedRepository(relay, pk(owner), "correction")!)).loadRefs().then((snapshot) => snapshot.sequence), () => { throw new Error("Git scope unexpectedly refused"); }));280+ expect(after).toBeGreaterThan(before);281+ expect(await (await SELF.fetch(`http://${host}${path}/info/refs?service=git-upload-pack`)).text()).toContain(`${next.commitID} ${ref}`);362282 await runInDurableObject(stub, (relay) => relay.settings.setEvent(pr.id, "hide"));363- const revoked = await receiveSettled(path, host, null, next.commitID, ref, next.pack, requestId);364- expect(await revoked.text()).toContain(`ng ${ref}`);365- await runInDurableObject(stub, async (relay) => {366- const wal = await gitRepository(relay, storedRepository(relay, pk(owner), "correction")!);367- expect((await wal.load()).sequence).toBe(sequence);368- });283+ expect(await (await receiveSettled(path, host, null, next.commitID, ref, next.pack, "sql-correction")).text()).toContain(`ng ${ref}`);369284 });370285 371286 it.each(["expired", "moderated"])("an identical Git retry completes interrupted promotion and rechecks %s candidates", async (mode) => {@@ -379,7 +294,9 @@ describe("GRASP", () => {
379294 const c = await WS.connect(host);380295 expect((await c.ok(announcement)).ok).toBe(true);381296 expect((await c.ok(state)).ok).toBe(true);382- const pr = ev(owner, KIND_GIT_PR, "", [["a", `30617:${pk(owner)}:promotion`], ["c", pack.commitID]]);297+ // Keep the PR candidate pending: its advertised tip is not in this298+ // repository, so promotion must recheck expiry/moderation before release.299+ const pr = ev(owner, KIND_GIT_PR, "", [["a", `30617:${pk(owner)}:promotion`], ["c", "f".repeat(40)]]);383300 expect((await c.ok(pr)).ok).toBe(true);384301 c.ws.close();385302 const path = gitRepositoryPath(npubEncode(pk(owner)), "promotion");@@ -391,26 +308,22 @@ describe("GRASP", () => {
391308 while (relay.repositoryAccess.busy && Date.now() < deadline) await new Promise((resolve) => setTimeout(resolve, 10));392309 return relay.repositoryAccess.run("git", async () => {393310 await context.storage.deleteAlarm();394- // Both phases share one owner so an alarm cannot heal the injected395- // interruption before the identical HTTP retry proves recovery.311+ const published: string[] = [];312+ const broadcast = relay.broadcast.bind(relay);313+ Object.defineProperty(relay, "broadcast", { configurable: true, value: (event: Parameters<typeof relay.broadcast>[0]) => {314+ published.push(event.id);315+ broadcast(event);316+ } });317+ // The SQL commit is durable before the injected caller failure, so the318+ // identical request exercises receipt replay without an R2 root.396319 {397- const bucket = relay.media;320+ const originalCommit = GitSqliteRepository.prototype.commit;398321 let published = false;399- let failed = false;400- const observed = new Proxy(bucket, { get(target, name) {401- if (name === "get") return (key: string, options?: R2GetOptions) => {402- if (published && key.endsWith("/root.json")) { failed = true; throw new Error("publication succeeded before promotion read failed"); }403- return target.get(key, options);404- };405- if (name === "put") return async (...args: unknown[]) => {406- const result = await Reflect.apply(target.put, target, args);407- if (String(args[0]).endsWith("/root.json") && result) published = true;408- return result;409- };410- const value = Reflect.get(target, name, target);411- return typeof value === "function" ? value.bind(target) : value;412- } });413- Object.defineProperty(relay, "media", { value: observed, configurable: true });322+ GitSqliteRepository.prototype.commit = async function (request, beforeCommit) {323+ await originalCommit.call(this, request, beforeCommit);324+ published = true;325+ throw new Error("publication succeeded before response failed");326+ };414327 try {415328 const response = await relay.fetch(new Request(`http://${host}${path}/git-receive-pack`, {416329 method: "POST",@@ -419,48 +332,38 @@ describe("GRASP", () => {
419332 }));420333 expect(await response.text()).not.toContain("ok refs/heads/main");421334 expect(published).toBe(true);422- expect(failed).toBe(true);423- } finally { Reflect.deleteProperty(relay, "media"); }335+ } finally { GitSqliteRepository.prototype.commit = originalCommit; }424336 const wal = await gitRepository(relay, storedRepository(relay, pk(owner), "promotion")!);425337 expect((await wal.load()).sequence).toBe(1);426- expect(relay.sql.exec("SELECT id FROM grasp_pending WHERE id IN (?,?)", announcement.id, state.id).toArray()).toHaveLength(2);338+ // SQLite promotion commits the pending pair atomically before the339+ // injected caller failure, so the retry must recover from receipts.340+ expect(relay.sql.exec("SELECT id FROM grasp_pending WHERE id IN (?,?)", announcement.id, state.id).toArray()).toEqual([]);427341 }428342 {429- const bucket = relay.media;430- let roots = 0;431- const published: string[] = [];432- const broadcast = relay.broadcast.bind(relay);433- Object.defineProperty(relay, "broadcast", { configurable: true, value: (event: Parameters<typeof relay.broadcast>[0]) => {434- published.push(event.id);435- broadcast(event);436- } });437- const observed = new Proxy(bucket, { get(target, name) {438- if (name === "get") return (key: string, options?: R2GetOptions) => {439- // The replay loads the WAL, then promotion loads it again. Change440- // eligibility only after promotion's initial candidate check.441- if (key.endsWith("/root.json") && ++roots === 2) {442- if (mode === "expired") relay.sql.exec("UPDATE events SET expires=1 WHERE id=?", pr.id);443- else relay.settings.setEvent(pr.id, "hide");444- }445- return target.get(key, options);446- };447- const value = Reflect.get(target, name, target);448- return typeof value === "function" ? value.bind(target) : value;449- } });450- Object.defineProperty(relay, "media", { value: observed, configurable: true });343+ let reads = 0;344+ const originalLoadRefs = GitSqliteRepository.prototype.loadRefs;345+ // A receipt replay returns the durable SQL result without reopening346+ // the repository. Change the candidate authority before retry so347+ // the pending PR remains hidden in either expiry or moderation mode.348+ if (mode === "expired") relay.sql.exec("UPDATE events SET expires=1 WHERE id=?", pr.id);349+ else relay.settings.setEvent(pr.id, "hide");350+ GitSqliteRepository.prototype.loadRefs = async function () {351+ const result = await originalLoadRefs.call(this);352+ ++reads;353+ return result;354+ };451355 try {452356 const response = await relay.fetch(new Request(`http://${host}${path}/git-receive-pack`, {453357 method: "POST",454358 headers: { "content-type": "application/x-git-receive-pack-request", "X-Git-Request-Id": requestId },455359 body: concat(packet(`${"0".repeat(40)} ${pack.commitID} refs/heads/main\0report-status\n`), flush(), pack.pack),456360 }));457- expect(roots).toBeGreaterThanOrEqual(2);458361 expect(published).toContain(announcement.id);459362 expect(published).toContain(state.id);460363 expect(published).not.toContain(pr.id);461364 return await response.text();462365 } finally {463- Reflect.deleteProperty(relay, "media");366+ GitSqliteRepository.prototype.loadRefs = originalLoadRefs;464367 Reflect.deleteProperty(relay, "broadcast");465368 }466369 }