- commit
- c33701a0162394c53061d136fdf22ac2c39295eb
- parent
- 5d1179c2a5dbdc3784e9dfb02173c0bd4ba79601
- author
- Dami <dami.etoile@gmail.com>
| M | src/ | 47 | +++++------ |
| A | src/ | 140 | ++++++++++++++++++++++++++++++ |
| M | src/ | 6 | +- |
| M | test/ | 6 | +- |
| A | test/ | 95 | ++++++++++++++++++++ |
@@ -11,6 +11,7 @@ import { type Relay } from "./relay.ts";
1111 import { KIND_PUSH_REGISTRATION } from "./kinds.ts";1212 import { archiveCurrent, isListKind } from "./list-history.ts";1313 import { Settings } from "./settings.ts";14+import { exportGitBackup, prepareGitBackup, restoreGitBackup, type GitBackup, type PreparedGit } from "./git-backup.ts";1415 1516 export const BACKUP_FORMAT = "bind.ws/relay-backup/1";1617 // JSON parsing, base64 expansion and integrity copies coexist in the Worker@@ -29,6 +30,7 @@ export type BackupArchive = {
2930 events: string[];3031 blobs: (Payload & { sha256: string; type: string; uploader: string; uploaded: number })[];3132 git: (Payload & { key: string })[];33+ sqlGit?: GitBackup;3234 state?: { listHistory: string[]; hidden: string[]; hosted: string[]; pending: { id: string; until: number }[]; prRefs?: { repo: string; ref: string; until: number }[] };3335 };3436 @@ -99,21 +101,10 @@ export async function createBackup(relay: Relay, id: string): Promise<{ manifest
99101 blobs.push({ ...b, data: toB64(data), sha256: b.sha256, size: data.length });100102 }101103 const git: BackupArchive["git"] = [];102- let cursor: string | undefined;103- for (;;) {104- const listed = await relay.media.list({ prefix: `${relay.slug}/git/`, cursor, limit: 1000 });105- for (const item of listed.objects) {106- if (git.length + blobs.length >= BACKUP_MAX_OBJECTS) return "restricted: backup object limit reached; no complete archive";107- if (item.size > 4 * 1024 * 1024) return "restricted: Git object exceeds the backup object limit";108- if (!reserve(item.size)) return "restricted: backup exceeds its bounded size or object limit";109- const obj = await relay.media.get(item.key);110- if (!obj) return `error: Git object ${item.key} disappeared during backup`;111- const data = new Uint8Array(await obj.arrayBuffer());112- git.push({ key: item.key.slice(relay.slug.length + 1), sha256: digest(data), size: data.length, data: toB64(data) });113- }114- if (!listed.truncated) break;115- cursor = listed.cursor;116- }104+ const sqlGit = exportGitBackup(relay, reserve);105+ if (typeof sqlGit === "string") return sqlGit;106+ const verifiedGit = await prepareGitBackup(sqlGit);107+ if (typeof verifiedGit === "string") return verifiedGit;117108 const config = exportConfig(relay.settings, relay.slug);118109 if (!reserve(enc.encode(JSON.stringify(config)).length)) return "restricted: backup exceeds its bounded size or object limit";119110 const state = { listHistory: [] as string[], hidden: [...relay.settings.hiddenEvents], hosted: relay.sql.exec<{ id: string }>(`SELECT id FROM grasp_hosted`).toArray().map((r) => r.id), pending: relay.sql.exec<{ id: string; until: number }>(`SELECT id,until FROM grasp_pending`).toArray(), prRefs: relay.sql.exec<{ repo: string; ref: string; until: number }>(`SELECT repo,ref,until FROM grasp_pr_refs`).toArray() };@@ -122,7 +113,7 @@ export async function createBackup(relay: Relay, id: string): Promise<{ manifest
122113 state.listHistory.push(row.raw);123114 }124115 if (!reserve(enc.encode(JSON.stringify({ ...state, listHistory: [] })).length, state.hidden.length + state.hosted.length + state.pending.length + state.prRefs.length)) return "restricted: backup state exceeds its bounded size";125- const archive = { format: BACKUP_FORMAT, manifest: { id, slug: relay.slug, owner, relayIdentity: relay.identity.pubkey, createdAt: now(), bytes: 0, events: events.length, blobs: blobs.length, git: git.length, archiveSha256: "" }, config, events, blobs, git, state } as BackupArchive;116+ const archive = { format: BACKUP_FORMAT, manifest: { id, slug: relay.slug, owner, relayIdentity: relay.identity.pubkey, createdAt: now(), bytes: 0, events: events.length, blobs: blobs.length, git: sqlGit.repositories.reduce((count, repo) => count + repo.objects.length, 0), archiveSha256: "" }, config, events, blobs, git, sqlGit, state } as BackupArchive;126117 archive.manifest.archiveSha256 = "0".repeat(64);127118 for (let i = 0; i < 4; i++) archive.manifest.bytes = bytesOf(archive).length;128119 archive.manifest.archiveSha256 = digest(unsignedBytes(archive));@@ -146,7 +137,8 @@ export async function deleteBackup(relay: Relay, id: string) {
146137 return true;147138 }148139 149-const checkedArchive = (bytes: Uint8Array): BackupArchive | string => {140+const preparedArchives = new WeakMap<BackupArchive, PreparedGit[]>();141+const checkedArchive = async (bytes: Uint8Array): Promise<BackupArchive | string> => {150142 if (bytes.length > BACKUP_MAX_BYTES) return "restricted: backup exceeds 8 MiB";151143 let archive: BackupArchive;152144 try { archive = JSON.parse(dec.decode(bytes)) as BackupArchive; } catch { return "invalid: backup is not JSON"; }@@ -158,7 +150,7 @@ const checkedArchive = (bytes: Uint8Array): BackupArchive | string => {
158150 catch { return "invalid: backup contains malformed event JSON"; }159151 }160152 for (const b of archive.blobs) { let data: Uint8Array; try { data = fromB64(b.data); } catch { return "invalid: malformed blob data"; } if (data.length !== b.size || digest(data) !== b.sha256) return "invalid: blob integrity check failed"; }161- for (const g of archive.git) { let data: Uint8Array; try { data = fromB64(g.data); } catch { return "invalid: malformed Git data"; } if (data.length !== g.size || digest(data) !== g.sha256 || typeof g.key !== "string" || !g.key.startsWith("git/")) return "invalid: Git integrity check failed"; }153+ if (archive.git.length) return "unsupported: R2 Git archives cannot be restored into SQLite";162154 const state = archive.state;163155 if (state) {164156 if (!Array.isArray(state.listHistory) || !Array.isArray(state.hidden) || !Array.isArray(state.hosted) || !Array.isArray(state.pending)) return "invalid: backup state malformed";@@ -171,6 +163,10 @@ const checkedArchive = (bytes: Uint8Array): BackupArchive | string => {
171163 catch { return "invalid: backup list history malformed"; }172164 }173165 }166+ const usedEntries = archive.events.length + archive.blobs.length + (state?.prRefs?.length ?? 0) + (state?.listHistory.length ?? 0) + (state?.hidden.length ?? 0) + (state?.hosted.length ?? 0) + (state?.pending.length ?? 0);167+ const git = await prepareGitBackup(archive.sqlGit, BACKUP_MAX_OBJECTS - usedEntries);168+ if (typeof git === "string") return git;169+ preparedArchives.set(archive, git);174170 return archive;175171 };176172 @@ -178,9 +174,9 @@ const checkedArchive = (bytes: Uint8Array): BackupArchive | string => {
178174 // signed and validated in the archive; direct storage preserves private data179175 // and avoids applying the target's ordinary write policy to old history.180176 export async function restoreBackup(relay: Relay, bytes: Uint8Array, caller: string): Promise<unknown | string> {181- const archive = checkedArchive(bytes);177+ const archive = await checkedArchive(bytes);182178 if (typeof archive === "string") return archive;183- if (relay.settings.policy.owner !== "" || relay.settings.isLeased() || relay.sql.exec(`SELECT 1 FROM events LIMIT 1`).toArray().length || relay.sql.exec(`SELECT 1 FROM blobs LIMIT 1`).toArray().length || relay.sql.exec(`SELECT 1 FROM grasp_objects LIMIT 1`).toArray().length || (relay.slug !== "" && (await relay.media.list({ prefix: `${relay.slug}/`, limit: 1 })).objects.length)) return "restricted: restore requires a fresh, unclaimed relay";179+ if (relay.settings.policy.owner !== "" || relay.settings.isLeased() || relay.sql.exec(`SELECT 1 FROM events LIMIT 1`).toArray().length || relay.sql.exec(`SELECT 1 FROM blobs LIMIT 1`).toArray().length || relay.sql.exec(`SELECT 1 FROM git_sqlite_meta UNION ALL SELECT 1 FROM git_sqlite_objects UNION ALL SELECT 1 FROM git_sqlite_catalog LIMIT 1`).toArray().length || (relay.slug !== "" && (await relay.media.list({ prefix: `${relay.slug}/`, limit: 1 })).objects.length)) return "restricted: restore requires a fresh, unclaimed relay";184180 if (archive.manifest.owner !== caller) return "restricted: target signer is not the backup owner";185181 const config = archive.config as Record<string, unknown>;186182 const parsed = parseConfig(config, relay.settings.policy);@@ -192,7 +188,6 @@ export async function restoreBackup(relay: Relay, bytes: Uint8Array, caller: str
192188 const key = `${relay.slug}/${b.sha256}`;193189 await relay.media.put(key, data, { httpMetadata: { contentType: b.type } }); staged.push(key);194190 }195- for (const g of archive.git) { const key = `${relay.slug}/${g.key}`; await relay.media.put(key, fromB64(g.data)); staged.push(key); }196191 } catch (error) {197192 await relay.media.delete(staged).catch(() => {});198193 return "error: restore storage staging failed";@@ -216,7 +211,7 @@ export async function restoreBackup(relay: Relay, bytes: Uint8Array, caller: str
216211 for (const row of archive.state?.pending ?? []) relay.sql.exec(`INSERT OR IGNORE INTO grasp_pending(id,until) SELECT id,? FROM events WHERE id=?`, row.until, row.id);217212 for (const row of archive.state?.prRefs ?? []) relay.sql.exec(`INSERT OR IGNORE INTO grasp_pr_refs(repo,ref,until) VALUES(?,?,?)`, row.repo, row.ref, row.until);218213 for (const b of archive.blobs) relay.sql.exec(`INSERT OR REPLACE INTO blobs(sha256,size,type,uploader,uploaded) VALUES(?,?,?,?,?)`, b.sha256, b.size, b.type, b.uploader, b.uploaded);219- for (const g of archive.git) relay.sql.exec(`INSERT OR REPLACE INTO grasp_objects(key,owner,size) VALUES(?,?,?)`, `${relay.slug}/${g.key}`, caller, g.size);214+ restoreGitBackup(relay, preparedArchives.get(archive)!);220215 });221216 } catch (error) {222217 relay.settings = new Settings(relay.sql);@@ -229,7 +224,7 @@ export async function restoreBackup(relay: Relay, bytes: Uint8Array, caller: str
229224 await relay.publishMembership();230225 await relay.publishDiscovery();231226 if (relay.sql.exec("SELECT 1 FROM grasp_pending UNION ALL SELECT 1 FROM grasp_pr_refs LIMIT 1").toArray().length || relay.settings.policy.features.grasp02) await relay.ensureAlarm(now() + 1);232- return { restored: true, owner: caller, sourceRelayIdentity: archive.manifest.relayIdentity, targetRelayIdentity: relay.identity.pubkey, events: archive.events.length, blobs: archive.blobs.length, git: archive.git.length, bytes: bytes.length };227+ return { restored: true, owner: caller, sourceRelayIdentity: archive.manifest.relayIdentity, targetRelayIdentity: relay.identity.pubkey, events: archive.events.length, blobs: archive.blobs.length, git: archive.manifest.git, bytes: bytes.length };233228 }234229 235230 export async function backupDownload(relay: Relay, req: Request, id: string): Promise<Response> {@@ -270,13 +265,13 @@ export async function restoreBackupRequest(relay: Relay, req: Request): Promise<
270265 const auth = verifyNIP98(req.headers.get("authorization") ?? "", req.url, req.method, body);271266 if (typeof auth === "string") return Response.json({ error: auth }, { status: 401 });272267 if (new URL(req.url).pathname === "/backups/preview") {273- const parsed = checkedArchive(bytes);268+ const parsed = await checkedArchive(bytes);274269 if (typeof parsed === "string") return Response.json({ error: parsed }, { status: 400 });275270 const cfg = parseConfig(parsed.config, relay.settings.policy);276271 if (typeof cfg === "string") return Response.json({ error: cfg }, { status: 400 });277- if (relay.settings.policy.owner !== "" || relay.settings.isLeased() || relay.sql.exec(`SELECT 1 FROM events LIMIT 1`).toArray().length || relay.sql.exec(`SELECT 1 FROM blobs LIMIT 1`).toArray().length || relay.sql.exec(`SELECT 1 FROM grasp_objects LIMIT 1`).toArray().length || (relay.slug !== "" && (await relay.media.list({ prefix: `${relay.slug}/`, limit: 1 })).objects.length)) return Response.json({ error: "restricted: restore requires a fresh, unclaimed relay" }, { status: 403 });272+ if (relay.settings.policy.owner !== "" || relay.settings.isLeased() || relay.sql.exec(`SELECT 1 FROM events LIMIT 1`).toArray().length || relay.sql.exec(`SELECT 1 FROM blobs LIMIT 1`).toArray().length || relay.sql.exec(`SELECT 1 FROM git_sqlite_meta UNION ALL SELECT 1 FROM git_sqlite_objects UNION ALL SELECT 1 FROM git_sqlite_catalog LIMIT 1`).toArray().length || (relay.slug !== "" && (await relay.media.list({ prefix: `${relay.slug}/`, limit: 1 })).objects.length)) return Response.json({ error: "restricted: restore requires a fresh, unclaimed relay" }, { status: 403 });278273 if (parsed.manifest.owner !== auth.pubkey) return Response.json({ error: "restricted: target signer is not the backup owner" }, { status: 403 });279- return Response.json({ result: { preview: true, source: parsed.manifest, targetIsFresh: true, config: parsed.config, events: parsed.events.length, blobs: parsed.blobs.length, git: parsed.git.length, bytes: bytes.length } });274+ return Response.json({ result: { preview: true, source: parsed.manifest, targetIsFresh: true, config: parsed.config, events: parsed.events.length, blobs: parsed.blobs.length, git: parsed.manifest.git, bytes: bytes.length } });280275 }281276 const result = await restoreBackup(relay, bytes, auth.pubkey);282277 return typeof result === "string" ? Response.json({ error: result }, { status: result.startsWith("invalid:") ? 400 : 403 }) : Response.json({ result });@@ -0,0 +1,140 @@
1+// Portable SQLite Git snapshots retain compressed objects and retry receipts.2+// Preparation verifies identities, hashes, graphs and limits before restore3+// enters the relay transaction; only validated rows reach publication.4+import { sha256 } from "@noble/hashes/sha2.js";5+import { sha1 } from "@noble/hashes/legacy.js";6+import { bytesToHex } from "@noble/hashes/utils.js";7+import { unzlibSync } from "fflate";8+import { objectLinks, validateRefName, repositoryStoragePrefix, repositoryAddress, type GitObjectType, type RefUpdate } from "ntig";9+import { npubEncode } from "nostr-tools/nip19";10+import { DEFAULT_GIT_SQLITE_LIMITS as LIMITS } from "./git-sqlite.ts";11+import { gitSql } from "./git-catalog.ts";12+import type { Relay } from "./relay.ts";13+14+const enc = new TextEncoder();15+const hex = /^[0-9a-f]{40}$/;16+const hash = /^[0-9a-f]{64}$/;17+const digest = (value: unknown) => bytesToHex(sha256(enc.encode(JSON.stringify(value))));18+const size = (value: string) => enc.encode(value).length;19+const base64 = (bytes: Uint8Array) => { let value = ""; for (let at = 0; at < bytes.length; at += 32768) value += String.fromCharCode(...bytes.subarray(at, at + 32768)); return btoa(value); };20+const decoded = (value: string) => Uint8Array.from(atob(value), character => character.charCodeAt(0));21+22+type ObjectRow = { oid: string; type: GitObjectType; size: number; chunks: string[] };23+type RecordRow = { id: string; sequence: number; requestHash: string; recordHash: string; packHash: string | null; updates: RefUpdate[] };24+type Repository = { key: string; owner: string; identifier: string; alternative: boolean; objects: ObjectRow[]; refs: Record<string, string>; receipts: RecordRow[] };25+export type GitBackup = { format: "bind.ws/git-sqlite/1"; repositories: Repository[] };26+export type PreparedGit = { repo: Repository; objects: { row: ObjectRow; chunks: Uint8Array[]; links: ReturnType<typeof objectLinks> }[]; raw: number; compressed: number; metadata: number; tip: string | null };27+28+// exportGitBackup reserves archive memory before reading payloads from SQLite.29+export function exportGitBackup(relay: Relay, reserve: (bytes: number, entries?: number) => boolean): GitBackup | string {30+ const result: GitBackup = { format: "bind.ws/git-sqlite/1", repositories: [] };31+ for (const row of gitSql<{ repository: string; compressed_bytes: number; metadata_bytes: number; object_count: number }>(relay, "SELECT repository,compressed_bytes,metadata_bytes,object_count FROM git_sqlite_meta ORDER BY repository LIMIT 12001").toArray()) {32+ if (!reserve(row.compressed_bytes + row.metadata_bytes, row.object_count + 1)) return "restricted: Git backup exceeds its bounded size or object limit";33+ const identity = gitSql<{ owner: string; identifier: string; alternative: number }>(relay, "SELECT owner,identifier,alternative FROM git_sqlite_catalog WHERE repository=?", row.repository).toArray()[0];34+ if (!identity) return "error: Git repository identity is missing";35+ const repo: Repository = { key: row.repository, owner: identity.owner, identifier: identity.identifier, alternative: !!identity.alternative, objects: [], refs: {}, receipts: [] };36+ for (const object of gitSql<{ oid: string; type: GitObjectType; size: number }>(relay, "SELECT oid,type,raw_size AS size FROM git_sqlite_objects WHERE repository=? ORDER BY oid LIMIT 4097", row.repository).toArray()) {37+ const chunks = gitSql<{ data: ArrayBuffer }>(relay, "SELECT data FROM git_sqlite_object_chunks WHERE repository=? AND oid=? ORDER BY chunk LIMIT 10", row.repository, object.oid).toArray().map(chunk => base64(new Uint8Array(chunk.data)));38+ if (!reserve(0, chunks.length)) return "restricted: Git backup object limit reached";39+ repo.objects.push({ ...object, chunks });40+ }41+ const refs = gitSql<{ name: string; oid: string }>(relay, "SELECT name,oid FROM git_sqlite_refs WHERE repository=? ORDER BY name LIMIT 1025", row.repository).toArray();42+ repo.refs = Object.fromEntries(refs.map(ref => [ref.name, ref.oid]));43+ repo.receipts = gitSql<{ id: string; sequence: number; requestHash: string; recordHash: string; packHash: string | null; updates: string }>(relay, "SELECT id,sequence,request_hash AS requestHash,record_hash AS recordHash,pack_hash AS packHash,updates FROM git_sqlite_receipts WHERE repository=? ORDER BY sequence LIMIT 12001", row.repository).toArray().map(receipt => ({ ...receipt, updates: JSON.parse(receipt.updates) as RefUpdate[] }));44+ if (!reserve(0, refs.length + repo.receipts.length)) return "restricted: Git backup object limit reached";45+ result.repositories.push(repo);46+ }47+ return result;48+}49+50+// prepareGitBackup derives links from verified bodies and replays the receipt51+// chain so neither forged metadata nor a different namespace survives restore.52+export async function prepareGitBackup(input: unknown, maxEntries = 12000): Promise<PreparedGit[] | string> {53+ if (input === undefined) return [];54+ try {55+ const backup = input as GitBackup;56+ if (!backup || backup.format !== "bind.ws/git-sqlite/1" || !Array.isArray(backup.repositories) || backup.repositories.length > maxEntries) throw new Error();57+ const result: PreparedGit[] = [], keys = new Set<string>();58+ let entries = 0;59+ for (const repo of backup.repositories) {60+ if (!repo || !hash.test(repo.key) || keys.has(repo.key) || !hash.test(repo.owner) || typeof repo.identifier !== "string" || typeof repo.alternative !== "boolean" || !Array.isArray(repo.objects) || repo.objects.length > LIMITS.maxObjects || !Array.isArray(repo.receipts) || !repo.refs || typeof repo.refs !== "object" || Array.isArray(repo.refs)) throw new Error();61+ const prefix = await repositoryStoragePrefix(repositoryAddress(npubEncode(repo.owner), repo.identifier), repo.alternative);62+ if (prefix.split("/").filter(Boolean).at(-1) !== repo.key) throw new Error("namespace");63+ keys.add(repo.key);64+ entries += 1 + repo.objects.length + Object.keys(repo.refs).length + repo.receipts.length;65+ if (entries > maxEntries || Object.keys(repo.refs).length > LIMITS.maxRefs) throw new Error("limit");66+ const prepared: PreparedGit = { repo, objects: [], raw: 0, compressed: 0, metadata: 0, tip: null };67+ const objects = new Map<string, { type: GitObjectType; links: ReturnType<typeof objectLinks> }>();68+ let edges = 0;69+ for (const row of repo.objects) {70+ if (!row || !hex.test(row.oid) || objects.has(row.oid) || !["blob", "tree", "commit", "tag"].includes(row.type) || !Number.isSafeInteger(row.size) || row.size < 0 || row.size > LIMITS.maxObjectBytes || !Array.isArray(row.chunks) || !row.chunks.length || row.chunks.length > 9) throw new Error();71+ prepared.raw += row.size; entries += row.chunks.length;72+ if (prepared.raw > LIMITS.maxRawBytes || entries > maxEntries) throw new Error("limit");73+ const chunks = row.chunks.map(value => {74+ if (typeof value !== "string" || value.length > Math.ceil(LIMITS.chunkBytes / 3) * 4 || !/^[A-Za-z0-9+/]*={0,2}$/.test(value)) throw new Error();75+ const bytes = decoded(value);76+ if (!bytes.length || bytes.length > LIMITS.chunkBytes) throw new Error();77+ return bytes;78+ });79+ const compressedSize = chunks.reduce((total, chunk) => total + chunk.length, 0);80+ if (compressedSize > LIMITS.maxObjectBytes + 65536) throw new Error("limit");81+ prepared.compressed += compressedSize;82+ if (prepared.compressed > LIMITS.maxCompressedBytes) throw new Error("limit");83+ const compressed = new Uint8Array(compressedSize); let at = 0;84+ for (const chunk of chunks) { compressed.set(chunk, at); at += chunk.length; }85+ const data = unzlibSync(compressed, { out: new Uint8Array(row.size + 1) });86+ if (data.length !== row.size || bytesToHex(sha1.create().update(enc.encode(`${row.type} ${data.length}\0`)).update(data).digest()) !== row.oid) throw new Error("hash");87+ const links = objectLinks({ oid: row.oid, type: row.type, data }, LIMITS.maxGraphEdges - edges);88+ edges += links.length;89+ if (edges > LIMITS.maxGraphEdges) throw new Error("limit");90+ objects.set(row.oid, { type: row.type, links });91+ prepared.objects.push({ row, chunks, links });92+ prepared.metadata += size(repo.key) + 80 + chunks.length * (size(repo.key) + 48) + links.length * (size(repo.key) + 96);93+ }94+ for (const object of objects.values()) for (const link of object.links) if (objects.get(link.oid)?.type !== link.type) throw new Error("graph");95+ const refs = new Map<string, string>(), ids = new Set<string>();96+ for (const [index, receipt] of repo.receipts.entries()) {97+ if (!receipt || !/^[a-zA-Z0-9_.:-]{1,128}$/.test(receipt.id) || ids.has(receipt.id) || receipt.sequence !== index + 1 || !hash.test(receipt.requestHash) || !hash.test(receipt.recordHash) || (receipt.packHash !== null && !hash.test(receipt.packHash)) || !Array.isArray(receipt.updates) || !receipt.updates.length || receipt.updates.length > LIMITS.maxRefs) throw new Error("receipt");98+ ids.add(receipt.id);99+ const names = new Set<string>();100+ let previous = "";101+ for (const update of receipt.updates) {102+ validateRefName(update.name);103+ if (names.has(update.name) || update.name < previous || (update.old !== null && (!hex.test(update.old) || !objects.has(update.old))) || (update.new !== null && (!hex.test(update.new) || !objects.has(update.new))) || (refs.get(update.name) ?? null) !== update.old) throw new Error("receipt refs");104+ previous = update.name; names.add(update.name);105+ if (update.new === null) refs.delete(update.name); else refs.set(update.name, update.new);106+ }107+ if (digest({ id: receipt.id, updates: receipt.updates, pack: receipt.packHash }) !== receipt.requestHash || digest({ sequence: receipt.sequence, parent: prepared.tip, id: receipt.id, digest: receipt.requestHash }) !== receipt.recordHash) throw new Error("receipt hash");108+ prepared.tip = receipt.recordHash;109+ prepared.metadata += size(repo.key) + size(JSON.stringify(receipt.updates)) + size(receipt.id) + 160;110+ }111+ if (!repo.receipts.length || refs.size !== Object.keys(repo.refs).length) throw new Error("refs");112+ for (const [name, oid] of Object.entries(repo.refs)) {113+ validateRefName(name);114+ if (refs.get(name) !== oid || !objects.has(oid) || (name.startsWith("refs/heads/") && objects.get(oid)?.type !== "commit")) throw new Error("refs");115+ const parts = name.split("/");116+ for (let index = 2; index < parts.length; index++) if (refs.has(parts.slice(0, index).join("/"))) throw new Error("refs");117+ prepared.metadata += size(repo.key) + size(name) + 40;118+ }119+ if (prepared.metadata > LIMITS.maxMetadataBytes) throw new Error("limit");120+ result.push(prepared);121+ }122+ return result;123+ } catch { return "invalid: SQLite Git backup identity, objects, receipts or limits failed validation"; }124+}125+126+// restoreGitBackup runs inside the caller's transaction after preparation.127+export function restoreGitBackup(relay: Relay, prepared: PreparedGit[]) {128+ if (gitSql(relay, "SELECT 1 FROM git_sqlite_meta UNION ALL SELECT 1 FROM git_sqlite_objects UNION ALL SELECT 1 FROM git_sqlite_catalog LIMIT 1").toArray().length) throw new Error("error: Git restore requires an empty target");129+ for (const { repo, objects, raw, compressed, metadata, tip } of prepared) {130+ gitSql(relay, "INSERT INTO git_sqlite_catalog(repository,owner,identifier,alternative) VALUES(?,?,?,?)", repo.key, repo.owner, repo.identifier, repo.alternative ? 1 : 0);131+ for (const { row, chunks, links } of objects) {132+ gitSql(relay, "INSERT INTO git_sqlite_objects(repository,oid,type,raw_size,compressed_size,chunks) VALUES(?,?,?,?,?,?)", repo.key, row.oid, row.type, row.size, chunks.reduce((total, chunk) => total + chunk.length, 0), chunks.length);133+ for (const [index, chunk] of chunks.entries()) gitSql(relay, "INSERT INTO git_sqlite_object_chunks(repository,oid,chunk,data) VALUES(?,?,?,?)", repo.key, row.oid, index, chunk.slice().buffer);134+ for (const [index, link] of links.entries()) gitSql(relay, "INSERT INTO git_sqlite_object_links(repository,oid,position,link_oid,link_type) VALUES(?,?,?,?,?)", repo.key, row.oid, index, link.oid, link.type);135+ }136+ for (const [name, oid] of Object.entries(repo.refs)) gitSql(relay, "INSERT INTO git_sqlite_refs(repository,name,oid) VALUES(?,?,?)", repo.key, name, oid);137+ for (const receipt of repo.receipts) gitSql(relay, "INSERT INTO git_sqlite_receipts(repository,id,sequence,request_hash,record_hash,updates,pack_hash) VALUES(?,?,?,?,?,?,?)", repo.key, receipt.id, receipt.sequence, receipt.requestHash, receipt.recordHash, JSON.stringify(receipt.updates), receipt.packHash);138+ gitSql(relay, "INSERT INTO git_sqlite_meta(repository,sequence,tip,compressed_bytes,raw_bytes,object_count,metadata_bytes) VALUES(?,?,?,?,?,?,?)", repo.key, repo.receipts.length, tip, compressed, raw, objects.length, metadata);139+ }140+}@@ -1,5 +1,5 @@
1-// GRASP's accepted Nostr authority and purgatory. Git refs remain in ntig;2-// these tables record only visibility, cleanup deadlines and billed objects.1+// GRASP's accepted Nostr authority and purgatory. Git objects and refs live2+// in the SQLite backend; these tables record visibility and cleanup deadlines.33 import { now, tag, tagValues, type Event } from "./event.ts";44 import { KIND_REPO, KIND_REPO_STATE, KIND_GIT_PR, KIND_GIT_PR_UPDATE } from "./kinds.ts";55 import { featureOn } from "./settings.ts";@@ -16,7 +16,6 @@ CREATE TABLE IF NOT EXISTS grasp_pending (id TEXT PRIMARY KEY REFERENCES events(
1616 CREATE TRIGGER IF NOT EXISTS grasp_removed AFTER DELETE ON events BEGIN DELETE FROM grasp_pending WHERE id=old.id; DELETE FROM grasp_hosted WHERE id=old.id; END;1717 CREATE TABLE IF NOT EXISTS grasp_tick (id INTEGER PRIMARY KEY CHECK(id=1), coordinate TEXT NOT NULL);1818 CREATE TABLE IF NOT EXISTS grasp_pr_refs (repo TEXT NOT NULL, ref TEXT NOT NULL, until INTEGER NOT NULL, PRIMARY KEY(repo,ref));19-CREATE TABLE IF NOT EXISTS grasp_objects (key TEXT PRIMARY KEY, owner TEXT NOT NULL, size INTEGER NOT NULL);2019 CREATE TABLE IF NOT EXISTS grasp_git_sync (id TEXT PRIMARY KEY, due INTEGER NOT NULL DEFAULT 0, attempts INTEGER NOT NULL DEFAULT 0, error TEXT NOT NULL DEFAULT '');2120 CREATE TABLE IF NOT EXISTS grasp_event_sync (id TEXT PRIMARY KEY, source TEXT NOT NULL, scope TEXT NOT NULL, windows TEXT NOT NULL, live_since INTEGER NOT NULL, due INTEGER NOT NULL DEFAULT 0, failures INTEGER NOT NULL DEFAULT 0, error TEXT NOT NULL DEFAULT '');2221 CREATE INDEX IF NOT EXISTS grasp_event_sync_due ON grasp_event_sync(due,id);@@ -110,7 +109,6 @@ export function holdGrasp(relay: Relay, e: Event, host: string): boolean {
110109 return true;111110 }112111 export const graspVisible = (relay: Relay, id: string) => relay.sql.exec(`SELECT 1 FROM grasp_pending WHERE id=?`, id).toArray().length === 0;113-export const graspBytes = (relay: Relay) => relay.sql.exec<{ n: number }>(`SELECT coalesce(sum(size),0) AS n FROM grasp_objects`).one().n;114112 115113 // requiredObjects returns the tips an event needs before it leaves purgatory.116114 export function requiredObjects(relay: Relay, e: Event): string[] | null {@@ -78,7 +78,7 @@ describe("portable backups", () => {
7878 expect(response.status).toBe(413);7979 });8080 81- it("round trips configuration, events, a media blob and a Git object, with preview and exclusions", async () => {81+ it("round trips configuration, events and a media blob, with preview and exclusions", async () => {8282 const source = "backup-source.bind.ws";8383 const owner = generateSecretKey(), writer = generateSecretKey();8484 await rpc(source, owner, "claim");@@ -96,7 +96,6 @@ describe("portable backups", () => {
9696 relay.sql.exec(`INSERT INTO grasp_pr_refs(repo,ref,until) VALUES(?,?,?)`, `pr:${pk(writer)}:repo`, `refs/nostr/${"a".repeat(64)}`, 9999999999);9797 relay.sql.exec(`INSERT INTO grasp_pr_refs(repo,ref,until) VALUES(?,?,?)`, `pr:${pk(writer)}:repo`, `refs/nostr/${"b".repeat(64)}`, 0);9898 await storeBlob(relay, new TextEncoder().encode("site bytes"), "text/plain", pk(writer), 10);99- await relay.media.put("backup-source/git/test-object", new TextEncoder().encode("git bytes"));10099 await relay.store.save(ev(owner, 30390, '{"callback":"https://secret.invalid"}'), 10);101100 const result = await createBackup(relay, "roundtrip");102101 expect(typeof result).not.toBe("string");@@ -114,7 +113,7 @@ describe("portable backups", () => {
114113 return result as { blobs: number; git: number };115114 });116115 expect(preview.blobs).toBe(1);117- expect(preview.git).toBe(1);116+ expect(preview.git).toBe(0);118117 await runInDurableObject(env.RELAY.getByName("backup-target"), async (relay: Relay) => {119118 expect(relay.settings.policy.name).toBe("Recovered");120119 expect(relay.sql.exec(`SELECT repo,ref,until FROM grasp_pr_refs ORDER BY ref`).toArray()).toEqual([{ repo: `pr:${pk(writer)}:repo`, ref: `refs/nostr/${"a".repeat(64)}`, until: 9999999999 }, { repo: `pr:${pk(writer)}:repo`, ref: `refs/nostr/${"b".repeat(64)}`, until: 0 }]);@@ -122,7 +121,6 @@ describe("portable backups", () => {
122121 expect(relay.settings.hiddenEvents.size).toBe(1);123122 expect(relay.sql.exec(`SELECT count(*) n FROM grasp_pending`).one().n).toBe(1);124123 expect((await relay.media.get("backup-target/" + ""))).toBeNull();125- expect((await relay.media.get("backup-target/" + "git/test-object"))?.size).toBe(9);126124 });127125 });128126 @@ -0,0 +1,95 @@
1+import { env, runInDurableObject } from "cloudflare:test";2+import { describe, expect, it } from "vitest";3+import { encodePack, type GitObject } from "ntig";4+import { generateSecretKey } from "nostr-tools/pure";5+import { sha256 } from "@noble/hashes/sha2.js";6+import { bytesToHex } from "@noble/hashes/utils.js";7+import { BACKUP_MAX_BYTES, createBackup, restoreBackup } from "../../src/backups.ts";8+import { gitRepository } from "../../src/grasp.ts";9+import type { RepositoryAnnouncement } from "../../src/grasp-policy.ts";10+import type { Relay } from "../../src/relay.ts";11+import { pk, rpc } from "../helpers/relay.ts";12+13+const text = new TextEncoder();14+const digest = (bytes: Uint8Array) => bytesToHex(sha256(bytes));15+const hex = (value: string) => { const output = new Uint8Array(value.length / 2); for (let i = 0; i < output.length; i++) output[i] = Number.parseInt(value.slice(i * 2, i * 2 + 2), 16); return output; };16+async function oid(type: string, data: Uint8Array) {17+ const header = text.encode(`${type} ${data.length}\0`); const input = new Uint8Array(header.length + data.length); input.set(header); input.set(data, header.length);18+ return bytesToHex(new Uint8Array(await crypto.subtle.digest("SHA-1", input)));19+}20+async function gitCommit(message: string, parent: string | null = null) {21+ const blobData = text.encode(`${message}\n`); const blob = await oid("blob", blobData);22+ const treeData = new Uint8Array(32); treeData.set(text.encode("100644 file\0")); treeData.set(hex(blob), 12); const tree = await oid("tree", treeData);23+ const commitData = text.encode(`tree ${tree}\n${parent ? `parent ${parent}\n` : ""}author Backup <backup@example.com> 1 +0000\ncommitter Backup <backup@example.com> 1 +0000\n\n${message}\n`); const commit = await oid("commit", commitData);24+ const objects: GitObject[] = [{ oid: blob, type: "blob", data: blobData }, { oid: tree, type: "tree", data: treeData }, { oid: commit, type: "commit", data: commitData }];25+ return { commit, pack: await encodePack(objects) };26+}27+const repository = (owner: string, identifier: string): RepositoryAnnouncement => ({ id: `repo:${owner}:${identifier}`, owner, identifier, clone: [], relays: [], maintainers: [], private: false });28+29+async function archiveBytes(relay: Relay, key: string) {30+ const object = await relay.media.get(`${relay.slug}/backups/${key}.json`); return new Uint8Array(await object!.arrayBuffer());31+}32+function updateArchiveHash(archive: Record<string, any>) {33+ const copy = structuredClone(archive); copy.manifest.archiveSha256 = ""; archive.manifest.archiveSha256 = digest(new TextEncoder().encode(JSON.stringify(copy)));34+}35+36+describe("SQLite Git backups", () => {37+ it("round trips a cataloged member-owned repository and permits further push and replay", async () => {38+ const ownerSecret = generateSecretKey(); const owner = pk(ownerSecret); const member = pk(generateSecretKey()); const source = `sql-git-backup-source-${owner.slice(0, 8)}`; const target = `sql-git-backup-target-${owner.slice(0, 8)}`;39+ await rpc(`${source}.bind.ws`, ownerSecret, "claim");40+ const archive = await runInDurableObject(env.RELAY.getByName(source), async (relay: Relay) => {41+ const announcement = repository(member, "member-repo"); const git = await gitRepository(relay, announcement);42+ const first = await gitCommit("first"); await git.commit({ id: "first", updates: [{ name: "refs/heads/main", old: null, new: first.commit }], pack: first.pack });43+ const second = await gitCommit("second", first.commit); await git.commit({ id: "second", updates: [{ name: "refs/heads/main", old: first.commit, new: second.commit }], pack: second.pack });44+ const catalog = relay.sql.exec<{ owner: string; identifier: string }>("SELECT owner,identifier FROM git_sqlite_catalog WHERE repository=?", git.prefix).one();45+ expect(catalog).toEqual({ owner: member, identifier: "member-repo" });46+ expect(typeof await createBackup(relay, "sql-roundtrip")).not.toBe("string");47+ return archiveBytes(relay, "sql-roundtrip");48+ });49+ const restored = await runInDurableObject(env.RELAY.getByName(target), async (relay: Relay) => {50+ relay.slug = `${target}.bind.ws`;51+ const result = await restoreBackup(relay, archive, owner);52+ expect(typeof result).not.toBe("string");53+ const row = relay.sql.exec<{ repository: string; owner: string; identifier: string }>("SELECT c.repository,c.owner,c.identifier FROM git_sqlite_catalog c").one();54+ expect(row.owner).toBe(member); expect(row.identifier).toBe("member-repo");55+ const git = await gitRepository(relay, repository(member, "member-repo"));56+ expect((await git.loadRefs()).refs["refs/heads/main"]).toBeTruthy();57+ const replay = await git.commit({ id: "second", updates: [{ name: "refs/heads/main", old: "0".repeat(40), new: "f".repeat(40) }] }).catch((error) => error);58+ expect(replay).toBeInstanceOf(Error);59+ const oldTip = (await git.loadRefs()).refs["refs/heads/main"];60+ const third = await gitCommit("third", oldTip);61+ const thirdRequest = { id: "third", updates: [{ name: "refs/heads/main", old: oldTip, new: third.commit }], pack: third.pack };62+ await git.commit(thirdRequest);63+ expect((await git.commit(thirdRequest)).replayed).toBe(true);64+ });65+ expect(restored).toBeUndefined();66+ });67+68+ it("rejects rehashed structural tampering and namespace mismatch before target writes", async () => {69+ const owner = pk(generateSecretKey()); const source = `sql-git-backup-tamper-source-${owner.slice(0, 8)}`; const target = `sql-git-backup-tamper-target-${owner.slice(0, 8)}`;70+ const original = await runInDurableObject(env.RELAY.getByName(source), async (relay: Relay) => {71+ relay.settings.update({ owner });72+ const git = await gitRepository(relay, repository(owner, "tamper")); const first = await gitCommit("tamper"); await git.commit({ id: "tamper", updates: [{ name: "refs/heads/main", old: null, new: first.commit }], pack: first.pack }); await createBackup(relay, "tamper"); return archiveBytes(relay, "tamper");73+ });74+ const tampered = JSON.parse(new TextDecoder().decode(original)) as Record<string, any>;75+ tampered.sqlGit.repositories[0].key = "f".repeat(64); updateArchiveHash(tampered);76+ const damaged = JSON.parse(new TextDecoder().decode(original)) as Record<string, any>;77+ const chunk = damaged.sqlGit.repositories[0].objects[0].chunks[0]; const middle = Math.floor(chunk.length / 2);78+ damaged.sqlGit.repositories[0].objects[0].chunks[0] = `${chunk.slice(0, middle)}${chunk[middle] === "A" ? "B" : "A"}${chunk.slice(middle + 1)}`; updateArchiveHash(damaged);79+ await runInDurableObject(env.RELAY.getByName(target), async (relay: Relay) => {80+ relay.slug = `${target}.bind.ws`;81+ for (const candidate of [tampered, damaged]) {82+ const result = await restoreBackup(relay, text.encode(JSON.stringify(candidate)), owner);83+ expect(String(result)).toMatch(/^invalid:/);84+ expect(relay.settings.policy.owner).toBe("");85+ expect(relay.sql.exec("SELECT 1 FROM git_sqlite_objects LIMIT 1").toArray()).toHaveLength(0);86+ }87+ });88+ });89+90+ it("rejects a snapshot over the portable size bound before touching a target", async () => {91+ const owner = pk(generateSecretKey());92+ const result = await runInDurableObject(env.RELAY.getByName("sql-git-backup-size"), async (relay: Relay) => restoreBackup(relay, new Uint8Array(BACKUP_MAX_BYTES + 1), owner));93+ expect(result).toMatch(/^restricted: backup exceeds/);94+ });95+});