1// Portable SQLite Git snapshots retain compressed objects and retry receipts.2// Preparation verifies identities, hashes, graphs and limits before restore3// enters the relay transaction; only validated rows reach publication.4import { sha256 } from "@noble/hashes/sha2.js";5import { sha1 } from "@noble/hashes/legacy.js";6import { bytesToHex } from "@noble/hashes/utils.js";7import { unzlibSync } from "fflate";8import { objectLinks, validateRefName, repositoryStoragePrefix, repositoryAddress, type GitObjectType, type RefUpdate } from "ntig";9import { npubEncode } from "nostr-tools/nip19";10import { DEFAULT_GIT_SQLITE_LIMITS as LIMITS, type GitSqliteLimits } from "./git-sqlite.ts";11import { gitSql } from "./git-catalog.ts";12import type { Relay } from "./relay.ts";1314const enc = new TextEncoder();15const hex = /^[0-9a-f]{40}$/;16const hash = /^[0-9a-f]{64}$/;17const digest = (value: unknown) => bytesToHex(sha256(enc.encode(JSON.stringify(value))));18const size = (value: string) => enc.encode(value).length;19const base64 = (bytes: Uint8Array) => { let value = ""; for (let at = 0; at < bytes.length; at += 32768) value += String.fromCharCode(...bytes.subarray(at, at + 32768)); return btoa(value); };20const decoded = (value: string) => Uint8Array.from(atob(value), character => character.charCodeAt(0));2122type ObjectRow = { oid: string; type: GitObjectType; size: number; chunks: string[] };23type RecordRow = { id: string; sequence: number; requestHash: string; recordHash: string; packHash: string | null; updates: RefUpdate[] };24type Repository = { key: string; owner: string; identifier: string; alternative: boolean; objects: ObjectRow[]; refs: Record<string, string>; receipts: RecordRow[] };25export type GitBackup = { format: "bind.ws/git-sqlite/1"; repositories: Repository[] };26export type PreparedGit = { repo: Repository; objects: { row: ObjectRow; chunks: Uint8Array[]; links: ReturnType<typeof objectLinks> }[]; raw: number; compressed: number; metadata: number; tip: string | null };2728// exportGitBackup reserves archive memory before reading payloads from SQLite.29export function exportGitBackup(relay: Relay, reserve: (bytes: number, entries?: number) => boolean, limits: Readonly<GitSqliteLimits> = LIMITS, maxEntries = 12000): GitBackup | string {30 const result: GitBackup = { format: "bind.ws/git-sqlite/1", repositories: [] };31 for (const row of gitSql<{ repository: string; compressed_bytes: number; metadata_bytes: number; object_count: number }>(relay, "SELECT repository,compressed_bytes,metadata_bytes,object_count FROM git_sqlite_meta ORDER BY repository LIMIT ?", maxEntries + 1).toArray()) {32 if (!reserve(row.compressed_bytes + row.metadata_bytes, row.object_count + 1)) return "restricted: Git backup exceeds its bounded size or object limit";33 const identity = gitSql<{ owner: string; identifier: string; alternative: number }>(relay, "SELECT owner,identifier,alternative FROM git_sqlite_catalog WHERE repository=?", row.repository).toArray()[0];34 if (!identity) return "error: Git repository identity is missing";35 const repo: Repository = { key: row.repository, owner: identity.owner, identifier: identity.identifier, alternative: !!identity.alternative, objects: [], refs: {}, receipts: [] };36 for (const object of gitSql<{ oid: string; type: GitObjectType; size: number }>(relay, "SELECT oid,type,raw_size AS size FROM git_sqlite_objects WHERE repository=? ORDER BY oid LIMIT ?", row.repository, limits.maxObjects + 1).toArray()) {37 const chunks = gitSql<{ data: ArrayBuffer }>(relay, "SELECT data FROM git_sqlite_object_chunks WHERE repository=? AND oid=? ORDER BY chunk LIMIT ?", row.repository, object.oid, Math.ceil((limits.maxObjectBytes + 65536) / limits.chunkBytes) + 1).toArray().map(chunk => base64(new Uint8Array(chunk.data)));38 if (!reserve(0, chunks.length)) return "restricted: Git backup object limit reached";39 repo.objects.push({ ...object, chunks });40 }41 const refs = gitSql<{ name: string; oid: string }>(relay, "SELECT name,oid FROM git_sqlite_refs WHERE repository=? ORDER BY name LIMIT ?", row.repository, limits.maxRefs + 1).toArray();42 repo.refs = Object.fromEntries(refs.map(ref => [ref.name, ref.oid]));43 repo.receipts = gitSql<{ id: string; sequence: number; requestHash: string; recordHash: string; packHash: string | null; updates: string }>(relay, "SELECT id,sequence,request_hash AS requestHash,record_hash AS recordHash,pack_hash AS packHash,updates FROM git_sqlite_receipts WHERE repository=? ORDER BY sequence LIMIT ?", row.repository, maxEntries + 1).toArray().map(receipt => ({ ...receipt, updates: JSON.parse(receipt.updates) as RefUpdate[] }));44 if (!reserve(0, refs.length + repo.receipts.length)) return "restricted: Git backup object limit reached";45 result.repositories.push(repo);46 }47 return result;48}4950// prepareGitBackup derives links from verified bodies and replays the receipt51// chain so neither forged metadata nor a different namespace survives restore.52export async function prepareGitBackup(input: unknown, maxEntries = 12000, limits: Readonly<GitSqliteLimits> = LIMITS): Promise<PreparedGit[] | string> {53 if (input === undefined) return [];54 try {55 const backup = input as GitBackup;56 if (!backup || backup.format !== "bind.ws/git-sqlite/1" || !Array.isArray(backup.repositories) || backup.repositories.length > maxEntries) throw new Error();57 const result: PreparedGit[] = [], keys = new Set<string>();58 let entries = 0;59 for (const repo of backup.repositories) {60 if (!repo || !hash.test(repo.key) || keys.has(repo.key) || !hash.test(repo.owner) || typeof repo.identifier !== "string" || typeof repo.alternative !== "boolean" || !Array.isArray(repo.objects) || repo.objects.length > limits.maxObjects || !Array.isArray(repo.receipts) || !repo.refs || typeof repo.refs !== "object" || Array.isArray(repo.refs)) throw new Error();61 const prefix = await repositoryStoragePrefix(repositoryAddress(npubEncode(repo.owner), repo.identifier), repo.alternative);62 if (prefix.split("/").filter(Boolean).at(-1) !== repo.key) throw new Error("namespace");63 keys.add(repo.key);64 entries += 1 + repo.objects.length + Object.keys(repo.refs).length + repo.receipts.length;65 if (entries > maxEntries || Object.keys(repo.refs).length > limits.maxRefs) throw new Error("limit");66 const prepared: PreparedGit = { repo, objects: [], raw: 0, compressed: 0, metadata: 0, tip: null };67 const objects = new Map<string, { type: GitObjectType; links: ReturnType<typeof objectLinks> }>();68 let edges = 0;69 for (const row of repo.objects) {70 if (!row || !hex.test(row.oid) || objects.has(row.oid) || !["blob", "tree", "commit", "tag"].includes(row.type) || !Number.isSafeInteger(row.size) || row.size < 0 || row.size > limits.maxObjectBytes || !Array.isArray(row.chunks) || !row.chunks.length || row.chunks.length > Math.ceil((limits.maxObjectBytes + 65536) / limits.chunkBytes)) throw new Error();71 prepared.raw += row.size; entries += row.chunks.length;72 if (prepared.raw > limits.maxRawBytes || entries > maxEntries) throw new Error("limit");73 const chunks = row.chunks.map(value => {74 if (typeof value !== "string" || value.length > Math.ceil(limits.chunkBytes / 3) * 4 || !/^[A-Za-z0-9+/]*={0,2}$/.test(value)) throw new Error();75 const bytes = decoded(value);76 if (!bytes.length || bytes.length > limits.chunkBytes) throw new Error();77 return bytes;78 });79 const compressedSize = chunks.reduce((total, chunk) => total + chunk.length, 0);80 if (compressedSize > limits.maxObjectBytes + 65536) throw new Error("limit");81 prepared.compressed += compressedSize;82 if (prepared.compressed > limits.maxCompressedBytes) throw new Error("limit");83 const compressed = new Uint8Array(compressedSize); let at = 0;84 for (const chunk of chunks) { compressed.set(chunk, at); at += chunk.length; }85 const data = unzlibSync(compressed, { out: new Uint8Array(row.size + 1) });86 if (data.length !== row.size || bytesToHex(sha1.create().update(enc.encode(`${row.type} ${data.length}\0`)).update(data).digest()) !== row.oid) throw new Error("hash");87 const links = objectLinks({ oid: row.oid, type: row.type, data }, limits.maxGraphEdges - edges);88 edges += links.length;89 if (edges > limits.maxGraphEdges) throw new Error("limit");90 objects.set(row.oid, { type: row.type, links });91 prepared.objects.push({ row, chunks, links });92 prepared.metadata += size(repo.key) + 80 + chunks.length * (size(repo.key) + 48) + links.length * (size(repo.key) + 96);93 }94 for (const object of objects.values()) for (const link of object.links) if (objects.get(link.oid)?.type !== link.type) throw new Error("graph");95 const refs = new Map<string, string>(), ids = new Set<string>();96 for (const [index, receipt] of repo.receipts.entries()) {97 if (!receipt || !/^[a-zA-Z0-9_.:-]{1,128}$/.test(receipt.id) || ids.has(receipt.id) || receipt.sequence !== index + 1 || !hash.test(receipt.requestHash) || !hash.test(receipt.recordHash) || (receipt.packHash !== null && !hash.test(receipt.packHash)) || !Array.isArray(receipt.updates) || !receipt.updates.length || receipt.updates.length > limits.maxRefs) throw new Error("receipt");98 ids.add(receipt.id);99 const names = new Set<string>();100 let previous = "";101 for (const update of receipt.updates) {102 validateRefName(update.name);103 if (names.has(update.name) || update.name < previous || (update.old !== null && (!hex.test(update.old) || !objects.has(update.old))) || (update.new !== null && (!hex.test(update.new) || !objects.has(update.new))) || (refs.get(update.name) ?? null) !== update.old) throw new Error("receipt refs");104 previous = update.name; names.add(update.name);105 if (update.new === null) refs.delete(update.name); else refs.set(update.name, update.new);106 }107 if (digest({ id: receipt.id, updates: receipt.updates, pack: receipt.packHash }) !== receipt.requestHash || digest({ sequence: receipt.sequence, parent: prepared.tip, id: receipt.id, digest: receipt.requestHash }) !== receipt.recordHash) throw new Error("receipt hash");108 prepared.tip = receipt.recordHash;109 prepared.metadata += size(repo.key) + size(JSON.stringify(receipt.updates)) + size(receipt.id) + 160;110 }111 if (!repo.receipts.length || refs.size !== Object.keys(repo.refs).length) throw new Error("refs");112 for (const [name, oid] of Object.entries(repo.refs)) {113 validateRefName(name);114 if (refs.get(name) !== oid || !objects.has(oid) || (name.startsWith("refs/heads/") && objects.get(oid)?.type !== "commit")) throw new Error("refs");115 const parts = name.split("/");116 for (let index = 2; index < parts.length; index++) if (refs.has(parts.slice(0, index).join("/"))) throw new Error("refs");117 prepared.metadata += size(repo.key) + size(name) + 40;118 }119 if (prepared.metadata > limits.maxMetadataBytes) throw new Error("limit");120 result.push(prepared);121 }122 return result;123 } catch { return "invalid: SQLite Git backup identity, objects, receipts or limits failed validation"; }124}125126// restoreGitBackup runs inside the caller's transaction after preparation.127export function restoreGitBackup(relay: Relay, prepared: PreparedGit[]) {128 if (gitSql(relay, "SELECT 1 FROM git_sqlite_meta UNION ALL SELECT 1 FROM git_sqlite_objects UNION ALL SELECT 1 FROM git_sqlite_catalog LIMIT 1").toArray().length) throw new Error("error: Git restore requires an empty target");129 for (const { repo, objects, raw, compressed, metadata, tip } of prepared) {130 gitSql(relay, "INSERT INTO git_sqlite_catalog(repository,owner,identifier,alternative) VALUES(?,?,?,?)", repo.key, repo.owner, repo.identifier, repo.alternative ? 1 : 0);131 for (const { row, chunks, links } of objects) {132 gitSql(relay, "INSERT INTO git_sqlite_objects(repository,oid,type,raw_size,compressed_size,chunks) VALUES(?,?,?,?,?,?)", repo.key, row.oid, row.type, row.size, chunks.reduce((total, chunk) => total + chunk.length, 0), chunks.length);133 for (const [index, chunk] of chunks.entries()) gitSql(relay, "INSERT INTO git_sqlite_object_chunks(repository,oid,chunk,data) VALUES(?,?,?,?)", repo.key, row.oid, index, chunk.slice().buffer);134 for (const [index, link] of links.entries()) gitSql(relay, "INSERT INTO git_sqlite_object_links(repository,oid,position,link_oid,link_type) VALUES(?,?,?,?,?)", repo.key, row.oid, index, link.oid, link.type);135 }136 for (const [name, oid] of Object.entries(repo.refs)) gitSql(relay, "INSERT INTO git_sqlite_refs(repository,name,oid) VALUES(?,?,?)", repo.key, name, oid);137 for (const receipt of repo.receipts) gitSql(relay, "INSERT INTO git_sqlite_receipts(repository,id,sequence,request_hash,record_hash,updates,pack_hash) VALUES(?,?,?,?,?,?,?)", repo.key, receipt.id, receipt.sequence, receipt.requestHash, receipt.recordHash, JSON.stringify(receipt.updates), receipt.packHash);138 gitSql(relay, "INSERT INTO git_sqlite_meta(repository,sequence,tip,compressed_bytes,raw_bytes,object_count,metadata_bytes) VALUES(?,?,?,?,?,?,?)", repo.key, repo.receipts.length, tip, compressed, raw, objects.length, metadata);139 }140}