* blossom: HEAD /upload and PUT /mirror (BUD-06, BUD-04) HEAD /upload answers from the headers and the policy whether an upload would be accepted, with the reason in X-Reason and no body. PUT /mirror takes a URL and copies the blob in: relays on this host are reached through their object, anything else over https, the size cap applies before and while reading, the hash must match the token's x tag, and the row and R2 object are written the way an upload writes them. * docs, landing: mention the upload check and mirroring
- commit
- b4c4b036b3c0f660013aa075de6e17b8f8c7fb83
- parent
- 372d1ac25a102f04dc3620bf0a2d0df4397c4c87
- author
- FelineStateMachine <dami.etoile@gmail.com>
| M | docs/ | 2 | +- |
| M | docs/ | 2 | +- |
| M | src/ | 110 | ++++++++++++++++++++++- |
| M | src/ | 4 | +- |
| M | src/ | 2 | +- |
| A | test/ | 147 | ++++++++++++++++++++++++++++++ |
@@ -58,7 +58,7 @@ Members can have a name at your relay, such as `alice@<name>.bind.ws`. Set it on
5858 5959 ## Media6060 61-Your relay stores images and files too. Clients that support Blossom upload to `https://<name>.bind.ws` and get a link by hash. Set the maximum upload size on the Rules tab.61+Your relay stores images and files too. Clients that support Blossom upload to `https://<name>.bind.ws` and get a link by hash. A client can ask first whether an upload would be accepted, and can have the relay copy a file from another server by URL instead of uploading it again. Set the maximum upload size on the Rules tab.6262 6363 ## Scripts6464 @@ -65,4 +65,4 @@ On claim, the object generates a keypair, advertised as `self` in NIP-11. It sig
6565 6666 ## Protocol surface6767 68-NIP-01, 09, 11, 13, 17/59 private kinds, 40, 42, 45 with HLL, 50, 56, 62, 67, 70, 77, 86, 98, plus NIP-05, NIP-43, NIP-57 and Blossom BUD-01/02. `test/conformance` is a black-box suite that runs against any relay URL.68+NIP-01, 09, 11, 13, 17/59 private kinds, 40, 42, 45 with HLL, 50, 56, 62, 67, 70, 77, 86, 98, plus NIP-05, NIP-43, NIP-57 and Blossom BUD-01/02/04/06. `test/conformance` is a black-box suite that runs against any relay URL.@@ -4,6 +4,8 @@
44 // remove them. Authorization is a kind 24242 event in the Authorization header.55 //66 // PUT /upload t=upload, x=<sha256 of body> -> descriptor7+// HEAD /upload t=upload; X-SHA-256, X-Content-Type, X-Content-Length (BUD-06)8+// PUT /mirror t=upload, x=<sha256>; body {url} -> descriptor (BUD-04)79 // GET /<sha256>[.ext] public810 // HEAD /<sha256>[.ext] public911 // DELETE /<sha256> t=delete, x=<sha256>; uploader or owner@@ -11,6 +13,7 @@
1113 import { sha256 } from "@noble/hashes/sha2.js";1214 import { tagValues, validate, type Event } from "./event.ts";1315 import { bytesToHex } from "./negentropy.ts";16+import { localName } from "./pull.ts";1417 import type { Relay } from "./relay.ts";1518 1619 export type Blob = {@@ -26,7 +29,9 @@ const EXT: Record<string, string> = {
2629 "video/mp4": "mp4", "video/webm": "webm", "video/quicktime": "mov", "audio/mpeg": "mp3", "audio/ogg": "ogg", "audio/wav": "wav", "audio/mp4": "m4a",2730 "application/pdf": "pdf", "text/plain": "txt", "application/json": "json", "text/markdown": "md",2831 };32+const TYPE_BY_EXT: Record<string, string> = Object.fromEntries(Object.entries(EXT).map(([type, ext]) => [ext, type]));2933 const SHA_RE = /^\/([0-9a-f]{64})(?:\.[a-z0-9]{1,8})?$/;34+const HEX64 = /^[0-9a-f]{64}$/;3035 3136 // verifyBlossom checks a kind 24242 event for the given action.3237 export function verifyBlossom(header: string, action: "upload" | "delete", now: number): Event | string {@@ -61,6 +66,51 @@ export function blobBytes(sql: SqlStorage): number {
6166 return sql.exec<{ n: number | null }>(`SELECT sum(size) AS n FROM blobs`).one().n ?? 0;6267 }6368 69+// fetchOrigin gets a blob for mirroring. Our own relays are reached through70+// their object, which also works in wrangler dev and tests; anything else is71+// fetched over https. Returns a Response or a reason.72+async function fetchOrigin(relay: Relay, raw: string): Promise<Response | string> {73+ let u: URL;74+ try {75+ u = new URL(raw);76+ } catch {77+ return "invalid: url is not a URL";78+ }79+ const local = localName(u, relay.domain);80+ try {81+ if (local) return await relay.relays.getByName(local).fetch(u.href, { headers: { "x-relay-name": local } });82+ if (u.protocol !== "https:") return "invalid: only https urls can be mirrored";83+ return await fetch(u.href, { redirect: "follow" });84+ } catch (err) {85+ return "error: could not fetch the origin: " + (err instanceof Error ? err.message : String(err));86+ }87+}88+89+// readCapped collects a body, giving up as soon as it passes max bytes.90+async function readCapped(body: ReadableStream<Uint8Array> | null, max: number): Promise<Uint8Array | "too big"> {91+ if (!body) return new Uint8Array();92+ const parts: Uint8Array[] = [];93+ let total = 0;94+ const reader = body.getReader();95+ for (;;) {96+ const { done, value } = await reader.read();97+ if (done) break;98+ total += value.length;99+ if (total > max) {100+ await reader.cancel();101+ return "too big";102+ }103+ parts.push(value);104+ }105+ const out = new Uint8Array(total);106+ let at = 0;107+ for (const p of parts) {108+ out.set(p, at);109+ at += p.length;110+ }111+ return out;112+}113+64114 export async function blossom(relay: Relay, req: Request): Promise<Response> {65115 const url = new URL(req.url);66116 const host = url.host;@@ -70,13 +120,71 @@ export async function blossom(relay: Relay, req: Request): Promise<Response> {
70120 new Response(JSON.stringify(b), { status, headers: { "content-type": "application/json", "access-control-allow-origin": "*", ...extra } });71121 const reason = (msg: string, status: number) => json({ error: msg }, status, { "x-reason": msg });72122 const key = (sha: string) => `${relay.slug}/${sha}`;123+ const maxBytes = () => relay.settings.policy.maxBlobMB * 1024 * 1024;124+ const tooBig = () => `invalid: blob exceeds ${relay.settings.policy.maxBlobMB} MB`;125+126+ // BUD-06: would this upload be accepted? No body either way.127+ if (req.method === "HEAD" && url.pathname === "/upload") {128+ const head = (status: number, msg = "") => new Response(null, { status, headers: { "access-control-allow-origin": "*", ...(msg ? { "x-reason": msg } : {}) } });129+ const sha = (req.headers.get("x-sha-256") ?? "").trim().toLowerCase();130+ if (!HEX64.test(sha)) return head(400, "invalid: X-SHA-256 must be a lowercase hex sha256");131+ const lengthHeader = req.headers.get("x-content-length");132+ if (lengthHeader === null || lengthHeader.trim() === "") return head(411, "invalid: X-Content-Length is required");133+ const length = Number(lengthHeader);134+ if (!Number.isInteger(length) || length < 0) return head(400, "invalid: X-Content-Length must be a whole number of bytes");135+ const auth = verifyBlossom(req.headers.get("authorization") ?? "", "upload", now);136+ if (typeof auth === "string") return head(401, auth);137+ const gate = relay.mayUpload(auth.pubkey, host);138+ if (gate) return head(403, gate);139+ if (length === 0) return head(400, "invalid: empty body");140+ if (length > maxBytes()) return head(413, tooBig());141+ const claimed = tagValues(auth, "x");142+ if (claimed.length && !claimed.includes(sha)) return head(400, "invalid: token x tag does not match the blob");143+ return head(200);144+ }145+146+ // BUD-04: copy a blob from a URL instead of receiving the bytes.147+ if (req.method === "PUT" && url.pathname === "/mirror") {148+ const auth = verifyBlossom(req.headers.get("authorization") ?? "", "upload", now);149+ if (typeof auth === "string") return reason(auth, 401);150+ const gate = relay.mayUpload(auth.pubkey, host);151+ if (gate) return reason(gate, 403);152+ let from = "";153+ try {154+ from = String((JSON.parse(await req.text()) as { url?: unknown }).url ?? "");155+ } catch {156+ return reason("invalid: body is not JSON", 400);157+ }158+ if (!from) return reason("invalid: body needs a url", 400);159+ const origin = await fetchOrigin(relay, from);160+ if (typeof origin === "string") return reason(origin, origin.startsWith("invalid:") ? 400 : 502);161+ if (!origin.ok) return reason(`error: the origin answered ${origin.status}`, 502);162+ const declared = Number(origin.headers.get("content-length"));163+ if (declared > maxBytes()) return reason(tooBig(), 413);164+ const body = await readCapped(origin.body, maxBytes());165+ if (body === "too big") return reason(tooBig(), 413);166+ if (body.length === 0) return reason("error: the origin sent an empty body", 502);167+ const sha = bytesToHex(sha256(body));168+ const claimed = tagValues(auth, "x");169+ if (claimed.length && !claimed.includes(sha)) return reason("invalid: the mirrored blob does not match the token x tag", 409);170+ const originType = (origin.headers.get("content-type") ?? "").split(";")[0].trim().toLowerCase();171+ const ext = new URL(from).pathname.split(".").pop()?.toLowerCase() ?? "";172+ const type = originType && originType !== "application/octet-stream" ? originType : (TYPE_BY_EXT[ext] ?? originType) || "application/octet-stream";173+ const existing = sql.exec<Blob>(`SELECT * FROM blobs WHERE sha256=?`, sha).toArray()[0];174+ if (!existing) {175+ await relay.media.put(key(sha), body, { httpMetadata: { contentType: type } });176+ sql.exec(`INSERT INTO blobs(sha256,size,type,uploader,uploaded) VALUES(?,?,?,?,?)`, sha, body.length, type, auth.pubkey, now);177+ }178+ relay.meterBytes(body.length, 0);179+ return json(descriptor(host, existing ?? { sha256: sha, size: body.length, type, uploader: auth.pubkey, uploaded: now }), existing ? 200 : 201);180+ }73181 74182 if (req.method === "PUT" && url.pathname === "/upload") {75183 const auth = verifyBlossom(req.headers.get("authorization") ?? "", "upload", now);76184 if (typeof auth === "string") return reason(auth, 401);77185 const gate = relay.mayUpload(auth.pubkey, host);78186 if (gate) return reason(gate, 403);79- const max = relay.settings.policy.maxBlobMB * 1024 * 1024;187+ const max = maxBytes();80188 const declared = Number(req.headers.get("content-length"));81189 if (declared > max) return reason(`invalid: blob exceeds ${relay.settings.policy.maxBlobMB} MB`, 413);82190 const body = new Uint8Array(await req.arrayBuffer());@@ -91,7 +91,7 @@ const PROTOCOL: [string, string][] = [
9191 ["05", "Members with a name are you@name.bind.ws."],9292 ["43", "The relay signs and publishes its member list."],9393 ["57", "Zaps to the relay add fuel. The receipt is stored on the relay."],94- ["Blossom", "Upload and fetch files by hash on the same host."],94+ ["Blossom", "Upload and fetch files by hash on the same host. Check an upload before sending it, or mirror a file from a URL."],9595 ];9696 9797 function table(heads: string[], rows: string[][], right: number[] = []): string {@@ -113,7 +113,7 @@ export function landing(req: Request, env: Env): Response {
113113 ["POST /", "NIP-86", "NIP-98, owner", "claim and the management methods"],114114 ["POST /lease", "lease", "optional NIP-98", `A temporary relay for ${leaseDays(env)} days. A signature reserves the claim for that key.`],115115 ["POST /events /query /count", "HTTP bridge", "NIP-98", "The signer has the same access as an authenticated socket."],116- ["PUT /upload, GET /<sha256>", "Blossom", "kind 24242", "Stored in R2. Counted as files."],116+ ["PUT /upload, GET /<sha256>", "Blossom", "kind 24242", "Stored in R2. Counted as files. HEAD /upload checks first; PUT /mirror copies from a URL."],117117 ["GET /.well-known/nostr.json", "NIP-05", "none", "Members that have a name."],118118 ["GET /fuel, POST /fuel/invoice", "NIP-57", "none", "Gauges are public. The invoice comes from a signed zap request."],119119 ]);@@ -493,7 +493,7 @@ export class Relay extends DurableObject<Env> {
493493 return Response.json({ terms: this.settings.policy.joinTerms }, { headers: { "access-control-allow-origin": "*" } });494494 }495495 if (url.pathname === "/api/invites/claim" && req.method === "POST") return this.claimInviteRequest(req);496- if (url.pathname === "/upload" || url.pathname.startsWith("/list/") || isBlobPath(url.pathname)) return blossom(this, req);496+ if (url.pathname === "/upload" || url.pathname === "/mirror" || url.pathname.startsWith("/list/") || isBlobPath(url.pathname)) return blossom(this, req);497497 if (url.pathname === "/fuel" && req.method === "GET") {498498 return Response.json({ ...this.fuelStatus(), credits: this.fuel.recentCredits() }, { headers: { "access-control-allow-origin": "*" } });499499 }@@ -0,0 +1,147 @@
1+// Blossom beyond upload and fetch: BUD-06 asks whether an upload would be2+// accepted, BUD-04 copies a blob from a URL, ours or anyone's.3+import { SELF } from "cloudflare:test";4+import { describe, it, expect } from "vitest";5+import { finalizeEvent, generateSecretKey, getPublicKey } from "nostr-tools/pure";6+import { getToken } from "nostr-tools/nip98";7+import { sha256 } from "@noble/hashes/sha2.js";8+import { bytesToHex } from "../src/negentropy.ts";9+10+const now = () => Math.floor(Date.now() / 1000);11+const ev = (sk: Uint8Array, kind: number, content: string, tags: string[][] = []) => finalizeEvent({ kind, content, tags, created_at: now() }, sk);12+13+async function rpc(host: string, sk: Uint8Array, method: string, ...params: unknown[]) {14+ const url = `http://${host}/`;15+ const payload = { method, params };16+ const token = await getToken(url, "POST", (e) => finalizeEvent(e, sk), true, payload);17+ const resp = await SELF.fetch(url, { method: "POST", headers: { "content-type": "application/nostr+json+rpc", authorization: token }, body: JSON.stringify(payload) });18+ return { status: resp.status, ...(await resp.json<any>()) };19+}20+21+const token = (sk: Uint8Array, sha?: string) =>22+ "Nostr " + btoa(JSON.stringify(ev(sk, 24242, "upload", [["t", "upload"], ...(sha ? [["x", sha]] : []), ["expiration", String(now() + 300)]])));23+24+async function upload(host: string, sk: Uint8Array, text: string) {25+ const body = new TextEncoder().encode(text);26+ const sha = bytesToHex(sha256(body));27+ const resp = await SELF.fetch(`http://${host}/upload`, { method: "PUT", headers: { authorization: token(sk, sha), "content-type": "text/plain" }, body });28+ return { status: resp.status, sha, body: await resp.json<any>() };29+}30+31+async function head(host: string, headers: Record<string, string>) {32+ const resp = await SELF.fetch(`http://${host}/upload`, { method: "HEAD", headers });33+ return { status: resp.status, reason: resp.headers.get("x-reason") ?? "", text: await resp.text() };34+}35+36+async function mirror(host: string, sk: Uint8Array, url: unknown, sha?: string, raw?: string) {37+ const resp = await SELF.fetch(`http://${host}/mirror`, { method: "PUT", headers: { authorization: token(sk, sha), "content-type": "application/json" }, body: raw ?? JSON.stringify({ url }) });38+ const text = await resp.text();39+ return { status: resp.status, body: text ? JSON.parse(text) : null, reason: resp.headers.get("x-reason") ?? "" };40+}41+42+describe("BUD-06 upload requirements", () => {43+ it("answers HEAD /upload from the headers and the policy, with no body", async () => {44+ const host = "buds-head.bind.ws";45+ const owner = generateSecretKey();46+ const stranger = generateSecretKey();47+ await rpc(host, owner, "claim");48+ const bytes = new TextEncoder().encode("a small file");49+ const sha = bytesToHex(sha256(bytes));50+ const ok = { "x-sha-256": sha, "x-content-type": "text/plain", "x-content-length": String(bytes.length), authorization: token(owner, sha) };51+52+ let r = await head(host, ok);53+ expect([r.status, r.reason, r.text]).toEqual([200, "", ""]);54+ r = await head(host, { ...ok, "x-sha-256": "nope" });55+ expect([r.status, r.reason.split(":")[0]]).toEqual([400, "invalid"]);56+ const { "x-content-length": _, ...noLength } = ok;57+ r = await head(host, noLength);58+ expect(r.status).toBe(411);59+ r = await head(host, { ...ok, "x-content-length": "twelve" });60+ expect(r.status).toBe(400);61+ const { authorization: __, ...noAuth } = ok;62+ r = await head(host, noAuth);63+ expect(r.status).toBe(401);64+ expect(r.reason).toMatch(/^auth-required/);65+ r = await head(host, { ...ok, "x-content-length": String(26 * 1024 * 1024) });66+ expect(r.status).toBe(413);67+ expect(r.reason).toMatch(/25 MB/);68+ r = await head(host, { ...ok, authorization: token(owner, "ab".repeat(32)) });69+ expect(r.status).toBe(400);70+71+ await rpc(host, owner, "setpolicy", { writes: "owner" });72+ r = await head(host, { ...ok, authorization: token(stranger, sha) });73+ expect(r.status).toBe(403);74+ expect(r.reason).toMatch(/^restricted/);75+76+ // Already stored is still a 200: the PUT would answer with the descriptor.77+ expect((await upload(host, owner, "a small file")).status).toBe(200);78+ expect((await head(host, ok)).status).toBe(200);79+ });80+});81+82+describe("BUD-04 mirror", () => {83+ it("copies a blob from another relay on this host, once", async () => {84+ const a = "buds-src.bind.ws";85+ const b = "buds-dst.bind.ws";86+ const alice = generateSecretKey();87+ const bob = generateSecretKey();88+ await rpc(a, alice, "claim");89+ await rpc(b, bob, "claim");90+ const up = await upload(a, alice, "the same bytes, elsewhere");91+ expect(up.status).toBe(200);92+ expect(up.body.url).toBe(`https://${a}/${up.sha}.txt`);93+94+ let r = await mirror(b, bob, up.body.url, up.sha);95+ expect(r.status, r.reason).toBe(201);96+ expect(r.body).toEqual({ url: `https://${b}/${up.sha}.txt`, sha256: up.sha, size: up.body.size, type: "text/plain", uploaded: expect.any(Number) });97+ const got = await SELF.fetch(`http://${b}/${up.sha}`);98+ expect(got.status).toBe(200);99+ expect(await got.text()).toBe("the same bytes, elsewhere");100+ const list = (await rpc(b, bob, "listblobs")).result;101+ expect(list.map((x: any) => [x.sha256, x.uploader])).toEqual([[up.sha, getPublicKey(bob)]]);102+103+ r = await mirror(b, bob, up.body.url, up.sha);104+ expect(r.status).toBe(200);105+ expect(r.body.sha256).toBe(up.sha);106+ expect((await rpc(b, bob, "listblobs")).result.length).toBe(1);107+ });108+109+ it("refuses what it should: wrong hash, bad body, bad url, no auth, policy", async () => {110+ const a = "buds-src2.bind.ws";111+ const b = "buds-dst2.bind.ws";112+ const alice = generateSecretKey();113+ const bob = generateSecretKey();114+ const stranger = generateSecretKey();115+ await rpc(a, alice, "claim");116+ await rpc(b, bob, "claim");117+ const up = await upload(a, alice, "bytes to mirror");118+119+ let r = await mirror(b, bob, up.body.url, "ab".repeat(32));120+ expect([r.status, r.reason.split(":")[0]]).toEqual([409, "invalid"]);121+ expect((await rpc(b, bob, "listblobs")).result).toEqual([]);122+ r = await mirror(b, bob, undefined, undefined, "not json");123+ expect(r.status).toBe(400);124+ r = await mirror(b, bob, "");125+ expect(r.status).toBe(400);126+ r = await mirror(b, bob, "ftp:/nope");127+ expect(r.status).toBe(400);128+ r = await mirror(b, bob, "http://example.com/" + up.sha);129+ expect([r.status, r.reason]).toEqual([400, "invalid: only https urls can be mirrored"]);130+ r = await mirror(b, bob, `https://${a}/${"cd".repeat(32)}`);131+ expect([r.status, r.reason]).toEqual([502, "error: the origin answered 404"]);132+ const noAuth = await SELF.fetch(`http://${b}/mirror`, { method: "PUT", body: JSON.stringify({ url: up.body.url }) });133+ expect(noAuth.status).toBe(401);134+135+ await rpc(b, bob, "setpolicy", { writes: "owner" });136+ r = await mirror(b, stranger, up.body.url, up.sha);137+ expect(r.status).toBe(403);138+ await rpc(b, bob, "setpolicy", { writes: "open", maxBlobMB: 1 });139+ // The origin declares its size, so an oversized blob is refused before any bytes move.140+ await rpc(a, alice, "setpolicy", { maxBlobMB: 2 });141+ const big = await SELF.fetch(`http://${a}/upload`, { method: "PUT", headers: { authorization: token(alice), "content-type": "application/octet-stream" }, body: new Uint8Array(1024 * 1024 + 1) });142+ expect(big.status).toBe(200);143+ r = await mirror(b, bob, (await big.json<any>()).url);144+ expect(r.status).toBe(413);145+ });146+147+});