- commit
- 67cc056da1a84168e4eafe1eb9fd11248918fe9f
- parent
- cf6ae187df2a33a81ac7471621dca06aa0fcd4b5
- author
- Dami <dami.etoile@gmail.com>
| M | docs/ | 4 | + |
| M | internal/ | 40 | ++++++++- |
| M | internal/ | 144 | +++++++++++++++++++++++++++++- |
| A | internal/ | bin | |
| A | internal/ | bin | |
| M | internal/ | 25 | +++++ |
| M | scripts/ | 14 | +++ |
| M | scripts/ | 45 | +++++++++ |
@@ -6,6 +6,10 @@ The relay installs as an app from the browser menu on Android, iOS and desktop.
66 77 Share a photo, file or link from another app and choose the relay. Shared files wait in the Upload panel on the Files page until you press **Upload**, so nothing is stored without your signature. A shared link opens the **Import from URL** control with the address filled in.88 9+## Large uploads10+11+On browsers that offer it, a file of 8 MiB or more uploads through the browser's background transfer, so the upload continues after the app closes. Every request is signed before the transfer starts, and the browser shows its own progress notice. Open the Files page later to see the result; for an encrypted file the share link appears there. On other browsers the upload runs while the page stays open.12+913 ## Nostr links1014 1115 The app registers for `web+nostr:` links. A link to an `npub`, `nprofile`, `note`, `nevent` or `naddr` opens the matching page: the author's events, the event page or the article. Plain `nostr:` links open the same way through `/open?target=`.@@ -31,7 +31,10 @@
3131 return descriptor;3232 };3333 34- const uploadTask = async (source, options = {}, controller) => {34+ // resolveTarget checks the source and options and returns the blob URL35+ // the upload addresses. Both the live uploader and the background plan36+ // use it so they agree on every detail.37+ const resolveTarget = async (source, options) => {3538 const bytes = await asBytes(source);3639 const hash = options.hash || hex(await sha256(bytes));3740 if (!/^[0-9a-f]{64}$/.test(hash)) throw new Error("expected a lowercase SHA-256 hash");@@ -55,7 +58,38 @@
5558 const target = new RegExp("/" + hash + "(?:\\.[a-z0-9]{1,8})?/?$").test(path)5659 ? base5760 : base.replace(/\/$/, "") + "/" + hash + (extension ? "." + extension : "");58- const url = target;61+ return {bytes, hash, type, url: target};62+ };63+64+ // plan lists the signed chunk requests for one upload without sending65+ // them, so a browser can hand the transfer to Background Fetch and finish66+ // it after the page closes. Each chunk carries its own authorization.67+ const plan = async (source, options = {}) => {68+ const {bytes, hash, type, url} = await resolveTarget(source, options);69+ const chunkSize =70+ Number.isSafeInteger(options.chunkSize) && options.chunkSize > 0 ? options.chunkSize : 5 * 1024 * 1024;71+ const requests = [];72+ for (let offset = 0; offset < bytes.byteLength || (bytes.byteLength === 0 && offset === 0); offset += chunkSize) {73+ const chunk = bytes.slice(offset, Math.min(offset + chunkSize, bytes.byteLength));74+ const headers = {75+ "upload-type": type,76+ "upload-length": String(bytes.byteLength),77+ "upload-offset": String(offset),78+ "content-type": "application/octet-stream"79+ };80+ if (options.authorize) {81+ const value = await options.authorize(url, "PATCH", chunk, {hash, type});82+ if (typeof value === "string") headers.authorization = value;83+ else if (value && typeof value === "object") Object.assign(headers, value);84+ }85+ requests.push({url, method: "PATCH", headers, body: chunk});86+ if (bytes.byteLength === 0) break;87+ }88+ return {hash, url, size: bytes.byteLength, requests};89+ };90+91+ const uploadTask = async (source, options = {}, controller) => {92+ const {bytes, hash, type, url} = await resolveTarget(source, options);5993 const fetcher = options.fetch || globalThis.fetch;6094 if (typeof fetcher !== "function") throw new Error("fetch is unavailable");6195 const chunkSize =@@ -248,7 +282,7 @@
248282 return task;249283 };250284 251- const api = Object.freeze({upload});285+ const api = Object.freeze({upload, plan});252286 globalThis.tiny = globalThis.tiny || {};253287 globalThis.tiny.blossom = {...globalThis.tiny.blossom, upload: api};254288 })();@@ -145,6 +145,11 @@
145145 // file gets a fresh random key, a folder or a large file becomes encrypted146146 // manifests, and the share link carries the key only in its fragment.147147 const LARGE_FILE_BYTES = 64 * 1024 * 1024;148+ // Transfers at least this large go through Background Fetch where the149+ // browser offers it, so closing the app does not stop them.150+ const BACKGROUND_BYTES = 8 * 1024 * 1024;151+ const backgroundAvailable = () =>152+ "BackgroundFetchManager" in globalThis && Boolean(navigator.serviceWorker?.controller);148153 const {b64url} = globalThis.tiny.util;149154 class FileUpload extends HTMLElement {150155 connectedCallback() {@@ -176,6 +181,116 @@
176181 this.querySelector("[data-cancel]").addEventListener("click", () => this.controller?.abort());177182 this.querySelector("[data-retry]").addEventListener("click", () => this.run().catch(() => {}));178183 this.intake().catch(error => this.say(error.message, true));184+ this.resumeBackground().catch(() => {});185+ }186+ // storeBackground hands one blob to the browser's Background Fetch with187+ // every request signed up front, then waits for the parked descriptor.188+ async storeBackground(bytes, {hash, type, name, fragment}) {189+ const registration = await navigator.serviceWorker.ready;190+ const base = new URL(tiny.localPath("/"), location.href).href;191+ const authorize = (target, method, body) => tiny.authorization(target, method, body);192+ let requests;193+ const probe = await fetch(base, {method: "OPTIONS"}).catch(() => null);194+ if (probe && /(?:^|,|\s)PATCH(?:,|\s|$)/i.test(probe.headers.get("allow") || "")) {195+ const planned = await upload().plan(bytes, {url: base, hash, type, authorize});196+ requests = planned.requests.map(197+ item => new Request(item.url, {method: item.method, headers: item.headers, body: item.body})198+ );199+ } else {200+ const target = base.replace(/\/$/, "") + "/upload";201+ requests = [202+ new Request(target, {203+ method: "PUT",204+ headers: {"content-type": type, authorization: await authorize(target, "PUT", bytes)},205+ body: bytes206+ })207+ ];208+ }209+ const id = "tiny-upload-" + crypto.randomUUID();210+ localStorage.setItem("tiny.bg." + id, JSON.stringify({hash, fragment: fragment || "", name, at: Date.now()}));211+ const task = await registration.backgroundFetch.fetch(id, requests, {212+ title: "Uploading " + name,213+ icons: [214+ {src: new URL(tiny.localPath("/icon-192.png"), location.href).href, sizes: "192x192", type: "image/png"}215+ ],216+ uploadTotal: bytes.byteLength217+ });218+ this.say("Uploading in the background. You can close this page.");219+ return this.awaitBackground(task, hash, id);220+ }221+ awaitBackground(task, hash, id) {222+ return new Promise((resolve, reject) => {223+ const settle = async () => {224+ if (task.result === "") {225+ if (task.uploadTotal)226+ this.say("Uploading in the background " + Math.round((task.uploaded / task.uploadTotal) * 100) + "%…");227+ return false;228+ }229+ task.removeEventListener?.("progress", settle);230+ try {231+ resolve(232+ await this.backgroundDescriptor(id, hash, task.result === "success" ? "" : task.failureReason || "failed")233+ );234+ } catch (error) {235+ reject(error);236+ }237+ return true;238+ };239+ task.addEventListener("progress", settle);240+ settle();241+ });242+ }243+ // backgroundDescriptor reads the parked result, verifies the hash and244+ // forgets the transfer.245+ async backgroundDescriptor(id, hash, failure) {246+ const cache = await caches.open("tiny-uploads");247+ const key = new URL(tiny.localPath("/uploads/" + id), location.href).href;248+ const response = await cache.match(key);249+ await cache.delete(key);250+ localStorage.removeItem("tiny.bg." + id);251+ if (failure) throw Error("Background upload " + failure + ".");252+ const descriptor = response ? await response.json().catch(() => null) : null;253+ if (!descriptor || descriptor.sha256 !== hash) throw Error("Relay returned an unexpected hash.");254+ return descriptor;255+ }256+ // resumeBackground picks up transfers started on an earlier visit.257+ async resumeBackground() {258+ if (!backgroundAvailable()) return;259+ const pending = Object.keys(localStorage).filter(key => key.startsWith("tiny.bg."));260+ if (!pending.length) return;261+ const registration = await navigator.serviceWorker.ready;262+ for (const key of pending) {263+ const id = key.slice("tiny.bg.".length);264+ let info;265+ try {266+ info = JSON.parse(localStorage.getItem(key));267+ } catch {268+ localStorage.removeItem(key);269+ continue;270+ }271+ const task = await registration.backgroundFetch.get(id).catch(() => null);272+ const finish = descriptor => {273+ if (!info.fragment) {274+ this.say("Background upload of " + info.name + " finished.");275+ return tiny.navigate?.(location.href);276+ }277+ const link = new URL(tiny.localPath("/file"), location.href);278+ link.search = "?hash=" + descriptor.sha256;279+ link.hash = info.fragment;280+ return this.showLink(link.href);281+ };282+ try {283+ if (task && task.result === "") await this.awaitBackground(task, info.hash, id).then(finish);284+ else285+ await this.backgroundDescriptor(286+ id,287+ info.hash,288+ task && task.result !== "success" ? task.failureReason || "failed" : ""289+ ).then(finish);290+ } catch (error) {291+ this.say(error.message, true);292+ }293+ }179294 }180295 // intake collects files parked by the service worker for a share from181296 // another app. They wait here until the person presses Upload.@@ -190,9 +305,14 @@
190305 const response = await cache.match("/share/" + id + "/" + index);191306 if (!response) continue;192307 const blob = await response.blob();193- files.push(new File([blob], decodeURIComponent(response.headers.get("x-name") || "shared"), {type: response.headers.get("content-type") || blob.type}));308+ files.push(309+ new File([blob], decodeURIComponent(response.headers.get("x-name") || "shared"), {310+ type: response.headers.get("content-type") || blob.type311+ })312+ );194313 }195- for (const key of await cache.keys()) if (new URL(key.url).pathname.startsWith("/share/" + id + "/")) await cache.delete(key);314+ for (const key of await cache.keys())315+ if (new URL(key.url).pathname.startsWith("/share/" + id + "/")) await cache.delete(key);196316 history.replaceState?.(null, "", location.pathname + location.search);197317 if (files.length) {198318 this.choose(files);@@ -311,10 +431,13 @@
311431 for (const [index, file] of files.entries()) {312432 checkAbort(this.controller.signal);313433 this.say("Uploading " + file.name + " (" + (index + 1) + " of " + files.length + ")…");314- await tiny.signedFetch("/upload", "PUT", await file.arrayBuffer(), {315- contentType: file.type || "application/octet-stream",316- signal: this.controller.signal317- });434+ const bytes = new Uint8Array(await file.arrayBuffer());435+ const type = file.type || "application/octet-stream";436+ if (files.length === 1 && bytes.byteLength >= BACKGROUND_BYTES && backgroundAvailable()) {437+ await this.storeBackground(bytes, {hash: await tiny.sha256hex(bytes), type, name: file.name});438+ continue;439+ }440+ await tiny.signedFetch("/upload", "PUT", bytes, {contentType: type, signal: this.controller.signal});318441 }319442 }320443 // storeSealed encrypts one file with a fresh AES-GCM key. The ciphertext@@ -345,7 +468,14 @@
345468 const {ciphertext, fragment, state} = this.pending;346469 const hash = await tiny.sha256hex(ciphertext);347470 let descriptor;348- if (upload()?.upload) {471+ if (ciphertext.byteLength >= BACKGROUND_BYTES && backgroundAvailable() && upload()?.plan) {472+ descriptor = await this.storeBackground(ciphertext, {473+ hash,474+ type: "application/octet-stream",475+ name: file.name,476+ fragment477+ });478+ } else if (upload()?.upload) {349479 const task = upload().upload(ciphertext, {350480 url: new URL(tiny.localPath("/"), location.href).href,351481 hash,Binary file.
Binary file.
@@ -41,6 +41,31 @@ const receiveShare = async (request, url) => {
4141 return Response.redirect(url.pathname.replace(/\/share$/, "/files") + "#share=" + id, 303);4242 };4343 44+// Background uploads finish after the page closes. The last response is the45+// blob descriptor; it is parked for the Files page and the person is told.46+const uploadResult = async (registration, ok, detail) => {47+ const cache = await caches.open("tiny-uploads");48+ let response = Response.json({error: detail || "upload failed"}, {status: 500});49+ if (ok) {50+ const records = await registration.matchAll();51+ const last = records[records.length - 1];52+ if (last) response = (await last.responseReady).clone();53+ }54+ await cache.put(new Request(new URL("/uploads/" + registration.id, self.registration.scope).href), response);55+};56+self.addEventListener("backgroundfetchsuccess", event => {57+ event.waitUntil(uploadResult(event.registration, true).then(() => event.updateUI({title: "Upload complete"})));58+});59+self.addEventListener("backgroundfetchfail", event => {60+ event.waitUntil(uploadResult(event.registration, false, event.registration.failureReason).then(() => event.updateUI({title: "Upload failed"})));61+});62+self.addEventListener("backgroundfetchabort", event => {63+ event.waitUntil(uploadResult(event.registration, false, "aborted"));64+});65+self.addEventListener("backgroundfetchclick", event => {66+ event.waitUntil(self.clients.openWindow(new URL("files", self.registration.scope).href));67+});68+4469 // Notifications are opt-in per device; a push only arrives after the person4570 // enabled them on this browser. The payload is the relay's short summary.4671 self.addEventListener("push", event => {@@ -125,3 +125,17 @@ test("keeps the response body readable after headers arrive", async () => {
125125 assert.equal(result.descriptor.sha256, hash);126126 } finally { await new Promise(resolve => server.close(resolve)); }127127 });128+129+test("plan lists signed chunk requests without sending them", async () => {130+ const bytes = new Uint8Array(12);131+ const calls = [];132+ const planned = await sandbox.tiny.blossom.upload.plan(bytes, {url: "https://blossom.example/", chunkSize: 5, authorize: async (url, method, body) => { calls.push([method, body.byteLength]); return "Nostr " + url; }});133+ assert.equal(planned.size, 12);134+ assert.equal(planned.requests.length, 3);135+ assert.equal(planned.requests.map(request => request.headers["upload-offset"]).join(","), "0,5,10");136+ assert.equal(planned.requests.map(request => request.body.byteLength).join(","), "5,5,2");137+ assert.equal(planned.requests[0].headers["upload-length"], "12");138+ assert.equal(planned.requests[0].url, "https://blossom.example/" + planned.hash);139+ assert.equal(JSON.stringify(calls), JSON.stringify([["PATCH", 5], ["PATCH", 5], ["PATCH", 2]]));140+ assert.equal(planned.requests[2].headers.authorization, "Nostr https://blossom.example/" + planned.hash);141+});@@ -285,3 +285,48 @@ test("retry keeps the original ciphertext and key after interrupted encrypted up
285285 assert.equal(new URL(link).hash.slice(1), originalFragment);286286 assert.equal(upload.pending, null);287287 });288+289+test("large uploads hand signed requests to Background Fetch and read the parked descriptor", async () => {290+ const {window, FileUpload} = load(true);291+ window.tiny.localPath = path => path;292+ window.tiny.sha256hex = async bytes => Buffer.from(await webcrypto.subtle.digest("SHA-256", bytes)).toString("hex");293+ window.tiny.authorization = async (url, method, body) => "Nostr " + method + ":" + new URL(url).pathname + ":" + body.byteLength;294+ window.tiny.navigate = async () => {};295+ const bytes = new Uint8Array(9 * 1024 * 1024);296+ bytes[0] = 7;297+ const hash = await window.tiny.sha256hex(bytes);298+ const descriptor = {sha256: hash, size: bytes.byteLength};299+ let started;300+ const task = {result: "", uploaded: 0, uploadTotal: bytes.byteLength, listeners: [], addEventListener(name, fn) { this.listeners.push(fn); }, removeEventListener() {}};301+ const parked = new Map();302+ globalThis.caches = {open: async () => ({match: async key => parked.get(key), delete: async key => parked.delete(key), put: async (key, value) => parked.set(key, value)})};303+ globalThis.BackgroundFetchManager = class {};304+ const store = new Map();305+ globalThis.localStorage = {getItem: key => store.get(key) ?? null, setItem: (key, value) => store.set(key, value), removeItem: key => store.delete(key), get length() { return store.size; }};306+ Object.defineProperty(globalThis, "localStorage", {configurable: true, value: globalThis.localStorage});307+ Object.keys = ((original => target => target === globalThis.localStorage ? [...store.keys()] : original(target)))(Object.keys);308+ Object.defineProperty(globalThis, "navigator", {configurable: true, value: {clipboard: {writeText: async () => { throw Error("unavailable"); }}, serviceWorker: {controller: {}, ready: Promise.resolve({backgroundFetch: {fetch: async (id, requests, options) => { started = {id, requests, options}; return task; }, get: async () => null}})}}});309+ const originalFetch = globalThis.fetch;310+ globalThis.fetch = async (_url, init = {}) => new Response(null, {status: 204, headers: {Allow: "PATCH, PUT"}});311+ try {312+ const upload = new FileUpload(); window.document.append(upload); upload.connectedCallback();313+ const form = upload.querySelector("form");314+ form.elements.namedItem("file").files = [new File([bytes], "big.bin", {type: "application/octet-stream"})];315+ const run = upload.run(form);316+ await new Promise(resolve => setTimeout(resolve, 50));317+ assert.ok(started, "background fetch was not started");318+ assert.equal(started.requests.length, 2);319+ assert.equal(started.requests[0].method, "PATCH");320+ assert.equal(started.requests[1].headers.get("upload-offset"), String(5 * 1024 * 1024));321+ assert.match(started.requests[0].headers.get("authorization"), /^Nostr PATCH:/);322+ assert.equal(started.options.uploadTotal, bytes.byteLength);323+ assert.equal(store.get("tiny.bg." + started.id) !== undefined, true);324+ parked.set("https://relay.test/uploads/" + started.id, Response.json(descriptor));325+ task.result = "success"; task.uploaded = bytes.byteLength;326+ for (const fn of task.listeners) await fn();327+ await run;328+ assert.equal(store.has("tiny.bg." + started.id), false);329+ assert.equal(parked.size, 0);330+ assert.equal(upload.out.textContent, "Stored 1 file.");331+ } finally { globalThis.fetch = originalFetch; }332+});