1// NIP-5A's Blossom lookup, shared by alarm-driven mirroring and a cache2// miss at the site door. Bytes are checked before storage or delivery.3import { sha256 } from "@noble/hashes/sha2.js";4import { blobBlocked, fetchOrigin, readCapped, storeBlob } from "./blossom.ts";5import { now, tagValues, type Event } from "./event.ts";6import { bytesToHex } from "./negentropy.ts";7import { checkSite, manifest, parseSite, siteLabel, sitePaths, siteType } from "./sites.ts";8import { featureOn } from "./settings.ts";9import { localName } from "./pull.ts";10import type { Relay } from "./relay.ts";11import type { Job } from "./jobs.ts";1213// Proxy responses are bounded because verification precedes delivery.14// Mirroring has the same ceiling; local R2 serving verifies as a stream.15export const SITE_FETCH_MAX = 32 * 1024 * 1024;16const MAX_SERVERS = 10; // per list, so author servers always get a turn too1718function safeServer(relay: Relay, raw: string): URL | null {19 try {20 const u = new URL(raw);21 const local = localName(u, relay.domain);22 if (u.username || u.password || u.hash || (u.protocol !== "https:" && !(local && u.protocol === "http:"))) return null;23 if (local === relay.slug) return null; // our own files were already checked24 if (!local && (u.port && u.port !== "443" || /^\[|^[\d.]+$/.test(u.hostname) || !u.hostname.includes(".") || /\.(localhost|local|internal)$/.test(u.hostname))) return null;25 return u;26 } catch { return null; }27}28function servers(relay: Relay, e: Event): string[] {29 const raw = relay.store.query({ authors: [e.pubkey], kinds: [10063], tags: {} }, { pubkeys: [] }, 1, now()).rows[0];30 const author = raw ? tagValues(JSON.parse(raw) as Event, "server") : [];31 return [...new Set([...tagValues(e, "server").slice(0, MAX_SERVERS), ...author.slice(0, MAX_SERVERS)])];32}3334export async function remoteSiteBlob(relay: Relay, e: Event, mapping: string[]): Promise<{ bytes: Uint8Array; type: string; length: string | null } | null> {35 if (!featureOn(relay.settings.policy, "sites") || relay.fuelStatus().outOfFuel || blobBlocked(relay, mapping[2])) return null;36 const max = Math.min(SITE_FETCH_MAX, relay.settings.policy.maxBlobMB * 1024 * 1024);37 for (const server of servers(relay, e)) {38 const origin = safeServer(relay, server);39 if (!origin) continue;40 origin.pathname = origin.pathname.replace(/\/$/, "") + "/" + mapping[2];41 origin.search = "";42 let target: URL | null = origin;43 // Redirect targets go through the same checks. No caller credentials44 // are forwarded to a server or a sibling relay.45 for (let redirects = 0; target && redirects <= 3; redirects++) {46 let response: Response | string;47 try {48 response = await fetchOrigin(relay, target.href, { redirect: "manual", headers: { "accept-encoding": "identity" }, signal: AbortSignal.timeout(5000) });49 if (typeof response === "string") break;50 if ([301, 302, 303, 307, 308].includes(response.status)) {51 const location = response.headers.get("location");52 await response.body?.cancel();53 target = location ? safeServer(relay, new URL(location, target).href) : null;54 continue;55 }56 if (!response.ok || (response.headers.has("content-encoding") && response.headers.get("content-encoding") !== "identity")) { await response.body?.cancel(); break; }57 const length = response.headers.get("content-length");58 if (length !== null && (!/^\d+$/.test(length) || Number(length) > max)) { await response.body?.cancel(); break; }59 const bytes = await readCapped(response.body, max);60 if (bytes === "too big") { relay.meterBytes(max, 0); break; }61 relay.meterBytes(bytes.length, 0);62 if ((length !== null && Number(length) !== bytes.length) || bytesToHex(sha256(bytes)) !== mapping[2]) break;63 const type = response.headers.get("content-type") ?? (length === null ? siteType(mapping[1]) : "");64 return { bytes, type, length };65 } catch { break; }66 }67 }68 return null;69}7071// queueMirrors drains the SQL queue, which survives full jobs and hibernation.72// Turning mirroring off discards pending work; proxy reads still work.73export async function queueMirrors(relay: Relay): Promise<void> {74 if (!featureOn(relay.settings.policy, "sites") || !relay.settings.policy.features.sites.mirror) {75 relay.sql.exec(`DELETE FROM site_mirror_queue`);76 return;77 }78 const rows = relay.sql.exec<{ event_id: string }>(`SELECT event_id FROM site_mirror_queue LIMIT 10`).toArray();79 for (const row of rows) {80 const raw = relay.store.query({ ids: [row.event_id], tags: {} }, { pubkeys: [] }, 1, now()).rows[0];81 const e = raw ? JSON.parse(raw) as Event : null;82 if (e && !checkSite(e)) {83 const jobs = await relay.jobs();84 if (!jobs.some((j) => j.kind === "mirror" && j.site === e.id)) {85 const job = await relay.addChecked({ kind: "mirror", label: "mirror", site: e.id, relays: servers(relay, e), filter: {}, every: 0, running: false, startedAt: 0, rounds: 0, failures: 0, relayIndex: 0, cursor: 0, stored: 0, skipped: 0, blobs: 0, sent: 0, refused: 0, last: null });86 if (typeof job === "string") return;87 }88 }89 relay.sql.exec(`DELETE FROM site_mirror_queue WHERE event_id=?`, row.event_id);90 }91}9293export async function runMirrorRound(relay: Relay, job: Job): Promise<{ more: boolean; error: string }> {94 job.rounds++;95 if (!featureOn(relay.settings.policy, "sites") || !relay.settings.policy.features.sites.mirror) return { more: false, error: "" };96 const raw = relay.store.query({ ids: [job.site ?? ""], tags: {} }, { pubkeys: [] }, 1, now()).rows[0];97 if (!raw) return { more: false, error: "" }; // deleted, replaced, hidden or expired98 const e = JSON.parse(raw) as Event;99 if (checkSite(e)) return { more: false, error: "invalid manifest" };100 const paths = [...new Map(sitePaths(e).map((t) => [t[2], t])).values()];101 if (job.cursor >= paths.length) return { more: false, error: "" };102 const mapping = paths[job.cursor];103 if (relay.sql.exec(`SELECT 1 FROM blobs WHERE sha256=?`, mapping[2]).toArray().length || blobBlocked(relay, mapping[2])) {104 job.skipped++; job.cursor++;105 return { more: job.cursor < paths.length, error: "" };106 }107 const gate = relay.mayUpload(e.pubkey, `${relay.slug}.${relay.domain}`);108 if (gate) return { more: false, error: gate };109 const remote = await remoteSiteBlob(relay, e, mapping);110 if (!remote) return { more: false, error: "not found: no server returned the verified site file " + mapping[2] };111 const site = parseSite(siteLabel(e));112 if (!site || manifest(relay, site)?.id !== e.id || blobBlocked(relay, mapping[2]) || !featureOn(relay.settings.policy, "sites") || !relay.settings.policy.features.sites.mirror || relay.mayUpload(e.pubkey, `${relay.slug}.${relay.domain}`)) return { more: false, error: "" };113 const { created } = await storeBlob(relay, remote.bytes, remote.type, e.pubkey, now());114 if (created) job.blobs++; else job.skipped++;115 job.cursor++;116 return { more: job.cursor < paths.length, error: "" };117}