media, search, articles and dm join the Haven four. A replica preset takes a source relay and keeps one standing pull of its kinds; applying a preset again replaces that job. Cheap names plus standing pulls make a search replica or an articles mirror a one-click name.
- commit
- 0b3adcefcdc0dd71124e3982ea58ada7f5e72e56
- parent
- 9729b29b212565c217da6ea98ee470816a7ae1f4
- author
- Dami <dami.etoile@gmail.com>
| M | src/ | 4 | +- |
| M | src/ | 63 | ++++++++++---------------- |
| M | src/ | 73 | ++++++++++++++++++++++++++++++ |
| M | src/ | 2 | +- |
| M | test/ | 44 | +++++++++++++++++- |
@@ -10,7 +10,7 @@ import { Socket, dial, checkPullURL, runPullRound, type PullFilter, type PullJob
1010 import type { Relay } from "./relay.ts";1111 1212 export type JobKind = "pull" | "push";13-export type JobLabel = "pull" | "backfill" | "push";13+export type JobLabel = "pull" | "backfill" | "push" | "replica";1414 export const EVERY = [0, 1, 6, 24] as const; // hours; 0 runs once1515 1616 export interface JobResult {@@ -65,7 +65,7 @@ export function checkJob(raw: unknown, relay: Relay): JobSpec | string {
6565 const r = (raw && typeof raw === "object" ? raw : {}) as Record<string, unknown>;6666 const kind = r.kind === "pull" || r.kind === "push" ? r.kind : null;6767 if (!kind) return "invalid: kind must be pull or push";68- const label: JobLabel = r.label === "backfill" && kind === "pull" ? "backfill" : kind;68+ const label: JobLabel = kind === "pull" && (r.label === "backfill" || r.label === "replica") ? r.label : kind;6969 const list = Array.isArray(r.relays) ? r.relays : [];7070 const relays: string[] = [];7171 for (const u of list) {@@ -13,7 +13,7 @@ import { relaysFromList } from "./jobs.ts";
1313 import { notify, notifySettings } from "./notify.ts";1414 import { DUMP_NAME_RE, deleteDump, dumpBytes, listDumps, writeDump } from "./dumps.ts";1515 import { can, METHOD_ACTIONS } from "./roles.ts";16-import { PRESETS, applyPreset } from "./presets.ts";16+import { PRESETS, applyPreset, findPreset } from "./presets.ts";1717 1818 // verifyNIP98 checks an "Authorization: Nostr <base64 event>" header against1919 // the request: kind 27235, fresh, u tag naming this URL, method tag, and a@@ -383,47 +383,32 @@ export async function manage(relay: Relay, req: Request): Promise<Response> {
383383 await relay.publishMembership();384384 return reply({ result: s.policy });385385 case "listpresets":386- return reply({ result: PRESETS.map((x) => ({ name: x.name, title: x.title, about: x.about })) });386+ return reply({ result: PRESETS.map((x) => ({ name: x.name, title: x.title, about: x.about, source: x.source, every: x.every })) });387387 case "applypreset": {388- // (name): writes, reads, directory, kind rules and keep-for rules in one go.389- const err = applyPreset(s, str(0));388+ // (name, {source?}): writes, reads, directory, kind rules and keep-for389+ // rules in one go. A replica preset also gets a standing pull of its390+ // kinds from the source; an earlier replica job is replaced.391+ const preset = findPreset(str(0));392+ if (!preset) return reply({ error: "invalid: no preset named " + str(0) }, 400);393+ const opts = params[1] && typeof params[1] === "object" ? (params[1] as { source?: unknown }) : {};394+ const source = typeof opts.source === "string" ? opts.source.trim() : "";395+ if (preset.source === "required" && !source) return reply({ error: "invalid: this preset needs a source relay to mirror" }, 400);396+ if (source && !preset.source) return reply({ error: "invalid: this preset does not mirror a source" }, 400);397+ if (source) {398+ const bad = checkPullURL(source, relay.slug, relay.domain);399+ if (bad) return reply({ error: bad }, 400);400+ }401+ const err = applyPreset(s, preset.name);390402 if (err) return reply({ error: err }, 400);403+ for (const j of await relay.jobs()) if (j.label === "replica") await relay.removeJob(j.id);404+ let job = null;405+ if (source) {406+ const r = await relay.addJob({ kind: "pull", label: "replica", relays: [source], filter: { kinds: preset.allow }, every: preset.every ?? 24 });407+ if (typeof r === "string") return reply({ error: r }, r.startsWith("invalid") ? 400 : 409);408+ job = r;409+ }391410 await relay.publishMembership();392- return reply({ result: s.policy });393- }394- case "pullfrom": {395- // (url): copy what another relay has that this one lacks. Runs in396- // the background; pullstatus reports on it.397- const url = str(0).trim();398- const bad = checkPullURL(url, relay.slug, relay.domain);399- if (bad) return reply({ error: bad }, 400);400- const err = await relay.pullStart(url);401- if (err) return reply({ error: err }, 409);402- return reply({ result: { started: true, url } });403- }404- case "pullstatus":405- return reply({ result: await relay.pullStatus() });406- case "listjobs":407- return reply({ result: await relay.jobs() });408- case "addjob": {409- // ({kind, relays, filter?, every?, label?}): a pull or push, once or standing.410- const r = await relay.addJob(params[0]);411- if (typeof r === "string") return reply({ error: r }, r.startsWith("invalid") ? 400 : 409);412- return reply({ result: r });413- }414- case "removejob":415- return reply({ result: await relay.removeJob(str(0)) });416- case "runjob":417- return reply({ result: await relay.runJob(str(0)) });418- case "backfill": {419- // (relays?): fetch the owner's own history from the relays in their420- // kind 10002 stored here, or from the given list.421- const given = Array.isArray(params[0]) ? (params[0] as unknown[]).filter((u): u is string => typeof u === "string" && u.trim() !== "") : [];422- const relays = given.length ? given : relaysFromList(relay, p.owner);423- if (relays.length === 0) return reply({ error: "invalid: no relay list (kind 10002) is stored here; give relays to fetch from" }, 400);424- const r = await relay.addJob({ kind: "pull", label: "backfill", relays, filter: { authors: [p.owner] }, every: 0 });425- if (typeof r === "string") return reply({ error: r }, r.startsWith("invalid") ? 400 : 409);426- return reply({ result: r });411+ return reply({ result: { ...s.policy, job } });427412 }428413 case "transferowner": {429414 // (pubkey): hands the relay to a member; the old owner stays as a moderator.@@ -6,6 +6,7 @@
66 // rules and keep-for rules together and leaves limits, identity, people and77 // bans alone.88 import { DEFAULT_POLICY, type Policy, type Settings } from "./settings.ts";9+import { SEARCH_KINDS } from "./store.ts";910 1011 export interface Preset {1112 name: string;@@ -17,8 +18,16 @@ export interface Preset {
1718 allow: number[];1819 block: number[];1920 retention: { kind: number | null; days: number }[];21+ // A replica keeps itself in step with a source relay through a standing22+ // pull of its kinds. "required" presets refuse to apply without one.23+ source?: "required" | "optional";24+ every?: number; // hours between pulls2025 }2126 27+// The kinds a search replica carries: what the full-text index covers,28+// minus drafts, which have no business on a public replica.29+const SEARCHABLE = [...SEARCH_KINDS].filter((k) => k !== 30024).sort((a, b) => a - b);30+2231 export const PRESETS: Preset[] = [2332 {2433 name: "default",@@ -92,6 +101,70 @@ export const PRESETS: Preset[] = [
92101 block: [],93102 retention: [],94103 },104+ // Kind-scoped names: cheap names plus standing pulls make single-purpose105+ // relays and specialised replicas nearly free. One name per job.106+ {107+ // A Blossom-only name, alice-media: the files live here, the notes do108+ // not. Members write, so their uploads pass the upload gate, but the109+ // only events they may post are profiles and Blossom server lists, so110+ // the name stays a media host and nothing else.111+ name: "media",112+ title: "Media",113+ about: "A media host. Members upload files; the only events accepted are profiles and Blossom server lists. Anyone reads.",114+ writes: "allowlist",115+ reads: "open",116+ directoryPublic: true,117+ allow: [0, 10063],118+ block: [],119+ retention: [],120+ },121+ {122+ // A search replica, alice-search: a standing pull of the searchable123+ // kinds from a source relay every six hours, readable by all, written124+ // only by the owner, so a client can point search at it without the125+ // source relay ever being asked.126+ name: "search",127+ title: "Search replica",128+ about: "A read-only copy of another relay's notes, threads, comments, highlights, articles and wiki pages, refreshed every six hours. Anyone reads.",129+ writes: "owner",130+ reads: "open",131+ directoryPublic: true,132+ allow: SEARCHABLE,133+ block: [],134+ retention: [],135+ source: "required",136+ every: 6,137+ },138+ {139+ // An articles name: long-form only, with the profiles that render the140+ // bylines. Pulls from a source when one is given, otherwise the owner141+ // publishes here directly.142+ name: "articles",143+ title: "Articles",144+ about: "Long-form articles and the profiles behind them, nothing else. Only you write, anyone reads. Give a source relay to mirror its articles daily.",145+ writes: "owner",146+ reads: "open",147+ directoryPublic: true,148+ allow: [0, 30023],149+ block: [],150+ retention: [],151+ source: "optional",152+ every: 24,153+ },154+ {155+ // A private-message inbox, alice-dm: anyone may drop a gift wrap, only156+ // members read them. Wraps carry no author, so an allow list of just157+ // the wrap kind and profiles keeps everything else out.158+ name: "dm",159+ title: "DM inbox",160+ about: "An inbox for private messages. Anyone drops gift wraps, only members read them. Nothing else is accepted.",161+ writes: "open",162+ reads: "members",163+ directoryPublic: false,164+ allow: [0, 1059],165+ block: [],166+ retention: [],167+ },95168 ];96169 97170 export function findPreset(name: string): Preset | undefined {@@ -56,7 +56,7 @@ const inList = (col: string) => `${col} IN (SELECT value FROM json_each(?))`;
5656 // searchable says which kinds go into the full-text index: the ones whose5757 // content is prose. Reactions, JSON blobs and private kinds are skipped; it5858 // keeps the index small and every event a few rows cheaper to write.59-const SEARCH_KINDS = new Set([0, 1, 11, 1111, 9802, 30023, 30024, 30818]);59+export const SEARCH_KINDS = new Set([0, 1, 11, 1111, 9802, 30023, 30024, 30818]);6060 export function searchable(kind: number): boolean {6161 return SEARCH_KINDS.has(kind) && !isPrivate(kind);6262 }@@ -30,11 +30,14 @@ describe("presets", () => {
3030 const owner = generateSecretKey();3131 await rpc(host, owner, "claim");3232 const list = (await rpc(host, owner, "listpresets")).result;33- expect(list.map((p: any) => p.name)).toEqual(["default", "outbox", "inbox", "private", "chat"]);33+ expect(list.map((p: any) => p.name)).toEqual(["default", "outbox", "inbox", "private", "chat", "media", "search", "articles", "dm"]);3434 for (const p of list) expect(p.about.length).toBeGreaterThan(10);35+ expect(list.find((p: any) => p.name === "search").source).toBe("required");36+ expect(list.find((p: any) => p.name === "articles").source).toBe("optional");37+ expect(list.find((p: any) => p.name === "media").source).toBeUndefined();3538 for (const preset of PRESETS) {36- const r = await rpc(host, owner, "applypreset", preset.name);37- expect(r.status, preset.name).toBe(200);39+ const r = await rpc(host, owner, "applypreset", preset.name, preset.source === "required" ? { source: "wss://elsewhere.bind.ws" } : undefined);40+ expect(r.status, preset.name + " " + JSON.stringify(r)).toBe(200);3841 expect(r.result.writes).toBe(preset.writes);3942 expect(r.result.reads).toBe(preset.reads);4043 expect(r.result.directoryPublic).toBe(preset.directoryPublic);@@ -51,6 +54,41 @@ describe("presets", () => {
5154 expect((await rpc(host, owner, "applypreset", "haven")).status).toBe(400);5255 });5356 57+ it("a replica preset needs a source and keeps one standing pull of its kinds", async () => {58+ const host = "presets-replica.bind.ws";59+ const owner = generateSecretKey();60+ await rpc(host, owner, "claim");61+ let r = await rpc(host, owner, "applypreset", "search");62+ expect(r.status).toBe(400);63+ expect(r.error).toMatch(/needs a source/);64+ r = await rpc(host, owner, "applypreset", "search", { source: "wss://" + host });65+ expect(r.status).toBe(400);66+ expect(r.error).toMatch(/itself/);67+ r = await rpc(host, owner, "applypreset", "media", { source: "wss://elsewhere.bind.ws" });68+ expect(r.status).toBe(400);69+ expect(r.error).toMatch(/does not mirror/);70+ r = await rpc(host, owner, "applypreset", "search", { source: "wss://elsewhere.bind.ws" });71+ expect(r.status, JSON.stringify(r)).toBe(200);72+ expect(r.result.job.kind).toBe("pull");73+ expect(r.result.job.label).toBe("replica");74+ expect(r.result.job.every).toBe(6);75+ expect(r.result.job.relays).toEqual(["wss://elsewhere.bind.ws"]);76+ expect(r.result.job.filter.kinds).toEqual([0, 1, 11, 1111, 9802, 30023, 30818]);77+ expect((await rpc(host, owner, "listallowedkinds")).result).toEqual([0, 1, 11, 1111, 9802, 30023, 30818]);78+ // Applying again replaces the standing job rather than adding one.79+ r = await rpc(host, owner, "applypreset", "articles", { source: "wss://other.bind.ws" });80+ expect(r.status).toBe(200);81+ expect(r.result.job.every).toBe(24);82+ const jobs = (await rpc(host, owner, "listjobs")).result.filter((j: any) => j.label === "replica");83+ expect(jobs.length).toBe(1);84+ expect(jobs[0].filter.kinds).toEqual([0, 30023]);85+ // Articles without a source is a plain rule bundle; the replica job goes away.86+ r = await rpc(host, owner, "applypreset", "articles");87+ expect(r.status).toBe(200);88+ expect(r.result.job).toBeNull();89+ expect((await rpc(host, owner, "listjobs")).result.filter((j: any) => j.label === "replica")).toEqual([]);90+ });91+5492 it("needs the rules action: a moderator may list presets but not apply one", async () => {5593 const host = "presets-mod.bind.ws";5694 const owner = generateSecretKey();