- commit
- 5fa7a165078c20aa1033b3e6c18771abbf210ada
- parent
- 799f014549985817ef4434f4c9bf3ca20b072d10
- author
- Dami <dami.etoile@gmail.com>
| M | docs/ | 4 | ++ |
| M | src/ | 36 | ++++++++++++++------ |
| M | test/ | 55 | +++++++++++++++++++++++++++++- |
@@ -111,6 +111,10 @@ The edge mapping may remain cached for up to 60 seconds. The local record
111111 checks that the selected site still exists before it serves a request, so a112112 stale mapping cannot serve a removed target.113113 114+Manifest replacements coalesce into one index write per hostname in each115+outbox batch. The route stays present while a throttled write waits for a116+retry; replacing a manifest does not delete and recreate its hostname.117+114118 ## Deliberate limits115119 116120 bind.ws does not advertise NIP-5A in `supported_nips`: NIP-11 still defines@@ -105,23 +105,37 @@ CREATE TRIGGER IF NOT EXISTS site_removed AFTER DELETE ON events WHEN old.kind I
105105 END;106106 `;107107 108-// syncSiteIndex never clears another relay's or publication's mapping.109-// A failed KV write leaves the SQL entry intact for the next alarm.108+// syncSiteIndex coalesces replacements into one KV write per label, keeping109+// the old route usable if a write is throttled. Failed groups stay queued.110110 export async function syncSiteIndex(relay: Relay): Promise<boolean> {111111 if (!relay.hosts || !relay.slug) return false;112112 const rows = relay.sql.exec<{ seq: number; raw: string; removed: number }>(`SELECT * FROM site_outbox ORDER BY seq LIMIT 100`).toArray();113+ const groups = new Map<string, { seq: number; e: Event; removed: number }[]>();113114 for (const row of rows) {114115 const e = JSON.parse(row.raw) as Event;115- if (!checkSite(e)) {116- const key = siteKey(siteLabel(e));117- if (row.removed || (expiration(e) > 0 && expiration(e) <= now())) {118- const existing = await relay.hosts.get(key);119- if (existing === JSON.stringify({ name: relay.slug, event: e.id })) await relay.hosts.delete(key);120- } else {121- await relay.hosts.put(key, JSON.stringify({ name: relay.slug, event: e.id }));122- }116+ if (checkSite(e)) { relay.sql.exec(`DELETE FROM site_outbox WHERE seq=?`, row.seq); continue; }117+ const label = siteLabel(e);118+ const group = groups.get(label) ?? [];119+ group.push({ seq: row.seq, e, removed: row.removed });120+ groups.set(label, group);121+ }122+ for (const [label, group] of groups) {123+ const last = group[group.length - 1];124+ const removed = last.removed || (expiration(last.e) > 0 && expiration(last.e) <= now());125+ // A batch can end between a replacement's removal and insertion. A126+ // different live version keeps the route; teardown removes the same127+ // version explicitly even while it still exists in the events table.128+ const current = manifest(relay, parseSite(label)!);129+ const target = removed ? current && current.id !== last.e.id ? current : null : current;130+ const key = siteKey(label);131+ const existing = await relay.hosts.get(key);132+ if (target) {133+ const value = JSON.stringify({ name: relay.slug, event: target.id });134+ if (existing !== value) await relay.hosts.put(key, value);135+ } else if (group.some(({ e }) => existing === JSON.stringify({ name: relay.slug, event: e.id }))) {136+ await relay.hosts.delete(key);123137 }124- relay.sql.exec(`DELETE FROM site_outbox WHERE seq=?`, row.seq);138+ for (const row of group) relay.sql.exec(`DELETE FROM site_outbox WHERE seq=?`, row.seq);125139 }126140 return relay.sql.exec(`SELECT 1 FROM site_outbox LIMIT 1`).toArray().length > 0;127141 }@@ -1,7 +1,8 @@
11 import { SELF, env, runInDurableObject } from "cloudflare:test";22 import { describe, it, expect } from "vitest";33 import { generateSecretKey } from "nostr-tools/pure";4-import { aggregate, base36, checkSite, parseSite, siteKey, siteLabel, unbase36 } from "../../src/sites.ts";4+import { aggregate, base36, checkSite, parseSite, siteKey, siteLabel, syncSiteIndex, unbase36 } from "../../src/sites.ts";5+import type { Relay } from "../../src/relay.ts";56 import { upload } from "../helpers/media.ts";67 import { ev, pk, rpc, now, nip98, info } from "../helpers/relay.ts";78 import { WS } from "../helpers/ws.ts";@@ -149,6 +150,58 @@ async function syncIndex(name: string) {
149150 }150151 151152 describe("site hostname index cleanup", () => {153+ it("keeps a replacement route through a throttled write and a split outbox batch", async () => {154+ const name = "idx-retry", sk = generateSecretKey();155+ await rpc(name + ".bind.ws", sk, "claim");156+ await runInDurableObject(env.RELAY.getByName(name), async (r) => {157+ r.store.onSitesChanged = () => {};158+ let value: string | null = null, writes = 0, deletes = 0, fail = false;159+ const hosts = {160+ get: async () => value,161+ put: async (_key: string, next: string) => { if (fail) throw new Error("KV write limit"); writes++; value = next; },162+ delete: async () => { deletes++; value = null; },163+ };164+ const subject = { sql: r.sql, store: r.store, slug: name, hosts } as unknown as Relay;165+ const first = ev(sk, 15128, "first", paths, now() - 3);166+ const second = ev(sk, 15128, "second", paths, now() - 2);167+ const third = ev(sk, 15128, "third", paths, now() - 1);168+ expect(r.store.save(first, now())).toBe("");169+ await syncSiteIndex(subject);170+ const original = value;171+ writes = 0;172+ expect(r.store.save(second, now())).toBe("");173+ expect(r.store.save(third, now())).toBe("");174+ fail = true;175+ await expect(syncSiteIndex(subject)).rejects.toThrow("KV write limit");176+ expect(value).toBe(original);177+ expect(deletes).toBe(0);178+ expect(r.sql.exec("SELECT * FROM site_outbox").toArray()).toHaveLength(4);179+ fail = false;180+ await syncSiteIndex(subject);181+ expect(value).toBe(JSON.stringify({ name, event: third.id }));182+ expect(writes).toBe(1);183+ expect(deletes).toBe(0);184+ expect(r.sql.exec("SELECT * FROM site_outbox").toArray()).toHaveLength(0);185+186+ // The first batch ends on the old version's removal; the successor187+ // is already durable, and its insertion remains for the next batch.188+ for (let i = 0; i < 99; i++) r.sql.exec("INSERT INTO site_outbox(raw,removed) VALUES(?,1)", JSON.stringify(third));189+ const fourth = ev(sk, 15128, "fourth", paths);190+ expect(r.store.save(fourth, now())).toBe("");191+ writes = 0;192+ expect(await syncSiteIndex(subject)).toBe(true);193+ expect(value).toBe(JSON.stringify({ name, event: fourth.id }));194+ expect(await syncSiteIndex(subject)).toBe(false);195+ expect(writes).toBe(1);196+ expect(deletes).toBe(0);197+ for (let i = 0; i < 100; i++) r.sql.exec("INSERT INTO site_outbox(raw,removed) VALUES(?,0)", JSON.stringify(first));198+ writes = 0;199+ expect(await syncSiteIndex(subject)).toBe(false);200+ expect(value).toBe(JSON.stringify({ name, event: fourth.id }));201+ expect(writes).toBe(0);202+ });203+ });204+152205 it("clears a mapping when management deletes its manifest", async () => {153206 const name = "idx-manage";154207 const sk = generateSecretKey();