- commit
- 5b67fbe6c462226d2647efde6df45162f2418779
- parent
- c745c85560301c73a56516a4089edb24c2a5be61
- author
- Dami <dami.etoile@gmail.com>
| M | README.md | 21 | ++++- |
| M | worker/ | 28 | ++++- |
| A | worker/ | 28 | +++++ |
| M | worker/ | 2 | +- |
| M | worker/ | 19 | ++- |
| A | worker/ | 161 | ++++++++++++++++++++++++++++++ |
| M | worker/ | 33 | +++++- |
| A | worker/ | 52 | ++++++++++ |
| M | worker/ | 4 | + |
| M | worker/ | 9 | ++ |
@@ -117,7 +117,9 @@ A push is a stateless three-step protocol, and the manifest itself is the state:
117117 appending history118118 119119 Serving is one D1 read (address → manifest, expiry, password) and two R2 reads, with `ETag`120-revalidation and a 60-second cache. Metadata lives in three tables:120+revalidation and a 60-second tiered Workers Cache. Cache hits skip Worker execution and the D1/R2121+reads entirely. Public responses are tagged per site and purged when a push, revert, deletion, TTL,122+or password policy changes; protected responses and errors are never cached. Metadata lives in three tables:121123 122124 | table | keys |123125 | -------- | ------------------------------------------------------------------------ |@@ -213,3 +215,20 @@ nzip auth --server http://localhost:8787 --token dev-token-local-only
213215 214216 Test the GC cron: `npx wrangler dev --test-scheduled`, then215217 `curl "http://localhost:8787/__scheduled?cron=0+4+*+*+*"`.218+219+### Security observability220+221+Workers Logs are enabled by default for structured scan telemetry, with automatic invocation logs222+disabled to keep volume and cost bounded. The Worker retains complete request sequences for a223+deterministic 1% sample of scanner identities. Events are emitted as `security.request` and include224+the request class, bounded path, response result, derived vault/site slots for four-hex addresses,225+country, Cloudflare colo, ASN, and an HMAC-derived `scanner_id`; raw client IPs are never logged.226+227+Inspect these events under **Workers & Pages → nzip → Observability**, filtering on228+`event = "security.request"`. The `sample_rate` field records the sampling factor for aggregate229+estimates. Normal successful asset requests are omitted, while bare address probes, unlock attempts,230+bad API requests, invalid paths, and missing address assets are eligible.231+232+Rejected `429` floods receive a second 1% per-request sample after the identity sample. This keeps233+the allowed probe sequence intact while preventing a single blocked client from producing unbounded234+log volume; each event's `sample_rate` reflects the effective rate.@@ -37,6 +37,7 @@ import {
3737 type SiteRow,3838 } from "./db.ts";3939 import { type Env, err, json, siteUrl } from "./env.ts";40+import { purgeSiteCache } from "./cache.ts";4041 import { hashPassword } from "./password.ts";4142 4243 const HEX64 = /^[0-9a-f]{64}$/;@@ -91,7 +92,7 @@ async function handleBlobPut(req: Request, env: Env, hash: string): Promise<Resp
9192 return json({ ok: true, hash });9293 }9394 94-async function handleCommit(req: Request, env: Env): Promise<Response> {95+async function handleCommit(req: Request, env: Env, ctx: ExecutionContext): Promise<Response> {9596 const body = await req.json<CommitRequest>();9697 const passwordHash = await passwordHashFor(body.password);9798 const bytes = canonicalManifestBytes(body.manifest);@@ -151,6 +152,7 @@ async function handleCommit(req: Request, env: Env): Promise<Response> {
151152 });152153 153154 const addressStr = formatAddress(address);155+ await purgeSiteCache(ctx, addressStr);154156 return json<CommitResponse>({155157 address: addressStr,156158 url: siteUrl(env, addressStr),@@ -193,7 +195,12 @@ async function sourceManifest(env: Env, site: SiteRow): Promise<Manifest | Respo
193195 return parseManifest(new Uint8Array(await object.arrayBuffer()));194196 }195197 196-export async function api(req: Request, env: Env, url: URL): Promise<Response> {198+export async function api(199+ req: Request,200+ env: Env,201+ url: URL,202+ ctx: ExecutionContext,203+): Promise<Response> {197204 const parts = url.pathname.split("/").filter(Boolean); // ["api", ...]198205 const method = req.method;199206 @@ -203,7 +210,7 @@ export async function api(req: Request, env: Env, url: URL): Promise<Response> {
203210 return await handlePrepare(req, env);204211 }205212 if (parts[1] === "push" && parts[2] === "commit" && method === "POST") {206- return await handleCommit(req, env);213+ return await handleCommit(req, env, ctx);207214 }208215 if (parts[1] === "blob" && parts.length === 3 && method === "PUT") {209216 return await handleBlobPut(req, env, parts[2]);@@ -280,7 +287,9 @@ export async function api(req: Request, env: Env, url: URL): Promise<Response> {
280287 }281288 if (parts.length === 3 && method === "DELETE") {282289 await env.DB.prepare("DELETE FROM sites WHERE address = ?").bind(site.address).run();283- return json({ ok: true, address: formatAddress(site.address) });290+ const address = formatAddress(site.address);291+ await purgeSiteCache(ctx, address);292+ return json({ ok: true, address });284293 }285294 if (parts.length === 3 && method === "PATCH") {286295 const body = await req.json<PatchSiteRequest>();@@ -305,10 +314,11 @@ export async function api(req: Request, env: Env, url: URL): Promise<Response> {
305314 .bind(...values).run();306315 }307316 const updated = await getSiteByAddress(env, site.address);317+ await purgeSiteCache(ctx, formatAddress(site.address));308318 return json(await siteRowToDetail(env, updated!));309319 }310320 if (parts.length === 4 && parts[3] === "revert" && method === "POST") {311- return await handleRevert(req, env, site);321+ return await handleRevert(req, env, site, ctx);312322 }313323 }314324 @@ -321,7 +331,12 @@ export async function api(req: Request, env: Env, url: URL): Promise<Response> {
321331 }322332 }323333 324-async function handleRevert(req: Request, env: Env, site: SiteRow): Promise<Response> {334+async function handleRevert(335+ req: Request,336+ env: Env,337+ site: SiteRow,338+ ctx: ExecutionContext,339+): Promise<Response> {325340 const body = await req.json<RevertRequest>().catch(() => ({} as RevertRequest));326341 const history = await siteHistory(env, site.address); // newest first327342 @@ -346,6 +361,7 @@ async function handleRevert(req: Request, env: Env, site: SiteRow): Promise<Resp
346361 });347362 348363 const addressStr = formatAddress(site.address);364+ await purgeSiteCache(ctx, addressStr);349365 return json({350366 address: addressStr,351367 url: siteUrl(env, addressStr),@@ -0,0 +1,28 @@
1+export function siteCacheTag(address: string): string {2+ return `nzip-site-${address}`;3+}4+5+/** Purge every cached URL for a site after content or policy changes. */6+export async function purgeSiteCache(7+ ctx: ExecutionContext,8+ address: string,9+): Promise<void> {10+ if (!ctx.cache) return;11+12+ try {13+ const result = await ctx.cache.purge({ tags: [siteCacheTag(address)] });14+ if (!result.success) {15+ console.error({16+ event: "cache.purge_failed",17+ address,18+ errors: result.errors,19+ });20+ }21+ } catch (error) {22+ console.error({23+ event: "cache.purge_failed",24+ address,25+ error: error instanceof Error ? error.message : String(error),26+ });27+ }28+}@@ -17,7 +17,7 @@ export function json<T = unknown>(body: T, status = 200, headers: HeadersInit =
1717 }1818 1919 export function err(message: string, status: number): Response {20- return json({ error: message }, status);20+ return json({ error: message }, status, { "cache-control": "no-store" });2121 }2222 2323 export function siteUrl(env: Env, address: string): string {@@ -3,6 +3,7 @@ import { checkBearer } from "./auth.ts";
33 import type { Env } from "./env.ts";44 import { err } from "./env.ts";55 import { runGc } from "./gc.ts";6+import { logSecurityRequest } from "./observability.ts";67 import { serve } from "./serve.ts";78 89 function tooManyRequests(): Response {@@ -13,17 +14,21 @@ function tooManyRequests(): Response {
1314 }1415 1516 export default {16- async fetch(req: Request, env: Env): Promise<Response> {17+ async fetch(req: Request, env: Env, ctx: ExecutionContext): Promise<Response> {1718 const url = new URL(req.url);19+ const finish = (response: Response): Response => {20+ ctx.waitUntil(logSecurityRequest(req, env, url, response));21+ return response;22+ };1823 1924 if (url.pathname === "/api" || url.pathname.startsWith("/api/")) {20- if (!checkBearer(req, env)) return err("unauthorized", 401);21- return await api(req, env, url);25+ if (!checkBearer(req, env)) return finish(err("unauthorized", 401));26+ return finish(await api(req, env, url, ctx));2227 }2328 2429 const isUnlock = req.method === "POST" && /^\/[0-9a-f]{4}\/__unlock$/.test(url.pathname);2530 if (req.method !== "GET" && req.method !== "HEAD" && !isUnlock) {26- return err("method not allowed", 405);31+ return finish(err("method not allowed", 405));2732 }2833 2934 // Guessing defenses. The address space is only 16 bits, so throttle the two@@ -33,13 +38,13 @@ export default {
3338 if (isUnlock) {3439 const address = url.pathname.slice(1, 5);3540 const { success } = await env.RL_UNLOCK.limit({ key: `${ip}:${address}` });36- if (!success) return tooManyRequests();41+ if (!success) return finish(tooManyRequests());3742 } else if (/^\/[0-9a-f]{4}\/?$/.test(url.pathname)) {3843 const { success } = await env.RL_ENUM.limit({ key: ip });39- if (!success) return tooManyRequests();44+ if (!success) return finish(tooManyRequests());4045 }4146 42- return await serve(req, env, url);47+ return finish(await serve(req, env, url));4348 },4449 4550 async scheduled(_controller: ScheduledController, env: Env, ctx: ExecutionContext): Promise<void> {@@ -0,0 +1,161 @@
1+import type { Env } from "./env.ts";2+3+const ADDRESS_PATH = /^\/([0-9a-f]{4})(\/.*)?$/;4+const EXACT_ADDRESS_PATH = /^\/([0-9a-f]{4})\/?$/;5+const UNLOCK_PATH = /^\/([0-9a-f]{4})\/__unlock$/;6+const KNOWN_SERVICE_PATHS = new Set(["/", "/favicon.ico", "/robots.txt"]);7+const MAX_LOGGED_PATH_LENGTH = 256;8+const SCANNER_SAMPLE_RATE = 0.01;9+const SAMPLE_SPACE = 0x1_0000;10+const SAMPLE_CUTOFF = Math.floor(SAMPLE_SPACE * SCANNER_SAMPLE_RATE);11+12+type RequestClass = "address" | "address_asset" | "unlock" | "api" | "invalid";13+14+export interface SecurityRequestEvent {15+ event: "security.request";16+ sample_rate: number;17+ path_class: RequestClass;18+ path: string;19+ path_truncated?: true;20+ method: string;21+ status: number;22+ result: string;23+ address?: string;24+ vault_slot?: number;25+ site_slot?: number;26+}27+28+function resultForStatus(status: number): string {29+ if (status === 401) return "unauthorized";30+ if (status === 404) return "not_found";31+ if (status === 405) return "method_not_allowed";32+ if (status === 410) return "gone";33+ if (status === 429) return "rate_limited";34+ if (status >= 500) return "server_error";35+ if (status >= 400) return "client_error";36+ if (status >= 300) return "redirect";37+ return "served";38+}39+40+/** Build a bounded, low-cardinality event for requests useful in scan analysis. */41+export function buildSecurityRequestEvent(42+ req: Request,43+ url: URL,44+ response: Response,45+): SecurityRequestEvent | null {46+ const path = url.pathname;47+ const unlock = path.match(UNLOCK_PATH);48+ const exactAddress = path.match(EXACT_ADDRESS_PATH);49+ const addressPath = path.match(ADDRESS_PATH);50+51+ let pathClass: RequestClass;52+ if (path === "/api" || path.startsWith("/api/")) {53+ if (response.status < 400) return null;54+ pathClass = "api";55+ } else if (unlock) {56+ pathClass = "unlock";57+ } else if (exactAddress) {58+ // Bare address hits are the enumeration surface, whether they hit or miss.59+ pathClass = "address";60+ } else if (addressPath) {61+ if (response.status < 400) return null;62+ pathClass = "address_asset";63+ } else {64+ if (response.status < 400 || KNOWN_SERVICE_PATHS.has(path)) return null;65+ pathClass = "invalid";66+ }67+68+ const address = (unlock ?? exactAddress ?? addressPath)?.[1];69+ const loggedPath = path.slice(0, MAX_LOGGED_PATH_LENGTH);70+ return {71+ event: "security.request",72+ sample_rate: SCANNER_SAMPLE_RATE,73+ path_class: pathClass,74+ path: loggedPath,75+ ...(loggedPath.length < path.length76+ ? { path_truncated: true as const }77+ : {}),78+ method: req.method,79+ status: response.status,80+ result: resultForStatus(response.status),81+ ...(address82+ ? {83+ address,84+ vault_slot: Number.parseInt(address[0], 16),85+ site_slot: Number.parseInt(address.slice(1), 16),86+ }87+ : {}),88+ };89+}90+91+async function scannerDigest(ip: string, secret: string): Promise<Uint8Array> {92+ const encoder = new TextEncoder();93+ const key = await crypto.subtle.importKey(94+ "raw",95+ encoder.encode(secret),96+ { name: "HMAC", hash: "SHA-256" },97+ false,98+ ["sign"],99+ );100+ return new Uint8Array(101+ await crypto.subtle.sign("HMAC", key, encoder.encode(ip)),102+ );103+}104+105+function hex(bytes: Uint8Array): string {106+ return Array.from(bytes, (byte) => byte.toString(16).padStart(2, "0")).join(107+ "",108+ );109+}110+111+/**112+ * Retain complete sequences for a deterministic ~1% sample of scanner identities.113+ * This is more useful for enumeration detection, and much cheaper, than sampling114+ * individual requests independently. The raw client IP is never logged.115+ */116+export async function logSecurityRequest(117+ req: Request,118+ env: Env,119+ url: URL,120+ response: Response,121+): Promise<void> {122+ const event = buildSecurityRequestEvent(req, url, response);123+ if (!event) return;124+125+ try {126+ const ip = req.headers.get("cf-connecting-ip") ?? "local";127+ const digest = await scannerDigest(ip, env.NZIP_TOKEN);128+ const bucket = (digest[0] << 8) | digest[1];129+ if (bucket >= SAMPLE_CUTOFF) return;130+131+ // Once a limiter has rejected a client, a flood can continue without bound.132+ // Sample those repeated 429s again by request while retaining all allowed133+ // probes from the selected scanner identity.134+ let effectiveSampleRate = SCANNER_SAMPLE_RATE;135+ if (response.status === 429) {136+ const requestIdentity = req.headers.get("cf-ray") ?? crypto.randomUUID();137+ const requestDigest = await scannerDigest(138+ `${ip}:${requestIdentity}`,139+ env.NZIP_TOKEN,140+ );141+ const requestBucket = (requestDigest[0] << 8) | requestDigest[1];142+ if (requestBucket >= SAMPLE_CUTOFF) return;143+ effectiveSampleRate *= SCANNER_SAMPLE_RATE;144+ }145+146+ const cf = req.cf;147+ console.warn({148+ ...event,149+ sample_rate: effectiveSampleRate,150+ scanner_id: hex(digest.slice(0, 8)),151+ ...(cf?.country ? { country: cf.country } : {}),152+ ...(cf?.colo ? { colo: cf.colo } : {}),153+ ...(typeof cf?.asn === "number" ? { asn: cf.asn } : {}),154+ });155+ } catch (error) {156+ console.error({157+ event: "security.logging_failed",158+ error: error instanceof Error ? error.message : String(error),159+ });160+ }161+}@@ -1,5 +1,6 @@
11 import { parseAddress, parseManifest } from "../../shared/mod.ts";22 import { getSiteByAddress, type SiteRow } from "./db.ts";3+import { siteCacheTag } from "./cache.ts";34 import type { Env } from "./env.ts";45 import { FAVICON_PNG } from "./favicon.ts";56 import { hasValidUnlockCookie, makeUnlockCookie, verifyPassword } from "./password.ts";@@ -32,6 +33,20 @@ function htmlResponse(body: string, status: number, headers: HeadersInit = {}):
3233 });3334 }3435 36+function publicSiteHeaders(address: string): HeadersInit {37+ return {38+ "cache-control": "public, max-age=60",39+ "cache-tag": siteCacheTag(address),40+ };41+}42+43+function publicSiteRedirect(url: string, address: string): Response {44+ return new Response(null, {45+ status: 302,46+ headers: { location: url, ...publicSiteHeaders(address) },47+ });48+}49+3550 function unlockForm(address: string, error?: string): string {3651 return `<!doctype html><meta charset="utf-8"><meta name="viewport" content="width=device-width, initial-scale=1, viewport-fit=cover"><meta name="robots" content="noindex"><title>unlock — nzip</title>3752 <style>*{box-sizing:border-box}body{background:#121110;color:#d6cfc2;font-family:ui-monospace,monospace;display:grid;place-items:center;min-height:100vh;min-height:100dvh;margin:0;padding:max(24px,env(safe-area-inset-top)) max(16px,env(safe-area-inset-right)) max(24px,env(safe-area-inset-bottom)) max(16px,env(safe-area-inset-left))}form{width:min(100%,420px);text-align:center}b{color:#ffb347;font-size:24px;display:block;margin-bottom:20px}.field{display:flex;width:100%}input{background:#1a1815;border:1px solid #2e2a24;color:#d6cfc2;min-width:0;flex:1;padding:12px 14px;font:inherit;font-size:16px;line-height:20px;outline:none;border-radius:0}input:focus{border-color:#ffb347}button{background:#ffb347;border:0;color:#121110;min-height:46px;padding:12px 18px;font:inherit;font-weight:700;cursor:pointer;border-radius:0}.e{color:#f7768e;margin-top:12px;font-size:13px}@media(max-width:360px){.field{display:grid;gap:10px}button{width:100%}}</style>@@ -61,7 +76,9 @@ async function handleUnlock(req: Request, env: Env, site: SiteRow, address: stri
6176 export async function serve(req: Request, env: Env, url: URL): Promise<Response> {6277 const path = url.pathname;6378 64- if (path === "/") return htmlResponse(landingPage(env), 200);79+ if (path === "/") {80+ return htmlResponse(landingPage(env), 200, { "cache-control": "public, max-age=3600" });81+ }6582 // Served for the browsers' default probe; pushed sites under /{4hex}/ still6683 // control their own icons through their HTML. PNG bytes at the .ico path are6784 // accepted everywhere.@@ -72,7 +89,7 @@ export async function serve(req: Request, env: Env, url: URL): Promise<Response>
7289 }7390 if (path === "/robots.txt") {7491 return new Response("User-agent: *\nAllow: /\n", {75- headers: { "content-type": "text/plain" },92+ headers: { "content-type": "text/plain", "cache-control": "public, max-age=86400" },7693 });7794 }7895 @@ -109,7 +126,7 @@ export async function serve(req: Request, env: Env, url: URL): Promise<Response>
109126 let explicitIndexPath = false;110127 if (m[2] === undefined) {111128 if (filePaths.length !== 1) {112- return Response.redirect(new URL(`/${addressStr}/`, url).toString(), 302);129+ return publicSiteRedirect(new URL(`/${addressStr}/`, url).toString(), addressStr);113130 }114131 assetPath = filePaths[0];115132 } else {@@ -128,13 +145,13 @@ export async function serve(req: Request, env: Env, url: URL): Promise<Response>
128145 : path.slice(0, -"index.html".length);129146 const canonicalUrl = new URL(canonicalPath, url);130147 canonicalUrl.search = url.search;131- return Response.redirect(canonicalUrl.toString(), 302);148+ return publicSiteRedirect(canonicalUrl.toString(), addressStr);132149 }133150 134151 let entry = manifest.files[assetPath];135152 if (!entry && manifest.files[`${assetPath}/index.html`]) {136153 // Directory hit without trailing slash: redirect so relative URLs resolve.137- return Response.redirect(new URL(`${path}/`, url).toString(), 302);154+ return publicSiteRedirect(new URL(`${path}/`, url).toString(), addressStr);138155 }139156 if (!entry) return htmlResponse(NOT_FOUND_PAGE, 404);140157 @@ -144,7 +161,10 @@ export async function serve(req: Request, env: Env, url: URL): Promise<Response>
144161 // Never validate a previously cached protected response with 304: doing so145162 // would let the browser reuse its stored body after the site was locked.146163 if (!protectedSite && req.headers.get("if-none-match") === etag) {147- return new Response(null, { status: 304, headers: { etag, "cache-control": cacheControl } });164+ return new Response(null, {165+ status: 304,166+ headers: { etag, ...publicSiteHeaders(addressStr) },167+ });148168 }149169 150170 const blob = await env.CONTENT.get(`blob/${entry.h}`);@@ -158,6 +178,7 @@ export async function serve(req: Request, env: Env, url: URL): Promise<Response>
158178 // Public addresses are mutable on re-push. Protected content must never159179 // survive the password check in a browser or intermediary cache.160180 "cache-control": cacheControl,181+ ...(protectedSite ? {} : { "cache-tag": siteCacheTag(addressStr) }),161182 },162183 });163184 }@@ -0,0 +1,52 @@
1+import assert from "node:assert/strict";2+import test from "node:test";3+4+import { buildSecurityRequestEvent } from "../src/observability.ts";5+6+function eventFor(path, status, method = "GET") {7+ const url = new URL(path, "https://n.zip");8+ return buildSecurityRequestEvent(9+ new Request(url, { method }),10+ url,11+ new Response(null, { status }),12+ );13+}14+15+test("four-hex address probes include derived vault and site locations", () => {16+ assert.deepEqual(eventFor("/0123", 404), {17+ event: "security.request",18+ sample_rate: 0.01,19+ path_class: "address",20+ path: "/0123",21+ method: "GET",22+ status: 404,23+ result: "not_found",24+ address: "0123",25+ vault_slot: 0,26+ site_slot: 0x123,27+ });28+});29+30+test("scanner-shaped invalid paths are classified without an address", () => {31+ assert.deepEqual(eventFor("/.env", 404), {32+ event: "security.request",33+ sample_rate: 0.01,34+ path_class: "invalid",35+ path: "/.env",36+ method: "GET",37+ status: 404,38+ result: "not_found",39+ });40+});41+42+test("successful assets and service paths do not produce security events", () => {43+ assert.equal(eventFor("/0123/app.js", 200), null);44+ assert.equal(eventFor("/", 200), null);45+ assert.equal(eventFor("/favicon.ico", 200), null);46+});47+48+test("missing assets and unlock attempts retain their request class", () => {49+ assert.equal(eventFor("/0123/missing.js", 404)?.path_class, "address_asset");50+ assert.equal(eventFor("/0123/__unlock", 429, "POST")?.path_class, "unlock");51+ assert.equal(eventFor("/0123/__unlock", 429, "POST")?.result, "rate_limited");52+});@@ -52,6 +52,8 @@ test("single-file root index redirects to the bare site URL", async () => {
5252 );5353 5454 assert.equal(response.status, 302);55+ assert.equal(response.headers.get("cache-control"), "public, max-age=60");56+ assert.equal(response.headers.get("cache-tag"), "nzip-site-2f9b");5557 assert.equal(5658 response.headers.get("location"),5759 "https://n.zip/2f9b?view=full",@@ -92,6 +94,8 @@ test("canonical directory URL serves its index without redirecting", async () =>
9294 9395 assert.equal(response.status, 200);9496 assert.equal(response.headers.get("location"), null);97+ assert.equal(response.headers.get("cache-control"), "public, max-age=60");98+ assert.equal(response.headers.get("cache-tag"), "nzip-site-2f9b");9599 });96100 97101 test("missing index path still returns not found", async () => {@@ -3,6 +3,15 @@
33 "name": "nzip",44 "main": "src/index.ts",55 "compatibility_date": "2026-06-01",6+ "cache": { "enabled": true },7+ "observability": {8+ "enabled": true,9+ "logs": {10+ "enabled": true,11+ "head_sampling_rate": 1,12+ "invocation_logs": false13+ }14+ },615 "routes": [{ "pattern": "share.example.com", "custom_domain": true }],716 "r2_buckets": [817 {