1// What forge mirror puts in the offline copy (README.md, The pages mirror):2// the forge's own pages, less the ones there is one of per file. Those are3// most of www, and every page links the stylesheet by its hash, so each4// style change rewrote them all and the next sync uploaded them all again.5// The copy only has to keep the forge readable while it is down.6//7// in everything else: /, owner pages, topics, insights and their exports,8// /design/, /assets/, feeds, search indexes, and each repo's home,9// log, commits, reviews, releases, refs, sites, wiki, file lists10// out /<owner>/<repo>/blob/** (file pages)11// /<owner>/<repo>/tree/<branch>/<dir>/** (directory pages; each12// branch's top-level listing stays)13// /<owner>/<repo>/raw/** (images), except those a page in the copy14// shows, such as a README's15// /sites/**, /previews/**, /hosts/**, and *.old and *.tmp.* leftovers16// and /<owner>/<repo>/releases/download/** go in a pass of their own17//18// node lib/mirror.ts <www> the files in, one path per line, relative to www19import fs from 'node:fs';20import path from 'node:path';21import { fileURLToPath } from 'node:url';2223const SKIP_TOP = new Set(['sites', 'previews', 'hosts']);2425// Whether a directory (its path's segments) can hold anything that goes in.26function walkInto(s: string[]): boolean {27 const name = s[s.length - 1];28 if (name.endsWith('.old') || name.includes('.tmp.')) return false;29 if (s.length === 1) return !SKIP_TOP.has(name);30 if (s.length === 3) return name !== 'blob';31 if (s.length === 4 && s[2] === 'releases') return name !== 'download';32 if (s.length === 5 && s[2] === 'tree') return false;33 return true;34}3536// The URLs a page's attributes point to.37const ATTR = /\b(?:src|srcset|href)="([^"]+)"/g;38function urlsIn(html: string): string[] {39 const out: string[] = [];40 for (const m of html.matchAll(ATTR)) {41 for (const part of m[1].split(',')) out.push(part.trim().split(/\s+/)[0].replaceAll('&', '&'));42 }43 return out;44}4546function fromUrl(u: string): string | null {47 if (!u.startsWith('/') || u.startsWith('//')) return null;48 try {49 return u.replace(/[?#].*$/, '').slice(1).split('/').map(decodeURIComponent).join('/');50 } catch {51 return null;52 }53}5455export function mirrorFiles(www: string): string[] {56 const keep: string[] = [];57 const raw = new Set<string>();58 const walk = (s: string[]): void => {59 for (const e of fs.readdirSync(path.join(www, ...s), { withFileTypes: true })) {60 const t = [...s, e.name];61 // rclone skips symlinks, as it always has here.62 if (e.isDirectory()) {63 if (walkInto(t)) walk(t);64 } else if (e.isFile()) {65 if (t.length > 3 && t[2] === 'raw') raw.add(t.join('/'));66 else if (t.length === 5 && t[2] === 'tree' ? t[4] === 'index.html' : true) keep.push(t.join('/'));67 }68 }69 };70 walk([]);71 // The images the pages that go in show, a README's or a wiki page's.72 const shown = new Set<string>();73 for (const f of keep) {74 if (!f.endsWith('.html')) continue;75 for (const u of urlsIn(fs.readFileSync(path.join(www, f), 'utf8'))) {76 const p = fromUrl(u);77 if (p && raw.has(p)) shown.add(p);78 }79 }80 return [...keep, ...shown];81}8283// Run directly (realpath: /tmp is a symlink on macOS, and import.meta.url is resolved).84if (process.argv[1] && fs.realpathSync(process.argv[1]) === fileURLToPath(import.meta.url)) {85 const www = process.argv[2];86 if (!www) {87 process.stderr.write('usage: node lib/mirror.ts <www>\n');88 process.exit(2);89 }90 process.stdout.write(mirrorFiles(www).map((f) => f + '\n').join(''));91}