Each build writes /<owner>/<repo>/code.db, an SQLite FTS5 trigram index of the default branch's text files in chunks of eight lines, made with node:sqlite and kept in the cache by tree id: an unchanged tree reuses the file, a changed one starts from the last index and replaces only the files whose blobs changed. Binary, oversized, lock, minified and image files are left out. The browser reads it through a small read-only range VFS on the official @sqlite.org/sqlite-wasm, in a worker that loads the first time someone searches code. It fetches both ends of the file, then the term index's pages for the query's trigrams, then the matching chunks' pages, each step in parallel: a cold query is three round trips and some 250-400K on this repo, a warm one up to one. The overlay gains a Code section of path:line hits; queries under three characters can't use the index. That lifts the README's "file contents are not searched": the index is a static file, so it works from Caddy and from the pages mirror alike.
- commit
- c28b495df900408e9dab14f1a625f89c122bf904
- parent
- 63cbbf2b44d09b477a1bd426142b2d086f487f1e
- author
- cat <admin@dev.site>
- kind
- feat risk score 1.7 of 3, Jev's estimate
| M | eleventy.config.ts | 10 | + |
| M | lib/ | 29 | +- |
| A | lib/ | 213 | +++++++++++ |
| M | lib/ | 11 | +- |
| M | lib/ | 7 | +- |
| M | lib/ | 1 | + |
| M | lib/ | 2 | + |
| M | package-lock.json | 13 | + |
| M | package.json | 1 | + |
| M | site/ | 8 | +- |
| A | site/ | 603 | ++++++++++++++++++++++++++++++ |
| M | site/ | 220 | +++++++++-- |
| A | test/ | 191 | ++++++++++ |
| A | test/ | 34 | ++ |
| A | test/ | 21 | + |
| A | test/ | 44 | ++ |
| A | test/ | 23 | + |
@@ -38,10 +38,20 @@ export default function (cfg: EleventyConfig) {
3838 shipAssets(out, forge?.live ?? null, forge?.code ?? codeVersion());3939 if (!forge) return;4040 // Raw files (images, per-branch file lists) are bytes, not templates.41+ // Code indexes are hard links to the cache's (same filesystem), so an42+ // unchanged one stays the same file.4143 for (const f of forge.raw) {4244 const dst = path.join(out, f.path);4345 fs.mkdirSync(path.dirname(dst), { recursive: true });4446 if (f.data) fs.writeFileSync(dst, f.data);47+ else if (f.file) {48+ fs.rmSync(dst, { force: true });49+ try {50+ fs.linkSync(f.file, dst);51+ } catch {52+ fs.copyFileSync(f.file, dst);53+ }54+ }4555 }4656 // Unchanged pages: hard links to the live site's files (same filesystem).4757 let linked = 0;@@ -1,6 +1,7 @@
11 // Assets under content-hashed names, /assets/style.<hash>.css, so browsers22 // can cache them forever: the stylesheet, the logo, the vendored scripts, our3-// TypeScript with its types stripped, and mermaid. 11ty's bundle plugin3+// TypeScript with its types stripped, mermaid, and sqlite-wasm (its script4+// and its .wasm, for code search). 11ty's bundle plugin45 // writes the templates' own CSS next to them, also named by a hash.56 //67 // Pages from older builds keep linking older names: forge build swaps one@@ -16,6 +17,14 @@ import { fileURLToPath } from 'node:url';
1617 const root = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..');1718 const src = path.join(root, 'site', 'assets');1819 const MERMAID = path.join(root, 'node_modules', 'mermaid', 'dist', 'mermaid.min.js');20+const SQLITE = path.join(root, 'node_modules', '@sqlite.org', 'sqlite-wasm', 'dist');21+// Files from node_modules: short name, the name they ship as, the source.22+const VENDOR: [string, string, string][] = [23+ ['mermaid', 'mermaid.js', MERMAID],24+ // The browser build, an ES module that codesearch.ts imports in its worker.25+ ['sqlite3', 'sqlite3.js', path.join(SQLITE, 'index.mjs')],26+ ['sqlite3wasm', 'sqlite3.wasm', path.join(SQLITE, 'sqlite3.wasm')],27+];1928 const VERSIONS = 'versions.json';2029 // Code versions whose files stay: enough for a rebuild of every repo to finish2130 // and for a repo whose build fails to keep its styles for a few deploys.@@ -29,26 +38,30 @@ export interface Asset {
2938 data: Buffer;3039 }3140 32-// By short name: style, logo, fixi, ssexi, diffs, keys, live, search, ui, mermaid.41+// By short name: style, logo, fixi, ssexi, diffs, keys, live, search,42+// codesearch, ui, mermaid, sqlite3, sqlite3wasm.3343 export type Assets = Record<string, Asset>;3444 3545 let memo: { key: string; assets: Assets } | null = null;3646 3747 export function assets(): Assets {38- const files = [...fs.readdirSync(src).filter((f) => /\.(css|svg|js|ts)$/.test(f)).sort().map((f) => path.join(src, f)), MERMAID];48+ const files: [string, string, string][] = [49+ ...fs.readdirSync(src).filter((f) => /\.(css|svg|js|ts)$/.test(f)).sort().map((f): [string, string, string] => [f.split('.')[0], f, path.join(src, f)]),50+ ...VENDOR,51+ ];3952 // Rebuilt only when a source changes (the dev server rebuilds in one process).40- const key = files.map((f) => `${f} ${fs.statSync(f).mtimeMs}`).join('\n');53+ const key = files.map(([, , f]) => `${f} ${fs.statSync(f).mtimeMs}`).join('\n');4154 if (memo?.key === key) return memo.assets;4255 const out: Assets = {};43- for (const f of files) {56+ for (const [short, base, f] of files) {4457 let data = fs.readFileSync(f);45- let ext = path.extname(f);46- const name = path.basename(f).split('.')[0];58+ let ext = path.extname(base);59+ const name = base.split('.')[0];4760 if (ext === '.ts') {4861 data = Buffer.from(stripTypeScriptTypes(data.toString('utf8')));4962 ext = '.js';5063 }51- out[name] = { file: `${name}.${hash(data)}${ext}`, data };64+ out[short] = { file: `${name}.${hash(data)}${ext}`, data };5265 }5366 memo = { key, assets: out };5467 return out;@@ -0,0 +1,213 @@
1+// The code index: every text file on a repo's default branch in a SQLite FTS52+// trigram index, /<owner>/<repo>/code.db, which the browser queries over HTTP3+// range requests (site/assets/codesearch.ts) without downloading it.4+//5+// file(id, path, source) source: line anchors are on the /source/ page6+// chunk(id, file, line, text) CHUNK lines of a file from line `line`7+// code fts5 over chunk.text: trigram, detail=full8+//9+// Rows are chunks of lines rather than lines: an index of lines is about the10+// same size but has eight times the rows, and the browser finds the matching11+// line inside a chunk itself. detail=full keeps positions, which substring12+// (phrase) queries need; columnsize=0 drops the docsize table that only13+// ranking reads (results come in path order). After the inserts, 'optimize'14+// merges the index into one segment and VACUUM lays each table out in order,15+// so a doclist or a run of chunks is a run of pages, one range request.16+//17+// Skipped: binary files (a NUL in the first 8000 bytes, as git decides),18+// files over the highlight cap, lockfiles, minified files and source maps,19+// images, symlinks and submodules.20+//21+// Each index is kept in $FORGE_CACHE/code/<owner>/<repo>/<tree>.<key>.db and22+// hard-linked into the pages, so a push that leaves the default branch's tree23+// alone reuses the file. A new tree starts from the previous index: chunks of24+// files whose blob is unchanged are kept, the rest are deleted and inserted.25+import fs from 'node:fs';26+import path from 'node:path';27+import { createHash } from 'node:crypto';28+import { createRequire } from 'node:module';29+import type { DatabaseSync } from 'node:sqlite';30+import { isBinary } from './git.ts';31+import { MAX_HIGHLIGHT } from './highlight.ts';32+import { IMAGE } from './repo.ts';33+import type { Config, Repo, TreeEntry } from './types.ts';34+35+// Bump when the schema or what gets indexed changes.36+const VERSION = 1;37+export const CHUNK = 8;38+// 16K pages keep every table at two levels (its root, then leaves) up to39+// some 25M, so the browser can tell which page a row is on from the root.40+const PAGE = 16384;41+const MARKDOWN = /\.(md|markdown|mdown|mkd)$/i;42+const LOCKFILES = new Set([43+ 'package-lock.json', 'npm-shrinkwrap.json', 'yarn.lock', 'pnpm-lock.yaml', 'bun.lock', 'deno.lock', 'composer.lock', 'gemfile.lock',44+ 'cargo.lock', 'go.sum', 'poetry.lock', 'pipfile.lock', 'uv.lock', 'pdm.lock', 'flake.lock', 'mix.lock', 'podfile.lock', 'packages.lock.json',45+ 'pubspec.lock', 'gradle.lockfile', 'package.resolved',46+]);47+const NOISE = /\.(min\.(js|css|mjs)|map)$/i;48+49+const require = createRequire(import.meta.url);50+let sqlite: typeof import('node:sqlite') | null = null;51+// node:sqlite warns that it is experimental on first use; builds stay quiet.52+function db(file: string): DatabaseSync {53+ if (!sqlite) {54+ const emit = process.emitWarning;55+ process.emitWarning = ((w: string | Error, ...rest: never[]) => (/SQLite/.test(String(w)) ? undefined : emit.call(process, w, ...rest))) as typeof emit;56+ try {57+ sqlite = require('node:sqlite') as typeof import('node:sqlite');58+ } finally {59+ process.emitWarning = emit;60+ }61+ }62+ return new sqlite.DatabaseSync(file);63+}64+65+export type Skip = 'binary' | 'large' | 'lockfile' | 'generated' | 'image' | 'link';66+67+// Why a file isn't indexed, from its tree entry alone; binary needs the bytes.68+export function skipEntry(e: TreeEntry): Skip | null {69+ if (e.mode === '120000' || e.mode === '160000') return 'link';70+ if ((e.size ?? Infinity) > MAX_HIGHLIGHT) return 'large';71+ const base = e.name.toLowerCase();72+ if (LOCKFILES.has(base)) return 'lockfile';73+ if (NOISE.test(base)) return 'generated';74+ if (IMAGE.test(base)) return 'image';75+ return null;76+}77+78+export interface CodeStats {79+ files: number;80+ chunks: number;81+ bytes: number;82+ reused: number;83+ skipped: Partial<Record<Skip, number>>;84+ ms: number;85+}86+87+// One file's place in the index, so the next tree can keep its chunks.88+type Layout = Record<string, [oid: string, id: number, first: number, count: number]>;89+90+const key = (cfg: Config): string => createHash('sha1').update(`${VERSION} ${CHUNK} ${PAGE} ${cfg.maxFiles}`).digest('hex').slice(0, 8);91+92+// The index of the default branch's tree, built if it isn't cached, or null93+// when the branch has no tree. Older indexes of the repo are removed (pages94+// still being swapped out keep theirs: they are hard links).95+export function codeIndex(cfg: Config, repo: Repo): { file: string; stats: CodeStats | null } | null {96+ const tree = repo.trees.get(repo.defaultBranch);97+ const oid = tree?.dirOids.get('');98+ if (!tree || !oid) return null;99+ const dir = path.join(cfg.cacheDir, 'code', repo.slug);100+ const name = `${oid}.${key(cfg)}`;101+ const file = path.join(dir, `${name}.db`);102+ if (fs.existsSync(file)) return { file, stats: null };103+ fs.mkdirSync(dir, { recursive: true });104+ const old = fs.readdirSync(dir).filter((f) => f.endsWith('.db') && f !== `${name}.db`);105+ // The newest earlier index of this schema, to start from.106+ const base = old.filter((f) => f.endsWith(`.${key(cfg)}.db`) && fs.existsSync(path.join(dir, f.replace(/\.db$/, '.json'))))107+ .map((f) => ({ f, t: fs.statSync(path.join(dir, f)).mtimeMs })).sort((a, b) => b.t - a.t)[0]?.f;108+ const tmp = `${file}.${process.pid}`;109+ const stats = build(repo, tree.blobs, tmp, base ? path.join(dir, base) : null);110+ fs.writeFileSync(file.replace(/\.db$/, '.json'), JSON.stringify(stats.layout));111+ fs.renameSync(tmp, file);112+ // Earlier indexes, and scratch files of builds that died.113+ for (const f of fs.readdirSync(dir)) if (f !== `${name}.db` && f !== `${name}.json`) fs.rmSync(path.join(dir, f), { force: true });114+ return { file, stats: stats.stats };115+}116+117+function build(repo: Repo, entries: TreeEntry[], out: string, base: string | null): { stats: CodeStats; layout: Layout } {118+ const t0 = performance.now();119+ const stats: CodeStats = { files: 0, chunks: 0, bytes: 0, reused: 0, skipped: {}, ms: 0 };120+ const skip = (why: Skip): void => void (stats.skipped[why] = (stats.skipped[why] ?? 0) + 1);121+ const want = entries.filter((e) => {122+ const why = skipEntry(e);123+ if (why) skip(why);124+ return !why;125+ });126+ // Unchanged files keep their rows when there is an earlier index to start from.127+ let prev: Layout = {};128+ fs.rmSync(out, { force: true });129+ if (base) {130+ try {131+ prev = JSON.parse(fs.readFileSync(base.replace(/\.db$/, '.json'), 'utf8'));132+ fs.copyFileSync(base, out);133+ } catch {134+ prev = {};135+ fs.rmSync(out, { force: true });136+ }137+ }138+ const layout: Layout = {};139+ const d = db(out);140+ try {141+ if (!fs.statSync(out).size) {142+ d.exec(`pragma page_size = ${PAGE};143+ create table file(id integer primary key, path text not null, source integer not null);144+ create table chunk(id integer primary key, file integer not null, line integer not null, text text not null);145+ create virtual table code using fts5(text, content='chunk', content_rowid='id', tokenize='trigram', detail=full, columnsize=0);`);146+ }147+ // A scratch file until it is renamed into place: no journal, no syncs.148+ d.exec('pragma journal_mode = off; pragma synchronous = off; pragma temp_store = memory; begin');149+ const keep = new Set(want.filter((e) => prev[e.path]?.[0] === e.oid).map((e) => e.path));150+ // Rows of files that changed or went: out of the index, then the tables.151+ const delFts = d.prepare(`insert into code(code, rowid, text) select 'delete', id, text from chunk where file = ?`);152+ const delChunks = d.prepare('delete from chunk where file = ?');153+ const delFile = d.prepare('delete from file where id = ?');154+ for (const [p, [, id]] of Object.entries(prev)) {155+ if (keep.has(p)) continue;156+ delFts.run(id);157+ delChunks.run(id);158+ delFile.run(id);159+ }160+ // New rows go after the kept ones; ids are never reused.161+ let nextFile = Math.max(0, ...Object.values(prev).map((v) => v[1])) + 1;162+ let nextChunk = Math.max(0, ...Object.values(prev).map((v) => v[2] + v[3] - 1)) + 1;163+ const insFile = d.prepare('insert into file(id, path, source) values (?, ?, ?)');164+ const insChunk = d.prepare('insert into chunk(id, file, line, text) values (?, ?, ?, ?)');165+ const insFts = d.prepare('insert into code(rowid, text) values (?, ?)');166+ for (const e of want) {167+ if (!keep.has(e.path)) continue;168+ layout[e.path] = prev[e.path];169+ stats.bytes += e.size ?? 0;170+ }171+ stats.reused = stats.files = keep.size;172+ const todo = want.filter((e) => !keep.has(e.path));173+ // Read blobs in batches so a big tree never sits in memory at once.174+ for (let i = 0; i < todo.length;) {175+ const batch: TreeEntry[] = [];176+ for (let size = 0; i < todo.length && size < 32 << 20; i++) {177+ batch.push(todo[i]);178+ size += todo[i].size ?? 0;179+ }180+ const blobs = repo.loadBlobs(batch.map((e) => e.oid));181+ for (const e of batch) {182+ const data = blobs.get(e.oid)?.data;183+ if (!data) continue;184+ if (isBinary(data)) {185+ skip('binary');186+ continue;187+ }188+ const lines = data.toString('utf8').split('\n');189+ if (lines.at(-1) === '') lines.pop();190+ const id = nextFile++;191+ const first = nextChunk;192+ insFile.run(id, e.path, MARKDOWN.test(e.path) ? 1 : 0);193+ for (let l = 0; l < lines.length; l += CHUNK) {194+ const text = lines.slice(l, l + CHUNK).join('\n');195+ insChunk.run(nextChunk, id, l + 1, text);196+ insFts.run(nextChunk, text);197+ nextChunk++;198+ }199+ layout[e.path] = [e.oid, id, first, nextChunk - first];200+ stats.files++;201+ stats.bytes += data.length;202+ }203+ }204+ d.exec('commit');205+ d.exec(`insert into code(code) values ('optimize')`);206+ d.exec('vacuum');207+ stats.chunks = (d.prepare('select count(*) n from chunk').get() as { n: number }).n;208+ } finally {209+ d.close();210+ }211+ stats.ms = Math.round(performance.now() - t0);212+ return { stats, layout };213+}@@ -17,6 +17,7 @@ import * as html from './render.ts';
1717 import { Spans, build, codeVersion } from './metrics.ts';1818 import { RepoCache } from './cache.ts';1919 import { repoIndex, reposIndex } from './search.ts';20+import { codeIndex } from './code-index.ts';2021 import { labelCommits, labelKey, releases, repoTopic } from './jev-commits.ts';2122 import { annotateReviews, answersStamp, isMechanical, pairChanges } from './jev-reviews.ts';2223 import { HighlightCache } from './highlight.ts';@@ -106,6 +107,8 @@ export interface RawFile {
106107 path: string;107108 oid?: string;108109 data?: Buffer;110+ // A file to link instead (a code index in the cache).111+ file?: string;109112 }110113 111114 export interface Forge {@@ -337,7 +340,11 @@ export function loadForge(env: NodeJS.ProcessEnv = process.env): Forge {
337340 for (const w of repo.wiki) {338341 add(pages.wiki, page(u.wiki(w.slug).path, `${w.title} - wiki - ${repo.slug}`, 'wiki', { wiki: w, pages: repo.wiki.map((x) => ({ title: x.title, slug: x.slug, href: u.wiki(x.slug).href })) }));339342 }340- const index = repoIndex(repo);343+ // The default branch's code index (lib/code-index.ts), kept by tree id.344+ const codeDb = spans.time('code', () => codeIndex(cfg, repo));345+ if (codeDb) raw.push({ path: u.code.path, file: codeDb.file });346+ if (codeDb?.stats) spans.count('code indexed', codeDb.stats.files - codeDb.stats.reused);347+ const index = repoIndex(repo, codeDb ? u.code.href : null);341348 if (want(u.search.path, `search ${fingerprint(index)}`)) raw.push({ path: u.search.path, data: Buffer.from(index) });342349 343350 // The heavy reads, for exactly the pages that will render.@@ -362,7 +369,7 @@ export function loadForge(env: NodeJS.ProcessEnv = process.env): Forge {
362369 const owners = ownerNames.map((owner) => ({ owner, repos: summaries.filter((s) => s.owner === owner) }));363370 // Repo search on the index and owner pages.364371 raw.push({ path: '/repos.json', data: Buffer.from(reposIndex(summaries)) });365- last = { config: cfg, repos: summaries, owners, pages, raw: raw.filter((r) => r.data), live, code, skipped, pageCaches };372+ last = { config: cfg, repos: summaries, owners, pages, raw: raw.filter((r) => r.data || r.file), live, code, skipped, pageCaches };366373 return last;367374 }368375 @@ -2,7 +2,8 @@
22 // time the search opens. Per repo: commits, reviews and wiki pages; file paths33 // come from the default branch's file list, already emitted for the finder.44 // For the forge: repo names and descriptions. Bodies and wiki text are capped5-// to keep the files small, and file contents aren't indexed at all.5+// to keep the files small. File contents are in the code index6+// (lib/code-index.ts), which `code` points at when the default branch has one.67 import type { Repo, RepoSummary } from './types.ts';78 89 const BODY = 240;@@ -17,7 +18,7 @@ function plain(w: { html: string; title: string }): string {
1718 return (text.startsWith(w.title) ? text.slice(w.title.length) : text).trim();1819 }1920 20-export function repoIndex(repo: Repo): string {21+export function repoIndex(repo: Repo, code: string | null = null): string {2122 const u = repo.urls;2223 const commits = [...repo.commits.values()]2324 .sort((a, b) => Date.parse(b.author.date) - Date.parse(a.author.date))@@ -25,7 +26,7 @@ export function repoIndex(repo: Repo): string {
2526 const reviews = [...repo.reviews].reverse().map((r) => [r.id, r.title, r.branch, r.author ?? '', r.status]);2627 const wiki = repo.wiki.map((w) => [u.wiki(w.slug).href, w.title, plain(w).slice(0, WIKI)]);2728 const files = repo.trees.has(repo.defaultBranch) ? u.files(repo.defaultBranch).href : null;28- return JSON.stringify({ blob: u.blobBase(repo.defaultBranch).href, files, commits, reviews, wiki, home: u.home.href });29+ return JSON.stringify({ blob: u.blobBase(repo.defaultBranch).href, files, code, commits, reviews, wiki, home: u.home.href });2930 }3031 3132 export function reposIndex(repos: RepoSummary[]): string {@@ -45,6 +45,8 @@ export function repoUrls(cfg: Config, owner: string, name: string): RepoUrls {
4545 files: (b) => both(`${base}files/${encRef(b)}.txt`),4646 // The search index (lib/search.ts), fetched when the search opens.4747 search: both(`${base}search.json`),48+ // The default branch's code index (lib/code-index.ts), read by range requests.49+ code: both(`${base}code.db`),4850 commit: (sha) => both(`${base}commit/${sha}/`),4951 refs: both(`${base}refs/`),5052 // Branch lists, fetched on demand so a new branch doesn't change every page.@@ -7,6 +7,7 @@
77 "name": "forge",88 "dependencies": {99 "@11ty/eleventy": "^3.1.7",10+ "@sqlite.org/sqlite-wasm": "^3.53.4-build2",1011 "@typesafe-ai/sdk": "^0.7.0",1112 "highlight.js": "^11.12.0",1213 "markdown-it": "^14.1.0",@@ -329,6 +330,18 @@
329330 "url": "https://github.com/sponsors/sindresorhus"330331 }331332 },333+ "node_modules/@sqlite.org/sqlite-wasm": {334+ "version": "3.53.4-build2",335+ "resolved": "https://registry.npmjs.org/@sqlite.org/sqlite-wasm/-/sqlite-wasm-3.53.4-build2.tgz",336+ "integrity": "sha512-kQVArGJA4z7s56Dm8t+eTBXad4LiohuVj9pfgDNOcxa+Ic5xMrWlLPTq6cT0hw5S4MIvtpsraW+B0wD+IPvC7A==",337+ "license": "Apache-2.0",338+ "workspaces": [339+ "demos/*"340+ ],341+ "engines": {342+ "node": ">=22"343+ }344+ },332345 "node_modules/@types/d3": {333346 "version": "7.4.3",334347 "resolved": "https://registry.npmjs.org/@types/d3/-/d3-7.4.3.tgz",@@ -1,5 +1,7 @@
11 {# The header's search button. It and "/" open the search overlay (search.ts);2- "t" opens the find-file overlay (keys.ts), which looks the same. -#}2+ "t" opens the find-file overlay (keys.ts), which looks the same. In a repo3+ it names the code search worker and sqlite-wasm, which load only when4+ someone searches file contents. -#}35 {% css %}46 /* search: the header button, and the overlay it and "/" open */57 .search-open { display: inline-flex; align-items: center; gap: 6px; flex: none; background: var(--bg); color: var(--muted); font-size: 0.9em; padding: 4px 9px; }@@ -16,9 +18,11 @@
1618 #search li.sel a, #search li.sel button { background: var(--hover); box-shadow: inset 2px 0 0 var(--accent); }1719 #search .t, #search .m { display: block; overflow: hidden; text-overflow: ellipsis; white-space: nowrap; }1820 #search .t.mono { font: 0.9em var(--mono); }21+#search .m.mono { font-family: var(--mono); white-space: pre; }1922 #search .m { font-size: 0.82em; color: var(--muted); }2023 #search b { color: var(--link); font-weight: 600; }2124 #search li.more button { border: 0; background: none; color: var(--link); padding: 3px 8px; width: 100%; text-align: left; border-radius: 4px; }25+#search li.note { padding: 5px 8px; font-size: 0.85em; color: var(--muted); }2226 #search .hint { margin: 8px 8px 2px; font-size: 0.85em; color: var(--muted); }2327 #search .hint:empty { display: none; }2428 /* file finder */@@ -29,4 +33,4 @@
2933 #finder li.sel a { background: var(--hover); }3034 #finder li b { color: var(--link); }3135 {% endcss %}32-<button type="button" class="search-open" hidden data-search="{{ r.urls.search.href if r else '/repos.json' }}"{% if o and o.owner %} data-owner="{{ o.owner }}"{% endif %} title="Search (/)" aria-label="Search"><svg viewBox="0 0 16 16" width="15" height="15" aria-hidden="true"><circle cx="6.8" cy="6.8" r="4.6" fill="none" stroke="currentColor" stroke-width="1.6"/><path d="M10.3 10.3 14 14" stroke="currentColor" stroke-width="1.6" stroke-linecap="round"/></svg><kbd class="hide-sm">/</kbd></button>36+<button type="button" class="search-open" hidden data-search="{{ r.urls.search.href if r else '/repos.json' }}"{% if o and o.owner %} data-owner="{{ o.owner }}"{% endif %}{% if r %} data-code="{{ assets.codesearch }}" data-sqlite="{{ assets.sqlite3 }}" data-wasm="{{ assets.sqlite3wasm }}"{% endif %} title="Search (/)" aria-label="Search"><svg viewBox="0 0 16 16" width="15" height="15" aria-hidden="true"><circle cx="6.8" cy="6.8" r="4.6" fill="none" stroke="currentColor" stroke-width="1.6"/><path d="M10.3 10.3 14 14" stroke="currentColor" stroke-width="1.6" stroke-linecap="round"/></svg><kbd class="hide-sm">/</kbd></button>@@ -0,0 +1,603 @@
1+// Code search: a repo's code index (lib/code-index.ts) is a SQLite FTS52+// trigram index of the default branch, a static file the browser never3+// downloads whole. sqlite-wasm opens it through a read-only VFS whose reads4+// are HTTP range requests, kept for the session, so a query fetches only the5+// pages it touches. search.ts starts this file as a module worker the first6+// time someone searches code; test/code-search-bench.ts imports it in Node to7+// measure the same code path.8+//9+// SQLite reads synchronously, and a worker may block on XHR, so each page10+// SQLite misses is a request and a round trip. Most of a query's pages can be11+// named before SQLite asks for them, though, and those are fetched at once:12+// the index leaf of each term (from FTS5's small %_idx table), then the page13+// of each matching chunk (from the chunk table's root page).14+15+// Bytes [from, to) of url (from < 0: the last -from bytes), or the whole16+// file if the server ignores ranges; where they start, the file's size, and17+// its validator, which must not change while it is open.18+export interface Ranged {19+ whole: boolean;20+ data: Uint8Array;21+ start: number;22+ size: number;23+ tag: string;24+}25+export type GetRange = (url: string, from: number, to: number) => Ranged;26+export type FetchRange = (url: string, from: number, to: number) => Promise<Ranged>;27+28+const bytes = (from: number, to: number): string => (from < 0 ? `bytes=${from}` : `bytes=${from}-${to - 1}`);29+30+function ranged(url: string, status: number, header: (h: string) => string | null, data: Uint8Array): Ranged {31+ if (status !== 200 && status !== 206) throw new Error(`${url}: HTTP ${status}`);32+ // Ranges of a compressed body aren't ranges of the file.33+ const enc = header('Content-Encoding');34+ if (enc && enc !== 'identity') throw new Error(`${url} is served with Content-Encoding: ${enc}`);35+ const whole = status === 200;36+ const m = /(\d+)-\d+\/(\d+)\s*$/.exec(header('Content-Range') ?? '');37+ if (!whole && !m) throw new Error(`${url}: no Content-Range`);38+ return { whole, data, start: whole ? 0 : Number(m![1]), size: whole ? data.length : Number(m![2]), tag: header('ETag') ?? header('Last-Modified') ?? '' };39+}40+41+// What SQLite's reads cost: a blocking XHR, which only workers may make.42+export const xhrRange: GetRange = (url, from, to) => {43+ const x = new XMLHttpRequest();44+ x.open('GET', url, false);45+ x.responseType = 'arraybuffer';46+ x.setRequestHeader('Range', bytes(from, to));47+ x.send();48+ return ranged(url, x.status, (h) => x.getResponseHeader(h), new Uint8Array(x.response as ArrayBuffer));49+};50+51+// What prefetches cost: fetches, several at a time.52+export const fetchRange: FetchRange = async (url, from, to) => {53+ const r = await fetch(url, { headers: { Range: bytes(from, to) } });54+ return ranged(url, r.status, (h) => r.headers.get(h), new Uint8Array(await r.arrayBuffer()));55+};56+57+// The file is read in blocks of one page (lib/code-index.ts writes 16K58+// pages). A read that continues the last one doubles the next, up to 3259+// blocks, since FTS5 walks long doclists in page order. Opening reads both60+// ends at once: 8 blocks of head (the schema, every table's root page and61+// the file table, which VACUUM puts first) and 4 of tail (FTS5's %_idx,62+// which it puts last). An index of up to 256K is read whole.63+export const BLOCK = 16 << 10;64+const RUN = 32;65+const HEAD = 8;66+const TAIL = 4;67+const WHOLE = 256 << 10;68+69+// The rebuilt index replaced the file under an open database.70+export class Changed extends Error {}71+72+export class RangeFile {73+ url: string;74+ get: GetRange;75+ getAsync: FetchRange;76+ block: number;77+ size = -1;78+ tag = '';79+ // HTTP requests, bytes, and round trips (a batch of prefetches is one).80+ requests = 0;81+ bytes = 0;82+ rounds = 0;83+ blocks = new Map<number, Uint8Array>();84+ whole: Uint8Array | null = null;85+ private next = -1;86+ private run = 1;87+88+ constructor(url: string, get: GetRange, getAsync: FetchRange, block = BLOCK) {89+ this.url = url;90+ this.get = get;91+ this.getAsync = getAsync;92+ this.block = block;93+ }94+95+ // Both ends of the file, which also say how big it is.96+ async start(): Promise<void> {97+ if (this.size >= 0) return;98+ this.rounds++;99+ const [head, tail] = await Promise.all([this.getAsync(this.url, 0, HEAD * this.block), this.getAsync(this.url, -TAIL * this.block, 0)]);100+ this.keep(head);101+ if (!head.whole) this.keep(tail);102+ await this.rest();103+ }104+105+ // Without start(), just the head.106+ open(): void {107+ if (this.size >= 0) return;108+ this.fetch(0, HEAD);109+ }110+111+ // A small index, whole.112+ private async rest(): Promise<void> {113+ if (this.whole || this.size > WHOLE) return;114+ const n = Math.ceil(this.size / this.block);115+ await this.prefetch(Array.from({ length: n }, (_, i) => i * this.block));116+ }117+118+ private keep(r: Ranged): void {119+ this.requests++;120+ this.bytes += r.data.length;121+ if (this.tag && r.tag !== this.tag) throw new Changed(`${this.url} changed`);122+ this.tag = r.tag;123+ this.size = r.size;124+ if (r.whole) {125+ this.whole = r.data;126+ return;127+ }128+ // Whole blocks only: a tail may start inside one.129+ let i = (this.block - (r.start % this.block)) % this.block;130+ for (; i < r.data.length; i += this.block) this.blocks.set((r.start + i) / this.block, r.data.subarray(i, i + this.block));131+ }132+133+ private span(first: number, count: number): [number, number] {134+ const from = first * this.block;135+ return [from, this.size < 0 ? from + count * this.block : Math.min(from + count * this.block, this.size)];136+ }137+138+ private fetch(first: number, count: number): void {139+ this.rounds++;140+ this.keep(this.get(this.url, ...this.span(first, count)));141+ this.next = first + count;142+ }143+144+ has(offset: number): boolean {145+ return !!this.whole || this.blocks.has(Math.floor(offset / this.block));146+ }147+148+ // Fetch the blocks holding these offsets, all at once.149+ async prefetch(offsets: number[]): Promise<void> {150+ const want = [...new Set(offsets.filter((o) => o >= 0 && o < this.size && !this.has(o)).map((o) => Math.floor(o / this.block)))].sort((a, b) => a - b);151+ if (!want.length) return;152+ // Neighbouring blocks go in one request.153+ const runs: [number, number][] = [];154+ for (const b of want) {155+ const r = runs.at(-1);156+ if (r && r[0] + r[1] === b) r[1]++;157+ else runs.push([b, 1]);158+ }159+ this.rounds++;160+ const got = await Promise.all(runs.map(([b, n]) => this.getAsync(this.url, ...this.span(b, n))));161+ for (const r of got) this.keep(r);162+ }163+164+ // Fill dest from offset; returns how many bytes there were.165+ read(dest: Uint8Array, offset: number): number {166+ this.open();167+ const end = Math.min(offset + dest.length, this.size);168+ if (this.whole) {169+ dest.set(this.whole.subarray(offset, Math.max(offset, end)));170+ return Math.max(0, end - offset);171+ }172+ const last = Math.floor((end - 1) / this.block);173+ for (let b = Math.floor(offset / this.block); b <= last; b++) {174+ if (!this.blocks.has(b)) {175+ let n = 1;176+ while (b + n <= last && !this.blocks.has(b + n)) n++;177+ this.run = b === this.next ? Math.min(this.run * 2, RUN) : 1;178+ this.fetch(b, Math.max(n, this.run));179+ }180+ const data = this.blocks.get(b);181+ if (!data) throw new Error(`${this.url}: no bytes at ${b * this.block}`);182+ const start = b * this.block;183+ const lo = Math.max(offset, start);184+ const hi = Math.min(end, start + data.length);185+ dest.set(data.subarray(lo - start, hi - start), lo - offset);186+ }187+ return Math.max(0, end - offset);188+ }189+}190+191+// The parts of sqlite-wasm this file uses.192+type Ptr = number | bigint;193+interface Struct {194+ pointer: Ptr;195+ $iVersion: number;196+ $szOsFile: number;197+ $mxPathname: number;198+ $pMethods: Ptr;199+ $xRandomness: Ptr;200+ $xSleep: Ptr;201+ dispose(): void;202+}203+interface Stmt {204+ bind(values: unknown[]): Stmt;205+ step(): boolean;206+ get(row: unknown[]): unknown[];207+ finalize(): void;208+}209+export interface Db {210+ prepare(sql: string): Stmt;211+ close(): void;212+}213+export interface Sqlite3 {214+ capi: Record<string, number> & {215+ sqlite3_vfs: new (p?: Ptr) => Struct;216+ sqlite3_io_methods: new () => Struct;217+ sqlite3_file: (new (p: Ptr) => Struct) & { structInfo: { sizeof: number } };218+ sqlite3_vfs_find(name: string | null): Ptr;219+ };220+ wasm: {221+ heap8u(): Uint8Array;222+ poke32(p: Ptr, v: number): void;223+ poke64(p: Ptr, v: bigint): void;224+ poke(p: Ptr, v: number, type: string): void;225+ cstrToJs(p: Ptr): string;226+ cstrncpy(to: Ptr, from: Ptr, n: number): number;227+ };228+ vfs: { installVfs(opt: { io?: { struct: Struct; methods: object }; vfs?: { struct: Struct; methods: object; name?: string } }): void };229+ oo1: { DB: new (opt: { filename: string; flags: string; vfs: string }) => Db };230+}231+232+export const VFS = 'forge-range';233+234+// The files SQLite may open, by name, and the last error a read hit.235+export interface RangeVfs {236+ files: Map<string, RangeFile>;237+ error(): unknown;238+}239+240+// A read-only VFS over RangeFiles. Nothing else exists: no journal, no WAL,241+// and the file says it is immutable, so SQLite takes no locks and checks for242+// neither.243+export function installVfs(sqlite3: Sqlite3): RangeVfs {244+ const { capi, wasm } = sqlite3;245+ const files = new Map<string, RangeFile>();246+ const open = new Map<number, RangeFile>();247+ let last: unknown = null;248+ const io = new capi.sqlite3_io_methods();249+ io.$iVersion = 1;250+ const readOnly = (): number => capi.SQLITE_READONLY;251+ sqlite3.vfs.installVfs({252+ io: {253+ struct: io,254+ methods: {255+ xClose: (f: Ptr) => (open.delete(Number(f)), 0),256+ xRead(f: Ptr, dest: Ptr, n: number, at: number | bigint): number {257+ const out = wasm.heap8u().subarray(Number(dest), Number(dest) + n);258+ try {259+ const got = open.get(Number(f))!.read(out, Number(at));260+ if (got === n) return 0;261+ out.fill(0, got);262+ return capi.SQLITE_IOERR_SHORT_READ;263+ } catch (e) {264+ last = e;265+ return capi.SQLITE_IOERR_READ;266+ }267+ },268+ xWrite: readOnly,269+ xTruncate: readOnly,270+ xSync: () => 0,271+ xFileSize(f: Ptr, out: Ptr): number {272+ const file = open.get(Number(f))!;273+ try {274+ file.open();275+ } catch (e) {276+ last = e;277+ return capi.SQLITE_IOERR_FSTAT;278+ }279+ wasm.poke64(out, BigInt(file.size));280+ return 0;281+ },282+ xLock: () => 0,283+ xUnlock: () => 0,284+ xCheckReservedLock: (_f: Ptr, out: Ptr) => (wasm.poke32(out, 0), 0),285+ xFileControl: () => capi.SQLITE_NOTFOUND,286+ xSectorSize: () => 4096,287+ xDeviceCharacteristics: () => capi.SQLITE_IOCAP_IMMUTABLE,288+ },289+ },290+ });291+ const vfs = new capi.sqlite3_vfs();292+ const base = capi.sqlite3_vfs_find(null);293+ vfs.$iVersion = 2;294+ vfs.$szOsFile = capi.sqlite3_file.structInfo.sizeof;295+ vfs.$mxPathname = 1024;296+ if (base) {297+ const d = new capi.sqlite3_vfs(base);298+ vfs.$xRandomness = d.$xRandomness;299+ vfs.$xSleep = d.$xSleep;300+ d.dispose();301+ }302+ sqlite3.vfs.installVfs({303+ vfs: {304+ struct: vfs,305+ name: VFS,306+ methods: {307+ xOpen(_v: Ptr, name: Ptr, f: Ptr, _flags: number, outFlags: Ptr): number {308+ const file = name ? files.get(wasm.cstrToJs(name)) : undefined;309+ if (!file) return capi.SQLITE_CANTOPEN;310+ open.set(Number(f), file);311+ const s = new capi.sqlite3_file(f);312+ s.$pMethods = io.pointer;313+ s.dispose();314+ if (outFlags) wasm.poke32(outFlags, capi.SQLITE_OPEN_READONLY);315+ return 0;316+ },317+ xDelete: readOnly,318+ xAccess: (_v: Ptr, _name: Ptr, _flags: number, out: Ptr) => (wasm.poke32(out, 0), 0),319+ xFullPathname: (_v: Ptr, name: Ptr, n: number, out: Ptr) => (wasm.cstrncpy(out, name, n) < n ? 0 : capi.SQLITE_CANTOPEN),320+ xGetLastError: () => 0,321+ xCurrentTime: (_v: Ptr, out: Ptr) => (wasm.poke(out, 2440587.5 + Date.now() / 864e5, 'double'), 0),322+ xCurrentTimeInt64: (_v: Ptr, out: Ptr) => (wasm.poke(out, 210866760000000 + Date.now(), 'i64'), 0),323+ },324+ },325+ });326+ return {327+ files,328+ error: () => {329+ const e = last;330+ last = null;331+ return e;332+ },333+ };334+}335+336+// The child pages of a table b-tree's interior page that hold these rowids,337+// or null if the page is a leaf (https://sqlite.org/fileformat2.html).338+function childPages(page: Uint8Array, header: number, rowids: number[]): number[] | null {339+ if (page[header] !== 5) return null;340+ const u32 = (at: number): number => page[at] * 2 ** 24 + (page[at + 1] << 16) + (page[at + 2] << 8) + page[at + 3];341+ // Cells are a child's page number, then a varint: the largest rowid under it.342+ const cells: [number, number][] = [];343+ const n = (page[header + 3] << 8) | page[header + 4];344+ for (let i = 0; i < n; i++) {345+ const at = (page[header + 12 + 2 * i] << 8) | page[header + 13 + 2 * i];346+ let key = 0;347+ for (let j = at + 4, k = 0; k < 9; j++, k++) {348+ if (k === 8) {349+ key = key * 256 + page[j];350+ break;351+ }352+ key = key * 128 + (page[j] & 0x7f);353+ if (page[j] < 0x80) break;354+ }355+ cells.push([key, u32(at)]);356+ }357+ const right = u32(header + 8);358+ return rowids.map((r) => {359+ let lo = 0;360+ let hi = cells.length;361+ while (lo < hi) {362+ const mid = (lo + hi) >> 1;363+ if (cells[mid][0] < r) lo = mid + 1;364+ else hi = mid;365+ }366+ return lo < cells.length ? cells[lo][1] : right;367+ });368+}369+370+export interface Hit {371+ path: string;372+ line: number;373+ text: string;374+ // Line anchors live on the source page (markdown renders as a document).375+ source: boolean;376+}377+378+export interface Result {379+ hits: Hit[];380+ // More chunks matched than were read.381+ more: boolean;382+ // What this query fetched.383+ requests: number;384+ bytes: number;385+ rounds: number;386+}387+388+// Trigrams can't find anything shorter.389+export const MIN = 3;390+// Chunks read for a first look (the overlay shows six lines until asked for391+// more), and when asked.392+export const FIRST = 12;393+export const MORE = 100;394+// A line is shown around its match, at most this long.395+const WIDTH = 160;396+// FTS5 leaves (about 4K each) of a term's doclist fetched ahead.397+const DOCLIST = 32;398+399+// An FTS5 string, quotes doubled.400+const str = (s: string): string => `"${s.replaceAll('"', '""')}"`;401+402+// The trigrams a query looks up. As a phrase, every trigram of the string is403+// a term, and each term is a lookup in another part of the index. Those that404+// just cover it (every third one, and the last) at NEAR distance find the405+// same chunks with a third of the lookups; lines are checked for the exact406+// string afterwards anyway.407+export function cover(q: string): string[] {408+ const c = [...q];409+ const at: number[] = [];410+ // Up to 5 characters, the phrase itself: every trigram, at most 3.411+ for (let i = 0; i + 3 < c.length; i += c.length <= 5 ? 1 : 3) at.push(i);412+ at.push(c.length - 3);413+ return at.map((i) => c.slice(i, i + 3).join(''));414+}415+416+// The FTS5 query for a literal substring, ignoring case.417+export function match(q: string): string {418+ const n = [...q].length;419+ // NEAR counts the tokens between the first phrase and the last.420+ return n <= 5 ? str(q) : `NEAR(${cover(q).map(str).join(' ')}, ${n - 4})`;421+}422+423+export class CodeIndex {424+ sqlite3: Sqlite3;425+ vfs: RangeVfs;426+ url: string;427+ get: GetRange;428+ getAsync: FetchRange;429+ block: number;430+ file!: RangeFile;431+ db: Db | null = null;432+ private n = 0;433+ // What the file had fetched when the last query ended; opening reads too.434+ private seen = { requests: 0, bytes: 0, rounds: 0 };435+ // From the schema, for prefetching: page size, root pages, FTS5 segments.436+ private layout: { page: number; chunk: number; data: number; segs: number[] } | null = null;437+438+ constructor(sqlite3: Sqlite3, vfs: RangeVfs, url: string, get: GetRange = xhrRange, getAsync: FetchRange = fetchRange, block = BLOCK) {439+ this.sqlite3 = sqlite3;440+ this.vfs = vfs;441+ this.url = url;442+ this.get = get;443+ this.getAsync = getAsync;444+ this.block = block;445+ this.open();446+ }447+448+ private open(): void {449+ this.file = new RangeFile(this.url, this.get, this.getAsync, this.block);450+ this.seen = { requests: 0, bytes: 0, rounds: 0 };451+ this.layout = null;452+ this.db = null;453+ }454+455+ // The database, once both ends of the file are in.456+ private async ready(): Promise<void> {457+ if (this.db) return;458+ await this.file.start();459+ const name = `code-${++this.n}.db`;460+ this.vfs.files.set(name, this.file);461+ this.db = new this.sqlite3.oo1.DB({ filename: name, flags: 'r', vfs: VFS });462+ }463+464+ close(): void {465+ this.db?.close();466+ this.db = null;467+ this.vfs.files.delete(`code-${this.n}.db`);468+ }469+470+ private all(sql: string, bind: unknown[] = []): unknown[][] {471+ const st = this.db!.prepare(sql);472+ const rows: unknown[][] = [];473+ try {474+ if (bind.length) st.bind(bind);475+ while (st.step()) rows.push(st.get([]));476+ } finally {477+ st.finalize();478+ }479+ return rows;480+ }481+482+ // Lines containing q, ignoring case, by path. If the index was rebuilt483+ // meanwhile, start over on the new one, once.484+ async search(q: string, limit = FIRST): Promise<Result> {485+ q = q.trim().replaceAll('\0', '');486+ if ([...q].length < MIN) return { hits: [], more: false, requests: 0, bytes: 0, rounds: 0 };487+ try {488+ return await this.query(q, limit);489+ } catch (e) {490+ const why = e instanceof Changed ? e : this.vfs.error();491+ if (!(why instanceof Changed)) throw why ?? e;492+ this.close();493+ this.open();494+ return await this.query(q, limit);495+ }496+ }497+498+ // The pages of a table's rows, from its root page: exact for a table of499+ // two levels, which 16K pages give up to some 25M.500+ private pages(root: number, rowids: number[]): number[] {501+ const { page } = this.layout!;502+ const buf = new Uint8Array(page);503+ this.file.read(buf, (root - 1) * page);504+ return childPages(buf, root === 1 ? 100 : 0, rowids) ?? [];505+ }506+507+ private async prefetch(root: number, rowids: number[]): Promise<void> {508+ const { page } = this.layout!;509+ if (rowids.length) await this.file.prefetch(this.pages(root, rowids).map((p) => (p - 1) * page));510+ }511+512+ private async query(q: string, limit: number): Promise<Result> {513+ await this.ready();514+ this.layout ??= {515+ page: this.all('pragma page_size')[0][0] as number,516+ chunk: this.all(`select rootpage from sqlite_schema where name = 'chunk'`)[0][0] as number,517+ data: this.all(`select rootpage from sqlite_schema where name = 'code_data'`)[0][0] as number,518+ segs: this.all('select distinct segid from code_idx').map((r) => r[0] as number),519+ };520+ // The index rows the query will read, from %_idx: each term's doclist,521+ // from the leaf it starts on to the one the next term starts on (its522+ // first DOCLIST leaves), and its doclist index. %_idx keys a leaf by its523+ // first term, with '0' in front for the main index, and stores its page524+ // number shifted left once. A leaf's row is segid << 37 | its page525+ // number; a doclist index's has bit 36 set too. Row 10 is the structure.526+ const enc = new TextEncoder();527+ const wanted = [1, 10];528+ for (const t of cover(q.toLowerCase())) {529+ const key = enc.encode('0' + t);530+ for (const seg of this.layout.segs) {531+ const first = (this.all('select pgno >> 1 from code_idx where segid = ? and term <= ? order by term desc limit 1', [seg, key])[0]?.[0] as number | undefined) ?? 1;532+ const next = (this.all('select pgno >> 1 from code_idx where segid = ? and term > ? order by term limit 1', [seg, key])[0]?.[0] as number | undefined) ?? first;533+ const id = seg * 2 ** 37;534+ for (let p = first; p <= Math.min(next, first + DOCLIST - 1); p++) wanted.push(id + p);535+ if (next > first) wanted.push(id + 2 ** 36 + first);536+ }537+ }538+ await this.prefetch(this.layout.data, wanted);539+540+ const ids = this.all('select rowid from code where code match ? limit ?', [match(q), limit + 1]).map((r) => r[0] as number);541+ const more = ids.length > limit;542+ ids.length = Math.min(ids.length, limit);543+ await this.prefetch(this.layout.chunk, ids);544+545+ const needle = q.toLowerCase();546+ const hits: Hit[] = [];547+ const rows = ids.length ? this.all(`select f.path, f.source, c.line, c.text from chunk c join file f on f.id = c.file where c.id in (${ids.map(() => '?').join(',')})`, ids) : [];548+ for (const [path, source, line, text] of rows as [string, number, number, string][]) {549+ text.split('\n').forEach((l, i) => {550+ const at = l.toLowerCase().indexOf(needle);551+ if (at >= 0) hits.push({ path, line: line + i, text: around(l, at), source: !!source });552+ });553+ }554+ // Rows come in index order, which an incremental build doesn't keep in path order.555+ hits.sort((a, b) => (a.path < b.path ? -1 : a.path > b.path ? 1 : a.line - b.line));556+ const { requests, bytes, rounds } = this.file;557+ const r = { hits, more, requests: requests - this.seen.requests, bytes: bytes - this.seen.bytes, rounds: rounds - this.seen.rounds };558+ this.seen = { requests, bytes, rounds };559+ return r;560+ }561+}562+563+// The line without its indent, cut to WIDTH around the match at `at`.564+function around(line: string, at: number): string {565+ const text = line.trim();566+ if (text.length <= WIDTH) return text;567+ const i = at - (line.length - line.trimStart().length);568+ const start = Math.max(0, Math.min(i - 40, text.length - WIDTH));569+ return (start ? '…' : '') + text.slice(start, start + WIDTH) + (start + WIDTH < text.length ? '…' : '');570+}571+572+// As a worker: {sqlite, wasm, url} first, then {id, q, limit}; each query573+// answers {id, ...Result} or {id, error}, in order.574+interface Scope {575+ postMessage(m: unknown): void;576+ addEventListener(type: 'message', fn: (e: MessageEvent) => void): void;577+}578+if (typeof (globalThis as { WorkerGlobalScope?: unknown }).WorkerGlobalScope === 'function') {579+ const scope = globalThis as unknown as Scope;580+ let index: Promise<CodeIndex> | null = null;581+ let queue: Promise<void> = Promise.resolve();582+ scope.addEventListener('message', (e) => {583+ const m = e.data as { sqlite?: string; wasm?: string; url?: string; id?: number; q?: string; limit?: number };584+ if (m.sqlite && m.wasm && m.url) {585+ const { sqlite, wasm, url } = m;586+ index = (async () => {587+ const init = (await import(sqlite)).default as (opt: object) => Promise<Sqlite3>;588+ const sqlite3 = await init({ locateFile: () => wasm, print: () => {}, printErr: () => {} });589+ return new CodeIndex(sqlite3, installVfs(sqlite3), url);590+ })();591+ return;592+ }593+ // One query at a time: they share the database and its prefetches.594+ queue = queue.then(async () => {595+ try {596+ if (!index) throw new Error('code search was not started');597+ scope.postMessage({ id: m.id, ...(await (await index).search(m.q ?? '', m.limit)) });598+ } catch (err) {599+ scope.postMessage({ id: m.id, error: String((err as Error)?.message ?? err) });600+ }601+ });602+ });603+}@@ -3,6 +3,11 @@
33 // pages, repo names and descriptions. The index is a static JSON file (see44 // lib/search.ts) fetched the first time the search opens. Every term must55 // match somewhere; matches in titles rank above matches in text.6+//7+// From 3 characters it also searches file contents on the default branch:8+// the query, as one literal string, goes to a worker (codesearch.ts) that9+// queries the repo's code index with sqlite-wasm over range requests. The10+// worker and sqlite-wasm load the first time someone searches code.611 (() => {712 const button = document.querySelector<HTMLButtonElement>('button[data-search]');813 if (!button) return;@@ -13,6 +18,7 @@
1318 home?: string;1419 blob?: string;1520 files?: string | null;21+ code?: string | null;1622 commits?: Row[];1723 reviews?: Row[];1824 wiki?: Row[];@@ -37,6 +43,52 @@
3743 3844 const KINDS = ['Repositories', 'Reviews', 'Commits', 'Wiki', 'Files'];3945 const PER_KIND = 6;46+47+ // Code search: lines from the worker, for the query they answer.48+ interface CodeHit {49+ path: string;50+ line: number;51+ text: string;52+ source: boolean;53+ }54+ interface CodeResult {55+ q: string;56+ hits: CodeHit[];57+ more: boolean;58+ limit: number;59+ error?: string;60+ }61+ // As in codesearch.ts: trigrams need 3 characters; 12 chunks for a first62+ // look, 100 when asked for more.63+ const CODE_MIN = 3;64+ const CODE_FIRST = 12;65+ const CODE_MORE = 100;66+ let codeUrl: string | null = null;67+ let blobBase = '';68+ let worker: Worker | null = null;69+ let asked = 0;70+ const waiting = new Map<number, (r: Omit<CodeResult, 'q' | 'limit'>) => void>();71+ function codeQuery(q: string, limit: number): Promise<Omit<CodeResult, 'q' | 'limit'>> {72+ if (!worker) {73+ const abs = (u: string): string => new URL(u, location.href).href;74+ worker = new Worker(abs(button!.dataset.code!), { type: 'module' });75+ worker.postMessage({ sqlite: abs(button!.dataset.sqlite!), wasm: abs(button!.dataset.wasm!), url: abs(codeUrl!) });76+ worker.addEventListener('message', (e: MessageEvent) => {77+ const m = e.data as { id: number; hits?: CodeHit[]; more?: boolean; error?: string };78+ waiting.get(m.id)?.({ hits: m.hits ?? [], more: !!m.more, error: m.error });79+ waiting.delete(m.id);80+ });81+ // A worker that can't start fails everything asked of it.82+ worker.addEventListener('error', () => {83+ for (const done of waiting.values()) done({ hits: [], more: false, error: 'Code search could not be loaded.' });84+ waiting.clear();85+ worker = null;86+ });87+ }88+ const id = ++asked;89+ worker.postMessage({ id, q, limit });90+ return new Promise((done) => waiting.set(id, done));91+ }4092 const enc = (p: string): string => p.split('/').map(encodeURIComponent).join('/');4193 const rec = (kind: string, href: string, title: string, meta = '', text = '', key = '', mono = false): Rec =>4294 ({ kind, href, title, meta, text, key, mono, low: [title.toLowerCase(), meta.toLowerCase(), text.toLowerCase()] });@@ -57,6 +109,8 @@
57109 list.push(rec('Commits', `${home}commit/${sha}/`, String(subject), `${String(sha).slice(0, 7)} ${author}, ${date}`, String(body), String(sha)));58110 }59111 for (const [href, title, text] of ix.wiki ?? []) list.push(rec('Wiki', String(href), String(title), '', String(text)));112+ codeUrl = ix.code ?? null;113+ blobBase = ix.blob ?? '';60114 if (ix.files) {61115 const paths = (await (await fetch(ix.files)).text()).split('\n').filter(Boolean);62116 for (const p of paths) list.push(rec('Files', `${ix.blob}${enc(p)}/`, p, '', '', '', true));@@ -129,8 +183,116 @@
129183 let rows: { li: HTMLLIElement; go: () => void }[] = [];130184 const expanded = new Set<string>();131185 186+ // The last code results that still apply, and the pending query's timer.187+ let code: CodeResult | null = null;188+ let timer = 0;189+ const go = (href: string): void => {190+ close();191+ location.href = href;192+ };193+194+ const heading = (kind: string, count: string): void => {195+ const head = document.createElement('li');196+ head.className = 'group';197+ head.textContent = `${kind} `;198+ const n = document.createElement('span');199+ n.textContent = count;200+ head.append(n);201+ ol.append(head);202+ };203+ // A row that opens a link: its title, and a second line if any.204+ const link = (href: string, title: Node[], mono: boolean, extra: Node[] | null, extraMono = false): void => {205+ const li = document.createElement('li');206+ const a = document.createElement('a');207+ a.href = href;208+ const t = document.createElement('span');209+ t.className = mono ? 't mono' : 't';210+ t.append(...title);211+ a.append(t);212+ if (extra?.length) {213+ const m = document.createElement('span');214+ m.className = extraMono ? 'm mono' : 'm';215+ m.append(...extra);216+ a.append(m);217+ }218+ li.append(a);219+ ol.append(li);220+ rows.push({ li, go: () => go(href) });221+ };222+ // A row that shows more of a kind.223+ const moreRow = (label: string, fn: () => void): void => {224+ const li = document.createElement('li');225+ li.className = 'more';226+ const b = document.createElement('button');227+ b.type = 'button';228+ b.textContent = label;229+ const run = (): void => {230+ fn();231+ draw();232+ input.focus();233+ };234+ b.addEventListener('click', run);235+ li.append(b);236+ ol.append(li);237+ rows.push({ li, go: run });238+ };239+ const note = (text: string): void => {240+ const li = document.createElement('li');241+ li.className = 'note';242+ li.textContent = text;243+ ol.append(li);244+ };245+246+ // Lines in files: the query as one string, ignoring case. While a longer247+ // query runs, the last results narrow to the lines that still match.248+ const drawCode = (q: string): boolean => {249+ const low = q.toLowerCase();250+ const fresh = code?.q === q;251+ if (fresh && code!.error) {252+ heading('Code', '');253+ note(code!.error);254+ return true;255+ }256+ const hits = !code ? [] : fresh ? code.hits : code.hits.filter((h) => h.text.toLowerCase().includes(low));257+ if (!hits.length) {258+ if (fresh) return false;259+ heading('Code', '');260+ note('Searching file contents');261+ return true;262+ }263+ const more = fresh && code!.more;264+ heading('Code', `${hits.length}${more ? '+' : ''}`);265+ const shown = expanded.has('Code') ? hits : hits.slice(0, PER_KIND);266+ for (const h of shown) {267+ link(`${blobBase}${enc(h.path)}/${h.source ? 'source/' : ''}#l${h.line}`, [document.createTextNode(`${h.path}:${h.line}`)], true, marked(h.text, [low]), true);268+ }269+ const rest = hits.length - shown.length;270+ if (rest || (more && code!.limit < CODE_MORE)) {271+ moreRow(rest ? `${rest}${more ? '+' : ''} more` : 'more', () => {272+ expanded.add('Code');273+ if (more && code!.limit < CODE_MORE) runCode(q, CODE_MORE);274+ });275+ } else if (more) note('More lines match than are shown here; a longer query narrows them.');276+ return true;277+ };278+ const runCode = (q: string, limit: number): void => {279+ codeQuery(q, limit).then((r) => {280+ // Results for an older query still help while they narrow to this one.281+ if (!box.isConnected || !input.value.trim().toLowerCase().includes(q.toLowerCase())) return;282+ code = { q, limit, ...r };283+ draw();284+ });285+ };286+ const queueCode = (): void => {287+ clearTimeout(timer);288+ const q = input.value.trim();289+ if (!codeUrl || [...q].length < CODE_MIN || code?.q === q) return;290+ timer = window.setTimeout(() => runCode(q, CODE_FIRST), 120);291+ };292+132293 const draw = (): void => {133294 const terms = input.value.toLowerCase().split(/\s+/).filter(Boolean);295+ const q = input.value.trim();134296 ol.replaceChildren();135297 rows = [];136298 if (!all) {@@ -138,58 +300,27 @@
138300 return;139301 }140302 if (!terms.length) {141- hint.textContent = repos ? 'Repository names and descriptions.' : 'Searches commit messages, reviews, wiki pages and file paths on the default branch, not file contents. Press t to find a file on this branch.';303+ hint.textContent = repos ? 'Repository names and descriptions.' : codeUrl304+ ? 'Searches commit messages, reviews, wiki pages, file paths and, from 3 characters, the files themselves, on the default branch. Press t to find a file on this branch.'305+ : 'Searches commit messages, reviews, wiki pages and file paths on the default branch. Press t to find a file on this branch.';142306 return;143307 }144308 const hits = all.map((r) => score(r, terms)).filter((h): h is Hit => !!h);145309 for (const kind of KINDS) {146310 const list = hits.filter((h) => h.r.kind === kind).sort((a, b) => b.s - a.s);147311 if (!list.length) continue;148- const head = document.createElement('li');149- head.className = 'group';150- head.textContent = `${kind} `;151- const n = document.createElement('span');152- n.textContent = String(list.length);153- head.append(n);154- ol.append(head);312+ heading(kind, String(list.length));155313 const shown = expanded.has(kind) ? list : list.slice(0, PER_KIND);156314 for (const h of shown) {157- const li = document.createElement('li');158- const a = document.createElement('a');159- a.href = h.r.href;160- const t = document.createElement('span');161- t.className = h.r.mono ? 't mono' : 't';162- t.append(...marked(h.r.title, terms));163- a.append(t);164315 const extra = h.inText ? snippet(h.r.text, terms) : h.r.meta || h.r.text.slice(0, 120);165- if (extra) {166- const m = document.createElement('span');167- m.className = 'm';168- m.append(...marked(extra, terms));169- a.append(m);170- }171- li.append(a);172- ol.append(li);173- rows.push({ li, go: () => (location.href = h.r.href) });174- }175- if (shown.length < list.length) {176- const li = document.createElement('li');177- li.className = 'more';178- const b = document.createElement('button');179- b.type = 'button';180- b.textContent = `${list.length - shown.length} more`;181- const go = (): void => {182- expanded.add(kind);183- draw();184- input.focus();185- };186- b.addEventListener('click', go);187- li.append(b);188- ol.append(li);189- rows.push({ li, go });316+ link(h.r.href, marked(h.r.title, terms), h.r.mono, extra ? marked(extra, terms) : null);190317 }318+ if (shown.length < list.length) moreRow(`${list.length - shown.length} more`, () => expanded.add(kind));191319 }192- hint.textContent = rows.length ? '' : 'No matches.';320+ const short = !!codeUrl && [...q].length < CODE_MIN;321+ const busy = !!codeUrl && !short && drawCode(q);322+ hint.textContent = rows.length || busy ? '' : short ? 'No matches. File contents are searched from 3 characters.' : 'No matches.';323+ if (rows.length && short) hint.textContent = 'File contents are searched from 3 characters.';193324 sel = Math.min(sel, Math.max(0, rows.length - 1));194325 rows[sel]?.li.classList.add('sel');195326 };@@ -199,12 +330,16 @@
199330 rows[sel]?.li.classList.add('sel');200331 rows[sel]?.li.scrollIntoView({ block: 'nearest' });201332 };202- const close = (): void => box.remove();333+ const close = (): void => {334+ clearTimeout(timer);335+ box.remove();336+ };203337 204338 input.addEventListener('input', () => {205339 sel = 0;206340 expanded.clear();207341 draw();342+ queueCode();208343 });209344 input.addEventListener('keydown', (e) => {210345 if (e.key === 'Escape') close();@@ -222,6 +357,7 @@
222357 loading.then((list) => {223358 all = list;224359 draw();360+ queueCode();225361 }, () => {226362 loading = null;227363 failed = true;@@ -0,0 +1,191 @@
1+// Code search, measured and queried the way the browser does it: the built2+// www served by demo/serve.ts, and site/assets/codesearch.ts (sqlite-wasm,3+// the range VFS, the query) reading code.db over HTTP. Node has no4+// synchronous XHR, so a worker thread fetches each range while this one5+// waits on an Atomics flag, the way a browser worker waits on its XHR.6+//7+// node test/code-search-bench.ts bench <state> [query...] sizes, requests, bytes8+// node test/code-search-bench.ts http <state> <owner/repo> <query> [limit]9+// node test/code-search-bench.ts sql <code.db> <query> node:sqlite, no HTTP10+//11+// <state> is a forge's .forge directory (www/ inside). `http` and `sql` print12+// "path:line: text" per hit; `http` then prints what the query fetched and13+// how the server answered, for code.db and for sqlite-wasm's .wasm.14+import fs from 'node:fs';15+import net from 'node:net';16+import path from 'node:path';17+import zlib from 'node:zlib';18+import { spawn, type ChildProcess } from 'node:child_process';19+import { Worker } from 'node:worker_threads';20+import { fileURLToPath } from 'node:url';21+import { CodeIndex, installVfs, match, MORE, type GetRange, type Sqlite3 } from '../site/assets/codesearch.ts';22+23+const here = path.dirname(fileURLToPath(import.meta.url));24+const [mode, ...args] = process.argv.slice(2);25+26+// Ranges are at most 32 blocks of 16K; a whole small file is under 256K.27+const MAX = 4 << 20;28+const FETCHER = `29+const { parentPort } = require('node:worker_threads');30+parentPort.on('message', async ({ url, from, to, sab }) => {31+ const ctl = new Int32Array(sab, 0, 4);32+ const body = new Uint8Array(sab, 1024);33+ const meta = new Uint8Array(sab, 16, 1008);34+ try {35+ const r = await fetch(url, { headers: { Range: from < 0 ? 'bytes=' + from : 'bytes=' + from + '-' + (to - 1) } });36+ const data = new Uint8Array(await r.arrayBuffer());37+ if (data.length > body.length) throw new Error('response too large: ' + data.length);38+ body.set(data);39+ const info = JSON.stringify({ status: r.status, range: r.headers.get('content-range'), type: r.headers.get('content-type'), enc: r.headers.get('content-encoding'), tag: r.headers.get('etag') ?? r.headers.get('last-modified') ?? '' });40+ meta.set(new TextEncoder().encode(info).subarray(0, 1000));41+ ctl[1] = data.length;42+ ctl[2] = Math.min(info.length, 1000);43+ } catch (e) {44+ const info = JSON.stringify({ error: String(e) });45+ meta.set(new TextEncoder().encode(info).subarray(0, 1000));46+ ctl[1] = -1;47+ ctl[2] = Math.min(info.length, 1000);48+ }49+ Atomics.store(ctl, 0, 1);50+ Atomics.notify(ctl, 0);51+});`;52+53+// A blocking range fetch.54+function syncRange(): { get: GetRange; stop: () => void } {55+ const w = new Worker(FETCHER, { eval: true });56+ const sab = new SharedArrayBuffer(1024 + MAX);57+ const ctl = new Int32Array(sab, 0, 4);58+ const get: GetRange = (url, from, to) => {59+ Atomics.store(ctl, 0, 0);60+ w.postMessage({ url, from, to, sab });61+ Atomics.wait(ctl, 0, 0);62+ const info = JSON.parse(new TextDecoder().decode(new Uint8Array(sab, 16, ctl[2]).slice()));63+ if (ctl[1] < 0) throw new Error(info.error);64+ if (info.status !== 200 && info.status !== 206) throw new Error(`${url}: HTTP ${info.status}`);65+ if (info.enc && info.enc !== 'identity') throw new Error(`${url} is served with Content-Encoding: ${info.enc}`);66+ const data = new Uint8Array(sab, 1024, ctl[1]).slice();67+ const whole = info.status === 200;68+ const m = /(\d+)-\d+\/(\d+)$/.exec(info.range ?? '');69+ return { whole, data, start: whole ? 0 : Number(m?.[1]), size: whole ? data.length : Number(m?.[2]), tag: info.tag };70+ };71+ return { get, stop: () => void w.terminate() };72+}73+74+// demo/serve.ts on a free port (it also takes the next one, for events).75+async function serve(state: string): Promise<{ base: string; stop: () => void }> {76+ const free = (): Promise<number> => new Promise((ok, no) => {77+ const s = net.createServer().once('error', no).listen(0, '127.0.0.1', () => {78+ const p = (s.address() as net.AddressInfo).port;79+ s.close(() => ok(p));80+ });81+ });82+ for (let tries = 0; ; tries++) {83+ const port = await free();84+ const child: ChildProcess = spawn(process.execPath, [path.join(here, '..', 'demo', 'serve.ts'), String(port)], { env: { ...process.env, FORGE_STATE: state }, stdio: ['ignore', 'pipe', 'pipe'] });85+ const up = await new Promise<boolean>((ok) => {86+ child.stdout!.on('data', (d: Buffer) => d.toString().includes('demo forge on') && ok(true));87+ child.on('exit', () => ok(false));88+ });89+ if (up) return { base: `http://127.0.0.1:${port}`, stop: () => void child.kill() };90+ if (tries > 5) throw new Error('demo/serve.ts did not start');91+ }92+}93+94+async function wasm(): Promise<Sqlite3> {95+ const init = (await import('@sqlite.org/sqlite-wasm')).default as unknown as (o: object) => Promise<Sqlite3>;96+ return init({ print: () => {}, printErr: () => {} });97+}98+99+const kb = (n: number): string => (n < 1024 ? `${n}B` : n < 1 << 20 ? `${(n / 1024).toFixed(1)}K` : `${(n / (1 << 20)).toFixed(2)}M`);100+101+// node:sqlite, without its experimental warning.102+function open(db: string): import('node:sqlite').DatabaseSync {103+ const emit = process.emitWarning;104+ process.emitWarning = (() => {}) as typeof emit;105+ const { DatabaseSync } = process.getBuiltinModule('node:sqlite') as typeof import('node:sqlite');106+ process.emitWarning = emit;107+ return new DatabaseSync(db, { readOnly: true });108+}109+110+function sqlHits(db: string, q: string): string[] {111+ const d = open(db);112+ const rows = d.prepare('select f.path, c.line, c.text from code join chunk c on c.id = code.rowid join file f on f.id = c.file where code match ?').all(match(q)) as { path: string; line: number; text: string }[];113+ const out: string[] = [];114+ for (const r of rows) r.text.split('\n').forEach((l, i) => l.toLowerCase().includes(q.toLowerCase()) && out.push(`${r.path}:${r.line + i}: ${l.trim()}`));115+ return out.sort();116+}117+118+if (mode === 'sql') {119+ for (const h of sqlHits(args[0], args[1])) console.log(h);120+} else if (mode === 'http') {121+ const [state, slug, q, limit] = args;122+ const srv = await serve(state);123+ const f = syncRange();124+ try {125+ const sqlite3 = await wasm();126+ const ix = new CodeIndex(sqlite3, installVfs(sqlite3), `${srv.base}/${slug}/code.db`, f.get);127+ const r = await ix.search(q, Number(limit || MORE));128+ for (const h of r.hits) console.log(`${h.path}:${h.line}: ${h.text}`);129+ console.log(`requests ${r.requests} rounds ${r.rounds} bytes ${r.bytes}`);130+ const head = await fetch(`${srv.base}/${slug}/code.db`, { headers: { Range: 'bytes=0-99' } });131+ console.log(`range ${head.status} ${head.headers.get('content-range')} type ${head.headers.get('content-type')} encoding ${head.headers.get('content-encoding') ?? 'none'} length ${(await head.arrayBuffer()).byteLength}`);132+ // What the browser instantiates sqlite-wasm from.133+ const wasmFile = fs.readdirSync(path.join(state, 'www', 'assets')).find((a) => a.endsWith('.wasm'));134+ const w = await fetch(`${srv.base}/assets/${wasmFile}`, { method: 'HEAD' });135+ console.log(`wasm ${w.status} type ${w.headers.get('content-type')} cache ${w.headers.get('cache-control')}`);136+ } finally {137+ f.stop();138+ srv.stop();139+ }140+} else if (mode === 'bench') {141+ const [state, ...qs] = args;142+ const queries = qs.length ? qs : ['function', 'escapeHtml', 'createHash', 'readFileSync(', 'TODO', 'import fs from', 'Cache-Control', 'zzqqxx'];143+ const www = path.join(state, 'www');144+ const dbs: string[] = [];145+ for (const owner of fs.readdirSync(www)) {146+ const d = path.join(www, owner);147+ if (!fs.statSync(d).isDirectory()) continue;148+ for (const repo of fs.readdirSync(d)) if (fs.existsSync(path.join(d, repo, 'code.db'))) dbs.push(`${owner}/${repo}`);149+ }150+ const assets = fs.readdirSync(path.join(www, 'assets'));151+ for (const a of assets.filter((f) => /^(sqlite3|codesearch)\./.test(f))) {152+ const data = fs.readFileSync(path.join(www, 'assets', a));153+ console.log(`asset ${a}: ${kb(data.length)}, ${kb(zlib.gzipSync(data).length)} gzipped (once, then cached for good)`);154+ }155+ const srv = await serve(state);156+ const f = syncRange();157+ try {158+ const sqlite3 = await wasm();159+ const vfs = installVfs(sqlite3);160+ for (const slug of dbs) {161+ const file = path.join(www, slug, 'code.db');162+ const d = open(file);163+ const { files, text } = d.prepare('select (select count(*) from file) files, (select sum(length(cast(text as blob))) from chunk) text').get() as { files: number; text: number };164+ d.close();165+ const size = fs.statSync(file).size;166+ console.log(`\n${slug}: code.db ${kb(size)} for ${files} files, ${kb(text)} of text (${(size / text).toFixed(1)}x)`);167+ console.log('query'.padEnd(16) + 'hits'.padStart(5) + ' cold: rounds requests bytes again after the others: rounds requests bytes');168+ const url = `${srv.base}/${slug}/code.db`;169+ for (const q of queries) {170+ // Cold: a fresh index, as on a page that never searched code.171+ const cold = new CodeIndex(sqlite3, vfs, url, f.get);172+ const c = await cold.search(q);173+ const again = await cold.search(q);174+ cold.close();175+ // After the others: the queries before it warmed some pages.176+ const later = new CodeIndex(sqlite3, vfs, url, f.get);177+ for (const o of queries) if (o !== q) await later.search(o);178+ const l = await later.search(q);179+ later.close();180+ const col = (n: number | string, w: number): string => String(n).padStart(w);181+ console.log(`${q.padEnd(16)}${col(c.hits.length + (c.more ? '+' : ''), 5)} ${col(c.rounds, 13)} ${col(c.requests, 8)} ${col(kb(c.bytes), 6)} ${col(again.requests, 6)} ${col(l.rounds, 25)} ${col(l.requests, 8)} ${col(kb(l.bytes), 6)}`);182+ }183+ }184+ } finally {185+ f.stop();186+ srv.stop();187+ }188+} else {189+ console.error('usage: node test/code-search-bench.ts bench <state> [query...] | http <state> <owner/repo> <query> [limit] | sql <code.db> <query>');190+ process.exit(2);191+}@@ -0,0 +1,34 @@
1+# A build writes the default branch's code index, search.json points at it,2+# and a substring query finds the file and line, ignoring case.3+. "$(dirname "$0")/../lib.sh"4+user dami5+repo dami/greet "say hello"6+clone dami dami/greet "$T/w/dami"7+mkdir -p "$T/w/dami/src"8+printf 'one\ntwo\n' >"$T/w/dami/notes.txt"9+i=110+while [ $i -le 20 ]; do printf 'line %s\n' "$i"; i=$((i + 1)); done >"$T/w/dami/src/main.go"11+printf 'func Greet(name string) string {\n\treturn "Hello, " + name\n}\n' >>"$T/w/dami/src/main.go"12+printf '# Notes\n\nSee greetHelper here.\n' >"$T/w/dami/README.md"13+at "$T/w/dami" commit "first"14+push_as dami "$T/w/dami" origin main15+build16+db=$FORGE_WWW/dami/greet/code.db17+exists "$db"18+has "$FORGE_WWW/dami/greet/search.json" '"code":"/dami/greet/code.db"'19+# The rest of the index is unchanged.20+has "$FORGE_WWW/dami/greet/search.json" '"files":"/dami/greet/files/main.txt"'21+bench() { node "$T_HOME/test/code-search-bench.ts" "$@"; }22+ok bench sql "$db" 'return "hello, "'23+eq "$(output)" 'src/main.go:22: return "Hello, " + name'24+# Line 22 is in the third chunk of eight lines; markdown links to its source page.25+ok bench sql "$db" 'GREETHELPER'26+eq "$(output)" 'README.md:3: See greetHelper here.'27+ok bench sql "$db" 'nothing like this'28+eq "$(output)" ''29+# Pages name the worker and sqlite-wasm, and the assets exist.30+has "$FORGE_WWW/dami/greet/index.html" 'data-code="/assets/codesearch\.[^"]+\.js" data-sqlite="/assets/sqlite3\.[^"]+\.js" data-wasm="/assets/sqlite3\.[^"]+\.wasm"'31+lacks "$FORGE_WWW/index.html" 'data-code='32+ls "$FORGE_WWW/assets" >"$T/assets"33+has "$T/assets" '^sqlite3\.[0-9A-Za-z_-]{10}\.wasm$'34+has "$FORGE_WWW/assets/versions.json" 'sqlite3\.[0-9A-Za-z_-]{10}\.wasm'@@ -0,0 +1,21 @@
1+# The browser's code path, headless: demo/serve.ts serves code.db in byte2+# ranges (206) and sqlite-wasm's .wasm as application/wasm, and the range3+# VFS in site/assets/codesearch.ts finds a line through it.4+. "$(dirname "$0")/../lib.sh"5+user dami6+repo dami/greet7+clone dami dami/greet "$T/w/dami"8+i=19+while [ $i -le 30 ]; do printf 'filler line %s\n' "$i"; i=$((i + 1)); done >"$T/w/dami/greet.sh"10+printf 'echo "Hello, $1"\n' >>"$T/w/dami/greet.sh"11+at "$T/w/dami" commit "first"12+push_as dami "$T/w/dami" origin main13+build14+ok node "$T_HOME/test/code-search-bench.ts" http "$FORGE_ROOT/.forge" dami/greet 'hello, $1'15+has "$T_LOG" '^greet\.sh:31: echo "Hello, \$1"$'16+has "$T_LOG" '^requests [1-9][0-9]* rounds [1-9][0-9]* bytes [1-9][0-9]*$'17+has "$T_LOG" '^range 206 bytes 0-99/[0-9]+ type application/vnd\.sqlite3 encoding none length 100$'18+has "$T_LOG" '^wasm 200 type application/wasm cache public, max-age=31536000, immutable$'19+# Too short for trigrams: nothing, and nothing fetched.20+ok node "$T_HOME/test/code-search-bench.ts" http "$FORGE_ROOT/.forge" dami/greet 'he'21+has "$T_LOG" '^requests 0 '@@ -0,0 +1,44 @@
1+# The code index is kept by the default branch's tree: a push that leaves2+# that tree alone keeps the very same file, and a push that changes it3+# rebuilds the index from the previous one, keeping unchanged files' rows.4+. "$(dirname "$0")/../lib.sh"5+user dami6+repo dami/greet7+clone dami dami/greet "$T/w/dami"8+printf 'alpha one\n' >"$T/w/dami/a.txt"9+printf 'beta two\n' >"$T/w/dami/b.txt"10+at "$T/w/dami" commit "first"11+push_as dami "$T/w/dami" origin main12+build13+db=$FORGE_WWW/dami/greet/code.db14+inode() { ls -i "$1" | awk '{print $1}'; }15+first=$(inode "$db")16+# A review branch doesn't touch main's tree.17+at "$T/w/dami" git switch -q -c dami/topic18+at "$T/w/dami" commit "topic"19+push_as dami "$T/w/dami" origin dami/topic20+build21+eq "$(inode "$db")" "$first"22+# A new commit on main with the same tree (an empty commit) keeps it too.23+at "$T/w/dami" git switch -q main24+tick25+at "$T/w/dami" git commit -q --allow-empty -m "empty"26+push_as dami "$T/w/dami" origin main27+build28+eq "$(inode "$db")" "$first"29+# Changing main rebuilds: the new text is found, the old is gone, and the30+# unchanged file is still there.31+printf 'alpha uno\n' >"$T/w/dami/a.txt"32+at "$T/w/dami" commit "change" a.txt33+push_as dami "$T/w/dami" origin main34+build35+[ "$(inode "$db")" != "$first" ] || fail "code.db was not rebuilt"36+bench() { node "$T_HOME/test/code-search-bench.ts" sql "$db" "$1"; }37+ok bench 'alpha uno'38+eq "$(output)" 'a.txt:1: alpha uno'39+ok bench 'alpha one'40+eq "$(output)" ''41+ok bench 'beta two'42+eq "$(output)" 'b.txt:1: beta two'43+# The cache keeps only the current index (and its layout).44+eq "$(ls "$FORGE_CACHE/code/dami/greet" | wc -l | tr -d ' ')" 2@@ -0,0 +1,23 @@
1+# The code index leaves out binary files, files over the highlight cap (256K),2+# lockfiles, minified files and images; text files are in it.3+. "$(dirname "$0")/../lib.sh"4+user dami5+repo dami/greet6+clone dami dami/greet "$T/w/dami"7+cd "$T/w/dami"8+printf 'needle-ok\n' >ok.txt9+printf 'needle-bin\000\001\002\n' >data.bin10+# Just over 256K of text.11+awk 'BEGIN { for (i = 0; i < 9000; i++) print "needle-big filler filler filler" }' >big.txt12+printf '{ "needle-lock": 1 }\n' >package-lock.json13+printf 'var needle_min="needle-min";\n' >app.min.js14+printf '<svg><text>needle-svg</text></svg>\n' >logo.svg15+cd - >/dev/null16+at "$T/w/dami" commit "files"17+push_as dami "$T/w/dami" origin main18+build19+[ "$(wc -c <"$T/w/dami/big.txt")" -gt 262144 ] || fail "big.txt is under 256K"20+ok node "$T_HOME/test/code-search-bench.ts" sql "$FORGE_WWW/dami/greet/code.db" needle-21+eq "$(output)" 'ok.txt:1: needle-ok'22+# The file pages are still there; only the index skips them.23+exists "$FORGE_WWW/dami/greet/blob/main/big.txt/index.html"