| M | README.md | 31 | ++++++- |
| M | lib/ | 26 | +++++- |
| M | site/ | 2 | +- |
| M | site/ | 50 | +++++++++--- |
| A | site/ | 97 | ++++++++++++++++++++++ |
| M | site/ | 100 | ++++++++--------------- |
| M | test/ | 4 | + |
| A | test/ | 133 | ++++++++++++++++++++++++++++++ |
| A | test/ | 57 | +++++++++++++ |
@@ -555,6 +555,21 @@ first time the search opens, and matching runs in the browser. Commit bodies
555555 are cut to 240 characters and wiki pages to 2000, to keep the index small.556556 `t` still finds files by name on the current branch.557557 558+Every section reads a query the same way (`site/assets/query.ts`): terms559+split on whitespace, in any order, and a quoted phrase (`"fix the search"`)560+is one term, spaces and all, that must appear as written, except that a run561+of whitespace in it matches any run of whitespace: `"fix the"` finds562+`fix the`, and `"fix the"` finds `fix the`. Every term must563+match, ignoring case, and results mark each match, a phrase as a whole. A564+quote opens a phrase only at the start of a term (in `it's"` it is just a565+character), an unclosed quote runs to the end of the query as a phrase, so a566+phrase narrows as you type it, and empty quotes are no term. So567+`fix search` finds "Search fix for the box" and `"fix the search"` doesn't.568+`node test/search-phrases.ts` runs the same code in Node, for569+`test/scenarios/search-phrases.sh`. Pages load `search.ts` as a module; it570+and `codesearch.ts` import `./query.ts`, which a build rewrites to that571+file's content-hashed name.572+558573 #### Code search559574 560575 File contents are in a second index per repo, `/<owner>/<repo>/code.db`,@@ -569,10 +584,18 @@ pages mirror on R2 through the edge, whose fallback passes the header on.
569584 Once the query has a word of 3 characters (a trigram can't match less), it570585 goes to a worker, and a Code section lists each line that has every word,571586 ignoring case, as `path:line`, linked to that line of the file's page (the572-source page for markdown). Words split on whitespace, as in the other573-sections, in any order, and a quoted phrase (`"import fs from"`) is one574-literal string, spaces and all. Words of 3 characters or more are looked up575-in the index; shorter ones only filter the lines it finds. It loads nothing587+source page for markdown). Its words are the terms the other sections read,588+so a quoted phrase (`"import fs from"`) is one literal string, spaces and589+all, in which a run of whitespace matches any run, tabs and alignment too;590+a word inside another goes, since a line with the longer one has it.591+Words of 3 characters or more are looked up592+in the index; shorter ones only filter the lines it finds. A phrase is593+looked up by its pieces between whitespace, those of 3 characters or more,594+since no trigram may span whitespace that can be any length: all of them595+within `NEAR` distance of the phrase's length plus 32 characters per gap,596+and the lines found are checked with any run of whitespace for a space. A597+phrase without such a piece (`"x = 1"`) is looked up as written, so it finds598+only single-spaced lines. It loads nothing576599 until then: the first code query starts the worker, which loads577600 `sqlite3.wasm` and sqlite-wasm's script, both content-hashed assets. Results578601 come in path order; the first look reads 12 chunks, and "more" reads up to@@ -42,7 +42,7 @@ export interface Asset {
4242 data: Buffer;4343 }4444 45-// By short name: style, logo, fixi, ssexi, diffs, keys, live, search,45+// By short name: style, logo, fixi, ssexi, diffs, keys, live, search, query,4646 // codesearch, login, ui, mermaid, sqlite3, sqlite3wasm, webauthn.4747 export type Assets = Record<string, Asset>;4848 @@ -56,17 +56,31 @@ export function assets(): Assets {
5656 // Rebuilt only when a source changes (the dev server rebuilds in one process).5757 const key = files.map(([, , f]) => `${f} ${fs.statSync(f).mtimeMs}`).join('\n');5858 if (memo?.key === key) return memo.assets;59- const out: Assets = {};60- for (const [short, base, f] of files) {59+ // Our scripts import each other as './query.ts', which Node and tsc read60+ // as it is; shipped, that is the other file's hashed name, so a script's61+ // name changes with what it imports.62+ const made = new Map<string, Asset>();63+ const make = ([short, base, f]: [string, string, string]): Asset => {64+ const done = made.get(short);65+ if (done) return done;6166 let data = fs.readFileSync(f);6267 let ext = path.extname(base);6368 const name = base.split('.')[0];6469 if (ext === '.ts') {65- data = Buffer.from(stripTypeScriptTypes(data.toString('utf8')));70+ const js = stripTypeScriptTypes(data.toString('utf8')).replace(/(\bfrom\s*)'\.\/([\w-]+\.ts)'/g, (_, from: string, dep: string) => {71+ const d = files.find(([, b, p]) => b === dep && path.dirname(p) === src);72+ if (!d) throw new Error(`${base} imports ./${dep}, which is not in site/assets`);73+ return `${from}'./${make(d).file}'`;74+ });75+ data = Buffer.from(js);6676 ext = '.js';6777 }68- out[short] = { file: `${name}.${hash(data)}${ext}`, data };69- }78+ const a = { file: `${name}.${hash(data)}${ext}`, data };79+ made.set(short, a);80+ return a;81+ };82+ const out: Assets = {};83+ for (const f of files) out[f[0]] = make(f);7084 memo = { key, assets: out };7185 return out;7286 }@@ -21,7 +21,7 @@
2121 <script src="{{ assets.ssexi }}" defer></script>2222 <script src="{{ assets.live }}" defer></script>2323 <script src="{{ assets.keys }}" defer></script>24-<script src="{{ assets.search }}" defer></script>24+<script type="module" src="{{ assets.search }}"></script>2525 <script src="{{ assets.ui }}" data-mermaid="{{ assets.mermaid }}" defer></script>2626 {% block scripts %}{% endblock %}2727 {% block head %}{{ m.meta(p.title if p and p.title else forge.config.name, r.description if r else '') }}{% endblock %}@@ -12,6 +12,8 @@
1212 // the index leaf of each term (from FTS5's small %_idx table), then the page1313 // of each matching chunk (from the chunk table's root page).1414 15+import { find, narrow, pieces, terms } from './query.ts';16+1517 // Bytes [from, to) of url (from < 0: the last -from bytes), or the whole1618 // file if the server ignores ranges; where they start, the file's size, and1719 // its validator, which must not change while it is open.@@ -416,22 +418,46 @@ export function cover(q: string): string[] {
416418 return at.map((i) => c.slice(i, i + 3).join(''));417419 }418420 421+// The strings the index looks up for a word: the word itself, but for a422+// phrase its pieces between whitespace of MIN characters or more, since a423+// space in a phrase stands for any run of whitespace and no trigram may span424+// one. A phrase without such a piece (`"x = 1"`) is looked up as written,425+// single spaces and all, so it finds only lines spaced that way.426+function lookups(w: string): string[] {427+ const ps = pieces(w).filter((p) => [...p].length >= MIN);428+ return ps.length ? ps : [w];429+}430+431+// Extra whitespace characters allowed per gap in a phrase: its pieces must432+// be near each other in the index, with this much room for runs of spaces433+// longer than one (alignment). Lines are checked afterwards anyway.434+const GAP = 32;435+419436 // The FTS5 query for a literal substring, ignoring case.420-export function match(q: string): string {437+function literal(q: string): string {421438 const n = [...q].length;422439 // NEAR counts the tokens between the first phrase and the last.423440 return n <= 5 ? str(q) : `NEAR(${cover(q).map(str).join(' ')}, ${n - 4})`;424441 }425442 426-// A query's words, lower case, as the other search sections split theirs:427-// on whitespace, except that a quoted phrase ("like this") is one literal428-// string, spaces and all (an unclosed quote runs to the end). A word that429-// is part of another goes, since every line with the longer one has it.430-export function words(q: string): string[] {431- const all = [...q.toLowerCase().replaceAll('\0', '').matchAll(/"([^"]*)"?|(\S+)/g)].map((m) => m[1] ?? m[2]).filter((w) => w.trim());432- return [...new Set(all)].filter((w, i, ws) => !ws.some((o, j) => j !== i && o.length > w.length && o.includes(w)));443+// The FTS5 query for a word: a literal substring, ignoring case, but in a444+// phrase the trigrams of each piece, all within NEAR distance of the phrase445+// with GAP more characters per gap.446+export function match(w: string): string {447+ const ps = lookups(w);448+ if (ps.length === 1) return literal(ps[0]);449+ const gaps = pieces(w).length - 1;450+ return `NEAR(${ps.flatMap(cover).map(str).join(' ')}, ${[...w].length - 4 + GAP * gaps})`;433451 }434452 453+// A query's words, lower case, its terms as query.ts reads them: split on454+// whitespace, except that a quoted phrase ("like this") is one literal455+// string, spaces and all, a run of whitespace in it matching any run (an456+// unclosed quote runs to the end).457+// A word that is part of another goes, since every line with the longer one458+// has it, and so do NULs, which an FTS5 string can't hold.459+export const words = (q: string): string[] => narrow(terms(q.replaceAll('\0', '')));460+435461 // The words the trigram index can look up; shorter ones only filter lines.436462 export const indexed = (ws: string[]): string[] => ws.filter((w) => [...w].length >= MIN);437463 @@ -443,9 +469,9 @@ export function lineMatch(line: string, ws: string[]): number {
443469 const low = line.toLowerCase();444470 let first = -1;445471 for (const w of ws) {446- const at = low.indexOf(w);447- if (at < 0) return -1;448- if (first < 0 || at < first) first = at;472+ const at = find(low, w);473+ if (!at) return -1;474+ if (first < 0 || at[0] < first) first = at[0];449475 }450476 return first;451477 }@@ -556,7 +582,7 @@ export class CodeIndex {
556582 // number; a doclist index's has bit 36 set too. Row 10 is the structure.557583 const enc = new TextEncoder();558584 const wanted = [1, 10];559- for (const t of new Set(indexed(ws).flatMap(cover))) {585+ for (const t of new Set(indexed(ws).flatMap(lookups).flatMap(cover))) {560586 const key = enc.encode('0' + t);561587 for (const seg of this.layout.segs) {562588 const first = (this.all('select pgno >> 1 from code_idx where segid = ? and term <= ? order by term desc limit 1', [seg, key])[0]?.[0] as number | undefined) ?? 1;@@ -0,0 +1,97 @@
1+// A search query, as every section of the search reads it: search.ts for2+// commits, reviews, wiki pages, file paths and repos, and codesearch.ts for3+// the lines of files. A query is terms split on whitespace, except that a4+// quoted phrase ("like this") is one term, spaces and all, which must appear5+// as written, except that a run of whitespace in it matches any run of6+// whitespace ("x = 1" finds `x = 1`, and the other way round). A quote7+// opens a phrase only at the start of a term (in `it's"` or `a"b` it is just8+// a character), an unclosed one runs to the end of the query, and empty9+// quotes are no term. Every term must match, ignoring case. Nothing here10+// touches the page, so test/search-phrases.ts checks it in Node as the11+// browser runs it.12+13+// A query's terms, lower case, each once; a run of whitespace in a phrase14+// is kept as one space, which find() matches with any run.15+export function terms(q: string): string[] {16+ const all = [...q.toLowerCase().matchAll(/"([^"]*)"?|(\S+)/g)].map((m) => m[2] ?? m[1].replace(/\s+/g, ' ')).filter((t) => t.trim());17+ return [...new Set(all)];18+}19+20+// The terms a search within one line needs: a term inside another goes,21+// since every line with the longer one has it.22+export const narrow = (ts: string[]): string[] => ts.filter((t, i) => !ts.some((o, j) => j !== i && o.length > t.length && o.includes(t)));23+24+// A term's pieces: a phrase's parts between whitespace (a word is one).25+export const pieces = (t: string): string[] => t.split(' ').filter(Boolean);26+27+// Where a term is in a lower case text, from `from` on: [start, end), or28+// null. A space in a term (only phrases have them) matches any run of29+// whitespace; terms without one are a plain indexOf.30+const patterns = new Map<string, RegExp>();31+export function find(low: string, t: string, from = 0): [number, number] | null {32+ if (!t.includes(' ')) {33+ const i = low.indexOf(t, from);34+ return i < 0 ? null : [i, i + t.length];35+ }36+ let re = patterns.get(t);37+ if (!re) {38+ if (patterns.size > 200) patterns.clear();39+ re = new RegExp(t.split(' ').map((p) => p.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')).join('\\s+'), 'g');40+ patterns.set(t, re);41+ }42+ re.lastIndex = from;43+ const m = re.exec(low);44+ return m ? [m.index, m.index + m[0].length] : null;45+}46+47+// Whether a text has every term, ignoring case.48+export function hasAll(text: string, ts: string[]): boolean {49+ const low = text.toLowerCase();50+ return ts.every((t) => find(low, t));51+}52+53+const wordStart = (s: string, i: number): boolean => i === 0 || /[^\p{L}\p{N}]/u.test(s[i - 1]);54+55+// How well a record matches, from its title, meta line and text (lower56+// case), and a key that matches by prefix (a commit's full sha); null unless57+// every term is in one of them. Matches in titles rank above matches in58+// text, and those at the start of a word above those inside one.59+export function score(low: [string, string, string], key: string, ts: string[]): { s: number; inText: boolean } | null {60+ let s = 0;61+ let inText = false;62+ for (const t of ts) {63+ const at = find(low[0], t);64+ if (at) s += wordStart(low[0], at[0]) ? 6 : 4;65+ else if (t.length >= 4 && key.startsWith(t)) s += 5;66+ else if (find(low[1], t)) s += 2;67+ else if (find(low[2], t)) {68+ s += 1;69+ inText = true;70+ } else return null;71+ }72+ return { s, inText };73+}74+75+// A text cut into runs, each marked if it is part of a term's match: a76+// phrase is marked whole, and overlapping matches make one run. Text whose77+// lower case is another length (a few letters) stays one plain run.78+export function marks(s: string, ts: string[]): [string, boolean][] {79+ const low = s.toLowerCase();80+ if (low.length !== s.length) return [[s, false]];81+ const hit = new Uint8Array(s.length);82+ for (const t of ts) for (let m = find(low, t); m; m = find(low, t, m[0] + 1)) hit.fill(1, m[0], m[1]);83+ const out: [string, boolean][] = [];84+ for (let i = 0, j = 0; i < s.length; i = j) {85+ while (j < s.length && hit[j] === hit[i]) j++;86+ out.push([s.slice(i, j), !!hit[i]]);87+ }88+ return out;89+}90+91+// The part of a long text around its first match.92+export function snippet(text: string, ts: string[]): string {93+ const low = text.toLowerCase();94+ const at = Math.min(...ts.map((t) => find(low, t)?.[0] ?? -1).filter((i) => i >= 0));95+ const start = Math.max(0, at - 30);96+ return (start ? '…' : '') + text.slice(start, start + 120) + (start + 120 < text.length ? '…' : '');97+}@@ -1,14 +1,18 @@
11 // Search, on "/" or the header button. In a repo it covers commits, reviews,22 // wiki pages and file paths on the default branch; on the index and owner33 // pages, repo names and descriptions. The index is a static JSON file (see4-// lib/search.ts) fetched the first time the search opens. Every term must5-// match somewhere; matches in titles rank above matches in text.4+// lib/search.ts) fetched the first time the search opens. A query is terms5+// split on whitespace, "a phrase" in quotes being one, read the same way in6+// every section (query.ts). Every term must match somewhere; matches in7+// titles rank above matches in text.68 //79 // With a word of 3 characters it also searches file contents on the default8-// branch, for lines with every word ("a phrase" in quotes is one): the9-// query goes to a worker (codesearch.ts) that queries the repo's code index10-// with sqlite-wasm over range requests. The worker and sqlite-wasm load the11-// first time someone searches code.10+// branch, for lines with every term: the query goes to a worker11+// (codesearch.ts) that queries the repo's code index with sqlite-wasm over12+// range requests. The worker and sqlite-wasm load the first time someone13+// searches code.14+import { hasAll, marks, narrow, score as match, snippet, terms } from './query.ts';15+1216 (() => {1317 const button = document.querySelector<HTMLButtonElement>('button[data-search]');1418 if (!button) return;@@ -60,20 +64,13 @@
6064 error?: string;6165 }6266 // As in codesearch.ts: trigrams need 3 characters; 12 chunks for a first63- // look, 100 when asked for more. A query's words split on whitespace, but64- // a quoted phrase is one, spaces and all; a word inside another goes.67+ // look, 100 when asked for more. A line has every term, so a term inside68+ // another goes.6569 const CODE_MIN = 3;6670 const CODE_FIRST = 12;6771 const CODE_MORE = 100;68- const codeWords = (q: string): string[] => {69- const all = [...q.toLowerCase().matchAll(/"([^"]*)"?|(\S+)/g)].map((m) => m[1] ?? m[2]).filter((w) => w.trim());70- return [...new Set(all)].filter((w, i, ws) => !ws.some((o, j) => j !== i && o.length > w.length && o.includes(w)));71- };72+ const codeWords = (q: string): string[] => narrow(terms(q));7273 const codeable = (q: string): boolean => codeWords(q).some((w) => [...w].length >= CODE_MIN);73- const hasAll = (text: string, ws: string[]): boolean => {74- const low = text.toLowerCase();75- return ws.every((w) => low.includes(w));76- };7774 let codeUrl: string | null = null;7875 let blobBase = '';7976 let worker: Worker | null = null;@@ -129,50 +126,21 @@
129126 return list;130127 }131128 132- const wordStart = (s: string, i: number): boolean => i === 0 || /[^\p{L}\p{N}]/u.test(s[i - 1]);133-134- function score(r: Rec, terms: string[]): Hit | null {135- let s = 0;136- let inText = false;137- for (const t of terms) {138- const i = r.low[0].indexOf(t);139- if (i >= 0) s += wordStart(r.low[0], i) ? 6 : 4;140- else if (t.length >= 4 && r.key.startsWith(t)) s += 5;141- else if (r.low[1].includes(t)) s += 2;142- else if (r.low[2].includes(t)) {143- s += 1;144- inText = true;145- } else return null;146- }147- return { r, s, inText };148- }149-150- // Text with every occurrence of a term in <b>, built as nodes (never HTML).151- function marked(s: string, terms: string[]): Node[] {152- const low = s.toLowerCase();153- if (low.length !== s.length) return [document.createTextNode(s)];154- const hit = new Uint8Array(s.length);155- for (const t of terms) for (let i = low.indexOf(t); i >= 0; i = low.indexOf(t, i + 1)) hit.fill(1, i, i + t.length);156- const out: Node[] = [];157- for (let i = 0, j = 0; i < s.length; i = j) {158- while (j < s.length && hit[j] === hit[i]) j++;159- const text = document.createTextNode(s.slice(i, j));160- if (!hit[i]) out.push(text);161- else {162- const b = document.createElement('b');163- b.append(text);164- out.push(b);165- }166- }167- return out;129+ function score(r: Rec, ts: string[]): Hit | null {130+ const m = match(r.low, r.key, ts);131+ return m && { r, ...m };168132 }169133 170- // The part of a long text around its first match.171- function snippet(text: string, terms: string[]): string {172- const low = text.toLowerCase();173- const at = Math.min(...terms.map((t) => low.indexOf(t)).filter((i) => i >= 0));174- const start = Math.max(0, at - 30);175- return (start ? '…' : '') + text.slice(start, start + 120) + (start + 120 < text.length ? '…' : '');134+ // Text with every match of a term (a phrase whole) in <b>, built as nodes135+ // (never HTML).136+ function marked(s: string, ts: string[]): Node[] {137+ return marks(s, ts).map(([text, hit]) => {138+ const t = document.createTextNode(text);139+ if (!hit) return t;140+ const b = document.createElement('b');141+ b.append(t);142+ return b;143+ });176144 }177145 178146 function open(): void {@@ -304,7 +272,7 @@
304272 };305273 306274 const draw = (): void => {307- const terms = input.value.toLowerCase().split(/\s+/).filter(Boolean);275+ const ts = terms(input.value);308276 const q = input.value.trim();309277 ol.replaceChildren();310278 rows = [];@@ -312,21 +280,21 @@
312280 hint.textContent = failed ? 'The search index could not be loaded.' : 'Loading the search index';313281 return;314282 }315- if (!terms.length) {316- hint.textContent = repos ? 'Repository names and descriptions.' : codeUrl317- ? 'Searches commit messages, reviews, wiki pages, file paths and, for words of 3 characters or more, the lines of the files themselves ("a phrase" in quotes), on the default branch. Press t to find a file on this branch.'318- : 'Searches commit messages, reviews, wiki pages and file paths on the default branch. Press t to find a file on this branch.';283+ if (!ts.length) {284+ hint.textContent = repos ? 'Repository names and descriptions; "a phrase" in quotes matches as written.' : codeUrl285+ ? 'Searches commit messages, reviews, wiki pages, file paths and, for words of 3 characters or more, the lines of the files themselves, on the default branch; "a phrase" in quotes matches as written. Press t to find a file on this branch.'286+ : 'Searches commit messages, reviews, wiki pages and file paths on the default branch; "a phrase" in quotes matches as written. Press t to find a file on this branch.';319287 return;320288 }321- const hits = all.map((r) => score(r, terms)).filter((h): h is Hit => !!h);289+ const hits = all.map((r) => score(r, ts)).filter((h): h is Hit => !!h);322290 for (const kind of KINDS) {323291 const list = hits.filter((h) => h.r.kind === kind).sort((a, b) => b.s - a.s);324292 if (!list.length) continue;325293 heading(kind, String(list.length));326294 const shown = expanded.has(kind) ? list : list.slice(0, PER_KIND);327295 for (const h of shown) {328- const extra = h.inText ? snippet(h.r.text, terms) : h.r.meta || h.r.text.slice(0, 120);329- link(h.r.href, marked(h.r.title, terms), h.r.mono, extra ? marked(extra, terms) : null);296+ const extra = h.inText ? snippet(h.r.text, ts) : h.r.meta || h.r.text.slice(0, 120);297+ link(h.r.href, marked(h.r.title, ts), h.r.mono, extra ? marked(extra, ts) : null);330298 }331299 if (shown.length < list.length) moreRow(`${list.length - shown.length} more`, () => expanded.add(kind));332300 }@@ -51,3 +51,7 @@ lacks "$FORGE_WWW/index.html" 'data-code='
5151 ls "$FORGE_WWW/assets" >"$T/assets"5252 has "$T/assets" '^sqlite3\.[0-9A-Za-z_-]{10}\.wasm$'5353 has "$FORGE_WWW/assets/versions.json" 'sqlite3\.[0-9A-Za-z_-]{10}\.wasm'54+# The worker imports the query parser (query.ts) by its shipped, hashed name.55+w=$(ls "$FORGE_WWW"/assets/codesearch.*.js)56+has "$w" "^import \{ [a-z, ]+ \} from '\./query\.[0-9A-Za-z_-]{10}\.js';$"57+exists "$FORGE_WWW/assets/$(sed -n "s|^import .* from '\./\(query\.[^']*\)';\$|\1|p" "$w")"@@ -0,0 +1,133 @@
1+# The search reads a query the same way in every section: terms split on2+# whitespace, a quoted phrase is one term that must appear as written3+# (ignoring case, and a run of whitespace in it matching any run), and every4+# term must match. Commits, reviews, wiki pages and repos match through5+# site/assets/query.ts, as the browser does, and code search's words come6+# out the same and find lines the same way.7+. "$(dirname "$0")/../lib.sh"8+user dami9+repo dami/greet "say hello world"10+repo dami/other "hello there, world"11+clone dami dami/greet "$T/w/dami"12+at "$T/w/dami" commit "Fix the search box"13+at "$T/w/dami" commit "Search fix for the box"14+tick15+at "$T/w/dami" git commit -q --allow-empty -m "Tidy the box" -m "This is to fix the search for good."16+at "$T/w/dami" commit "box search fix"17+printf 'width = height\ndepth = length\ntotal\tcount\nx = 1\ny = 2\n' >"$T/w/dami/sizes.txt"18+at "$T/w/dami" commit "Align the columns"19+push_as dami "$T/w/dami" origin main20+at "$T/w/dami" git switch -q -c dami/wide21+at "$T/w/dami" commit "Make the search box wider"22+push_as dami "$T/w/dami" origin dami/wide23+build24+ix=$FORGE_WWW/dami/greet/search.json25+exists "$ix"26+terms() { node "$T_HOME/test/search-phrases.ts" terms "$@"; }27+match() { node "$T_HOME/test/search-phrases.ts" match "$@"; }28+29+# The parser. A phrase is one term, spaces and all; words are terms of their30+# own; an unclosed quote runs to the end, as a phrase; empty quotes (or only31+# spaces in them) are no term; a quote inside a word is a character. Code32+# search reads the same terms, less any inside another (a line with the33+# longer one has it).34+ok terms 'fix search'35+eq "$(output)" 'fix search => [fix] [search] | code [fix] [search]'36+ok terms '"Fix the Search"'37+eq "$(output)" '"Fix the Search" => [fix the search] | code [fix the search]'38+ok terms '"the search'39+eq "$(output)" '"the search => [the search] | code [the search]'40+# A run of whitespace in a phrase is kept as one space, which matches any41+# run of whitespace (below); spaces at its ends stay, one each.42+ok terms '"fix the search"' '" fix "'43+eq "$(output)" '"fix the search" => [fix the search] | code [fix the search]44+" fix " => [ fix ] | code [ fix ]'45+ok terms '""' '" "' '"'46+eq "$(output)" '"" => - | code -47+" " => - | code -48+" => - | code -'49+ok terms '"search box" fix' 'fix "search box"'50+eq "$(output)" '"search box" fix => [search box] [fix] | code [search box] [fix]51+fix "search box" => [fix] [search box] | code [fix] [search box]'52+ok terms 'search "search box"' 'Fix FIX'53+eq "$(output)" 'search "search box" => [search] [search box] | code [search box]54+Fix FIX => [fix] | code [fix]'55+ok terms "it's\"" 'a"b c' '"fix"box'56+eq "$(output)" "it's\" => [it's\"] | code [it's\"]57+a\"b c => [a\"b] [c] | code [a\"b] [c]58+\"fix\"box => [fix] [box] | code [fix] [box]"59+60+# Words: every commit with both, in any order, in the subject or the body.61+ok match "$ix" 'fix search'62+eq "$(output | grep -c '^Commits: ')" 463+# A phrase: only the subject and the body that have it as written, ignoring64+# case, and it is marked whole.65+ok match "$ix" '"fix the search"'66+eq "$(output | grep '^Commits: ' | sed 's/ | .*//')" 'Commits: [Fix the search] box67+Commits: Tidy the box'68+has "$T_LOG" '^Commits: Tidy the box \| This is to \[fix the search\] for good\.$'69+# A run of whitespace in a phrase matches any run, both ways: spaced out,70+# it finds single spaces, and single spaces find a spaced-out subject,71+# marked whole.72+ok match "$ix" '"fix the search"'73+eq "$(output | grep '^Commits: ' | sed 's/ | .*//')" 'Commits: [Fix the search] box74+Commits: Tidy the box'75+ok match "$ix" '"align the columns"'76+eq "$(output | sed 's/ | .*//')" 'Commits: [Align the columns]'77+ok match "$ix" '"align the columns"'78+eq "$(output | sed 's/ | .*//')" 'Commits: [Align the columns]'79+ok match "$ix" '"align thecolumns"'80+eq "$(output)" ''81+# Unclosed, it is the same phrase.82+ok match "$ix" '"FIX THE SEARCH'83+eq "$(output | grep '^Commits: ' | sed 's/ | .*//')" 'Commits: [Fix the search] box84+Commits: Tidy the box'85+# Mixed: the phrase and the word must both match.86+ok match "$ix" '"search box" fix'87+eq "$(output)" "$(match "$ix" 'fix "search box"')"88+eq "$(output | sed 's/ | .*//')" 'Commits: [Fix] the [search box]'89+# Reviews too (and the review's commit, which the index has as well).90+ok match "$ix" '"search box"'91+eq "$(output | sed 's/ | .*//')" 'Reviews: #1 Make the [search box] wider92+Commits: Make the [search box] wider93+Commits: Fix the [search box]'94+# Nothing to look for: empty quotes match nothing, as an empty query does.95+ok match "$ix" '""'96+eq "$(output)" ''97+ok match "$ix" '"box search fix"'98+eq "$(output | sed 's/ | .*//')" 'Commits: [box search fix]'99+ok match "$ix" '"fix the box"'100+eq "$(output)" ''101+102+# Repos on the index page: names and descriptions, the same way.103+ok match "$FORGE_WWW/repos.json" 'hello world'104+eq "$(output)" 'Repositories: dami/greet | say [hello] [world]105+Repositories: dami/other | [hello] there, [world]'106+ok match "$FORGE_WWW/repos.json" '"hello world"'107+eq "$(output)" 'Repositories: dami/greet | say [hello world]'108+109+# Code search finds lines the same way, both ways, tabs too: the index110+# looks up each piece of 3 characters or more, near the others, and the111+# line check takes any run of whitespace for a space. A phrase without such112+# a piece is looked up as written, so it finds single-spaced lines only.113+db=$FORGE_WWW/dami/greet/code.db114+code() { node "$T_HOME/test/code-search-bench.ts" sql "$db" "$1"; }115+ok code '"width = height"'116+eq "$(output)" 'sizes.txt:1: width = height'117+ok code '"depth = length"'118+eq "$(output)" 'sizes.txt:2: depth = length'119+ok code '"total count"'120+eq "$(output)" "$(printf 'sizes.txt:3: total\tcount')"121+ok code '"WIDTH = height" "depth"'122+eq "$(output)" ''123+ok code '"x = 1"'124+eq "$(output)" 'sizes.txt:4: x = 1'125+ok code '"y = 2"'126+eq "$(output)" ''127+128+# Pages load search.js as a module, which imports query.js by its shipped name.129+has "$FORGE_WWW/dami/greet/index.html" '<script type="module" src="/assets/search\.[0-9A-Za-z_-]{10}\.js"></script>'130+has "$FORGE_WWW/index.html" '<script type="module" src="/assets/search\.[0-9A-Za-z_-]{10}\.js"></script>'131+s=$(ls "$FORGE_WWW"/assets/search.*.js)132+has "$s" "^import \{ [a-zA-Z, ]+ \} from '\./query\.[0-9A-Za-z_-]{10}\.js';$"133+exists "$FORGE_WWW/assets/$(sed -n "s|^import .* from '\./\(query\.[^']*\)';\$|\1|p" "$s")"@@ -0,0 +1,57 @@
1+// How the search reads a query and what it matches, run in Node with the2+// browser's own code: site/assets/query.ts, which search.ts matches every3+// section with, and codesearch.ts's words for file contents.4+//5+// node test/search-phrases.ts terms <query>... each query's terms, then code search's6+// node test/search-phrases.ts match <search.json> <query> what the sections list7+//8+// `terms` prints a line per query: `<query> => [term] [term] | code [word]`,9+// with - for none.10+// `match` builds the records from a built search.json (or repos.json) the11+// way search.ts's load() does, less the file list, and prints a line per12+// hit in rank order within its kind: `<kind>: <title> | <second line>`,13+// with each marked run in [brackets].14+import fs from 'node:fs';15+import { marks, score, snippet, terms } from '../site/assets/query.ts';16+import { words } from '../site/assets/codesearch.ts';17+18+const [mode, ...args] = process.argv.slice(2);19+const show = (ts: string[]): string => ts.map((t) => `[${t}]`).join(' ') || '-';20+21+interface Rec {22+ kind: string;23+ title: string;24+ meta: string;25+ text: string;26+ key: string;27+}28+29+if (mode === 'terms') {30+ for (const q of args) console.log(`${q} => ${show(terms(q))} | code ${show(words(q))}`);31+} else if (mode === 'match') {32+ const [file, q] = args;33+ type Row = (string | number)[];34+ const ix = JSON.parse(fs.readFileSync(file, 'utf8')) as { commits?: Row[]; reviews?: Row[]; wiki?: Row[]; repos?: Row[] };35+ const recs: Rec[] = [];36+ const rec = (kind: string, title: string, meta = '', text = '', key = ''): number => recs.push({ kind, title, meta, text, key });37+ for (const [slug, desc] of ix.repos ?? []) rec('Repositories', String(slug), '', String(desc));38+ for (const [id, title, branch, author, status] of ix.reviews ?? []) rec('Reviews', `#${id} ${title}`, `${branch}, ${author ? author + ', ' : ''}${status}`);39+ for (const [sha, subject, author, date, body] of ix.commits ?? []) rec('Commits', String(subject), `${String(sha).slice(0, 7)} ${author}, ${date}`, String(body), String(sha));40+ for (const [, title, text] of ix.wiki ?? []) rec('Wiki', String(title), '', String(text));41+ const ts = terms(q);42+ const marked = (s: string): string => marks(s, ts).map(([t, hit]) => (hit ? `[${t}]` : t)).join('');43+ for (const kind of ['Repositories', 'Reviews', 'Commits', 'Wiki']) {44+ const hits = recs45+ .filter((r) => r.kind === kind && ts.length)46+ .map((r) => ({ r, m: score([r.title.toLowerCase(), r.meta.toLowerCase(), r.text.toLowerCase()], r.key, ts) }))47+ .filter((h) => h.m)48+ .sort((a, b) => b.m!.s - a.m!.s);49+ for (const { r, m } of hits) {50+ const extra = m!.inText ? snippet(r.text, ts) : r.meta || r.text.slice(0, 120);51+ console.log(`${kind}: ${marked(r.title)}${extra ? ` | ${marked(extra)}` : ''}`);52+ }53+ }54+} else {55+ console.error('usage: node test/search-phrases.ts terms <query>... | match <search.json> <query>');56+ process.exit(2);57+}