Files, commits, reviews, wiki and repos read a query like code search: a quoted phrase is one term, matched as written, with one shared parser (query.ts).
revisions4
| r4 | cat | 9a4fbb2 | what changed since r3 | |
| r3 | cat | bcfdd90 | what changed since r2 behavior change | |
| r2 | cat | 851ac97 | what changed since r1 behavior change | |
| r1 | cat | 4ff8b62 |
No comments yet: forge comment 3 adds one.
1/4 Read code search queries with a parser of its own module
Code search's query parsing (words split on whitespace, a quoted phrase as one literal string, a word inside another dropped) moves out of codesearch.ts into site/assets/query.ts, so the other search sections can share it next instead of keeping their own copy in search.ts. Site scripts had no imports, since a build ships each under a hashed name. A build now rewrites a script's `from './x.ts'` import to the other file's hashed name, so the importer's name changes with what it imports. Code search behaves and fetches exactly as before (test/code-search-bench.ts gives the same rounds, requests and bytes); code-search-index checks that the shipped worker imports query.js by the name it is shipped under.
| M | lib/ | 26 | ++++++++++++++++++++------ |
| M | site/ | 16 | ++++++++-------- |
| A | site/ | 16 | ++++++++++++++++ |
| M | test/ | 4 | ++++ |
@@ -42,7 +42,7 @@ export interface Asset {
4242 data: Buffer;4343 }4444 45-// By short name: style, logo, fixi, ssexi, diffs, keys, live, search,45+// By short name: style, logo, fixi, ssexi, diffs, keys, live, search, query,4646 // codesearch, login, ui, mermaid, sqlite3, sqlite3wasm, webauthn.4747 export type Assets = Record<string, Asset>;4848 @@ -56,17 +56,31 @@ export function assets(): Assets {
5656 // Rebuilt only when a source changes (the dev server rebuilds in one process).5757 const key = files.map(([, , f]) => `${f} ${fs.statSync(f).mtimeMs}`).join('\n');5858 if (memo?.key === key) return memo.assets;59- const out: Assets = {};60- for (const [short, base, f] of files) {59+ // Our scripts import each other as './query.ts', which Node and tsc read60+ // as it is; shipped, that is the other file's hashed name, so a script's61+ // name changes with what it imports.62+ const made = new Map<string, Asset>();63+ const make = ([short, base, f]: [string, string, string]): Asset => {64+ const done = made.get(short);65+ if (done) return done;6166 let data = fs.readFileSync(f);6267 let ext = path.extname(base);6368 const name = base.split('.')[0];6469 if (ext === '.ts') {65- data = Buffer.from(stripTypeScriptTypes(data.toString('utf8')));70+ const js = stripTypeScriptTypes(data.toString('utf8')).replace(/(\bfrom\s*)'\.\/([\w-]+\.ts)'/g, (_, from: string, dep: string) => {71+ const d = files.find(([, b, p]) => b === dep && path.dirname(p) === src);72+ if (!d) throw new Error(`${base} imports ./${dep}, which is not in site/assets`);73+ return `${from}'./${make(d).file}'`;74+ });75+ data = Buffer.from(js);6676 ext = '.js';6777 }68- out[short] = { file: `${name}.${hash(data)}${ext}`, data };69- }78+ const a = { file: `${name}.${hash(data)}${ext}`, data };79+ made.set(short, a);80+ return a;81+ };82+ const out: Assets = {};83+ for (const f of files) out[f[0]] = make(f);7084 memo = { key, assets: out };7185 return out;7286 }@@ -12,6 +12,8 @@
1212 // the index leaf of each term (from FTS5's small %_idx table), then the page1313 // of each matching chunk (from the chunk table's root page).1414 15+import { narrow, terms } from './query.ts';16+1517 // Bytes [from, to) of url (from < 0: the last -from bytes), or the whole1618 // file if the server ignores ranges; where they start, the file's size, and1719 // its validator, which must not change while it is open.@@ -423,14 +425,12 @@ export function match(q: string): string {
423425 return n <= 5 ? str(q) : `NEAR(${cover(q).map(str).join(' ')}, ${n - 4})`;424426 }425427 426-// A query's words, lower case, as the other search sections split theirs:427-// on whitespace, except that a quoted phrase ("like this") is one literal428-// string, spaces and all (an unclosed quote runs to the end). A word that429-// is part of another goes, since every line with the longer one has it.430-export function words(q: string): string[] {431- const all = [...q.toLowerCase().replaceAll('\0', '').matchAll(/"([^"]*)"?|(\S+)/g)].map((m) => m[1] ?? m[2]).filter((w) => w.trim());432- return [...new Set(all)].filter((w, i, ws) => !ws.some((o, j) => j !== i && o.length > w.length && o.includes(w)));433-}428+// A query's words, lower case, its terms as query.ts reads them: split on429+// whitespace, except that a quoted phrase ("like this") is one literal430+// string, spaces and all (an unclosed quote runs to the end).431+// A word that is part of another goes, since every line with the longer one432+// has it, and so do NULs, which an FTS5 string can't hold.433+export const words = (q: string): string[] => narrow(terms(q.replaceAll('\0', '')));434434 435435 // The words the trigram index can look up; shorter ones only filter lines.436436 export const indexed = (ws: string[]): string[] => ws.filter((w) => [...w].length >= MIN);@@ -0,0 +1,16 @@
1+// A search query's terms, as codesearch.ts reads them for the lines of2+// files: split on whitespace, except that a quoted phrase ("like this") is3+// one term, spaces and all, which must appear as written. A quote opens a4+// phrase only at the start of a term (in `it's"` or `a"b` it is just a5+// character), an unclosed one runs to the end of the query, and empty quotes6+// are no term. Every term must match, ignoring case.7+8+// A query's terms, lower case, each once.9+export function terms(q: string): string[] {10+ const all = [...q.toLowerCase().matchAll(/"([^"]*)"?|(\S+)/g)].map((m) => m[1] ?? m[2]).filter((t) => t.trim());11+ return [...new Set(all)];12+}13+14+// The terms a search within one line needs: a term inside another goes,15+// since every line with the longer one has it.16+export const narrow = (ts: string[]): string[] => ts.filter((t, i) => !ts.some((o, j) => j !== i && o.length > t.length && o.includes(t)));@@ -51,3 +51,7 @@ lacks "$FORGE_WWW/index.html" 'data-code='
5151 ls "$FORGE_WWW/assets" >"$T/assets"5252 has "$T/assets" '^sqlite3\.[0-9A-Za-z_-]{10}\.wasm$'5353 has "$FORGE_WWW/assets/versions.json" 'sqlite3\.[0-9A-Za-z_-]{10}\.wasm'54+# The worker imports the query parser (query.ts) by its shipped, hashed name.55+w=$(ls "$FORGE_WWW"/assets/codesearch.*.js)56+has "$w" "^import \{ narrow, terms \} from '\./query\.[0-9A-Za-z_-]{10}\.js';$"57+exists "$FORGE_WWW/assets/$(sed -n "s|^import .* from '\./\(query\.[^']*\)';\$|\1|p" "$w")"2/4 Match quoted phrases in every search section
The search overlay's sections (files, commits, reviews, wiki, and repos on the index and owner pages) split a query on whitespace only, while code search took "a quoted phrase" as one literal string. Now all read a query with query.ts: a quoted phrase is one term that must appear as written, ignoring case; unquoted words are terms of their own; every term must match. An unclosed quote runs to the end as a phrase, so a phrase narrows as it is typed; empty quotes are no term; a quote inside a word is a character. Results mark a phrase as a whole. Scoring, marking and snippets move into query.ts beside the parser, untouched otherwise, so that test/search-phrases.ts can run them in Node on a built search.json. Pages load search.js as a module to import it. The search-phrases scenario records the parser over a table of queries (phrase against words, unclosed and empty quotes, mixed), what commits, reviews and repos match and how they are marked, and that the shipped search.js imports query.js by its hashed name.
| M | README.md | 20 | +++++- |
| M | site/ | 2 | +- |
| M | site/ | 67 | ++++++++++++++++++-- |
| M | site/ | 100 | ++++++++++-------------------- |
| A | test/ | 94 | ++++++++++++++++++++++++++++ |
| A | test/ | 57 | +++++++++++++++++ |
@@ -555,6 +555,19 @@ first time the search opens, and matching runs in the browser. Commit bodies
555555 are cut to 240 characters and wiki pages to 2000, to keep the index small.556556 `t` still finds files by name on the current branch.557557 558+Every section reads a query the same way (`site/assets/query.ts`): terms559+split on whitespace, in any order, and a quoted phrase (`"fix the search"`)560+is one term, spaces and all, that must appear as written. Every term must561+match, ignoring case, and results mark each match, a phrase as a whole. A562+quote opens a phrase only at the start of a term (in `it's"` it is just a563+character), an unclosed quote runs to the end of the query as a phrase, so a564+phrase narrows as you type it, and empty quotes are no term. So565+`fix search` finds "Search fix for the box" and `"fix the search"` doesn't.566+`node test/search-phrases.ts` runs the same code in Node, for567+`test/scenarios/search-phrases.sh`. Pages load `search.ts` as a module; it568+and `codesearch.ts` import `./query.ts`, which a build rewrites to that569+file's content-hashed name.570+558571 #### Code search559572 560573 File contents are in a second index per repo, `/<owner>/<repo>/code.db`,@@ -569,9 +582,10 @@ pages mirror on R2 through the edge, whose fallback passes the header on.
569582 Once the query has a word of 3 characters (a trigram can't match less), it570583 goes to a worker, and a Code section lists each line that has every word,571584 ignoring case, as `path:line`, linked to that line of the file's page (the572-source page for markdown). Words split on whitespace, as in the other573-sections, in any order, and a quoted phrase (`"import fs from"`) is one574-literal string, spaces and all. Words of 3 characters or more are looked up585+source page for markdown). Its words are the terms the other sections read,586+so a quoted phrase (`"import fs from"`) is one literal string, spaces and587+all; a word inside another goes, since a line with the longer one has it.588+Words of 3 characters or more are looked up575589 in the index; shorter ones only filter the lines it finds. It loads nothing576590 until then: the first code query starts the worker, which loads577591 `sqlite3.wasm` and sqlite-wasm's script, both content-hashed assets. Results@@ -21,7 +21,7 @@
2121 <script src="{{ assets.ssexi }}" defer></script>2222 <script src="{{ assets.live }}" defer></script>2323 <script src="{{ assets.keys }}" defer></script>24-<script src="{{ assets.search }}" defer></script>24+<script type="module" src="{{ assets.search }}"></script>2525 <script src="{{ assets.ui }}" data-mermaid="{{ assets.mermaid }}" defer></script>2626 {% block scripts %}{% endblock %}2727 {% block head %}{{ m.meta(p.title if p and p.title else forge.config.name, r.description if r else '') }}{% endblock %}@@ -1,9 +1,12 @@
1-// A search query's terms, as codesearch.ts reads them for the lines of2-// files: split on whitespace, except that a quoted phrase ("like this") is3-// one term, spaces and all, which must appear as written. A quote opens a4-// phrase only at the start of a term (in `it's"` or `a"b` it is just a5-// character), an unclosed one runs to the end of the query, and empty quotes6-// are no term. Every term must match, ignoring case.1+// A search query, as every section of the search reads it: search.ts for2+// commits, reviews, wiki pages, file paths and repos, and codesearch.ts for3+// the lines of files. A query is terms split on whitespace, except that a4+// quoted phrase ("like this") is one term, spaces and all, which must appear5+// as written. A quote opens a phrase only at the start of a term (in6+// `it's"` or `a"b` it is just a character), an unclosed one runs to the end7+// of the query, and empty quotes are no term. Every term must match, ignoring8+// case. Nothing here touches the page, so test/search-phrases.ts checks it9+// in Node as the browser runs it.710 811 // A query's terms, lower case, each once.912 export function terms(q: string): string[] {@@ -14,3 +17,55 @@ export function terms(q: string): string[] {
1417 // The terms a search within one line needs: a term inside another goes,1518 // since every line with the longer one has it.1619 export const narrow = (ts: string[]): string[] => ts.filter((t, i) => !ts.some((o, j) => j !== i && o.length > t.length && o.includes(t)));20+21+// Whether a text has every term, ignoring case.22+export function hasAll(text: string, ts: string[]): boolean {23+ const low = text.toLowerCase();24+ return ts.every((t) => low.includes(t));25+}26+27+const wordStart = (s: string, i: number): boolean => i === 0 || /[^\p{L}\p{N}]/u.test(s[i - 1]);28+29+// How well a record matches, from its title, meta line and text (lower30+// case), and a key that matches by prefix (a commit's full sha); null unless31+// every term is in one of them. Matches in titles rank above matches in32+// text, and those at the start of a word above those inside one.33+export function score(low: [string, string, string], key: string, ts: string[]): { s: number; inText: boolean } | null {34+ let s = 0;35+ let inText = false;36+ for (const t of ts) {37+ const i = low[0].indexOf(t);38+ if (i >= 0) s += wordStart(low[0], i) ? 6 : 4;39+ else if (t.length >= 4 && key.startsWith(t)) s += 5;40+ else if (low[1].includes(t)) s += 2;41+ else if (low[2].includes(t)) {42+ s += 1;43+ inText = true;44+ } else return null;45+ }46+ return { s, inText };47+}48+49+// A text cut into runs, each marked if it is part of a term's match: a50+// phrase is marked whole, and overlapping matches make one run. Text whose51+// lower case is another length (a few letters) stays one plain run.52+export function marks(s: string, ts: string[]): [string, boolean][] {53+ const low = s.toLowerCase();54+ if (low.length !== s.length) return [[s, false]];55+ const hit = new Uint8Array(s.length);56+ for (const t of ts) for (let i = low.indexOf(t); i >= 0; i = low.indexOf(t, i + 1)) hit.fill(1, i, i + t.length);57+ const out: [string, boolean][] = [];58+ for (let i = 0, j = 0; i < s.length; i = j) {59+ while (j < s.length && hit[j] === hit[i]) j++;60+ out.push([s.slice(i, j), !!hit[i]]);61+ }62+ return out;63+}64+65+// The part of a long text around its first match.66+export function snippet(text: string, ts: string[]): string {67+ const low = text.toLowerCase();68+ const at = Math.min(...ts.map((t) => low.indexOf(t)).filter((i) => i >= 0));69+ const start = Math.max(0, at - 30);70+ return (start ? '…' : '') + text.slice(start, start + 120) + (start + 120 < text.length ? '…' : '');71+}@@ -1,14 +1,18 @@
11 // Search, on "/" or the header button. In a repo it covers commits, reviews,22 // wiki pages and file paths on the default branch; on the index and owner33 // pages, repo names and descriptions. The index is a static JSON file (see4-// lib/search.ts) fetched the first time the search opens. Every term must5-// match somewhere; matches in titles rank above matches in text.4+// lib/search.ts) fetched the first time the search opens. A query is terms5+// split on whitespace, "a phrase" in quotes being one, read the same way in6+// every section (query.ts). Every term must match somewhere; matches in7+// titles rank above matches in text.68 //79 // With a word of 3 characters it also searches file contents on the default8-// branch, for lines with every word ("a phrase" in quotes is one): the9-// query goes to a worker (codesearch.ts) that queries the repo's code index10-// with sqlite-wasm over range requests. The worker and sqlite-wasm load the11-// first time someone searches code.10+// branch, for lines with every term: the query goes to a worker11+// (codesearch.ts) that queries the repo's code index with sqlite-wasm over12+// range requests. The worker and sqlite-wasm load the first time someone13+// searches code.14+import { hasAll, marks, narrow, score as match, snippet, terms } from './query.ts';15+1216 (() => {1317 const button = document.querySelector<HTMLButtonElement>('button[data-search]');1418 if (!button) return;@@ -60,20 +64,13 @@
6064 error?: string;6165 }6266 // As in codesearch.ts: trigrams need 3 characters; 12 chunks for a first63- // look, 100 when asked for more. A query's words split on whitespace, but64- // a quoted phrase is one, spaces and all; a word inside another goes.67+ // look, 100 when asked for more. A line has every term, so a term inside68+ // another goes.6569 const CODE_MIN = 3;6670 const CODE_FIRST = 12;6771 const CODE_MORE = 100;68- const codeWords = (q: string): string[] => {69- const all = [...q.toLowerCase().matchAll(/"([^"]*)"?|(\S+)/g)].map((m) => m[1] ?? m[2]).filter((w) => w.trim());70- return [...new Set(all)].filter((w, i, ws) => !ws.some((o, j) => j !== i && o.length > w.length && o.includes(w)));71- };72+ const codeWords = (q: string): string[] => narrow(terms(q));7273 const codeable = (q: string): boolean => codeWords(q).some((w) => [...w].length >= CODE_MIN);73- const hasAll = (text: string, ws: string[]): boolean => {74- const low = text.toLowerCase();75- return ws.every((w) => low.includes(w));76- };7774 let codeUrl: string | null = null;7875 let blobBase = '';7976 let worker: Worker | null = null;@@ -129,50 +126,21 @@
129126 return list;130127 }131128 132- const wordStart = (s: string, i: number): boolean => i === 0 || /[^\p{L}\p{N}]/u.test(s[i - 1]);133-134- function score(r: Rec, terms: string[]): Hit | null {135- let s = 0;136- let inText = false;137- for (const t of terms) {138- const i = r.low[0].indexOf(t);139- if (i >= 0) s += wordStart(r.low[0], i) ? 6 : 4;140- else if (t.length >= 4 && r.key.startsWith(t)) s += 5;141- else if (r.low[1].includes(t)) s += 2;142- else if (r.low[2].includes(t)) {143- s += 1;144- inText = true;145- } else return null;146- }147- return { r, s, inText };148- }149-150- // Text with every occurrence of a term in <b>, built as nodes (never HTML).151- function marked(s: string, terms: string[]): Node[] {152- const low = s.toLowerCase();153- if (low.length !== s.length) return [document.createTextNode(s)];154- const hit = new Uint8Array(s.length);155- for (const t of terms) for (let i = low.indexOf(t); i >= 0; i = low.indexOf(t, i + 1)) hit.fill(1, i, i + t.length);156- const out: Node[] = [];157- for (let i = 0, j = 0; i < s.length; i = j) {158- while (j < s.length && hit[j] === hit[i]) j++;159- const text = document.createTextNode(s.slice(i, j));160- if (!hit[i]) out.push(text);161- else {162- const b = document.createElement('b');163- b.append(text);164- out.push(b);165- }166- }167- return out;129+ function score(r: Rec, ts: string[]): Hit | null {130+ const m = match(r.low, r.key, ts);131+ return m && { r, ...m };168132 }169133 170- // The part of a long text around its first match.171- function snippet(text: string, terms: string[]): string {172- const low = text.toLowerCase();173- const at = Math.min(...terms.map((t) => low.indexOf(t)).filter((i) => i >= 0));174- const start = Math.max(0, at - 30);175- return (start ? '…' : '') + text.slice(start, start + 120) + (start + 120 < text.length ? '…' : '');134+ // Text with every match of a term (a phrase whole) in <b>, built as nodes135+ // (never HTML).136+ function marked(s: string, ts: string[]): Node[] {137+ return marks(s, ts).map(([text, hit]) => {138+ const t = document.createTextNode(text);139+ if (!hit) return t;140+ const b = document.createElement('b');141+ b.append(t);142+ return b;143+ });176144 }177145 178146 function open(): void {@@ -304,7 +272,7 @@
304272 };305273 306274 const draw = (): void => {307- const terms = input.value.toLowerCase().split(/\s+/).filter(Boolean);275+ const ts = terms(input.value);308276 const q = input.value.trim();309277 ol.replaceChildren();310278 rows = [];@@ -312,21 +280,21 @@
312280 hint.textContent = failed ? 'The search index could not be loaded.' : 'Loading the search index';313281 return;314282 }315- if (!terms.length) {316- hint.textContent = repos ? 'Repository names and descriptions.' : codeUrl317- ? 'Searches commit messages, reviews, wiki pages, file paths and, for words of 3 characters or more, the lines of the files themselves ("a phrase" in quotes), on the default branch. Press t to find a file on this branch.'318- : 'Searches commit messages, reviews, wiki pages and file paths on the default branch. Press t to find a file on this branch.';283+ if (!ts.length) {284+ hint.textContent = repos ? 'Repository names and descriptions; "a phrase" in quotes matches as written.' : codeUrl285+ ? 'Searches commit messages, reviews, wiki pages, file paths and, for words of 3 characters or more, the lines of the files themselves, on the default branch; "a phrase" in quotes matches as written. Press t to find a file on this branch.'286+ : 'Searches commit messages, reviews, wiki pages and file paths on the default branch; "a phrase" in quotes matches as written. Press t to find a file on this branch.';319287 return;320288 }321- const hits = all.map((r) => score(r, terms)).filter((h): h is Hit => !!h);289+ const hits = all.map((r) => score(r, ts)).filter((h): h is Hit => !!h);322290 for (const kind of KINDS) {323291 const list = hits.filter((h) => h.r.kind === kind).sort((a, b) => b.s - a.s);324292 if (!list.length) continue;325293 heading(kind, String(list.length));326294 const shown = expanded.has(kind) ? list : list.slice(0, PER_KIND);327295 for (const h of shown) {328- const extra = h.inText ? snippet(h.r.text, terms) : h.r.meta || h.r.text.slice(0, 120);329- link(h.r.href, marked(h.r.title, terms), h.r.mono, extra ? marked(extra, terms) : null);296+ const extra = h.inText ? snippet(h.r.text, ts) : h.r.meta || h.r.text.slice(0, 120);297+ link(h.r.href, marked(h.r.title, ts), h.r.mono, extra ? marked(extra, ts) : null);330298 }331299 if (shown.length < list.length) moreRow(`${list.length - shown.length} more`, () => expanded.add(kind));332300 }@@ -0,0 +1,94 @@
1+# The search reads a query the same way in every section: terms split on2+# whitespace, a quoted phrase is one term that must appear as written3+# (ignoring case), and every term must match. Commits, reviews, wiki pages4+# and repos match through site/assets/query.ts, as the browser does, and5+# code search's words come out the same.6+. "$(dirname "$0")/../lib.sh"7+user dami8+repo dami/greet "say hello world"9+repo dami/other "hello there, world"10+clone dami dami/greet "$T/w/dami"11+at "$T/w/dami" commit "Fix the search box"12+at "$T/w/dami" commit "Search fix for the box"13+tick14+at "$T/w/dami" git commit -q --allow-empty -m "Tidy the box" -m "This is to fix the search for good."15+at "$T/w/dami" commit "box search fix"16+push_as dami "$T/w/dami" origin main17+at "$T/w/dami" git switch -q -c dami/wide18+at "$T/w/dami" commit "Make the search box wider"19+push_as dami "$T/w/dami" origin dami/wide20+build21+ix=$FORGE_WWW/dami/greet/search.json22+exists "$ix"23+terms() { node "$T_HOME/test/search-phrases.ts" terms "$@"; }24+match() { node "$T_HOME/test/search-phrases.ts" match "$@"; }25+26+# The parser. A phrase is one term, spaces and all; words are terms of their27+# own; an unclosed quote runs to the end, as a phrase; empty quotes (or only28+# spaces in them) are no term; a quote inside a word is a character. Code29+# search reads the same terms, less any inside another (a line with the30+# longer one has it).31+ok terms 'fix search'32+eq "$(output)" 'fix search => [fix] [search] | code [fix] [search]'33+ok terms '"Fix the Search"'34+eq "$(output)" '"Fix the Search" => [fix the search] | code [fix the search]'35+ok terms '"the search'36+eq "$(output)" '"the search => [the search] | code [the search]'37+ok terms '""' '" "' '"'38+eq "$(output)" '"" => - | code -39+" " => - | code -40+" => - | code -'41+ok terms '"search box" fix' 'fix "search box"'42+eq "$(output)" '"search box" fix => [search box] [fix] | code [search box] [fix]43+fix "search box" => [fix] [search box] | code [fix] [search box]'44+ok terms 'search "search box"' 'Fix FIX'45+eq "$(output)" 'search "search box" => [search] [search box] | code [search box]46+Fix FIX => [fix] | code [fix]'47+ok terms "it's\"" 'a"b c' '"fix"box'48+eq "$(output)" "it's\" => [it's\"] | code [it's\"]49+a\"b c => [a\"b] [c] | code [a\"b] [c]50+\"fix\"box => [fix] [box] | code [fix] [box]"51+52+# Words: every commit with both, in any order, in the subject or the body.53+ok match "$ix" 'fix search'54+eq "$(output | grep -c '^Commits: ')" 455+# A phrase: only the subject and the body that have it as written, ignoring56+# case, and it is marked whole.57+ok match "$ix" '"fix the search"'58+eq "$(output | grep '^Commits: ' | sed 's/ | .*//')" 'Commits: [Fix the search] box59+Commits: Tidy the box'60+has "$T_LOG" '^Commits: Tidy the box \| This is to \[fix the search\] for good\.$'61+# Unclosed, it is the same phrase.62+ok match "$ix" '"FIX THE SEARCH'63+eq "$(output | grep '^Commits: ' | sed 's/ | .*//')" 'Commits: [Fix the search] box64+Commits: Tidy the box'65+# Mixed: the phrase and the word must both match.66+ok match "$ix" '"search box" fix'67+eq "$(output)" "$(match "$ix" 'fix "search box"')"68+eq "$(output | sed 's/ | .*//')" 'Commits: [Fix] the [search box]'69+# Reviews too (and the review's commit, which the index has as well).70+ok match "$ix" '"search box"'71+eq "$(output | sed 's/ | .*//')" 'Reviews: #1 Make the [search box] wider72+Commits: Make the [search box] wider73+Commits: Fix the [search box]'74+# Nothing to look for: empty quotes match nothing, as an empty query does.75+ok match "$ix" '""'76+eq "$(output)" ''77+ok match "$ix" '"box search fix"'78+eq "$(output | sed 's/ | .*//')" 'Commits: [box search fix]'79+ok match "$ix" '"fix the box"'80+eq "$(output)" ''81+82+# Repos on the index page: names and descriptions, the same way.83+ok match "$FORGE_WWW/repos.json" 'hello world'84+eq "$(output)" 'Repositories: dami/greet | say [hello] [world]85+Repositories: dami/other | [hello] there, [world]'86+ok match "$FORGE_WWW/repos.json" '"hello world"'87+eq "$(output)" 'Repositories: dami/greet | say [hello world]'88+89+# Pages load search.js as a module, which imports query.js by its shipped name.90+has "$FORGE_WWW/dami/greet/index.html" '<script type="module" src="/assets/search\.[0-9A-Za-z_-]{10}\.js"></script>'91+has "$FORGE_WWW/index.html" '<script type="module" src="/assets/search\.[0-9A-Za-z_-]{10}\.js"></script>'92+s=$(ls "$FORGE_WWW"/assets/search.*.js)93+has "$s" "^import \{ hasAll, marks, narrow, score as match, snippet, terms \} from '\./query\.[0-9A-Za-z_-]{10}\.js';$"94+exists "$FORGE_WWW/assets/$(sed -n "s|^import .* from '\./\(query\.[^']*\)';\$|\1|p" "$s")"@@ -0,0 +1,57 @@
1+// How the search reads a query and what it matches, run in Node with the2+// browser's own code: site/assets/query.ts, which search.ts matches every3+// section with, and codesearch.ts's words for file contents.4+//5+// node test/search-phrases.ts terms <query>... each query's terms, then code search's6+// node test/search-phrases.ts match <search.json> <query> what the sections list7+//8+// `terms` prints a line per query: `<query> => [term] [term] | code [word]`,9+// with - for none.10+// `match` builds the records from a built search.json (or repos.json) the11+// way search.ts's load() does, less the file list, and prints a line per12+// hit in rank order within its kind: `<kind>: <title> | <second line>`,13+// with each marked run in [brackets].14+import fs from 'node:fs';15+import { marks, score, snippet, terms } from '../site/assets/query.ts';16+import { words } from '../site/assets/codesearch.ts';17+18+const [mode, ...args] = process.argv.slice(2);19+const show = (ts: string[]): string => ts.map((t) => `[${t}]`).join(' ') || '-';20+21+interface Rec {22+ kind: string;23+ title: string;24+ meta: string;25+ text: string;26+ key: string;27+}28+29+if (mode === 'terms') {30+ for (const q of args) console.log(`${q} => ${show(terms(q))} | code ${show(words(q))}`);31+} else if (mode === 'match') {32+ const [file, q] = args;33+ type Row = (string | number)[];34+ const ix = JSON.parse(fs.readFileSync(file, 'utf8')) as { commits?: Row[]; reviews?: Row[]; wiki?: Row[]; repos?: Row[] };35+ const recs: Rec[] = [];36+ const rec = (kind: string, title: string, meta = '', text = '', key = ''): number => recs.push({ kind, title, meta, text, key });37+ for (const [slug, desc] of ix.repos ?? []) rec('Repositories', String(slug), '', String(desc));38+ for (const [id, title, branch, author, status] of ix.reviews ?? []) rec('Reviews', `#${id} ${title}`, `${branch}, ${author ? author + ', ' : ''}${status}`);39+ for (const [sha, subject, author, date, body] of ix.commits ?? []) rec('Commits', String(subject), `${String(sha).slice(0, 7)} ${author}, ${date}`, String(body), String(sha));40+ for (const [, title, text] of ix.wiki ?? []) rec('Wiki', String(title), '', String(text));41+ const ts = terms(q);42+ const marked = (s: string): string => marks(s, ts).map(([t, hit]) => (hit ? `[${t}]` : t)).join('');43+ for (const kind of ['Repositories', 'Reviews', 'Commits', 'Wiki']) {44+ const hits = recs45+ .filter((r) => r.kind === kind && ts.length)46+ .map((r) => ({ r, m: score([r.title.toLowerCase(), r.meta.toLowerCase(), r.text.toLowerCase()], r.key, ts) }))47+ .filter((h) => h.m)48+ .sort((a, b) => b.m!.s - a.m!.s);49+ for (const { r, m } of hits) {50+ const extra = m!.inText ? snippet(r.text, ts) : r.meta || r.text.slice(0, 120);51+ console.log(`${kind}: ${marked(r.title)}${extra ? ` | ${marked(extra)}` : ''}`);52+ }53+ }54+} else {55+ console.error('usage: node test/search-phrases.ts terms <query>... | match <search.json> <query>');56+ process.exit(2);57+}3/4 Count a run of whitespace in a quoted phrase as one space
The search indexes collapse whitespace (commit bodies and wiki text are flattened to single spaces), so a phrase typed with two spaces, like "fix the", could never match. Now a run of whitespace inside a quoted phrase is one space, in every section and in code search alike, which keeps the two reading a query the same way. Spaces at a phrase's ends stay, one each. Code lines keep their own whitespace, so a phrase no longer finds code aligned with several spaces. The bench's default queries fetch and find exactly what they did. The search-phrases scenario adds spaced-out phrases to its table and checks that one matches the same commits as its single-spaced form.
| M | README.md | 8 | ++++++-- |
| M | site/ | 3 | ++- |
| M | site/ | 14 | ++++++++------ |
| M | test/ | 9 | +++++++++ |
@@ -557,7 +557,9 @@ are cut to 240 characters and wiki pages to 2000, to keep the index small.
557557 558558 Every section reads a query the same way (`site/assets/query.ts`): terms559559 split on whitespace, in any order, and a quoted phrase (`"fix the search"`)560-is one term, spaces and all, that must appear as written. Every term must560+is one term, spaces and all, that must appear as written, except that a run561+of whitespace in it counts as one space, as in the indexes, which collapse562+whitespace (`"fix the"` is `"fix the"`). Every term must561563 match, ignoring case, and results mark each match, a phrase as a whole. A562564 quote opens a phrase only at the start of a term (in `it's"` it is just a563565 character), an unclosed quote runs to the end of the query as a phrase, so a@@ -584,7 +586,9 @@ goes to a worker, and a Code section lists each line that has every word,
584586 ignoring case, as `path:line`, linked to that line of the file's page (the585587 source page for markdown). Its words are the terms the other sections read,586588 so a quoted phrase (`"import fs from"`) is one literal string, spaces and587-all; a word inside another goes, since a line with the longer one has it.589+all, a run of whitespace as one space as everywhere (so it won't find code590+aligned with several spaces); a word inside another goes, since a line with591+the longer one has it.588592 Words of 3 characters or more are looked up589593 in the index; shorter ones only filter the lines it finds. It loads nothing590594 until then: the first code query starts the worker, which loads@@ -427,7 +427,8 @@ export function match(q: string): string {
427427 428428 // A query's words, lower case, its terms as query.ts reads them: split on429429 // whitespace, except that a quoted phrase ("like this") is one literal430-// string, spaces and all (an unclosed quote runs to the end).430+// string, spaces and all, a run of them as one (an unclosed quote runs to431+// the end).431432 // A word that is part of another goes, since every line with the longer one432433 // has it, and so do NULs, which an FTS5 string can't hold.433434 export const words = (q: string): string[] => narrow(terms(q.replaceAll('\0', '')));@@ -2,15 +2,17 @@
22 // commits, reviews, wiki pages, file paths and repos, and codesearch.ts for33 // the lines of files. A query is terms split on whitespace, except that a44 // quoted phrase ("like this") is one term, spaces and all, which must appear5-// as written. A quote opens a phrase only at the start of a term (in6-// `it's"` or `a"b` it is just a character), an unclosed one runs to the end7-// of the query, and empty quotes are no term. Every term must match, ignoring8-// case. Nothing here touches the page, so test/search-phrases.ts checks it9-// in Node as the browser runs it.5+// as written, except that a run of whitespace in it is one space, as in the6+// indexes (lib/search.ts flattens bodies and wiki text). A quote opens a7+// phrase only at the start of a term (in `it's"` or `a"b` it is just a8+// character), an unclosed one runs to the end of the query, and empty quotes9+// are no term. Every term must match, ignoring case. Nothing here touches10+// the page, so test/search-phrases.ts checks it in Node as the browser runs11+// it.1012 1113 // A query's terms, lower case, each once.1214 export function terms(q: string): string[] {13- const all = [...q.toLowerCase().matchAll(/"([^"]*)"?|(\S+)/g)].map((m) => m[1] ?? m[2]).filter((t) => t.trim());15+ const all = [...q.toLowerCase().matchAll(/"([^"]*)"?|(\S+)/g)].map((m) => m[2] ?? m[1].replace(/\s+/g, ' ')).filter((t) => t.trim());1416 return [...new Set(all)];1517 }1618 @@ -34,6 +34,11 @@ ok terms '"Fix the Search"'
3434 eq "$(output)" '"Fix the Search" => [fix the search] | code [fix the search]'3535 ok terms '"the search'3636 eq "$(output)" '"the search => [the search] | code [the search]'37+# A run of whitespace in a phrase is one space, as in the indexes; spaces38+# at its ends stay (one each).39+ok terms '"fix the search"' '" fix "'40+eq "$(output)" '"fix the search" => [fix the search] | code [fix the search]41+" fix " => [ fix ] | code [ fix ]'3742 ok terms '""' '" "' '"'3843 eq "$(output)" '"" => - | code -3944 " " => - | code -@@ -58,6 +63,10 @@ ok match "$ix" '"fix the search"'
5863 eq "$(output | grep '^Commits: ' | sed 's/ | .*//')" 'Commits: [Fix the search] box5964 Commits: Tidy the box'6065 has "$T_LOG" '^Commits: Tidy the box \| This is to \[fix the search\] for good\.$'66+# Spaced out, it is the same phrase.67+ok match "$ix" '"fix the search"'68+eq "$(output | grep '^Commits: ' | sed 's/ | .*//')" 'Commits: [Fix the search] box69+Commits: Tidy the box'6170 # Unclosed, it is the same phrase.6271 ok match "$ix" '"FIX THE SEARCH'6372 eq "$(output | grep '^Commits: ' | sed 's/ | .*//')" 'Commits: [Fix the search] box4/4 Let whitespace in a quoted phrase match any run of whitespace
Collapsing whitespace in a phrase lost code search's exact-spacing matches: a phrase could no longer find code aligned with several spaces. Now a run of whitespace in a phrase matches any run of whitespace, in every section and in code search, so "x = 1" and "x = 1" read alike and both find lines spaced either way. query.ts matches a phrase with a pattern that takes \s+ for each space (terms without spaces are still a plain indexOf). Code search can't look up trigrams that span whitespace of any length, so it looks up a phrase's pieces of 3 characters or more, all within NEAR distance of the phrase plus 32 characters per gap, and checks lines with the same pattern. A phrase without such a piece is looked up as written, finding only single-spaced lines. The bench's default queries fetch and find exactly what they did; a quoted phrase with spaces costs about what it did, a lookup or two more for its pieces. The search-phrases scenario checks both directions in a commit subject and in code search, a tab, and the short-piece limit.
| M | README.md | 17 | ++++++--- |
| M | site/ | 41 | ++++++++++++++++++---- |
| M | site/ | 54 | ++++++++++++++++++++++-------- |
| M | test/ | 2 | +- |
| M | test/ | 44 | +++++++++++++++++++++---- |
@@ -558,8 +558,8 @@ are cut to 240 characters and wiki pages to 2000, to keep the index small.
558558 Every section reads a query the same way (`site/assets/query.ts`): terms559559 split on whitespace, in any order, and a quoted phrase (`"fix the search"`)560560 is one term, spaces and all, that must appear as written, except that a run561-of whitespace in it counts as one space, as in the indexes, which collapse562-whitespace (`"fix the"` is `"fix the"`). Every term must561+of whitespace in it matches any run of whitespace: `"fix the"` finds562+`fix the`, and `"fix the"` finds `fix the`. Every term must563563 match, ignoring case, and results mark each match, a phrase as a whole. A564564 quote opens a phrase only at the start of a term (in `it's"` it is just a565565 character), an unclosed quote runs to the end of the query as a phrase, so a@@ -586,11 +586,16 @@ goes to a worker, and a Code section lists each line that has every word,
586586 ignoring case, as `path:line`, linked to that line of the file's page (the587587 source page for markdown). Its words are the terms the other sections read,588588 so a quoted phrase (`"import fs from"`) is one literal string, spaces and589-all, a run of whitespace as one space as everywhere (so it won't find code590-aligned with several spaces); a word inside another goes, since a line with591-the longer one has it.589+all, in which a run of whitespace matches any run, tabs and alignment too;590+a word inside another goes, since a line with the longer one has it.592591 Words of 3 characters or more are looked up593-in the index; shorter ones only filter the lines it finds. It loads nothing592+in the index; shorter ones only filter the lines it finds. A phrase is593+looked up by its pieces between whitespace, those of 3 characters or more,594+since no trigram may span whitespace that can be any length: all of them595+within `NEAR` distance of the phrase's length plus 32 characters per gap,596+and the lines found are checked with any run of whitespace for a space. A597+phrase without such a piece (`"x = 1"`) is looked up as written, so it finds598+only single-spaced lines. It loads nothing594599 until then: the first code query starts the worker, which loads595600 `sqlite3.wasm` and sqlite-wasm's script, both content-hashed assets. Results596601 come in path order; the first look reads 12 chunks, and "more" reads up to@@ -12,7 +12,7 @@
1212 // the index leaf of each term (from FTS5's small %_idx table), then the page1313 // of each matching chunk (from the chunk table's root page).1414 15-import { narrow, terms } from './query.ts';15+import { find, narrow, pieces, terms } from './query.ts';1616 1717 // Bytes [from, to) of url (from < 0: the last -from bytes), or the whole1818 // file if the server ignores ranges; where they start, the file's size, and@@ -418,17 +418,42 @@ export function cover(q: string): string[] {
418418 return at.map((i) => c.slice(i, i + 3).join(''));419419 }420420 421+// The strings the index looks up for a word: the word itself, but for a422+// phrase its pieces between whitespace of MIN characters or more, since a423+// space in a phrase stands for any run of whitespace and no trigram may span424+// one. A phrase without such a piece (`"x = 1"`) is looked up as written,425+// single spaces and all, so it finds only lines spaced that way.426+function lookups(w: string): string[] {427+ const ps = pieces(w).filter((p) => [...p].length >= MIN);428+ return ps.length ? ps : [w];429+}430+431+// Extra whitespace characters allowed per gap in a phrase: its pieces must432+// be near each other in the index, with this much room for runs of spaces433+// longer than one (alignment). Lines are checked afterwards anyway.434+const GAP = 32;435+421436 // The FTS5 query for a literal substring, ignoring case.422-export function match(q: string): string {437+function literal(q: string): string {423438 const n = [...q].length;424439 // NEAR counts the tokens between the first phrase and the last.425440 return n <= 5 ? str(q) : `NEAR(${cover(q).map(str).join(' ')}, ${n - 4})`;426441 }427442 443+// The FTS5 query for a word: a literal substring, ignoring case, but in a444+// phrase the trigrams of each piece, all within NEAR distance of the phrase445+// with GAP more characters per gap.446+export function match(w: string): string {447+ const ps = lookups(w);448+ if (ps.length === 1) return literal(ps[0]);449+ const gaps = pieces(w).length - 1;450+ return `NEAR(${ps.flatMap(cover).map(str).join(' ')}, ${[...w].length - 4 + GAP * gaps})`;451+}452+428453 // A query's words, lower case, its terms as query.ts reads them: split on429454 // whitespace, except that a quoted phrase ("like this") is one literal430-// string, spaces and all, a run of them as one (an unclosed quote runs to431-// the end).455+// string, spaces and all, a run of whitespace in it matching any run (an456+// unclosed quote runs to the end).432457 // A word that is part of another goes, since every line with the longer one433458 // has it, and so do NULs, which an FTS5 string can't hold.434459 export const words = (q: string): string[] => narrow(terms(q.replaceAll('\0', '')));@@ -444,9 +469,9 @@ export function lineMatch(line: string, ws: string[]): number {
444469 const low = line.toLowerCase();445470 let first = -1;446471 for (const w of ws) {447- const at = low.indexOf(w);448- if (at < 0) return -1;449- if (first < 0 || at < first) first = at;472+ const at = find(low, w);473+ if (!at) return -1;474+ if (first < 0 || at[0] < first) first = at[0];450475 }451476 return first;452477 }@@ -557,7 +582,7 @@ export class CodeIndex {
557582 // number; a doclist index's has bit 36 set too. Row 10 is the structure.558583 const enc = new TextEncoder();559584 const wanted = [1, 10];560- for (const t of new Set(indexed(ws).flatMap(cover))) {585+ for (const t of new Set(indexed(ws).flatMap(lookups).flatMap(cover))) {561586 const key = enc.encode('0' + t);562587 for (const seg of this.layout.segs) {563588 const first = (this.all('select pgno >> 1 from code_idx where segid = ? and term <= ? order by term desc limit 1', [seg, key])[0]?.[0] as number | undefined) ?? 1;@@ -2,15 +2,16 @@
22 // commits, reviews, wiki pages, file paths and repos, and codesearch.ts for33 // the lines of files. A query is terms split on whitespace, except that a44 // quoted phrase ("like this") is one term, spaces and all, which must appear5-// as written, except that a run of whitespace in it is one space, as in the6-// indexes (lib/search.ts flattens bodies and wiki text). A quote opens a7-// phrase only at the start of a term (in `it's"` or `a"b` it is just a8-// character), an unclosed one runs to the end of the query, and empty quotes9-// are no term. Every term must match, ignoring case. Nothing here touches10-// the page, so test/search-phrases.ts checks it in Node as the browser runs11-// it.5+// as written, except that a run of whitespace in it matches any run of6+// whitespace ("x = 1" finds `x = 1`, and the other way round). A quote7+// opens a phrase only at the start of a term (in `it's"` or `a"b` it is just8+// a character), an unclosed one runs to the end of the query, and empty9+// quotes are no term. Every term must match, ignoring case. Nothing here10+// touches the page, so test/search-phrases.ts checks it in Node as the11+// browser runs it.1212 13-// A query's terms, lower case, each once.13+// A query's terms, lower case, each once; a run of whitespace in a phrase14+// is kept as one space, which find() matches with any run.1415 export function terms(q: string): string[] {1516 const all = [...q.toLowerCase().matchAll(/"([^"]*)"?|(\S+)/g)].map((m) => m[2] ?? m[1].replace(/\s+/g, ' ')).filter((t) => t.trim());1617 return [...new Set(all)];@@ -20,10 +21,33 @@ export function terms(q: string): string[] {
2021 // since every line with the longer one has it.2122 export const narrow = (ts: string[]): string[] => ts.filter((t, i) => !ts.some((o, j) => j !== i && o.length > t.length && o.includes(t)));2223 24+// A term's pieces: a phrase's parts between whitespace (a word is one).25+export const pieces = (t: string): string[] => t.split(' ').filter(Boolean);26+27+// Where a term is in a lower case text, from `from` on: [start, end), or28+// null. A space in a term (only phrases have them) matches any run of29+// whitespace; terms without one are a plain indexOf.30+const patterns = new Map<string, RegExp>();31+export function find(low: string, t: string, from = 0): [number, number] | null {32+ if (!t.includes(' ')) {33+ const i = low.indexOf(t, from);34+ return i < 0 ? null : [i, i + t.length];35+ }36+ let re = patterns.get(t);37+ if (!re) {38+ if (patterns.size > 200) patterns.clear();39+ re = new RegExp(t.split(' ').map((p) => p.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')).join('\\s+'), 'g');40+ patterns.set(t, re);41+ }42+ re.lastIndex = from;43+ const m = re.exec(low);44+ return m ? [m.index, m.index + m[0].length] : null;45+}46+2347 // Whether a text has every term, ignoring case.2448 export function hasAll(text: string, ts: string[]): boolean {2549 const low = text.toLowerCase();26- return ts.every((t) => low.includes(t));50+ return ts.every((t) => find(low, t));2751 }2852 2953 const wordStart = (s: string, i: number): boolean => i === 0 || /[^\p{L}\p{N}]/u.test(s[i - 1]);@@ -36,11 +60,11 @@ export function score(low: [string, string, string], key: string, ts: string[]):
3660 let s = 0;3761 let inText = false;3862 for (const t of ts) {39- const i = low[0].indexOf(t);40- if (i >= 0) s += wordStart(low[0], i) ? 6 : 4;63+ const at = find(low[0], t);64+ if (at) s += wordStart(low[0], at[0]) ? 6 : 4;4165 else if (t.length >= 4 && key.startsWith(t)) s += 5;42- else if (low[1].includes(t)) s += 2;43- else if (low[2].includes(t)) {66+ else if (find(low[1], t)) s += 2;67+ else if (find(low[2], t)) {4468 s += 1;4569 inText = true;4670 } else return null;@@ -55,7 +79,7 @@ export function marks(s: string, ts: string[]): [string, boolean][] {
5579 const low = s.toLowerCase();5680 if (low.length !== s.length) return [[s, false]];5781 const hit = new Uint8Array(s.length);58- for (const t of ts) for (let i = low.indexOf(t); i >= 0; i = low.indexOf(t, i + 1)) hit.fill(1, i, i + t.length);82+ for (const t of ts) for (let m = find(low, t); m; m = find(low, t, m[0] + 1)) hit.fill(1, m[0], m[1]);5983 const out: [string, boolean][] = [];6084 for (let i = 0, j = 0; i < s.length; i = j) {6185 while (j < s.length && hit[j] === hit[i]) j++;@@ -67,7 +91,7 @@ export function marks(s: string, ts: string[]): [string, boolean][] {
6791 // The part of a long text around its first match.6892 export function snippet(text: string, ts: string[]): string {6993 const low = text.toLowerCase();70- const at = Math.min(...ts.map((t) => low.indexOf(t)).filter((i) => i >= 0));94+ const at = Math.min(...ts.map((t) => find(low, t)?.[0] ?? -1).filter((i) => i >= 0));7195 const start = Math.max(0, at - 30);7296 return (start ? '…' : '') + text.slice(start, start + 120) + (start + 120 < text.length ? '…' : '');7397 }@@ -53,5 +53,5 @@ has "$T/assets" '^sqlite3\.[0-9A-Za-z_-]{10}\.wasm$'
5353 has "$FORGE_WWW/assets/versions.json" 'sqlite3\.[0-9A-Za-z_-]{10}\.wasm'5454 # The worker imports the query parser (query.ts) by its shipped, hashed name.5555 w=$(ls "$FORGE_WWW"/assets/codesearch.*.js)56-has "$w" "^import \{ narrow, terms \} from '\./query\.[0-9A-Za-z_-]{10}\.js';$"56+has "$w" "^import \{ [a-z, ]+ \} from '\./query\.[0-9A-Za-z_-]{10}\.js';$"5757 exists "$FORGE_WWW/assets/$(sed -n "s|^import .* from '\./\(query\.[^']*\)';\$|\1|p" "$w")"@@ -1,8 +1,9 @@
11 # The search reads a query the same way in every section: terms split on22 # whitespace, a quoted phrase is one term that must appear as written3-# (ignoring case), and every term must match. Commits, reviews, wiki pages4-# and repos match through site/assets/query.ts, as the browser does, and5-# code search's words come out the same.3+# (ignoring case, and a run of whitespace in it matching any run), and every4+# term must match. Commits, reviews, wiki pages and repos match through5+# site/assets/query.ts, as the browser does, and code search's words come6+# out the same and find lines the same way.67 . "$(dirname "$0")/../lib.sh"78 user dami89 repo dami/greet "say hello world"@@ -13,6 +14,8 @@ at "$T/w/dami" commit "Search fix for the box"
1314 tick1415 at "$T/w/dami" git commit -q --allow-empty -m "Tidy the box" -m "This is to fix the search for good."1516 at "$T/w/dami" commit "box search fix"17+printf 'width = height\ndepth = length\ntotal\tcount\nx = 1\ny = 2\n' >"$T/w/dami/sizes.txt"18+at "$T/w/dami" commit "Align the columns"1619 push_as dami "$T/w/dami" origin main1720 at "$T/w/dami" git switch -q -c dami/wide1821 at "$T/w/dami" commit "Make the search box wider"@@ -34,8 +37,8 @@ ok terms '"Fix the Search"'
3437 eq "$(output)" '"Fix the Search" => [fix the search] | code [fix the search]'3538 ok terms '"the search'3639 eq "$(output)" '"the search => [the search] | code [the search]'37-# A run of whitespace in a phrase is one space, as in the indexes; spaces38-# at its ends stay (one each).40+# A run of whitespace in a phrase is kept as one space, which matches any41+# run of whitespace (below); spaces at its ends stay, one each.3942 ok terms '"fix the search"' '" fix "'4043 eq "$(output)" '"fix the search" => [fix the search] | code [fix the search]4144 " fix " => [ fix ] | code [ fix ]'@@ -63,10 +66,18 @@ ok match "$ix" '"fix the search"'
6366 eq "$(output | grep '^Commits: ' | sed 's/ | .*//')" 'Commits: [Fix the search] box6467 Commits: Tidy the box'6568 has "$T_LOG" '^Commits: Tidy the box \| This is to \[fix the search\] for good\.$'66-# Spaced out, it is the same phrase.69+# A run of whitespace in a phrase matches any run, both ways: spaced out,70+# it finds single spaces, and single spaces find a spaced-out subject,71+# marked whole.6772 ok match "$ix" '"fix the search"'6873 eq "$(output | grep '^Commits: ' | sed 's/ | .*//')" 'Commits: [Fix the search] box6974 Commits: Tidy the box'75+ok match "$ix" '"align the columns"'76+eq "$(output | sed 's/ | .*//')" 'Commits: [Align the columns]'77+ok match "$ix" '"align the columns"'78+eq "$(output | sed 's/ | .*//')" 'Commits: [Align the columns]'79+ok match "$ix" '"align thecolumns"'80+eq "$(output)" ''7081 # Unclosed, it is the same phrase.7182 ok match "$ix" '"FIX THE SEARCH'7283 eq "$(output | grep '^Commits: ' | sed 's/ | .*//')" 'Commits: [Fix the search] box@@ -95,9 +106,28 @@ Repositories: dami/other | [hello] there, [world]'
95106 ok match "$FORGE_WWW/repos.json" '"hello world"'96107 eq "$(output)" 'Repositories: dami/greet | say [hello world]'97108 109+# Code search finds lines the same way, both ways, tabs too: the index110+# looks up each piece of 3 characters or more, near the others, and the111+# line check takes any run of whitespace for a space. A phrase without such112+# a piece is looked up as written, so it finds single-spaced lines only.113+db=$FORGE_WWW/dami/greet/code.db114+code() { node "$T_HOME/test/code-search-bench.ts" sql "$db" "$1"; }115+ok code '"width = height"'116+eq "$(output)" 'sizes.txt:1: width = height'117+ok code '"depth = length"'118+eq "$(output)" 'sizes.txt:2: depth = length'119+ok code '"total count"'120+eq "$(output)" "$(printf 'sizes.txt:3: total\tcount')"121+ok code '"WIDTH = height" "depth"'122+eq "$(output)" ''123+ok code '"x = 1"'124+eq "$(output)" 'sizes.txt:4: x = 1'125+ok code '"y = 2"'126+eq "$(output)" ''127+98128 # Pages load search.js as a module, which imports query.js by its shipped name.99129 has "$FORGE_WWW/dami/greet/index.html" '<script type="module" src="/assets/search\.[0-9A-Za-z_-]{10}\.js"></script>'100130 has "$FORGE_WWW/index.html" '<script type="module" src="/assets/search\.[0-9A-Za-z_-]{10}\.js"></script>'101131 s=$(ls "$FORGE_WWW"/assets/search.*.js)102-has "$s" "^import \{ hasAll, marks, narrow, score as match, snippet, terms \} from '\./query\.[0-9A-Za-z_-]{10}\.js';$"132+has "$s" "^import \{ [a-zA-Z, ]+ \} from '\./query\.[0-9A-Za-z_-]{10}\.js';$"103133 exists "$FORGE_WWW/assets/$(sed -n "s|^import .* from '\./\(query\.[^']*\)';\$|\1|p" "$s")"