1/**2 * Source normalization for dedup identity (NOT a minifier): strips comments3 * and collapses whitespace so formatting-only changes hash identically, while4 * every literal, identifier, and operator survives byte-for-byte. The5 * normalized form is hashed into blobs.norm_hash; two uploads with the same6 * norm_hash are the same strategy for Browse dedup purposes.7 *8 * Dependency-free scanner, aware of: line/block comments, single/double9 * strings, template literals (with nested ${...}), and regex literals (via10 * the standard prev-token heuristic). It must be deterministic above all —11 * a misjudged regex boundary degrades invariance, never correctness.12 */1314const isWord = (ch: string): boolean => {15 const c = ch.charCodeAt(0);16 return (17 (c >= 48 && c <= 57) || // 0-918 (c >= 65 && c <= 90) || // A-Z19 (c >= 97 && c <= 122) || // a-z20 c === 95 || // _21 c === 36 || // $22 c > 127 // non-ASCII identifiers pass through conservatively23 );24};2526const isSpace = (ch: string): boolean => {27 const c = ch.charCodeAt(0);28 return c === 32 || c === 9 || c === 10 || c === 13 || c === 11 || c === 12 || c === 0xfeff;29};3031/** Tokens after which a `/` starts a regex literal rather than division. */32const REGEX_PRECEDERS = new Set([33 '(', ',', '=', ':', '[', '!', '&', '|', '?', '{', '}', ';', '\n',34 'return', 'typeof', 'instanceof', 'in', 'of', 'new', 'delete', 'void',35 'throw', 'case', 'do', 'else', 'yield', 'await',36]);3738export function normalizeSource(source: string): string {39 const out: string[] = [];40 const n = source.length;41 let i = 0;42 /** Last significant token (string form) for the regex heuristic. */43 let prevToken = '\n';44 let pendingSpace = false;4546 const push = (text: string): void => {47 if (pendingSpace) {48 const prev = out.length > 0 ? out[out.length - 1]! : '';49 // A space survives only where dropping it would fuse two word-ish50 // tokens (identifiers, keywords, numbers) into one.51 if (prev.length > 0 && isWord(prev[prev.length - 1]!) && isWord(text[0]!)) {52 out.push(' ');53 }54 pendingSpace = false;55 }56 out.push(text);57 };5859 while (i < n) {60 const c = source[i]!;6162 // Whitespace: remember that a gap existed, emit nothing yet.63 if (isSpace(c)) {64 pendingSpace = true;65 i++;66 continue;67 }6869 // Comments.70 if (c === '/' && i + 1 < n && source[i + 1] === '/') {71 while (i < n && source[i] !== '\n') i++;72 pendingSpace = true;73 continue;74 }75 if (c === '/' && i + 1 < n && source[i + 1] === '*') {76 i += 2;77 while (i + 1 < n && !(source[i] === '*' && source[i + 1] === '/')) i++;78 i = Math.min(n, i + 2);79 pendingSpace = true;80 continue;81 }8283 // String literals.84 if (c === '"' || c === "'") {85 const start = i;86 i++;87 while (i < n && source[i] !== c) {88 if (source[i] === '\\') i++;89 i++;90 }91 i = Math.min(n, i + 1);92 const text = source.slice(start, i);93 push(text);94 prevToken = text;95 continue;96 }9798 // Template literals: kept verbatim (inner whitespace is content; the99 // formatting of interpolated expressions is deliberately NOT collapsed —100 // determinism beats maximal invariance here).101 if (c === '`') {102 const start = i;103 i++;104 let depth = 0;105 while (i < n) {106 const t = source[i]!;107 if (t === '\\') {108 i += 2;109 continue;110 }111 if (depth === 0 && t === '`') {112 i++;113 break;114 }115 if (depth === 0 && t === '$' && source[i + 1] === '{') {116 depth = 1;117 i += 2;118 continue;119 }120 if (depth > 0) {121 if (t === '{') depth++;122 else if (t === '}') depth--;123 }124 i++;125 }126 const text = source.slice(start, i);127 push(text);128 prevToken = text;129 continue;130 }131132 // Regex literal vs division.133 if (c === '/' && REGEX_PRECEDERS.has(prevToken)) {134 const start = i;135 i++;136 let inClass = false;137 let closed = false;138 while (i < n) {139 const t = source[i]!;140 if (t === '\\') {141 i += 2;142 continue;143 }144 if (t === '[') inClass = true;145 else if (t === ']') inClass = false;146 else if (t === '/' && !inClass) {147 i++;148 closed = true;149 break;150 } else if (t === '\n') break; // not a regex after all; bail safely151 i++;152 }153 if (closed) while (i < n && isWord(source[i]!)) i++; // flags154 const text = source.slice(start, i);155 push(text);156 prevToken = text;157 continue;158 }159160 // Identifier / keyword / number.161 if (isWord(c)) {162 const start = i;163 while (i < n && isWord(source[i]!)) i++;164 const text = source.slice(start, i);165 push(text);166 prevToken = text;167 continue;168 }169170 // Punctuator: single char is enough for the regex heuristic.171 push(c);172 prevToken = c;173 i++;174 }175176 return out.join('');177}