1// The pre-receive secret check. A public repo shows everyone what is pushed2// to it, and a private one keeps it for its readers, and for everyone once it3// is made public, so a push that adds a credential is refused before it4// lands, not rotated after. Private repos are checked like public ones. Each5// new commit's added lines are scanned, since history keeps a secret even6// after a later commit removes it:7//8// - patterns for formats that are a credential on sight (private keys, AWS,9// GitHub, Slack, Stripe, Anthropic, OpenAI, Google, ...) refuse on their own;10// - Jev reads the rest as lists of added lines grouped by file and gives the11// probability that they add a real credential: hardcoded passwords,12// connection strings, .env values, tokens without a known prefix. At or over13// `git config forge.secretthreshold` (default 0.7) the push is refused. On14// test cases, real credentials scored 0.89 to 0.97 and placeholders, env15// reads, fixtures and hashes 0.03 to 0.48; 0.7 sits in the gap.16//17// It fails soft. Jev gets one try of ~4s per request and ~6s for the whole18// check, and an error or timeout lets the push through with a warning.19// Without TYPESAFE_API_KEY, past 200 new commits, or for five minutes after20// the API timed out or was unreachable (see ask()), only the patterns run.21// Imports (-o noreview) and `git push -o allow-secret` skip the check.22//23// Making a private repo public (ssh <host> public) runs it over the repo's24// whole history first: a push's check only reads commits the repo doesn't25// have yet, which there are none of then.26//27// node lib/jev-secrets.ts <updates pre-receive's "old new ref" lines, GIT_DIR set28// node lib/jev-secrets.ts --history every commit in the repo (historyCommits), GIT_DIR set29//30// Exit 3 means a finding, printed on stderr; anything else accepts the push.31// Not 1, which node also uses when a module fails to load: a broken import32// must not refuse every push.33import { spawn } from 'node:child_process';34import fs from 'node:fs';35import { fileURLToPath } from 'node:url';36import { revList, tryRun } from './git.ts';37import { ask, choice, enabled, noul } from './jev.ts';3839const FOUND = 3; // exit status for a finding40const DEADLINE = 6000; // ms for the whole check41const TIMEOUT = 4000; // ms for one Jev request, which isn't retried42const LINES_PER_ASK = 300;43const FILES_PER_ASK = 40;44const MAX_ASKS = 10;45const MAX_COMMITS = 200; // past this, patterns only46const MAX_FILE = '1m'; // larger files are skipped like binaries47const MAX_LINE = 400; // characters of a line that Jev reads48export const THRESHOLD = 0.7;4950export interface Line {51 n: number;52 text: string;53 commit: string;54}5556// The lines one commit adds to one file.57export interface Added {58 commit: string;59 path: string;60 lines: Line[];61}6263export interface Finding {64 path: string;65 line: string;66 commit: string;67 by: string;68}6970const warn = (s: string): void => {71 process.stderr.write(`forge: secret check: ${s}\n`);72};7374export function pushOptions(env: NodeJS.ProcessEnv = process.env): string[] {75 return Array.from({ length: Number(env.GIT_PUSH_OPTION_COUNT) || 0 }, (_, i) => env[`GIT_PUSH_OPTION_${i}`] ?? '');76}7778export function threshold(gitDir: string): number {79 const v = Number(tryRun(gitDir, ['config', '--get', 'forge.secretthreshold'])?.trim());80 return v > 0 ? v : THRESHOLD;81}8283// Commits the push brings that the repo doesn't have. Only branches and tags:84// notes are CI attestations, refs/sites/* and refs/site-previews/* hold85// built output, and refs/releases/* release files, mostly binaries, which86// pre-receive checks against their signed SHA256SUMS instead.87export function newCommits(gitDir: string, updates: string): string[] {88 const tips = updates.split('\n').map((l) => l.trim().split(/\s+/))89 .filter(([, sha, ref]) => ref && !/^0+$/.test(sha) && /^refs\/(heads|tags)\//.test(ref))90 .map(([, sha]) => sha);91 return tips.length ? revList(gitDir, [...tips, '--not', '--all']) : [];92}9394// Every commit that going public shows: those on branches and tags, as the95// push check reads them, and the earlier revisions of reviews96// (refs/forge/keep), which review pages show.97export function historyCommits(gitDir: string): string[] {98 return revList(gitDir, ['--branches', '--tags', '--glob=refs/forge/keep/*']);99}100101const unquote = (s: string): string => {102 s = s.replace(/\t$/, '');103 if (!s.startsWith('"')) return s;104 try {105 return JSON.parse(s) as string;106 } catch {107 return s.slice(1, -1);108 }109};110111// The lines each commit adds against its first parent. A merge shows what it112// adds over every parent (git's combined diff), so merging main into a branch113// doesn't scan main again; the other side's new commits are scanned on their114// own. Binary files, and text files over MAX_FILE, show no lines.115export async function addedLines(gitDir: string, commits: string[]): Promise<Added[]> {116 const out: Added[] = [];117 if (!commits.length) return out;118 const git = spawn('git', [119 '-c', 'core.quotepath=false', '-c', `core.bigFileThreshold=${MAX_FILE}`, '-c', 'log.showRoot=true', '--git-dir', gitDir,120 'log', '--stdin', '--no-walk=unsorted', '--format=commit %H', '--cc', '-U0', '-M',121 '--no-color', '--no-ext-diff', '--no-textconv', '--src-prefix=a/', '--dst-prefix=b/',122 ], { stdio: ['pipe', 'pipe', 'ignore'] });123 const exit = new Promise<number | null>((resolve) => git.on('close', resolve));124 git.stdin.on('error', () => {});125 git.stdin.end(commits.join('\n') + '\n');126 let commit = '';127 let file: Added | null = null;128 let width = 0; // parents in the current hunk; 0 outside hunks129 let n = 0; // the next line number on the new side130 const parse = (s: string): void => {131 const hunk = /^(@@+) (?:-\S+ )+\+(\d+)(?:,\d+)? @@+/.exec(s);132 if (hunk) {133 width = hunk[1].length - 1;134 n = Number(hunk[2]);135 return;136 }137 if (width && /^[-+ \\]/.test(s)) {138 const mark = s.slice(0, width);139 if (mark.includes('-') || mark[0] === '\\') return;140 // A line new to every parent; " +" or "+ " came from one of them.141 if (file && !mark.includes(' ')) file.lines.push({ n, text: s.slice(width).replace(/\r$/, ''), commit });142 n++;143 return;144 }145 width = 0;146 if (s.startsWith('commit ')) commit = s.slice(7);147 else if (s.startsWith('+++ ')) {148 file = s === '+++ /dev/null' ? null : { commit, path: unquote(s.slice(4)).replace(/^b\//, ''), lines: [] };149 if (file) out.push(file);150 }151 };152 git.stdout.setEncoding('utf8');153 let rest = '';154 for await (const chunk of git.stdout) {155 const lines = (rest + chunk).split('\n');156 rest = lines.pop() ?? '';157 lines.forEach(parse);158 }159 if (rest) parse(rest);160 const code = await exit;161 if (code !== 0) throw new Error(`git log exited with ${code}`);162 return out.filter((f) => f.lines.length);163}164165// Formats that are a credential on sight; a match refuses the push by itself.166export const PATTERNS: [string, RegExp][] = [167 ['private key', /-----BEGIN (?:[A-Z0-9]+ )*PRIVATE KEY(?: BLOCK)?-----/],168 ['AWS access key id', /\b(?:AKIA|ASIA|ABIA|ACCA)[A-Z0-9]{16}\b/],169 ['AWS secret access key', /aws_?secret_?access_?key["']?\s*[:=]\s*["']?[A-Za-z0-9/+]{40}(?![A-Za-z0-9/+])/i],170 ['GitHub token', /\b(?:gh[pousr]_[A-Za-z0-9]{36,}|github_pat_[A-Za-z0-9_]{60,})/],171 ['GitLab token', /\bglpat-[A-Za-z0-9_-]{20,}/],172 ['Slack token', /\bxox[abposre]-[A-Za-z0-9-]{10,}/],173 ['Slack webhook', /hooks\.slack\.com\/services\/T[A-Z0-9]+\/B[A-Z0-9]+\/[A-Za-z0-9]{20,}/],174 ['Discord webhook', /discord(?:app)?\.com\/api\/webhooks\/\d+\/[A-Za-z0-9_-]{60,}/],175 ['Stripe live key', /\b[rs]k_live_[A-Za-z0-9]{20,}/],176 ['Anthropic API key', /\bsk-ant-[A-Za-z0-9_-]{32,}/],177 ['OpenAI API key', /\bsk-(?:proj|svcacct|admin)-[A-Za-z0-9_-]{32,}|\bsk-[A-Za-z0-9]{20}T3BlbkFJ[A-Za-z0-9]{20}/],178 ['Google API key', /\bAIza[A-Za-z0-9_-]{35}/],179 ['Google OAuth client secret', /\bGOCSPX-[A-Za-z0-9_-]{28}/],180 ['npm token', /\bnpm_[A-Za-z0-9]{36}\b/],181 ['PyPI token', /\bpypi-AgEIcHlwaS5vcmc[A-Za-z0-9_-]{50,}/],182 ['Hugging Face token', /\bhf_[A-Za-z0-9]{34,}\b/],183 ['SendGrid key', /\bSG\.[A-Za-z0-9_-]{22}\.[A-Za-z0-9_-]{43}\b/],184 ['DigitalOcean token', /\bdo[opr]_v1_[a-f0-9]{64}\b/],185 ['Shopify token', /\bshp(?:at|ca|pa|ss)_[a-fA-F0-9]{32}\b/],186];187188// Documented examples and placeholders in those formats (AKIAIOSFODNN7EXAMPLE, ghp_xxxx...).189const PLACEHOLDER = /example|sample|dummy|fake|placeholder|redacted|your|here|xxxx|(.)\1{5,}/i;190// A key block counts when key material follows: on the line (an escaped \n in191// JSON) or within the next few lines, not in code that only looks for the header.192const KEY_BODY = /[A-Za-z0-9+/]{40,}/;193194function real(name: string, m: RegExpExecArray, f: Added, i: number): boolean {195 if (name !== 'private key') return !PLACEHOLDER.test(m[0]);196 const l = f.lines[i];197 if (KEY_BODY.test(l.text.slice(m.index + m[0].length))) return true;198 return f.lines.slice(i + 1).some((b) => b.n - l.n <= 6 && KEY_BODY.test(b.text));199}200201export function patternFindings(added: Added[]): Finding[] {202 const out: Finding[] = [];203 for (const f of added) {204 f.lines.forEach((l, i) => {205 for (const [name, re] of PATTERNS) {206 const m = re.exec(l.text);207 if (m && real(name, m, f, i)) {208 out.push({ path: f.path, line: String(l.n), commit: l.commit, by: `${name} (pattern)` });209 break;210 }211 }212 });213 }214 return out;215}216217// Files Jev doesn't read: lockfiles and generated text, full of hashes that218// look like keys and never hold one. The patterns still scan them.219const NOT_FOR_JEV = /(^|\/)(package-lock\.json|npm-shrinkwrap\.json|yarn\.lock|pnpm-lock\.yaml|bun\.lock|Cargo\.lock|go\.sum|poetry\.lock|uv\.lock|Gemfile\.lock|composer\.lock|flake\.lock)$|\.(min\.js|min\.css|map|svg)$/;220// Where credentials usually go. When a push adds more than Jev can read, these go first.221const LIKELY = /(^|\/)\.env|\.(env|ya?ml|json|toml|ini|cfg|conf|properties|tfvars|xml)$|config|setting|secret|credential|docker-compose/i;222223export interface Part {224 path: string;225 lines: Line[];226}227228// Added lines as Jev reads them: per file, deduplicated across commits and229// without blank lines, cut into requests of about LINES_PER_ASK lines, at most230// MAX_ASKS of them.231export function batches(added: Added[]): Part[][] {232 const byPath = new Map<string, Part>();233 const seen = new Set<string>();234 for (const f of added) {235 if (NOT_FOR_JEV.test(f.path)) continue;236 for (const l of f.lines) {237 const key = `${f.path}\0${l.text}`;238 if (!l.text.trim() || seen.has(key)) continue;239 seen.add(key);240 if (!byPath.has(f.path)) byPath.set(f.path, { path: f.path, lines: [] });241 byPath.get(f.path)!.lines.push(l);242 }243 }244 const parts = [...byPath.values()].sort((a, b) => Number(LIKELY.test(b.path)) - Number(LIKELY.test(a.path)));245 const out: Part[][] = [];246 let cur: Part[] = [];247 let room = LINES_PER_ASK;248 for (const p of parts) {249 for (let i = 0; i < p.lines.length;) {250 if (!room || cur.length >= FILES_PER_ASK) {251 out.push(cur);252 if (out.length >= MAX_ASKS) return out;253 cur = [];254 room = LINES_PER_ASK;255 }256 const lines = p.lines.slice(i, i + room);257 cur.push({ path: p.path, lines });258 i += lines.length;259 room -= lines.length;260 }261 }262 if (cur.length) out.push(cur);263 return out;264}265266export function state(batch: Part[]): unknown {267 return {268 files: batch.map((p) => ({269 path: p.path,270 added_lines: p.lines.map((l) => `${l.n}: ${l.text.length > MAX_LINE ? l.text.slice(0, MAX_LINE) + '...' : l.text}`),271 })),272 };273}274275export function questions(paths: string[]) {276 return {277 secret: noul(278 'These are the lines commits add to a git repository that is public or may be made public, grouped by file, each starting with its line number. ' +279 'Do the added lines put a real credential into the repository: a password, API key, access token, private key, or a connection string or URL with a password or token in it?',280 {281 true: 'At least one added line contains a literal secret value that looks real and would let anyone who reads this repository log in or call an API: a hardcoded password, a token or key string, a database URL with a password, a webhook URL with its secret token, or a .env or config entry with a real secret value.',282 false: 'No added line contains a real secret value. These are not real: placeholders and examples (changeme, <your-password>, xxx, your-api-key, example.com), test fixtures and mock values, environment variable names and reads (process.env.X, os.environ["X"], ${X}), empty values, redacted or masked values (****, REDACTED), hashes and checksums, public keys, and code that handles credentials without containing one.',283 },284 ),285 file: choice(286 'Which file has the added line with the real credential?',287 Object.fromEntries([...paths.map((p): [string, string | null] => [p, null]), ['none', 'No file adds a real credential']]),288 ),289 };290}291292// Lines worth pointing at in a file Jev picked: assignments to secret-sounding293// names, URLs with a password, long opaque strings.294const HINT = /(pass(word|wd|phrase)?|pwd|secret|token|api_?key|apikey|auth|credential|private_?key|access_?key|dsn|conn)\w*["']?\s*[:=]|\w+:\/\/[^\s/:@]+:[^\s/@]+@|[A-Za-z0-9+/_-]{32,}/i;295296function pointAt(batch: Part[], picked: Record<string, number>, p: number): Finding[] {297 const files = batch.length === 1 ? batch : batch.filter((f) => (picked[f.path] ?? 0) >= 0.25);298 const out: Finding[] = [];299 for (const f of files.length ? files : batch) {300 const hits = f.lines.filter((l) => HINT.test(l.text.replace(/-/g, '_'))).slice(0, 3);301 const lines = hits.length ? hits : [f.lines[0]];302 for (const l of lines) out.push({ path: f.path, line: hits.length ? String(l.n) : `${l.n}-${f.lines[f.lines.length - 1].n}`, commit: l.commit, by: `credential (Jev, ${p.toFixed(2)})` });303 }304 return out;305}306307// Ask about every batch at once. ask() reports a failure on stderr as308// "jev: ..."; those are held back so a failure prints one warning line.309export async function jevFindings(all: Part[][], min: number, ms: number): Promise<{ findings: Finding[]; failed: string | null }> {310 const findings: Finding[] = [];311 const errors: string[] = [];312 let answered = 0;313 const write = process.stderr.write;314 process.stderr.write = ((s: string | Uint8Array, ...rest: unknown[]) =>315 typeof s === 'string' && s.startsWith('jev: ') ? errors.push(s.slice(5).trim()) > 0 : write.apply(process.stderr, [s, ...rest] as never)) as typeof write;316 try {317 const asks = all.map(async (batch) => {318 const res = await ask(state(batch), questions(batch.map((p) => p.path)), { timeout: Math.min(TIMEOUT, ms), retries: 0 });319 if (!res) return;320 answered++;321 const p = res.answers.secret.noul;322 if (p >= min) findings.push(...pointAt(batch, res.answers.file.probabilities, p));323 });324 await Promise.race([Promise.all(asks), new Promise((resolve) => setTimeout(resolve, ms).unref())]);325 } finally {326 process.stderr.write = write;327 }328 const failed = answered < all.length ? errors[0]?.replace(/\.$/, '') ?? `no answer within ${ms}ms` : null;329 return { findings, failed };330}331332function report(found: Finding[], history: boolean): void {333 const seen = new Set<string>();334 const rows = found.map((f) => ` ${f.path}:${f.line} in ${f.commit.slice(0, 7)}: ${f.by}`).filter((r) => !seen.has(r) && seen.add(r));335 process.stderr.write([336 history ? "forge: secret check: this repo's history has what looks like a credential:" : 'forge: secret check: this push adds what looks like a credential:',337 ...rows.slice(0, 20),338 ...(rows.length > 20 ? [` and ${rows.length - 20} more`] : []),339 ...(history340 ? [341 'Making the repo public would show it to everyone. Take the secret out of every commit',342 'that adds it (git rebase -i, or git filter-repo), force-push, and rotate the secret.',343 'If it is not a real secret: ssh <host> public <owner>/<repo> --force',344 ]345 : [346 'A public repo shows it to everyone, and a private one keeps it in its history. Take the',347 'secret out of every commit that adds it (git commit --amend for the last commit,',348 'git rebase -i for older ones), read it from the environment or an untracked file',349 'instead, push again, and rotate the secret.',350 'If it is not a real secret: git push -o allow-secret <remote> <branch>',351 ]),352 ].join('\n') + '\n');353}354355// updates: pre-receive's lines; null checks the whole history (--history).356export async function check(gitDir: string, updates: string | null, env: NodeJS.ProcessEnv = process.env): Promise<number> {357 const t0 = Date.now();358 const opts = updates == null ? [] : pushOptions(env);359 if (opts.includes('noreview')) return 0;360 if (opts.includes('allow-secret')) {361 warn('skipped (-o allow-secret)');362 return 0;363 }364 const commits = updates == null ? historyCommits(gitDir) : newCommits(gitDir, updates);365 const added = await addedLines(gitDir, commits);366 const found = patternFindings(added);367 let warning: string | null = null;368 const todo = enabled(env) ? batches(added) : [];369 if (todo.length && commits.length > MAX_COMMITS) {370 warning = `${commits.length} ${updates == null ? '' : 'new '}commits; checked known key formats only`;371 } else if (todo.length) {372 const left = DEADLINE - (Date.now() - t0);373 if (left < 1000) warning = 'no time left for Jev; checked known key formats only';374 else {375 const r = await jevFindings(todo, threshold(gitDir), left);376 found.push(...r.findings);377 if (r.failed) warning = `${r.failed.startsWith('not asked') ? `Jev ${r.failed}` : `Jev failed (${r.failed})`}; checked known key formats only`;378 }379 }380 if (found.length) {381 report(found, updates == null);382 return FOUND;383 }384 if (warning) warn(warning);385 return 0;386}387388// Run directly (realpath: /tmp is a symlink on macOS, and import.meta.url is resolved).389if (process.argv[1] && fs.realpathSync(process.argv[1]) === fileURLToPath(import.meta.url)) {390 let code = 0;391 try {392 code = await check(process.env.GIT_DIR || '.', process.argv[2] === '--history' ? null : fs.readFileSync(0, 'utf8'));393 } catch (err) {394 warn(`${err instanceof Error ? err.message : String(err)}; push not checked`);395 }396 // Exit now: a Jev request past the deadline may still hold a socket open.397 process.exit(code);398}