make stress fetches the datasets, runs every benchmark built with -tags stress, prints a summary table and appends the run to .deps/stress/results/runs.jsonl: one JSON line per benchmark with the git SHA, dirty flag, Go version, OS, CPU, ns/op, B/op, allocs/op and custom metrics (internal/stress/benchrec). BENCH, BENCHTIME and PKGS narrow a run. make stress-report reads the records with DuckDB and shows the latest run against the previous one on the same machine and against a baseline, flags changes past THRESHOLD (10%) and MIN_DELTA_NS, pivots the headline benchmarks over the last eight runs, and with FAIL_ON_REGRESSION=1 exits 1 on a regression.
- commit
- 35703a7dd3e0e5d1fe6350d31b15506669a10450
- parent
- 8124ca1acbc8f41cd13327c91754e26858c12d66
- author
- Dami <dami.etoile@gmail.com>
| M | Makefile | 16 | ++- |
| A | internal/ | 214 | ++++++++++++++++++++++++++++++ |
| A | scripts/ | 36 | +++++ |
| A | scripts/ | 38 | +++++ |
| A | scripts/ | 71 | ++++++++++ |
| A | scripts/ | 37 | +++++ |
@@ -7,7 +7,7 @@ GHOSTTY_SRC := $(DEPS)/ghostty-src
77 GHOSTTY_OUT := $(DEPS)/ghostty88 GHOSTTY_STAMP := $(GHOSTTY_OUT)/.built-$(GHOSTTY_COMMIT)99 10-10+1111 1212 build:1313 CGO_ENABLED=0 go build -o bin/012 ./cmd/012@@ -31,6 +31,20 @@ e2e: $(GHOSTTY_STAMP)
3131 screens: $(GHOSTTY_STAMP)3232 cd e2e && PKG_CONFIG_PATH=$(GHOSTTY_OUT)/share/pkgconfig go test -count=1 -run TestScreens ./... -update3333 34+# Stress: fetch real datasets into .deps/stress, run the benchmarks built35+# with -tags stress, print a summary and record the run in36+# .deps/stress/results/runs.jsonl. BENCH, BENCHTIME and PKGS narrow it;37+# see docs/limits.md. Plain `go test ./...` never runs these.38+stress:39+ scripts/stress/run.sh40+41+stress-data:42+ scripts/stress-data.sh43+44+# Compare the latest run with the previous one and a baseline (DuckDB).45+stress-report:46+ scripts/stress/report.sh47+3448 # Differential tests: formulas and number formats against excelize's3549 # calculation engine. A separate module, so the binary never depends on it.3650 oracle:@@ -0,0 +1,214 @@
1+// Command benchrec turns `go test -bench` output into run records, one2+// JSON line per benchmark, appended to a results file that DuckDB (and3+// ClickHouse) read; see scripts/stress and docs/observability.md. It also4+// prints a compact table of the run.5+//6+// go test -bench . -benchmem ./... | go run ./internal/stress/benchrec -out runs.jsonl7+//8+// GIT_SHA and GIT_DIRTY describe the code measured; the rest is read9+// from the output and the machine.10+package main11+12+import (13+ "bufio"14+ "cmp"15+ "encoding/json"16+ "flag"17+ "fmt"18+ "io"19+ "maps"20+ "os"21+ "regexp"22+ "runtime"23+ "slices"24+ "strconv"25+ "strings"26+ "text/tabwriter"27+ "time"28+)29+30+// Record is one benchmark result of one run. The same columns are the31+// stress_results table in ClickHouse (deploy/observability).32+type Record struct {33+ RunID string `json:"run_id"`34+ TS string `json:"ts"`35+ GitSHA string `json:"git_sha"`36+ GitDirty bool `json:"git_dirty"`37+ GoVersion string `json:"go_version"`38+ GOOS string `json:"goos"`39+ GOARCH string `json:"goarch"`40+ CPU string `json:"cpu"`41+ Host string `json:"host"`42+ Pkg string `json:"pkg"`43+ Name string `json:"name"` // without the -GOMAXPROCS suffix44+ Procs int `json:"procs"`45+ N int64 `json:"n"`46+ NsOp float64 `json:"ns_op"`47+ BOp float64 `json:"b_op"`48+ AllocsOp float64 `json:"allocs_op"`49+ MBs float64 `json:"mb_s"`50+ Metrics map[string]float64 `json:"metrics"` // custom b.ReportMetric units51+}52+53+func main() {54+ out := flag.String("out", "", "append records to this JSONL file")55+ flag.Parse()56+ recs, err := parse(os.Stdin, meta())57+ if err != nil {58+ fmt.Fprintln(os.Stderr, "benchrec:", err)59+ os.Exit(1)60+ }61+ if len(recs) == 0 {62+ fmt.Fprintln(os.Stderr, "benchrec: no benchmark results in the input")63+ os.Exit(1)64+ }65+ if *out != "" {66+ if err := appendJSONL(*out, recs); err != nil {67+ fmt.Fprintln(os.Stderr, "benchrec:", err)68+ os.Exit(1)69+ }70+ }71+ summary(os.Stdout, recs)72+ if *out != "" {73+ fmt.Printf("\n%d results of run %s appended to %s\n", len(recs), recs[0].RunID, *out)74+ }75+}76+77+// meta describes the run: when, which code, which machine.78+func meta() Record {79+ now := time.Now().UTC()80+ host, _ := os.Hostname()81+ sha := os.Getenv("GIT_SHA")82+ r := Record{83+ TS: now.Format(time.RFC3339), GitSHA: sha, GitDirty: os.Getenv("GIT_DIRTY") == "1",84+ GoVersion: runtime.Version(), GOOS: runtime.GOOS, GOARCH: runtime.GOARCH, Host: host,85+ }86+ r.RunID = now.Format("20060102T150405Z") + "-" + cmp.Or(sha, "unknown")87+ return r88+}89+90+var benchLine = regexp.MustCompile(`^(Benchmark\S+?)(?:-(\d+))?\s+(\d+)\s+(.*)$`)91+92+// parse reads benchmark output: "pkg:" and "cpu:" headers, then result93+// lines of an iteration count and value-unit pairs.94+func parse(r io.Reader, base Record) ([]Record, error) {95+ var out []Record96+ pkg, cpu := "", ""97+ sc := bufio.NewScanner(r)98+ sc.Buffer(make([]byte, 1<<20), 1<<20)99+ for sc.Scan() {100+ line := sc.Text()101+ switch {102+ case strings.HasPrefix(line, "pkg: "):103+ pkg = strings.TrimPrefix(line, "pkg: ")104+ continue105+ case strings.HasPrefix(line, "cpu: "):106+ cpu = strings.TrimPrefix(line, "cpu: ")107+ continue108+ }109+ m := benchLine.FindStringSubmatch(line)110+ if m == nil {111+ continue112+ }113+ rec := base114+ rec.Pkg, rec.CPU, rec.Name = pkg, cpu, strings.TrimPrefix(m[1], "Benchmark")115+ rec.Procs, _ = strconv.Atoi(m[2])116+ rec.N, _ = strconv.ParseInt(m[3], 10, 64)117+ rec.Metrics = map[string]float64{}118+ f := strings.Fields(m[4])119+ for i := 0; i+1 < len(f); i += 2 {120+ v, err := strconv.ParseFloat(f[i], 64)121+ if err != nil {122+ return nil, fmt.Errorf("%s: value %q", line, f[i])123+ }124+ switch unit := f[i+1]; unit {125+ case "ns/op":126+ rec.NsOp = v127+ case "B/op":128+ rec.BOp = v129+ case "allocs/op":130+ rec.AllocsOp = v131+ case "MB/s":132+ rec.MBs = v133+ default:134+ rec.Metrics[unit] = v135+ }136+ }137+ out = append(out, rec)138+ }139+ return out, sc.Err()140+}141+142+func appendJSONL(path string, recs []Record) error {143+ f, err := os.OpenFile(path, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0o644)144+ if err != nil {145+ return err146+ }147+ enc := json.NewEncoder(f)148+ for _, r := range recs {149+ if err := enc.Encode(r); err != nil {150+ f.Close()151+ return err152+ }153+ }154+ return f.Close()155+}156+157+// summary prints one line per benchmark: time per op in readable units,158+// allocations, and custom metrics.159+func summary(w io.Writer, recs []Record) {160+ tw := tabwriter.NewWriter(w, 0, 0, 2, ' ', 0)161+ fmt.Fprintf(tw, "benchmark\ttime/op\tallocs/op\tmem/op\tother\t\n")162+ pkg := ""163+ for _, r := range recs {164+ if r.Pkg != pkg {165+ pkg = r.Pkg166+ fmt.Fprintf(tw, "%s\t\t\t\t\t\n", pkg)167+ }168+ var other []string169+ if r.MBs > 0 {170+ other = append(other, fmt.Sprintf("%.1f MB/s", r.MBs))171+ }172+ for _, k := range slices.Sorted(maps.Keys(r.Metrics)) {173+ other = append(other, fmt.Sprintf("%s %s", num(r.Metrics[k]), k))174+ }175+ fmt.Fprintf(tw, " %s\t%s\t%s\t%s\t%s\t\n", r.Name, duration(r.NsOp), num(r.AllocsOp), bytes(r.BOp), strings.Join(other, ", "))176+ }177+ tw.Flush()178+}179+180+func duration(ns float64) string {181+ switch {182+ case ns >= 1e9:183+ return fmt.Sprintf("%.2f s", ns/1e9)184+ case ns >= 1e6:185+ return fmt.Sprintf("%.2f ms", ns/1e6)186+ case ns >= 1e3:187+ return fmt.Sprintf("%.1f us", ns/1e3)188+ }189+ return fmt.Sprintf("%.0f ns", ns)190+}191+192+func bytes(b float64) string {193+ switch {194+ case b >= 1<<30:195+ return fmt.Sprintf("%.1f GB", b/(1<<30))196+ case b >= 1<<20:197+ return fmt.Sprintf("%.1f MB", b/(1<<20))198+ case b >= 1<<10:199+ return fmt.Sprintf("%.1f KB", b/(1<<10))200+ }201+ return fmt.Sprintf("%.0f B", b)202+}203+204+func num(v float64) string {205+ switch {206+ case v >= 1e6:207+ return fmt.Sprintf("%.1fM", v/1e6)208+ case v >= 1e4:209+ return fmt.Sprintf("%.0fk", v/1e3)210+ case v == float64(int64(v)):211+ return strconv.FormatInt(int64(v), 10)212+ }213+ return strconv.FormatFloat(v, 'f', 1, 64)214+}@@ -0,0 +1,36 @@
1+-- Loads the stress run records (JSONL written by internal/stress/benchrec)2+-- into a table named results. The same columns are the stress_results3+-- table in ClickHouse (deploy/observability/clickhouse/init.sql).4+--5+-- Set the file first: SET VARIABLE results = '.deps/stress/results/runs.jsonl';6+7+CREATE OR REPLACE TEMP TABLE results AS8+SELECT *9+FROM read_json(getvariable('results'),10+ format = 'newline_delimited',11+ columns = {12+ run_id: 'VARCHAR', ts: 'TIMESTAMP', git_sha: 'VARCHAR', git_dirty: 'BOOLEAN',13+ go_version: 'VARCHAR', goos: 'VARCHAR', goarch: 'VARCHAR', cpu: 'VARCHAR', host: 'VARCHAR',14+ pkg: 'VARCHAR', name: 'VARCHAR', procs: 'INTEGER', n: 'BIGINT',15+ ns_op: 'DOUBLE', b_op: 'DOUBLE', allocs_op: 'DOUBLE', mb_s: 'DOUBLE',16+ metrics: 'MAP(VARCHAR, DOUBLE)'17+ });18+19+-- Durations in readable units.20+CREATE OR REPLACE TEMP MACRO fmt_ns(ns) AS21+ CASE22+ WHEN ns IS NULL THEN ''23+ WHEN ns >= 1e9 THEN printf('%.2f s', ns / 1e9)24+ WHEN ns >= 1e6 THEN printf('%.2f ms', ns / 1e6)25+ WHEN ns >= 1e3 THEN printf('%.1f us', ns / 1e3)26+ ELSE printf('%.0f ns', ns)27+ END;28+29+-- Runs, newest first, numbered: 1 is the latest.30+CREATE OR REPLACE TEMP TABLE runs AS31+SELECT run_id, min(ts) AS ts, any_value(git_sha) AS git_sha, bool_or(git_dirty) AS dirty,32+ any_value(cpu) AS cpu, any_value(goos) AS goos, any_value(go_version) AS go_version,33+ count(*) AS benchmarks,34+ row_number() OVER (ORDER BY min(ts) DESC) AS age35+FROM results36+GROUP BY run_id;@@ -0,0 +1,38 @@
1+2+# report.sh - compare the latest stress run with the previous one and a3+# baseline, using DuckDB over the JSONL run records. No server needed.4+#5+# THRESHOLD=0.10 flag changes over 10%6+# MIN_DELTA_NS=2000 ignore changes under 2 us (noise)7+# BASELINE=run_id compare with this run (default: the oldest run)8+# FAIL_ON_REGRESSION=1 exit 1 when the latest run regressed against the previous9+set -euo pipefail10+11+root=$(cd "$(dirname "$0")/../.." && pwd)12+results=${RESULTS:-"$root/.deps/stress/results/runs.jsonl"}13+if [ ! -s "$results" ]; then14+ echo "stress-report: no results yet in $results; run make stress first" >&215+ exit 116+fi17+if ! command -v duckdb >/dev/null 2>&1; then18+ echo "stress-report: needs the duckdb CLI (brew install duckdb)" >&219+ exit 120+fi21+22+vars="SET VARIABLE results = '$results';23+SET VARIABLE threshold = ${THRESHOLD:-0.10};24+SET VARIABLE min_delta_ns = ${MIN_DELTA_NS:-2000};25+SET VARIABLE baseline = '${BASELINE:-}';"26+27+cat <(echo "$vars") "$root/scripts/stress/load.sql" "$root/scripts/stress/report.sql" | duckdb28+29+if [ "${FAIL_ON_REGRESSION:-}" = 1 ]; then30+ n=$(cat <(echo "$vars") "$root/scripts/stress/load.sql" "$root/scripts/stress/report.sql" <(echo "31+.mode list32+.header off33+SELECT count(*) FROM compared WHERE kind = 'previous' AND flag = 'REGRESSION';") | duckdb | tail -1)34+ if [ "$n" != 0 ]; then35+ echo "stress-report: $n regressions against the previous run" >&236+ exit 137+ fi38+fi@@ -0,0 +1,71 @@
1+-- Stress report: the latest run against the previous run and a baseline2+-- on the same CPU, regressions flagged, and trends of the headline3+-- benchmarks. Run by scripts/stress/report.sh after load.sql, with4+-- variables threshold (fraction, e.g. 0.10), min_delta_ns (smaller5+-- changes are noise) and baseline (a run_id, or '' for the oldest run).6+7+.mode box8+.maxrows 4009+10+.print11+.print # Runs12+SELECT age, run_id, strftime(ts, '%Y-%m-%d %H:%M') AS ts, git_sha, dirty, cpu, go_version, benchmarks13+FROM runs ORDER BY age LIMIT 10;14+15+-- The runs compared: the latest, the one before it on the same CPU and16+-- OS, and the baseline.17+CREATE OR REPLACE TEMP TABLE picked AS18+WITH latest AS (SELECT * FROM runs WHERE age = 1),19+same AS (SELECT r.* FROM runs r, latest l WHERE r.cpu = l.cpu AND r.goos = l.goos)20+SELECT21+ (SELECT run_id FROM latest) AS latest,22+ (SELECT run_id FROM same WHERE age > 1 ORDER BY age LIMIT 1) AS previous,23+ coalesce(nullif(getvariable('baseline'), ''), (SELECT run_id FROM same ORDER BY age DESC LIMIT 1)) AS baseline;24+25+CREATE OR REPLACE TEMP TABLE compared AS26+WITH l AS (SELECT r.* FROM results r, picked p WHERE r.run_id = p.latest)27+SELECT l.pkg, l.name, kind, o.ns_op AS before_ns, l.ns_op AS after_ns,28+ (l.ns_op - o.ns_op) / o.ns_op AS change,29+ l.allocs_op - o.allocs_op AS allocs_change,30+ CASE31+ WHEN (l.ns_op - o.ns_op) / o.ns_op > getvariable('threshold')32+ AND l.ns_op - o.ns_op > getvariable('min_delta_ns') THEN 'REGRESSION'33+ WHEN (o.ns_op - l.ns_op) / o.ns_op > getvariable('threshold')34+ AND o.ns_op - l.ns_op > getvariable('min_delta_ns') THEN 'faster'35+ ELSE ''36+ END AS flag37+FROM l38+JOIN (39+ SELECT r.*, 'previous' AS kind FROM results r, picked p WHERE r.run_id = p.previous40+ UNION ALL41+ SELECT r.*, 'baseline' AS kind FROM results r, picked p WHERE r.run_id = p.baseline AND p.baseline <> p.latest42+) o USING (pkg, name);43+44+.print45+.print # Latest against the previous run on this machine (changes past the threshold)46+SELECT pkg, name, fmt_ns(before_ns) AS before, fmt_ns(after_ns) AS after,47+ printf('%+.1f%%', change * 100) AS change, allocs_change, flag48+FROM compared WHERE kind = 'previous' AND flag <> ''49+ORDER BY flag DESC, change DESC;50+51+.print52+.print # Latest against the baseline53+SELECT pkg, name, fmt_ns(before_ns) AS before, fmt_ns(after_ns) AS after,54+ printf('%+.1f%%', change * 100) AS change, allocs_change, flag55+FROM compared WHERE kind = 'baseline' AND flag <> ''56+ORDER BY flag DESC, change DESC;57+58+.print59+.print # Headline benchmarks over the last 8 runs (time per op)60+CREATE OR REPLACE TEMP TABLE headline AS61+SELECT r.name, r.run_id, u.age, fmt_ns(r.ns_op) AS t62+FROM results r JOIN runs u USING (run_id)63+WHERE u.age <= 8 AND regexp_matches(r.name,64+ '^(Edit/(chain|fanin|running|volatile)|RecalcAll/(dense|fanin)|Frame/(dense-8192x256|longtext-8192x500)/200x60|Keystroke/(arrow/dense-8192x256|extend-data/dense-8192x256)/80x24|Import/(owid|airport)|JEV/1000|Open/dense-8192x256|Save/dense-8192x256)');65+PIVOT headline ON age USING first(t) GROUP BY name ORDER BY name;66+67+.print68+.print # Regressions69+SELECT kind, count(*) FILTER (WHERE flag = 'REGRESSION') AS regressions,70+ count(*) FILTER (WHERE flag = 'faster') AS faster, count(*) AS compared71+FROM compared GROUP BY kind ORDER BY kind DESC;@@ -0,0 +1,37 @@
1+2+# run.sh - what `make stress` does: fetch the datasets, run the stress3+# benchmarks (built with -tags stress), print a summary table, and append4+# the run to .deps/stress/results/runs.jsonl for make stress-report. When5+# the observability stack is up (make obs-up), the run also goes to6+# ClickHouse.7+#8+# BENCH=Edit|Frame only these benchmarks (a go test -bench pattern)9+# BENCHTIME=500ms per benchmark (slow ones still run once)10+# PKGS="./internal/sheet"11+set -euo pipefail12+13+root=$(cd "$(dirname "$0")/../.." && pwd)14+cd "$root"15+"$root/scripts/stress-data.sh"16+17+out="$root/.deps/stress/results"18+mkdir -p "$out"19+raw="$out/raw-$(date -u +%Y%m%dT%H%M%SZ).txt"20+pkgs=${PKGS:-"./internal/sheet ./internal/ui ./internal/fileio ./internal/telemetry"}21+22+# shellcheck disable=SC2086 # pkgs is a list23+STRESS_DIR="$root/.deps/stress" go test -tags stress -run '^$' \24+ -bench "${BENCH:-.}" -benchmem -benchtime "${BENCHTIME:-500ms}" -timeout 60m $pkgs 2>&1 |25+ tee "$raw" | awk '/^Benchmark/ { print " " $1; fflush() } /^(FAIL|panic|--- FAIL)/ { print; fflush() }'26+27+sha=$(git -C "$root" rev-parse --short HEAD 2>/dev/null || echo unknown)28+dirty=029+if [ -n "$(git -C "$root" status --porcelain --untracked-files=no 2>/dev/null)" ]; then30+ dirty=131+fi32+GIT_SHA=$sha GIT_DIRTY=$dirty go run ./internal/stress/benchrec -out "$out/runs.jsonl" <"$raw"33+echo "raw output: $raw"34+35+if curl -fsS -o /dev/null "${CLICKHOUSE_URL:-http://localhost:8123}/ping" 2>/dev/null; then36+ "$root/scripts/stress/clickhouse-load.sh"37+fi