1-- Stress report: the latest run against the previous run and a baseline2-- on the same CPU, regressions flagged, and trends of the headline3-- benchmarks. Run by scripts/stress/report.sh after load.sql, with4-- variables threshold (fraction, e.g. 0.10), min_delta_ns (smaller5-- changes are noise) and baseline (a run_id, or '' for the oldest run that6-- covered most of the same benchmarks).78.mode box9.maxrows 4001011.print12.print # Runs13SELECT age, run_id, strftime(ts, '%Y-%m-%d %H:%M') AS ts, git_sha, dirty, cpu, go_version, benchmarks14FROM runs ORDER BY age LIMIT 10;1516-- The runs compared: the latest, the one before it on the same CPU and17-- OS, and the baseline.18CREATE OR REPLACE TEMP TABLE picked AS19WITH latest AS (SELECT * FROM runs WHERE age = 1),20same AS (SELECT r.* FROM runs r, latest l WHERE r.cpu = l.cpu AND r.goos = l.goos)21SELECT22 (SELECT run_id FROM latest) AS latest,23 (SELECT run_id FROM same WHERE age > 1 ORDER BY age LIMIT 1) AS previous,24 -- By default the oldest run that covered most of what the latest did.25 coalesce(nullif(getvariable('baseline'), ''),26 (SELECT run_id FROM same WHERE benchmarks >= 0.9 * (SELECT benchmarks FROM latest) ORDER BY age DESC LIMIT 1)) AS baseline;2728CREATE OR REPLACE TEMP TABLE compared AS29WITH l AS (SELECT r.* FROM bench r, picked p WHERE r.run_id = p.latest)30SELECT l.pkg, l.name, kind, o.ns_op AS before_ns, l.ns_op AS after_ns,31 (l.ns_op - o.ns_op) / o.ns_op AS change,32 l.allocs_op - o.allocs_op AS allocs_change,33 CASE34 WHEN (l.ns_op - o.ns_op) / o.ns_op > getvariable('threshold')35 AND l.ns_op - o.ns_op > getvariable('min_delta_ns') THEN 'REGRESSION'36 WHEN (o.ns_op - l.ns_op) / o.ns_op > getvariable('threshold')37 AND o.ns_op - l.ns_op > getvariable('min_delta_ns') THEN 'faster'38 ELSE ''39 END AS flag40FROM l41JOIN (42 SELECT r.*, 'previous' AS kind FROM bench r, picked p WHERE r.run_id = p.previous43 UNION ALL44 SELECT r.*, 'baseline' AS kind FROM bench r, picked p WHERE r.run_id = p.baseline AND p.baseline <> p.latest45) o USING (pkg, name);4647.print48.print # Latest against the previous run on this machine (changes past the threshold)49SELECT pkg, name, fmt_ns(before_ns) AS before, fmt_ns(after_ns) AS after,50 printf('%+.1f%%', change * 100) AS change, allocs_change, flag51FROM compared WHERE kind = 'previous' AND flag <> ''52ORDER BY flag DESC, change DESC;5354.print55.print # Latest against the baseline56SELECT pkg, name, fmt_ns(before_ns) AS before, fmt_ns(after_ns) AS after,57 printf('%+.1f%%', change * 100) AS change, allocs_change, flag58FROM compared WHERE kind = 'baseline' AND flag <> ''59ORDER BY flag DESC, change DESC;6061.print62.print # Headline benchmarks over the last 8 runs (time per op)63CREATE OR REPLACE TEMP TABLE headline AS64SELECT r.name, r.run_id, u.age, fmt_ns(r.ns_op) AS t65FROM bench r JOIN runs u USING (run_id)66WHERE u.age <= 8 AND regexp_matches(r.name,67 '^(Edit/(chain|fanin|running|volatile)|RecalcAll/(dense|fanin)|Frame/(dense-8192x256|longtext-8192x500)/200x60|Keystroke/(arrow/dense-8192x256|extend-data/dense-8192x256)/80x24|Import/(owid|airport)|JEV/1000|Open/dense-8192x256|Save/dense-8192x256)');68PIVOT headline ON age USING first(t) GROUP BY name ORDER BY name;6970.print71.print # Regressions72SELECT kind, count(*) FILTER (WHERE flag = 'REGRESSION') AS regressions,73 count(*) FILTER (WHERE flag = 'faster') AS faster, count(*) AS compared74FROM compared GROUP BY kind ORDER BY kind DESC;