12# report.sh - compare the latest stress run with the previous one, a3# baseline and the last tagged release, using DuckDB over the JSONL run4# records. No server needed. Exits 1 when the latest run regressed5# against the last release's recorded run.6#7# THRESHOLD=0.10 flag changes over 10% (plus the benchmark's noise8# against the release)9# MIN_DELTA_NS=2000 ignore changes under 2 us (noise)10# BASELINE=run_id compare with this run (default: the oldest run)11# RELEASE=v1.2.3 compare with this tag's runs (default: the newest12# tag reachable from HEAD that has a recorded run)13# RELEASE_RUN=run_id compare with this run instead of a release's14# FAIL_ON_REGRESSION=1 also exit 1 when the latest run regressed against the previous15set -euo pipefail1617root=$(cd "$(dirname "$0")/../.." && pwd)18results=${RESULTS:-"$root/.deps/stress/results/runs.jsonl"}19if [ ! -s "$results" ]; then20 echo "stress-report: no results yet in $results; run make stress first" >&221 exit 122fi23if ! command -v duckdb >/dev/null 2>&1; then24 echo "stress-report: needs the duckdb CLI (brew install duckdb)" >&225 exit 126fi2728# Release tags, newest version first, as 'tag commit' pairs.29if [ -n "${RELEASE:-}" ]; then30 tags=$(git -C "$root" rev-parse -q --verify "refs/tags/$RELEASE^{commit}" | sed "s/^/$RELEASE /") || {31 echo "stress-report: no tag $RELEASE" >&232 exit 133 }34else35 tags=$(git -C "$root" tag -l 'v*' --merged HEAD --sort=-v:refname \36 --format='%(refname:short) %(if)%(*objectname)%(then)%(*objectname)%(else)%(objectname)%(end)' 2>/dev/null || true)37fi38list=$(printf '%s\n' "$tags" | awk 'NF == 2 { printf "%s'\''%s'\''", (n++ ? ", " : ""), $0 }')3940vars="SET VARIABLE results = '$results';41SET VARIABLE threshold = ${THRESHOLD:-0.10};42SET VARIABLE min_delta_ns = ${MIN_DELTA_NS:-2000};43SET VARIABLE baseline = '${BASELINE:-}';44SET VARIABLE release_run = '${RELEASE_RUN:-}';45SET VARIABLE release_tags = [$list]::VARCHAR[];"4647sql() {48 cat <(echo "$vars") "$root/scripts/stress/load.sql" "$@"49}5051sql "$root/scripts/stress/report.sql" "$root/scripts/stress/release.sql" | duckdb5253count() {54 sql "$root/scripts/stress/report.sql" "$root/scripts/stress/release.sql" <(echo "55.mode list56.header off57$1") | duckdb | tail -158}5960status=061if [ "$(count 'SELECT count(*) FROM release_pick;')" = 0 ]; then62 echo "stress-report: no recorded run of a release to compare with; record one with make stress on a tagged commit" >&263else64 n=$(count "SELECT count(*) FROM release_compared WHERE flag = 'REGRESSION';")65 if [ "$n" != 0 ]; then66 echo "stress-report: $n regressions against the last release" >&267 status=168 fi69fi70if [ "${FAIL_ON_REGRESSION:-}" = 1 ]; then71 n=$(count "SELECT count(*) FROM compared WHERE kind = 'previous' AND flag = 'REGRESSION';")72 if [ "$n" != 0 ]; then73 echo "stress-report: $n regressions against the previous run" >&274 status=175 fi76fi77exit $status