12# stress-data.sh - fetch real public datasets for stress-testing 012's importers.3#4# Downloads every file listed in scripts/stress-data.tsv into .deps/stress/5# (override with STRESS_DIR), verifies SHA-256, and is idempotent: files that6# are already present with the right checksum are left alone, files with a7# wrong checksum are re-downloaded, and a download that still does not match8# fails loudly. All URLs are pinned to a commit hash or a versioned release.9#10# The manifest (scripts/stress-data.tsv, tab-separated, one header row:11# name, url, sha256, bytes, license, shape, purpose) is the single source of12# truth; Go code can read it too. The table below is a summary of it.13#14# | file | exercises | source (pinned) | license | bytes | rows x cols |15# |--------------------------------|--------------------------------------------------|---------------------------------------------------|--------------------------|---------|------------------------------|16# | owid-energy-data.csv | wide CSV (130 cols) + rows past the 8192 limit | github owid/energy-data @7e387a1 | CC-BY-4.0 | 9229369 | 23377 x 130 |17# | airport-codes.csv | very long CSV (10x row limit), accented Latin | github datasets/airport-codes @08b5eef | ODC-PDDL-1.0 | 8809882 | 86134 x 13 |18# | co2-ppm-daily.csv | numeric CSV past the row limit, dates | github datasets/co2-ppm-daily @ac476b2 | ODC-PDDL-1.0 (NOAA) | 347788 | 18304 x 2 |19# | country-codes.csv | Arabic/CJK/Cyrillic text, leading-zero codes | github datasets/country-codes @6a595f1 | ODC-PDDL-1.0 | 134003 | 249 x 56 |20# | FormulaEvalTestData_Copy.xlsx | XLSX formulas (1189 on sheet 1) + number formats | github apache/poi @942d95d test-data/spreadsheet | Apache-2.0 | 65011 | sheet 1 A2:AL1504 (788 rows) |21# | Chinook_Sqlite.sqlite | SQLite tables (PlaylistTrack past row limit) | github lerocha/chinook-database release v1.4.5 | MIT | 1067008 | 11 tables, max 8715 x 2 |22# | alltypes_tiny_pages.parquet | Parquet, every primitive type, tiny pages | github apache/parquet-testing @56653c4 | Apache-2.0 | 454233 | 7300 x 13 |23#24# Total: about 20 MB. Every file is openly licensed (CC-BY-4.0, public domain25# dedications, Apache-2.0 or MIT); see the license column of the manifest.26#27# Usage: scripts/stress-data.sh (from anywhere)28# STRESS_DIR=/tmp/stress scripts/stress-data.sh2930set -eu3132script_dir=$(cd "$(dirname "$0")" && pwd)33repo_root=$(cd "$script_dir/.." && pwd)34manifest="$script_dir/stress-data.tsv"35dest=${STRESS_DIR:-"$repo_root/.deps/stress"}3637if [ ! -f "$manifest" ]; then38 echo "stress-data: manifest not found: $manifest" >&239 exit 140fi4142if command -v shasum >/dev/null 2>&1; then43 sha256() { shasum -a 256 "$1" | awk '{print $1}'; }44elif command -v sha256sum >/dev/null 2>&1; then45 sha256() { sha256sum "$1" | awk '{print $1}'; }46else47 echo "stress-data: need shasum or sha256sum" >&248 exit 149fi5051filesize() { wc -c <"$1" | tr -d ' '; }5253mkdir -p "$dest"5455failed=056first=157while IFS=$'\t' read -r name url want bytes _license _shape _purpose; do58 if [ "$first" = 1 ]; then # header row59 first=060 continue61 fi62 [ -n "$name" ] || continue63 path="$dest/$name"6465 if [ -f "$path" ] && [ "$(sha256 "$path")" = "$want" ]; then66 printf '%-32s %10s ok\n' "$name" "$(filesize "$path")"67 continue68 fi6970 if [ -f "$path" ]; then71 echo "stress-data: $name: checksum mismatch on disk, re-downloading" >&272 fi7374 tmp="$path.part"75 rm -f "$tmp"76 if ! curl -fL --retry 3 --silent --show-error -o "$tmp" "$url"; then77 rm -f "$tmp"78 echo "stress-data: ERROR: $name: download failed: $url" >&279 failed=180 continue81 fi8283 got=$(sha256 "$tmp")84 if [ "$got" != "$want" ]; then85 echo "stress-data: ERROR: $name: SHA-256 mismatch after download" >&286 echo " url: $url" >&287 echo " expected: $want ($bytes bytes)" >&288 echo " got: $got ($(filesize "$tmp") bytes)" >&289 echo " the upstream file changed or the download is corrupt; kept as $tmp" >&290 failed=191 continue92 fi9394 mv -f "$tmp" "$path"95 printf '%-32s %10s downloaded\n' "$name" "$(filesize "$path")"96done <"$manifest"9798if [ "$failed" != 0 ]; then99 echo "stress-data: FAILED: one or more files could not be fetched and verified (see above)" >&2100 exit 1101fi