Imports report throughput, cells kept and live heap per cell for each file in scripts/stress-data.tsv (skipped until the script has run); exports write 8192x26 and 8192x256 numeric sheets as CSV, TSV, XLSX and SQLite and report file sizes.
- commit
- 91fd92fa48a4b6f963355ef4fc6a8af8da1657ea
- parent
- be396942492b6e64bf8219c6c4443577e0191ded
- author
- Dami <dami.etoile@gmail.com>
| A | internal/ | 111 | ++++++++++++++++++++++++++++++ |
@@ -0,0 +1,111 @@
1+//go:build stress2+3+// Stress benchmarks of import and export: the real datasets fetched by4+// scripts/stress-data.sh into .deps/stress (skipped when missing), and5+// synthetic sheets written in every export format. Only built with6+// -tags stress (see `make stress` and docs/limits.md).7+package fileio8+9+import (10+ "context"11+ "encoding/csv"12+ "os"13+ "path/filepath"14+ "runtime"15+ "strconv"16+ "testing"17+18+ "012/internal/sheet"19+ "012/internal/stress"20+)21+22+// stressDir is where scripts/stress-data.sh puts the datasets.23+func stressDir() string {24+ if d := os.Getenv("STRESS_DIR"); d != "" {25+ return d26+ }27+ return filepath.Join("..", "..", ".deps", "stress")28+}29+30+// datasets reads the manifest's file names.31+func datasets(b *testing.B) []string {32+ f, err := os.Open(filepath.Join("..", "..", "scripts", "stress-data.tsv"))33+ if err != nil {34+ b.Fatal(err)35+ }36+ defer f.Close()37+ r := csv.NewReader(f)38+ r.Comma, r.LazyQuotes, r.FieldsPerRecord = '\t', true, -139+ rows, err := r.ReadAll()40+ if err != nil {41+ b.Fatal(err)42+ }43+ var names []string44+ for _, row := range rows[1:] {45+ names = append(names, row[0])46+ }47+ return names48+}49+50+// BenchmarkImport imports each dataset, reporting throughput, cells51+// kept, and the live heap of the imported sheet per cell.52+func BenchmarkImport(b *testing.B) {53+ for _, name := range datasets(b) {54+ path := filepath.Join(stressDir(), name)55+ b.Run(name, func(b *testing.B) {56+ st, err := os.Stat(path)57+ if err != nil {58+ b.Skip("run scripts/stress-data.sh first")59+ }60+ opt := Options{}61+ if k, _ := KindOf(name); k == SQLite {62+ opt.Table = "PlaylistTrack" // the biggest table63+ }64+ var res *Result65+ for b.Loop() {66+ if res, err = Import(context.Background(), path, opt); err != nil {67+ b.Fatal(err)68+ }69+ }70+ b.SetBytes(st.Size())71+ cells := res.Sheet.Len()72+ b.ReportMetric(float64(cells), "cells")73+ b.ReportMetric(float64(cells)*float64(b.N)/b.Elapsed().Seconds(), "cells/s")74+ res = nil75+ before := liveHeap()76+ res, _ = Import(context.Background(), path, opt)77+ b.ReportMetric(float64(liveHeap()-before)/float64(max(cells, 1)), "B/cell")78+ runtime.KeepAlive(res)79+ })80+ }81+}82+83+// BenchmarkExport writes a dense numeric sheet in each export format and84+// reports the file size.85+func BenchmarkExport(b *testing.B) {86+ for _, size := range []struct{ rows, cols int }{{sheet.MaxRows, 26}, {sheet.MaxRows, 256}} {87+ s := stress.Dense(size.rows, size.cols)88+ snap := Snap(s, sheet.Rect{}, "data")89+ for _, k := range []Kind{CSV, TSV, XLSX, SQLite} {90+ b.Run(k.String()+"/"+strconv.Itoa(size.rows)+"x"+strconv.Itoa(size.cols), func(b *testing.B) {91+ path := filepath.Join(b.TempDir(), "out"+k.Ext())92+ for b.Loop() {93+ os.Remove(path)94+ if _, err := Export(context.Background(), path, k, snap, ExportOptions{Table: "data"}); err != nil {95+ b.Fatal(err)96+ }97+ }98+ if st, err := os.Stat(path); err == nil {99+ b.ReportMetric(float64(st.Size())/(1<<20), "MB-file")100+ }101+ })102+ }103+ }104+}105+106+func liveHeap() uint64 {107+ runtime.GC()108+ var ms runtime.MemStats109+ runtime.ReadMemStats(&ms)110+ return ms.HeapAlloc111+}