1//go:build stress23// Stress benchmarks of import and export: the real datasets fetched by4// scripts/stress-data.sh into .deps/stress (skipped when missing), and5// synthetic sheets written in every export format. Only built with6// -tags stress (see `make stress` and docs/contributing/limits.md).7package fileio89import (10 "context"11 "encoding/csv"12 "os"13 "path/filepath"14 "runtime"15 "runtime/metrics"16 "strconv"17 "testing"18 "time"1920 "src.rest/cat/012/internal/sheet"21 "src.rest/cat/012/internal/stress"22)2324// stressDir is where scripts/stress-data.sh puts the datasets.25func stressDir() string {26 if d := os.Getenv("STRESS_DIR"); d != "" {27 return d28 }29 return filepath.Join("..", "..", ".deps", "stress")30}3132// datasets reads the manifest's file names.33func datasets(b *testing.B) []string {34 f, err := os.Open(filepath.Join("..", "..", "scripts", "stress-data.tsv"))35 if err != nil {36 b.Fatal(err)37 }38 defer f.Close()39 r := csv.NewReader(f)40 r.Comma, r.LazyQuotes, r.FieldsPerRecord = '\t', true, -141 rows, err := r.ReadAll()42 if err != nil {43 b.Fatal(err)44 }45 var names []string46 for _, row := range rows[1:] {47 names = append(names, row[0])48 }49 return names50}5152// BenchmarkImport imports each dataset, reporting throughput, cells53// kept, and the live heap of the imported sheet per cell.54func BenchmarkImport(b *testing.B) {55 for _, name := range datasets(b) {56 path := filepath.Join(stressDir(), name)57 b.Run(name, func(b *testing.B) {58 st, err := os.Stat(path)59 if err != nil {60 b.Skip("run scripts/stress-data.sh first")61 }62 opt := Options{}63 if k, _ := KindOf(name); k == SQLite {64 opt.Table = "PlaylistTrack" // the biggest table65 }66 benchImport(b, path, st.Size(), opt)67 })68 }69}7071// BenchmarkImportXLSX imports synthetic workbooks written by 012's72// exporter: numbers, text and a column of formulas, each a full 819273// rows.74func BenchmarkImportXLSX(b *testing.B) {75 for _, tc := range []struct {76 name string77 s *sheet.Sheet78 }{79 {"numbers-8192x26", stress.Dense(stress.Rows, 26)},80 {"table-8192x26", stress.Table(stress.Rows-1, 26)},81 {"formulas-8192", stress.Chain(stress.Rows)},82 } {83 b.Run(tc.name, func(b *testing.B) {84 path := filepath.Join(b.TempDir(), tc.name+".xlsx")85 if _, err := Export(context.Background(), path, XLSX, Snap(tc.s, sheet.Rect{}, "data"), ExportOptions{}); err != nil {86 b.Fatal(err)87 }88 st, err := os.Stat(path)89 if err != nil {90 b.Fatal(err)91 }92 benchImport(b, path, st.Size(), Options{})93 })94 }95}9697// benchImport imports path in a loop and reports throughput, cells kept,98// the live heap of the imported sheet per cell (B/cell) and the peak99// heap while importing, per cell (peak-B/cell), sampled.100func benchImport(b *testing.B, path string, size int64, opt Options) {101 var res *Result102 var err error103 for b.Loop() {104 if res, err = Import(context.Background(), path, opt); err != nil {105 b.Fatal(err)106 }107 }108 b.SetBytes(size)109 cells := res.Sheet.Len()110 b.ReportMetric(float64(cells), "cells")111 b.ReportMetric(float64(cells)*float64(b.N)/b.Elapsed().Seconds(), "cells/s")112 res = nil113 before := liveHeap()114 stop := samplePeak()115 res, _ = Import(context.Background(), path, opt)116 peak := stop()117 b.ReportMetric(float64(liveHeap()-before)/float64(max(cells, 1)), "B/cell")118 b.ReportMetric(float64(peak-min(peak, before))/float64(max(cells, 1)), "peak-B/cell")119 runtime.KeepAlive(res)120}121122// samplePeak samples the heap every 100 microseconds until the returned123// function is called, which returns the most seen: live objects and124// garbage not yet collected, what the process holds.125func samplePeak() func() uint64 {126 sample := []metrics.Sample{{Name: "/memory/classes/heap/objects:bytes"}}127 read := func() uint64 {128 metrics.Read(sample)129 return sample[0].Value.Uint64()130 }131 done := make(chan struct{})132 peak := make(chan uint64)133 go func() {134 hi := read()135 t := time.NewTicker(100 * time.Microsecond)136 defer t.Stop()137 for {138 select {139 case <-done:140 peak <- max(hi, read())141 return142 case <-t.C:143 hi = max(hi, read())144 }145 }146 }()147 return func() uint64 {148 close(done)149 return <-peak150 }151}152153// BenchmarkExport writes a dense numeric sheet in each export format and154// reports the file size.155func BenchmarkExport(b *testing.B) {156 for _, size := range []struct{ rows, cols int }{{stress.Rows, 26}, {stress.Rows, 256}} {157 s := stress.Dense(size.rows, size.cols)158 snap := Snap(s, sheet.Rect{}, "data")159 for _, k := range []Kind{CSV, TSV, XLSX, SQLite} {160 b.Run(k.String()+"/"+strconv.Itoa(size.rows)+"x"+strconv.Itoa(size.cols), func(b *testing.B) {161 path := filepath.Join(b.TempDir(), "out"+k.Ext())162 for b.Loop() {163 os.Remove(path)164 if _, err := Export(context.Background(), path, k, snap, ExportOptions{Table: "data"}); err != nil {165 b.Fatal(err)166 }167 }168 if st, err := os.Stat(path); err == nil {169 b.ReportMetric(float64(st.Size())/(1<<20), "MB-file")170 }171 })172 }173 }174}175176func liveHeap() uint64 {177 runtime.GC()178 var ms runtime.MemStats179 runtime.ReadMemStats(&ms)180 return ms.HeapAlloc181}