BenchmarkPivot (-tags stress) times an edit under a pivot of a full table: 1 to 4 ms. Groups are looked up by a comparable key struct, and column groups through a tree of them, instead of strings built per row.
- commit
- 4f28187e0b2848071623a1423aec6ca905a9b71a
- parent
- d7dad192ad4c07fb5fb4ae8051c017c7b8c4d359
- author
- Dami <dami.etoile@gmail.com>
| M | docs/ | 22 | +++++++ |
| A | internal/ | 69 | +++++++++++++++++++++++ |
| M | internal/ | 92 | ++++++++++++++++-------------- |
| M | internal/ | 2 | +- |
@@ -29,6 +29,7 @@ it lags, and past a second it stalls.
2929 | JEV | Up to about 1000 JEV cells: 0.3 ms of CPU per answer | 4000 JEV cells: 1.3 ms per answer, 5 s of CPU to answer them all | | Every answer recalculates every JEV cell (they're volatile) |3030 | Formula depth | 10,000 nested parentheses or IFs: under 5 ms | | No explicit limit; recursion grows the stack | Recursive parser and evaluator |3131 | Find, filter, sort, fill | Filter or sort 8191 rows: 1 to 32 ms; find over 213 k cells: 38 ms; fill 8192 rows: 4 ms | Replace all over 213 k cells: 280 ms | | Per-cell string conversion and regexp |32+| Pivot tables | A pivot over 8191 rows recomputes after an edit to its source in 1 to 4 ms | | Results past 8192 x 256 (the pivot shows #REF!) | Reading each source cell of its fields: a map lookup each |3233 3334 ## Sheet size3435 @@ -187,6 +188,27 @@ Cell reads now go through one more function call to resolve the sheet,
187188 which is most of the 3 to 5% on read-heavy shapes; undo steps key their188189 before-images by sheet and address, which is BigUndo's extra allocation.189190 191+## Pivot tables192+193+A pivot table is recomputed whole whenever a cell of its source range is194+recalculated: it reads every source row once, groups it, and rewrites195+only the result cells that changed, so what reads the results196+recalculates only when they move. `BenchmarkPivot` edits one cell of a197+full 8191-row table (an id, a category of 8, a number and one of 1000198+items, twice) under a pivot and times the edit, the recalculation and199+the pivot together:200+201+| Pivot | Result rows | Edit | Allocations |202+|---|---|---|---|203+| Rows: 8 categories; SUM and COUNTA | 10 | 0.95 ms | 196 |204+| Rows: 8 x 8 categories with subtotals; columns: 8; SUM and AVERAGE | 75 | 2.7 to 2.9 ms | 2.5 k |205+| Rows: 1000 items sorted by their sum; SUM and COUNTUNIQUE | 1001 | 3.5 to 4.2 ms | 14 k |206+| Frequency table of the 1000 items | 1001 | 2.5 to 2.8 ms | 24 k |207+208+Groups are found by a comparable key struct rather than a string built209+per row: that took the 8-category pivot from 8.4 k to 196 allocations210+per edit, the others from 32 to 39 k to the figures above.211+190212 ## Hotspots found and fixed191213 192214 Each with its benchmark before and after, in the commit that fixed it.@@ -0,0 +1,69 @@
1+//go:build stress2+3+package sheet_test4+5+import (6+ "testing"7+8+ "github.com/FelineStateMachine/012/internal/sheet"9+ "github.com/FelineStateMachine/012/internal/stress"10+)11+12+// BenchmarkPivot is a pivot over a full 8192-row table recomputing after13+// one edit to its source: the edit, the recalculation and the pivot's14+// regrouping and rewriting of its results. The table's columns are an15+// id, a category of 8, a number and one of 1000 items, twice.16+func BenchmarkPivot(b *testing.B) {17+ num := func(col int) sheet.PivotValue { return sheet.PivotValue{Col: col, Summarize: sheet.SumBy} }18+ shapes := []struct {19+ name string20+ make func(src *sheet.Sheet, r sheet.Rect) sheet.Pivot21+ }{22+ {"rows-8", func(src *sheet.Sheet, r sheet.Rect) sheet.Pivot {23+ p := sheet.NewPivot(src, r)24+ p.Rows = []sheet.PivotGroup{{Col: 1}}25+ p.Values = []sheet.PivotValue{num(2), {Col: 3, Summarize: sheet.CountABy}}26+ return p27+ }},28+ {"rows-8x8-cols-8", func(src *sheet.Sheet, r sheet.Rect) sheet.Pivot {29+ p := sheet.NewPivot(src, r)30+ p.Rows = []sheet.PivotGroup{{Col: 1}, {Col: 5}}31+ p.Columns = []sheet.PivotGroup{{Col: 5}}32+ p.Values = []sheet.PivotValue{num(2), {Col: 6, Summarize: sheet.AverageBy}}33+ return p34+ }},35+ {"rows-1000-unique", func(src *sheet.Sheet, r sheet.Rect) sheet.Pivot {36+ p := sheet.NewPivot(src, r)37+ p.Rows = []sheet.PivotGroup{{Col: 3, Desc: true, SortBy: 1}}38+ p.Values = []sheet.PivotValue{num(2), {Col: 7, Summarize: sheet.CountUniqueBy}}39+ return p40+ }},41+ {"frequency-1000", func(src *sheet.Sheet, r sheet.Rect) sheet.Pivot {42+ return sheet.FrequencyPivot(src, r, 3)43+ }},44+ }45+ for _, sh := range shapes {46+ b.Run(sh.name, func(b *testing.B) {47+ src := stress.Table(sheet.MaxRows-1, 8)48+ r := sheet.NewRect(sheet.Addr{}, sheet.Addr{Col: 7, Row: sheet.MaxRows - 1})49+ pv, err := src.Book().CreatePivot(src, r, "", sh.make(src, r))50+ if err != nil {51+ b.Fatal(err)52+ }53+ if pv.PivotError() != "" {54+ b.Fatal(pv.PivotError())55+ }56+ edit := sheet.Addr{Col: 2, Row: 4000}57+ inputs := [2]string{"1234.5", "4321.5"}58+ i := 059+ for b.Loop() {60+ if err := src.Set(edit, inputs[i%2]); err != nil {61+ b.Fatal(err)62+ }63+ i++64+ }65+ out, _ := pv.PivotRange()66+ b.ReportMetric(float64(out.To.Row+1), "rows-out")67+ })68+ }69+}@@ -3,7 +3,6 @@ package sheet
33 import (44 "cmp"55 "slices"6- "strconv"76 "strings"87 )98 @@ -89,7 +88,7 @@ type accum struct {
8988 rows, counta, n int9089 sum, lo, hi float649190 err Value // the first error, which SUM and the rest show92- uniq map[string]struct{}91+ uniq map[groupKey]struct{}9392 }9493 9594 func (a *accum) add(v Value, unique bool) {@@ -112,9 +111,9 @@ func (a *accum) add(v Value, unique bool) {
112111 a.counta++113112 if unique {114113 if a.uniq == nil {115- a.uniq = map[string]struct{}{}114+ a.uniq = map[groupKey]struct{}{}116115 }117- a.uniq[valueKey(v, false)] = struct{}{}116+ a.uniq[keyOf(v, false)] = struct{}{}118117 }119118 }120119 @@ -126,25 +125,22 @@ func (a *accum) or(v Value) Value {
126125 return v127126 }128127 129-// valueKey identifies a value for grouping: numbers (dates too) by value,130-// text ignoring case when fold is set, as Sheets groups "east" with131-// "East". Numbers and text never share a key, so 1 and "1" are two132-// groups.133-func valueKey(v Value, fold bool) string {134- switch v.Kind {135- case Number:136- return "n" + strconv.FormatFloat(v.Num, 'g', -1, 64)137- case Text:138- if fold {139- return "t" + strings.ToLower(v.Str)140- }141- return "t" + v.Str142- case Bool:143- return "b" + strconv.FormatFloat(v.Num, 'g', -1, 64)144- case Error:145- return "e" + v.Str128+// groupKey identifies a value for grouping: numbers (dates too) by value,129+// text ignoring case when folded, as Sheets groups "east" with "East".130+// Numbers and text never share a key, so 1 and "1" are two groups. It is131+// a comparable struct, so looking a group up allocates nothing.132+type groupKey struct {133+ kind Kind134+ num float64135+ str string136+}137+138+func keyOf(v Value, fold bool) groupKey {139+ k := groupKey{kind: v.Kind, num: v.Num, str: v.Str}140+ if fold && v.Kind == Text {141+ k.str = strings.ToLower(v.Str)146142 }147- return ""143+ return k148144 }149145 150146 // pivotNode is a group of source rows: the root holds them all, its@@ -154,7 +150,7 @@ type pivotNode struct {
154150 label Value155151 format Format156152 kids []*pivotNode157- byKey map[string]*pivotNode153+ byKey map[groupKey]*pivotNode158154 acc map[int][]accum159155 }160156 @@ -165,6 +161,12 @@ type colGroup struct {
165161 formats []Format166162 }167163 164+// colNode finds column groups by each column field's value in turn.165+type colNode struct {166+ kids map[groupKey]*colNode167+ group *colGroup168+}169+168170 // pivotCalc is a pivot being computed.169171 type pivotCalc struct {170172 w *Workbook@@ -172,14 +174,14 @@ type pivotCalc struct {
172174 src *Sheet173175 root *pivotNode174176 cols []*colGroup // sorted once gathered175- colBy map[string]*colGroup177+ colRoot colNode176178 unique []bool // by value: whether to gather distinct values177179 formats []Format // by value: the format results show in178180 records int // source rows summarized179181 }180182 181183 func newPivotCalc(w *Workbook, p *Pivot, src *Sheet) *pivotCalc {182- c := &pivotCalc{w: w, p: p, src: src, root: &pivotNode{}, colBy: map[string]*colGroup{}}184+ c := &pivotCalc{w: w, p: p, src: src, root: &pivotNode{}}183185 for _, v := range p.Values {184186 c.unique = append(c.unique, v.Summarize == CountUniqueBy)185187 c.formats = append(c.formats, c.valueFormat(v))@@ -258,12 +260,12 @@ func pivotTests(fs []PivotFilter) []colTest {
258260 // child returns the group of n for the value at a, adding it if new.259261 func (n *pivotNode) child(src *Sheet, a Addr) *pivotNode {260262 v := src.Value(a)261- k := valueKey(v, true)263+ k := keyOf(v, true)262264 if kid, ok := n.byKey[k]; ok {263265 return kid264266 }265267 if n.byKey == nil {266- n.byKey = map[string]*pivotNode{}268+ n.byKey = map[groupKey]*pivotNode{}267269 }268270 kid := &pivotNode{label: v, format: src.DisplayFormat(a)}269271 n.byKey[k] = kid@@ -277,24 +279,30 @@ func (c *pivotCalc) colGroupOf(row int) int {
277279 if len(c.p.Columns) == 0 || len(c.p.Values) == 0 {278280 return -1279281 }280- var b strings.Builder282+ n := &c.colRoot281283 for _, g := range c.p.Columns {282- b.WriteString(valueKey(c.src.Value(Addr{Col: g.Col, Row: row}), true))283- b.WriteByte(0)284- }285- k := b.String()286- if cg, ok := c.colBy[k]; ok {287- return cg.id284+ k := keyOf(c.src.Value(Addr{Col: g.Col, Row: row}), true)285+ kid := n.kids[k]286+ if kid == nil {287+ if n.kids == nil {288+ n.kids = map[groupKey]*colNode{}289+ }290+ kid = &colNode{}291+ n.kids[k] = kid292+ }293+ n = kid288294 }289- cg := &colGroup{id: len(c.cols)}290- for _, g := range c.p.Columns {291- a := Addr{Col: g.Col, Row: row}292- cg.labels = append(cg.labels, c.src.Value(a))293- cg.formats = append(cg.formats, c.src.DisplayFormat(a))295+ if n.group == nil {296+ cg := &colGroup{id: len(c.cols)}297+ for _, g := range c.p.Columns {298+ a := Addr{Col: g.Col, Row: row}299+ cg.labels = append(cg.labels, c.src.Value(a))300+ cg.formats = append(cg.formats, c.src.DisplayFormat(a))301+ }302+ n.group = cg303+ c.cols = append(c.cols, cg)294304 }295- c.colBy[k] = cg296- c.cols = append(c.cols, cg)297- return cg.id305+ return n.group.id298306 }299307 300308 // add gathers row's values into n, for its column group and for all.@@ -107,7 +107,7 @@ func (l *pivotLayout) header() {
107107 108108 func sameLabels(a, b []Value) bool {109109 for i := range a {110- if valueKey(a[i], true) != valueKey(b[i], true) {110+ if keyOf(a[i], true) != keyOf(b[i], true) {111111 return false112112 }113113 }