Ranges read whole as arrays are shared by the formulas of a recalculation (Reader.Forget empties them as one starts and ends), so a thousand FILTERs of one column read it once. An anchor computing the array it already spilled, with nothing changed in its cells, writes nothing. UNIQUE groups a column by its values without building a key per row, and comparing text ignores ASCII case byte by byte, allocating nothing. On 1000 FILTER and UNIQUE arrays over 8192 rows an edit went from 414 to 57 ms and 9.2 M to 8 k allocations. Arithmetic on two numbers is done at once in the operator, a cell's formula saves the evaluation state only when it nests in another's, and the walks every entered formula takes switch on the tree themselves again, bringing formula-heavy benchmarks back within 3% of the code before arrays. Co-Authored-By: Claude <noreply@anthropic.com>
- commit
- b1aa3d3a98cb3f470668bcf9357610c0fa00aff5
- parent
- 2fc0c66812e856cab1c41c54d04caaa17821d35d
- author
- Dami <dami.etoile@gmail.com>
| M | internal/ | 32 | ++++++++-- |
| M | internal/ | 19 | ++++++- |
| M | internal/ | 52 | +++++++++++----- |
| M | internal/ | 94 | +++++++++++++++++++----------- |
| M | internal/ | 24 | ++++++-- |
| M | internal/ | 17 | +++++ |
| M | internal/ | 6 | ++- |
| M | internal/ | 1 | + |
| M | internal/ | 38 | ++++++++++-- |
| M | internal/ | 28 | +++++++++- |
@@ -121,13 +121,24 @@ func EachChild(n Node, fn func(Node)) {
121121 }122122 }123123 124-// WalkNames calls fn for every name in n.124+// WalkNames calls fn for every name in n. It walks the tree itself125+// rather than through EachChild, as every formula entered is walked.125126 func WalkNames(n Node, fn func(Name)) {126- if nn, ok := n.(Name); ok {127- fn(nn)128- return127+ switch n := n.(type) {128+ case Name:129+ fn(n)130+ case Unary:131+ WalkNames(n.X, fn)132+ case Binary:133+ WalkNames(n.L, fn)134+ WalkNames(n.R, fn)135+ case Call:136+ for _, a := range n.Args {137+ WalkNames(a, fn)138+ }139+ case Array, Invoke:140+ EachChild(n, func(k Node) { WalkNames(k, fn) })129141 }130- EachChild(n, func(k Node) { WalkNames(k, fn) })131142 }132143 133144 // WalkRefs calls fn for every single-cell reference and range in n, with@@ -138,7 +149,16 @@ func WalkRefs(n Node, ref func(string, Addr), rng func(string, Rect)) {
138149 ref(n.Sheet, n.Addr)139150 case Range:140151 rng(n.Sheet, n.Rect)141- default:152+ case Unary:153+ WalkRefs(n.X, ref, rng)154+ case Binary:155+ WalkRefs(n.L, ref, rng)156+ WalkRefs(n.R, ref, rng)157+ case Call:158+ for _, a := range n.Args {159+ WalkRefs(a, ref, rng)160+ }161+ case Array, Invoke:142162 EachChild(n, func(k Node) { WalkRefs(k, ref, rng) })143163 }144164 }@@ -188,11 +188,28 @@ func arrayArg(n Node, get lookup) (*Array, *Value) {
188188 }189189 190190 // rangeArray reads the range r on sheet as an array: its cells up to191-// the last one holding something, blank past them.191+// the last one holding something, blank past them. Arrays are never192+// changed once made, so a range read before (see Forget) is shared.192193 func (rd *Reader) rangeArray(sheet string, r Rect) Value {193194 if r.From == r.To {194195 return rd.cell(sheet, r.From)195196 }197+ k := rangeKey{sheet, r}198+ if a, ok := rd.ranges[k]; ok {199+ return rd.arrayValue(a)200+ }201+ v := rd.readRange(sheet, r)202+ if a := rd.arrayOf(v); a != nil {203+ if rd.ranges == nil {204+ rd.ranges = map[rangeKey]*Array{}205+ }206+ rd.ranges[k] = a207+ }208+ return v209+}210+211+// readRange reads a range as an array.212+func (rd *Reader) readRange(sheet string, r Rect) Value {196213 a := &Array{Rows: r.To.Row - r.From.Row + 1, Cols: r.To.Col - r.From.Col + 1}197214 b, any, exists := rd.book.Bounds(sheet, r)198215 switch {@@ -208,19 +208,8 @@ func unique(args []Node, get lookup) Value {
208208 if byCol {209209 a = a.transposed()210210 }211- n := min(a.DRows+1, a.Rows) // the data rows, and one blank row past them212- groups := map[string][]int{}213- var order []string214- for r := range n {215- k := rowKey(a, r)216- if _, seen := groups[k]; !seen {217- order = append(order, k)218- }219- groups[k] = append(groups[k], r)220- }221211 var keep []int222- for _, k := range order {223- g := groups[k]212+ for _, g := range groupRows(a, min(a.DRows+1, a.Rows)) { // the data rows, and one blank row past them224213 if once && (len(g) > 1 || g[0] == a.DRows && a.Rows > a.DRows+1) {225214 continue // repeated, or the blank row, which repeats past the data226215 }@@ -236,14 +225,45 @@ func unique(args []Node, get lookup) Value {
236225 return get.arrayValue(out)237226 }238227 228+// groupRows groups the first n rows of a that UNIQUE takes to be the229+// same, in the order each first comes: the rows of each group.230+func groupRows(a *Array, n int) [][]int {231+ if a.Cols == 1 { // a column's values are their own keys232+ return groupBy(n, func(r int) Value { return uniqueValue(a.At(r, 0)) })233+ }234+ return groupBy(n, func(r int) string { return rowKey(a, r) })235+}236+237+// groupBy groups rows 0..n-1 by key, in the order each key first comes.238+func groupBy[K comparable](n int, key func(int) K) [][]int {239+ var groups [][]int240+ index := map[K]int{}241+ for r := range n {242+ k := key(r)243+ i, found := index[k]244+ if !found {245+ i = len(groups)246+ index[k] = i247+ groups = append(groups, nil)248+ }249+ groups[i] = append(groups[i], r)250+ }251+ return groups252+}253+254+// uniqueValue is v as UNIQUE compares it: empty text is a blank.255+func uniqueValue(v Value) Value {256+ if v.Kind == value.Text && v.Str == "" {257+ return Value{}258+ }259+ return v260+}261+239262 // rowKey is a key equal for rows UNIQUE takes to be the same.240263 func rowKey(a *Array, r int) string {241264 var b []byte242265 for c := range a.Cols {243- v := a.At(r, c)244- if v.Kind == value.Text && v.Str == "" {245- v = Value{} // empty text and a blank are the same246- }266+ v := uniqueValue(a.At(r, c))247267 b = append(b, byte(v.Kind))248268 if v.Kind == value.Text || v.Kind == value.Error {249269 b = append(b, v.Str...)@@ -34,24 +34,34 @@ func EvalCell(n Node, get *Reader, at Addr) (Value, *Array) {
3434 // saving the state of the formula it nests in (a cell read while another3535 // is evaluated).3636 func evalTop(n Node, get *Reader, at Addr, arrays bool) (Value, *Array) {37- saved := get.evalState37+ nested := get.nest > 038+ var saved evalState39+ if nested {40+ saved = get.evalState41+ }3842 get.evalState = evalState{here: at, wantArr: arrays}3943 get.nest++40- // Deferred, so an evaluation the engine abandons part way (its41- // evaluate.go) leaves the Reader as it found it.42- defer get.leave(saved)4344 v := eval(n, get)44- return get.first(v), get.arrayOf(v)45+ var a *Array46+ if v.Kind == value.Array {47+ v, a = get.first(v), get.arrayOf(v)48+ }49+ if get.nest--; get.nest == 0 && len(get.arena) > 0 {50+ clear(get.arena)51+ get.arena = get.arena[:0]52+ }53+ if nested {54+ get.evalState = saved55+ }56+ return v, a4557 }4658 47-// leave gives back the evaluation state saved, emptying the arena when48-// the outermost formula is done.49-func (rd *Reader) leave(saved evalState) {50- if rd.nest--; rd.nest == 0 {51- clear(rd.arena)52- rd.arena = rd.arena[:0]53- }54- rd.evalState = saved59+// Reset forgets the evaluations in progress, which the engine abandoned60+// part way (its evaluate.go), so the next starts afresh.61+func (rd *Reader) Reset() {62+ rd.evalState, rd.nest = evalState{}, 063+ clear(rd.arena)64+ rd.arena = rd.arena[:0]5565 }5666 5767 func eval(n Node, get lookup) Value {@@ -87,31 +97,30 @@ func eval(n Node, get lookup) Value {
8797 case decBinary:8898 return evalBinary(formula.Binary(n), get, true)8999 case formula.Call:90- return evalCall(n, get)100+ // In an array context a function of one value is mapped over the101+ // arrays it's given (lift.go); otherwise its arguments are read as102+ // it reads them, and what it returns is one value unless the103+ // caller asked for an array.104+ *get.depth++ // see the engine's evaluate.go; operators count in theirs105+ f := funcOf(n)106+ var v Value107+ if get.lift > 0 && f.arrays != takesArrays {108+ v = liftCall(f, n.Args, get)109+ } else {110+ want := get.wantArr111+ get.wantArr = f.arrays != liftScalar112+ v = f.call(n.Args, get)113+ get.wantArr = want114+ }115+ *get.depth--116+ if v.Kind == value.Array {117+ return get.reduce(v)118+ }119+ return v91120 }92121 return get.reduce(evalOther(n, get))93122 }94123 95-// evalCall calls a function. In an array context a function of one96-// value is mapped over the arrays it's given (lift.go); otherwise its97-// arguments are read as it reads them, and what it returns is one value98-// unless the caller asked for an array.99-func evalCall(n formula.Call, get lookup) Value {100- *get.depth++ // see the engine's evaluate.go; operators count in theirs101- f := funcOf(n)102- var v Value103- if get.lift > 0 && f.arrays != takesArrays {104- v = liftCall(f, n.Args, get)105- } else {106- want := get.wantArr107- get.wantArr = f.arrays != liftScalar108- v = f.call(n.Args, get)109- get.wantArr = want110- }111- *get.depth--112- return get.reduce(v)113-}114-115124 // evalOther evaluates what formulas rarely hold: array literals, names116125 // LET and LAMBDA bind, LAMBDA calls, and arguments standing in for117126 // arrays being mapped over.@@ -179,8 +188,23 @@ func evalBinary(n formula.Binary, get lookup, dec bool) Value {
179188 *get.depth++180189 l, r := eval(n.L, get), eval(n.R, get)181190 *get.depth--182- if l.Kind == value.Array || r.Kind == value.Array {191+ switch {192+ case l.Kind == value.Array || r.Kind == value.Array:183193 return get.reduce(get.elementwise(l, r, func(l, r Value) Value { return binaryOp(n.Op, l, r, dec) }))194+ case l.Kind == value.Error:195+ return l196+ case r.Kind == value.Error:197+ return r198+ case !dec && l.Kind == value.Number && r.Kind == value.Number:199+ // Arithmetic on numbers, the commonest case, at once.200+ switch n.Op {201+ case "+":202+ return num(l.Num + r.Num)203+ case "-":204+ return num(l.Num - r.Num)205+ case "*":206+ return num(l.Num * r.Num)207+ }184208 }185209 return binaryOp(n.Op, l, r, dec)186210 }@@ -8,12 +8,26 @@ import (
88 // IsVolatile reports whether n calls a function whose result changes99 // without its inputs changing (TODAY, NOW, RAND).1010 func IsVolatile(n Node) bool {11- if c, ok := n.(formula.Call); ok && funcOf(c).Volatile {12- return true11+ switch n := n.(type) {12+ case formula.Unary:13+ return IsVolatile(n.X)14+ case formula.Binary:15+ return IsVolatile(n.L) || IsVolatile(n.R)16+ case formula.Call:17+ if funcOf(n).Volatile {18+ return true19+ }20+ for _, a := range n.Args {21+ if IsVolatile(a) {22+ return true23+ }24+ }25+ case formula.Array, formula.Invoke:26+ volatile := false27+ formula.EachChild(n, func(k Node) { volatile = volatile || IsVolatile(k) })28+ return volatile1329 }14- volatile := false15- formula.EachChild(n, func(k Node) { volatile = volatile || IsVolatile(k) })16- return volatile30+ return false1731 }1832 1933 // InferFormat picks the format Sheets shows a formula's result in when@@ -57,8 +57,25 @@ type Reader struct {
5757 // (array.go), emptied when the outermost is done; nest counts them.5858 arena []any5959 nest int60+ // ranges are the ranges read whole as arrays since Forget, shared by61+ // the formulas reading them: a thousand FILTERs of one column read it62+ // once. See Forget.63+ ranges map[rangeKey]*Array6064 }6165 66+// rangeKey is a range as a formula names it.67+type rangeKey struct {68+ sheet string69+ r Rect70+}71+72+// Forget drops the ranges read as arrays. The engine calls it as each73+// recalculation starts and ends: within one, a range's values don't74+// change once read, since reading it evaluates its dirty cells first (a75+// range holding a cell being evaluated is a cycle, an error either way),76+// and spilled cells are written between passes.77+func (rd *Reader) Forget() { clear(rd.ranges) }78+6279 // evalState is where a formula's evaluation is: saved while a cell it6380 // reads is evaluated through the same Reader, and given back after.6481 type evalState struct {@@ -62,6 +62,7 @@ func (w *Workbook) evaluate() {
6262 }6363 for _, s := range w.sheets {6464 s.calcGet.read, s.calcGet.memo = nil, nil65+ s.calcGet.lib.Forget()6566 s.calc, s.calcGet, s.calcFmt = nil, nil, nil6667 }6768 }@@ -119,7 +120,9 @@ func (e *evaluator) formula(s *Sheet, a Addr, c *Cell) {
119120 w.evaluating = loc{s, a}120121 v, arr := functions.EvalCell(w.arith(expr), s.calcGet.lib, a)121122 c.Value = v122- w.noteSpill(loc{s, a}, arr, v)123+ if arr != nil || s.spills != nil {124+ w.noteSpill(loc{s, a}, arr, v)125+ }123126 w.evaluating = outer124127 c.auto = functions.InferFormat(expr, s.calcFmt)125128 w.depth--@@ -176,6 +179,7 @@ func (e *evaluator) abandon(finished *bool) {
176179 s.calc[a] = dirty177180 }178181 }182+ s.calcGet.lib.Reset()179183 }180184 for _, p := range e.pending {181185 p.s.calc[p.a] = deferred@@ -212,6 +212,7 @@ func (w *Workbook) recalcReader(s *Sheet, e *evaluator, memo *aggMemo) *reader {
212212 s.recalcs = rd213213 }214214 rd.read, rd.memo, rd.lastName, rd.last = e.compute, memo, "", nil215+ rd.lib.Forget()215216 return rd216217 }217218 @@ -29,6 +29,12 @@ import (
2929 type spill struct {3030 area Rect // the cells it covers, the anchor first; when blocked, those it needs3131 why string // why the anchor shows #REF! instead, or ""32+ // arr and auto are the array written and its format, so the same33+ // array computed again isn't written again, unless stale says a cell34+ // of area changed since.35+ arr *functions.Array36+ auto Format37+ stale bool3238 }3339 3440 // ErrSpillEdit is what typing into a spilled cell says: its value@@ -112,6 +118,10 @@ func (s *Sheet) applySpill(a Addr, p pendingSpill) ([]loc, bool) {
112118 if c == nil || !c.IsFormula() || p.arr == nil {113119 return s.dropSpill(a), false114120 }121+ old := s.spills[a]122+ if old != nil && old.why == "" && !old.stale && old.auto == c.auto && sameArray(old.arr, p.arr) {123+ return nil, false // written already, and nothing has changed in its way124+ }115125 area, why := s.spillArea(a, p.arr)116126 if why != "" {117127 changed := s.dropSpill(a)@@ -120,10 +130,10 @@ func (s *Sheet) applySpill(a Addr, p pendingSpill) ([]loc, bool) {
120130 return changed, p.v != ErrRef121131 }122132 var changed []loc123- if old := s.spills[a]; old != nil && old.why == "" {133+ if old != nil && old.why == "" {124134 changed = s.clearSpilled(old.area, area)125135 }126- s.setSpill(a, &spill{area: area})136+ s.setSpill(a, &spill{area: area, arr: p.arr, auto: c.auto})127137 for r := range area.To.Row - a.Row + 1 {128138 for col := range area.To.Col - a.Col + 1 {129139 if r == 0 && col == 0 {@@ -181,11 +191,10 @@ func (s *Sheet) writeSpilled(a Addr, v Value, auto Format) bool {
181191 s.setDerived(a, formattingOnly(f, st))182192 return true183193 }184- c := &Cell{Input: derivedInput(v), Value: v, Format: f, Style: st, auto: auto, spilled: true}185- if old != nil && old.spilled && old.Value == c.Value && old.Input == c.Input && old.auto == c.auto {186- return false194+ if old != nil && old.spilled && old.Value == v && old.auto == auto {195+ return false // its input is its value's187196 }188- s.setDerived(a, c)197+ s.setDerived(a, &Cell{Input: derivedInput(v), Value: v, Format: f, Style: st, auto: auto, spilled: true})189198 return true190199 }191200 @@ -252,11 +261,28 @@ func (s *Sheet) spillTouched(a Addr, c *Cell) {
252261 }253262 s.spillAt.readers(a, func(anchor Addr) {254263 if anchor != a {264+ s.spills[anchor].stale = true255265 s.wb.markDirty(loc{s, anchor})256266 }257267 })258268 }259269 270+// sameArray reports whether two arrays hold the same values.271+func sameArray(x, y *functions.Array) bool {272+ if x == y {273+ return true274+ }275+ if x == nil || y == nil || x.Rows != y.Rows || x.Cols != y.Cols || x.DRows != y.DRows || x.DCols != y.DCols || x.Fill != y.Fill {276+ return false277+ }278+ for i, v := range x.V {279+ if y.V[i] != v {280+ return false281+ }282+ }283+ return true284+}285+260286 // markDirty has the step being made or undone recalculate the cell at261287 // l, without recording it for undo.262288 func (w *Workbook) markDirty(l loc) {@@ -7,6 +7,7 @@
77 package value88 99 import (10+ "cmp"1011 "math"1112 "strings"1213 @@ -111,6 +112,31 @@ func AsText(v Value) string {
111112 return v.String()112113 }113114 115+// compareText orders text ignoring case, as upper case: byte by byte116+// while both are ASCII, which most text is, so comparing a column of it117+// allocates nothing.118+func compareText(a, b string) int {119+ for i := 0; i < len(a) && i < len(b); i++ {120+ x, y := a[i], b[i]121+ if x >= utf8RuneSelf || y >= utf8RuneSelf {122+ return strings.Compare(strings.ToUpper(a[i:]), strings.ToUpper(b[i:]))123+ }124+ if x >= 'a' && x <= 'z' {125+ x -= 'a' - 'A'126+ }127+ if y >= 'a' && y <= 'z' {128+ y -= 'a' - 'A'129+ }130+ if x != y {131+ return cmp.Compare(x, y)132+ }133+ }134+ return cmp.Compare(len(a), len(b))135+}136+137+// utf8RuneSelf is where multi-byte UTF-8 starts.138+const utf8RuneSelf = 0x80139+114140 // Compare orders values like Sheets: numbers < text < booleans, text is115141 // case-insensitive, and a blank equals 0 or "".116142 func Compare(l, r Value) int {@@ -133,7 +159,7 @@ func Compare(l, r Value) int {
133159 return d134160 }135161 if l.Kind == Text {136- return strings.Compare(strings.ToUpper(l.Str), strings.ToUpper(r.Str))162+ return compareText(l.Str, r.Str)137163 }138164 switch {139165 case l.Num < r.Num: