A chain of formulas nests once per cell, and the goroutine's stack grows with every level, so a formula's evaluation keeps its frame small: the array it spills waits on the Reader (Reader.Spilled) rather than being returned, the evaluation state outside array contexts and LET is saved in two locals, and array operators, spilling and the rest run out of line. Chains are back to the speed of the code before arrays (they were 12 to 20% slower). The cells a spill covers go in Snapshot.Spills beside the cells rather than on every SnapCell, which had made SQLite exports 14% slower. JEV questions are found through array literals and LAMBDA calls too. Co-Authored-By: Claude <noreply@anthropic.com>
- commit
- ab4a7d9f00c669e6a40df761ff49b5a7b716a8fa
- parent
- 4c6ea120133362ca1d5b5ba8401e6e9149b09aa4
- author
- Dami <dami.etoile@gmail.com>
| M | internal/ | 15 | +++- |
| M | internal/ | 15 | ++-- |
| M | internal/ | 119 | +++++++++++++++++++++--------- |
| M | internal/ | 2 | + |
| M | internal/ | 7 | +- |
| M | internal/ | 18 | +--- |
| M | internal/ | 11 | ++- |
@@ -37,6 +37,10 @@ type Snapshot struct {
3737 Filter *sheet.Filter3838 HiddenRows map[int]bool3939 40+ // Spills are the cells each formula whose array spills covers, by41+ // the formula's cell, for formats that keep formulas (XLSX).42+ Spills map[sheet.Addr]sheet.Rect43+4044 // Notes are the cells' notes in the range, for formats that keep4145 // them (XLSX, as comments). A note may be on a cell with no contents.4246 Notes map[sheet.Addr]string@@ -57,8 +61,7 @@ type SnapCell struct {
5761 Own sheet.Format // the cell's own format5862 Style sheet.Style5963 Formula bool60- Sheets []string // the sheets a formula names, as written61- Spill sheet.Rect // the cells a formula's array spills into, when it spills64+ Sheets []string // the sheets a formula names, as written6265 }6366 6467 // Text is the cell as displayed, without a width limit.@@ -102,7 +105,12 @@ func Snap(s *sheet.Sheet, r sheet.Rect, name string) *Snapshot {
102105 continue103106 }104107 c := s.Cell(a)105- spill, _ := s.SpillArea(a)108+ if area, ok := s.SpillArea(a); ok {109+ if snap.Spills == nil {110+ snap.Spills = map[sheet.Addr]sheet.Rect{}111+ }112+ snap.Spills[a] = area113+ }106114 snap.Cells[a] = SnapCell{107115 Input: c.Input,108116 Value: c.Value,@@ -111,7 +119,6 @@ func Snap(s *sheet.Sheet, r sheet.Rect, name string) *Snapshot {
111119 Style: s.CellStyle(a),112120 Formula: c.IsFormula(),113121 Sheets: s.NamedSheets(a),114- Spill: spill,115122 }116123 }117124 return snap@@ -29,10 +29,11 @@ type xlsxWriter struct {
2929 // refuse.3030 values valueCount3131 missing valueCount32- missingSheet string // the sheet missing's example names33- known map[string]bool // keys of the sheets written34- renamed map[string]string // the names of sheets written under another, by key35- dynamic bool // a dynamic array formula was written: see xlsxarray.go32+ missingSheet string // the sheet missing's example names33+ known map[string]bool // keys of the sheets written34+ renamed map[string]string // the names of sheets written under another, by key35+ dynamic bool // a dynamic array formula was written: see xlsxarray.go36+ spills map[sheet.Addr]sheet.Rect // the sheet being written's Snapshot.Spills3637 }3738 3839 // valueCount counts formulas written as values, keeping the first one's@@ -66,6 +67,7 @@ func (w *xlsxWriter) unknownSheet(c SnapCell) (string, bool) {
6667 // Errors stick in bw.6768 func (w *xlsxWriter) sheet(bw *bufio.Writer, ws string, snap *Snapshot, active bool) int {6869 r := snap.Range70+ w.spills = snap.Spills6971 bw.WriteString(xmlHead + `<worksheet xmlns="` + sheetMain + `" xmlns:r="` + officeRel + `">`)7072 if len(snap.HiddenRows) > 0 {7173 bw.WriteString(`<sheetPr filterMode="1"/>`) // a filter is hiding rows@@ -140,7 +142,7 @@ func (w *xlsxWriter) cell(b []byte, ws string, a sheet.Addr, c SnapCell) []byte
140142 switch {141143 case fx != "" && arrays:142144 b = append(b, `<f t="array" ref="`...)143- b = append(b, arrayRef(a, c.Spill)...)145+ b = append(b, arrayRef(a, w.spills[a])...)144146 b = append(b, `">`...)145147 b = appendEscaped(b, fx, false)146148 b = append(b, "</f>"...)@@ -180,7 +182,8 @@ func (w *xlsxWriter) formula(ws string, a sheet.Addr, c SnapCell) (string, bool)
180182 if !ok {181183 w.values.add(w.multi, ws, a)182184 }183- return fx, arrays || c.Spill != (sheet.Rect{})185+ _, spills := w.spills[a]186+ return fx, arrays || spills184187 }185188 186189 // cellValue is a cell's value as Excel stores it: the type attribute,@@ -18,54 +18,98 @@ func Eval(n Node, get *Reader) Value { return EvalAt(n, get, get.here) }
1818 // column (implicit intersection, as Sheets and Excel do). Evaluations1919 // nest, one cell's formula reading another's through the same Reader, so2020 // the cell before is given back afterwards.21-func EvalAt(n Node, get *Reader, at Addr) Value {22- v, _ := evalTop(n, get, at, false)23- return v24-}21+func EvalAt(n Node, get *Reader, at Addr) Value { return evalTop(n, get, at, false) }2522 26-// EvalCell computes the formula in the cell at, as EvalAt, and the array27-// it computes when that is several values: the cell shows the first, and28-// the engine spills the array from it. A 1x1 array is one value. A range29-// that is the whole formula is an array, as Sheets spills =B2:B9.30-func EvalCell(n Node, get *Reader, at Addr) (Value, *Array) {31- if r, ok := n.(formula.Range); ok {32- n = formula.Call{Fn: arrayFormula, Args: []Node{r}}33- }34- return evalTop(n, get, at, true)35-}23+// EvalCell computes the formula in the cell at, as EvalAt, keeping the24+// array it computes when that is several values for Spilled: the cell25+// shows the first, and the engine spills the array from it. A 1x1 array26+// is one value. A range that is the whole formula is an array, as Sheets27+// spills =B2:B9.28+func EvalCell(n Node, get *Reader, at Addr) Value { return evalTop(n, get, at, true) }29+30+// Spilled returns the array the formula EvalCell computed last spills,31+// or nil when it computed one value.32+func (rd *Reader) Spilled() *Array { return rd.spilled }33+34+// wholeRange is a range that is a cell's whole formula read through35+// ARRAYFORMULA, so it spills.36+//37+//go:noinline38+func wholeRange(n Node) Node { return formula.Call{Fn: arrayFormula, Args: []Node{n}} }3639 3740 // evalTop evaluates a cell's formula with a fresh evaluation state,38-// saving the state of the formula it nests in (a cell read while another39-// is evaluated).40-func evalTop(n Node, get *Reader, at Addr, arrays bool) (Value, *Array) {41- nested := get.nest > 042- var saved evalState43- if nested {44- saved = get.evalState41+// giving the state of the formula it nests in (a cell read while another42+// is evaluated) back afterwards. A chain of formulas nests once per43+// cell, and the goroutine's stack grows with every level, so evalTop44+// keeps its frame small: the common state, outside any array context or45+// LET, in two locals, the rest on the Reader's own stack.46+func evalTop(n Node, get *Reader, at Addr, arrays bool) Value {47+ if _, ok := n.(formula.Range); ok && arrays {48+ n = wholeRange(n)4549 }46- get.evalState = evalState{here: at, wantArr: arrays}50+ here, want := get.here, get.wantArr51+ rich := get.lift > 0 || get.scope != nil52+ if rich {53+ get.push()54+ }55+ get.here, get.wantArr = at, arrays4756 get.nest++4857 v := eval(n, get)49- var a *Array58+ get.spilled = nil5059 if v.Kind == value.Array {51- v, a = get.first(v), get.arrayOf(v)60+ v = get.spill(v)5261 }53- if get.nest--; get.nest == 0 && len(get.arena) > 0 {54- clear(get.arena)55- get.arena = get.arena[:0]62+ get.here, get.wantArr = here, want63+ if rich {64+ get.pop()5665 }57- if nested {58- get.evalState = saved66+ if get.nest--; get.nest == 0 && len(get.arena) > 0 {67+ get.empty()5968 }60- return v, a69+ return v70+}71+72+// push saves the evaluation state on the Reader's stack, and pop gives73+// it back.74+//75+//go:noinline76+func (rd *Reader) push() {77+ rd.outer = append(rd.outer, rd.evalState)78+ rd.lift, rd.scope = 0, nil79+}80+81+//go:noinline82+func (rd *Reader) pop() {83+ last := len(rd.outer) - 184+ rd.evalState = rd.outer[last]85+ rd.outer[last] = evalState{}86+ rd.outer = rd.outer[:last]87+}88+89+// spill keeps the array v stands for, for Spilled, and returns its90+// first value.91+//92+//go:noinline93+func (rd *Reader) spill(v Value) Value {94+ rd.spilled = rd.arrayOf(v)95+ return rd.first(v)96+}97+98+// empty empties the arena.99+//100+//go:noinline101+func (rd *Reader) empty() {102+ clear(rd.arena)103+ rd.arena = rd.arena[:0]61104 }62105 63106 // Reset forgets the evaluations in progress, which the engine abandoned64107 // part way (its evaluate.go), so the next starts afresh.65108 func (rd *Reader) Reset() {66- rd.evalState, rd.nest = evalState{}, 067- clear(rd.arena)68- rd.arena = rd.arena[:0]109+ rd.evalState, rd.nest, rd.spilled = evalState{}, 0, nil110+ clear(rd.outer)111+ rd.outer = rd.outer[:0]112+ rd.empty()69113 }70114 71115 func eval(n Node, get lookup) Value {@@ -194,7 +238,7 @@ func evalBinary(n formula.Binary, get lookup, dec bool) Value {
194238 *get.depth--195239 switch {196240 case l.Kind == value.Array || r.Kind == value.Array:197- return get.reduce(get.elementwise(l, r, func(l, r Value) Value { return binaryOp(n.Op, l, r, dec) }))241+ return get.binaryArrays(n.Op, l, r, dec)198242 case l.Kind == value.Error:199243 return l200244 case r.Kind == value.Error:@@ -213,6 +257,13 @@ func evalBinary(n formula.Binary, get lookup, dec bool) Value {
213257 return binaryOp(n.Op, l, r, dec)214258 }215259 260+// binaryArrays applies a binary operator to arrays, value by value.261+//262+//go:noinline263+func (rd *Reader) binaryArrays(op string, l, r Value, dec bool) Value {264+ return rd.reduce(rd.elementwise(l, r, func(l, r Value) Value { return binaryOp(op, l, r, dec) }))265+}266+216267 func binaryOp(op string, l, r Value, dec bool) Value {217268 if l.Kind == value.Error {218269 return l@@ -57,6 +57,8 @@ type Reader struct {
5757 // (array.go), emptied when the outermost is done; nest counts them.5858 arena []any5959 nest int60+ outer []evalState // the states of the formulas nest counts, but the innermost61+ spilled *Array // the array the last formula computed; see Spilled6062 // ranges are the ranges read whole as arrays since Forget, shared by6163 // the formulas reading them: a thousand FILTERs of one column read it6264 // once. See Forget.@@ -118,10 +118,9 @@ func (e *evaluator) formula(s *Sheet, a Addr, c *Cell) {
118118 expr := s.bound(c)119119 outer := w.evaluating120120 w.evaluating = loc{s, a}121- v, arr := functions.EvalCell(w.arith(expr), s.calcGet.lib, a)122- c.Value = v123- if arr != nil || s.spills != nil {124- w.noteSpill(loc{s, a}, arr, v)121+ c.Value = functions.EvalCell(w.arith(expr), s.calcGet.lib, a)122+ if s.calcGet.lib.Spilled() != nil || s.spills != nil {123+ w.noteSpill(s, a, c)125124 }126125 w.evaluating = outer127126 c.auto = functions.InferFormat(expr, s.calcFmt)@@ -59,22 +59,14 @@ func (s *Sheet) RemoteCalls(a Addr) []RemoteCall {
5959 var calls []RemoteCall6060 var walk func(Node)6161 walk = func(n Node) {62- switch n := n.(type) {63- case formula.Unary:64- walk(n.X)65- case formula.Binary:66- walk(n.L)67- walk(n.R)68- case formula.Call:69- if f := funcOf(n); f.Remote() {70- if call, err := f.Question(n.Args, get); err == nil {71- calls = append(calls, call)62+ if call, ok := n.(formula.Call); ok {63+ if f := funcOf(call); f.Remote() {64+ if q, err := f.Question(call.Args, get); err == nil {65+ calls = append(calls, q)7266 }7367 }74- for _, arg := range n.Args {75- walk(arg)76- }7768 }69+ formula.EachChild(n, walk)7870 }7971 walk(s.bound(c))8072 return calls@@ -58,10 +58,13 @@ type pendingSpill struct {
5858 v Value // what the formula computed, before spilling5959 }6060 61-// noteSpill records what the formula at l computed, when it spills or62-// did.63-func (w *Workbook) noteSpill(l loc, arr *functions.Array, v Value) {64- if arr == nil && l.s.spills[l.a] == nil {61+// noteSpill records what the formula of c, at a on s, computed, when it62+// spills or did.63+//64+//go:noinline65+func (w *Workbook) noteSpill(s *Sheet, a Addr, c *Cell) {66+ arr, v, l := s.calcGet.lib.Spilled(), c.Value, loc{s, a}67+ if arr == nil && s.spills[a] == nil {6568 return6669 }6770 if w.spillWork == nil {