SUM-like functions hand the engine their aggregate and it adds each cell of a range as it reads it (Book.Fold), calling Agg.Add directly, rather than filling a chunk of values for the library to walk: a lone SUM over a range costs what it did inside the engine. Lookups walk the positions of a line's cells with a cursor instead of nested callbacks, reading each value they reach.
- commit
- 719e2beff43d02a809c666898a3723cfcc3502b1
- parent
- 46e1e3a812c763df268fd54fe789fc06d7d5c1f3
- author
- Dami <dami.etoile@gmail.com>
| M | internal/ | 26 | ++++++--- |
| M | internal/ | 85 | +++++++++++++++++++++++------- |
| M | internal/ | 7 | +- |
| M | internal/ | 71 | ++++++++++---------------- |
| M | internal/ | 34 | ++++++++++++ |
@@ -247,19 +247,18 @@ func NewAgg() Agg { return Agg{prod: 1, min: math.Inf(1), max: math.Inf(-1)} }
247247 // SUM("a") is #VALUE!. A range read first is served from the running248248 // aggregates shared by the recalculation (Book.RangeAgg; the engine's249249 // rangememo.go), so a thousand SUM(A:A) or a column of running totals250-// read each cell once.250+// read each cell once; other ranges the engine adds up itself251+// (Book.Fold), as it reads them.251252 func aggregate(done func(Agg) Value) func([]Node, lookup) Value {252253 return func(args []Node, get lookup) Value {253254 s := NewAgg()254255 for _, arg := range args {255- if rn, ok := arg.(formula.Range); ok && s.count == 0 {256- if a, e, ok := get.book.RangeAgg(rn.Sheet, rn.Rect); ok {257- if e != nil {258- return *e259- }260- s = a261- continue256+ if rn, ok := arg.(formula.Range); ok {257+ var e *Value258+ if s, e = rangeAgg(rn, s, get); e != nil {259+ return *e262260 }261+ continue263262 }264263 if e := eachOf(arg, get, s.Add); e != nil {265264 return *e@@ -269,6 +268,17 @@ func aggregate(done func(Agg) Value) func([]Node, lookup) Value {
269268 }270269 }271270 271+// rangeAgg adds the range rn to s: from the running aggregates when s is272+// empty and the range is shared, and otherwise as the engine reads it.273+func rangeAgg(rn formula.Range, s Agg, get lookup) (Agg, *Value) {274+ if s.count == 0 {275+ if a, e, ok := get.book.RangeAgg(rn.Sheet, rn.Rect); ok {276+ return a, e277+ }278+ }279+ return get.book.Fold(rn.Sheet, rn.Rect, s)280+}281+272282 func logical(test func(trues, n int) bool) func([]Node, lookup) Value {273283 return func(args []Node, get lookup) Value {274284 trues, n := 0, 0@@ -31,6 +31,11 @@ type Book interface {
3131 // a recalculation (see Agg), with the first error in it, or false3232 // when r should be read directly.3333 RangeAgg(sheet string, r Rect) (Agg, *Value, bool)34+ // Fold adds the cells of r that hold something to s, row by row, as35+ // SUM-like functions read a range (Agg.Add), evaluating them as Cell36+ // does and stopping at the first error, which it returns. On a sheet37+ // that doesn't exist the range is one #REF!.38+ Fold(sheet string, r Rect, s Agg) (Agg, *Value)3439 // Ask looks up the answer to a remote question (JEV functions). The3540 // value is an error when there is none to give: Pending while it is3641 // on its way, ErrNoRemote when nothing answers them.@@ -138,28 +143,66 @@ func (rd *Reader) cells(sheet string, r Rect, fn func(Addr, Value) bool) {
138143 139144 // stored calls fn with every cell of r that holds something, row by row,140145 // until fn returns false, without evaluating any. A sheet that doesn't141-// exist holds none. Lookups stop at the first match, so each call starts142-// with a small chunk, however long earlier ranges were, and doubles it143-// as the range goes on: finding a key near the top of a whole column144-// costs the cells before it.146+// exist holds none.145147 func (rd *Reader) stored(sheet string, r Rect, fn func(Addr) bool) {146- b := rd.take()147- defer rd.give()148- size := firstChunk149- for from := r.From; from.Row <= r.To.Row; {150- n := rd.book.Scan(sheet, r, from, b.addrs[:size], nil)151- for i := range n {152- if !fn(b.addrs[i]) {153- return154- }155- }156- if n < size {157- return158- }159- from = next(r, b.addrs[n-1])160- if size < maxChunk {161- size *= 2162- b.fit(size)148+ c := rd.walk(sheet, r)149+ defer c.close()150+ for a, ok := c.next(); ok && fn(a); a, ok = c.next() {151+ }152+}153+154+// cursor walks the cells of a range that hold something, row by row,155+// without evaluating any: stored as a loop, for searches that stop part156+// way and read values as they go. Searches stop at their match, so a157+// walk starts with a small chunk, however long earlier ranges were, and158+// doubles it as the range goes on: finding a key near the top of a159+// whole column costs the cells before it. A cursor holds a buffer until160+// it is closed.161+type cursor struct {162+ rd *Reader163+ b *scanBuf164+ sheet string165+ r Rect166+ from Addr // where the next chunk starts167+ size int // how many cells it asks for168+ n, i int // the cells in the buffer, and the next of them169+}170+171+// walk starts a cursor over the stored cells of r.172+func (rd *Reader) walk(sheet string, r Rect) cursor {173+ return cursor{rd: rd, b: rd.take(), sheet: sheet, r: r, from: r.From, size: firstChunk}174+}175+176+// next returns the next cell, or false when there are no more.177+func (c *cursor) next() (Addr, bool) {178+ if c.i < c.n {179+ c.i++180+ return c.b.addrs[c.i-1], true181+ }182+ return c.fill()183+}184+185+// fill reads the next chunk and returns its first cell.186+func (c *cursor) fill() (Addr, bool) {187+ if c.from.Row > c.r.To.Row {188+ return Addr{}, false189+ }190+ c.n, c.i = c.rd.book.Scan(c.sheet, c.r, c.from, c.b.addrs[:c.size], nil), 1191+ if c.n <= 0 {192+ c.from.Row = c.r.To.Row + 1193+ return Addr{}, false194+ }195+ if c.n < c.size {196+ c.from.Row = c.r.To.Row + 1 // that was the last197+ } else {198+ c.from = next(c.r, c.b.addrs[c.n-1])199+ if c.size < maxChunk {200+ c.size *= 2201+ c.b.fit(c.size)163202 }164203 }204+ return c.b.addrs[0], true165205 }206+207+// close gives the cursor's buffer back.208+func (c *cursor) close() { c.rd.give() }@@ -57,9 +57,10 @@ func (b *fakeBook) Scan(sheet string, r Rect, from Addr, addrs []Addr, vals []Va
5757 return n5858 }5959 60-func (b *fakeBook) Bounds(string, Rect) (Rect, bool, bool) { return Rect{}, false, true }61-func (b *fakeBook) RangeAgg(string, Rect) (Agg, *Value, bool) { return Agg{}, nil, false }62-func (b *fakeBook) Ask(RemoteCall) (RemoteAnswer, Value) { return RemoteAnswer{}, ErrNoRemote }60+func (b *fakeBook) Bounds(string, Rect) (Rect, bool, bool) { return Rect{}, false, true }61+func (b *fakeBook) RangeAgg(string, Rect) (Agg, *Value, bool) { return Agg{}, nil, false }62+func (b *fakeBook) Fold(_ string, _ Rect, s Agg) (Agg, *Value) { return s, nil }63+func (b *fakeBook) Ask(RemoteCall) (RemoteAnswer, Value) { return RemoteAnswer{}, ErrNoRemote }6364 func newFakeReader(cells map[Addr]Value) (*Reader, *fakeBook) {6465 b := &fakeBook{cells: cells}6566 depth := 0@@ -30,7 +30,7 @@ func lineSeq(m matrix, get lookup, vertical bool) seq {
3030 }3131 return seq{3232 n: line.size(), data: min(line.dataLen(), line.size()), line: line, blank: line.blank,33- sparse: !line.ref || !get.dense,33+ sparse: line.ref && !get.dense, // a single value is one entry, searched as data3434 get: get, vertical: vertical,3535 r: Rect{From: line.origin, To: Addr{Col: line.origin.Col + line.cols - 1, Row: line.origin.Row + line.rows - 1}},3636 }@@ -39,14 +39,8 @@ func lineSeq(m matrix, get lookup, vertical bool) seq {
3939 // at is entry i's value.4040 func (q *seq) at(i int) Value { return q.line.at(i) }4141 42-// value is the value of entry i, whose cell is at a when the line is a43-// range.44-func (q *seq) value(i int, a Addr) Value {45- if !q.line.ref {46- return q.at(i)47- }48- return q.get.cell(q.line.sheet, a)49-}42+// value is the value of the entry whose cell is at a.43+func (q *seq) value(a Addr) Value { return q.get.cell(q.line.sheet, a) }5044 5145 // index is the entry of the line's cell at a.5246 func (q *seq) index(a Addr) int {@@ -56,22 +50,13 @@ func (q *seq) index(a Addr) int {
5650 return a.Col - q.r.From.Col5751 }5852 59-// entries calls fn with the entries holding cells, in order, and where60-// their cells are, until fn returns false. It evaluates nothing: a search61-// reads the value of each entry it reaches (q.value), and no more.62-func (q *seq) entries(fn func(i int, a Addr) bool) {63- if !q.line.ref {64- fn(0, Addr{})65- return66- }67- q.get.stored(q.line.sheet, q.r, func(a Addr) bool { return fn(q.index(a), a) })68-}53+// walk walks the cells of a sparse line's entries, in order. It54+// evaluates nothing: a search reads the value of each entry it reaches55+// (q.value), and no more, as reading one at a time did.56+func (q *seq) walk() cursor { return q.get.walk(q.line.sheet, q.r) }6957 70-// positions is the entries holding cells, in order.58+// positions is the sparse line's entries holding cells, in order.7159 func (q *seq) positions() []int {72- if !q.line.ref {73- return []int{0}74- }7560 var out []int7661 q.get.stored(q.line.sheet, q.r, func(a Addr) bool {7762 out = append(out, q.index(a))@@ -84,23 +69,20 @@ func (q *seq) positions() []int {
8469 func (q *seq) findExact(key Value, wild bool) int {8570 if q.sparse {8671 blank := lookupEqual(key, q.blank, wild)87- next, found := 0, -1 // next is the first entry not looked at88- q.entries(func(i int, a Addr) bool {72+ next := 0 // the first entry not looked at73+ c := q.walk()74+ defer c.close()75+ for a, ok := c.next(); ok; a, ok = c.next() {76+ i := q.index(a)8977 switch {9078 case i > next && blank:91- found = next // a blank before this cell92- case lookupEqual(key, q.value(i, a), wild):93- found = i94- default:95- next = i + 196- return true79+ return next // a blank before this cell80+ case lookupEqual(key, q.value(a), wild):81+ return i9782 }98- return false99- })100- switch {101- case found >= 0:102- return found103- case blank && next < q.n:83+ next = i + 184+ }85+ if blank && next < q.n {10486 return next10587 }10688 return -1@@ -122,17 +104,18 @@ func (q *seq) findExact(key Value, wild bool) int {
122104 func (q *seq) findSorted(key Value, dir int) int {123105 found := -1124106 if q.sparse && key.Kind != q.blank.Kind { // blanks are skipped: only the cells count125- q.entries(func(i int, a Addr) bool {126- v := q.value(i, a)107+ c := q.walk()108+ defer c.close()109+ for a, ok := c.next(); ok; a, ok = c.next() {110+ v := q.value(a)127111 if v.Kind != key.Kind {128- return true112+ continue129113 }130114 if compare(v, key)*dir > 0 {131- return false115+ return found132116 }133- found = i134- return true135- })117+ found = q.index(a)118+ }136119 return found137120 }138121 for i := range q.data {@@ -77,6 +77,40 @@ func (rd *reader) RangeAgg(sheet string, r Rect) (functions.Agg, *Value, bool) {
7777 return run.through(r.To.Row)7878 }7979 80+// Fold adds the cells of r to s as SUM-like functions read a range; see81+// functions.Book. The engine walks the cells and adds each as it reads82+// it, so a range costs no more than a call to Agg.Add per cell.83+func (rd *reader) Fold(sheet string, r Rect, s functions.Agg) (functions.Agg, *Value) {84+ t := rd.sheet(sheet)85+ if t == nil {86+ return s, s.Add(ErrRef, false)87+ }88+ var e *Value89+ add := func(a Addr) bool {90+ e = s.Add(rd.read(t, a), false)91+ return e == nil92+ }93+ switch {94+ case denseReads:95+ for a := r.From; a.Row <= r.To.Row && add(a); {96+ if a.Col++; a.Col > r.To.Col {97+ a = Addr{Col: r.From.Col, Row: a.Row + 1}98+ }99+ }100+ case r.From.Col == r.To.Col:101+ t.cells.colScan(r.From.Col, r.From.Row, r.To.Row, func(row int) bool {102+ return add(Addr{Col: r.From.Col, Row: row})103+ })104+ default:105+ for a := range t.cells.inRange(r) {106+ if !add(a) {107+ break108+ }109+ }110+ }111+ return s, e112+}113+80114 // extend reads the run's columns from its next row through to.81115 func (run *runAgg) extend(rd *reader, t *Sheet, key aggKey, to int) {82116 run.busy = true