Index which rows of each column hold cells (bitmaps of 1024 rows beside the cell map), so a range yields its cells in row order without visiting blank addresses. Aggregates read ranges sparsely; SUMIF, COUNTIFS, SUMPRODUCT, COUNTBLANK and the lookups walk only the data area of their ranges and account for the blank cells past it at once, keeping ROWS, INDEX and the positions they return at the range's full size. Within a recalculation, SUM-like functions share running aggregates per run of columns and first row, checkpointed after each row, so a thousand SUM(A1:A8192) read the column once and running totals cost O(n). Range users are found through interval trees per column (and one for wide ranges) instead of a fixed array of 256 column buckets. A test compares every clipped read with reading every address.
- commit
- c114f3207a388da596438aa314fe99d443c757df
- parent
- bdc52596b78b33bf8de50f034204c3732688d78f
- author
- Dami <dami.etoile@gmail.com>
| M | internal/ | 121 | ++++++++--- |
| M | internal/ | 112 | ++++++---- |
| M | internal/ | 67 | ++++-- |
| M | internal/ | 84 | -------- |
| M | internal/ | 77 | ------- |
| M | internal/ | 7 | +- |
| A | internal/ | 217 | ++++++++++++++++++++ |
| M | internal/ | 164 | +++++++++++++-- |
| M | internal/ | 4 | +- |
| A | internal/ | 126 | +++++++++++ |
| A | internal/ | 101 | +++++++++ |
| M | internal/ | 57 | +++-- |
| M | internal/ | 333 | ++++++++++++++++++++++++++++-- |
| A | internal/ | 85 | ++++++++ |
| M | internal/ | 8 | +- |
@@ -66,13 +66,20 @@ func boolArg(args []Node, i int, def bool, get lookup) (bool, *Value) {
6666 }6767 6868 // matrix is a rectangular block of values: a range, or a single value69-// treated as a 1x1 block.69+// treated as a 1x1 block. Only its top-left dataRows x dataCols may hold70+// data: every cell past them is blank, so functions that walk a range71+// (SUMIF(A:A, ...), MATCH, COUNTBLANK) visit what the sheet holds rather72+// than the million rows of a whole column, while ROWS, INDEX and the73+// positions they return keep the range's full size.7074 type matrix struct {7175 rows, cols int7276 cell func(r, c int) Value7377 origin Addr // top-left cell when the block is a reference7478 sheet string // the reference's sheet, as written7579 ref bool80+81+ dataRows, dataCols int82+ blank Value // every cell outside the data: blank, or #REF! for a missing sheet7683 }7784 7885 func (m matrix) at(i int) Value { return m.cell(i/m.cols, i%m.cols) }@@ -82,6 +89,14 @@ func (m matrix) size() int { return m.rows * m.cols }
8289 // vector reports whether m is a single row or column.8390 func (m matrix) vector() bool { return m.rows == 1 || m.cols == 1 }8491 92+// dataLen is how many of a vector's leading entries may hold data.93+func (m matrix) dataLen() int {94+ if m.cols == 1 {95+ return m.dataRows96+ }97+ return m.dataCols98+}99+85100 func matrixArg(n Node, get lookup) matrix {86101 switch n := n.(type) {87102 case formula.Range:@@ -90,17 +105,32 @@ func matrixArg(n Node, get lookup) matrix {
90105 return rectMatrix(n.Sheet, Rect{From: n.Addr, To: n.Addr}, get)91106 }92107 v := eval(n, get)93- return matrix{rows: 1, cols: 1, cell: func(int, int) Value { return v }}108+ return matrix{rows: 1, cols: 1, cell: func(int, int) Value { return v }, dataRows: 1, dataCols: 1, blank: v}94109 }95110 96111 func rectMatrix(sheet string, r Rect, get lookup) matrix {97- return matrix{112+ m := matrix{98113 rows: r.To.Row - r.From.Row + 1, cols: r.To.Col - r.From.Col + 1,99- cell: func(row, col int) Value {100- return get.cell(sheet, Addr{Col: r.From.Col + col, Row: r.From.Row + row})101- },102114 origin: r.From, sheet: sheet, ref: true,103115 }116+ switch t := get.sheet(sheet); {117+ case t == nil:118+ m.blank = ErrRef119+ case denseReads:120+ m.dataRows, m.dataCols = m.rows, m.cols121+ default:122+ if b, ok := t.cells.bounds(r); ok {123+ m.dataRows, m.dataCols = b.To.Row-r.From.Row+1, b.To.Col-r.From.Col+1124+ }125+ }126+ rows, cols, blank := m.dataRows, m.dataCols, m.blank127+ m.cell = func(row, col int) Value {128+ if row >= rows || col >= cols {129+ return blank130+ }131+ return get.cell(sheet, Addr{Col: r.From.Col + col, Row: r.From.Row + row})132+ }133+ return m104134 }105135 106136 // resized returns a block of rows x cols from the same top-left cell, as@@ -134,8 +164,8 @@ func nums(args []Node, get lookup) ([]float64, *Value) {
134164 return out, e135165 }136166 137-// texts flattens args to strings for CONCATENATE and TEXTJOIN: every cell138-// of a range, blanks as "".167+// texts flattens args to strings for CONCATENATE and TEXTJOIN: the cells168+// of ranges that hold something, and any other argument, blank or not.139169 func texts(args []Node, get lookup) ([]string, *Value) {140170 var out []string141171 e := each(args, get, func(v Value, _ bool) *Value {@@ -148,6 +178,51 @@ func texts(args []Node, get lookup) ([]string, *Value) {
148178 return out, e149179 }150180 181+// textsWithBlanks is texts with every blank cell of a range as "", as182+// TEXTJOIN keeping empty entries needs, but at most limit of them in all:183+// past that many the joined text is too long anyway.184+func textsWithBlanks(args []Node, get lookup, limit int) ([]string, *Value) {185+ var out []string186+ blanks := func(n int) {187+ for ; n > 0 && len(out) < limit; n-- {188+ out = append(out, "")189+ }190+ }191+ for _, arg := range args {192+ rn, ok := arg.(formula.Range)193+ if !ok {194+ if e := eachOf(arg, get, func(v Value, _ bool) *Value {195+ if v.Kind == Error {196+ return errOf(v)197+ }198+ out = append(out, text(v))199+ return nil200+ }); e != nil {201+ return nil, e202+ }203+ continue204+ }205+ r, last := rn.Rect, -1206+ width := r.To.Col - r.From.Col + 1207+ var e *Value208+ get.cells(rn.Sheet, r, func(a Addr, v Value) bool {209+ if v.Kind == Error {210+ e = errOf(v)211+ return false212+ }213+ p := (a.Row-r.From.Row)*width + a.Col - r.From.Col214+ blanks(p - last - 1)215+ out, last = append(out, text(v)), p216+ return true217+ })218+ if e != nil {219+ return nil, e220+ }221+ blanks(width*(r.To.Row-r.From.Row+1) - last - 1)222+ }223+ return out, nil224+}225+151226 func str(s string) Value { return Value{Kind: Text, Str: s} }152227 153228 func runeLen(s string) int { return utf8.RuneCountInString(s) }@@ -275,31 +350,21 @@ func cmpResult(op string, d int) bool {
275350 return d >= 0276351 }277352 278-// criteriaMask evaluates (range, criterion) pairs starting at args[i],279-// returning which cells of the first range satisfy all of them. All280-// ranges must be the same size.281-func criteriaMask(args []Node, i int, get lookup) (rows, cols int, mask []bool, err *Value) {353+// criteriaArgs reads the (range, criterion) pairs starting at args[i].354+// All ranges must be the same size.355+func criteriaArgs(args []Node, i int, get lookup) ([]matrix, []criterion, *Value) {356+ var ms []matrix357+ var cs []criterion282358 for ; i+1 < len(args); i += 2 {283359 m := matrixArg(args[i], get)284360 cv := eval(args[i+1], get)285361 if cv.Kind == Error {286- return 0, 0, nil, &cv362+ return nil, nil, &cv287363 }288- c := newCriterion(cv)289- if mask == nil {290- rows, cols = m.rows, m.cols291- mask = make([]bool, m.size())292- for k := range mask {293- mask[k] = true294- }295- } else if m.rows != rows || m.cols != cols {296- return 0, 0, nil, &ErrValue297- }298- for k := range mask {299- if mask[k] && !c.test(m.at(k)) {300- mask[k] = false301- }364+ if len(ms) > 0 && (m.rows != ms[0].rows || m.cols != ms[0].cols) {365+ return nil, nil, &ErrValue302366 }367+ ms, cs = append(ms, m), append(cs, newCriterion(cv))303368 }304- return rows, cols, mask, nil369+ return ms, cs, nil305370 }@@ -159,68 +159,84 @@ type agg struct {
159159 min, max float64160160 }161161 162-// each calls fn for every value in args: every cell of a range, or the163-// value of any other argument. direct is false for cells of a range or a164-// reference: as in Sheets, SUM(A1) ignores text in A1 but SUM("a") is165-// #VALUE!.162+// each calls fn for every value in args: every cell of a range that163+// holds something (blank cells, which every caller skips, aren't164+// visited), or the value of any other argument. direct is false for cells165+// of a range or a reference: as in Sheets, SUM(A1) ignores text in A1 but166+// SUM("a") is #VALUE!.166167 func each(args []Node, get lookup, fn func(v Value, direct bool) *Value) *Value {167168 for _, arg := range args {168- if ref, ok := arg.(formula.Ref); ok {169- if e := fn(get.cell(ref.Sheet, ref.Addr), false); e != nil {170- return e171- }172- continue173- }174- if _, ok := arg.(formula.Empty); ok {175- continue176- }177- if rn, ok := arg.(formula.Range); ok {178- var e *Value179- get.cells(rn.Sheet, rn.Rect, func(v Value) bool {180- e = fn(v, false)181- return e == nil182- })183- if e != nil {184- return e185- }186- continue187- }188- if e := fn(eval(arg, get), true); e != nil {169+ if e := eachOf(arg, get, fn); e != nil {189170 return e190171 }191172 }192173 return nil193174 }194175 176+func eachOf(arg Node, get lookup, fn func(v Value, direct bool) *Value) *Value {177+ switch arg := arg.(type) {178+ case formula.Ref:179+ return fn(get.cell(arg.Sheet, arg.Addr), false)180+ case formula.Empty:181+ return nil182+ case formula.Range:183+ var e *Value184+ get.cells(arg.Sheet, arg.Rect, func(_ Addr, v Value) bool {185+ e = fn(v, false)186+ return e == nil187+ })188+ return e189+ }190+ return fn(eval(arg, get), true)191+}192+193+// add counts v into the aggregate, with SUM's rules: blanks are skipped,194+// errors returned, and text counts only when given directly.195+func (s *agg) add(v Value, direct bool) *Value {196+ switch v.Kind {197+ case Error:198+ return errOf(v)199+ case Empty:200+ return nil201+ }202+ s.count++203+ if v.Kind != Number && !direct {204+ return nil205+ }206+ f, err := toNum(v)207+ if err != nil {208+ return err209+ }210+ s.nums++211+ s.sum += f212+ s.prod *= f213+ s.min, s.max = math.Min(s.min, f), math.Max(s.max, f)214+ return nil215+}216+217+func newAgg() agg { return agg{prod: 1, min: math.Inf(1), max: math.Inf(-1)} }218+195219 // aggregate builds SUM-like functions with Sheets semantics: in ranges only196220 // numbers count and text is ignored; direct arguments are coerced, so197-// SUM("a") is #VALUE!.221+// SUM("a") is #VALUE!. A range read first is served from the running222+// aggregates shared by the recalculation (rangememo.go), so a thousand223+// SUM(A:A) or a column of running totals read each cell once.198224 func aggregate(done func(agg) Value) func([]Node, lookup) Value {199225 return func(args []Node, get lookup) Value {200- s := agg{prod: 1, min: math.Inf(1), max: math.Inf(-1)}201- e := each(args, get, func(v Value, direct bool) *Value {202- switch v.Kind {203- case Error:204- return errOf(v)205- case Empty:206- return nil207- }208- s.count++209- if v.Kind != Number && !direct {210- return nil226+ s := newAgg()227+ for _, arg := range args {228+ if rn, ok := arg.(formula.Range); ok && s.count == 0 {229+ if a, e, ok := get.rangeAgg(rn.Sheet, rn.Rect); ok {230+ if e != nil {231+ return *e232+ }233+ s = a234+ continue235+ }211236 }212- f, err := toNum(v)213- if err != nil {214- return err237+ if e := eachOf(arg, get, s.add); e != nil {238+ return *e215239 }216- s.nums++217- s.sum += f218- s.prod *= f219- s.min, s.max = math.Min(s.min, f), math.Max(s.max, f)220- return nil221- })222- if e != nil {223- return *e224240 }225241 return done(s)226242 }@@ -1,6 +1,7 @@
11 package sheet22 33 import (4+ "slices"45 "strings"56 )67 @@ -40,14 +41,15 @@ func match(args []Node, get lookup) Value {
4041 case !m.vector():4142 return ErrNA4243 }44+ q := seq{n: m.size(), data: min(m.dataLen(), m.size()), at: m.at, blank: m.blank}4345 var i int4446 switch {4547 case kind == 0:46- i = findExact(key, m.size(), m.at, true)48+ i = q.findExact(key, true)4749 case kind > 0:48- i = findSorted(key, m.size(), m.at, 1)50+ i = q.findSorted(key, 1)4951 default:50- i = findSorted(key, m.size(), m.at, -1)52+ i = q.findSorted(key, -1)5153 }5254 if i < 0 {5355 return ErrNA@@ -114,11 +116,11 @@ func tableLookup(args []Node, get lookup, vertical bool) Value {
114116 if err != nil {115117 return *err116118 }117- n, across := m.rows, m.cols118- at := func(i int) Value { return m.cell(i, 0) }119+ q := seq{n: m.rows, data: min(m.dataRows, m.rows), at: func(i int) Value { return m.cell(i, 0) }, blank: m.blank}120+ across := m.cols119121 if !vertical {120- n, across = m.cols, m.rows121- at = func(i int) Value { return m.cell(0, i) }122+ q = seq{n: m.cols, data: min(m.dataCols, m.cols), at: func(i int) Value { return m.cell(0, i) }, blank: m.blank}123+ across = m.rows122124 }123125 switch {124126 case idx < 1:@@ -128,9 +130,9 @@ func tableLookup(args []Node, get lookup, vertical bool) Value {
128130 }129131 var i int130132 if sorted {131- i = findSorted(key, n, at, 1)133+ i = q.findSorted(key, 1)132134 } else {133- i = findExact(key, n, at, true)135+ i = q.findExact(key, true)134136 }135137 if i < 0 {136138 return ErrNA@@ -156,31 +158,45 @@ func lookupEqual(key, v Value, wild bool) bool {
156158 return key.Num == v.Num157159 }158160 161+// seq is the entries of a row or column searched by a lookup: n of them,162+// of which only the first data may hold data; the rest are all blank.163+type seq struct {164+ n, data int165+ at func(int) Value166+ blank Value167+}168+159169 // findExact returns the first index whose value equals key, or -1.160-func findExact(key Value, n int, at func(int) Value, wild bool) int {161- for i := range n {162- if lookupEqual(key, at(i), wild) {170+func (q seq) findExact(key Value, wild bool) int {171+ for i := range q.data {172+ if lookupEqual(key, q.at(i), wild) {163173 return i164174 }165175 }176+ if q.data < q.n && lookupEqual(key, q.blank, wild) {177+ return q.data178+ }166179 return -1167180 }168181 169182 // findSorted is the approximate match of sorted lookups: with dir 1 the170183 // last value <= key in ascending data, with dir -1 the last value >= key171184 // in descending data. Values of another kind are skipped.172-func findSorted(key Value, n int, at func(int) Value, dir int) int {185+func (q seq) findSorted(key Value, dir int) int {173186 found := -1174- for i := range n {175- v := at(i)187+ for i := range q.data {188+ v := q.at(i)176189 if v.Kind != key.Kind {177190 continue178191 }179192 if c := compare(v, key) * dir; c > 0 {180- break193+ return found181194 }182195 found = i183196 }197+ if v := q.blank; q.data < q.n && v.Kind == key.Kind && compare(v, key)*dir <= 0 {198+ found = q.n - 1 // the blanks past the data all match; the last wins199+ }184200 return found185201 }186202 @@ -216,11 +232,20 @@ func xlookup(args []Node, get lookup) Value {
216232 default:217233 return ErrValue218234 }219- order := make([]int, n)220- for i := range order {221- order[i] = i222- if search < 0 {223- order[i] = n - 1 - i235+ // The entries past the data are all blank, so only the first of them236+ // in search order can be the answer.237+ data := min(look.dataLen(), n)238+ order := make([]int, 0, data+1)239+ for i := range data {240+ order = append(order, i)241+ }242+ if data < n {243+ order = append(order, data)244+ }245+ if search < 0 {246+ slices.Reverse(order)247+ if data < n {248+ order[0] = n - 1224249 }225250 }226251 best := -1@@ -133,87 +133,3 @@ func toMultiple(round func(float64) float64) func([]Node, lookup) Value {
133133 return num(round(q) * f)134134 }135135 }136-137-func sumIf(args []Node, get lookup) Value {138- rng := matrixArg(args[0], get)139- cv := eval(args[1], get)140- if cv.Kind == Error {141- return cv142- }143- c := newCriterion(cv)144- sum := rng145- if len(args) > 2 {146- sum = matrixArg(args[2], get).resized(rng.rows, rng.cols, get)147- if sum.rows != rng.rows || sum.cols != rng.cols {148- return ErrValue149- }150- }151- total := 0.0152- for i := range rng.size() {153- if !c.test(rng.at(i)) {154- continue155- }156- v := sum.at(i)157- switch v.Kind {158- case Error:159- return v160- case Number:161- total += v.Num162- }163- }164- return num(total)165-}166-167-func sumIfs(args []Node, get lookup) Value {168- sum := matrixArg(args[0], get)169- rows, cols, mask, err := criteriaMask(args, 1, get)170- switch {171- case err != nil:172- return *err173- case rows != sum.rows || cols != sum.cols:174- return ErrValue175- }176- total := 0.0177- for i, ok := range mask {178- if !ok {179- continue180- }181- v := sum.at(i)182- switch v.Kind {183- case Error:184- return v185- case Number:186- total += v.Num187- }188- }189- return num(total)190-}191-192-// sumProduct multiplies same-sized arrays entry by entry and sums; text193-// and blanks count as 0.194-func sumProduct(args []Node, get lookup) Value {195- ms := make([]matrix, len(args))196- for i, a := range args {197- ms[i] = matrixArg(a, get)198- if ms[i].rows != ms[0].rows || ms[i].cols != ms[0].cols {199- return ErrValue200- }201- }202- total := 0.0203- for k := range ms[0].size() {204- p := 1.0205- for _, m := range ms {206- v := m.at(k)207- switch v.Kind {208- case Error:209- return v210- case Number:211- p *= v.Num212- default:213- p = 0214- }215- }216- total += p217- }218- return num(total)219-}@@ -40,47 +40,6 @@ func init() {
4040 4141 // Evaluators for the table above, in its order.4242 43-func averageIf(args []Node, get lookup) Value {44- rng := matrixArg(args[0], get)45- cv := eval(args[1], get)46- if cv.Kind == Error {47- return cv48- }49- c := newCriterion(cv)50- avg := rng51- if len(args) > 2 {52- avg = matrixArg(args[2], get).resized(rng.rows, rng.cols, get)53- }54- mask := make([]bool, rng.size())55- for i := range mask {56- mask[i] = c.test(rng.at(i))57- }58- return averageMasked(avg, mask)59-}60-61-func averageIfs(args []Node, get lookup) Value {62- avg := matrixArg(args[0], get)63- rows, cols, mask, err := criteriaMask(args, 1, get)64- switch {65- case err != nil:66- return *err67- case rows != avg.rows || cols != avg.cols:68- return ErrValue69- }70- return averageMasked(avg, mask)71-}72-73-func countBlank(args []Node, get lookup) Value {74- m := matrixArg(args[0], get)75- n := 076- for i := range m.size() {77- if v := m.at(i); v.Kind == Empty || v.Kind == Text && v.Str == "" {78- n++79- }80- }81- return num(float64(n))82-}83-8443 func median(x []float64) Value {8544 if len(x) == 0 {8645 return ErrNum@@ -140,42 +99,6 @@ func rank(args []Node, get lookup) Value {
14099 return num(float64(rank))141100 }142101 143-func countIfs(args []Node, get lookup) Value {144- _, _, mask, err := criteriaMask(args, 0, get)145- if err != nil {146- return *err147- }148- n := 0149- for _, ok := range mask {150- if ok {151- n++152- }153- }154- return num(float64(n))155-}156-157-// averageMasked averages the numbers of m where mask is set.158-func averageMasked(m matrix, mask []bool) Value {159- sum, n := 0.0, 0160- for i, ok := range mask {161- if !ok {162- continue163- }164- v := m.at(i)165- switch v.Kind {166- case Error:167- return v168- case Number:169- sum += v.Num170- n++171- }172- }173- if n == 0 {174- return ErrDiv0175- }176- return num(sum / float64(n))177-}178-179102 // withNums passes the numbers of all arguments, with aggregate180103 // semantics, to f.181104 func withNums(f func([]float64) Value) func([]Node, lookup) Value {@@ -82,7 +82,12 @@ func textJoin(args []Node, get lookup) Value {
8282 if err != nil {8383 return *err8484 }85- parts, err := texts(args[2:], get)85+ var parts []string86+ if skip || delim == "" {87+ parts, err = texts(args[2:], get)88+ } else {89+ parts, err = textsWithBlanks(args[2:], get, maxText+2)90+ }8691 if err != nil {8792 return *err8893 }@@ -0,0 +1,217 @@
1+package sheet2+3+// Functions over aligned ranges (SUMIF, COUNTIFS, SUMPRODUCT...) walk4+// only the part of their ranges that may hold data, the union of the5+// ranges' data areas; the cells past it are all blank, so they are6+// accounted for at once. SUMIF(A:A, "", B:B) over ten filled rows looks at7+// ten rows, however tall the sheet.8+9+// dataArea is the union of the data areas of same-sized matrices.10+func dataArea(ms ...matrix) (rows, cols int) {11+ for _, m := range ms {12+ rows = max(rows, min(m.dataRows, m.rows))13+ cols = max(cols, min(m.dataCols, m.cols))14+ }15+ return rows, cols16+}17+18+// masked is which cells of aligned ranges meet criteria: over the data19+// area (rows x cols, row-major), and for the cells past it, which are20+// the same everywhere.21+type masked struct {22+ rows, cols int23+ pass []bool24+ tail int // cells past the data area25+ tailPass bool // whether they meet the criteria26+}27+28+// applyCriteria tests each criterion on its range, over the data area of29+// the ranges and of also, which is aligned with them.30+func applyCriteria(ms []matrix, cs []criterion, also ...matrix) masked {31+ rows, cols := dataArea(append(also, ms...)...)32+ k := masked{rows: rows, cols: cols, pass: make([]bool, rows*cols), tailPass: true}33+ for i := range k.pass {34+ k.pass[i] = true35+ }36+ if len(ms) > 0 {37+ k.tail = ms[0].size() - rows*cols38+ }39+ for j, m := range ms {40+ for i, ok := range k.pass {41+ if ok && !cs[j].test(m.cell(i/cols, i%cols)) {42+ k.pass[i] = false43+ }44+ }45+ k.tailPass = k.tailPass && cs[j].test(m.blank)46+ }47+ return k48+}49+50+// sumMasked adds the numbers of m where the criteria pass.51+func sumMasked(m matrix, k masked) Value {52+ total := 0.053+ for i, ok := range k.pass {54+ if !ok {55+ continue56+ }57+ switch v := m.cell(i/k.cols, i%k.cols); v.Kind {58+ case Error:59+ return v60+ case Number:61+ total += v.Num62+ }63+ }64+ if k.tail > 0 && k.tailPass && m.blank.Kind == Error {65+ return m.blank66+ }67+ return num(total)68+}69+70+// averageMasked averages the numbers of m where the criteria pass.71+func averageMasked(m matrix, k masked) Value {72+ sum, n := 0.0, 073+ for i, ok := range k.pass {74+ if !ok {75+ continue76+ }77+ switch v := m.cell(i/k.cols, i%k.cols); v.Kind {78+ case Error:79+ return v80+ case Number:81+ sum += v.Num82+ n++83+ }84+ }85+ switch {86+ case k.tail > 0 && k.tailPass && m.blank.Kind == Error:87+ return m.blank88+ case n == 0:89+ return ErrDiv090+ }91+ return num(sum / float64(n))92+}93+94+func sumIf(args []Node, get lookup) Value {95+ rng := matrixArg(args[0], get)96+ cv := eval(args[1], get)97+ if cv.Kind == Error {98+ return cv99+ }100+ sum := rng101+ if len(args) > 2 {102+ sum = matrixArg(args[2], get).resized(rng.rows, rng.cols, get)103+ if sum.rows != rng.rows || sum.cols != rng.cols {104+ return ErrValue105+ }106+ }107+ return sumMasked(sum, applyCriteria([]matrix{rng}, []criterion{newCriterion(cv)}, sum))108+}109+110+func sumIfs(args []Node, get lookup) Value {111+ sum := matrixArg(args[0], get)112+ ms, cs, err := criteriaArgs(args, 1, get)113+ switch {114+ case err != nil:115+ return *err116+ case len(ms) == 0 || ms[0].rows != sum.rows || ms[0].cols != sum.cols:117+ return ErrValue118+ }119+ return sumMasked(sum, applyCriteria(ms, cs, sum))120+}121+122+func averageIf(args []Node, get lookup) Value {123+ rng := matrixArg(args[0], get)124+ cv := eval(args[1], get)125+ if cv.Kind == Error {126+ return cv127+ }128+ avg := rng129+ if len(args) > 2 {130+ avg = matrixArg(args[2], get).resized(rng.rows, rng.cols, get)131+ }132+ return averageMasked(avg, applyCriteria([]matrix{rng}, []criterion{newCriterion(cv)}, avg))133+}134+135+func averageIfs(args []Node, get lookup) Value {136+ avg := matrixArg(args[0], get)137+ ms, cs, err := criteriaArgs(args, 1, get)138+ switch {139+ case err != nil:140+ return *err141+ case len(ms) == 0 || ms[0].rows != avg.rows || ms[0].cols != avg.cols:142+ return ErrValue143+ }144+ return averageMasked(avg, applyCriteria(ms, cs, avg))145+}146+147+func countIfs(args []Node, get lookup) Value {148+ ms, cs, err := criteriaArgs(args, 0, get)149+ if err != nil {150+ return *err151+ }152+ k := applyCriteria(ms, cs)153+ n := 0154+ for _, ok := range k.pass {155+ if ok {156+ n++157+ }158+ }159+ if k.tailPass {160+ n += k.tail161+ }162+ return num(float64(n))163+}164+165+func countBlank(args []Node, get lookup) Value {166+ m := matrixArg(args[0], get)167+ isBlank := func(v Value) bool { return v.Kind == Empty || v.Kind == Text && v.Str == "" }168+ rows, cols := dataArea(m)169+ n := 0170+ for r := range rows {171+ for c := range cols {172+ if isBlank(m.cell(r, c)) {173+ n++174+ }175+ }176+ }177+ if isBlank(m.blank) {178+ n += m.size() - rows*cols179+ }180+ return num(float64(n))181+}182+183+// sumProduct multiplies same-sized arrays entry by entry and sums; text184+// and blanks count as 0.185+func sumProduct(args []Node, get lookup) Value {186+ ms := make([]matrix, len(args))187+ for i, a := range args {188+ ms[i] = matrixArg(a, get)189+ if ms[i].rows != ms[0].rows || ms[i].cols != ms[0].cols {190+ return ErrValue191+ }192+ }193+ rows, cols := dataArea(ms...)194+ total := 0.0195+ for k := range rows * cols {196+ p := 1.0197+ for _, m := range ms {198+ switch v := m.cell(k/cols, k%cols); v.Kind {199+ case Error:200+ return v201+ case Number:202+ p *= v.Num203+ default:204+ p = 0205+ }206+ }207+ total += p208+ }209+ if ms[0].size() > rows*cols {210+ for _, m := range ms {211+ if m.blank.Kind == Error {212+ return m.blank213+ }214+ }215+ }216+ return num(total)217+}@@ -1,23 +1,45 @@
11 package sheet22 3-// rangeIndex finds the formulas whose range references may contain a4-// cell, bucketed by column. Recalculation asks it about every changed5-// cell; scanning every range user instead made a change cost O(changed x6-// range users), so filling a column of 8192 running totals checked 677-// million ranges. A formula whose ranges span several columns is in each8-// of their buckets.3+import (4+ "cmp"5+ "slices"6+)7+8+// rangeIndex finds the formulas whose range references contain a cell.9+// Recalculation asks it about every changed cell; scanning every range10+// user instead made a change cost O(changed x range users), so filling a11+// column of 8192 running totals checked 67 million ranges.12+//13+// Ranges up to wideCols columns are indexed under each column they cover,14+// wider ones (whole rows) once; within each, an interval tree over rows15+// finds the ranges holding a row in O(log n + found), and formulas with16+// the same range (a thousand SUM(A:A)) share one entry. Only the columns17+// that ranges cover take memory.918 type rangeIndex struct {10- byCol [MaxCols]map[Addr]struct{}19+ byCol map[int]*intervals20+ wide intervals1121 }1222 13-// add indexes the formula at a under the columns its ranges cover.23+// wideCols is the widest range indexed column by column.24+const wideCols = 3225+26+// add indexes the formula at a under the ranges it reads.1427 func (x *rangeIndex) add(a Addr, ranges []Rect) {1528 for _, r := range ranges {29+ if r.To.Col-r.From.Col >= wideCols {30+ x.wide.add(r, a)31+ continue32+ }33+ if x.byCol == nil {34+ x.byCol = make(map[int]*intervals)35+ }1636 for c := max(r.From.Col, 0); c <= min(r.To.Col, MaxCols-1); c++ {17- if x.byCol[c] == nil {18- x.byCol[c] = make(map[Addr]struct{})37+ iv := x.byCol[c]38+ if iv == nil {39+ iv = &intervals{}40+ x.byCol[c] = iv1941 }20- x.byCol[c][a] = struct{}{}42+ iv.add(r, a)2143 }2244 }2345 }@@ -25,17 +47,123 @@ func (x *rangeIndex) add(a Addr, ranges []Rect) {
2547 // remove undoes add for the same ranges.2648 func (x *rangeIndex) remove(a Addr, ranges []Rect) {2749 for _, r := range ranges {50+ if r.To.Col-r.From.Col >= wideCols {51+ x.wide.remove(r, a)52+ continue53+ }2854 for c := max(r.From.Col, 0); c <= min(r.To.Col, MaxCols-1); c++ {29- delete(x.byCol[c], a)55+ if iv := x.byCol[c]; iv != nil {56+ iv.remove(r, a)57+ if len(iv.users) == 0 {58+ delete(x.byCol, c)59+ }60+ }61+ }62+ }63+}64+65+// readers calls fn with each formula one of whose ranges contains a; a66+// formula with two such ranges may come twice.67+func (x *rangeIndex) readers(a Addr, fn func(Addr)) {68+ if iv := x.byCol[a.Col]; iv != nil {69+ iv.stab(a, fn)70+ }71+ x.wide.stab(a, fn)72+}73+74+// intervals holds ranges and the formulas reading each, with a tree over75+// their rows built when first asked after a change.76+type intervals struct {77+ users map[Rect]map[Addr]struct{}78+ sorted []Rect // by first row; nil when stale79+ maxEnd []int // the last row of any range in the subtree rooted at each index80+}81+82+func (iv *intervals) add(r Rect, a Addr) {83+ if iv.users == nil {84+ iv.users = make(map[Rect]map[Addr]struct{})85+ }86+ u := iv.users[r]87+ if u == nil {88+ u = make(map[Addr]struct{})89+ iv.users[r] = u90+ iv.sorted = nil91+ }92+ u[a] = struct{}{}93+}94+95+func (iv *intervals) remove(r Rect, a Addr) {96+ u := iv.users[r]97+ if u == nil {98+ return99+ }100+ delete(u, a)101+ if len(u) == 0 {102+ delete(iv.users, r)103+ iv.sorted = nil104+ }105+}106+107+// stab calls fn with the users of every range containing a.108+func (iv *intervals) stab(a Addr, fn func(Addr)) {109+ if len(iv.users) == 0 {110+ return111+ }112+ if iv.sorted == nil {113+ iv.build()114+ }115+ iv.query(0, len(iv.sorted), a, fn)116+}117+118+func (iv *intervals) build() {119+ iv.sorted = iv.sorted[:0]120+ for r := range iv.users {121+ iv.sorted = append(iv.sorted, r)122+ }123+ slices.SortFunc(iv.sorted, func(p, q Rect) int {124+ if c := cmp.Compare(p.From.Row, q.From.Row); c != 0 {125+ return c126+ }127+ if c := cmp.Compare(p.To.Row, q.To.Row); c != 0 {128+ return c129+ }130+ if c := cmp.Compare(p.From.Col, q.From.Col); c != 0 {131+ return c30132 }133+ return cmp.Compare(p.To.Col, q.To.Col)134+ })135+ iv.maxEnd = slices.Grow(iv.maxEnd[:0], len(iv.sorted))[:len(iv.sorted)]136+ iv.fill(0, len(iv.sorted))137+}138+139+// fill computes maxEnd for the implicit tree over sorted[lo:hi], whose140+// root is its middle element, and returns it.141+func (iv *intervals) fill(lo, hi int) int {142+ if lo >= hi {143+ return -131144 }145+ mid := (lo + hi) / 2146+ m := max(iv.sorted[mid].To.Row, iv.fill(lo, mid), iv.fill(mid+1, hi))147+ iv.maxEnd[mid] = m148+ return m32149 }33150 34-// candidates returns the formulas with a range over column col; callers35-// still test whether one of their ranges contains the cell.36-func (x *rangeIndex) candidates(col int) map[Addr]struct{} {37- if col < 0 || col >= MaxCols {38- return nil151+func (iv *intervals) query(lo, hi int, a Addr, fn func(Addr)) {152+ for lo < hi {153+ mid := (lo + hi) / 2154+ if iv.maxEnd[mid] < a.Row {155+ return // nothing below reaches the row156+ }157+ iv.query(lo, mid, a, fn)158+ r := iv.sorted[mid]159+ if r.From.Row > a.Row {160+ return // this and everything after start below the row161+ }162+ if r.Contains(a) {163+ for u := range iv.users[r] {164+ fn(u)165+ }166+ }167+ lo = mid + 139168 }40- return x.byCol[col]41169 }@@ -45,7 +45,7 @@ func TestRangeUsersByColumn(t *testing.T) {
4545 s := New()4646 s.Set(at("E1"), "=SUM(A1:A5)")4747 s.Set(at("E1"), "=SUM(B1:B5)")48- if n := len(s.rangeUsers.candidates(0)); n != 0 {49- t.Errorf("%d stale range users in column A", n)48+ if iv := s.rangeUsers.byCol[0]; iv != nil {49+ t.Errorf("%d stale range users in column A", len(iv.users))5050 }5151 }@@ -0,0 +1,126 @@
1+package sheet2+3+import "slices"4+5+// Running aggregates: within one recalculation, the SUM-like functions6+// (SUM, AVERAGE, COUNT, COUNTA, MIN, MAX, PRODUCT) share what they read7+// of a range. For each run of columns and first row, the aggregate is8+// kept as a checkpoint after every row with data, read forward only as9+// far as some formula has asked. A thousand SUM(A1:A8192) read column A10+// once; 8192 running totals SUM($A$1:An) extend the same run one row at11+// a time, so they cost O(n) reads instead of O(n^2). Each result is12+// accumulated in the same order as reading the range directly, so it is13+// the same to the last bit.14+15+// memoRows is the shortest range, in rows, whose aggregate is shared:16+// shorter ones are read directly.17+const memoRows = 1618+19+type aggKey struct {20+ s *Sheet21+ c0, c1, r0 int22+}23+24+// runAgg is a run: the rows read so far with data, and the aggregate25+// through each.26+type runAgg struct {27+ rows []int28+ cum []agg29+ next int // every row before next has been read30+31+ errRow int32+ err *Value // the first error met, which ends the run33+34+ busy bool // being extended: a formula it reads may ask for it again35+}36+37+type aggMemo struct {38+ w *Workbook39+ m map[aggKey]*runAgg40+}41+42+// rangeAgg returns the aggregate of the range r on the named sheet, or43+// false when it should be read directly: no recalculation is running, the44+// range is short, its sheet doesn't exist, or a circular reference makes45+// the order of evaluation matter.46+func (rd *reader) rangeAgg(sheet string, r Rect) (agg, *Value, bool) {47+ memo := rd.memo48+ if memo == nil || denseReads || r.To.Row-r.From.Row+1 < memoRows || memo.w.Circular {49+ return agg{}, nil, false50+ }51+ t := rd.sheet(sheet)52+ if t == nil {53+ return agg{}, nil, false54+ }55+ key := aggKey{t, r.From.Col, r.To.Col, r.From.Row}56+ run := memo.m[key]57+ if run == nil {58+ run = &runAgg{next: r.From.Row}59+ memo.m[key] = run60+ }61+ if run.err == nil && r.To.Row >= run.next {62+ if run.busy {63+ return agg{}, nil, false64+ }65+ circular := memo.w.Circular66+ run.extend(rd, t, key, r.To.Row)67+ if memo.w.Circular != circular {68+ delete(memo.m, key) // it read a cell of a cycle69+ }70+ }71+ return run.through(r.To.Row)72+}73+74+// extend reads the run's columns from its next row through to.75+func (run *runAgg) extend(rd *reader, t *Sheet, key aggKey, to int) {76+ run.busy = true77+ defer func() { run.busy = false }()78+ cur, row := newAgg(), -179+ if n := len(run.cum); n > 0 {80+ cur = run.cum[n-1]81+ }82+ visit := func(a Addr) bool {83+ if a.Row != row {84+ if row >= 0 {85+ run.rows, run.cum = append(run.rows, row), append(run.cum, cur)86+ }87+ row, run.next = a.Row, a.Row88+ }89+ if e := cur.add(rd.read(t, a), false); e != nil {90+ run.errRow, run.err = a.Row, e91+ return false92+ }93+ return true94+ }95+ if key.c0 == key.c1 {96+ t.cells.colScan(key.c0, run.next, to, func(r int) bool { return visit(Addr{Col: key.c0, Row: r}) })97+ } else {98+ for a := range t.cells.inRange(Rect{From: Addr{Col: key.c0, Row: run.next}, To: Addr{Col: key.c1, Row: to}}) {99+ if !visit(a) {100+ break101+ }102+ }103+ }104+ if run.err != nil {105+ return106+ }107+ if row >= 0 {108+ run.rows, run.cum = append(run.rows, row), append(run.cum, cur)109+ }110+ run.next = to + 1111+}112+113+// through is the aggregate from the run's first row through row to.114+func (run *runAgg) through(to int) (agg, *Value, bool) {115+ if run.err != nil && to >= run.errRow {116+ return agg{}, run.err, true117+ }118+ i, found := slices.BinarySearch(run.rows, to)119+ if found {120+ return run.cum[i], nil, true121+ }122+ if i == 0 {123+ return newAgg(), nil, true124+ }125+ return run.cum[i-1], nil, true126+}@@ -0,0 +1,101 @@
1+package sheet2+3+import (4+ "fmt"5+ "math/rand/v2"6+ "testing"7+)8+9+// Formulas read ranges sparsely (only the cells that hold something),10+// clip positional reads to the data and share running aggregates across a11+// recalculation. Each formula here gives exactly what reading every12+// address of its ranges gives, on sheets with blanks, text, errors and13+// empty strings scattered through the data.14+func TestRangeReadsMatchDense(t *testing.T) {15+ formulas := []string{16+ "=SUM(A1:A2000)", "=SUM(A1:E2000)", "=AVERAGE(B3:B1500)", "=COUNT(A:A)", "=COUNTA(A1:E3000)",17+ "=MIN(C1:C2000)", "=MAX(A1:C2000)", "=PRODUCT(D1:D40)", "=SUM(A1:A5, 1.5, A6:A900)",18+ `=SUMIF(A1:A2000, "")`, `=SUMIF(A1:A2000, ">5", C1:C2000)`, `=SUMIF(B1:B2000, "<>x", C1)`,19+ `=COUNTIF(A1:A2000, "")`, `=COUNTIF(B1:B2000, "<>b")`, `=COUNTIF(A1:C2000, ">3")`,20+ "=COUNTBLANK(A1:E2000)", `=COUNTIFS(A1:A2000, "", B1:B2000, "")`, `=SUMIFS(C1:C2000, A1:A2000, "<>")`,21+ `=AVERAGEIF(A1:A2000, ">0")`, `=AVERAGEIFS(C1:C2000, B1:B2000, "")`,22+ "=SUMPRODUCT(A1:A2000, C1:C2000)", "=SUMPRODUCT(A1:B2000, C1:D2000)",23+ "=MATCH(7, A1:A2000, 0)", "=MATCH(Z99, A1:A2000, 0)", "=MATCH(Z99, A1:A2000)", "=MATCH(1E9, C1:C2000)",24+ "=MATCH(Z99, A1:A2000, -1)", "=MATCH(-1, C1:C2000, -1)",25+ "=VLOOKUP(Z99, A1:C2000, 3, FALSE)", "=VLOOKUP(5, A1:C2000, 2)", "=HLOOKUP(Z99, A1:E2000, 2, FALSE)",26+ "=XLOOKUP(Z99, A1:A2000, C1:C2000)", "=XLOOKUP(Z99, A1:A2000, C1:C2000, , 0, -1)",27+ "=XLOOKUP(3, A1:A2000, C1:C2000, , 1)", "=XLOOKUP(3, A1:A2000, C1:C2000, , -1, -1)",28+ `=TEXTJOIN(",", FALSE, B1:B60)`, `=LEN(TEXTJOIN("-", FALSE, B1:C2000))`, `=TEXTJOIN(",", TRUE, B1:B2000)`,29+ "=CONCATENATE(B1:B80)", "=MEDIAN(A1:A2000)", "=RANK(C5, C1:C2000)", "=ROWS(A:A)", "=COLUMNS(1:1)",30+ "=INDEX(A:A, 70000)", "=MATCH(Z99, F1:F2000, 0)", "=MATCH(Z99, F1:F2000)", "=MATCH(Z99, F1:F2000, -1)",31+ "=XLOOKUP(Z99, F1:F2000, C1:C2000)", "=XLOOKUP(Z99, F1:F2000, C1:C2000, , 0, -1)", "=XLOOKUP(Z99, F1:F2000, C1:C2000, , 1)",32+ "=VLOOKUP(Z99, F1:G2000, 2, FALSE)", "=VLOOKUP(Z99, F1:G2000, 2)", `=COUNTIF(F1:F2000, "")`, "=COUNTBLANK(F1:F2000)",33+ `=LEN(TEXTJOIN(",", FALSE, F1:F2000))`, `=SUMIF(F1:F2000, "", G1:G2000)`, `=SUMIF(F1:F2000, "<>1", A1:A2000)`, "=SUM(Nope!A1:A2000)", "=COUNTBLANK(Nope!A1:A20)", `=COUNTIF(Nope!A1:A20, "")`,34+ }35+ for seed := range 6 {36+ rng := rand.New(rand.NewPCG(uint64(seed), 7))37+ s := New()38+ fill := []string{"", "", "", "1", "2.5", "-3", "7", "x", "b", "TRUE", `=""`, "=1/0", "'5"}39+ if seed%2 == 1 {40+ fill = fill[:10] // no errors: aggregates get past them41+ }42+ for row := range 300 {43+ for col := range 5 {44+ if in := fill[rng.IntN(len(fill))]; in != "" {45+ s.Set(Addr{Col: col, Row: row * (1 + seed%3)}, in)46+ }47+ }48+ }49+ for row := range 60 {50+ s.Set(Addr{Col: 5, Row: row}, fill[3+rng.IntN(len(fill)-3)])51+ s.Set(Addr{Col: 6, Row: row + 1}, "4")52+ }53+ for i, f := range formulas {54+ s.Set(Addr{Col: 7 + i/200, Row: i % 200}, f)55+ }56+ // Running totals, down a column and over itself.57+ for row := range 120 {58+ s.Set(Addr{Col: 11, Row: row}, fmt.Sprintf("=SUM($A$1:A%d)", row+1))59+ s.Set(Addr{Col: 12, Row: row}, fmt.Sprintf("=SUM($M$1:M%d)+1", row))60+ s.Set(Addr{Col: 13, Row: row}, fmt.Sprintf("=COUNTA($A$1:$C%d)+MAX($C$3:C%d)", row+20, row+20))61+ }62+ s.Set(Addr{Col: 12, Row: 0}, "1")63+ s.RecalcAll()64+ sparse := snapshotValues(s)65+ denseReads = true66+ s.RecalcAll()67+ dense := snapshotValues(s)68+ denseReads = false69+ for a, v := range dense {70+ if sparse[a] != v {71+ t.Errorf("seed %d: %s %q = %+v, reading every cell gives %+v", seed, a, s.Cell(a).Input, sparse[a], v)72+ }73+ }74+ }75+}76+77+func snapshotValues(s *Sheet) map[Addr]Value {78+ out := map[Addr]Value{}79+ for a, c := range s.cells.all() {80+ out[a] = c.Value81+ }82+ return out83+}84+85+// A range whose running aggregate is being read by the formulas it86+// contains still gives each formula its own value.87+func TestRunningAggregateReentry(t *testing.T) {88+ s := New()89+ s.Set(at("A1"), "1")90+ for row := 2; row <= 40; row++ {91+ s.Set(Addr{Col: 0, Row: row - 1}, fmt.Sprintf("=SUM($A$1:A%d)", row-1))92+ }93+ // A(n) = 2^(n-2) for n >= 2.94+ if got := s.Value(at("A40")).Num; got != 1<<38 {95+ t.Errorf("A40 = %v, want %v", got, 1<<38)96+ }97+ s.Set(at("A1"), "2")98+ if got := s.Value(at("A40")).Num; got != 1<<39 {99+ t.Errorf("after the edit A40 = %v, want %v", got, 1<<39)100+ }101+}@@ -127,11 +127,7 @@ func (m *marking) readersOf(l loc) {
127127 for d := range s.dependents[a] {128128 m.push(loc{s, d})129129 }130- for u := range s.rangeUsers.candidates(a.Col) {131- if s.cells.get(u).rangeHas(a) {132- m.push(loc{s, u})133- }134- }130+ s.rangeUsers.readers(a, func(u Addr) { m.push(loc{s, u}) })135131 for _, nu := range m.named {136132 if nu.s == s && nu.r.Contains(a) {137133 for u := range nu.users {@@ -149,17 +145,6 @@ func (m *marking) readersOf(l loc) {
149145 }150146 }151147 152-// rangeHas reports whether one of the ranges the formula reads on its own153-// sheet contains a.154-func (c *Cell) rangeHas(a Addr) bool {155- for _, r := range c.ranges {156- if r.Contains(a) {157- return true158- }159- }160- return false161-}162-163148 // evaluate computes the cells marked dirty in each sheet's calc. Cells164149 // are evaluated lazily in dependency order: reading a dirty cell165150 // evaluates it first. A cell that is reached again while it is still@@ -204,10 +189,12 @@ func (w *Workbook) evaluate() {
204189 s.calc[a] = done205190 return c.Value206191 }192+ memo := &aggMemo{w: w, m: map[aggKey]*runAgg{}}207193 for _, s := range w.sheets {208194 s.version++209195 s.hidden.valid = false // values may have changed what the filter hides210196 s.calcGet = w.lookupOn(s, compute)197+ s.calcGet.memo = memo211198 s.calcFmt = w.formatFrom(s)212199 }213200 for _, s := range w.sheets {@@ -243,6 +230,7 @@ type reader struct {
243230 w *Workbook244231 s *Sheet245232 read func(*Sheet, Addr) Value233+ memo *aggMemo // running aggregates shared by a recalculation; nil otherwise246234 lastName string247235 last *Sheet248236 }@@ -271,23 +259,42 @@ func (rd *reader) cell(sheet string, a Addr) Value {
271259 return rd.read(t, a)272260 }273261 274-// cells calls fn with the value of every cell of r, row by row, until fn275-// returns false.276-func (rd *reader) cells(sheet string, r Rect, fn func(Value) bool) {262+// cells calls fn with the address and value of every cell of r that263+// holds something, row by row, until fn returns false: blank cells are264+// skipped without being visited, so a whole column costs what it holds.265+// A range on a sheet that doesn't exist reads as one #REF!.266+func (rd *reader) cells(sheet string, r Rect, fn func(Addr, Value) bool) {277267 t := rd.sheet(sheet)278- for row := r.From.Row; row <= r.To.Row; row++ {279- for col := r.From.Col; col <= r.To.Col; col++ {280- v := ErrRef281- if t != nil {282- v = rd.read(t, Addr{Col: col, Row: row})268+ switch {269+ case t == nil:270+ fn(r.From, ErrRef)271+ case denseReads:272+ for row := r.From.Row; row <= r.To.Row; row++ {273+ for col := r.From.Col; col <= r.To.Col; col++ {274+ if a := (Addr{Col: col, Row: row}); !fn(a, rd.read(t, a)) {275+ return276+ }283277 }284- if !fn(v) {278+ }279+ case r.From.Col == r.To.Col:280+ t.cells.colScan(r.From.Col, r.From.Row, r.To.Row, func(row int) bool {281+ a := Addr{Col: r.From.Col, Row: row}282+ return fn(a, rd.read(t, a))283+ })284+ default:285+ for a := range t.cells.inRange(r) {286+ if !fn(a, rd.read(t, a)) {285287 return286288 }287289 }288290 }289291 }290292 293+// denseReads makes formulas read every address of their ranges, as they294+// did before ranges were clipped to what they hold, so tests can check295+// that the two agree.296+var denseReads bool297+291298 // values reads current values for formulas on s, across sheets.292299 func (w *Workbook) values(s *Sheet) lookup {293300 return w.lookupOn(s, func(t *Sheet, a Addr) Value { return t.Value(a) })@@ -3,6 +3,8 @@ package sheet
33 import (44 "iter"55 "maps"6+ "math/bits"7+ "slices"68 )79 810 // cellStore holds a sheet's cells: those with contents or formatting, by@@ -11,23 +13,116 @@ import (
1113 // (to column blocks of compact values, with formulas and formats in side1214 // tables; see docs/limits.md) without touching it.1315 //14-// Iteration order is unspecified. Cells may be set or deleted while15-// iterating, with a map's rules: a deleted cell not yet reached isn't16-// yielded, a new one may or may not be.16+// Next to the map, an occupancy index records which rows of each column17+// hold a cell, as bitmaps of 1024 rows, so reading a range costs what the18+// range holds rather than its area: a whole column of a million rows with19+// ten cells is ten visits, in row order.20+//21+// Iteration order of all is unspecified; inRange yields in row-major22+// order. Cells may be set or deleted while iterating, with a map's rules:23+// a deleted cell not yet reached isn't yielded, a new one may or may not24+// be.1725 type cellStore struct {18- m map[Addr]*Cell26+ m map[Addr]*Cell27+ cols map[int]*colIndex28+ colIDs []int // the columns holding cells, ascending29+}30+31+// Rows of a column are indexed in blocks of blockRows, one bit each.32+const (33+ blockShift = 1034+ blockRows = 1 << blockShift35+ blockWords = blockRows / 6436+)37+38+type rowBlock struct {39+ bits [blockWords]uint6440+ n int1941 }2042 21-func newCellStore() cellStore { return cellStore{m: make(map[Addr]*Cell)} }43+// colIndex is the occupancy of one column: its blocks by number, and the44+// numbers in order.45+type colIndex struct {46+ blocks map[int]*rowBlock47+ ids []int48+ n int49+}50+51+func newCellStore() cellStore {52+ return cellStore{m: make(map[Addr]*Cell), cols: make(map[int]*colIndex)}53+}2254 2355 // get returns the cell at a, or nil.2456 func (st *cellStore) get(a Addr) *Cell { return st.m[a] }2557 2658 // set stores c, which isn't nil, at a.27-func (st *cellStore) set(a Addr, c *Cell) { st.m[a] = c }59+func (st *cellStore) set(a Addr, c *Cell) {60+ if _, ok := st.m[a]; !ok {61+ st.mark(a)62+ }63+ st.m[a] = c64+}2865 2966 // delete removes the cell at a, if any.30-func (st *cellStore) delete(a Addr) { delete(st.m, a) }67+func (st *cellStore) delete(a Addr) {68+ if _, ok := st.m[a]; ok {69+ st.unmark(a)70+ delete(st.m, a)71+ }72+}73+74+func (st *cellStore) mark(a Addr) {75+ ci := st.cols[a.Col]76+ if ci == nil {77+ ci = &colIndex{blocks: make(map[int]*rowBlock)}78+ st.cols[a.Col] = ci79+ st.colIDs = insertSorted(st.colIDs, a.Col)80+ }81+ id := a.Row >> blockShift82+ b := ci.blocks[id]83+ if b == nil {84+ b = &rowBlock{}85+ ci.blocks[id] = b86+ ci.ids = insertSorted(ci.ids, id)87+ }88+ bit := a.Row & (blockRows - 1)89+ b.bits[bit>>6] |= 1 << (bit & 63)90+ b.n++91+ ci.n++92+}93+94+func (st *cellStore) unmark(a Addr) {95+ ci := st.cols[a.Col]96+ id := a.Row >> blockShift97+ b := ci.blocks[id]98+ bit := a.Row & (blockRows - 1)99+ b.bits[bit>>6] &^= 1 << (bit & 63)100+ b.n--101+ ci.n--102+ if b.n == 0 {103+ delete(ci.blocks, id)104+ ci.ids = removeSorted(ci.ids, id)105+ }106+ if ci.n == 0 {107+ delete(st.cols, a.Col)108+ st.colIDs = removeSorted(st.colIDs, a.Col)109+ }110+}111+112+func insertSorted(s []int, v int) []int {113+ i, found := slices.BinarySearch(s, v)114+ if found {115+ return s116+ }117+ return slices.Insert(s, i, v)118+}119+120+func removeSorted(s []int, v int) []int {121+ if i, found := slices.BinarySearch(s, v); found {122+ return slices.Delete(s, i, i+1)123+ }124+ return s125+}31126 32127 // len is the number of cells stored, formatting-only ones included.33128 func (st *cellStore) len() int { return len(st.m) }@@ -35,27 +130,221 @@ func (st *cellStore) len() int { return len(st.m) }
35130 // all yields every stored cell.36131 func (st *cellStore) all() iter.Seq2[Addr, *Cell] { return maps.All(st.m) }37132 38-// inRange yields the stored cells in r, visiting whichever is cheaper:39-// each address of r (a lookup each), or every stored cell (iterating the40-// map costs about a quarter of a lookup per cell).133+// colsIn returns the stored columns from c0 to c1, ascending. The slice134+// is shared: callers mustn't keep it across changes.135+func (st *cellStore) colsIn(c0, c1 int) []int {136+ i, _ := slices.BinarySearch(st.colIDs, c0)137+ j, _ := slices.BinarySearch(st.colIDs, c1+1)138+ return st.colIDs[i:j]139+}140+141+// colCells is the number of cells stored in column c.142+func (st *cellStore) colCells(c int) int {143+ if ci := st.cols[c]; ci != nil {144+ return ci.n145+ }146+ return 0147+}148+149+// inRange yields the stored cells in r in row-major order, visiting only150+// the rows of each column that hold cells.41151 func (st *cellStore) inRange(r Rect) iter.Seq2[Addr, *Cell] {42152 return func(yield func(Addr, *Cell) bool) {43- area := (r.To.Col - r.From.Col + 1) * (r.To.Row - r.From.Row + 1)44- if area <= len(st.m)/4 {45- for row := r.From.Row; row <= r.To.Row; row++ {46- for col := r.From.Col; col <= r.To.Col; col++ {47- a := Addr{Col: col, Row: row}48- if c := st.m[a]; c != nil && !yield(a, c) {49- return50- }51- }52- }153+ cols := slices.Clone(st.colsIn(r.From.Col, r.To.Col))154+ if len(cols) == 0 || r.From.Row > r.To.Row {53155 return54156 }55- for a, c := range st.m {56- if r.Contains(a) && !yield(a, c) {157+ b0, b1 := r.From.Row>>blockShift, r.To.Row>>blockShift158+ for _, id := range st.blockIDs(cols, b0, b1) {159+ var in []colBlock160+ for _, c := range cols {161+ if b := st.cols[c].blocks[id]; b != nil {162+ in = append(in, colBlock{c, b})163+ }164+ }165+ if !st.yieldBlock(in, id, r, yield) {57166 return58167 }59168 }60169 }61170 }171+172+type colBlock struct {173+ col int174+ b *rowBlock175+}176+177+// blockIDs returns the numbers from b0 to b1 of the blocks that any of178+// cols has, ascending.179+func (st *cellStore) blockIDs(cols []int, b0, b1 int) []int {180+ if len(cols) == 1 {181+ ids := st.cols[cols[0]].ids182+ i, _ := slices.BinarySearch(ids, b0)183+ j, _ := slices.BinarySearch(ids, b1+1)184+ return slices.Clone(ids[i:j])185+ }186+ var out []int187+ for _, c := range cols {188+ ids := st.cols[c].ids189+ i, _ := slices.BinarySearch(ids, b0)190+ j, _ := slices.BinarySearch(ids, b1+1)191+ out = append(out, ids[i:j]...)192+ }193+ slices.Sort(out)194+ return slices.Compact(out)195+}196+197+// yieldBlock yields the cells of block id in the columns in, row by row,198+// limited to the rows of r.199+func (st *cellStore) yieldBlock(in []colBlock, id int, r Rect, yield func(Addr, *Cell) bool) bool {200+ base := id << blockShift201+ lo, hi := max(r.From.Row-base, 0), min(r.To.Row-base, blockRows-1)202+ for w := lo >> 6; w <= hi>>6; w++ {203+ var any uint64204+ for _, cb := range in {205+ any |= cb.b.bits[w]206+ }207+ any &= wordMask(w, lo, hi)208+ for any != 0 {209+ bit := bits.TrailingZeros64(any)210+ any &^= 1 << bit211+ row := base + w<<6 + bit212+ for _, cb := range in {213+ if cb.b.bits[w]&(1<<bit) == 0 {214+ continue215+ }216+ a := Addr{Col: cb.col, Row: row}217+ if c := st.m[a]; c != nil && !yield(a, c) {218+ return false219+ }220+ }221+ }222+ }223+ return true224+}225+226+// wordMask keeps the bits of word w that fall in rows lo..hi of a block.227+func wordMask(w, lo, hi int) uint64 {228+ m := ^uint64(0)229+ if first := w << 6; lo > first {230+ m &= ^uint64(0) << (lo - first)231+ }232+ if last := w<<6 + 63; hi < last {233+ m &= ^uint64(0) >> (last - hi)234+ }235+ return m236+}237+238+// colScan calls fn with each row from r0 to r1 of column c that holds a239+// cell, in order, until fn returns false, and reports whether it went to240+// the end. It allocates nothing; the column's cells must not be added241+// or removed meanwhile.242+func (st *cellStore) colScan(c, r0, r1 int, fn func(row int) bool) bool {243+ ci := st.cols[c]244+ if ci == nil || r0 > r1 {245+ return true246+ }247+ i, _ := slices.BinarySearch(ci.ids, r0>>blockShift)248+ for ; i < len(ci.ids) && ci.ids[i] <= r1>>blockShift; i++ {249+ id := ci.ids[i]250+ b, base := ci.blocks[id], id<<blockShift251+ lo, hi := max(r0-base, 0), min(r1-base, blockRows-1)252+ for w := lo >> 6; w <= hi>>6; w++ {253+ x := b.bits[w] & wordMask(w, lo, hi)254+ for x != 0 {255+ bit := bits.TrailingZeros64(x)256+ x &^= 1 << bit257+ if !fn(base + w<<6 + bit) {258+ return false259+ }260+ }261+ }262+ }263+ return true264+}265+266+// nextRow returns the first row of column c at or after row (dir 1), or267+// at or before it (dir -1), that holds a cell.268+func (st *cellStore) nextRow(c, row, dir int) (int, bool) {269+ ci := st.cols[c]270+ if ci == nil || row < 0 {271+ return 0, false272+ }273+ id := row >> blockShift274+ i, _ := slices.BinarySearch(ci.ids, id)275+ if dir > 0 {276+ for ; i < len(ci.ids); i++ {277+ b := ci.ids[i]278+ from := 0279+ if b == id {280+ from = row & (blockRows - 1)281+ }282+ if r, ok := ci.blocks[b].first(from); ok {283+ return b<<blockShift + r, true284+ }285+ }286+ return 0, false287+ }288+ if i == len(ci.ids) || ci.ids[i] != id {289+ i--290+ }291+ for ; i >= 0; i-- {292+ b := ci.ids[i]293+ to := blockRows - 1294+ if b == id {295+ to = row & (blockRows - 1)296+ }297+ if r, ok := ci.blocks[b].last(to); ok {298+ return b<<blockShift + r, true299+ }300+ }301+ return 0, false302+}303+304+// first is the lowest set row at or after from.305+func (b *rowBlock) first(from int) (int, bool) {306+ for w := from >> 6; w < blockWords; w++ {307+ x := b.bits[w]308+ if w == from>>6 {309+ x &= ^uint64(0) << (from & 63)310+ }311+ if x != 0 {312+ return w<<6 + bits.TrailingZeros64(x), true313+ }314+ }315+ return 0, false316+}317+318+// last is the highest set row at or before to.319+func (b *rowBlock) last(to int) (int, bool) {320+ for w := to >> 6; w >= 0; w-- {321+ x := b.bits[w]322+ if w == to>>6 {323+ x &= ^uint64(0) >> (63 - to&63)324+ }325+ if x != 0 {326+ return w<<6 + 63 - bits.LeadingZeros64(x), true327+ }328+ }329+ return 0, false330+}331+332+// bounds returns the smallest range holding every stored cell of r, and333+// false if r holds none. It costs a few steps per stored column of r.334+func (st *cellStore) bounds(r Rect) (Rect, bool) {335+ out, found := Rect{}, false336+ for _, c := range st.colsIn(r.From.Col, r.To.Col) {337+ top, ok := st.nextRow(c, r.From.Row, 1)338+ if !ok || top > r.To.Row {339+ continue340+ }341+ bottom, _ := st.nextRow(c, r.To.Row, -1)342+ if !found {343+ out, found = Rect{From: Addr{Col: c, Row: top}, To: Addr{Col: c, Row: bottom}}, true344+ continue345+ }346+ out.From.Row, out.To.Row = min(out.From.Row, top), max(out.To.Row, bottom)347+ out.To.Col = c348+ }349+ return out, found350+}@@ -0,0 +1,85 @@
1+package sheet2+3+import (4+ "math/rand/v2"5+ "slices"6+ "testing"7+)8+9+// The occupancy index agrees with the map: inRange yields exactly the10+// stored cells of a range in row-major order, bounds is their bounding11+// box and nextRow finds the neighbours, through random sets and deletes12+// that cross block boundaries.13+func TestCellStoreIndex(t *testing.T) {14+ rng := rand.New(rand.NewPCG(1, 2))15+ st := newCellStore()16+ rows := []int{0, 1, 63, 64, 1023, 1024, 1025, 5000, 70000, MaxRows - 1}17+ for i := range 3000 {18+ a := Addr{Col: rng.IntN(40), Row: rows[rng.IntN(len(rows))] + rng.IntN(3)}19+ a.Row = min(a.Row, MaxRows-1)20+ if rng.IntN(3) == 0 {21+ st.delete(a)22+ } else {23+ st.set(a, &Cell{Input: "x"})24+ }25+ if i%100 == 0 {26+ checkStore(t, &st, rng)27+ }28+ }29+ for a := range st.all() {30+ st.delete(a)31+ }32+ if len(st.cols) != 0 || len(st.colIDs) != 0 {33+ t.Errorf("empty store keeps %d columns", len(st.cols))34+ }35+}36+37+func checkStore(t *testing.T, st *cellStore, rng *rand.Rand) {38+ t.Helper()39+ for range 20 {40+ r := NewRect(Addr{Col: rng.IntN(45), Row: rng.IntN(80000)}, Addr{Col: rng.IntN(45), Row: rng.IntN(MaxRows)})41+ var want []Addr42+ for a := range st.all() {43+ if r.Contains(a) {44+ want = append(want, a)45+ }46+ }47+ sortAddrs(want)48+ var got []Addr49+ for a := range st.inRange(r) {50+ got = append(got, a)51+ }52+ if !slices.Equal(got, want) {53+ t.Fatalf("inRange(%v): %d cells, want %d", r, len(got), len(want))54+ }55+ b, ok := st.bounds(r)56+ if ok != (len(want) > 0) {57+ t.Fatalf("bounds(%v) found %v with %d cells", r, ok, len(want))58+ }59+ for _, a := range want {60+ if !b.Contains(a) {61+ t.Fatalf("bounds(%v) = %v misses %v", r, b, a)62+ }63+ }64+ if ok && (!slices.ContainsFunc(want, func(a Addr) bool { return a.Row == b.From.Row }) ||65+ !slices.ContainsFunc(want, func(a Addr) bool { return a.Col == b.To.Col })) {66+ t.Fatalf("bounds(%v) = %v isn't tight", r, b)67+ }68+ c, row := rng.IntN(40), rng.IntN(MaxRows)69+ next, ok := st.nextRow(c, row, 1)70+ want1, ok1 := MaxRows, false71+ prev, okp := st.nextRow(c, row, -1)72+ want2, ok2 := -1, false73+ for a := range st.all() {74+ if a.Col == c && a.Row >= row && a.Row < want1 {75+ want1, ok1 = a.Row, true76+ }77+ if a.Col == c && a.Row <= row && a.Row > want2 {78+ want2, ok2 = a.Row, true79+ }80+ }81+ if ok != ok1 || ok && next != want1 || okp != ok2 || okp && prev != want2 {82+ t.Fatalf("nextRow(%d, %d) = %d %v / %d %v, want %d %v / %d %v", c, row, next, ok, prev, okp, want1, ok1, want2, ok2)83+ }84+ }85+}@@ -47,13 +47,7 @@ func (s *Sheet) Dependents(a Addr) []Target {
4747 for d := range s.dependents[a] {4848 seen[loc{s, d}] = true4949 }50- for u := range s.rangeUsers.candidates(a.Col) {51- for _, r := range s.cells.get(u).ranges {52- if r.Contains(a) {53- seen[loc{s, u}] = true54- }55- }56- }50+ s.rangeUsers.readers(a, func(u Addr) { seen[loc{s, u}] = true })5751 for k, users := range w.nameUsers {5852 if n, ok := w.names[k]; ok && !n.Lost && n.Sheet == s && n.Range.Contains(a) {5953 for u := range users {