SUMIF, COUNTIFS, AVERAGEIF(S), SUMPRODUCT and COUNTBLANK visit the cells any of their ranges holds and count the blanks between at once; MATCH, VLOOKUP, HLOOKUP and XLOOKUP walk the cells of the searched column (or row) in order and treat the blanks between as one. Ten thousand values spread down a million rows are ten thousand visits: the sparse-1M stress shape (1000 each of SUM(A:A), running totals and VLOOKUP over A:B) edits in 2.4 ms. A FanIn shape over whole columns joins the stress shapes.
- commit
- 0273b4f3fc5196dac05a2dff29bbdeea8033e480
- parent
- 3953e111662c62d394e9ae8f7a64bde7bee9d708
- author
- Dami <dami.etoile@gmail.com>
| M | internal/ | 134 | ++++++++++++++++++++--- |
| M | internal/ | 172 | +++++++++++++++++++----------- |
| M | internal/ | 36 | ++++++ |
@@ -1,6 +1,7 @@
11 package sheet22 33 import (4+ "iter"45 "slices"56 "strings"67 )@@ -41,7 +42,7 @@ func match(args []Node, get lookup) Value {
4142 case !m.vector():4243 return ErrNA4344 }44- q := seq{n: m.size(), data: min(m.dataLen(), m.size()), at: m.at, blank: m.blank}45+ q := lineSeq(m, get, m.cols == 1)4546 var i int4647 switch {4748 case kind == 0:@@ -116,10 +117,9 @@ func tableLookup(args []Node, get lookup, vertical bool) Value {
116117 if err != nil {117118 return *err118119 }119- q := seq{n: m.rows, data: min(m.dataRows, m.rows), at: func(i int) Value { return m.cell(i, 0) }, blank: m.blank}120+ q := lineSeq(m, get, vertical)120121 across := m.cols121122 if !vertical {122- q = seq{n: m.cols, data: min(m.dataCols, m.cols), at: func(i int) Value { return m.cell(0, i) }, blank: m.blank}123123 across = m.rows124124 }125125 switch {@@ -159,15 +159,72 @@ func lookupEqual(key, v Value, wild bool) bool {
159159 }160160 161161 // seq is the entries of a row or column searched by a lookup: n of them,162-// of which only the first data may hold data; the rest are all blank.162+// of which only the first data may hold data. With sparse, only those in163+// stored hold anything; every other entry is blank.163164 type seq struct {164165 n, data int165166 at func(int) Value166167 blank Value168+ stored iter.Seq[int] // the entries holding cells, in order, when sparse169+ sparse bool170+}171+172+// lineSeq is the first column (vertical) or row of m, as a lookup173+// searches it: through the cells it holds, when it is a range.174+func lineSeq(m matrix, get lookup, vertical bool) seq {175+ line := m176+ if m.ref {177+ to := Addr{Col: m.origin.Col, Row: m.origin.Row + m.rows - 1}178+ if !vertical {179+ to = Addr{Col: m.origin.Col + m.cols - 1, Row: m.origin.Row}180+ }181+ line = rectMatrix(m.sheet, Rect{From: m.origin, To: to}, get)182+ }183+ q := seq{n: line.size(), data: min(line.dataLen(), line.size()), at: line.at, blank: line.blank}184+ t := get.sheet(line.sheet)185+ switch {186+ case !line.ref:187+ q.sparse, q.stored = true, func(yield func(int) bool) { yield(0) }188+ case denseReads:189+ case t == nil:190+ q.sparse, q.stored = true, func(func(int) bool) {}191+ default:192+ q.sparse = true193+ r := Rect{From: line.origin, To: Addr{Col: line.origin.Col + line.cols - 1, Row: line.origin.Row + line.rows - 1}}194+ q.stored = func(yield func(int) bool) {195+ if vertical {196+ t.cells.colScan(r.From.Col, r.From.Row, r.To.Row, func(row int) bool { return yield(row - r.From.Row) })197+ return198+ }199+ for a := range t.cells.inRange(r) {200+ if !yield(a.Col - r.From.Col) {201+ return202+ }203+ }204+ }205+ }206+ return q167207 }168208 169209 // findExact returns the first index whose value equals key, or -1.170210 func (q seq) findExact(key Value, wild bool) int {211+ if q.sparse {212+ blank := lookupEqual(key, q.blank, wild)213+ next := 0 // the first entry not looked at214+ for i := range q.stored {215+ if i > next && blank {216+ return next // a blank before this cell217+ }218+ if lookupEqual(key, q.at(i), wild) {219+ return i220+ }221+ next = i + 1222+ }223+ if blank && next < q.n {224+ return next225+ }226+ return -1227+ }171228 for i := range q.data {172229 if lookupEqual(key, q.at(i), wild) {173230 return i@@ -184,6 +241,19 @@ func (q seq) findExact(key Value, wild bool) int {
184241 // in descending data. Values of another kind are skipped.185242 func (q seq) findSorted(key Value, dir int) int {186243 found := -1244+ if q.sparse && key.Kind != q.blank.Kind { // blanks are skipped: only the cells count245+ for i := range q.stored {246+ v := q.at(i)247+ if v.Kind != key.Kind {248+ continue249+ }250+ if compare(v, key)*dir > 0 {251+ return found252+ }253+ found = i254+ }255+ return found256+ }187257 for i := range q.data {188258 v := q.at(i)189259 if v.Kind != key.Kind {@@ -200,6 +270,45 @@ func (q seq) findSorted(key Value, dir int) int {
200270 return found201271 }202272 273+// order is the entries a search visits, first to last (last to first274+// with reverse): those that hold cells, and of the blanks, which are all275+// the same, only the first met.276+func (q seq) order(reverse bool) []int {277+ var out []int278+ if !q.sparse {279+ for i := range q.data {280+ out = append(out, i)281+ }282+ if q.data < q.n {283+ out = append(out, q.data)284+ }285+ if reverse {286+ slices.Reverse(out)287+ if q.data < q.n {288+ out[0] = q.n - 1289+ }290+ }291+ return out292+ }293+ out = slices.Collect(q.stored)294+ gap := -1 // the first blank in search order295+ if !reverse {296+ for gap = 0; gap < len(out) && out[gap] == gap; gap++ {297+ }298+ } else {299+ for gap = q.n - 1; len(out) > 0 && q.n-1-gap < len(out) && out[len(out)-1-(q.n-1-gap)] == gap; gap-- {300+ }301+ }302+ if gap >= 0 && gap < q.n {303+ i, _ := slices.BinarySearch(out, gap)304+ out = slices.Insert(out, i, gap)305+ }306+ if reverse {307+ slices.Reverse(out)308+ }309+ return out310+}311+203312 func xlookup(args []Node, get lookup) Value {204313 key := eval(args[0], get)205314 if key.Kind == Error {@@ -232,22 +341,7 @@ func xlookup(args []Node, get lookup) Value {
232341 default:233342 return ErrValue234343 }235- // The entries past the data are all blank, so only the first of them236- // in search order can be the answer.237- data := min(look.dataLen(), n)238- order := make([]int, 0, data+1)239- for i := range data {240- order = append(order, i)241- }242- if data < n {243- order = append(order, data)244- }245- if search < 0 {246- slices.Reverse(order)247- if data < n {248- order[0] = n - 1249- }250- }344+ order := lineSeq(look, get, look.cols == 1).order(search < 0)251345 best := -1252346 for _, i := range order {253347 v := look.at(i)@@ -1,60 +1,116 @@
11 package sheet22 3-// Functions over aligned ranges (SUMIF, COUNTIFS, SUMPRODUCT...) walk4-// only the part of their ranges that may hold data, the union of the5-// ranges' data areas; the cells past it are all blank, so they are6-// accounted for at once. SUMIF(A:A, "", B:B) over ten filled rows looks at7-// ten rows, however tall the sheet.8-9-// dataArea is the union of the data areas of same-sized matrices.10-func dataArea(ms ...matrix) (rows, cols int) {3+import (4+ "cmp"5+ "slices"6+)7+8+// Functions over aligned ranges (SUMIF, COUNTIFS, SUMPRODUCT...) visit9+// only the cells their ranges hold; the rest are blank in all of them,10+// so they are accounted for at once. SUMIF(A:A, "", B:B) over ten filled11+// rows looks at ten cells, however tall the sheet, and ten cells spread12+// over a million rows are still ten.13+14+// pos is a cell of same-sized matrices, by row and column from their15+// top-left corner.16+type pos struct{ r, c int }17+18+// storedPos returns the cells of m that hold something, row by row, and19+// false when every cell of its data area must be read instead (as tests20+// ask with denseReads).21+func (m matrix) storedPos(get lookup) ([]pos, bool) {22+ if !m.ref {23+ return []pos{{}}, true24+ }25+ t := get.sheet(m.sheet)26+ switch {27+ case denseReads:28+ return nil, false29+ case t == nil:30+ return nil, true31+ }32+ r := Rect{From: m.origin, To: Addr{Col: m.origin.Col + m.cols - 1, Row: m.origin.Row + m.rows - 1}}33+ var out []pos34+ for a := range t.cells.inRange(r) {35+ out = append(out, pos{a.Row - r.From.Row, a.Col - r.From.Col})36+ }37+ return out, true38+}39+40+// cellsOf returns the cells to visit of same-sized matrices: those any of41+// them holds, in row-major order. Every other cell is blank in all of42+// them.43+func cellsOf(get lookup, ms ...matrix) []pos {44+ var all []pos45+ for _, m := range ms {46+ p, ok := m.storedPos(get)47+ if !ok {48+ return denseArea(ms...)49+ }50+ all = append(all, p...)51+ }52+ if len(ms) > 1 {53+ slices.SortFunc(all, func(a, b pos) int { return cmp.Or(cmp.Compare(a.r, b.r), cmp.Compare(a.c, b.c)) })54+ all = slices.Compact(all)55+ }56+ return all57+}58+59+// denseArea is every cell of the union of the matrices' data areas.60+func denseArea(ms ...matrix) []pos {61+ rows, cols := 0, 01162 for _, m := range ms {1263 rows = max(rows, min(m.dataRows, m.rows))1364 cols = max(cols, min(m.dataCols, m.cols))1465 }15- return rows, cols66+ out := make([]pos, 0, rows*cols)67+ for r := range rows {68+ for c := range cols {69+ out = append(out, pos{r, c})70+ }71+ }72+ return out1673 }1774 18-// masked is which cells of aligned ranges meet criteria: over the data19-// area (rows x cols, row-major), and for the cells past it, which are20-// the same everywhere.75+// masked is which cells of aligned ranges meet criteria: those visited76+// that do, and whether the rest, all blank, do.2177 type masked struct {22- rows, cols int23- pass []bool24- tail int // cells past the data area25- tailPass bool // whether they meet the criteria78+ pass []pos79+ tail int // cells not visited80+ tailPass bool // whether they meet the criteria2681 }2782 28-// applyCriteria tests each criterion on its range, over the data area of29-// the ranges and of also, which is aligned with them.30-func applyCriteria(ms []matrix, cs []criterion, also ...matrix) masked {31- rows, cols := dataArea(append(also, ms...)...)32- k := masked{rows: rows, cols: cols, pass: make([]bool, rows*cols), tailPass: true}33- for i := range k.pass {34- k.pass[i] = true35- }36- if len(ms) > 0 {37- k.tail = ms[0].size() - rows*cols83+// applyCriteria tests each criterion on its range, over the cells the84+// ranges and also (aligned with them) hold.85+func applyCriteria(get lookup, ms []matrix, cs []criterion, also ...matrix) masked {86+ cells := cellsOf(get, append(also, ms...)...)87+ k := masked{tail: ms[0].size() - len(cells), tailPass: true}88+ for _, p := range cells {89+ if passes(ms, cs, p) {90+ k.pass = append(k.pass, p)91+ }3892 }3993 for j, m := range ms {40- for i, ok := range k.pass {41- if ok && !cs[j].test(m.cell(i/cols, i%cols)) {42- k.pass[i] = false43- }44- }4594 k.tailPass = k.tailPass && cs[j].test(m.blank)4695 }4796 return k4897 }4998 99+// passes reports whether the cell at p meets every criterion.100+func passes(ms []matrix, cs []criterion, p pos) bool {101+ for j, m := range ms {102+ if !cs[j].test(m.cell(p.r, p.c)) {103+ return false104+ }105+ }106+ return true107+}108+50109 // sumMasked adds the numbers of m where the criteria pass.51110 func sumMasked(m matrix, k masked) Value {52111 total := 0.053- for i, ok := range k.pass {54- if !ok {55- continue56- }57- switch v := m.cell(i/k.cols, i%k.cols); v.Kind {112+ for _, p := range k.pass {113+ switch v := m.cell(p.r, p.c); v.Kind {58114 case Error:59115 return v60116 case Number:@@ -70,11 +126,8 @@ func sumMasked(m matrix, k masked) Value {
70126 // averageMasked averages the numbers of m where the criteria pass.71127 func averageMasked(m matrix, k masked) Value {72128 sum, n := 0.0, 073- for i, ok := range k.pass {74- if !ok {75- continue76- }77- switch v := m.cell(i/k.cols, i%k.cols); v.Kind {129+ for _, p := range k.pass {130+ switch v := m.cell(p.r, p.c); v.Kind {78131 case Error:79132 return v80133 case Number:@@ -104,7 +157,7 @@ func sumIf(args []Node, get lookup) Value {
104157 return ErrValue105158 }106159 }107- return sumMasked(sum, applyCriteria([]matrix{rng}, []criterion{newCriterion(cv)}, sum))160+ return sumMasked(sum, applyCriteria(get, []matrix{rng}, []criterion{newCriterion(cv)}, sum))108161 }109162 110163 func sumIfs(args []Node, get lookup) Value {@@ -116,7 +169,7 @@ func sumIfs(args []Node, get lookup) Value {
116169 case len(ms) == 0 || ms[0].rows != sum.rows || ms[0].cols != sum.cols:117170 return ErrValue118171 }119- return sumMasked(sum, applyCriteria(ms, cs, sum))172+ return sumMasked(sum, applyCriteria(get, ms, cs, sum))120173 }121174 122175 func averageIf(args []Node, get lookup) Value {@@ -129,7 +182,7 @@ func averageIf(args []Node, get lookup) Value {
129182 if len(args) > 2 {130183 avg = matrixArg(args[2], get).resized(rng.rows, rng.cols, get)131184 }132- return averageMasked(avg, applyCriteria([]matrix{rng}, []criterion{newCriterion(cv)}, avg))185+ return averageMasked(avg, applyCriteria(get, []matrix{rng}, []criterion{newCriterion(cv)}, avg))133186 }134187 135188 func averageIfs(args []Node, get lookup) Value {@@ -141,7 +194,7 @@ func averageIfs(args []Node, get lookup) Value {
141194 case len(ms) == 0 || ms[0].rows != avg.rows || ms[0].cols != avg.cols:142195 return ErrValue143196 }144- return averageMasked(avg, applyCriteria(ms, cs, avg))197+ return averageMasked(avg, applyCriteria(get, ms, cs, avg))145198 }146199 147200 func countIfs(args []Node, get lookup) Value {@@ -149,13 +202,8 @@ func countIfs(args []Node, get lookup) Value {
149202 if err != nil {150203 return *err151204 }152- k := applyCriteria(ms, cs)153- n := 0154- for _, ok := range k.pass {155- if ok {156- n++157- }158- }205+ k := applyCriteria(get, ms, cs)206+ n := len(k.pass)159207 if k.tailPass {160208 n += k.tail161209 }@@ -165,17 +213,15 @@ func countIfs(args []Node, get lookup) Value {
165213 func countBlank(args []Node, get lookup) Value {166214 m := matrixArg(args[0], get)167215 isBlank := func(v Value) bool { return v.Kind == Empty || v.Kind == Text && v.Str == "" }168- rows, cols := dataArea(m)216+ cells := cellsOf(get, m)169217 n := 0170- for r := range rows {171- for c := range cols {172- if isBlank(m.cell(r, c)) {173- n++174- }218+ for _, p := range cells {219+ if isBlank(m.cell(p.r, p.c)) {220+ n++175221 }176222 }177223 if isBlank(m.blank) {178- n += m.size() - rows*cols224+ n += m.size() - len(cells)179225 }180226 return num(float64(n))181227 }@@ -190,12 +236,12 @@ func sumProduct(args []Node, get lookup) Value {
190236 return ErrValue191237 }192238 }193- rows, cols := dataArea(ms...)239+ cells := cellsOf(get, ms...)194240 total := 0.0195- for k := range rows * cols {241+ for _, pc := range cells {196242 p := 1.0197243 for _, m := range ms {198- switch v := m.cell(k/cols, k%cols); v.Kind {244+ switch v := m.cell(pc.r, pc.c); v.Kind {199245 case Error:200246 return v201247 case Number:@@ -206,7 +252,7 @@ func sumProduct(args []Node, get lookup) Value {
206252 }207253 total += p208254 }209- if ms[0].size() > rows*cols {255+ if ms[0].size() > len(cells) {210256 for _, m := range ms {211257 if m.blank.Kind == Error {212258 return m.blank@@ -77,6 +77,40 @@ func FanIn(n, k int) *sheet.Sheet {
7777 return s7878 }7979 80+// FanInColumns is FanIn with whole-column ranges, =SUM(A:A): the same81+// data, read through a range of a million rows.82+func FanInColumns(n, k int) *sheet.Sheet {83+ s := sheet.New()84+ for row := range n {85+ load(s, at(0, row), fmt.Sprint(row%100))86+ }87+ for i := range k {88+ load(s, at(1+i/n, i%n), "=SUM(A:A)")89+ }90+ s.RecalcAll()91+ return s92+}93+94+// Sparse is a tall, sparse sheet: n numbers every stride rows down95+// column A, a label beside each, and k of each of three formulas over96+// whole columns: =SUM(A:A), running totals to every thousandth row, and97+// =VLOOKUP over A:B. Its cost should follow its n cells, not the million98+// rows they span.99+func Sparse(n, stride, k int) *sheet.Sheet {100+ s := sheet.New()101+ for i := range n {102+ load(s, at(0, i*stride), fmt.Sprint(i%100))103+ load(s, at(1, i*stride), fmt.Sprintf("item %d", i))104+ }105+ for i := range k {106+ load(s, at(2, i), "=SUM(A:A)")107+ load(s, at(3, i), fmt.Sprintf("=SUM($A$1:A%d)", (i+1)*1000))108+ load(s, at(4, i), fmt.Sprintf("=VLOOKUP(%d, A:B, 2, FALSE)", i%100))109+ }110+ s.RecalcAll()111+ return s112+}113+80114 // FanOut is one cell, A1, read directly by k formulas.81115 func FanOut(k int) *sheet.Sheet {82116 s := sheet.New()@@ -220,5 +254,7 @@ func Shapes() []Shape {
220254 {"running-8192", func() *sheet.Sheet { return RunningTotals(Rows) }, at(0, 0), "3"},221255 {"volatile-8192", func() *sheet.Sheet { return Volatile(Rows) }, at(200, 0), "1"},222256 {"names-1000", func() *sheet.Sheet { return Names(1000) }, at(0, 50), "9"},257+ {"fanin-1000xSUM(A:A)", func() *sheet.Sheet { return FanInColumns(Rows, 1000) }, at(0, 4000), "7"},258+ {"sparse-1M", func() *sheet.Sheet { return Sparse(10000, 100, 1000) }, at(0, 500000), "5"},223259 }224260 }