Changing a selection over a big sheet read every cell in it (2 M map lookups for a whole 8192 x 256 sheet). Keep Sum, Count and Nums per block of 64 rows of each column, built on first use and marked stale when a cell is stored, deleted or recalculated, so a selection is summed a block at a time plus the rows at its ends.
- commit
- ce91bf34953aeb2b29fd158027eafd35bf0ee8bd
- parent
- bdc52596b78b33bf8de50f034204c3732688d78f
- author
- Dami <dami.etoile@gmail.com>
| M | internal/ | 40 | ------- |
| M | internal/ | 1 | + |
| A | internal/ | 163 | ++++++++++++++++++++++++++++++ |
| A | internal/ | 84 | +++++++++++++++ |
| M | internal/ | 16 | ++- |
@@ -54,43 +54,3 @@ func (s *Sheet) Edge(a Addr, dc, dr int) Addr {
5454 next = n5555 }5656 }57-58-// Stats summarizes the values in r, as shown in the status line for a59-// selection. Count is non-blank cells, Nums is numeric cells.60-type Stats struct {61- Sum float6462- Count, Nums int63-}64-65-// statsCache is the last RangeStats result: the status line asks for the66-// selection's statistics on every frame, and over a whole sheet of two67-// million cells they take tens of milliseconds.68-type statsCache struct {69- r Rect70- version uint6471- st Stats72- ok bool73-}74-75-// RangeStats computes Stats over r. The result is kept until a cell or76-// value changes.77-func (s *Sheet) RangeStats(r Rect) Stats {78- if c := s.stats; c.ok && c.r == r && c.version == s.version {79- return c.st80- }81- var st Stats82- add := func(c *Cell) {83- st.Count++84- if c.Value.Kind == Number {85- st.Nums++86- st.Sum += c.Value.Num87- }88- }89- for _, c := range s.cells.inRange(r) {90- if !c.Blank() {91- add(c)92- }93- }94- s.stats = statsCache{r: r, version: s.version, st: st, ok: true}95- return st96-}@@ -202,6 +202,7 @@ func (w *Workbook) evaluate() {
202202 }203203 }204204 s.calc[a] = done205+ s.cells.changed(a)205206 return c.Value206207 }207208 for _, s := range w.sheets {@@ -0,0 +1,163 @@
1+package sheet2+3+// Stats summarizes the values in r, as shown in the status line for a4+// selection. Count is non-blank cells, Nums is numeric cells.5+type Stats struct {6+ Sum float647+ Count, Nums int8+}9+10+func (st *Stats) add(c *Cell) {11+ if c.Blank() {12+ return13+ }14+ st.Count++15+ if c.Value.Kind == Number {16+ st.Nums++17+ st.Sum += c.Value.Num18+ }19+}20+21+// statsCache is the last RangeStats result: the status line asks for the22+// selection's statistics on every frame.23+type statsCache struct {24+ r Rect25+ version uint6426+ st Stats27+ ok bool28+}29+30+// RangeStats computes Stats over r. The result is kept until a cell or31+// value changes. A selection over a large, well-filled sheet is summed32+// from the cells' statistics index (see statsIndex), so changing it costs33+// a block per column plus the rows at its ends, not a read of every cell.34+func (s *Sheet) RangeStats(r Rect) Stats {35+ if c := s.stats; c.ok && c.r == r && c.version == s.version {36+ return c.st37+ }38+ var st Stats39+ area := (r.To.Col - r.From.Col + 1) * (r.To.Row - r.From.Row + 1)40+ if area <= statsDirect || s.cells.len() <= statsSparse {41+ for _, c := range s.cells.inRange(r) {42+ st.add(c)43+ }44+ } else {45+ st = s.cells.rangeStats(r)46+ }47+ s.stats = statsCache{r: r, version: s.version, st: st, ok: true}48+ return st49+}50+51+const (52+ // statsBlock is how many rows of a column one entry of the index53+ // sums: at most twice that many cells per column are read at a54+ // selection's ends.55+ statsBlock = 6456+ // statsBlocks is the number of blocks down a column.57+ statsBlocks = (MaxRows + statsBlock - 1) / statsBlock58+ // statsDirect: selections of at most this many cells are summed cell59+ // by cell.60+ statsDirect = 409661+ // statsSparse: sheets of at most this many cells are summed by62+ // visiting their cells, which is cheaper than the index.63+ statsSparse = 1638464+)65+66+// blockStats is Stats over one block of a column.67+type blockStats struct {68+ sum float6469+ count, nums int3270+}71+72+// statsIndex keeps Stats for every block of statsBlock rows of every73+// column, so a large selection is summed a block at a time. It's built on74+// first use, from one pass over the cells. After that, storing or75+// deleting a cell, or recalculating its value, marks its block stale, and76+// stale blocks are summed again (a lookup per row) when a selection next77+// covers them. Blocks are recomputed rather than adjusted, so sums don't78+// drift with rounding.79+type statsIndex struct {80+ blocks []blockStats // by col*statsBlocks + row/statsBlock81+ stale []uint64 // a bit per block82+}83+84+func blockOf(a Addr) int { return a.Col*statsBlocks + a.Row/statsBlock }85+86+// touch marks the block holding a stale. A nil index has nothing to mark.87+func (x *statsIndex) touch(a Addr) {88+ if x != nil {89+ i := blockOf(a)90+ x.stale[i>>6] |= 1 << (i & 63)91+ }92+}93+94+// buildStats makes the index from one pass over every cell.95+func (st *cellStore) buildStats() *statsIndex {96+ n := MaxCols * statsBlocks97+ x := &statsIndex{blocks: make([]blockStats, n), stale: make([]uint64, (n+63)/64)}98+ for a, c := range st.m {99+ if c.Blank() {100+ continue101+ }102+ b := &x.blocks[blockOf(a)]103+ b.count++104+ if c.Value.Kind == Number {105+ b.nums++106+ b.sum += c.Value.Num107+ }108+ }109+ st.stats = x110+ return x111+}112+113+// block returns the statistics of block b of column col, summing it114+// again if it's stale.115+func (st *cellStore) block(col, b int) blockStats {116+ x := st.stats117+ i := col*statsBlocks + b118+ if x.stale[i>>6]&(1<<(i&63)) != 0 {119+ var s Stats120+ st.scan(col, b*statsBlock, min((b+1)*statsBlock, MaxRows)-1, &s)121+ x.blocks[i] = blockStats{sum: s.Sum, count: int32(s.Count), nums: int32(s.Nums)}122+ x.stale[i>>6] &^= 1 << (i & 63)123+ }124+ return x.blocks[i]125+}126+127+// scan adds the cells of column col from row from to row to into s.128+func (st *cellStore) scan(col, from, to int, s *Stats) {129+ for row := from; row <= to; row++ {130+ if c := st.m[Addr{Col: col, Row: row}]; c != nil {131+ s.add(c)132+ }133+ }134+}135+136+// rangeStats sums r from the index: in each column, the whole blocks r137+// covers, and the rows at its ends outside them cell by cell.138+func (st *cellStore) rangeStats(r Rect) Stats {139+ if st.stats == nil {140+ st.buildStats()141+ }142+ var s Stats143+ first := (r.From.Row + statsBlock - 1) / statsBlock // first whole block144+ last := first // one past the last whole block145+ for last < statsBlocks && min((last+1)*statsBlock, MaxRows)-1 <= r.To.Row {146+ last++147+ }148+ for col := r.From.Col; col <= r.To.Col; col++ {149+ if first >= last {150+ st.scan(col, r.From.Row, r.To.Row, &s)151+ continue152+ }153+ st.scan(col, r.From.Row, first*statsBlock-1, &s)154+ for b := first; b < last; b++ {155+ bs := st.block(col, b)156+ s.Sum += bs.sum157+ s.Count += int(bs.count)158+ s.Nums += int(bs.nums)159+ }160+ st.scan(col, last*statsBlock, r.To.Row, &s)161+ }162+ return s163+}@@ -0,0 +1,84 @@
1+package sheet2+3+import (4+ "fmt"5+ "math"6+ "testing"7+)8+9+// bruteStats sums r cell by cell, as RangeStats did before the index.10+func bruteStats(s *Sheet, r Rect) Stats {11+ var st Stats12+ for row := r.From.Row; row <= r.To.Row; row++ {13+ for col := r.From.Col; col <= r.To.Col; col++ {14+ if c := s.Cell(Addr{Col: col, Row: row}); c != nil {15+ st.add(c)16+ }17+ }18+ }19+ return st20+}21+22+func sameStats(a, b Stats) bool {23+ return a.Count == b.Count && a.Nums == b.Nums && math.Abs(a.Sum-b.Sum) <= 1e-9*math.Max(1, math.Abs(b.Sum))24+}25+26+// TestRangeStatsIndex checks the block index against summing cell by27+// cell, over selections that start and end inside blocks and on their28+// edges, as cells are set, recalculated, deleted and undone.29+func TestRangeStatsIndex(t *testing.T) {30+ s := New()31+ for row := range 2000 {32+ for col := range 10 {33+ s.Load(Addr{Col: col, Row: row}, fmt.Sprint(row*10+col), Format{}, Style{})34+ }35+ }36+ s.Load(at("L1"), "=SUM(A1:A2000)", Format{}, Style{})37+ s.Load(at("K5"), "text", Format{}, Style{})38+ s.RecalcAll()39+ rects := []Rect{40+ NewRect(at("A1"), at("L8192")),41+ NewRect(at("A1"), at("IV8192")),42+ NewRect(at("B2"), at("K1999")),43+ NewRect(at("A64"), at("C129")), // rows 63..128: one whole block44+ NewRect(at("A65"), at("C128")), // rows 64..127: exactly one block45+ NewRect(at("C10"), at("F100")), // no whole block46+ NewRect(at("A1"), at("A8192")), // one column47+ NewRect(at("A100"), at("J120")), // small: cell by cell48+ }49+ check := func(when string) {50+ t.Helper()51+ for _, r := range rects {52+ if got, want := s.RangeStats(r), bruteStats(s, r); !sameStats(got, want) {53+ t.Errorf("%s: RangeStats(%s) = %+v, want %+v", when, r, got, want)54+ }55+ }56+ }57+ check("built")58+ if s.cells.stats == nil {59+ t.Fatal("the statistics index wasn't built")60+ }61+ s.Set(at("A1"), "1000000") // recalculates L1 as well62+ check("after an edit")63+ s.Set(at("C70"), "")64+ check("after clearing a cell")65+ s.Set(at("Z8000"), "7")66+ check("after a cell outside the data")67+ s.Undo()68+ s.Undo()69+ check("after undo")70+ s.Redo()71+ check("after redo")72+}73+74+func TestRangeStatsSparse(t *testing.T) {75+ s := New()76+ s.Set(at("A1"), "2")77+ s.Set(at("IV8192"), "3")78+ if got := s.RangeStats(NewRect(at("A1"), at("IV8192"))); got != (Stats{Sum: 5, Count: 2, Nums: 2}) {79+ t.Errorf("whole sparse sheet = %+v", got)80+ }81+ if s.cells.stats != nil {82+ t.Error("a sparse sheet built the statistics index")83+ }84+}@@ -15,7 +15,8 @@ import (
1515 // iterating, with a map's rules: a deleted cell not yet reached isn't1616 // yielded, a new one may or may not be.1717 type cellStore struct {18- m map[Addr]*Cell18+ m map[Addr]*Cell19+ stats *statsIndex // statistics by block, once a selection needs them; see stats.go1920 }2021 2122 func newCellStore() cellStore { return cellStore{m: make(map[Addr]*Cell)} }@@ -24,10 +25,19 @@ func newCellStore() cellStore { return cellStore{m: make(map[Addr]*Cell)} }
2425 func (st *cellStore) get(a Addr) *Cell { return st.m[a] }2526 2627 // set stores c, which isn't nil, at a.27-func (st *cellStore) set(a Addr, c *Cell) { st.m[a] = c }28+func (st *cellStore) set(a Addr, c *Cell) {29+ st.m[a] = c30+ st.stats.touch(a)31+}2832 2933 // delete removes the cell at a, if any.30-func (st *cellStore) delete(a Addr) { delete(st.m, a) }34+func (st *cellStore) delete(a Addr) {35+ delete(st.m, a)36+ st.stats.touch(a)37+}38+39+// changed notes that the value of the cell at a was recalculated.40+func (st *cellStore) changed(a Addr) { st.stats.touch(a) }3141 3242 // len is the number of cells stored, formatting-only ones included.3343 func (st *cellStore) len() int { return len(st.m) }