A selection covering a whole block of 1024 rows adds the block's total rather than its 16 entries: extending a selection over 8192 x 256 takes 0.41 ms a key, from main's 0.47 ms. limits.md gets main and the merge back to back.
- commit
- 100955bb936c9c26436c4dbec3c1d8487a8c1853
- parent
- bfb1a1824065b3ff1696937ca965d4e58f678276
- author
- Dami <dami.etoile@gmail.com>
| M | docs/ | 32 | ++++++++++++++++++++++++++++-- |
| M | internal/ | 30 | ++++++++++++++++++++++++++-- |
@@ -26,7 +26,7 @@ it lags, and past a second it stalls.
2626 | Incremental recalc | A change that makes formulas read under about 500,000 cells in total (fan-out, chains and volatiles of 8192 cells: 1 to 2 ms); 1000 SUMs over a full column: 0.7 ms; 8192 running totals: 2.7 ms | | | About 20 ns per cell read: two map lookups |2727 | Full recalc | Any sheet: under 40 ms for 213 k numbers; 1000 full-column SUMs 1.1 ms; 8192 running totals 2.6 ms | | | Same as above |2828 | Rendering | Any sheet at up to 200 x 60: 1 ms a frame; 400 x 120: 4.5 ms | 20 charts at 400 x 120: 7 ms | | View building styled strings, then Bubble Tea parsing and diffing them |29-| Selection statistics | Any selection: extending one over all 2 M cells costs 0.5 ms a key | | | Per column with data, 64 rows at a time from an index on the blocks of filled cells, plus the rows at the selection's ends |29+| Selection statistics | Any selection: extending one over all 2 M cells costs 0.4 ms a key | | | Per column with data, 1024 or 64 rows at a time from an index on the blocks of filled cells, plus the rows at the selection's ends |3030 | Imports | CSV, SQLite, Parquet: 1 to 3 M cells/s (two million cells in about 1 s); XLSX numbers or text: 0.7 to 1.1 M cells/s | XLSX with formulas: 0.4 to 0.6 M cells/s | Data past `max-cells` or the grid (dropped, with a note); XLSX files past the reader's limits (refused) | Building cells one at a time; XML decoding; XLSX formula translation |3131 | Undo | One step of any size: undo costs what the edit cost | History capped at 100 steps and 256 MB of before-images: 100 whole-column steps hold 250 MB | | Whole-cell before-images, about 300 B per cell per step |3232 | JEV | 4000 JEV cells: 4 us of CPU per answer, 15 ms to answer them all | | | An answer recalculates the cells that asked it; answers within a frame recalculate together |@@ -452,7 +452,7 @@ Before and after, back to back (`-benchtime 1s`, medians of one run):
452452 | BigUndo: clear 8192 x 26, undo | 159 ms | 164 ms |453453 | Sort 8191 rows | 31 ms | 32 ms |454454 | Pivot over 8191 rows, 8 categories | 0.89 ms | 0.77 ms |455-| Extend a selection of 8192 x 256 by a row | 31 ms | 35 ms (measured alone) |455+| Extend a selection of 8192 x 256 by a row | 31 ms | 35 ms (measured alone; 0.41 ms once merged with main's selection statistics, below) |456456 457457 The index costs a few percent on edits and loads (keeping two bitmaps458458 current per cell), and running aggregates cost memory for their@@ -460,6 +460,34 @@ checkpoints (a 1000-SUM fan-in allocates 2 MB per recalculation instead
460460 of 0.2 MB). A range is shared only from its second read in a461461 recalculation, so ranges read once cost nothing extra.462462 463+### Merged with main464+465+Step A was merged with main at 7431646 (selection statistics by block,466+JEV recalc by question, 120 fps, depth limits, the streaming XLSX467+reader). Main's dense statistics index, one entry per 64 rows of every468+column, would have been 268 M entries at the new grid; it now lives on469+the blocks of the index of filled cells, an entry per 64 rows and a470+total per 1024, made where there is data. Main and the merge back to471+back (`-benchtime 1s`):472+473+| Benchmark | Main | Merged |474+|---|---|---|475+| Extend a selection of 8192 x 256 by a row, 80 x 24 | 0.47 ms | 0.41 ms |476+| Select all of 8192 x 256, 80 x 24 | 0.28 ms | 0.28 ms |477+| JEV, 4000 cells answered | 14.5 ms | 14.3 ms |478+| Import XLSX 8192 x 26 numbers / table / 8192 formulas | 187 / 313 / 13.6 ms | 189 / 312 / 13.8 ms |479+| Import the POI formula tests (XLSX) | 9.7 ms | 9.5 ms |480+| Edit fan-in / running totals | 186 / 777 ms | 0.73 / 2.7 ms |481+| Sort 8191 rows | 29.7 ms | 32.3 ms |482+| BigUndo | 150 ms | 156 ms |483+| Open 8192 x 26 | 117 ms | 121 ms |484+485+CSV imports take longer only because they keep more: the airport codes486+(855 k cells, 0.78 s) and OWID energy (1.04 M cells, 0.84 s) now come in487+whole. XLSX imports set column widths and styles for the data's columns488+(at least 256), without an undo step each: setting 16,384 of them one489+step at a time took the POI file from 9.7 to 44 ms before the fix.490+463491 ## Hotspots found and fixed464492 465493 Each with its benchmark before and after, in the commit that fixed it.@@ -81,14 +81,17 @@ type wordStats struct {
8181 // blockStats is the statistics of a block of the filled index, made on8282 // first use.8383 type blockStats struct {84- words [blockWords]wordStats85- ok uint16 // a bit per word: its entry is current84+ words [blockWords]wordStats85+ ok uint16 // a bit per word: its entry is current86+ total wordStats87+ totalOK bool // total is the sum of the words' entries, all current8688 }8789 8890 // touch marks the entry holding a stale.8991 func (st *cellStore) touch(a Addr) {9092 if b := st.filled.col(a.Col).block(a.Row >> blockShift); b != nil && b.stats != nil {9193 b.stats.ok &^= 1 << ((a.Row & (blockRows - 1)) >> 6)94+ b.stats.totalOK = false9295 }9396 }9497 @@ -111,6 +114,22 @@ func (st *cellStore) word(col, id int, b *rowBlock, w int) wordStats {
111114 return x.words[w]112115 }113116 117+// blockTotal returns the statistics of the whole block id of column col.118+func (st *cellStore) blockTotal(col, id int, b *rowBlock) wordStats {119+ if b.stats != nil && b.stats.totalOK {120+ return b.stats.total121+ }122+ var t wordStats123+ for w := range blockWords {124+ ws := st.word(col, id, b, w)125+ t.sum += ws.sum126+ t.count += ws.count127+ t.nums += ws.nums128+ }129+ b.stats.total, b.stats.totalOK = t, true130+ return t131+}132+114133 // scanWord adds the filled cells of word w of block id, among the rows115134 // mask keeps, into s.116135 func (st *cellStore) scanWord(col, id int, b *rowBlock, w int, mask uint64, s *Stats) {@@ -137,6 +156,13 @@ func (st *cellStore) rangeStats(r Rect) Stats {
137156 b := ci.blocks[id]138157 base := id << blockShift139158 lo, hi := max(r.From.Row-base, 0), min(r.To.Row-base, blockRows-1)159+ if lo == 0 && hi == blockRows-1 {160+ t := st.blockTotal(col, id, b)161+ s.Sum += t.sum162+ s.Count += int(t.count)163+ s.Nums += int(t.nums)164+ continue165+ }140166 for w := lo >> 6; w <= hi>>6; w++ {141167 if mask := wordMask(w, lo, hi); mask != ^uint64(0) {142168 st.scanWord(col, id, b, w, mask, &s)