- commit
- bd785aa20552c7f6e61d101b4ed703f123dc6fd2
- parent
- 9245d8004321527969d413a43b16a844600ef538
- parent
- 6ef84602a33593c2ab1a1d5e28e83b19bfb33912
- author
- Dami <dami.etoile@gmail.com>
@@ -32,14 +32,13 @@ Sizes: S (a day or two), M (about a week), L (weeks).
3232 ### 3. Scale3333 3434 The grid is Excel's, 1,048,576 x 16,384, and memory is bounded by the35-`max-cells` budget rather than the grid. The budget rises in measured36-steps; see [Bounds of support](docs/contributing/limits.md#what-would-raise-the-bounds).35+`max-cells` budget (ten million cells) rather than the grid. The budget36+rises in measured steps; see [Bounds of support](docs/contributing/limits.md#what-would-raise-the-bounds).3737 3838 | Item | Result | Size |3939 |---|---|---|40-| Compact column storage behind `cellStore`: typed value blocks, formulas and formats in side tables (about 20 to 40 B per cell instead of 300) | A budget of 10 M+ cells in a few hundred MB, cell reads 2 to 3 ns | L |4140 | A streaming or binary `.012` format next to the readable JSON one | Open and save scale with the data | M |42-| Smaller undo steps: formatting changes as diffs | More history in the same memory | S |41+| Smaller undo steps: plain cells' before-images as slots, formatting changes as diffs | More history in the same memory; clearing a full sheet costs what the sheet does | S to M |4342 | Linked, paged read-only ranges over Parquet and SQLite that feed pivots and formulas by streaming | Sources too big for any grid | L |4443 4544 ### 4. Macros, keys and the terminal@@ -78,6 +77,7 @@ by a gateway dialing the iroh ticket.
7877 - Dynamic arrays that spill (FILTER, SORT, UNIQUE, SEQUENCE and more, ARRAYFORMULA), LET and LAMBDA, SPLIT and the REGEX functions: [Arrays and spills](docs/formulas/arrays.md)7978 - Opt-in decimal arithmetic for money: [Decimal arithmetic](docs/formulas/decimal.md)8079 - An Excel-sized grid with a `max-cells` budget; operations cost the data, not the grid: [Bounds of support](docs/contributing/limits.md#sheet-size)80+- Compact column storage: 20 B a number, a ten-million-cell `max-cells` budget, 3 ns cell reads: [Bounds of support](docs/contributing/limits.md#sheet-size)8181 8282 **Finding and using features**8383 @@ -92,12 +92,19 @@ was typed, its parsed expression, its computed value, and its format and
9292 style.9393 9494 - **Cells.** Each sheet keeps its cells in a `cellStore` (`store.go`):95- get, set, delete, count, iterate all or a range. Nothing else touches96- the map beneath, so the representation can change without the rest of97- the engine noticing. Beside it, occupancy indexes (`occupancy.go`,95+ get, set, delete, count, iterate all or a range, and narrower reads96+ (a value, presence, formatting, the formula cells). Nothing else97+ touches the representation beneath. Occupancy indexes (`occupancy.go`,9898 bitmaps of 1024 rows per column) record which cells are stored and9999 which have contents, so a range of the million-row grid is read at the100- cost of what it holds. Formats of whole columns and rows, and of the100+ cost of what it holds; the stored index's blocks hold the cells101+ themselves, a 16-byte slot each in row order (`slot.go`). A plain cell102+ (a number, boolean or text as typed, with a format and style) lives in103+ its slot, with its text in a table of strings and its formatting in a104+ table of looks; formulas, notes, and what pivots and spills write are105+ whole `Cell`s in a side table, which recalculation updates in place.106+ `get` hands out a plain cell as a `Cell` made for the caller, a copy107+ whose changes reach nothing; `set` is the one way to change a cell. Formats of whole columns and rows, and of the101108 whole sheet, live on the lines (`lines.go`); a cell falls back on them.102109 Copy, paste and move carry the formatting cells show (`clipfmt.go`):103110 whole lines as line formats, blocks as the cells' own. A line's format@@ -273,9 +280,9 @@ setting (`SetDecimal`, which marks formulas with `functions.Decimalize`).
273280 The package boundaries leave the bounds in [Bounds of support](limits.md) room to274281 move without touching callers:275282 276-- **Storage.** Compact cell storage (column blocks of values, formulas and277- formats in side tables) replaces `cellStore`'s map; its methods are the278- whole contract.283+- **Storage.** `cellStore`'s methods are the whole contract, so what is284+ kept whole or in a slot can change (spilled and pivot cells in slots,285+ plain before-images in the history) without touching callers.279286 - **Range reads.** Shared range results, prefix sums for running totals280287 and column-block scans go behind the engine's `Scan` and `RangeAgg`,281288 the places functions read ranges.@@ -117,13 +117,21 @@ modified.
117117 118118 Code reads and writes cells through a narrow API (`cellStore` in119119 `internal/sheet/store.go`: get, set, delete, count, iterate everything or120-a range, a column being a range), not the map underneath. That lets the121-store change shape (compact column blocks, side tables for formulas and122-formats; see [Bounds of support](limits.md)) without touching the rest of the123-engine. Beside the map, occupancy indexes (`occupancy.go`) say which rows124-of each column hold a cell, and which hold contents, so a range yields its125-cells in row order at the cost of what it holds, and the used range, data126-edges and the next filled cell are found without scanning.120+a range, a column being a range), never the representation underneath:121+column blocks of 16-byte slots holding plain cells' values, with text,122+formats and whole formula cells in side tables (see123+[Bounds of support](limits.md#sheet-size)). `get` returns a `*Cell`:124+a formula's own, which recalculation updates, or for a plain cell one125+made for the caller, which it may keep but whose changes reach nothing;126+every change goes through `set`. Making that `Cell` costs an allocation127+or three, so code that visits many cells asks for only what it needs:128+`value` and `peek` for values, `has` and `filledAt` for presence,129+`look` for formatting, `richAt` and `richCells` for formulas and notes,130+and the key iterators (`keys`, `keysIn`, `anyKeysIn`) for addresses. The131+stored occupancy index (`occupancy.go`) holds the blocks; a second says132+which rows hold contents, so a range yields its cells in row order at133+the cost of what it holds, and the used range, data edges and the next134+filled cell are found without scanning.127135 128136 ### 7. Read ranges as ranges129137 @@ -25,51 +25,63 @@ it lags, and past a second it stalls.
2525 2626 | Dimension | Comfortable | Degraded | Unsupported | Dominant cost |2727 |---|---|---|---|---|28-| Sheet size | A grid of 1,048,576 x 16,384 (A..XFD), with up to `max-cells` cells (two million by default): navigation, drawing and every command cost what the cells cost, not the grid | Loading and saving two million cells: 1 to 2 s; 600 MB of heap | More than `max-cells` cells: imports keep whole rows up to it and say what they dropped; larger pastes and fills are refused | Heap per cell (about 300 B), JSON file format |29-| Incremental recalc | A change that makes formulas read under about 500,000 cells in total (fan-out, chains and volatiles of 8192 cells: 1 to 2 ms); 1000 SUMs over a full column: 0.75 ms; 8192 running totals: 2.8 ms | 60 criteria functions (SUMIF, COUNTIFS, AVERAGEIF) over whole columns of 8192 rows: 37 ms an edit | | About 20 ns per cell read: two map lookups |30-| Full recalc | Any sheet: under 40 ms for 213 k numbers; 1000 full-column SUMs 1 ms; 8192 running totals 2.5 ms | 60 whole-column criteria functions over 8192 rows: 59 ms | | Same as above |28+| Sheet size | A grid of 1,048,576 x 16,384 (A..XFD), with up to `max-cells` cells (ten million by default): navigation, drawing and every command cost what the cells cost, not the grid; ten million numbers take 205 MB and build in 2.2 s | Opening a `.012` file of ten million cells: 8 s and 2.7 GB at its peak; saving it: 3.7 s | More than `max-cells` cells: imports keep whole rows up to it and say what they dropped; larger pastes and fills are refused | JSON file format, decoded whole; heap per cell (about 20 B for numbers, 20 to 60 B for real data, 750 B for formulas) |29+| Incremental recalc | A change that makes formulas read under about 500,000 cells in total (fan-out, chains and volatiles of 8192 cells: 1.3 to 1.9 ms); 1000 SUMs over a full column: 0.53 ms; 8192 running totals: 2.7 ms | 60 criteria functions (SUMIF, COUNTIFS, AVERAGEIF) over whole columns of 8192 rows: 29 ms an edit | | About 3 ns per cell read: an index into the column's block |30+| Full recalc | Any sheet: numbers and text hold their values and cost nothing; 1000 full-column SUMs 0.5 ms; 8192 running totals 1.8 ms | 60 whole-column criteria functions over 8192 rows: 44 ms | | Same as above |3131 | Rendering | Any sheet at up to 200 x 60: 1 ms a frame; 400 x 120: 4.5 ms; a color scale on every cell shown adds 0.4 ms at 200 x 60 | 20 charts at 400 x 120: 7 ms | | View building styled strings, then Bubble Tea parsing and diffing them |32-| Selection statistics | Any selection: extending one over all 2 M cells costs 0.4 ms a key | | | Per column with data, 1024 or 64 rows at a time from an index on the blocks of filled cells, plus the rows at the selection's ends |33-| Imports | CSV, SQLite, Parquet: 1 to 3 M cells/s (two million cells in about 1 s); XLSX numbers or text: 0.7 to 1.1 M cells/s | XLSX with formulas: 0.4 to 0.6 M cells/s | Data past `max-cells` or the grid (dropped, with a note); XLSX files past the reader's limits (refused) | Building cells one at a time; XML decoding; XLSX formula translation |34-| Undo | One step of any size: undo costs what the edit cost | History capped at 100 steps and 256 MB of before-images: 100 whole-column steps hold 250 MB | | Whole-cell before-images, about 300 B per cell per step |32+| Selection statistics | Any selection: extending one over all 2 M cells costs 0.3 ms a key | | | Per column with data, 1024 or 64 rows at a time from an index on the blocks of filled cells, plus the rows at the selection's ends |33+| Imports | CSV, SQLite, Parquet: 2 to 4 M cells/s (a million cells in 0.25 to 0.4 s); XLSX numbers or text: 0.8 to 1.4 M cells/s | XLSX with formulas: 0.4 to 0.6 M cells/s | Data past `max-cells` or the grid (dropped, with a note); XLSX files past the reader's limits (refused) | Building cells one at a time; XML decoding; XLSX formula translation |34+| Undo | One step of any size: undo costs what the edit cost | History capped at 100 steps and 256 MB of before-images: 100 whole-column steps hold 280 MB | | Whole-cell before-images, about 300 B per cell per step |3535 | JEV | 4000 JEV cells: 4 us of CPU per answer, 15 ms to answer them all | | | An answer recalculates the cells that asked it; answers within a frame recalculate together |3636 | Formula depth | 1000 nested parentheses or IFs: under 0.5 ms; chains of formulas through every cell of a sheet | | More than 1024 levels of nesting in one formula: a parse error | Recursive parser; evaluation puts off cells past 65,536 levels |3737 | Macros | Replaying 1000 recorded actions: 2.2 ms, one undo step; a script's call to the sheet: about 1.4 us | | Scripts past 10 M Starlark steps: stopped, with the line | One message per call to the sheet, served in batches on the UI goroutine |3838 | Find, filter, sort, fill | Filter or sort 8191 rows: 1 to 32 ms; find over 213 k cells: 38 ms; fill 8192 rows: 4 ms | Replace all over 213 k cells: 280 ms | | Per-cell string conversion and regexp |3939 | SSH sessions (`012 serve`) | 50 sessions typing at once: frames within one frame interval (p95 8.7 ms), 1.3 MiB per session plus its sheets (upper bound) | | More than `--max-sessions` (8 by default): turned away | Bubble Tea's 120 fps pacing; per session, the terminal's cell buffers |40-| Pivot tables | A pivot over 8191 rows recomputes after an edit to its source in 1 to 4 ms | | Results past the grid (the pivot shows #REF!) | Reading each source cell of its fields: a map lookup each |41-| Arrays and spills | FILTER, SORT or UNIQUE over 8192 rows: about 0.1 ms each per edit; 1000 of them spilling 516 k cells, an edit recomputing 500: 55 ms | Full recalculation of those 1000: 204 ms | An array past `maxArray` values (2,097,152) stored: `#VALUE!`; a spill past the sheet's edge or `max-cells` cells: `#REF!` | Computing each array's values; writing only the spilled cells that changed |40+| Pivot tables | A pivot over 8191 rows recomputes after an edit to its source in 0.8 to 2.5 ms | | Results past the grid (the pivot shows #REF!) | Grouping the source rows |41+| Arrays and spills | FILTER, SORT or UNIQUE over 8192 rows: about 0.1 ms each per edit; 1000 of them spilling 516 k cells, an edit recomputing 500: 52 ms | Full recalculation of those 1000: 217 ms | An array past `maxArray` values (2,097,152) stored: `#VALUE!`; a spill past the sheet's edge or `max-cells` cells: `#REF!` | Computing each array's values; writing only the spilled cells that changed |4242 4343 ## Sheet size4444 4545 Hard limits: `sheet.MaxRows = 1,048,576`, `sheet.MaxCols = 16,384`4646 (A..XFD), as in Excel. Memory is bounded by `max-cells` (the config file,47-two million cells by default): imports keep whole rows up to it, pastes47+ten million cells by default): imports keep whole rows up to it, pastes4848 and fills write at most that many cells at once. WK1 files keep their own49-8192 x 256. Past about two million cells the costs below grow with the50-cells; the grid itself costs nothing (see [The grid](#the-grid)).49+8192 x 256. The costs below grow with the cells; the grid itself costs50+nothing (see [The grid](#the-grid)). At ten million cells the file51+format is what degrades: a `.012` file is JSON decoded whole.5152 52-| Measure | 8192 x 26 (213 k cells) | 8192 x 256 (2.1 M cells) |53-|---|---|---|54-| Build through `Load`, then full recalc | 90 ms | 1.4 s |55-| Live heap | 64 MB | 602 MB (about 300 B per number cell) |56-| Save `.012` (JSON, one line per cell) | 74 ms, 4.6 MB | 1.04 s, 47 MB |57-| Open `.012` | 120 to 130 ms (116 MB allocated) | 1.9 to 2 s (1.3 GB allocated) |58-| Export CSV / TSV / XLSX / SQLite | 41 / 55 / 62 / 40 ms | 0.71 / 0.66 / 0.81 / 0.72 s |59-| Frame at 200 x 60, whole | 1.06 ms | 1.05 ms |53+| Measure | 8192 x 26 (213 k cells) | 8192 x 256 (2.1 M cells) | 1,000,000 x 10 (10 M cells) |54+|---|---|---|---|55+| Build through `Load`, then full recalc | 48 ms | 0.47 s | 2.2 s (396 MB peak resident) |56+| Live heap | 4.4 MB | 43 MB (20.5 B per number cell) | 205 MB |57+| Save `.012` (JSON, one line per cell) | 77 ms, 4.6 MB | 0.83 s, 47 MB | 3.7 s, 236 MB |58+| Open `.012` | 108 ms (118 MB allocated) | 1.5 s (1.3 GB allocated) | 8.1 to 8.3 s (5.7 GB allocated, 2.7 GB peak resident) |59+| Export CSV / TSV / XLSX / SQLite | 33 / 31 / 45 / 38 ms | 0.51 / 0.52 / 0.65 / 0.68 s | |60+| Edit a number, undo and redo it | 0.7 us, 1.2 us | | 0.7 us, 1.2 us |61+| Frame at 200 x 60, whole | 0.88 ms | 0.87 ms | |6062 6163 XLSX exports are written by 012 with archive/zip: 1.9 MB allocated for6264 8192 x 26 and 17 MB for 8192 x 256, whose time is mostly compression.6365 6466 Heap per non-blank cell after loading (`BenchmarkMemory`,65-`BenchmarkImport`): numbers 301 B, formulas 707 B (the parsed tree and66-reference lists), 200-character text 790 B, imported CSV 275 to 300 B.67-A `Cell` struct is 216 B; the rest is the map entry, the input string68-and a boxed number for the parsed literal.67+`BenchmarkImport`): numbers 20.5 B, opened from a file too; formulas 75068+B (the parsed tree and reference lists); imported CSV, SQLite and69+Parquet 21 to 63 B; text about 60 B plus its length, once per distinct string70+(a column of 90 texts repeated takes 27 B a cell). The store71+(`internal/sheet/store.go`, `slot.go`) keeps each column in blocks of72+1024 rows, a 16-byte slot per cell with contents or formatting: a plain73+cell (a number, boolean or text as typed, with a format and style) holds74+its value there, its text in a table of strings kept once each, its75+format and style in a table of looks, and its input only when that76+isn't the value's own text ("1.50" is kept as 1.5 printed with two77+decimals). Formulas, notes, and the results pivots and spills write are78+whole `Cell`s (216 B) in a side table. Reading a cell's value is an79+index into its column's block: 2.8 ns (`BenchmarkRead`), 8.6 ns a cell80+read by a SUM in a full recalculation.6981 7082 ## The grid7183 72-Nothing costs what the grid costs. Beside the cell map, occupancy84+Nothing costs what the grid costs. The cells' occupancy7385 indexes (bitmaps of 1024 rows per column) say which cells are stored and7486 which have contents; ranges are read through them, so a whole column of7587 ten cells is ten visits. Formats of whole columns, rows and the sheet@@ -101,7 +113,7 @@ What each part of 012 visits when given a range or selection:
101113 | XLSX column widths and styles | the data's width (at least 256), a style on A and XFD as the sheet's |102114 103115 A sheet of 10,000 numbers spread down a million rows under 3000104-whole-column formulas (sparse-1M) edits in 2.5 ms, draws a 200 x 60116+whole-column formulas (sparse-1M) edits in 1.6 ms, draws a 200 x 60105117 frame in 0.68 ms, and answers arrow keys at XFD1048576 in 0.65 ms.106118 Typing an entry under 1000 SUMs over a column, through to its frame at107119 200 x 60, takes 0.91 ms.@@ -119,19 +131,22 @@ read once cost nothing extra.
119131 120132 | Topology | Edit | Full recalc | Undo + redo |121133 |---|---|---|---|122-| Independent cell in 8192 x 26 numbers | 0.66 us | 32 to 38 ms | 0.8 us |123-| Chain: A(n) = A(n-1)+1 down 8192 rows | 1.85 ms | 1.1 to 1.4 ms | 3.4 ms |124-| Fan-out: one cell read by 8192 formulas | 1.4 ms | 0.9 ms | 2.7 ms |125-| Volatile: 8192 TODAY() and RAND() | 1.3 ms | 0.8 ms | 2.5 ms |126-| 1000 named ranges, one formula each | 48 us | 2.8 ms | 99 us |127-| Fan-in: 1000 x SUM(A1:A8192), or SUM(A:A) | 0.75 ms | 1 ms | 1.4 ms |128-| Running totals: 8192 x SUM($A$1:An) | 2.85 ms | 2.5 ms | 5.4 ms |129-| Sparse: 10,000 numbers spread down a million rows, 1000 each of SUM(A:A), running totals and VLOOKUP over A:B | 2.45 ms | 3.6 ms | 5.1 ms |130-| Criteria: 60 SUMIF, COUNTIFS and AVERAGEIF over whole columns of 8192 rows | 37 ms | 59 ms | |131-| Lookup: 300 VLOOKUP, MATCH and XLOOKUP into 8192 keys | 25 ms | 26.5 ms | |132-133-Every volatile formula is recomputed on every change, as in Sheets, so134-8192 volatile cells add about 1.2 ms to every edit anywhere. Criteria134+| Independent cell in 8192 x 26 numbers | 0.7 us | nothing to compute | 1.2 us |135+| Chain: A(n) = A(n-1)+1 down 8192 rows | 1.9 ms | 1 ms | 3.8 ms |136+| Fan-out: one cell read by 8192 formulas | 1.3 ms | 0.8 ms | 2.6 ms |137+| Volatile: 8192 TODAY() and RAND() | 1.3 ms | 0.8 ms | 2.6 ms |138+| 1000 named ranges, one formula each | 55 us | 1.2 ms | 110 us |139+| Fan-in: 1000 x SUM(A1:A8192), or SUM(A:A) | 0.53 ms | 0.48 ms | 1.1 to 1.2 ms |140+| Running totals: 8192 x SUM($A$1:An) | 2.7 ms | 1.8 ms | 5.8 ms |141+| Sparse: 10,000 numbers spread down a million rows, 1000 each of SUM(A:A), running totals and VLOOKUP over A:B | 1.6 ms | 1.5 ms | 3.2 ms |142+| Criteria: 60 SUMIF, COUNTIFS and AVERAGEIF over whole columns of 8192 rows | 29 ms | 44 ms | 59 ms |143+| Lookup: 300 VLOOKUP, MATCH and XLOOKUP into 8192 keys | 12 ms | 12 ms | 27 ms |144+| 10 M numbers (1,000,000 x 10) | 0.7 us | nothing to compute | 1.2 us |145+146+A full recalculation computes only formulas and what pivots and spills147+write: numbers and text hold their values from the moment they are148+stored. Every volatile formula is recomputed on every change, as in149+Sheets, so 8192 volatile cells add about 1.2 ms to every edit anywhere. Criteria135150 and lookups cost what their ranges hold, not what the grid holds: the136151 criteria collect positions from the occupancy index, and lookups walk137152 the stored entries with a cursor.@@ -169,7 +184,7 @@ terminal (`BenchmarkFrame`).
169184 | Sheet | 80 x 24 | 200 x 60 | 400 x 120 |170185 |---|---|---|---|171186 | Empty | 0.19 ms | 0.89 ms | 3.2 ms |172-| 8192 x 256 numbers | 0.20 ms | 1.05 ms | 4.1 ms |187+| 8192 x 256 numbers | 0.18 ms | 0.87 ms | 3.4 ms |173188 | 8192 rows of 500-character unicode text | 0.34 ms | 1.37 ms | 4.4 ms |174189 | 1000 named ranges | 0.30 ms | 1.06 ms | 3.5 ms |175190 | 20 charts (text) | 0.82 ms | 2.9 ms | 7.3 ms |@@ -189,7 +204,7 @@ every cell on screen a shade), at 200 x 60 (`BenchmarkFrame`,
189204 | | Without rules | With the scale |190205 |---|---|---|191206 | A frame, or an arrow key through to it | 0.89 ms | 1.28 ms |192-| Typing a number and Enter, through to the frame | 0.91 ms | 2.9 ms (the scale's percentile over 213 k numbers) |207+| Typing a number and Enter, through to the frame | 0.89 ms | 2.3 ms (the scale's percentile over 213 k numbers) |193208 194209 Shades and rule colors keep their escape codes, so a plain cell on one195210 costs a string concatenation, not a style render. A custom formula is@@ -202,8 +217,8 @@ slices, 0.44 ms, is the worst). An image for kitty graphics is redrawn
202217 when its data, size or theme changes: 25 to 260 us at 24 x 10 cells and203218 1 to 1.5 ms at 120 x 40, a pie 8.6 ms (`internal/chart`,204219 `BenchmarkDraw`, `BenchmarkImage`). Key presses add little: an arrow key205-through to its frame is 0.21 ms at 80 x 24 and 1.06 ms at 200 x 60;206-Page Down 1.26 ms.220+through to its frame is 0.19 ms at 80 x 24 and 0.9 ms at 200 x 60;221+Page Down 1.1 ms.207222 208223 ## Selection statistics209224 @@ -223,9 +238,9 @@ a selection over the whole sheet, through to its frame
223238 224239 | Key, 8192 x 256 numbers | 80 x 24 | 200 x 60 |225240 |---|---|---|226-| Shift+Up and Shift+Down with the data selected (`extend-data`) | 0.41 to 0.53 ms | 2.1 ms |227-| Ctrl+Shift+Down and Ctrl+Shift+Up over the first row (`extend-edge`) | 0.43 ms | 1.7 ms |228-| Ctrl+A, the selection standing (`select-data`) | 0.28 ms | 1.8 ms |241+| Shift+Up and Shift+Down with the data selected (`extend-data`) | 0.33 ms | 1.85 ms |242+| Ctrl+Shift+Down and Ctrl+Shift+Up over the first row (`extend-edge`) | 0.29 ms | 1.56 ms |243+| Ctrl+A, the selection standing (`select-data`) | 0.29 ms | 1.78 ms |229244 230245 The index costs memory only on sheets that had a large selection, in231246 proportion to their data, and a nil check per stored or recalculated@@ -238,13 +253,13 @@ Real, openly licensed datasets fetched by `scripts/stress-data.sh`
238253 239254 | File | Shape | Import | Throughput | Heap per cell |240255 |---|---|---|---|---|241-| OWID energy (CSV) | 23,377 x 130, 1.04 M cells | 0.84 to 0.95 s | 1.1 to 1.2 M cells/s | 277 B |242-| Airport codes (CSV) | 86,134 x 13, 855 k cells | 0.78 to 0.80 s | 1.1 M cells/s | 274 B |243-| NOAA daily CO2 (CSV) | 18,304 x 2 | 6 ms | 2.7 M cells/s | 294 B |244-| Country codes (CSV, Arabic, CJK) | 249 x 56 | 5.6 ms | 2.2 M cells/s | 272 B |245-| Apache POI formula tests (XLSX) | 788 rows, 1189 formulas, 3828 cells | 9.5 to 9.7 ms | 395 k cells/s | 425 B |246-| Chinook PlaylistTrack (SQLite) | 8715 x 2 | 5.5 ms | 3.0 M cells/s | 293 B |247-| Parquet alltypes_tiny_pages | 7300 x 13 | 40 ms | 2.4 M cells/s | 276 B |256+| OWID energy (CSV) | 23,377 x 130, 1.04 M cells | 0.25 s | 4.2 M cells/s | 23 B |257+| Airport codes (CSV) | 86,134 x 13, 855 k cells | 0.39 s | 2.2 M cells/s | 55 B |258+| NOAA daily CO2 (CSV) | 18,304 x 2 | 12 ms | 3.1 M cells/s | 63 B |259+| Country codes (CSV, Arabic, CJK) | 249 x 56 | 5.4 ms | 2.3 M cells/s | 59 B |260+| Apache POI formula tests (XLSX) | 788 rows, 1189 formulas, 3828 cells | 8.5 to 10 ms | 380 to 450 k cells/s | 239 B |261+| Chinook PlaylistTrack (SQLite) | 8715 x 2 | 6.1 ms | 2.9 M cells/s | 21 B |262+| Parquet alltypes_tiny_pages | 7300 x 13 | 27 ms | 3.5 M cells/s | 44 B |248263 249264 Past `max-cells` (or the grid), imports keep whole rows up to it and250265 say how many rows they left out. Parquet files and SQLite tables stop@@ -263,10 +278,10 @@ importing, per cell kept, and includes garbage not yet collected:
263278 264279 | Workbook | Time | Allocated | Allocations | Heap per cell after | Peak heap per cell |265280 |---|---|---|---|---|---|266-| 8192 x 26 numbers (213 k cells) | 189 to 199 ms | 170 MB | 3.3 M | 290 B | 589 B |267-| 8192 x 26 table, text and numbers | 296 to 312 ms | 234 MB | 7.4 M | 297 B | 722 B |268-| 8192 formulas (`=A1+1` down a column) | 13.8 ms | 17 MB | 321 k | 746 B | 2188 B |269-| Apache POI formula tests | 9.5 to 9.7 ms | 9.7 MB | 179 k | 425 B | 1288 B |281+| 8192 x 26 numbers (213 k cells) | 147 to 154 ms | 171 MB | 3.3 M | 20.5 B | 114 B |282+| 8192 x 26 table, text and numbers | 262 to 275 ms | 246 MB | 7.7 M | 21 B | 119 B |283+| 8192 formulas (`=A1+1` down a column) | 14.5 ms | 18.6 MB | 321 k | 755 B | 2061 B |284+| Apache POI formula tests | 8.5 to 10 ms | 9 MB | 162 k | 239 B | 965 B |270285 271286 The reader refuses files past `xlsxLimits` (in `internal/fileio/xlsxpkg.go`),272287 with a message naming the limit: 10,000 files in the zip; 1 GB@@ -282,9 +297,12 @@ entities, so entity bombs fail as unknown entities.
282297 283298 ## Undo284299 285-Each step keeps a copy of every cell it changed. Undoing costs about300+Each step keeps a copy of every cell it changed, as a whole `Cell`301+(about 300 B, where the sheet keeps a number in 20). Undoing costs about286302 what the change cost, since it recalculates the same cells. Clearing287-213 k cells and undoing it takes 158 to 164 ms.303+213 k cells and undoing it takes 144 ms; clearing all ten million cells304+of a full `max-cells` sheet would keep about 3 GB of before-images, the305+one step the history always keeps however large.288306 289307 The history is bounded by steps (`MaxUndo = 100`) and by an estimate of290308 the memory its before-images hold (`MaxUndoBytes = 256 MB`); past either,@@ -298,9 +316,9 @@ measured heap within about 10% (`BenchmarkHistoryFull`,
298316 299317 | Benchmark | Result |300318 |---|---|301-| 100 edits rewriting a whole column (819 k before-images) | 250 MB held (239 MB estimated), all 100 kept |302-| 12 edits rewriting all of 8192 x 26 (2.6 M before-images) | 230 MB held (248 MB estimated), the last 4 kept |303-| Undo + redo of one cell (dense, names) | 1.0 us, 106 us |319+| 100 edits rewriting a whole column (819 k before-images) | 281 MB held (252 MB estimated), all 100 kept |320+| 12 edits rewriting all of 8192 x 26 (2.6 M before-images) | 199 MB held (196 MB estimated), the last 3 kept |321+| Undo + redo of one cell (dense, names) | 1.2 us, 110 us |304322 305323 ## JEV306324 @@ -372,8 +390,8 @@ read-heavy recalculation.
372390 373391 | Benchmark (`internal/sheet`, `go test -bench SheetEdit`) | Edit |374392 |---|---|375-| 5000 formulas on one sheet reading `A(n)` and `SUM(A1:A10)` | 2.7 ms |376-| The same 5000 formulas reading `Data!A(n)` and `SUM(Data!A1:A10)` | 3.5 ms |393+| 5000 formulas on one sheet reading `A(n)` and `SUM(A1:A10)` | 1.65 ms |394+| The same 5000 formulas reading `Data!A(n)` and `SUM(Data!A1:A10)` | 2 ms |377395 378396 ## Serving over SSH379397 @@ -388,10 +406,10 @@ for 120 frames a second, as the local app does (`-benchtime 200x`):
388406 389407 | Sessions, sheet | Heap per session | Key to frame p50 / p95 / max | CPU per frame |390408 |---|---|---|---|391-| 10, new sheet | 1.30 MiB | 5.1 / 8.3 / 9.6 ms | 0.67 ms |392-| 10, 1000 x 26 numbers | 8.5 MiB | 4.3 / 8.3 / 8.5 ms | 0.71 ms |409+| 10, new sheet | 1.35 MiB | 5.1 / 8.3 / 9.8 ms | 0.83 ms |410+| 10, 1000 x 26 numbers | 1.9 MiB | 3.9 / 8.3 / 10.8 ms | 0.79 ms |393411 | 50, new sheet | 1.30 MiB | 5.0 / 8.7 / 17.8 ms | 0.53 ms |394-| 50, 1000 x 26 numbers | 8.5 MiB | 5.1 / 8.9 / 12.8 ms | 0.66 ms |412+| 50, 1000 x 26 numbers | 1.9 MiB | 5.1 / 8.9 / 12.8 ms | 0.66 ms |395413 396414 Latency doesn't move from 10 to 50 sessions: it's Bubble Tea's frame397415 pacing (at most one frame every 8.3 ms, so a key waits half a frame on@@ -400,7 +418,7 @@ average), not load. A heap profile with 50 sessions open
400418 Bubble Tea's render buffers (about 1 MiB at 120 x 40, growing with the401419 window), 130 KiB in its input key table and under 50 KiB in both ends402420 of the SSH connection, so the figure is close to the server's own. On403-top come about 300 B per cell of the sheets a session has open;421+top come 20 to 60 B per cell of data of the sheets a session has open;404422 sessions opening the same file each hold a copy. CPU per frame covers405423 both ends of the connection (the encryption twice, the client reading406424 the frame); at 0.7 ms, 50 sessions typing continuously at 120 frames a@@ -420,10 +438,10 @@ times the edit, the recalculation and the pivot together:
420438 421439 | Pivot | Result rows | Edit | Allocations |422440 |---|---|---|---|423-| Rows: 8 categories; SUM and COUNTA | 10 | 0.77 to 0.95 ms | 196 |424-| Rows: 8 x 8 categories with subtotals; columns: 8; SUM and AVERAGE | 75 | 2.7 to 2.9 ms | 2.5 k |425-| Rows: 1000 items sorted by their sum; SUM and COUNTUNIQUE | 1001 | 3.5 to 4.2 ms | 14 k |426-| Frequency table of the 1000 items | 1001 | 2.5 to 2.8 ms | 24 k |441+| Rows: 8 categories; SUM and COUNTA | 10 | 0.79 ms | 196 |442+| Rows: 8 x 8 categories with subtotals; columns: 8; SUM and AVERAGE | 75 | 2.2 ms | 2.5 k |443+| Rows: 1000 items sorted by their sum; SUM and COUNTUNIQUE | 1001 | 2.5 ms | 14 k |444+| Frequency table of the 1000 items | 1001 | 1.8 ms | 24 k |427445 428446 ## Arrays and spills429447 @@ -448,10 +466,10 @@ FILTERs, 63 of whose arrays change:
448466 449467 | Measure | Time | Allocated |450468 |---|---|---|451-| Edit a number | 55 ms | 161 MB, 8.3 k allocations |452-| Undo and redo it | 110 ms | |453-| Full recalculation | 204 ms | |454-| Build (load and recalculate) | 310 ms | |469+| Edit a number | 52 ms | 161 MB, 7.9 k allocations |470+| Undo and redo it | 103 ms | |471+| Full recalculation | 217 ms | |472+| Build (load and recalculate) | 298 ms | |455473 456474 Most of an edit is the FILTERs' own work: comparing 8192 categories each457475 (`B1:B8192="alpha"`, about 20 ns a value) and picking the rows that@@ -481,32 +499,25 @@ computation.
481499 482500 ## What would raise the bounds483501 484-In order of value for effort. The first two are what raising485-`max-cells` toward Google Sheets' 10 M cells would take; without them a486-10 M-cell sheet would need about 3 GB of heap and take 10 s to open.487-488-1. **Compact cell storage** (L, 2 to 3 weeks). Cells live in a489- `map[Addr]*Cell` behind `cellStore` (`internal/sheet/store.go`), 300 B490- each: a 216 B struct carrying formula-only491- fields (expression, reference lists, inferred format) on every cell,492- plus the map entry, the input string and a boxed literal. Storing493- columns in row blocks (say 1024 rows) of compact values, with494- formulas, formats and styles in side tables keyed by address, and495- the input string kept only when it differs from the canonical text496- of the value, would bring numbers to about 20 to 40 B per cell and497- turn every cell read in recalc from two hash lookups (20 ns) into an498- index (2 to 3 ns). Cells are already per sheet in a workbook, so the499- storage can change sheet by sheet, and nothing outside `cellStore`500- touches the map, so the change stays inside it.501-2. **Streaming, compact file format** (M, 1 week). The `.012` file is502- JSON decoded whole (1.3 GB allocated to open 47 MB). A streaming503- decoder over the same format would roughly halve open time and cut504- allocation tenfold; a columnar or gzip-compressed variant would cut505- the size about fourfold. The format is `internal/sheet/file.go`, apart506- from the cell store.507-3. **Smaller undo steps** (S, 1 day). The history is capped by the508- memory its before-images hold; storing formatting-only changes as509- diffs rather than whole cells would let it keep more of them.502+In order of value for effort. `max-cells` stands at ten million, as503+Google Sheets' limit does; the sheet itself would hold several times504+that in a few GB, and these are what stand in the way.505+506+1. **Streaming, compact file format** (M, 1 week). The `.012` file is507+ JSON decoded whole: ten million cells take 8 s and 2.7 GB at the508+ peak to open, 5.7 GB allocated, for a sheet that then holds 205 MB.509+ A streaming decoder over the same format would roughly halve open510+ time and cut allocation tenfold; a columnar or gzip-compressed511+ variant would cut the size about fourfold. The format is512+ `internal/sheet/file.go`, apart from the cell store.513+2. **Smaller undo steps** (S to M). A step keeps a whole `Cell` for each514+ cell it changed, about 300 B where the store keeps a number in 20,515+ and formatting-only changes as whole cells too. Keeping plain cells'516+ before-images as slots, and formatting changes as diffs, would let a517+ step over a full sheet cost what the sheet does.518+3. **Compact spilled and pivot cells.** What arrays and pivots write is519+ kept as whole `Cell`s (about 300 B each, 516 k of them in the arrays520+ stress shape); their values and inferred formats would fit in slots.510521 511522 ## Measuring512523 @@ -531,5 +542,6 @@ overhead), `internal/fileio/stress_test.go` (imports and exports),
531542 `internal/chart/stress_test.go` (charts as text and images),532543 `internal/serve/stress_test.go` (concurrent SSH sessions) and533544 `e2e/stress_test.go`; the synthetic sheets are built by534-`internal/stress`. Results accumulate in `.deps/stress/results` and, with545+`internal/stress`, up to `dense-1Mx10`, ten million numbers, the546+default `max-cells`. Results accumulate in `.deps/stress/results` and, with535547 the observability stack up, in ClickHouse.@@ -49,9 +49,9 @@ which kept their range, and undo brings the removed ones back.
4949 5050 **Size.** A sheet is 1,048,576 rows by 16,384 columns (A to XFD), as in5151 Excel. Imports keep at most `max-cells` cells ([Configuration](../reference/config.md#max-cells),52-two million by default, about 600 MB): whole rows, as many as fit, and52+ten million by default, a few hundred MB): whole rows, as many as fit, and5353 the context line says how many rows were left out, e.g. `only the first54-166,666 rows fit in max-cells (2,000,000 cells); 12,000 rows left out`.54+833,333 rows fit in max-cells (10,000,000 cells); 12,000 rows left out`.5555 Data past the grid's edges is left out the same way. WK1 files keep their5656 own 8,192 by 256. See [Bounds of support](../contributing/limits.md#imports) for speeds.5757 @@ -71,7 +71,7 @@ start.
7171 | [`chart-images`](#chart-images) | `true` | `O12_CHART_IMAGES` |7272 | [`notifications`](#notifications) | `true` | `O12_NOTIFICATIONS` |7373 | [`keymap`](#keymap) | `default` | `O12_KEYMAP` |74-| [`max-cells`](#max-cells) | `2000000` | `O12_MAX_CELLS` |74+| [`max-cells`](#max-cells) | `10000000` | `O12_MAX_CELLS` |7575 | [`jev-api-key-command`](#jev-api-key-command) | | |7676 | [`jev-credential-store`](#jev-credential-store) | `true` | `O12_JEV_CREDENTIAL_STORE` |7777 | [`jev-base-url`](#jev-base-url) | | `TYPESAFE_BASE_URL` |@@ -137,12 +137,12 @@ Keys in the grid. `default` works like Google Sheets; `vim` adds hjkl, counts, o
137137 138138 #### `max-cells`139139 140-The most cells an import keeps, and a paste or fill writes at once. A sheet takes about 300 bytes a cell, so the default of two million is about 600 MB. Imports keep whole rows up to the budget and say how many they left out; larger pastes and fills are refused. The grid itself is 1,048,576 rows by 16,384 columns (A to XFD) whatever this is.140+The most cells an import keeps, and a paste or fill writes at once. Numbers and text take 20 to 60 bytes a cell and formulas about 750, so the default of ten million cells of data is 200 to 600 MB. Imports keep whole rows up to the budget and say how many they left out; larger pastes and fills are refused. The grid itself is 1,048,576 rows by 16,384 columns (A to XFD) whatever this is.141141 142142 | | |143143 |---|---|144144 | Type | number |145-| Default | `2000000` |145+| Default | `10000000` |146146 | Environment | `O12_MAX_CELLS` |147147 | Applies | File > Settings > Reload config |148148 @@ -190,8 +190,8 @@ check protects the local app from other programs writing the file.
190190 - An idle session is closed without saving over its file; unsaved191191 changes go to a recovery file instead.192192 - A dropped connection loses unsaved changes.193-- Each session holds about 1.3 MiB of heap on a new sheet, plus about194- 300 B per cell of the sheets it opens; two sessions opening one file195- hold two copies. With 50 sessions typing at once on loopback, frames193+- Each session holds about 1.3 MiB of heap on a new sheet, plus 20 to194+ 60 B per cell of data of the sheets it opens (a formula takes about195+ 750); two sessions opening one file hold two copies. With 50 sessions typing at once on loopback, frames196196 still arrive within one frame interval (p95 16.6 ms), the same as with197197 10: see [Bounds of support](../contributing/limits.md#serving-over-ssh).@@ -93,9 +93,10 @@ var Options = []Option{
9393 Desc: "Keys in the grid. `default` works like Google Sheets; `vim` adds hjkl, counts, operators, " +9494 "visual selection and a : command line (File > Settings > Vim keys)."},9595 96- {Name: "max-cells", Kind: Int, Group: GroupData, Default: "2000000", Env: []string{"O12_MAX_CELLS"}, Live: true,97- Desc: "The most cells an import keeps, and a paste or fill writes at once. A sheet takes about " +98- "300 bytes a cell, so the default of two million is about 600 MB. Imports keep whole rows " +96+ {Name: "max-cells", Kind: Int, Group: GroupData, Default: "10000000", Env: []string{"O12_MAX_CELLS"}, Live: true,97+ Desc: "The most cells an import keeps, and a paste or fill writes at once. Numbers and text " +98+ "take 20 to 60 bytes a cell and formulas about 750, so the default of ten million cells of " +99+ "data is 200 to 600 MB. Imports keep whole rows " +99100 "up to the budget and say how many they left out; larger pastes and fills are refused. " +100101 "The grid itself is 1,048,576 rows by 16,384 columns (A to XFD) whatever this is."},101102 @@ -13,21 +13,21 @@ const materializeLimit = 1 << 16
1313 // formula (=DATE() shows a date, =SUM(B2:B4) of currency shows1414 // currency).1515 func (s *Sheet) DisplayFormat(a Addr) Format {16- c := s.cells.get(a)16+ f, sty, ok := s.cells.look(a)1717 switch {18- case c != nil && !c.Format.IsZero():19- return c.Format18+ case !f.IsZero():19+ return f2020 case s.lines.none():21- case c != nil && c.Style.own:21+ case ok && sty.own:2222 default:2323 if f := s.inherited(a).Format; !f.IsZero() {2424 return f2525 }2626 }27- if c == nil {28- return Format{}27+ if c := s.cells.richAt(a); c != nil { // only formulas infer formats28+ return c.auto2929 }30- return c.auto30+ return Format{}3131 }3232 3333 // SetFormat gives every cell in r the number format f, including blank@@ -191,7 +191,7 @@ func (s *Sheet) clipToUsed(r Rect) Rect {
191191 // nothing more touches it. A blank cell with no filled neighbors is a192192 // region of its own.193193 func (s *Sheet) Region(a Addr) Rect {194- filled := func(a Addr) bool { return a.Valid() && !s.cells.get(a).Blank() }194+ filled := func(a Addr) bool { return a.Valid() && s.cells.filledAt(a) }195195 seen := map[Addr]bool{}196196 var queue []Addr197197 visit := func(p Addr) {@@ -253,8 +253,8 @@ func (s *Sheet) GuessChart(r Rect) Chart {
253253 from++254254 }255255 texts, nums := 0, 0256- for _, cell := range s.cells.inRange(Rect{From: Addr{Col: from, Row: r.From.Row}, To: Addr{Col: r.To.Col, Row: r.From.Row}}) {257- switch cell.Value.Kind {256+ for _, v := range s.cells.valuesIn(Rect{From: Addr{Col: from, Row: r.From.Row}, To: Addr{Col: r.To.Col, Row: r.From.Row}}) {257+ switch v.Kind {258258 case Text:259259 texts++260260 case Number, Bool:@@ -200,7 +200,7 @@ func (s *Sheet) formatTargets(f *clipFormats, p pasteLayout) []Addr {
200200 if len(targets) > materializeLimit {201201 return targets202202 }203- if a := (Addr{Col: c, Row: row}); s.cells.get(a) == nil {203+ if a := (Addr{Col: c, Row: row}); !s.cells.has(a) {204204 targets = append(targets, a)205205 }206206 }@@ -289,7 +289,7 @@ func (s *Sheet) plainSource(src, dst Rect, same bool) {
289289 if seen++; seen > materializeLimit {290290 return291291 }292- if a := (Addr{Col: c, Row: row}); !(same && dst.Contains(a)) && s.cells.get(a) == nil {292+ if a := (Addr{Col: c, Row: row}); !(same && dst.Contains(a)) && !s.cells.has(a) {293293 s.placeFormat(a, lineFmt{})294294 }295295 }@@ -68,14 +68,17 @@ func (w *Workbook) evaluate() {
6868 }6969 7070 // compute returns the value of the cell at a, evaluating it first if it's71-// dirty. Every cell a formula reads comes through here, so it looks each72-// map up once: a SUM over 8192 cells makes 8192 calls.71+// dirty. Every cell a formula reads comes through here: a SUM over 819272+// cells makes 8192 calls. A plain cell's value is its entry's, never73+// out of date, so it costs an index into the store; only rich cells74+// look up their recalculation state.7375 func (e *evaluator) compute(s *Sheet, a Addr) Value {7476 w := e.w75- c := s.cells.get(a)77+ c, v, _ := s.cells.peek(a)78+ if c == nil { // plain or blank79+ return v80+ }7681 switch st := s.calc[a]; {77- case c == nil:78- return Value{}7982 case st == visiting, st == deferred:8083 w.Circular = true8184 return ErrRef@@ -198,8 +198,8 @@ func (s *Sheet) writeBody(b *bytes.Buffer, indent, names string) error {
198198 return err199199 }200200 addrs := make([]Addr, 0, s.cells.len())201- for a, c := range s.cells.all() {202- if c.saved() != nil {201+ for a := range s.cells.keys() {202+ if c := s.cells.richAt(a); c == nil || c.saved() != nil {203203 addrs = append(addrs, a)204204 }205205 }@@ -294,11 +294,10 @@ func (s *Sheet) rowPasses(row int, tests []colTest) bool {
294294 // ShownText is the cell's value as its format displays it, with no width295295 // limit: what filters and the values list compare.296296 func (s *Sheet) ShownText(a Addr) string {297- c := s.cells.get(a)298- if c.Blank() {297+ if !s.cells.filledAt(a) {299298 return ""300299 }301- return FormatText(c.Value, s.DisplayFormat(a))300+ return FormatText(s.cells.value(a), s.DisplayFormat(a))302301 }303302 304303 // Matches reports whether a cell with value v, shown as shown, meets the@@ -162,7 +162,7 @@ func (s *Sheet) record(a Addr) {
162162 }163163 l := loc{s, a}164164 if _, seen := st.cells[l]; !seen {165- st.keep(l, s.cells.get(a).clone())165+ st.keep(l, s.cells.copyOf(a))166166 }167167 s.wb.hist.dirty = append(s.wb.hist.dirty, l)168168 }@@ -245,7 +245,7 @@ func (w *Workbook) dropUnchanged(st *step) {
245245 st.dropUnchangedLines()246246 st.dropUnchangedRules()247247 for l, c := range st.cells {248- if c == nil && l.s.cells.get(l.a) == nil {248+ if c == nil && !l.s.cells.has(l.a) {249249 delete(st.cells, l)250250 st.cellBytes -= entryBytes251251 }@@ -421,7 +421,7 @@ func (w *Workbook) swap(undo bool) (Change, bool) {
421421 }422422 changed := make([]loc, 0, len(st.cells))423423 for l, c := range st.cells {424- inv.keep(l, l.s.cells.get(l.a).clone())424+ inv.keep(l, l.s.cells.copyOf(l.a))425425 l.s.place(l.a, c.clone())426426 changed = append(changed, l)427427 }@@ -12,15 +12,15 @@ const MaxUndoBytes = 256 << 20
1212 // undoBudget is MaxUndoBytes; tests lower it.1313 var undoBudget int64 = MaxUndoBytes1414 15-// Estimated heap held by the parts of a step. A before-image is a shallow16-// copy of a Cell (216 B) plus its map entry and boxed literal; a formula17-// also keeps its parsed tree and reference lists alive, which grow with18-// its text. The constants match the heap measured by the stress15+// Estimated heap held by the parts of a step. A before-image is a Cell16+// (216 B) plus its map entry, its input and boxed literal (a plain cell's17+// are made for it from the store, see store.go); a formula also keeps its18+// parsed tree and reference lists alive, which grow with its text. The constants match the heap measured by the stress1919 // benchmarks (BenchmarkHistoryFull) within about 10%.2020 const (2121 stepBytes = 512 // the step and its empty maps2222 entryBytes = 64 // a map entry: the key and a pointer23- cellBytes = 240 // a copied Cell and its boxed literal23+ cellBytes = 256 // a Cell, its input's header and its boxed literal2424 formulaBytes = 256 // a parsed formula's tree and references, plus perFormulaByte per byte of text2525 perFormulaByte = 162626 chartBytes = 128@@ -73,7 +73,7 @@ func (w *Workbook) deletedSize(st *step) int64 {
7373 var n int647474 for _, s := range st.sheets.order {7575 if !slices.Contains(w.sheets, s) {76- n += int64(s.Len()) * (entryBytes + cellBytes)76+ n += s.cells.size()7777 }7878 }7979 return n@@ -109,7 +109,7 @@ func TestUndoBytesCountDeletedSheet(t *testing.T) {
109109 if err := w.DeleteSheet(data); err != nil {110110 t.Fatal(err)111111 }112- if got := w.HistoryBytes(); got < 500*cellBytes {112+ if got := w.HistoryBytes(); got < 500*slotBytes {113113 t.Errorf("deleting a 500-cell sheet is estimated at %d bytes", got)114114 }115115 }@@ -153,7 +153,7 @@ func (s *Sheet) repoint(old *Sheet, d Rect, byRow bool) (Rect, bool) {
153153 return d, true154154 }155155 now := s.Region(d.From)156- if now.From != d.From || s.cells.get(now.From).Blank() && now.From == now.To {156+ if now.From != d.From || !s.cells.filledAt(now.From) && now.From == now.To {157157 return d, false158158 }159159 if byRow && now.To.Row-now.From.Row != d.To.Row-d.From.Row ||@@ -91,20 +91,17 @@ func (s *Sheet) SheetFormat() (Format, Style) { return s.lines.sheet.Format, s.l
9191 // effective is the format (without what formulas infer) and style the9292 // cell at a shows: its own, falling back part by part on its line's.9393 func (s *Sheet) effective(a Addr) lineFmt {94- c := s.cells.get(a)95- if c != nil && c.Style.own {96- st := c.Style97- st.own = false98- return lineFmt{c.Format, st}94+ f, sty, ok := s.cells.look(a)95+ if ok && sty.own {96+ sty.own = false97+ return lineFmt{f, sty}9998 }10099 l := s.inherited(a)101- if c != nil {102- if !c.Format.IsZero() {103- l.Format = c.Format104- }105- if !c.Style.IsZero() {106- l.Style = c.Style107- }100+ if !f.IsZero() {101+ l.Format = f102+ }103+ if !sty.IsZero() {104+ l.Style = sty108105 }109106 return l110107 }@@ -266,7 +263,7 @@ func (s *Sheet) formatLines(r Rect, fn func(*lineFmt)) bool {
266263 if !cols {267264 a = Addr{Col: n, Row: across.From.Row + i} // a row's cells in formatted column n268265 }269- if s.cells.get(a) == nil {266+ if !s.cells.has(a) {270267 targets = append(targets, a)271268 }272269 }@@ -104,7 +104,7 @@ func (c *looksCache) look(s *Sheet, a Addr) Look {
104104 k := s.rules.validations[i].Kind105105 l.Checkbox, l.Dropdown = k == ValidCheckbox, k.Dropdown()106106 l.Checked = l.Checkbox && v.Kind == Bool && v.Num != 0107- l.Invalid = !s.cells.get(a).Blank() && !c.check(s, i, a, v, s.DisplayFormat(a))107+ l.Invalid = s.cells.filledAt(a) && !c.check(s, i, a, v, s.DisplayFormat(a))108108 }109109 return l110110 }@@ -264,11 +264,11 @@ func (c *looksCache) scale(s *Sheet, i int) *scaleStats {
264264 lo, hi, n := math.Inf(1), math.Inf(-1), 0265265 var vals []float64266266 for _, r := range f.Ranges {267- for _, cell := range s.cells.inRange(r) {268- if cell.Value.Kind != Number {267+ for _, v := range s.cells.valuesIn(r) {268+ if v.Kind != Number {269269 continue270270 }271- v := cell.Value.Num271+ v := v.Num272272 lo, hi, n = min(lo, v), max(hi, v), n+1273273 if keep {274274 vals = append(vals, v)@@ -104,7 +104,7 @@ func (mv sheetMove) readers() map[loc]*Cell {
104104 }105105 }106106 for _, t := range []*Sheet{mv.from, mv.to} {107- for a := range t.cells.all() {107+ for a := range t.cells.richCells() {108108 add(loc{t, a})109109 }110110 }@@ -17,7 +17,7 @@ const MaxNote = 4096
1717 1818 // Note returns the note on the cell at a, or "".1919 func (s *Sheet) Note(a Addr) string {20- if c := s.cells.get(a); c != nil {20+ if c := s.cells.richAt(a); c != nil { // plain cells have no note2121 return c.Note2222 }2323 return ""@@ -91,8 +91,8 @@ func (s *Sheet) LoadNote(a Addr, text string) {
9191 // It walks the cells the sheet holds, not r's addresses.9292 func (s *Sheet) NotesIn(r Rect) []Addr {9393 var out []Addr94- for a := range s.cells.anyInRange(r) {95- if s.cells.get(a).Note != "" {94+ for a := range s.cells.anyKeysIn(r) {95+ if s.Note(a) != "" {9696 out = append(out, a)9797 }9898 }@@ -15,6 +15,7 @@ import (
1515 type occupancy struct {1616 cols []*colIndex // by column; nil for none1717 colIDs []int // the columns holding cells, ascending18+ n int // the cells marked1819 }1920 2021 // Rows of a column are indexed in blocks of blockRows, one bit each.@@ -25,9 +26,28 @@ const (
2526 )2627 2728 type rowBlock struct {28- bits [blockWords]uint6429+ bits [blockWords]uint6430+ // rank counts the rows set in the words before each, so a row's31+ // position among the block's is a popcount away (index).32+ rank [blockWords]uint162933 n int3034 stats *blockStats // the filled index's statistics, made on first use; see stats.go35+ vals []slot // the stored index's cells, one per set row in order; see slot.go36+}37+38+// index is the position of row bit of the block among its set rows, and39+// whether it is set.40+func (b *rowBlock) index(bit int) (int, bool) {41+ w, m := bit>>6, uint64(1)<<(bit&63)42+ x := b.bits[w]43+ return int(b.rank[w]) + bits.OnesCount64(x&(m-1)), x&m != 044+}45+46+// count adds d to the rank of the words after word w.47+func (b *rowBlock) count(w, d int) {48+ for j := w + 1; j < blockWords; j++ {49+ b.rank[j] = uint16(int(b.rank[j]) + d)50+ }3151 }3252 3353 // colIndex is the occupancy of one column: its blocks by number (nil@@ -61,8 +81,8 @@ func (o *occupancy) has(a Addr) bool {
6181 return b != nil && b.bits[bit>>6]&(1<<(bit&63)) != 06282 }6383 64-// mark records a cell at a, which had none.65-func (o *occupancy) mark(a Addr) {84+// mark records a cell at a, which had none, and returns its block.85+func (o *occupancy) mark(a Addr) *rowBlock {6686 ci := o.col(a.Col)6787 if ci == nil {6888 ci = &colIndex{}@@ -84,8 +104,11 @@ func (o *occupancy) mark(a Addr) {
84104 }85105 bit := a.Row & (blockRows - 1)86106 b.bits[bit>>6] |= 1 << (bit & 63)107+ b.count(bit>>6, 1)87108 b.n++88109 ci.n++110+ o.n++111+ return b89112 }90113 91114 // unmark records that a no longer holds a cell.@@ -95,8 +118,10 @@ func (o *occupancy) unmark(a Addr) {
95118 b := ci.blocks[id]96119 bit := a.Row & (blockRows - 1)97120 b.bits[bit>>6] &^= 1 << (bit & 63)121+ b.count(bit>>6, -1)98122 b.n--99123 ci.n--124+ o.n--100125 if b.n == 0 {101126 ci.blocks[id] = nil102127 ci.ids = removeSorted(ci.ids, id)@@ -16,9 +16,9 @@ var (
1616 ErrFillTooBig = errors.New("That would write more cells than max-cells allows (see File > Settings)")1717 )1818 19-// DefaultMaxCells is the max-cells setting's default: about 600 MB of20-// cells at 300 bytes each.21-const DefaultMaxCells = 2_000_00019+// DefaultMaxCells is the max-cells setting's default: about 200 MB of20+// numbers at 20 bytes each (store.go).21+const DefaultMaxCells = 10_000_0002222 2323 // maxCells caps how many cells a paste or fill writes at once, and how2424 // many an import keeps, so an accidental whole-sheet selection or a huge@@ -65,13 +65,13 @@ func (s *Sheet) remap(label string, focus Rect, cell func(Addr) (Addr, bool), rn
6565 for l, c := range others {6666 l.s.place(l.a, c)6767 }68- for a, c := range s.cells.all() {69- if next[a] != c {70- s.place(a, next[a])68+ for a := range s.cells.keys() {69+ if c := next[a]; !s.cells.holds(a, c) {70+ s.place(a, c)7171 }7272 }7373 for a, c := range next {74- if s.cells.get(a) != c {74+ if !s.cells.holds(a, c) {7575 s.place(a, c)7676 }7777 }@@ -111,7 +111,7 @@ func (s *Sheet) insert(rows bool, at, n int) error {
111111 if rows {112112 edge = rowRect(first, size-1)113113 }114- for range s.cells.inRange(edge) {114+ for range s.cells.keysIn(edge) {115115 return ErrPushedOff116116 }117117 s.restructure(rows, formula.Span{At: at, N: n, Size: size})@@ -290,7 +290,7 @@ func (s *Sheet) Paste(c *Clip, dst Rect, values bool) (Rect, error) {
290290 func (s *Sheet) pasteCell(a Addr, c *Cell, dc, dr int, values bool) {291291 switch {292292 case c == nil:293- if s.cells.get(a) != nil {293+ if s.cells.has(a) {294294 s.place(a, nil)295295 }296296 case values:@@ -411,7 +411,7 @@ func (s *Sheet) seriesStart(r Rect, down bool) (Rect, bool) {
411411 }412412 used := map[int]bool{}413413 for _, a := range s.cellsIn(r) {414- if !s.cells.get(a).Blank() {414+ if s.cells.filledAt(a) {415415 used[line(a)] = true416416 }417417 }@@ -387,7 +387,7 @@ func (s *Sheet) writeDerived(want []pivotCell) []loc {
387387 next[pc.a] = c388388 }389389 var changed []loc390- for a, c := range s.cells.all() {390+ for a, c := range s.cells.richCells() {391391 if c.derived && next[a] == nil {392392 s.setDerived(a, nil)393393 changed = append(changed, loc{s, a})@@ -102,7 +102,7 @@ func (rd *reader) Fold(sheet string, r Rect, s functions.Agg) (functions.Agg, *V
102102 return add(Addr{Col: r.From.Col, Row: row})103103 })104104 default:105- for a := range t.cells.inRange(r) {105+ for a := range t.cells.keysIn(r) {106106 if !add(a) {107107 break108108 }@@ -135,7 +135,7 @@ func (run *runAgg) extend(rd *reader, t *Sheet, key aggKey, to int) {
135135 if key.c0 == key.c1 {136136 t.cells.colScan(key.c0, run.next, to, func(r int) bool { return visit(Addr{Col: key.c0, Row: r}) })137137 } else {138- for a := range t.cells.inRange(Rect{From: Addr{Col: key.c0, Row: run.next}, To: Addr{Col: key.c1, Row: to}}) {138+ for a := range t.cells.keysIn(Rect{From: Addr{Col: key.c0, Row: run.next}, To: Addr{Col: key.c1, Row: to}}) {139139 if !visit(a) {140140 break141141 }@@ -24,17 +24,18 @@ func (w *Workbook) RecalcAll() {
2424 // after a sheet is added, renamed or deleted, when references by name may2525 // resolve differently. Every cell is dirty, so there is nothing to2626 // propagate: tracing dependents from each cell cost O(cells x range27-// users).27+// users). Plain cells hold their values already (store.go), so only rich28+// ones are marked.2829 func (w *Workbook) recalcAll() {2930 w.gen++3031 start := w.recalcStart()3132 n := 03233 for _, s := range w.sheets {33- s.calc = make(map[Addr]int, s.cells.len())34- for a := range s.cells.all() {34+ s.calc = make(map[Addr]int, len(s.cells.rich)-len(s.cells.richFree))35+ for a := range s.cells.richCells() {3536 s.calc[a] = dirty3637 }37- n += len(s.calc)38+ n += s.cells.len()3839 }3940 w.evaluate()4041 w.settleSpills(nil)@@ -97,7 +97,7 @@ func (w *Workbook) RecalcAnswered(calls []RemoteCall) {
9797 for _, c := range calls {9898 k := c.Key()9999 for l := range w.waiting[k] {100- if l.s.live && l.s.cells.get(l.a) != nil {100+ if l.s.live && l.s.cells.has(l.a) {101101 changed = append(changed, l)102102 }103103 }@@ -108,7 +108,7 @@ func (s *Sheet) fill(l fillLane) {
108108 fallthrough109109 case c != nil:110110 s.pasteCell(to, c, to.Col-seq[k].Col, to.Row-seq[k].Row, false)111- case s.cells.get(to) != nil:111+ case s.cells.has(to):112112 s.place(to, nil)113113 }114114 }@@ -118,30 +118,21 @@ func New() *Sheet {
118118 // formatting. Use Blank to test for contents.119119 func (s *Sheet) Cell(a Addr) *Cell { return s.cells.get(a) }120120 121+// Filled reports whether the cell at a has contents: !Cell(a).Blank(),122+// without making the cell.123+func (s *Sheet) Filled(a Addr) bool { return s.cells.filledAt(a) }124+121125 // Value returns the computed value at a.122-func (s *Sheet) Value(a Addr) Value {123- if c := s.cells.get(a); c != nil {124- return c.Value125- }126- return Value{}127-}126+func (s *Sheet) Value(a Addr) Value { return s.cells.value(a) }128127 129128 // Len returns the number of non-blank cells.130-func (s *Sheet) Len() int {131- n := 0132- for _, c := range s.cells.all() {133- if !c.Blank() {134- n++135- }136- }137- return n138-}129+func (s *Sheet) Len() int { return s.cells.filledLen() }139130 140131 // Addrs returns every non-blank cell in row-major order.141132 func (s *Sheet) Addrs() []Addr {142- out := make([]Addr, 0, s.cells.len())143- for a, c := range s.cells.all() {144- if !c.Blank() {133+ out := make([]Addr, 0, s.cells.filledLen())134+ for a := range s.cells.keys() {135+ if s.cells.filledAt(a) {145136 out = append(out, a)146137 }147138 }@@ -245,12 +236,8 @@ func (s *Sheet) Set(a Addr, input string) error {
245236 // put stores an entry without recalculating, keeping the cell's246237 // formatting.247238 func (s *Sheet) put(a Addr, input string) error {248- var f Format249- var st Style250- var note string251- if old := s.cells.get(a); old != nil {252- f, st, note = old.Format, old.Style, old.Note253- }239+ f, st, _ := s.cells.look(a)240+ note := s.Note(a)254241 if f.IsZero() && !st.own && s.inherited(a).Format.Kind == FmtText {255242 f = Format{Kind: FmtText} // typed into a plain text column: text256243 }@@ -388,8 +375,9 @@ func (s *Sheet) place(a Addr, c *Cell) {
388375 func (s *Sheet) EraseRange(r Rect) {389376 s.change("clear "+r.String(), r, func() {390377 for _, a := range s.cellsIn(r) {391- if c := s.cells.get(a); !c.Blank() {392- s.place(a, c.leftover())378+ if s.cells.filledAt(a) {379+ f, st, _ := s.cells.look(a)380+ s.place(a, formattingOnly(f, st).withNote(s.Note(a)))393381 }394382 }395383 })@@ -398,7 +386,7 @@ func (s *Sheet) EraseRange(r Rect) {
398386 // cellsIn returns the cells in r that have contents or formatting.399387 func (s *Sheet) cellsIn(r Rect) []Addr {400388 var out []Addr401- for a := range s.cells.anyInRange(r) {389+ for a := range s.cells.anyKeysIn(r) {402390 out = append(out, a)403391 }404392 return out@@ -414,9 +402,9 @@ func (s *Sheet) unlink(a Addr) {
414402 // unindex removes the dependency edges of the cell at a, reporting415403 // whether there is one.416404 func (s *Sheet) unindex(a Addr) bool {417- old := s.cells.get(a)405+ old := s.cells.richAt(a) // plain cells have no edges418406 if old == nil {419- return false407+ return s.cells.has(a)420408 }421409 for _, r := range old.refs {422410 delete(s.dependents[r], a)@@ -0,0 +1,284 @@
1+package sheet2+3+import (4+ "math"5+ "strconv"6+ "strings"7+8+ "github.com/FelineStateMachine/012/internal/formula"9+)10+11+// A slot is how cellStore keeps one cell: 16 bytes in its column's12+// block. Most cells are plain, an entry typed or imported that is a13+// number, a boolean or text, with at most a format and a style; their14+// slot holds the value itself, the text in the store's string table, the15+// format and style in its table of looks, and the input only when it16+// isn't the value's own text. Everything else (formulas, notes, what17+// pivots and spills write, text typed with a leading ') is rich: the18+// slot points at a whole Cell in the store's side table.19+type slot struct {20+ num float64 // slotNum's value; slotBool's, 1 or 021+ // ref is slotText's text in strs; slotNum's or slotBool's input in22+ // strs when it isn't the value's text (0 when it is); slotRich's cell23+ // in rich.24+ ref uint3225+ look uint16 // the format and style in looks; 0 for neither26+ kind uint827+ // dec is how a slotNum without ref prints its input: the value with28+ // dec-1 decimals, or its shortest text when 0.29+ dec uint830+}31+32+// Kinds of slot.33+const (34+ slotBlank uint8 = iota // formatting only35+ slotNum36+ slotBool37+ slotText38+ slotRich39+)40+41+// maxDec bounds the decimals a number's input may have to be kept as a42+// count rather than as text.43+const maxDec = 4044+45+// look is a cell's format and style, kept once per store however many46+// cells share it.47+type look struct {48+ f Format49+ st Style50+}51+52+// strTable holds the text of plain cells, each distinct string once,53+// counted by the cells using it; entry 0 is unused.54+type strTable struct {55+ strs []string56+ refs []uint3257+ index map[string]uint3258+ free []uint3259+}60+61+// add counts one more use of s and returns its entry.62+func (t *strTable) add(s string) uint32 {63+ if i, ok := t.index[s]; ok {64+ t.refs[i]++65+ return i66+ }67+ if t.index == nil {68+ t.index = map[string]uint32{}69+ t.strs, t.refs = []string{""}, []uint32{0}70+ }71+ var i uint3272+ if n := len(t.free); n > 0 {73+ i, t.free = t.free[n-1], t.free[:n-1]74+ t.strs[i], t.refs[i] = s, 175+ } else {76+ i = uint32(len(t.strs))77+ t.strs, t.refs = append(t.strs, s), append(t.refs, 1)78+ }79+ t.index[s] = i80+ return i81+}82+83+// release counts one fewer use of entry i, if it is one.84+func (t *strTable) release(i uint32) {85+ if i == 0 {86+ return87+ }88+ if t.refs[i]--; t.refs[i] == 0 {89+ delete(t.index, t.strs[i])90+ t.strs[i] = ""91+ t.free = append(t.free, i)92+ }93+}94+95+// lookID is the entry of l in the table of looks, added if new, and96+// false when the table is full.97+func (st *cellStore) lookID(f Format, sty Style) (uint16, bool) {98+ l := look{f, sty}99+ if l == (look{}) {100+ return 0, true101+ }102+ if i, ok := st.lookIdx[l]; ok {103+ return i, true104+ }105+ if len(st.looks) > math.MaxUint16 {106+ return 0, false107+ }108+ if st.lookIdx == nil {109+ st.lookIdx = map[look]uint16{}110+ st.looks = []look{{}}111+ }112+ i := uint16(len(st.looks))113+ st.looks = append(st.looks, l)114+ st.lookIdx[l] = i115+ return i, true116+}117+118+// lookOf is sl's format and style.119+func (st *cellStore) lookOf(sl slot) look {120+ if sl.look == 0 {121+ return look{}122+ }123+ return st.looks[sl.look]124+}125+126+// plainSlot is c as a plain slot, or false if c is rich. It adds the127+// strings it uses to the table.128+func (st *cellStore) plainSlot(c *Cell) (slot, bool) {129+ if c.Note != "" || c.derived || c.spilled || !c.auto.IsZero() || strings.HasPrefix(c.Input, "=") {130+ return slot{}, false131+ }132+ lk, ok := st.lookID(c.Format, c.Style)133+ if !ok {134+ return slot{}, false135+ }136+ sl := slot{look: lk}137+ switch e := c.expr.(type) {138+ case nil:139+ switch {140+ case c.Input == "":141+ sl.kind = slotBlank142+ case c.Format.Kind != FmtText && strings.HasPrefix(c.Input, "'"):143+ return slot{}, false // its value isn't its input144+ default:145+ sl.kind, sl.ref = slotText, st.strs.add(c.Input)146+ }147+ case formula.Num:148+ if math.IsNaN(e.V) || math.IsInf(e.V, 0) {149+ return slot{}, false // its value is #NUM!150+ }151+ sl.kind, sl.num = slotNum, e.V152+ if d, ok := numDec(e.V, c.Input); ok {153+ sl.dec = d154+ } else {155+ sl.ref = st.strs.add(c.Input)156+ }157+ case formula.Bool:158+ sl.kind = slotBool159+ if e.V {160+ sl.num = 1161+ }162+ if c.Input != boolText(e.V) {163+ sl.ref = st.strs.add(c.Input)164+ }165+ default:166+ return slot{}, false167+ }168+ return sl, true169+}170+171+// numDec is how input prints v, as slot.dec, or false if it isn't v172+// printed plainly.173+func numDec(v float64, input string) (uint8, bool) {174+ var buf [64]byte175+ if string(strconv.AppendFloat(buf[:0], v, 'f', -1, 64)) == input {176+ return 0, true177+ }178+ i := strings.IndexByte(input, '.')179+ if i < 0 || len(input)-i-1 >= maxDec {180+ return 0, false181+ }182+ d := len(input) - i - 1183+ if string(strconv.AppendFloat(buf[:0], v, 'f', d, 64)) == input {184+ return uint8(d + 1), true185+ }186+ return 0, false187+}188+189+func boolText(b bool) string {190+ if b {191+ return "TRUE"192+ }193+ return "FALSE"194+}195+196+// input is the entry of a plain slot.197+func (st *cellStore) input(sl slot) string {198+ switch {199+ case sl.kind == slotBlank:200+ return ""201+ case sl.ref != 0:202+ return st.strs.strs[sl.ref]203+ case sl.kind == slotBool:204+ return boolText(sl.num != 0)205+ }206+ return strconv.FormatFloat(sl.num, 'f', int(sl.dec)-1, 64)207+}208+209+// slotValue is the value of a plain slot.210+func (st *cellStore) slotValue(sl slot) Value {211+ switch sl.kind {212+ case slotNum:213+ return Value{Kind: Number, Num: sl.num}214+ case slotBool:215+ return Value{Kind: Bool, Num: sl.num}216+ case slotText:217+ return Value{Kind: Text, Str: st.strs.strs[sl.ref]}218+ }219+ return Value{}220+}221+222+// view is a plain slot as a Cell of its own: changing it changes nothing223+// stored.224+func (st *cellStore) view(sl slot) *Cell {225+ lk := st.lookOf(sl)226+ c := &Cell{Input: st.input(sl), Value: st.slotValue(sl), Format: lk.f, Style: lk.st}227+ switch sl.kind {228+ case slotNum:229+ c.expr = formula.Num{V: sl.num}230+ case slotBool:231+ c.expr = formula.Bool{V: sl.num != 0}232+ }233+ return c234+}235+236+// releaseSlot counts one fewer use of what sl refers to.237+func (st *cellStore) releaseSlot(sl slot) {238+ switch sl.kind {239+ case slotRich:240+ st.rich[sl.ref] = richCell{}241+ st.richFree = append(st.richFree, sl.ref)242+ case slotNum, slotBool, slotText:243+ st.strs.release(sl.ref)244+ }245+}246+247+// Estimated heap of the store's parts, for the undo history's budget248+// (historysize.go): a slot with its share of the blocks and indexes, as249+// BenchmarkMemory measures it, and a string's entry in the table.250+const (251+ slotBytes = 20252+ strBytes = 64253+)254+255+// size estimates the heap the cells hold: a slot each, and the rich256+// cells and strings on top.257+func (st *cellStore) size() int64 {258+ n := int64(st.len()) * slotBytes259+ for _, rc := range st.rich {260+ n += cellSize(rc.c)261+ }262+ for _, s := range st.strs.strs {263+ n += strBytes + int64(len(s))264+ }265+ return n266+}267+268+// richCell is a rich cell and where it is.269+type richCell struct {270+ c *Cell271+ a Addr272+}273+274+// addRich stores c, at a, in the side table and returns its entry.275+func (st *cellStore) addRich(a Addr, c *Cell) uint32 {276+ if n := len(st.richFree); n > 0 {277+ i := st.richFree[n-1]278+ st.richFree = st.richFree[:n-1]279+ st.rich[i] = richCell{c, a}280+ return i281+ }282+ st.rich = append(st.rich, richCell{c, a})283+ return uint32(len(st.rich) - 1)284+}@@ -0,0 +1,144 @@
1+package sheet2+3+import (4+ "reflect"5+ "strings"6+ "testing"7+8+ "github.com/FelineStateMachine/012/internal/formula"9+)10+11+// literalValue is what recalculation computes for an entry that isn't a12+// formula (evaluate.go).13+func literalValue(c *Cell) Value {14+ switch e := c.expr.(type) {15+ case formula.Num:16+ return num(e.V)17+ case formula.Bool:18+ return boolean(e.V)19+ }20+ switch {21+ case c.Input == "":22+ return Value{}23+ case c.Format.Kind == FmtText:24+ return Value{Kind: Text, Str: c.Input}25+ }26+ return Value{Kind: Text, Str: strings.TrimPrefix(c.Input, "'")}27+}28+29+// roundTripCells are cells of every kind: entries of each sort under30+// formats, styles and notes, with the values recalculation gives them.31+func roundTripCells() []*Cell {32+ inputs := []string{33+ "1", "1.50", "-0", "0.1", "1e5", "007", "1.0000000000000000001", "123456789012345678901234567890",34+ "$1,200", "12%", "9/26/2026", "2026-09-26", "13:45", "TRUE", "true", "FALSE", "hello", "'123", "'",35+ "=A1", "=5", "=TRUE", "+5", "-5", "+A1", " 7 ", "ünïcode", "#N/A", "1e400", "",36+ }37+ formats := []Format{{}, Preset(FmtCurrency), {Kind: FmtText}, {Kind: FmtDate, Pattern: "yyyy-mm-dd"}}38+ styles := []Style{{}, {Bold: true}, {Align: AlignRight, own: true}}39+ var out []*Cell40+ for i := range len(inputs) * len(formats) * len(styles) * 4 {41+ input, f := inputs[i%len(inputs)], formats[i/len(inputs)%len(formats)]42+ sty, rest := styles[i/len(inputs)/len(formats)%len(styles)], i/len(inputs)/len(formats)/len(styles)43+ c, err := newCell(input, f, sty, rest&1 == 1)44+ if rest&2 == 2 {45+ c = c.withNote("a note")46+ }47+ if err != nil || c == nil {48+ continue49+ }50+ c.Value = literalValue(c)51+ if c.expr != nil && !c.literal() {52+ c.Value = Value{} // a formula: only recalculation sets it53+ }54+ out = append(out, c)55+ }56+ return out57+}58+59+// Every cell comes back from the store as it went in, whether it is kept60+// as a plain slot or whole: its entry exactly as typed, its value, format,61+// style, note and expression.62+func TestSlotRoundTrip(t *testing.T) {63+ st := newCellStore()64+ a := Addr{Col: 3, Row: 1500}65+ cells := roundTripCells()66+ for _, c := range cells {67+ want := c.clone()68+ st.set(a, c)69+ checkSame(t, st.get(a), want)70+ checkSame(t, st.copyOf(a), want)71+ if v := st.value(a); v != want.Value {72+ t.Errorf("%q: value %v, want %v", want.Input, v, want.Value)73+ }74+ if got := st.filledAt(a); got != !want.Blank() {75+ t.Errorf("%q: filled %v", want.Input, got)76+ }77+ if !st.holds(a, st.get(a)) {78+ t.Errorf("%q: doesn't hold its own view", want.Input)79+ }80+ }81+ st.delete(a)82+ if st.len() != 0 || len(st.strs.index) != 0 || len(st.rich) != len(st.richFree) {83+ t.Errorf("after %d cells: %d stored, %d strings, %d rich", len(cells), st.len(), len(st.strs.index), len(st.rich)-len(st.richFree))84+ }85+}86+87+// literal reports whether c's expression is a typed number or boolean,88+// which recalculation doesn't evaluate as a formula.89+func (c *Cell) literal() bool {90+ switch c.expr.(type) {91+ case formula.Num, formula.Bool:92+ return !strings.HasPrefix(c.Input, "=")93+ }94+ return false95+}96+97+func checkSame(t *testing.T, got, want *Cell) {98+ t.Helper()99+ if got.Input != want.Input || got.Value != want.Value || got.Format != want.Format || got.Style != want.Style ||100+ got.Note != want.Note || got.auto != want.auto || got.derived != want.derived || got.spilled != want.spilled ||101+ !reflect.DeepEqual(got.expr, want.expr) || got.IsFormula() != want.IsFormula() {102+ t.Errorf("stored %+v\ncame back %+v", want, got)103+ }104+}105+106+// Cells sharing a string keep it while any of them uses it, and a cell107+// turning from plain to rich and back keeps its place.108+func TestSlotStrings(t *testing.T) {109+ st := newCellStore()110+ a1, a2 := Addr{}, Addr{Row: 1}111+ st.set(a1, &Cell{Input: "same"})112+ st.set(a2, &Cell{Input: "same"})113+ if len(st.strs.index) != 1 {114+ t.Fatalf("%d strings for one text", len(st.strs.index))115+ }116+ st.delete(a1)117+ if c := st.get(a2); c.Input != "same" || c.Value.Str != "same" {118+ t.Errorf("the other cell is %+v", c)119+ }120+ rich := &Cell{Input: "same", Note: "n"}121+ st.set(a2, rich)122+ if st.get(a2) != rich || st.richAt(a2) != rich || len(st.strs.index) != 0 {123+ t.Errorf("rich cell not kept whole, %d strings", len(st.strs.index))124+ }125+ num, _ := newCell("12.50", Format{}, Style{}, true)126+ st.set(a2, num)127+ if c := st.get(a2); c.Input != "12.50" || c.Value.Num != 12.5 || st.richAt(a2) != nil {128+ t.Errorf("back to plain: %+v", c)129+ }130+ if len(st.rich) != len(st.richFree) {131+ t.Errorf("%d rich cells left", len(st.rich)-len(st.richFree))132+ }133+}134+135+// A plain number costs its slot: no string, no Cell.136+func TestSlotCompact(t *testing.T) {137+ s := New()138+ for row := range 2048 {139+ s.Load(Addr{Row: row}, []string{"1", "2.5", "3.10", "-4"}[row%4], Format{}, Style{})140+ }141+ if n := len(s.cells.rich) + len(s.cells.strs.strs); n != 0 {142+ t.Errorf("%d rich cells and strings for plain numbers", n)143+ }144+}@@ -12,14 +12,12 @@ type Stats struct {
1212 Count, Nums int1313 }1414 15-func (st *Stats) add(c *Cell) {16- if c.Blank() {17- return18- }15+// add counts a cell with contents whose value is v.16+func (st *Stats) add(v Value) {1917 st.Count++20- if c.Value.Kind == Number {18+ if v.Kind == Number {2119 st.Nums++22- st.Sum += c.Value.Num20+ st.Sum += v.Num2321 }2422 }2523 @@ -43,8 +41,10 @@ func (s *Sheet) RangeStats(r Rect) Stats {
4341 var st Stats4442 area := (r.To.Col - r.From.Col + 1) * (r.To.Row - r.From.Row + 1)4543 if area <= statsDirect || s.cells.len() <= statsSparse {46- for _, c := range s.cells.anyInRange(r) {47- st.add(c)44+ for a := range s.cells.anyKeysIn(r) {45+ if s.cells.filledAt(a) {46+ st.add(s.cells.value(a))47+ }4848 }4949 } else {5050 st = s.cells.rangeStats(r)@@ -135,9 +135,7 @@ func (st *cellStore) blockTotal(col, id int, b *rowBlock) wordStats {
135135 func (st *cellStore) scanWord(col, id int, b *rowBlock, w int, mask uint64, s *Stats) {136136 base := id<<blockShift + w<<6137137 for x := b.bits[w] & mask; x != 0; x &= x - 1 {138- if c := st.m[Addr{Col: col, Row: base + bits.TrailingZeros64(x)}]; c != nil {139- s.add(c)140- }138+ s.add(st.value(Addr{Col: col, Row: base + bits.TrailingZeros64(x)}))141139 }142140 }143141 @@ -11,8 +11,8 @@ func bruteStats(s *Sheet, r Rect) Stats {
1111 var st Stats1212 for row := r.From.Row; row <= r.To.Row; row++ {1313 for col := r.From.Col; col <= r.To.Col; col++ {14- if c := s.Cell(Addr{Col: col, Row: row}); c != nil {15- st.add(c)14+ if c := s.Cell(Addr{Col: col, Row: row}); !c.Blank() {15+ st.add(c.Value)1616 }1717 }1818 }@@ -95,8 +95,8 @@ func TestRangeStatsFullGrid(t *testing.T) {
9595 brute := func(r Rect) Stats {9696 var st Stats9797 for a, c := range s.cells.all() {98- if r.Contains(a) {99- st.add(c)98+ if r.Contains(a) && !c.Blank() {99+ st.add(c.Value)100100 }101101 }102102 return st@@ -2,71 +2,285 @@ package sheet
22 33 import (44 "iter"5- "maps"5+ "math"66 "math/bits"77 "slices"8+ "strconv"9+10+ "github.com/FelineStateMachine/012/internal/formula"811 )912 1013 // cellStore holds a sheet's cells: those with contents or formatting, by1114 // address. The rest of the engine reads and writes cells only through12-// these methods, never the map beneath, so the representation can change13-// (to column blocks of compact values, with formulas and formats in side14-// tables; see docs/contributing/limits.md) without touching it.15+// these methods, never the representation beneath.16+//17+// Cells live in columns of blocks of blockRows rows: the stored18+// occupancy index (occupancy.go) has a bitmap per block of the rows19+// holding a cell, and each block keeps their slots (slot.go) in row20+// order, so a cell is found by indexing a column, a block and a popcount,21+// without hashing. A plain number costs its 16-byte slot. Rich cells22+// (formulas, notes, pivots' and spills' results) are whole Cells in a23+// side table the slot points at.24+//25+// get returns a rich cell's own Cell, which recalculation updates in26+// place, as ever, and a plain cell as a Cell made for the caller (a27+// view): the caller may keep it, but changing it changes nothing28+// stored; set is the one way to change a cell. Views cost an allocation29+// or three, so what reads many cells asks for what it needs: peek and30+// value for values, has and filledAt for presence, richAt for formulas,31+// look for formatting, and the key iterators for addresses.1532 //16-// Next to the map, two occupancy indexes (occupancy.go) record which17-// cells are stored and which have contents, so a range is read at the18-// cost of what it holds, and the used range and data edges are found19-// without scanning.33+// A second occupancy index records which cells have contents, so the34+// used range and data edges are found without scanning.2035 //21-// Iteration order of all is unspecified; inRange yields in row-major22-// order. Cells may be set or deleted while iterating, with a map's rules:23-// a deleted cell not yet reached isn't yielded, a new one may or may not24-// be.36+// Iteration yields column by column, except inRange and keysIn, which37+// yield in row-major order. Cells may be set or deleted while iterating,38+// with a map's rules: a deleted cell not yet reached isn't yielded, a new39+// one may or may not be.2540 type cellStore struct {26- m map[Addr]*Cell27- stored occupancy // every cell in m41+ stored occupancy // every cell, with the blocks holding their slots2842 filled occupancy // the cells with contents (not Blank), with their statistics; see stats.go2943 statsUsed bool // a selection has been summed from the statistics index44+45+ strs strTable // plain cells' text46+ looks []look // formats and styles, by slot.look47+ lookIdx map[look]uint16 // looks' entries48+ rich []richCell // rich cells, by slot.ref49+ richFree []uint32 // entries of rich not in use50+}51+52+func newCellStore() cellStore { return cellStore{} }53+54+// find returns the block holding a's slot and its position there, or nil.55+func (st *cellStore) find(a Addr) (*rowBlock, int) {56+ b := st.stored.col(a.Col).block(a.Row >> blockShift)57+ if b == nil {58+ return nil, 059+ }60+ i, ok := b.index(a.Row & (blockRows - 1))61+ if !ok {62+ return nil, 063+ }64+ return b, i65+}66+67+// get returns the cell at a, or nil: a rich cell itself, a plain one as68+// a view.69+func (st *cellStore) get(a Addr) *Cell {70+ b, i := st.find(a)71+ if b == nil {72+ return nil73+ }74+ return st.cellOf(b.vals[i])75+}76+77+func (st *cellStore) cellOf(sl slot) *Cell {78+ if sl.kind == slotRich {79+ return st.rich[sl.ref].c80+ }81+ return st.view(sl)82+}83+84+// peek returns the rich cell at a, or else the plain cell's value, and85+// whether a holds a cell at all. It allocates nothing: recalculation86+// reads every cell through it.87+func (st *cellStore) peek(a Addr) (*Cell, Value, bool) {88+ b, i := st.find(a)89+ if b == nil {90+ return nil, Value{}, false91+ }92+ sl := b.vals[i]93+ if sl.kind == slotRich {94+ return st.rich[sl.ref].c, Value{}, true95+ }96+ return nil, st.slotValue(sl), true97+}98+99+// value returns the value of the cell at a.100+func (st *cellStore) value(a Addr) Value {101+ c, v, _ := st.peek(a)102+ if c != nil {103+ return c.Value104+ }105+ return v30106 }31107 32-func newCellStore() cellStore { return cellStore{m: make(map[Addr]*Cell)} }108+// has reports whether a holds a cell, if only formatting.109+func (st *cellStore) has(a Addr) bool { return st.stored.has(a) }33110 34-// get returns the cell at a, or nil.35-func (st *cellStore) get(a Addr) *Cell { return st.m[a] }111+// filledAt reports whether the cell at a has contents: !get(a).Blank().112+func (st *cellStore) filledAt(a Addr) bool { return st.filled.has(a) }36113 37-// set stores c, which isn't nil, at a.114+// richAt returns the cell at a if it is rich (a formula, a note, a115+// pivot's or a spill's result), or nil.116+func (st *cellStore) richAt(a Addr) *Cell {117+ b, i := st.find(a)118+ if b == nil || b.vals[i].kind != slotRich {119+ return nil120+ }121+ return st.rich[b.vals[i].ref].c122+}123+124+// look returns the format and style of the cell at a, and whether there125+// is one.126+func (st *cellStore) look(a Addr) (Format, Style, bool) {127+ b, i := st.find(a)128+ if b == nil {129+ return Format{}, Style{}, false130+ }131+ if sl := b.vals[i]; sl.kind == slotRich {132+ c := st.rich[sl.ref].c133+ return c.Format, c.Style, true134+ } else {135+ l := st.lookOf(sl)136+ return l.f, l.st, true137+ }138+}139+140+// holds reports whether the cell at a is c: the same Cell if rich, the141+// same contents and formatting if plain; nil for none.142+func (st *cellStore) holds(a Addr, c *Cell) bool {143+ b, i := st.find(a)144+ if b == nil || c == nil {145+ return b == nil && c == nil146+ }147+ sl := b.vals[i]148+ if sl.kind == slotRich {149+ return st.rich[sl.ref].c == c150+ }151+ if l := st.lookOf(sl); c.Note != "" || c.derived || c.spilled || !c.auto.IsZero() || c.Format != l.f || c.Style != l.st {152+ return false153+ }154+ switch sl.kind {155+ case slotText:156+ return c.expr == nil && c.Input == st.strs.strs[sl.ref]157+ case slotNum:158+ e, ok := c.expr.(formula.Num)159+ return ok && math.Float64bits(e.V) == math.Float64bits(sl.num) && st.inputIs(sl, c.Input)160+ case slotBool:161+ e, ok := c.expr.(formula.Bool)162+ return ok && e.V == (sl.num != 0) && st.inputIs(sl, c.Input)163+ }164+ return c.expr == nil && c.Input == ""165+}166+167+// inputIs reports whether input is the entry of the plain number or168+// boolean sl, without making it.169+func (st *cellStore) inputIs(sl slot, input string) bool {170+ switch {171+ case sl.ref != 0:172+ return input == st.strs.strs[sl.ref]173+ case sl.kind == slotBool:174+ return input == boolText(sl.num != 0)175+ }176+ var buf [64]byte177+ return string(strconv.AppendFloat(buf[:0], sl.num, 'f', int(sl.dec)-1, 64)) == input178+}179+180+// copyOf is the cell at a as a Cell the caller owns, or nil.181+func (st *cellStore) copyOf(a Addr) *Cell {182+ b, i := st.find(a)183+ if b == nil {184+ return nil185+ }186+ if sl := b.vals[i]; sl.kind == slotRich {187+ return st.rich[sl.ref].c.clone()188+ } else {189+ return st.view(sl)190+ }191+}192+193+// set stores c, which isn't nil, at a. A rich c is kept as it is, so194+// recalculation can update it in place; a plain one is copied into a195+// slot, and its value is its entry's from then on.38196 func (st *cellStore) set(a Addr, c *Cell) {39- old, ok := st.m[a]40- if !ok {41- st.stored.mark(a)197+ b, i := st.find(a)198+ if b == nil {199+ b = st.stored.mark(a)200+ i, _ = b.index(a.Row & (blockRows - 1))201+ b.vals = slices.Insert(b.vals, i, slot{kind: slotBlank})202+ }203+ old := b.vals[i]204+ sl, ok := st.plainSlot(c) // before releasing old, which may share its strings205+ switch {206+ case ok:207+ st.releaseSlot(old)208+ case old.kind == slotRich:209+ st.rich[old.ref] = richCell{c, a}210+ sl = old211+ default:212+ st.releaseSlot(old)213+ sl = slot{kind: slotRich, ref: st.addRich(a, c)}42214 }43- switch was, is := ok && !old.Blank(), !c.Blank(); {215+ b.vals[i] = sl216+ st.filledAs(a, st.filled.has(a), !c.Blank())217+ st.touch(a)218+}219+220+// filledAs updates the index of filled cells for a cell at a that was221+// and is filled or not.222+func (st *cellStore) filledAs(a Addr, was, is bool) {223+ switch {44224 case is && !was:45225 st.filled.mark(a)46226 case was && !is:47227 st.filled.unmark(a)48228 }49- st.m[a] = c50- st.touch(a)51229 }52230 53231 // delete removes the cell at a, if any.54232 func (st *cellStore) delete(a Addr) {55- if old, ok := st.m[a]; ok {56- st.touch(a)57- st.stored.unmark(a)58- if !old.Blank() {59- st.filled.unmark(a)60- }61- delete(st.m, a)233+ b, i := st.find(a)234+ if b == nil {235+ return62236 }237+ st.touch(a)238+ old := b.vals[i]239+ if st.filled.has(a) {240+ st.filled.unmark(a)241+ }242+ b.vals = slices.Delete(b.vals, i, i+1)243+ st.stored.unmark(a)244+ st.releaseSlot(old)63245 }64246 65247 // len is the number of cells stored, formatting-only ones included.66-func (st *cellStore) len() int { return len(st.m) }248+func (st *cellStore) len() int { return st.stored.n }249+250+// filledLen is the number of cells with contents.251+func (st *cellStore) filledLen() int { return st.filled.n }252+253+// all yields every stored cell, plain ones as views.254+func (st *cellStore) all() iter.Seq2[Addr, *Cell] { return st.cellsOf(st.keys()) }255+256+// keys yields the address of every stored cell, column by column.257+func (st *cellStore) keys() iter.Seq[Addr] {258+ return st.anyKeysIn(Rect{To: Addr{Col: MaxCols - 1, Row: MaxRows - 1}})259+}67260 68-// all yields every stored cell.69-func (st *cellStore) all() iter.Seq2[Addr, *Cell] { return maps.All(st.m) }261+// richCells yields every rich cell: the formulas, notes and results of262+// pivots and spills.263+func (st *cellStore) richCells() iter.Seq2[Addr, *Cell] {264+ return func(yield func(Addr, *Cell) bool) {265+ for i := 0; i < len(st.rich); i++ {266+ if rc := st.rich[i]; rc.c != nil && !yield(rc.a, rc.c) {267+ return268+ }269+ }270+ }271+}272+273+// cellsOf yields the cells at the addresses of keys, skipping any no274+// longer stored.275+func (st *cellStore) cellsOf(keys iter.Seq[Addr]) iter.Seq2[Addr, *Cell] {276+ return func(yield func(Addr, *Cell) bool) {277+ for a := range keys {278+ if c := st.get(a); c != nil && !yield(a, c) {279+ return280+ }281+ }282+ }283+}70284 71285 // bounds returns the smallest range holding every stored cell of r, and72286 // false if r holds none.@@ -86,8 +300,24 @@ func (st *cellStore) colFill(c, r0, r1 int, dst []Addr) int {
86300 87301 // inRange yields the stored cells in r in row-major order, visiting only88302 // the rows of each column that hold cells.89-func (st *cellStore) inRange(r Rect) iter.Seq2[Addr, *Cell] {90- return func(yield func(Addr, *Cell) bool) {303+func (st *cellStore) inRange(r Rect) iter.Seq2[Addr, *Cell] { return st.cellsOf(st.keysIn(r)) }304+305+// valuesIn yields the values of the stored cells in r in row-major306+// order, without making views of them.307+func (st *cellStore) valuesIn(r Rect) iter.Seq2[Addr, Value] {308+ return func(yield func(Addr, Value) bool) {309+ for a := range st.keysIn(r) {310+ if !yield(a, st.value(a)) {311+ return312+ }313+ }314+ }315+}316+317+// keysIn yields the addresses of the stored cells in r in row-major318+// order.319+func (st *cellStore) keysIn(r Rect) iter.Seq[Addr] {320+ return func(yield func(Addr) bool) {91321 cols := slices.Clone(st.stored.colsIn(r.From.Col, r.To.Col))92322 if len(cols) == 0 || r.From.Row > r.To.Row {93323 return@@ -96,8 +326,8 @@ func (st *cellStore) inRange(r Rect) iter.Seq2[Addr, *Cell] {
96326 for _, id := range st.stored.blockIDs(cols, b0, b1) {97327 var in []colBlock98328 for _, c := range cols {99- if b := st.stored.cols[c].block(id); b != nil {100- in = append(in, colBlock{c, b})329+ if st.stored.col(c).block(id) != nil {330+ in = append(in, colBlock{col: c})101331 }102332 }103333 if !st.yieldBlock(in, id, r, yield) {@@ -112,7 +342,7 @@ func (st *cellStore) inRange(r Rect) iter.Seq2[Addr, *Cell] {
112342 func (st *cellStore) rangeFill(r Rect, from Addr, dst []Addr) int {113343 n := 0114344 if from.Col > r.From.Col { // the rest of a row first115- for a := range st.inRange(Rect{From: from, To: Addr{Col: r.To.Col, Row: from.Row}}) {345+ for a := range st.keysIn(Rect{From: from, To: Addr{Col: r.To.Col, Row: from.Row}}) {116346 dst[n] = a117347 if n++; n == len(dst) {118348 return n@@ -120,7 +350,7 @@ func (st *cellStore) rangeFill(r Rect, from Addr, dst []Addr) int {
120350 }121351 from = Addr{Col: r.From.Col, Row: from.Row + 1}122352 }123- for a := range st.inRange(Rect{From: from, To: r.To}) {353+ for a := range st.keysIn(Rect{From: from, To: r.To}) {124354 dst[n] = a125355 if n++; n == len(dst) {126356 break@@ -129,19 +359,27 @@ func (st *cellStore) rangeFill(r Rect, from Addr, dst []Addr) int {
129359 return n130360 }131361 132-// anyInRange yields the stored cells in r in no particular order,133-// visiting whichever is cheaper: the index, row by row, or every stored134-// cell (iterating the map costs a quarter of a lookup per cell) when r135-// is more than a quarter as large as the sheet's cells are many.362+// anyInRange yields the stored cells in r in no particular order.136363 func (st *cellStore) anyInRange(r Rect) iter.Seq2[Addr, *Cell] {137- area := (r.To.Col - r.From.Col + 1) * (r.To.Row - r.From.Row + 1)138- if area <= len(st.m)/4 {139- return st.inRange(r)140- }141- return func(yield func(Addr, *Cell) bool) {142- for a, c := range st.m {143- if r.Contains(a) && !yield(a, c) {144- return364+ return st.cellsOf(st.anyKeysIn(r))365+}366+367+// anyKeysIn yields the addresses of the stored cells in r column by368+// column, which walks each block's bitmap once.369+func (st *cellStore) anyKeysIn(r Rect) iter.Seq[Addr] {370+ return func(yield func(Addr) bool) {371+ if r.From.Row > r.To.Row {372+ return373+ }374+ b0, b1 := r.From.Row>>blockShift, r.To.Row>>blockShift375+ for _, c := range slices.Clone(st.stored.colsIn(r.From.Col, r.To.Col)) {376+ if st.stored.col(c) == nil {377+ continue378+ }379+ for _, id := range st.stored.blockIDs([]int{c}, b0, b1) {380+ if !st.yieldBlock([]colBlock{{col: c}}, id, r, yield) {381+ return382+ }145383 }146384 }147385 }@@ -152,15 +390,20 @@ type colBlock struct {
152390 b *rowBlock153391 }154392 155-// yieldBlock yields the cells of block id in the columns in, row by row,156-// limited to the rows of r.157-func (st *cellStore) yieldBlock(in []colBlock, id int, r Rect, yield func(Addr, *Cell) bool) bool {393+// yieldBlock yields the addresses of the cells of block id in the394+// columns in, row by row, limited to the rows of r. Each word is read395+// from the blocks as they are then, and each address checked before it396+// is yielded, so cells may be set and deleted meanwhile.397+func (st *cellStore) yieldBlock(in []colBlock, id int, r Rect, yield func(Addr) bool) bool {158398 base := id << blockShift159399 lo, hi := max(r.From.Row-base, 0), min(r.To.Row-base, blockRows-1)160400 for w := lo >> 6; w <= hi>>6; w++ {161401 var occ uint64162- for _, cb := range in {163- occ |= cb.b.bits[w]402+ for i := range in {403+ in[i].b = st.stored.col(in[i].col).block(id)404+ if in[i].b != nil {405+ occ |= in[i].b.bits[w]406+ }164407 }165408 occ &= wordMask(w, lo, hi)166409 for occ != 0 {@@ -168,11 +411,10 @@ func (st *cellStore) yieldBlock(in []colBlock, id int, r Rect, yield func(Addr,
168411 occ &^= 1 << bit169412 row := base + w<<6 + bit170413 for _, cb := range in {171- if cb.b.bits[w]&(1<<bit) == 0 {414+ if cb.b == nil || cb.b.bits[w]&(1<<bit) == 0 {172415 continue173416 }174- a := Addr{Col: cb.col, Row: row}175- if c := st.m[a]; c != nil && !yield(a, c) {417+ if a := (Addr{Col: cb.col, Row: row}); st.stored.has(a) && !yield(a) {176418 return false177419 }178420 }@@ -6,53 +6,67 @@ import (
66 "testing"77 )88 9-// The occupancy index agrees with the map: inRange yields exactly the10-// stored cells of a range in row-major order, bounds is their bounding11-// box and nextRow finds the neighbours, through random sets and deletes9+// The store agrees with a map of what was stored: get returns each10+// cell's entry, inRange yields exactly the stored cells of a range in11+// row-major order, bounds is their bounding box and nextRow finds the12+// neighbours, through random sets and deletes of plain and rich cells1213 // that cross block boundaries.1314 func TestCellStoreIndex(t *testing.T) {1415 rng := rand.New(rand.NewPCG(1, 2))1516 st := newCellStore()17+ model := map[Addr]string{}1618 rows := []int{0, 1, 63, 64, 1023, 1024, 1025, 5000, 70000, MaxRows - 1}19+ inputs := []string{"x", "", "5", "2.50", "=A1", "TRUE"} // "" is formatting only1720 for i := range 3000 {1821 a := Addr{Col: rng.IntN(40), Row: rows[rng.IntN(len(rows))] + rng.IntN(3)}1922 a.Row = min(a.Row, MaxRows-1)2023 if rng.IntN(3) == 0 {2124 st.delete(a)25+ delete(model, a)2226 } else {23- input := "x"24- if rng.IntN(4) == 0 {25- input = "" // formatting only27+ input := inputs[rng.IntN(len(inputs))]28+ c, err := newCell(input, Format{}, Style{Bold: input == ""}, true)29+ if err != nil {30+ t.Fatal(err)2631 }27- st.set(a, &Cell{Input: input})32+ st.set(a, c)33+ model[a] = input2834 }2935 if i%100 == 0 {30- checkStore(t, &st, rng)36+ checkStore(t, &st, model, rng)3137 }3238 }3339 for a := range st.all() {3440 st.delete(a)3541 }36- if slices.ContainsFunc(st.stored.cols, func(c *colIndex) bool { return c != nil }) || len(st.stored.colIDs) != 0 || len(st.filled.colIDs) != 0 {42+ if st.len() != 0 || slices.ContainsFunc(st.stored.cols, func(c *colIndex) bool { return c != nil }) || len(st.stored.colIDs) != 0 || len(st.filled.colIDs) != 0 {3743 t.Errorf("empty store keeps %d columns", len(st.stored.colIDs))3844 }3945 }4046 41-func checkStore(t *testing.T, st *cellStore, rng *rand.Rand) {47+func checkStore(t *testing.T, st *cellStore, model map[Addr]string, rng *rand.Rand) {4248 t.Helper()49+ if st.len() != len(model) {50+ t.Fatalf("%d cells stored, want %d", st.len(), len(model))51+ }52+ for a, input := range model {53+ if c := st.get(a); c == nil || c.Input != input {54+ t.Fatalf("get(%v) = %+v, want %q", a, c, input)55+ }56+ }4357 for range 20 {4458 r := NewRect(Addr{Col: rng.IntN(45), Row: rng.IntN(80000)}, Addr{Col: rng.IntN(45), Row: rng.IntN(MaxRows)})45- checkRange(t, st, r)46- checkFilled(t, st, r)47- checkNextRow(t, st, rng.IntN(40), rng.IntN(MaxRows))59+ checkRange(t, st, model, r)60+ checkFilled(t, st, model, r)61+ checkNextRow(t, st, model, rng.IntN(40), rng.IntN(MaxRows))4862 }4963 }5064 51-// checkRange compares inRange and bounds with the map.52-func checkRange(t *testing.T, st *cellStore, r Rect) {65+// checkRange compares inRange and bounds with the model.66+func checkRange(t *testing.T, st *cellStore, model map[Addr]string, r Rect) {5367 t.Helper()5468 var want []Addr55- for a := range st.all() {69+ for a := range model {5670 if r.Contains(a) {5771 want = append(want, a)5872 }@@ -65,6 +79,14 @@ func checkRange(t *testing.T, st *cellStore, r Rect) {
6579 if !slices.Equal(got, want) {6680 t.Fatalf("inRange(%v): %d cells, want %d", r, len(got), len(want))6781 }82+ got = got[:0]83+ for a := range st.anyKeysIn(r) {84+ got = append(got, a)85+ }86+ sortAddrs(got)87+ if !slices.Equal(got, want) {88+ t.Fatalf("anyKeysIn(%v): %d cells, want %d", r, len(got), len(want))89+ }6890 b, ok := st.bounds(r)6991 if ok != (len(want) > 0) {7092 t.Fatalf("bounds(%v) found %v with %d cells", r, ok, len(want))@@ -80,15 +102,15 @@ func checkRange(t *testing.T, st *cellStore, r Rect) {
80102 }81103 }82104 83-// checkFilled compares the index of cells with contents with the map.84-func checkFilled(t *testing.T, st *cellStore, r Rect) {105+// checkFilled compares the index of cells with contents with the model.106+func checkFilled(t *testing.T, st *cellStore, model map[Addr]string, r Rect) {85107 t.Helper()86108 var filled []Addr87- for a, c := range st.all() {88- if r.Contains(a) && !c.Blank() {109+ for a, input := range model {110+ if r.Contains(a) && input != "" {89111 filled = append(filled, a)90112 }91- if st.filled.has(a) == c.Blank() {113+ if st.filled.has(a) != (input != "") {92114 t.Fatalf("filled index says %v for %v", st.filled.has(a), a)93115 }94116 }@@ -98,14 +120,14 @@ func checkFilled(t *testing.T, st *cellStore, r Rect) {
98120 }99121 }100122 101-// checkNextRow compares nextRow both ways with the map.102-func checkNextRow(t *testing.T, st *cellStore, c, row int) {123+// checkNextRow compares nextRow both ways with the model.124+func checkNextRow(t *testing.T, st *cellStore, model map[Addr]string, c, row int) {103125 t.Helper()104126 next, ok := st.stored.nextRow(c, row, 1)105127 prev, okp := st.stored.nextRow(c, row, -1)106128 want1, ok1 := MaxRows, false107129 want2, ok2 := -1, false108- for a := range st.all() {130+ for a := range model {109131 if a.Col != c {110132 continue111133 }@@ -120,3 +142,34 @@ func checkNextRow(t *testing.T, st *cellStore, c, row int) {
120142 t.Fatalf("nextRow(%d, %d) = %d %v / %d %v, want %d %v / %d %v", c, row, next, ok, prev, okp, want1, ok1, want2, ok2)121143 }122144 }145+146+// Cells set and deleted while iterating follow a map's rules: a cell147+// deleted before it is reached isn't yielded, and every cell stored148+// throughout is yielded once.149+func TestCellStoreChangeWhileIterating(t *testing.T) {150+ st := newCellStore()151+ for row := range 3000 {152+ for col := range 3 {153+ st.set(Addr{Col: col, Row: row}, &Cell{Input: "x"})154+ }155+ }156+ seen := map[Addr]bool{}157+ for a := range st.keysIn(NewRect(Addr{}, Addr{Col: 2, Row: 2999})) {158+ if seen[a] {159+ t.Fatalf("%v yielded twice", a)160+ }161+ seen[a] = true162+ if a.Col == 0 {163+ st.delete(Addr{Col: 2, Row: a.Row}) // not yet reached164+ st.delete(Addr{Col: 1, Row: a.Row + 1}) // nor this165+ }166+ }167+ for a := range seen {168+ if !st.has(a) {169+ t.Fatalf("deleted %v was yielded", a)170+ }171+ }172+ if len(seen) != st.len() {173+ t.Errorf("%d yielded, %d stored", len(seen), st.len())174+ }175+}@@ -9,6 +9,7 @@ import (
99 "bytes"1010 "fmt"1111 "runtime"12+ "sync"1213 "testing"1314 1415 "github.com/FelineStateMachine/012/internal/sheet"@@ -243,17 +244,34 @@ func saveShapes() []namedBuild {
243244 {"dense-8192x26", func() *sheet.Sheet { return stress.Dense(stress.Rows, 26) }},244245 {"dense-8192x256", func() *sheet.Sheet { return stress.Dense(stress.Rows, stress.Cols) }},245246 {"chain-8192", func() *sheet.Sheet { return stress.Chain(stress.Rows) }},247+ {"dense-1Mx10", func() *sheet.Sheet { return stress.Dense(stress.MaxRows, stress.MaxCols) }},246248 }247249 }248250 249-// BenchmarkMemory reports the live heap per non-blank cell after a load.251+// BenchmarkMemory reports the live heap per non-blank cell after a load:252+// built through Load, or opened from a .012 file written beforehand.250253 func BenchmarkMemory(b *testing.B) {251- for _, sh := range []namedBuild{252- {"numbers", func() *sheet.Sheet { return stress.Dense(stress.Rows, 64) }},253- {"formulas", func() *sheet.Sheet { return stress.Chain(stress.Rows) }},254- {"text-200", func() *sheet.Sheet { return stress.LongText(stress.Rows, 200) }},254+ for _, sh := range []struct {255+ name string256+ prep func()257+ build func() *sheet.Sheet258+ }{259+ {"numbers", nil, func() *sheet.Sheet { return stress.Dense(stress.Rows, 64) }},260+ {"numbers-1Mx10", nil, func() *sheet.Sheet { return stress.Dense(stress.MaxRows, stress.MaxCols) }},261+ {"formulas", nil, func() *sheet.Sheet { return stress.Chain(stress.Rows) }},262+ {"text-200", nil, func() *sheet.Sheet { return stress.LongText(stress.Rows, 200) }},263+ {"open-8192x256", func() { denseFile() }, func() *sheet.Sheet {264+ s, err := sheet.Read(bytes.NewReader(denseFile()))265+ if err != nil {266+ b.Fatal(err)267+ }268+ return s269+ }},255270 } {256271 b.Run(sh.name, func(b *testing.B) {272+ if sh.prep != nil {273+ sh.prep()274+ }257275 for b.Loop() {258276 before := heap()259277 s := sh.build()@@ -265,6 +283,46 @@ func BenchmarkMemory(b *testing.B) {
265283 }266284 }267285 286+// denseFile is the .012 file of 8192x256 numbers.287+var denseFile = sync.OnceValue(func() []byte {288+ var buf bytes.Buffer289+ if err := stress.Dense(stress.Rows, stress.Cols).Write(&buf); err != nil {290+ panic(err)291+ }292+ return buf.Bytes()293+})294+295+// BenchmarkRead is the cost of reading a cell's value, per cell of 8192296+// x 26 numbers: through Sheet.Value, and as formulas read cells in a297+// recalculation (a SUM of each column, read directly).298+func BenchmarkRead(b *testing.B) {299+ s := stress.Dense(stress.Rows, 26)300+ cells := float64(stress.Rows * 26)301+ b.Run("value", func(b *testing.B) {302+ sum := 0.0303+ for b.Loop() {304+ for row := range stress.Rows {305+ for col := range 26 {306+ sum += s.Value(sheet.Addr{Col: col, Row: row}).Num307+ }308+ }309+ }310+ b.ReportMetric(float64(b.Elapsed().Nanoseconds())/float64(b.N)/cells, "ns/cell")311+ })312+ b.Run("recalc", func(b *testing.B) {313+ for col := range 26 {314+ name := sheet.ColName(col)315+ if err := s.Set(sheet.Addr{Col: col, Row: stress.Rows}, fmt.Sprintf("=SUM(%s1:%s%d)", name, name, stress.Rows)); err != nil {316+ b.Fatal(err)317+ }318+ }319+ for b.Loop() {320+ s.RecalcAll()321+ }322+ b.ReportMetric(float64(b.Elapsed().Nanoseconds())/float64(b.N)/cells, "ns/cell")323+ })324+}325+268326 // heap is the live heap after a full collection.269327 func heap() uint64 {270328 runtime.GC()@@ -407,7 +407,7 @@ func (s *Sheet) sourceItems(v Validation) []string {
407407 }408408 seen := map[string]bool{}409409 var out []string410- for a := range t.cells.inRange(r) {410+ for a := range t.cells.keysIn(r) {411411 text := strings.TrimSpace(t.ShownText(a))412412 key := strings.ToLower(text)413413 if text == "" || seen[key] {@@ -347,7 +347,7 @@ func (w *Workbook) attach(s *Sheet) {
347347 }348348 s.live = true349349 w.byKey[formula.SheetKey(s.name)] = s350- for a, c := range s.cells.all() {350+ for a, c := range s.cells.richCells() {351351 w.index(loc{s, a}, c)352352 }353353 w.structural = true@@ -363,7 +363,7 @@ func (w *Workbook) detach(s *Sheet) {
363363 if w.byKey[formula.SheetKey(s.name)] == s {364364 delete(w.byKey, formula.SheetKey(s.name))365365 }366- for a, c := range s.cells.all() {366+ for a, c := range s.cells.richCells() {367367 w.unindex(loc{s, a}, c)368368 }369369 w.structural = true@@ -30,6 +30,13 @@ const (
3030 Cols = 2563131 )3232 33+// MaxRows x MaxCols is sheet.DefaultMaxCells of numbers, the most an34+// import keeps by default: a tall table, as imports make.35+const (36+ MaxRows = 1_000_00037+ MaxCols = 1038+)39+3340 func at(col, row int) sheet.Addr { return sheet.Addr{Col: col, Row: row} }3441 3542 func load(s *sheet.Sheet, a sheet.Addr, input string) {@@ -333,5 +340,6 @@ func Shapes() []Shape {
333340 {"criteria-60xSUMIF8192", func() *sheet.Sheet { return Criteria(Rows, 60) }, at(0, 4000), "7"},334341 {"lookup-300xVLOOKUP8192", func() *sheet.Sheet { return Lookup(Rows, 300) }, at(0, 4000), "4000"},335342 {"arrays-1000xFILTER8192", func() *sheet.Sheet { return Arrays(Rows, 1000) }, at(0, 4000), "7"},343+ {"dense-1Mx10", func() *sheet.Sheet { return Dense(MaxRows, MaxCols) }, at(0, 500000), "5"},336344 }337345 }@@ -71,9 +71,7 @@ func (m *Model) autofit(c int) {
7171 }7272 widest := map[int]int{}7373 for _, a := range m.sheet.Addrs() {74- if cell := m.sheet.Cell(a); cell != nil {75- widest[a.Col] = max(widest[a.Col], ansi.StringWidth(cell.Value.String()))76- }74+ widest[a.Col] = max(widest[a.Col], ansi.StringWidth(m.sheet.Value(a).String()))7775 }7876 for _, k := range cols {7977 // One column of padding each side, and never narrower than the@@ -52,8 +52,8 @@ func Layout(s *sheet.Sheet, row, lo, ncols, minCol, maxCol int) []Span {
5252 l.x[c-l.first+1] = l.x[c-l.first] + s.ColWidth(c)5353 }5454 for c := l.first; c <= l.last; c++ {55- if cell := l.content(c); cell != nil {56- l.place(c, cell)55+ if v, ok := l.content(c); ok {56+ l.place(c, v)5757 }5858 }5959 keepGaps(l.out)@@ -71,14 +71,18 @@ type rowLayout struct {
7171 out []Span7272 }7373 74-// content is the cell in column c of the row, or nil if it's blank.75-func (l *rowLayout) content(c int) *sheet.Cell {76- if cell := l.s.Cell(sheet.Addr{Col: c, Row: l.row}); !cell.Blank() {77- return cell74+// content is the value in column c of the row, and false if the cell is75+// blank.76+func (l *rowLayout) content(c int) (sheet.Value, bool) {77+ if a := (sheet.Addr{Col: c, Row: l.row}); l.s.Filled(a) {78+ return l.s.Value(a), true7879 }79- return nil80+ return sheet.Value{}, false8081 }8182 83+// filled reports whether the cell in column c of the row has contents.84+func (l *rowLayout) filled(c int) bool { return l.s.Filled(sheet.Addr{Col: c, Row: l.row}) }85+8286 // col returns where column c starts and ends, relative to column first.8387 func (l *rowLayout) col(c int) (int, int) { return l.x[c-l.first], l.x[c-l.first+1] }8488 @@ -103,9 +107,9 @@ func (l *rowLayout) reach(minCol, maxCol int) (first, last int) {
103107 104108 // place lays out the text of the cell in column c over the columns it105109 // reaches.106-func (l *rowLayout) place(c int, cell *sheet.Cell) {110+func (l *rowLayout) place(c int, v sheet.Value) {107111 x0, x1 := l.col(c)108- text, align, pad := l.display(c, cell, x1-x0)112+ text, align, pad := l.display(c, v, x1-x0)109113 tw := ansi.StringWidth(text)110114 start := x0 + pad111115 switch align {@@ -116,7 +120,7 @@ func (l *rowLayout) place(c int, cell *sheet.Cell) {
116120 case sheet.AlignCenter:117121 start = x0 + (x1-x0-tw)/2118122 }119- from, to := l.room(c, cell, start, tw)123+ from, to := l.room(c, v, start, tw)120124 l.claimed = to121125 cut := textCutter{s: text}122126 style := l.s.CellStyle(sheet.Addr{Col: c, Row: l.row})@@ -135,15 +139,15 @@ func (l *rowLayout) place(c int, cell *sheet.Cell) {
135139 136140 // display formats the cell in column c for a column w wide, returning137141 // the text, its alignment and the padding on its aligned side.138-func (l *rowLayout) display(c int, cell *sheet.Cell, w int) (string, sheet.Align, int) {142+func (l *rowLayout) display(c int, v sheet.Value, w int) (string, sheet.Align, int) {139143 f := l.s.DisplayFormat(sheet.Addr{Col: c, Row: l.row})140- text, align := sheet.Display(cell.Value, f, w)144+ text, align := sheet.Display(v, f, w)141145 pad := 1142146 // A number one character too wide (12/31/2026 in a default column)143147 // may use the padding when nothing is to its right, rather than144148 // turning into #s.145- if cell.Value.Kind == sheet.Number && strings.Trim(text, "#") == "" && l.content(c+1) == nil {146- if wider, _ := sheet.Display(cell.Value, f, w+1); strings.Trim(wider, "#") != "" {149+ if v.Kind == sheet.Number && strings.Trim(text, "#") == "" && !l.filled(c+1) {150+ if wider, _ := sheet.Display(v, f, w+1); strings.Trim(wider, "#") != "" {147151 text, pad = wider, 0148152 }149153 }@@ -156,16 +160,16 @@ func (l *rowLayout) display(c int, cell *sheet.Cell, w int) (string, sheet.Align
156160 // room returns where the text of the cell in column c, tw wide from157161 // start, may go: its own column and, for text, the blank neighbors it158162 // runs into, but not over text already placed to its left.159-func (l *rowLayout) room(c int, cell *sheet.Cell, start, tw int) (from, to int) {163+func (l *rowLayout) room(c int, v sheet.Value, start, tw int) (from, to int) {160164 x0, x1 := l.col(c)161165 from, to = max(x0, l.claimed), x1162- if cell.Value.Kind != sheet.Text {166+ if v.Kind != sheet.Text {163167 return from, to164168 }165- for k := c + 1; k <= l.last && start+tw > to && l.content(k) == nil; k++ {169+ for k := c + 1; k <= l.last && start+tw > to && !l.filled(k); k++ {166170 _, to = l.col(k)167171 }168- for k := c - 1; k >= l.first && start < from && l.content(k) == nil; k-- {172+ for k := c - 1; k >= l.first && start < from && !l.filled(k); k-- {169173 kx0, _ := l.col(k)170174 if kx0 < l.claimed {171175 return l.claimed, to@@ -200,7 +200,7 @@ func (m *Model) toggleCheckboxes() tea.Cmd {
200200 continue201201 }202202 for _, vr := range v.Ranges {203- targets = appendCells(targets, intersect(vr, r), m.cur, func(a sheet.Addr) bool { return on || !s.Cell(a).Blank() })203+ targets = appendCells(targets, intersect(vr, r), m.cur, func(a sheet.Addr) bool { return on || s.Filled(a) })204204 }205205 }206206 }@@ -233,7 +233,7 @@ var vimMotions = map[string]motion{
233233 "0": nav("home"), "^": nav("home"),234234 "$": func(m *Model, a *sheet.Addr, _ int, _ bool) {235235 last := sheet.Addr{Col: sheet.MaxCols - 1, Row: a.Row}236- if m.sheet.Cell(last).Blank() {236+ if !m.sheet.Filled(last) {237237 last = m.sheet.Edge(last, -1, 0)238238 }239239 a.Col = last.Col