# Conflicts: # docs/limits.md
- commit
- 13271c287e48b94c61b7ee65aa07113fe62c9fd0
- parent
- 75179dabab27931f63cf9e821fab462e946c1885
- parent
- 327871c53ba597dfde8a19285d4b51a10613af2c
- author
- Dami <dami.etoile@gmail.com>
| A | CLAUDE.md | 23 | + |
| M | cmd/ | 4 | +- |
| M | cmd/ | 16 | - |
| M | docs/ | 2 | +- |
| M | docs/ | 7 | +- |
| M | docs/ | 2 | +- |
| M | docs/ | 506 | +++++++++++------------------- |
| M | docs/ | 8 | +- |
| M | internal/ | 1 | - |
| M | internal/ | 10 | - |
| M | internal/ | 4 | +- |
| M | internal/ | 14 | - |
| M | internal/ | 3 | +- |
| M | internal/ | 8 | - |
| M | internal/ | 9 | +- |
| M | internal/ | 2 | +- |
| M | internal/ | 2 | +- |
| M | internal/ | 2 | +- |
| A | scripts/ | 163 | ++++++++++ |
| M | scripts/ | 25 | +- |
| A | staticcheck.conf | 4 | + |
@@ -0,0 +1,23 @@
1+# Working on 0122+3+Read [docs/UX.md](docs/UX.md), [docs/extending.md](docs/extending.md) and4+[docs/testing.md](docs/testing.md) before changing code.5+6+## No broken windows7+8+Noise that stays becomes invisible, and then real problems hide in it.9+10+- `make check` passes before every push: gofmt, vet, `make lint`11+ (staticcheck in every module, doclint, cognitive complexity at most 25,12+ Go files at most 500 lines), unit tests, the excelize oracle and the13+ libghostty e2e tests. A red check is fixed first, not worked around.14+- A warning is fixed where it points, or turned off in its tool's config15+ with the reason written next to it (`staticcheck.conf`). Nothing is left16+ standing because "it's always been there".17+- Docs and comments describe the code as it is and why. How it got there18+ (what changed, what it did before, before and after numbers, which commit <!-- doclint:allow: names the phrases the rule forbids -->19+ or merge, who did it) belongs in commit messages; doclint enforces this.20+ Saying why the code stays compatible with a planned change is fine: that21+ explains the code today. Plans go in ROADMAP.md.22+- No scratch files, debug tests or temporary samples in commits.23+- Visual changes are reviewed in the golden gallery (`make screens`).@@ -133,8 +133,8 @@ func jevConfig(c *config.Config, key string) jev.Config {
133133 }134134 135135 // startJEV finds the API key and connects, returning notes for the136-// context line: why the key couldn't be read, or that a .env file's key137-// is no longer read.136+// context line: why the key couldn't be read, or that a .env file holds137+// a key, which 012 doesn't read.138138 //139139 // jev-api-key-command runs only when the environment and the credential140140 // store have no key, and then only when a sheet first asks JEV@@ -2,7 +2,6 @@ package main
22 33 import (44 "log/slog"5- "os"65 "runtime/debug"76 "time"87 @@ -23,21 +22,6 @@ func telemetryConfig(c *config.Config) telemetry.Config {
2322 return tc2423 }2524 26-// telemetryFlags takes the telemetry flags (--log, --otlp) out of args,27-// and reads the rest of telemetry's settings from the config file and28-// environment. 012 serve uses it before parsing its own flags.29-func telemetryFlags(args []string) (telemetry.Config, []string, error) {30- flags, rest, err := config.ParseFlags(args)31- if err != nil {32- return telemetry.Config{}, nil, err33- }34- path, err := config.DefaultPath()35- if err != nil {36- return telemetry.Config{}, nil, err37- }38- return telemetryConfig(config.Load(path, os.Getenv, flags)), rest, nil39-}40-4125 // startTelemetry opens the log and reports every recalculation and pivot4226 // refresh to it, as spans nested in their workbook's trace.4327 func startTelemetry(c telemetry.Config) (func() error, error) {@@ -203,7 +203,7 @@ and returns plain values instead:
203203 The engine fills a chunk from the occupancy index without a call per cell204204 (`colFill`, `rangeFill`), and keeps each sheet's reader between205205 recalculations, so the buffers are made once. See206-[limits.md](limits.md#the-function-library) for before and after.206+[limits.md](limits.md#the-function-library) for its costs.207207 208208 What stays in the engine is what needs cells or the workbook: the209209 running aggregates' storage and extension (`rangememo.go`), the links a@@ -169,7 +169,8 @@ builds can safely ignore (decimal arithmetic) don't raise the version.
169169 170170 Every seam above has a telemetry span and a `-tags stress` benchmark171171 (`make stress`, `make stress-report`), and [limits.md](limits.md) records172-the bounds. A change to a seam reports before and after numbers.172+the bounds. A change to a seam comes with its numbers, measured against173+the code it replaces.173174 174175 Spans nest without a `context.Context` through the engine: start a UI175176 span through the model's trace (`m.spans.Start`), so it holds what it@@ -184,8 +185,8 @@ global: `012 serve` runs many programs in one process (see
184185 `make lint`: no function over cognitive complexity 25, no Go file over 500185186 lines. A package that keeps growing past a few thousand lines gets split186187 along a boundary where dependencies point one way, as the function187-library (`internal/functions`) and values (`internal/value`) were split188-from the engine: the lower package defines the small interface it needs188+library (`internal/functions`) and values (`internal/value`) sit below189+the engine: the lower package defines the small interface it needs189190 (`functions.Book`) and the engine implements it, keeping its API through190191 aliases so callers don't change.191192 @@ -65,7 +65,7 @@ environment only: `jev-base-url` (or `TYPESAFE_BASE_URL`), which must be
6565 https unless it's this machine, and `jev-model` (or6666 `TYPESAFE_DEFAULT_MODEL`). See [config.md](config.md).6767 68-**`.env` files are no longer read.** A `.env` next to a downloaded sheet68+**012 doesn't read `.env` files.** A `.env` next to a downloaded sheet6969 could set `TYPESAFE_BASE_URL` and send your key to someone else's server.7070 If the directory 012 starts in has a `.env` with `TYPESAFE_API_KEY`, the7171 context line suggests `012 config set-key`; 012 never reads the key from@@ -7,12 +7,10 @@ real terminal emulator), and the telemetry described in
77 [observability.md](observability.md), which tracks how they move.88 99 Machine: Apple M5 Pro (18 cores), 48 GB, macOS, Go 1.27.1, arm64. Times10-are single-threaded: the engine and the UI run on one goroutine. Last full11-run: commit 4dbe37d (after the engine and UI restructure), `BENCHTIME=2s`12-on an otherwise idle machine; it matched the numbers below within noise.13-The grid step (see [Step A: the grid](#step-a-the-grid)) was measured14-before and after back to back, `-benchtime 1s`, commits bdc5259 and15-bb4ff53.10+are single-threaded: the engine and the UI run on one goroutine. Figures11+are medians of `-benchtime 1s` or `BENCHTIME=2s` runs on an otherwise12+idle machine; a range means runs differed by that much, and differences13+under about 5% are noise.1614 1715 The yardstick is a keystroke through to its frame: under 16 ms feels1816 instant (one frame at 60 Hz), under 100 ms feels responsive, past that@@ -23,8 +21,8 @@ it lags, and past a second it stalls.
2321 | Dimension | Comfortable | Degraded | Unsupported | Dominant cost |2422 |---|---|---|---|---|2523 | Sheet size | A grid of 1,048,576 x 16,384 (A..XFD), with up to `max-cells` cells (two million by default): navigation, drawing and every command cost what the cells cost, not the grid | Loading and saving two million cells: 1 to 2 s; 600 MB of heap | More than `max-cells` cells: imports keep whole rows up to it and say what they dropped; larger pastes and fills are refused | Heap per cell (about 300 B), JSON file format |26-| Incremental recalc | A change that makes formulas read under about 500,000 cells in total (fan-out, chains and volatiles of 8192 cells: 1 to 2 ms); 1000 SUMs over a full column: 0.7 ms; 8192 running totals: 2.7 ms | | | About 20 ns per cell read: two map lookups |27-| Full recalc | Any sheet: under 40 ms for 213 k numbers; 1000 full-column SUMs 1.1 ms; 8192 running totals 2.6 ms | | | Same as above |24+| Incremental recalc | A change that makes formulas read under about 500,000 cells in total (fan-out, chains and volatiles of 8192 cells: 1 to 2 ms); 1000 SUMs over a full column: 0.75 ms; 8192 running totals: 2.8 ms | 60 criteria functions (SUMIF, COUNTIFS, AVERAGEIF) over whole columns of 8192 rows: 37 ms an edit | | About 20 ns per cell read: two map lookups |25+| Full recalc | Any sheet: under 40 ms for 213 k numbers; 1000 full-column SUMs 1 ms; 8192 running totals 2.5 ms | 60 whole-column criteria functions over 8192 rows: 59 ms | | Same as above |2826 | Rendering | Any sheet at up to 200 x 60: 1 ms a frame; 400 x 120: 4.5 ms | 20 charts at 400 x 120: 7 ms | | View building styled strings, then Bubble Tea parsing and diffing them |2927 | Selection statistics | Any selection: extending one over all 2 M cells costs 0.4 ms a key | | | Per column with data, 1024 or 64 rows at a time from an index on the blocks of filled cells, plus the rows at the selection's ends |3028 | Imports | CSV, SQLite, Parquet: 1 to 3 M cells/s (two million cells in about 1 s); XLSX numbers or text: 0.7 to 1.1 M cells/s | XLSX with formulas: 0.4 to 0.6 M cells/s | Data past `max-cells` or the grid (dropped, with a note); XLSX files past the reader's limits (refused) | Building cells one at a time; XML decoding; XLSX formula translation |@@ -43,21 +41,19 @@ Hard limits: `sheet.MaxRows = 1,048,576`, `sheet.MaxCols = 16,384`
4341 two million cells by default): imports keep whole rows up to it, pastes4442 and fills write at most that many cells at once. WK1 files keep their own4543 8192 x 256. Past about two million cells the costs below grow with the46-cells, as they always did; only the grid is no longer a cost.44+cells; the grid itself costs nothing (see [The grid](#the-grid)).4745 4846 | Measure | 8192 x 26 (213 k cells) | 8192 x 256 (2.1 M cells) |4947 |---|---|---|5048 | Build through `Load`, then full recalc | 90 ms | 1.4 s |5149 | Live heap | 64 MB | 602 MB (about 300 B per number cell) |5250 | Save `.012` (JSON, one line per cell) | 74 ms, 4.6 MB | 1.04 s, 47 MB |53-| Open `.012` | 134 ms (116 MB allocated) | 2.08 s (1.3 GB allocated) |51+| Open `.012` | 120 to 130 ms (116 MB allocated) | 1.9 to 2 s (1.3 GB allocated) |5452 | Export CSV / TSV / XLSX / SQLite | 41 / 55 / 62 / 40 ms | 0.71 / 0.66 / 0.81 / 0.72 s |5553 | Frame at 200 x 60, whole | 1.06 ms | 1.05 ms |5654 57-XLSX exports are written by 012 with archive/zip since excelize went:58-measured back to back with the excelize writer, 8192 x 26 went from 33559-to 62 ms and 379 to 1.9 MB allocated, 8192 x 256 from 0.97 to 0.81 s60-(mostly compression) and 259 to 17 MB allocated.55+XLSX exports are written by 012 with archive/zip: 1.9 MB allocated for56+8192 x 26 and 17 MB for 8192 x 256, whose time is mostly compression.6157 6258 Heap per non-blank cell after loading (`BenchmarkMemory`,6359 `BenchmarkImport`): numbers 301 B, formulas 707 B (the parsed tree and@@ -65,6 +61,51 @@ reference lists), 200-character text 790 B, imported CSV 275 to 300 B.
6561 A `Cell` struct is 216 B; the rest is the map entry, the input string6662 and a boxed number for the parsed literal.6763 64+## The grid65+66+Nothing costs what the grid costs. Beside the cell map, occupancy67+indexes (bitmaps of 1024 rows per column) say which cells are stored and68+which have contents; ranges are read through them, so a whole column of69+ten cells is ten visits. Formats of whole columns, rows and the sheet70+live on the lines, not on cells. `TestCommandsCostTheDataNotTheGrid`71+(`internal/ui`) runs every command with the whole sheet, a whole column72+and a whole row selected on the full grid and fails past 250 ms or 8 MB;73+each takes 0.1 to 4 ms.74+75+What each part of 012 visits when given a range or selection:76+77+| Where | Visits |78+|---|---|79+| Range reads for aggregates (`Book.Fold` and `Book.Scan`) | the stored cells, in row order |80+| SUMIF(S), COUNTIF(S), AVERAGEIF(S), SUMPRODUCT, COUNTBLANK | the cells any range holds; the blanks between counted at once |81+| MATCH, VLOOKUP, HLOOKUP, XLOOKUP | the stored entries, the blanks between as one; ROWS and INDEX keep the full size |82+| Range users (`rangeIndex`), found per changed cell | interval trees per used column, whole rows in one |83+| Formatting whole columns or rows | column, row and sheet formats |84+| Used range, Ctrl+arrow, text running into view | the index of filled cells |85+| Filter over whole columns | rows with data, one test for the blank rest, stepped over at once |86+| Sort | the rows with cells |87+| Paste, fill of blanks | the clip's cells per tile; what blanks land on is cleared |88+| Formats pasted or moved | whole columns and rows: their line formats, and the cells where formatted lines cross them; a block: its blank cells get formats of their own up to 65,536 cells, then only the stored cells do |89+| Formulas re-inferring after a line's format changes | the dependency indexes, not the line's cells |90+| Copy to the system clipboard | the cells; refused past `max-cells` |91+| Pivot sources, JEV value ranges, chart detection | the rows with data; JEV ranges past 4096 cells trimmed to their data |92+| Exports of a selection, script reads | trimmed to the data |93+| Row numbers, name box | 7 digits (the header widens past 9999), XFD |94+| Column width reset, set | one undo step; a reset touches only columns with a width |95+| XLSX column widths and styles | the data's width (at least 256), a style on A and XFD as the sheet's |96+97+A sheet of 10,000 numbers spread down a million rows under 300098+whole-column formulas (sparse-1M) edits in 2.5 ms, draws a 200 x 6099+frame in 0.68 ms, and answers arrow keys at XFD1048576 in 0.65 ms.100+Typing an entry under 1000 SUMs over a column, through to its frame at101+200 x 60, takes 0.91 ms.102+103+The index costs a few percent on edits and loads (keeping two bitmaps104+current per cell), and running aggregates cost memory for their105+checkpoints (a 1000-SUM fan-in allocates 2 MB per recalculation). A106+range is shared only from its second read in a recalculation, so ranges107+read once cost nothing extra.108+68109 ## Formula topologies69110 70111 `BenchmarkEdit` sets one cell and recalculates what depends on it;@@ -72,17 +113,22 @@ and a boxed number for the parsed literal.
72113 73114 | Topology | Edit | Full recalc | Undo + redo |74115 |---|---|---|---|75-| Independent cell in 8192 x 26 numbers | 0.5 us | 38 ms | 0.8 us |76-| Chain: A(n) = A(n-1)+1 down 8192 rows | 1.7 ms | 1.1 ms | 3.4 ms |77-| Fan-out: one cell read by 8192 formulas | 1.3 ms | 0.9 ms | 2.7 ms |78-| Volatile: 8192 TODAY() and RAND() | 1.25 ms | 0.8 ms | 2.5 ms |116+| Independent cell in 8192 x 26 numbers | 0.66 us | 32 to 38 ms | 0.8 us |117+| Chain: A(n) = A(n-1)+1 down 8192 rows | 1.85 ms | 1.1 to 1.4 ms | 3.4 ms |118+| Fan-out: one cell read by 8192 formulas | 1.4 ms | 0.9 ms | 2.7 ms |119+| Volatile: 8192 TODAY() and RAND() | 1.3 ms | 0.8 ms | 2.5 ms |79120 | 1000 named ranges, one formula each | 48 us | 2.8 ms | 99 us |80-| Fan-in: 1000 x SUM(A1:A8192), or SUM(A:A) | 0.72 ms | 1.1 ms | 1.4 ms |81-| Running totals: 8192 x SUM($A$1:An) | 2.7 ms | 2.6 ms | 5.4 ms |82-| Sparse: 10,000 numbers spread down a million rows, 1000 each of SUM(A:A), running totals and VLOOKUP over A:B | 2.6 ms | 3.9 ms | 5.1 ms |121+| Fan-in: 1000 x SUM(A1:A8192), or SUM(A:A) | 0.75 ms | 1 ms | 1.4 ms |122+| Running totals: 8192 x SUM($A$1:An) | 2.85 ms | 2.5 ms | 5.4 ms |123+| Sparse: 10,000 numbers spread down a million rows, 1000 each of SUM(A:A), running totals and VLOOKUP over A:B | 2.45 ms | 3.6 ms | 5.1 ms |124+| Criteria: 60 SUMIF, COUNTIFS and AVERAGEIF over whole columns of 8192 rows | 37 ms | 59 ms | |125+| Lookup: 300 VLOOKUP, MATCH and XLOOKUP into 8192 keys | 25 ms | 26.5 ms | |83126 84127 Every volatile formula is recomputed on every change, as in Sheets, so85-8192 volatile cells add about 1.2 ms to every edit anywhere.128+8192 volatile cells add about 1.2 ms to every edit anywhere. Criteria129+and lookups cost what their ranges hold, not what the grid holds: the130+criteria collect positions from the occupancy index, and lookups walk131+the stored entries with a cursor.86132 87133 Through a real terminal (`make stress-e2e`: 200 x 60, 8192 x 26 imported,88134 200 SUMs over column A): arrow keys p50 8.3 ms, p95 10.2 ms; an entry@@ -90,11 +136,23 @@ that recalculates the 200 SUMs p50 62 to 70 ms, p95 67 to 75 ms. The
90136 arrow floor is Bubble Tea's renderer, which draws at most 120 frames a91137 second (`ui.FrameRate`, its maximum; the default is 60), so a key waits92138 up to 8 ms for the next frame; 012's own share of an arrow key is 0.293-to 1 ms. At 60 frames a second the same arrows took p50 16.6 ms, p95139+to 1 ms. At 60 frames a second the same arrows take p50 16.6 ms, p9594140 18.4 ms. The renderer writes a frame only when the view changed, so an95141 idle 012 costs the same at either rate: 0.6 to 2.2 ms of CPU a second96142 on an empty sheet over 30 s, within the noise of measuring it.97143 144+## The function library145+146+Functions live in `internal/functions`, behind a `Book` interface whose147+methods pass only values (see [architecture.md](architecture.md#functions)):148+SUM-like functions have the engine add a range up (`Fold`), other range149+walks read chunks into buffers the `Reader` reuses (`Scan`), and lookups150+walk positions with a cursor, so no function makes a call or an151+allocation per cell it reads. The criteria and lookup shapes in152+[Formula topologies](#formula-topologies) exercise the paths other than153+`Fold`. Plain arithmetic, 1000 formulas and a SUM, recalculates in 20 us154+in binary floating point and 156 us in decimal.155+98156 ## Rendering99157 100158 Only visible rows and columns are drawn, so frame time depends on the@@ -118,33 +176,35 @@ whatever its data: only the categories that fit are drawn (a pie of 8192
118176 slices, 0.44 ms, is the worst). An image for kitty graphics is redrawn119177 when its data, size or theme changes: 25 to 260 us at 24 x 10 cells and120178 1 to 1.5 ms at 120 x 40, a pie 8.6 ms (`internal/chart`,121-`BenchmarkDraw`, `BenchmarkImage`). Key presses add little: an arrow key through to its frame122-is 0.21 ms at 80 x 24 and 1.06 ms at 200 x 60; Page Down 1.26 ms.179+`BenchmarkDraw`, `BenchmarkImage`). Key presses add little: an arrow key180+through to its frame is 0.21 ms at 80 x 24 and 1.06 ms at 200 x 60;181+Page Down 1.26 ms.182+183+## Selection statistics123184 124185 With the whole of an 8192 x 256 sheet selected, the status line's Sum,125186 Avg and Count cover 2.1 M cells. On a sheet of more than 16,384 cells,126187 a selection of more than 4096 is summed from an index of each column's127-sum, count and count of numbers in blocks of 64 rows (`internal/sheet/stats.go`):128-the whole blocks it covers, plus the rows at its ends a cell at a time.129-The index is built on first use, in one pass over the cells (the 30 ms130-the whole sum used to take every time), and a stored, deleted or131-recalculated cell marks its block to be summed again when a selection188+sum, count and count of numbers (`internal/sheet/stats.go`), kept on the189+blocks of the index of filled cells: an entry per 64 rows and a total190+per 1024, made only where there is data. A selection adds up the whole191+blocks it covers, plus the rows at its ends a cell at a time. The index192+is built on first use, in one pass over the cells, and a stored, deleted193+or recalculated cell marks its block to be summed again when a selection132194 next covers it. Sums are recomputed, never adjusted, so they don't133-drift. The result is still cached until a cell changes. Each key that134-changes a selection over the whole sheet, through to its frame135-(`BenchmarkKeystroke`, 3 runs each, same machine):195+drift. The result is cached until a cell changes. Each key that changes196+a selection over the whole sheet, through to its frame197+(`BenchmarkKeystroke`):136198 137-| Key, 8192 x 256 numbers | 80 x 24 before | after | 200 x 60 before | after |138-|---|---|---|---|---|139-| Shift+Up and Shift+Down with the data selected (`extend-data`) | 31 to 44 ms | 0.53 ms | 32 to 46 ms | 2.1 ms |140-| Ctrl+Shift+Down and Ctrl+Shift+Up over the first row (`extend-edge`) | 16 to 23 ms | 0.43 ms | 18 to 24 ms | 1.7 ms |141-| Ctrl+A, the selection standing (`select-data`) | 0.30 ms | 0.30 ms | 1.8 ms | 1.8 ms |199+| Key, 8192 x 256 numbers | 80 x 24 | 200 x 60 |200+|---|---|---|201+| Shift+Up and Shift+Down with the data selected (`extend-data`) | 0.41 to 0.53 ms | 2.1 ms |202+| Ctrl+Shift+Down and Ctrl+Shift+Up over the first row (`extend-edge`) | 0.43 ms | 1.7 ms |203+| Ctrl+A, the selection standing (`select-data`) | 0.28 ms | 1.8 ms |142204 143-A CPU profile of `extend-data` before put 54% of the time in144-`RangeStats` walking the map; the rest is building the benchmark's sheet.145-The index costs 0.5 MB per sheet that has one (32,768 blocks of 16 B),146-nothing on sheets that never had a large selection, and a nil check per147-stored or recalculated cell.205+The index costs memory only on sheets that had a large selection, in206+proportion to their data, and a nil check per stored or recalculated207+cell.148208 149209 ## Imports150210 @@ -153,11 +213,11 @@ Real, openly licensed datasets fetched by `scripts/stress-data.sh`
153213 154214 | File | Shape | Import | Throughput | Heap per cell |155215 |---|---|---|---|---|156-| OWID energy (CSV) | 23,377 x 130, 345 k cells kept | 195 ms | 1.8 M cells/s, 47 MB/s | 277 B |157-| Airport codes (CSV) | 86,134 x 13, first 8192 rows kept | 83 ms | 107 MB/s | 274 B |216+| OWID energy (CSV) | 23,377 x 130, 1.04 M cells | 0.84 to 0.95 s | 1.1 to 1.2 M cells/s | 277 B |217+| Airport codes (CSV) | 86,134 x 13, 855 k cells | 0.78 to 0.80 s | 1.1 M cells/s | 274 B |158218 | NOAA daily CO2 (CSV) | 18,304 x 2 | 6 ms | 2.7 M cells/s | 294 B |159219 | Country codes (CSV, Arabic, CJK) | 249 x 56 | 5.6 ms | 2.2 M cells/s | 272 B |160-| Apache POI formula tests (XLSX) | 788 rows, 1189 formulas, 3828 cells | 9.7 ms | 395 k cells/s | 425 B |220+| Apache POI formula tests (XLSX) | 788 rows, 1189 formulas, 3828 cells | 9.5 to 9.7 ms | 395 k cells/s | 425 B |161221 | Chinook PlaylistTrack (SQLite) | 8715 x 2 | 5.5 ms | 3.0 M cells/s | 293 B |162222 | Parquet alltypes_tiny_pages | 7300 x 13 | 40 ms | 2.4 M cells/s | 276 B |163223 @@ -165,27 +225,23 @@ Past `max-cells` (or the grid), imports keep whole rows up to it and
165225 say how many rows they left out. Parquet files and SQLite tables stop166226 reading at the last row and take the count of the rest from the file;167227 CSV, TSV and SQLite queries are read to their end to count it, without168-keeping it. The table above is from before the grid grew, when files169-were cut at row 8192; now the airport codes (855 k cells, 0.80 s) and170-OWID energy (1.04 M cells, 0.95 s) come in whole, at the same rate of171-about a million cells a second. Every importer streams,172-so memory follows the sheet, not the file. XLSX used to go through173-excelize, which held each worksheet in memory while its rows were read;174-012's own reader streams it a token at a time, keeping only the shared175-strings (in one buffer, 4 bytes a string besides the text), the cell176-formats, and the first cell of each shared formula.228+keeping it. Every importer streams, so memory follows the sheet, not the229+file. 012's XLSX reader streams a worksheet a token at a time, keeping230+only the shared strings (in one buffer, 4 bytes a string besides the231+text), the cell formats, and the first cell of each shared formula.232+XLSX imports set column widths and styles for the data's columns (at233+least 256) without an undo step each.177234 178235 XLSX imports of full sheets written by 012's exporter179-(`BenchmarkImportXLSX`) and the POI file, before (excelize) and after180-(012's reader), on an Apple M5 Pro. Peak heap is sampled while236+(`BenchmarkImportXLSX`) and the POI file. Peak heap is sampled while181237 importing, per cell kept, and includes garbage not yet collected:182238 183239 | Workbook | Time | Allocated | Allocations | Heap per cell after | Peak heap per cell |184240 |---|---|---|---|---|---|185-| 8192 x 26 numbers (213 k cells) | 480 -> 199 ms | 491 -> 170 MB | 8.4 -> 3.3 M | 304 -> 290 B | 1365 -> 589 B |186-| 8192 x 26 table, text and numbers | 642 -> 296 ms | 644 -> 234 MB | 14.4 -> 7.4 M | 297 -> 297 B | 1432 -> 722 B |187-| 8192 formulas (`=A1+1` down a column) | 33 -> 13.9 ms | 38 -> 17 MB | 718 -> 321 k | 752 -> 746 B | 4790 -> 2188 B |188-| Apache POI formula tests | 21.1 -> 9.7 ms | 22 -> 9.7 MB | 373 -> 179 k | 429 -> 425 B | 2343 -> 1288 B |241+| 8192 x 26 numbers (213 k cells) | 189 to 199 ms | 170 MB | 3.3 M | 290 B | 589 B |242+| 8192 x 26 table, text and numbers | 296 to 312 ms | 234 MB | 7.4 M | 297 B | 722 B |243+| 8192 formulas (`=A1+1` down a column) | 13.8 ms | 17 MB | 321 k | 746 B | 2188 B |244+| Apache POI formula tests | 9.5 to 9.7 ms | 9.7 MB | 179 k | 425 B | 1288 B |189245 190246 The reader refuses files past `xlsxLimits` (in `internal/fileio/xlsxpkg.go`),191247 with a message naming the limit: 10,000 files in the zip; 1 GB@@ -203,26 +259,23 @@ entities, so entity bombs fail as unknown entities.
203259 204260 Each step keeps a copy of every cell it changed. Undoing costs about205261 what the change cost, since it recalculates the same cells. Clearing206-213 k cells and undoing it takes 158 ms.262+213 k cells and undoing it takes 158 to 164 ms.207263 208264 The history is bounded by steps (`MaxUndo = 100`) and by an estimate of209265 the memory its before-images hold (`MaxUndoBytes = 256 MB`); past either,210266 the oldest steps are dropped first, and the newest step is kept however211267 large it is. Each cell is counted as it is recorded (a fixed cost per212268 cell plus its text, more for a formula, whose parsed tree stays alive),213-so the budget costs nothing to check. The estimate follows the measured214-heap within about 10% (`BenchmarkHistoryFull`, `BenchmarkHistoryWide`,215-3 runs each, same machine):216-217-| Benchmark | Before | After |218-|---|---|---|219-| 100 edits rewriting a whole column (819 k before-images) | 250 MB held, all 100 kept | 250 MB held (239 MB estimated), all 100 kept |220-| 12 edits rewriting all of 8192 x 26 (2.6 M before-images) | 705 MB held, all 12 kept | 230 MB held (248 MB estimated), the last 4 kept |221-| BigUndo: clear 8192 x 26, undo | 154 ms, 430 k allocs | 158 ms, 430 k allocs |222-| Undo + redo of one cell (dense, names) | 0.98 us, 106 us | 1.0 us, 106 us |269+so the budget costs nothing to check; counting costs about 20 ns a270+cell, 3% of clearing a sheet and undoing it. The estimate follows the271+measured heap within about 10% (`BenchmarkHistoryFull`,272+`BenchmarkHistoryWide`):223273 224-Counting cells as they're recorded costs about 20 ns a cell, which is225-BigUndo's 3%; everything else is unchanged.274+| Benchmark | Result |275+|---|---|276+| 100 edits rewriting a whole column (819 k before-images) | 250 MB held (239 MB estimated), all 100 kept |277+| 12 edits rewriting all of 8192 x 26 (2.6 M before-images) | 230 MB held (248 MB estimated), the last 4 kept |278+| Undo + redo of one cell (dense, names) | 1.0 us, 106 us |226279 227280 ## JEV228281 @@ -234,20 +287,19 @@ formulas waiting for it and what reads them
234287 first since the last recalculation schedules one a frame later235288 (`ui.FrameInterval`), which covers every answer stored meanwhile, so236289 answers arriving together recalculate once. With a fake client237-answering instantly (`BenchmarkJEV`, 3 runs each, same machine; the238-benchmark doesn't wait the frame, so answers queued together, up to 8,239-recalculate together):290+answering instantly (`BenchmarkJEV`; the benchmark doesn't wait the291+frame, so answers queued together, up to 8, recalculate together):240292 241-| JEV cells | Per answer before | after | All answers before | after |242-|---|---|---|---|---|243-| 100 | 29 us | 3.5 us | 2.9 ms | 0.35 ms |244-| 1000 | 270 us | 3.6 us | 0.27 s | 3.6 ms |245-| 4000 | 1.2 ms | 3.8 us | 4.9 s | 15 ms |293+| JEV cells | Per answer | All answers |294+|---|---|---|295+| 100 | 3.5 us | 0.35 ms |296+| 1000 | 3.6 us | 3.6 ms |297+| 4000 | 3.8 us | 14 to 15 ms |246298 247299 Against the real service the network dominates: at most 8 questions are248-in flight, so 4000 questions at 300 ms each take 2.5 minutes, and the249-recalculation is now a few microseconds per answer whatever the number250-of JEV cells. Other changes still recalculate every JEV cell, as every300+in flight, so 4000 questions at 300 ms each take 2.5 minutes, while the301+recalculation costs a few microseconds per answer whatever the number302+of JEV cells. Other changes recalculate every JEV cell, as every251303 volatile formula, looking the answers up in the cache.252304 253305 ## Formula depth@@ -260,65 +312,44 @@ with the caret at the level past the cap, as other parse errors are.
260312 Excel allows 64 nested functions; 1024 is far beyond a formula written261313 by hand, and bounds the recursion of everything that walks a formula262314 (the printer, reference rewriting, the evaluator within one formula). A263-`.012` file holding a deeper formula (none could have been written by264-hand) fails to open with that message and the cell; imports keep such a265-formula's text or cached value, as for any formula they can't read.315+`.012` file holding a deeper formula fails to open with that message and316+the cell; imports keep such a formula's text or cached value, as for any317+formula they can't read.266318 267319 Evaluation recurses once per link of a chain of formulas, which no268-parser cap bounds: a chain through all 2.1 M cells of a sheet used to269-overflow the goroutine's 1 GB stack and kill the program. Evaluation now270-counts its depth in cells and operators; a cell reached past 65,536271-levels is put off: the evaluation in progress is abandoned (its cells go272-back to dirty), the cell is evaluated from the top of the stack, and the273-abandoned evaluation is retried and finds it done. Values are exactly as274-before, a cycle longer than the limit is still a cycle, and the stack275-stays under 32 MB (`TestFullSheetChain`, run with276-`SHEET_FULL_CHAIN=1`, checks it under a 64 MB limit: 4 s for 2.1 M277-cells, building included).278-279-Same machine, 8 runs each with before and after interleaved, other280-work running (differences under 10% are noise):281-282-| Benchmark | Before | After |283-|---|---|---|284-| Parse (`BenchmarkParse`) | 2.36 us | 2.41 us |285-| 1000 nested parentheses / IFs, parse and evaluate | 195 / 425 us | 203 / 438 us |286-| 10,000 nested parentheses / IFs | 1.8 / 4.4 ms, accepted | 1.1 / 2.0 ms to refuse |287-| RecalcAll chain of 8192 / dense 8192 x 26 | 1.33 / 50 ms | 1.28 to 1.38 / 41 to 51 ms |288-| Edit chain of 8192 / 8192 volatiles | 2.36 / 1.68 ms | 2.34 / 1.72 ms |320+parser cap bounds, so it counts its depth in cells and operators: a cell321+reached past 65,536 levels is put off. The evaluation in progress is322+abandoned (its cells go back to dirty), the cell is evaluated from the323+top of the stack, and the abandoned evaluation is retried and finds it324+done. Values are the same as with unbounded recursion, a cycle longer325+than the limit is still a cycle, and the stack stays under 32 MB326+(`TestFullSheetChain`, run with `SHEET_FULL_CHAIN=1`, checks it under a327+64 MB limit: 4 s for a chain through all 2.1 M cells, building328+included).329+330+| Benchmark | Time |331+|---|---|332+| Parse (`BenchmarkParse`) | 2.4 us |333+| 1000 nested parentheses / IFs, parse and evaluate | 203 / 438 us |334+| 10,000 nested parentheses / IFs | 1.1 / 2.0 ms to refuse |289335 290336 ## Several sheets291337 292338 A workbook recalculates all its sheets together. Formulas that name a293339 sheet (`Data!A1`) are indexed by the sheet names they use: a changed cell294-on a sheet some formula names scans those cross-sheet formulas, as range295-users were scanned before they were indexed by column, so a change there296-costs O(changed cells x cross-sheet formulas); cells on sheets no formula297-names skip the scan. Each sheet's lookup, which resolves sheet names, is298-made once per recalculation, and the recalculation state stays on each299-sheet keyed by address, so a single-sheet workbook computes as before.340+on a sheet some formula names scans those cross-sheet formulas, so a341+change there costs O(changed cells x cross-sheet formulas); cells on342+sheets no formula names skip the scan. Each sheet's lookup, which343+resolves sheet names, is made once per recalculation, and the344+recalculation state stays on each sheet keyed by address. Every cell345+read goes through one function call to resolve its sheet, 3 to 5% of a346+read-heavy recalculation.300347 301348 | Benchmark (`internal/sheet`, `go test -bench SheetEdit`) | Edit |302349 |---|---|303350 | 5000 formulas on one sheet reading `A(n)` and `SUM(A1:A10)` | 2.7 ms |304351 | The same 5000 formulas reading `Data!A(n)` and `SUM(Data!A1:A10)` | 3.5 ms |305352 306-The stress shapes before and after workbooks (5 runs each, same machine;307-differences under about 5% are noise):308-309-| Benchmark | Before | After |310-|---|---|---|311-| RecalcAll dense 8192 x 26 | 36.1 ms | 36.7 ms |312-| RecalcAll fan-in / running | 170 / 706 ms | 177 / 742 ms |313-| Edit chain / fan-in / running | 1.64 / 168 / 694 ms | 1.92 / 173 / 726 ms |314-| Undo chain | 3.4 ms | 4.0 ms |315-| BigUndo | 140 ms, 431 k allocs | 158 ms, 856 k allocs |316-| Open 8192 x 26 / 8192 x 256 | 120 ms / 1.85 s | 120 ms / 1.91 s |317-318-Cell reads now go through one more function call to resolve the sheet,319-which is most of the 3 to 5% on read-heavy shapes; undo steps key their320-before-images by sheet and address, which is BigUndo's extra allocation.321-322353 ## Serving over SSH323354 324355 `BenchmarkSessions` (`internal/serve/stress_test.go`) opens N sessions,@@ -327,25 +358,15 @@ opening the same 1000 x 26 sheet of numbers in every one, then has every
327358 session press an arrow key at once, 200 times, timing each key from the328359 client's write to the first bytes of its frame arriving. The clients329360 run in the same process, so the heap and CPU figures include their side330-of the connections and are upper bounds for the server. Medians of 3331-runs, `-benchtime 200x`, at 60 frames a second:361+of the connections and are upper bounds for the server. Sessions ask362+for 120 frames a second, as the local app does (`-benchtime 200x`):332363 333364 | Sessions, sheet | Heap per session | Key to frame p50 / p95 / max | CPU per frame |334365 |---|---|---|---|335-| 10, new sheet | 1.30 MiB | 10.7 / 16.6 / 17.1 ms | 0.95 ms |336-| 10, 1000 x 26 numbers | 8.5 MiB | 11.2 / 16.6 / 18.0 ms | 1.15 ms |337-| 50, new sheet | 1.30 MiB | 9.2 / 16.3 / 19.2 ms | 0.53 ms |338-| 50, 1000 x 26 numbers | 8.5 MiB | 9.4 / 16.7 / 20.5 ms | 0.69 ms |339-340-Sessions now ask for 120 frames a second, as the local app does. One341-run each, back to back, before and after:342-343-| Sessions, sheet | p50 / p95 / max at 60 fps | at 120 fps | CPU per frame, 60 / 120 fps |344-|---|---|---|---|345-| 10, new sheet | 10.8 / 16.6 / 16.8 ms | 5.1 / 8.3 / 9.6 ms | 0.78 / 0.67 ms |346-| 10, 1000 x 26 numbers | 9.7 / 16.6 / 20.7 ms | 4.3 / 8.3 / 8.5 ms | 1.41 / 0.71 ms |347-| 50, new sheet | 9.3 / 16.3 / 18.5 ms | 5.0 / 8.7 / 17.8 ms | 0.58 / 0.53 ms |348-| 50, 1000 x 26 numbers | 9.5 / 16.7 / 19.7 ms | 5.1 / 8.9 / 12.8 ms | 0.71 / 0.66 ms |366+| 10, new sheet | 1.30 MiB | 5.1 / 8.3 / 9.6 ms | 0.67 ms |367+| 10, 1000 x 26 numbers | 8.5 MiB | 4.3 / 8.3 / 8.5 ms | 0.71 ms |368+| 50, new sheet | 1.30 MiB | 5.0 / 8.7 / 17.8 ms | 0.53 ms |369+| 50, 1000 x 26 numbers | 8.5 MiB | 5.1 / 8.9 / 12.8 ms | 0.66 ms |349370 350371 Latency doesn't move from 10 to 50 sessions: it's Bubble Tea's frame351372 pacing (at most one frame every 8.3 ms, so a key waits half a frame on@@ -355,34 +376,30 @@ Bubble Tea's render buffers (about 1 MiB at 120 x 40, growing with the
355376 window), 130 KiB in its input key table and under 50 KiB in both ends356377 of the SSH connection, so the figure is close to the server's own. On357378 top come about 300 B per cell of the sheets a session has open;358-sessions opening the same file each hold a copy. CPU per frame covers both ends of the connection (the359-encryption twice, the client reading the frame); at 0.7 ms, 50 sessions360-typing continuously at 60 frames a second would keep about two cores361-busy, and at 120 about four: frames are drawn only when a session's362-screen changes, so the rate is the most keys can cause, not a cost of363-being connected.379+sessions opening the same file each hold a copy. CPU per frame covers380+both ends of the connection (the encryption twice, the client reading381+the frame); at 0.7 ms, 50 sessions typing continuously at 120 frames a382+second would keep about four cores busy: frames are drawn only when a383+session's screen changes, so the rate is the most keys can cause, not a384+cost of being connected.364385 365386 ## Pivot tables366387 367388 A pivot table is recomputed whole whenever a cell of its source range is368-recalculated: it reads every source row once, groups it, and rewrites369-only the result cells that changed, so what reads the results370-recalculates only when they move. `BenchmarkPivot` edits one cell of a371-full 8191-row table (an id, a category of 8, a number and one of 1000372-items, twice) under a pivot and times the edit, the recalculation and373-the pivot together:389+recalculated: it reads every source row once, groups it by a comparable390+key struct (no string built per row), and rewrites only the result cells391+that changed, so what reads the results recalculates only when they392+move. `BenchmarkPivot` edits one cell of a full 8191-row table (an id, a393+category of 8, a number and one of 1000 items, twice) under a pivot and394+times the edit, the recalculation and the pivot together:374395 375396 | Pivot | Result rows | Edit | Allocations |376397 |---|---|---|---|377-| Rows: 8 categories; SUM and COUNTA | 10 | 0.95 ms | 196 |398+| Rows: 8 categories; SUM and COUNTA | 10 | 0.77 to 0.95 ms | 196 |378399 | Rows: 8 x 8 categories with subtotals; columns: 8; SUM and AVERAGE | 75 | 2.7 to 2.9 ms | 2.5 k |379400 | Rows: 1000 items sorted by their sum; SUM and COUNTUNIQUE | 1001 | 3.5 to 4.2 ms | 14 k |380401 | Frequency table of the 1000 items | 1001 | 2.5 to 2.8 ms | 24 k |381402 382-Groups are found by a comparable key struct rather than a string built383-per row: that took the 8-category pivot from 8.4 k to 196 allocations384-per edit, the others from 32 to 39 k to the figures above.385-386403 ## Macros387404 388405 A macro's script runs on its own goroutine; every call it makes to the@@ -404,151 +421,11 @@ found through the sheet's index. Every run stops after 10
404421 million Starlark steps (`macro.DefaultMaxSteps`), about a second of pure405422 computation.406423 407-## Step A: the grid408-409-The grid grew from 8192 x 256 to 1,048,576 x 16,384 once nothing cost410-what the grid costs. Beside the cell map, occupancy indexes (bitmaps of411-1024 rows per column) say which cells are stored and which have412-contents; ranges are read through them, so a whole column of ten cells is413-ten visits. Formats of whole columns, rows and the sheet live on the414-lines. `TestCommandsCostTheDataNotTheGrid` (`internal/ui`) runs every415-command with the whole sheet, a whole column and a whole row selected on416-the full grid and fails past 250 ms or 8 MB; each takes 0.1 to 4 ms.417-Pasting whole columns or rows sets line formats; a pasted block whose418-source or destination has line formats gives its blank cells formatting419-of their own up to 65,536 cells, as formatting a block does, and past420-that only the cells stored.421-422-Code that cost the grid, and what it does now:423-424-| Where | Was | Now |425-|---|---|---|426-| Range reads for aggregates (then `reader.cells`; now `Book.Fold` and `Book.Scan`) | every address of the range | the stored cells, in row order |427-| SUMIF(S), COUNTIF(S), AVERAGEIF(S), SUMPRODUCT, COUNTBLANK | a mask over every cell | the cells any range holds; the blanks between counted at once |428-| MATCH, VLOOKUP, HLOOKUP, XLOOKUP | every entry of the column searched | the stored entries, the blanks between as one; ROWS and INDEX keep the full size |429-| Range users (`rangeIndex`) | a fixed array of 256 column buckets, scanned per change | interval trees per used column, whole rows in one |430-| Formatting whole columns or rows | up to 65,536 cells made, then only existing ones | column, row and sheet formats |431-| Used range, Ctrl+arrow, text running into view | every cell, or every row or column between | the index of filled cells |432-| Filter over whole columns | every row tested and hidden one by one | rows with data, one test for the blank rest, stepped over at once |433-| Sort | every row of the range | the rows with cells |434-| Paste, fill of blanks | every address of the destination | the clip's cells per tile; what blanks land on is cleared |435-| Copy to the system clipboard | every blank row between values | refused past `max-cells` cells |436-| Pivot sources, JEV value ranges, chart detection | every row of the range | the rows with data; JEV ranges past 4096 cells trimmed to their data |437-| Exports of a selection, script reads | every row of whole columns | trimmed to the data |438-| Row numbers, name box | 5 digits, IV | 7 digits (the header widens past 9999), XFD |439-| Column width reset, set | an undo step per column of the selection | one step; a reset touches only columns with a width |440-| XLSX column widths and styles | 256 columns | the data's width (at least 256), a style on A and XFD as the sheet's |441-442-Before and after, back to back (`-benchtime 1s`, medians of one run):443-444-| Benchmark | Before | After |445-|---|---|---|446-| Edit fan-in, 1000 x SUM(A1:A8192) | 176 ms | 0.72 ms |447-| Edit running totals, 8192 x SUM($A$1:An) | 0.73 s | 2.7 ms |448-| RecalcAll fan-in / running totals | 178 / 739 ms | 1.1 / 2.6 ms |449-| Undo + redo fan-in / running totals | 352 ms / 1.47 s | 1.4 / 5.4 ms |450-| Typing an entry under 1000 SUMs, through to its frame (200 x 60) | 59 ms | 0.91 ms |451-| Edit sparse-1M (10,000 numbers over a million rows, 3000 whole-column formulas) | | 2.6 ms |452-| Frame at 200 x 60 on sparse-1M; arrow keys at XFD1048576 | | 0.68 ms; 0.65 ms |453-| Edit a cell of 8192 x 26 numbers | 619 ns | 662 ns |454-| Open 8192 x 26 / 8192 x 256 | 127 ms / 1.97 s | 130 ms / 1.99 s |455-| Build 8192 x 26 through `Load` | 88 ms | 90 ms |456-| BigUndo: clear 8192 x 26, undo | 159 ms | 164 ms |457-| Sort 8191 rows | 31 ms | 32 ms |458-| Pivot over 8191 rows, 8 categories | 0.89 ms | 0.77 ms |459-| Extend a selection of 8192 x 256 by a row | 31 ms | 35 ms (measured alone; 0.41 ms once merged with main's selection statistics, below) |460-461-The index costs a few percent on edits and loads (keeping two bitmaps462-current per cell), and running aggregates cost memory for their463-checkpoints (a 1000-SUM fan-in allocates 2 MB per recalculation instead464-of 0.2 MB). A range is shared only from its second read in a465-recalculation, so ranges read once cost nothing extra.466-467-### Merged with main468-469-Step A was merged with main at 7431646 (selection statistics by block,470-JEV recalc by question, 120 fps, depth limits, the streaming XLSX471-reader). Main's dense statistics index, one entry per 64 rows of every472-column, would have been 268 M entries at the new grid; it now lives on473-the blocks of the index of filled cells, an entry per 64 rows and a474-total per 1024, made where there is data. Main and the merge back to475-back (`-benchtime 1s`):476-477-| Benchmark | Main | Merged |478-|---|---|---|479-| Extend a selection of 8192 x 256 by a row, 80 x 24 | 0.47 ms | 0.41 ms |480-| Select all of 8192 x 256, 80 x 24 | 0.28 ms | 0.28 ms |481-| JEV, 4000 cells answered | 14.5 ms | 14.3 ms |482-| Import XLSX 8192 x 26 numbers / table / 8192 formulas | 187 / 313 / 13.6 ms | 189 / 312 / 13.8 ms |483-| Import the POI formula tests (XLSX) | 9.7 ms | 9.5 ms |484-| Edit fan-in / running totals | 186 / 777 ms | 0.73 / 2.7 ms |485-| Sort 8191 rows | 29.7 ms | 32.3 ms |486-| BigUndo | 150 ms | 156 ms |487-| Open 8192 x 26 | 117 ms | 121 ms |488-489-CSV imports take longer only because they keep more: the airport codes490-(855 k cells, 0.78 s) and OWID energy (1.04 M cells, 0.84 s) now come in491-whole. XLSX imports set column widths and styles for the data's columns492-(at least 256), without an undo step each: setting 16,384 of them one493-step at a time took the POI file from 9.7 to 44 ms before the fix.494-495-## The function library496-497-The function library moved out of the engine into `internal/functions`,498-behind a `Book` interface whose methods pass only values (see499-[architecture.md](architecture.md#functions)): SUM-like functions have500-the engine add a range up (`Fold`), other range walks read chunks into501-buffers the `Reader` reuses (`Scan`), and lookups walk positions with a502-cursor. The criteria and lookup shapes (`criteria-60xSUMIF8192`: SUMIF,503-COUNTIFS and AVERAGEIF over whole columns of 8192 rows;504-`lookup-300xVLOOKUP8192`: VLOOKUP, MATCH and XLOOKUP into 8192 keys) were505-added for it. Before and after, back to back (`-benchtime 1s`, medians of506-three alternating runs; differences under about 3% are noise):507-508-| Benchmark | Before | After |509-|---|---|---|510-| Edit fan-in, 1000 x SUM(A1:A8192) / SUM(A:A) | 742 / 741 us | 749 / 750 us |511-| Edit running totals | 2.79 ms | 2.85 ms |512-| Edit sparse-1M | 2.64 ms, 9611 allocs | 2.45 ms, 3610 allocs |513-| Edit criteria / lookup | 40.7 / 26.3 ms | 37.1 / 25.3 ms |514-| Edit chain / fan-out / volatile / dense | 1.88 / 1.39 / 1.29 ms / 673 ns | 1.85 / 1.40 / 1.29 ms / 662 ns |515-| RecalcAll fan-in / running / sparse-1M | 0.97 / 2.50 / 3.82 ms | 0.97 / 2.51 / 3.62 ms |516-| RecalcAll criteria / lookup | 66.6 / 27.5 ms | 58.8 / 26.5 ms |517-| RecalcAll dense 8192 x 26 | 32.5 ms | 31.8 ms |518-| Arithmetic, binary / decimal (1000 formulas and a SUM) | 20.6 / 160 us | 19.8 / 156 us |519-520-Allocations are the same or fewer everywhere: lookups no longer make a521-closure per search, and criteria collect positions from the occupancy522-index without a call per cell.523-524-## Hotspots found and fixed525-526-Each with its benchmark before and after, in the commit that fixed it.527-528-| Hotspot | Evidence | Before | After |529-|---|---|---|---|530-| SUM, AVERAGE, COUNT allocated twice per cell read: `return &v` of the `Value` parameter moved it to the heap on every call | CPU profile of a fan-in edit: `toNum` and `aggregate.func1.1` allocating 3 GB | Edit/fanin 320 ms, 16.4 M allocs | 203 ms, 2 k allocs |531-| Every changed cell scanned every formula with a range reference | Running totals edit: 67 M `Rect.Contains` calls | Edit/running 1.83 s | 0.70 s |532-| `RecalcAll` traced dependents from every cell (it found nothing new) | RecalcAll/names spent its time in the name scan | RecalcAll/names 157 ms, fanin 395 ms | 2.8 ms, 172 ms |533-| Overflowing text cut with `ansi.Cut` per column, rescanning the text each time | CPU profile of View on long text: 90% in `ansi.Cut` | View/longtext 400 x 120: 40.5 ms | 2.1 ms |534-| `Names()` upper-cased both names in every sort comparison, every frame | 21 k allocations per View with 1000 names | View/names 80 x 24: 662 us | 197 us |535-| Status line summed a whole-sheet selection with 2 M map lookups, every frame | CPU profile: `RangeStats` 87% of a frame | 91 ms per frame | 0.3 ms (cached), 29 ms when the selection changes |536-| JEV cache keys encoded with `encoding/json` | CPU profile of answers: JSON 60% of recalc | JEV/1000 575 ms, JEV/4000 9.3 s | 263 ms, 5.1 s |537-| Aggregates read every cell of every range, 1000 identical SUMs a thousand times, running totals O(n^2) | Fan-in and running-total edits | Edit/fanin 176 ms, running 0.73 s | 0.72 ms, 2.7 ms (running aggregates shared per recalculation) |538-| Range users bucketed by a fixed array of 256 columns, scanned whole per changed cell | 16,384 columns; whole-row ranges in every bucket | | Interval trees per column, wide ranges in one tree |539-| A range read visited every address, blank or not | A whole column is a million addresses | | Occupancy index: only stored cells are visited |540-| Changing a selection summed every cell in it | CPU profile of Shift+Up over the whole sheet: `RangeStats` 54% | extend-data 80 x 24: 31 to 44 ms | 0.53 ms |541-| Every JEV answer recalculated every JEV cell | Per-answer cost grew with the number of JEV cells | JEV/4000 4.9 s | 15 ms |542-| Bubble Tea drew at most 60 frames a second | Arrow keys through a terminal waited for the next frame | p50 16.6 ms, p95 18.4 ms | 8.3 ms, 10.2 ms |543-| Lazy evaluation recursed once per link of a chain | A chain through all 2 M cells of a sheet overflowed the 1 GB stack and killed the program | crash | 4 s to build and evaluate, under 32 MB of stack |544-545424 ## What would raise the bounds546425 547-In order of value for effort. The grid is Excel's since step A; the548-first two are what raising `max-cells` toward Google Sheets' 10 M cells549-would take; without them a 10 M-cell sheet would need about 3 GB of550-heap and take 10 s to open. (Shared range results, which was the second,551-is done: see Step A.)426+In order of value for effort. The first two are what raising427+`max-cells` toward Google Sheets' 10 M cells would take; without them a428+10 M-cell sheet would need about 3 GB of heap and take 10 s to open.552429 553430 1. **Compact cell storage** (L, 2 to 3 weeks). Cells live in a554431 `map[Addr]*Cell` behind `cellStore` (`internal/sheet/store.go`), 300 B@@ -569,20 +446,9 @@ is done: see Step A.)
569446 allocation tenfold; a columnar or gzip-compressed variant would cut570447 the size about fourfold. The format is `internal/sheet/file.go`, apart571448 from the cell store.572-3. **Incremental selection statistics.** Done: per-column block573- aggregates, marked stale as cells change (see [Rendering](#rendering));574- extending a whole-sheet selection went from 31 to 44 ms to 0.53 ms.575-4. **JEV recalc by question.** Done: an answer recalculates the cells576- waiting for it, and answers within a frame recalculate together (see577- [JEV](#jev)); answering 4000 questions went from 4.9 s of CPU to578- 15 ms.579-5. **Smaller undo steps** (S, 1 day). The history is capped by the449+3. **Smaller undo steps** (S, 1 day). The history is capped by the580450 memory its before-images hold; storing formatting-only changes as581451 diffs rather than whole cells would let it keep more of them.582-6. **Frame rate.** Done: 120 frames a second, Bubble Tea's maximum,583- locally and over SSH; arrow keys through a terminal went from p50584- 16.6 ms to 8.3 ms, with no more CPU when idle.585-7. **Recursion limits.** Done: see [Formula depth](#formula-depth).586452 587453 ## Measuring588454 @@ -593,6 +459,8 @@ make stress-report # latest against previous and baseline, with trends
593459 make stress-e2e # key press to screen through libghostty594460 ```595461 462+`make stress` fetches the datasets with `scripts/stress-data.sh`, which463+checks each against its checksum in `scripts/stress-data.tsv`.596464 `TestCommandsCostTheDataNotTheGrid` runs with `go test ./internal/ui`597465 (it skips under `-short`) and guards the grid: a command whose cost grows598466 with its selection's area fails it.@@ -2,7 +2,7 @@
22 33 ```sh44 make check # all of the below that must pass before a push: gofmt, vet, lint, test, oracle, e2e5-make lint # go vet, cognitive complexity at most 25, Go files at most 500 lines5+make lint # go vet, staticcheck, cognitive complexity at most 25, Go files at most 500 lines, doclint (docs describe the code, not its history)66 make test # engine, file formats and UI unit tests77 make fuzz # fuzz the formula parser and the CSV, .wk1 and XLSX readers88 make oracle # compare formulas and number formats with excelize@@ -93,3 +93,9 @@ It needs vhs 0.12+, ttyd and ffmpeg. The JEV tape talks to `demos/fakejev`, a
9393 local stand-in the target starts on 127.0.0.1, never the real service. On9494 macOS, `demos/lib/ttyd` wraps ttyd so Alt+letter reaches the app. VHS has no9595 mouse commands, so the tapes use the keyboard.96+97+## No broken windows98+99+`make check` must pass before a push, and warnings are fixed or turned100+off with a written reason rather than left standing. The rules are in101+[CLAUDE.md](../CLAUDE.md).@@ -99,16 +99,6 @@ func (w *xlsxWriter) sheet(bw *bufio.Writer, ws string, snap *Snapshot, active b
9999 return r.To.Row - r.From.Row + 1100100 }101101 102-func mapKeys(m map[int]int) func(func(int) bool) {103- return func(yield func(int) bool) {104- for k := range m {105- if !yield(k) {106- return107- }108- }109- }110-}111-112102 // cell appends snapshot cell c at a as a <c> element. A formula's value113103 // is written as its cached result; a formula with no Excel equivalent,114104 // or naming a sheet that isn't written, is written as its value alone,@@ -8,8 +8,8 @@ import "github.com/FelineStateMachine/012/internal/value"
88 //99 // Every method takes and returns plain values, never a callback: a1010 // function value or pointer passed through an interface escapes to the11-// heap, so a callback per range read would cost an allocation where the12-// engine used to cost none. Ranges are read instead in chunks, into11+// heap, so a callback per range read would cost an allocation that an12+// engine reading its own cells doesn't pay. Ranges are read instead in chunks, into1313 // buffers the Reader keeps and reuses (Scan), and each function walks1414 // them with callbacks of its own that stay on the stack.1515 type Book interface {@@ -144,20 +144,6 @@ func DotEnvHasKey(dir string) bool {
144144 return false145145 }146146 147-// LoadConfig reads TYPESAFE_API_KEY, TYPESAFE_BASE_URL and148-// TYPESAFE_DEFAULT_MODEL from the environment only, reporting false when149-// there's no key or the base URL isn't https (or localhost). The dirs150-// are ignored: .env files are no longer read. New code resolves the key151-// with ResolveKey and takes the rest from internal/config.152-func LoadConfig(dirs ...string) (Config, bool) {153- c := Config{APIKey: strings.TrimSpace(os.Getenv(KeyEnv)), BaseURL: strings.TrimSpace(os.Getenv("TYPESAFE_BASE_URL")),154- Model: strings.TrimSpace(os.Getenv("TYPESAFE_DEFAULT_MODEL"))}155- if c.BaseURL != "" && config.CheckBaseURL(c.BaseURL) != nil {156- return c, false157- }158- return c, c.APIKey != ""159-}160-161147 // Lazy is a Client that connects the first time a question is asked,162148 // with connect, and remembers the result, error included. 012 uses it for163149 // jev-api-key-command, so a password manager is only asked for the key@@ -41,8 +41,9 @@ func TestKeysAreDistinct(t *testing.T) {
4141 m[l] = l + "!"4242 }4343 c := sheet.RemoteCall{Kind: "choice", State: "x", Instructions: "q", Criteria: m}44+ want := Key(c)4445 for range 20 {45- if Key(c) != Key(c) {46+ if got := Key(c); got != want {4647 t.Fatal("map order changed the key")4748 }4849 }@@ -130,14 +130,6 @@ func (o *occupancy) colsIn(c0, c1 int) []int {
130130 return o.colIDs[i:j]131131 }132132 133-// colCells is the number of cells stored in column c.134-func (o *occupancy) colCells(c int) int {135- if ci := o.col(c); ci != nil {136- return ci.n137- }138- return 0139-}140-141133 // blockIDs returns the numbers from b0 to b1 of the blocks that any of142134 // cols has, ascending.143135 func (o *occupancy) blockIDs(cols []int, b0, b1 int) []int {@@ -83,10 +83,9 @@ func LookupFunc(name string) (*FuncDef, bool) { return functions.LookupFunc(name
8383 // Funcs returns every function, sorted by name.8484 func Funcs() []*FuncDef { return functions.Funcs() }8585 86-func num(v float64) Value { return value.Num(v) }87-func boolean(b bool) Value { return value.Boolean(b) }88-func toNum(v Value) (float64, *Value) { return value.ToNum(v) }89-func text(v Value) string { return value.AsText(v) }90-func compare(l, r Value) int { return value.Compare(l, r) }86+func num(v float64) Value { return value.Num(v) }87+func boolean(b bool) Value { return value.Boolean(b) }88+func text(v Value) string { return value.AsText(v) }89+func compare(l, r Value) int { return value.Compare(l, r) }9190 9291 func clampInt(v, lo, hi int) int { return max(lo, min(v, hi)) }@@ -210,7 +210,7 @@ func TestChartImages(t *testing.T) {
210210 t.Errorf("unchanged chart sent again")211211 }212212 press(t, m, "<f5>", "B2", "<enter>", "99", "<enter>")213- _, cmd = m.Update(nil)213+ m.Update(nil)214214 raw = nil215215 m.term.sent[firstImageID] = "stale"216216 collect(m.term.syncImages(m.sheet, m.displayCharts, &m.th, m.spans))@@ -72,7 +72,7 @@ func (t *terminal) handle(msg tea.Msg) tea.Cmd {
7272 // Ask for the cell size (for image proportions) and the7373 // palette colors charts draw with.7474 var q strings.Builder75- q.WriteString(ansi.WindowOp(ansi.RequestCellSizeWinOp))75+ q.WriteString(ansi.WindowOp(16)) // 16: report the cell size in pixels7676 for i := range 16 {7777 fmt.Fprintf(&q, "\x1b]4;%d;?\x07", i)7878 }@@ -109,7 +109,7 @@ func (e *pivotEditor) pivot(m pivotHost) sheet.Pivot {
109109 return p110110 }111111 112-// items lists the editor's lines for the pivot as it is now.112+// items lists the editor's lines for the pivot's current definition.113113 func (e *pivotEditor) items(m pivotHost) []pivotItem {114114 p := e.pivot(m)115115 items := []pivotItem{{kind: itemSource}}@@ -0,0 +1,163 @@
1+// Command doclint keeps documentation about the code as it is. It reads2+// Go comments and Markdown prose (not code, strings or code blocks) and3+// reports wording that narrates how the code got here: what changed,4+// what it did before, what's temporary, who did it and when. History5+// belongs in commit messages; docs and comments say what is and why.6+//7+// Saying why something stays compatible with a planned change is fine8+// ("kept a pointer so the store can change shape"), since it explains9+// the code today. A line that must use a flagged word says so with10+// "doclint:allow" and a reason, in a comment on that line (or an HTML11+// comment in Markdown).12+//13+// Usage: go run ./scripts/doclint [dirs or files...]14+package main15+16+import (17+ "bufio"18+ "fmt"19+ "go/parser"20+ "go/token"21+ "io/fs"22+ "os"23+ "path/filepath"24+ "regexp"25+ "strings"26+)27+28+// rules are the phrasings that describe development rather than the code.29+var rules = []struct {30+ re *regexp.Regexp31+ why string32+ markdown bool // only in Markdown, where the phrase can't be about code33+ notTests bool // not in tests, which narrate the steps they take34+}{35+ {re: re(`\bno longer (read|supported|used|needed|exists?|works?|possible|available|goes|runs)\b`), why: "describes a past state"},36+ {re: re(`\bused to (be|have|take|go|cost|read|write|live|run|do|make|scan|walk|say|show)\b|\bpreviously\b|\bformerly\b|\boriginally\b`), why: "describes a past state"},37+ {re: regexp.MustCompile(`\b(it|this|that|they|which|012|[a-z]+s) now (uses?|takes?|goes|lives?|sits?|keeps?|returns?|comes?|reads?|writes?|has|have|is|are|gets?|works?)\b|\b(is|are) now\b`), why: `"now" narrates a change`, notTests: true},38+ {re: re(`\b(was|were|has been|have been) (moved|renamed|replaced|rewritten|split|merged)\b|\b(moved|renamed) (from|out of|into)\b`), why: "narrates a change"},39+ {re: re(`\bfor now\b|\btemporarily\b|\bat the moment\b|\bfor the time being\b|\buntil (it|this|that) lands\b`), why: "describes a transient state"},40+ {re: regexp.MustCompile(`\b(TODO|FIXME|XXX|HACK)\b`), why: "a work note; put it in ROADMAP.md instead"},41+ {re: regexp.MustCompile(`\(new\)|\bnew in v?\d|\bas of v\d|\bsince v\d`), why: "dates the text"},42+ {re: re(`\bbefore (and|->|→|/) after\b|\bthis (commit|change|PR)\b|\bin the commit that\b`), why: "narrates a change", markdown: true},43+ {re: re(`\b(the lead|an? agent|merged with main|merge of main)\b`), why: "narrates who changed it"},44+}45+46+func re(s string) *regexp.Regexp { return regexp.MustCompile(`(?i)` + s) }47+48+// skip lists paths whose purpose is history or plans: the roadmap tracks49+// what's done and ahead, NOTICE and LICENSE are legal text.50+var skip = map[string]bool{"ROADMAP.md": true, "NOTICE": true, "LICENSE": true}51+52+type finding struct {53+ file string54+ line int55+ text string56+ why string57+}58+59+func main() {60+ roots := os.Args[1:]61+ if len(roots) == 0 {62+ roots = []string{"."}63+ }64+ var found []finding65+ for _, root := range roots {66+ filepath.WalkDir(root, func(path string, d fs.DirEntry, err error) error {67+ if err != nil {68+ return err69+ }70+ name := d.Name()71+ if d.IsDir() {72+ if path != root && (strings.HasPrefix(name, ".") || name == "testdata" || name == "node_modules") {73+ return filepath.SkipDir74+ }75+ return nil76+ }77+ if skip[filepath.ToSlash(path)] || skip[name] {78+ return nil79+ }80+ switch filepath.Ext(name) {81+ case ".go":82+ found = append(found, goComments(path)...)83+ case ".md":84+ found = append(found, markdown(path)...)85+ }86+ return nil87+ })88+ }89+ for _, f := range found {90+ fmt.Printf("%s:%d: %s: %s\n", f.file, f.line, f.why, strings.TrimSpace(f.text))91+ }92+ if len(found) > 0 {93+ fmt.Printf("doclint: %d lines describe development rather than the code\n", len(found))94+ os.Exit(1)95+ }96+}97+98+// check reports the rules text breaks, unless it's allowed.99+func check(file string, line int, text string) []finding {100+ if strings.Contains(text, "doclint:allow") {101+ return nil102+ }103+ md := strings.HasSuffix(file, ".md")104+ test := strings.HasSuffix(file, "_test.go")105+ var out []finding106+ for _, r := range rules {107+ if r.markdown && !md || r.notTests && test {108+ continue109+ }110+ if r.re.MatchString(text) {111+ out = append(out, finding{file, line, text, r.why})112+ break113+ }114+ }115+ return out116+}117+118+func goComments(path string) []finding {119+ fset := token.NewFileSet()120+ f, err := parser.ParseFile(fset, path, nil, parser.ParseComments)121+ if err != nil {122+ return nil // not ours to judge; the compiler reports it123+ }124+ var out []finding125+ for _, g := range f.Comments {126+ for _, c := range g.List {127+ line := fset.Position(c.Pos()).Line128+ for i, l := range strings.Split(c.Text, "\n") {129+ out = append(out, check(path, line+i, l)...)130+ }131+ }132+ }133+ return out134+}135+136+func markdown(path string) []finding {137+ f, err := os.Open(path)138+ if err != nil {139+ return nil140+ }141+ defer f.Close()142+ var out []finding143+ fenced := false144+ sc := bufio.NewScanner(f)145+ sc.Buffer(make([]byte, 1<<20), 1<<20)146+ for n := 1; sc.Scan(); n++ {147+ l := sc.Text()148+ if strings.HasPrefix(strings.TrimSpace(l), "```") {149+ fenced = !fenced150+ continue151+ }152+ if fenced {153+ continue154+ }155+ out = append(out, check(path, n, stripCode(l))...)156+ }157+ return out158+}159+160+var inlineCode = regexp.MustCompile("`[^`]*`")161+162+// stripCode drops inline code spans, which quote code rather than narrate.163+func stripCode(l string) string { return inlineCode.ReplaceAllString(l, "``") }@@ -1,16 +1,32 @@
11 2-# Code shape limits, checked by make lint:2+# Checked by make lint:3+# - staticcheck, with staticcheck.conf, in every module34 # - no function with cognitive complexity over 25 (gocognit)45 # - no Go file over 500 lines5-# gocognit runs with go run at a pinned version, so it never enters go.mod.6+# - no docs or comments narrating history (scripts/doclint)7+# The tools run with go run at pinned versions, so they never enter go.mod.68 set -eu79 cd "$(dirname "$0")/.."810 911 GOCOGNIT=github.com/uudashr/gocognit/cmd/gocognit@v1.2.112+STATICCHECK=honnef.co/go/tools/cmd/staticcheck@2026.2.11013 MAX_COGNIT=251114 MAX_LINES=5001215 fail=01316 17+# Warnings are fixed, or turned off in staticcheck.conf with a reason;18+# none are left standing.19+for mod in . oracle e2e; do20+ if [ "$mod" = e2e ] && [ -z "${PKG_CONFIG_PATH:-}" ]; then21+ PKG_CONFIG_PATH="$(pwd)/.deps/ghostty/share/pkgconfig"22+ export PKG_CONFIG_PATH23+ fi24+ if ! (cd "$mod" && go run $STATICCHECK ./...); then25+ echo "staticcheck: findings in $mod"26+ fail=127+ fi28+done29+1430 dirs=$(find cmd internal demos e2e oracle -name '*.go' -exec dirname {} \; | sort -u)1531 over=$(go run $GOCOGNIT -over $MAX_COGNIT $dirs || true)1632 if [ -n "$over" ]; then@@ -26,4 +42,9 @@ if [ -n "$long" ]; then
2642 fail=12743 fi2844 45+# Docs and comments describe the code as it is, not how it got here.46+if ! go run ./scripts/doclint; then47+ fail=148+fi49+2950 exit $fail