- commit
- 0f0a99af8608bcd372e0f9b3a1196403e98a28b4
- parent
- 6ce1133cb4b24a4cfbfdd266942bb94887b4d733
- parent
- 2d53d2dee28ed51bb050118f152869e68c74d671
- author
- Dami <dami.etoile@gmail.com>
| M | Makefile | 4 | +- |
| M | ROADMAP.md | 3 | +- |
| M | deploy/ | 2 | +- |
| M | docs/ | 16 | ++++++--- |
| M | docs/ | 54 | ++++++++++++++---------------- |
| M | docs/ | 5 | ++- |
| M | docs/ | 2 | - |
| M | docs/ | 10 | +++--- |
| M | scripts/ | 45 | ++++++++++++++++--------- |
@@ -58,7 +58,7 @@ screens: $(GHOSTTY_STAMP)
5858 # Stress: fetch real datasets into .deps/stress, run the benchmarks built5959 # with -tags stress, print a summary and record the run in6060 # .deps/stress/results/runs.jsonl. BENCH, BENCHTIME and PKGS narrow it;61-# see docs/limits.md. Plain `go test ./...` never runs these.61+# see docs/contributing/limits.md. Plain `go test ./...` never runs these.6262 stress:6363 scripts/stress/run.sh6464 @@ -72,7 +72,7 @@ stress-report:
7272 scripts/stress/report.sh7373 7474 # The observability stack (deploy/observability): an OpenTelemetry75-# Collector, ClickHouse and Grafana in Docker. See docs/observability.md.75+# Collector, ClickHouse and Grafana in Docker. See docs/contributing/observability.md.7676 obs-up:7777 scripts/obs.sh up7878 @@ -39,6 +39,7 @@ rises in measured steps; see [Bounds of support](docs/contributing/limits.md#wha
3939 |---|---|---|4040 | A streaming or binary `.012` format next to the readable JSON one | Open and save scale with the data | M |4141 | Smaller undo steps: plain cells' before-images as slots, formatting changes as diffs | More history in the same memory; clearing a full sheet costs what the sheet does | S to M |42+| Compact spilled and pivot cells: derived cells in the compact slot form instead of whole `Cell`s | Large spills and pivots cost what plain cells cost ([Bounds of support](docs/contributing/limits.md#what-would-raise-the-bounds)) | M |4243 | Linked, paged read-only ranges over Parquet and SQLite that feed pivots and formulas by streaming | Sources too big for any grid | L |4344 4445 ### 4. Macros, keys and the terminal@@ -77,7 +78,7 @@ by a gateway dialing the iroh ticket.
7778 - Dynamic arrays that spill (FILTER, SORT, UNIQUE, SEQUENCE and more, ARRAYFORMULA), LET and LAMBDA, SPLIT and the REGEX functions: [Arrays and spills](docs/formulas/arrays.md)7879 - Opt-in decimal arithmetic for money: [Decimal arithmetic](docs/formulas/decimal.md)7980 - An Excel-sized grid with a `max-cells` budget; operations cost the data, not the grid: [Bounds of support](docs/contributing/limits.md#sheet-size)80-- Compact column storage: 20 B a number, a ten-million-cell `max-cells` budget, 3 ns cell reads: [Bounds of support](docs/contributing/limits.md#sheet-size)81+- Compact column storage and a ten-million-cell `max-cells` budget: [Bounds of support](docs/contributing/limits.md#sheet-size)8182 8283 **Finding and using features**8384 @@ -1,5 +1,5 @@
11 # Local observability stack for 012: an OpenTelemetry Collector feeding2-# ClickHouse, and Grafana on top. See docs/observability.md.2+# ClickHouse, and Grafana on top. See docs/contributing/observability.md.33 #44 # make obs-up start it and print the environment for 01255 # make obs-status containers, endpoints and row counts@@ -100,12 +100,16 @@ style.
100100 cost of what it holds; the stored index's blocks hold the cells101101 themselves, a 16-byte slot each in row order (`slot.go`). A plain cell102102 (a number, boolean or text as typed, with a format and style) lives in103- its slot, with its text in a table of strings and its formatting in a104- table of looks; formulas, notes, and what pivots and spills write are105- whole `Cell`s in a side table, which recalculation updates in place.106- `get` hands out a plain cell as a `Cell` made for the caller, a copy107- whose changes reach nothing; `set` is the one way to change a cell. Formats of whole columns and rows, and of the108- whole sheet, live on the lines (`lines.go`); a cell falls back on them.103+ its slot, with its text in a table of strings kept once each, its104+ formatting in a table of looks, and its input only when that isn't105+ the value's own text ("1.50" is kept as 1.5 printed with two106+ decimals). Formulas, notes, and what pivots and spills write are whole107+ `Cell`s in a side table, which recalculation updates in place. `get`108+ hands out a plain cell as a `Cell` made for the caller, a copy whose109+ changes reach nothing; `set` is the one way to change a cell. What110+ each costs is in [Bounds of support](limits.md#sheet-size). Formats of111+ whole columns and rows, and of the whole sheet, live on the lines112+ (`lines.go`); a cell falls back on them.109113 Copy, paste and move carry the formatting cells show (`clipfmt.go`):110114 whole lines as line formats, blocks as the cells' own. A line's format111115 changing recalculates the formulas reading any cell of it, found through@@ -29,7 +29,7 @@ it lags, and past a second it stalls.
2929 | Incremental recalc | A change that makes formulas read under about 500,000 cells in total (fan-out, chains and volatiles of 8192 cells: 1.3 to 1.9 ms); 1000 SUMs over a full column: 0.53 ms; 8192 running totals: 2.7 ms | 60 criteria functions (SUMIF, COUNTIFS, AVERAGEIF) over whole columns of 8192 rows: 29 ms an edit | | About 3 ns per cell read: an index into the column's block |3030 | Full recalc | Any sheet: numbers and text hold their values and cost nothing; 1000 full-column SUMs 0.5 ms; 8192 running totals 1.8 ms | 60 whole-column criteria functions over 8192 rows: 44 ms | | Same as above |3131 | Rendering | Any sheet at up to 200 x 60: 1 ms a frame; 400 x 120: 4.5 ms; a color scale on every cell shown adds 0.4 ms at 200 x 60 | 20 charts at 400 x 120: 7 ms | | View building styled strings, then Bubble Tea parsing and diffing them |32-| Selection statistics | Any selection: extending one over all 2 M cells costs 0.3 ms a key | | | Per column with data, 1024 or 64 rows at a time from an index on the blocks of filled cells, plus the rows at the selection's ends |32+| Selection statistics | Any selection: extending one over all 2.1 M cells of 8192 x 256 costs 0.3 ms a key | | | Per column with data, 1024 or 64 rows at a time from an index on the blocks of filled cells, plus the rows at the selection's ends |3333 | Imports | CSV, SQLite, Parquet: 2 to 4 M cells/s (a million cells in 0.25 to 0.4 s); XLSX numbers or text: 0.8 to 1.4 M cells/s | XLSX with formulas: 0.4 to 0.6 M cells/s | Data past `max-cells` or the grid (dropped, with a note); XLSX files past the reader's limits (refused) | Building cells one at a time; XML decoding; XLSX formula translation |3434 | Undo | One step of any size: undo costs what the edit cost | History capped at 100 steps and 256 MB of before-images: 100 whole-column steps hold 280 MB | | Whole-cell before-images, about 300 B per cell per step |3535 | JEV | 4000 JEV cells: 4 us of CPU per answer, 15 ms to answer them all | | | An answer recalculates the cells that asked it; answers within a frame recalculate together |@@ -67,17 +67,14 @@ Heap per non-blank cell after loading (`BenchmarkMemory`,
6767 `BenchmarkImport`): numbers 20.5 B, opened from a file too; formulas 7506868 B (the parsed tree and reference lists); imported CSV, SQLite and6969 Parquet 21 to 63 B; text about 60 B plus its length, once per distinct string70-(a column of 90 texts repeated takes 27 B a cell). The store71-(`internal/sheet/store.go`, `slot.go`) keeps each column in blocks of72-1024 rows, a 16-byte slot per cell with contents or formatting: a plain73-cell (a number, boolean or text as typed, with a format and style) holds74-its value there, its text in a table of strings kept once each, its75-format and style in a table of looks, and its input only when that76-isn't the value's own text ("1.50" is kept as 1.5 printed with two77-decimals). Formulas, notes, and the results pivots and spills write are78-whole `Cell`s (216 B) in a side table. Reading a cell's value is an79-index into its column's block: 2.8 ns (`BenchmarkRead`), 8.6 ns a cell80-read by a SUM in a full recalculation.70+(a column of 90 texts repeated takes 27 B a cell). A plain cell (a71+number, boolean or text as typed, with a format and style) is a 16-byte72+slot in its column's block of 1024 rows; formulas, notes, and the73+results pivots and spills write are whole `Cell`s, about 300 B each with74+their entry and input (the `Cell` itself is 216 B). How the store lays75+them out is in [Architecture](architecture.md#the-engine). Reading a76+cell's value is an index into its column's block: 2.8 ns77+(`BenchmarkRead`), 8.6 ns a cell read by a SUM in a full recalculation.8178 8279 ## The grid8380 @@ -499,25 +496,26 @@ computation.
499496 500497 ## What would raise the bounds501498 502-In order of value for effort. `max-cells` stands at ten million, as503-Google Sheets' limit does; the sheet itself would hold several times499+In order of value for effort; sizes and scheduling are in the500+[roadmap](../../ROADMAP.md#3-scale). `max-cells` stands at ten million,501+as Google Sheets' limit does; the sheet itself would hold several times504502 that in a few GB, and these are what stand in the way.505503 506-1. **Streaming, compact file format** (M, 1 week). The `.012` file is507- JSON decoded whole: ten million cells take 8 s and 2.7 GB at the508- peak to open, 5.7 GB allocated, for a sheet that then holds 205 MB.509- A streaming decoder over the same format would roughly halve open510- time and cut allocation tenfold; a columnar or gzip-compressed511- variant would cut the size about fourfold. The format is512- `internal/sheet/file.go`, apart from the cell store.513-2. **Smaller undo steps** (S to M). A step keeps a whole `Cell` for each514- cell it changed, about 300 B where the store keeps a number in 20,515- and formatting-only changes as whole cells too. Keeping plain cells'516- before-images as slots, and formatting changes as diffs, would let a517- step over a full sheet cost what the sheet does.504+1. **Streaming, compact file format.** The `.012` file is JSON decoded505+ whole: ten million cells take 8 s and 2.7 GB at the peak to open,506+ 5.7 GB allocated, for a sheet that then holds 205 MB. A streaming507+ decoder over the same format would roughly halve open time and cut508+ allocation tenfold; a columnar or gzip-compressed variant would cut509+ the size about fourfold. The format is `internal/sheet/file.go`,510+ apart from the cell store.511+2. **Smaller undo steps.** A step keeps a whole `Cell` for each cell it512+ changed, formatting-only changes included (see [Undo](#undo)).513+ Keeping plain cells' before-images as slots, and formatting changes514+ as diffs, would let a step over a full sheet cost what the sheet515+ does.518516 3. **Compact spilled and pivot cells.** What arrays and pivots write is519- kept as whole `Cell`s (about 300 B each, 516 k of them in the arrays520- stress shape); their values and inferred formats would fit in slots.517+ kept as whole `Cell`s (516 k of them in the arrays stress shape);518+ their values and inferred formats would fit in slots.521519 522520 ## Measuring523521 @@ -47,8 +47,9 @@ while typing enters the same entry into every selected cell.
4747 4848 ## Undo4949 50-Ctrl+Z undoes and Ctrl+Y or Ctrl+Shift+Z redoes, 100 steps deep, across51-every sheet: undo shows the sheet a step changed. Every change is covered:50+Ctrl+Z undoes and Ctrl+Y or Ctrl+Shift+Z redoes, 100 steps deep (fewer51+when steps are very large, the oldest dropped first: see52+[Bounds of support](../contributing/limits.md#undo)), across every sheet: undo shows the sheet a step changed. Every change is covered:5253 typing, clearing, paste, fill, sort, insert and delete, formats, widths,5354 names, charts, freeze, filters, pivot tables, and adding, deleting,5455 renaming, moving and duplicating sheets. A multi-cell change is one step,@@ -29,5 +29,3 @@ colors that follow light and dark.
2929 | Curly, colored underlines | error cells | kitty, Ghostty, WezTerm, iTerm2, Alacritty |3030 | Background color query and mode 2031 | light and dark themes, following system changes | most terminals; 2031 in Ghostty and kitty |3131 | Synchronized output (2026) | flicker-free redraws | kitty, Ghostty, WezTerm, iTerm2, Alacritty |32-33-Everything degrades quietly where it isn't supported.@@ -190,8 +190,8 @@ check protects the local app from other programs writing the file.
190190 - An idle session is closed without saving over its file; unsaved191191 changes go to a recovery file instead.192192 - A dropped connection loses unsaved changes.193-- Each session holds about 1.3 MiB of heap on a new sheet, plus 20 to194- 60 B per cell of data of the sheets it opens (a formula takes about195- 750); two sessions opening one file hold two copies. With 50 sessions typing at once on loopback, frames196- still arrive within one frame interval (p95 16.6 ms), the same as with197- 10: see [Bounds of support](../contributing/limits.md#serving-over-ssh).193+- Each session holds its screen's buffers, about a megabyte, plus the194+ sheets it opens; two sessions opening one file hold two copies. Fifty195+ sessions typing at once get their frames as fast as ten do: see196+ [Bounds of support](../contributing/limits.md#serving-over-ssh) for197+ the measurements.@@ -154,28 +154,41 @@ func checkFrontMatter(docs map[string]*doc) []string {
154154 155155 var goDocPath = regexp.MustCompile(`\bdocs/[a-z0-9_/-]+\.md\b`)156156 157-// checkGoPaths reports doc paths named in Go code, in messages and158-// comments, that don't exist.157+// checkGoPaths reports doc paths named outside the docs, in Go code158+// (messages and comments), the Makefile, scripts and deploy configs,159+// that don't exist.159160 func checkGoPaths() []string {160161 var out []string161- for _, root := range []string{"cmd", "internal", "e2e", "oracle", "demos", "scripts"} {162+ for _, f := range []string{"Makefile", "CLAUDE.md"} {163+ out = append(out, missingDocPaths(f)...)164+ }165+ for _, root := range []string{"cmd", "internal", "e2e", "oracle", "demos", "scripts", "deploy"} {162166 filepath.WalkDir(root, func(p string, d fs.DirEntry, err error) error {163- if err != nil || d.IsDir() || filepath.Ext(p) != ".go" {164- return nil165- }166- data, err := os.ReadFile(p)167- if err != nil {168- return nil169- }170- for i, l := range strings.Split(string(data), "\n") {171- for _, m := range goDocPath.FindAllString(l, -1) {172- if _, err := os.Stat(filepath.FromSlash(m)); err != nil {173- out = append(out, fmt.Sprintf("%s:%d: %s: no such doc", p, i+1, m))174- }175- }167+ if err == nil && !d.IsDir() && pathBearing[filepath.Ext(p)] {168+ out = append(out, missingDocPaths(p)...)176169 }177170 return nil178171 })179172 }180173 return out181174 }175+176+// pathBearing are the kinds of file outside docs/ that name doc pages.177+var pathBearing = map[string]bool{".go": true, ".sh": true, ".yml": true, ".yaml": true, ".tape": true, ".sql": true, ".json": true}178+179+// missingDocPaths reports the doc paths file p names that don't exist.180+func missingDocPaths(p string) []string {181+ data, err := os.ReadFile(p)182+ if err != nil {183+ return nil184+ }185+ var out []string186+ for i, l := range strings.Split(string(data), "\n") {187+ for _, m := range goDocPath.FindAllString(l, -1) {188+ if _, err := os.Stat(filepath.FromSlash(m)); err != nil {189+ out = append(out, fmt.Sprintf("%s:%d: %s: no such doc", p, i+1, m))190+ }191+ }192+ }193+ return out194+}