# Conflicts: # internal/sheet/cellfile.go # internal/sheet/history.go # internal/ui/model.go # internal/ui/sort.go
- commit
- 39374af3c3cc4ea1e6637906008f1279dde65784
- parent
- 7ee6cd111031ef82ea6e0a876a23b09a44467e04
- parent
- 7e6ca6ecb3677812cdd0a37663ce125b49d73fc1
- author
- Dami <dami.etoile@gmail.com>
@@ -42,6 +42,7 @@ lint:
4242 4343 fuzz:4444 go test ./internal/sheet -run '^$$' -fuzz FuzzParse -fuzztime 60s45+ go test ./internal/sheet -run '^$$' -fuzz FuzzRead -fuzztime 60s4546 go test ./internal/fileio -run '^$$' -fuzz FuzzReadDelimited -fuzztime 60s4647 go test ./internal/fileio -run '^$$' -fuzz FuzzReadWK1 -fuzztime 60s4748 go test ./internal/fileio -run '^$$' -fuzz 'FuzzReadXLSX$$' -fuzztime 60s@@ -36,8 +36,6 @@ rises in measured steps; see [Bounds of support](docs/contributing/limits.md#wha
3636 3737 | Item | Result | Size |3838 |---|---|---|39-| A streaming or binary `.012` format next to the readable JSON one | Open and save scale with the data | M |40-| Smaller undo steps: plain cells' before-images as slots, formatting changes as diffs | More history in the same memory; clearing a full sheet costs what the sheet does | S to M |4139 | Compact spilled and pivot cells: derived cells in the compact slot form instead of whole `Cell`s | Large spills and pivots cost what plain cells cost ([Bounds of support](docs/contributing/limits.md#what-would-raise-the-bounds)) | M |4240 | Linked, paged read-only ranges over Parquet and SQLite that feed pivots and formulas by streaming | Sources too big for any grid | L |4341 @@ -80,6 +78,7 @@ by a gateway dialing the iroh ticket.
8078 - A locale per file, as Sheets' File > Settings > Locale: decimal commas, date order, currency and `;` in formulas, typed and shown while files store en-US's form: [Locale](docs/sheets/locale.md)8179 - An Excel-sized grid with a `max-cells` budget; operations cost the data, not the grid: [Bounds of support](docs/contributing/limits.md#sheet-size)8280 - Compact column storage and a ten-million-cell `max-cells` budget: [Bounds of support](docs/contributing/limits.md#sheet-size)81+- Undo steps in the compact form: clearing a full ten-million-cell sheet holds about what the sheet does, and a step past 1 GB asks first: [Bounds of support](docs/contributing/limits.md#undo)8382 8483 **Finding and using features**8584 @@ -110,6 +109,7 @@ by a gateway dialing the iroh ticket.
110109 **Files**111110 112111 - Import CSV, TSV, XLSX, SQLite, Parquet and Lotus `.wk1`; export CSV, TSV, XLSX and SQLite; import locations; save-as and overwrite checks: [Files](docs/files/README.md)112+- `.012` files read and written as a stream, cells straight into and out of the store: [The .012 format](docs/files/format.md#reading-and-writing)113113 114114 **Upkeep**115115 @@ -167,9 +167,16 @@ style.
167167 - **Undo.** Every mutation goes through a small set of paths that snapshot168168 the cells, widths, names, charts, view state and sheet list they change,169169 on any sheet, so a step can be reversed exactly; multi-cell operations170- are one step. A deleted sheet keeps its cells, so undo brings it back.171- The history keeps at most 100 steps and about 256 MB of before-images,172- counted as they're recorded, dropping the oldest steps first.170+ are one step. A step keeps each sheet's before-images as an image171+ (`historyimage.go`) in the store's own form: plain cells as slots in172+ columns of blocks, their text and formatting in the image's tables,173+ and whole `Cell`s only for rich ones, so clearing a whole sheet holds174+ about what the sheet does. A deleted or replaced sheet keeps its175+ cells, so undo brings it back. The history keeps at most 100 steps and176+ about 256 MB of before-images, counted as they're recorded, dropping177+ the oldest steps first; a change whose step would pass 1 GB178+ (`UndoCost`, `MaxStepBytes`) asks first, and runs `WithoutUndo` if told179+ to go on.173180 - **Pivot tables.** A sheet may hold a pivot (`pivot.go`): a definition174181 naming its source by sheet name, as formulas do, and a region of175182 derived cells from A1 that the engine owns. When a recalculation marks@@ -309,13 +316,15 @@ The package boundaries leave the bounds in [Bounds of support](limits.md) room t
309316 move without touching callers:310317 311318 - **Storage.** `cellStore`'s methods are the whole contract, so what is312- kept whole or in a slot can change (spilled and pivot cells in slots,313- plain before-images in the history) without touching callers.319+ kept whole or in a slot can change (spilled and pivot cells in slots)320+ without touching callers.314321 - **Range reads.** Shared range results, prefix sums for running totals315322 and column-block scans go behind the engine's `Scan` and `RangeAgg`,316323 the places functions read ranges.317-- **Files.** A streaming or columnar `.012` format replaces `file.go`'s318- whole-document JSON; the format is already separate from the store.324+- **Files.** The `.012` reader and writer stream (`fileread.go`,325+ `filescan.go`) and meet the store only through its methods, so another326+ encoding would sit beside them; [The .012 format](../files/format.md#reading-and-writing)327+ says why there is one.319328 - **Depth limits.** `internal/formula`'s parser caps nesting at320329 `formula.MaxDepth` (1024 levels), so a pathological formula fails to321330 parse before anything evaluates it; evaluation puts off cells past@@ -25,13 +25,13 @@ it lags, and past a second it stalls.
2525 2626 | Dimension | Comfortable | Degraded | Unsupported | Dominant cost |2727 |---|---|---|---|---|28-| Sheet size | A grid of 1,048,576 x 16,384 (A..XFD), with up to `max-cells` cells (ten million by default): navigation, drawing and every command cost what the cells cost, not the grid; ten million numbers take 205 MB and build in 2.2 s | Opening a `.012` file of ten million cells: 8 s and 2.7 GB at its peak; saving it: 3.7 s | More than `max-cells` cells: imports keep whole rows up to it and say what they dropped; larger pastes and fills are refused | JSON file format, decoded whole; heap per cell (about 20 B for numbers, 20 to 60 B for real data, 750 B for formulas) |28+| Sheet size | A grid of 1,048,576 x 16,384 (A..XFD), with up to `max-cells` cells (ten million by default): navigation, drawing and every command cost what the cells cost, not the grid; ten million numbers take 205 MB, build in 1.8 s and save in 0.54 s | Opening a `.012` file of ten million cells: 1.2 s, 311 MB of heap at its peak | More than `max-cells` cells: imports keep whole rows up to it and say what they dropped; larger pastes and fills are refused | Parsing each cell's entry from the file's JSON; heap per cell (about 20 B for numbers, 20 to 60 B for real data, 750 B for formulas) |2929 | Incremental recalc | A change that makes formulas read under about 500,000 cells in total (fan-out, chains and volatiles of 8192 cells: 1.3 to 1.9 ms); 1000 SUMs over a full column: 0.53 ms; 8192 running totals: 2.7 ms | 60 criteria functions (SUMIF, COUNTIFS, AVERAGEIF) over whole columns of 8192 rows: 29 ms an edit | | About 3 ns per cell read: an index into the column's block |3030 | Full recalc | Any sheet: numbers and text hold their values and cost nothing; 1000 full-column SUMs 0.5 ms; 8192 running totals 1.8 ms | 60 whole-column criteria functions over 8192 rows: 44 ms | | Same as above |3131 | Rendering | Any sheet at up to 200 x 60: 1 ms a frame; 400 x 120: 4.5 ms; a color scale on every cell shown adds 0.4 ms at 200 x 60, borders on every cell and wrapped text 0.4 ms | 20 charts at 400 x 120: 7 ms | | View building styled strings, then Bubble Tea parsing and diffing them |3232 | Selection statistics | Any selection: extending one over all 2.1 M cells of 8192 x 256 costs 0.3 ms a key | | | Per column with data, 1024 or 64 rows at a time from an index on the blocks of filled cells, plus the rows at the selection's ends |3333 | Imports | CSV, SQLite, Parquet: 2 to 4 M cells/s (a million cells in 0.25 to 0.4 s); XLSX numbers or text: 0.8 to 1.4 M cells/s | XLSX with formulas: 0.4 to 0.6 M cells/s | Data past `max-cells` or the grid (dropped, with a note); XLSX files past the reader's limits (refused) | Building cells one at a time; XML decoding; XLSX formula translation |34-| Undo | One step of any size: undo costs what the edit cost | History capped at 100 steps and 256 MB of before-images: 100 whole-column steps hold 280 MB | | Whole-cell before-images, about 300 B per cell per step |34+| Undo | One step of any size: undo costs what the edit cost; clearing all ten million cells of a full sheet holds 193 MB | History capped at 100 steps and 256 MB of before-images: 100 whole-column steps hold 15 MB | A step past 1 GB (millions of formulas or long distinct texts at once): asks, and runs without undo if told to | Before-images in the store's form: a slot (20 B) per plain cell, a whole `Cell` per formula |3535 | JEV | 4000 JEV cells: 4 us of CPU per answer, 15 ms to answer them all | | | An answer recalculates the cells that asked it; answers within a frame recalculate together |3636 | Formula depth | 1000 nested parentheses or IFs: under 0.5 ms; chains of formulas through every cell of a sheet | | More than 1024 levels of nesting in one formula: a parse error | Recursive parser; evaluation puts off cells past 65,536 levels |3737 | Macros | Replaying 1000 recorded actions: 2.2 ms, one undo step; a script's call to the sheet: about 1.4 us | | Scripts past 10 M Starlark steps: stopped, with the line | One message per call to the sheet, served in batches on the UI goroutine |@@ -47,17 +47,19 @@ Hard limits: `sheet.MaxRows = 1,048,576`, `sheet.MaxCols = 16,384`
4747 ten million cells by default): imports keep whole rows up to it, pastes4848 and fills write at most that many cells at once. WK1 files keep their own4949 8192 x 256. The costs below grow with the cells; the grid itself costs50-nothing (see [The grid](#the-grid)). At ten million cells the file51-format is what degrades: a `.012` file is JSON decoded whole.50+nothing (see [The grid](#the-grid)). A `.012` file is read and written51+as a stream, cells straight into and out of the store ([The .01252+format](../files/format.md#reading-and-writing)), so opening costs about53+the memory the workbook then holds.5254 5355 | Measure | 8192 x 26 (213 k cells) | 8192 x 256 (2.1 M cells) | 1,000,000 x 10 (10 M cells) |5456 |---|---|---|---|55-| Build through `Load`, then full recalc | 48 ms | 0.47 s | 2.2 s (396 MB peak resident) |57+| Build through `Load`, then full recalc | 48 ms | 0.47 s | 1.8 s |5658 | Live heap | 4.4 MB | 43 MB (20.5 B per number cell) | 205 MB |57-| Save `.012` (JSON, one line per cell) | 77 ms, 4.6 MB | 0.83 s, 47 MB | 3.7 s, 236 MB |58-| Open `.012` | 108 ms (118 MB allocated) | 1.5 s (1.3 GB allocated) | 8.1 to 8.3 s (5.7 GB allocated, 2.7 GB peak resident) |59+| Save `.012` (JSON, one line per cell; heap used at the peak) | 12 ms, 4.6 MB (0.3 MB) | 0.14 s, 47 MB (0.1 MB) | 0.54 s, 236 MB (0.7 MB) |60+| Open `.012` (heap at the peak, the workbook included) | 24 ms (15 MB) | 0.24 s (60 MB) | 1.2 s (311 MB; 0.9 GB resident, the file's bytes and a sheet built beside it included) |5961 | Export CSV / TSV / XLSX / SQLite | 33 / 31 / 45 / 38 ms | 0.51 / 0.52 / 0.65 / 0.68 s | |60-| Edit a number, undo and redo it | 0.7 us, 1.2 us | | 0.7 us, 1.2 us |62+| Edit a number, undo and redo it | 0.6 us, 1.2 us | | 0.6 us, 1.2 us |6163 | Frame at 200 x 60, whole | 0.88 ms | 0.87 ms | |6264 6365 XLSX exports are written by 012 with archive/zip: 1.9 MB allocated for@@ -314,28 +316,41 @@ entities, so entity bombs fail as unknown entities.
314316 315317 ## Undo316318 317-Each step keeps a copy of every cell it changed, as a whole `Cell`318-(about 300 B, where the sheet keeps a number in 20). Undoing costs about319-what the change cost, since it recalculates the same cells. Clearing320-213 k cells and undoing it takes 144 ms; clearing all ten million cells321-of a full `max-cells` sheet would keep about 3 GB of before-images, the322-one step the history always keeps however large.319+Each step keeps the cells it changed as they were, in the form the320+sheet keeps them (`historyimage.go`): a plain cell as its 16-byte slot321+in columns of blocks, its text and formatting in the step's own tables,322+and a whole `Cell` only for a formula, a note or a derived cell. A step323+of one cell or a few keeps them in a list. Undoing costs about what the324+change cost, since it recalculates the same cells. Clearing 213 k cells325+and undoing it takes 86 ms; clearing all ten million cells of a full326+`max-cells` sheet of numbers holds 193 MB, and with its undo takes 6.2327+s. A deleted or replaced sheet is kept whole, so its step costs nothing328+more.323329 324330 The history is bounded by steps (`MaxUndo = 100`) and by an estimate of325331 the memory its before-images hold (`MaxUndoBytes = 256 MB`); past either,326332 the oldest steps are dropped first, and the newest step is kept however327-large it is. Each cell is counted as it is recorded (a fixed cost per328-cell plus its text, more for a formula, whose parsed tree stays alive),329-so the budget costs nothing to check; counting costs about 20 ns a330-cell, 3% of clearing a sheet and undoing it. The estimate follows the331-measured heap within about 10% (`BenchmarkHistoryFull`,332-`BenchmarkHistoryWide`):333+large it is. Each cell is counted as it is recorded (a slot, a block's334+share, its text once per step, more for a formula, whose parsed tree335+stays alive), so the budget costs nothing to check. The estimate follows336+the measured heap within about 10% (`BenchmarkHistoryFull`,337+`BenchmarkHistoryWide`, `BenchmarkClearMax`):333338 334339 | Benchmark | Result |335340 |---|---|336-| 100 edits rewriting a whole column (819 k before-images) | 281 MB held (252 MB estimated), all 100 kept |337-| 12 edits rewriting all of 8192 x 26 (2.6 M before-images) | 199 MB held (196 MB estimated), the last 3 kept |338-| Undo + redo of one cell (dense, names) | 1.2 us, 110 us |341+| 100 edits rewriting a whole column (819 k before-images) | 15.9 MB held (15.9 MB estimated), all 100 kept |342+| 12 edits rewriting all of 8192 x 26 (2.6 M before-images) | 49.3 MB held (49.4 MB estimated), all 12 kept |343+| Clearing all of 1,000,000 x 10 numbers | 193 MB held (193 MB estimated) |344+| Undo + redo of one cell (dense, names) | 1.2 us, 109 us |345+346+A change whose step would hold more than `MaxStepBytes` (1 GB), as347+`Sheet.UndoCost` estimates it before the change (a slot a cell, the348+formulas whole, the range's share of the strings), asks first on the349+context line: "This can't be undone: it would take N MB of undo350+history." Enter runs it `WithoutUndo`, recording nothing and forgetting351+the history, which could no longer be undone past it. Plain cells cost352+about 20 B each, so only millions of formulas or of long distinct texts353+changed at once come near it.339354 340355 ## JEV341356 @@ -516,26 +531,14 @@ computation.
516531 517532 ## What would raise the bounds518533 519-In order of value for effort; sizes and scheduling are in the520-[roadmap](../../ROADMAP.md#3-scale). `max-cells` stands at ten million,521-as Google Sheets' limit does; the sheet itself would hold several times522-that in a few GB, and these are what stand in the way.523-524-1. **Streaming, compact file format.** The `.012` file is JSON decoded525- whole: ten million cells take 8 s and 2.7 GB at the peak to open,526- 5.7 GB allocated, for a sheet that then holds 205 MB. A streaming527- decoder over the same format would roughly halve open time and cut528- allocation tenfold; a columnar or gzip-compressed variant would cut529- the size about fourfold. The format is `internal/sheet/file.go`,530- apart from the cell store.531-2. **Smaller undo steps.** A step keeps a whole `Cell` for each cell it532- changed, formatting-only changes included (see [Undo](#undo)).533- Keeping plain cells' before-images as slots, and formatting changes534- as diffs, would let a step over a full sheet cost what the sheet535- does.536-3. **Compact spilled and pivot cells.** What arrays and pivots write is537- kept as whole `Cell`s (516 k of them in the arrays stress shape);538- their values and inferred formats would fit in slots.534+Sizes and scheduling are in the [roadmap](../../ROADMAP.md#3-scale).535+`max-cells` stands at ten million, as Google Sheets' limit does; the536+sheet itself would hold several times that in a few GB, and opening,537+saving and undoing cost about what the cells do ([Sheet size](#sheet-size),538+[Undo](#undo)). What stands in the way is derived cells: what arrays539+and pivots write is kept as whole `Cell`s (516 k of them in the arrays540+stress shape), where their values and inferred formats would fit in541+slots.539542 540543 ## Measuring541544 @@ -9,7 +9,7 @@ sidebar_position: 5
99 make check # all of the below that must pass before a push: gofmt, vet, lint, test, oracle, e2e1010 make lint # go vet, staticcheck, cognitive complexity at most 25, Go files at most 500 lines, doclint and doccheck (see Docs below)1111 make test # engine, file formats and UI unit tests12-make fuzz # fuzz the formula parser and the CSV, .wk1 and XLSX readers12+make fuzz # fuzz the formula parser and the .012, CSV, .wk1 and XLSX readers1313 make oracle # compare formulas and number formats with excelize1414 make e2e # run the real binary in a terminal emulator (needs Zig and pkg-config)1515 make screens # rewrite the golden screens and build the review gallery@@ -19,7 +19,9 @@ make stress # benchmarks on synthetic and real data (see limits.md)
1919 ## Unit tests2020 2121 The engine is tested directly: parsing, every function, recalculation,22-undo, reference rewriting, file round trips. The UI is tested by sending22+undo, reference rewriting, file round trips. `FuzzRead` holds the23+streaming `.012` reader to encoding/json decoding the whole file: both24+refuse a file or both read it into workbooks that save the same bytes. The UI is tested by sending2325 Bubble Tea messages (keys, mouse, paste, window size) to the model and2426 reading what `View` renders, without a terminal.2527 @@ -37,6 +37,29 @@ file opens: formula results, [pivot tables](#pivot-tables)' results, and
3737 the arrays formulas [spill](../formulas/arrays.md), whose cells are kept3838 only for their formatting and notes.3939 40+## Reading and writing41+42+012 reads and writes the file as a stream (`internal/sheet/fileread.go`,43+`filescan.go`). Each cell goes into its sheet as its line is read, a44+number typed plainly straight into the 16-byte slot the sheet keeps it45+in, so opening a file takes about the memory of the workbook, however46+large the file; the other fields are small and decoded whole. Saving47+writes the cells in row-major order as it reads them from the sheet,48+straight into the file. Ten million numbers (236 MB of JSON) open in49+1.2 s and save in 0.5 s ([Bounds of support](../contributing/limits.md#sheet-size)).50+51+Any JSON with the same fields reads the same: the order of keys,52+spacing, and the case of field names don't matter. A cell given twice53+takes its last entry, though an invalid earlier one is refused; the54+`version` or the `sheets` list given twice around cells is refused.55+56+There is one format. A binary one that stored the slots themselves57+would open ten million numbers in about 0.3 s at best (storing ten58+million slots alone takes 0.17 s) where the JSON takes 1.2 s: a second59+saved on sheets near `max-cells`, and nothing a smaller sheet would60+notice, for the loss of what the JSON gives, diffs, merges, and a file61+anyone can read and fix.62+4063 ## Versions4164 4265 012 writes the lowest version that holds the workbook, so older builds@@ -54,6 +54,9 @@ typing, clearing, paste, fill, sort, insert and delete, formats, widths,
5454 names, charts, freeze, filters, pivot tables, and adding, deleting,5555 renaming, moving and duplicating sheets. A multi-cell change is one step,5656 and the context line says what was undone, e.g. `Undid: clear B3:B5`.57+A change whose undo would take more than 1 GB of memory (clearing58+millions of formulas at once) asks first: Enter makes it without undo,59+forgetting the steps before it, and Esc backs out.5760 5861 ## Rows and columns5962 @@ -3,6 +3,7 @@ package sheet
33 import (44 "encoding/json"55 "fmt"6+ "strconv"67 )78 89 // fileCell is a cell with formatting or a note as the file writes it; a@@ -135,3 +136,59 @@ func decodeBorders(fb *fileBorders) (Borders, error) {
135136 }136137 return b, nil137138 }139+140+// appendSaved appends the cell at a, which holds one, as the file141+// writes it, or reports false for a cell the file leaves out (a pivot's142+// or a spill's result). A plain cell without formatting is written from143+// its slot, without making a Cell.144+func (st *cellStore) appendSaved(buf []byte, a Addr) ([]byte, bool, error) {145+ b, i := st.find(a)146+ if sl := b.vals[i]; sl.kind != slotRich && sl.look == 0 {147+ return st.appendInput(buf, sl), true, nil148+ }149+ c := st.get(a).saved()150+ if c == nil {151+ return buf, false, nil152+ }153+ raw, err := encodeCell(c)154+ return append(buf, raw...), true, err155+}156+157+// appendInput appends a plain slot's entry as a JSON string.158+func (st *cellStore) appendInput(buf []byte, sl slot) []byte {159+ switch {160+ case sl.kind == slotBlank:161+ return append(buf, '"', '"')162+ case sl.ref != 0:163+ return appendJSONString(buf, st.strs.strs[sl.ref])164+ case sl.kind == slotBool:165+ return append(append(append(buf, '"'), boolText(sl.num != 0)...), '"')166+ }167+ buf = strconv.AppendFloat(append(buf, '"'), sl.num, 'f', int(sl.dec)-1, 64)168+ return append(buf, '"')169+}170+171+// appendJSONString appends s as json.Marshal writes it: as it is between172+// quotes when it is printable ASCII that needs no escape (HTML's173+// characters included, which json.Marshal escapes), and through174+// json.Marshal otherwise.175+func appendJSONString(buf []byte, s string) []byte {176+ for i := 0; i < len(s); i++ {177+ if c := s[i]; c < 0x20 || c > 0x7e || c == '"' || c == '\\' || c == '<' || c == '>' || c == '&' {178+ raw, _ := json.Marshal(s)179+ return append(buf, raw...)180+ }181+ }182+ return append(append(append(buf, '"'), s...), '"')183+}184+185+// appendAddr appends a's A1 name, as a.String() writes it.186+func appendAddr(buf []byte, a Addr) []byte {187+ var col [4]byte188+ i := len(col)189+ for c := a.Col + 1; c > 0; c = (c - 1) / 26 {190+ i--191+ col[i] = byte('A' + (c-1)%26)192+ }193+ return strconv.AppendInt(append(buf, col[i:]...), int64(a.Row+1), 10)194+}@@ -1,7 +1,7 @@
11 package sheet22 33 import (4- "bytes"4+ "bufio"55 "encoding/json"66 "fmt"77 "io"@@ -9,8 +9,6 @@ import (
99 "slices"1010 "strconv"1111 "strings"12-13- "github.com/FelineStateMachine/012/internal/locale"1412 )1513 1614 // FileExt is the extension of the native worksheet format.@@ -87,20 +85,22 @@ func (s *Sheet) Write(w io.Writer) error { return s.wb.Write(w) }
8785 8886 // Write saves the workbook as JSON, storing each cell's input as typed8987 // and its formatting. Cells go one per line in row-major order so diffs90-// read naturally.88+// read naturally. It streams: cells are written as they are read from89+// the store, through a buffer of its own, so writing costs about nothing90+// on top of the workbook whatever its size.9191 func (w *Workbook) Write(out io.Writer) error {92- var b bytes.Buffer92+ b := bufio.NewWriterSize(out, 64<<10)9393 if w.single() {9494 s := w.sheets[0]9595 version := 29696 if v := s.view; len(w.names) > 0 || v.frozenRows > 0 || v.frozenCols > 0 || v.filter != nil {9797 version = 39898 }99- fmt.Fprintf(&b, "{\n \"version\": %d,\n", version)99+ fmt.Fprintf(b, "{\n \"version\": %d,\n", version)100100 if s.name != "Sheet1" {101- fmt.Fprintf(&b, " \"name\": %s,\n", jsonString(s.name))101+ fmt.Fprintf(b, " \"name\": %s,\n", jsonString(s.name))102102 }103- if err := s.writeBody(&b, " ", w.headLines()); err != nil {103+ if err := s.writeBody(b, " ", w.headLines()); err != nil {104104 return err105105 }106106 b.WriteString("\n}\n")@@ -109,31 +109,30 @@ func (w *Workbook) Write(out io.Writer) error {
109109 if w.hasPivots() {110110 version = fileVersion111111 }112- fmt.Fprintf(&b, "{\n \"version\": %d,\n", version)112+ fmt.Fprintf(b, "{\n \"version\": %d,\n", version)113113 if head := w.headLines(); head != "" {114114 b.WriteString(" " + head + ",\n")115115 }116116 if w.Active() > 0 {117- fmt.Fprintf(&b, " \"active\": %d,\n", w.Active())117+ fmt.Fprintf(b, " \"active\": %d,\n", w.Active())118118 }119119 b.WriteString(` "sheets": [`)120120 for i, s := range w.sheets {121121 if i > 0 {122122 b.WriteByte(',')123123 }124- fmt.Fprintf(&b, "\n {\n \"name\": %s,\n", jsonString(s.name))124+ fmt.Fprintf(b, "\n {\n \"name\": %s,\n", jsonString(s.name))125125 if s.tabHidden {126126 b.WriteString(" \"hidden\": true,\n")127127 }128- if err := s.writeBody(&b, " ", ""); err != nil {128+ if err := s.writeBody(b, " ", ""); err != nil {129129 return err130130 }131131 b.WriteString("\n }")132132 }133133 b.WriteString("\n ]\n}\n")134134 }135- _, err := out.Write(b.Bytes())136- return err135+ return b.Flush()137136 }138137 139138 // single reports whether the workbook fits the single-sheet format of@@ -190,7 +189,7 @@ func (w *Workbook) namesLine() string {
190189 // writeBody writes a sheet's fields, each line starting with indent, the191190 // names line (if any) after the widths as version 3 has it, and no192191 // newline after the last field.193-func (s *Sheet) writeBody(b *bytes.Buffer, indent, names string) error {192+func (s *Sheet) writeBody(b *bufio.Writer, indent, names string) error {194193 if len(s.widths) > 0 {195194 b.WriteString(indent + `"widths": {`)196195 for i, c := range slices.Sorted(maps.Keys(s.widths)) {@@ -211,25 +210,31 @@ func (s *Sheet) writeBody(b *bytes.Buffer, indent, names string) error {
211210 if err := s.writeView(b, indent); err != nil {212211 return err213212 }214- addrs := make([]Addr, 0, s.cells.len())215- for a := range s.cells.keys() {216- if c := s.cells.richAt(a); c == nil || c.saved() != nil {217- addrs = append(addrs, a)218- }219- }220- sortAddrs(addrs)221213 b.WriteString(indent + `"cells": {`)222- for i, a := range addrs {223- raw, err := encodeCell(s.cells.get(a).saved())214+ var line []byte215+ n := 0216+ for a := range s.cells.keysIn(Rect{To: Addr{Col: MaxCols - 1, Row: MaxRows - 1}}) {217+ line = append(append(line[:0], '\n'), indent...)218+ line = appendAddr(append(line, ' ', ' ', '"'), a)219+ line = append(line, '"', ':', ' ')220+ var ok bool221+ var err error222+ line, ok, err = s.cells.appendSaved(line, a)224223 if err != nil {225224 return err226225 }227- if i > 0 {226+ if !ok {227+ continue228+ }229+ if n > 0 {228230 b.WriteByte(',')229231 }230- fmt.Fprintf(b, "\n%s %q: %s", indent, a.String(), raw)232+ n++233+ if _, err := b.Write(line); err != nil {234+ return err235+ }231236 }232- if len(addrs) > 0 {237+ if n > 0 {233238 b.WriteString("\n" + indent)234239 }235240 b.WriteString("}")@@ -238,7 +243,7 @@ func (s *Sheet) writeBody(b *bytes.Buffer, indent, names string) error {
238243 239244 // writeObjects writes the fields after the cells: the charts, one per240245 // line, the pivot table's definition, and the rules.241-func (s *Sheet) writeObjects(b *bytes.Buffer, indent string) error {246+func (s *Sheet) writeObjects(b *bufio.Writer, indent string) error {242247 if len(s.charts) > 0 {243248 b.WriteString(",\n" + indent + `"charts": [`)244249 for i, c := range s.charts {@@ -282,60 +287,6 @@ func ReadTraced(r io.Reader, trace any) (*Sheet, error) {
282287 // version.283288 func ReadBook(r io.Reader) (*Workbook, error) { return readBook(r, nil) }284289 285-func readBook(r io.Reader, trace any) (*Workbook, error) {286- var f fileFormat287- if err := json.NewDecoder(r).Decode(&f); err != nil {288- return nil, err289- }290- if f.Version < 1 || f.Version > fileVersion {291- return nil, fmt.Errorf("unsupported file version %d", f.Version)292- }293- bodies := f.Sheets294- if f.Version < 4 {295- bodies = []fileSheet{f.fileSheet}296- if bodies[0].Name == "" {297- bodies[0].Name = "Sheet1"298- }299- }300- if len(bodies) == 0 {301- return nil, fmt.Errorf("the file has no sheets")302- }303- w := emptyBook()304- w.trace = trace305- for _, body := range bodies {306- if err := w.checkName(nil, body.Name); err != nil {307- return nil, fmt.Errorf("sheet %q: %w", body.Name, err)308- }309- w.insert(w.newSheet(body.Name), len(w.sheets))310- }311- if err := w.readNames(f.Names); err != nil {312- return nil, err313- }314- for i, body := range bodies {315- if err := w.sheets[i].read(body, f.Version); err != nil {316- if len(bodies) > 1 {317- err = fmt.Errorf("sheet %s: %w", body.Name, err)318- }319- return nil, err320- }321- }322- w.active = clampInt(f.Active, 0, len(w.sheets)-1)323- w.settleHidden()324- // Anything but "decimal" (say, a mode from a later build) computes in325- // binary, as the file would in a build without the setting.326- w.decimal = f.Arithmetic == "decimal"327- // A locale this build doesn't know follows the default.328- if l, ok := locale.Lookup(f.Locale); ok {329- w.locale = l330- }331- if err := w.readMacros(f.Macros); err != nil {332- return nil, err333- }334- w.macroOrigin = f.MacroOrigin335- w.RecalcAll()336- return w, nil337-}338-339290 // readNames defines the named ranges of a file. A range without a sheet340291 // is on the first sheet, as in single-sheet files.341292 func (w *Workbook) readNames(names map[string]string) error {@@ -365,8 +316,9 @@ func (w *Workbook) readNames(names map[string]string) error {
365316 return nil366317 }367318 368-// read fills an empty sheet from its part of a file.369-func (s *Sheet) read(f fileSheet, version int) error {319+// read fills a sheet, its cells read (fileread.go), from the rest of320+// its part of a file.321+func (s *Sheet) read(f fileSheet) error {370322 s.tabHidden = f.Hidden371323 for name, width := range f.Widths {372324 c, ok := ParseCol(name)@@ -385,25 +337,6 @@ func (s *Sheet) read(f fileSheet, version int) error {
385337 if err := s.readLines(f.Lines); err != nil {386338 return err387339 }388- for name, raw := range f.Cells {389- a, ok := ParseAddr(name)390- if !ok {391- return fmt.Errorf("invalid cell %q", name)392- }393- input, fm, st, note, err := decodeNoted(raw)394- if err != nil {395- return fmt.Errorf("%s: %w", name, err)396- }397- // Version 1 had no formats, so there entries imply them, as if398- // typed again; since version 2 the stored format wins.399- c, err := newCell(input, fm, st, version < 2)400- if err != nil {401- return fmt.Errorf("%s: %w", name, err)402- }403- if c = c.withNote(CleanNote(note)); c != nil {404- s.place(a, c)405- }406- }407340 for i, fc := range f.Charts {408341 c, err := decodeChart(fc)409342 if err != nil {@@ -0,0 +1,414 @@
1+package sheet2+3+import (4+ "bytes"5+ "encoding/json"6+ "errors"7+ "fmt"8+ "io"9+ "strconv"10+ "strings"11+12+ "github.com/FelineStateMachine/012/internal/locale"13+)14+15+// The reader streams the file (see scanner): cells go straight into16+// their sheet as they are read, so opening a file allocates about what17+// the workbook holds rather than a tree of the whole document. The18+// other fields of the workbook and of each sheet are small; they are19+// kept as written and decoded with encoding/json once the stream ends.20+//21+// Keys match fields as encoding/json matches them, case aside, and a22+// field given twice takes its last value, cells adding up, except that23+// the version and the list of sheets, which say where cells go, are24+// refused twice once cells were read by them. Cells come25+// after "version" in every file Write makes; any read before it (a file26+// written by hand) are kept as written until the version is known.27+type fileReader struct {28+ sc *scanner29+ w *Workbook30+ version int31+ // hasVersion is set once "version" is read, and streamed once cells32+ // were read with it, which it may then no longer change.33+ hasVersion, streamed bool34+ listed bool // the list of sheets was read35+ single *Sheet // versions 1 to 3: the sheet of the top-level cells36+ pending []json.RawMessage // top-level cells read before the version, in order37+ sheets []readSheet // version 4: the sheets read38+ sheetsRaw json.RawMessage // "sheets" read before the version39+}40+41+// readSheet is a sheet of a version 4 file, its cells read, and its42+// other fields.43+type readSheet struct {44+ s *Sheet45+ body fileSheet46+}47+48+// field is a member of an object, its value as written.49+type field struct {50+ key string51+ raw json.RawMessage52+}53+54+func readBook(r io.Reader, trace any) (*Workbook, error) {55+ rd := &fileReader{sc: newScanner(r), w: emptyBook()}56+ f, err := rd.top()57+ if err != nil {58+ return nil, err59+ }60+ if f.Version < 1 || f.Version > fileVersion {61+ return nil, fmt.Errorf("unsupported file version %d", f.Version)62+ }63+ if err := rd.late(); err != nil {64+ return nil, err65+ }66+ w := rd.w67+ w.trace = trace68+ list := rd.sheets69+ if f.Version < 4 {70+ if rd.single == nil {71+ rd.single = w.newSheet("")72+ }73+ list = []readSheet{{rd.single, f.fileSheet}}74+ if list[0].body.Name == "" {75+ list[0].body.Name = "Sheet1"76+ }77+ }78+ if len(list) == 0 {79+ return nil, errors.New("the file has no sheets")80+ }81+ for _, rs := range list {82+ if err := w.checkName(nil, rs.body.Name); err != nil {83+ return nil, fmt.Errorf("sheet %q: %w", rs.body.Name, err)84+ }85+ rs.s.name = rs.body.Name86+ w.insert(rs.s, len(w.sheets))87+ }88+ if err := w.readNames(f.Names); err != nil {89+ return nil, err90+ }91+ for _, rs := range list {92+ if err := rs.s.read(rs.body); err != nil {93+ if len(list) > 1 {94+ err = fmt.Errorf("sheet %s: %w", rs.body.Name, err)95+ }96+ return nil, err97+ }98+ }99+ w.active = clampInt(f.Active, 0, len(w.sheets)-1)100+ w.settleHidden()101+ // Anything but "decimal" (say, a mode from a later build) computes in102+ // binary, as the file would in a build without the setting.103+ w.decimal = f.Arithmetic == "decimal"104+ // A locale this build doesn't know follows the default.105+ if l, ok := locale.Lookup(f.Locale); ok {106+ w.locale = l107+ }108+ if err := w.readMacros(f.Macros); err != nil {109+ return nil, err110+ }111+ w.macroOrigin = f.MacroOrigin112+ w.RecalcAll()113+ return w, nil114+}115+116+// top reads the file's object, streaming the cells, and returns its117+// other fields.118+func (rd *fileReader) top() (fileFormat, error) {119+ var f fileFormat120+ if null, err := rd.sc.null(); err != nil || null {121+ return f, err122+ }123+ var rest []field124+ err := rd.sc.members(func(key string) error {125+ switch {126+ case strings.EqualFold(key, "version"):127+ return rd.readVersion()128+ case strings.EqualFold(key, "cells"):129+ return rd.topCells()130+ case strings.EqualFold(key, "sheets"):131+ return rd.topSheets()132+ }133+ raw, err := rd.sc.raw()134+ rest = append(rest, field{key, raw})135+ return err136+ })137+ if err == nil {138+ err = unmarshalFields(rest, &f)139+ }140+ f.Version = rd.version141+ return f, err142+}143+144+func (rd *fileReader) readVersion() error {145+ raw, err := rd.sc.raw()146+ if err != nil {147+ return err148+ }149+ v := rd.version150+ if err := json.Unmarshal(raw, &v); err != nil {151+ return err152+ }153+ if rd.streamed && v != rd.version {154+ return errTwice155+ }156+ rd.version, rd.hasVersion = v, true157+ return nil158+}159+160+// topCells reads the top-level cells: the only sheet's, before version 4.161+func (rd *fileReader) topCells() error {162+ if !rd.hasVersion || len(rd.pending) > 0 {163+ raw, err := rd.sc.raw()164+ rd.pending = append(rd.pending, raw)165+ return err166+ }167+ if rd.version >= 4 {168+ raw, err := rd.sc.raw()169+ if err == nil {170+ err = checkCells(raw)171+ }172+ return err173+ }174+ return rd.cells(rd.singleSheet(), rd.sc)175+}176+177+func (rd *fileReader) singleSheet() *Sheet {178+ if rd.single == nil {179+ rd.single = rd.w.newSheet("")180+ }181+ return rd.single182+}183+184+// checkCells checks cells a file of this version ignores, as decoding185+// them into the file's fields would.186+func checkCells(raw json.RawMessage) error {187+ var m map[string]json.RawMessage188+ return json.Unmarshal(raw, &m)189+}190+191+// topSheets reads the list of sheets of version 4.192+func (rd *fileReader) topSheets() error {193+ if !rd.hasVersion || rd.version < 4 {194+ raw, err := rd.sc.raw()195+ if err != nil {196+ return err197+ }198+ if !rd.hasVersion {199+ if rd.sheetsRaw != nil {200+ return errTwice201+ }202+ rd.sheetsRaw = raw203+ return nil204+ }205+ var ignored []fileSheet206+ return json.Unmarshal(raw, &ignored)207+ }208+ return rd.readSheets(rd.sc)209+}210+211+// errTwice is returned for a version or a list of sheets given twice212+// once cells were read by the first.213+var errTwice = errors.New("the file gives its version or its sheets twice")214+215+// readSheets reads a list of sheets from sc.216+func (rd *fileReader) readSheets(sc *scanner) error {217+ if rd.listed {218+ return errTwice219+ }220+ rd.listed, rd.sheetsRaw = true, nil221+ if null, err := sc.null(); err != nil || null {222+ return err223+ }224+ return sc.elements(func() error {225+ s := rd.w.newSheet("")226+ var rest []field227+ if null, err := sc.null(); err != nil || null {228+ rd.sheets = append(rd.sheets, readSheet{s: s})229+ return err230+ }231+ name := ""232+ err := sc.members(func(key string) error {233+ if strings.EqualFold(key, "cells") {234+ err := rd.cells(s, sc)235+ if err != nil && name != "" {236+ err = fmt.Errorf("sheet %s: %w", name, err)237+ }238+ return err239+ }240+ raw, err := sc.raw()241+ if strings.EqualFold(key, "name") {242+ json.Unmarshal(raw, &name) // for errors only: the body decodes it243+ }244+ rest = append(rest, field{key, raw})245+ return err246+ })247+ var body fileSheet248+ if err == nil {249+ err = unmarshalFields(rest, &body)250+ }251+ rd.sheets = append(rd.sheets, readSheet{s, body})252+ return err253+ })254+}255+256+// late reads what came before the version, now that it is known.257+func (rd *fileReader) late() error {258+ for _, raw := range rd.pending {259+ if rd.version >= 4 {260+ if err := checkCells(raw); err != nil {261+ return err262+ }263+ continue264+ }265+ if err := rd.cells(rd.singleSheet(), newScanner(bytes.NewReader(raw))); err != nil {266+ return err267+ }268+ }269+ if rd.sheetsRaw == nil {270+ return nil271+ }272+ if rd.version < 4 {273+ var ignored []fileSheet274+ return json.Unmarshal(rd.sheetsRaw, &ignored)275+ }276+ return rd.readSheets(newScanner(bytes.NewReader(rd.sheetsRaw)))277+}278+279+// cells reads a sheet's cells from sc into s. Version 1 had no formats,280+// so there entries imply them, as if typed again; since version 2 the281+// stored format wins.282+func (rd *fileReader) cells(s *Sheet, sc *scanner) error {283+ rd.streamed = true284+ if null, err := sc.null(); err != nil || null {285+ return err286+ }287+ implied := rd.version < 2288+ return sc.fields(func(key []byte, plain bool) error {289+ a, name, err := cellKey(key, plain)290+ if err != nil {291+ return err292+ }293+ if err := s.readCell(sc, a, implied); err != nil {294+ if name == "" {295+ name = a.String()296+ }297+ return fmt.Errorf("%s: %w", name, err)298+ }299+ return nil300+ })301+}302+303+// cellKey is the cell a key of "cells" names, and its name when it isn't304+// a's (for errors). A plain key is parsed from its bytes, which are305+// valid only until the next read, without making a string of it.306+func cellKey(key []byte, plain bool) (Addr, string, error) {307+ if plain {308+ a, ok := ParseAddr(string(key))309+ if !ok {310+ return a, "", fmt.Errorf("invalid cell %q", key)311+ }312+ return a, "", nil313+ }314+ var name string315+ if err := json.Unmarshal(quote(key), &name); err != nil {316+ return Addr{}, "", err317+ }318+ a, ok := ParseAddr(name)319+ if !ok {320+ return a, "", fmt.Errorf("invalid cell %q", name)321+ }322+ return a, name, nil323+}324+325+// readCell reads the cell at a from sc: a string for an entry alone, an326+// object for one with formatting or a note (see fileFormat).327+func (s *Sheet) readCell(sc *scanner, a Addr, implied bool) error {328+ c, err := sc.peek()329+ if err != nil {330+ return err331+ }332+ var input, note string333+ var fm Format334+ var st Style335+ if c == '"' {336+ body, plain, err := sc.text()337+ if err != nil {338+ return err339+ }340+ if plain && s.loadNum(a, body, Format{}, Style{}) {341+ return nil342+ }343+ if input, err = unquote(body, plain); err != nil {344+ return err345+ }346+ } else {347+ raw, err := sc.raw()348+ if err != nil {349+ return err350+ }351+ if input, fm, st, note, err = decodeNoted(raw); err != nil {352+ return err353+ }354+ if note == "" && s.loadNum(a, []byte(input), fm, st) {355+ return nil356+ }357+ }358+ cell, err := newCell(input, fm, st, implied)359+ if err != nil {360+ return err361+ }362+ // An empty entry leaves no cell, even where the key came before.363+ if cell = cell.withNote(CleanNote(note)); cell != nil || s.cells.has(a) {364+ s.place(a, cell)365+ }366+ return nil367+}368+369+// loadNum stores a number typed plainly ("12.5", "-3", "0.50", the most370+// common entry by far) at a, from a file, straight into its slot: what371+// newCell and place would store, without making a Cell or a string. It372+// reports false for any other entry, a cell a holds with contents, or a373+// style that wraps or draws borders (place indexes those by row), all374+// of which go the general way. A sheet being read has no undo step,375+// pivot or spill yet for place to see to.376+func (s *Sheet) loadNum(a Addr, input []byte, f Format, st Style) bool {377+ if f.Kind == FmtText || st.shapes() || s.cells.richAt(a) != nil {378+ return false379+ }380+ d, ok := plainForm(string(input))381+ if !ok {382+ return false383+ }384+ v, err := strconv.ParseFloat(string(input), 64)385+ if err != nil {386+ return false387+ }388+ lk, ok := s.cells.lookID(f, st)389+ if !ok {390+ return false391+ }392+ s.version++393+ s.cells.setSlot(a, slot{kind: slotNum, num: v, dec: d, look: lk})394+ return true395+}396+397+// unmarshalFields decodes an object's fields, as written, into v.398+func unmarshalFields(fields []field, v any) error {399+ if len(fields) == 0 {400+ return nil401+ }402+ var b bytes.Buffer403+ b.WriteByte('{')404+ for i, f := range fields {405+ if i > 0 {406+ b.WriteByte(',')407+ }408+ b.WriteString(jsonString(f.key))409+ b.WriteByte(':')410+ b.Write(f.raw)411+ }412+ b.WriteByte('}')413+ return json.Unmarshal(b.Bytes(), v)414+}@@ -0,0 +1,282 @@
1+package sheet2+3+import (4+ "bufio"5+ "encoding/json"6+ "errors"7+ "fmt"8+ "io"9+ "unicode/utf8"10+)11+12+// scanner reads JSON a value at a time from a stream, for the .01213+// reader (fileread.go): the cells are read one by one straight into the14+// sheet, and everything else, which is small, is captured whole and15+// given to encoding/json, so it decodes as it always has. Strings16+// without escapes are read without encoding/json; the rest go through17+// it, so text decodes exactly as it would there.18+type scanner struct {19+ r *bufio.Reader20+ buf []byte // scratch for a string longer than the reader's buffer or escaped21+ key []byte // the key being read22+}23+24+func newScanner(r io.Reader) *scanner { return &scanner{r: bufio.NewReaderSize(r, 64<<10)} }25+26+// errSyntax is returned for anything that isn't the JSON expected.27+var errSyntax = errors.New("invalid JSON")28+29+// syntax describes a byte that wasn't expected.30+func syntax(c byte, want string) error {31+ return fmt.Errorf("%w: %q where %s was expected", errSyntax, c, want)32+}33+34+// eofErr is err, turning a clean EOF in the middle of a value into35+// io.ErrUnexpectedEOF, as encoding/json does.36+func eofErr(err error) error {37+ if err == io.EOF {38+ return io.ErrUnexpectedEOF39+ }40+ return err41+}42+43+// peek skips white space and returns the next byte without reading it.44+func (sc *scanner) peek() (byte, error) {45+ for {46+ c, err := sc.r.ReadByte()47+ if err != nil {48+ return 0, eofErr(err)49+ }50+ switch c {51+ case ' ', '\t', '\n', '\r':52+ continue53+ }54+ return c, sc.r.UnreadByte()55+ }56+}57+58+// next skips white space and reads the next byte.59+func (sc *scanner) next() (byte, error) {60+ if _, err := sc.peek(); err != nil {61+ return 0, err62+ }63+ return sc.r.ReadByte()64+}65+66+// expect reads the next byte, which must be c.67+func (sc *scanner) expect(c byte) error {68+ got, err := sc.next()69+ if err == nil && got != c {70+ err = syntax(got, fmt.Sprintf("%q", c))71+ }72+ return err73+}74+75+// null reads the literal null if it comes next, and reports whether it76+// did.77+func (sc *scanner) null() (bool, error) {78+ c, err := sc.peek()79+ if err != nil || c != 'n' {80+ return false, err81+ }82+ raw, err := sc.raw()83+ if err == nil && string(raw) != "null" {84+ err = fmt.Errorf("%w: %q", errSyntax, raw)85+ }86+ return err == nil, err87+}88+89+// members reads an object, calling fn with each key once the reader is90+// at its value, which fn must read.91+func (sc *scanner) members(fn func(key string) error) error {92+ return sc.fields(func(key []byte, plain bool) error {93+ s := string(key)94+ if !plain {95+ if err := json.Unmarshal(quote(key), &s); err != nil {96+ return err97+ }98+ }99+ return fn(s)100+ })101+}102+103+// fields is members giving each key as written between its quotes, and104+// whether that is its text (see text); the bytes are valid until the105+// value is read.106+func (sc *scanner) fields(fn func(key []byte, plain bool) error) error {107+ if err := sc.expect('{'); err != nil {108+ return err109+ }110+ return sc.list('}', func() error {111+ text, plain, err := sc.text()112+ if err != nil {113+ return err114+ }115+ sc.key = append(sc.key[:0], text...) // reading on may move text116+ if err := sc.expect(':'); err != nil {117+ return err118+ }119+ return fn(sc.key, plain)120+ })121+}122+123+// elements reads an array, calling fn for each element, which fn must124+// read.125+func (sc *scanner) elements(fn func() error) error {126+ if err := sc.expect('['); err != nil {127+ return err128+ }129+ return sc.list(']', fn)130+}131+132+// list reads the items of an object or array up to its closing byte,133+// with fn reading each.134+func (sc *scanner) list(end byte, fn func() error) error {135+ c, err := sc.peek()136+ if err != nil {137+ return err138+ }139+ if c == end {140+ _, err = sc.r.ReadByte()141+ return err142+ }143+ for {144+ if err := fn(); err != nil {145+ return err146+ }147+ c, err := sc.next()148+ switch {149+ case err != nil:150+ return err151+ case c == end:152+ return nil153+ case c != ',':154+ return syntax(c, fmt.Sprintf(`"," or %q`, end))155+ }156+ }157+}158+159+// unquote is the text of a string written as body between its quotes,160+// plain or not (see text).161+func unquote(body []byte, plain bool) (string, error) {162+ if plain {163+ return string(body), nil164+ }165+ var s string166+ err := json.Unmarshal(quote(body), &s)167+ return s, err168+}169+170+// quote is body between quotes, in a new slice.171+func quote(body []byte) []byte {172+ out := make([]byte, 0, len(body)+2)173+ return append(append(append(out, '"'), body...), '"')174+}175+176+// text reads a string as it is written between its quotes, and reports177+// whether it is plain: no escapes, control characters or invalid UTF-8,178+// so those bytes are its text. They are valid until the next read.179+func (sc *scanner) text() ([]byte, bool, error) {180+ if err := sc.expect('"'); err != nil {181+ return nil, false, err182+ }183+ part, err := sc.r.ReadSlice('"')184+ if err == nil && !escapedQuote(part) { // the common case: no copy185+ body := part[:len(part)-1]186+ return body, plainText(body), nil187+ }188+ sc.buf = append(sc.buf[:0], part...)189+ for err != nil || escapedQuote(sc.buf) {190+ if err != nil && err != bufio.ErrBufferFull {191+ return nil, false, eofErr(err)192+ }193+ part, err = sc.r.ReadSlice('"')194+ sc.buf = append(sc.buf, part...)195+ }196+ body := sc.buf[:len(sc.buf)-1]197+ return body, plainText(body), nil198+}199+200+// plainText reports whether a string's body as written is its text.201+func plainText(body []byte) bool {202+ for _, c := range body {203+ if c < 0x20 || c == '\\' {204+ return false205+ }206+ }207+ return utf8.Valid(body)208+}209+210+// escapedQuote reports whether the quote ending s is escaped: preceded211+// by an odd number of backslashes.212+func escapedQuote(s []byte) bool {213+ n := 0214+ for i := len(s) - 2; i >= 0 && s[i] == '\\'; i-- {215+ n++216+ }217+ return n%2 == 1218+}219+220+// raw reads the next value whole, as it is written. It checks only221+// that brackets balance outside strings; encoding/json checks the rest222+// when the value is decoded.223+func (sc *scanner) raw() (json.RawMessage, error) {224+ c, err := sc.peek()225+ if err != nil {226+ return nil, err227+ }228+ switch c {229+ case '"':230+ body, _, err := sc.text()231+ return quote(body), err232+ case '{', '[':233+ return sc.nested()234+ }235+ var out []byte236+ for {237+ c, err := sc.r.ReadByte()238+ if err == io.EOF && len(out) > 0 {239+ return out, nil240+ }241+ if err != nil {242+ return nil, eofErr(err)243+ }244+ switch c {245+ case ',', '}', ']', ' ', '\t', '\n', '\r':246+ return out, sc.r.UnreadByte()247+ }248+ out = append(out, c)249+ }250+}251+252+// nested reads an object or array whole.253+func (sc *scanner) nested() (json.RawMessage, error) {254+ var out []byte255+ depth := 0256+ for {257+ c, err := sc.r.ReadByte()258+ if err != nil {259+ return nil, eofErr(err)260+ }261+ switch c {262+ case '"':263+ if err := sc.r.UnreadByte(); err != nil {264+ return nil, err265+ }266+ body, _, err := sc.text()267+ if err != nil {268+ return nil, err269+ }270+ out = append(append(append(out, '"'), body...), '"')271+ continue272+ case '{', '[':273+ depth++274+ case '}', ']':275+ depth--276+ }277+ out = append(out, c)278+ if depth == 0 {279+ return out, nil280+ }281+ }282+}@@ -0,0 +1,272 @@
1+package sheet2+3+import (4+ "bytes"5+ "encoding/json"6+ "errors"7+ "fmt"8+ "strings"9+ "testing"10+11+ "github.com/FelineStateMachine/012/internal/locale"12+)13+14+// readWhole reads a file the way encoding/json reads it whole, into15+// fileFormat, as the reference the streaming reader is held to. It16+// reports ambiguous for a file whose result depends on map order: a17+// sheet naming one cell by two keys ("A1" and "a1").18+func readWhole(data []byte) (w *Workbook, ambiguous bool, err error) {19+ var f fileFormat20+ if err := json.NewDecoder(bytes.NewReader(data)).Decode(&f); err != nil {21+ return nil, false, err22+ }23+ if f.Version < 1 || f.Version > fileVersion {24+ return nil, false, fmt.Errorf("unsupported file version %d", f.Version)25+ }26+ bodies := f.Sheets27+ if f.Version < 4 {28+ bodies = []fileSheet{f.fileSheet}29+ if bodies[0].Name == "" {30+ bodies[0].Name = "Sheet1"31+ }32+ }33+ if len(bodies) == 0 {34+ return nil, false, errors.New("the file has no sheets")35+ }36+ w = emptyBook()37+ for _, body := range bodies {38+ if err := w.checkName(nil, body.Name); err != nil {39+ return nil, false, err40+ }41+ w.insert(w.newSheet(body.Name), len(w.sheets))42+ }43+ if err := w.readNames(f.Names); err != nil {44+ return nil, false, err45+ }46+ for i, body := range bodies {47+ s := w.sheets[i]48+ amb, err := placeWhole(s, body.Cells, f.Version)49+ if err != nil {50+ return nil, false, err51+ }52+ ambiguous = ambiguous || amb53+ if err := s.read(body); err != nil {54+ return nil, false, err55+ }56+ }57+ w.active = clampInt(f.Active, 0, len(w.sheets)-1)58+ w.settleHidden()59+ w.decimal = f.Arithmetic == "decimal"60+ if l, ok := locale.Lookup(f.Locale); ok {61+ w.locale = l62+ }63+ if err := w.readMacros(f.Macros); err != nil {64+ return nil, false, err65+ }66+ w.macroOrigin = f.MacroOrigin67+ w.RecalcAll()68+ return w, ambiguous, nil69+}70+71+// placeWhole places a sheet's cells decoded whole, and reports whether72+// two keys name one cell.73+func placeWhole(s *Sheet, cells map[string]json.RawMessage, version int) (ambiguous bool, err error) {74+ seen := map[Addr]bool{}75+ for name, raw := range cells {76+ a, ok := ParseAddr(name)77+ if !ok {78+ return false, fmt.Errorf("invalid cell %q", name)79+ }80+ ambiguous = ambiguous || seen[a]81+ seen[a] = true82+ input, fm, st, note, err := decodeNoted(raw)83+ if err != nil {84+ return false, err85+ }86+ c, err := newCell(input, fm, st, version < 2)87+ if err != nil {88+ return false, err89+ }90+ if c = c.withNote(CleanNote(note)); c != nil {91+ s.place(a, c)92+ }93+ }94+ return ambiguous, nil95+}96+97+// written is the file w writes.98+func written(t testing.TB, w *Workbook) string {99+ var b bytes.Buffer100+ if err := w.Write(&b); err != nil {101+ t.Fatal(err)102+ }103+ return b.String()104+}105+106+// sameAsWhole checks that the streaming reader reads data as readWhole107+// does: both fail, or both make workbooks that write the same file.108+func sameAsWhole(t *testing.T, data []byte) {109+ want, ambiguous, werr := readWhole(data)110+ got, gerr := ReadBook(bytes.NewReader(data))111+ switch {112+ case errors.Is(gerr, errTwice):113+ // Refused, where decoding the whole file took the last (and for114+ // sheets, merged the two lists).115+ case gerr != nil && werr == nil && duplicateKeys(data):116+ // Streaming reads every entry of a cell given twice, and refuses117+ // an invalid one that decoding the whole file overwrote.118+ case (werr == nil) != (gerr == nil):119+ t.Fatalf("reading %q: streaming %v, whole %v", data, gerr, werr)120+ case werr == nil && !ambiguous:121+ if g, w := written(t, got), written(t, want); g != w {122+ t.Fatalf("reading %q: streaming writes\n%s\nwhole writes\n%s", data, g, w)123+ }124+ }125+}126+127+// duplicateKeys reports whether an object in data gives a key twice.128+func duplicateKeys(data []byte) bool {129+ dec := json.NewDecoder(bytes.NewReader(data))130+ type level struct {131+ keys map[string]bool132+ isKey bool // the next token in this object is a key133+ }134+ var stack []*level135+ for {136+ tok, err := dec.Token()137+ if err != nil {138+ return false139+ }140+ top := (*level)(nil)141+ if len(stack) > 0 {142+ top = stack[len(stack)-1]143+ }144+ switch tok {145+ case json.Delim('{'), json.Delim('['):146+ if top != nil && top.keys != nil {147+ top.isKey = true // the object or array is the value148+ }149+ l := &level{}150+ if tok == json.Delim('{') {151+ l.keys, l.isKey = map[string]bool{}, true152+ }153+ stack = append(stack, l)154+ continue155+ case json.Delim('}'), json.Delim(']'):156+ stack = stack[:len(stack)-1]157+ if len(stack) == 0 {158+ return false159+ }160+ continue161+ }162+ if top == nil || top.keys == nil {163+ continue164+ }165+ if top.isKey {166+ k := tok.(string)167+ if top.keys[k] {168+ return true169+ }170+ top.keys[k] = true171+ }172+ top.isKey = !top.isKey173+ }174+}175+176+// streamSeeds are files to read: what Write makes of sheets holding the177+// parser's fuzz seeds and every kind of field, and files written by hand178+// with keys out of order, in other cases, escaped and given twice.179+func streamSeeds(t testing.TB) [][]byte {180+ s := New()181+ inputs := []string{"=A1*2", "=SUM(A1:B3)", "=IF(AND(A1>2,B1),1,NA())", `="a"&"b"`, "=1.5E-3^2", "$1,200", "12%", "@SUM(A1..B3)", "=$A$1+A$2*$B3", "=SUM($A1:B$3)", "=#REF!+1", "=A$$1",182+ "Rent", "TRUE", "1.50", "'007", "<b>&</b>", "tab\tquote\"back\\slash", "café 
", strings.Repeat("long ", 20000)}183+ for i, in := range inputs {184+ s.Set(Addr{Col: i % 4, Row: i / 4}, in)185+ }186+ s.SetFormat(NewRect(Addr{Row: 10}, Addr{Col: 2, Row: 12}), Format{Kind: FmtCurrency, Decimals: 2})187+ s.SetStyle(NewRect(Addr{Row: 11}, Addr{Col: 3, Row: 11}), func(st *Style) { st.Bold = true })188+ s.SetNote(Addr{Col: 5, Row: 1}, "a note")189+ s.SetColWidth(1, 20)190+ one := []byte(written(t, s.Book()))191+ if _, err := s.Book().AddSheet("Other", 1); err != nil {192+ t.Fatal(err)193+ }194+ s.Book().Sheet(1).Set(Addr{}, "=SUM(Sheet1!B1:B5)")195+ two := []byte(written(t, s.Book()))196+ // Many cells, so keys and values straddle the reader's buffer.197+ big := New()198+ for r := range 20000 {199+ big.Load(Addr{Row: r}, fmt.Sprintf("%d.%02d", r, r%100), Format{}, Style{})200+ big.Load(Addr{Col: 1, Row: r}, fmt.Sprintf("text %d", r%300), Format{}, Style{Bold: r%7 == 0})201+ }202+ return [][]byte{one, two, []byte(written(t, big.Book())),203+ []byte(`{"version": 1, "widths": {"A": 14}, "cells": {"A1": "Rent", "B1": "$1,450", "B2": "=B1*12"}}`),204+ []byte(`{"cells": {"A1": "5", "B1": "=A1+1"}, "version": 2}`),205+ []byte(`{"Cells": {"A1": "5"}, "VERSION": 3, "cells": {"A1": "6", "A2": {"input": "7", "bold": true}}}`),206+ []byte(`{"sheets": [{"cells": {"A1": "1"}, "name": "S"}], "version": 4}`),207+ []byte(`{"version": 4, "sheets": [{"name": "A", "cells": {"A1": "1"}}, null], "sheets": [{"name": "B", "cells": {"B2": "A\n"}}]}`),208+ []byte(`{"version": 2, "cells": {"A1": "x", "a1": "y"}}`),209+ []byte(`{"version": 2, "cells": {"A1": "=(", "A1": "1", "B1": "2", "B1": ""}}`),210+ []byte(`{"version": 2, "cells": null, "sheets": 5}`),211+ []byte(`{"version": 4, "cells": "x", "sheets": []}`),212+ []byte(`{"version": 2, "cells": {"A1": "😀", "A2": "bad \xff", "ZZ9": 5}}`),213+ []byte(`{"version": 2, "cells": {"A1": "1"}} trailing`),214+ []byte(`null`), []byte(`[]`), []byte(`{"version": 2, "cells": {"A1": "1",}}`), []byte(`{"version": 2, "cells": {"A1"`),215+ }216+}217+218+// The streaming reader reads every seed as encoding/json reading the219+// whole file did.220+func TestStreamReadsAsWhole(t *testing.T) {221+ for _, data := range streamSeeds(t) {222+ sameAsWhole(t, data)223+ }224+}225+226+func FuzzRead(f *testing.F) {227+ for _, data := range streamSeeds(f) {228+ f.Add(data)229+ }230+ f.Fuzz(func(t *testing.T, data []byte) {231+ if len(data) > 1<<16 {232+ return233+ }234+ sameAsWhole(t, data)235+ })236+}237+238+// Cells are written as encodeCell writes each, in row-major order, one239+// per line.240+func TestWriteCellsAsEncoded(t *testing.T) {241+ data := streamSeeds(t)[0]242+ s, err := Read(bytes.NewReader(data))243+ if err != nil {244+ t.Fatal(err)245+ }246+ var addrs []Addr247+ for a := range s.cells.keys() {248+ addrs = append(addrs, a)249+ }250+ sortAddrs(addrs)251+ var want strings.Builder252+ want.WriteString(`"cells": {`)253+ for i, a := range addrs {254+ raw, err := encodeCell(s.cells.get(a).saved())255+ if err != nil {256+ t.Fatal(err)257+ }258+ if i > 0 {259+ want.WriteByte(',')260+ }261+ fmt.Fprintf(&want, "\n %q: %s", a.String(), raw)262+ }263+ want.WriteString("\n }")264+ if got := written(t, s.Book()); !strings.Contains(got, want.String()) || string(data) != got {265+ t.Errorf("cells written as\n%s\nwant\n%s", got, want.String())266+ }267+ for _, a := range []Addr{{}, {Col: 25, Row: 9}, {Col: 26}, {Col: 701, Row: 99}, {Col: 702}, {Col: MaxCols - 1, Row: MaxRows - 1}} {268+ if got := string(appendAddr(nil, a)); got != a.String() {269+ t.Errorf("appendAddr %q, want %q", got, a.String())270+ }271+ }272+}@@ -12,11 +12,12 @@ const MaxUndo = 100
1212 // before the step; undoing swaps those back in and records the state it1313 // replaced as the redo step.1414 //15-// Steps snapshot whole cells (a shallow copy of the struct), so fields16-// added to Cell later are covered without changes here. That relies on17-// one rule: a stored *Cell is never modified in place except for its18-// computed Value. To change a cell, build a new one and store it with19-// place.15+// Steps keep cells' before-images in the compact form the sheet keeps16+// its cells (see historyimage.go): plain cells as slots, rich ones as a17+// shallow copy of their Cell, so fields added to Cell later are covered18+// without changes here. That relies on one rule: a stored *Cell is never19+// modified in place except for its computed Value. To change a cell,20+// build a new one and store it with place.2021 type history struct {2122 undo, redo []*step2223 open *step // the step being built; nil outside change@@ -24,6 +25,12 @@ type history struct {
2425 dirty []loc2526 lastID int2627 bytes int64 // estimated heap held by the undo steps; see historysize.go28+ // off is set while a change runs without undo (WithoutUndo): its29+ // step records no before-images.30+ off bool31+ // base is the state ID with no step to undo: 0 until a change ran32+ // without undo, then that change's.33+ base int2734 // mergeWidths lets the next width-only step join the top one, so a2835 // live preview of a column width or row height and its final value2936 // (or its cancellation) are one step. Seal ends the run.@@ -39,7 +46,7 @@ type step struct {
3946 label string // what the step did, e.g. "clear B3:B5"4047 sheet *Sheet // the sheet the UI shows when the step is undone or redone4148 focus Rect // what the UI selects there42- cells map[loc]*Cell // before the step; nil for blank49+ cells map[*Sheet]*image // before the step, by sheet4350 widths map[colKey]int // before the step; 0 for the default width4451 heights map[rowKey]int // rows' heights before the step, 0 for none; nil until one changes4552 lines map[lineKey]lineFmt // column and row formats before the step@@ -68,12 +75,12 @@ type colKey struct {
6875 }6976 7077 func newStep(label string, s *Sheet, focus Rect) *step {71- return &step{label: label, sheet: s, focus: focus, cells: map[loc]*Cell{}, widths: map[colKey]int{}, lines: map[lineKey]lineFmt{},78+ return &step{label: label, sheet: s, focus: focus, cells: map[*Sheet]*image{}, widths: map[colKey]int{}, lines: map[lineKey]lineFmt{},7279 names: map[string]*Name{}, views: map[*Sheet]*viewState{}, charts: map[*Sheet][]Chart{}, pivots: map[*Sheet]*Pivot{}, rules: map[*Sheet]rulesState{}}7380 }7481 7582 func (st *step) empty() bool {76- return len(st.cells) == 0 && len(st.widths) == 0 && len(st.heights) == 0 && st.widthOnly()83+ return len(st.widths) == 0 && len(st.heights) == 0 && st.widthOnly()7784 }7885 7986 // widthOnly reports whether the step changed nothing but column widths@@ -163,11 +170,10 @@ func (s *Sheet) record(a Addr) {
163170 if st == nil {164171 return165172 }166- l := loc{s, a}167- if _, seen := st.cells[l]; !seen {168- st.keep(l, s.cells.copyOf(a))173+ if !s.wb.hist.off {174+ st.keep(s, a)169175 }170- s.wb.hist.dirty = append(s.wb.hist.dirty, l)176+ s.wb.hist.dirty = append(s.wb.hist.dirty, loc{s, a})171177 }172178 173179 // push adds a finished step to the undo stack and clears redo, dropping@@ -194,10 +200,11 @@ func (w *Workbook) push(st *step) {
194200 func (w *Workbook) dropUnchanged(st *step) {195201 st.dropUnchangedLines()196202 st.dropUnchangedRules()197- for l, c := range st.cells {198- if c == nil && !l.s.cells.has(l.a) {199- delete(st.cells, l)200- st.cellBytes -= entryBytes203+ for s, img := range st.cells {204+ st.cellBytes -= img.dropBlanks(s)205+ if img.len() == 0 {206+ delete(st.cells, s)207+ st.cellBytes -= imageBytes201208 }202209 }203210 for k, n := range st.names {@@ -331,7 +338,7 @@ func (w *Workbook) StateID() int {
331338 if top := w.hist.top(); top != nil {332339 return top.id333340 }334- return 0341+ return w.hist.base335342 }336343 337344 // ClearHistory forgets all undo and redo steps.@@ -375,11 +382,13 @@ func (w *Workbook) swap(undo bool) (Change, bool) {
375382 inv.sheets = w.sheetList()376383 w.setSheets(st.sheets)377384 }378- changed := make([]loc, 0, len(st.cells))379- for l, c := range st.cells {380- inv.keep(l, l.s.cells.copyOf(l.a))381- l.s.place(l.a, c.clone())382- changed = append(changed, l)385+ var changed []loc386+ for s, img := range st.cells {387+ img.each(func(a Addr, c *Cell) {388+ inv.keep(s, a)389+ s.place(a, c)390+ changed = append(changed, loc{s, a})391+ })383392 }384393 for k, width := range st.widths {385394 inv.widths[k] = k.s.widths[k.col]@@ -441,20 +450,3 @@ func (w *Workbook) swap(undo bool) (Change, bool) {
441450 macrosOnly := st.macros != nil && len(st.cells) == 0 && len(st.lines) == 0 && len(st.names) == 0 && len(st.charts) == 0 && len(st.pivots) == 0 && len(st.rules) == 0 && st.sheets == nil442451 return Change{Label: st.label, Focus: st.focus, Sheet: st.sheet, Tabs: st.sheets != nil, Macros: macrosOnly}, true443452 }444-445-// colRect is the range covering whole columns from..to.446-func colRect(from, to int) Rect {447- return Rect{From: Addr{Col: from}, To: Addr{Col: to, Row: MaxRows - 1}}448-}449-450-// rowRect is the range covering whole rows from..to.451-func rowRect(from, to int) Rect {452- return Rect{From: Addr{Row: from}, To: Addr{Col: MaxCols - 1, Row: to}}453-}454-455-func union(a, b Rect) Rect {456- return Rect{457- From: Addr{Col: min(a.From.Col, b.From.Col), Row: min(a.From.Row, b.From.Row)},458- To: Addr{Col: max(a.To.Col, b.To.Col), Row: max(a.To.Row, b.To.Row)},459- }460-}@@ -0,0 +1,248 @@
1+package sheet2+3+import "slices"4+5+// An image holds an undo step's before-images of one sheet's cells in6+// the form the sheet keeps them (store.go, slot.go): columns of blocks7+// of 16-byte slots, the text in a table of strings kept once each, the8+// formatting in a table of looks, and whole Cells only for rich cells9+// (formulas, notes, what pivots and spills write). A step that clears10+// or rewrites a whole sheet so holds about what the sheet does, 20 B11+// per number, rather than a Cell per cell. A cell that held nothing12+// before the step is a bit in blank.13+//14+// The columns are indexed by position (occupancy.go), which costs a few15+// KB for the first cell far down a sheet, so an image holds its first16+// smallImage cells in a list and moves them into columns past that:17+// the typical step, one cell or a few, costs what they do.18+type image struct {19+ small []imageCell // the cells while there are at most smallImage20+ few [2]imageCell // small's first cells, without an allocation21+ big bool // the cells are in cells and blank instead22+ // cells holds the tables of strings, looks and rich cells, and the23+ // columns once big; nil until one is needed, as a plain number24+ // needs none.25+ cells *cellStore26+ blank occupancy27+ // looks maps the sheet's looks to the image's: entry i is the28+ // image's look for the sheet's look i, plus one; 0 for not yet29+ // seen. A store's looks are only ever added, so the map holds.30+ looks []uint1631+}32+33+// imageCell is a before-image in an image's list: the slot, in the34+// image's tables, or blank for a cell that held nothing.35+type imageCell struct {36+ a Addr37+ sl slot38+ blank bool39+}40+41+// smallImage is how many cells an image lists before it keeps them in42+// columns: a list's linear search stays cheaper than an index up to43+// about this many.44+const smallImage = 3245+46+// Estimated heap of an image's parts beyond slotBytes a cell, as47+// BenchmarkClearMax and BenchmarkHistoryWide measure it.48+const (49+ blockBytes = 256 // a block of blockRows rows, made for its first cell50+ richBytes = 24 // a rich cell's entry in the side table51+)52+53+// seen reports whether the image holds the cell at a, blank or not.54+func (img *image) seen(a Addr) bool {55+ if img.big {56+ return img.cells.stored.has(a) || img.blank.has(a) // big has cells57+ }58+ for i := range img.small {59+ if img.small[i].a == a {60+ return true61+ }62+ }63+ return false64+}65+66+// len is the number of cells the image holds, blank ones included.67+func (img *image) len() int {68+ n := len(img.small) + img.blank.n69+ if img.cells != nil {70+ n += img.cells.stored.n71+ }72+ return n73+}74+75+// store is img.cells, made if need be.76+func (img *image) store() *cellStore {77+ if img.cells == nil {78+ img.cells = &cellStore{}79+ }80+ return img.cells81+}82+83+// tables is img.cells to read from: an empty store when there is none,84+// as a slot then refers to no table.85+func (img *image) tables() *cellStore {86+ if img.cells == nil {87+ return &noTables88+ }89+ return img.cells90+}91+92+// noTables is an empty store, never written.93+var noTables cellStore94+95+// keep records the cell at a of src, the sheet's store, as it stands,96+// and returns the heap that adds.97+func (img *image) keep(src *cellStore, a Addr) int64 {98+ b, i := src.find(a)99+ if b == nil {100+ return img.put(a, slot{}, true)101+ }102+ sl := b.vals[i]103+ if sl.kind == slotRich {104+ return img.keepRich(a, src.rich[sl.ref].c.clone())105+ }106+ n := int64(slotBytes)107+ if sl.look != 0 {108+ lk, ok := img.look(src, sl.look)109+ if !ok { // the image's table of looks is full110+ return img.keepRich(a, src.view(sl))111+ }112+ sl.look = lk113+ }114+ if sl.ref != 0 {115+ s := src.strs.strs[sl.ref]116+ t := &img.store().strs117+ sl.ref = t.add(s)118+ if t.refs[sl.ref] == 1 {119+ n += strBytes + int64(len(s))120+ }121+ }122+ return n + img.put(a, sl, false)123+}124+125+// keepRich records c, a rich cell or one made to stand for a plain one,126+// as the before-image at a.127+func (img *image) keepRich(a Addr, c *Cell) int64 {128+ n := int64(slotBytes+richBytes) + cellSize(c)129+ return n + img.put(a, slot{kind: slotRich, ref: img.store().addRich(a, c)}, false)130+}131+132+// put stores sl at a, or a blank there, which the image doesn't hold,133+// and returns the heap new blocks take.134+func (img *image) put(a Addr, sl slot, blank bool) int64 {135+ if !img.big {136+ if img.small == nil {137+ img.small = img.few[:0]138+ }139+ if len(img.small) < smallImage {140+ img.small = append(img.small, imageCell{a, sl, blank})141+ return 0142+ }143+ n := img.grow()144+ return n + img.put(a, sl, blank)145+ }146+ o := &img.store().stored147+ if blank {148+ o = &img.blank149+ }150+ var n int64151+ if o.col(a.Col).block(a.Row>>blockShift) == nil {152+ n = blockBytes153+ }154+ b := o.mark(a)155+ if !blank {156+ i, _ := b.index(a.Row & (blockRows - 1))157+ b.vals = slices.Insert(b.vals, i, sl)158+ }159+ return n160+}161+162+// grow moves the listed cells into columns and returns the heap their163+// blocks take.164+func (img *image) grow() int64 {165+ img.big = true166+ var n int64167+ for _, ic := range img.small {168+ n += img.put(ic.a, ic.sl, ic.blank)169+ }170+ img.small = nil171+ return n172+}173+174+// look is the image's entry for the sheet's look id, added if new, and175+// false when the image's table is full.176+func (img *image) look(src *cellStore, id uint16) (uint16, bool) {177+ if int(id) < len(img.looks) && img.looks[id] != 0 {178+ return img.looks[id] - 1, true179+ }180+ l := src.looks[id]181+ lk, ok := img.store().lookID(l.f, l.st)182+ if !ok {183+ return 0, false184+ }185+ if int(id) >= len(img.looks) {186+ img.looks = slices.Grow(img.looks, int(id)+1-len(img.looks))[:int(id)+1]187+ }188+ img.looks[id] = lk + 1189+ return lk, true190+}191+192+// each calls fn with every cell the image holds and its before-image as193+// a Cell to place, nil for blank: a rich cell itself (the image is194+// dropped once restored), a plain one made for the caller. fn mustn't195+// change the image.196+func (img *image) each(fn func(a Addr, c *Cell)) {197+ st := img.tables()198+ for _, ic := range img.small {199+ var c *Cell200+ if !ic.blank {201+ c = st.cellOf(ic.sl)202+ }203+ fn(ic.a, c)204+ }205+ for _, col := range st.stored.colIDs {206+ st.stored.colScan(col, 0, MaxRows-1, func(row int) bool {207+ a := Addr{Col: col, Row: row}208+ b, i := st.find(a)209+ fn(a, st.cellOf(b.vals[i]))210+ return true211+ })212+ }213+ for _, col := range img.blank.colIDs {214+ img.blank.colScan(col, 0, MaxRows-1, func(row int) bool {215+ fn(Addr{Col: col, Row: row}, nil)216+ return true217+ })218+ }219+}220+221+// dropBlanks forgets the blank before-images of cells that are still222+// blank in s, and returns the heap that frees: the blocks emptied.223+func (img *image) dropBlanks(s *Sheet) int64 {224+ img.small = slices.DeleteFunc(img.small, func(ic imageCell) bool { return ic.blank && !s.cells.has(ic.a) })225+ var drop []Addr226+ for _, c := range img.blank.colIDs {227+ img.blank.colScan(c, 0, MaxRows-1, func(row int) bool {228+ if a := (Addr{Col: c, Row: row}); !s.cells.has(a) {229+ drop = append(drop, a)230+ }231+ return true232+ })233+ }234+ blocks := img.blank.blocks()235+ for _, a := range drop {236+ img.blank.unmark(a)237+ }238+ return int64(blocks-img.blank.blocks()) * blockBytes239+}240+241+// blocks is the number of blocks the occupancy holds.242+func (o *occupancy) blocks() int {243+ n := 0244+ for _, c := range o.colIDs {245+ n += len(o.cols[c].ids)246+ }247+ return n248+}@@ -0,0 +1,169 @@
1+package sheet2+3+import (4+ "bytes"5+ "fmt"6+ "testing"7+)8+9+// scaleBook is a workbook of two sheets, the first holding rows rows of10+// every kind of cell a before-image keeps: numbers as typed, text,11+// formatted numbers, formulas, booleans, notes and formatting alone.12+func scaleBook(t testing.TB, rows int) *Sheet {13+ s := New()14+ cur := Format{Kind: FmtCurrency, Decimals: 2}15+ for r := range rows {16+ n := r + 117+ load := func(col int, in string, f Format, st Style) {18+ if err := s.Load(Addr{Col: col, Row: r}, in, f, st); err != nil {19+ t.Fatal(err)20+ }21+ }22+ load(0, fmt.Sprintf("%d.50", r%1000), Format{}, Style{})23+ load(1, fmt.Sprintf("item %d", r%500), Format{}, Style{Bold: r%3 == 0})24+ load(2, fmt.Sprint(r*7%10000), cur, Style{})25+ load(3, fmt.Sprintf("=A%d*2+C%d", n, n), Format{}, Style{})26+ switch r % 4 {27+ case 0:28+ load(4, "TRUE", Format{}, Style{})29+ case 1:30+ load(4, "", Format{Kind: FmtPercent}, Style{Italic: true})31+ }32+ }33+ for r := 0; r < rows; r += 100 {34+ if err := s.SetNote(Addr{Col: 1, Row: r}, fmt.Sprintf("note %d", r)); err != nil {35+ t.Fatal(err)36+ }37+ }38+ other, err := s.Book().AddSheet("Other", 1)39+ if err != nil {40+ t.Fatal(err)41+ }42+ other.Set(Addr{}, "=SUM(Sheet1!A:A)")43+ s.RecalcAll()44+ s.ClearHistory()45+ return s46+}47+48+// saved is the workbook as its file, and the value of Other!A1, which49+// reads every number of the first sheet.50+func saved(t *testing.T, s *Sheet) string {51+ t.Helper()52+ var b bytes.Buffer53+ if err := s.Book().Write(&b); err != nil {54+ t.Fatal(err)55+ }56+ w := s.Book()57+ return b.String() + fmt.Sprint(w.Sheet(w.Len()-1).Value(Addr{}))58+}59+60+// Every whole-range operation over a large sheet undoes to the workbook61+// as it was and redoes to the workbook as it made it, and its step is62+// estimated at a slot or so a plain cell, not a Cell.63+func TestUndoWholeRangeAtScale(t *testing.T) {64+ const rows = 1000065+ s := scaleBook(t, rows)66+ w := s.Book()67+ all := NewRect(Addr{}, Addr{Col: 4, Row: rows - 1})68+ ops := []struct {69+ name string70+ op func(t *testing.T)71+ }{72+ {"clear", func(*testing.T) { s.EraseRange(all) }},73+ {"format", func(*testing.T) { s.SetFormat(all, Format{Kind: FmtPercent, Decimals: 1}) }},74+ {"style", func(*testing.T) { s.SetStyle(all, func(st *Style) { st.Underline = true }) }},75+ {"clear formatting", func(*testing.T) { s.ClearFormatting(all) }},76+ {"fill", func(t *testing.T) {77+ if err := s.FillEntry(all, all.From, "7"); err != nil {78+ t.Fatal(err)79+ }80+ }},81+ {"insert rows", func(t *testing.T) {82+ if err := s.InsertRows(10, 5); err != nil {83+ t.Fatal(err)84+ }85+ }},86+ {"delete rows", func(*testing.T) { s.DeleteRows(3, 100) }},87+ {"insert columns", func(t *testing.T) {88+ if err := s.InsertCols(1, 2); err != nil {89+ t.Fatal(err)90+ }91+ }},92+ {"delete columns", func(*testing.T) { s.DeleteCols(0, 1) }},93+ {"sort", func(*testing.T) { s.SortRange(all, []SortKey{{Col: 2, Desc: true}}) }},94+ {"replace sheet", func(t *testing.T) {95+ src := New()96+ src.Set(Addr{}, "42")97+ if _, err := w.ReplaceSheet(s, src, "import"); err != nil {98+ t.Fatal(err)99+ }100+ }},101+ {"delete sheet", func(t *testing.T) {102+ if err := w.DeleteSheet(s); err != nil {103+ t.Fatal(err)104+ }105+ }},106+ }107+ for _, o := range ops {108+ before := saved(t, s)109+ o.op(t)110+ after := saved(t, s)111+ if after == before {112+ t.Fatalf("%s changed nothing", o.name)113+ }114+ if n := w.HistoryBytes(); n > int64(rows*5)*(slotBytes+32)+int64(rows)*(richBytes+cellBytes+formulaBytes+perFormulaByte*16) {115+ t.Errorf("%s: step estimated at %d bytes", o.name, n)116+ }117+ if _, ok := w.Undo(); !ok || saved(t, s) != before {118+ t.Fatalf("%s: undo doesn't restore the workbook", o.name)119+ }120+ if _, ok := w.Redo(); !ok || saved(t, s) != after {121+ t.Fatalf("%s: redo doesn't make it again", o.name)122+ }123+ w.Undo()124+ w.ClearHistory()125+ }126+}127+128+// A step over a sheet of plain numbers holds about a slot a cell.129+func TestUndoPlainCellsAsSlots(t *testing.T) {130+ s := New()131+ const n = 50000132+ for r := range n {133+ s.Load(Addr{Row: r}, fmt.Sprintf("%d.25", r), Format{}, Style{})134+ }135+ s.RecalcAll()136+ r := NewRect(Addr{}, Addr{Row: n - 1})137+ cost := s.UndoCost(r)138+ s.EraseRange(r)139+ got := s.Book().HistoryBytes()140+ if got > n*(slotBytes+4) {141+ t.Errorf("clearing %d numbers is estimated at %d bytes", n, got)142+ }143+ if cost < got*9/10 || cost > got*11/10 {144+ t.Errorf("clearing %d numbers was to cost %d bytes, and took %d", n, cost, got)145+ }146+}147+148+// A change too large to undo runs without recording, forgets the history149+// and still reads as a change to the workbook.150+func TestWithoutUndo(t *testing.T) {151+ s := New()152+ s.Set(Addr{}, "1")153+ w := s.Book()154+ id := w.StateID()155+ w.WithoutUndo(func() { s.Set(Addr{}, "2") })156+ if w.CanUndo() || w.CanRedo() || w.HistoryBytes() != 0 {157+ t.Error("the history outlived a change without undo")158+ }159+ if inputs(s)["A1"] != "2" || w.StateID() == id || w.StateID() == 0 {160+ t.Errorf("A1 %q, state %d (was %d)", inputs(s)["A1"], w.StateID(), id)161+ }162+ s.Set(Addr{}, "3")163+ if s.Undo(); inputs(s)["A1"] != "2" {164+ t.Errorf("undo after: A1 %q", inputs(s)["A1"])165+ }166+ if cost := s.UndoCost(NewRect(Addr{}, Addr{Col: 9, Row: 9})); cost < slotBytes || cost > 1024 {167+ t.Errorf("one cell's undo cost %d", cost)168+ }169+}@@ -5,21 +5,26 @@ import "slices"
55 // MaxUndoBytes caps the memory the undo history's before-images hold, as66 // estimated by step.size. When a new step takes the history past it, the77 // oldest steps are dropped first; the newest step is always kept, however8-// large, so any single change can be undone. 256 MB keeps MaxUndo steps9-// that each rewrite a whole column (about 250 MB, see docs/contributing/limits.md).8+// large, so any single change can be undone (a step past MaxStepBytes9+// asks first). 256 MB holds a step clearing a full max-cells sheet of10+// numbers, about 195 MB, and MaxUndo steps rewriting a whole column of11+// 8192 take 16 MB (see docs/contributing/limits.md).1012 const MaxUndoBytes = 256 << 201113 1214 // undoBudget is MaxUndoBytes; tests lower it.1315 var undoBudget int64 = MaxUndoBytes1416 15-// Estimated heap held by the parts of a step. A before-image is a Cell16-// (216 B) plus its map entry, its input and boxed literal (a plain cell's17-// are made for it from the store, see store.go); a formula also keeps its18-// parsed tree and reference lists alive, which grow with its text. The constants match the heap measured by the stress19-// benchmarks (BenchmarkHistoryFull) within about 10%.17+// Estimated heap held by the parts of a step. A plain cell's18+// before-image is a slot (slotBytes) and its text, once per image (see19+// historyimage.go); a rich one is also a Cell (216 B) with its input,20+// and a formula keeps its parsed tree and reference lists alive, which21+// grow with its text. The constants match the heap measured by the22+// stress benchmarks (BenchmarkHistoryFull, BenchmarkHistoryWide,23+// BenchmarkClearMax) within about 10%.2024 const (2125 stepBytes = 512 // the step and its empty maps2226 entryBytes = 64 // a map entry: the key and a pointer27+ imageBytes = 256 // a sheet's image, empty2328 cellBytes = 256 // a Cell, its input's header and its boxed literal2429 formulaBytes = 256 // a parsed formula's tree and references, plus perFormulaByte per byte of text2530 perFormulaByte = 16@@ -27,10 +32,18 @@ const (
2732 ruleBytes = 256 // a rule with its ranges and values2833 )2934 30-// keep records c as the before-image of the cell at l and counts it.31-func (st *step) keep(l loc, c *Cell) {32- st.cells[l] = c33- st.cellBytes += entryBytes + cellSize(c)35+// keep records the cell at a of s as it stands, unless the step holds36+// it already, and counts it.37+func (st *step) keep(s *Sheet, a Addr) {38+ img := st.cells[s]39+ if img == nil {40+ img = &image{}41+ st.cells[s] = img42+ st.cellBytes += imageBytes43+ }44+ if !img.seen(a) {45+ st.cellBytes += img.keep(&s.cells, a)46+ }3447 }3548 3649 // size estimates the heap a finished step holds: its cells, counted as@@ -108,3 +121,57 @@ func (h *history) popUndo() *step {
108121 109122 // HistoryBytes estimates the memory the undo steps hold.110123 func (w *Workbook) HistoryBytes() int64 { return w.hist.bytes }124+125+// MaxStepBytes is the most undo history one change may hold without126+// asking: the UI asks before a change its UndoCost puts over it, and runs127+// it WithoutUndo if told to go on. Plain cells cost about 20 B each, so128+// only millions of formulas or of distinct long texts come near it.129+const MaxStepBytes = 1 << 30130+131+// UndoCost estimates the undo history a change to every cell of r would132+// hold: a slot a cell, the rich cells in r whole, and r's share of the133+// sheet's strings. The UI asks it before every command that edits, so134+// it costs the fewer of r's cells and the sheet's rich cells.135+func (s *Sheet) UndoCost(r Rect) int64 {136+ st := &s.cells137+ cells := 0138+ for _, c := range st.stored.colsIn(r.From.Col, r.To.Col) {139+ cells += st.stored.countIn(c, r.From.Row, r.To.Row)140+ }141+ if cells == 0 {142+ return 0143+ }144+ n := int64(cells)*slotBytes + st.strs.bytes*int64(cells)/int64(st.len())145+ add := func(c *Cell) { n += richBytes + cellSize(c) }146+ if cells < len(st.rich)-len(st.richFree) { // the cheaper way to find r's rich cells147+ for a := range st.anyKeysIn(r) {148+ if c := st.richAt(a); c != nil {149+ add(c)150+ }151+ }152+ return n153+ }154+ for _, rc := range st.rich {155+ if rc.c != nil && r.Contains(rc.a) {156+ add(rc.c)157+ }158+ }159+ return n160+}161+162+// WithoutUndo runs fn, a change too large to undo (see MaxStepBytes),163+// recording no before-images, and forgets the undo history, which could164+// no longer be undone past it. Inside a Batch it runs fn as part of it.165+func (w *Workbook) WithoutUndo(fn func()) {166+ h := &w.hist167+ if h.depth > 0 {168+ fn()169+ return170+ }171+ h.off = true172+ defer func() {173+ id := h.lastID + 1174+ w.hist = history{lastID: id, base: id}175+ }()176+ fn()177+}@@ -22,9 +22,9 @@ func TestUndoByteBudget(t *testing.T) {
2222 fillColumn(t, s, 1000, "x")2323 w := s.Book()2424 w.ClearHistory()25- fillColumn(t, s, 1000, "0") // before-images of 1000 cells25+ fillColumn(t, s, 1000, "0") // before-images of 1000 cells, slots and one string2626 one := w.HistoryBytes()27- if one < 1000*cellBytes/2 || one > 1000*(entryBytes+cellBytes+64) {27+ if one < 1000*slotBytes || one > 1000*slotBytes+8<<10 {2828 t.Fatalf("a 1000-cell step is estimated at %d bytes", one)2929 }3030 defer func(b int64) { undoBudget = b }(undoBudget)@@ -120,8 +120,8 @@ func historySum(w *Workbook) int64 {
120120 var n int64121121 for _, st := range w.hist.undo {122122 var cells int64123- for _, c := range st.cells {124- cells += entryBytes + cellSize(c)123+ for _, img := range st.cells {124+ cells += img.walkSize()125125 }126126 if cells != st.cellBytes {127127 return -1@@ -130,3 +130,22 @@ func historySum(w *Workbook) int64 {
130130 }131131 return n132132 }133+134+// walkSize is what keep counts for the image, found by walking it.135+func (img *image) walkSize() int64 {136+ t := img.tables()137+ n := int64(imageBytes + t.stored.n*slotBytes)138+ for _, ic := range img.small {139+ if !ic.blank {140+ n += slotBytes141+ }142+ }143+ n += int64(t.stored.blocks()+img.blank.blocks()) * blockBytes144+ for _, rc := range t.rich {145+ n += richBytes + cellSize(rc.c)146+ }147+ for _, s := range t.strs.strs[min(1, len(t.strs.strs)):] {148+ n += strBytes + int64(len(s))149+ }150+ return n151+}@@ -116,6 +116,11 @@ func TestShapers(t *testing.T) {
116116 if !s.RuleAbove(7) || s.RuleAbove(6) || s.RuleAbove(4) {117117 t.Error("rule above row 8 only")118118 }119+ // A number read from a file is indexed too.120+ s.Set(at("A7"), "12.5")121+ if got := roundTrip(t, s); !got.RuleAbove(7) || !slices.Equal(got.WrappedIn(4), []int{2}) {122+ t.Error("the file's borders and wrapping aren't indexed")123+ }119124 s.ClearFormatting(rng("A1:Z99"))120125 if s.Shaped() {121126 t.Error("still shaped after clearing")@@ -1,7 +1,7 @@
11 package sheet22 33 import (4- "bytes"4+ "bufio"55 "encoding/json"66 "fmt"77 "maps"@@ -23,7 +23,7 @@ import (
2323 // needs no new version.2424 2525 // writeLines writes the "lines" field, if the sheet has line formats.26-func (s *Sheet) writeLines(b *bytes.Buffer, indent string) error {26+func (s *Sheet) writeLines(b *bufio.Writer, indent string) error {2727 if s.lines.none() {2828 return nil2929 }@@ -115,3 +115,20 @@ func (s *Sheet) sheetEdge(a Addr, dc, dr int) Addr {
115115 }116116 return a117117 }118+119+// colRect is the range covering whole columns from..to.120+func colRect(from, to int) Rect {121+ return Rect{From: Addr{Col: from}, To: Addr{Col: to, Row: MaxRows - 1}}122+}123+124+// rowRect is the range covering whole rows from..to.125+func rowRect(from, to int) Rect {126+ return Rect{From: Addr{Row: from}, To: Addr{Col: MaxCols - 1, Row: to}}127+}128+129+func union(a, b Rect) Rect {130+ return Rect{131+ From: Addr{Col: min(a.From.Col, b.From.Col), Row: min(a.From.Row, b.From.Row)},132+ To: Addr{Col: max(a.To.Col, b.To.Col), Row: max(a.To.Row, b.To.Row)},133+ }134+}@@ -330,3 +330,27 @@ func (o *occupancy) bounds(r Rect) (Rect, bool) {
330330 }331331 return out, found332332 }333+334+// countIn is the number of cells in rows r0 to r1 of column c, counted335+// by block: whole blocks by their count, the ends by popcount.336+func (o *occupancy) countIn(c, r0, r1 int) int {337+ ci := o.col(c)338+ if ci == nil || r0 > r1 {339+ return 0340+ }341+ n := 0342+ i, _ := slices.BinarySearch(ci.ids, r0>>blockShift)343+ for ; i < len(ci.ids) && ci.ids[i] <= r1>>blockShift; i++ {344+ id := ci.ids[i]345+ b, base := ci.blocks[id], id<<blockShift346+ lo, hi := max(r0-base, 0), min(r1-base, blockRows-1)347+ if lo == 0 && hi == blockRows-1 {348+ n += b.n349+ continue350+ }351+ for w := lo >> 6; w <= hi>>6; w++ {352+ n += bits.OnesCount64(b.bits[w] & wordMask(w, lo, hi))353+ }354+ }355+ return n356+}@@ -1,7 +1,7 @@
11 package sheet22 33 import (4- "bytes"4+ "bufio"55 "maps"66 "slices"77 "strconv"@@ -296,7 +296,7 @@ func (s *Sheet) heightsIn(lo, hi int) map[int]int {
296296 }297297 298298 // writeHeights writes the "heights" field: rows by number, in lines.299-func (s *Sheet) writeHeights(b *bytes.Buffer, indent string) {299+func (s *Sheet) writeHeights(b *bufio.Writer, indent string) {300300 if len(s.heights) == 0 {301301 return302302 }@@ -1,7 +1,7 @@
11 package sheet22 33 import (4- "bytes"4+ "bufio"55 "encoding/json"66 "fmt"77 "strings"@@ -172,7 +172,7 @@ func (fv fileValidation) rule() (Validation, error) {
172172 173173 // writeRules writes the sheet's rules, one per line, after its other174174 // fields.175-func (s *Sheet) writeRules(b *bytes.Buffer, indent string) {175+func (s *Sheet) writeRules(b *bufio.Writer, indent string) {176176 list := func(key string, lines []string) {177177 if len(lines) == 0 {178178 return@@ -56,6 +56,7 @@ type strTable struct {
5656 refs []uint325757 index map[string]uint325858 free []uint3259+ bytes int64 // the estimated heap the strings hold, strBytes and their text each5960 }6061 6162 // add counts one more use of s and returns its entry.@@ -77,6 +78,7 @@ func (t *strTable) add(s string) uint32 {
7778 t.strs, t.refs = append(t.strs, s), append(t.refs, 1)7879 }7980 t.index[s] = i81+ t.bytes += strBytes + int64(len(s))8082 return i8183 }8284 @@ -86,6 +88,7 @@ func (t *strTable) release(i uint32) {
8688 return8789 }8890 if t.refs[i]--; t.refs[i] == 0 {91+ t.bytes -= strBytes + int64(len(t.strs[i]))8992 delete(t.index, t.strs[i])9093 t.strs[i] = ""9194 t.free = append(t.free, i)@@ -171,6 +174,9 @@ func (st *cellStore) plainSlot(c *Cell) (slot, bool) {
171174 // numDec is how input prints v, as slot.dec, or false if it isn't v172175 // printed plainly.173176 func numDec(v float64, input string) (uint8, bool) {177+ if d, ok := plainDec(v, input); ok {178+ return d, true179+ }174180 var buf [64]byte175181 if string(strconv.AppendFloat(buf[:0], v, 'f', -1, 64)) == input {176182 return 0, true@@ -186,6 +192,60 @@ func numDec(v float64, input string) (uint8, bool) {
186192 return 0, false187193 }188194 195+// plainDec is numDec for the common input, a plain decimal (see196+// plainForm) that is v, found without printing v. It reports false for197+// any other input, which numDec then prints v to compare.198+func plainDec(v float64, input string) (uint8, bool) {199+ d, ok := plainForm(input)200+ if !ok {201+ return 0, false202+ }203+ if p, err := strconv.ParseFloat(input, 64); err != nil || math.Float64bits(p) != math.Float64bits(v) {204+ return 0, false205+ }206+ return d, true207+}208+209+// plainForm reports whether input is a plain decimal, of at most 15210+// significant digits without a sign but "-", leading zeros or an211+// exponent, and returns how it prints its value as slot.dec. Such a212+// decimal is its value's shortest text when it doesn't end in a zero213+// after the point, and its value printed with as many decimals as it214+// has otherwise: 15 digits always come back from a float64.215+func plainForm(input string) (uint8, bool) {216+ s := input217+ if s != "" && s[0] == '-' {218+ s = s[1:]219+ }220+ if s == "" || s[0] == '0' && len(s) > 1 && s[1] != '.' {221+ return 0, false222+ }223+ point, digits, lead := -1, 0, true224+ for i := 0; i < len(s); i++ {225+ switch c := s[i]; {226+ case c == '.' && point < 0 && i > 0 && i < len(s)-1:227+ point = i228+ case c >= '0' && c <= '9':229+ if lead = lead && c == '0'; !lead {230+ digits++231+ }232+ default:233+ return 0, false234+ }235+ }236+ d := 0237+ if point >= 0 {238+ d = len(s) - point - 1239+ }240+ if digits > 15 || d >= maxDec {241+ return 0, false242+ }243+ if d == 0 || s[len(s)-1] != '0' {244+ return 0, true245+ }246+ return uint8(d + 1), true247+}248+189249 func boolText(b bool) string {190250 if b {191251 return "TRUE"@@ -259,10 +319,7 @@ func (st *cellStore) size() int64 {
259319 for _, rc := range st.rich {260320 n += cellSize(rc.c)261321 }262- for _, s := range st.strs.strs {263- n += strBytes + int64(len(s))264- }265- return n322+ return n + st.strs.bytes266323 }267324 268325 // richCell is a rich cell and where it is.@@ -0,0 +1,62 @@
1+package sheet2+3+import (4+ "fmt"5+ "math"6+ "math/rand/v2"7+ "strconv"8+ "strings"9+ "testing"10+)11+12+// printedDec is numDec found only by printing v, as for inputs13+// plainDec leaves.14+func printedDec(v float64, input string) (uint8, bool) {15+ if strconv.FormatFloat(v, 'f', -1, 64) == input {16+ return 0, true17+ }18+ i := strings.IndexByte(input, '.')19+ if i < 0 || len(input)-i-1 >= maxDec {20+ return 0, false21+ }22+ d := len(input) - i - 123+ if strconv.FormatFloat(v, 'f', d, 64) == input {24+ return uint8(d + 1), true25+ }26+ return 0, false27+}28+29+// Where plainDec answers without printing, it answers as printing does.30+func TestPlainDecAsPrinted(t *testing.T) {31+ inputs := []string{"0", "-0", "0.0", "-0.0", "0.50", "1.5", "100", "100.0", "12.30", "007", "0.1", "0.000", "1e5", ".5", "5.",32+ "123456789012345", "1234567890123456", "0.000000000000000000001", "99999999999999.9", "-42.4200", "1,200", "$5", "1.2.3", "--1", "+1"}33+ r := rand.New(rand.NewPCG(3, 4))34+ for range 20000 {35+ switch r.IntN(3) {36+ case 0:37+ inputs = append(inputs, fmt.Sprintf("%.*f", r.IntN(8), (r.Float64()-0.5)*math.Pow(10, float64(r.IntN(18)))))38+ case 1:39+ inputs = append(inputs, strconv.FormatInt(r.Int64N(1e17)-5e16, 10))40+ default:41+ inputs = append(inputs, strconv.FormatFloat(r.NormFloat64()*1e6, 'f', -1, 64))42+ }43+ }44+ answered := 045+ for _, in := range inputs {46+ v, err := strconv.ParseFloat(in, 64)47+ if err != nil {48+ continue49+ }50+ d, ok := plainDec(v, in)51+ if !ok {52+ continue53+ }54+ answered++55+ if pd, pok := printedDec(v, in); !pok || pd != d {56+ t.Errorf("%q: plainDec %d, printed %d %v", in, d, pd, pok)57+ }58+ }59+ if answered < len(inputs)/5 {60+ t.Errorf("plainDec answered %d of %d", answered, len(inputs))61+ }62+}@@ -217,6 +217,21 @@ func (st *cellStore) set(a Addr, c *Cell) {
217217 st.touch(a)218218 }219219 220+// setSlot stores sl, a plain slot whose strings and look st already221+// counts, at a: set for a slot made directly, as a file's numbers are.222+func (st *cellStore) setSlot(a Addr, sl slot) {223+ b, i := st.find(a)224+ if b == nil {225+ b = st.stored.mark(a)226+ i, _ = b.index(a.Row & (blockRows - 1))227+ b.vals = slices.Insert(b.vals, i, slot{kind: slotBlank})228+ }229+ st.releaseSlot(b.vals[i])230+ b.vals[i] = sl231+ st.filledAs(a, st.filled.has(a), sl.kind != slotBlank)232+ st.touch(a)233+}234+220235 // filledAs updates the index of filled cells for a cell at a that was221236 // and is filled or not.222237 func (st *cellStore) filledAs(a Addr, was, is bool) {@@ -9,8 +9,11 @@ import (
99 "bytes"1010 "fmt"1111 "runtime"12+ "runtime/metrics"1213 "sync"14+ "sync/atomic"1315 "testing"16+ "time"1417 1518 "github.com/FelineStateMachine/012/internal/sheet"1619 "github.com/FelineStateMachine/012/internal/stress"@@ -82,6 +85,26 @@ func BenchmarkBigUndo(b *testing.B) {
8285 }8386 }8487 88+// BenchmarkClearMax clears all of a full max-cells sheet of numbers (1M89+// x 10) as one step and undoes it, then reports the heap such a step90+// holds (what forgetting it frees) and the budget's estimate of it.91+func BenchmarkClearMax(b *testing.B) {92+ s := stress.Dense(stress.MaxRows, stress.MaxCols)93+ all := sheet.NewRect(sheet.Addr{}, sheet.Addr{Col: stress.MaxCols - 1, Row: stress.MaxRows - 1})94+ s.ClearHistory()95+ for b.Loop() {96+ s.EraseRange(all)97+ s.Undo()98+ s.ClearHistory()99+ }100+ s.EraseRange(all)101+ est := s.Book().HistoryBytes()102+ held := heap()103+ s.ClearHistory()104+ b.ReportMetric(float64(held-heap())/(1<<20), "MB-held")105+ b.ReportMetric(float64(est)/(1<<20), "MB-estimate")106+}107+85108 // BenchmarkHistoryFull makes MaxUndo whole-column edits, the most the86109 // history keeps, and reports the heap they hold.87110 func BenchmarkHistoryFull(b *testing.B) {@@ -200,24 +223,29 @@ func BenchmarkFillSeries(b *testing.B) {
200223 }201224 }202225 203-// BenchmarkSave writes the native .012 file and reports its size.226+// BenchmarkSave writes the native .012 file and reports its size and227+// the most heap in use while writing it, above what the sheet holds.204228 func BenchmarkSave(b *testing.B) {205229 for _, sh := range saveShapes() {206230 b.Run(sh.name, func(b *testing.B) {207231 s := sh.build()208- var buf bytes.Buffer209- for b.Loop() {210- buf.Reset()211- if err := s.Write(&buf); err != nil {212- b.Fatal(err)232+ var out counter233+ peak := peakHeap(func() {234+ for b.Loop() {235+ out = 0236+ if err := s.Write(&out); err != nil {237+ b.Fatal(err)238+ }213239 }214- }215- b.ReportMetric(float64(buf.Len())/(1<<20), "MB-file")240+ })241+ b.ReportMetric(float64(out)/(1<<20), "MB-file")242+ b.ReportMetric(float64(peak)/(1<<20), "MB-peak")216243 })217244 }218245 }219246 220-// BenchmarkOpen reads the native .012 file back.247+// BenchmarkOpen reads the native .012 file back, and reports the most248+// heap in use while reading it, the sheet read included.221249 func BenchmarkOpen(b *testing.B) {222250 for _, sh := range saveShapes() {223251 b.Run(sh.name, func(b *testing.B) {@@ -225,15 +253,55 @@ func BenchmarkOpen(b *testing.B) {
225253 if err := sh.build().Write(&buf); err != nil {226254 b.Fatal(err)227255 }228- for b.Loop() {229- if _, err := sheet.Read(bytes.NewReader(buf.Bytes())); err != nil {230- b.Fatal(err)256+ peak := peakHeap(func() {257+ for b.Loop() {258+ if _, err := sheet.Read(bytes.NewReader(buf.Bytes())); err != nil {259+ b.Fatal(err)260+ }231261 }232- }262+ })263+ b.ReportMetric(float64(peak)/(1<<20), "MB-peak")233264 })234265 }235266 }236267 268+// counter is a writer that keeps only the count of bytes written.269+type counter int64270+271+func (c *counter) Write(p []byte) (int, error) {272+ *c += counter(len(p))273+ return len(p), nil274+}275+276+// peakHeap runs fn and returns the most heap in use meanwhile above what277+// was live before, sampled every millisecond.278+func peakHeap(fn func()) uint64 {279+ base := heap()280+ sample := []metrics.Sample{{Name: "/memory/classes/heap/objects:bytes"}}281+ var peak atomic.Uint64282+ done := make(chan struct{})283+ var wg sync.WaitGroup284+ wg.Go(func() {285+ t := time.NewTicker(time.Millisecond)286+ defer t.Stop()287+ for {288+ metrics.Read(sample)289+ if v := sample[0].Value.Uint64(); v > peak.Load() {290+ peak.Store(v)291+ }292+ select {293+ case <-done:294+ return295+ case <-t.C:296+ }297+ }298+ })299+ fn()300+ close(done)301+ wg.Wait()302+ return peak.Load() - min(base, peak.Load())303+}304+237305 type namedBuild struct {238306 name string239307 build func() *sheet.Sheet@@ -1,7 +1,7 @@
11 package sheet22 33 import (4- "bytes"4+ "bufio"55 "encoding/json"66 "fmt"77 "reflect"@@ -159,7 +159,7 @@ type fileCriteria struct {
159159 160160 // writeView adds the frozen panes, the filter and the protected ranges to161161 // a file being written.162-func (s *Sheet) writeView(b *bytes.Buffer, indent string) error {162+func (s *Sheet) writeView(b *bufio.Writer, indent string) error {163163 var keys []string164164 var parts []any165165 if v := s.view; v.frozenRows > 0 || v.frozenCols > 0 {@@ -191,6 +191,9 @@ func (m *Model) runCommand(id string) tea.Cmd {
191191 if m.askCommand(c) {192192 return nil193193 }194+ if c.edits != nil && m.askUndoCost(c.edits(m), func(m *Model) tea.Cmd { return m.runCommand(c.id) }) {195+ return nil196+ }194197 if m.rec != nil {195198 return m.recordingCommand(c) // macrorec.go196199 }@@ -2,6 +2,7 @@ package ui
22 33 import (44 "fmt"5+ "io"56 "io/fs"67 "log/slog"78 "os"@@ -189,14 +190,16 @@ func (m *Model) saveAs(name string, check bool) tea.Cmd {
189190 }190191 191192 // saveCmd writes the worksheet to path atomically: to a temporary file192-// first, then renamed over the target. With expect set, it writes only193-// if the file on disk is still that version.193+// beside it first, then renamed over the target. The file is written194+// here, as the workbook stands, streaming (see Workbook.Write), so a195+// large workbook isn't held twice in memory; the rename happens in the196+// background. With expect set, it renames only if the file on disk is197+// still that version.194198 func saveCmd(s *sheet.Sheet, name, path string, expect *stamp, spans *telemetry.Trace) tea.Cmd {195199 span := spans.Start("save")196- var buf strings.Builder197- err := s.Write(&buf)200+ tmp, n, err := writeTemp(path, s.Write)198201 if telemetry.Enabled() {199- span.End(slog.Int("cells", s.Len()), slog.Int("bytes", buf.Len()))202+ span.End(slog.Int("cells", s.Len()), slog.Int64("bytes", n))200203 }201204 return func() tea.Msg {202205 if err != nil {@@ -204,37 +207,50 @@ func saveCmd(s *sheet.Sheet, name, path string, expect *stamp, spans *telemetry.
204207 }205208 // A file deleted meanwhile is simply written again.206209 if now := diskStamp(path); expect != nil && now != (stamp{}) && !now.equal(*expect) {210+ os.Remove(tmp)207211 return savedMsg{name: name, conflict: true}208212 }209- if err := writeAtomic(path, buf.String()); err != nil {213+ if err := os.Rename(tmp, path); err != nil {214+ os.Remove(tmp)210215 return savedMsg{name: name, err: err}211216 }212217 return savedMsg{name: name, stamp: diskStamp(path)}213218 }214219 }215220 216-// writeAtomic writes data to path through a temporary file of its own217-// beside it, so two writers never share one, then renames it over path.218-func writeAtomic(path, data string) error {221+// writeTemp writes a file with write into a temporary file of its own222+// beside path, so two writers never share one, and returns its name and223+// size. On an error it removes the file.224+func writeTemp(path string, write func(io.Writer) error) (string, int64, error) {219225 f, err := os.CreateTemp(filepath.Dir(path), "."+filepath.Base(path)+".*.tmp")220226 if err != nil {221- return err227+ return "", 0, err222228 }223229 tmp := f.Name()224- _, err = f.WriteString(data)230+ cw := &countWriter{w: f}231+ err = write(cw)225232 if err == nil {226233 err = f.Chmod(0o644)227234 }228235 if cerr := f.Close(); err == nil {229236 err = cerr230237 }231- if err == nil {232- err = os.Rename(tmp, path)233- }234238 if err != nil {235239 os.Remove(tmp)236240 }237- return err241+ return tmp, cw.n, err242+}243+244+// countWriter counts the bytes written through it.245+type countWriter struct {246+ w io.Writer247+ n int64248+}249+250+func (c *countWriter) Write(p []byte) (int, error) {251+ n, err := c.w.Write(p)252+ c.n += int64(n)253+ return n, err238254 }239255 240256 // loadCmd reads the file in the background, the span of it and its@@ -84,14 +84,15 @@ type Model struct {
8484 recovered string // the recovery file restored into this book, removed once it's saved: recovery.go8585 8686 mode mode87- protectOK bool // an edit to a protected range was agreed to: protect.go87+ protectOK bool // an edit to a protected range was agreed to: protect.go88+ undoOK bool // a change too large to undo was agreed to: undocost.go89+ note string // feedback on the last action, e.g. "Undid: clear B3"90+ warn string // like note, for something that went wrong, e.g. a macro's error91+ errMsg string // the message ERROR mode shows8892 // borderLine is the line Format > Borders draws with: layoutfmt.go.8993 borderLine sheet.Line9094 // painted is what borders drew this frame, by role: gridlines.go.9195 painted map[paintKey]string92- note string // feedback on the last action, e.g. "Undid: clear B3"93- warn string // like note, for something that went wrong, e.g. a macro's error94- errMsg string // the message ERROR mode shows9596 9697 // Components. Each owns its state and the handling of the input it9798 // takes; Model routes messages to them and composes what they draw.@@ -69,7 +69,8 @@ func (m *Model) sort(r sheet.Rect, keys []sheet.SortKey) tea.Cmd {
6969 m.fail(sheet.ErrSortMerged.Error())7070 return nil7171 }72- if m.askProtected(r, func(m *Model) tea.Cmd { return m.sort(r, keys) }) {72+ retry := func(m *Model) tea.Cmd { return m.sort(r, keys) }73+ if m.askProtected(r, retry) || m.askUndoCost(r, retry) {7374 return nil7475 }7576 span := m.spans.Start("sort", slog.Int("rows", r.To.Row-r.From.Row+1), slog.Int("cols", r.To.Col-r.From.Col+1), slog.Int("keys", len(keys)))@@ -0,0 +1,57 @@
1+package ui2+3+import (4+ "fmt"5+6+ tea "charm.land/bubbletea/v2"7+8+ "github.com/FelineStateMachine/012/internal/sheet"9+)10+11+// A change whose undo step would hold more than sheet.MaxStepBytes12+// (millions of formulas, or of distinct long texts, cleared or rewritten13+// at once) asks first on the context line rather than holding gigabytes14+// for undo: Enter runs it without undo, forgetting the history, and Esc15+// backs out. Commands are checked through what they edit, as protected16+// ranges are, and so is sorting. Macros that run don't ask.17+18+// maxStepBytes is sheet.MaxStepBytes; tests lower it.19+var maxStepBytes int64 = sheet.MaxStepBytes20+21+// askUndoCost reports whether changing r would take more undo history22+// than maxStepBytes, and if so asks whether to go on: Enter runs retry23+// without undo, and retry isn't asked again.24+func (m *Model) askUndoCost(r sheet.Rect, retry func(*Model) tea.Cmd) bool {25+ if m.undoOK || m.macros.run != nil {26+ return false27+ }28+ cost := m.sheet.UndoCost(r)29+ if cost <= maxStepBytes {30+ return false31+ }32+ protectOK := m.protectOK // agreed to before this asked33+ m.openOverlay(&choiceBar{34+ m: m, msg: "This can't be undone: it would take " + byteSize(cost) + " of undo history.", warn: true,35+ desc: "Going on runs it without undo and forgets the changes before it",36+ choices: []choice{37+ {key: "enter", label: "Go on without undo", run: func(m *Model) tea.Cmd {38+ m.undoOK, m.protectOK = true, protectOK39+ defer func() { m.undoOK, m.protectOK = false, false }()40+ var cmd tea.Cmd41+ m.book().WithoutUndo(func() { cmd = retry(m) })42+ m.syncChanged()43+ return cmd44+ }},45+ {key: "esc", label: "Cancel", run: func(*Model) tea.Cmd { return nil }},46+ },47+ })48+ return true49+}50+51+// byteSize is n bytes in MB, or in GB from a thousand MB.52+func byteSize(n int64) string {53+ if mb := n >> 20; mb < 1000 {54+ return fmt.Sprintf("%d MB", mb)55+ }56+ return fmt.Sprintf("%.1f GB", float64(n)/(1<<30))57+}@@ -0,0 +1,39 @@
1+package ui2+3+import (4+ "strings"5+ "testing"6+7+ "github.com/FelineStateMachine/012/internal/sheet"8+)9+10+// A change whose undo step would pass the ceiling asks first: Esc backs11+// out, Enter runs it without undo and forgets the history.12+func TestUndoCostAsks(t *testing.T) {13+ defer func(n int64) { maxStepBytes = n }(maxStepBytes)14+ maxStepBytes = 10015+ m := newModel()16+ press(t, m, "1", "<enter>", "2", "<enter>", "3", "<enter>", "4", "<enter>", "5", "<enter>", "6", "<enter>", "7", "<enter>")17+ press(t, m, "<up>", "<shift+up>", "<shift+up>", "<shift+up>", "<shift+up>", "<shift+up>", "<shift+up>")18+ m.runCommand("clear")19+ l := line(m, contextLine)20+ if _, ok := m.overlay.(*choiceBar); !ok || !strings.Contains(l, "This can't be undone: it would take 0 MB of undo history.") {21+ t.Fatalf("clear didn't ask: %q", l)22+ }23+ press(t, m, "<esc>")24+ if m.sheet.Value(addr("A1")).Num != 1 || !m.sheet.CanUndo() {25+ t.Fatalf("Esc went on: A1 %v", m.sheet.Value(addr("A1")))26+ }27+ m.runCommand("clear")28+ press(t, m, "<enter>")29+ if m.sheet.Value(addr("A1")).Kind != sheet.Empty || m.sheet.CanUndo() || !m.changed {30+ t.Fatalf("after Enter: A1 %v, can undo %v, changed %v", m.sheet.Value(addr("A1")), m.sheet.CanUndo(), m.changed)31+ }32+ // Under the ceiling nothing asks.33+ maxStepBytes = sheet.MaxStepBytes34+ press(t, m, "8", "<enter>", "<up>")35+ m.runCommand("clear")36+ if m.overlay != nil || m.sheet.Value(addr("A1")).Kind != sheet.Empty || !m.sheet.CanUndo() {37+ t.Errorf("small clear: overlay %T, A1 %v", m.overlay, m.sheet.Value(addr("A1")))38+ }39+}@@ -49,7 +49,8 @@ func parseNumberFormat(s string) (float64, Format, bool) {
4949 s = strings.ReplaceAll(s, ",", "")5050 }5151 v, err := strconv.ParseFloat(s, 64)52- if err != nil || strings.ContainsAny(s, "xXpP_") || strings.EqualFold(s, "inf") {52+ dot, exp, odd := scanNumber(s)53+ if err != nil || odd || strings.EqualFold(s, "inf") {5354 return 0, Format{}, false5455 }5556 if pct {@@ -66,7 +67,7 @@ func parseNumberFormat(s string) (float64, Format, bool) {
6667 }6768 // The format Sheets infers: two decimals if any were typed.6869 dec := 069- if strings.Contains(s, ".") {70+ if dot {7071 dec = 27172 }7273 var f Format@@ -75,7 +76,7 @@ func parseNumberFormat(s string) (float64, Format, bool) {
7576 f = Format{Kind: FmtCurrency, Decimals: dec}7677 case pct:7778 f = Format{Kind: FmtPercent, Decimals: dec}78- case strings.ContainsAny(s, "eE"):79+ case exp:7980 f = Preset(FmtScientific)8081 case grouped:8182 f = Format{Kind: FmtNumber, Decimals: dec}@@ -83,6 +84,23 @@ func parseNumberFormat(s string) (float64, Format, bool) {
8384 return v, f, true8485 }8586 87+// scanNumber reports whether s has a decimal point, an exponent, and88+// what ParseFloat accepts but an entry isn't: hexadecimal, underscores.89+// One pass, as every number loaded goes through it.90+func scanNumber(s string) (dot, exp, odd bool) {91+ for i := 0; i < len(s); i++ {92+ switch s[i] {93+ case '.':94+ dot = true95+ case 'e', 'E':96+ exp = true97+ case 'x', 'X', 'p', 'P', '_':98+ odd = true99+ }100+ }101+ return dot, exp, odd102+}103+86104 // validGrouping checks thousands separators: groups of three digits87105 // before the decimal point, and none after it (1.234,5 is text).88106 func validGrouping(s string) bool {