The reader scans the JSON a value at a time: each cell goes straight into its sheet as it is read, and the other fields, which are small, are captured as written and decoded with encoding/json. Keys match as encoding/json matches them; cells read before the version (a file written by hand) wait for it. A version or list of sheets given twice once cells were read by the first is refused. The writer streams through a bufio.Writer, cells in row-major order from the store's own iteration, plain cells without formatting written from their slots. The bytes are the same. FuzzRead holds the streaming reader to encoding/json decoding the whole file, seeded with the parser's fuzz inputs and hand-written files. dense-1Mx10: open 8.2 s -> 2.6 s, peak heap 2.3 GB -> 545 MB; save 3.7 s -> 0.54 s, peak heap 1244 MB -> 0.7 MB. dense-8192x256: open 1.43 s -> 0.54 s (539 -> 121 MB peak); save 0.80 s -> 0.14 s.
- commit
- cc3626c2467468d766d9b1b0d0a97c76015c375f
- parent
- 7fbf09775997ff2cfc4e0460b8cf86cb226492d6
- author
- Dami <dami.etoile@gmail.com>
| M | Makefile | 1 | + |
| M | internal/ | 57 | +++++ |
| M | internal/ | 133 | +++--------- |
| A | internal/ | 336 | ++++++++++++++++++++++++++++++ |
| A | internal/ | 265 | ++++++++++++++++++++++++ |
| A | internal/ | 208 | +++++++++++++++++++ |
| M | internal/ | 4 | +- |
| M | internal/ | 4 | +- |
| M | internal/ | 4 | +- |
@@ -36,6 +36,7 @@ lint:
3636 3737 fuzz:3838 go test ./internal/sheet -run '^$$' -fuzz FuzzParse -fuzztime 60s39+ go test ./internal/sheet -run '^$$' -fuzz FuzzRead -fuzztime 60s3940 go test ./internal/fileio -run '^$$' -fuzz FuzzReadDelimited -fuzztime 60s4041 go test ./internal/fileio -run '^$$' -fuzz FuzzReadWK1 -fuzztime 60s4142 go test ./internal/fileio -run '^$$' -fuzz 'FuzzReadXLSX$$' -fuzztime 60s@@ -3,6 +3,7 @@ package sheet
33 import (44 "encoding/json"55 "fmt"6+ "strconv"67 )78 89 // fileCell is a cell with formatting or a note as the file writes it; a@@ -90,3 +91,59 @@ func decodeFormatted(fc fileCell) (string, Format, Style, error) {
9091 st := Style{Bold: fc.Bold, Italic: fc.Italic, Underline: fc.Underline, Strikethrough: fc.Strikethrough, Align: al, own: fc.Own}9192 return fc.Input, f, st, nil9293 }94+95+// appendSaved appends the cell at a, which holds one, as the file96+// writes it, or reports false for a cell the file leaves out (a pivot's97+// or a spill's result). A plain cell without formatting is written from98+// its slot, without making a Cell.99+func (st *cellStore) appendSaved(buf []byte, a Addr) ([]byte, bool, error) {100+ b, i := st.find(a)101+ if sl := b.vals[i]; sl.kind != slotRich && sl.look == 0 {102+ return st.appendInput(buf, sl), true, nil103+ }104+ c := st.get(a).saved()105+ if c == nil {106+ return buf, false, nil107+ }108+ raw, err := encodeCell(c)109+ return append(buf, raw...), true, err110+}111+112+// appendInput appends a plain slot's entry as a JSON string.113+func (st *cellStore) appendInput(buf []byte, sl slot) []byte {114+ switch {115+ case sl.kind == slotBlank:116+ return append(buf, '"', '"')117+ case sl.ref != 0:118+ return appendJSONString(buf, st.strs.strs[sl.ref])119+ case sl.kind == slotBool:120+ return append(append(append(buf, '"'), boolText(sl.num != 0)...), '"')121+ }122+ buf = strconv.AppendFloat(append(buf, '"'), sl.num, 'f', int(sl.dec)-1, 64)123+ return append(buf, '"')124+}125+126+// appendJSONString appends s as json.Marshal writes it: as it is between127+// quotes when it is printable ASCII that needs no escape (HTML's128+// characters included, which json.Marshal escapes), and through129+// json.Marshal otherwise.130+func appendJSONString(buf []byte, s string) []byte {131+ for i := 0; i < len(s); i++ {132+ if c := s[i]; c < 0x20 || c > 0x7e || c == '"' || c == '\\' || c == '<' || c == '>' || c == '&' {133+ raw, _ := json.Marshal(s)134+ return append(buf, raw...)135+ }136+ }137+ return append(append(append(buf, '"'), s...), '"')138+}139+140+// appendAddr appends a's A1 name, as a.String() writes it.141+func appendAddr(buf []byte, a Addr) []byte {142+ var col [4]byte143+ i := len(col)144+ for c := a.Col + 1; c > 0; c = (c - 1) / 26 {145+ i--146+ col[i] = byte('A' + (c-1)%26)147+ }148+ return strconv.AppendInt(append(buf, col[i:]...), int64(a.Row+1), 10)149+}@@ -1,7 +1,7 @@
11 package sheet22 33 import (4- "bytes"4+ "bufio"55 "encoding/json"66 "fmt"77 "io"@@ -77,20 +77,22 @@ func (s *Sheet) Write(w io.Writer) error { return s.wb.Write(w) }
7777 7878 // Write saves the workbook as JSON, storing each cell's input as typed7979 // and its formatting. Cells go one per line in row-major order so diffs80-// read naturally.80+// read naturally. It streams: cells are written as they are read from81+// the store, through a buffer of its own, so writing costs about nothing82+// on top of the workbook whatever its size.8183 func (w *Workbook) Write(out io.Writer) error {82- var b bytes.Buffer84+ b := bufio.NewWriterSize(out, 64<<10)8385 if w.single() {8486 s := w.sheets[0]8587 version := 28688 if v := s.view; len(w.names) > 0 || v.frozenRows > 0 || v.frozenCols > 0 || v.filter != nil {8789 version = 38890 }89- fmt.Fprintf(&b, "{\n \"version\": %d,\n", version)91+ fmt.Fprintf(b, "{\n \"version\": %d,\n", version)9092 if s.name != "Sheet1" {91- fmt.Fprintf(&b, " \"name\": %s,\n", jsonString(s.name))93+ fmt.Fprintf(b, " \"name\": %s,\n", jsonString(s.name))9294 }93- if err := s.writeBody(&b, " ", w.headLines()); err != nil {95+ if err := s.writeBody(b, " ", w.headLines()); err != nil {9496 return err9597 }9698 b.WriteString("\n}\n")@@ -99,31 +101,30 @@ func (w *Workbook) Write(out io.Writer) error {
99101 if w.hasPivots() {100102 version = fileVersion101103 }102- fmt.Fprintf(&b, "{\n \"version\": %d,\n", version)104+ fmt.Fprintf(b, "{\n \"version\": %d,\n", version)103105 if head := w.headLines(); head != "" {104106 b.WriteString(" " + head + ",\n")105107 }106108 if w.Active() > 0 {107- fmt.Fprintf(&b, " \"active\": %d,\n", w.Active())109+ fmt.Fprintf(b, " \"active\": %d,\n", w.Active())108110 }109111 b.WriteString(` "sheets": [`)110112 for i, s := range w.sheets {111113 if i > 0 {112114 b.WriteByte(',')113115 }114- fmt.Fprintf(&b, "\n {\n \"name\": %s,\n", jsonString(s.name))116+ fmt.Fprintf(b, "\n {\n \"name\": %s,\n", jsonString(s.name))115117 if s.tabHidden {116118 b.WriteString(" \"hidden\": true,\n")117119 }118- if err := s.writeBody(&b, " ", ""); err != nil {120+ if err := s.writeBody(b, " ", ""); err != nil {119121 return err120122 }121123 b.WriteString("\n }")122124 }123125 b.WriteString("\n ]\n}\n")124126 }125- _, err := out.Write(b.Bytes())126- return err127+ return b.Flush()127128 }128129 129130 // single reports whether the workbook fits the single-sheet format of@@ -177,7 +178,7 @@ func (w *Workbook) namesLine() string {
177178 // writeBody writes a sheet's fields, each line starting with indent, the178179 // names line (if any) after the widths as version 3 has it, and no179180 // newline after the last field.180-func (s *Sheet) writeBody(b *bytes.Buffer, indent, names string) error {181+func (s *Sheet) writeBody(b *bufio.Writer, indent, names string) error {181182 if len(s.widths) > 0 {182183 b.WriteString(indent + `"widths": {`)183184 for i, c := range slices.Sorted(maps.Keys(s.widths)) {@@ -197,25 +198,31 @@ func (s *Sheet) writeBody(b *bytes.Buffer, indent, names string) error {
197198 if err := s.writeView(b, indent); err != nil {198199 return err199200 }200- addrs := make([]Addr, 0, s.cells.len())201- for a := range s.cells.keys() {202- if c := s.cells.richAt(a); c == nil || c.saved() != nil {203- addrs = append(addrs, a)204- }205- }206- sortAddrs(addrs)207201 b.WriteString(indent + `"cells": {`)208- for i, a := range addrs {209- raw, err := encodeCell(s.cells.get(a).saved())202+ var line []byte203+ n := 0204+ for a := range s.cells.keysIn(Rect{To: Addr{Col: MaxCols - 1, Row: MaxRows - 1}}) {205+ line = append(append(line[:0], '\n'), indent...)206+ line = appendAddr(append(line, ' ', ' ', '"'), a)207+ line = append(line, '"', ':', ' ')208+ var ok bool209+ var err error210+ line, ok, err = s.cells.appendSaved(line, a)210211 if err != nil {211212 return err212213 }213- if i > 0 {214+ if !ok {215+ continue216+ }217+ if n > 0 {214218 b.WriteByte(',')215219 }216- fmt.Fprintf(b, "\n%s %q: %s", indent, a.String(), raw)220+ n++221+ if _, err := b.Write(line); err != nil {222+ return err223+ }217224 }218- if len(addrs) > 0 {225+ if n > 0 {219226 b.WriteString("\n" + indent)220227 }221228 b.WriteString("}")@@ -224,7 +231,7 @@ func (s *Sheet) writeBody(b *bytes.Buffer, indent, names string) error {
224231 225232 // writeObjects writes the fields after the cells: the charts, one per226233 // line, the pivot table's definition, and the rules.227-func (s *Sheet) writeObjects(b *bytes.Buffer, indent string) error {234+func (s *Sheet) writeObjects(b *bufio.Writer, indent string) error {228235 if len(s.charts) > 0 {229236 b.WriteString(",\n" + indent + `"charts": [`)230237 for i, c := range s.charts {@@ -268,56 +275,6 @@ func ReadTraced(r io.Reader, trace any) (*Sheet, error) {
268275 // version.269276 func ReadBook(r io.Reader) (*Workbook, error) { return readBook(r, nil) }270277 271-func readBook(r io.Reader, trace any) (*Workbook, error) {272- var f fileFormat273- if err := json.NewDecoder(r).Decode(&f); err != nil {274- return nil, err275- }276- if f.Version < 1 || f.Version > fileVersion {277- return nil, fmt.Errorf("unsupported file version %d", f.Version)278- }279- bodies := f.Sheets280- if f.Version < 4 {281- bodies = []fileSheet{f.fileSheet}282- if bodies[0].Name == "" {283- bodies[0].Name = "Sheet1"284- }285- }286- if len(bodies) == 0 {287- return nil, fmt.Errorf("the file has no sheets")288- }289- w := emptyBook()290- w.trace = trace291- for _, body := range bodies {292- if err := w.checkName(nil, body.Name); err != nil {293- return nil, fmt.Errorf("sheet %q: %w", body.Name, err)294- }295- w.insert(w.newSheet(body.Name), len(w.sheets))296- }297- if err := w.readNames(f.Names); err != nil {298- return nil, err299- }300- for i, body := range bodies {301- if err := w.sheets[i].read(body, f.Version); err != nil {302- if len(bodies) > 1 {303- err = fmt.Errorf("sheet %s: %w", body.Name, err)304- }305- return nil, err306- }307- }308- w.active = clampInt(f.Active, 0, len(w.sheets)-1)309- w.settleHidden()310- // Anything but "decimal" (say, a mode from a later build) computes in311- // binary, as the file would in a build without the setting.312- w.decimal = f.Arithmetic == "decimal"313- if err := w.readMacros(f.Macros); err != nil {314- return nil, err315- }316- w.macroOrigin = f.MacroOrigin317- w.RecalcAll()318- return w, nil319-}320-321278 // readNames defines the named ranges of a file. A range without a sheet322279 // is on the first sheet, as in single-sheet files.323280 func (w *Workbook) readNames(names map[string]string) error {@@ -347,8 +304,9 @@ func (w *Workbook) readNames(names map[string]string) error {
347304 return nil348305 }349306 350-// read fills an empty sheet from its part of a file.351-func (s *Sheet) read(f fileSheet, version int) error {307+// read fills a sheet, its cells read (fileread.go), from the rest of308+// its part of a file.309+func (s *Sheet) read(f fileSheet) error {352310 s.tabHidden = f.Hidden353311 for name, width := range f.Widths {354312 c, ok := ParseCol(name)@@ -360,25 +318,6 @@ func (s *Sheet) read(f fileSheet, version int) error {
360318 if err := s.readLines(f.Lines); err != nil {361319 return err362320 }363- for name, raw := range f.Cells {364- a, ok := ParseAddr(name)365- if !ok {366- return fmt.Errorf("invalid cell %q", name)367- }368- input, fm, st, note, err := decodeNoted(raw)369- if err != nil {370- return fmt.Errorf("%s: %w", name, err)371- }372- // Version 1 had no formats, so there entries imply them, as if373- // typed again; since version 2 the stored format wins.374- c, err := newCell(input, fm, st, version < 2)375- if err != nil {376- return fmt.Errorf("%s: %w", name, err)377- }378- if c = c.withNote(CleanNote(note)); c != nil {379- s.place(a, c)380- }381- }382321 for i, fc := range f.Charts {383322 c, err := decodeChart(fc)384323 if err != nil {@@ -0,0 +1,336 @@
1+package sheet2+3+import (4+ "bytes"5+ "encoding/json"6+ "errors"7+ "fmt"8+ "io"9+ "strings"10+)11+12+// The reader streams the file (see scanner): cells go straight into13+// their sheet as they are read, so opening a file allocates about what14+// the workbook holds rather than a tree of the whole document. The15+// other fields of the workbook and of each sheet are small; they are16+// kept as written and decoded with encoding/json once the stream ends.17+//18+// Keys match fields as encoding/json matches them, case aside, and a19+// field given twice takes its last value, cells adding up, except that20+// the version and the list of sheets, which say where cells go, are21+// refused twice once cells were read by them. Cells come22+// after "version" in every file Write makes; any read before it (a file23+// written by hand) are kept as written until the version is known.24+type fileReader struct {25+ sc *scanner26+ w *Workbook27+ version int28+ // hasVersion is set once "version" is read, and streamed once cells29+ // were read with it, which it may then no longer change.30+ hasVersion, streamed bool31+ listed bool // the list of sheets was read32+ single *Sheet // versions 1 to 3: the sheet of the top-level cells33+ pending []json.RawMessage // top-level cells read before the version, in order34+ sheets []readSheet // version 4: the sheets read35+ sheetsRaw json.RawMessage // "sheets" read before the version36+}37+38+// readSheet is a sheet of a version 4 file, its cells read, and its39+// other fields.40+type readSheet struct {41+ s *Sheet42+ body fileSheet43+}44+45+// field is a member of an object, its value as written.46+type field struct {47+ key string48+ raw json.RawMessage49+}50+51+func readBook(r io.Reader, trace any) (*Workbook, error) {52+ rd := &fileReader{sc: newScanner(r), w: emptyBook()}53+ f, err := rd.top()54+ if err != nil {55+ return nil, err56+ }57+ if f.Version < 1 || f.Version > fileVersion {58+ return nil, fmt.Errorf("unsupported file version %d", f.Version)59+ }60+ if err := rd.late(); err != nil {61+ return nil, err62+ }63+ w := rd.w64+ w.trace = trace65+ list := rd.sheets66+ if f.Version < 4 {67+ if rd.single == nil {68+ rd.single = w.newSheet("")69+ }70+ list = []readSheet{{rd.single, f.fileSheet}}71+ if list[0].body.Name == "" {72+ list[0].body.Name = "Sheet1"73+ }74+ }75+ if len(list) == 0 {76+ return nil, errors.New("the file has no sheets")77+ }78+ for _, rs := range list {79+ if err := w.checkName(nil, rs.body.Name); err != nil {80+ return nil, fmt.Errorf("sheet %q: %w", rs.body.Name, err)81+ }82+ rs.s.name = rs.body.Name83+ w.insert(rs.s, len(w.sheets))84+ }85+ if err := w.readNames(f.Names); err != nil {86+ return nil, err87+ }88+ for _, rs := range list {89+ if err := rs.s.read(rs.body); err != nil {90+ if len(list) > 1 {91+ err = fmt.Errorf("sheet %s: %w", rs.body.Name, err)92+ }93+ return nil, err94+ }95+ }96+ w.active = clampInt(f.Active, 0, len(w.sheets)-1)97+ w.settleHidden()98+ // Anything but "decimal" (say, a mode from a later build) computes in99+ // binary, as the file would in a build without the setting.100+ w.decimal = f.Arithmetic == "decimal"101+ if err := w.readMacros(f.Macros); err != nil {102+ return nil, err103+ }104+ w.macroOrigin = f.MacroOrigin105+ w.RecalcAll()106+ return w, nil107+}108+109+// top reads the file's object, streaming the cells, and returns its110+// other fields.111+func (rd *fileReader) top() (fileFormat, error) {112+ var f fileFormat113+ if null, err := rd.sc.null(); err != nil || null {114+ return f, err115+ }116+ var rest []field117+ err := rd.sc.members(func(key string) error {118+ switch {119+ case strings.EqualFold(key, "version"):120+ return rd.readVersion()121+ case strings.EqualFold(key, "cells"):122+ return rd.topCells()123+ case strings.EqualFold(key, "sheets"):124+ return rd.topSheets()125+ }126+ raw, err := rd.sc.raw()127+ rest = append(rest, field{key, raw})128+ return err129+ })130+ if err == nil {131+ err = unmarshalFields(rest, &f)132+ }133+ f.Version = rd.version134+ return f, err135+}136+137+func (rd *fileReader) readVersion() error {138+ raw, err := rd.sc.raw()139+ if err != nil {140+ return err141+ }142+ v := rd.version143+ if err := json.Unmarshal(raw, &v); err != nil {144+ return err145+ }146+ if rd.streamed && v != rd.version {147+ return errTwice148+ }149+ rd.version, rd.hasVersion = v, true150+ return nil151+}152+153+// topCells reads the top-level cells: the only sheet's, before version 4.154+func (rd *fileReader) topCells() error {155+ if !rd.hasVersion || len(rd.pending) > 0 {156+ raw, err := rd.sc.raw()157+ rd.pending = append(rd.pending, raw)158+ return err159+ }160+ if rd.version >= 4 {161+ raw, err := rd.sc.raw()162+ if err == nil {163+ err = checkCells(raw)164+ }165+ return err166+ }167+ return rd.cells(rd.singleSheet(), rd.sc)168+}169+170+func (rd *fileReader) singleSheet() *Sheet {171+ if rd.single == nil {172+ rd.single = rd.w.newSheet("")173+ }174+ return rd.single175+}176+177+// checkCells checks cells a file of this version ignores, as decoding178+// them into the file's fields would.179+func checkCells(raw json.RawMessage) error {180+ var m map[string]json.RawMessage181+ return json.Unmarshal(raw, &m)182+}183+184+// topSheets reads the list of sheets of version 4.185+func (rd *fileReader) topSheets() error {186+ if !rd.hasVersion || rd.version < 4 {187+ raw, err := rd.sc.raw()188+ if err != nil {189+ return err190+ }191+ if !rd.hasVersion {192+ if rd.sheetsRaw != nil {193+ return errTwice194+ }195+ rd.sheetsRaw = raw196+ return nil197+ }198+ var ignored []fileSheet199+ return json.Unmarshal(raw, &ignored)200+ }201+ return rd.readSheets(rd.sc)202+}203+204+// errTwice is returned for a version or a list of sheets given twice205+// once cells were read by the first.206+var errTwice = errors.New("the file gives its version or its sheets twice")207+208+// readSheets reads a list of sheets from sc.209+func (rd *fileReader) readSheets(sc *scanner) error {210+ if rd.listed {211+ return errTwice212+ }213+ rd.listed, rd.sheetsRaw = true, nil214+ if null, err := sc.null(); err != nil || null {215+ return err216+ }217+ return sc.elements(func() error {218+ s := rd.w.newSheet("")219+ var rest []field220+ if null, err := sc.null(); err != nil || null {221+ rd.sheets = append(rd.sheets, readSheet{s: s})222+ return err223+ }224+ name := ""225+ err := sc.members(func(key string) error {226+ if strings.EqualFold(key, "cells") {227+ err := rd.cells(s, sc)228+ if err != nil && name != "" {229+ err = fmt.Errorf("sheet %s: %w", name, err)230+ }231+ return err232+ }233+ raw, err := sc.raw()234+ if strings.EqualFold(key, "name") {235+ json.Unmarshal(raw, &name) // for errors only: the body decodes it236+ }237+ rest = append(rest, field{key, raw})238+ return err239+ })240+ var body fileSheet241+ if err == nil {242+ err = unmarshalFields(rest, &body)243+ }244+ rd.sheets = append(rd.sheets, readSheet{s, body})245+ return err246+ })247+}248+249+// late reads what came before the version, now that it is known.250+func (rd *fileReader) late() error {251+ for _, raw := range rd.pending {252+ if rd.version >= 4 {253+ if err := checkCells(raw); err != nil {254+ return err255+ }256+ continue257+ }258+ if err := rd.cells(rd.singleSheet(), newScanner(bytes.NewReader(raw))); err != nil {259+ return err260+ }261+ }262+ if rd.sheetsRaw == nil {263+ return nil264+ }265+ if rd.version < 4 {266+ var ignored []fileSheet267+ return json.Unmarshal(rd.sheetsRaw, &ignored)268+ }269+ return rd.readSheets(newScanner(bytes.NewReader(rd.sheetsRaw)))270+}271+272+// cells reads a sheet's cells from sc into s. Version 1 had no formats,273+// so there entries imply them, as if typed again; since version 2 the274+// stored format wins.275+func (rd *fileReader) cells(s *Sheet, sc *scanner) error {276+ rd.streamed = true277+ if null, err := sc.null(); err != nil || null {278+ return err279+ }280+ implied := rd.version < 2281+ return sc.members(func(name string) error {282+ input, fm, st, note, err := cellValue(sc)283+ if err != nil {284+ return fmt.Errorf("%s: %w", name, err)285+ }286+ a, ok := ParseAddr(name)287+ if !ok {288+ return fmt.Errorf("invalid cell %q", name)289+ }290+ c, err := newCell(input, fm, st, implied)291+ if err != nil {292+ return fmt.Errorf("%s: %w", name, err)293+ }294+ if c = c.withNote(CleanNote(note)); c != nil {295+ s.place(a, c)296+ }297+ return nil298+ })299+}300+301+// cellValue reads a cell's entry, formatting and note: a string for an302+// entry alone, an object otherwise (see fileFormat).303+func cellValue(sc *scanner) (string, Format, Style, string, error) {304+ c, err := sc.peek()305+ if err != nil {306+ return "", Format{}, Style{}, "", err307+ }308+ if c == '"' {309+ input, err := sc.str()310+ return input, Format{}, Style{}, "", err311+ }312+ raw, err := sc.raw()313+ if err != nil {314+ return "", Format{}, Style{}, "", err315+ }316+ return decodeNoted(raw)317+}318+319+// unmarshalFields decodes an object's fields, as written, into v.320+func unmarshalFields(fields []field, v any) error {321+ if len(fields) == 0 {322+ return nil323+ }324+ var b bytes.Buffer325+ b.WriteByte('{')326+ for i, f := range fields {327+ if i > 0 {328+ b.WriteByte(',')329+ }330+ b.WriteString(jsonString(f.key))331+ b.WriteByte(':')332+ b.Write(f.raw)333+ }334+ b.WriteByte('}')335+ return json.Unmarshal(b.Bytes(), v)336+}@@ -0,0 +1,265 @@
1+package sheet2+3+import (4+ "bufio"5+ "encoding/json"6+ "errors"7+ "fmt"8+ "io"9+ "unicode/utf8"10+)11+12+// scanner reads JSON a value at a time from a stream, for the .01213+// reader (fileread.go): the cells are read one by one straight into the14+// sheet, and everything else, which is small, is captured whole and15+// given to encoding/json, so it decodes as it always has. Strings16+// without escapes are read without encoding/json; the rest go through17+// it, so text decodes exactly as it would there.18+type scanner struct {19+ r *bufio.Reader20+ buf []byte // scratch for a string longer than the reader's buffer or escaped21+}22+23+func newScanner(r io.Reader) *scanner { return &scanner{r: bufio.NewReaderSize(r, 64<<10)} }24+25+// errSyntax is returned for anything that isn't the JSON expected.26+var errSyntax = errors.New("invalid JSON")27+28+// syntax describes a byte that wasn't expected.29+func syntax(c byte, want string) error {30+ return fmt.Errorf("%w: %q where %s was expected", errSyntax, c, want)31+}32+33+// eofErr is err, turning a clean EOF in the middle of a value into34+// io.ErrUnexpectedEOF, as encoding/json does.35+func eofErr(err error) error {36+ if err == io.EOF {37+ return io.ErrUnexpectedEOF38+ }39+ return err40+}41+42+// peek skips white space and returns the next byte without reading it.43+func (sc *scanner) peek() (byte, error) {44+ for {45+ c, err := sc.r.ReadByte()46+ if err != nil {47+ return 0, eofErr(err)48+ }49+ switch c {50+ case ' ', '\t', '\n', '\r':51+ continue52+ }53+ return c, sc.r.UnreadByte()54+ }55+}56+57+// next skips white space and reads the next byte.58+func (sc *scanner) next() (byte, error) {59+ if _, err := sc.peek(); err != nil {60+ return 0, err61+ }62+ return sc.r.ReadByte()63+}64+65+// expect reads the next byte, which must be c.66+func (sc *scanner) expect(c byte) error {67+ got, err := sc.next()68+ if err == nil && got != c {69+ err = syntax(got, fmt.Sprintf("%q", c))70+ }71+ return err72+}73+74+// null reads the literal null if it comes next, and reports whether it75+// did.76+func (sc *scanner) null() (bool, error) {77+ c, err := sc.peek()78+ if err != nil || c != 'n' {79+ return false, err80+ }81+ raw, err := sc.raw()82+ if err == nil && string(raw) != "null" {83+ err = fmt.Errorf("%w: %q", errSyntax, raw)84+ }85+ return err == nil, err86+}87+88+// members reads an object, calling fn with each key once the reader is89+// at its value, which fn must read.90+func (sc *scanner) members(fn func(key string) error) error {91+ if err := sc.expect('{'); err != nil {92+ return err93+ }94+ return sc.list('}', func() error {95+ key, err := sc.str()96+ if err != nil {97+ return err98+ }99+ if err := sc.expect(':'); err != nil {100+ return err101+ }102+ return fn(key)103+ })104+}105+106+// elements reads an array, calling fn for each element, which fn must107+// read.108+func (sc *scanner) elements(fn func() error) error {109+ if err := sc.expect('['); err != nil {110+ return err111+ }112+ return sc.list(']', fn)113+}114+115+// list reads the items of an object or array up to its closing byte,116+// with fn reading each.117+func (sc *scanner) list(end byte, fn func() error) error {118+ c, err := sc.peek()119+ if err != nil {120+ return err121+ }122+ if c == end {123+ _, err = sc.r.ReadByte()124+ return err125+ }126+ for {127+ if err := fn(); err != nil {128+ return err129+ }130+ c, err := sc.next()131+ switch {132+ case err != nil:133+ return err134+ case c == end:135+ return nil136+ case c != ',':137+ return syntax(c, fmt.Sprintf(`"," or %q`, end))138+ }139+ }140+}141+142+// str reads a string.143+func (sc *scanner) str() (string, error) {144+ body, plain, err := sc.text()145+ if err != nil || plain {146+ return string(body), err147+ }148+ var s string149+ err = json.Unmarshal(quote(body), &s)150+ return s, err151+}152+153+// quote is body between quotes, in a new slice.154+func quote(body []byte) []byte {155+ out := make([]byte, 0, len(body)+2)156+ return append(append(append(out, '"'), body...), '"')157+}158+159+// text reads a string as it is written between its quotes, and reports160+// whether it is plain: no escapes, control characters or invalid UTF-8,161+// so those bytes are its text. They are valid until the next read.162+func (sc *scanner) text() ([]byte, bool, error) {163+ if err := sc.expect('"'); err != nil {164+ return nil, false, err165+ }166+ part, err := sc.r.ReadSlice('"')167+ if err == nil && !escapedQuote(part) { // the common case: no copy168+ body := part[:len(part)-1]169+ return body, plainText(body), nil170+ }171+ sc.buf = append(sc.buf[:0], part...)172+ for err != nil || escapedQuote(sc.buf) {173+ if err != nil && err != bufio.ErrBufferFull {174+ return nil, false, eofErr(err)175+ }176+ part, err = sc.r.ReadSlice('"')177+ sc.buf = append(sc.buf, part...)178+ }179+ body := sc.buf[:len(sc.buf)-1]180+ return body, plainText(body), nil181+}182+183+// plainText reports whether a string's body as written is its text.184+func plainText(body []byte) bool {185+ for _, c := range body {186+ if c < 0x20 || c == '\\' {187+ return false188+ }189+ }190+ return utf8.Valid(body)191+}192+193+// escapedQuote reports whether the quote ending s is escaped: preceded194+// by an odd number of backslashes.195+func escapedQuote(s []byte) bool {196+ n := 0197+ for i := len(s) - 2; i >= 0 && s[i] == '\\'; i-- {198+ n++199+ }200+ return n%2 == 1201+}202+203+// raw reads the next value whole, as it is written. It checks only204+// that brackets balance outside strings; encoding/json checks the rest205+// when the value is decoded.206+func (sc *scanner) raw() (json.RawMessage, error) {207+ c, err := sc.peek()208+ if err != nil {209+ return nil, err210+ }211+ switch c {212+ case '"':213+ body, _, err := sc.text()214+ return quote(body), err215+ case '{', '[':216+ return sc.nested()217+ }218+ var out []byte219+ for {220+ c, err := sc.r.ReadByte()221+ if err == io.EOF && len(out) > 0 {222+ return out, nil223+ }224+ if err != nil {225+ return nil, eofErr(err)226+ }227+ switch c {228+ case ',', '}', ']', ' ', '\t', '\n', '\r':229+ return out, sc.r.UnreadByte()230+ }231+ out = append(out, c)232+ }233+}234+235+// nested reads an object or array whole.236+func (sc *scanner) nested() (json.RawMessage, error) {237+ var out []byte238+ depth := 0239+ for {240+ c, err := sc.r.ReadByte()241+ if err != nil {242+ return nil, eofErr(err)243+ }244+ switch c {245+ case '"':246+ if err := sc.r.UnreadByte(); err != nil {247+ return nil, err248+ }249+ body, _, err := sc.text()250+ if err != nil {251+ return nil, err252+ }253+ out = append(append(append(out, '"'), body...), '"')254+ continue255+ case '{', '[':256+ depth++257+ case '}', ']':258+ depth--259+ }260+ out = append(out, c)261+ if depth == 0 {262+ return out, nil263+ }264+ }265+}@@ -0,0 +1,208 @@
1+package sheet2+3+import (4+ "bytes"5+ "encoding/json"6+ "errors"7+ "fmt"8+ "strings"9+ "testing"10+)11+12+// readWhole reads a file the way encoding/json reads it whole, into13+// fileFormat, as the reference the streaming reader is held to. It14+// reports ambiguous for a file whose result depends on map order: a15+// sheet naming one cell by two keys ("A1" and "a1").16+func readWhole(data []byte) (w *Workbook, ambiguous bool, err error) {17+ var f fileFormat18+ if err := json.NewDecoder(bytes.NewReader(data)).Decode(&f); err != nil {19+ return nil, false, err20+ }21+ if f.Version < 1 || f.Version > fileVersion {22+ return nil, false, fmt.Errorf("unsupported file version %d", f.Version)23+ }24+ bodies := f.Sheets25+ if f.Version < 4 {26+ bodies = []fileSheet{f.fileSheet}27+ if bodies[0].Name == "" {28+ bodies[0].Name = "Sheet1"29+ }30+ }31+ if len(bodies) == 0 {32+ return nil, false, errors.New("the file has no sheets")33+ }34+ w = emptyBook()35+ for _, body := range bodies {36+ if err := w.checkName(nil, body.Name); err != nil {37+ return nil, false, err38+ }39+ w.insert(w.newSheet(body.Name), len(w.sheets))40+ }41+ if err := w.readNames(f.Names); err != nil {42+ return nil, false, err43+ }44+ for i, body := range bodies {45+ s := w.sheets[i]46+ amb, err := placeWhole(s, body.Cells, f.Version)47+ if err != nil {48+ return nil, false, err49+ }50+ ambiguous = ambiguous || amb51+ if err := s.read(body); err != nil {52+ return nil, false, err53+ }54+ }55+ w.active = clampInt(f.Active, 0, len(w.sheets)-1)56+ w.settleHidden()57+ w.decimal = f.Arithmetic == "decimal"58+ if err := w.readMacros(f.Macros); err != nil {59+ return nil, false, err60+ }61+ w.macroOrigin = f.MacroOrigin62+ w.RecalcAll()63+ return w, ambiguous, nil64+}65+66+// placeWhole places a sheet's cells decoded whole, and reports whether67+// two keys name one cell.68+func placeWhole(s *Sheet, cells map[string]json.RawMessage, version int) (ambiguous bool, err error) {69+ seen := map[Addr]bool{}70+ for name, raw := range cells {71+ a, ok := ParseAddr(name)72+ if !ok {73+ return false, fmt.Errorf("invalid cell %q", name)74+ }75+ ambiguous = ambiguous || seen[a]76+ seen[a] = true77+ input, fm, st, note, err := decodeNoted(raw)78+ if err != nil {79+ return false, err80+ }81+ c, err := newCell(input, fm, st, version < 2)82+ if err != nil {83+ return false, err84+ }85+ if c = c.withNote(CleanNote(note)); c != nil {86+ s.place(a, c)87+ }88+ }89+ return ambiguous, nil90+}91+92+// written is the file w writes.93+func written(t testing.TB, w *Workbook) string {94+ var b bytes.Buffer95+ if err := w.Write(&b); err != nil {96+ t.Fatal(err)97+ }98+ return b.String()99+}100+101+// sameAsWhole checks that the streaming reader reads data as readWhole102+// does: both fail, or both make workbooks that write the same file.103+func sameAsWhole(t *testing.T, data []byte) {104+ want, ambiguous, werr := readWhole(data)105+ got, gerr := ReadBook(bytes.NewReader(data))106+ switch {107+ case errors.Is(gerr, errTwice):108+ // Refused, where decoding the whole file took the last (and for109+ // sheets, merged the two lists).110+ case (werr == nil) != (gerr == nil):111+ t.Fatalf("reading %q: streaming %v, whole %v", data, gerr, werr)112+ case werr == nil && !ambiguous:113+ if g, w := written(t, got), written(t, want); g != w {114+ t.Fatalf("reading %q: streaming writes\n%s\nwhole writes\n%s", data, g, w)115+ }116+ }117+}118+119+// streamSeeds are files to read: what Write makes of sheets holding the120+// parser's fuzz seeds and every kind of field, and files written by hand121+// with keys out of order, in other cases, escaped and given twice.122+func streamSeeds(t testing.TB) [][]byte {123+ s := New()124+ inputs := []string{"=A1*2", "=SUM(A1:B3)", "=IF(AND(A1>2,B1),1,NA())", `="a"&"b"`, "=1.5E-3^2", "$1,200", "12%", "@SUM(A1..B3)", "=$A$1+A$2*$B3", "=SUM($A1:B$3)", "=#REF!+1", "=A$$1",125+ "Rent", "TRUE", "1.50", "'007", "<b>&</b>", "tab\tquote\"back\\slash", "café
", strings.Repeat("long ", 20000)}126+ for i, in := range inputs {127+ s.Set(Addr{Col: i % 4, Row: i / 4}, in)128+ }129+ s.SetFormat(NewRect(Addr{Row: 10}, Addr{Col: 2, Row: 12}), Format{Kind: FmtCurrency, Decimals: 2})130+ s.SetStyle(NewRect(Addr{Row: 11}, Addr{Col: 3, Row: 11}), func(st *Style) { st.Bold = true })131+ s.SetNote(Addr{Col: 5, Row: 1}, "a note")132+ s.SetColWidth(1, 20)133+ one := []byte(written(t, s.Book()))134+ if _, err := s.Book().AddSheet("Other", 1); err != nil {135+ t.Fatal(err)136+ }137+ s.Book().Sheet(1).Set(Addr{}, "=SUM(Sheet1!B1:B5)")138+ two := []byte(written(t, s.Book()))139+ return [][]byte{one, two,140+ []byte(`{"version": 1, "widths": {"A": 14}, "cells": {"A1": "Rent", "B1": "$1,450", "B2": "=B1*12"}}`),141+ []byte(`{"cells": {"A1": "5", "B1": "=A1+1"}, "version": 2}`),142+ []byte(`{"Cells": {"A1": "5"}, "VERSION": 3, "cells": {"A1": "6", "A2": {"input": "7", "bold": true}}}`),143+ []byte(`{"sheets": [{"cells": {"A1": "1"}, "name": "S"}], "version": 4}`),144+ []byte(`{"version": 4, "sheets": [{"name": "A", "cells": {"A1": "1"}}, null], "sheets": [{"name": "B", "cells": {"B2": "A\n"}}]}`),145+ []byte(`{"version": 2, "cells": {"A1": "x", "a1": "y"}}`),146+ []byte(`{"version": 2, "cells": null, "sheets": 5}`),147+ []byte(`{"version": 4, "cells": "x", "sheets": []}`),148+ []byte(`{"version": 2, "cells": {"A1": "😀", "A2": "bad \xff", "ZZ9": 5}}`),149+ []byte(`{"version": 2, "cells": {"A1": "1"}} trailing`),150+ []byte(`null`), []byte(`[]`), []byte(`{"version": 2, "cells": {"A1": "1",}}`), []byte(`{"version": 2, "cells": {"A1"`),151+ }152+}153+154+// The streaming reader reads every seed as encoding/json reading the155+// whole file did.156+func TestStreamReadsAsWhole(t *testing.T) {157+ for _, data := range streamSeeds(t) {158+ sameAsWhole(t, data)159+ }160+}161+162+func FuzzRead(f *testing.F) {163+ for _, data := range streamSeeds(f) {164+ f.Add(data)165+ }166+ f.Fuzz(func(t *testing.T, data []byte) {167+ if len(data) > 1<<16 {168+ return169+ }170+ sameAsWhole(t, data)171+ })172+}173+174+// Cells are written as encodeCell writes each, in row-major order, one175+// per line.176+func TestWriteCellsAsEncoded(t *testing.T) {177+ data := streamSeeds(t)[0]178+ s, err := Read(bytes.NewReader(data))179+ if err != nil {180+ t.Fatal(err)181+ }182+ var addrs []Addr183+ for a := range s.cells.keys() {184+ addrs = append(addrs, a)185+ }186+ sortAddrs(addrs)187+ var want strings.Builder188+ want.WriteString(`"cells": {`)189+ for i, a := range addrs {190+ raw, err := encodeCell(s.cells.get(a).saved())191+ if err != nil {192+ t.Fatal(err)193+ }194+ if i > 0 {195+ want.WriteByte(',')196+ }197+ fmt.Fprintf(&want, "\n %q: %s", a.String(), raw)198+ }199+ want.WriteString("\n }")200+ if got := written(t, s.Book()); !strings.Contains(got, want.String()) || string(data) != got {201+ t.Errorf("cells written as\n%s\nwant\n%s", got, want.String())202+ }203+ for _, a := range []Addr{{}, {Col: 25, Row: 9}, {Col: 26}, {Col: 701, Row: 99}, {Col: 702}, {Col: MaxCols - 1, Row: MaxRows - 1}} {204+ if got := string(appendAddr(nil, a)); got != a.String() {205+ t.Errorf("appendAddr %q, want %q", got, a.String())206+ }207+ }208+}@@ -1,7 +1,7 @@
11 package sheet22 33 import (4- "bytes"4+ "bufio"55 "encoding/json"66 "fmt"77 "maps"@@ -23,7 +23,7 @@ import (
2323 // needs no new version.2424 2525 // writeLines writes the "lines" field, if the sheet has line formats.26-func (s *Sheet) writeLines(b *bytes.Buffer, indent string) error {26+func (s *Sheet) writeLines(b *bufio.Writer, indent string) error {2727 if s.lines.none() {2828 return nil2929 }@@ -1,7 +1,7 @@
11 package sheet22 33 import (4- "bytes"4+ "bufio"55 "encoding/json"66 "fmt"77 "strings"@@ -172,7 +172,7 @@ func (fv fileValidation) rule() (Validation, error) {
172172 173173 // writeRules writes the sheet's rules, one per line, after its other174174 // fields.175-func (s *Sheet) writeRules(b *bytes.Buffer, indent string) {175+func (s *Sheet) writeRules(b *bufio.Writer, indent string) {176176 list := func(key string, lines []string) {177177 if len(lines) == 0 {178178 return@@ -1,7 +1,7 @@
11 package sheet22 33 import (4- "bytes"4+ "bufio"55 "encoding/json"66 "fmt"77 "reflect"@@ -153,7 +153,7 @@ type fileCriteria struct {
153153 154154 // writeView adds the frozen panes, the filter and the protected ranges to155155 // a file being written.156-func (s *Sheet) writeView(b *bytes.Buffer, indent string) error {156+func (s *Sheet) writeView(b *bufio.Writer, indent string) error {157157 var keys []string158158 var parts []any159159 if v := s.view; v.frozenRows > 0 || v.frozenCols > 0 {