The reader stores a number typed plainly ("12.5", "-3", "0.50") as its
slot without making a Cell or a string: its key is parsed from the
reader's bytes, and plainForm tells how it prints without formatting it.
numDec takes the same shortcut for every entry, and parseNumberFormat
scans an entry once rather than three times.
A later empty entry for a cell given twice leaves no cell, as decoding
the whole file did (found by FuzzRead; its input is kept as a seed).
dense-1Mx10 open: 2.6 s -> 1.15 s, peak heap 545 -> 326 MB, 3.3 GB ->
0.5 GB allocated. dense-8192x256: 0.54 -> 0.24 s.- commit
- c1c3881977a881f32f9c2ab156747db68b2c3597
- parent
- 4ebe5b5a607788fba2c4b509015c844ede60ec4d
- author
- Dami <dami.etoile@gmail.com>
- kind
- refactor risk score 2.1 of 3, Jev's estimate
| M | internal/ | 113 | ++++++++++++++++++++++++------ |
| M | internal/ | 33 | +++++++-- |
| M | internal/ | 61 | ++++++++++++++++- |
| M | internal/ | 57 | +++++++++++++++ |
| A | internal/ | 62 | ++++++++++++++++ |
| M | internal/ | 15 | ++++ |
| A | internal/ | 2 | + |
| M | internal/ | 24 | ++++++- |
@@ -6,6 +6,7 @@ import (
66 "errors"77 "fmt"88 "io"9+ "strconv"910 "strings"1011 1112 "github.com/FelineStateMachine/012/internal/locale"@@ -284,42 +285,112 @@ func (rd *fileReader) cells(s *Sheet, sc *scanner) error {
284285 return err285286 }286287 implied := rd.version < 2287- return sc.members(func(name string) error {288- input, fm, st, note, err := cellValue(sc)288+ return sc.fields(func(key []byte, plain bool) error {289+ a, name, err := cellKey(key, plain)289290 if err != nil {290- return fmt.Errorf("%s: %w", name, err)291- }292- a, ok := ParseAddr(name)293- if !ok {294- return fmt.Errorf("invalid cell %q", name)291+ return err295292 }296- c, err := newCell(input, fm, st, implied)297- if err != nil {293+ if err := s.readCell(sc, a, implied); err != nil {294+ if name == "" {295+ name = a.String()296+ }298297 return fmt.Errorf("%s: %w", name, err)299298 }300- if c = c.withNote(CleanNote(note)); c != nil {301- s.place(a, c)302- }303299 return nil304300 })305301 }306302 307-// cellValue reads a cell's entry, formatting and note: a string for an308-// entry alone, an object otherwise (see fileFormat).309-func cellValue(sc *scanner) (string, Format, Style, string, error) {303+// cellKey is the cell a key of "cells" names, and its name when it isn't304+// a's (for errors). A plain key is parsed from its bytes, which are305+// valid only until the next read, without making a string of it.306+func cellKey(key []byte, plain bool) (Addr, string, error) {307+ if plain {308+ a, ok := ParseAddr(string(key))309+ if !ok {310+ return a, "", fmt.Errorf("invalid cell %q", key)311+ }312+ return a, "", nil313+ }314+ var name string315+ if err := json.Unmarshal(quote(key), &name); err != nil {316+ return Addr{}, "", err317+ }318+ a, ok := ParseAddr(name)319+ if !ok {320+ return a, "", fmt.Errorf("invalid cell %q", name)321+ }322+ return a, name, nil323+}324+325+// readCell reads the cell at a from sc: a string for an entry alone, an326+// object for one with formatting or a note (see fileFormat).327+func (s *Sheet) readCell(sc *scanner, a Addr, implied bool) error {310328 c, err := sc.peek()311329 if err != nil {312- return "", Format{}, Style{}, "", err330+ return err313331 }332+ var input, note string333+ var fm Format334+ var st Style314335 if c == '"' {315- input, err := sc.str()316- return input, Format{}, Style{}, "", err336+ body, plain, err := sc.text()337+ if err != nil {338+ return err339+ }340+ if plain && s.loadNum(a, body, Format{}, Style{}) {341+ return nil342+ }343+ if input, err = unquote(body, plain); err != nil {344+ return err345+ }346+ } else {347+ raw, err := sc.raw()348+ if err != nil {349+ return err350+ }351+ if input, fm, st, note, err = decodeNoted(raw); err != nil {352+ return err353+ }354+ if note == "" && s.loadNum(a, []byte(input), fm, st) {355+ return nil356+ }317357 }318- raw, err := sc.raw()358+ cell, err := newCell(input, fm, st, implied)319359 if err != nil {320- return "", Format{}, Style{}, "", err360+ return err361+ }362+ // An empty entry leaves no cell, even where the key came before.363+ if cell = cell.withNote(CleanNote(note)); cell != nil || s.cells.has(a) {364+ s.place(a, cell)365+ }366+ return nil367+}368+369+// loadNum stores a number typed plainly ("12.5", "-3", "0.50", the most370+// common entry by far) at a, from a file, straight into its slot: what371+// newCell and place would store, without making a Cell or a string. It372+// reports false for any other entry, or a cell a holds with contents,373+// which go the general way. A sheet being read has no undo step, pivot374+// or spill yet for place to see to.375+func (s *Sheet) loadNum(a Addr, input []byte, f Format, st Style) bool {376+ if f.Kind == FmtText || s.cells.richAt(a) != nil {377+ return false378+ }379+ d, ok := plainForm(string(input))380+ if !ok {381+ return false382+ }383+ v, err := strconv.ParseFloat(string(input), 64)384+ if err != nil {385+ return false386+ }387+ lk, ok := s.cells.lookID(f, st)388+ if !ok {389+ return false321390 }322- return decodeNoted(raw)391+ s.version++392+ s.cells.setSlot(a, slot{kind: slotNum, num: v, dec: d, look: lk})393+ return true323394 }324395 325396 // unmarshalFields decodes an object's fields, as written, into v.@@ -18,6 +18,7 @@ import (
1818 type scanner struct {1919 r *bufio.Reader2020 buf []byte // scratch for a string longer than the reader's buffer or escaped21+ key []byte // the key being read2122 }2223 2324 func newScanner(r io.Reader) *scanner { return &scanner{r: bufio.NewReaderSize(r, 64<<10)} }@@ -88,18 +89,34 @@ func (sc *scanner) null() (bool, error) {
8889 // members reads an object, calling fn with each key once the reader is8990 // at its value, which fn must read.9091 func (sc *scanner) members(fn func(key string) error) error {92+ return sc.fields(func(key []byte, plain bool) error {93+ s := string(key)94+ if !plain {95+ if err := json.Unmarshal(quote(key), &s); err != nil {96+ return err97+ }98+ }99+ return fn(s)100+ })101+}102+103+// fields is members giving each key as written between its quotes, and104+// whether that is its text (see text); the bytes are valid until the105+// value is read.106+func (sc *scanner) fields(fn func(key []byte, plain bool) error) error {91107 if err := sc.expect('{'); err != nil {92108 return err93109 }94110 return sc.list('}', func() error {95- key, err := sc.str()111+ text, plain, err := sc.text()96112 if err != nil {97113 return err98114 }115+ sc.key = append(sc.key[:0], text...) // reading on may move text99116 if err := sc.expect(':'); err != nil {100117 return err101118 }102- return fn(key)119+ return fn(sc.key, plain)103120 })104121 }105122 @@ -139,14 +156,14 @@ func (sc *scanner) list(end byte, fn func() error) error {
139156 }140157 }141158 142-// str reads a string.143-func (sc *scanner) str() (string, error) {144- body, plain, err := sc.text()145- if err != nil || plain {146- return string(body), err159+// unquote is the text of a string written as body between its quotes,160+// plain or not (see text).161+func unquote(body []byte, plain bool) (string, error) {162+ if plain {163+ return string(body), nil147164 }148165 var s string149- err = json.Unmarshal(quote(body), &s)166+ err := json.Unmarshal(quote(body), &s)150167 return s, err151168 }152169 @@ -112,6 +112,9 @@ func sameAsWhole(t *testing.T, data []byte) {
112112 case errors.Is(gerr, errTwice):113113 // Refused, where decoding the whole file took the last (and for114114 // sheets, merged the two lists).115+ case gerr != nil && werr == nil && duplicateKeys(data):116+ // Streaming reads every entry of a cell given twice, and refuses117+ // an invalid one that decoding the whole file overwrote.115118 case (werr == nil) != (gerr == nil):116119 t.Fatalf("reading %q: streaming %v, whole %v", data, gerr, werr)117120 case werr == nil && !ambiguous:@@ -121,6 +124,55 @@ func sameAsWhole(t *testing.T, data []byte) {
121124 }122125 }123126 127+// duplicateKeys reports whether an object in data gives a key twice.128+func duplicateKeys(data []byte) bool {129+ dec := json.NewDecoder(bytes.NewReader(data))130+ type level struct {131+ keys map[string]bool132+ isKey bool // the next token in this object is a key133+ }134+ var stack []*level135+ for {136+ tok, err := dec.Token()137+ if err != nil {138+ return false139+ }140+ top := (*level)(nil)141+ if len(stack) > 0 {142+ top = stack[len(stack)-1]143+ }144+ switch tok {145+ case json.Delim('{'), json.Delim('['):146+ if top != nil && top.keys != nil {147+ top.isKey = true // the object or array is the value148+ }149+ l := &level{}150+ if tok == json.Delim('{') {151+ l.keys, l.isKey = map[string]bool{}, true152+ }153+ stack = append(stack, l)154+ continue155+ case json.Delim('}'), json.Delim(']'):156+ stack = stack[:len(stack)-1]157+ if len(stack) == 0 {158+ return false159+ }160+ continue161+ }162+ if top == nil || top.keys == nil {163+ continue164+ }165+ if top.isKey {166+ k := tok.(string)167+ if top.keys[k] {168+ return true169+ }170+ top.keys[k] = true171+ }172+ top.isKey = !top.isKey173+ }174+}175+124176 // streamSeeds are files to read: what Write makes of sheets holding the125177 // parser's fuzz seeds and every kind of field, and files written by hand126178 // with keys out of order, in other cases, escaped and given twice.@@ -141,13 +193,20 @@ func streamSeeds(t testing.TB) [][]byte {
141193 }142194 s.Book().Sheet(1).Set(Addr{}, "=SUM(Sheet1!B1:B5)")143195 two := []byte(written(t, s.Book()))144- return [][]byte{one, two,196+ // Many cells, so keys and values straddle the reader's buffer.197+ big := New()198+ for r := range 20000 {199+ big.Load(Addr{Row: r}, fmt.Sprintf("%d.%02d", r, r%100), Format{}, Style{})200+ big.Load(Addr{Col: 1, Row: r}, fmt.Sprintf("text %d", r%300), Format{}, Style{Bold: r%7 == 0})201+ }202+ return [][]byte{one, two, []byte(written(t, big.Book())),145203 []byte(`{"version": 1, "widths": {"A": 14}, "cells": {"A1": "Rent", "B1": "$1,450", "B2": "=B1*12"}}`),146204 []byte(`{"cells": {"A1": "5", "B1": "=A1+1"}, "version": 2}`),147205 []byte(`{"Cells": {"A1": "5"}, "VERSION": 3, "cells": {"A1": "6", "A2": {"input": "7", "bold": true}}}`),148206 []byte(`{"sheets": [{"cells": {"A1": "1"}, "name": "S"}], "version": 4}`),149207 []byte(`{"version": 4, "sheets": [{"name": "A", "cells": {"A1": "1"}}, null], "sheets": [{"name": "B", "cells": {"B2": "A\n"}}]}`),150208 []byte(`{"version": 2, "cells": {"A1": "x", "a1": "y"}}`),209+ []byte(`{"version": 2, "cells": {"A1": "=(", "A1": "1", "B1": "2", "B1": ""}}`),151210 []byte(`{"version": 2, "cells": null, "sheets": 5}`),152211 []byte(`{"version": 4, "cells": "x", "sheets": []}`),153212 []byte(`{"version": 2, "cells": {"A1": "😀", "A2": "bad \xff", "ZZ9": 5}}`),@@ -171,6 +171,9 @@ func (st *cellStore) plainSlot(c *Cell) (slot, bool) {
171171 // numDec is how input prints v, as slot.dec, or false if it isn't v172172 // printed plainly.173173 func numDec(v float64, input string) (uint8, bool) {174+ if d, ok := plainDec(v, input); ok {175+ return d, true176+ }174177 var buf [64]byte175178 if string(strconv.AppendFloat(buf[:0], v, 'f', -1, 64)) == input {176179 return 0, true@@ -186,6 +189,60 @@ func numDec(v float64, input string) (uint8, bool) {
186189 return 0, false187190 }188191 192+// plainDec is numDec for the common input, a plain decimal (see193+// plainForm) that is v, found without printing v. It reports false for194+// any other input, which numDec then prints v to compare.195+func plainDec(v float64, input string) (uint8, bool) {196+ d, ok := plainForm(input)197+ if !ok {198+ return 0, false199+ }200+ if p, err := strconv.ParseFloat(input, 64); err != nil || math.Float64bits(p) != math.Float64bits(v) {201+ return 0, false202+ }203+ return d, true204+}205+206+// plainForm reports whether input is a plain decimal, of at most 15207+// significant digits without a sign but "-", leading zeros or an208+// exponent, and returns how it prints its value as slot.dec. Such a209+// decimal is its value's shortest text when it doesn't end in a zero210+// after the point, and its value printed with as many decimals as it211+// has otherwise: 15 digits always come back from a float64.212+func plainForm(input string) (uint8, bool) {213+ s := input214+ if s != "" && s[0] == '-' {215+ s = s[1:]216+ }217+ if s == "" || s[0] == '0' && len(s) > 1 && s[1] != '.' {218+ return 0, false219+ }220+ point, digits, lead := -1, 0, true221+ for i := 0; i < len(s); i++ {222+ switch c := s[i]; {223+ case c == '.' && point < 0 && i > 0 && i < len(s)-1:224+ point = i225+ case c >= '0' && c <= '9':226+ if lead = lead && c == '0'; !lead {227+ digits++228+ }229+ default:230+ return 0, false231+ }232+ }233+ d := 0234+ if point >= 0 {235+ d = len(s) - point - 1236+ }237+ if digits > 15 || d >= maxDec {238+ return 0, false239+ }240+ if d == 0 || s[len(s)-1] != '0' {241+ return 0, true242+ }243+ return uint8(d + 1), true244+}245+189246 func boolText(b bool) string {190247 if b {191248 return "TRUE"@@ -0,0 +1,62 @@
1+package sheet2+3+import (4+ "fmt"5+ "math"6+ "math/rand/v2"7+ "strconv"8+ "strings"9+ "testing"10+)11+12+// printedDec is numDec found only by printing v, as for inputs13+// plainDec leaves.14+func printedDec(v float64, input string) (uint8, bool) {15+ if strconv.FormatFloat(v, 'f', -1, 64) == input {16+ return 0, true17+ }18+ i := strings.IndexByte(input, '.')19+ if i < 0 || len(input)-i-1 >= maxDec {20+ return 0, false21+ }22+ d := len(input) - i - 123+ if strconv.FormatFloat(v, 'f', d, 64) == input {24+ return uint8(d + 1), true25+ }26+ return 0, false27+}28+29+// Where plainDec answers without printing, it answers as printing does.30+func TestPlainDecAsPrinted(t *testing.T) {31+ inputs := []string{"0", "-0", "0.0", "-0.0", "0.50", "1.5", "100", "100.0", "12.30", "007", "0.1", "0.000", "1e5", ".5", "5.",32+ "123456789012345", "1234567890123456", "0.000000000000000000001", "99999999999999.9", "-42.4200", "1,200", "$5", "1.2.3", "--1", "+1"}33+ r := rand.New(rand.NewPCG(3, 4))34+ for range 20000 {35+ switch r.IntN(3) {36+ case 0:37+ inputs = append(inputs, fmt.Sprintf("%.*f", r.IntN(8), (r.Float64()-0.5)*math.Pow(10, float64(r.IntN(18)))))38+ case 1:39+ inputs = append(inputs, strconv.FormatInt(r.Int64N(1e17)-5e16, 10))40+ default:41+ inputs = append(inputs, strconv.FormatFloat(r.NormFloat64()*1e6, 'f', -1, 64))42+ }43+ }44+ answered := 045+ for _, in := range inputs {46+ v, err := strconv.ParseFloat(in, 64)47+ if err != nil {48+ continue49+ }50+ d, ok := plainDec(v, in)51+ if !ok {52+ continue53+ }54+ answered++55+ if pd, pok := printedDec(v, in); !pok || pd != d {56+ t.Errorf("%q: plainDec %d, printed %d %v", in, d, pd, pok)57+ }58+ }59+ if answered < len(inputs)/5 {60+ t.Errorf("plainDec answered %d of %d", answered, len(inputs))61+ }62+}@@ -217,6 +217,21 @@ func (st *cellStore) set(a Addr, c *Cell) {
217217 st.touch(a)218218 }219219 220+// setSlot stores sl, a plain slot whose strings and look st already221+// counts, at a: set for a slot made directly, as a file's numbers are.222+func (st *cellStore) setSlot(a Addr, sl slot) {223+ b, i := st.find(a)224+ if b == nil {225+ b = st.stored.mark(a)226+ i, _ = b.index(a.Row & (blockRows - 1))227+ b.vals = slices.Insert(b.vals, i, slot{kind: slotBlank})228+ }229+ st.releaseSlot(b.vals[i])230+ b.vals[i] = sl231+ st.filledAs(a, st.filled.has(a), sl.kind != slotBlank)232+ st.touch(a)233+}234+220235 // filledAs updates the index of filled cells for a cell at a that was221236 // and is filled or not.222237 func (st *cellStore) filledAs(a Addr, was, is bool) {@@ -49,7 +49,8 @@ func parseNumberFormat(s string) (float64, Format, bool) {
4949 s = strings.ReplaceAll(s, ",", "")5050 }5151 v, err := strconv.ParseFloat(s, 64)52- if err != nil || strings.ContainsAny(s, "xXpP_") || strings.EqualFold(s, "inf") {52+ dot, exp, odd := scanNumber(s)53+ if err != nil || odd || strings.EqualFold(s, "inf") {5354 return 0, Format{}, false5455 }5556 if pct {@@ -66,7 +67,7 @@ func parseNumberFormat(s string) (float64, Format, bool) {
6667 }6768 // The format Sheets infers: two decimals if any were typed.6869 dec := 069- if strings.Contains(s, ".") {70+ if dot {7071 dec = 27172 }7273 var f Format@@ -75,7 +76,7 @@ func parseNumberFormat(s string) (float64, Format, bool) {
7576 f = Format{Kind: FmtCurrency, Decimals: dec}7677 case pct:7778 f = Format{Kind: FmtPercent, Decimals: dec}78- case strings.ContainsAny(s, "eE"):79+ case exp:7980 f = Preset(FmtScientific)8081 case grouped:8182 f = Format{Kind: FmtNumber, Decimals: dec}@@ -83,6 +84,23 @@ func parseNumberFormat(s string) (float64, Format, bool) {
8384 return v, f, true8485 }8586 87+// scanNumber reports whether s has a decimal point, an exponent, and88+// what ParseFloat accepts but an entry isn't: hexadecimal, underscores.89+// One pass, as every number loaded goes through it.90+func scanNumber(s string) (dot, exp, odd bool) {91+ for i := 0; i < len(s); i++ {92+ switch s[i] {93+ case '.':94+ dot = true95+ case 'e', 'E':96+ exp = true97+ case 'x', 'X', 'p', 'P', '_':98+ odd = true99+ }100+ }101+ return dot, exp, odd102+}103+86104 // validGrouping checks thousands separators: groups of three digits87105 // before the decimal point, and none after it (1.234,5 is text).88106 func validGrouping(s string) bool {