cat / 012
clone

a spreadsheet for the terminal: lotus 1-2-3 looks, google sheets keys and formulas, one go binary

Raise the grid to Excel's 1,048,576 rows by 16,384 columns

Dami committed 1c459ce
Columns now run to XFD, with three-letter names in references, and
formulas take whole columns (A:C, $A:$A) and rows (2:5), which stay
whole when copied or when lines are inserted and deleted; ranges print
that way, A1:A1048576 as A:A.

For the engine to cost what the data costs at that size:

- A second occupancy index holds the cells with contents, so the used
  range, data-edge jumps and the next filled column of a row are found
  by index rather than by walking a million rows. A filter over whole
  columns tests the rows with data and one blank row for the rest, and
  steps over hidden blank rows at once.
- Formatting whole columns or rows keeps the format on the line, as
  Sheets does, rather than materializing a cell per address: a cell
  shows its own format, else its row's, else its column's, part by part;
  one the fallback can't express keeps its own whole. Line formats undo,
  move with inserted lines and are saved in an additive "lines" field.
- Paste clears what the clip's blanks land on and places each clip cell
  per tile; fills of blank lanes only clear; clipboard text stops at the
  last value; pivots and JEV read only their data rows.
- Pastes and fills are capped by a max-cells budget (two million by
  default) instead of a fixed 1 M.
commit
1c459ce04e8eeda23b0cb4e64fee22ebf4ca5219
parent
c114f3207a388da596438aa314fe99d443c757df
author
Dami <dami.etoile@gmail.com>
31 files changed, +1427 -428
Minternal/formula/formula_test.go33+++-
Minternal/formula/parse.go31+++
Minternal/formula/print.go2+-
Minternal/formula/ref.go109++++++++++-
Minternal/formula/rewrite.go15+-
Minternal/sheet/cellformat.go72++++---
Minternal/sheet/file.go15+-
Minternal/sheet/filter.go56+++++-
Minternal/sheet/format.go5+
Minternal/sheet/format_test.go2+-
Minternal/sheet/functions_jev.go48+++--
Minternal/sheet/history.go29++-
Minternal/sheet/historysize.go2+-
Ainternal/sheet/linefile.go83++++++++
Ainternal/sheet/lines.go312++++++++++++++++++++++++++++++
Ainternal/sheet/lines_test.go127++++++++++++
Minternal/sheet/load.go4+
Minternal/sheet/nav.go107++++++++--
Minternal/sheet/nav_test.go10+-
Ainternal/sheet/occupancy.go266++++++++++++++++++++++++++
Minternal/sheet/ops.go68+++++-
Minternal/sheet/pivot.go2+-
Minternal/sheet/pivot_stress_test.go4+-
Minternal/sheet/pivotcalc.go16+-
Minternal/sheet/refs_test.go10+-
Minternal/sheet/series.go39+++-
Minternal/sheet/sheet.go4+
Minternal/sheet/store.go279+++------------------------
Minternal/sheet/store_test.go26++-
Minternal/sheet/stress_test.go36++--
Ainternal/sheet/vast_test.go43++++
M internal/formula/formula_test.go +29 -4
@@ -41,11 +41,14 @@ func TestAddr(t *testing.T) {
4141 {"z10", Addr{25, 9}, true},4242 {"AA1", Addr{26, 0}, true},4343 {"IV8192", Addr{255, 8191}, true},44+ {"ZZ1", Addr{701, 0}, true},45+ {"AAA1", Addr{702, 0}, true},46+ {"XFD1048576", Addr{16383, 1048575}, true},4447 {"$B$2", Addr{1, 1}, true},45- {"IW1", Addr{}, false},48+ {"XFE1", Addr{}, false},4649 {"A0", Addr{}, false},47- {"A8193", Addr{}, false},48- {"ABC1", Addr{}, false},50+ {"A1048577", Addr{}, false},51+ {"ABCD1", Addr{}, false},4952 {"A+1", Addr{}, false},5053 {"1A", Addr{}, false},5154 }
@@ -69,6 +72,16 @@ func TestParseRange(t *testing.T) {
6972 t.Errorf("ParseRange(%q) = %v, %v", in, r, ok)7073 }7174 }75+ for in, want := range map[string]string{"B:A": "A:B", "$3:2": "2:3", "XFD:XFD": "XFD:XFD", "A1:A1048576": "A:A"} {76+ if r, ok := ParseRange(in); !ok || r.String() != want {77+ t.Errorf("ParseRange(%q) = %v, %v; want %s", in, r, ok, want)78+ }79+ }80+ for _, in := range []string{"A:1", "0:1", "1:1048577", "XFE:XFE", "A:"} {81+ if r, ok := ParseRange(in); ok {82+ t.Errorf("ParseRange(%q) = %v, want no range", in, r)83+ }84+ }7285 }7386 7487 func TestParseAbsoluteRefs(t *testing.T) {
@@ -119,6 +132,13 @@ func TestPrintFormula(t *testing.T) {
119132 {"=IF(A1>=2,TRUE,profit)", "=IF(A1>=2,TRUE,profit)"},120133 {"=SUM(#REF!)", "=SUM(#REF!)"},121134 {"+A1", "=+A1"},135+ {"=SUM(a:c)", "=SUM(A:C)"},136+ {"=SUM($C:a)", "=SUM(A:$C)"},137+ {"=SUM(A1:A1048576)", "=SUM(A:A)"},138+ {"=SUM(2:5)+SUM($3:$3)", "=SUM(2:5)+SUM($3:$3)"},139+ {"=SUM(Data!B:B)+SUM('Q 1'!1:2)", "=SUM(Data!B:B)+SUM('Q 1'!1:2)"},140+ {"=SUM(Data!XFD:Data!XFD)", "=SUM(Data!XFD:XFD)"},141+ {"=ZZZ1+AAA1:XFD9", "=ZZZ1+AAA1:XFD9"},122142 }123143 for _, tt := range tests {124144 n, err := Parse(tt.in, testFuncs)
@@ -146,7 +166,12 @@ func TestShiftRefs(t *testing.T) {
146166 {"=A2", 0, -2, "=#REF!"},147167 {"=SUM(A1:B2)+1", -1, 0, "=SUM(#REF!)+1"},148168 {"=$A2", -1, 0, "=$A2"},149- {"=IV1", 1, 0, "=#REF!"},169+ {"=XFD1", 1, 0, "=#REF!"},170+ {"=SUM(A:A)", 2, 7, "=SUM(C:C)"},171+ {"=SUM($A:B)", 1, 7, "=SUM($A:C)"},172+ {"=SUM(3:4)", 5, 2, "=SUM(5:6)"},173+ {"=A1048576", 0, 1, "=#REF!"},174+ {"=ZZ1", 1, 0, "=AAA1"},150175 // A mixed range can flip when one corner is anchored.151176 {"=SUM($C1:D1)", -2, 0, "=SUM(B1:$C1)"},152177 }
M internal/formula/parse.go +31 -0
@@ -102,6 +102,9 @@ func (p *parser) prefix() (Node, error) {
102102 t := p.next()103103 switch t.kind {104104 case tokNum:105+ if r, ok := p.lines(t, ""); ok {106+ return r, nil107+ }105108 v, err := strconv.ParseFloat(t.text, 64)106109 if err != nil {107110 return nil, &ParseError{t.pos, "Invalid number " + t.text}
@@ -113,6 +116,9 @@ func (p *parser) prefix() (Node, error) {
113116 return p.ident(t, "")114117 case tokSheet:115118 ref := p.next()119+ if r, ok := p.lines(ref, t.text); ok {120+ return r, nil121+ }116122 if ref.kind != tokIdent {117123 return nil, &ParseError{ref.pos, "Expected a cell after " + QuoteSheet(t.text) + "!"}118124 }
@@ -160,6 +166,9 @@ func (p *parser) prefixOp(t token) (Node, error) {
160166 // ident parses a reference, a range, a boolean or a name. sheet is the161167 // sheet the reference was qualified with, if any.162168 func (p *parser) ident(t token, sheet string) (Node, error) {169+ if r, ok := p.lines(t, sheet); ok {170+ return r, nil171+ }163172 a, abs, isRef := ParseRef(t.text)164173 if !isRef {165174 if sheet != "" {
@@ -195,6 +204,28 @@ func (p *parser) ident(t token, sheet string) (Node, error) {
195204 return r, nil196205 }197206 207+// lines parses whole columns (A:C, $A:$A) or rows (2:5), starting at t,208+// if that's what comes.209+func (p *parser) lines(t token, sheet string) (Range, bool) {210+ if !p.isOp(":") || t.kind != tokIdent && t.kind != tokNum {211+ return Range{}, false212+ }213+ end := p.toks[p.pos+1]214+ if end.kind == tokSheet && SheetKey(end.text) == SheetKey(sheet) && p.pos+2 < len(p.toks) {215+ end = p.toks[p.pos+2]216+ }217+ if end.kind != tokIdent && end.kind != tokNum {218+ return Range{}, false219+ }220+ r, abs, ok := ParseLines(t.text, end.text)221+ if !ok {222+ return Range{}, false223+ }224+ for p.next() != end {225+ }226+ return Range{r, abs, sheet}, true227+}228+198229 // NewRange builds a normalized range from two corners as written. Each199230 // absolute marker stays with its column or row, so $B1:A$2 becomes A1:$B$2.200231 func NewRange(a, b Addr, aAbs, bAbs Abs) Range {
M internal/formula/print.go +1 -1
M internal/formula/ref.go +99 -10
@@ -5,11 +5,11 @@ import (
55 "strings"66 )77 8-// Worksheet bounds, matching Lotus 1-2-3 Release 2 (A..IV, 1..8192). A9-// reference past them isn't a reference: IW1 reads as a name.8+// Worksheet bounds, matching Excel (A..XFD, 1..1048576). A reference9+// past them isn't a reference: XFE1 reads as a name.1010 const (11- MaxCols = 25612- MaxRows = 819211+ MaxCols = 1638412+ MaxRows = 10485761313 )1414 1515 // Addr identifies a cell by zero-based column and row.
@@ -27,17 +27,24 @@ func (a Addr) Valid() bool {
2727 return a.Col >= 0 && a.Col < MaxCols && a.Row >= 0 && a.Row < MaxRows2828 }2929 30-// ColName converts a zero-based column index to letters: 0 -> A, 26 -> AA.30+// ColName converts a zero-based column index to letters: 0 -> A, 26 -> AA,31+// 702 -> AAA.3132 func ColName(c int) string {3233 if c < 26 {3334 return string(rune('A' + c))3435 }35- return string(rune('A'+c/26-1)) + string(rune('A'+c%26))36+ var b [8]byte37+ i := len(b)38+ for c++; c > 0; c = (c - 1) / 26 {39+ i--40+ b[i] = byte('A' + (c-1)%26)41+ }42+ return string(b[i:])3643 }3744 3845 // ParseCol converts column letters (case-insensitive) to a zero-based index.3946 func ParseCol(s string) (int, bool) {40- if len(s) == 0 || len(s) > 2 {47+ if len(s) == 0 || len(s) > 3 {4148 return 0, false4249 }4350 c := 0
@@ -126,15 +133,26 @@ func (r Rect) Contains(a Addr) bool {
126133 a.Row >= r.From.Row && a.Row <= r.To.Row127134 }128135 129-// String returns the range as A1:B3, or A1 for a single cell.136+// AllRows reports whether r spans every row: whole columns, A:A.137+func (r Rect) AllRows() bool { return r.From.Row == 0 && r.To.Row == MaxRows-1 }138+139+// AllCols reports whether r spans every column: whole rows, 1:1.140+func (r Rect) AllCols() bool { return r.From.Col == 0 && r.To.Col == MaxCols-1 }141+142+// String returns the range as A1:B3, A1 for a single cell, or A:C and143+// 2:5 for whole columns and rows.130144 func (r Rect) String() string {131- if r.From == r.To {145+ switch {146+ case r.From == r.To:132147 return r.From.String()148+ case r.AllRows(), r.AllCols():149+ return RangeString(r, [2]Abs{})133150 }134151 return r.From.String() + ":" + r.To.String()135152 }136153 137-// ParseRange parses "A1", "A1:B3" or 1-2-3 style "A1..B3".154+// ParseRange parses "A1", "A1:B3", 1-2-3 style "A1..B3", or whole155+// columns "A:C" and rows "2:5".138156 func ParseRange(s string) (Rect, bool) {139157 s = strings.TrimSpace(s)140158 sep := strings.Index(s, "..")
@@ -146,11 +164,82 @@ func ParseRange(s string) (Rect, bool) {
146164 a, ok := ParseAddr(s)147165 return Rect{From: a, To: a}, ok148166 }167+ if r, _, ok := ParseLines(s[:sep], s[sep+width:]); ok {168+ return r, true169+ }149170 a, ok1 := ParseAddr(s[:sep])150171 b, ok2 := ParseAddr(s[sep+width:])151172 return NewRect(a, b), ok1 && ok2152173 }153174 175+// ParseLines parses the ends of whole columns ("A", "$C") or whole rows176+// ("2", "$5") as the range they span, with their absolute markers.177+func ParseLines(from, to string) (Rect, [2]Abs, bool) {178+ c0, abs0, ok0 := parseLine(from, false)179+ c1, abs1, ok1 := parseLine(to, false)180+ if ok0 && ok1 {181+ r := NewRect(Addr{Col: c0}, Addr{Col: c1, Row: MaxRows - 1})182+ return r, orderAbs(c0 > c1, abs0, abs1), true183+ }184+ r0, abs0, ok0 := parseLine(from, true)185+ r1, abs1, ok1 := parseLine(to, true)186+ if ok0 && ok1 {187+ r := NewRect(Addr{Row: r0}, Addr{Col: MaxCols - 1, Row: r1})188+ return r, orderAbs(r0 > r1, abs0, abs1), true189+ }190+ return Rect{}, [2]Abs{}, false191+}192+193+func orderAbs(swap bool, a, b Abs) [2]Abs {194+ if swap {195+ return [2]Abs{b, a}196+ }197+ return [2]Abs{a, b}198+}199+200+// parseLine parses a column's letters or a row's number, with an201+// optional $.202+func parseLine(s string, row bool) (int, Abs, bool) {203+ rest, abs := strings.CutPrefix(s, "$")204+ if !row {205+ c, ok := ParseCol(rest)206+ if abs {207+ return c, AbsCol, ok208+ }209+ return c, 0, ok210+ }211+ if rest == "" || strings.ContainsFunc(rest, func(r rune) bool { return r < '0' || r > '9' }) {212+ return 0, 0, false213+ }214+ n, err := strconv.Atoi(rest)215+ if err != nil || n < 1 || n > MaxRows {216+ return 0, 0, false217+ }218+ if abs {219+ return n - 1, AbsRow, true220+ }221+ return n - 1, 0, true222+}223+224+// RangeString writes a range as written in a formula, with its absolute225+// markers: A1:B3, $A$1:B3, or whole columns (A:C) and rows (2:5).226+func RangeString(r Rect, abs [2]Abs) string {227+ switch {228+ case r.AllRows():229+ return lineString(ColName(r.From.Col), abs[0]&AbsCol) + ":" + lineString(ColName(r.To.Col), abs[1]&AbsCol)230+ case r.AllCols():231+ return lineString(strconv.Itoa(r.From.Row+1), abs[0]&AbsRow) + ":" + lineString(strconv.Itoa(r.To.Row+1), abs[1]&AbsRow)232+ }233+ return RefString(r.From, abs[0]) + ":" + RefString(r.To, abs[1])234+}235+236+func lineString(s string, abs Abs) string {237+ if abs != 0 {238+ return "$" + s239+ }240+ return s241+}242+154243 // LooksLikeRef reports whether an upper-case name reads as a cell in A1 or155244 // R1C1 style, even beyond this sheet's edges, so names stay unambiguous in156245 // other spreadsheets too.
M internal/formula/rewrite.go +13 -2
M internal/sheet/cellformat.go +39 -33
@@ -11,24 +11,33 @@ import "github.com/FelineStateMachine/012/internal/formula"
1111 const materializeLimit = 1 << 161212 1313 // DisplayFormat returns the format the cell at a is shown with: its own,14-// or for Automatic, the one inferred from its formula (=DATE() shows a15-// date, =SUM(B2:B4) of currency shows currency).14+// its row's or column's, or for Automatic, the one inferred from its15+// formula (=DATE() shows a date, =SUM(B2:B4) of currency shows16+// currency).1617 func (s *Sheet) DisplayFormat(a Addr) Format {1718 c := s.cells.get(a)1819 switch {19- case c == nil:20+ case c != nil && !c.Format.IsZero():21+ return c.Format22+ case s.lines.cols == nil && s.lines.rows == nil:23+ case c != nil && c.Style.own:24+ default:25+ if f := s.inherited(a).Format; !f.IsZero() {26+ return f27+ }28+ }29+ if c == nil {2030 return Format{}21- case c.Format.IsZero():22- return c.auto2331 }24- return c.Format32+ return c.auto2533 }2634 2735 // SetFormat gives every cell in r the number format f, including blank28-// cells, which keep it for when something is typed.36+// cells, which keep it for when something is typed. Whole columns and37+// rows keep it as their line's format rather than on each cell.2938 func (s *Sheet) SetFormat(r Rect, f Format) {3039 s.change("format "+r.String()+" as "+f.Kind.label(), r, func() {31- s.eachCell(r, !f.IsZero(), func(_ Addr, c *Cell) { c.Format = f })40+ s.eachFormat(r, !f.IsZero(), func(_ Addr, l *lineFmt) { l.Format = f })3241 })3342 }3443
@@ -37,7 +46,7 @@ func (s *Sheet) SetFormat(r Rect, f Format) {
3746 // label ("bold B2:B5") wrap it in Batch.3847 func (s *Sheet) SetStyle(r Rect, fn func(*Style)) {3948 s.change("style "+r.String(), r, func() {40- s.eachCell(r, true, func(_ Addr, c *Cell) { fn(&c.Style) })49+ s.eachFormat(r, true, func(_ Addr, l *lineFmt) { fn(&l.Style) })4150 })4251 }4352
@@ -45,7 +54,7 @@ func (s *Sheet) SetStyle(r Rect, fn func(*Style)) {
4554 // Sheets' Format > Clear formatting.4655 func (s *Sheet) ClearFormatting(r Rect) {4756 s.change("clear formatting "+r.String(), r, func() {48- s.eachCell(r, false, func(_ Addr, c *Cell) { c.Format, c.Style = Format{}, Style{} })57+ s.eachFormat(r, false, func(_ Addr, l *lineFmt) { *l = lineFmt{} })4958 })5059 }5160
@@ -57,20 +66,28 @@ func (s *Sheet) AdjustDecimals(r Rect, delta int) {
5766 label = "fewer decimals in "5867 }5968 s.change(label+r.String(), r, func() {60- s.eachCell(r, false, func(a Addr, c *Cell) {61- if !c.Blank() {62- c.Format = s.DisplayFormat(a).WithDecimals(delta, c.Value.Num)69+ s.eachCell(r, false, func(a Addr, l *lineFmt) {70+ if c := s.cells.get(a); !c.Blank() {71+ l.Format = s.DisplayFormat(a).WithDecimals(delta, c.Value.Num)6372 }6473 })6574 })6675 }6776 68-// eachCell applies fn to a copy of each cell in r and places the copy,69-// so the change can be undone. With create, blank cells get a cell to70-// hold the formatting (up to materializeLimit cells); cells left with71-// neither contents nor formatting are removed. Cells that fn leaves72-// unchanged are not touched.73-func (s *Sheet) eachCell(r Rect, create bool, fn func(Addr, *Cell)) {77+// eachFormat applies fn to the formatting of r: to its lines' formats78+// when r is whole columns or rows, and to each of its cells otherwise.79+func (s *Sheet) eachFormat(r Rect, create bool, fn func(Addr, *lineFmt)) {80+ if !s.formatLines(r, func(l *lineFmt) { fn(Addr{}, l) }) {81+ s.eachCell(r, create, fn)82+ }83+}84+85+// eachCell applies fn to what each cell in r shows and has the cell86+// show the result, placing a copy so the change can be undone. With87+// create, blank cells get a cell to hold the formatting (up to88+// materializeLimit cells); cells left with neither contents nor89+// formatting are removed. Cells that fn leaves unchanged are not touched.90+func (s *Sheet) eachCell(r Rect, create bool, fn func(Addr, *lineFmt)) {7491 area := (r.To.Col - r.From.Col + 1) * (r.To.Row - r.From.Row + 1)7592 var addrs []Addr7693 if create && area <= materializeLimit {
@@ -83,20 +100,9 @@ func (s *Sheet) eachCell(r Rect, create bool, fn func(Addr, *Cell)) {
83100 addrs = s.cellsIn(r)84101 }85102 for _, a := range addrs {86- old := s.cells.get(a)87- c := old.clone()88- if c == nil {89- c = &Cell{}90- }91- fn(a, c)92- switch {93- case c.Blank() && c.Format.IsZero() && c.Style.IsZero():94- if old != nil {95- s.place(a, nil)96- }97- case old == nil || c.Format != old.Format || c.Style != old.Style:98- s.place(a, c)99- }103+ want := s.effective(a)104+ fn(a, &want)105+ s.placeFormat(a, want)100106 }101107 }102108
M internal/sheet/file.go +12 -3
@@ -59,8 +59,9 @@ type fileFormat struct {
5959 6060 // fileSheet is one sheet of a file.6161 type fileSheet struct {62- Name string `json:"name,omitempty"`63- Widths map[string]int `json:"widths,omitempty"`62+ Name string `json:"name,omitempty"`63+ Widths map[string]int `json:"widths,omitempty"`64+ Lines map[string]json.RawMessage `json:"lines,omitempty"` // column and row formats, see linefile.go6465 fileView6566 Cells map[string]json.RawMessage `json:"cells"`6667 Charts []fileChart `json:"charts,omitempty"`
@@ -119,6 +120,7 @@ type fileCell struct {
119120 Underline bool `json:"underline,omitempty"`120121 Strikethrough bool `json:"strikethrough,omitempty"`121122 Align string `json:"align,omitempty"`123+ Own bool `json:"own,omitempty"` // not taking its column's or row's formatting122124 }123125 124126 func encodeCell(c *Cell) (json.RawMessage, error) {
@@ -133,6 +135,7 @@ func encodeCell(c *Cell) (json.RawMessage, error) {
133135 Underline: c.Style.Underline,134136 Strikethrough: c.Style.Strikethrough,135137 Align: c.Style.Align.String(),138+ Own: c.Style.own,136139 }137140 if !c.Format.IsZero() {138141 fc.Format = c.Format.Kind.String()
@@ -168,7 +171,7 @@ func decodeCell(raw json.RawMessage) (string, Format, Style, error) {
168171 if !ok {169172 return "", Format{}, Style{}, fmt.Errorf("unknown alignment %q", fc.Align)170173 }171- st := Style{Bold: fc.Bold, Italic: fc.Italic, Underline: fc.Underline, Strikethrough: fc.Strikethrough, Align: al}174+ st := Style{Bold: fc.Bold, Italic: fc.Italic, Underline: fc.Underline, Strikethrough: fc.Strikethrough, Align: al, own: fc.Own}172175 return fc.Input, f, st, nil173176 }174177
@@ -285,6 +288,9 @@ func (s *Sheet) writeBody(b *bytes.Buffer, indent, names string) error {
285288 }286289 b.WriteString("},\n")287290 }291+ if err := s.writeLines(b, indent); err != nil {292+ return err293+ }288294 if names != "" {289295 b.WriteString(indent + names + ",\n")290296 }
@@ -441,6 +447,9 @@ func (s *Sheet) read(f fileSheet, version int) error {
441447 }442448 s.setWidth(c, width)443449 }450+ if err := s.readLines(f.Lines); err != nil {451+ return err452+ }444453 for name, raw := range f.Cells {445454 a, ok := ParseAddr(name)446455 if !ok {
M internal/sheet/filter.go +51 -5
@@ -171,10 +171,14 @@ func (s *Sheet) setFilter(label string, f *Filter) {
171171 }172172 173173 // hiddenCache remembers which rows the filter hides until values or the174-// filter change, so navigation and drawing stay cheap.174+// filter change, so navigation and drawing stay cheap. The rows of the175+// filter's range past its last filled row are all blank, so they are176+// hidden or shown together: a filter on whole columns tests the rows177+// with data and one blank row, not a million.175178 type hiddenCache struct {176179 valid bool177180 rows map[int]struct{}181+ tail Rect // the blank rows at the end of the range, when hidden (From.Row > To.Row otherwise)178182 }179183 180184 // RowHidden reports whether the filter hides row r.
@@ -183,6 +187,9 @@ func (s *Sheet) RowHidden(r int) bool {
183187 return false184188 }185189 s.updateHidden()190+ if r >= s.hidden.tail.From.Row && r <= s.hidden.tail.To.Row {191+ return true192+ }186193 _, hid := s.hidden.rows[r]187194 return hid188195 }
@@ -193,24 +200,57 @@ func (s *Sheet) HiddenRows() int {
193200 return 0194201 }195202 s.updateHidden()196- return len(s.hidden.rows)203+ return len(s.hidden.rows) + max(s.hidden.tail.To.Row-s.hidden.tail.From.Row+1, 0)204+}205+206+// NextShownRow returns the first row from r+d on (d is 1 or -1) that the207+// filter doesn't hide, jumping over the hidden blank rows at the end of208+// its range at once, and false past the edge of the sheet.209+func (s *Sheet) NextShownRow(r, d int) (int, bool) {210+ for r += d; r >= 0 && r < MaxRows; r += d {211+ if !s.RowHidden(r) {212+ return r, true213+ }214+ if t := s.hidden.tail; r >= t.From.Row && r <= t.To.Row {215+ r = t.To.Row216+ if d < 0 {217+ r = t.From.Row218+ }219+ }220+ }221+ return r, false222+}223+224+// filterData is the part of the filter's range that may hold data: its225+// rows through the last filled one.226+func (s *Sheet) filterData(r Rect) Rect {227+ b, ok := s.cells.filled.bounds(r)228+ r.To.Row = r.From.Row229+ if ok {230+ r.To.Row = b.To.Row231+ }232+ return r197233 }198234 199235 func (s *Sheet) updateHidden() {200236 if s.hidden.valid {201237 return202238 }203- s.hidden = hiddenCache{valid: true, rows: map[int]struct{}{}}239+ s.hidden = hiddenCache{valid: true, rows: map[int]struct{}{}, tail: Rect{From: Addr{Row: 1}}}204240 f := s.view.filter205241 if f == nil || len(f.Cols) == 0 {206242 return207243 }208244 tests := s.criteriaTests(f, -1)209- for row := f.Range.From.Row + 1; row <= f.Range.To.Row; row++ {245+ data := s.filterData(f.Range)246+ for row := f.Range.From.Row + 1; row <= data.To.Row; row++ {210247 if !s.rowPasses(row, tests) {211248 s.hidden.rows[row] = struct{}{}212249 }213250 }251+ if blank := data.To.Row + 1; blank <= f.Range.To.Row && !s.rowPasses(blank, tests) {252+ s.hidden.tail = Rect{From: Addr{Row: blank}, To: Addr{Row: f.Range.To.Row}}253+ }214254 }215255 216256 // colTest is one column's criteria, ready to run.
@@ -347,7 +387,8 @@ func (s *Sheet) valuesList(r Rect, col int, tests []colTest, hide []string, skip
347387 }348388 counts := map[string]int{}349389 values := map[string]Value{}350- for row := r.From.Row + 1; row <= r.To.Row; row++ {390+ data := s.filterData(r)391+ for row := r.From.Row + 1; row <= data.To.Row; row++ {351392 if !s.rowPasses(row, tests) || skipBlank && s.rowBlank(row, r) {352393 continue353394 }
@@ -358,6 +399,11 @@ func (s *Sheet) valuesList(r Rect, col int, tests []colTest, hide []string, skip
358399 }359400 counts[t]++360401 }402+ // The rows past the data are blank: they count as one value.403+ if blank := data.To.Row + 1; blank <= r.To.Row && !skipBlank && s.rowPasses(blank, tests) {404+ counts[""] += r.To.Row - blank + 1405+ values[""] = Value{}406+ }361407 // Unchecked values stay listed even when no row has them any more, so362408 // they can be checked again.363409 for h := range hidden {
M internal/sheet/format.go +5 -0
M internal/sheet/format_test.go +1 -1
M internal/sheet/functions_jev.go +32 -16
@@ -72,28 +72,41 @@ func remoteEval(build func([]Node, lookup) (RemoteCall, error), result func(Remo
7272 }7373 }7474 75+// jevWhole is the largest range a JEV function's value sees whole.76+const jevWhole = 409677+7578 // inputError carries a spreadsheet error found in a JEV function's inputs.7679 type inputError struct{ v Value }7780 7881 func (e inputError) Error() string { return e.v.Str }7982 8083 // jevState turns the value argument into what the model sees: text,81-// numbers and booleans as themselves, a range as rows of them.84+// numbers and booleans as themselves, a range as rows of them. A range85+// of more than jevWhole cells leaves off the blank rows and columns past86+// its data, so a whole column is its data.8287 func jevState(n Node, get lookup) (any, error) {8388 if rn, ok := n.(formula.Range); ok {84- var rows [][]any85- for r := rn.Rect.From.Row; r <= rn.Rect.To.Row; r++ {89+ m := rectMatrix(rn.Sheet, rn.Rect, get)90+ if m.blank.Kind == Error {91+ return nil, inputError{m.blank}92+ }93+ rows, cols := m.rows, m.cols94+ if m.size() > jevWhole {95+ rows, cols = min(max(m.dataRows, 1), m.rows), min(max(m.dataCols, 1), m.cols)96+ }97+ var out [][]any98+ for r := range rows {8699 var row []any87- for c := rn.Rect.From.Col; c <= rn.Rect.To.Col; c++ {88- v := get.cell(rn.Sheet, Addr{Col: c, Row: r})100+ for c := range cols {101+ v := m.cell(r, c)89102 if v.Kind == Error {90103 return nil, inputError{v}91104 }92105 row = append(row, plain(v))93106 }94- rows = append(rows, row)107+ out = append(out, row)95108 }96- return rows, nil109+ return out, nil97110 }98111 v := eval(n, get)99112 if v.Kind == Error {
@@ -139,16 +152,19 @@ func jevQuestion(n Node, get lookup) (string, error) {
139152 func jevList(n Node, get lookup) ([]string, error) {140153 var out []string141154 if rn, ok := n.(formula.Range); ok {142- for r := rn.Rect.From.Row; r <= rn.Rect.To.Row; r++ {143- for c := rn.Rect.From.Col; c <= rn.Rect.To.Col; c++ {144- v := get.cell(rn.Sheet, Addr{Col: c, Row: r})145- if v.Kind == Error {146- return nil, inputError{v}147- }148- if s := strings.TrimSpace(text(v)); s != "" {149- out = append(out, s)150- }155+ var err error156+ get.cells(rn.Sheet, rn.Rect, func(_ Addr, v Value) bool {157+ if v.Kind == Error {158+ err = inputError{v}159+ return false151160 }161+ if s := strings.TrimSpace(text(v)); s != "" {162+ out = append(out, s)163+ }164+ return true165+ })166+ if err != nil {167+ return nil, err152168 }153169 return out, nil154170 }
M internal/sheet/history.go +20 -9
@@ -36,12 +36,13 @@ type step struct {
3636 // cellBytes is the estimated heap held by cells, kept as they are3737 // recorded so sizing a step doesn't walk them again.3838 cellBytes int6439- label string // what the step did, e.g. "clear B3:B5"40- sheet *Sheet // the sheet the UI shows when the step is undone or redone41- focus Rect // what the UI selects there42- cells map[loc]*Cell // before the step; nil for blank43- widths map[colKey]int // before the step; 0 for the default width44- names map[string]*Name // before the step, by key; nil for undefined39+ label string // what the step did, e.g. "clear B3:B5"40+ sheet *Sheet // the sheet the UI shows when the step is undone or redone41+ focus Rect // what the UI selects there42+ cells map[loc]*Cell // before the step; nil for blank43+ widths map[colKey]int // before the step; 0 for the default width44+ lines map[lineKey]lineFmt // column and row formats before the step45+ names map[string]*Name // before the step, by key; nil for undefined4546 views map[*Sheet]*viewState4647 // charts holds each touched sheet's charts before the step.4748 charts map[*Sheet][]Chart
@@ -62,7 +63,7 @@ type colKey struct {
6263 }6364 6465 func newStep(label string, s *Sheet, focus Rect) *step {65- return &step{label: label, sheet: s, focus: focus, cells: map[loc]*Cell{}, widths: map[colKey]int{},66+ return &step{label: label, sheet: s, focus: focus, cells: map[loc]*Cell{}, widths: map[colKey]int{}, lines: map[lineKey]lineFmt{},6667 names: map[string]*Name{}, views: map[*Sheet]*viewState{}, charts: map[*Sheet][]Chart{}, pivots: map[*Sheet]*Pivot{}}6768 }6869
@@ -72,7 +73,7 @@ func (st *step) empty() bool {
7273 7374 // widthOnly reports whether the step changed nothing but column widths.7475 func (st *step) widthOnly() bool {75- return len(st.cells) == 0 && len(st.names) == 0 && len(st.views) == 0 && len(st.charts) == 0 && len(st.pivots) == 0 && st.sheets == nil && st.decimal == nil &&76+ return len(st.cells) == 0 && len(st.lines) == 0 && len(st.names) == 0 && len(st.views) == 0 && len(st.charts) == 0 && len(st.pivots) == 0 && st.sheets == nil && st.decimal == nil &&7677 st.macros == nil7778 }7879
@@ -243,6 +244,11 @@ func (w *Workbook) dropUnchanged(st *step) {
243244 delete(st.widths, k)244245 }245246 }247+ for k, l := range st.lines {248+ if k.s.line(k.row, k.n) == l {249+ delete(st.lines, k)250+ }251+ }246252 for l, c := range st.cells {247253 if c == nil && l.s.cells.get(l.a) == nil {248254 delete(st.cells, l)
@@ -403,6 +409,11 @@ func (w *Workbook) swap(undo bool) (Change, bool) {
403409 inv.widths[k] = k.s.widths[k.col]404410 k.s.setWidth(k.col, width)405411 }412+ for k, l := range st.lines {413+ inv.lines[k] = k.s.line(k.row, k.n)414+ k.s.setLine(k.row, k.n, l)415+ changed = append(changed, k.s.lineCells(k.row, k.n)...)416+ }406417 for k, n := range st.names {407418 inv.names[k] = w.namePtr(k)408419 changed = append(changed, w.putName(k, n)...)
@@ -442,7 +453,7 @@ func (w *Workbook) swap(undo bool) (Change, bool) {
442453 w.pushUndo(inv)443454 }444455 h.mergeWidths = false445- macrosOnly := st.macros != nil && len(st.cells) == 0 && len(st.names) == 0 && len(st.charts) == 0 && len(st.pivots) == 0 && st.sheets == nil456+ macrosOnly := st.macros != nil && len(st.cells) == 0 && len(st.lines) == 0 && len(st.names) == 0 && len(st.charts) == 0 && len(st.pivots) == 0 && st.sheets == nil446457 return Change{Label: st.label, Focus: st.focus, Sheet: st.sheet, Tabs: st.sheets != nil, Macros: macrosOnly}, true447458 }448459
M internal/sheet/historysize.go +1 -1
A internal/sheet/linefile.go +83 -0
@@ -0,0 +1,83 @@
1+package sheet2+3+import (4+ "bytes"5+ "encoding/json"6+ "fmt"7+ "maps"8+ "slices"9+ "strings"10+11+ "github.com/FelineStateMachine/012/internal/formula"12+)13+14+// Column and row formats in the file: a "lines" field after the widths,15+// keyed by columns or rows as a range writes them, runs of lines with the16+// same formatting together, each value a formatted blank cell:17+//18+// "lines": {"B:B": {"format": "currency", "decimals": 2}, "D:F": {"bold": true}, "1:1": {"bold": true}},19+//20+// Earlier builds ignore the field and show those cells unformatted, so it21+// needs no new version.22+23+// writeLines writes the "lines" field, if the sheet has line formats.24+func (s *Sheet) writeLines(b *bytes.Buffer, indent string) error {25+ if s.lines.cols == nil && s.lines.rows == nil {26+ return nil27+ }28+ b.WriteString(indent + `"lines": {`)29+ first := true30+ for _, row := range []bool{false, true} {31+ m := s.lines.cols32+ if row {33+ m = s.lines.rows34+ }35+ ns := slices.Sorted(maps.Keys(m))36+ for i := 0; i < len(ns); {37+ j := i + 138+ for j < len(ns) && ns[j] == ns[j-1]+1 && m[ns[j]] == m[ns[i]] {39+ j++40+ }41+ l := m[ns[i]]42+ raw, err := encodeCell(&Cell{Format: l.Format, Style: l.Style})43+ if err != nil {44+ return err45+ }46+ if !first {47+ b.WriteString(", ")48+ }49+ first = false50+ fmt.Fprintf(b, "%q: %s", s.lineRect(row, ns[i], ns[j-1]).String(), raw)51+ i = j52+ }53+ }54+ b.WriteString("},\n")55+ return nil56+}57+58+// readLines sets the line formats of a file's "lines" field.59+func (s *Sheet) readLines(lines map[string]json.RawMessage) error {60+ for key, raw := range lines {61+ from, to, found := strings.Cut(key, ":")62+ if !found {63+ to = from64+ }65+ r, _, ok := formula.ParseLines(from, to)66+ if !ok {67+ return fmt.Errorf("invalid lines %q", key)68+ }69+ input, f, st, err := decodeCell(raw)70+ if err != nil || input != "" {71+ return fmt.Errorf("lines %s: invalid format", key)72+ }73+ row := !r.AllRows()74+ n0, n1 := r.From.Col, r.To.Col75+ if row {76+ n0, n1 = r.From.Row, r.To.Row77+ }78+ for n := n0; n <= n1; n++ {79+ s.setLine(row, n, lineFmt{f, st})80+ }81+ }82+ return nil83+}
A internal/sheet/lines.go +312 -0
@@ -0,0 +1,312 @@
1+package sheet2+3+import (4+ "maps"5+ "slices"6+7+ "github.com/FelineStateMachine/012/internal/formula"8+)9+10+// Column and row formats, as in Sheets: formatting whole columns or rows11+// stores the format once for the line instead of on each of its million12+// cells, and cells take it when they have none of their own. A cell's13+// format and style come from the cell, else its row, else its column;14+// each part (number format, text style) falls back on its own, so a15+// currency cell in a bold column shows bold currency. A cell whose16+// formatting differs from its line's in a way the fallback can't express17+// (Automatic in a currency column) keeps its own formatting whole, marked18+// by Style.own.19+20+// lineFmt is the formatting of a column, a row, or what a cell shows.21+type lineFmt struct {22+ Format Format23+ Style Style24+}25+26+func (l lineFmt) IsZero() bool { return l == lineFmt{} }27+28+// lineFormats are a sheet's column and row formats; nil maps when none.29+type lineFormats struct {30+ cols map[int]lineFmt31+ rows map[int]lineFmt32+}33+34+// lineKey is a column (row false) or row of a sheet, for undo.35+type lineKey struct {36+ s *Sheet37+ row bool38+ n int39+}40+41+// line is a column's (row false) or row's formatting.42+func (s *Sheet) line(row bool, n int) lineFmt {43+ if row {44+ return s.lines.rows[n]45+ }46+ return s.lines.cols[n]47+}48+49+// inherited is what a cell at a shows without formatting of its own: its50+// row's format and style, else its column's.51+func (s *Sheet) inherited(a Addr) lineFmt {52+ if s.lines.rows == nil && s.lines.cols == nil {53+ return lineFmt{}54+ }55+ l := s.lines.rows[a.Row]56+ if c, ok := s.lines.cols[a.Col]; ok {57+ if l.Format.IsZero() {58+ l.Format = c.Format59+ }60+ if l.Style.IsZero() {61+ l.Style = c.Style62+ }63+ }64+ return l65+}66+67+// effective is the format (without what formulas infer) and style the68+// cell at a shows: its own, falling back part by part on its line's.69+func (s *Sheet) effective(a Addr) lineFmt {70+ c := s.cells.get(a)71+ if c != nil && c.Style.own {72+ st := c.Style73+ st.own = false74+ return lineFmt{c.Format, st}75+ }76+ l := s.inherited(a)77+ if c != nil {78+ if !c.Format.IsZero() {79+ l.Format = c.Format80+ }81+ if !c.Style.IsZero() {82+ l.Style = c.Style83+ }84+ }85+ return l86+}87+88+// CellFormat returns the number format the cell at a has, its own or89+// its row's or column's, without the one a formula infers.90+func (s *Sheet) CellFormat(a Addr) Format { return s.effective(a).Format }91+92+// CellStyle returns the text style the cell at a shows: its own, or its93+// row's or column's.94+func (s *Sheet) CellStyle(a Addr) Style { return s.effective(a).Style }95+96+// own is the formatting a cell stores to show want where it would97+// otherwise show inh.98+func own(want, inh lineFmt) (Format, Style) {99+ f, st := want.Format, want.Style100+ switch {101+ case want == inh:102+ return Format{}, Style{}103+ case f.IsZero() && !inh.Format.IsZero(), st.IsZero() && !inh.Style.IsZero():104+ st.own = true // Automatic, or plain text, over a formatted line105+ return f, st106+ }107+ if f == inh.Format {108+ f = Format{}109+ }110+ if st == inh.Style {111+ st = Style{}112+ }113+ return f, st114+}115+116+// ColFormats and RowFormats return the columns and rows that have a117+// format or style of their own, by index.118+func (s *Sheet) ColFormats() map[int]Format { return lineFormatsOf(s.lines.cols) }119+func (s *Sheet) RowFormats() map[int]Format { return lineFormatsOf(s.lines.rows) }120+121+// ColStyles and RowStyles are the same for text styles.122+func (s *Sheet) ColStyles() map[int]Style { return lineStylesOf(s.lines.cols) }123+func (s *Sheet) RowStyles() map[int]Style { return lineStylesOf(s.lines.rows) }124+125+func lineFormatsOf(m map[int]lineFmt) map[int]Format {126+ out := map[int]Format{}127+ for n, l := range m {128+ if !l.Format.IsZero() {129+ out[n] = l.Format130+ }131+ }132+ return out133+}134+135+func lineStylesOf(m map[int]lineFmt) map[int]Style {136+ out := map[int]Style{}137+ for n, l := range m {138+ if !l.Style.IsZero() {139+ out[n] = l.Style140+ }141+ }142+ return out143+}144+145+// LoadLineFormat gives a whole column (row false) or row a format and146+// style, as an importer does, without recording undo or touching its147+// cells.148+func (s *Sheet) LoadLineFormat(row bool, n int, f Format, st Style) {149+ if row && n >= 0 && n < MaxRows || !row && n >= 0 && n < MaxCols {150+ s.setLine(row, n, lineFmt{f, st})151+ }152+}153+154+func (s *Sheet) lineRect(row bool, from, to int) Rect {155+ if row {156+ return rowRect(from, to)157+ }158+ return colRect(from, to)159+}160+161+// setLine sets a column's or row's formatting, recording it for undo and,162+// within a step, marking its cells changed, so formulas reading them infer163+// their format again.164+func (s *Sheet) setLine(row bool, n int, l lineFmt) {165+ m := &s.lines.cols166+ if row {167+ m = &s.lines.rows168+ }169+ if (*m)[n] == l {170+ return171+ }172+ if st := s.wb.hist.open; st != nil {173+ k := lineKey{s, row, n}174+ if _, seen := st.lines[k]; !seen {175+ st.lines[k] = (*m)[n]176+ }177+ }178+ if l.IsZero() {179+ delete(*m, n)180+ if len(*m) == 0 {181+ *m = nil182+ }183+ } else {184+ if *m == nil {185+ *m = map[int]lineFmt{}186+ }187+ (*m)[n] = l188+ }189+ s.version++190+ if s.wb.hist.open != nil {191+ s.wb.hist.dirty = append(s.wb.hist.dirty, s.lineCells(row, n)...)192+ }193+}194+195+// lineCells are the stored cells of a column or row.196+func (s *Sheet) lineCells(row bool, n int) []loc {197+ var out []loc198+ for _, a := range s.cellsIn(s.lineRect(row, n, n)) {199+ out = append(out, loc{s, a})200+ }201+ return out202+}203+204+// formatLines applies fn to the formatting of r, which is whole columns205+// or rows: to their line formats and to the cells in r that show206+// something else. It reports false, doing nothing, for any other range.207+func (s *Sheet) formatLines(r Rect, fn func(*lineFmt)) bool {208+ cols, rows := r.AllRows(), r.AllCols()209+ if !cols && !rows {210+ return false211+ }212+ // The cells that may show something other than their line: those213+ // stored, and where a formatted line crosses r the other way.214+ targets := s.cellsIn(r)215+ crossing, across := s.lines.rows, Rect{From: Addr{Col: r.From.Col}, To: Addr{Col: r.To.Col}}216+ if !cols {217+ crossing, across = s.lines.cols, Rect{From: Addr{Row: r.From.Row}, To: Addr{Row: r.To.Row}}218+ }219+ if !(cols && rows) {220+ for _, n := range slices.Sorted(maps.Keys(crossing)) {221+ if len(targets) > materializeLimit {222+ break223+ }224+ for i := range across.To.Col - across.From.Col + across.To.Row - across.From.Row + 1 {225+ a := Addr{Col: across.From.Col + i, Row: n} // a column's cells in formatted row n226+ if !cols {227+ a = Addr{Col: n, Row: across.From.Row + i} // a row's cells in formatted column n228+ }229+ if s.cells.get(a) == nil {230+ targets = append(targets, a)231+ }232+ }233+ }234+ }235+ want := make([]lineFmt, len(targets))236+ for i, a := range targets {237+ want[i] = s.effective(a)238+ fn(&want[i])239+ }240+ s.eachLine(r, fn)241+ for i, a := range targets {242+ s.placeFormat(a, want[i])243+ }244+ return true245+}246+247+// eachLine applies fn to the line formats of r's whole columns, or rows;248+// for the whole sheet, to every column and to the rows that have one.249+func (s *Sheet) eachLine(r Rect, fn func(*lineFmt)) {250+ apply := func(row bool, n int, l lineFmt) {251+ fn(&l)252+ s.setLine(row, n, l)253+ }254+ if r.AllRows() {255+ for c := r.From.Col; c <= r.To.Col; c++ {256+ apply(false, c, s.lines.cols[c])257+ }258+ if r.AllCols() {259+ for _, n := range slices.Sorted(maps.Keys(s.lines.rows)) {260+ apply(true, n, s.lines.rows[n])261+ }262+ }263+ return264+ }265+ for n := r.From.Row; n <= r.To.Row; n++ {266+ apply(true, n, s.lines.rows[n])267+ }268+}269+270+// placeFormat has the cell at a show want, giving it formatting of its271+// own only where its lines don't already show it, and placing a copy so272+// the change can be undone. A cell left with neither contents nor273+// formatting is removed.274+func (s *Sheet) placeFormat(a Addr, want lineFmt) {275+ old := s.cells.get(a)276+ c := old.clone()277+ if c == nil {278+ c = &Cell{}279+ }280+ c.Format, c.Style = own(want, s.inherited(a))281+ switch {282+ case c.Blank() && c.Format.IsZero() && c.Style.IsZero():283+ if old != nil {284+ s.place(a, nil)285+ }286+ case old == nil || c.Format != old.Format || c.Style != old.Style:287+ s.place(a, c)288+ }289+}290+291+// shiftLines moves column (or row) formats along with inserted or292+// deleted columns (rows).293+func (s *Sheet) shiftLines(rows bool, sp formula.Span) {294+ m := s.lines.cols295+ if rows {296+ m = s.lines.rows297+ }298+ next := map[int]lineFmt{}299+ for n, l := range m {300+ if to, ok := sp.Point(n); ok {301+ next[to] = l302+ }303+ }304+ for _, n := range slices.Collect(maps.Keys(m)) {305+ if _, ok := next[n]; !ok {306+ s.setLine(rows, n, lineFmt{})307+ }308+ }309+ for n, l := range next {310+ s.setLine(rows, n, l)311+ }312+}
A internal/sheet/lines_test.go +127 -0
@@ -0,0 +1,127 @@
1+package sheet2+3+import (4+ "bytes"5+ "testing"6+)7+8+func col(c int) Rect { return colRect(c, c) }9+10+// Formatting whole columns and rows keeps the format on the line: no11+// cells are made, every cell of the line shows it, and the latest12+// formatting wins where lines cross or a cell differs.13+func TestLineFormats(t *testing.T) {14+ s := New()15+ s.Set(at("B2"), "5")16+ s.SetFormat(at1("B2"), Preset(FmtPercent))17+ s.SetFormat(col(1), Preset(FmtCurrency))18+ if n := s.cells.len(); n != 1 {19+ t.Errorf("%d cells stored, want only B2", n)20+ }21+ for _, a := range []string{"B2", "B1", "B1048576"} {22+ if f := s.DisplayFormat(at(a)); f.Kind != FmtCurrency {23+ t.Errorf("%s shows %v, want currency", a, f.Kind)24+ }25+ }26+ if c := s.Cell(at("B2")); !c.Format.IsZero() {27+ t.Errorf("B2 keeps its own %v under the column's format", c.Format.Kind)28+ }29+30+ // Bold the column, then take the bold off one cell of it.31+ s.SetStyle(col(0), func(st *Style) { st.Bold = true })32+ s.SetStyle(at1("A5"), func(st *Style) { st.Bold = false })33+ if s.CellStyle(at("A5")).Bold || !s.CellStyle(at("A6")).Bold {34+ t.Errorf("A5 bold %v, A6 bold %v", s.CellStyle(at("A5")).Bold, s.CellStyle(at("A6")).Bold)35+ }36+ // Styling a row changes what its cells show, column formats included.37+ s.SetStyle(rowRect(2, 2), func(st *Style) { st.Italic = true })38+ if st := s.CellStyle(at("A3")); !st.Italic || !st.Bold {39+ t.Errorf("A3 = %+v, want the column's bold and the row's italic", st)40+ }41+ s.SetStyle(col(0), func(st *Style) { st.Underline = true })42+ if st := s.CellStyle(at("A3")); !st.Italic || !st.Underline {43+ t.Errorf("A3 after underlining column A = %+v", st)44+ }45+ if st := s.CellStyle(at("C3")); !st.Italic || st.Underline {46+ t.Errorf("C3 = %+v, want the row's italic only", st)47+ }48+49+ // Typing into a plain text column keeps the entry as text.50+ s.SetFormat(col(3), Format{Kind: FmtText})51+ s.Set(at("D7"), "0012")52+ if v := s.Value(at("D7")); v.Kind != Text || v.Str != "0012" {53+ t.Errorf("D7 = %+v, want the text 0012", v)54+ }55+56+ // Saved and loaded, the formats read the same.57+ var buf bytes.Buffer58+ if err := s.Write(&buf); err != nil {59+ t.Fatal(err)60+ }61+ back, err := Read(&buf)62+ if err != nil {63+ t.Fatalf("%v\n%s", err, buf.String())64+ }65+ for _, a := range []string{"A3", "A5", "A6", "C3", "B9", "D7", "Z1"} {66+ if got, want := back.effective(at(a)), s.effective(at(a)); got != want {67+ t.Errorf("%s after loading = %+v, want %+v", a, got, want)68+ }69+ if got, want := back.DisplayFormat(at(a)), s.DisplayFormat(at(a)); got != want {70+ t.Errorf("%s format after loading = %+v, want %+v", a, got, want)71+ }72+ }73+}74+75+func at1(a string) Rect { return Rect{From: at(a), To: at(a)} }76+77+// Line formats undo, redo, move with inserted columns and clear.78+func TestLineFormatsUndoAndInsert(t *testing.T) {79+ s := New()80+ s.Set(at("C4"), "7")81+ s.SetFormat(col(2), Preset(FmtCurrency))82+ s.Undo()83+ if f := s.DisplayFormat(at("C9")); !f.IsZero() {84+ t.Errorf("after undo C9 shows %v", f.Kind)85+ }86+ s.Redo()87+ if f := s.DisplayFormat(at("C9")); f.Kind != FmtCurrency {88+ t.Errorf("after redo C9 shows %v", f.Kind)89+ }90+ if err := s.InsertCols(0, 2); err != nil {91+ t.Fatal(err)92+ }93+ if f := s.DisplayFormat(at("E9")); f.Kind != FmtCurrency {94+ t.Errorf("after inserting two columns E9 shows %v", f.Kind)95+ }96+ if f := s.DisplayFormat(at("C9")); !f.IsZero() {97+ t.Errorf("after inserting two columns C9 shows %v", f.Kind)98+ }99+ s.ClearFormatting(col(4))100+ if len(s.lines.cols) != 0 || s.DisplayFormat(at("E4")).Kind == FmtCurrency {101+ t.Errorf("clearing column E left %v", s.lines.cols)102+ }103+}104+105+// Formatting the whole sheet formats each column once and makes no106+// cells.107+func TestWholeSheetFormat(t *testing.T) {108+ s := New()109+ s.Set(at("XFD1048576"), "1")110+ all := Rect{To: Addr{Col: MaxCols - 1, Row: MaxRows - 1}}111+ s.SetStyle(all, func(st *Style) { st.Bold = true })112+ if len(s.lines.cols) != MaxCols || s.cells.len() != 1 {113+ t.Errorf("%d column formats, %d cells", len(s.lines.cols), s.cells.len())114+ }115+ if !s.CellStyle(at("XFD1048576")).Bold || !s.CellStyle(at("Q77")).Bold {116+ t.Error("not bold everywhere")117+ }118+ var buf bytes.Buffer119+ s.Write(&buf)120+ if !bytes.Contains(buf.Bytes(), []byte(`"A:XFD": {"bold":true}`)) {121+ t.Errorf("the columns aren't written as one run:\n%.300s", buf.String())122+ }123+ s.ClearFormatting(all)124+ if len(s.lines.cols) != 0 {125+ t.Errorf("%d column formats after clearing", len(s.lines.cols))126+ }127+}
M internal/sheet/load.go +4 -0
M internal/sheet/nav.go +84 -23
@@ -3,32 +3,29 @@ package sheet
33 // UsedRange returns the smallest range from A1 covering every non-blank44 // cell, and false if the sheet is empty.55 func (s *Sheet) UsedRange() (Rect, bool) {6- var last Addr7- found := false8- for a, c := range s.cells.all() {9- if c.Blank() {10- continue11- }12- found = true13- last.Col = max(last.Col, a.Col)14- last.Row = max(last.Row, a.Row)15- }16- return Rect{To: last}, found6+ b, ok := s.cells.filled.bounds(Rect{To: Addr{Col: MaxCols - 1, Row: MaxRows - 1}})7+ return Rect{To: b.To}, ok178 }189 10+// FilledBounds returns the smallest range holding every non-blank cell11+// of r, and false if there is none.12+func (s *Sheet) FilledBounds(r Rect) (Rect, bool) { return s.cells.filled.bounds(r) }13+1914 // Edge returns where a data-edge jump (Ctrl+arrow in Excel, End+arrow in2015 // 1-2-3) from a in direction (dc, dr) lands. Inside a block of filled cells2116 // it stops at the block's last cell; from a blank cell, or at the end of a2217 // block, it goes to the next filled cell. With nothing filled ahead it2318 // stops at the edge of the worksheet. Rows the filter hides are skipped.19+// Blank stretches are jumped over through the index of filled cells, so20+// a jump costs what the block it walks holds, not the empty rows past it.2421 func (s *Sheet) Edge(a Addr, dc, dr int) Addr {25- filled := func(a Addr) bool { return !s.cells.get(a).Blank() }22+ filled := func(a Addr) bool { return s.cells.filled.has(a) }2623 step := func(a Addr) Addr {27- n := Addr{Col: a.Col + dc, Row: a.Row + dr}28- for dr != 0 && n.Valid() && s.RowHidden(n.Row) {29- n.Row += dr24+ if dr != 0 {25+ r, _ := s.NextShownRow(a.Row, dr) // off the sheet when there is none26+ return Addr{Col: a.Col, Row: r}3027 }31- return n28+ return Addr{Col: a.Col + dc, Row: a.Row}3229 }3330 next := step(a)3431 if !next.Valid() {
@@ -43,16 +40,80 @@ func (s *Sheet) Edge(a Addr, dc, dr int) Addr {
4340 next = n4441 }4542 }46- for {47- if filled(next) {48- return next43+ if f, ok := s.nextFilled(a, dc, dr); ok {44+ return f45+ }46+ return s.sheetEdge(a, dc, dr)47+}48+49+// nextFilled is the first filled cell past a in direction (dc, dr), on50+// a row the filter doesn't hide.51+func (s *Sheet) nextFilled(a Addr, dc, dr int) (Addr, bool) {52+ fl := &s.cells.filled53+ if dr != 0 {54+ for row := a.Row + dr; ; {55+ r, ok := fl.nextRow(a.Col, row, dr)56+ if !ok {57+ return Addr{}, false58+ }59+ if !s.RowHidden(r) {60+ return Addr{Col: a.Col, Row: r}, true61+ }62+ row = r + dr63+ }64+ }65+ limit := MaxCols - 166+ if dc < 0 {67+ limit = 068+ }69+ c, ok := s.NextFilledCol(a.Row, a.Col+dc, dc, limit)70+ return Addr{Col: c, Row: a.Row}, ok71+}72+73+// NextFilledCol returns the nearest column of row, from col on in74+// direction dir (1 or -1) and no further than limit, whose cell has75+// contents, looking only at the columns that hold any.76+func (s *Sheet) NextFilledCol(row, col, dir, limit int) (int, bool) {77+ fl := &s.cells.filled78+ lo, hi := col, limit79+ if dir < 0 {80+ lo, hi = limit, col81+ }82+ if lo > hi {83+ return 0, false84+ }85+ cols := fl.colsIn(max(lo, 0), min(hi, MaxCols-1))86+ for i := range cols {87+ c := cols[i]88+ if dir < 0 {89+ c = cols[len(cols)-1-i]90+ }91+ if fl.has(Addr{Col: c, Row: row}) {92+ return c, true93+ }94+ }95+ return 0, false96+}97+98+// sheetEdge is the last cell from a in direction (dc, dr): the sheet's99+// last (or first) column, or its last (or first) row the filter doesn't100+// hide, or a itself when every row past it is hidden.101+func (s *Sheet) sheetEdge(a Addr, dc, dr int) Addr {102+ switch {103+ case dc > 0:104+ a.Col = MaxCols - 1105+ case dc < 0:106+ a.Col = 0107+ case dr > 0:108+ if r, ok := s.NextShownRow(MaxRows, -1); ok && r > a.Row {109+ a.Row = r49110 }50- n := step(next)51- if !n.Valid() {52- return next111+ default:112+ if r, ok := s.NextShownRow(-1, 1); ok && r < a.Row {113+ a.Row = r53114 }54- next = n55115 }116+ return a56117 }57118 58119 // Stats summarizes the values in r, as shown in the status line for a
M internal/sheet/nav_test.go +5 -5
A internal/sheet/occupancy.go +266 -0
@@ -0,0 +1,266 @@
1+package sheet2+3+import (4+ "math/bits"5+ "slices"6+)7+8+// occupancy records which rows of each column hold a cell, as bitmaps of9+// blockRows rows, so the cells of a range are found in row order at the10+// cost of what it holds, not its area: a whole column of a million rows11+// with ten cells is ten visits.12+type occupancy struct {13+ cols map[int]*colIndex14+ colIDs []int // the columns holding cells, ascending15+}16+17+// Rows of a column are indexed in blocks of blockRows, one bit each.18+const (19+ blockShift = 1020+ blockRows = 1 << blockShift21+ blockWords = blockRows / 6422+)23+24+type rowBlock struct {25+ bits [blockWords]uint6426+ n int27+}28+29+// colIndex is the occupancy of one column: its blocks by number, and the30+// numbers in order.31+type colIndex struct {32+ blocks map[int]*rowBlock33+ ids []int34+ n int35+}36+37+// has reports whether a holds a cell.38+func (o *occupancy) has(a Addr) bool {39+ ci := o.cols[a.Col]40+ if ci == nil {41+ return false42+ }43+ b := ci.blocks[a.Row>>blockShift]44+ bit := a.Row & (blockRows - 1)45+ return b != nil && b.bits[bit>>6]&(1<<(bit&63)) != 046+}47+48+// mark records a cell at a, which had none.49+func (o *occupancy) mark(a Addr) {50+ ci := o.cols[a.Col]51+ if ci == nil {52+ ci = &colIndex{blocks: make(map[int]*rowBlock)}53+ if o.cols == nil {54+ o.cols = make(map[int]*colIndex)55+ }56+ o.cols[a.Col] = ci57+ o.colIDs = insertSorted(o.colIDs, a.Col)58+ }59+ id := a.Row >> blockShift60+ b := ci.blocks[id]61+ if b == nil {62+ b = &rowBlock{}63+ ci.blocks[id] = b64+ ci.ids = insertSorted(ci.ids, id)65+ }66+ bit := a.Row & (blockRows - 1)67+ b.bits[bit>>6] |= 1 << (bit & 63)68+ b.n++69+ ci.n++70+}71+72+// unmark records that a no longer holds a cell.73+func (o *occupancy) unmark(a Addr) {74+ ci := o.cols[a.Col]75+ id := a.Row >> blockShift76+ b := ci.blocks[id]77+ bit := a.Row & (blockRows - 1)78+ b.bits[bit>>6] &^= 1 << (bit & 63)79+ b.n--80+ ci.n--81+ if b.n == 0 {82+ delete(ci.blocks, id)83+ ci.ids = removeSorted(ci.ids, id)84+ }85+ if ci.n == 0 {86+ delete(o.cols, a.Col)87+ o.colIDs = removeSorted(o.colIDs, a.Col)88+ }89+}90+91+func insertSorted(s []int, v int) []int {92+ i, found := slices.BinarySearch(s, v)93+ if found {94+ return s95+ }96+ return slices.Insert(s, i, v)97+}98+99+func removeSorted(s []int, v int) []int {100+ if i, found := slices.BinarySearch(s, v); found {101+ return slices.Delete(s, i, i+1)102+ }103+ return s104+}105+106+// colsIn returns the stored columns from c0 to c1, ascending. The slice107+// is shared: callers mustn't keep it across changes.108+func (o *occupancy) colsIn(c0, c1 int) []int {109+ i, _ := slices.BinarySearch(o.colIDs, c0)110+ j, _ := slices.BinarySearch(o.colIDs, c1+1)111+ return o.colIDs[i:j]112+}113+114+// colCells is the number of cells stored in column c.115+func (o *occupancy) colCells(c int) int {116+ if ci := o.cols[c]; ci != nil {117+ return ci.n118+ }119+ return 0120+}121+122+// blockIDs returns the numbers from b0 to b1 of the blocks that any of123+// cols has, ascending.124+func (o *occupancy) blockIDs(cols []int, b0, b1 int) []int {125+ if len(cols) == 1 {126+ ids := o.cols[cols[0]].ids127+ i, _ := slices.BinarySearch(ids, b0)128+ j, _ := slices.BinarySearch(ids, b1+1)129+ return slices.Clone(ids[i:j])130+ }131+ var out []int132+ for _, c := range cols {133+ ids := o.cols[c].ids134+ i, _ := slices.BinarySearch(ids, b0)135+ j, _ := slices.BinarySearch(ids, b1+1)136+ out = append(out, ids[i:j]...)137+ }138+ slices.Sort(out)139+ return slices.Compact(out)140+}141+142+// wordMask keeps the bits of word w that fall in rows lo..hi of a block.143+func wordMask(w, lo, hi int) uint64 {144+ m := ^uint64(0)145+ if first := w << 6; lo > first {146+ m &= ^uint64(0) << (lo - first)147+ }148+ if last := w<<6 + 63; hi < last {149+ m &= ^uint64(0) >> (last - hi)150+ }151+ return m152+}153+154+// colScan calls fn with each row from r0 to r1 of column c that holds a155+// cell, in order, until fn returns false, and reports whether it went to156+// the end. It allocates nothing; the column's cells must not be added157+// or removed meanwhile.158+func (o *occupancy) colScan(c, r0, r1 int, fn func(row int) bool) bool {159+ ci := o.cols[c]160+ if ci == nil || r0 > r1 {161+ return true162+ }163+ i, _ := slices.BinarySearch(ci.ids, r0>>blockShift)164+ for ; i < len(ci.ids) && ci.ids[i] <= r1>>blockShift; i++ {165+ id := ci.ids[i]166+ b, base := ci.blocks[id], id<<blockShift167+ lo, hi := max(r0-base, 0), min(r1-base, blockRows-1)168+ for w := lo >> 6; w <= hi>>6; w++ {169+ x := b.bits[w] & wordMask(w, lo, hi)170+ for x != 0 {171+ bit := bits.TrailingZeros64(x)172+ x &^= 1 << bit173+ if !fn(base + w<<6 + bit) {174+ return false175+ }176+ }177+ }178+ }179+ return true180+}181+182+// nextRow returns the first row of column c at or after row (dir 1), or183+// at or before it (dir -1), that holds a cell.184+func (o *occupancy) nextRow(c, row, dir int) (int, bool) {185+ ci := o.cols[c]186+ if ci == nil || row < 0 {187+ return 0, false188+ }189+ id := row >> blockShift190+ i, _ := slices.BinarySearch(ci.ids, id)191+ if dir > 0 {192+ for ; i < len(ci.ids); i++ {193+ b := ci.ids[i]194+ from := 0195+ if b == id {196+ from = row & (blockRows - 1)197+ }198+ if r, ok := ci.blocks[b].first(from); ok {199+ return b<<blockShift + r, true200+ }201+ }202+ return 0, false203+ }204+ if i == len(ci.ids) || ci.ids[i] != id {205+ i--206+ }207+ for ; i >= 0; i-- {208+ b := ci.ids[i]209+ to := blockRows - 1210+ if b == id {211+ to = row & (blockRows - 1)212+ }213+ if r, ok := ci.blocks[b].last(to); ok {214+ return b<<blockShift + r, true215+ }216+ }217+ return 0, false218+}219+220+// first is the lowest set row at or after from.221+func (b *rowBlock) first(from int) (int, bool) {222+ for w := from >> 6; w < blockWords; w++ {223+ x := b.bits[w]224+ if w == from>>6 {225+ x &= ^uint64(0) << (from & 63)226+ }227+ if x != 0 {228+ return w<<6 + bits.TrailingZeros64(x), true229+ }230+ }231+ return 0, false232+}233+234+// last is the highest set row at or before to.235+func (b *rowBlock) last(to int) (int, bool) {236+ for w := to >> 6; w >= 0; w-- {237+ x := b.bits[w]238+ if w == to>>6 {239+ x &= ^uint64(0) >> (63 - to&63)240+ }241+ if x != 0 {242+ return w<<6 + 63 - bits.LeadingZeros64(x), true243+ }244+ }245+ return 0, false246+}247+248+// bounds returns the smallest range holding every stored cell of r, and249+// false if r holds none. It costs a few steps per stored column of r.250+func (o *occupancy) bounds(r Rect) (Rect, bool) {251+ out, found := Rect{}, false252+ for _, c := range o.colsIn(r.From.Col, r.To.Col) {253+ top, ok := o.nextRow(c, r.From.Row, 1)254+ if !ok || top > r.To.Row {255+ continue256+ }257+ bottom, _ := o.nextRow(c, r.To.Row, -1)258+ if !found {259+ out, found = Rect{From: Addr{Col: c, Row: top}, To: Addr{Col: c, Row: bottom}}, true260+ continue261+ }262+ out.From.Row, out.To.Row = min(out.From.Row, top), max(out.To.Row, bottom)263+ out.To.Col = c264+ }265+ return out, found266+}
M internal/sheet/ops.go +54 -14
@@ -4,6 +4,7 @@ import (
44 "errors"55 "fmt"66 "strconv"7+ "sync/atomic"78 89 "github.com/FelineStateMachine/012/internal/formula"910 )
@@ -12,12 +13,31 @@ import (
1213 var (1314 ErrPushedOff = errors.New("There's data at the edge of the sheet that would be pushed off")1415 ErrPasteEdge = errors.New("The paste doesn't fit: it would go past the edge of the sheet")15- ErrFillTooBig = errors.New("The selection is too large to fill")16+ ErrFillTooBig = errors.New("That would write more cells than max-cells allows (see File > Settings)")1617 )1718 18-// maxFill caps how many cells a paste or fill writes at once, so an19-// accidental whole-sheet selection can't stall the program.20-const maxFill = 1 << 2019+// DefaultMaxCells is the max-cells setting's default: about 600 MB of20+// cells at 300 bytes each.21+const DefaultMaxCells = 2_000_00022+23+// maxCells caps how many cells a paste or fill writes at once, and how24+// many an import keeps, so an accidental whole-sheet selection or a huge25+// file can't stall the program or exhaust memory. It is the max-cells26+// setting of the config file, process-wide like the rest of it.27+var maxCells atomic.Int6428+29+func init() { maxCells.Store(DefaultMaxCells) }30+31+// SetMaxCells sets the cell budget; n < 1 restores the default.32+func SetMaxCells(n int) {33+ if n < 1 {34+ n = DefaultMaxCells35+ }36+ maxCells.Store(int64(n))37+}38+39+// MaxCells returns the cell budget.40+func MaxCells() int { return int(maxCells.Load()) }2141 2242 // remap moves every stored cell with cell (dropping those it rejects) and2343 // rewrites every reference to this sheet's cells, here and on other
@@ -123,6 +143,7 @@ func (s *Sheet) restructure(rows bool, sp formula.Span) {
123143 if !rows {124144 s.shiftWidths(sp)125145 }146+ s.shiftLines(rows, sp)126147 s.shiftView(rows, sp)127148 s.shiftPivots(rows, sp)128149 })
@@ -169,9 +190,15 @@ func (c *Clip) Size() (cols, rows int) {
169190 }170191 171192 // Text returns the clip's values as displayed in General format, row by172-// row, for the system clipboard.193+// row, for the system clipboard. Blank rows and columns past the last194+// value are left off, so copying whole columns gives their data.173195 func (c *Clip) Text() [][]string {174- cols, rows := c.Size()196+ cols, rows := 0, 0197+ for off, cell := range c.cells {198+ if !cell.Blank() {199+ cols, rows = max(cols, off.Col+1), max(rows, off.Row+1)200+ }201+ }175202 out := make([][]string, rows)176203 for r := range out {177204 out[r] = make([]string, cols)
@@ -199,20 +226,33 @@ func (s *Sheet) Paste(c *Clip, dst Rect, values bool) (Rect, error) {
199226 if !dst.To.Valid() {200227 return Rect{}, ErrPasteEdge201228 }202- if w*h > maxFill {229+ tilesAcross, tilesDown := w/cols, h/rows230+ if len(c.cells)*tilesAcross*tilesDown > MaxCells() {203231 return Rect{}, ErrFillTooBig204232 }205233 label := "paste into " + dst.String()206234 if values {207235 label = "paste values into " + dst.String()208236 }237+ // Blank parts of the clip clear what they land on, and each of its238+ // cells is placed in every tile: the cost is the cells written and239+ // cleared, not dst's area.240+ offset := func(a Addr) Addr {241+ return Addr{Col: (a.Col - dst.From.Col) % cols, Row: (a.Row - dst.From.Row) % rows}242+ }209243 s.change(label, dst, func() {210- for r := range h {211- for col := range w {212- off := Addr{Col: col % cols, Row: r % rows}213- to := Addr{Col: dst.From.Col + col, Row: dst.From.Row + r}214- from := Addr{Col: c.Src.From.Col + off.Col, Row: c.Src.From.Row + off.Row}215- s.pasteCell(to, c.cells[off], to.Col-from.Col, to.Row-from.Row, values)244+ for _, a := range s.cellsIn(dst) {245+ if c.cells[offset(a)] == nil {246+ s.place(a, nil)247+ }248+ }249+ for off, cell := range c.cells {250+ from := Addr{Col: c.Src.From.Col + off.Col, Row: c.Src.From.Row + off.Row}251+ for tr := range tilesDown {252+ for tc := range tilesAcross {253+ to := Addr{Col: dst.From.Col + tc*cols + off.Col, Row: dst.From.Row + tr*rows + off.Row}254+ s.pasteCell(to, cell, to.Col-from.Col, to.Row-from.Row, values)255+ }216256 }217257 }218258 })
@@ -294,7 +334,7 @@ func (s *Sheet) FillRight(r Rect) (Rect, error) {
294334 // origin and copied to each cell, adjusting references (Ctrl+Enter).295335 func (s *Sheet) FillEntry(r Rect, origin Addr, input string) error {296336 w, h := r.To.Col-r.From.Col+1, r.To.Row-r.From.Row+1297- if w*h > maxFill {337+ if w*h > MaxCells() {298338 return ErrFillTooBig299339 }300340 if _, _, err := classify(input); err != nil {
M internal/sheet/pivot.go +1 -1
M internal/sheet/pivot_stress_test.go +2 -2
M internal/sheet/pivotcalc.go +5 -11
M internal/sheet/refs_test.go +5 -5
M internal/sheet/series.go +28 -11
@@ -23,23 +23,20 @@ import (
2323 // is continued on its own. It returns the range filled, as one undo step.2424 func (s *Sheet) FillSeries(src, dst Rect) (Rect, error) {2525 w, h := dst.To.Col-dst.From.Col+1, dst.To.Row-dst.From.Row+126- if w*h > maxFill {26+ if w*h > MaxCells() {2727 return Rect{}, ErrFillTooBig2828 }2929 vertical := dst.From.Col == src.From.Col && dst.To.Col == src.To.Col3030 if !vertical && (dst.From.Row != src.From.Row || dst.To.Row != src.To.Row) || dst == src {3131 return src, nil3232 }33- var lanes [][2][]Addr // per lane: the source cells in fill order, then the targets33+ var lanes []fillLane3434 lane := func(from, step Addr, n int, to Addr, m int) {35- var seq, targets []Addr35+ var seq []Addr3636 for i := range n {3737 seq = append(seq, Addr{Col: from.Col + step.Col*i, Row: from.Row + step.Row*i})3838 }39- for i := range m {40- targets = append(targets, Addr{Col: to.Col + step.Col*i, Row: to.Row + step.Row*i})41- }42- lanes = append(lanes, [2][]Addr{seq, targets})39+ lanes = append(lanes, fillLane{seq, to, step, m})4340 }4441 n := src.To.Row - src.From.Row + 14542 if !vertical {
@@ -65,20 +62,40 @@ func (s *Sheet) FillSeries(src, dst Rect) (Rect, error) {
6562 }6663 s.change("fill "+dst.String(), dst, func() {6764 for _, l := range lanes {68- s.fillLane(l[0], l[1])65+ s.fill(l)6966 }7067 })7168 return dst, nil7269 }7370 74-// fillLane continues the cells at seq into targets.75-func (s *Sheet) fillLane(seq, targets []Addr) {71+// fillLane is one column (or row) of a fill: the source cells in fill72+// order, and n targets from to on by step.73+type fillLane struct {74+ seq []Addr75+ to, step Addr76+ n int77+}78+79+// fill continues the cells of a lane into its targets. A lane of blank80+// cells only clears what its targets hold.81+func (s *Sheet) fill(l fillLane) {82+ seq := l.seq7683 cells := make([]*Cell, len(seq))84+ blank := true7785 for i, a := range seq {7886 cells[i] = s.cells.get(a)87+ blank = blank && cells[i] == nil88+ }89+ if blank {90+ last := Addr{Col: l.to.Col + l.step.Col*(l.n-1), Row: l.to.Row + l.step.Row*(l.n-1)}91+ for _, a := range s.cellsIn(NewRect(l.to, last)) {92+ s.place(a, nil)93+ }94+ return7995 }8096 next := detectSeries(cells)81- for i, to := range targets {97+ for i := range l.n {98+ to := Addr{Col: l.to.Col + l.step.Col*i, Row: l.to.Row + l.step.Row*i}8299 k := i % len(seq)83100 c := cells[k]84101 switch {
M internal/sheet/sheet.go +4 -0
M internal/sheet/store.go +31 -248
@@ -13,10 +13,10 @@ import (
1313 // (to column blocks of compact values, with formulas and formats in side1414 // tables; see docs/limits.md) without touching it.1515 //16-// Next to the map, an occupancy index records which rows of each column17-// hold a cell, as bitmaps of 1024 rows, so reading a range costs what the18-// range holds rather than its area: a whole column of a million rows with19-// ten cells is ten visits, in row order.16+// Next to the map, two occupancy indexes (occupancy.go) record which17+// cells are stored and which have contents, so a range is read at the18+// cost of what it holds, and the used range and data edges are found19+// without scanning.2020 //2121 // Iteration order of all is unspecified; inRange yields in row-major2222 // order. Cells may be set or deleted while iterating, with a map's rules:
@@ -24,141 +24,70 @@ import (
2424 // be.2525 type cellStore struct {2626 m map[Addr]*Cell27- cols map[int]*colIndex28- colIDs []int // the columns holding cells, ascending27+ stored occupancy // every cell in m28+ filled occupancy // the cells with contents (not Blank)2929 }3030 31-// Rows of a column are indexed in blocks of blockRows, one bit each.32-const (33- blockShift = 1034- blockRows = 1 << blockShift35- blockWords = blockRows / 6436-)37-38-type rowBlock struct {39- bits [blockWords]uint6440- n int41-}42-43-// colIndex is the occupancy of one column: its blocks by number, and the44-// numbers in order.45-type colIndex struct {46- blocks map[int]*rowBlock47- ids []int48- n int49-}50-51-func newCellStore() cellStore {52- return cellStore{m: make(map[Addr]*Cell), cols: make(map[int]*colIndex)}53-}31+func newCellStore() cellStore { return cellStore{m: make(map[Addr]*Cell)} }5432 5533 // get returns the cell at a, or nil.5634 func (st *cellStore) get(a Addr) *Cell { return st.m[a] }5735 5836 // set stores c, which isn't nil, at a.5937 func (st *cellStore) set(a Addr, c *Cell) {60- if _, ok := st.m[a]; !ok {61- st.mark(a)38+ old, ok := st.m[a]39+ if !ok {40+ st.stored.mark(a)41+ }42+ switch was, is := ok && !old.Blank(), !c.Blank(); {43+ case is && !was:44+ st.filled.mark(a)45+ case was && !is:46+ st.filled.unmark(a)6247 }6348 st.m[a] = c6449 }6550 6651 // delete removes the cell at a, if any.6752 func (st *cellStore) delete(a Addr) {68- if _, ok := st.m[a]; ok {69- st.unmark(a)53+ if old, ok := st.m[a]; ok {54+ st.stored.unmark(a)55+ if !old.Blank() {56+ st.filled.unmark(a)57+ }7058 delete(st.m, a)7159 }7260 }7361 74-func (st *cellStore) mark(a Addr) {75- ci := st.cols[a.Col]76- if ci == nil {77- ci = &colIndex{blocks: make(map[int]*rowBlock)}78- st.cols[a.Col] = ci79- st.colIDs = insertSorted(st.colIDs, a.Col)80- }81- id := a.Row >> blockShift82- b := ci.blocks[id]83- if b == nil {84- b = &rowBlock{}85- ci.blocks[id] = b86- ci.ids = insertSorted(ci.ids, id)87- }88- bit := a.Row & (blockRows - 1)89- b.bits[bit>>6] |= 1 << (bit & 63)90- b.n++91- ci.n++92-}93-94-func (st *cellStore) unmark(a Addr) {95- ci := st.cols[a.Col]96- id := a.Row >> blockShift97- b := ci.blocks[id]98- bit := a.Row & (blockRows - 1)99- b.bits[bit>>6] &^= 1 << (bit & 63)100- b.n--101- ci.n--102- if b.n == 0 {103- delete(ci.blocks, id)104- ci.ids = removeSorted(ci.ids, id)105- }106- if ci.n == 0 {107- delete(st.cols, a.Col)108- st.colIDs = removeSorted(st.colIDs, a.Col)109- }110-}111-112-func insertSorted(s []int, v int) []int {113- i, found := slices.BinarySearch(s, v)114- if found {115- return s116- }117- return slices.Insert(s, i, v)118-}119-120-func removeSorted(s []int, v int) []int {121- if i, found := slices.BinarySearch(s, v); found {122- return slices.Delete(s, i, i+1)123- }124- return s125-}126-12762 // len is the number of cells stored, formatting-only ones included.12863 func (st *cellStore) len() int { return len(st.m) }12964 13065 // all yields every stored cell.13166 func (st *cellStore) all() iter.Seq2[Addr, *Cell] { return maps.All(st.m) }13267 133-// colsIn returns the stored columns from c0 to c1, ascending. The slice134-// is shared: callers mustn't keep it across changes.135-func (st *cellStore) colsIn(c0, c1 int) []int {136- i, _ := slices.BinarySearch(st.colIDs, c0)137- j, _ := slices.BinarySearch(st.colIDs, c1+1)138- return st.colIDs[i:j]139-}68+// bounds returns the smallest range holding every stored cell of r, and69+// false if r holds none.70+func (st *cellStore) bounds(r Rect) (Rect, bool) { return st.stored.bounds(r) }14071 141-// colCells is the number of cells stored in column c.142-func (st *cellStore) colCells(c int) int {143- if ci := st.cols[c]; ci != nil {144- return ci.n145- }146- return 072+// colScan calls fn with each row from r0 to r1 of column c that holds a73+// cell; see occupancy.colScan.74+func (st *cellStore) colScan(c, r0, r1 int, fn func(row int) bool) bool {75+ return st.stored.colScan(c, r0, r1, fn)14776 }14877 14978 // inRange yields the stored cells in r in row-major order, visiting only15079 // the rows of each column that hold cells.15180 func (st *cellStore) inRange(r Rect) iter.Seq2[Addr, *Cell] {15281 return func(yield func(Addr, *Cell) bool) {153- cols := slices.Clone(st.colsIn(r.From.Col, r.To.Col))82+ cols := slices.Clone(st.stored.colsIn(r.From.Col, r.To.Col))15483 if len(cols) == 0 || r.From.Row > r.To.Row {15584 return15685 }15786 b0, b1 := r.From.Row>>blockShift, r.To.Row>>blockShift158- for _, id := range st.blockIDs(cols, b0, b1) {87+ for _, id := range st.stored.blockIDs(cols, b0, b1) {15988 var in []colBlock16089 for _, c := range cols {161- if b := st.cols[c].blocks[id]; b != nil {90+ if b := st.stored.cols[c].blocks[id]; b != nil {16291 in = append(in, colBlock{c, b})16392 }16493 }
@@ -174,26 +103,6 @@ type colBlock struct {
174103 b *rowBlock175104 }176105 177-// blockIDs returns the numbers from b0 to b1 of the blocks that any of178-// cols has, ascending.179-func (st *cellStore) blockIDs(cols []int, b0, b1 int) []int {180- if len(cols) == 1 {181- ids := st.cols[cols[0]].ids182- i, _ := slices.BinarySearch(ids, b0)183- j, _ := slices.BinarySearch(ids, b1+1)184- return slices.Clone(ids[i:j])185- }186- var out []int187- for _, c := range cols {188- ids := st.cols[c].ids189- i, _ := slices.BinarySearch(ids, b0)190- j, _ := slices.BinarySearch(ids, b1+1)191- out = append(out, ids[i:j]...)192- }193- slices.Sort(out)194- return slices.Compact(out)195-}196-197106 // yieldBlock yields the cells of block id in the columns in, row by row,198107 // limited to the rows of r.199108 func (st *cellStore) yieldBlock(in []colBlock, id int, r Rect, yield func(Addr, *Cell) bool) bool {
@@ -222,129 +131,3 @@ func (st *cellStore) yieldBlock(in []colBlock, id int, r Rect, yield func(Addr,
222131 }223132 return true224133 }225-226-// wordMask keeps the bits of word w that fall in rows lo..hi of a block.227-func wordMask(w, lo, hi int) uint64 {228- m := ^uint64(0)229- if first := w << 6; lo > first {230- m &= ^uint64(0) << (lo - first)231- }232- if last := w<<6 + 63; hi < last {233- m &= ^uint64(0) >> (last - hi)234- }235- return m236-}237-238-// colScan calls fn with each row from r0 to r1 of column c that holds a239-// cell, in order, until fn returns false, and reports whether it went to240-// the end. It allocates nothing; the column's cells must not be added241-// or removed meanwhile.242-func (st *cellStore) colScan(c, r0, r1 int, fn func(row int) bool) bool {243- ci := st.cols[c]244- if ci == nil || r0 > r1 {245- return true246- }247- i, _ := slices.BinarySearch(ci.ids, r0>>blockShift)248- for ; i < len(ci.ids) && ci.ids[i] <= r1>>blockShift; i++ {249- id := ci.ids[i]250- b, base := ci.blocks[id], id<<blockShift251- lo, hi := max(r0-base, 0), min(r1-base, blockRows-1)252- for w := lo >> 6; w <= hi>>6; w++ {253- x := b.bits[w] & wordMask(w, lo, hi)254- for x != 0 {255- bit := bits.TrailingZeros64(x)256- x &^= 1 << bit257- if !fn(base + w<<6 + bit) {258- return false259- }260- }261- }262- }263- return true264-}265-266-// nextRow returns the first row of column c at or after row (dir 1), or267-// at or before it (dir -1), that holds a cell.268-func (st *cellStore) nextRow(c, row, dir int) (int, bool) {269- ci := st.cols[c]270- if ci == nil || row < 0 {271- return 0, false272- }273- id := row >> blockShift274- i, _ := slices.BinarySearch(ci.ids, id)275- if dir > 0 {276- for ; i < len(ci.ids); i++ {277- b := ci.ids[i]278- from := 0279- if b == id {280- from = row & (blockRows - 1)281- }282- if r, ok := ci.blocks[b].first(from); ok {283- return b<<blockShift + r, true284- }285- }286- return 0, false287- }288- if i == len(ci.ids) || ci.ids[i] != id {289- i--290- }291- for ; i >= 0; i-- {292- b := ci.ids[i]293- to := blockRows - 1294- if b == id {295- to = row & (blockRows - 1)296- }297- if r, ok := ci.blocks[b].last(to); ok {298- return b<<blockShift + r, true299- }300- }301- return 0, false302-}303-304-// first is the lowest set row at or after from.305-func (b *rowBlock) first(from int) (int, bool) {306- for w := from >> 6; w < blockWords; w++ {307- x := b.bits[w]308- if w == from>>6 {309- x &= ^uint64(0) << (from & 63)310- }311- if x != 0 {312- return w<<6 + bits.TrailingZeros64(x), true313- }314- }315- return 0, false316-}317-318-// last is the highest set row at or before to.319-func (b *rowBlock) last(to int) (int, bool) {320- for w := to >> 6; w >= 0; w-- {321- x := b.bits[w]322- if w == to>>6 {323- x &= ^uint64(0) >> (63 - to&63)324- }325- if x != 0 {326- return w<<6 + 63 - bits.LeadingZeros64(x), true327- }328- }329- return 0, false330-}331-332-// bounds returns the smallest range holding every stored cell of r, and333-// false if r holds none. It costs a few steps per stored column of r.334-func (st *cellStore) bounds(r Rect) (Rect, bool) {335- out, found := Rect{}, false336- for _, c := range st.colsIn(r.From.Col, r.To.Col) {337- top, ok := st.nextRow(c, r.From.Row, 1)338- if !ok || top > r.To.Row {339- continue340- }341- bottom, _ := st.nextRow(c, r.To.Row, -1)342- if !found {343- out, found = Rect{From: Addr{Col: c, Row: top}, To: Addr{Col: c, Row: bottom}}, true344- continue345- }346- out.From.Row, out.To.Row = min(out.From.Row, top), max(out.To.Row, bottom)347- out.To.Col = c348- }349- return out, found350-}
M internal/sheet/store_test.go +21 -5
@@ -20,7 +20,11 @@ func TestCellStoreIndex(t *testing.T) {
2020 if rng.IntN(3) == 0 {2121 st.delete(a)2222 } else {23- st.set(a, &Cell{Input: "x"})23+ input := "x"24+ if rng.IntN(4) == 0 {25+ input = "" // formatting only26+ }27+ st.set(a, &Cell{Input: input})2428 }2529 if i%100 == 0 {2630 checkStore(t, &st, rng)
@@ -29,8 +33,8 @@ func TestCellStoreIndex(t *testing.T) {
2933 for a := range st.all() {3034 st.delete(a)3135 }32- if len(st.cols) != 0 || len(st.colIDs) != 0 {33- t.Errorf("empty store keeps %d columns", len(st.cols))36+ if len(st.stored.cols) != 0 || len(st.stored.colIDs) != 0 || len(st.filled.cols) != 0 {37+ t.Errorf("empty store keeps %d columns", len(st.stored.cols))3438 }3539 }3640
@@ -65,10 +69,22 @@ func checkStore(t *testing.T, st *cellStore, rng *rand.Rand) {
6569 !slices.ContainsFunc(want, func(a Addr) bool { return a.Col == b.To.Col })) {6670 t.Fatalf("bounds(%v) = %v isn't tight", r, b)6771 }72+ var filled []Addr73+ for a, c := range st.all() {74+ if r.Contains(a) && !c.Blank() {75+ filled = append(filled, a)76+ }77+ if st.filled.has(a) == c.Blank() {78+ t.Fatalf("filled index says %v for %v", st.filled.has(a), a)79+ }80+ }81+ if fb, ok := st.filled.bounds(r); ok != (len(filled) > 0) || ok && !slices.ContainsFunc(filled, func(a Addr) bool { return a.Row == fb.To.Row }) {82+ t.Fatalf("filled bounds(%v) = %v %v", r, fb, ok)83+ }6884 c, row := rng.IntN(40), rng.IntN(MaxRows)69- next, ok := st.nextRow(c, row, 1)85+ next, ok := st.stored.nextRow(c, row, 1)7086 want1, ok1 := MaxRows, false71- prev, okp := st.nextRow(c, row, -1)87+ prev, okp := st.stored.nextRow(c, row, -1)7288 want2, ok2 := -1, false7389 for a := range st.all() {7490 if a.Col == c && a.Row >= row && a.Row < want1 {
M internal/sheet/stress_test.go +18 -18
@@ -73,8 +73,8 @@ func BenchmarkUndo(b *testing.B) {
7373 // BenchmarkBigUndo clears a dense 8192x26 block and undoes it: the7474 // largest single step a user makes by hand.7575 func BenchmarkBigUndo(b *testing.B) {76- s := stress.Dense(sheet.MaxRows, 26)77- all := sheet.NewRect(sheet.Addr{}, sheet.Addr{Col: 25, Row: sheet.MaxRows - 1})76+ s := stress.Dense(stress.Rows, 26)77+ all := sheet.NewRect(sheet.Addr{}, sheet.Addr{Col: 25, Row: stress.Rows - 1})7878 for b.Loop() {7979 s.EraseRange(all)8080 s.Undo()
@@ -85,10 +85,10 @@ func BenchmarkBigUndo(b *testing.B) {
8585 // history keeps, and reports the heap they hold.8686 func BenchmarkHistoryFull(b *testing.B) {8787 for b.Loop() {88- s := stress.Dense(sheet.MaxRows, 4)88+ s := stress.Dense(stress.Rows, 4)8989 before := heap()9090 for i := range sheet.MaxUndo {91- col := sheet.NewRect(sheet.Addr{Col: i % 4}, sheet.Addr{Col: i % 4, Row: sheet.MaxRows - 1})91+ col := sheet.NewRect(sheet.Addr{Col: i % 4}, sheet.Addr{Col: i % 4, Row: stress.Rows - 1})9292 if err := s.FillEntry(col, col.From, fmt.Sprint(i)); err != nil {9393 b.Fatal(err)9494 }
@@ -103,9 +103,9 @@ func BenchmarkHistoryFull(b *testing.B) {
103103 // 8192x26 block, more than the history's byte budget holds, and reports104104 // the heap the history keeps.105105 func BenchmarkHistoryWide(b *testing.B) {106- all := sheet.NewRect(sheet.Addr{}, sheet.Addr{Col: 25, Row: sheet.MaxRows - 1})106+ all := sheet.NewRect(sheet.Addr{}, sheet.Addr{Col: 25, Row: stress.Rows - 1})107107 for b.Loop() {108- s := stress.Dense(sheet.MaxRows, 26)108+ s := stress.Dense(stress.Rows, 26)109109 before := heap()110110 for i := range 12 {111111 if err := s.FillEntry(all, all.From, fmt.Sprint(i)); err != nil {
@@ -135,8 +135,8 @@ func BenchmarkNested(b *testing.B) {
135135 }136136 137137 func BenchmarkSort(b *testing.B) {138- s := stress.Table(sheet.MaxRows-1, 8)139- r := sheet.NewRect(sheet.Addr{Row: 1}, sheet.Addr{Col: 7, Row: sheet.MaxRows - 1})138+ s := stress.Table(stress.Rows-1, 8)139+ r := sheet.NewRect(sheet.Addr{Row: 1}, sheet.Addr{Col: 7, Row: stress.Rows - 1})140140 desc := false141141 for b.Loop() {142142 s.SortRange(r, []sheet.SortKey{{Col: 2, Desc: desc}})
@@ -145,8 +145,8 @@ func BenchmarkSort(b *testing.B) {
145145 }146146 147147 func BenchmarkFilter(b *testing.B) {148- s := stress.Table(sheet.MaxRows-1, 8)149- s.CreateFilter(sheet.NewRect(sheet.Addr{}, sheet.Addr{Col: 7, Row: sheet.MaxRows - 1}))148+ s := stress.Table(stress.Rows-1, 8)149+ s.CreateFilter(sheet.NewRect(sheet.Addr{}, sheet.Addr{Col: 7, Row: stress.Rows - 1}))150150 b.Run("apply", func(b *testing.B) {151151 arg := [2]string{"5000", "2500"}152152 i := 0
@@ -164,7 +164,7 @@ func BenchmarkFilter(b *testing.B) {
164164 }165165 166166 func BenchmarkFind(b *testing.B) {167- s := stress.Dense(sheet.MaxRows, 26)167+ s := stress.Dense(stress.Rows, 26)168168 b.Run("find", func(b *testing.B) {169169 for b.Loop() {170170 if _, err := s.Find("99.5", sheet.FindOptions{}); err != nil {
@@ -189,7 +189,7 @@ func BenchmarkFillSeries(b *testing.B) {
189189 s.Set(sheet.Addr{}, "1")190190 s.Set(sheet.Addr{Row: 1}, "2")191191 src := sheet.NewRect(sheet.Addr{}, sheet.Addr{Row: 1})192- dst := sheet.NewRect(sheet.Addr{}, sheet.Addr{Row: sheet.MaxRows - 1})192+ dst := sheet.NewRect(sheet.Addr{}, sheet.Addr{Row: stress.Rows - 1})193193 for b.Loop() {194194 if _, err := s.FillSeries(src, dst); err != nil {195195 b.Fatal(err)
@@ -238,18 +238,18 @@ type namedBuild struct {
238238 239239 func saveShapes() []namedBuild {240240 return []namedBuild{241- {"dense-8192x26", func() *sheet.Sheet { return stress.Dense(sheet.MaxRows, 26) }},242- {"dense-8192x256", func() *sheet.Sheet { return stress.Dense(sheet.MaxRows, sheet.MaxCols) }},243- {"chain-8192", func() *sheet.Sheet { return stress.Chain(sheet.MaxRows) }},241+ {"dense-8192x26", func() *sheet.Sheet { return stress.Dense(stress.Rows, 26) }},242+ {"dense-8192x256", func() *sheet.Sheet { return stress.Dense(stress.Rows, stress.Cols) }},243+ {"chain-8192", func() *sheet.Sheet { return stress.Chain(stress.Rows) }},244244 }245245 }246246 247247 // BenchmarkMemory reports the live heap per non-blank cell after a load.248248 func BenchmarkMemory(b *testing.B) {249249 for _, sh := range []namedBuild{250- {"numbers", func() *sheet.Sheet { return stress.Dense(sheet.MaxRows, 64) }},251- {"formulas", func() *sheet.Sheet { return stress.Chain(sheet.MaxRows) }},252- {"text-200", func() *sheet.Sheet { return stress.LongText(sheet.MaxRows, 200) }},250+ {"numbers", func() *sheet.Sheet { return stress.Dense(stress.Rows, 64) }},251+ {"formulas", func() *sheet.Sheet { return stress.Chain(stress.Rows) }},252+ {"text-200", func() *sheet.Sheet { return stress.LongText(stress.Rows, 200) }},253253 } {254254 b.Run(sh.name, func(b *testing.B) {255255 for b.Loop() {
A internal/sheet/vast_test.go +43 -0
@@ -0,0 +1,43 @@
1+package sheet2+3+import "testing"4+5+// A filter over whole columns tests the rows with data and treats the6+// blank rows past them as one: hiding them, counting them and stepping7+// over them cost nothing per row.8+func TestFilterWholeColumns(t *testing.T) {9+ s := New()10+ for i, in := range []string{"h", "1", "", "3", "x"} {11+ if in != "" {12+ s.Set(Addr{Row: i}, in)13+ }14+ }15+ s.CreateFilter(Rect{To: Addr{Col: 1, Row: MaxRows - 1}})16+ s.FilterColumn(0, Criteria{Cond: Condition{CondNotEmpty, ""}})17+ if n := s.HiddenRows(); n != MaxRows-4 {18+ t.Errorf("HiddenRows = %d, want %d", n, MaxRows-4)19+ }20+ if r, ok := s.NextShownRow(3, 1); !ok || r != 4 {21+ t.Errorf("NextShownRow(3, 1) = %d, %v", r, ok)22+ }23+ if r, ok := s.NextShownRow(4, 1); ok {24+ t.Errorf("NextShownRow(4, 1) = %d, want none", r)25+ }26+ if r, ok := s.NextShownRow(MaxRows, -1); !ok || r != 4 {27+ t.Errorf("NextShownRow from the bottom = %d, %v", r, ok)28+ }29+ if got := s.Edge(at("A5"), 0, 1); got != at("A5") {30+ t.Errorf("Edge down from the last shown row = %v", got)31+ }32+ if got := s.Edge(at("B1"), 0, 1); got != at("B5") {33+ t.Errorf("Edge down column B = %v, want B5", got)34+ }35+ vals := s.FilterValues(1)36+ if len(vals) != 1 || vals[0].Text != "" || vals[0].Count != 3 {37+ t.Errorf("FilterValues(B) = %+v", vals)38+ }39+ s.FilterColumn(0, Criteria{})40+ if vals := s.FilterValues(0); vals[len(vals)-1].Text != "" || vals[len(vals)-1].Count != MaxRows-4 {41+ t.Errorf("blanks in FilterValues(A) = %+v", vals[len(vals)-1])42+ }43+}