The occupancy index finds columns and blocks by slice index instead of hashing, and replacing a cell no longer removes and re-adds it, so an edit costs what it did before the index. A range read once is read directly: only its second read starts a shared running aggregate, so a thousand SUMs of different regions allocate nothing extra. Rows blank across a pivot's source stop at their first filled column, sorting places cells without building the result as a map, charts read at most 256 series and find labels and headers through the index, script reads of large ranges trim to their data, and pastes from the terminal past max-cells are refused. SUMIF, SUMIFS and SUMPRODUCT keep their terms functions for decimal arithmetic.
- commit
- fb81461eeb96948648ef5db9cc265e8266a6140d
- parent
- 877f0bee302d6ea47d494863769600bf002ae33e
- author
- Dami <dami.etoile@gmail.com>
| M | internal/ | 23 | ++++--- |
| M | internal/ | 92 | +++++++++++++++++++++--------- |
| M | internal/ | 58 | ++++++++++++------- |
| M | internal/ | 11 | +++- |
| M | internal/ | 12 | ++-- |
| M | internal/ | 15 | ++++- |
| M | internal/ | 20 | +++---- |
| M | internal/ | 2 | +- |
| M | internal/ | 4 | +- |
| M | internal/ | 4 | + |
| M | internal/ | 10 | +++ |
| M | internal/ | 10 | ++- |
@@ -97,8 +97,12 @@ type ChartData struct {
9797 }9898 9999 // maxChartPoints caps the categories a chart reads, so a chart over whole100-// columns stays quick to draw.101-const maxChartPoints = 500100+// columns stays quick to draw, and maxChartSeries its series (the columns101+// 1-2-3's grid had), for one over whole rows.102+const (103+ maxChartPoints = 500104+ maxChartSeries = 256105+)102106 103107 // ChartData reads a chart's cells. Series names and labels are the cells'104108 // text as displayed.@@ -119,7 +123,7 @@ func (s *Sheet) ChartData(c Chart) ChartData {
119123 if c.Labels && m > 1 {120124 col0 = 1121125 }122- n = min(n, first+maxChartPoints)126+ n, m = min(n, first+maxChartPoints), min(m, col0+maxChartSeries)123127 var d ChartData124128 for i := first; i < n; i++ {125129 label := strconv.Itoa(i - first + 1)@@ -231,11 +235,10 @@ func (s *Sheet) GuessChart(r Rect) Chart {
231235 isText := func(a Addr) bool { return s.Value(a).Kind == Text }232236 if r.To.Col > r.From.Col {233237 labels := false234- for row := r.From.Row + 1; row <= r.To.Row; row++ {235- if isText(Addr{Col: r.From.Col, Row: row}) {236- labels = true237- }238- }238+ s.cells.filled.colScan(r.From.Col, r.From.Row+1, r.To.Row, func(row int) bool {239+ labels = isText(Addr{Col: r.From.Col, Row: row})240+ return !labels241+ })239242 c.Labels = labels || r.From.Row == r.To.Row && isText(r.From)240243 }241244 if r.To.Row > r.From.Row {@@ -244,8 +247,8 @@ func (s *Sheet) GuessChart(r Rect) Chart {
244247 from++245248 }246249 texts, nums := 0, 0247- for col := from; col <= r.To.Col; col++ {248- switch s.Value(Addr{Col: col, Row: r.From.Row}).Kind {250+ for _, cell := range s.cells.inRange(Rect{From: Addr{Col: from, Row: r.From.Row}, To: Addr{Col: r.To.Col, Row: r.From.Row}}) {251+ switch cell.Value.Kind {249252 case Text:250253 texts++251254 case Number, Bool:@@ -106,21 +106,21 @@ func passes(ms []matrix, cs []criterion, p pos) bool {
106106 return true107107 }108108 109-// sumMasked adds the numbers of m where the criteria pass.110-func sumMasked(m matrix, k masked) Value {111- total := 0.0109+// sumMasked calls add with the numbers of m where the criteria pass, or110+// returns the first error there.111+func sumMasked(m matrix, k masked, add func(float64)) *Value {112112 for _, p := range k.pass {113113 switch v := m.cell(p.r, p.c); v.Kind {114114 case Error:115- return v115+ return &v116116 case Number:117- total += v.Num117+ add(v.Num)118118 }119119 }120120 if k.tail > 0 && k.tailPass && m.blank.Kind == Error {121- return m.blank121+ return &m.blank122122 }123- return num(total)123+ return nil124124 }125125 126126 // averageMasked averages the numbers of m where the criteria pass.@@ -144,32 +144,69 @@ func averageMasked(m matrix, k masked) Value {
144144 return num(sum / float64(n))145145 }146146 147+// sumIf, sumIfs and sumProduct add in binary; their terms functions148+// visit the same terms for decimal arithmetic too.147149 func sumIf(args []Node, get lookup) Value {150+ total := 0.0151+ if e := sumIfTerms(args, get, func(f float64) { total += f }); e != nil {152+ return *e153+ }154+ return num(total)155+}156+157+func sumIfs(args []Node, get lookup) Value {158+ total := 0.0159+ if e := sumIfsTerms(args, get, func(f float64) { total += f }); e != nil {160+ return *e161+ }162+ return num(total)163+}164+165+func sumProduct(args []Node, get lookup) Value {166+ total := 0.0167+ e := sumProductTerms(args, get, func(fs []float64) {168+ p := 1.0169+ for _, f := range fs {170+ p *= f171+ }172+ total += p173+ })174+ if e != nil {175+ return *e176+ }177+ return num(total)178+}179+180+// sumIfTerms calls add with each number SUMIF sums, or returns the error181+// it gives.182+func sumIfTerms(args []Node, get lookup, add func(float64)) *Value {148183 rng := matrixArg(args[0], get)149184 cv := eval(args[1], get)150185 if cv.Kind == Error {151- return cv186+ return &cv152187 }153188 sum := rng154189 if len(args) > 2 {155190 sum = matrixArg(args[2], get).resized(rng.rows, rng.cols, get)156191 if sum.rows != rng.rows || sum.cols != rng.cols {157- return ErrValue192+ return &ErrValue158193 }159194 }160- return sumMasked(sum, applyCriteria(get, []matrix{rng}, []criterion{newCriterion(cv)}, sum))195+ return sumMasked(sum, applyCriteria(get, []matrix{rng}, []criterion{newCriterion(cv)}, sum), add)161196 }162197 163-func sumIfs(args []Node, get lookup) Value {198+// sumIfsTerms calls add with each number SUMIFS sums, or returns the199+// error it gives.200+func sumIfsTerms(args []Node, get lookup, add func(float64)) *Value {164201 sum := matrixArg(args[0], get)165202 ms, cs, err := criteriaArgs(args, 1, get)166203 switch {167204 case err != nil:168- return *err205+ return err169206 case len(ms) == 0 || ms[0].rows != sum.rows || ms[0].cols != sum.cols:170- return ErrValue207+ return &ErrValue171208 }172- return sumMasked(sum, applyCriteria(get, ms, cs, sum))209+ return sumMasked(sum, applyCriteria(get, ms, cs, sum), add)173210 }174211 175212 func averageIf(args []Node, get lookup) Value {@@ -226,38 +263,39 @@ func countBlank(args []Node, get lookup) Value {
226263 return num(float64(n))227264 }228265 229-// sumProduct multiplies same-sized arrays entry by entry and sums; text230-// and blanks count as 0.231-func sumProduct(args []Node, get lookup) Value {266+// sumProductTerms calls term with the factors of each entry of267+// same-sized arrays that any of them holds (text and blanks count as 0;268+// entries blank in all of them are 0 and left out), or returns the269+// error SUMPRODUCT gives. The slice is reused between calls.270+func sumProductTerms(args []Node, get lookup, term func(fs []float64)) *Value {232271 ms := make([]matrix, len(args))233272 for i, a := range args {234273 ms[i] = matrixArg(a, get)235274 if ms[i].rows != ms[0].rows || ms[i].cols != ms[0].cols {236- return ErrValue275+ return &ErrValue237276 }238277 }239278 cells := cellsOf(get, ms...)240- total := 0.0279+ fs := make([]float64, len(ms))241280 for _, pc := range cells {242- p := 1.0243- for _, m := range ms {281+ for i, m := range ms {244282 switch v := m.cell(pc.r, pc.c); v.Kind {245283 case Error:246- return v284+ return &v247285 case Number:248- p *= v.Num286+ fs[i] = v.Num249287 default:250- p = 0288+ fs[i] = 0251289 }252290 }253- total += p291+ term(fs)254292 }255293 if ms[0].size() > len(cells) {256294 for _, m := range ms {257295 if m.blank.Kind == Error {258- return m.blank296+ return &m.blank259297 }260298 }261299 }262- return num(total)300+ return nil263301 }@@ -9,9 +9,12 @@ import (
99 // blockRows rows, so the cells of a range are found in row order at the1010 // cost of what it holds, not its area: a whole column of a million rows1111 // with ten cells is ten visits.12+//13+// Columns and their blocks are found by index in slices grown to the14+// last one used, so marking a cell costs no hashing.1215 type occupancy struct {13- cols map[int]*colIndex14- colIDs []int // the columns holding cells, ascending16+ cols []*colIndex // by column; nil for none17+ colIDs []int // the columns holding cells, ascending1518 }1619 1720 // Rows of a column are indexed in blocks of blockRows, one bit each.@@ -26,40 +29,55 @@ type rowBlock struct {
2629 n int2730 }2831 29-// colIndex is the occupancy of one column: its blocks by number, and the30-// numbers in order.32+// colIndex is the occupancy of one column: its blocks by number (nil33+// for none), and the numbers in order.3134 type colIndex struct {32- blocks map[int]*rowBlock35+ blocks []*rowBlock3336 ids []int3437 n int3538 }3639 40+// col is column c's index, or nil.41+func (o *occupancy) col(c int) *colIndex {42+ if c < 0 || c >= len(o.cols) {43+ return nil44+ }45+ return o.cols[c]46+}47+48+// block is block id of the column, or nil.49+func (ci *colIndex) block(id int) *rowBlock {50+ if ci == nil || id < 0 || id >= len(ci.blocks) {51+ return nil52+ }53+ return ci.blocks[id]54+}55+3756 // has reports whether a holds a cell.3857 func (o *occupancy) has(a Addr) bool {39- ci := o.cols[a.Col]40- if ci == nil {41- return false42- }43- b := ci.blocks[a.Row>>blockShift]58+ b := o.col(a.Col).block(a.Row >> blockShift)4459 bit := a.Row & (blockRows - 1)4560 return b != nil && b.bits[bit>>6]&(1<<(bit&63)) != 04661 }4762 4863 // mark records a cell at a, which had none.4964 func (o *occupancy) mark(a Addr) {50- ci := o.cols[a.Col]65+ ci := o.col(a.Col)5166 if ci == nil {52- ci = &colIndex{blocks: make(map[int]*rowBlock)}53- if o.cols == nil {54- o.cols = make(map[int]*colIndex)67+ ci = &colIndex{}68+ if a.Col >= len(o.cols) {69+ o.cols = slices.Grow(o.cols, a.Col+1-len(o.cols))[:a.Col+1]5570 }5671 o.cols[a.Col] = ci5772 o.colIDs = insertSorted(o.colIDs, a.Col)5873 }5974 id := a.Row >> blockShift60- b := ci.blocks[id]75+ b := ci.block(id)6176 if b == nil {6277 b = &rowBlock{}78+ if id >= len(ci.blocks) {79+ ci.blocks = slices.Grow(ci.blocks, id+1-len(ci.blocks))[:id+1]80+ }6381 ci.blocks[id] = b6482 ci.ids = insertSorted(ci.ids, id)6583 }@@ -79,11 +97,11 @@ func (o *occupancy) unmark(a Addr) {
7997 b.n--8098 ci.n--8199 if b.n == 0 {82- delete(ci.blocks, id)100+ ci.blocks[id] = nil83101 ci.ids = removeSorted(ci.ids, id)84102 }85103 if ci.n == 0 {86- delete(o.cols, a.Col)104+ o.cols[a.Col] = nil87105 o.colIDs = removeSorted(o.colIDs, a.Col)88106 }89107 }@@ -113,7 +131,7 @@ func (o *occupancy) colsIn(c0, c1 int) []int {
113131 114132 // colCells is the number of cells stored in column c.115133 func (o *occupancy) colCells(c int) int {116- if ci := o.cols[c]; ci != nil {134+ if ci := o.col(c); ci != nil {117135 return ci.n118136 }119137 return 0@@ -156,7 +174,7 @@ func wordMask(w, lo, hi int) uint64 {
156174 // the end. It allocates nothing; the column's cells must not be added157175 // or removed meanwhile.158176 func (o *occupancy) colScan(c, r0, r1 int, fn func(row int) bool) bool {159- ci := o.cols[c]177+ ci := o.col(c)160178 if ci == nil || r0 > r1 {161179 return true162180 }@@ -182,7 +200,7 @@ func (o *occupancy) colScan(c, r0, r1 int, fn func(row int) bool) bool {
182200 // nextRow returns the first row of column c at or after row (dir 1), or183201 // at or before it (dir -1), that holds a cell.184202 func (o *occupancy) nextRow(c, row, dir int) (int, bool) {185- ci := o.cols[c]203+ ci := o.col(c)186204 if ci == nil || row < 0 {187205 return 0, false188206 }@@ -226,10 +226,15 @@ func (c *pivotCalc) gather() {
226226 c.sortCols()227227 }228228 229-// rowBlank reports whether row is blank across r's columns.229+// rowBlank reports whether row is blank across r's columns, looking at230+// the columns that hold anything.230231 func (s *Sheet) rowBlank(row int, r Rect) bool {231- _, filled := s.cells.filled.bounds(Rect{From: Addr{Col: r.From.Col, Row: row}, To: Addr{Col: r.To.Col, Row: row}})232- return !filled232+ for _, c := range s.cells.filled.colsIn(r.From.Col, r.To.Col) {233+ if s.cells.filled.has(Addr{Col: c, Row: row}) {234+ return false235+ }236+ }237+ return true233238 }234239 235240 // pivotTests prepares a pivot's filters as a filter's column tests.@@ -4,9 +4,9 @@ import "slices"
44 55 // Running aggregates: within one recalculation, the SUM-like functions66 // (SUM, AVERAGE, COUNT, COUNTA, MIN, MAX, PRODUCT) share what they read7-// of a range. For each run of columns and first row, the aggregate is8-// kept as a checkpoint after every row with data, read forward only as9-// far as some formula has asked. A thousand SUM(A1:A8192) read column A7+// of a range. For each run of columns and first row read more than once,8+// the aggregate is kept as a checkpoint after every row with data, read9+// forward only as far as some formula has asked. A thousand SUM(A1:A8192) read column A1010 // once; 8192 running totals SUM($A$1:An) extend the same run one row at1111 // a time, so they cost O(n) reads instead of O(n^2). Each result is1212 // accumulated in the same order as reading the range directly, so it is@@ -55,8 +55,10 @@ func (rd *reader) rangeAgg(sheet string, r Rect) (agg, *Value, bool) {
5555 key := aggKey{t, r.From.Col, r.To.Col, r.From.Row}5656 run := memo.m[key]5757 if run == nil {58- run = &runAgg{next: r.From.Row}59- memo.m[key] = run58+ // A range read once (a SUM of each named region) is read directly;59+ // only the second read of a run starts sharing it.60+ memo.m[key] = &runAgg{next: r.From.Row}61+ return agg{}, nil, false6062 }6163 if run.err == nil && r.To.Row >= run.next {6264 if run.busy {@@ -309,10 +309,11 @@ func (s *Sheet) place(a Addr, c *Cell) {
309309 if s.pivot.def != nil && (s.pivot.out.Contains(a) || s.pivot.err != "" && s.pivot.blocked.Contains(a)) {310310 s.pivot.stale = true311311 }312- s.unlink(a)313312 if c == nil {313+ s.unlink(a)314314 return315315 }316+ s.unindex(a) // the old cell's edges; set replaces it in the store316317 s.cells.set(a, c)317318 for _, r := range c.refs {318319 if s.dependents[r] == nil {@@ -352,9 +353,17 @@ func (s *Sheet) cellsIn(r Rect) []Addr {
352353 353354 // unlink removes the cell at a and its dependency edges.354355 func (s *Sheet) unlink(a Addr) {356+ if s.unindex(a) {357+ s.cells.delete(a)358+ }359+}360+361+// unindex removes the dependency edges of the cell at a, reporting362+// whether there is one.363+func (s *Sheet) unindex(a Addr) bool {355364 old := s.cells.get(a)356365 if old == nil {357- return366+ return false358367 }359368 for _, r := range old.refs {360369 delete(s.dependents[r], a)@@ -367,5 +376,5 @@ func (s *Sheet) unlink(a Addr) {
367376 }368377 s.rangeUsers.remove(a, old.ranges)369378 delete(s.volatile, a)370- s.cells.delete(a)379+ return true371380 }@@ -98,23 +98,19 @@ func (s *Sheet) SortRange(r Rect, keys []SortKey) {
9898 }9999 }100100 dst := s.sortedRows(r, rows, keys)101- next := make(map[Addr]*Cell, len(old))102- for a, c := range old {103- to := Addr{Col: a.Col, Row: dst[a.Row]}104- if to.Row != a.Row {105- c = c.rewritten(formula.Shift(0, to.Row-a.Row))106- }107- next[to] = c101+ src := make(map[int]int, len(dst)) // where each row's new contents come from102+ for from, to := range dst {103+ src[to] = from108104 }109105 s.change("sort "+r.String(), r, func() {110106 for a := range old {111- if next[a] == nil {112- s.place(a, nil)107+ if from, ok := src[a.Row]; !ok || old[Addr{Col: a.Col, Row: from}] == nil {108+ s.place(a, nil) // nothing moves here113109 }114110 }115- for a, c := range next {116- if old[a] != c {117- s.place(a, c)111+ for a, c := range old {112+ if to := dst[a.Row]; to != a.Row {113+ s.place(Addr{Col: a.Col, Row: to}, c.rewritten(formula.Shift(0, to-a.Row)))118114 }119115 }120116 })@@ -87,7 +87,7 @@ func (st *cellStore) inRange(r Rect) iter.Seq2[Addr, *Cell] {
8787 for _, id := range st.stored.blockIDs(cols, b0, b1) {8888 var in []colBlock8989 for _, c := range cols {90- if b := st.stored.cols[c].blocks[id]; b != nil {90+ if b := st.stored.cols[c].block(id); b != nil {9191 in = append(in, colBlock{c, b})9292 }9393 }@@ -33,8 +33,8 @@ func TestCellStoreIndex(t *testing.T) {
3333 for a := range st.all() {3434 st.delete(a)3535 }36- if len(st.stored.cols) != 0 || len(st.stored.colIDs) != 0 || len(st.filled.cols) != 0 {37- t.Errorf("empty store keeps %d columns", len(st.stored.cols))36+ if slices.ContainsFunc(st.stored.cols, func(c *colIndex) bool { return c != nil }) || len(st.stored.colIDs) != 0 || len(st.filled.colIDs) != 0 {37+ t.Errorf("empty store keeps %d columns", len(st.stored.colIDs))3838 }3939 }4040 @@ -139,6 +139,10 @@ func (m *Model) pasteText(content string) bool {
139139 width = max(width, len(row))140140 }141141 r := sheet.Rect{From: m.cur, To: sheet.Addr{Col: m.cur.Col + width - 1, Row: m.cur.Row + len(rows) - 1}}142+ if width*len(rows) > sheet.MaxCells() {143+ m.fail(sheet.ErrFillTooBig.Error())144+ return true145+ }142146 if !r.To.Valid() {143147 m.fail(sheet.ErrPasteEdge.Error())144148 return true@@ -6,6 +6,8 @@ import (
66 "testing"77 88 tea "charm.land/bubbletea/v2"9+10+ "github.com/FelineStateMachine/012/internal/sheet"911 )1012 1113 func init() {@@ -169,6 +171,14 @@ func TestPasteTSVFromTerminal(t *testing.T) {
169171 if m.sheet.Len() != 0 {170172 t.Errorf("paste should undo in one step, %d cells left", m.sheet.Len())171173 }174+ // A block of more than max-cells cells is refused.175+ sheet.SetMaxCells(5)176+ defer sheet.SetMaxCells(0)177+ send(m, tea.PasteMsg{Content: "Qty\tPrice\r\n2\t3\r\n\t=B2*C2\r\n"})178+ if m.sheet.Len() != 0 || m.mode != modeError || !strings.Contains(m.errMsg, "max-cells") {179+ t.Errorf("a paste past max-cells wrote %d cells, error %q", m.sheet.Len(), m.errMsg)180+ }181+ press(t, m, "<esc>")172182 // A single line still starts an entry.173183 send(m, tea.PasteMsg{Content: "hello\n"})174184 if m.mode != modeEnter || m.line.text() != "hello" {@@ -87,14 +87,16 @@ func wholeRef(r sheet.Rect) string {
8787 }8888 8989 // readRange is r to read: a large range (whole columns, say) is trimmed90-// to the used range. Small ones aren't, as finding the used range takes91-// a pass over the sheet.90+// to its last row and column with contents. Small ones are read whole,91+// blanks and all, as written.9292 func readRange(s *sheet.Sheet, r sheet.Rect) (sheet.Rect, error) {9393 if (r.To.Col-r.From.Col+1)*(r.To.Row-r.From.Row+1) <= 1<<12 {9494 return r, nil9595 }96- if used, ok := s.UsedRange(); ok && (r.To.Col > used.To.Col || r.To.Row > used.To.Row) {97- r.To = sheet.Addr{Col: max(min(r.To.Col, used.To.Col), r.From.Col), Row: max(min(r.To.Row, used.To.Row), r.From.Row)}96+ data, ok := s.FilledBounds(r)97+ r.To = r.From98+ if ok {99+ r.To = data.To98100 }99101 if (r.To.Col-r.From.Col+1)*(r.To.Row-r.From.Row+1) > maxReadCells {100102 return r, fmt.Errorf("%s has more than %d cells to read at once", r, maxReadCells)