cat / 012
clone

a spreadsheet for the terminal: lotus 1-2-3 looks, google sheets keys and formulas, one go binary

Cycles are found from the formulas as written

Dami committed 6ae7be2
Evaluation found a cycle only where it walked one, reaching a cell it
was still evaluating, so a cycle behind an IF branch not taken, a range
read that stops at an error, or INDEX reading one cell of its range
went unseen, and its cells showed values that depended on the order
they were evaluated in: typed, they could show one thing and, reopened,
another. FuzzRandomEdits found such a workbook within a second.

Before evaluating, a recalculation now finds the cycles among the
formulas it recalculates from their references as written (cells,
ranges, names, tables bound to their rows, regions, other sheets), as
Sheets and Excel do, and every cell of a cycle is #REF!. Only those
formulas are looked at: a cycle through one of them runs through
formulas that read it, which recalculate too. Ranges reach the formulas
in them through a segment tree over each column's formulas, so a range
costs the logarithm of the formulas in its columns and running totals
over computed cells stay linear; Tarjan's algorithm, without recursion,
finds the components. When every formula references only cells before
it, row by row (a chain, running totals), there is nothing to look for.
The buffers are kept between recalculations. make speed stays inside
its margins (chain-8192 3.24x against 3.46x, running-8192 3.12x against
3.02x, allocations unchanged).

The array whose formula reads its own cells through VLOOKUP is now a
cycle of formulas, the range VLOOKUP reads holding the array's anchor,
and its explanation says so. randKnown, whose one seed was such a
cycle, goes: -randedit.seeds=20000 passes.
commit
6ae7be21e055a80e0d5670417ecf804a77d427be
parent
1e7ce1dcf37e85c874c7e9b40ce7419224dd9793
author
Dami <dami.etoile@gmail.com>
kind
fix risk score 2.1 of 3, Jev's estimate
10 files changed, +515 -27
Mdocs/contributing/architecture.md9+-
Mdocs/contributing/testing.md2-
Mdocs/formulas/README.md2+-
Ainternal/sheet/cyclefind.go377++++++++++++++++++++++++++++++
Ainternal/sheet/cyclefind_test.go116+++++++++
Minternal/sheet/evaluate.go4+-
Minternal/sheet/randedit_test.go16+-
Minternal/sheet/spill_test.go12+-
Ainternal/sheet/testdata/fuzz/FuzzRandomEdits/5b93384d423fd67f2+
Minternal/sheet/workbook.go2+
M docs/contributing/architecture.md +7 -2
M docs/contributing/testing.md +0 -2
M docs/formulas/README.md +1 -1
A internal/sheet/cyclefind.go +377 -0
@@ -0,0 +1,377 @@
1+package sheet2+3+import (4+ "slices"5+6+ "github.com/FelineStateMachine/012/internal/formula"7+)8+9+// Cycles are found from the formulas as written, as Sheets and Excel10+// find them, before a recalculation evaluates anything: a formula is11+// part of a cycle when what it references (cells, ranges, names, tables,12+// regions, other sheets) leads back to it, whether or not evaluating it13+// reads that far: an IF branch not taken, a range read that stops at an14+// error, INDEX reading one cell of its range. Evaluation meeting a cell15+// it is evaluating finds the cycles it walks (evaluate.go), but only16+// those, so what the cells of a cycle it didn't walk showed depended on17+// the order they were evaluated in. Every cell of a cycle is #REF!.18+//19+// Only the formulas to recalculate are looked at: a cycle through one of20+// them runs through formulas that read it, which recalculate too.21+//22+// The graph's nodes are those formulas, each leading to the formulas it23+// references. A range leads to the formulas in it through a segment tree24+// over each column's formulas, whose nodes lead to their halves, so a25+// range costs the logarithm of the formulas in its columns rather than26+// their number, and a column of running totals over computed cells27+// stays linear. Tarjan's algorithm finds the strongly connected28+// components; the formulas of a component with more than one node, or29+// that reference themselves, are a cycle's. The graph's buffers are kept30+// for the next recalculation, up to maxCycleKeep formulas.31+32+// maxCycleKeep is the most formulas whose graph's buffers are kept.33+const maxCycleKeep = 1 << 1634+35+// cycleGraph is the graph of the formulas to recalculate: formulas are36+// nodes 0 to formulas-1, in sheet order and by column and row within a37+// sheet, and segment nodes follow. The edges leaving node v are38+// edges[start[v]:start[v+1]].39+type cycleGraph struct {40+ w *Workbook41+ formulas int3242+ sheets []cycleSheet43+ keys []uint64 // each sheet's formulas' keys, sorted44+ cols []cycleCol // each sheet's columns holding formulas, in order45+ last *cycleSheet46+ start []int3247+ edges []int3248+ tarjan49+}50+51+// cycleSheet is a sheet's formulas in the graph: keys[from:to] (column in52+// the high half, row in the low), which are nodes from to to, and its53+// columns, cols[col0:col1].54+type cycleSheet struct {55+ s *Sheet56+ from, to int3257+ col0, col1 int3258+}59+60+// cycleCol is a column's formulas: keys[from:from+m], and the node of its61+// segment tree's first inner node (the tree has m-1).62+type cycleCol struct {63+ col int64+ from, m int3265+ seg int3266+}67+68+func cycleKey(a Addr) uint64 { return uint64(a.Col)<<32 | uint64(uint32(a.Row)) }69+70+func keyAddr(k uint64) Addr { return Addr{Col: int(k >> 32), Row: int(uint32(k))} }71+72+// findCycles returns the formulas to recalculate, dirty in each sheet's73+// calc, that are part of a cycle, or nil when none is.74+func (w *Workbook) findCycles() map[loc]bool {75+ g := w.cycles76+ if g == nil {77+ g = &cycleGraph{w: w}78+ }79+ var cycle map[loc]bool80+ if g.collect() {81+ g.link()82+ cycle = g.components()83+ }84+ g.reset()85+ w.cycles = nil86+ if cap(g.keys) <= maxCycleKeep {87+ w.cycles = g88+ }89+ return cycle90+}91+92+// reset empties the graph for the next recalculation, keeping its93+// buffers.94+func (g *cycleGraph) reset() {95+ clear(g.sheets)96+ g.sheets, g.keys, g.cols, g.start, g.edges = g.sheets[:0], g.keys[:0], g.cols[:0], g.start[:0], g.edges[:0]97+ g.formulas, g.last = 0, nil98+ g.tarjan = tarjan{index: g.index[:0], low: g.low[:0], on: g.on[:0], stack: g.stack[:0], path: g.path[:0], next: g.next[:0]}99+}100+101+// collect gathers the formulas to recalculate, reporting whether a102+// cycle may run through them: not when there are none, or when each103+// references only cells before it, row by row, as a column of running104+// totals or a chain down a column does, since then no reference leads105+// back.106+func (g *cycleGraph) collect() bool {107+ back := true108+ for _, s := range g.w.sheets {109+ from := len(g.keys)110+ for a, st := range s.calc {111+ if c := s.cells.richAt(a); st == dirty && c != nil && c.expr != nil && !c.derived && !c.spilled && readsCells(c) {112+ g.keys = append(g.keys, cycleKey(a))113+ back = back && readsBefore(c, a)114+ }115+ }116+ if len(g.keys) > from {117+ g.sheets = append(g.sheets, cycleSheet{s: s, from: int32(from), to: int32(len(g.keys))})118+ }119+ }120+ g.formulas = int32(len(g.keys))121+ if back {122+ return false123+ }124+ for _, cs := range g.sheets {125+ slices.Sort(g.keys[cs.from:cs.to])126+ }127+ seg := g.formulas128+ for i := range g.sheets {129+ cs := &g.sheets[i]130+ cs.col0 = int32(len(g.cols))131+ for j := cs.from; j < cs.to; j++ {132+ col := int(g.keys[j] >> 32)133+ if len(g.cols) == int(cs.col0) || g.cols[len(g.cols)-1].col != col {134+ g.cols = append(g.cols, cycleCol{col: col, from: j})135+ }136+ g.cols[len(g.cols)-1].m++137+ }138+ cs.col1 = int32(len(g.cols))139+ for c := cs.col0; c < cs.col1; c++ {140+ g.cols[c].seg = seg141+ seg += max(g.cols[c].m-1, 0)142+ }143+ }144+ g.start = slices.Grow(g.start, int(seg)+1)[:seg+1]145+ return true146+}147+148+// readsBefore reports whether the formula c, at a, references only149+// cells of its own sheet before a, row by row.150+func readsBefore(c *Cell, a Addr) bool {151+ if len(c.xrefs) > 0 || len(c.names) > 0 {152+ return false153+ }154+ for _, r := range c.refs {155+ if !before(r, a) {156+ return false157+ }158+ }159+ for _, r := range c.ranges {160+ if !before(r.To, a) {161+ return false162+ }163+ }164+ return true165+}166+167+// readsCells reports whether the formula c references anything.168+func readsCells(c *Cell) bool {169+ return len(c.refs) > 0 || len(c.ranges) > 0 || len(c.xrefs) > 0 || len(c.names) > 0170+}171+172+// link writes the edges: each formula's to what it references, then173+// each segment node's to its halves.174+func (g *cycleGraph) link() {175+ for _, cs := range g.sheets {176+ for j := cs.from; j < cs.to; j++ {177+ g.start[j] = int32(len(g.edges))178+ g.references(cs.s, keyAddr(g.keys[j]))179+ }180+ }181+ v := g.formulas182+ for _, col := range g.cols {183+ for p := int32(1); p < col.m; p++ {184+ g.start[v] = int32(len(g.edges))185+ g.edges = append(g.edges, col.node(2*p), col.node(2*p+1))186+ v++187+ }188+ }189+ g.start[v] = int32(len(g.edges))190+}191+192+// node is the node at position p of the column's segment tree: an inner193+// node below m, a formula from m on.194+func (col cycleCol) node(p int32) int32 {195+ if p >= col.m {196+ return col.from + p - col.m197+ }198+ return col.seg + p - 1199+}200+201+// references adds the edges from the formula at a on s to the formulas202+// it references. One naming a name, a table or a region is bound first,203+// as evaluating it is, so a table's row (Sales[@Price]) leads to that204+// row alone.205+func (g *cycleGraph) references(s *Sheet, a Addr) {206+ c := s.cells.richAt(a)207+ if len(c.names) > 0 {208+ formula.WalkRefs(s.bound(a, c),209+ func(sheet string, r Addr) { g.cell(g.w.resolve(s, sheet), r) },210+ func(sheet string, r Rect) { g.rect(g.w.resolve(s, sheet), r) })211+ return212+ }213+ for _, r := range c.refs {214+ g.cell(s, r)215+ }216+ for _, r := range c.ranges {217+ g.rect(s, r)218+ }219+ for _, x := range c.xrefs {220+ g.rect(g.w.byKey[x.key], x.r)221+ }222+}223+224+// sheet is t's formulas in the graph, if it has any.225+func (g *cycleGraph) sheet(t *Sheet) (*cycleSheet, bool) {226+ if g.last != nil && g.last.s == t {227+ return g.last, true228+ }229+ for i := range g.sheets {230+ if g.sheets[i].s == t {231+ g.last = &g.sheets[i]232+ return g.last, true233+ }234+ }235+ return nil, false236+}237+238+// cell adds an edge to the formula at a on t, if it is one.239+func (g *cycleGraph) cell(t *Sheet, a Addr) {240+ if cs, ok := g.sheet(t); ok {241+ if i, found := slices.BinarySearch(g.keys[cs.from:cs.to], cycleKey(a)); found {242+ g.edges = append(g.edges, cs.from+int32(i))243+ }244+ }245+}246+247+// rect adds edges to the formulas of r on t: in each of its columns, to248+// the fewest segment nodes that cover them.249+func (g *cycleGraph) rect(t *Sheet, r Rect) {250+ cs, ok := g.sheet(t)251+ switch {252+ case !ok:253+ return254+ case r.From == r.To:255+ g.cell(t, r.From)256+ return257+ }258+ cols := g.cols[cs.col0:cs.col1]259+ c0, _ := slices.BinarySearchFunc(cols, r.From.Col, func(c cycleCol, col int) int { return c.col - col })260+ for _, col := range cols[c0:] {261+ if col.col > r.To.Col {262+ break263+ }264+ keys := g.keys[col.from : col.from+col.m]265+ lo, _ := slices.BinarySearch(keys, cycleKey(Addr{Col: col.col, Row: r.From.Row}))266+ hi, _ := slices.BinarySearch(keys, cycleKey(Addr{Col: col.col, Row: r.To.Row})+1)267+ for lp, hp := int32(lo)+col.m, int32(hi)+col.m; lp < hp; lp, hp = lp/2, hp/2 {268+ if lp&1 == 1 {269+ g.edges = append(g.edges, col.node(lp))270+ lp++271+ }272+ if hp&1 == 1 {273+ hp--274+ g.edges = append(g.edges, col.node(hp))275+ }276+ }277+ }278+}279+280+// components runs Tarjan's algorithm from each formula, without281+// recursion, and returns the formulas in cycles.282+func (g *cycleGraph) components() map[loc]bool {283+ n := len(g.start) - 1284+ g.index = slices.Grow(g.index, n)[:n]285+ g.low = slices.Grow(g.low, n)[:n]286+ g.on = slices.Grow(g.on, n)[:n]287+ clear(g.index)288+ for v := range g.formulas {289+ if g.index[v] == 0 {290+ g.visit(v)291+ }292+ }293+ return g.cycle294+}295+296+// tarjan is the state of Tarjan's algorithm: each node's index, from 1297+// (0 while unvisited), and lowest index reached, the nodes on the stack,298+// the path being followed, with the next edge of each, and the formulas299+// found in cycles.300+type tarjan struct {301+ index, low []int32302+ on []bool303+ stack, path []int32304+ next []int32305+ count int32306+ cycle map[loc]bool307+}308+309+func (g *cycleGraph) push(v int32) {310+ g.count++311+ g.index[v], g.low[v] = g.count, g.count312+ g.stack = append(g.stack, v)313+ g.on[v] = true314+ g.path = append(g.path, v)315+ g.next = append(g.next, g.start[v])316+}317+318+func (g *cycleGraph) visit(root int32) {319+ g.push(root)320+ for len(g.path) > 0 {321+ top := len(g.path) - 1322+ v := g.path[top]323+ if e := g.next[top]; e < g.start[v+1] {324+ g.next[top]++325+ switch u := g.edges[e]; {326+ case g.index[u] == 0:327+ g.push(u)328+ case g.on[u]:329+ g.low[v] = min(g.low[v], g.index[u])330+ }331+ continue332+ }333+ g.path, g.next = g.path[:top], g.next[:top]334+ if top > 0 {335+ p := g.path[top-1]336+ g.low[p] = min(g.low[p], g.low[v])337+ }338+ if g.low[v] == g.index[v] {339+ g.pop(v)340+ }341+ }342+}343+344+// pop takes the component rooted at v off the stack, keeping its345+// formulas when it is a cycle.346+func (g *cycleGraph) pop(v int32) {347+ i := len(g.stack) - 1348+ for g.stack[i] != v {349+ i--350+ }351+ comp := g.stack[i:]352+ g.stack = g.stack[:i]353+ for _, u := range comp {354+ g.on[u] = false355+ }356+ if len(comp) == 1 && !slices.Contains(g.edges[g.start[v]:g.start[v+1]], v) {357+ return358+ }359+ if g.cycle == nil {360+ g.cycle = map[loc]bool{}361+ }362+ for _, u := range comp {363+ if u < g.formulas {364+ g.cycle[g.loc(u)] = true365+ }366+ }367+}368+369+// loc is the cell of the formula node v.370+func (g *cycleGraph) loc(v int32) loc {371+ for _, cs := range g.sheets {372+ if v < cs.to {373+ return loc{cs.s, keyAddr(g.keys[v])}374+ }375+ }376+ panic("sheet: no formula node")377+}
A internal/sheet/cyclefind_test.go +116 -0
@@ -0,0 +1,116 @@
1+package sheet2+3+import (4+ "maps"5+ "strings"6+ "testing"7+)8+9+// Cycles evaluation doesn't walk are cycles all the same, #REF! in every10+// cell, typed in any order or opened (cyclefind.go).11+func TestCyclesFoundAsWritten(t *testing.T) {12+ for _, tc := range []cycleCase{13+ {"an IF branch not taken", map[string]string{"A1": "=IF(FALSE,B1,1)", "B1": "=A1+1"}, []string{"A1", "B1"}, nil},14+ {"a range read cut short by an error", map[string]string{"A1": "=SUM(B1:B3)", "B1": "=1/0", "B3": "=A1"}, []string{"A1", "B3"}, map[string]string{"B1": "#DIV/0!"}},15+ {"INDEX reading one cell", map[string]string{"A1": "=INDEX(A2:A5,1)", "A2": "7", "A4": "=A1"}, []string{"A1", "A4"}, nil},16+ {"itself, behind a branch", map[string]string{"C3": "=IF(TRUE,1,C3)", "D3": "=IFERROR(C3,5)"}, []string{"C3"}, map[string]string{"D3": "5"}},17+ {"another sheet", map[string]string{"A1": "=IF(FALSE,Data!A1,1)", "Data!A1": "=Sheet1!A1"}, []string{"A1", "Data!A1"}, nil},18+ {"a name", map[string]string{"A1": "=IF(FALSE,SUM(Near),1)", "B2": "=A1"}, []string{"A1", "B2"}, nil},19+ } {20+ orders := [][]string{nil, nil}21+ for a := range tc.cells {22+ orders[0] = append(orders[0], a)23+ orders[1] = append([]string{a}, orders[1]...)24+ }25+ for _, order := range orders {26+ tc.check(t, order)27+ }28+ }29+}30+31+// cycleCase is cells to type, those of the cycle they make, and what32+// others show.33+type cycleCase struct {34+ name string35+ cells map[string]string36+ cycle []string37+ others map[string]string38+}39+40+// check types the cells in order, on a workbook with a sheet Data and a41+// name Near for A1:B2, and checks what they show, and show once opened.42+func (tc cycleCase) check(t *testing.T, order []string) {43+ t.Helper()44+ s := New()45+ w := s.Book()46+ w.AddSheet("Data", 1)47+ w.DefineName("Near", s, rect("A1:B2"))48+ for _, a := range order {49+ setIn(t, w, a, tc.cells[a])50+ }51+ reopened := roundTrip(t, s).Book()52+ if !reopened.Circular {53+ t.Errorf("%s: not circular", tc.name)54+ }55+ want := map[string]string{}56+ maps.Copy(want, tc.others)57+ for _, a := range tc.cycle {58+ want[a] = "#REF!"59+ }60+ for _, sh := range []*Workbook{w, reopened} {61+ for a, v := range want {62+ if got := show(t, sh, qualified(a)); got != v {63+ t.Errorf("%s, typed %v: %s = %s, want %s", tc.name, order, a, got, v)64+ }65+ }66+ }67+}68+69+// setIn types in into the cell a names, on Sheet1 unless it names a70+// sheet.71+func setIn(t *testing.T, w *Workbook, a, in string) {72+ t.Helper()73+ s := w.Sheet(0)74+ if sheet, cell, ok := strings.Cut(a, "!"); ok {75+ s, a = w.Lookup(sheet), cell76+ }77+ if err := s.Set(at(a), in); err != nil {78+ t.Fatal(err)79+ }80+}81+82+func qualified(a string) string {83+ if strings.Contains(a, "!") {84+ return a85+ }86+ return "Sheet1!" + a87+}88+89+// Formulas reading each other without a cycle aren't taken for one:90+// a table's formulas reading their own row, running totals over a91+// column of formulas, and a total above the column it adds up.92+func TestNoCycleWithout(t *testing.T) {93+ s := salesSheet(t)94+ s.Set(at("D1"), "Total")95+ w := s.Book()96+ w.ResizeTable("Sales", rect("A1:D4"))97+ for row := 2; row <= 4; row++ {98+ s.Set(Addr{Col: 3, Row: row - 1}, "=Sales[@Units]*Sales[@Amount]")99+ }100+ for row := 1; row <= 30; row++ {101+ s.Set(Addr{Col: 6, Row: row}, "=F"+itoa(row+1)+"*2")102+ s.Set(Addr{Col: 7, Row: row}, "=SUM(G$2:G"+itoa(row+1)+")")103+ }104+ s.Set(at("G1"), "=SUM(G2:G31)")105+ for _, sh := range []*Workbook{w, roundTrip(t, s).Book()} {106+ if sh.Circular {107+ t.Error("circular")108+ }109+ if got := show(t, sh, "Sheet1!D4"); got != "150" {110+ t.Errorf("D4 = %s", got)111+ }112+ if got := show(t, sh, "Sheet1!H31"); got != "0" {113+ t.Errorf("H31 = %s", got)114+ }115+ }116+}
M internal/sheet/evaluate.go +2 -2
M internal/sheet/randedit_test.go +1 -15
M internal/sheet/spill_test.go +7 -5
A internal/sheet/testdata/fuzz/FuzzRandomEdits/5b93384d423fd67f +2 -0
M internal/sheet/workbook.go +2 -0