A cell's statements before the last that assign names hand them back
beside the output (nushell.ExecVars), kept as Output.Vars, so later
cells read $files from a cell whose output is something else. The
notebook works out what a run takes (notebook.Prepare), which the UI and
the headless runner both use. Old notebook sheets' multi-statement
commands convert wrapped in do {}, keeping their names.- commit
- a164d6787e3770517fa77814c86cd7082ac90b89
- parent
- 50f2bb290288055700a22df4cb9ced0a57a4f2a2
- author
- Dami <dami.etoile@gmail.com>
- kind
- fix risk score 2.1 of 3, Jev's estimate
| M | internal/ | 1 | - |
| A | internal/ | 43 | ++++++ |
| M | internal/ | 4 | +- |
| M | internal/ | 207 | +++++++++++++++++++++++++----- |
| M | internal/ | 20 | +++- |
| M | internal/ | 2 | +- |
| M | internal/ | 36 | +++++ |
| M | internal/ | 5 | +- |
| M | internal/ | 5 | +- |
@@ -140,7 +140,6 @@ func regionNUON(t *sheet.Sheet, r sheet.Region) ([]byte, error) {
140140 return encodeNUON(snap)141141 }142142 143-144143 // rangeRef is the sheet and range $sheet.ref reads: of the sheet it145144 // names, or else of the sheet shown when the file was saved, or the146145 // first sheet that isn't a notebook.@@ -0,0 +1,43 @@
1+package headless2+3+import (4+ "context"5+ "encoding/base64"6+ "testing"7+ "time"8+9+ "github.com/FelineStateMachine/012/internal/notebook"10+ "github.com/FelineStateMachine/012/internal/sheet"11+)12+13+// --notebooks runs a cell of several statements as the screen does: its14+// own variable isn't a cell it reads, and a later cell reads it from its15+// run.16+func TestRunNotebookStatements(t *testing.T) {17+ w := sheet.NewBook()18+ nb, err := w.AddNotebook("Notebook", w.Sheet(0))19+ if err != nil {20+ t.Fatal(err)21+ }22+ src := "files = ls | where type == file # the files\n$files | where size > 1kb"23+ nb.SetNotebookCells("add cells", []notebook.Cell{24+ {Kind: notebook.Code, Source: "$files | length"},25+ {Kind: notebook.Code, Source: src},26+ })27+ cmd, _, _ := notebook.Parse(src).Command()28+ enc := func(s string) string { return base64.StdEncoding.EncodeToString([]byte(s)) }29+ nu := fakeNu{30+ cmd: "__out " + enc("[[name]; [b]]") + "\nfiles " + enc("[[name]; [a], [b]]") + "\n",31+ "$files | length": "2",32+ }33+ if failed := RunNotebooks(context.Background(), w, NotebookOptions{Runner: nu, Timeout: time.Second}); len(failed) != 0 {34+ t.Fatalf("failed: %q", failed)35+ }36+ cells := nb.NotebookCells()37+ if o := w.Output(cells[1].ID); o == nil || string(o.NUON) != "[[name]; [b]]" || string(o.Vars["files"]) != "[[name]; [a], [b]]" {38+ t.Errorf("the cell of two statements: %+v", o)39+ }40+ if o := w.Output(cells[0].ID); o == nil || string(o.NUON) != "2" || o.Reads["files"] == 0 {41+ t.Errorf("the cell reading $files: %+v", o)42+ }43+}@@ -11,6 +11,8 @@ type Run struct {
1111 // variables it hands back beside the output.1212 Command string1313 Exports []string14+ // Stream is what nu runs to stream the cell (Source.StreamCommand).15+ Stream string1416 // Tables are the other cells' outputs and variables it reads, by1517 // name, as NUON, and Reads the Seqs of the outputs they came from.1618 Tables map[string][]byte@@ -30,7 +32,7 @@ type Run struct {
3032 func Prepare(cells []Cell, i int, output func(id int) *Output) (Run, error) {3133 src := cells[i].Parse()3234 cmd, exports, ranges := src.Command()33- run := Run{Command: cmd, Exports: exports, Tables: map[string][]byte{}, Reads: map[string]int{}, Ranges: ranges, Selection: src.ReadsSelection()}35+ run := Run{Command: cmd, Exports: exports, Stream: src.StreamCommand(), Tables: map[string][]byte{}, Reads: map[string]int{}, Ranges: ranges, Selection: src.ReadsSelection()}3436 vars := Vars(cells)3537 for _, name := range src.Refs() {3638 j, ok := vars[name]@@ -5,59 +5,196 @@ import (
55 "testing"66 )77 8-func TestSplitName(t *testing.T) {9- for _, tc := range []struct{ src, name, pipe string }{10- {"sales = open sales.csv", "sales", "open sales.csv"},11- {"big=\n$sales | first", "big", "\n$sales | first"},12- {"ls | where name == x", "", "ls | where name == x"},13- {"a == b", "", "a == b"},14- {"in = ls", "", "in = ls"},15- {"9x = ls", "", "9x = ls"},16- {"let x = 1", "", "let x = 1"},17- {"__s = 1", "", "__s = 1"},8+func TestName(t *testing.T) {9+ for _, tc := range []struct{ src, name string }{10+ {"sales = open sales.csv", "sales"},11+ {"big=\n$sales | first", "big"},12+ {"ls | where name == x", ""},13+ {"a == b", ""},14+ {"a =~ b", ""},15+ {"in = ls", ""},16+ {"9x = ls", ""},17+ {"let x = 1", ""},18+ {"__s = 1", ""},19+ {"files = ls\n$files | length", ""},20+ {"files = ls\nn = $files | length", "n"},21+ {"n = ls\n| where size > 1kb", "n"},22+ {"n = ls |\n where size > 1kb", "n"},23+ {"n = ls\n# keep the big ones\n| where size > 1kb", "n"},24+ {"n = ls\n\n| length", ""},25+ {"n = ls; $n | length", ""},26+ {"n = [\n1\n2\n]", "n"},27+ {"n = ls # all of them", "n"},28+ {"# n = ls", ""},1829 } {19- name, p := SplitName(tc.src)20- if name != tc.name || p != tc.pipe {21- t.Errorf("SplitName(%q) = %q, %q", tc.src, name, p)30+ if got := (Cell{Source: tc.src}).Name(); got != tc.name {31+ t.Errorf("Name(%q) = %q, want %q", tc.src, got, tc.name)2232 }2333 }24- if got := WithName("ls", "files"); got != "files = ls" {25- t.Errorf("WithName %q", got)26- }27- if got := WithName("files = ls", ""); got != "ls" {28- t.Errorf("WithName without %q", got)29- }3034 if (Cell{Kind: Note, Source: "x = y"}).Name() != "" {3135 t.Error("a note cell has a name")3236 }3337 }3438 35-func TestRefs(t *testing.T) {36- got := Refs("$r1 | where x == $in.a and $nu.home | $r_2.x | $selection | $sheet.A1")37- if !slices.Equal(got, []string{"r1", "r_2"}) {39+func TestWithName(t *testing.T) {40+ for _, tc := range []struct{ src, name, want string }{41+ {"ls", "files", "files = ls"},42+ {"files = ls", "", "ls"},43+ {"files = ls", "all", "all = ls"},44+ {"files = ls\n$files | length", "n", "files = ls\nn = $files | length"},45+ {"files = ls\nn = $files | length", "", "files = ls\n$files | length"},46+ {"", "x", "x = "},47+ {"# just a note", "x", "# just a note\nx = "},48+ } {49+ if got := WithName(tc.src, tc.name); got != tc.want {50+ t.Errorf("WithName(%q, %q) = %q, want %q", tc.src, tc.name, got, tc.want)51+ }52+ }53+}54+55+// The owner's cell: files is the cell's own, read by its second line, not56+// a cell it reads; the cell has no name, and hands files to later cells.57+func TestOwnersCell(t *testing.T) {58+ src := "files = ls | where type == file\n$files | where size > 1kb | sort-by size --reverse"59+ p := Parse(src)60+ if p.Name() != "" || len(p.Refs()) != 0 || !slices.Equal(p.Assigned(), []string{"files"}) {61+ t.Errorf("name %q refs %v assigned %v", p.Name(), p.Refs(), p.Assigned())62+ }63+ cmd, exports, _ := p.Command()64+ want := "let files = ls | where type == file\n{__out: ($files | where size > 1kb | sort-by size --reverse), files: $files}"65+ if cmd != want || !slices.Equal(exports, []string{"files"}) {66+ t.Errorf("command %q exports %v", cmd, exports)67+ }68+ if got := p.StreamCommand(); got != "let files = ls | where type == file\n$files | where size > 1kb | sort-by size --reverse" {69+ t.Errorf("stream command %q", got)70+ }71+ cs := cells(src, "$files | length")72+ order, cycle := Order(cs, []int{0, 1})73+ if !slices.Equal(order, []int{0, 1}) || len(cycle) != 0 {74+ t.Errorf("order %v cycle %v", order, cycle)75+ }76+ run, err := Prepare(cs, 0, func(int) *Output { return nil })77+ if err != nil || run.Command != want || len(run.Tables) != 0 {78+ t.Errorf("prepare %+v %v", run, err)79+ }80+ // The second cell reads files from the first's run.81+ outs := map[int]*Output{1: {NUON: []byte("[]"), Seq: 4, Vars: map[string][]byte{"files": []byte("[[name]; [a]]")}}}82+ out := func(id int) *Output { return outs[id] }83+ run, err = Prepare(cs, 1, out)84+ if err != nil || string(run.Tables["files"]) != "[[name]; [a]]" || run.Reads["files"] != 4 {85+ t.Errorf("prepare the reader %+v %v", run, err)86+ }87+ // An output read from a file keeps no variables: the first cell runs88+ // before the second.89+ outs[1].Vars = nil90+ if got := Inputs(cs, 1, out); !slices.Equal(got, []int{0}) {91+ t.Errorf("inputs %v", got)92+ }93+ if _, err := Prepare(cs, 1, out); err == nil {94+ t.Error("prepared a cell reading a variable its cell didn't hand back")95+ }96+}97+98+func TestComments(t *testing.T) {99+ src := "# x = ls\n# $y | $sheet.A1:B2 | $selection\nls # $z\n| where name == \"a#b\" # $w\n| where name != 'c # $v'\n| where name != r#'d # $u'#\n| append [1#2]\n| append $\"($q # $t\n)\""100+ p := Parse(src)101+ if p.Name() != "" || !slices.Equal(p.Refs(), []string{"q"}) || p.ReadsSelection() || len(p.Ranges) != 0 {102+ t.Errorf("name %q refs %v selection %v ranges %v", p.Name(), p.Refs(), p.ReadsSelection(), p.Ranges)103+ }104+ var comments []string105+ for _, c := range p.Comments {106+ comments = append(comments, src[c[0]:c[1]])107+ }108+ if !slices.Equal(comments, []string{"# x = ls", "# $y | $sheet.A1:B2 | $selection", "# $z", "# $w", "# $t"}) {109+ t.Errorf("comments %q", comments)110+ }111+ if len(p.Stmts) != 1 {112+ t.Errorf("statements %+v", p.Stmts)113+ }114+ cmd, _, _ := p.Command()115+ if len(cmd) != len(src) || cmd[:8] != " " {116+ t.Errorf("command %q", cmd)117+ }118+ // Inside brackets a comment may follow a , or a :.119+ if got := Parse("[1,#x\n2]").Comments; len(got) != 1 {120+ t.Errorf("comments in a list %v", got)121+ }122+ if got := Parse("echo x,#y").Comments; len(got) != 0 {123+ t.Errorf("a comment in a word %v", got)124+ }125+}126+127+func TestStatements(t *testing.T) {128+ src := "a = ls\nb = $a | length; c = $b + 1\nlet d = 2\n$c + $d + $e"129+ p := Parse(src)130+ if !slices.Equal(p.Assigned(), []string{"a", "b", "c"}) || !slices.Equal(p.Refs(), []string{"e"}) {131+ t.Errorf("assigned %v refs %v", p.Assigned(), p.Refs())132+ }133+ cmd, exports, _ := p.Command()134+ want := "let a = ls\nlet b = $a | length; let c = $b + 1\nlet d = 2\n{__out: ($c + $d + $e), a: $a, b: $b, c: $c}"135+ if cmd != want || !slices.Equal(exports, []string{"a", "b", "c"}) {136+ t.Errorf("command %q exports %v", cmd, exports)137+ }138+ // The last statement's name is the output: it's not exported apart.139+ cmd, exports, _ = Parse("x = 1\nx = $x + 1").Command()140+ if cmd != "let x = 1\n$x + 1" || len(exports) != 0 {141+ t.Errorf("command %q exports %v", cmd, exports)142+ }143+ // $x on the line assigning x reads the x from before.144+ if got := Parse("x = $x | first").Refs(); !slices.Equal(got, []string{"x"}) {38145 t.Errorf("refs %v", got)39146 }40- if !ReadsSelection("$selection | math sum") || ReadsSelection("$selections") {41- t.Error("ReadsSelection")147+}148+149+// A variable of the cell's own shadows another cell's name: the cell150+// doesn't read that cell.151+func TestLocalShadows(t *testing.T) {152+ cs := cells("sales = open sales.csv", "sales = [[a]; [1]]\n$sales | length", "n = $sales | length")153+ if got := Reads(cs, Vars(cs), 1); len(got) != 0 {154+ t.Errorf("the shadowing cell reads %v", got)155+ }156+ if got := Reads(cs, Vars(cs), 2); !slices.Equal(got, []int{0}) {157+ t.Errorf("the third cell reads %v", got)158+ }159+ if got := Dependents(cs, 0); !slices.Equal(got, []int{2}) {160+ t.Errorf("dependents %v", got)161+ }162+ if Vars(cs)["sales"] != 0 {163+ t.Errorf("sales is the first cell's, as its name: %v", Vars(cs))164+ }165+ // A cell reading another's name before assigning it reads that cell.166+ cs = cells("sales = ls", "big = $sales | first\nsales = 1\n$sales")167+ if got := Reads(cs, Vars(cs), 1); !slices.Equal(got, []int{0}) {168+ t.Errorf("reads %v", got)42169 }43170 }44171 45-func TestBind(t *testing.T) {46- got, refs := Bind("$sheet.A1:C9 | append $sheet.'Q1 data'!B2:B5 | append $sheet.A1:C9.name | $x$sheet.A1 | $sheet.")47- want := "$__sheet1 | append $__sheet2 | append $__sheet1.name | $x$sheet.A1 | $sheet."172+func TestRanges(t *testing.T) {173+ src := "$sheet.A1:C9 | append $sheet.'Q1 data'!B2:B5 | append $sheet.A1:C9.name | $x$sheet.A1 | $sheet. | append \"$sheet.A1\""174+ p := Parse(src)175+ var spans []string176+ for _, s := range p.Ranges {177+ spans = append(spans, src[s[0]:s[1]])178+ }179+ if !slices.Equal(spans, []string{"$sheet.A1:C9", "$sheet.'Q1 data'!B2:B5", "$sheet.A1:C9"}) {180+ t.Errorf("ranges %q", spans)181+ }182+ got, _, refs := p.Command()183+ want := "$__sheet1 | append $__sheet2 | append $__sheet1.name | $x$sheet.A1 | $sheet. | append \"$sheet.A1\""48184 if got != want {49- t.Errorf("Bind = %q", got)185+ t.Errorf("command = %q", got)50186 }51187 if len(refs) != 2 || refs[0].Ref != "A1:C9" || refs[1].Ref != "'Q1 data'!B2:B5" || refs[1].Var != "__sheet2" {52188 t.Errorf("refs %+v", refs)53189 }54- src := "$sheet.A1:C9 | append $sheet.'Q1 data'!B2:B5 | append $sheet.A1:C9.name | $x$sheet.A1 | $sheet."55- var spans []string56- for _, s := range RangeSpans(src) {57- spans = append(spans, src[s[0]:s[1]])190+ if !slices.Equal(p.Refs(), []string{"x"}) {191+ t.Errorf("refs %v", p.Refs())58192 }59- if !slices.Equal(spans, []string{"$sheet.A1:C9", "$sheet.'Q1 data'!B2:B5", "$sheet.A1:C9"}) {60- t.Errorf("RangeSpans %q", spans)193+ if !Parse("$selection | math sum").ReadsSelection() || Parse("$selections").ReadsSelection() {194+ t.Error("ReadsSelection")195+ }196+ if got := Parse("$r1 | where x == $in.a and $nu.home | $r_2.x | $selection | $sheet.A1").Refs(); !slices.Equal(got, []string{"r1", "r_2"}) {197+ t.Errorf("refs %v", got)61198 }62199 }63200 @@ -95,7 +232,7 @@ func TestStale(t *testing.T) {
95232 out := func(id int) *Output { return outs[id] }96233 run := func(i int, seq int) {97234 o := &Output{NUON: []byte("x"), Seq: seq, Count: seq, Source: cs[i].Source, Reads: map[string]int{}}98- for _, n := range Refs(cs[i].Pipeline()) {235+ for _, n := range cs[i].Parse().Refs() {99236 if j, ok := Names(cs)[n]; ok && outs[cs[j].ID] != nil {100237 o.Reads[n] = outs[cs[j].ID].Seq101238 }@@ -101,8 +101,9 @@ func skipSpaces(view []byte, i, to int, spaces string) int {
101101 102102 func isSpace(c byte) bool { return c == ' ' || c == '\t' || c == '\r' || c == '\n' }103103 104-// Output is the name of the variable that holds a cell's output in the105-// record a run hands back when it also hands back variables.104+// OutputVar is the name of the variable that holds a cell's output in105+// the record a run hands back when it also hands back variables106+// (nushell.ExecVars).106107 const OutputVar = "__out"107108 108109 // Command is what nu runs for the source: each statement but the last@@ -113,6 +114,18 @@ const OutputVar = "__out"
113114 // last doesn't, those are exports, and the command ends in a record of114115 // the output (as __out) and each of them.115116 func (s Source) Command() (cmd string, exports []string, ranges []SheetRef) {117+ return s.command(true)118+}119+120+// StreamCommand is what nu runs for the source as a stream: Command121+// without the record, so the last statement's values stream as they122+// come, and the names the statements before it assign stay the run's.123+func (s Source) StreamCommand() string {124+ cmd, _, _ := s.command(false)125+ return cmd126+}127+128+func (s Source) command(record bool) (cmd string, exports []string, ranges []SheetRef) {116129 var edits []edit117130 for _, r := range s.Ranges {118131 ref := s.Text[r[0]+len(sheetVar) : r[1]]@@ -139,6 +152,9 @@ func (s Source) Command() (cmd string, exports []string, ranges []SheetRef) {
139152 if last.Name != "" {140153 edits = append(edits, edit{last.From, last.Body, ""})141154 }155+ if !record {156+ exports = nil157+ }142158 if len(exports) > 0 {143159 var rec strings.Builder144160 rec.WriteString(")")@@ -9,8 +9,8 @@ package nushell
99 1010 import (1111 "bytes"12- "encoding/base64"1312 "context"13+ "encoding/base64"1414 "errors"1515 "fmt"1616 "io"@@ -53,6 +53,42 @@ func TestExec(t *testing.T) {
5353 }5454 }5555 56+// A run handing back variables prints a line for each after what the57+// pipeline printed itself.58+func TestExecVars(t *testing.T) {59+ job := Job{Command: "x", Vars: []string{"files"}}60+ f := &fake{out: "printed\n__out WzEsIDJd\nfiles WzNd\n"}61+ out, vars, err := ExecVars(context.Background(), f, job, time.Second, 0)62+ if err != nil || string(out) != "[1, 2]" || string(vars["files"]) != "[3]" || len(vars) != 1 {63+ t.Errorf("%q %q %v", out, vars, err)64+ }65+ if !strings.Contains(f.script, `items {|k, v|`) || strings.Contains(f.script, "| to nuon\n") {66+ t.Errorf("script %s", f.script)67+ }68+ if _, _, err := ExecVars(context.Background(), &fake{out: "[1, 2]"}, job, time.Second, 0); err == nil {69+ t.Error("no variables came back, and no error")70+ }71+ out, vars, err = ExecVars(context.Background(), &fake{out: "[1, 2]"}, Job{Command: "x"}, time.Second, 0)72+ if err != nil || string(out) != "[1, 2]" || vars != nil {73+ t.Errorf("without variables %q %v %v", out, vars, err)74+ }75+}76+77+// The real nu hands back a cell's variables beside its output.78+func TestNuVars(t *testing.T) {79+ if _, err := exec.LookPath("nu"); err != nil {80+ t.Skip("nu isn't installed")81+ }82+ job := Job{83+ Command: "let files = [[name, size]; [a, 2kb], [b, 10b]]\n{__out: ($files | where size > 1kb | get name), files: ($files | length)}",84+ Vars: []string{"files"},85+ }86+ out, vars, err := ExecVars(context.Background(), Nu{}, job, 30*time.Second, 0)87+ if err != nil || string(out) != "[a]" || string(vars["files"]) != "2" {88+ t.Errorf("%q %q %v", out, vars, err)89+ }90+}91+5692 func TestMessage(t *testing.T) {5793 stderr := "Error: nu::shell::column_not_found\n\n × Cannot find column 'xyz'\n ╭─[source:1:6]\n"5894 if got := Message(stderr, errors.New("exit status 1")); got != "Cannot find column 'xyz'" {@@ -167,8 +167,11 @@ func (w *Workbook) convertOld(old []oldRegion) {
167167 for _, o := range old {168168 fr := o.fr169169 src := fr.Command170- if fr.Input != "" {170+ switch {171+ case fr.Input != "":171172 src = "$sheet." + quoteRef(fr.Input) + " | do {\n" + src + "\n}"173+ case len(notebook.Parse(src).Stmts) > 1: // one statement, so the name is its output's174+ src = "do {\n" + src + "\n}"172175 }173176 nb.regions.cells = append(nb.regions.cells, notebook.Cell{ID: w.NewCellID(), Source: fr.Name + " = " + src})174177 at, _ := ParseAddr(fr.At)@@ -267,7 +267,8 @@ func TestOldNotebookConverted(t *testing.T) {
267267 {"name": "Shell 1", "notebook": true, "cells": {}, "regions": [268268 {"name": "r1", "command": "ls", "at": "A1", "rows": 3, "cols": 2},269269 {"name": "big", "command": "$r1 | where size > 1kb", "at": "A6", "reads": ["r1"]},270- {"name": "sel", "command": "$in | math sum", "at": "A9", "input": "Data 1!A1:A4"}270+ {"name": "sel", "command": "$in | math sum", "at": "A9", "input": "Data 1!A1:A4"},271+ {"name": "two", "command": "let x = 1\n$x + 1", "at": "A12"}271272 ]},272273 {"name": "Data 1", "cells": {"B1": "=SUM(nu.big)"}}273274 ]}`@@ -280,7 +281,7 @@ func TestOldNotebookConverted(t *testing.T) {
280281 t.Fatalf("notebook %v at %d", nb, w.Index(nb))281282 }282283 cells := nb.NotebookCells()283- want := []string{"r1 = ls", "big = $r1 | where size > 1kb", "sel = $sheet.'Data 1'!A1:A4 | do {\n$in | math sum\n}"}284+ want := []string{"r1 = ls", "big = $r1 | where size > 1kb", "sel = $sheet.'Data 1'!A1:A4 | do {\n$in | math sum\n}", "two = do {\nlet x = 1\n$x + 1\n}"}284285 for i, c := range cells {285286 if c.Source != want[i] {286287 t.Errorf("cell %d: %q, want %q", i+1, c.Source, want[i])