The highlighter and nu's prelude read names, comments and raw strings from notebook.Parse, masking each name = in place; the context line says which names a cell sets; a multi-statement cell streams its last statement. UI, nbview and e2e tests (real nu) for the owner's cell.
- commit
- 02a76805863e2aefa4ac6a944fe164a49dc8bbe2
- parent
- a164d6787e3770517fa77814c86cd7082ac90b89
- author
- Dami <dami.etoile@gmail.com>
- kind
- feat risk score 2.1 of 3, Jev's estimate
| M | e2e/ | 29 | ++++++++ |
| M | internal/ | 10 | ++- |
| M | internal/ | 11 | ++- |
| M | internal/ | 9 | +- |
| A | internal/ | 112 | ++++++++++++++++++++++++++++++ |
| M | internal/ | 2 | +- |
| M | internal/ | 12 | ++- |
| A | internal/ | 59 | ++++++++++++++++ |
| M | internal/ | 7 | -- |
@@ -46,6 +46,35 @@ func TestNotebookWithNu(t *testing.T) {
4646 s.waitFor("[1]:")4747 }4848 49+// A cell of two statements, the second reading the variable the first50+// assigns, runs with the real nu: its output is the second's, and a51+// later cell reads the variable.52+func TestNotebookStatementsWithNu(t *testing.T) {53+ needNu(t)54+ dir := t.TempDir()55+ os.WriteFile(filepath.Join(dir, "big.txt"), []byte(strings.Repeat("x", 3000)), 0o644)56+ os.WriteFile(filepath.Join(dir, "small.txt"), []byte("x"), 0o644)57+ os.Mkdir(filepath.Join(dir, "sub"), 0o755)58+ s := startWith(t, options{dir: dir, startsOn: "EDIT"}, "nu")59+ s.keys("files = ls | where type == file # not sub", "<enter>", "$files | where size > 1kb | sort-by size --reverse", "<shift+enter>")60+ s.waitFor("Out[1]:")61+ s.waitFor("big.txt")62+ s.keys("<enter>", "$files | length", "<shift+enter>")63+ s.waitFor("Out[2]:")64+ scr := s.screen()65+ if strings.Contains(scr, "failed") || strings.Contains(scr, "small.txt") {66+ t.Errorf("screen:\n%s", scr)67+ }68+ s.eventually("the second cell reads 2 files", func() bool {69+ for _, l := range strings.Split(s.screen(), "\n") {70+ if strings.Contains(l, "Out[2]:") && strings.HasSuffix(strings.TrimSpace(l), " 2") {71+ return true72+ }73+ }74+ return false75+ })76+}77+4978 func TestNotebookSendsToSheetWithNu(t *testing.T) {5079 needNu(t)5180 s := startWith(t, options{startsOn: "EDIT"}, "nu")@@ -21,13 +21,17 @@ import (
2121 // ($selection) and ranges of sheets ($sheet.A1:C9), each handed to nu as2222 // NUON, never spliced into the pipeline.2323 24-// jobFor is what running cell c takes: the outputs, linked files and25-// ranges it reads, as NUON.26-func (m *Model) jobFor(s *sheet.Sheet, c notebook.Cell, run *nbRun) (nushell.Job, error) {24+// jobFor is what running cell c takes: the outputs, variables, linked25+// files and ranges it reads, as NUON. As a stream, the cell hands back26+// no variables but its output (notebook.Source.StreamCommand).27+func (m *Model) jobFor(s *sheet.Sheet, c notebook.Cell, run *nbRun, stream bool) (nushell.Job, error) {2728 cells := s.NotebookCells()2829 i := slices.IndexFunc(cells, func(x notebook.Cell) bool { return x.ID == c.ID })2930 p, err := notebook.Prepare(cells, i, m.book().Output)3031 job := nushell.Job{Command: p.Command, Vars: p.Exports, Tables: p.Tables, Config: m.configBool("nu-config", false)}32+ if stream {33+ job.Command, job.Vars = p.Stream, nil34+ }3135 run.reads = p.Reads3236 if err != nil {3337 return job, err@@ -1,6 +1,7 @@
11 package ui22 33 import (4+ "slices"45 "strings"56 67 "github.com/FelineStateMachine/012/internal/sheet"@@ -63,8 +64,14 @@ func (m *Model) syncLang(s *sheet.Sheet, v *nbview.View) {
6364 func (m *Model) nbWords(s *sheet.Sheet) []nbview.Word {6465 var out []nbview.Word6566 for _, c := range s.NotebookCells() {66- if n := c.Name(); n != "" {67- out = append(out, nbview.Word{Text: "$" + n, Desc: "a cell's output"})67+ name := c.Name()68+ if name != "" {69+ out = append(out, nbview.Word{Text: "$" + name, Desc: "a cell's output"})70+ }71+ for _, n := range c.Parse().Assigned() {72+ if n != name && !slices.ContainsFunc(out, func(w nbview.Word) bool { return w.Text == "$"+n }) {73+ out = append(out, nbview.Word{Text: "$" + n, Desc: "a cell's variable"})74+ }6875 }6976 }7077 for _, t := range m.book().Sheets() {@@ -107,6 +107,7 @@ type (
107107 nbDoneMsg struct {108108 run *nbRun109109 nuon []byte110+ vars map[string][]byte // the variables it handed back110111 err error111112 }112113 // nbWordsMsg brings nu's command names.@@ -294,7 +295,7 @@ func (m *Model) nextCell() tea.Cmd {
294295 }295296 m.stopStream(q.id) // run once, it no longer follows296297 run := &nbRun{nbQueued: q, source: c.Source, start: time.Now()}297- job, err := m.jobFor(q.s, c, run)298+ job, err := m.jobFor(q.s, c, run, false)298299 if err != nil {299300 m.failCell(q, c, err.Error())300301 continue@@ -336,13 +337,13 @@ func (m *Model) startCell(run *nbRun, job nushell.Job) tea.Cmd {
336337 exec := func() tea.Msg {337338 defer cancel()338339 span := parent.Start("nu", slog.Int("tables", len(job.Tables)))339- out, err := nushell.Exec(ctx, runner, job, timeout, maxOutput)340+ out, vars, err := nushell.ExecVars(ctx, runner, job, timeout, maxOutput)340341 if err != nil {341342 span.Fail(err)342343 } else {343344 span.End(slog.Int("bytes", len(out)))344345 }345- return nbDoneMsg{run: run, nuon: out, err: err}346+ return nbDoneMsg{run: run, nuon: out, vars: vars, err: err}346347 }347348 return tea.Batch(m.roomOwned(exec), m.tickCell(run.gen))348349 }@@ -388,7 +389,7 @@ func (m *Model) finishCell(msg nbDoneMsg) tea.Cmd {
388389 }389390 m.nb.runs.queue = nil390391 } else {391- o.NUON = msg.nuon392+ o.NUON, o.Vars = msg.nuon, msg.vars392393 if o.NUON == nil {393394 o.NUON = []byte{}394395 }@@ -0,0 +1,112 @@
1+package ui2+3+import (4+ "encoding/base64"5+ "slices"6+ "strings"7+ "testing"8+9+ "github.com/FelineStateMachine/012/internal/notebook"10+)11+12+// ownersCell is a cell of two statements: files is its own variable,13+// read by its second line.14+const ownersCell = "files = ls | where type == file\n$files | where size > 1kb | sort-by size --reverse"15+16+// varsOut is what nu prints for a run handing back variables: a line17+// for the output and each variable, its NUON as base64.18+func varsOut(out string, vars ...string) string {19+ enc := func(s string) string { return base64.StdEncoding.EncodeToString([]byte(s)) }20+ lines := []string{"__out " + enc(out)}21+ for i := 0; i+1 < len(vars); i += 2 {22+ lines = append(lines, vars[i]+" "+enc(vars[i+1]))23+ }24+ return strings.Join(lines, "\n") + "\n"25+}26+27+// nuCommand is what nu runs for src.28+func nuCommand(src string) string {29+ cmd, _, _ := notebook.Parse(src).Command()30+ return cmd31+}32+33+// The owner's cell runs as a script: files is the cell's own, not a cell34+// it reads, its output is its last line's, it has no name, and a later35+// cell reads $files from its run.36+func TestNotebookStatements(t *testing.T) {37+ big := "[[name, size]; [c.txt, 5kb], [a.txt, 2kb]]"38+ m, nu := notebookModel(t, map[string]string{39+ nuCommand(ownersCell): varsOut(big, "files", lsOut),40+ "$files | length": "3",41+ })42+ write(t, m, ownersCell)43+ write(t, m, "$files | length")44+ press(t, m, "<ctrl+enter>") // the second runs the first first45+ if got := nu.ran(); len(got) != 2 || !strings.HasPrefix(got[0], "let files = ls") {46+ t.Fatalf("ran %q", got)47+ }48+ if !slices.Equal(nu.jobs[0].Vars, []string{"files"}) || len(nu.jobs[0].Tables) != 0 {49+ t.Errorf("the first job %+v", nu.jobs[0])50+ }51+ if got := string(nu.jobs[1].Tables["files"]); got != lsOut {52+ t.Errorf("$files reached the second cell as %q", got)53+ }54+ s := screen(m)55+ if strings.Contains(s, "failed") || strings.Contains(s, "─ files ─") || !strings.Contains(s, "c.txt") {56+ t.Errorf("screen:\n%s", s)57+ }58+ cs := m.sheet.NotebookCells()59+ if o := m.book().Output(cs[0].ID); o == nil || string(o.NUON) != big || string(o.Vars["files"]) != lsOut {60+ t.Errorf("output %+v", o)61+ }62+ press(t, m, "<up>", "<up>", "<up>")63+ if i, _ := m.nbView().Selected(); i != 0 {64+ t.Fatalf("selected %d", i)65+ }66+ if got := line(m, contextLine); !strings.Contains(got, "sets $files") || strings.Contains(got, "reads") {67+ t.Errorf("context line %q", got)68+ }69+ // Running it again makes the reader stale.70+ press(t, m, "<ctrl+enter>")71+ if st := m.cellState(m.sheet, cs[1].ID); !st.Stale {72+ t.Error("the reader isn't stale")73+ }74+ // Naming the cell names its last statement.75+ press(t, m, "n")76+ send(m, pasteMsg("big"))77+ press(t, m, "<enter>")78+ if got := m.sheet.NotebookCells()[0].Source; got != "files = ls | where type == file\nbig = $files | where size > 1kb | sort-by size --reverse" {79+ t.Errorf("named %q", got)80+ }81+}82+83+// Comments are nu's: a commented-out assignment names nothing and a84+// commented-out $name reads nothing.85+func TestNotebookComments(t *testing.T) {86+ src := "# files = ls\nls # | where name == $nope"87+ m, nu := notebookModel(t, map[string]string{nuCommand(src): lsOut})88+ write(t, m, src)89+ press(t, m, "<ctrl+enter>")90+ if len(nu.jobs) != 1 || len(nu.jobs[0].Tables) != 0 || strings.Contains(nu.jobs[0].Command, "#") {91+ t.Fatalf("ran %+v", nu.jobs)92+ }93+ if strings.Contains(screen(m), "failed") || m.sheet.NotebookCells()[0].Name() != "" {94+ t.Errorf("screen:\n%s", screen(m))95+ }96+}97+98+// A cell of several statements streams its last one's values, its own99+// variables bound first and none handed back.100+func TestNotebookStreamStatements(t *testing.T) {101+ streamPolls = nil102+ m, _ := notebookModel(t, nil)103+ nu := newStreamNu()104+ m.SetShellRunner(nu)105+ write(t, m, "log = open app.log\n$log | lines")106+ press(t, m, "f")107+ defer m.stopCells()108+ script := <-nu.scripts109+ if !strings.Contains(script, "let log = open app.log\n$log | lines") || strings.Contains(script, "__out") {110+ t.Errorf("stream script %s", script)111+ }112+}@@ -77,7 +77,7 @@ func (m *Model) startStream(q nbQueued) tea.Cmd {
7777 return nil7878 }7979 run := &nbRun{nbQueued: q, source: c.Source, start: time.Now()}80- job, err := m.jobFor(q.s, c, run)80+ job, err := m.jobFor(q.s, c, run, true)8181 if err != nil {8282 m.failCell(q, c, err.Error())8383 return nil@@ -26,8 +26,9 @@ type prepared struct {
2626 2727 // prepare masks src and declares what it reads: `sales = ls` is asked2828 // about as ` ls`, $sheet.A1:C9 as $__s1_______, and the prelude29-// declares $__s1, $selection, $sheet, names and the names the cell30-// assigns, each `let x: any = []` so nu takes it for whatever it holds.29+// declares $__s1, $selection, $sheet, names and the names the cell's30+// lines assign for the lines after them, each `let x: any = []` so nu31+// takes it for whatever it holds.3132 func prepare(src string, names []string) prepared {3233 b := []byte(src)3334 parsed := notebook.Parse(src)@@ -39,7 +40,12 @@ func prepare(src string, names []string) prepared {
3940 }4041 }4142 }42- names = append(parsed.Assigned(), names...)43+ names = slices.Clone(names)44+ for k, st := range parsed.Stmts {45+ if st.Name != "" && k < len(parsed.Stmts)-1 { // assigned for the lines after46+ names = append(names, st.Name)47+ }48+ }4349 var pre strings.Builder4450 declared := map[string]bool{}4551 declare := func(name, value string) {@@ -0,0 +1,59 @@
1+package nbview2+3+import (4+ "context"5+ "fmt"6+ "strings"7+ "testing"8+9+ "github.com/FelineStateMachine/012/internal/notebook"10+)11+12+// A cell of several statements: each name assigned shows as a variable,13+// a comment's text as a comment, whatever it holds, and # in a string14+// or a raw string is the string's.15+func TestHighlightsStatements(t *testing.T) {16+ src := "files = ls # all\n# big = $files\nbig = $files | where name != \"a#b\" | append r#'c # d'#"17+ var got []string18+ for _, s := range (Tokens{}).Highlight(context.Background(), src) {19+ got = append(got, fmt.Sprintf("%s:%d", src[s.From:s.To], s.Kind))20+ }21+ want := fmt.Sprint([]string{"files:2", "=:5", "ls:0", "# all:6", "# big = $files:6", "big:2", "=:5", "$files:2", "|:5", "where:0",22+ "!=:5", `"a#b":1`, "|:5", "append:0", "r#'c # d'#:1"})23+ if fmt.Sprint(got) != want {24+ t.Errorf("spans %v\nwant %v", got, want)25+ }26+}27+28+// nu is asked about a cell of several statements with each `name =`29+// blanked, every byte in its place, and the names its lines assign for30+// the lines after declared.31+func TestPrepareStatements(t *testing.T) {32+ src := "files = ls # x = 1\nbig = $files | first"33+ p := prepare(src, []string{"sales"})34+ if body := p.text[p.at:]; body != " ls # x = 1\n $files | first" {35+ t.Errorf("masked %q", body)36+ }37+ pre := p.text[:p.at]38+ if !strings.Contains(pre, "let files: any = []\n") || strings.Contains(pre, "let big") || strings.Contains(pre, "let x") {39+ t.Errorf("prelude %q", pre)40+ }41+}42+43+// The context line says what a cell of several statements reads and44+// sets: not its own variables.45+func TestHeadStatements(t *testing.T) {46+ h := newHost("files = ls | where type == file\n$files | where size > 1kb | sort-by size --reverse", "$files | length")47+ v := newView(h, 60, 12)48+ v.Select(0, false)49+ if name, text := v.Head(); name != "cell 1" || text != "a nushell pipeline; sets $files" {50+ t.Errorf("head %q %q", name, text)51+ }52+ v.Select(1, false)53+ if _, text := v.Head(); text != "reads $files" {54+ t.Errorf("head of the reader %q", text)55+ }56+ if (notebook.Cell{Source: h.cells[0].Source}).Name() != "" {57+ t.Error("the cell is named")58+ }59+}@@ -194,13 +194,6 @@ func wordEnd(src string, i int) int {
194194 return i195195 }196196 197-func lineEnd(src string, i int) int {198- for i < len(src) && src[i] != '\n' {199- i++200- }201- return i202-}203-204197 // quoted is where a string starting at i ends: its closing quote, a205198 // double-quoted string's escapes skipped, or the end.206199 func quoted(src string, i int) int {