- commit
- 3f5a5322a1356387aae8525bb8d2f2001d797d3d
- parent
- c7a0f1b1d65aaad0b383e13b3e7fb60a7d6b2805
- parent
- e2ec615e5dfa15f28d5ddc7e606184eaf3e82cac
- author
- Dami <dami.etoile@gmail.com>
| M | ROADMAP.md | 15 | +- |
| M | docs/ | 55 | +++++- |
| M | e2e/ | 29 | +++ |
| M | e2e/ | 22 | ++- |
| M | internal/ | 67 | +++--- |
| A | internal/ | 43 | ++++ |
| M | internal/ | 2 | +- |
| M | internal/ | 131 | +++++++++---- |
| A | internal/ | 56 | ++++++ |
| M | internal/ | 56 | +++--- |
| M | internal/ | 207 | +++++++++++++++++---- |
| M | internal/ | 103 | +---------- |
| A | internal/ | 297 | ++++++++++++++++++++++++++++++ |
| A | internal/ | 193 | +++++++++++++++++++ |
| M | internal/ | 40 | ++++- |
| M | internal/ | 36 | ++++ |
| M | internal/ | 5 | +- |
| M | internal/ | 5 | +- |
| M | internal/ | 39 | ++-- |
| M | internal/ | 11 | +- |
| M | internal/ | 9 | +- |
| A | internal/ | 112 | +++++++++++ |
| M | internal/ | 2 | +- |
| M | internal/ | 21 | ++- |
| M | internal/ | 48 | +++-- |
| A | internal/ | 59 | ++++++ |
| M | internal/ | 39 | +++- |
@@ -48,9 +48,11 @@ shown by `012 diff`, under the same trust rules as macros. Live mode is
4848 |---|---|---|4949 | Live mode: `012 --listen` and `012 mcp --attach`; the agent's cursor and name in the grid, its changes arriving as a suggestion (marked cells, accepted or rejected whole or by cell) unless direct edits are allowed, its own undo, a scope (sheet, range, read-only), elicitation to ask the person | Coworking with an agent in the grid | M to L |5050 51-## Later: other transports (shelved)51+## Shelved5252 53-Explored, not scheduled: carrying the same shared sessions over iroh53+Explored, not scheduled; the fit is still open.54+55+Other transports: carrying the same shared sessions over iroh5456 tickets (the Go transport in `FelineStateMachine/allons`5557 `local/transport/iroh`, which needs cgo and a prebuilt iroh-ffi archive, so5658 it would sit behind a build tag) and the web (`NimbleMarkets/go-booba`@@ -59,6 +61,15 @@ v2 support unverified). They would reuse the rooms of `012 serve`, whose
5961 server orders every participant's operations, rather than a design of6062 their own.6163 64+SQL cells and connections: SQL as a second cell language beside nu, on65+an in-process SQLite (pure Go) over the workbook's tables, outputs and66+linked files by default, with saved connections to SQLite files,67+Postgres and MySQL whose secrets stay in the keychain. Results would be68+outputs like a nu cell's (grid, `G`, `$name`, stale), `@name`69+parameters bound from cells, and `012 query` and an MCP `query` tool70+for scripts and agents. DuckDB would only be an external kernel, as it71+needs cgo.72+6273 ## Shipped6374 6475 **Engine**@@ -292,9 +292,11 @@ it the context line says what nu said:
292292 Tab asks `nu --ide-complete` too, after the notebook's own names, so it293293 completes flags, subcommands and paths as well as cells and commands.294294 nu doesn't know the variables a notebook binds, so it's asked about the295-cell with `$name` for each named cell and linked file, `$selection` and296-`$sheet` declared before it; a `$sheet.A1:C9` range reads as one297-variable, and `name =` isn't part of what nu reads.295+cell with `$name` for each name the cells assign and each linked file,296+`$selection` and `$sheet` declared before it; a `$sheet.A1:C9` range297+reads as one variable, and the cell's `name =`s ([Names and298+$name](#names-and-name)) aren't part of what nu reads, so what nu says299+is where it points in the cell.298300 299301 Questions to nu run in the background and never on each key: one300302 process for each question after a pause, at most two at a time, each@@ -346,7 +348,9 @@ printed, and the toolbar how many cells stream; streams run beside the
346348 cells run once, each on its own, without `nu-timeout`. `ii` (or `■`)347349 stops them all, keeping what they printed; running the cell once (`r`)348350 stops its stream first. A stream that ends on its own keeps its output349-as a run does, and one that fails says why.351+as a run does, and one that fails says why. A cell of several statements352+streams its last one's values; the names its earlier lines assign stay353+its own, as a stream hands later cells only its output.350354 351355 The output keeps a stream's last 10,000 values; a sheet it was sent to352356 keeps every row, up to `max-cells`. The rows arrive as the change@@ -355,10 +359,39 @@ undo history.
355359 356360 ## Names and $name357361 358-A cell that starts `name =` gives its output that name; `n` names a359-cell for you. Later cells read the output as `$name`, a nushell360-variable holding the value, and formulas on any sheet as `nu.name`, or361-a column of it as `name[column]`, once it's [sent to a sheet](#send-to-a-sheet):362+A cell holds one statement or several, a line each (or apart by `;`),363+as a nushell script does; a line starting with `|` goes on with the364+pipeline above it. `name =` before a pipeline, on any line, assigns `name` as365+nushell's `let` does, and the cell's later lines read366+it as `$name`. `#` starts a comment to the end of the line, outside a367+string, as in nushell: a commented-out line assigns and reads nothing.368+369+As in nushell and Jupyter, the cell's output is its last statement's370+value, and the names work across cells by one rule:371+372+- **The cell's name is its output's.** When the last statement is373+ an assignment (`name =`), the cell is named `name`; otherwise it has no374+ name until `n` names it, which puts `name =` before its last375+ statement. Other cells read the output as `$name`, and formulas on376+ any sheet as `nu.name`, or a column of it as `name[column]`, once377+ it's [sent to a sheet](#send-to-a-sheet).378+- **Every name assigned goes to later cells.** `$files` in another379+ cell reads the `files` a cell's earlier line assigned, as it was at380+ the end of that cell's run. A cell's name comes first: `$sales` reads381+ the cell named `sales`, and only when no cell is named so, the first382+ cell assigning `sales`.383+- **A cell's own names come first in it.** `$files` after the line384+ assigning `files` reads the cell's own, not another cell's, and isn't385+ a cell it reads.386+387+```nu388+files = ls | where type == file389+$files | where size > 1kb | sort-by size --reverse390+```391+392+This cell has no name (its last line assigns none) and reads no cell;393+its output is the big files, and later cells read `$files` too. Split394+over cells, each named after its output:362395 363396 ```nu364397 files = ls@@ -366,6 +399,12 @@ big = $files | where size > 1kb
366399 $big | get name | str join ", "367400 ```368401 402+The context line says what a cell reads, the names it sets for later403+cells and its own name (`reads $files; sets $big`). A file keeps each404+cell's output, not the other names it assigned: after opening one, a405+cell reading such a name runs the cell assigning it first, as it does406+a cell that hasn't run.407+369408 A name is letters, digits and `_`, not starting with a digit or `__`;370409 `in`, `env`, `nu`, `it`, `selection` and `sheet` are taken. Two cells371410 can't share a name: the second says so and doesn't run until it's@@ -46,6 +46,35 @@ func TestNotebookWithNu(t *testing.T) {
4646 s.waitFor("[1]:")4747 }4848 49+// A cell of two statements, the second reading the variable the first50+// assigns, runs with the real nu: its output is the second's, and a51+// later cell reads the variable.52+func TestNotebookStatementsWithNu(t *testing.T) {53+ needNu(t)54+ dir := t.TempDir()55+ os.WriteFile(filepath.Join(dir, "big.txt"), []byte(strings.Repeat("x", 3000)), 0o644)56+ os.WriteFile(filepath.Join(dir, "small.txt"), []byte("x"), 0o644)57+ os.Mkdir(filepath.Join(dir, "sub"), 0o755)58+ s := startWith(t, options{dir: dir, startsOn: "EDIT"}, "nu")59+ s.keys("files = ls | where type == file # not sub", "<enter>", "$files | where size > 1kb | sort-by size --reverse", "<shift+enter>")60+ s.waitFor("Out[1]:")61+ s.waitFor("big.txt")62+ s.keys("<enter>", "$files | length", "<shift+enter>")63+ s.waitFor("Out[2]:")64+ scr := s.screen()65+ if strings.Contains(scr, "failed") || strings.Contains(scr, "small.txt") {66+ t.Errorf("screen:\n%s", scr)67+ }68+ s.eventually("the second cell reads 2 files", func() bool {69+ for _, l := range strings.Split(s.screen(), "\n") {70+ if strings.Contains(l, "Out[2]:") && strings.HasSuffix(strings.TrimSpace(l), " 2") {71+ return true72+ }73+ }74+ return false75+ })76+}77+4978 func TestNotebookSendsToSheetWithNu(t *testing.T) {5079 needNu(t)5180 s := startWith(t, options{startsOn: "EDIT"}, "nu")@@ -59,7 +59,7 @@ func TestScreens(t *testing.T) {
5959 if got != string(want) {6060 actual := filepath.Join(t.TempDir(), sc.name+".html")6161 os.WriteFile(actual, []byte(got), 0o644)62- t.Errorf("screen %s changed; if intended, run make screens and review the gallery.\nplain text now:\n%s", sc.name, s.screen())62+ t.Errorf("screen %s changed; if intended, run make screens and review the gallery.\n%s\nplain text now:\n%s", sc.name, firstDiff(string(want), got), s.screen())6363 }6464 })6565 }@@ -194,3 +194,23 @@ h2{font:500 13px ui-monospace,monospace;color:#aaa;margin:0 0 8px}
194194 b.WriteString("</div>\n")195195 return os.WriteFile(filepath.Join(dir, "gallery.html"), []byte(b.String()), 0o644)196196 }197+198+// firstDiff shows the first line where a golden screen's HTML and the199+// screen now part, which the plain text can't show when only a color or200+// style changed.201+func firstDiff(want, got string) string {202+ w, g := strings.Split(want, "\n"), strings.Split(got, "\n")203+ for i := 0; i < len(w) || i < len(g); i++ {204+ var a, b string205+ if i < len(w) {206+ a = w[i]207+ }208+ if i < len(g) {209+ b = g[i]210+ }211+ if a != b {212+ return fmt.Sprintf("first difference, line %d:\n want %s\n got %s", i+1, a, b)213+ }214+ }215+ return "no line differs"216+}@@ -50,7 +50,7 @@ func runNotebook(ctx context.Context, w *sheet.Workbook, s *sheet.Sheet, o Noteb
5050 failed = append(failed, fmt.Sprintf("%s cell %d reads itself, through the cells it reads: it doesn't run", s.Name(), i+1))5151 }5252 for n, i := range order {53- if err := runCell(ctx, w, s, cells, i, n+1, o); err != nil {53+ if err := runCell(ctx, w, cells, i, n+1, o); err != nil {5454 return append(failed, fmt.Sprintf("%s cell %d: %s", s.Name(), i+1, err))5555 }5656 }@@ -59,7 +59,7 @@ func runNotebook(ctx context.Context, w *sheet.Workbook, s *sheet.Sheet, o Noteb
5959 6060 // runCell runs cell i, the count-th run, keeping its output or why it6161 // failed.62-func runCell(ctx context.Context, w *sheet.Workbook, s *sheet.Sheet, cells []notebook.Cell, i, count int, o NotebookOptions) error {62+func runCell(ctx context.Context, w *sheet.Workbook, cells []notebook.Cell, i, count int, o NotebookOptions) error {6363 c := cells[i]6464 out := ¬ebook.Output{Count: count, Source: c.Source}6565 defer func() {@@ -70,14 +70,14 @@ func runCell(ctx context.Context, w *sheet.Workbook, s *sheet.Sheet, cells []not
7070 out.Err = why7171 return errors.New(why)7272 }73- job, reads, err := jobFor(w, s, cells, c, o.NuConfig)73+ job, reads, err := jobFor(w, cells, i, o.NuConfig)7474 if err != nil {7575 out.Err = err.Error()7676 return err7777 }7878 out.Reads = reads7979 start := time.Now()80- data, err := nushell.Exec(ctx, o.Runner, job, o.Timeout, maxOutput)80+ data, vars, err := nushell.ExecVars(ctx, o.Runner, job, o.Timeout, maxOutput)8181 out.Took = time.Since(start)8282 if err != nil {8383 out.Err = err.Error()@@ -87,47 +87,31 @@ func runCell(ctx context.Context, w *sheet.Workbook, s *sheet.Sheet, cells []not
8787 }8888 return errors.New(out.Err)8989 }90- out.NUON = append([]byte{}, data...)90+ out.NUON, out.Vars = append([]byte{}, data...), vars9191 return nil9292 }9393 94-// jobFor is what running cell c takes: the outputs, linked files and95-// ranges it reads, as NUON, and the outputs it read by name.96-func jobFor(w *sheet.Workbook, s *sheet.Sheet, cells []notebook.Cell, c notebook.Cell, config bool) (nushell.Job, map[string]int, error) {97- pipeline, ranges := notebook.Bind(c.Pipeline())98- job := nushell.Job{Command: pipeline, Tables: map[string][]byte{}, Config: config}99- reads := map[string]int{}100- names := notebook.Names(cells)101- for _, name := range notebook.Refs(pipeline) {102- if j, ok := names[name]; ok {103- o := w.Output(cells[j].ID)104- switch {105- case o == nil || o.Unsaved:106- return job, nil, fmt.Errorf("it reads $%s, which hasn't run", name)107- case o.Failed():108- return job, nil, fmt.Errorf("it reads $%s, which failed", name)109- }110- job.Tables[name], reads[name] = o.NUON, o.Seq111- continue112- }94+// jobFor is what running cell i takes: the outputs, variables, linked95+// files and ranges it reads, as NUON, and the outputs it read by name.96+func jobFor(w *sheet.Workbook, cells []notebook.Cell, i int, config bool) (nushell.Job, map[string]int, error) {97+ p, err := notebook.Prepare(cells, i, w.Output)98+ job := nushell.Job{Command: p.Command, Vars: p.Exports, Tables: p.Tables, Config: config}99+ if err != nil {100+ return job, nil, err101+ }102+ for _, name := range p.Others {113103 if t, r, ok := w.Region(name); ok && r.Linked() && r.Name == name {114- rng, shown := t.RegionTable(r.Name)115- snap := fileio.Snap(t, rng, r.Name)116- if !shown {117- snap = fileio.Snap(t, sheet.Rect{From: r.At, To: r.At}, r.Name)118- snap.Cells = nil119- }120- data, err := encodeNUON(snap)104+ data, err := regionNUON(t, r)121105 if err != nil {122106 return job, nil, err123107 }124108 job.Tables[name] = data125109 }126110 }127- if notebook.ReadsSelection(pipeline) {111+ if p.Selection {128112 return job, nil, errors.New("it reads $selection, which only the screen has")129113 }130- for _, ref := range ranges {114+ for _, ref := range p.Ranges {131115 t, rng, err := rangeRef(w, ref.Ref)132116 if err != nil {133117 return job, nil, err@@ -138,7 +122,22 @@ func jobFor(w *sheet.Workbook, s *sheet.Sheet, cells []notebook.Cell, c notebook
138122 }139123 job.Tables[ref.Var] = data140124 }141- return job, reads, nil125+ return job, p.Reads, nil126+}127+128+// regionNUON is a linked file's table as NUON, read whole whatever its129+// filter shows, as on the screen.130+func regionNUON(t *sheet.Sheet, r sheet.Region) ([]byte, error) {131+ rng, ok := t.RegionTable(r.Name)132+ if !ok {133+ rng = sheet.Rect{From: r.At, To: r.At}134+ }135+ snap := fileio.Snap(t, rng, r.Name)136+ snap.HiddenRows = nil137+ if !ok {138+ snap.Cells = nil139+ }140+ return encodeNUON(snap)142141 }143142 144143 // rangeRef is the sheet and range $sheet.ref reads: of the sheet it@@ -0,0 +1,43 @@
1+package headless2+3+import (4+ "context"5+ "encoding/base64"6+ "testing"7+ "time"8+9+ "github.com/FelineStateMachine/012/internal/notebook"10+ "github.com/FelineStateMachine/012/internal/sheet"11+)12+13+// --notebooks runs a cell of several statements as the screen does: its14+// own variable isn't a cell it reads, and a later cell reads it from its15+// run.16+func TestRunNotebookStatements(t *testing.T) {17+ w := sheet.NewBook()18+ nb, err := w.AddNotebook("Notebook", w.Sheet(0))19+ if err != nil {20+ t.Fatal(err)21+ }22+ src := "files = ls | where type == file # the files\n$files | where size > 1kb"23+ nb.SetNotebookCells("add cells", []notebook.Cell{24+ {Kind: notebook.Code, Source: "$files | length"},25+ {Kind: notebook.Code, Source: src},26+ })27+ cmd, _, _ := notebook.Parse(src).Command()28+ enc := func(s string) string { return base64.StdEncoding.EncodeToString([]byte(s)) }29+ nu := fakeNu{30+ cmd: "__out " + enc("[[name]; [b]]") + "\nfiles " + enc("[[name]; [a], [b]]") + "\n",31+ "$files | length": "2",32+ }33+ if failed := RunNotebooks(context.Background(), w, NotebookOptions{Runner: nu, Timeout: time.Second}); len(failed) != 0 {34+ t.Fatalf("failed: %q", failed)35+ }36+ cells := nb.NotebookCells()37+ if o := w.Output(cells[1].ID); o == nil || string(o.NUON) != "[[name]; [b]]" || string(o.Vars["files"]) != "[[name]; [a], [b]]" {38+ t.Errorf("the cell of two statements: %+v", o)39+ }40+ if o := w.Output(cells[0].ID); o == nil || string(o.NUON) != "2" || o.Reads["files"] == 0 {41+ t.Errorf("the cell reading $files: %+v", o)42+ }43+}@@ -41,7 +41,7 @@ func RunCell(ctx context.Context, w *sheet.Workbook, name string, n int, o Noteb
4141 return CellRun{}, fmt.Errorf("%s cell %d is a note, which doesn't run", name, n)4242 }4343 out := CellRun{Notebook: s.Name(), Cell: n, Name: c.Name(), State: "ran"}44- if err := runCell(ctx, w, s, cells, n-1, 1, o); err != nil {44+ if err := runCell(ctx, w, cells, n-1, 1, o); err != nil {4545 out.State, out.Error = "failed", err.Error()4646 return out, nil4747 }@@ -7,12 +7,14 @@ import (
77 )88 99 // The cells' dependency graph: a cell reading $sales depends on the cell10-// named sales. A name belongs to the first code cell giving it; a later11-// one giving the same name can't run until it's renamed. Cells run in12-// the order asked, each after the cells it reads among those running,13-// and a cell reading itself, directly or through others, doesn't run.10+// giving sales (Vars). A cell's name belongs to the first code cell11+// giving it; a later one giving the same name can't run until it's12+// renamed. Cells run in the order asked, each after the cells it reads13+// among those running, and a cell reading itself, directly or through14+// others, doesn't run.1415 15-// Names maps each name to the index of the first code cell giving it.16+// Names maps each cell's name to the index of the first code cell17+// giving it.1618 func Names(cells []Cell) map[string]int {1719 out := map[string]int{}1820 for i, c := range cells {@@ -25,6 +27,21 @@ func Names(cells []Cell) map[string]int {
2527 return out2628 }2729 30+// Vars maps each variable the cells give others to the index of the31+// cell giving it: a cell's name to that cell (Names), and any other32+// name a statement assigns to the first code cell assigning it.33+func Vars(cells []Cell) map[string]int {34+ out := Names(cells)35+ for i, c := range cells {36+ for _, n := range c.Parse().Assigned() {37+ if _, ok := out[n]; !ok {38+ out[n] = i39+ }40+ }41+ }42+ return out43+}44+2845 // Taken says why cell i can't use its name: an earlier cell gives it2946 // (names ignore case there, as formulas' nu.name do), or "".3047 func Taken(cells []Cell, i int) string {@@ -40,13 +57,33 @@ func Taken(cells []Cell, i int) string {
4057 return ""4158 }4259 60+// ref is a variable a cell reads from another: its name, and the index61+// of the cell giving it.62+type ref struct {63+ name string64+ cell int65+}66+67+// refs are the variables cell i reads from other cells, in the order it68+// names them; vars is Vars(cells). A name only it assigns, read before69+// it does, is none.70+func refs(cells []Cell, vars map[string]int, i int) []ref {71+ var out []ref72+ for _, n := range cells[i].Parse().Refs() {73+ if j, ok := vars[n]; ok && j != i {74+ out = append(out, ref{n, j})75+ }76+ }77+ return out78+}79+4380 // Reads returns the indices of the cells cell i reads, in the order it44-// names them.45-func Reads(cells []Cell, names map[string]int, i int) []int {81+// names them; vars is Vars(cells).82+func Reads(cells []Cell, vars map[string]int, i int) []int {4683 var out []int47- for _, n := range Refs(cells[i].Pipeline()) {48- if j, ok := names[n]; ok && j != i && !slices.Contains(out, j) {49- out = append(out, j)84+ for _, r := range refs(cells, vars, i) {85+ if !slices.Contains(out, r.cell) {86+ out = append(out, r.cell)5087 }5188 }5289 return out@@ -56,7 +93,7 @@ func Reads(cells []Cell, names map[string]int, i int) []int {
5693 // among them, in want's order otherwise, and the cells left out for5794 // reading themselves, directly or through others.5895 func Order(cells []Cell, want []int) (order, cycle []int) {59- names := Names(cells)96+ vars := Vars(cells)6097 in := map[int]bool{}6198 for _, i := range want {6299 if cells[i].Kind == Code {@@ -70,7 +107,7 @@ func Order(cells []Cell, want []int) (order, cycle []int) {
70107 if !in[i] || done[i] {71108 continue72109 }73- ready := !slices.ContainsFunc(Reads(cells, names, i), func(j int) bool { return in[j] && !done[j] })110+ ready := !slices.ContainsFunc(Reads(cells, vars, i), func(j int) bool { return in[j] && !done[j] })74111 if ready {75112 order, done[i], progress = append(order, i), true, true76113 }@@ -90,12 +127,12 @@ func Order(cells []Cell, want []int) (order, cycle []int) {
90127 // Dependents returns the cells that read cell i, directly or through91128 // others, in the notebook's order.92129 func Dependents(cells []Cell, i int) []int {93- names := Names(cells)130+ vars := Vars(cells)94131 want := map[int]bool{i: true}95132 for changed := true; changed; {96133 changed = false97134 for j := range cells {98- if !want[j] && cells[j].Kind == Code && slices.ContainsFunc(Reads(cells, names, j), func(k int) bool { return want[k] }) {135+ if !want[j] && cells[j].Kind == Code && slices.ContainsFunc(Reads(cells, vars, j), func(k int) bool { return want[k] }) {99136 want[j], changed = true, true100137 }101138 }@@ -109,21 +146,35 @@ func Dependents(cells []Cell, i int) []int {
109146 return out110147 }111148 149+// Readable reports whether o, the output of cell c, holds the variable150+// name: c's output when c is named so, else a variable it assigns,151+// which a file doesn't keep.152+func Readable(c Cell, name string, o *Output) bool {153+ switch {154+ case o == nil || o.NUON == nil:155+ return false156+ case c.Name() == name:157+ return true158+ }159+ _, ok := o.Vars[name]160+ return ok161+}162+112163 // Inputs returns the cells cell i reads, directly or through others,113-// that have no output, in the notebook's order: what has to run before114-// it can.164+// whose outputs don't hold what it reads (Readable), in the notebook's165+// order: what has to run before it can.115166 func Inputs(cells []Cell, i int, output func(id int) *Output) []int {116- names := Names(cells)167+ vars := Vars(cells)117168 need := map[int]bool{}118169 var visit func(j int)119170 visit = func(j int) {120- for _, k := range Reads(cells, names, j) {121- if need[k] || k == i {171+ for _, r := range refs(cells, vars, j) {172+ if need[r.cell] || r.cell == i {122173 continue123174 }124- if o := output(cells[k].ID); o == nil || o.NUON == nil {125- need[k] = true126- visit(k)175+ if c := cells[r.cell]; !Readable(c, r.name, output(c.ID)) {176+ need[r.cell] = true177+ visit(r.cell)127178 }128179 }129180 }@@ -141,7 +192,7 @@ func Inputs(cells []Cell, i int, output func(id int) *Output) []int {
141192 // them now would give: their source changed since, or an output they142193 // read changed or is stale itself.143194 func Stale(cells []Cell, output func(id int) *Output) map[int]bool {144- names := Names(cells)195+ vars := Vars(cells)145196 all := make([]int, len(cells))146197 for i := range all {147198 all[i] = i@@ -155,18 +206,17 @@ func Stale(cells []Cell, output func(id int) *Output) map[int]bool {
155206 continue156207 }157208 s := o.Source != c.Source158- for _, name := range Refs(c.Pipeline()) {159- j, ok := names[name]209+ read := map[string]bool{}210+ for _, r := range refs(cells, vars, i) {211+ read[r.name] = true160212 seq := 0161- if ok {162- if d := output(cells[j].ID); d != nil {163- seq = d.Seq164- }165- s = s || stale[cells[j].ID]166- }167- if read, had := o.Reads[name]; had || ok {168- s = s || read != seq213+ if d := output(cells[r.cell].ID); d != nil {214+ seq = d.Seq169215 }216+ s = s || stale[cells[r.cell].ID] || o.Reads[r.name] != seq217+ }218+ for name := range o.Reads {219+ s = s || !read[name] // it read a cell it no longer does170220 }171221 if s {172222 stale[c.ID] = true@@ -179,20 +229,17 @@ func Stale(cells []Cell, output func(id int) *Output) map[int]bool {
179229 // have left: made from the source as it is, having read the outputs as180230 // they are, so none is stale until something changes.181231 func Settle(cells []Cell, outputs map[int]*Output) {182- names := Names(cells)183- for _, c := range cells {232+ vars := Vars(cells)233+ for i, c := range cells {184234 o := outputs[c.ID]185235 if o == nil {186236 continue187237 }188238 o.Source, o.Reads = c.Source, map[string]int{}189- for _, name := range Refs(c.Pipeline()) {190- if j, ok := names[name]; ok {191- if d := outputs[cells[j].ID]; d != nil {192- o.Reads[name] = d.Seq193- } else {194- o.Reads[name] = 0195- }239+ for _, r := range refs(cells, vars, i) {240+ o.Reads[r.name] = 0241+ if d := outputs[cells[r.cell].ID]; d != nil {242+ o.Reads[r.name] = d.Seq196243 }197244 }198245 }@@ -0,0 +1,56 @@
1+package notebook2+3+import "fmt"4+5+// Run is what running a code cell takes that the notebook knows: what6+// nu runs, and the variables it reads from other cells. The screen and7+// the headless runner each add what only they know: linked files, the8+// selection and ranges of sheets.9+type Run struct {10+ // Command is what nu runs (Source.Command), and Exports the11+ // variables it hands back beside the output.12+ Command string13+ Exports []string14+ // Stream is what nu runs to stream the cell (Source.StreamCommand).15+ Stream string16+ // Tables are the other cells' outputs and variables it reads, by17+ // name, as NUON, and Reads the Seqs of the outputs they came from.18+ Tables map[string][]byte19+ Reads map[string]int20+ // Others are the names it reads that no cell gives: linked files,21+ // or nothing nu will say so of.22+ Others []string23+ // Ranges are the ranges of sheets it reads, and Selection whether it24+ // reads $selection.25+ Ranges []SheetRef26+ Selection bool27+}28+29+// Prepare works out what running cell i takes, reading the outputs of30+// the cells it reads; the error says why it can't run: what it reads31+// hasn't run or failed.32+func Prepare(cells []Cell, i int, output func(id int) *Output) (Run, error) {33+ src := cells[i].Parse()34+ cmd, exports, ranges := src.Command()35+ run := Run{Command: cmd, Exports: exports, Stream: src.StreamCommand(), Tables: map[string][]byte{}, Reads: map[string]int{}, Ranges: ranges, Selection: src.ReadsSelection()}36+ vars := Vars(cells)37+ for _, name := range src.Refs() {38+ j, ok := vars[name]39+ if !ok || j == i {40+ run.Others = append(run.Others, name)41+ continue42+ }43+ o := output(cells[j].ID)44+ switch {45+ case o.Failed():46+ return run, fmt.Errorf("it reads $%s, which failed", name)47+ case o == nil || o.Unsaved || !Readable(cells[j], name, o):48+ return run, fmt.Errorf("it reads $%s, which hasn't run", name)49+ }50+ run.Tables[name], run.Reads[name] = o.NUON, o.Seq51+ if cells[j].Name() != name {52+ run.Tables[name] = o.Vars[name]53+ }54+ }55+ return run, nil56+}@@ -54,41 +54,37 @@ type Cell struct {
5454 Source string5555 }5656 57-// Name is the name the cell gives its output, from a source of the form58-// `name = pipeline`, or "".59-func (c Cell) Name() string {57+// Parse reads the cell's source; a note cell's has nothing to read.58+func (c Cell) Parse() Source {6059 if c.Kind != Code {61- return ""60+ return Source{Text: c.Source}6261 }63- name, _ := SplitName(c.Source)64- return name62+ return Parse(c.Source)6563 }6664 67-// Pipeline is what runs: the source without its name.68-func (c Cell) Pipeline() string {69- _, p := SplitName(c.Source)70- return p71-}72-73-// SplitName splits `name = pipeline` into the name and the pipeline; a74-// source that doesn't start so has no name. `==` isn't a name's `=`.75-func SplitName(src string) (name, pipeline string) {76- head, rest, ok := strings.Cut(src, "=")77- head = strings.TrimSpace(head)78- if !ok || strings.HasPrefix(rest, "=") || ValidName(head) != nil {79- return "", src80- }81- return head, strings.TrimLeft(rest, " \t")82-}65+// Name is the name the cell gives its output: what its last statement66+// assigns (`name = pipeline`), or "".67+func (c Cell) Name() string { return c.Parse().Name() }8368 84-// WithName is src given the name name, replacing any name it has; ""85-// takes its name away.69+// WithName is src given the name name, as the name its last statement70+// assigns, replacing any it has; "" takes its name away.8671 func WithName(src, name string) string {87- _, p := SplitName(src)88- if name == "" {89- return p72+ head := ""73+ if name != "" {74+ head = name + " = "75+ }76+ p := Parse(src)77+ if len(p.Stmts) == 0 {78+ if strings.TrimSpace(src) == "" {79+ return head80+ }81+ return src + "\n" + head82+ }83+ last := p.Stmts[len(p.Stmts)-1]84+ if last.Name != "" {85+ return src[:last.From] + head + src[last.Body:]9086 }91- return name + " = " + p87+ return src[:last.From] + head + src[last.From:]9288 }9389 9490 // Reserved are nushell's own variables and the notebook's: $selection@@ -141,6 +137,10 @@ type Output struct {
141137 // Seq identifies the output among the workbook's, for telling142138 // whether what a cell read has changed since.143139 Seq int140+ // Vars are the values, as NUON, of the variables the cell assigns141+ // other than its output, which later cells read by name. A file142+ // doesn't keep them: a cell reading one runs its cell first.143+ Vars map[string][]byte144144 // Took is how long the run took.145145 Took time.Duration146146 // Source is the cell's source when it ran.@@ -5,59 +5,196 @@ import (
55 "testing"66 )77 8-func TestSplitName(t *testing.T) {9- for _, tc := range []struct{ src, name, pipe string }{10- {"sales = open sales.csv", "sales", "open sales.csv"},11- {"big=\n$sales | first", "big", "\n$sales | first"},12- {"ls | where name == x", "", "ls | where name == x"},13- {"a == b", "", "a == b"},14- {"in = ls", "", "in = ls"},15- {"9x = ls", "", "9x = ls"},16- {"let x = 1", "", "let x = 1"},17- {"__s = 1", "", "__s = 1"},8+func TestName(t *testing.T) {9+ for _, tc := range []struct{ src, name string }{10+ {"sales = open sales.csv", "sales"},11+ {"big=\n$sales | first", "big"},12+ {"ls | where name == x", ""},13+ {"a == b", ""},14+ {"a =~ b", ""},15+ {"in = ls", ""},16+ {"9x = ls", ""},17+ {"let x = 1", ""},18+ {"__s = 1", ""},19+ {"files = ls\n$files | length", ""},20+ {"files = ls\nn = $files | length", "n"},21+ {"n = ls\n| where size > 1kb", "n"},22+ {"n = ls |\n where size > 1kb", "n"},23+ {"n = ls\n# keep the big ones\n| where size > 1kb", "n"},24+ {"n = ls\n\n| length", ""},25+ {"n = ls; $n | length", ""},26+ {"n = [\n1\n2\n]", "n"},27+ {"n = ls # all of them", "n"},28+ {"# n = ls", ""},1829 } {19- name, p := SplitName(tc.src)20- if name != tc.name || p != tc.pipe {21- t.Errorf("SplitName(%q) = %q, %q", tc.src, name, p)30+ if got := (Cell{Source: tc.src}).Name(); got != tc.name {31+ t.Errorf("Name(%q) = %q, want %q", tc.src, got, tc.name)2232 }2333 }24- if got := WithName("ls", "files"); got != "files = ls" {25- t.Errorf("WithName %q", got)26- }27- if got := WithName("files = ls", ""); got != "ls" {28- t.Errorf("WithName without %q", got)29- }3034 if (Cell{Kind: Note, Source: "x = y"}).Name() != "" {3135 t.Error("a note cell has a name")3236 }3337 }3438 35-func TestRefs(t *testing.T) {36- got := Refs("$r1 | where x == $in.a and $nu.home | $r_2.x | $selection | $sheet.A1")37- if !slices.Equal(got, []string{"r1", "r_2"}) {39+func TestWithName(t *testing.T) {40+ for _, tc := range []struct{ src, name, want string }{41+ {"ls", "files", "files = ls"},42+ {"files = ls", "", "ls"},43+ {"files = ls", "all", "all = ls"},44+ {"files = ls\n$files | length", "n", "files = ls\nn = $files | length"},45+ {"files = ls\nn = $files | length", "", "files = ls\n$files | length"},46+ {"", "x", "x = "},47+ {"# just a note", "x", "# just a note\nx = "},48+ } {49+ if got := WithName(tc.src, tc.name); got != tc.want {50+ t.Errorf("WithName(%q, %q) = %q, want %q", tc.src, tc.name, got, tc.want)51+ }52+ }53+}54+55+// The owner's cell: files is the cell's own, read by its second line, not56+// a cell it reads; the cell has no name, and hands files to later cells.57+func TestOwnersCell(t *testing.T) {58+ src := "files = ls | where type == file\n$files | where size > 1kb | sort-by size --reverse"59+ p := Parse(src)60+ if p.Name() != "" || len(p.Refs()) != 0 || !slices.Equal(p.Assigned(), []string{"files"}) {61+ t.Errorf("name %q refs %v assigned %v", p.Name(), p.Refs(), p.Assigned())62+ }63+ cmd, exports, _ := p.Command()64+ want := "let files = ls | where type == file\n{__out: ($files | where size > 1kb | sort-by size --reverse), files: $files}"65+ if cmd != want || !slices.Equal(exports, []string{"files"}) {66+ t.Errorf("command %q exports %v", cmd, exports)67+ }68+ if got := p.StreamCommand(); got != "let files = ls | where type == file\n$files | where size > 1kb | sort-by size --reverse" {69+ t.Errorf("stream command %q", got)70+ }71+ cs := cells(src, "$files | length")72+ order, cycle := Order(cs, []int{0, 1})73+ if !slices.Equal(order, []int{0, 1}) || len(cycle) != 0 {74+ t.Errorf("order %v cycle %v", order, cycle)75+ }76+ run, err := Prepare(cs, 0, func(int) *Output { return nil })77+ if err != nil || run.Command != want || len(run.Tables) != 0 {78+ t.Errorf("prepare %+v %v", run, err)79+ }80+ // The second cell reads files from the first's run.81+ outs := map[int]*Output{1: {NUON: []byte("[]"), Seq: 4, Vars: map[string][]byte{"files": []byte("[[name]; [a]]")}}}82+ out := func(id int) *Output { return outs[id] }83+ run, err = Prepare(cs, 1, out)84+ if err != nil || string(run.Tables["files"]) != "[[name]; [a]]" || run.Reads["files"] != 4 {85+ t.Errorf("prepare the reader %+v %v", run, err)86+ }87+ // An output read from a file keeps no variables: the first cell runs88+ // before the second.89+ outs[1].Vars = nil90+ if got := Inputs(cs, 1, out); !slices.Equal(got, []int{0}) {91+ t.Errorf("inputs %v", got)92+ }93+ if _, err := Prepare(cs, 1, out); err == nil {94+ t.Error("prepared a cell reading a variable its cell didn't hand back")95+ }96+}97+98+func TestComments(t *testing.T) {99+ src := "# x = ls\n# $y | $sheet.A1:B2 | $selection\nls # $z\n| where name == \"a#b\" # $w\n| where name != 'c # $v'\n| where name != r#'d # $u'#\n| append [1#2]\n| append $\"($q # $t\n)\""100+ p := Parse(src)101+ if p.Name() != "" || !slices.Equal(p.Refs(), []string{"q"}) || p.ReadsSelection() || len(p.Ranges) != 0 {102+ t.Errorf("name %q refs %v selection %v ranges %v", p.Name(), p.Refs(), p.ReadsSelection(), p.Ranges)103+ }104+ var comments []string105+ for _, c := range p.Comments {106+ comments = append(comments, src[c[0]:c[1]])107+ }108+ if !slices.Equal(comments, []string{"# x = ls", "# $y | $sheet.A1:B2 | $selection", "# $z", "# $w", "# $t"}) {109+ t.Errorf("comments %q", comments)110+ }111+ if len(p.Stmts) != 1 {112+ t.Errorf("statements %+v", p.Stmts)113+ }114+ cmd, _, _ := p.Command()115+ if len(cmd) != len(src) || cmd[:8] != " " {116+ t.Errorf("command %q", cmd)117+ }118+ // Inside brackets a comment may follow a , or a :.119+ if got := Parse("[1,#x\n2]").Comments; len(got) != 1 {120+ t.Errorf("comments in a list %v", got)121+ }122+ if got := Parse("echo x,#y").Comments; len(got) != 0 {123+ t.Errorf("a comment in a word %v", got)124+ }125+}126+127+func TestStatements(t *testing.T) {128+ src := "a = ls\nb = $a | length; c = $b + 1\nlet d = 2\n$c + $d + $e"129+ p := Parse(src)130+ if !slices.Equal(p.Assigned(), []string{"a", "b", "c"}) || !slices.Equal(p.Refs(), []string{"e"}) {131+ t.Errorf("assigned %v refs %v", p.Assigned(), p.Refs())132+ }133+ cmd, exports, _ := p.Command()134+ want := "let a = ls\nlet b = $a | length; let c = $b + 1\nlet d = 2\n{__out: ($c + $d + $e), a: $a, b: $b, c: $c}"135+ if cmd != want || !slices.Equal(exports, []string{"a", "b", "c"}) {136+ t.Errorf("command %q exports %v", cmd, exports)137+ }138+ // The last statement's name is the output: it's not exported apart.139+ cmd, exports, _ = Parse("x = 1\nx = $x + 1").Command()140+ if cmd != "let x = 1\n$x + 1" || len(exports) != 0 {141+ t.Errorf("command %q exports %v", cmd, exports)142+ }143+ // $x on the line assigning x reads the x from before.144+ if got := Parse("x = $x | first").Refs(); !slices.Equal(got, []string{"x"}) {38145 t.Errorf("refs %v", got)39146 }40- if !ReadsSelection("$selection | math sum") || ReadsSelection("$selections") {41- t.Error("ReadsSelection")147+}148+149+// A variable of the cell's own shadows another cell's name: the cell150+// doesn't read that cell.151+func TestLocalShadows(t *testing.T) {152+ cs := cells("sales = open sales.csv", "sales = [[a]; [1]]\n$sales | length", "n = $sales | length")153+ if got := Reads(cs, Vars(cs), 1); len(got) != 0 {154+ t.Errorf("the shadowing cell reads %v", got)155+ }156+ if got := Reads(cs, Vars(cs), 2); !slices.Equal(got, []int{0}) {157+ t.Errorf("the third cell reads %v", got)158+ }159+ if got := Dependents(cs, 0); !slices.Equal(got, []int{2}) {160+ t.Errorf("dependents %v", got)161+ }162+ if Vars(cs)["sales"] != 0 {163+ t.Errorf("sales is the first cell's, as its name: %v", Vars(cs))164+ }165+ // A cell reading another's name before assigning it reads that cell.166+ cs = cells("sales = ls", "big = $sales | first\nsales = 1\n$sales")167+ if got := Reads(cs, Vars(cs), 1); !slices.Equal(got, []int{0}) {168+ t.Errorf("reads %v", got)42169 }43170 }44171 45-func TestBind(t *testing.T) {46- got, refs := Bind("$sheet.A1:C9 | append $sheet.'Q1 data'!B2:B5 | append $sheet.A1:C9.name | $x$sheet.A1 | $sheet.")47- want := "$__sheet1 | append $__sheet2 | append $__sheet1.name | $x$sheet.A1 | $sheet."172+func TestRanges(t *testing.T) {173+ src := "$sheet.A1:C9 | append $sheet.'Q1 data'!B2:B5 | append $sheet.A1:C9.name | $x$sheet.A1 | $sheet. | append \"$sheet.A1\""174+ p := Parse(src)175+ var spans []string176+ for _, s := range p.Ranges {177+ spans = append(spans, src[s[0]:s[1]])178+ }179+ if !slices.Equal(spans, []string{"$sheet.A1:C9", "$sheet.'Q1 data'!B2:B5", "$sheet.A1:C9"}) {180+ t.Errorf("ranges %q", spans)181+ }182+ got, _, refs := p.Command()183+ want := "$__sheet1 | append $__sheet2 | append $__sheet1.name | $x$sheet.A1 | $sheet. | append \"$sheet.A1\""48184 if got != want {49- t.Errorf("Bind = %q", got)185+ t.Errorf("command = %q", got)50186 }51187 if len(refs) != 2 || refs[0].Ref != "A1:C9" || refs[1].Ref != "'Q1 data'!B2:B5" || refs[1].Var != "__sheet2" {52188 t.Errorf("refs %+v", refs)53189 }54- src := "$sheet.A1:C9 | append $sheet.'Q1 data'!B2:B5 | append $sheet.A1:C9.name | $x$sheet.A1 | $sheet."55- var spans []string56- for _, s := range RangeSpans(src) {57- spans = append(spans, src[s[0]:s[1]])190+ if !slices.Equal(p.Refs(), []string{"x"}) {191+ t.Errorf("refs %v", p.Refs())58192 }59- if !slices.Equal(spans, []string{"$sheet.A1:C9", "$sheet.'Q1 data'!B2:B5", "$sheet.A1:C9"}) {60- t.Errorf("RangeSpans %q", spans)193+ if !Parse("$selection | math sum").ReadsSelection() || Parse("$selections").ReadsSelection() {194+ t.Error("ReadsSelection")195+ }196+ if got := Parse("$r1 | where x == $in.a and $nu.home | $r_2.x | $selection | $sheet.A1").Refs(); !slices.Equal(got, []string{"r1", "r_2"}) {197+ t.Errorf("refs %v", got)61198 }62199 }63200 @@ -95,7 +232,7 @@ func TestStale(t *testing.T) {
95232 out := func(id int) *Output { return outs[id] }96233 run := func(i int, seq int) {97234 o := &Output{NUON: []byte("x"), Seq: seq, Count: seq, Source: cs[i].Source, Reads: map[string]int{}}98- for _, n := range Refs(cs[i].Pipeline()) {235+ for _, n := range cs[i].Parse().Refs() {99236 if j, ok := Names(cs)[n]; ok && outs[cs[j].ID] != nil {100237 o.Reads[n] = outs[cs[j].ID].Seq101238 }@@ -1,17 +1,14 @@
11 package notebook22 3-import (4- "slices"5- "strconv"6- "strings"7-)3+import "strings"84 95 // What a pipeline reads. A code cell reads another's output as $name,106 // the selection on a sheet as $selection, and a range of a sheet as117 // $sheet.A1:C9 (or $sheet.Sales!A1:C9, $sheet.'Q1 data'!A1:B5). Each128 // reaches nu as a variable holding a table, read from a NUON file, never139 // as text spliced into the pipeline: a $sheet range is renamed to a14-// variable of its own ($__sheet1) before the pipeline runs.10+// variable of its own ($__sheet1) before the pipeline runs11+// (Source.Command).1512 1613 // Selection is the variable that holds the selection.1714 const Selection = "selection"@@ -19,43 +16,6 @@ const Selection = "selection"
1916 // sheetVar starts a range of a sheet.2017 const sheetVar = "$sheet."2118 22-// Refs returns the names a pipeline reads as $name, in the order it23-// first names them, leaving out nushell's variables and the notebook's24-// own ($selection, $sheet).25-func Refs(pipeline string) []string {26- var out []string27- for i := 0; i < len(pipeline); i++ {28- if pipeline[i] != '$' {29- continue30- }31- j := i + 132- for j < len(pipeline) && isWord(pipeline[j]) {33- j++34- }35- name := pipeline[i+1 : j]36- if name != "" && ValidName(name) == nil && !slices.Contains(out, name) {37- out = append(out, name)38- }39- i = j - 140- }41- return out42-}43-44-// ReadsSelection reports whether a pipeline reads $selection.45-func ReadsSelection(pipeline string) bool {46- for i := 0; ; {47- k := strings.Index(pipeline[i:], "$"+Selection)48- if k < 0 {49- return false50- }51- end := i + k + 1 + len(Selection)52- if end == len(pipeline) || !isWord(pipeline[end]) {53- return true54- }55- i = end56- }57-}58-5919 // SheetRef is a range of a sheet a pipeline reads, $sheet.A1:C9.6020 type SheetRef struct {6121 // Ref is the range as written after $sheet.: "A1:C9", "Sales!A1:C9".@@ -64,63 +24,6 @@ type SheetRef struct {
6424 Var string6525 }6626 67-// Bind renames each range of a sheet the pipeline reads ($sheet.A1:C9)68-// to a variable of its own, returning the pipeline to run and the ranges69-// in the order they appear, the same range read twice once.70-func Bind(pipeline string) (string, []SheetRef) {71- var b strings.Builder72- var refs []SheetRef73- rest := pipeline74- for {75- k := strings.Index(rest, sheetVar)76- if k < 0 || k > 0 && isWord(rest[k-1]) {77- if k < 0 {78- b.WriteString(rest)79- return b.String(), refs80- }81- b.WriteString(rest[:k+len(sheetVar)])82- rest = rest[k+len(sheetVar):]83- continue84- }85- b.WriteString(rest[:k])86- rest = rest[k+len(sheetVar):]87- n := refLen(rest)88- if n == 0 {89- b.WriteString(sheetVar)90- continue91- }92- ref := rest[:n]93- rest = rest[n:]94- i := slices.IndexFunc(refs, func(r SheetRef) bool { return r.Ref == ref })95- if i < 0 {96- refs = append(refs, SheetRef{Ref: ref, Var: "__sheet" + strconv.Itoa(len(refs)+1)})97- i = len(refs) - 198- }99- b.WriteString("$" + refs[i].Var)100- }101-}102-103-// RangeSpans are where a pipeline reads ranges of sheets, by byte104-// offsets: each $sheet.A1:C9 whole, as Bind renames them.105-func RangeSpans(pipeline string) [][2]int {106- var out [][2]int107- for i := 0; ; {108- k := strings.Index(pipeline[i:], sheetVar)109- if k < 0 {110- return out111- }112- k += i113- i = k + len(sheetVar)114- if k > 0 && isWord(pipeline[k-1]) {115- continue116- }117- if n := refLen(pipeline[i:]); n > 0 {118- i += n119- out = append(out, [2]int{k, i})120- }121- }122-}123-12427 // refLen is how long the range at the start of s is: a sheet's name,12528 // quoted in ” if it needs to be, then ! and cells, or cells alone. A12629 // dot ends it, so a cell path may follow: $sheet.A1:C9.name.@@ -0,0 +1,297 @@
1+package notebook2+3+import (4+ "slices"5+ "strings"6+)7+8+// How 012 reads a code cell's source, as nu does: its comments and9+// strings apart from its code, and its statements. A statement is a10+// line, or a part of one between `;`, at the top level (outside11+// brackets); a pipeline goes on over the next line when that line12+// starts with `|` (comment lines between allowed, a blank line not), or13+// when its own line ends in `|` or in an assignment's `=`. A statement14+// `name = pipeline` assigns name, as nu's `let name = pipeline` does.15+16+// Source is a code cell's source as 012 reads it.17+type Source struct {18+ Text string19+ // Stmts are its statements, in order.20+ Stmts []Stmt21+ // Comments are its comments' bytes, # to the end of the line, and22+ // Strings its strings', quotes included.23+ Comments, Strings [][2]int24+ // Ranges are where it reads ranges of sheets, each $sheet.A1:C925+ // whole.26+ Ranges [][2]int27+ // code is Text with its comments blanked, and view is code with its28+ // strings blanked too, but for the code interpolated into them:29+ // where names are read.30+ code, view []byte31+}32+33+// Stmt is a statement of a source, by byte offsets into it.34+type Stmt struct {35+ // From and To are its bytes, without the spaces and comments around.36+ From, To int37+ // Body is where what it runs starts: after `name =`, or From.38+ Body int39+ // Name is what `name = pipeline` assigns, or "".40+ Name string41+ // Local is the variable `let name = ...` or `mut name = ...` binds,42+ // which only the cell's later statements read, or "".43+ Local string44+}45+46+// Parse reads src.47+func Parse(src string) Source {48+ s := &scanner{src: src, view: []byte(src)}49+ s.code(0, 0)50+ code := []byte(src)51+ for _, c := range s.comments {52+ blank(code, c[0], c[1])53+ }54+ return Source{Text: src, Stmts: statements(src, s.view), Comments: s.comments, Strings: s.strs, Ranges: s.ranges, code: code, view: s.view}55+}56+57+// Name is what the source's last statement assigns: the cell's output's58+// name, or "".59+func (s Source) Name() string {60+ if len(s.Stmts) == 0 {61+ return ""62+ }63+ return s.Stmts[len(s.Stmts)-1].Name64+}65+66+// Assigned are the names the statements assign, in the order they67+// first do.68+func (s Source) Assigned() []string {69+ var out []string70+ for _, st := range s.Stmts {71+ if st.Name != "" && !slices.Contains(out, st.Name) {72+ out = append(out, st.Name)73+ }74+ }75+ return out76+}77+78+// Refs are the names the source reads as $name from outside it, in the79+// order it first names them: not a name an earlier statement assigns or80+// binds with let, nor nushell's variables and the notebook's own81+// ($selection, $sheet), nor one in a comment or a string.82+func (s Source) Refs() []string {83+ var out []string84+ v := s.view85+ for i := 0; i < len(v); i++ {86+ if v[i] != '$' {87+ continue88+ }89+ j := i + 190+ for j < len(v) && isWord(v[j]) {91+ j++92+ }93+ name := string(v[i+1 : j])94+ if name != "" && ValidName(name) == nil && !slices.Contains(out, name) && !s.local(name, i) {95+ out = append(out, name)96+ }97+ i = j - 198+ }99+ return out100+}101+102+// local reports whether name, read at offset at, is a variable of the103+// cell's own: a statement before assigns or binds it.104+func (s Source) local(name string, at int) bool {105+ return slices.ContainsFunc(s.Stmts, func(st Stmt) bool {106+ return st.To <= at && (st.Name == name || st.Local == name)107+ })108+}109+110+// ReadsSelection reports whether the source reads $selection.111+func (s Source) ReadsSelection() bool {112+ v, want := string(s.view), "$"+Selection113+ for i := 0; ; {114+ k := strings.Index(v[i:], want)115+ if k < 0 {116+ return false117+ }118+ end := i + k + len(want)119+ if end == len(v) || !isWord(v[end]) {120+ return true121+ }122+ i = end123+ }124+}125+126+// Heads are the bytes of each assignment's `name =`, up to what it runs.127+func (s Source) Heads() [][2]int {128+ var out [][2]int129+ for _, st := range s.Stmts {130+ if st.Name != "" {131+ out = append(out, [2]int{st.From, st.Body})132+ }133+ }134+ return out135+}136+137+// scanner tells a source's comments and strings from its code.138+type scanner struct {139+ src string140+ view []byte141+ comments, strs, ranges [][2]int142+}143+144+// code reads code from i up to the bracket close that ends it (0 for145+// the end of the source), returning where it stopped.146+func (s *scanner) code(i int, close byte) int {147+ for i < len(s.src) {148+ c := s.src[i]149+ switch {150+ case close != 0 && c == close:151+ return i152+ case c == '(' || c == '[' || c == '{':153+ i = s.code(i+1, closer(c))154+ if i < len(s.src) {155+ i++156+ }157+ case c == '#' && s.tokenStart(i, close != 0):158+ i = s.comment(i)159+ case c == '"' || c == '\'' || c == '`':160+ i = s.str(i, i+1, c)161+ case c == '$':162+ i = s.dollar(i)163+ case c == 'r' && s.tokenStart(i, close != 0) && rawHashes(s.src[i+1:]) > 0:164+ i = s.raw(i)165+ default:166+ i++167+ }168+ }169+ return i170+}171+172+func closer(c byte) byte {173+ switch c {174+ case '(':175+ return ')'176+ case '[':177+ return ']'178+ }179+ return '}'180+}181+182+// tokenStart reports whether a token may start at i: after a space, a183+// pipe, a ; or an opening bracket, and inside brackets after a , or a184+// : too, where nu reads a # as a comment.185+func (s *scanner) tokenStart(i int, inside bool) bool {186+ if i == 0 {187+ return true188+ }189+ p := s.src[i-1]190+ return strings.IndexByte(" \t\r\n|;([{", p) >= 0 || inside && (p == ',' || p == ':')191+}192+193+func (s *scanner) comment(i int) int {194+ j := i195+ for j < len(s.src) && s.src[j] != '\n' {196+ j++197+ }198+ s.comments = append(s.comments, [2]int{i, j})199+ blank(s.view, i, j)200+ return j201+}202+203+// str reads a quoted string opened at start, its text from i: a double204+// quoted one has escapes, a single quoted or backquoted one none.205+func (s *scanner) str(start, i int, q byte) int {206+ for i < len(s.src) && s.src[i] != q {207+ if q == '"' && s.src[i] == '\\' {208+ i++209+ }210+ i++211+ }212+ i = min(i+1, len(s.src))213+ s.strs = append(s.strs, [2]int{start, i})214+ blank(s.view, start, i)215+ return i216+}217+218+// dollar reads what starts with $ at i: an interpolated string, a range219+// of a sheet, or a variable.220+func (s *scanner) dollar(i int) int {221+ rest := s.src[i+1:]222+ switch {223+ case strings.HasPrefix(rest, `"`) || strings.HasPrefix(rest, "'"):224+ return s.interpolated(i)225+ case strings.HasPrefix(s.src[i:], sheetVar) && (i == 0 || !isWord(s.src[i-1])):226+ from := i + len(sheetVar)227+ if n := refLen(s.src[from:]); n > 0 {228+ s.ranges = append(s.ranges, [2]int{i, from + n})229+ blank(s.view, from, from+n) // its quotes and brackets aren't code230+ return from + n231+ }232+ }233+ return i + 1234+}235+236+// interpolated reads $"..." or $'...' at i: text, blanked, and the code237+// in its parentheses, read as code.238+func (s *scanner) interpolated(i int) int {239+ q := s.src[i+1]240+ j := i + 2241+ blank(s.view, i, j)242+ for j < len(s.src) && s.src[j] != q {243+ switch {244+ case q == '"' && s.src[j] == '\\':245+ blank(s.view, j, min(j+2, len(s.src)))246+ j += 2247+ case s.src[j] == '(':248+ j = s.code(j+1, ')')249+ if j < len(s.src) {250+ j++251+ }252+ default:253+ blank(s.view, j, j+1)254+ j++255+ }256+ }257+ j = min(j+1, len(s.src))258+ blank(s.view, j-1, j)259+ s.strs = append(s.strs, [2]int{i, j})260+ return j261+}262+263+// rawHashes is how many # open a raw string at the start of s, r#'...'#264+// without its r, or 0.265+func rawHashes(s string) int {266+ n := 0267+ for n < len(s) && s[n] == '#' {268+ n++269+ }270+ if n == 0 || n == len(s) || s[n] != '\'' {271+ return 0272+ }273+ return n274+}275+276+// raw reads a raw string at i, to its ' and as many # as opened it.277+func (s *scanner) raw(i int) int {278+ n := rawHashes(s.src[i+1:])279+ end := "'" + strings.Repeat("#", n)280+ body := i + 2 + n281+ j := len(s.src)282+ if k := strings.Index(s.src[body:], end); k >= 0 {283+ j = body + k + len(end)284+ }285+ s.strs = append(s.strs, [2]int{i, j})286+ blank(s.view, i, j)287+ return j288+}289+290+// blank makes b[from:to] spaces, but for its line breaks.291+func blank(b []byte, from, to int) {292+ for i := from; i < to && i < len(b); i++ {293+ if b[i] != '\n' {294+ b[i] = ' '295+ }296+ }297+}@@ -0,0 +1,193 @@
1+package notebook2+3+import (4+ "slices"5+ "strconv"6+ "strings"7+)8+9+// statements splits a source into its statements, reading view, the10+// source with its comments and strings blanked.11+func statements(src string, view []byte) []Stmt {12+ var out []Stmt13+ depth, from := 0, 014+ for i := 0; i <= len(view); i++ {15+ if i < len(view) {16+ switch c := view[i]; {17+ case c == '(' || c == '[' || c == '{':18+ depth++19+ continue20+ case c == ')' || c == ']' || c == '}':21+ depth = max(depth-1, 0)22+ continue23+ case c != ';' && c != '\n', depth > 0, c == '\n' && continues(src, view, from, i):24+ continue25+ }26+ }27+ if st, ok := statement(view, from, i); ok {28+ out = append(out, st)29+ }30+ from = i + 131+ }32+ return out33+}34+35+// continues reports whether the statement from from goes on past the36+// line break at i: its line ends in | or in an assignment's =, or the37+// next line, past comment lines, starts with |.38+func continues(src string, view []byte, from, i int) bool {39+ before := strings.TrimRight(string(view[from:i]), " \t\r")40+ if strings.HasSuffix(before, "|") || strings.HasSuffix(before, "=") {41+ return true42+ }43+ for j := i + 1; j < len(view); {44+ end := j45+ for end < len(view) && view[end] != '\n' {46+ end++47+ }48+ line := strings.TrimSpace(string(view[j:end]))49+ switch {50+ case strings.HasPrefix(line, "|"):51+ return true52+ case line != "" || strings.TrimSpace(src[j:end]) == "":53+ return false // code, or a blank line54+ }55+ j = end + 1 // a comment line56+ }57+ return false58+}59+60+// statement is the statement in view[from:to], if there's one.61+func statement(view []byte, from, to int) (Stmt, bool) {62+ for from < to && isSpace(view[from]) {63+ from++64+ }65+ for to > from && isSpace(view[to-1]) {66+ to--67+ }68+ if from == to {69+ return Stmt{}, false70+ }71+ st := Stmt{From: from, To: to, Body: from}72+ name, rest := word(view, from, to)73+ rest = skipSpaces(view, rest, to, " \t")74+ switch {75+ case rest < to && view[rest] == '=' && ValidName(name) == nil && !opAt(view, rest+1, to):76+ st.Name, st.Body = name, skipSpaces(view, rest+1, to, " \t\r\n")77+ case name == "let" || name == "mut" || name == "const":78+ st.Local, _ = word(view, skipSpaces(view, rest, to, " \t"), to)79+ }80+ return st, true81+}82+83+// opAt reports whether an = ending at i is part of an operator: ==, =~.84+func opAt(view []byte, i, to int) bool { return i < to && (view[i] == '=' || view[i] == '~') }85+86+// word is the word of letters, digits and _ at from, and where it ends.87+func word(view []byte, from, to int) (string, int) {88+ i := from89+ for i < to && isWord(view[i]) {90+ i++91+ }92+ return string(view[from:i]), i93+}94+95+func skipSpaces(view []byte, i, to int, spaces string) int {96+ for i < to && strings.IndexByte(spaces, view[i]) >= 0 {97+ i++98+ }99+ return i100+}101+102+func isSpace(c byte) bool { return c == ' ' || c == '\t' || c == '\r' || c == '\n' }103+104+// OutputVar is the name of the variable that holds a cell's output in105+// the record a run hands back when it also hands back variables106+// (nushell.ExecVars).107+const OutputVar = "__out"108+109+// Command is what nu runs for the source: each statement but the last110+// that assigns a name as `let name = ...`, the last without its `name =`111+// (its value is the output), the ranges of sheets renamed to variables112+// of their own ($__sheet1), and the comments blanked, so what's left113+// keeps its place. When statements before the last assign names the114+// last doesn't, those are exports, and the command ends in a record of115+// the output (as __out) and each of them.116+func (s Source) Command() (cmd string, exports []string, ranges []SheetRef) {117+ return s.command(true)118+}119+120+// StreamCommand is what nu runs for the source as a stream: Command121+// without the record, so the last statement's values stream as they122+// come, and the names the statements before it assign stay the run's.123+func (s Source) StreamCommand() string {124+ cmd, _, _ := s.command(false)125+ return cmd126+}127+128+func (s Source) command(record bool) (cmd string, exports []string, ranges []SheetRef) {129+ var edits []edit130+ for _, r := range s.Ranges {131+ ref := s.Text[r[0]+len(sheetVar) : r[1]]132+ i := slices.IndexFunc(ranges, func(x SheetRef) bool { return x.Ref == ref })133+ if i < 0 {134+ ranges = append(ranges, SheetRef{Ref: ref, Var: "__sheet" + strconv.Itoa(len(ranges)+1)})135+ i = len(ranges) - 1136+ }137+ edits = append(edits, edit{r[0], r[1], "$" + ranges[i].Var})138+ }139+ if len(s.Stmts) == 0 {140+ return apply(s.code, edits), nil, ranges141+ }142+ last := s.Stmts[len(s.Stmts)-1]143+ for _, st := range s.Stmts[:len(s.Stmts)-1] {144+ if st.Name == "" {145+ continue146+ }147+ edits = append(edits, edit{st.From, st.Body, "let " + st.Name + " = "})148+ if st.Name != last.Name && !slices.Contains(exports, st.Name) {149+ exports = append(exports, st.Name)150+ }151+ }152+ if last.Name != "" {153+ edits = append(edits, edit{last.From, last.Body, ""})154+ }155+ if !record {156+ exports = nil157+ }158+ if len(exports) > 0 {159+ var rec strings.Builder160+ rec.WriteString(")")161+ for _, e := range exports {162+ rec.WriteString(", " + e + ": $" + e)163+ }164+ rec.WriteString("}")165+ edits = append(edits, edit{last.Body, last.Body, "{" + OutputVar + ": ("}, edit{last.To, last.To, rec.String()})166+ }167+ return apply(s.code, edits), exports, ranges168+}169+170+// edit replaces b[from:to] with text.171+type edit struct {172+ from, to int173+ text string174+}175+176+// apply makes edits to b, which don't overlap.177+func apply(b []byte, edits []edit) string {178+ slices.SortStableFunc(edits, func(x, y edit) int {179+ if x.from != y.from {180+ return x.from - y.from181+ }182+ return x.to - y.to183+ })184+ var out strings.Builder185+ at := 0186+ for _, e := range edits {187+ out.Write(b[at:e.from])188+ out.WriteString(e.text)189+ at = e.to190+ }191+ out.Write(b[at:])192+ return out.String()193+}@@ -10,6 +10,7 @@ package nushell
1010 import (1111 "bytes"1212 "context"13+ "encoding/base64"1314 "errors"1415 "fmt"1516 "io"@@ -33,6 +34,10 @@ type Job struct {
3334 Tables map[string][]byte3435 // Config runs nu with the user's config files rather than without.3536 Config bool37+ // Vars are the variables the run hands back beside its output: the38+ // command ends in a record of its output, as __out, and each of39+ // them, and the run prints each as a line of its own (ExecVars).40+ Vars []string3641 // IDE asks nu about the script rather than running it: the flags3742 // that ask (--ide-ast, or --ide-complete 14), nu reading the script3843 // from a file named after them, without config files or the@@ -77,10 +82,43 @@ func Script(job Job, names []string) string {
7782 for i, name := range names {7883 fmt.Fprintf(&b, "let %s = (open --raw $env.NU012_TABLE_%d | from nuon)\n", name, i)7984 }80- b.WriteString("do {\n" + job.Command + "\n} | to nuon\n")85+ b.WriteString("do {\n" + job.Command + "\n} | ")86+ if len(job.Vars) > 0 {87+ b.WriteString(`items {|k, v| $"($k) ($v | to nuon | encode base64)" } | str join "\n"` + "\n")88+ } else {89+ b.WriteString("to nuon\n")90+ }8191 return b.String()8292 }8393 94+// outVar names the output among a run's variables.95+const outVar = "__out"96+97+// ExecVars runs job as Exec does, returning its output and, when it98+// hands back variables (Job.Vars), their values, each as NUON. They're99+// the last lines it printed, each a name and its NUON as base64, so100+// what the pipeline printed itself before them doesn't mix in.101+func ExecVars(ctx context.Context, r Runner, job Job, timeout time.Duration, maxBytes int) ([]byte, map[string][]byte, error) {102+ out, err := Exec(ctx, r, job, timeout, maxBytes)103+ if err != nil || len(job.Vars) == 0 {104+ return out, nil, err105+ }106+ lines := strings.Split(string(out), "\n")107+ vars := map[string][]byte{}108+ for _, line := range lines[max(len(lines)-len(job.Vars)-1, 0):] {109+ name, enc, _ := strings.Cut(strings.TrimSpace(line), " ")110+ if data, err := base64.StdEncoding.DecodeString(enc); err == nil && (name == outVar || slices.Contains(job.Vars, name)) {111+ vars[name] = data112+ }113+ }114+ nuon, ok := vars[outVar]115+ if !ok || len(vars) != len(job.Vars)+1 {116+ return nil, nil, errors.New("the run didn't hand back its variables")117+ }118+ delete(vars, outVar)119+ return nuon, vars, nil120+}121+84122 // ErrTooLarge is a run that printed more than it may keep.85123 var ErrTooLarge = errors.New("the output is too large to keep")86124 @@ -53,6 +53,42 @@ func TestExec(t *testing.T) {
5353 }5454 }5555 56+// A run handing back variables prints a line for each after what the57+// pipeline printed itself.58+func TestExecVars(t *testing.T) {59+ job := Job{Command: "x", Vars: []string{"files"}}60+ f := &fake{out: "printed\n__out WzEsIDJd\nfiles WzNd\n"}61+ out, vars, err := ExecVars(context.Background(), f, job, time.Second, 0)62+ if err != nil || string(out) != "[1, 2]" || string(vars["files"]) != "[3]" || len(vars) != 1 {63+ t.Errorf("%q %q %v", out, vars, err)64+ }65+ if !strings.Contains(f.script, `items {|k, v|`) || strings.Contains(f.script, "| to nuon\n") {66+ t.Errorf("script %s", f.script)67+ }68+ if _, _, err := ExecVars(context.Background(), &fake{out: "[1, 2]"}, job, time.Second, 0); err == nil {69+ t.Error("no variables came back, and no error")70+ }71+ out, vars, err = ExecVars(context.Background(), &fake{out: "[1, 2]"}, Job{Command: "x"}, time.Second, 0)72+ if err != nil || string(out) != "[1, 2]" || vars != nil {73+ t.Errorf("without variables %q %v %v", out, vars, err)74+ }75+}76+77+// The real nu hands back a cell's variables beside its output.78+func TestNuVars(t *testing.T) {79+ if _, err := exec.LookPath("nu"); err != nil {80+ t.Skip("nu isn't installed")81+ }82+ job := Job{83+ Command: "let files = [[name, size]; [a, 2kb], [b, 10b]]\n{__out: ($files | where size > 1kb | get name), files: ($files | length)}",84+ Vars: []string{"files"},85+ }86+ out, vars, err := ExecVars(context.Background(), Nu{}, job, 30*time.Second, 0)87+ if err != nil || string(out) != "[a]" || string(vars["files"]) != "2" {88+ t.Errorf("%q %q %v", out, vars, err)89+ }90+}91+5692 func TestMessage(t *testing.T) {5793 stderr := "Error: nu::shell::column_not_found\n\n × Cannot find column 'xyz'\n ╭─[source:1:6]\n"5894 if got := Message(stderr, errors.New("exit status 1")); got != "Cannot find column 'xyz'" {@@ -167,8 +167,11 @@ func (w *Workbook) convertOld(old []oldRegion) {
167167 for _, o := range old {168168 fr := o.fr169169 src := fr.Command170- if fr.Input != "" {170+ switch {171+ case fr.Input != "":171172 src = "$sheet." + quoteRef(fr.Input) + " | do {\n" + src + "\n}"173+ case len(notebook.Parse(src).Stmts) > 1: // one statement, so the name is its output's174+ src = "do {\n" + src + "\n}"172175 }173176 nb.regions.cells = append(nb.regions.cells, notebook.Cell{ID: w.NewCellID(), Source: fr.Name + " = " + src})174177 at, _ := ParseAddr(fr.At)@@ -267,7 +267,8 @@ func TestOldNotebookConverted(t *testing.T) {
267267 {"name": "Shell 1", "notebook": true, "cells": {}, "regions": [268268 {"name": "r1", "command": "ls", "at": "A1", "rows": 3, "cols": 2},269269 {"name": "big", "command": "$r1 | where size > 1kb", "at": "A6", "reads": ["r1"]},270- {"name": "sel", "command": "$in | math sum", "at": "A9", "input": "Data 1!A1:A4"}270+ {"name": "sel", "command": "$in | math sum", "at": "A9", "input": "Data 1!A1:A4"},271+ {"name": "two", "command": "let x = 1\n$x + 1", "at": "A12"}271272 ]},272273 {"name": "Data 1", "cells": {"B1": "=SUM(nu.big)"}}273274 ]}`@@ -280,7 +281,7 @@ func TestOldNotebookConverted(t *testing.T) {
280281 t.Fatalf("notebook %v at %d", nb, w.Index(nb))281282 }282283 cells := nb.NotebookCells()283- want := []string{"r1 = ls", "big = $r1 | where size > 1kb", "sel = $sheet.'Data 1'!A1:A4 | do {\n$in | math sum\n}"}284+ want := []string{"r1 = ls", "big = $r1 | where size > 1kb", "sel = $sheet.'Data 1'!A1:A4 | do {\n$in | math sum\n}", "two = do {\nlet x = 1\n$x + 1\n}"}284285 for i, c := range cells {285286 if c.Source != want[i] {286287 t.Errorf("cell %d: %q, want %q", i+1, c.Source, want[i])@@ -5,6 +5,7 @@ import (
55 "context"66 "errors"77 "fmt"8+ "slices"89 "time"910 1011 tea "charm.land/bubbletea/v2"@@ -20,26 +21,22 @@ import (
2021 // ($selection) and ranges of sheets ($sheet.A1:C9), each handed to nu as2122 // NUON, never spliced into the pipeline.2223 23-// jobFor is what running cell c takes: the outputs, linked files and24-// ranges it reads, as NUON.25-func (m *Model) jobFor(s *sheet.Sheet, c notebook.Cell, run *nbRun) (nushell.Job, error) {26- pipeline, ranges := notebook.Bind(c.Pipeline())27- job := nushell.Job{Command: pipeline, Tables: map[string][]byte{}, Config: m.configBool("nu-config", false)}28- run.reads = map[string]int{}24+// jobFor is what running cell c takes: the outputs, variables, linked25+// files and ranges it reads, as NUON. As a stream, the cell hands back26+// no variables but its output (notebook.Source.StreamCommand).27+func (m *Model) jobFor(s *sheet.Sheet, c notebook.Cell, run *nbRun, stream bool) (nushell.Job, error) {2928 cells := s.NotebookCells()30- names := notebook.Names(cells)31- for _, name := range notebook.Refs(pipeline) {32- if j, ok := names[name]; ok {33- o := m.book().Output(cells[j].ID)34- switch {35- case o == nil || o.Unsaved:36- return job, fmt.Errorf("it reads $%s, which hasn't run", name)37- case o.Failed():38- return job, fmt.Errorf("it reads $%s, which failed", name)39- }40- job.Tables[name], run.reads[name] = o.NUON, o.Seq41- continue42- }29+ i := slices.IndexFunc(cells, func(x notebook.Cell) bool { return x.ID == c.ID })30+ p, err := notebook.Prepare(cells, i, m.book().Output)31+ job := nushell.Job{Command: p.Command, Vars: p.Exports, Tables: p.Tables, Config: m.configBool("nu-config", false)}32+ if stream {33+ job.Command, job.Vars = p.Stream, nil34+ }35+ run.reads = p.Reads36+ if err != nil {37+ return job, err38+ }39+ for _, name := range p.Others {4340 if t, r, ok := m.book().Region(name); ok && r.Linked() && r.Name == name {4441 data, err := regionNUON(t, r)4542 if err != nil {@@ -48,12 +45,12 @@ func (m *Model) jobFor(s *sheet.Sheet, c notebook.Cell, run *nbRun) (nushell.Job
4845 job.Tables[name] = data4946 }5047 }51- if notebook.ReadsSelection(pipeline) {48+ if p.Selection {5249 if err := m.bindSelection(&job, run); err != nil {5350 return job, err5451 }5552 }56- for _, ref := range ranges {53+ for _, ref := range p.Ranges {5754 t, rng, err := m.rangeRef(ref.Ref)5855 if err != nil {5956 return job, err@@ -1,6 +1,7 @@
11 package ui22 33 import (4+ "slices"45 "strings"56 67 "github.com/FelineStateMachine/012/internal/sheet"@@ -63,8 +64,14 @@ func (m *Model) syncLang(s *sheet.Sheet, v *nbview.View) {
6364 func (m *Model) nbWords(s *sheet.Sheet) []nbview.Word {6465 var out []nbview.Word6566 for _, c := range s.NotebookCells() {66- if n := c.Name(); n != "" {67- out = append(out, nbview.Word{Text: "$" + n, Desc: "a cell's output"})67+ name := c.Name()68+ if name != "" {69+ out = append(out, nbview.Word{Text: "$" + name, Desc: "a cell's output"})70+ }71+ for _, n := range c.Parse().Assigned() {72+ if n != name && !slices.ContainsFunc(out, func(w nbview.Word) bool { return w.Text == "$"+n }) {73+ out = append(out, nbview.Word{Text: "$" + n, Desc: "a cell's variable"})74+ }6875 }6976 }7077 for _, t := range m.book().Sheets() {@@ -107,6 +107,7 @@ type (
107107 nbDoneMsg struct {108108 run *nbRun109109 nuon []byte110+ vars map[string][]byte // the variables it handed back110111 err error111112 }112113 // nbWordsMsg brings nu's command names.@@ -294,7 +295,7 @@ func (m *Model) nextCell() tea.Cmd {
294295 }295296 m.stopStream(q.id) // run once, it no longer follows296297 run := &nbRun{nbQueued: q, source: c.Source, start: time.Now()}297- job, err := m.jobFor(q.s, c, run)298+ job, err := m.jobFor(q.s, c, run, false)298299 if err != nil {299300 m.failCell(q, c, err.Error())300301 continue@@ -336,13 +337,13 @@ func (m *Model) startCell(run *nbRun, job nushell.Job) tea.Cmd {
336337 exec := func() tea.Msg {337338 defer cancel()338339 span := parent.Start("nu", slog.Int("tables", len(job.Tables)))339- out, err := nushell.Exec(ctx, runner, job, timeout, maxOutput)340+ out, vars, err := nushell.ExecVars(ctx, runner, job, timeout, maxOutput)340341 if err != nil {341342 span.Fail(err)342343 } else {343344 span.End(slog.Int("bytes", len(out)))344345 }345- return nbDoneMsg{run: run, nuon: out, err: err}346+ return nbDoneMsg{run: run, nuon: out, vars: vars, err: err}346347 }347348 return tea.Batch(m.roomOwned(exec), m.tickCell(run.gen))348349 }@@ -388,7 +389,7 @@ func (m *Model) finishCell(msg nbDoneMsg) tea.Cmd {
388389 }389390 m.nb.runs.queue = nil390391 } else {391- o.NUON = msg.nuon392+ o.NUON, o.Vars = msg.nuon, msg.vars392393 if o.NUON == nil {393394 o.NUON = []byte{}394395 }@@ -0,0 +1,112 @@
1+package ui2+3+import (4+ "encoding/base64"5+ "slices"6+ "strings"7+ "testing"8+9+ "github.com/FelineStateMachine/012/internal/notebook"10+)11+12+// ownersCell is a cell of two statements: files is its own variable,13+// read by its second line.14+const ownersCell = "files = ls | where type == file\n$files | where size > 1kb | sort-by size --reverse"15+16+// varsOut is what nu prints for a run handing back variables: a line17+// for the output and each variable, its NUON as base64.18+func varsOut(out string, vars ...string) string {19+ enc := func(s string) string { return base64.StdEncoding.EncodeToString([]byte(s)) }20+ lines := []string{"__out " + enc(out)}21+ for i := 0; i+1 < len(vars); i += 2 {22+ lines = append(lines, vars[i]+" "+enc(vars[i+1]))23+ }24+ return strings.Join(lines, "\n") + "\n"25+}26+27+// nuCommand is what nu runs for src.28+func nuCommand(src string) string {29+ cmd, _, _ := notebook.Parse(src).Command()30+ return cmd31+}32+33+// The owner's cell runs as a script: files is the cell's own, not a cell34+// it reads, its output is its last line's, it has no name, and a later35+// cell reads $files from its run.36+func TestNotebookStatements(t *testing.T) {37+ big := "[[name, size]; [c.txt, 5kb], [a.txt, 2kb]]"38+ m, nu := notebookModel(t, map[string]string{39+ nuCommand(ownersCell): varsOut(big, "files", lsOut),40+ "$files | length": "3",41+ })42+ write(t, m, ownersCell)43+ write(t, m, "$files | length")44+ press(t, m, "<ctrl+enter>") // the second runs the first first45+ if got := nu.ran(); len(got) != 2 || !strings.HasPrefix(got[0], "let files = ls") {46+ t.Fatalf("ran %q", got)47+ }48+ if !slices.Equal(nu.jobs[0].Vars, []string{"files"}) || len(nu.jobs[0].Tables) != 0 {49+ t.Errorf("the first job %+v", nu.jobs[0])50+ }51+ if got := string(nu.jobs[1].Tables["files"]); got != lsOut {52+ t.Errorf("$files reached the second cell as %q", got)53+ }54+ s := screen(m)55+ if strings.Contains(s, "failed") || strings.Contains(s, "─ files ─") || !strings.Contains(s, "c.txt") {56+ t.Errorf("screen:\n%s", s)57+ }58+ cs := m.sheet.NotebookCells()59+ if o := m.book().Output(cs[0].ID); o == nil || string(o.NUON) != big || string(o.Vars["files"]) != lsOut {60+ t.Errorf("output %+v", o)61+ }62+ press(t, m, "<up>", "<up>", "<up>")63+ if i, _ := m.nbView().Selected(); i != 0 {64+ t.Fatalf("selected %d", i)65+ }66+ if got := line(m, contextLine); !strings.Contains(got, "sets $files") || strings.Contains(got, "reads") {67+ t.Errorf("context line %q", got)68+ }69+ // Running it again makes the reader stale.70+ press(t, m, "<ctrl+enter>")71+ if st := m.cellState(m.sheet, cs[1].ID); !st.Stale {72+ t.Error("the reader isn't stale")73+ }74+ // Naming the cell names its last statement.75+ press(t, m, "n")76+ send(m, pasteMsg("big"))77+ press(t, m, "<enter>")78+ if got := m.sheet.NotebookCells()[0].Source; got != "files = ls | where type == file\nbig = $files | where size > 1kb | sort-by size --reverse" {79+ t.Errorf("named %q", got)80+ }81+}82+83+// Comments are nu's: a commented-out assignment names nothing and a84+// commented-out $name reads nothing.85+func TestNotebookComments(t *testing.T) {86+ src := "# files = ls\nls # | where name == $nope"87+ m, nu := notebookModel(t, map[string]string{nuCommand(src): lsOut})88+ write(t, m, src)89+ press(t, m, "<ctrl+enter>")90+ if len(nu.jobs) != 1 || len(nu.jobs[0].Tables) != 0 || strings.Contains(nu.jobs[0].Command, "#") {91+ t.Fatalf("ran %+v", nu.jobs)92+ }93+ if strings.Contains(screen(m), "failed") || m.sheet.NotebookCells()[0].Name() != "" {94+ t.Errorf("screen:\n%s", screen(m))95+ }96+}97+98+// A cell of several statements streams its last one's values, its own99+// variables bound first and none handed back.100+func TestNotebookStreamStatements(t *testing.T) {101+ streamPolls = nil102+ m, _ := notebookModel(t, nil)103+ nu := newStreamNu()104+ m.SetShellRunner(nu)105+ write(t, m, "log = open app.log\n$log | lines")106+ press(t, m, "f")107+ defer m.stopCells()108+ script := <-nu.scripts109+ if !strings.Contains(script, "let log = open app.log\n$log | lines") || strings.Contains(script, "__out") {110+ t.Errorf("stream script %s", script)111+ }112+}@@ -77,7 +77,7 @@ func (m *Model) startStream(q nbQueued) tea.Cmd {
7777 return nil7878 }7979 run := &nbRun{nbQueued: q, source: c.Source, start: time.Now()}80- job, err := m.jobFor(q.s, c, run)80+ job, err := m.jobFor(q.s, c, run, true)8181 if err != nil {8282 m.failCell(q, c, err.Error())8383 return nil@@ -319,22 +319,31 @@ func (v *View) Head() (name, text string) {
319319 if v.onOut {320320 return name, v.outputSummary(c)321321 }322+ src := c.Parse()322323 var reads []string323- for _, r := range notebook.Refs(c.Pipeline()) {324+ for _, r := range src.Refs() {324325 reads = append(reads, "$"+r)325326 }326- if notebook.ReadsSelection(c.Pipeline()) {327+ if src.ReadsSelection() {327328 reads = append(reads, "$selection")328329 }329- if _, refs := notebook.Bind(c.Pipeline()); len(refs) > 0 {330- for _, r := range refs {331- reads = append(reads, "$sheet."+r.Ref)332- }330+ _, _, ranges := src.Command()331+ for _, r := range ranges {332+ reads = append(reads, "$sheet."+r.Ref)333333 }334334 text = "a nushell pipeline"335335 if len(reads) > 0 {336336 text = "reads " + strings.Join(reads, ", ")337337 }338+ var sets []string339+ for _, a := range src.Assigned() {340+ if a != c.Name() {341+ sets = append(sets, "$"+a)342+ }343+ }344+ if len(sets) > 0 {345+ text += "; sets " + strings.Join(sets, ", ")346+ }338347 if n := c.Name(); n != "" {339348 text += "; read as $" + n + ", nu." + n + " in formulas"340349 }@@ -16,28 +16,36 @@ import (
1616 1717 // prepared is the text nu is asked about: a prelude declaring the1818 // notebook's variables, then the source masked, starting at byte at.19+// Masking keeps every byte of the source in its place, so what nu says20+// of the text is where it says it in the source.1921 type prepared struct {20- text string21- at int22- head int // the bytes of the source that are its `name =`22+ text string23+ at int24+ heads [][2]int // the bytes of the source that are its `name =`s2325 }2426 2527 // prepare masks src and declares what it reads: `sales = ls` is asked2628 // about as ` ls`, $sheet.A1:C9 as $__s1_______, and the prelude27-// declares $__s1, $selection, $sheet and names, each `let x: any = []`28-// so nu takes it for whatever it holds.29+// declares $__s1, $selection, $sheet, names and the names the cell's30+// lines assign for the lines after them, each `let x: any = []` so nu31+// takes it for whatever it holds.2932 func prepare(src string, names []string) prepared {3033 b := []byte(src)31- name, pipeline := notebook.SplitName(src)32- head := 033- if name != "" {34- head = len(src) - len(pipeline)35- for i := range head {34+ parsed := notebook.Parse(src)35+ heads := parsed.Heads()36+ for _, h := range heads {37+ for i := h[0]; i < h[1]; i++ {3638 if b[i] != '\n' {3739 b[i] = ' '3840 }3941 }4042 }43+ names = slices.Clone(names)44+ for k, st := range parsed.Stmts {45+ if st.Name != "" && k < len(parsed.Stmts)-1 { // assigned for the lines after46+ names = append(names, st.Name)47+ }48+ }4149 var pre strings.Builder4250 declared := map[string]bool{}4351 declare := func(name, value string) {@@ -46,9 +54,9 @@ func prepare(src string, names []string) prepared {
4654 pre.WriteString("let " + name + ": any = " + value + "\n")4755 }4856 }49- for i, s := range notebook.RangeSpans(src) {57+ for i, s := range parsed.Ranges {5058 v := "__s" + strconv.Itoa(i+1)51- if s[0] < head || len(v)+1 > s[1]-s[0] {59+ if len(v)+1 > s[1]-s[0] {5260 continue5361 }5462 v += strings.Repeat("_", s[1]-s[0]-len(v)-1)@@ -62,22 +70,28 @@ func prepare(src string, names []string) prepared {
6270 declare(nm, "[]")6371 }6472 }65- return prepared{text: pre.String() + string(b), at: pre.Len(), head: head}73+ return prepared{text: pre.String() + string(b), at: pre.Len(), heads: heads}74+}75+76+// inHead reports whether src[from:to] is part of an assignment's head.77+func (p prepared) inHead(from, to int) bool {78+ return slices.ContainsFunc(p.heads, func(h [2]int) bool { return from < h[1] && to > h[0] })6679 }6780 6881 // spans are src's syntax roles from nu's shapes of p's text: the69-// tokenizer's for the cell's name and comments, which nu doesn't70-// report, and a number's unit (the kb of 1kb) as part of the number.82+// tokenizer's for the names the cell assigns and its comments, which nu83+// doesn't report, and a number's unit (the kb of 1kb) as part of the84+// number.7185 func (p prepared) spans(src string, shapes []nushell.Shape) []Span {7286 var out []Span7387 for _, s := range (Tokens{}).Highlight(context.Background(), src) {74- if s.To <= p.head || s.Kind == theme.SyntaxComment {88+ if p.inHead(s.From, s.To) || s.Kind == theme.SyntaxComment {7589 out = append(out, s)7690 }7791 }7892 for i, s := range shapes {7993 from, to := s.From-p.at, s.To-p.at80- if from < p.head || to > len(src) || from >= to {94+ if from < 0 || to > len(src) || from >= to || p.inHead(from, to) {8195 continue8296 }8397 kind, ok := shapeKind(s.Shape, src[from:to])@@ -0,0 +1,59 @@
1+package nbview2+3+import (4+ "context"5+ "fmt"6+ "strings"7+ "testing"8+9+ "github.com/FelineStateMachine/012/internal/notebook"10+)11+12+// A cell of several statements: each name assigned shows as a variable,13+// a comment's text as a comment, whatever it holds, and # in a string14+// or a raw string is the string's.15+func TestHighlightsStatements(t *testing.T) {16+ src := "files = ls # all\n# big = $files\nbig = $files | where name != \"a#b\" | append r#'c # d'#"17+ var got []string18+ for _, s := range (Tokens{}).Highlight(context.Background(), src) {19+ got = append(got, fmt.Sprintf("%s:%d", src[s.From:s.To], s.Kind))20+ }21+ want := fmt.Sprint([]string{"files:2", "=:5", "ls:0", "# all:6", "# big = $files:6", "big:2", "=:5", "$files:2", "|:5", "where:0",22+ "!=:5", `"a#b":1`, "|:5", "append:0", "r#'c # d'#:1"})23+ if fmt.Sprint(got) != want {24+ t.Errorf("spans %v\nwant %v", got, want)25+ }26+}27+28+// nu is asked about a cell of several statements with each `name =`29+// blanked, every byte in its place, and the names its lines assign for30+// the lines after declared.31+func TestPrepareStatements(t *testing.T) {32+ src := "files = ls # x = 1\nbig = $files | first"33+ p := prepare(src, []string{"sales"})34+ if body := p.text[p.at:]; body != " ls # x = 1\n $files | first" {35+ t.Errorf("masked %q", body)36+ }37+ pre := p.text[:p.at]38+ if !strings.Contains(pre, "let files: any = []\n") || strings.Contains(pre, "let big") || strings.Contains(pre, "let x") {39+ t.Errorf("prelude %q", pre)40+ }41+}42+43+// The context line says what a cell of several statements reads and44+// sets: not its own variables.45+func TestHeadStatements(t *testing.T) {46+ h := newHost("files = ls | where type == file\n$files | where size > 1kb | sort-by size --reverse", "$files | length")47+ v := newView(h, 60, 12)48+ v.Select(0, false)49+ if name, text := v.Head(); name != "cell 1" || text != "a nushell pipeline; sets $files" {50+ t.Errorf("head %q %q", name, text)51+ }52+ v.Select(1, false)53+ if _, text := v.Head(); text != "reads $files" {54+ t.Errorf("head of the reader %q", text)55+ }56+ if (notebook.Cell{Source: h.cells[0].Source}).Name() != "" {57+ t.Error("the cell is named")58+ }59+}@@ -71,12 +71,30 @@ var operators = []string{"and", "or", "not", "xor", "in", "not-in", "like", "not
7171 7272 // Highlight implements Highlighter.7373 func (Tokens) Highlight(_ context.Context, src string) []Span {74- t := tokenizer{src: src, command: true}75- if name, _ := notebook.SplitName(src); name != "" {76- t.i = strings.Index(src, name) + len(name)77- t.add(t.i-len(name), theme.SyntaxVariable) // the cell's name, as $name reads it74+ t := tokenizer{src: src, command: true, whole: map[int]Span{}}75+ p := notebook.Parse(src)76+ for _, st := range p.Stmts {77+ if st.Name != "" { // a name assigned, as $name reads it78+ t.whole[st.From] = Span{st.From, st.From + len(st.Name), theme.SyntaxVariable}79+ }80+ }81+ for _, c := range p.Comments {82+ t.whole[c[0]] = Span{c[0], c[1], theme.SyntaxComment}83+ }84+ for _, s := range p.Strings {85+ if src[s[0]] == 'r' { // a raw string, r#'...'#86+ t.whole[s[0]] = Span{s[0], s[1], theme.SyntaxString}87+ }7888 }7989 for t.i < len(t.src) {90+ if s, ok := t.whole[t.i]; ok {91+ t.out = append(t.out, s)92+ t.i = s.To93+ if s.Kind != theme.SyntaxComment {94+ t.command = false95+ }96+ continue97+ }8098 if !t.punct() {8199 t.word()82100 }@@ -91,6 +109,9 @@ type tokenizer struct {
91109 i int92110 command bool93111 out []Span112+ // whole are the tokens the source's reading knows (notebook.Parse),113+ // by where they start: names assigned, comments, raw strings.114+ whole map[int]Span94115 }95116 96117 func (t *tokenizer) add(from int, k theme.Syntax) { t.out = append(t.out, Span{from, t.i, k}) }@@ -109,9 +130,6 @@ func (t *tokenizer) punct() bool {
109130 t.command = true110131 case c == ' ' || c == '\t' || c == '\r' || c == ')' || c == '}' || c == '[' || c == ']' || c == ',':111132 t.i++112- case c == '#' && (i == 0 || isSpace(src[i-1])):113- t.i = lineEnd(src, i)114- t.add(start, theme.SyntaxComment)115133 case c == '"' || c == '\'' || c == '`':116134 t.i = quoted(src, i)117135 t.add(start, theme.SyntaxString)@@ -176,13 +194,6 @@ func wordEnd(src string, i int) int {
176194 return i177195 }178196 179-func lineEnd(src string, i int) int {180- for i < len(src) && src[i] != '\n' {181- i++182- }183- return i184-}185-186197 // quoted is where a string starting at i ends: its closing quote, a187198 // double-quoted string's escapes skipped, or the end.188199 func quoted(src string, i int) int {