--notebooks runs every notebook's code cells, as F9 does, where --regions ran command regions; opening a file sends the outputs it kept to their sheets, as the screen does, and saving keeps them within the nu-save caps. diff lines notebook cells up by their sources and dumps them for textconv; merge takes a notebook's cells as one part, and trust follows notebook sources and linked files.
- commit
- c4c579cc1bb331655a021b90fb9794a508c0bca6
- parent
- 7260ce3e364185f3e0ea6dcd6e829f41d6754891
- author
- Dami <dami.etoile@gmail.com>
- kind
- feat risk score 2.7 of 3, Jev's estimate
| M | ROADMAP.md | 2 | +- |
| M | cmd/ | 36 | ++-- |
| M | cmd/ | 2 | +- |
| M | cmd/ | 4 | +- |
| M | cmd/ | 53 | +++++--- |
| M | cmd/ | 42 | +++-- |
| M | docs/ | 10 | +- |
| M | docs/ | 22 | ++- |
| M | docs/ | 33 | +++-- |
| M | docs/ | 6 | +- |
| M | internal/ | 10 | +- |
| M | internal/ | 13 | +- |
| M | internal/ | 34 | ++++- |
| M | internal/ | 18 | ++- |
| M | internal/ | 11 | +- |
| M | internal/ | 2 | +- |
| M | internal/ | 16 | ++ |
| A | internal/ | 160 | ++++++++++++++++++++++ |
| M | internal/ | 6 | + |
| M | internal/ | 10 | +- |
| M | internal/ | 62 | ++++++-- |
| A | internal/ | 54 | +++++++ |
| A | internal/ | 220 | ++++++++++++++++++++++++++++++ |
| D | internal/ | 138 | ------------------- |
@@ -143,7 +143,7 @@ stream above rather than a design of their own.
143143 144144 - Import CSV, TSV, JSON, NUON, XLSX, SQLite, Parquet and Lotus `.wk1`; export CSV, TSV, JSON, NUON, XLSX and SQLite; import locations; save-as and overwrite checks: [Files](docs/files/README.md)145145 - `.012` files read and written as a stream, cells straight into and out of the store: [The .012 format](docs/files/format.md#reading-and-writing)146-- Commands for scripts without the screen: `012 get` (text, CSV, TSV, JSON or NUON), `012 set`, `012 recalc` and `012 export`, running regions and JEV only behind flags: [Scripts](docs/files/scripts.md)146+- Commands for scripts without the screen: `012 get` (text, CSV, TSV, JSON or NUON), `012 set`, `012 recalc` and `012 export`, running notebooks and JEV only behind flags: [Scripts](docs/files/scripts.md)147147 - `012 diff` cell by cell, as git's diff command or textconv, and `012 merge-driver` merging cell by cell with conflicts noted on the cells: [Diff and merge in git](docs/files/git.md)148148 - Following files: a linked region follows a CSV, TSV, JSON lines or NUON file as it grows and any importable file as it's rewritten, keeping every row or the last ones, its rows arriving as the change stream: [Following files](docs/files/following.md)149149 @@ -18,15 +18,15 @@ import (
1818 const jevTimeout = 60 * time.Second1919 2020 // evaluate runs what the flags ask of a workbook before it's read:21-// --regions runs its notebook regions' commands with nu, when the22-// workbook is trusted here or --trust says so, as the screen asks before23-// running them; --jev answers its JEV functions over the network. Region24-// failures are reported on standard error as they happen.21+// --notebooks runs its notebook cells with nu, when the workbook is22+// trusted here or --trust says so, as the screen asks before running23+// them; --jev answers its JEV functions over the network. Cells that24+// fail are reported on standard error.2525 func evaluate(f *headless.File, a cliArgs, e env) evalFailures {26- if a.has("trust") && !a.has("regions") {27- return evalFailures{stop: usageError("--trust goes with --regions", "")}26+ if a.has("trust") && !a.has("notebooks") {27+ return evalFailures{stop: usageError("--trust goes with --notebooks", "")}2828 }29- if !a.has("regions") && !a.has("jev") {29+ if !a.has("notebooks") && !a.has("jev") {3030 return evalFailures{}3131 }3232 cfg, err := loadConfig(e, nil)@@ -36,14 +36,14 @@ func evaluate(f *headless.File, a cliArgs, e env) evalFailures {
3636 ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt)3737 defer stop()3838 var out evalFailures39- if a.has("regions") {39+ if a.has("notebooks") {4040 if err := mayRun(f, cfg, a.has("trust")); err != nil {4141 return evalFailures{stop: err}4242 }43- out.regions = headless.RunRegions(ctx, f.Book, headless.RegionOptions{43+ out.cells = headless.RunNotebooks(ctx, f.Book, headless.NotebookOptions{4444 Runner: e.nuRunner(), Timeout: cfg.Duration("nu-timeout"), NuConfig: cfg.Bool("nu-config")})45- for _, r := range out.regions {46- fmt.Fprintln(e.stderr, "012: "+r)45+ for _, c := range out.cells {46+ fmt.Fprintln(e.stderr, "012: "+c)4747 }4848 }4949 if a.has("jev") {@@ -56,14 +56,14 @@ func evaluate(f *headless.File, a cliArgs, e env) evalFailures {
5656 return out5757 }5858 59-// mayRun reports why a workbook's commands may not run: shell = off, or60-// a workbook from another computer without --trust (or shell = on). A61-// trusted workbook is marked trusted here, which a command that saves62-// keeps, as answering Run on the screen does.59+// mayRun reports why a workbook's notebook cells may not run: shell =60+// off, or a workbook from another computer without --trust (or shell =61+// on). A trusted workbook is marked trusted here, which a command that62+// saves keeps, as answering Run on the screen does.6363 func mayRun(f *headless.File, cfg *config.Config, trust bool) error {6464 switch cfg.String("shell") {6565 case "off":66- return errors.New("shell commands are off (shell = off in your config)")66+ return errors.New("notebook cells don't run (shell = off in your config)")6767 case "on":6868 return nil6969 }@@ -72,7 +72,7 @@ func mayRun(f *headless.File, cfg *config.Config, trust bool) error {
7272 return nil7373 }7474 if !trust {75- return fmt.Errorf("%s was saved on another computer: its commands would run as you, with your files; --trust runs them", f.Path)75+ return fmt.Errorf("%s was saved on another computer: its notebook cells would run as you, with your files; --trust runs them", f.Path)7676 }7777 if machine != "" {7878 f.Book.SetMacroOrigin(machine)@@ -80,7 +80,7 @@ func mayRun(f *headless.File, cfg *config.Config, trust bool) error {
8080 return nil8181 }8282 83-// nuRunner runs region commands: nu, or the test's fake.83+// nuRunner runs notebook cells: nu, or the test's fake.8484 func (e env) nuRunner() nushell.Runner {8585 if e.nu != nil {8686 return e.nu@@ -12,7 +12,7 @@ import (
1212 "github.com/FelineStateMachine/012/internal/sheet"1313 )1414 15-const exportUsage = "usage: 012 export file.012 out.csv|tsv|xlsx|json|nuon|sqlite [ref] [--format kind] [--table name] [--regions] [--jev] [--trust]"15+const exportUsage = "usage: 012 export file.012 out.csv|tsv|xlsx|json|nuon|sqlite [ref] [--format kind] [--table name] [--notebooks] [--jev] [--trust]"1616 1717 // runExport is 012 export: a sheet, a range or (to formats holding1818 // several sheets) the whole workbook written as File > Download writes@@ -48,8 +48,8 @@ type env struct {
4848 // stdoutTTY is set when standard output is a terminal, where 0124949 // diff colors what it writes.5050 stdoutTTY bool51- // nu runs notebook regions' commands for 012 get, recalc and export52- // --regions; nil is the real nu.51+ // nu runs notebook cells for 012 get, recalc and export52+ // --notebooks; nil is the real nu.5353 nu nushell.Runner5454 }5555 @@ -8,22 +8,23 @@ import (
88 "strings"99 1010 "github.com/FelineStateMachine/012/internal/headless"11+ "github.com/FelineStateMachine/012/internal/notebook"1112 "github.com/FelineStateMachine/012/internal/sheet"1213 )1314 1415 // 012 get, set and recalc: workbook files read and changed without the1516 // screen, for scripts (see docs/files/scripts.md). 012 export is in16-// export.go, and what --regions, --jev and --trust run in evaluate.go.17+// export.go, and what --notebooks, --jev and --trust run in evaluate.go.1718 1819 const (19- getUsage = "usage: 012 get file.012 [ref] [--format text|csv|tsv|json|nuon] [--input] [--no-header] [--regions] [--jev] [--trust]"20+ getUsage = "usage: 012 get file.012 [ref] [--format text|csv|tsv|json|nuon] [--input] [--no-header] [--notebooks] [--jev] [--trust]"2021 setUsage = "usage: 012 set file.012 ref input [ref input ...] [--force]"21- recalcUsage = "usage: 012 recalc file.012 [--regions] [--jev] [--trust]"22+ recalcUsage = "usage: 012 recalc file.012 [--notebooks] [--jev] [--trust]"2223 )2324 24-// evalFlags are the flags that let a command run a workbook's commands25-// or ask JEV; nothing runs without them.26-var evalFlags = []string{"regions", "jev", "trust"}25+// evalFlags are the flags that let a command run a workbook's notebook26+// cells or ask JEV; nothing runs without them.27+var evalFlags = []string{"notebooks", "jev", "trust"}2728 2829 // runGet is 012 get: a cell's value or input, or a range's as a table.2930 func runGet(args []string, e env) error {@@ -56,16 +57,16 @@ func runGet(args []string, e env) error {
5657 if err := headless.Get(e.stdout, t, o); err != nil {5758 return err5859 }59- if !a.has("regions") && hasCommands(t.Sheet) {60- fmt.Fprintln(e.stderr, "012: note: "+t.Sheet.Name()+"'s notebook regions are empty unless --regions runs them")60+ if !a.has("notebooks") && hasOutputs(t.Sheet) {61+ fmt.Fprintln(e.stderr, "012: note: "+t.Sheet.Name()+" shows notebook outputs as the file kept them; --notebooks runs the cells again")6162 }6263 return failed.err()6364 }6465 65-// hasCommands reports whether s has regions filled by commands.66-func hasCommands(s *sheet.Sheet) bool {66+// hasOutputs reports whether notebook cells' outputs were sent to s.67+func hasOutputs(s *sheet.Sheet) bool {6768 for _, r := range s.Regions() {68- if !r.Linked() {69+ if r.Output {6970 return true7071 }7172 }@@ -99,7 +100,7 @@ func runSet(args []string, e env) error {
99100 for _, w := range warnings {100101 fmt.Fprintln(e.stderr, "012: warning: "+w)101102 }102- return f.Save()103+ return save(f, e)103104 }104105 105106 // runRecalc is 012 recalc: recalculate, save, and list the cells whose@@ -124,12 +125,24 @@ func runRecalc(args []string, e env) error {
124125 return err125126 }126127 problems, circular := headless.Recalc(f.Book)127- if err := f.Save(); err != nil {128+ if err := save(f, e); err != nil {128129 return err129130 }130131 return reportProblems(e.stdout, problems, circular, failed)131132 }132133 134+// save saves a workbook, keeping as much of its notebook outputs as the135+// config's nu-save-cell-kb and nu-save-notebook-kb allow, as the screen136+// saves it.137+func save(f *headless.File, e env) error {138+ cfg, err := loadConfig(e, nil)139+ if err != nil {140+ return err141+ }142+ f.Book.SetOutputCaps(notebook.Caps{Cell: cfg.Int("nu-save-cell-kb") << 10, Total: cfg.Int("nu-save-notebook-kb") << 10})143+ return f.Save()144+}145+133146 // reportProblems lists the cells showing errors, one a line, and says134147 // how many; the status is 1 when there are any or a region failed.135148 func reportProblems(w io.Writer, problems []headless.Problem, circular bool, failed evalFailures) error {@@ -159,20 +172,20 @@ func plural(n int, one, many string) string {
159172 return many160173 }161174 162-// evalFailures are what --regions and --jev couldn't do: a reason163-// nothing ran at all (fatal), or regions whose commands failed.175+// evalFailures are what --notebooks and --jev couldn't do: a reason176+// nothing ran at all (fatal), or notebook cells that failed.164177 type evalFailures struct {165- stop error166- regions []string178+ stop error179+ cells []string167180 }168181 169182 func (f evalFailures) fatal() error { return f.stop }170183 171184 // err is the status once the command's output is written: 1 when a172-// region failed, each already reported.185+// notebook cell failed, each already reported.173186 func (f evalFailures) err() error {174- if len(f.regions) == 0 {187+ if len(f.cells) == 0 {175188 return nil176189 }177- return &exitError{code: 1, err: errors.New(fmt.Sprint(len(f.regions)) + " " + plural(len(f.regions), "region", "regions") + " failed")}190+ return &exitError{code: 1, err: errors.New(fmt.Sprint(len(f.cells)) + " notebook " + plural(len(f.cells), "cell", "cells") + " failed")}178191 }@@ -9,6 +9,7 @@ import (
99 "strings"1010 "testing"1111 12+ "github.com/FelineStateMachine/012/internal/notebook"1213 "github.com/FelineStateMachine/012/internal/nushell"1314 "github.com/FelineStateMachine/012/internal/sheet"1415 )@@ -60,7 +61,7 @@ func TestGetSetRecalc(t *testing.T) {
6061 {[]string{"get", path, "Q3!A1"}, 1, `no sheet named "Q3"`},6162 {[]string{"get", path, "A1", "--format", "xml"}, 2, "--format xml: 012 get writes text, csv, tsv, json, nuon"},6263 {[]string{"get", path, "A1", "--bogus"}, 2, "unknown flag --bogus"},63- {[]string{"get", path, "A1", "--trust"}, 2, "--trust goes with --regions"},64+ {[]string{"get", path, "A1", "--trust"}, 2, "--trust goes with --notebooks"},6465 {[]string{"get", filepath.Join(t.TempDir(), "none.012"), "A1"}, 1, "no such file"},6566 {[]string{"set", path, "A1"}, 2, "usage: 012 set"},6667 {[]string{"set", path, "B4", "=SUM(B1"}, 1, "Sheet1!B4: Expected , or ) in SUM, at character 8 of =SUM(B1; " + path + " is unchanged"},@@ -137,13 +138,15 @@ func (f *fakeNu) Run(_ context.Context, _ nushell.Job, _ string, stdout io.Write
137138 return err138139 }139140 140-// notebookFile writes a workbook with a region, made on the computer141-// origin.141+// notebookFile writes a workbook whose notebook's cell files = ls is142+// sent to Sheet1!A1, made on the computer origin.142143 func notebookFile(t *testing.T, origin string) string {143144 t.Helper()144145 w := sheet.NewBook()145- s, _ := w.AddNotebook("Shell 1", nil)146- if err := s.AddRegion(sheet.Region{Name: "r1", Command: "ls"}); err != nil {146+ nb, _ := w.AddNotebook("Notebook", w.Sheet(0))147+ nb.SetNotebookCells("add cell", []notebook.Cell{{Kind: notebook.Code, Source: "files = ls"}})148+ a1, _ := sheet.ParseAddr("A1")149+ if err := w.Sheet(0).AddRegion(sheet.Region{Name: "files", At: a1, Output: true}); err != nil {147150 t.Fatal(err)148151 }149152 w.SetMacroOrigin(origin)@@ -154,41 +157,46 @@ func notebookFile(t *testing.T, origin string) string {
154157 return path155158 }156159 157-func TestRegionsNeedTrust(t *testing.T) {160+func TestNotebooksNeedTrust(t *testing.T) {158161 e, out, errOut := scriptEnv(t)159162 nu := &fakeNu{}160163 e.nu = nu161164 path := notebookFile(t, "another computer")162165 163- status(e, "get", path, "'Shell 1'!A2")164- if nu.ran != 0 || out.String() != "\n" || !strings.Contains(errOut.String(), "notebook regions are empty unless --regions runs them") {165- t.Errorf("without --regions: ran %d, %q %q", nu.ran, out, errOut)166+ status(e, "get", path, "A2")167+ if nu.ran != 0 || out.String() != "\n" || !strings.Contains(errOut.String(), "Sheet1 shows notebook outputs as the file kept them; --notebooks runs the cells again") {168+ t.Errorf("without --notebooks: ran %d, %q %q", nu.ran, out, errOut)166169 }167- if code, err := status(e, "get", path, "'Shell 1'!A2", "--regions"); code != 1 || nu.ran != 0 || !strings.Contains(err.Error(), "saved on another computer") {170+ if code, err := status(e, "get", path, "A2", "--notebooks"); code != 1 || nu.ran != 0 || !strings.Contains(err.Error(), "saved on another computer") {168171 t.Errorf("untrusted: %d %v, ran %d", code, err, nu.ran)169172 }170173 out.Reset()171- if code, err := status(e, "get", path, "'Shell 1'!A3", "--regions", "--trust"); code != 0 || nu.ran != 1 || out.String() != "a.txt\n" {174+ if code, err := status(e, "get", path, "A2", "--notebooks", "--trust"); code != 0 || nu.ran != 1 || out.String() != "a.txt\n" {172175 t.Errorf("--trust: %d %v, ran %d, %q", code, err, nu.ran, out)173176 }174- // get saves nothing, so the file still asks; recalc saves the trust.175- if _, err := status(e, "recalc", path, "--regions"); err == nil {177+ // get saves nothing, so the file still asks; recalc saves the trust,178+ // and the output, which get then reads without running anything.179+ if _, err := status(e, "recalc", path, "--notebooks"); err == nil {176180 t.Error("get --trust saved the trust")177181 }178- if code, err := status(e, "recalc", path, "--regions", "--trust"); code != 0 {182+ if code, err := status(e, "recalc", path, "--notebooks", "--trust"); code != 0 {179183 t.Fatal(err)180184 }181- if code, err := status(e, "get", path, "'Shell 1'!A2", "--regions"); code != 0 || nu.ran != 3 {185+ out.Reset()186+ if code, err := status(e, "get", path, "A2"); code != 0 || nu.ran != 2 || out.String() != "a.txt\n" {187+ t.Errorf("the output saved: %d %v, ran %d, %q", code, err, nu.ran, out)188+ }189+ if code, err := status(e, "get", path, "A2", "--notebooks"); code != 0 || nu.ran != 3 {182190 t.Errorf("trusted by recalc: %d %v, ran %d", code, err, nu.ran)183191 }184192 185193 writeConfig(t, "shell = off\n")186- if _, err := status(e, "get", path, "A1", "--regions", "--trust"); err == nil || !strings.Contains(err.Error(), "shell = off") {194+ if _, err := status(e, "get", path, "A1", "--notebooks", "--trust"); err == nil || !strings.Contains(err.Error(), "shell = off") {187195 t.Errorf("shell = off: %v", err)188196 }189197 writeConfig(t, "shell = on\n")190198 other := notebookFile(t, "elsewhere")191- if code, err := status(e, "get", other, "A1", "--regions"); code != 0 {199+ if code, err := status(e, "get", other, "A1", "--notebooks"); code != 0 {192200 t.Errorf("shell = on: %v", err)193201 }194202 }@@ -20,7 +20,7 @@ internal/locale the locales: separators, date order, currency, formula separato
2020 internal/fileio import and export: CSV, TSV, XLSX, SQLite, Parquet, Lotus .wk1, JSON, NUON; tables on streams2121 internal/nuon nushell's object notation: typed values, tables read a row at a time, written back2222 internal/live the sources of linked regions: files followed as they grow or are rewritten23-internal/headless workbook files without the screen: references, get, set, recalc, regions and JEV when asked, atomic saves23+internal/headless workbook files without the screen: references, get, set, recalc, notebooks and JEV when asked, atomic saves2424 internal/diff workbooks compared and merged cell by cell, from their files' fields2525 internal/chart chart layout, text rendering and kitty image encoding2626 internal/jev the API key's resolution, answer cache and TypeSafe client@@ -625,9 +625,11 @@ subpackage behind the same table row.
625625 `set`, `recalc` and `export` ([Scripts](../files/scripts.md)): it626626 resolves references, writes values through the same snapshots and627627 encoders as downloads, types entries through `Sheet.Set` in one628-`Batch`, and saves with a write-then-rename. It runs regions through629-`internal/nushell` and asks `internal/jev` only when its caller says630-so; `cmd/012` decides, by the flags and the macro-origin trust.628+`Batch`, and saves with a write-then-rename. It sends the outputs a629+file kept to their sheets as the screen does on opening, runs notebook630+cells through `internal/nushell` and asks `internal/jev` only when its631+caller says so; `cmd/012` decides, by the flags and the macro-origin632+trust.631633 632634 `internal/diff` reads a `.012` file as its JSON fields: the workbook's,633635 and each sheet's with its cells by address. Comparing and merging go@@ -25,7 +25,7 @@ Q4!A12 input + Total
2525 Q4!B3 format currency, decimals 2 → currency, bold, decimals 22626 Q4!B3 note check → checked2727 Q4 widths C 10 → 1428-Q4 region r2 command ls → ls -a28+Notebook cell 2 source ls → ls -a2929 name Sales range Q3!B2:B20 → Q4!B2:B213030 ```3131 @@ -40,15 +40,17 @@ What it compares:
4040 |---|---|4141 | Sheets | `added`, `removed`, `renamed from`, `moved from 3 to 1`. A sheet under a new name that holds at least half the cells of one gone is that sheet renamed |4242 | Cells | `input` (what was typed: a formula or an entry), `value` (what a formula computes, or an array spilled), `format` (the cell's formatting, by its fields in the file), `note` |43-| Regions | Each [notebook region](../nushell/notebooks.md) or [linked file](following.md) by name: added, removed, or which field changed |43+| Notebooks | Each [notebook](../nushell/notebooks.md) cell, lined up by its source: `added`, `removed`, or which field changed (`source`, `output`, `error`) |44+| Regions | Each [linked file](following.md) or notebook output sent to a sheet, by name: added, removed, or which field changed |4445 | Layout | Every other field of a sheet, by its name in [the file](format.md): `widths`, `heights` and `lines` key by key, the rest (`charts`, `conditionalFormats`, `freeze`) whole |4546 | Workbook | Named ranges, macros, the locale and decimal arithmetic |4647 4748 A value is shown only for formulas and cells an array filled, since a4849 typed entry's value is its input. Values that change on every4950 recalculation (`NOW()`, `RAND()` and what reads them, on any sheet)50-aren't compared, and neither is anything that needs a region's command51-or JEV run: `012 diff` only reads the two files. The sheet shown, the51+aren't compared, and neither is anything that needs a notebook cell or52+JEV run: `012 diff` only reads the two files, whose notebook outputs are53+compared as they were saved. The sheet shown, the5254 file's version, the notebook prompt's history and which computer5355 trusted its commands aren't changes either.5456 @@ -103,7 +105,7 @@ on its own:
103105 104106 ```mermaid105107 flowchart TD106- part["A part: a cell's input, format or note; a sheet's name; a column's width; a region; a named range"]108+ part["A part: a cell's input, format or note; a sheet's name; a column's width; a region; a notebook's cells; a named range"]107109 part --> same{"Did both sides change it?"}108110 same -- "One side, or neither" --> take["Take the side that changed it"]109111 same -- "Both, alike" --> take@@ -117,7 +119,9 @@ them): a sheet renamed on one side and edited on the other merges both.
117119 A sheet removed on one side goes, unless the other side changed it,118120 which is a conflict: ours stays (kept when ours changed it, gone when119121 ours removed it). Sheets added on both sides under one name merge cell120-by cell.122+by cell. A notebook's cells are one part, their sources compared: the123+side that changed them brings its cells and their outputs, and a124+notebook run on both sides with the same cells keeps ours' outputs.121125 122126 On conflicts the merge keeps ours for each, adds to the cell's note what123127 theirs had (`Merge conflict, kept ours; theirs had input 11`), so the@@ -143,6 +147,6 @@ Two limits:
143147 cells. Merge such a change in 012 instead: open both versions and copy144148 across.145149 - Trust in a file's commands follows [macros' rule](../sheets/macros.md#macros-from-other-computers):146- the merged file keeps ours' trust only when its macros and regions'147- commands are ours', or theirs were trusted on the same computer.148- Otherwise the next time the file runs a command, 012 asks first.150+ the merged file keeps ours' trust only when its macros, notebook cells151+ and linked files are ours', or theirs were trusted on the same152+ computer. Otherwise the next time the file runs one, 012 asks first.@@ -17,7 +17,7 @@ shell scripts, cron jobs, Makefiles and nushell pipelines:
1717 ```1818 1919 None of them runs a program or reaches the network unless a flag asks:20-see [Regions and JEV functions](#regions-and-jev-functions). The flags20+see [Notebooks and JEV functions](#notebooks-and-jev-functions). The flags2121 and exit statuses of every command are in2222 [Command line](../reference/command-line.md).2323 @@ -88,7 +88,7 @@ can't take, naming the cell:
8888 |---|---|8989 | A formula that doesn't parse | `Sheet1!B7: Expected , or ) in SUM, at character 11 of =SUM(A1:A6` |9090 | An entry a validation rule rejects | the rule's help, as the context line would |91-| A cell of an array's result, a pivot table or a region | the same message as on the screen |91+| A cell of an array's result, a pivot table, a linked file or a notebook's output | the same message as on the screen |9292 | A cell in a [protected range](../sheets/notes-protection.md) | `Sheet1!B7 is protected (B1:B9): --force sets it anyway` |9393 | A reference to a range, not one cell | `A1:B2 is the range A1:B2: name one cell` |9494 @@ -121,22 +121,27 @@ written as their values where the format has no formulas, and a note on
121121 standard error says how many. CSV, TSV, JSON and NUON leave out the rows122122 a filter hides, as a download does.123123 124-## Regions and JEV functions124+## Notebooks and JEV functions125125 126126 A `.012` file stores what was typed and computes the rest, so a few127127 things need more than the file:128128 129-- **Notebook regions' tables** come from their nushell commands, which130- these commands don't run: a region's table is empty, and `get` says so131- on standard error. `--regions` runs every region's command, each after132- those it reads, as Run all regions (F9) does. As on the screen, a133- [file saved on another computer](../nushell/notebooks.md#saving-and-trust)134- asks first: `--regions` refuses it unless `--trust` gives the answer135- Run would, and `recalc --regions --trust` saves that trust in the file136- as the screen does, so later runs don't need it. `shell = off` in the137- [configuration](../reference/config.md#shell) turns `--regions` off,138- `shell = on` runs every file's commands, and each command times out139- after `nu-timeout`.129+- **[Notebook](../nushell/notebooks.md) outputs** are kept in the file130+ as their cells last left them, and these commands read them as saved,131+ sent to sheets as on the screen; `get` notes on standard error when132+ the sheet it reads shows outputs. `--notebooks` runs every notebook's133+ code cells first, each after the cells it reads, as F9 does; a cell134+ that fails stops the rest of its notebook, and is reported. A cell135+ reading `$selection` fails, having no selection to read.136+- As on the screen, a notebook137+ [saved on another computer](../nushell/notebooks.md#saving-and-trust)138+ asks first: `--notebooks` refuses it unless `--trust` gives the answer139+ Run would, and `recalc --notebooks --trust` saves that trust in the140+ file as the screen does, so later runs don't need it. `shell = off` in141+ the [configuration](../reference/config.md#shell) turns `--notebooks`142+ off, `shell = on` runs every file's cells, each cell stops after143+ `nu-timeout`, and saving keeps as much of the outputs as144+ `nu-save-cell-kb` and `nu-save-notebook-kb` allow.140145 - **[Linked files](following.md)** are followed only on the screen; their141146 regions are empty here.142147 - **[JEV functions](../formulas/jev.md)** ask a hosted model over the@@ -48,8 +48,8 @@ flag.
4848 | `--no-header` | `get` | JSON and NUON columns named by their letters, the first row a record |4949 | `--force` | `set` | Sets cells in protected ranges |5050 | `--format kind`, `--table name` | `export` | The format when the file's extension doesn't say; SQLite's table |51-| `--regions` | `get`, `recalc`, `export` | Runs the notebook regions' commands first, when the file is trusted here |52-| `--trust` | with `--regions` | Runs the commands of a file saved on another computer |51+| `--notebooks` | `get`, `recalc`, `export` | Runs the notebooks' cells first, when the file is trusted here |52+| `--trust` | with `--notebooks` | Runs the cells of a file saved on another computer |5353 | `--jev` | `get`, `recalc`, `export` | Asks JEV for the JEV functions' answers first |5454 | `--format text\|json\|nuon` | `diff` | The form of the changes; text by default |5555 | `--color auto\|always\|never` | `diff` | Colors the text; auto colors on a terminal or git's pager unless `NO_COLOR` is set |@@ -59,5 +59,5 @@ flag.
5959 | Status | Means |6060 |---|---|6161 | 0 | Done; for `diff`, the workbooks are the same |62-| 1 | An error, said on standard error; for `recalc`, cells show errors (after saving); for `diff`, the workbooks differ; for `merge-driver`, conflicts; for `get`, `recalc` and `export` with `--regions`, a region's command failed (after writing) |62+| 1 | An error, said on standard error; for `recalc`, cells show errors (after saving); for `diff`, the workbooks differ; for `merge-driver`, conflicts; for `get`, `recalc` and `export` with `--notebooks`, a notebook cell failed (after writing) |6363 | 2 | The command was used wrongly (its usage is printed); for `diff` and `merge-driver`, trouble reading or merging the files |@@ -96,6 +96,7 @@ func formatText(raw json.RawMessage) string {
9696 // what the changes call the sheet.9797 func sheetDiff(a, b side, name string) []Change {9898 out := cellChanges(a, b, name)99+ out = append(out, notebookChanges(name, field(a.raw, "notebookCells"), field(b.raw, "notebookCells"))...)99100 out = append(out, listChanges(KindRegion, name, field(a.raw, "regions"), field(b.raw, "regions"))...)100101 keys := map[string]bool{}101102 for _, s := range []*rawSheet{a.raw, b.raw} {@@ -104,7 +105,7 @@ func sheetDiff(a, b side, name string) []Change {
104105 }105106 }106107 for _, k := range slices.Sorted(maps.Keys(keys)) {107- if k != "regions" {108+ if k != "regions" && k != "notebookCells" {108109 out = append(out, fieldChanges(KindLayout, name, k, a.raw.fields[k], b.raw.fields[k])...)109110 }110111 }@@ -191,16 +192,11 @@ func valueChange(a, b *sheet.Sheet, at sheet.Addr) (Change, bool) {
191192 return Change{Kind: KindCell, Field: "value", Old: oldV, New: newV, oldText: oldT, newText: newT}, true192193 }193194 194-// cellValue is a cell's value typed by its format, and as it shows. A195-// region's label line shows its name and command, which the region's196-// own changes say, so it has none.195+// cellValue is a cell's value typed by its format, and as it shows.197196 func cellValue(s *sheet.Sheet, at sheet.Addr) (nuon.Value, string) {198197 if s == nil || s.Value(at).Kind == sheet.Empty {199198 return nuon.NullValue(), ""200199 }201- if _, label, ok := s.RegionAt(at); ok && label {202- return nuon.NullValue(), ""203- }204200 return fileio.CellValue(fileio.SnapCell{Value: s.Value(at), Format: s.DisplayFormat(at)}), s.LocalText(at)205201 }206202 @@ -1,14 +1,15 @@
11 // Package diff compares and merges .012 workbooks cell by cell, for22 // 012 diff and 012 merge-driver (see docs/files/git.md). Compare lists33 // what changed between two workbooks: sheets added, removed, renamed or4-// moved, each cell's input, value, format and note, regions, named5-// ranges, macros and the other fields of sheets and workbooks. Merge4+// moved, each cell's input, value, format and note, notebook cells,5+// regions, named ranges, macros and the other fields of sheets and6+// workbooks. Merge67 // combines two workbooks changed from a common one, and conflicts only78 // where both changed the same part of the same cell or field.89 //910 // Both read files as their JSON fields (raw.go), so nothing a file holds1011 // goes unseen, and use the engine only to compute formulas' values,11-// which the file doesn't store. Nothing here runs a region's command or12+// which the file doesn't store. Nothing here runs a notebook cell or1213 // asks JEV: values come from what the formulas compute alone.1314 package diff1415 @@ -83,15 +84,15 @@ func (c Change) Where() string {
8384 return "sheet " + c.Sheet8485 case KindRegion:8586 return c.Sheet + " region " + c.Item86- case KindLayout:87+ case KindLayout, KindNotebook:8788 return c.Sheet + " " + c.Item8889 }8990 return c.Kind + " " + c.Item9091 }9192 9293 // Compare lists what changed from a to b: sheets first, then each93-// sheet's cells in row order, its regions and other fields, then the94-// workbook's names, macros and settings.94+// sheet's cells in row order, a notebook tab's cells, its regions and95+// other fields, then the workbook's names, macros and settings.9596 func Compare(a, b *Book) []Change {9697 var out []Change9798 p := matchSheets(a.raw, b.raw)@@ -130,12 +130,12 @@ func TestCompareShapes(t *testing.T) {
130130 }131131 132132 func TestCompareRegionsAndBook(t *testing.T) {133- a := []byte(`{"version": 4, "locale": "en-US", "macros": [{"name": "m1", "api": 1, "source": "x"}], "sheets": [{"name": "S", "notebook": true, "regions": [{"name": "r1", "command": "ls", "at": "A1"}, {"name": "r2", "command": "ps", "at": "A9"}], "cells": {}}]}`)134- b := []byte(`{"version": 4, "locale": "de-DE", "active": 0, "sheets": [{"name": "S", "notebook": true, "regions": [{"name": "r1", "command": "ls -a", "at": "A1"}, {"name": "r3", "command": "date", "at": "A20"}], "cells": {}}]}`)135- want := `S region r1 command ls → ls -a136-S region r2 removed - ps137-S region r3 added + date138-macro m1 removed - (no command)133+ a := []byte(`{"version": 4, "locale": "en-US", "macros": [{"name": "m1", "api": 1, "source": "x"}], "sheets": [{"name": "S", "regions": [{"name": "app", "at": "A1", "path": "app.csv"}, {"name": "old", "at": "D1", "path": "old.csv"}], "cells": {}}]}`)134+ b := []byte(`{"version": 4, "locale": "de-DE", "active": 0, "sheets": [{"name": "S", "regions": [{"name": "app", "at": "A1", "path": "app.log"}, {"name": "sales", "at": "H1", "output": true}], "cells": {}}]}`)135+ want := `S region app path app.csv → app.log136+S region old removed - old.csv137+S region sales added + a notebook cell's output138+macro m1 removed - x139139 workbook locale en-US → de-DE140140 `141141 if got := lines(t, a, b); got != want {@@ -143,6 +143,28 @@ workbook locale en-US → de-DE
143143 }144144 }145145 146+func TestCompareNotebooks(t *testing.T) {147+ nb := func(cells string) []byte {148+ return []byte(`{"version": 4, "sheets": [{"name": "Sheet1", "cells": {}}, {"name": "Notebook", "tab": "notebook", "notebookCells": [` + cells + `], "cells": {}}]}`)149+ }150+ a := nb(`{"kind": "note", "source": "# Sales"}, {"source": "sales = open s.csv", "output": "[[n]; [1]]"}, {"source": "$sales | length", "output": "1"}, {"source": "gone"}`)151+ b := nb(`{"source": "new = ls"}, {"kind": "note", "source": "# Sales"}, {"source": "sales = open s.csv", "output": "[[n]; [2]]"}, {"source": "$sales | length", "error": "boom"}`)152+ want := `Notebook cell 1 added + new = ls153+Notebook cell 3 output [[n]; [1]] → [[n]; [2]]154+Notebook cell 4 error + boom155+Notebook cell 4 output - 1156+Notebook cell 4 removed - gone157+`158+ if got := lines(t, a, b); got != want {159+ t.Errorf("got\n%s\nwant\n%s", got, want)160+ }161+ var out bytes.Buffer162+ Dump(&out, read(t, b))163+ if !strings.Contains(out.String(), "Notebook cell 3 sales = open s.csv = [[n]; [2]]\nNotebook cell 4 $sales | length error: boom\n") {164+ t.Errorf("dump:\n%s", out.String())165+ }166+}167+146168 func TestWriteChanges(t *testing.T) {147169 a := file(t, []string{"Sheet1"}, []map[string]string{{"A1": "=1+1", "B1": "x"}}, nil)148170 b := file(t, []string{"Sheet1"}, []map[string]string{{"A1": "=1+2"}}, nil)@@ -115,16 +115,20 @@ func named(raw json.RawMessage) map[string]json.RawMessage {
115115 }116116 117117 // summary is what the text output shows of a region or macro added or118-// removed: its command, the file it follows, or its shortcut.118+// removed: the file a region follows or that it's a notebook's output;119+// a macro's script's first line.119120 func summary(raw json.RawMessage) string {120121 fields, _ := object(raw)121- for _, k := range []string{"command", "path", "key"} {122- var s string123- if json.Unmarshal(fields[k], &s) == nil && s != "" {124- return s125- }122+ var s string123+ switch {124+ case json.Unmarshal(fields["path"], &s) == nil && s != "":125+ return s126+ case string(fields["output"]) == "true":127+ return "a notebook cell's output"128+ case json.Unmarshal(fields["source"], &s) == nil && s != "":129+ return firstLine(s)126130 }127- return "(no command)"131+ return string(raw)128132 }129133 130134 // rawChange is a change between two JSON values, nil for none.@@ -136,8 +136,8 @@ func (m *merger) books(b, o, t *rawBook) *rawBook {
136136 }137137 138138 // trust keeps the computer ours trusted its commands on only when the139-// merge's commands (macros, regions' commands and files) are ours', or140-// theirs were trusted on the same computer, so a merge never makes139+// merge's commands (macros, notebook cells and linked files) are ours',140+// or theirs were trusted on the same computer, so a merge never makes141141 // someone else's commands run without asking.142142 func trust(out, o, t *rawBook) {143143 if equal(o.top["macroOrigin"], t.top["macroOrigin"]) || code(out) == code(o) {@@ -149,15 +149,16 @@ func trust(out, o, t *rawBook) {
149149 delete(out.top, "macroOrigin")150150 }151151 152-// code is what a workbook can make 012 run or read: its macros and its153-// regions' commands and files, in one string to compare.152+// code is what a workbook can make 012 run or read: its macros, its153+// notebooks' cells and the files it links, in one string to compare.154154 func code(b *rawBook) string {155155 var parts []string156156 parts = append(parts, string(b.top["macros"]))157157 for _, s := range b.sheets {158+ parts = append(parts, string(sources(s.fields["notebookCells"])))158159 for _, r := range named(s.fields["regions"]) {159160 f, _ := object(r)160- parts = append(parts, string(f["command"])+string(f["path"])+string(f["input"])+string(f["query"]))161+ parts = append(parts, string(f["command"])+string(f["path"])+string(f["table"])+string(f["query"]))161162 }162163 }163164 slices.Sort(parts)@@ -114,7 +114,7 @@ func TestMergeSheets(t *testing.T) {
114114 115115 func TestMergeTrust(t *testing.T) {116116 book := func(origin, command string) []byte {117- return []byte(`{"version": 4, "macroOrigin": "` + origin + `", "sheets": [{"name": "S", "notebook": true, "regions": [{"name": "r1", "command": "` + command + `", "at": "A1"}], "cells": {}}]}`)117+ return []byte(`{"version": 4, "macroOrigin": "` + origin + `", "sheets": [{"name": "S", "cells": {}}, {"name": "N", "tab": "notebook", "notebookCells": [{"source": "` + command + `"}], "cells": {}}]}`)118118 }119119 base := book("here", "ls")120120 cases := []struct {@@ -122,6 +122,20 @@ func str(s string) json.RawMessage {
122122 return raw123123 }124124 125+// notebook merges a notebook tab's cells as one part: their sources,126+// with the outputs of the side taken.127+func (m *merger) notebook(sheetName string, b, o, t json.RawMessage) json.RawMessage {128+ sb, so, st := sources(b), sources(o), sources(t)129+ switch {130+ case equal(so, st), equal(sb, st):131+ return o132+ case equal(sb, so):133+ return t134+ }135+ m.said(sheetName+" notebook", "", "cells changed on both sides")136+ return o137+}138+125139 // sheet merges one sheet: its name, each cell and each field.126140 func (m *merger) sheet(b, o, t *rawSheet) *rawSheet {127141 out := &rawSheet{fields: map[string]json.RawMessage{}, cells: map[string]json.RawMessage{}}@@ -136,6 +150,8 @@ func (m *merger) sheet(b, o, t *rawSheet) *rawSheet {
136150 switch k {137151 case "regions":138152 v = m.list(b.fields[k], o.fields[k], t.fields[k], func(key string) string { return out.name + " region " + key })153+ case "notebookCells":154+ v = m.notebook(out.name, b.fields[k], o.fields[k], t.fields[k])139155 case "widths", "heights", "lines":140156 v = m.object(b.fields[k], o.fields[k], t.fields[k], func(key string) (string, string) { return out.name + " " + k, key })141157 default:@@ -0,0 +1,160 @@
1+package diff2+3+import (4+ "encoding/json"5+ "fmt"6+ "strings"7+)8+9+// A notebook tab keeps its cells in the file as a list, each with its10+// source and what its last run left (see docs/files/format.md). Cells11+// have no names of their own to match them by, so comparing lines them12+// up by their sources, and merging treats the list of sources as one13+// part, whose outputs follow the side it's taken from.14+15+// KindNotebook is a Change to a notebook tab's cell: its source, or16+// what its last run left.17+const KindNotebook = "notebook"18+19+// nbCell is a notebook cell as the file has it.20+type nbCell struct {21+ raw json.RawMessage22+ fields map[string]json.RawMessage23+ source string24+}25+26+func nbCells(raw json.RawMessage) []nbCell {27+ var list []json.RawMessage28+ json.Unmarshal(raw, &list)29+ out := make([]nbCell, len(list))30+ for i, e := range list {31+ out[i].raw = compact(e)32+ out[i].fields, _ = object(e)33+ json.Unmarshal(out[i].fields["source"], &out[i].source)34+ }35+ return out36+}37+38+// sources is the part of a notebook merging compares: each cell's kind39+// and source, without the outputs runs leave.40+func sources(raw json.RawMessage) json.RawMessage {41+ if raw == nil {42+ return nil43+ }44+ var parts []string45+ for _, c := range nbCells(raw) {46+ parts = append(parts, string(c.fields["kind"])+" "+c.source)47+ }48+ out, _ := json.Marshal(parts)49+ return out50+}51+52+// notebookChanges compares two notebooks' cells: those whose sources53+// match, in order, by what their runs left; the others as sources54+// changed, added or removed.55+func notebookChanges(sheetName string, a, b json.RawMessage) []Change {56+ ca, cb := nbCells(a), nbCells(b)57+ var out []Change58+ i, j := 0, 059+ for _, m := range commonSources(ca, cb) {60+ out = append(out, unmatched(sheetName, ca[i:m[0]], cb[j:m[1]], i, j)...)61+ out = append(out, cellFields(sheetName, m[1], ca[m[0]], cb[m[1]])...)62+ i, j = m[0]+1, m[1]+163+ }64+ return append(out, unmatched(sheetName, ca[i:], cb[j:], i, j)...)65+}66+67+// unmatched compares the cells between two matched ones: side by side68+// as sources changed, the rest added or removed. i and j are where they69+// start on each side.70+func unmatched(sheetName string, a, b []nbCell, i, j int) []Change {71+ var out []Change72+ for k := 0; k < len(a) || k < len(b); k++ {73+ switch {74+ case k >= len(a):75+ out = append(out, nbChange(sheetName, j+k, "added", nil, b[k].fields["source"]))76+ case k >= len(b):77+ out = append(out, nbChange(sheetName, i+k, "removed", a[k].fields["source"], nil))78+ default:79+ out = append(out, cellFields(sheetName, j+k, a[k], b[k])...)80+ }81+ }82+ return out83+}84+85+// cellFields compares two cells field by field.86+func cellFields(sheetName string, at int, a, b nbCell) []Change {87+ var out []Change88+ for _, k := range unionKeys(a.fields, b.fields) {89+ if !equal(a.fields[k], b.fields[k]) {90+ out = append(out, nbChange(sheetName, at, k, a.fields[k], b.fields[k]))91+ }92+ }93+ return out94+}95+96+func nbChange(sheetName string, at int, field string, a, b json.RawMessage) Change {97+ return rawChange(KindNotebook, sheetName, fmt.Sprintf("cell %d", at+1), field, a, b)98+}99+100+// commonSources pairs the cells of a and b whose sources are the same,101+// as many as keep their order (a longest common subsequence), by index.102+func commonSources(a, b []nbCell) [][2]int {103+ n, m := len(a), len(b)104+ lcs := make([][]int, n+1)105+ for i := range lcs {106+ lcs[i] = make([]int, m+1)107+ }108+ for i := n - 1; i >= 0; i-- {109+ for j := m - 1; j >= 0; j-- {110+ if a[i].source == b[j].source {111+ lcs[i][j] = lcs[i+1][j+1] + 1112+ } else {113+ lcs[i][j] = max(lcs[i+1][j], lcs[i][j+1])114+ }115+ }116+ }117+ var out [][2]int118+ for i, j := 0, 0; i < n && j < m; {119+ switch {120+ case a[i].source == b[j].source:121+ out = append(out, [2]int{i, j})122+ i, j = i+1, j+1123+ case lcs[i+1][j] >= lcs[i][j+1]:124+ i++125+ default:126+ j++127+ }128+ }129+ return out130+}131+132+// dumpNotebook writes a notebook's cells as Dump's lines: the source,133+// and the first line of what the last run printed or said.134+func dumpNotebook(sheetName string, raw json.RawMessage) []string {135+ var out []string136+ for i, c := range nbCells(raw) {137+ line := fmt.Sprintf("%s cell %d %s", sheetName, i+1, quoted(c.source))138+ var s string139+ switch {140+ case json.Unmarshal(c.fields["error"], &s) == nil && s != "":141+ line += " error: " + firstLine(s)142+ case json.Unmarshal(c.fields["output"], &s) == nil && s != "":143+ line += " = " + firstLine(s)144+ }145+ out = append(out, line)146+ }147+ return out148+}149+150+// firstLine is s's first line, at most 120 characters of it.151+func firstLine(s string) string {152+ s, _, cut := strings.Cut(s, "\n")153+ if r := []rune(s); len(r) > 120 {154+ s, cut = string(r[:120]), true155+ }156+ if cut {157+ s += "…"158+ }159+ return s160+}@@ -134,6 +134,12 @@ func Dump(w io.Writer, b *Book) error {
134134 q := sheet.QuoteSheet(s.name)135135 fmt.Fprintf(bw, "sheet %s\n", s.name)136136 for _, k := range slices.Sorted(maps.Keys(s.fields)) {137+ if k == "notebookCells" {138+ for _, line := range dumpNotebook(s.name, s.fields[k]) {139+ bw.WriteString(line + "\n")140+ }141+ continue142+ }137143 if k == "regions" {138144 for _, name := range slices.Sorted(maps.Keys(named(s.fields[k]))) {139145 fmt.Fprintf(bw, "%s region %s %s\n", s.name, name, named(s.fields[k])[name])@@ -3,7 +3,7 @@
33 // a .012 file, finds the cells a reference names, reads and writes44 // them, lists the errors formulas show and saves atomically. Nothing55 // here runs a program or reaches the network unless its caller asks:6-// notebook regions run through RunRegions and JEV functions through6+// notebook cells run through RunNotebooks and JEV functions through77 // AnswerJEV, which the command line calls only behind flags.88 package headless99 @@ -27,9 +27,10 @@ type File struct {
2727 mode fs.FileMode2828 }2929 30-// Open reads the workbook at path. With create, a path with no file31-// opens as an empty workbook that Save writes there, as opening a new32-// name in 012 does.30+// Open reads the workbook at path, with the notebook outputs it kept31+// sent on to their sheets, as the screen opens it. With create, a path32+// with no file opens as an empty workbook that Save writes there, as33+// opening a new name in 012 does.3334 func Open(path string, create bool) (*File, error) {3435 f := &File{Path: path, mode: 0o644}3536 file, err := os.Open(path)@@ -50,6 +51,7 @@ func Open(path string, create bool) (*File, error) {
5051 if f.Book, err = sheet.ReadBook(file); err != nil {5152 return nil, fmt.Errorf("%s: %w", path, err)5253 }54+ SyncOutputs(f.Book)5355 return f, nil5456 }5557 @@ -10,6 +10,7 @@ import (
1010 "testing"1111 "time"1212 13+ "github.com/FelineStateMachine/012/internal/notebook"1314 "github.com/FelineStateMachine/012/internal/nushell"1415 "github.com/FelineStateMachine/012/internal/sheet"1516 )@@ -255,30 +256,61 @@ func (f fakeNu) Run(_ context.Context, job nushell.Job, _ string, stdout io.Writ
255256 return err256257 }257258 258-func TestRunRegions(t *testing.T) {259+func TestRunNotebooks(t *testing.T) {259260 w := sheet.NewBook()260- s, err := w.AddNotebook("Shell 1", nil)261+ grid := w.Sheet(0)262+ grid.Set(addr("D1"), "7")263+ nb, err := w.AddNotebook("Notebook", grid)261264 if err != nil {262265 t.Fatal(err)263266 }264- if err := s.AddRegion(sheet.Region{Name: "r1", Command: "ls"}); err != nil {267+ nb.SetNotebookCells("add cells", []notebook.Cell{268+ {Kind: notebook.Code, Source: "files = ls"},269+ {Kind: notebook.Note, Source: "# Files"},270+ {Kind: notebook.Code, Source: "n = $files | length"},271+ {Kind: notebook.Code, Source: "$sheet.D1:D1"},272+ {Kind: notebook.Code, Source: "boom"},273+ {Kind: notebook.Code, Source: "last = 1"},274+ })275+ if err := grid.AddRegion(sheet.Region{Name: "files", At: addr("A1"), Output: true}); err != nil {265276 t.Fatal(err)266277 }267- if err := s.AddRegion(sheet.Region{Name: "r2", Command: "$r1 | length", Deps: []string{"r1"}, At: addr("A10")}); err != nil {268- t.Fatal(err)269- }270- if err := s.AddRegion(sheet.Region{Name: "r3", Command: "boom", At: addr("A20")}); err != nil {271- t.Fatal(err)272- }273- nu := fakeNu{"ls": "[[name, size]; [a, 1], [b, 2]]", "$r1 | length": "[[n]; [2]]"}274- failed := RunRegions(context.Background(), w, RegionOptions{Runner: nu, Timeout: time.Second})275- if len(failed) != 1 || !strings.Contains(failed[0], "region r3 (boom): Command `boom` not found") {276- t.Errorf("failed: %v", failed)278+ nu := fakeNu{"ls": "[[name, size]; [a, 1], [b, 2]]", "$files | length": "2", "$__sheet1": "[[D]; [7]]"}279+ failed := RunNotebooks(context.Background(), w, NotebookOptions{Runner: nu, Timeout: time.Second})280+ if len(failed) != 1 || failed[0] != "Notebook cell 5: Command `boom` not found" {281+ t.Errorf("failed: %q", failed)277282 }278- target, _ := Resolve(w, "'Shell 1'!A2:B4")283+ target, _ := Resolve(w, "A1:B3")279284 var b bytes.Buffer280285 Get(&b, target, GetOptions{Format: "csv"})281286 if b.String() != "name,size\na,1\nb,2\n" {282- t.Errorf("r1's table:\n%s", b.String())287+ t.Errorf("the output sent to Sheet1:\n%s", b.String())288+ }289+ cells := nb.NotebookCells()290+ if o := w.Output(cells[2].ID); o == nil || string(o.NUON) != "2" || o.Reads["files"] == 0 {291+ t.Errorf("$files | length: %+v", o)292+ }293+ if o := w.Output(cells[4].ID); o == nil || o.Err != "Command `boom` not found" {294+ t.Errorf("boom: %+v", o)295+ }296+ if o := w.Output(cells[5].ID); o != nil {297+ t.Errorf("a cell after a failure ran: %+v", o)298+ }299+ if o := w.Output(cells[1].ID); o != nil {300+ t.Errorf("a note ran: %+v", o)301+ }302+}303+304+func TestOpenSendsOutputs(t *testing.T) {305+ path := filepath.Join(t.TempDir(), "nb.012")306+ os.WriteFile(path, []byte(`{"version": 4, "sheets": [307+{"name": "Sheet1", "regions": [{"name": "files", "at": "B2", "output": true}], "cells": {}},308+{"name": "Notebook", "tab": "notebook", "notebookCells": [{"source": "files = ls", "output": "[[name]; [a.txt]]"}], "cells": {}}]}`), 0o644)309+ f, err := Open(path, false)310+ if err != nil {311+ t.Fatal(err)312+ }313+ if got := f.Book.Sheet(0).LocalText(addr("B3")); got != "a.txt" {314+ t.Errorf("B3 = %q, want the output the file kept", got)283315 }284316 }@@ -0,0 +1,54 @@
1+package headless2+3+import (4+ "context"5+ "sync"6+ "time"7+8+ "github.com/FelineStateMachine/012/internal/jev"9+ "github.com/FelineStateMachine/012/internal/sheet"10+)11+12+// jevParallel is how many questions are asked at once, as on the13+// screen.14+const jevParallel = 815+16+// maxJEVRounds bounds the rounds of questions: answers can make new17+// questions (a JEV function reading another's answer), and each round18+// asks those the last one made.19+const maxJEVRounds = 1620+21+// AnswerJEV answers w's JEV functions with client, asking what the22+// workbook's formulas ask, a few questions at a time, until none is23+// waiting, each within timeout. It returns how many were asked.24+func AnswerJEV(ctx context.Context, w *sheet.Workbook, client jev.Client, timeout time.Duration) int {25+ cache := jev.NewCache()26+ w.SetRemote(cache)27+ asked := 028+ for range maxJEVRounds {29+ calls := cache.Take(1 << 30)30+ if len(calls) == 0 {31+ break32+ }33+ asked += len(calls)34+ answers := make([]sheet.RemoteAnswer, len(calls))35+ var wg sync.WaitGroup36+ sem := make(chan struct{}, jevParallel)37+ for i, c := range calls {38+ wg.Add(1)39+ sem <- struct{}{}40+ go func() {41+ defer func() { <-sem; wg.Done() }()42+ ctx, cancel := context.WithTimeout(ctx, timeout)43+ defer cancel()44+ answers[i] = jev.Ask(ctx, client, c)45+ }()46+ }47+ wg.Wait()48+ for i, c := range calls {49+ cache.Store(c, answers[i])50+ }51+ w.RecalcAnswered(calls)52+ }53+ return asked54+}@@ -0,0 +1,220 @@
1+package headless2+3+import (4+ "bytes"5+ "context"6+ "errors"7+ "fmt"8+ "time"9+10+ "github.com/FelineStateMachine/012/internal/fileio"11+ "github.com/FelineStateMachine/012/internal/notebook"12+ "github.com/FelineStateMachine/012/internal/nushell"13+ "github.com/FelineStateMachine/012/internal/sheet"14+)15+16+// NotebookOptions say how RunNotebooks runs cells.17+type NotebookOptions struct {18+ Runner nushell.Runner // nu, or a fake in tests19+ Timeout time.Duration // for each cell20+ NuConfig bool // run nu with the user's config files21+}22+23+// maxOutput is the most a run may print, as on the screen.24+const maxOutput = 512 << 2025+26+// RunNotebooks runs the code cells of each notebook tab, each after the27+// cells it reads, as Run all does on the screen: each cell's output28+// replaces the one the file kept and goes on to the sheet it was sent29+// to. A failure stops its notebook's run, as on the screen. It returns30+// a line for each cell that failed or couldn't run.31+func RunNotebooks(ctx context.Context, w *sheet.Workbook, o NotebookOptions) []string {32+ var failed []string33+ for _, s := range w.Sheets() {34+ if s.IsNotebook() {35+ failed = append(failed, runNotebook(ctx, w, s, o)...)36+ }37+ }38+ return failed39+}40+41+func runNotebook(ctx context.Context, w *sheet.Workbook, s *sheet.Sheet, o NotebookOptions) []string {42+ cells := s.NotebookCells()43+ all := make([]int, len(cells))44+ for i := range all {45+ all[i] = i46+ }47+ order, cycle := notebook.Order(cells, all)48+ var failed []string49+ for _, i := range cycle {50+ failed = append(failed, fmt.Sprintf("%s cell %d reads itself, through the cells it reads: it doesn't run", s.Name(), i+1))51+ }52+ for n, i := range order {53+ if err := runCell(ctx, w, s, cells, i, n+1, o); err != nil {54+ return append(failed, fmt.Sprintf("%s cell %d: %s", s.Name(), i+1, err))55+ }56+ }57+ return failed58+}59+60+// runCell runs cell i, the count-th run, keeping its output or why it61+// failed.62+func runCell(ctx context.Context, w *sheet.Workbook, s *sheet.Sheet, cells []notebook.Cell, i, count int, o NotebookOptions) error {63+ c := cells[i]64+ out := ¬ebook.Output{Count: count, Source: c.Source}65+ defer func() {66+ w.SetOutput(c.ID, out)67+ feedOutput(w, c.Name())68+ }()69+ if why := notebook.Taken(cells, i); why != "" {70+ out.Err = why71+ return errors.New(why)72+ }73+ job, reads, err := jobFor(w, s, cells, c, o.NuConfig)74+ if err != nil {75+ out.Err = err.Error()76+ return err77+ }78+ out.Reads = reads79+ start := time.Now()80+ data, err := nushell.Exec(ctx, o.Runner, job, o.Timeout, maxOutput)81+ out.Took = time.Since(start)82+ if err != nil {83+ out.Err = err.Error()84+ var nuErr *nushell.Error85+ if errors.As(err, &nuErr) {86+ out.Err, out.Detail = nuErr.Msg, nuErr.Help()87+ }88+ return errors.New(out.Err)89+ }90+ out.NUON = append([]byte{}, data...)91+ return nil92+}93+94+// jobFor is what running cell c takes: the outputs, linked files and95+// ranges it reads, as NUON, and the outputs it read by name.96+func jobFor(w *sheet.Workbook, s *sheet.Sheet, cells []notebook.Cell, c notebook.Cell, config bool) (nushell.Job, map[string]int, error) {97+ pipeline, ranges := notebook.Bind(c.Pipeline())98+ job := nushell.Job{Command: pipeline, Tables: map[string][]byte{}, Config: config}99+ reads := map[string]int{}100+ names := notebook.Names(cells)101+ for _, name := range notebook.Refs(pipeline) {102+ if j, ok := names[name]; ok {103+ o := w.Output(cells[j].ID)104+ switch {105+ case o == nil || o.Unsaved:106+ return job, nil, fmt.Errorf("it reads $%s, which hasn't run", name)107+ case o.Failed():108+ return job, nil, fmt.Errorf("it reads $%s, which failed", name)109+ }110+ job.Tables[name], reads[name] = o.NUON, o.Seq111+ continue112+ }113+ if t, r, ok := w.Region(name); ok && r.Linked() && r.Name == name {114+ rng, shown := t.RegionTable(r.Name)115+ snap := fileio.Snap(t, rng, r.Name)116+ if !shown {117+ snap = fileio.Snap(t, sheet.Rect{From: r.At, To: r.At}, r.Name)118+ snap.Cells = nil119+ }120+ data, err := encodeNUON(snap)121+ if err != nil {122+ return job, nil, err123+ }124+ job.Tables[name] = data125+ }126+ }127+ if notebook.ReadsSelection(pipeline) {128+ return job, nil, errors.New("it reads $selection, which only the screen has")129+ }130+ for _, ref := range ranges {131+ t, rng, err := rangeRef(w, ref.Ref)132+ if err != nil {133+ return job, nil, err134+ }135+ data, err := encodeNUON(fileio.Snap(t, rng, t.Name()))136+ if err != nil {137+ return job, nil, err138+ }139+ job.Tables[ref.Var] = data140+ }141+ return job, reads, nil142+}143+144+// rangeRef is the sheet and range $sheet.ref reads: of the sheet it145+// names, or else of the sheet shown when the file was saved, or the146+// first sheet that isn't a notebook.147+func rangeRef(w *sheet.Workbook, ref string) (*sheet.Sheet, sheet.Rect, error) {148+ name, cells := sheet.SplitSheet(ref)149+ t := w.Sheet(w.Active())150+ if name != "" {151+ t = w.Lookup(name)152+ } else if t.IsNotebook() {153+ for _, s := range w.Sheets() {154+ if !s.IsNotebook() {155+ t = s156+ break157+ }158+ }159+ }160+ rng, ok := sheet.ParseRange(cells)161+ if t == nil || t.IsNotebook() || !ok {162+ return nil, sheet.Rect{}, fmt.Errorf("$sheet.%s isn't a range of a sheet", ref)163+ }164+ return t, rng, nil165+}166+167+func encodeNUON(snap *fileio.Snapshot) ([]byte, error) {168+ var b bytes.Buffer169+ _, err := fileio.Encode(&b, fileio.NUON, snap)170+ return b.Bytes(), err171+}172+173+// SyncOutputs sends the outputs a file kept to the sheets they were sent174+// to, as opening the file on the screen does: reading, not running.175+func SyncOutputs(w *sheet.Workbook) {176+ for _, name := range w.StaleOutputs() {177+ feedOutput(w, name)178+ }179+}180+181+// feedOutput sends the output of the cell named name to the region it182+// was sent to, if it was: its rows, none when it hasn't run, or why when183+// it failed or is gone.184+func feedOutput(w *sheet.Workbook, name string) {185+ if name == "" {186+ return187+ }188+ _, r, ok := w.Region(name)189+ if !ok || !r.Output {190+ return191+ }192+ op := sheet.LiveOp{Region: r.Name, Reset: true}193+ c, ok := cellNamed(w, r.Name)194+ switch o := w.Output(c.ID); {195+ case !ok:196+ op = sheet.LiveOp{Region: r.Name, Err: "its cell is gone"}197+ case o == nil || o.Unsaved:198+ case o.Failed():199+ op = sheet.LiveOp{Region: r.Name, Err: o.Err}200+ default:201+ rows, note, err := fileio.NUONRows(context.Background(), o.NUON, 0)202+ if err != nil {203+ op = sheet.LiveOp{Region: r.Name, Err: err.Error()}204+ break205+ }206+ op.Header, op.Rows, op.Note = rows.Header, rows.Rows, note207+ }208+ w.ApplyLive(op)209+}210+211+// cellNamed is the notebook cell that gives its output name.212+func cellNamed(w *sheet.Workbook, name string) (notebook.Cell, bool) {213+ for _, s := range w.Sheets() {214+ cells := s.NotebookCells()215+ if i, ok := notebook.Names(cells)[name]; ok {216+ return cells[i], true217+ }218+ }219+ return notebook.Cell{}, false220+}@@ -1,138 +0,0 @@
1-package headless2-3-import (4- "bytes"5- "context"6- "fmt"7- "sync"8- "time"9-10- "github.com/FelineStateMachine/012/internal/fileio"11- "github.com/FelineStateMachine/012/internal/jev"12- "github.com/FelineStateMachine/012/internal/nushell"13- "github.com/FelineStateMachine/012/internal/sheet"14-)15-16-// RegionOptions say how RunRegions runs commands.17-type RegionOptions struct {18- Runner nushell.Runner // nu, or a fake in tests19- Timeout time.Duration // for each command20- NuConfig bool // run nu with the user's config files21-}22-23-// RunRegions runs every notebook region's command with nu, each after24-// the regions it reads, as Run all regions does on the screen, and25-// shows each table. It returns what failed, one line per region; a26-// region that fails leaves the table it had, which in a file just read27-// is none. Linked files' regions are left empty: following a file is28-// the screen's.29-func RunRegions(ctx context.Context, w *sheet.Workbook, o RegionOptions) []string {30- var failed []string31- for _, name := range w.RunOrder() {32- s, r, ok := w.Region(name)33- if !ok || r.Linked() {34- continue35- }36- if err := runRegion(ctx, s, r, o); err != nil {37- failed = append(failed, fmt.Sprintf("region %s (%s): %s", r.Name, r.Command, err))38- }39- }40- return failed41-}42-43-func runRegion(ctx context.Context, s *sheet.Sheet, r sheet.Region, o RegionOptions) error {44- job, err := jobFor(s.Book(), r, o.NuConfig)45- if err != nil {46- return err47- }48- data, err := nushell.Exec(ctx, o.Runner, job, o.Timeout, 0)49- if err != nil {50- return err51- }52- return s.ShowRegion(r.Name, data)53-}54-55-// jobFor is what running r takes: the tables it reads, as NUON.56-func jobFor(w *sheet.Workbook, r sheet.Region, config bool) (nushell.Job, error) {57- job := nushell.Job{Command: r.Command, Tables: map[string][]byte{}, Config: config}58- for _, d := range r.Deps {59- s, dr, ok := w.Region(d)60- if !ok {61- return job, fmt.Errorf("it reads $%s, which isn't a region any more", d)62- }63- t, ok := s.RegionTable(dr.Name)64- if !ok {65- t = sheet.Rect{From: dr.At, To: dr.At}66- }67- snap := fileio.Snap(s, t, dr.Name)68- snap.HiddenRows = nil69- if !ok {70- snap.Cells = nil71- }72- data, err := encodeNUON(snap)73- if err != nil {74- return job, err75- }76- job.Tables[d] = data77- }78- if r.Input != "" {79- t, err := Resolve(w, r.Input)80- if err != nil {81- return job, fmt.Errorf("its input %s is gone", r.Input)82- }83- if job.Input, err = encodeNUON(fileio.Snap(t.Sheet, t.Range, t.Sheet.Name())); err != nil {84- return job, err85- }86- }87- return job, nil88-}89-90-func encodeNUON(snap *fileio.Snapshot) ([]byte, error) {91- var b bytes.Buffer92- _, err := fileio.Encode(&b, fileio.NUON, snap)93- return b.Bytes(), err94-}95-96-// jevParallel is how many questions are asked at once, as on the97-// screen.98-const jevParallel = 899-100-// maxJEVRounds bounds the rounds of questions: answers can make new101-// questions (a JEV function reading another's answer), and each round102-// asks those the last one made.103-const maxJEVRounds = 16104-105-// AnswerJEV answers w's JEV functions with client, asking what the106-// workbook's formulas ask, a few questions at a time, until none is107-// waiting, each within timeout. It returns how many were asked.108-func AnswerJEV(ctx context.Context, w *sheet.Workbook, client jev.Client, timeout time.Duration) int {109- cache := jev.NewCache()110- w.SetRemote(cache)111- asked := 0112- for range maxJEVRounds {113- calls := cache.Take(1 << 30)114- if len(calls) == 0 {115- break116- }117- asked += len(calls)118- answers := make([]sheet.RemoteAnswer, len(calls))119- var wg sync.WaitGroup120- sem := make(chan struct{}, jevParallel)121- for i, c := range calls {122- wg.Add(1)123- sem <- struct{}{}124- go func() {125- defer func() { <-sem; wg.Done() }()126- ctx, cancel := context.WithTimeout(ctx, timeout)127- defer cancel()128- answers[i] = jev.Ask(ctx, client, c)129- }()130- }131- wg.Wait()132- for i, c := range calls {133- cache.Store(c, answers[i])134- }135- w.RecalcAnswered(calls)136- }137- return asked138-}