notebook.RangeSpans says where a pipeline reads ranges of sheets, so the source nu is asked about keeps its offsets.
- commit
- 2f398c00210ca85e3ec0a00af1bc67ab85641d4a
- parent
- 84dcbde042ed2496505781a75cdebf9c63ea8a98
- author
- Dami <dami.etoile@gmail.com>
- kind
- feat risk score 1.3 of 3, Jev's estimate
@@ -51,6 +51,14 @@ func TestBind(t *testing.T) {
5151 if len(refs) != 2 || refs[0].Ref != "A1:C9" || refs[1].Ref != "'Q1 data'!B2:B5" || refs[1].Var != "__sheet2" {5252 t.Errorf("refs %+v", refs)5353 }54+ src := "$sheet.A1:C9 | append $sheet.'Q1 data'!B2:B5 | append $sheet.A1:C9.name | $x$sheet.A1 | $sheet."55+ var spans []string56+ for _, s := range RangeSpans(src) {57+ spans = append(spans, src[s[0]:s[1]])58+ }59+ if !slices.Equal(spans, []string{"$sheet.A1:C9", "$sheet.'Q1 data'!B2:B5", "$sheet.A1:C9"}) {60+ t.Errorf("RangeSpans %q", spans)61+ }5462 }5563 5664 func cells(srcs ...string) []Cell {@@ -100,6 +100,27 @@ func Bind(pipeline string) (string, []SheetRef) {
100100 }101101 }102102 103+// RangeSpans are where a pipeline reads ranges of sheets, by byte104+// offsets: each $sheet.A1:C9 whole, as Bind renames them.105+func RangeSpans(pipeline string) [][2]int {106+ var out [][2]int107+ for i := 0; ; {108+ k := strings.Index(pipeline[i:], sheetVar)109+ if k < 0 {110+ return out111+ }112+ k += i113+ i = k + len(sheetVar)114+ if k > 0 && isWord(pipeline[k-1]) {115+ continue116+ }117+ if n := refLen(pipeline[i:]); n > 0 {118+ i += n119+ out = append(out, [2]int{k, i})120+ }121+ }122+}123+103124 // refLen is how long the range at the start of s is: a sheet's name,104125 // quoted in ” if it needs to be, then ! and cells, or cells alone. A105126 // dot ends it, so a cell path may follow: $sheet.A1:C9.name.@@ -0,0 +1,219 @@
1+package nushell2+3+import (4+ "bufio"5+ "bytes"6+ "context"7+ "encoding/json"8+ "errors"9+ "fmt"10+ "strconv"11+ "strings"12+)13+14+// What nu says of a script without running it, through its IDE flags,15+// each reading the script from a file:16+//17+// nu --ide-ast file [{"type":"ast","span":{"start":0,"end":2},"shape":"shape_internalcall","content":"ls"}, ...]18+// nu --ide-check 20 file {"type":"diagnostic","severity":"Error","message":"Variable not found.","span":{"start":31,"end":35}}19+// one object a line, "hint" lines among them20+// nu --ide-complete 7 file {"completions": ["sort", "sort-by"]}21+//22+// Spans and offsets are bytes of the file. These are the shapes of nu23+// 0.116 (testdata/ide holds its answers); an older nu whose answers24+// don't read as these is reported as ErrOld, and a nu older than25+// MinVersion isn't asked at all.26+27+// MinVersion is the oldest nu asked about scripts.28+const MinVersion = "0.100.0"29+30+// ErrOld is a nu too old to ask, or one whose answers don't read.31+var ErrOld = errors.New("nu is too old to ask about pipelines")32+33+// Shape is a stretch of a script, by byte offsets, and nu's name for34+// what it is: shape_internalcall, shape_string.35+type Shape struct {36+ From, To int37+ Shape string38+}39+40+// Problem is what nu's check found wrong with a stretch of a script.41+type Problem struct {42+ From, To int43+ Severity string // Error or Warning44+ Msg string45+}46+47+// maxAnswer is the most of an answer kept: a script's shapes or48+// completions are far less.49+const maxAnswer = 4 << 2050+51+// ask runs job, asking nu about script, and returns what it printed.52+func ask(ctx context.Context, r Runner, script string, flags ...string) ([]byte, error) {53+ ctx, cancel := context.WithCancelCause(ctx)54+ defer cancel(nil)55+ out := &capped{max: maxAnswer, stop: func() { cancel(ErrTooLarge) }}56+ err := r.Run(ctx, Job{Command: script, IDE: flags}, script, out)57+ switch {58+ case ctx.Err() != nil:59+ return nil, context.Cause(ctx)60+ case err != nil:61+ return nil, err62+ }63+ return bytes.TrimSpace(out.b.Bytes()), nil64+}65+66+// Version is nu's version, checked against MinVersion: ErrOld when it's67+// older or doesn't read as one.68+func Version(ctx context.Context, r Runner) (string, error) {69+ out, err := ask(ctx, r, "", "--version")70+ if err != nil {71+ return "", err72+ }73+ v := string(out)74+ if !AtLeast(v, MinVersion) {75+ return v, fmt.Errorf("%w: %s", ErrOld, v)76+ }77+ return v, nil78+}79+80+// AtLeast reports whether version v is oldest or later; one that doesn't81+// read as major.minor.patch isn't.82+func AtLeast(v, oldest string) bool {83+ a, okA := semver(v)84+ b, okB := semver(oldest)85+ if !okA || !okB {86+ return false87+ }88+ for i := range a {89+ if a[i] != b[i] {90+ return a[i] > b[i]91+ }92+ }93+ return true94+}95+96+func semver(v string) ([3]int, bool) {97+ var out [3]int98+ v, _, _ = strings.Cut(strings.TrimSpace(v), "-") // 0.117.0-nightly99+ parts := strings.Split(v, ".")100+ if len(parts) != 3 {101+ return out, false102+ }103+ for i, p := range parts {104+ n, err := strconv.Atoi(p)105+ if err != nil {106+ return out, false107+ }108+ out[i] = n109+ }110+ return out, true111+}112+113+// AST is nu's shapes of script's tokens, in order.114+func AST(ctx context.Context, r Runner, script string) ([]Shape, error) {115+ out, err := ask(ctx, r, script, "--ide-ast")116+ if err != nil {117+ return nil, err118+ }119+ return ParseAST(out)120+}121+122+type span struct{ Start, End int }123+124+// ParseAST reads what --ide-ast printed.125+func ParseAST(out []byte) ([]Shape, error) {126+ var items []struct {127+ Type string128+ Span *span129+ Shape string130+ }131+ if err := json.Unmarshal(out, &items); err != nil {132+ return nil, fmt.Errorf("%w: %v", ErrOld, err)133+ }134+ shapes := make([]Shape, 0, len(items))135+ for _, it := range items {136+ if it.Type != "ast" || it.Span == nil || !strings.HasPrefix(it.Shape, "shape_") {137+ continue138+ }139+ shapes = append(shapes, Shape{From: it.Span.Start, To: it.Span.End, Shape: it.Shape})140+ }141+ return shapes, nil142+}143+144+// Check is what nu's check finds wrong with script, at most limit145+// problems, each once.146+func Check(ctx context.Context, r Runner, script string, limit int) ([]Problem, error) {147+ out, err := ask(ctx, r, script, "--ide-check", strconv.Itoa(limit))148+ if err != nil {149+ return nil, err150+ }151+ return ParseCheck(out)152+}153+154+// ParseCheck reads what --ide-check printed: its diagnostics, each once,155+// leaving out its hints (the types it inferred).156+func ParseCheck(out []byte) ([]Problem, error) {157+ var probs []Problem158+ sc := bufio.NewScanner(bytes.NewReader(out))159+ sc.Buffer(nil, maxAnswer)160+ for sc.Scan() {161+ line := bytes.TrimSpace(sc.Bytes())162+ if len(line) == 0 {163+ continue164+ }165+ var d struct {166+ Type, Severity, Message string167+ Span *span168+ }169+ if err := json.Unmarshal(line, &d); err != nil {170+ return nil, fmt.Errorf("%w: %v", ErrOld, err)171+ }172+ if d.Type != "diagnostic" || d.Span == nil {173+ continue174+ }175+ p := Problem{From: d.Span.Start, To: d.Span.End, Severity: d.Severity, Msg: d.Message}176+ if !containsProblem(probs, p) {177+ probs = append(probs, p)178+ }179+ }180+ return probs, sc.Err()181+}182+183+func containsProblem(probs []Problem, p Problem) bool {184+ for _, q := range probs {185+ if q == p {186+ return true187+ }188+ }189+ return false190+}191+192+// Complete is what nu would complete at offset, a byte of script, in193+// its order. nu says what goes there, not what it replaces.194+func Complete(ctx context.Context, r Runner, script string, offset int) ([]string, error) {195+ out, err := ask(ctx, r, script, "--ide-complete", strconv.Itoa(offset))196+ if err != nil {197+ return nil, err198+ }199+ return ParseComplete(out)200+}201+202+// ParseComplete reads what --ide-complete printed.203+func ParseComplete(out []byte) ([]string, error) {204+ if len(out) == 0 {205+ return nil, nil206+ }207+ var c struct {208+ Completions *[]string209+ }210+ if err := json.Unmarshal(out, &c); err != nil || c.Completions == nil {211+ return nil, fmt.Errorf("%w: %s", ErrOld, firstLine(out))212+ }213+ return *c.Completions, nil214+}215+216+func firstLine(b []byte) string {217+ line, _, _ := bytes.Cut(b, []byte("\n"))218+ return string(line)219+}@@ -0,0 +1,127 @@
1+package nushell2+3+import (4+ "context"5+ "errors"6+ "os"7+ "os/exec"8+ "path/filepath"9+ "slices"10+ "strings"11+ "testing"12+)13+14+// testdata/ide holds nu 0.116's answers about the scripts beside them15+// (ast.nu, check.nu, complete.nu at its end): what 012 relies on nu16+// saying.17+func recorded(t *testing.T, name string) []byte {18+ t.Helper()19+ data, err := os.ReadFile(filepath.Join("testdata", "ide", name))20+ if err != nil {21+ t.Fatal(err)22+ }23+ return data24+}25+26+func TestParseAST(t *testing.T) {27+ shapes, err := ParseAST(recorded(t, "ast.json"))28+ if err != nil {29+ t.Fatal(err)30+ }31+ src := string(recorded(t, "ast.nu"))32+ var got []string33+ for _, s := range shapes {34+ got = append(got, src[s.From:s.To]+"="+strings.TrimPrefix(s.Shape, "shape_"))35+ }36+ want := []string{"ls=internalcall", "|=pipe", "where=internalcall", "size=string", "size=variable", ">=operator",37+ "1=int", "kb=string", "|=pipe", "sort-by=internalcall", "name=string", "--reverse=flag"}38+ if !slices.Equal(got, want) {39+ t.Errorf("shapes\n got %q\nwant %q", got, want)40+ }41+ if _, err := ParseAST([]byte("Error: unknown flag")); !errors.Is(err, ErrOld) {42+ t.Errorf("unreadable answer: %v", err)43+ }44+}45+46+func TestParseCheck(t *testing.T) {47+ probs, err := ParseCheck(recorded(t, "check.jsonl"))48+ if err != nil {49+ t.Fatal(err)50+ }51+ src := string(recorded(t, "check.nu"))52+ var got []string53+ for _, p := range probs {54+ got = append(got, src[p.From:p.To]+": "+p.Msg)55+ }56+ want := []string{"(: Unclosed delimiter.", "--foo: The `math sum` command doesn't have flag `foo`.", ": Unclosed delimiter.",57+ "(: Type mismatch.", "where: Command does not support number input."}58+ if !slices.Equal(got, want) {59+ t.Errorf("problems\n got %q\nwant %q", got, want)60+ }61+ probs, err = ParseCheck([]byte(`{"type":"hint","typename":"any","position":{"start":4,"end":6}}` + "\n"))62+ if err != nil || len(probs) != 0 {63+ t.Errorf("hints are left out: %v %v", probs, err)64+ }65+}66+67+func TestParseComplete(t *testing.T) {68+ got, err := ParseComplete(recorded(t, "complete.json"))69+ if err != nil || !slices.Equal(got, []string{"sort", "sort-by", "source", "source-env"}) {70+ t.Errorf("completions %q %v", got, err)71+ }72+ if got, err := ParseComplete([]byte(`{"completions": []}`)); err != nil || len(got) != 0 {73+ t.Errorf("none %q %v", got, err)74+ }75+ if _, err := ParseComplete([]byte(`{"other": 1}`)); !errors.Is(err, ErrOld) {76+ t.Errorf("unreadable answer: %v", err)77+ }78+}79+80+func TestVersion(t *testing.T) {81+ f := &fake{out: string(recorded(t, "version.txt"))}82+ if v, err := Version(context.Background(), f); err != nil || v != "0.116.0" || !slices.Equal(f.job.IDE, []string{"--version"}) {83+ t.Errorf("version %q %v %q", v, err, f.job.IDE)84+ }85+ if _, err := Version(context.Background(), &fake{out: "0.99.1\n"}); !errors.Is(err, ErrOld) {86+ t.Errorf("an old nu: %v", err)87+ }88+ for v, want := range map[string]bool{"0.100.0": true, "0.117.0-nightly.3": true, "1.0.0": true, "0.99.9": false, "nu": false} {89+ if AtLeast(v, MinVersion) != want {90+ t.Errorf("AtLeast(%q) = %v", v, !want)91+ }92+ }93+}94+95+// The script asked about reaches the runner as it is, with the flags.96+func TestAskHandsTheScript(t *testing.T) {97+ f := &fake{out: `{"completions": ["sort"]}`}98+ if _, err := Complete(context.Background(), f, "ls | so", 7); err != nil {99+ t.Fatal(err)100+ }101+ if f.script != "ls | so" || !slices.Equal(f.job.IDE, []string{"--ide-complete", "7"}) {102+ t.Errorf("script %q flags %q", f.script, f.job.IDE)103+ }104+}105+106+// The real nu, when it's installed, says what the recorded answers do.107+func TestNuIDE(t *testing.T) {108+ if _, err := exec.LookPath("nu"); err != nil {109+ t.Skip("nu isn't installed")110+ }111+ ctx := context.Background()112+ if _, err := Version(ctx, Nu{}); err != nil {113+ t.Skipf("nu: %v", err)114+ }115+ shapes, err := AST(ctx, Nu{}, string(recorded(t, "ast.nu")))116+ if err != nil || len(shapes) < 5 || shapes[0] != (Shape{0, 2, "shape_internalcall"}) {117+ t.Errorf("ast %v %v", shapes, err)118+ }119+ probs, err := Check(ctx, Nu{}, string(recorded(t, "check.nu")), 20)120+ if err != nil || !slices.ContainsFunc(probs, func(p Problem) bool { return strings.Contains(p.Msg, "flag `foo`") }) {121+ t.Errorf("check %v %v", probs, err)122+ }123+ comp, err := Complete(ctx, Nu{}, "ls | so", 7)124+ if err != nil || !slices.Contains(comp, "sort-by") {125+ t.Errorf("complete %q %v", comp, err)126+ }127+}@@ -33,6 +33,11 @@ type Job struct {
3333 Tables map[string][]byte3434 // Config runs nu with the user's config files rather than without.3535 Config bool36+ // IDE asks nu about the script rather than running it: the flags37+ // that ask (--ide-ast, or --ide-complete 14), nu reading the script38+ // from a file named after them, without config files or the39+ // standard library (see ide.go).40+ IDE []string3641 }3742 3843 // Runner runs a job's script, writing what it prints to stdout. A test@@ -171,9 +176,9 @@ func (n Nu) Run(ctx context.Context, job Job, script string, stdout io.Writer) e
171176 }172177 env = append(env, "NU012_TABLE_"+strconv.Itoa(i)+"="+file)173178 }174- args := []string{"-c", script}175- if !job.Config {176- args = append([]string{"--no-config-file"}, args...)179+ args, err := nuArgs(job, script, dir)180+ if err != nil {181+ return err177182 }178183 cmd := exec.CommandContext(ctx, path, args...)179184 cmd.Env, cmd.Stdout = env, stdout@@ -190,6 +195,23 @@ func (n Nu) Run(ctx context.Context, job Job, script string, stdout io.Writer) e
190195 return nil191196 }192197 198+// nuArgs is nu's command line for job: the script to run, or the flags199+// that ask about it, the script in a file in dir.200+func nuArgs(job Job, script, dir string) ([]string, error) {201+ if len(job.IDE) > 0 {202+ file := dir + string(os.PathSeparator) + "cell.nu"203+ if err := os.WriteFile(file, []byte(script), 0o600); err != nil {204+ return nil, err205+ }206+ return append(append([]string{"--no-config-file", "--no-std-lib"}, job.IDE...), file), nil207+ }208+ args := []string{"-c", script}209+ if !job.Config {210+ args = append([]string{"--no-config-file"}, args...)211+ }212+ return args, nil213+}214+193215 // Message is the line of nu's standard error that says what went wrong:194216 // the one marked ×, or the first, or how the process ended.195217 func Message(stderr string, err error) string {@@ -0,0 +1,6 @@
1+{"type":"diagnostic","severity":"Error","message":"Unclosed delimiter.","span":{"start":31,"end":32}}2+{"type":"diagnostic","severity":"Error","message":"The `math sum` command doesn't have flag `foo`.","span":{"start":17,"end":22}}3+{"type":"diagnostic","severity":"Error","message":"Unclosed delimiter.","span":{"start":31,"end":32}}4+{"type":"diagnostic","severity":"Error","message":"Unclosed delimiter.","span":{"start":32,"end":32}}5+{"type":"diagnostic","severity":"Error","message":"Type mismatch.","span":{"start":31,"end":32}}6+{"type":"diagnostic","severity":"Error","message":"Command does not support number input.","span":{"start":25,"end":30}}@@ -0,0 +1 @@
1+0.116.0