- commit
- 96fbecea2ca8dbc8f5b781f26bfa041f207123c4
- parent
- 62a3e613539d08e782dfa36d25f0afcf63e8b965
- author
- Dami <dami.etoile@gmail.com>
- kind
- feat risk score 2.9 of 3, Jev's estimate
| M | internal/ | 68 | +++--- |
| M | internal/ | 2 | +- |
| M | internal/ | 131 | +++++++++---- |
| A | internal/ | 54 | +++++ |
| M | internal/ | 56 | +++--- |
| M | internal/ | 103 | +---------- |
| A | internal/ | 297 | ++++++++++++++++++++++++++++++ |
| A | internal/ | 177 | ++++++++++++++++++ |
| M | internal/ | 40 | ++++- |
| M | internal/ | 29 | +-- |
| M | internal/ | 21 | ++- |
| M | internal/ | 42 | +++-- |
| M | internal/ | 32 | +++- |
@@ -50,7 +50,7 @@ func runNotebook(ctx context.Context, w *sheet.Workbook, s *sheet.Sheet, o Noteb
5050 failed = append(failed, fmt.Sprintf("%s cell %d reads itself, through the cells it reads: it doesn't run", s.Name(), i+1))5151 }5252 for n, i := range order {53- if err := runCell(ctx, w, s, cells, i, n+1, o); err != nil {53+ if err := runCell(ctx, w, cells, i, n+1, o); err != nil {5454 return append(failed, fmt.Sprintf("%s cell %d: %s", s.Name(), i+1, err))5555 }5656 }@@ -59,7 +59,7 @@ func runNotebook(ctx context.Context, w *sheet.Workbook, s *sheet.Sheet, o Noteb
5959 6060 // runCell runs cell i, the count-th run, keeping its output or why it6161 // failed.62-func runCell(ctx context.Context, w *sheet.Workbook, s *sheet.Sheet, cells []notebook.Cell, i, count int, o NotebookOptions) error {62+func runCell(ctx context.Context, w *sheet.Workbook, cells []notebook.Cell, i, count int, o NotebookOptions) error {6363 c := cells[i]6464 out := ¬ebook.Output{Count: count, Source: c.Source}6565 defer func() {@@ -70,14 +70,14 @@ func runCell(ctx context.Context, w *sheet.Workbook, s *sheet.Sheet, cells []not
7070 out.Err = why7171 return errors.New(why)7272 }73- job, reads, err := jobFor(w, s, cells, c, o.NuConfig)73+ job, reads, err := jobFor(w, cells, i, o.NuConfig)7474 if err != nil {7575 out.Err = err.Error()7676 return err7777 }7878 out.Reads = reads7979 start := time.Now()80- data, err := nushell.Exec(ctx, o.Runner, job, o.Timeout, maxOutput)80+ data, vars, err := nushell.ExecVars(ctx, o.Runner, job, o.Timeout, maxOutput)8181 out.Took = time.Since(start)8282 if err != nil {8383 out.Err = err.Error()@@ -87,47 +87,31 @@ func runCell(ctx context.Context, w *sheet.Workbook, s *sheet.Sheet, cells []not
8787 }8888 return errors.New(out.Err)8989 }90- out.NUON = append([]byte{}, data...)90+ out.NUON, out.Vars = append([]byte{}, data...), vars9191 return nil9292 }9393 94-// jobFor is what running cell c takes: the outputs, linked files and95-// ranges it reads, as NUON, and the outputs it read by name.96-func jobFor(w *sheet.Workbook, s *sheet.Sheet, cells []notebook.Cell, c notebook.Cell, config bool) (nushell.Job, map[string]int, error) {97- pipeline, ranges := notebook.Bind(c.Pipeline())98- job := nushell.Job{Command: pipeline, Tables: map[string][]byte{}, Config: config}99- reads := map[string]int{}100- names := notebook.Names(cells)101- for _, name := range notebook.Refs(pipeline) {102- if j, ok := names[name]; ok {103- o := w.Output(cells[j].ID)104- switch {105- case o == nil || o.Unsaved:106- return job, nil, fmt.Errorf("it reads $%s, which hasn't run", name)107- case o.Failed():108- return job, nil, fmt.Errorf("it reads $%s, which failed", name)109- }110- job.Tables[name], reads[name] = o.NUON, o.Seq111- continue112- }94+// jobFor is what running cell i takes: the outputs, variables, linked95+// files and ranges it reads, as NUON, and the outputs it read by name.96+func jobFor(w *sheet.Workbook, cells []notebook.Cell, i int, config bool) (nushell.Job, map[string]int, error) {97+ p, err := notebook.Prepare(cells, i, w.Output)98+ job := nushell.Job{Command: p.Command, Vars: p.Exports, Tables: p.Tables, Config: config}99+ if err != nil {100+ return job, nil, err101+ }102+ for _, name := range p.Others {113103 if t, r, ok := w.Region(name); ok && r.Linked() && r.Name == name {114- rng, shown := t.RegionTable(r.Name)115- snap := fileio.Snap(t, rng, r.Name)116- if !shown {117- snap = fileio.Snap(t, sheet.Rect{From: r.At, To: r.At}, r.Name)118- snap.Cells = nil119- }120- data, err := encodeNUON(snap)104+ data, err := regionNUON(t, r)121105 if err != nil {122106 return job, nil, err123107 }124108 job.Tables[name] = data125109 }126110 }127- if notebook.ReadsSelection(pipeline) {111+ if p.Selection {128112 return job, nil, errors.New("it reads $selection, which only the screen has")129113 }130- for _, ref := range ranges {114+ for _, ref := range p.Ranges {131115 t, rng, err := rangeRef(w, ref.Ref)132116 if err != nil {133117 return job, nil, err@@ -138,9 +122,25 @@ func jobFor(w *sheet.Workbook, s *sheet.Sheet, cells []notebook.Cell, c notebook
138122 }139123 job.Tables[ref.Var] = data140124 }141- return job, reads, nil125+ return job, p.Reads, nil126+}127+128+// regionNUON is a linked file's table as NUON, read whole whatever its129+// filter shows, as on the screen.130+func regionNUON(t *sheet.Sheet, r sheet.Region) ([]byte, error) {131+ rng, ok := t.RegionTable(r.Name)132+ if !ok {133+ rng = sheet.Rect{From: r.At, To: r.At}134+ }135+ snap := fileio.Snap(t, rng, r.Name)136+ snap.HiddenRows = nil137+ if !ok {138+ snap.Cells = nil139+ }140+ return encodeNUON(snap)142141 }143142 143+144144 // rangeRef is the sheet and range $sheet.ref reads: of the sheet it145145 // names, or else of the sheet shown when the file was saved, or the146146 // first sheet that isn't a notebook.@@ -41,7 +41,7 @@ func RunCell(ctx context.Context, w *sheet.Workbook, name string, n int, o Noteb
4141 return CellRun{}, fmt.Errorf("%s cell %d is a note, which doesn't run", name, n)4242 }4343 out := CellRun{Notebook: s.Name(), Cell: n, Name: c.Name(), State: "ran"}44- if err := runCell(ctx, w, s, cells, n-1, 1, o); err != nil {44+ if err := runCell(ctx, w, cells, n-1, 1, o); err != nil {4545 out.State, out.Error = "failed", err.Error()4646 return out, nil4747 }@@ -7,12 +7,14 @@ import (
77 )88 99 // The cells' dependency graph: a cell reading $sales depends on the cell10-// named sales. A name belongs to the first code cell giving it; a later11-// one giving the same name can't run until it's renamed. Cells run in12-// the order asked, each after the cells it reads among those running,13-// and a cell reading itself, directly or through others, doesn't run.10+// giving sales (Vars). A cell's name belongs to the first code cell11+// giving it; a later one giving the same name can't run until it's12+// renamed. Cells run in the order asked, each after the cells it reads13+// among those running, and a cell reading itself, directly or through14+// others, doesn't run.1415 15-// Names maps each name to the index of the first code cell giving it.16+// Names maps each cell's name to the index of the first code cell17+// giving it.1618 func Names(cells []Cell) map[string]int {1719 out := map[string]int{}1820 for i, c := range cells {@@ -25,6 +27,21 @@ func Names(cells []Cell) map[string]int {
2527 return out2628 }2729 30+// Vars maps each variable the cells give others to the index of the31+// cell giving it: a cell's name to that cell (Names), and any other32+// name a statement assigns to the first code cell assigning it.33+func Vars(cells []Cell) map[string]int {34+ out := Names(cells)35+ for i, c := range cells {36+ for _, n := range c.Parse().Assigned() {37+ if _, ok := out[n]; !ok {38+ out[n] = i39+ }40+ }41+ }42+ return out43+}44+2845 // Taken says why cell i can't use its name: an earlier cell gives it2946 // (names ignore case there, as formulas' nu.name do), or "".3047 func Taken(cells []Cell, i int) string {@@ -40,13 +57,33 @@ func Taken(cells []Cell, i int) string {
4057 return ""4158 }4259 60+// ref is a variable a cell reads from another: its name, and the index61+// of the cell giving it.62+type ref struct {63+ name string64+ cell int65+}66+67+// refs are the variables cell i reads from other cells, in the order it68+// names them; vars is Vars(cells). A name only it assigns, read before69+// it does, is none.70+func refs(cells []Cell, vars map[string]int, i int) []ref {71+ var out []ref72+ for _, n := range cells[i].Parse().Refs() {73+ if j, ok := vars[n]; ok && j != i {74+ out = append(out, ref{n, j})75+ }76+ }77+ return out78+}79+4380 // Reads returns the indices of the cells cell i reads, in the order it44-// names them.45-func Reads(cells []Cell, names map[string]int, i int) []int {81+// names them; vars is Vars(cells).82+func Reads(cells []Cell, vars map[string]int, i int) []int {4683 var out []int47- for _, n := range Refs(cells[i].Pipeline()) {48- if j, ok := names[n]; ok && j != i && !slices.Contains(out, j) {49- out = append(out, j)84+ for _, r := range refs(cells, vars, i) {85+ if !slices.Contains(out, r.cell) {86+ out = append(out, r.cell)5087 }5188 }5289 return out@@ -56,7 +93,7 @@ func Reads(cells []Cell, names map[string]int, i int) []int {
5693 // among them, in want's order otherwise, and the cells left out for5794 // reading themselves, directly or through others.5895 func Order(cells []Cell, want []int) (order, cycle []int) {59- names := Names(cells)96+ vars := Vars(cells)6097 in := map[int]bool{}6198 for _, i := range want {6299 if cells[i].Kind == Code {@@ -70,7 +107,7 @@ func Order(cells []Cell, want []int) (order, cycle []int) {
70107 if !in[i] || done[i] {71108 continue72109 }73- ready := !slices.ContainsFunc(Reads(cells, names, i), func(j int) bool { return in[j] && !done[j] })110+ ready := !slices.ContainsFunc(Reads(cells, vars, i), func(j int) bool { return in[j] && !done[j] })74111 if ready {75112 order, done[i], progress = append(order, i), true, true76113 }@@ -90,12 +127,12 @@ func Order(cells []Cell, want []int) (order, cycle []int) {
90127 // Dependents returns the cells that read cell i, directly or through91128 // others, in the notebook's order.92129 func Dependents(cells []Cell, i int) []int {93- names := Names(cells)130+ vars := Vars(cells)94131 want := map[int]bool{i: true}95132 for changed := true; changed; {96133 changed = false97134 for j := range cells {98- if !want[j] && cells[j].Kind == Code && slices.ContainsFunc(Reads(cells, names, j), func(k int) bool { return want[k] }) {135+ if !want[j] && cells[j].Kind == Code && slices.ContainsFunc(Reads(cells, vars, j), func(k int) bool { return want[k] }) {99136 want[j], changed = true, true100137 }101138 }@@ -109,21 +146,35 @@ func Dependents(cells []Cell, i int) []int {
109146 return out110147 }111148 149+// Readable reports whether o, the output of cell c, holds the variable150+// name: c's output when c is named so, else a variable it assigns,151+// which a file doesn't keep.152+func Readable(c Cell, name string, o *Output) bool {153+ switch {154+ case o == nil || o.NUON == nil:155+ return false156+ case c.Name() == name:157+ return true158+ }159+ _, ok := o.Vars[name]160+ return ok161+}162+112163 // Inputs returns the cells cell i reads, directly or through others,113-// that have no output, in the notebook's order: what has to run before114-// it can.164+// whose outputs don't hold what it reads (Readable), in the notebook's165+// order: what has to run before it can.115166 func Inputs(cells []Cell, i int, output func(id int) *Output) []int {116- names := Names(cells)167+ vars := Vars(cells)117168 need := map[int]bool{}118169 var visit func(j int)119170 visit = func(j int) {120- for _, k := range Reads(cells, names, j) {121- if need[k] || k == i {171+ for _, r := range refs(cells, vars, j) {172+ if need[r.cell] || r.cell == i {122173 continue123174 }124- if o := output(cells[k].ID); o == nil || o.NUON == nil {125- need[k] = true126- visit(k)175+ if c := cells[r.cell]; !Readable(c, r.name, output(c.ID)) {176+ need[r.cell] = true177+ visit(r.cell)127178 }128179 }129180 }@@ -141,7 +192,7 @@ func Inputs(cells []Cell, i int, output func(id int) *Output) []int {
141192 // them now would give: their source changed since, or an output they142193 // read changed or is stale itself.143194 func Stale(cells []Cell, output func(id int) *Output) map[int]bool {144- names := Names(cells)195+ vars := Vars(cells)145196 all := make([]int, len(cells))146197 for i := range all {147198 all[i] = i@@ -155,18 +206,17 @@ func Stale(cells []Cell, output func(id int) *Output) map[int]bool {
155206 continue156207 }157208 s := o.Source != c.Source158- for _, name := range Refs(c.Pipeline()) {159- j, ok := names[name]209+ read := map[string]bool{}210+ for _, r := range refs(cells, vars, i) {211+ read[r.name] = true160212 seq := 0161- if ok {162- if d := output(cells[j].ID); d != nil {163- seq = d.Seq164- }165- s = s || stale[cells[j].ID]166- }167- if read, had := o.Reads[name]; had || ok {168- s = s || read != seq213+ if d := output(cells[r.cell].ID); d != nil {214+ seq = d.Seq169215 }216+ s = s || stale[cells[r.cell].ID] || o.Reads[r.name] != seq217+ }218+ for name := range o.Reads {219+ s = s || !read[name] // it read a cell it no longer does170220 }171221 if s {172222 stale[c.ID] = true@@ -179,20 +229,17 @@ func Stale(cells []Cell, output func(id int) *Output) map[int]bool {
179229 // have left: made from the source as it is, having read the outputs as180230 // they are, so none is stale until something changes.181231 func Settle(cells []Cell, outputs map[int]*Output) {182- names := Names(cells)183- for _, c := range cells {232+ vars := Vars(cells)233+ for i, c := range cells {184234 o := outputs[c.ID]185235 if o == nil {186236 continue187237 }188238 o.Source, o.Reads = c.Source, map[string]int{}189- for _, name := range Refs(c.Pipeline()) {190- if j, ok := names[name]; ok {191- if d := outputs[cells[j].ID]; d != nil {192- o.Reads[name] = d.Seq193- } else {194- o.Reads[name] = 0195- }239+ for _, r := range refs(cells, vars, i) {240+ o.Reads[r.name] = 0241+ if d := outputs[cells[r.cell].ID]; d != nil {242+ o.Reads[r.name] = d.Seq196243 }197244 }198245 }@@ -0,0 +1,54 @@
1+package notebook2+3+import "fmt"4+5+// Run is what running a code cell takes that the notebook knows: what6+// nu runs, and the variables it reads from other cells. The screen and7+// the headless runner each add what only they know: linked files, the8+// selection and ranges of sheets.9+type Run struct {10+ // Command is what nu runs (Source.Command), and Exports the11+ // variables it hands back beside the output.12+ Command string13+ Exports []string14+ // Tables are the other cells' outputs and variables it reads, by15+ // name, as NUON, and Reads the Seqs of the outputs they came from.16+ Tables map[string][]byte17+ Reads map[string]int18+ // Others are the names it reads that no cell gives: linked files,19+ // or nothing nu will say so of.20+ Others []string21+ // Ranges are the ranges of sheets it reads, and Selection whether it22+ // reads $selection.23+ Ranges []SheetRef24+ Selection bool25+}26+27+// Prepare works out what running cell i takes, reading the outputs of28+// the cells it reads; the error says why it can't run: what it reads29+// hasn't run or failed.30+func Prepare(cells []Cell, i int, output func(id int) *Output) (Run, error) {31+ src := cells[i].Parse()32+ cmd, exports, ranges := src.Command()33+ run := Run{Command: cmd, Exports: exports, Tables: map[string][]byte{}, Reads: map[string]int{}, Ranges: ranges, Selection: src.ReadsSelection()}34+ vars := Vars(cells)35+ for _, name := range src.Refs() {36+ j, ok := vars[name]37+ if !ok || j == i {38+ run.Others = append(run.Others, name)39+ continue40+ }41+ o := output(cells[j].ID)42+ switch {43+ case o.Failed():44+ return run, fmt.Errorf("it reads $%s, which failed", name)45+ case o == nil || o.Unsaved || !Readable(cells[j], name, o):46+ return run, fmt.Errorf("it reads $%s, which hasn't run", name)47+ }48+ run.Tables[name], run.Reads[name] = o.NUON, o.Seq49+ if cells[j].Name() != name {50+ run.Tables[name] = o.Vars[name]51+ }52+ }53+ return run, nil54+}@@ -54,41 +54,37 @@ type Cell struct {
5454 Source string5555 }5656 57-// Name is the name the cell gives its output, from a source of the form58-// `name = pipeline`, or "".59-func (c Cell) Name() string {57+// Parse reads the cell's source; a note cell's has nothing to read.58+func (c Cell) Parse() Source {6059 if c.Kind != Code {61- return ""60+ return Source{Text: c.Source}6261 }63- name, _ := SplitName(c.Source)64- return name62+ return Parse(c.Source)6563 }6664 67-// Pipeline is what runs: the source without its name.68-func (c Cell) Pipeline() string {69- _, p := SplitName(c.Source)70- return p71-}72-73-// SplitName splits `name = pipeline` into the name and the pipeline; a74-// source that doesn't start so has no name. `==` isn't a name's `=`.75-func SplitName(src string) (name, pipeline string) {76- head, rest, ok := strings.Cut(src, "=")77- head = strings.TrimSpace(head)78- if !ok || strings.HasPrefix(rest, "=") || ValidName(head) != nil {79- return "", src80- }81- return head, strings.TrimLeft(rest, " \t")82-}65+// Name is the name the cell gives its output: what its last statement66+// assigns (`name = pipeline`), or "".67+func (c Cell) Name() string { return c.Parse().Name() }8368 84-// WithName is src given the name name, replacing any name it has; ""85-// takes its name away.69+// WithName is src given the name name, as the name its last statement70+// assigns, replacing any it has; "" takes its name away.8671 func WithName(src, name string) string {87- _, p := SplitName(src)88- if name == "" {89- return p72+ head := ""73+ if name != "" {74+ head = name + " = "75+ }76+ p := Parse(src)77+ if len(p.Stmts) == 0 {78+ if strings.TrimSpace(src) == "" {79+ return head80+ }81+ return src + "\n" + head82+ }83+ last := p.Stmts[len(p.Stmts)-1]84+ if last.Name != "" {85+ return src[:last.From] + head + src[last.Body:]9086 }91- return name + " = " + p87+ return src[:last.From] + head + src[last.From:]9288 }9389 9490 // Reserved are nushell's own variables and the notebook's: $selection@@ -141,6 +137,10 @@ type Output struct {
141137 // Seq identifies the output among the workbook's, for telling142138 // whether what a cell read has changed since.143139 Seq int140+ // Vars are the values, as NUON, of the variables the cell assigns141+ // other than its output, which later cells read by name. A file142+ // doesn't keep them: a cell reading one runs its cell first.143+ Vars map[string][]byte144144 // Took is how long the run took.145145 Took time.Duration146146 // Source is the cell's source when it ran.@@ -1,17 +1,14 @@
11 package notebook22 3-import (4- "slices"5- "strconv"6- "strings"7-)3+import "strings"84 95 // What a pipeline reads. A code cell reads another's output as $name,106 // the selection on a sheet as $selection, and a range of a sheet as117 // $sheet.A1:C9 (or $sheet.Sales!A1:C9, $sheet.'Q1 data'!A1:B5). Each128 // reaches nu as a variable holding a table, read from a NUON file, never139 // as text spliced into the pipeline: a $sheet range is renamed to a14-// variable of its own ($__sheet1) before the pipeline runs.10+// variable of its own ($__sheet1) before the pipeline runs11+// (Source.Command).1512 1613 // Selection is the variable that holds the selection.1714 const Selection = "selection"@@ -19,43 +16,6 @@ const Selection = "selection"
1916 // sheetVar starts a range of a sheet.2017 const sheetVar = "$sheet."2118 22-// Refs returns the names a pipeline reads as $name, in the order it23-// first names them, leaving out nushell's variables and the notebook's24-// own ($selection, $sheet).25-func Refs(pipeline string) []string {26- var out []string27- for i := 0; i < len(pipeline); i++ {28- if pipeline[i] != '$' {29- continue30- }31- j := i + 132- for j < len(pipeline) && isWord(pipeline[j]) {33- j++34- }35- name := pipeline[i+1 : j]36- if name != "" && ValidName(name) == nil && !slices.Contains(out, name) {37- out = append(out, name)38- }39- i = j - 140- }41- return out42-}43-44-// ReadsSelection reports whether a pipeline reads $selection.45-func ReadsSelection(pipeline string) bool {46- for i := 0; ; {47- k := strings.Index(pipeline[i:], "$"+Selection)48- if k < 0 {49- return false50- }51- end := i + k + 1 + len(Selection)52- if end == len(pipeline) || !isWord(pipeline[end]) {53- return true54- }55- i = end56- }57-}58-5919 // SheetRef is a range of a sheet a pipeline reads, $sheet.A1:C9.6020 type SheetRef struct {6121 // Ref is the range as written after $sheet.: "A1:C9", "Sales!A1:C9".@@ -64,63 +24,6 @@ type SheetRef struct {
6424 Var string6525 }6626 67-// Bind renames each range of a sheet the pipeline reads ($sheet.A1:C9)68-// to a variable of its own, returning the pipeline to run and the ranges69-// in the order they appear, the same range read twice once.70-func Bind(pipeline string) (string, []SheetRef) {71- var b strings.Builder72- var refs []SheetRef73- rest := pipeline74- for {75- k := strings.Index(rest, sheetVar)76- if k < 0 || k > 0 && isWord(rest[k-1]) {77- if k < 0 {78- b.WriteString(rest)79- return b.String(), refs80- }81- b.WriteString(rest[:k+len(sheetVar)])82- rest = rest[k+len(sheetVar):]83- continue84- }85- b.WriteString(rest[:k])86- rest = rest[k+len(sheetVar):]87- n := refLen(rest)88- if n == 0 {89- b.WriteString(sheetVar)90- continue91- }92- ref := rest[:n]93- rest = rest[n:]94- i := slices.IndexFunc(refs, func(r SheetRef) bool { return r.Ref == ref })95- if i < 0 {96- refs = append(refs, SheetRef{Ref: ref, Var: "__sheet" + strconv.Itoa(len(refs)+1)})97- i = len(refs) - 198- }99- b.WriteString("$" + refs[i].Var)100- }101-}102-103-// RangeSpans are where a pipeline reads ranges of sheets, by byte104-// offsets: each $sheet.A1:C9 whole, as Bind renames them.105-func RangeSpans(pipeline string) [][2]int {106- var out [][2]int107- for i := 0; ; {108- k := strings.Index(pipeline[i:], sheetVar)109- if k < 0 {110- return out111- }112- k += i113- i = k + len(sheetVar)114- if k > 0 && isWord(pipeline[k-1]) {115- continue116- }117- if n := refLen(pipeline[i:]); n > 0 {118- i += n119- out = append(out, [2]int{k, i})120- }121- }122-}123-12427 // refLen is how long the range at the start of s is: a sheet's name,12528 // quoted in ” if it needs to be, then ! and cells, or cells alone. A12629 // dot ends it, so a cell path may follow: $sheet.A1:C9.name.@@ -0,0 +1,297 @@
1+package notebook2+3+import (4+ "slices"5+ "strings"6+)7+8+// How 012 reads a code cell's source, as nu does: its comments and9+// strings apart from its code, and its statements. A statement is a10+// line, or a part of one between `;`, at the top level (outside11+// brackets); a pipeline goes on over the next line when that line12+// starts with `|` (comment lines between allowed, a blank line not), or13+// when its own line ends in `|` or in an assignment's `=`. A statement14+// `name = pipeline` assigns name, as nu's `let name = pipeline` does.15+16+// Source is a code cell's source as 012 reads it.17+type Source struct {18+ Text string19+ // Stmts are its statements, in order.20+ Stmts []Stmt21+ // Comments are its comments' bytes, # to the end of the line, and22+ // Strings its strings', quotes included.23+ Comments, Strings [][2]int24+ // Ranges are where it reads ranges of sheets, each $sheet.A1:C925+ // whole.26+ Ranges [][2]int27+ // code is Text with its comments blanked, and view is code with its28+ // strings blanked too, but for the code interpolated into them:29+ // where names are read.30+ code, view []byte31+}32+33+// Stmt is a statement of a source, by byte offsets into it.34+type Stmt struct {35+ // From and To are its bytes, without the spaces and comments around.36+ From, To int37+ // Body is where what it runs starts: after `name =`, or From.38+ Body int39+ // Name is what `name = pipeline` assigns, or "".40+ Name string41+ // Local is the variable `let name = ...` or `mut name = ...` binds,42+ // which only the cell's later statements read, or "".43+ Local string44+}45+46+// Parse reads src.47+func Parse(src string) Source {48+ s := &scanner{src: src, view: []byte(src)}49+ s.code(0, 0)50+ code := []byte(src)51+ for _, c := range s.comments {52+ blank(code, c[0], c[1])53+ }54+ return Source{Text: src, Stmts: statements(src, s.view), Comments: s.comments, Strings: s.strs, Ranges: s.ranges, code: code, view: s.view}55+}56+57+// Name is what the source's last statement assigns: the cell's output's58+// name, or "".59+func (s Source) Name() string {60+ if len(s.Stmts) == 0 {61+ return ""62+ }63+ return s.Stmts[len(s.Stmts)-1].Name64+}65+66+// Assigned are the names the statements assign, in the order they67+// first do.68+func (s Source) Assigned() []string {69+ var out []string70+ for _, st := range s.Stmts {71+ if st.Name != "" && !slices.Contains(out, st.Name) {72+ out = append(out, st.Name)73+ }74+ }75+ return out76+}77+78+// Refs are the names the source reads as $name from outside it, in the79+// order it first names them: not a name an earlier statement assigns or80+// binds with let, nor nushell's variables and the notebook's own81+// ($selection, $sheet), nor one in a comment or a string.82+func (s Source) Refs() []string {83+ var out []string84+ v := s.view85+ for i := 0; i < len(v); i++ {86+ if v[i] != '$' {87+ continue88+ }89+ j := i + 190+ for j < len(v) && isWord(v[j]) {91+ j++92+ }93+ name := string(v[i+1 : j])94+ if name != "" && ValidName(name) == nil && !slices.Contains(out, name) && !s.local(name, i) {95+ out = append(out, name)96+ }97+ i = j - 198+ }99+ return out100+}101+102+// local reports whether name, read at offset at, is a variable of the103+// cell's own: a statement before assigns or binds it.104+func (s Source) local(name string, at int) bool {105+ return slices.ContainsFunc(s.Stmts, func(st Stmt) bool {106+ return st.To <= at && (st.Name == name || st.Local == name)107+ })108+}109+110+// ReadsSelection reports whether the source reads $selection.111+func (s Source) ReadsSelection() bool {112+ v, want := string(s.view), "$"+Selection113+ for i := 0; ; {114+ k := strings.Index(v[i:], want)115+ if k < 0 {116+ return false117+ }118+ end := i + k + len(want)119+ if end == len(v) || !isWord(v[end]) {120+ return true121+ }122+ i = end123+ }124+}125+126+// Heads are the bytes of each assignment's `name =`, up to what it runs.127+func (s Source) Heads() [][2]int {128+ var out [][2]int129+ for _, st := range s.Stmts {130+ if st.Name != "" {131+ out = append(out, [2]int{st.From, st.Body})132+ }133+ }134+ return out135+}136+137+// scanner tells a source's comments and strings from its code.138+type scanner struct {139+ src string140+ view []byte141+ comments, strs, ranges [][2]int142+}143+144+// code reads code from i up to the bracket close that ends it (0 for145+// the end of the source), returning where it stopped.146+func (s *scanner) code(i int, close byte) int {147+ for i < len(s.src) {148+ c := s.src[i]149+ switch {150+ case close != 0 && c == close:151+ return i152+ case c == '(' || c == '[' || c == '{':153+ i = s.code(i+1, closer(c))154+ if i < len(s.src) {155+ i++156+ }157+ case c == '#' && s.tokenStart(i, close != 0):158+ i = s.comment(i)159+ case c == '"' || c == '\'' || c == '`':160+ i = s.str(i, i+1, c)161+ case c == '$':162+ i = s.dollar(i)163+ case c == 'r' && s.tokenStart(i, close != 0) && rawHashes(s.src[i+1:]) > 0:164+ i = s.raw(i)165+ default:166+ i++167+ }168+ }169+ return i170+}171+172+func closer(c byte) byte {173+ switch c {174+ case '(':175+ return ')'176+ case '[':177+ return ']'178+ }179+ return '}'180+}181+182+// tokenStart reports whether a token may start at i: after a space, a183+// pipe, a ; or an opening bracket, and inside brackets after a , or a184+// : too, where nu reads a # as a comment.185+func (s *scanner) tokenStart(i int, inside bool) bool {186+ if i == 0 {187+ return true188+ }189+ p := s.src[i-1]190+ return strings.IndexByte(" \t\r\n|;([{", p) >= 0 || inside && (p == ',' || p == ':')191+}192+193+func (s *scanner) comment(i int) int {194+ j := i195+ for j < len(s.src) && s.src[j] != '\n' {196+ j++197+ }198+ s.comments = append(s.comments, [2]int{i, j})199+ blank(s.view, i, j)200+ return j201+}202+203+// str reads a quoted string opened at start, its text from i: a double204+// quoted one has escapes, a single quoted or backquoted one none.205+func (s *scanner) str(start, i int, q byte) int {206+ for i < len(s.src) && s.src[i] != q {207+ if q == '"' && s.src[i] == '\\' {208+ i++209+ }210+ i++211+ }212+ i = min(i+1, len(s.src))213+ s.strs = append(s.strs, [2]int{start, i})214+ blank(s.view, start, i)215+ return i216+}217+218+// dollar reads what starts with $ at i: an interpolated string, a range219+// of a sheet, or a variable.220+func (s *scanner) dollar(i int) int {221+ rest := s.src[i+1:]222+ switch {223+ case strings.HasPrefix(rest, `"`) || strings.HasPrefix(rest, "'"):224+ return s.interpolated(i)225+ case strings.HasPrefix(s.src[i:], sheetVar) && (i == 0 || !isWord(s.src[i-1])):226+ from := i + len(sheetVar)227+ if n := refLen(s.src[from:]); n > 0 {228+ s.ranges = append(s.ranges, [2]int{i, from + n})229+ blank(s.view, from, from+n) // its quotes and brackets aren't code230+ return from + n231+ }232+ }233+ return i + 1234+}235+236+// interpolated reads $"..." or $'...' at i: text, blanked, and the code237+// in its parentheses, read as code.238+func (s *scanner) interpolated(i int) int {239+ q := s.src[i+1]240+ j := i + 2241+ blank(s.view, i, j)242+ for j < len(s.src) && s.src[j] != q {243+ switch {244+ case q == '"' && s.src[j] == '\\':245+ blank(s.view, j, min(j+2, len(s.src)))246+ j += 2247+ case s.src[j] == '(':248+ j = s.code(j+1, ')')249+ if j < len(s.src) {250+ j++251+ }252+ default:253+ blank(s.view, j, j+1)254+ j++255+ }256+ }257+ j = min(j+1, len(s.src))258+ blank(s.view, j-1, j)259+ s.strs = append(s.strs, [2]int{i, j})260+ return j261+}262+263+// rawHashes is how many # open a raw string at the start of s, r#'...'#264+// without its r, or 0.265+func rawHashes(s string) int {266+ n := 0267+ for n < len(s) && s[n] == '#' {268+ n++269+ }270+ if n == 0 || n == len(s) || s[n] != '\'' {271+ return 0272+ }273+ return n274+}275+276+// raw reads a raw string at i, to its ' and as many # as opened it.277+func (s *scanner) raw(i int) int {278+ n := rawHashes(s.src[i+1:])279+ end := "'" + strings.Repeat("#", n)280+ body := i + 2 + n281+ j := len(s.src)282+ if k := strings.Index(s.src[body:], end); k >= 0 {283+ j = body + k + len(end)284+ }285+ s.strs = append(s.strs, [2]int{i, j})286+ blank(s.view, i, j)287+ return j288+}289+290+// blank makes b[from:to] spaces, but for its line breaks.291+func blank(b []byte, from, to int) {292+ for i := from; i < to && i < len(b); i++ {293+ if b[i] != '\n' {294+ b[i] = ' '295+ }296+ }297+}@@ -0,0 +1,177 @@
1+package notebook2+3+import (4+ "slices"5+ "strconv"6+ "strings"7+)8+9+// statements splits a source into its statements, reading view, the10+// source with its comments and strings blanked.11+func statements(src string, view []byte) []Stmt {12+ var out []Stmt13+ depth, from := 0, 014+ for i := 0; i <= len(view); i++ {15+ if i < len(view) {16+ switch c := view[i]; {17+ case c == '(' || c == '[' || c == '{':18+ depth++19+ continue20+ case c == ')' || c == ']' || c == '}':21+ depth = max(depth-1, 0)22+ continue23+ case c != ';' && c != '\n', depth > 0, c == '\n' && continues(src, view, from, i):24+ continue25+ }26+ }27+ if st, ok := statement(view, from, i); ok {28+ out = append(out, st)29+ }30+ from = i + 131+ }32+ return out33+}34+35+// continues reports whether the statement from from goes on past the36+// line break at i: its line ends in | or in an assignment's =, or the37+// next line, past comment lines, starts with |.38+func continues(src string, view []byte, from, i int) bool {39+ before := strings.TrimRight(string(view[from:i]), " \t\r")40+ if strings.HasSuffix(before, "|") || strings.HasSuffix(before, "=") {41+ return true42+ }43+ for j := i + 1; j < len(view); {44+ end := j45+ for end < len(view) && view[end] != '\n' {46+ end++47+ }48+ line := strings.TrimSpace(string(view[j:end]))49+ switch {50+ case strings.HasPrefix(line, "|"):51+ return true52+ case line != "" || strings.TrimSpace(src[j:end]) == "":53+ return false // code, or a blank line54+ }55+ j = end + 1 // a comment line56+ }57+ return false58+}59+60+// statement is the statement in view[from:to], if there's one.61+func statement(view []byte, from, to int) (Stmt, bool) {62+ for from < to && isSpace(view[from]) {63+ from++64+ }65+ for to > from && isSpace(view[to-1]) {66+ to--67+ }68+ if from == to {69+ return Stmt{}, false70+ }71+ st := Stmt{From: from, To: to, Body: from}72+ name, rest := word(view, from, to)73+ rest = skipSpaces(view, rest, to, " \t")74+ switch {75+ case rest < to && view[rest] == '=' && ValidName(name) == nil && !opAt(view, rest+1, to):76+ st.Name, st.Body = name, skipSpaces(view, rest+1, to, " \t\r\n")77+ case name == "let" || name == "mut" || name == "const":78+ st.Local, _ = word(view, skipSpaces(view, rest, to, " \t"), to)79+ }80+ return st, true81+}82+83+// opAt reports whether an = ending at i is part of an operator: ==, =~.84+func opAt(view []byte, i, to int) bool { return i < to && (view[i] == '=' || view[i] == '~') }85+86+// word is the word of letters, digits and _ at from, and where it ends.87+func word(view []byte, from, to int) (string, int) {88+ i := from89+ for i < to && isWord(view[i]) {90+ i++91+ }92+ return string(view[from:i]), i93+}94+95+func skipSpaces(view []byte, i, to int, spaces string) int {96+ for i < to && strings.IndexByte(spaces, view[i]) >= 0 {97+ i++98+ }99+ return i100+}101+102+func isSpace(c byte) bool { return c == ' ' || c == '\t' || c == '\r' || c == '\n' }103+104+// Output is the name of the variable that holds a cell's output in the105+// record a run hands back when it also hands back variables.106+const OutputVar = "__out"107+108+// Command is what nu runs for the source: each statement but the last109+// that assigns a name as `let name = ...`, the last without its `name =`110+// (its value is the output), the ranges of sheets renamed to variables111+// of their own ($__sheet1), and the comments blanked, so what's left112+// keeps its place. When statements before the last assign names the113+// last doesn't, those are exports, and the command ends in a record of114+// the output (as __out) and each of them.115+func (s Source) Command() (cmd string, exports []string, ranges []SheetRef) {116+ var edits []edit117+ for _, r := range s.Ranges {118+ ref := s.Text[r[0]+len(sheetVar) : r[1]]119+ i := slices.IndexFunc(ranges, func(x SheetRef) bool { return x.Ref == ref })120+ if i < 0 {121+ ranges = append(ranges, SheetRef{Ref: ref, Var: "__sheet" + strconv.Itoa(len(ranges)+1)})122+ i = len(ranges) - 1123+ }124+ edits = append(edits, edit{r[0], r[1], "$" + ranges[i].Var})125+ }126+ if len(s.Stmts) == 0 {127+ return apply(s.code, edits), nil, ranges128+ }129+ last := s.Stmts[len(s.Stmts)-1]130+ for _, st := range s.Stmts[:len(s.Stmts)-1] {131+ if st.Name == "" {132+ continue133+ }134+ edits = append(edits, edit{st.From, st.Body, "let " + st.Name + " = "})135+ if st.Name != last.Name && !slices.Contains(exports, st.Name) {136+ exports = append(exports, st.Name)137+ }138+ }139+ if last.Name != "" {140+ edits = append(edits, edit{last.From, last.Body, ""})141+ }142+ if len(exports) > 0 {143+ var rec strings.Builder144+ rec.WriteString(")")145+ for _, e := range exports {146+ rec.WriteString(", " + e + ": $" + e)147+ }148+ rec.WriteString("}")149+ edits = append(edits, edit{last.Body, last.Body, "{" + OutputVar + ": ("}, edit{last.To, last.To, rec.String()})150+ }151+ return apply(s.code, edits), exports, ranges152+}153+154+// edit replaces b[from:to] with text.155+type edit struct {156+ from, to int157+ text string158+}159+160+// apply makes edits to b, which don't overlap.161+func apply(b []byte, edits []edit) string {162+ slices.SortStableFunc(edits, func(x, y edit) int {163+ if x.from != y.from {164+ return x.from - y.from165+ }166+ return x.to - y.to167+ })168+ var out strings.Builder169+ at := 0170+ for _, e := range edits {171+ out.Write(b[at:e.from])172+ out.WriteString(e.text)173+ at = e.to174+ }175+ out.Write(b[at:])176+ return out.String()177+}@@ -9,6 +9,7 @@ package nushell
99 1010 import (1111 "bytes"12+ "encoding/base64"1213 "context"1314 "errors"1415 "fmt"@@ -33,6 +34,10 @@ type Job struct {
3334 Tables map[string][]byte3435 // Config runs nu with the user's config files rather than without.3536 Config bool37+ // Vars are the variables the run hands back beside its output: the38+ // command ends in a record of its output, as __out, and each of39+ // them, and the run prints each as a line of its own (ExecVars).40+ Vars []string3641 // IDE asks nu about the script rather than running it: the flags3742 // that ask (--ide-ast, or --ide-complete 14), nu reading the script3843 // from a file named after them, without config files or the@@ -77,10 +82,43 @@ func Script(job Job, names []string) string {
7782 for i, name := range names {7883 fmt.Fprintf(&b, "let %s = (open --raw $env.NU012_TABLE_%d | from nuon)\n", name, i)7984 }80- b.WriteString("do {\n" + job.Command + "\n} | to nuon\n")85+ b.WriteString("do {\n" + job.Command + "\n} | ")86+ if len(job.Vars) > 0 {87+ b.WriteString(`items {|k, v| $"($k) ($v | to nuon | encode base64)" } | str join "\n"` + "\n")88+ } else {89+ b.WriteString("to nuon\n")90+ }8191 return b.String()8292 }8393 94+// outVar names the output among a run's variables.95+const outVar = "__out"96+97+// ExecVars runs job as Exec does, returning its output and, when it98+// hands back variables (Job.Vars), their values, each as NUON. They're99+// the last lines it printed, each a name and its NUON as base64, so100+// what the pipeline printed itself before them doesn't mix in.101+func ExecVars(ctx context.Context, r Runner, job Job, timeout time.Duration, maxBytes int) ([]byte, map[string][]byte, error) {102+ out, err := Exec(ctx, r, job, timeout, maxBytes)103+ if err != nil || len(job.Vars) == 0 {104+ return out, nil, err105+ }106+ lines := strings.Split(string(out), "\n")107+ vars := map[string][]byte{}108+ for _, line := range lines[max(len(lines)-len(job.Vars)-1, 0):] {109+ name, enc, _ := strings.Cut(strings.TrimSpace(line), " ")110+ if data, err := base64.StdEncoding.DecodeString(enc); err == nil && (name == outVar || slices.Contains(job.Vars, name)) {111+ vars[name] = data112+ }113+ }114+ nuon, ok := vars[outVar]115+ if !ok || len(vars) != len(job.Vars)+1 {116+ return nil, nil, errors.New("the run didn't hand back its variables")117+ }118+ delete(vars, outVar)119+ return nuon, vars, nil120+}121+84122 // ErrTooLarge is a run that printed more than it may keep.85123 var ErrTooLarge = errors.New("the output is too large to keep")86124 @@ -5,6 +5,7 @@ import (
55 "context"66 "errors"77 "fmt"8+ "slices"89 "time"910 1011 tea "charm.land/bubbletea/v2"@@ -23,23 +24,15 @@ import (
2324 // jobFor is what running cell c takes: the outputs, linked files and2425 // ranges it reads, as NUON.2526 func (m *Model) jobFor(s *sheet.Sheet, c notebook.Cell, run *nbRun) (nushell.Job, error) {26- pipeline, ranges := notebook.Bind(c.Pipeline())27- job := nushell.Job{Command: pipeline, Tables: map[string][]byte{}, Config: m.configBool("nu-config", false)}28- run.reads = map[string]int{}2927 cells := s.NotebookCells()30- names := notebook.Names(cells)31- for _, name := range notebook.Refs(pipeline) {32- if j, ok := names[name]; ok {33- o := m.book().Output(cells[j].ID)34- switch {35- case o == nil || o.Unsaved:36- return job, fmt.Errorf("it reads $%s, which hasn't run", name)37- case o.Failed():38- return job, fmt.Errorf("it reads $%s, which failed", name)39- }40- job.Tables[name], run.reads[name] = o.NUON, o.Seq41- continue42- }28+ i := slices.IndexFunc(cells, func(x notebook.Cell) bool { return x.ID == c.ID })29+ p, err := notebook.Prepare(cells, i, m.book().Output)30+ job := nushell.Job{Command: p.Command, Vars: p.Exports, Tables: p.Tables, Config: m.configBool("nu-config", false)}31+ run.reads = p.Reads32+ if err != nil {33+ return job, err34+ }35+ for _, name := range p.Others {4336 if t, r, ok := m.book().Region(name); ok && r.Linked() && r.Name == name {4437 data, err := regionNUON(t, r)4538 if err != nil {@@ -48,12 +41,12 @@ func (m *Model) jobFor(s *sheet.Sheet, c notebook.Cell, run *nbRun) (nushell.Job
4841 job.Tables[name] = data4942 }5043 }51- if notebook.ReadsSelection(pipeline) {44+ if p.Selection {5245 if err := m.bindSelection(&job, run); err != nil {5346 return job, err5447 }5548 }56- for _, ref := range ranges {49+ for _, ref := range p.Ranges {5750 t, rng, err := m.rangeRef(ref.Ref)5851 if err != nil {5952 return job, err@@ -319,22 +319,31 @@ func (v *View) Head() (name, text string) {
319319 if v.onOut {320320 return name, v.outputSummary(c)321321 }322+ src := c.Parse()322323 var reads []string323- for _, r := range notebook.Refs(c.Pipeline()) {324+ for _, r := range src.Refs() {324325 reads = append(reads, "$"+r)325326 }326- if notebook.ReadsSelection(c.Pipeline()) {327+ if src.ReadsSelection() {327328 reads = append(reads, "$selection")328329 }329- if _, refs := notebook.Bind(c.Pipeline()); len(refs) > 0 {330- for _, r := range refs {331- reads = append(reads, "$sheet."+r.Ref)332- }330+ _, _, ranges := src.Command()331+ for _, r := range ranges {332+ reads = append(reads, "$sheet."+r.Ref)333333 }334334 text = "a nushell pipeline"335335 if len(reads) > 0 {336336 text = "reads " + strings.Join(reads, ", ")337337 }338+ var sets []string339+ for _, a := range src.Assigned() {340+ if a != c.Name() {341+ sets = append(sets, "$"+a)342+ }343+ }344+ if len(sets) > 0 {345+ text += "; sets " + strings.Join(sets, ", ")346+ }338347 if n := c.Name(); n != "" {339348 text += "; read as $" + n + ", nu." + n + " in formulas"340349 }@@ -16,28 +16,30 @@ import (
1616 1717 // prepared is the text nu is asked about: a prelude declaring the1818 // notebook's variables, then the source masked, starting at byte at.19+// Masking keeps every byte of the source in its place, so what nu says20+// of the text is where it says it in the source.1921 type prepared struct {20- text string21- at int22- head int // the bytes of the source that are its `name =`22+ text string23+ at int24+ heads [][2]int // the bytes of the source that are its `name =`s2325 }2426 2527 // prepare masks src and declares what it reads: `sales = ls` is asked2628 // about as ` ls`, $sheet.A1:C9 as $__s1_______, and the prelude27-// declares $__s1, $selection, $sheet and names, each `let x: any = []`28-// so nu takes it for whatever it holds.29+// declares $__s1, $selection, $sheet, names and the names the cell30+// assigns, each `let x: any = []` so nu takes it for whatever it holds.2931 func prepare(src string, names []string) prepared {3032 b := []byte(src)31- name, pipeline := notebook.SplitName(src)32- head := 033- if name != "" {34- head = len(src) - len(pipeline)35- for i := range head {33+ parsed := notebook.Parse(src)34+ heads := parsed.Heads()35+ for _, h := range heads {36+ for i := h[0]; i < h[1]; i++ {3637 if b[i] != '\n' {3738 b[i] = ' '3839 }3940 }4041 }42+ names = append(parsed.Assigned(), names...)4143 var pre strings.Builder4244 declared := map[string]bool{}4345 declare := func(name, value string) {@@ -46,9 +48,9 @@ func prepare(src string, names []string) prepared {
4648 pre.WriteString("let " + name + ": any = " + value + "\n")4749 }4850 }49- for i, s := range notebook.RangeSpans(src) {51+ for i, s := range parsed.Ranges {5052 v := "__s" + strconv.Itoa(i+1)51- if s[0] < head || len(v)+1 > s[1]-s[0] {53+ if len(v)+1 > s[1]-s[0] {5254 continue5355 }5456 v += strings.Repeat("_", s[1]-s[0]-len(v)-1)@@ -62,22 +64,28 @@ func prepare(src string, names []string) prepared {
6264 declare(nm, "[]")6365 }6466 }65- return prepared{text: pre.String() + string(b), at: pre.Len(), head: head}67+ return prepared{text: pre.String() + string(b), at: pre.Len(), heads: heads}68+}69+70+// inHead reports whether src[from:to] is part of an assignment's head.71+func (p prepared) inHead(from, to int) bool {72+ return slices.ContainsFunc(p.heads, func(h [2]int) bool { return from < h[1] && to > h[0] })6673 }6774 6875 // spans are src's syntax roles from nu's shapes of p's text: the69-// tokenizer's for the cell's name and comments, which nu doesn't70-// report, and a number's unit (the kb of 1kb) as part of the number.76+// tokenizer's for the names the cell assigns and its comments, which nu77+// doesn't report, and a number's unit (the kb of 1kb) as part of the78+// number.7179 func (p prepared) spans(src string, shapes []nushell.Shape) []Span {7280 var out []Span7381 for _, s := range (Tokens{}).Highlight(context.Background(), src) {74- if s.To <= p.head || s.Kind == theme.SyntaxComment {82+ if p.inHead(s.From, s.To) || s.Kind == theme.SyntaxComment {7583 out = append(out, s)7684 }7785 }7886 for i, s := range shapes {7987 from, to := s.From-p.at, s.To-p.at80- if from < p.head || to > len(src) || from >= to {88+ if from < 0 || to > len(src) || from >= to || p.inHead(from, to) {8189 continue8290 }8391 kind, ok := shapeKind(s.Shape, src[from:to])@@ -71,12 +71,30 @@ var operators = []string{"and", "or", "not", "xor", "in", "not-in", "like", "not
7171 7272 // Highlight implements Highlighter.7373 func (Tokens) Highlight(_ context.Context, src string) []Span {74- t := tokenizer{src: src, command: true}75- if name, _ := notebook.SplitName(src); name != "" {76- t.i = strings.Index(src, name) + len(name)77- t.add(t.i-len(name), theme.SyntaxVariable) // the cell's name, as $name reads it74+ t := tokenizer{src: src, command: true, whole: map[int]Span{}}75+ p := notebook.Parse(src)76+ for _, st := range p.Stmts {77+ if st.Name != "" { // a name assigned, as $name reads it78+ t.whole[st.From] = Span{st.From, st.From + len(st.Name), theme.SyntaxVariable}79+ }80+ }81+ for _, c := range p.Comments {82+ t.whole[c[0]] = Span{c[0], c[1], theme.SyntaxComment}83+ }84+ for _, s := range p.Strings {85+ if src[s[0]] == 'r' { // a raw string, r#'...'#86+ t.whole[s[0]] = Span{s[0], s[1], theme.SyntaxString}87+ }7888 }7989 for t.i < len(t.src) {90+ if s, ok := t.whole[t.i]; ok {91+ t.out = append(t.out, s)92+ t.i = s.To93+ if s.Kind != theme.SyntaxComment {94+ t.command = false95+ }96+ continue97+ }8098 if !t.punct() {8199 t.word()82100 }@@ -91,6 +109,9 @@ type tokenizer struct {
91109 i int92110 command bool93111 out []Span112+ // whole are the tokens the source's reading knows (notebook.Parse),113+ // by where they start: names assigned, comments, raw strings.114+ whole map[int]Span94115 }95116 96117 func (t *tokenizer) add(from int, k theme.Syntax) { t.out = append(t.out, Span{from, t.i, k}) }@@ -109,9 +130,6 @@ func (t *tokenizer) punct() bool {
109130 t.command = true110131 case c == ' ' || c == '\t' || c == '\r' || c == ')' || c == '}' || c == '[' || c == ']' || c == ',':111132 t.i++112- case c == '#' && (i == 0 || isSpace(src[i-1])):113- t.i = lineEnd(src, i)114- t.add(start, theme.SyntaxComment)115133 case c == '"' || c == '\'' || c == '`':116134 t.i = quoted(src, i)117135 t.add(start, theme.SyntaxString)