.nuon and .json import and download, typed by nuon's mapping; ImportReader reads a table from a stream, telling NUON, JSON, CSV and TSV apart, and Encode writes CSV, TSV, JSON or NUON to one.
- commit
- d270e2425ec48e0befda0172c5f99e544e2ee2e3
- parent
- 8ace0e657f9f366152090f8669c1cad6d1a80f8e
- author
- Dami <dami.etoile@gmail.com>
- kind
- feat risk score 2.0 of 3, Jev's estimate
| M | Makefile | 2 | + |
| M | docs/ | 2 | +- |
| M | docs/ | 5 | +- |
| M | internal/ | 68 | ++++---- |
| M | internal/ | 32 | ++++- |
| A | internal/ | 162 | ++++++++++++++++++++ |
| A | internal/ | 243 | ++++++++++++++++++++++++++++++ |
| A | internal/ | 221 | +++++++++++++++++++++++++++ |
| A | internal/ | 108 | +++++++++++++ |
@@ -47,6 +47,8 @@ fuzz:
4747 go test ./internal/fileio -run '^$$' -fuzz FuzzReadWK1 -fuzztime 60s4848 go test ./internal/fileio -run '^$$' -fuzz 'FuzzReadXLSX$$' -fuzztime 60s4949 go test ./internal/fileio -run '^$$' -fuzz FuzzReadXLSXParts -fuzztime 60s50+ go test ./internal/nuon -run '^$$' -fuzz FuzzParse -fuzztime 60s51+ go test ./internal/fileio -run '^$$' -fuzz FuzzReadNUON -fuzztime 60s5052 5153 e2e: $(GHOSTTY_STAMP)5254 cd e2e && PKG_CONFIG_PATH=$(GHOSTTY_OUT)/share/pkgconfig go test -count=1 ./...@@ -9,7 +9,7 @@ sidebar_position: 5
99 make check # all of the below that must pass before a push: gofmt, vet, lint, test, oracle, e2e1010 make lint # go vet, staticcheck, cognitive complexity at most 25, Go files at most 500 lines, doclint and doccheck (see Docs below)1111 make test # engine, file formats and UI unit tests12-make fuzz # fuzz the formula parser and the .012, CSV, .wk1 and XLSX readers12+make fuzz # fuzz the formula parser and the .012, CSV, .wk1, XLSX and NUON readers1313 make oracle # compare formulas and number formats with excelize1414 make e2e # run the real binary in a terminal emulator (needs Zig and pkg-config)1515 make screens # rewrite the golden screens and build the review gallery@@ -25,9 +25,12 @@ it back in its format.
2525 | SQLite | Pick a table or view, or type a query; a header row names the columns | The sheet shown or the selection as a table, first row as column names; a table of that name is replaced |2626 | Parquet | Every column, with dates and timestamps; lists joined with commas | |2727 | Lotus 1-2-3 `.wk1`, `.wks` | Numbers, labels with their alignment, formats, column widths, formulas translated (references, operators, `@SUM`, `@AVG`, `@IF`, `@ROUND`, `@PMT` and 60 more) or kept as values | |28+| JSON `.json` | A list of records (or records one after another, as NDJSON): keys become the header row, numbers, booleans and text keep their types, nested lists and records are their text | The sheet shown as a list of records named by its first row, numbers as numbers |29+| Nushell `.nuon` | A nushell table with its types: file sizes, durations and dates become numbers in the Size, Duration and Date time formats; see [Nushell](../terminal/nushell.md#types) | The sheet shown as a nushell table, first row as column names, types kept by the cells' formats |2830 2931 Formats other than XLSX import as one sheet named after the file (or the30-SQLite table).32+SQLite table). A table can also come in on standard input, and go back33+out on standard output: see [Nushell and pipelines](../terminal/nushell.md).3134 3235 File > Import asks where the data goes, as Sheets' Import location does3336 (a new, empty spreadsheet is simply replaced):@@ -162,6 +162,10 @@ func exportTSV(_ context.Context, name string, snap *Snapshot, _ ExportOptions)
162162 return exportDelimited(name, TSV, snap)163163 }164164 165+func encodeCSV(w io.Writer, snap *Snapshot) (int, error) { return encodeDelimited(w, CSV, snap) }166+167+func encodeTSV(w io.Writer, snap *Snapshot) (int, error) { return encodeDelimited(w, TSV, snap) }168+165169 func importDelimited(ctx context.Context, name string, k Kind, opt Options) (*Result, error) {166170 prog := opt.Progress167171 f, err := os.Open(name)@@ -191,10 +195,16 @@ func importDelimited(ctx context.Context, name string, k Kind, opt Options) (*Re
191195 // writes its numbers that way (see numberLocale). progress is called192196 // every few hundred rows.193197 func readDelimited(ctx context.Context, in io.Reader, k Kind, maxCells int, loc *locale.Locale, progress func(rows int)) (*sheet.Sheet, int, []string, error) {198+ s, rows, notes, _, err := readDelimitedAs(ctx, in, k, maxCells, loc, progress)199+ return s, rows, notes, err200+}201+202+// readDelimitedAs is readDelimited, also returning the delimiter.203+func readDelimitedAs(ctx context.Context, in io.Reader, k Kind, maxCells int, loc *locale.Locale, progress func(rows int)) (*sheet.Sheet, int, []string, rune, error) {194204 br := bufio.NewReaderSize(in, sniffSize)195205 text, enc, err := decode(br)196206 if err != nil {197- return nil, 0, nil, err207+ return nil, 0, nil, 0, err198208 }199209 tr := bufio.NewReaderSize(text, sniffSize)200210 sample, _ := tr.Peek(sniffSize)@@ -216,7 +226,7 @@ func readDelimited(ctx context.Context, in io.Reader, k Kind, maxCells int, loc
216226 // Report before reading on: a pipe may keep the next read waiting.217227 if row%256 == 0 {218228 if err := ctx.Err(); err != nil {219- return nil, 0, nil, err229+ return nil, 0, nil, 0, err220230 }221231 progress(row)222232 }@@ -227,9 +237,9 @@ func readDelimited(ctx context.Context, in io.Reader, k Kind, maxCells int, loc
227237 if err != nil {228238 var pe *csv.ParseError229239 if errors.As(err, &pe) {230- return nil, 0, nil, fmt.Errorf("line %d: %v", pe.Line, pe.Err)240+ return nil, 0, nil, 0, fmt.Errorf("line %d: %v", pe.Line, pe.Err)231241 }232- return nil, 0, nil, err242+ return nil, 0, nil, 0, err233243 }234244 for col, field := range rec {235245 if field != "" {@@ -249,7 +259,7 @@ func readDelimited(ctx context.Context, in io.Reader, k Kind, maxCells int, loc
249259 notes = append(notes, "read as "+enc)250260 }251261 s, notes := b.finish(notes)252- return s, row, notes, nil262+ return s, row, notes, comma, nil253263 }254264 255265 func delimiterName(r rune) string {@@ -270,37 +280,31 @@ func delimiterName(r rune) string {
270280 // separator is a comma.271281 // Rows are written as they are formatted, never all held at once.272282 func exportDelimited(name string, k Kind, snap *Snapshot) (*ExportResult, error) {273- rows := 0274- err := writeFile(name, func(out io.Writer) error {275- w := csv.NewWriter(out)276- w.Comma = csvComma(snap.Locale)277- if k == TSV {278- w.Comma = '\t'279- }280- for line := range snap.textRows() {281- if err := w.Write(line); err != nil {282- return err283- }284- rows++285- }286- w.Flush()287- return w.Error()288- })283+ res, err := exportText(name, snap, func(w io.Writer, snap *Snapshot) (int, error) { return encodeDelimited(w, k, snap) })289284 if err != nil {290285 return nil, err291286 }292- formulas := 0293- for _, c := range snap.Cells {294- if c.Formula {295- formulas++296- }297- }298- res := &ExportResult{Rows: rows}299287 if k == CSV && csvComma(snap.Locale) != ',' {300- res.Notes = append(res.Notes, "separated by semicolons")301- }302- if formulas > 0 {303- res.Notes = append(res.Notes, count(formulas, "formula", "formulas")+" saved as values")288+ res.Notes = append([]string{"separated by semicolons"}, res.Notes...)304289 }305290 return res, nil306291 }292+293+// encodeDelimited writes the rows of exportDelimited to out, returning294+// how many it wrote.295+func encodeDelimited(out io.Writer, k Kind, snap *Snapshot) (int, error) {296+ rows := 0297+ w := csv.NewWriter(out)298+ w.Comma = csvComma(snap.Locale)299+ if k == TSV {300+ w.Comma = '\t'301+ }302+ for line := range snap.textRows() {303+ if err := w.Write(line); err != nil {304+ return 0, err305+ }306+ rows++307+ }308+ w.Flush()309+ return rows, w.Error()310+}@@ -2,6 +2,7 @@ package fileio
22 33 import (44 "context"5+ "io"56 "path/filepath"67 "strings"78 )@@ -16,6 +17,8 @@ const (
1617 SQLite1718 Parquet1819 WK120+ JSON21+ NUON1922 )2023 2124 // A fileFormat is everything about a Kind: its names, its extensions,@@ -35,6 +38,9 @@ type fileFormat struct {
3538 write exporter3639 menu string // the Download menu's title, without the extension3740 about string // what downloading writes, for the command's description41+ // encode is set for text formats, which Encode writes to a stream42+ // (012 --pipe's standard output); it returns the rows written.43+ encode func(io.Writer, *Snapshot) (int, error)3844 3945 // book is set for formats that hold named sheets of their own:4046 // every sheet is exported, and an imported file keeps its sheets'@@ -55,11 +61,11 @@ type exporter func(ctx context.Context, name string, snap *Snapshot, opt ExportO
5561 // kinds are the formats, in Kind order, which is menu order.5662 var kinds = []fileFormat{{5763 kind: CSV, name: "CSV", noun: "CSV", label: "Comma-separated values", exts: []string{".csv"},58- read: importCSV, write: exportCSV, menu: "Comma-separated values",64+ read: importCSV, write: exportCSV, menu: "Comma-separated values", encode: encodeCSV,5965 about: "Save the values as shown, as comma-separated values (.csv)",6066 }, {6167 kind: TSV, name: "TSV", noun: "TSV", label: "Tab-separated values", exts: []string{".tsv", ".tab"},62- read: importTSV, write: exportTSV, menu: "Tab-separated values",68+ read: importTSV, write: exportTSV, menu: "Tab-separated values", encode: encodeTSV,6369 about: "Save the values as shown, as tab-separated values (.tsv)",6470 }, {6571 kind: XLSX, name: "XLSX", noun: "Excel", label: "Excel workbook", exts: []string{".xlsx", ".xlsm"},@@ -77,6 +83,14 @@ var kinds = []fileFormat{{
7783 }, {7884 kind: WK1, name: "WK1", noun: "1-2-3", label: "Lotus 1-2-3 worksheet", exts: []string{".wk1", ".wks"},7985 read: importWK1,86+}, {87+ kind: JSON, name: "JSON", noun: "JSON", label: "JSON list of records", exts: []string{".json"},88+ read: importJSON, write: exportJSON, menu: "JSON list of records", encode: encodeJSON,89+ about: "Save the sheet as a JSON list of records, named by its first row",90+}, {91+ kind: NUON, name: "NUON", noun: "NUON", label: "Nushell table (NUON)", exts: []string{".nuon"},92+ read: importNUON, write: exportNUON, menu: "Nushell table", encode: encodeNUON,93+ about: "Save the sheet as a nushell table that keeps its types, named by its first row",8094 }}8195 8296 // unknownFormat is what an invalid Kind reports.@@ -145,6 +159,20 @@ func (k Kind) About() string { return k.format().about }
145159 // sheets, so a download writes the whole workbook (see SnapBook).146160 func (k Kind) HoldsSheets() bool { return k.format().book }147161 162+// IsText reports whether this kind is text Encode writes to a stream:163+// CSV, TSV, JSON or NUON.164+func (k Kind) IsText() bool { return k.format().encode != nil }165+166+// KindNamed finds a kind by its short name, ignoring case: "nuon".167+func KindNamed(name string) (Kind, bool) {168+ for _, k := range kinds {169+ if strings.EqualFold(k.name, name) {170+ return k.kind, true171+ }172+ }173+ return 0, false174+}175+148176 // HasTables reports whether a file of this kind is a database of tables:149177 // importing picks one (see Tables), and a download writes the sheet or150178 // the selection as a named table (ExportOptions.Table).@@ -0,0 +1,162 @@
1+package fileio2+3+import (4+ "context"5+ "errors"6+ "io"7+ "math"8+ "os"9+ "time"10+11+ "github.com/FelineStateMachine/012/internal/numfmt"12+ "github.com/FelineStateMachine/012/internal/nuon"13+ "github.com/FelineStateMachine/012/internal/sheet"14+)15+16+// NUON and JSON tables become cells with their types (see17+// docs/terminal/nushell.md for the mapping): the header row holds the18+// column names, numbers stay numbers, file sizes are bytes in the Size19+// format, durations are elapsed time in the Duration format, dates are20+// date-time serials in the local time zone, and nested records and21+// lists are their NUON text. Rows are read and stored one at a time22+// (nuon.Reader), so a table streaming in is stored as it arrives.23+24+// zone is the time zone dates are shown in: the local one, which tests25+// replace.26+var zone = func() *time.Location { return time.Local }27+28+func importNUON(ctx context.Context, name string, opt Options) (*Result, error) {29+ return importTable(ctx, name, opt)30+}31+32+func importJSON(ctx context.Context, name string, opt Options) (*Result, error) {33+ return importTable(ctx, name, opt)34+}35+36+func importTable(ctx context.Context, name string, opt Options) (*Result, error) {37+ f, err := os.Open(name)38+ if err != nil {39+ return nil, err40+ }41+ defer f.Close()42+ defer context.AfterFunc(ctx, func() { f.Close() })()43+ var size int6444+ if st, err := f.Stat(); err == nil && st.Mode().IsRegular() {45+ size = st.Size()46+ }47+ counter := &countingReader{r: f}48+ res, _, err := readTable(ctx, counter, opt.MaxCells, func(rows int) {49+ opt.Progress.setRows(rows)50+ opt.Progress.setFrac(counter.n, size)51+ })52+ return res, err53+}54+55+// readTable reads a NUON or JSON table into a new sheet a row at a time,56+// keeping up to maxCells cells (see newBuilder), and reports whether the57+// text was all JSON. progress is called every few hundred rows.58+func readTable(ctx context.Context, in io.Reader, maxCells int, progress func(rows int)) (*Result, bool, error) {59+ r := nuon.NewReader(in)60+ b := newBuilder(ctx, maxCells)61+ t := tableCells{b: b, cols: map[string]int{}, zone: zone()}62+ row := 063+ for ; ; row++ {64+ if row%256 == 0 {65+ if err := ctx.Err(); err != nil {66+ return nil, false, err67+ }68+ progress(row)69+ }70+ fields, err := r.Next()71+ if errors.Is(err, io.EOF) {72+ break73+ }74+ if err != nil {75+ return nil, false, err76+ }77+ if b.isFull() {78+ b.fits(sheet.Addr{Row: row + 1}) // counted for the note79+ continue80+ }81+ t.header(r.Header())82+ t.row(row+1, fields)83+ }84+ progress(row)85+ s, notes := b.finish(nil)86+ return &Result{Sheet: s, Rows: row + 1, Notes: notes}, r.JSON(), nil87+}88+89+// tableCells stores a table's rows: each column's name in the header90+// row once it's first seen, and each value under its column.91+type tableCells struct {92+ b *builder93+ cols map[string]int94+ zone *time.Location95+}96+97+// header adds the table form's columns, in their order, before any row98+// names them.99+func (t *tableCells) header(cols []string) {100+ for _, c := range cols {101+ t.col(c)102+ }103+}104+105+// col is the column named c, added after the others if it's new.106+func (t *tableCells) col(c string) int {107+ i, ok := t.cols[c]108+ if !ok {109+ i = len(t.cols)110+ t.cols[c] = i111+ t.b.text(sheet.Addr{Col: i}, c, sheet.Format{}, sheet.Style{})112+ }113+ return i114+}115+116+func (t *tableCells) row(r int, fields nuon.Row) {117+ for _, f := range fields {118+ t.value(sheet.Addr{Col: t.col(f.Key), Row: r}, f.Value)119+ }120+}121+122+// Durations are elapsed time: a count of days, shown as hours, minutes123+// and seconds, with milliseconds when there's less than a second in it.124+var (125+ durationFormat = sheet.Format{Kind: sheet.FmtDuration}126+ durationFormatFrac = sheet.Format{Kind: sheet.FmtDuration, Pattern: "[h]:mm:ss.000"}127+)128+129+// nsPerDay is a day in nanoseconds, a date serial's unit.130+const nsPerDay = 86400e9131+132+// value stores one value as a cell.133+func (t *tableCells) value(a sheet.Addr, v nuon.Value) {134+ b, none := t.b, sheet.Style{}135+ switch v.Kind {136+ case nuon.Null:137+ case nuon.Bool:138+ b.boolean(a, v.Bool, none)139+ case nuon.Int:140+ b.number(a, float64(v.Int), sheet.Format{}, none)141+ case nuon.Float:142+ if math.IsNaN(v.Float) || math.IsInf(v.Float, 0) {143+ b.text(a, v.String(), sheet.Format{}, none)144+ return145+ }146+ b.number(a, v.Float, sheet.Format{}, none)147+ case nuon.String:148+ b.text(a, v.Str, sheet.Format{}, none)149+ case nuon.Filesize:150+ b.number(a, float64(v.Int), sheet.Preset(sheet.FmtSize), none)151+ case nuon.Duration:152+ f := durationFormat153+ if v.Int%1e9 != 0 {154+ f = durationFormatFrac155+ }156+ b.number(a, float64(v.Int)/nsPerDay, f, none)157+ case nuon.Date:158+ b.number(a, numfmt.SerialOf(v.Time.In(t.zone)), sheet.Preset(sheet.FmtDateTime), none)159+ default: // binary, lists and records160+ b.text(a, v.String(), sheet.Format{}, none)161+ }162+}@@ -0,0 +1,243 @@
1+package fileio2+3+import (4+ "bytes"5+ "context"6+ "os"7+ "os/exec"8+ "path/filepath"9+ "strings"10+ "testing"11+ "time"12+13+ "github.com/FelineStateMachine/012/internal/nuon"14+ "github.com/FelineStateMachine/012/internal/sheet"15+)16+17+// inZone shows dates in loc for the test.18+func inZone(t *testing.T, loc *time.Location) {19+ t.Helper()20+ old := zone21+ zone = func() *time.Location { return loc }22+ t.Cleanup(func() { zone = old })23+}24+25+var mountain = time.FixedZone("MDT", -6*3600)26+27+func readNUON(t *testing.T, text string) *Result {28+ t.Helper()29+ res, err := ImportReader(context.Background(), "stdin", strings.NewReader(text), Options{})30+ if err != nil {31+ t.Fatalf("%s: %v", text, err)32+ }33+ return res34+}35+36+func TestNUONCells(t *testing.T) {37+ inZone(t, mountain)38+ res := readNUON(t, `[[name, size, modified, took, ok, n, x, tags, meta, raw];39+ ["CLAUDE.md", 1646b, 2026-09-27T17:27:31+00:00, 90sec, true, 1.5, "12", [a, b], {k: 1}, 0x[DEAD]],40+ [go.mod, 2mb, 2026-01-02T03:04:05.250-06:00, 1500ms, false, -3, null, [], {}, 0x[]]]`)41+ s := res.Sheet42+ if res.Kind != NUON || s.Name() != "stdin" || res.Rows != 3 {43+ t.Errorf("kind %v, sheet %q, rows %d", res.Kind, s.Name(), res.Rows)44+ }45+ for a, want := range map[string]string{46+ "A1": "name", "B1": "size", "J1": "raw",47+ "A2": "CLAUDE.md", "B2": "1.6 kB", "C2": "9/27/2026 11:27:31", "D2": "0:01:30", "E2": "TRUE", "F2": "1.5",48+ "G2": "12", "H2": "[a, b]", "I2": "{k: 1}", "J2": "0x[DEAD]",49+ "B3": "2.0 MB", "C3": "1/2/2026 3:04:05", "D3": "0:00:01.500", "E3": "FALSE", "F3": "-3", "G3": "", "H3": "[]",50+ } {51+ if got := shown(s, addr(t, a)); got != want {52+ t.Errorf("%s shows %q, want %q", a, got, want)53+ }54+ }55+ if v := s.Value(addr(t, "G2")); v.Kind != sheet.Text {56+ t.Errorf("the string 12 became %v", v)57+ }58+ if f := s.CellFormat(addr(t, "B2")); f.Kind != sheet.FmtSize {59+ t.Errorf("size format %+v", f)60+ }61+}62+63+// TestNUONColumnsFromRecords names columns as records bring them.64+func TestNUONColumnsFromRecords(t *testing.T) {65+ s := readNUON(t, `[{a: 1}, {b: 2, a: 3}] {c: x}`).Sheet66+ for a, want := range map[string]string{"A1": "a", "B1": "b", "C1": "c", "A2": "1", "A3": "3", "B3": "2", "C4": "x"} {67+ if got := shown(s, addr(t, a)); got != want {68+ t.Errorf("%s shows %q, want %q", a, got, want)69+ }70+ }71+}72+73+func TestImportReaderSniffs(t *testing.T) {74+ for text, want := range map[string]Kind{75+ `[{"a": 1}]`: JSON,76+ "\n {\"a\": 1}\n": JSON,77+ `[{a: 1}]`: NUON,78+ `[[a]; [1]]`: NUON,79+ "a,b\n1,2\n": CSV,80+ "a\tb\n1\t2\n": TSV,81+ "\xEF\xBB\xBF[1]": JSON,82+ "": CSV,83+ "name;qty\nx;1\n": CSV,84+ } {85+ if res := readNUON(t, text); res.Kind != want {86+ t.Errorf("%q read as %v, want %v", text, res.Kind, want)87+ }88+ }89+ if res := readNUON(t, "a\tb\n1\t2\n"); len(res.Notes) != 0 {90+ t.Errorf("TSV notes %q", res.Notes)91+ }92+ if _, err := ImportReader(context.Background(), "stdin", strings.NewReader("[1, 2"), Options{}); err == nil {93+ t.Error("an unterminated list read")94+ }95+}96+97+func TestNUONMaxCells(t *testing.T) {98+ var b strings.Builder99+ b.WriteString("[[a, b];")100+ for i := range 10 {101+ b.WriteString(" [" + string(rune('0'+i)) + ", x]")102+ }103+ b.WriteString("]")104+ res, err := ImportReader(context.Background(), "stdin", strings.NewReader(b.String()), Options{MaxCells: 9})105+ if err != nil {106+ t.Fatal(err)107+ }108+ if n := res.Sheet.Len(); n != 8 {109+ t.Errorf("%d cells kept, want the header and 3 rows", n)110+ }111+ if len(res.Notes) != 1 || !strings.Contains(res.Notes[0], "only the first 4 rows fit in max-cells (9 cells); 7 rows left out") {112+ t.Errorf("notes %q", res.Notes)113+ }114+}115+116+// encode writes the whole sheet as k.117+func encode(t *testing.T, s *sheet.Sheet, k Kind) string {118+ t.Helper()119+ var b bytes.Buffer120+ if _, err := Encode(&b, k, Snap(s, sheet.Rect{}, s.Name())); err != nil {121+ t.Fatal(err)122+ }123+ return b.String()124+}125+126+func TestEncodeKeepsTypes(t *testing.T) {127+ inZone(t, mountain)128+ s := build(t, map[string]string{129+ "A1": "name", "B1": "when", "C1": "n", "D1": "ok", "E1": "", "F1": "t",130+ "A2": "a b", "B2": "9/27/2026 11:27:31", "C2": "1.5", "D2": "TRUE", "E2": "=1/0", "F2": "3:00 PM",131+ "A3": "'12", "B3": "9/27/2026", "C3": "42", "F3": "[1, 2]",132+ })133+ s.SetFormat(sheet.Rect{From: addr(t, "C3"), To: addr(t, "C3")}, sheet.Preset(sheet.FmtSize))134+ want := `[[name, when, n, ok, E, t]; ["a b", 2026-09-27T11:27:31-06:00, 1.5, true, "#DIV/0!", 54000000000000ns],135+["12", 2026-09-27T00:00:00-06:00, 42b, null, null, [1, 2]]]136+`137+ if got := encode(t, s, NUON); got != want {138+ t.Errorf("NUON:\n%s\nwant\n%s", got, want)139+ }140+ wantJSON := `[141+{"name":"a b","when":"2026-09-27T11:27:31-06:00","n":1.5,"ok":true,"E":"#DIV/0!","t":54000000000000},142+{"name":"12","when":"2026-09-27T00:00:00-06:00","n":42,"ok":null,"E":null,"t":[1,2]}143+]144+`145+ if got := encode(t, s, JSON); got != wantJSON {146+ t.Errorf("JSON:\n%s\nwant\n%s", got, wantJSON)147+ }148+ if got := encode(t, sheet.New(), NUON); got != "[]\n" {149+ t.Errorf("empty sheet %q", got)150+ }151+ if got := encode(t, s, CSV); !strings.HasPrefix(got, "name,when,n,ok,,t\n") {152+ t.Errorf("CSV %q", got)153+ }154+ if _, err := Encode(&bytes.Buffer{}, XLSX, Snap(s, sheet.Rect{}, "x")); err == nil {155+ t.Error("XLSX encoded to a stream")156+ }157+}158+159+// TestNUONRoundTrip reads a table and writes it back as the same NUON.160+func TestNUONRoundTrip(t *testing.T) {161+ inZone(t, mountain)162+ for _, text := range []string{163+ `[[name, size, modified, took, ok, n, tags, meta, raw]; ["CLAUDE.md", 1646b, 2026-09-27T11:27:31.351890-06:00, 90000000000ns, true, 1.5, [a, b], {k: 1}, 0x[DEAD]], [x, 0b, 2026-01-02T03:04:05-06:00, 1500000000ns, false, -3, [], {}, 0x[]]]`,164+ `[[a, b]; [1, null], ["x y", "true"]]`, // an empty string would come back null: a blank cell165+ } {166+ s := readNUON(t, text).Sheet167+ got, want := parse(t, encode(t, s, NUON)), parse(t, text)168+ if !nuon.Equal(got, want) {169+ t.Errorf("read\n%s\nwrote\n%s", want, got)170+ }171+ }172+}173+174+func parse(t *testing.T, text string) nuon.Value {175+ t.Helper()176+ v, err := nuon.Parse([]byte(text))177+ if err != nil {178+ t.Fatalf("%s: %v", text, err)179+ }180+ return v181+}182+183+// TestNUONThroughNu sends what nushell writes through a sheet and back184+// to nushell, which reads the same table, when nu is installed.185+func TestNUONThroughNu(t *testing.T) {186+ nu, err := exec.LookPath("nu")187+ if err != nil {188+ t.Skip("nu isn't installed")189+ }190+ inZone(t, time.Local)191+ run := func(script, in string) string {192+ cmd := exec.Command(nu, "--no-config-file", "--stdin", "-c", script)193+ cmd.Stdin = strings.NewReader(in)194+ out, err := cmd.Output()195+ if err != nil {196+ t.Fatalf("nu -c %q: %v", script, err)197+ }198+ return strings.TrimSpace(string(out))199+ }200+ table := `[[name, size, modified, took, ok, n]; [a, 1646b, 2026-09-27T11:27:31.351890163-06:00, 90sec, true, 1.5], [b, 2mb, 2026-01-02, 1.5sec, false, -3]]`201+ // Dates come back in the local zone, to the microsecond.202+ norm := ` | update modified {|r| $r.modified | date to-timezone local | format date "%+" }`203+ theirs := run(table+" | to nuon", "")204+ s := readNUON(t, theirs).Sheet205+ if got, want := run("from nuon"+norm+" | to nuon", encode(t, s, NUON)), run(table+norm+" | to nuon", ""); got != strings.ReplaceAll(want, ".351890163", ".351890") {206+ t.Errorf("nu reads\n%s\nwant\n%s", got, want)207+ }208+ if got := run("from json | length", encode(t, s, JSON)); got != "2" {209+ t.Errorf("nu reads %s JSON rows", got)210+ }211+}212+213+func TestNUONFiles(t *testing.T) {214+ dir := t.TempDir()215+ path := filepath.Join(dir, "files.nuon")216+ os.WriteFile(path, []byte(`[[name, size]; [a, 1kb]]`), 0o600)217+ res, err := Import(context.Background(), path, Options{})218+ if err != nil || res.Kind != NUON || res.Sheet.Name() != "files" || shown(res.Sheet, addr(t, "B2")) != "1.0 kB" {219+ t.Fatalf("import: %v %+v", err, res)220+ }221+ out := filepath.Join(dir, "out.json")222+ if _, err := Export(context.Background(), out, JSON, Snap(res.Sheet, sheet.Rect{}, "files"), ExportOptions{}); err != nil {223+ t.Fatal(err)224+ }225+ back, err := Import(context.Background(), out, Options{})226+ if err != nil || back.Kind != JSON || shown(back.Sheet, addr(t, "B2")) != "1000" {227+ t.Errorf("JSON back: %v, B2 %q", err, shown(back.Sheet, addr(t, "B2")))228+ }229+}230+231+// FuzzReadNUON holds the table reader to never panicking and to232+// storing only what fits.233+func FuzzReadNUON(f *testing.F) {234+ for _, s := range []string{`[[a, b]; [1, 2kb]]`, `[{a: [1 2]}, {b: 2026-09-27T11:27:31Z}]`, `{"a": 1}` + "\n" + `{"a": 2}`, "a,b\n1,2"} {235+ f.Add([]byte(s))236+ }237+ f.Fuzz(func(t *testing.T, data []byte) {238+ res, err := ImportReader(context.Background(), "stdin", bytes.NewReader(data), Options{MaxCells: 1000})239+ if err == nil && res.Sheet.Len() > 1000 {240+ t.Fatalf("%d cells", res.Sheet.Len())241+ }242+ })243+}@@ -0,0 +1,221 @@
1+package fileio2+3+import (4+ "bufio"5+ "context"6+ "io"7+ "math"8+ "strings"9+ "time"10+11+ "github.com/FelineStateMachine/012/internal/nuon"12+ "github.com/FelineStateMachine/012/internal/sheet"13+)14+15+// A table written as NUON or JSON takes its column names from the first16+// row of the range and has a record for each row after it. Cells keep17+// their types by their formats: numbers in the Size format are file18+// sizes, in the Duration format durations, in a date or date-time19+// format dates in the local time zone (a time of day alone is a20+// duration since midnight); other numbers are ints when whole and21+// floats otherwise. Text that is a list, a record or binary as NUON22+// writes it goes back as that value, so nested values read from NUON23+// return as they came.24+25+func exportNUON(_ context.Context, name string, snap *Snapshot, _ ExportOptions) (*ExportResult, error) {26+ return exportText(name, snap, encodeNUON)27+}28+29+func exportJSON(_ context.Context, name string, snap *Snapshot, _ ExportOptions) (*ExportResult, error) {30+ return exportText(name, snap, encodeJSON)31+}32+33+// exportText writes a snapshot to a file with encode.34+func exportText(name string, snap *Snapshot, encode func(io.Writer, *Snapshot) (int, error)) (*ExportResult, error) {35+ rows := 036+ err := writeFile(name, func(w io.Writer) error {37+ n, err := encode(w, snap)38+ rows = n39+ return err40+ })41+ if err != nil {42+ return nil, err43+ }44+ return &ExportResult{Rows: rows, Notes: formulaNotes(snap)}, nil45+}46+47+// formulaNotes says how many formulas were written as their values.48+func formulaNotes(snap *Snapshot) []string {49+ formulas := 050+ for _, c := range snap.Cells {51+ if c.Formula {52+ formulas++53+ }54+ }55+ if formulas == 0 {56+ return nil57+ }58+ return []string{count(formulas, "formula", "formulas") + " saved as values"}59+}60+61+// encodeNUON writes the snapshot as a NUON table, returning the rows62+// written, the header among them.63+func encodeNUON(w io.Writer, snap *Snapshot) (int, error) {64+ cols := snapColumns(snap)65+ tw := nuon.NewTableWriter(w, cols)66+ n := 167+ for row := range snapRecords(snap, len(cols)) {68+ if err := tw.Write(row); err != nil {69+ return 0, err70+ }71+ n++72+ }73+ return n, tw.Close()74+}75+76+// encodeJSON writes the snapshot as a JSON list of records, one to a77+// line.78+func encodeJSON(w io.Writer, snap *Snapshot) (int, error) {79+ bw := bufio.NewWriter(w)80+ cols := snapColumns(snap)81+ fields := make([]nuon.Field, len(cols))82+ for i, c := range cols {83+ fields[i].Key = c84+ }85+ n := 186+ var buf []byte87+ bw.WriteString("[")88+ for row := range snapRecords(snap, len(cols)) {89+ for i, v := range row {90+ fields[i].Value = v91+ }92+ buf = append(buf[:0], "\n"...)93+ if n > 1 {94+ buf = append(buf[:0], ",\n"...)95+ }96+ buf = nuon.AppendJSONRecord(buf, fields)97+ if _, err := bw.Write(buf); err != nil {98+ return 0, err99+ }100+ n++101+ }102+ if n > 1 {103+ bw.WriteString("\n")104+ }105+ bw.WriteString("]\n")106+ return n, bw.Flush()107+}108+109+// snapColumns are the names in the first row of the range, as shown; a110+// blank one is named after its column letter.111+func snapColumns(snap *Snapshot) []string {112+ r := snap.Range113+ cols := make([]string, r.To.Col-r.From.Col+1)114+ for col := r.From.Col; col <= r.To.Col; col++ {115+ name := ""116+ if c, ok := snap.Cells[sheet.Addr{Col: col, Row: r.From.Row}]; ok {117+ name = strings.TrimSpace(c.Text())118+ }119+ if name == "" {120+ name = sheet.ColName(col)121+ }122+ cols[col-r.From.Col] = name123+ }124+ return cols125+}126+127+// snapRecords yields the values of each row after the first. The slice128+// is reused from row to row.129+func snapRecords(snap *Snapshot, n int) func(yield func([]nuon.Value) bool) {130+ return func(yield func([]nuon.Value) bool) {131+ r, loc := snap.Range, zone()132+ row := make([]nuon.Value, n)133+ for y := r.From.Row + 1; y <= r.To.Row; y++ {134+ for x := r.From.Col; x <= r.To.Col; x++ {135+ c, ok := snap.Cells[sheet.Addr{Col: x, Row: y}]136+ row[x-r.From.Col] = nuon.NullValue()137+ if ok {138+ row[x-r.From.Col] = nuonValue(c, loc)139+ }140+ }141+ if !yield(row) {142+ return143+ }144+ }145+ }146+}147+148+// nuonValue is a cell as a NUON value, typed by its format.149+func nuonValue(c SnapCell, loc *time.Location) nuon.Value {150+ switch v := c.Value; v.Kind {151+ case sheet.Empty:152+ return nuon.NullValue()153+ case sheet.Bool:154+ return nuon.BoolValue(v.Num != 0)155+ case sheet.Text:156+ return textValue(v.Str)157+ case sheet.Number:158+ return numberValue(v.Num, c.Format, loc)159+ }160+ return nuon.StringValue(c.Value.String()) // an error, as it shows161+}162+163+// textValue is text as a string, or as the list, record or binary it164+// is written as.165+func textValue(s string) nuon.Value {166+ if strings.HasPrefix(s, "[") || strings.HasPrefix(s, "{") || strings.HasPrefix(s, "0x[") {167+ v, err := nuon.Parse([]byte(s))168+ if err == nil && (v.Kind == nuon.List || v.Kind == nuon.Record || v.Kind == nuon.Binary) && v.String() == s {169+ return v170+ }171+ }172+ return nuon.StringValue(s)173+}174+175+// maxExact is the largest magnitude below which every whole float64 is176+// exact, so it's written as an int.177+const maxExact = 1 << 53178+179+func numberValue(n float64, f sheet.Format, loc *time.Location) nuon.Value {180+ switch f.Kind {181+ case sheet.FmtSize:182+ if b, ok := wholeInt(n); ok {183+ return nuon.FilesizeValue(b)184+ }185+ case sheet.FmtDuration:186+ if ns, ok := wholeInt(n * nsPerDay); ok {187+ return nuon.DurationValue(time.Duration(ns))188+ }189+ case sheet.FmtTime:190+ if ns, ok := wholeInt(n * nsPerDay); ok && n >= 0 && n < 1 {191+ return nuon.DurationValue(time.Duration(ns))192+ }193+ return nuon.DateValue(serialTime(n, loc))194+ case sheet.FmtDate, sheet.FmtDateTime:195+ return nuon.DateValue(serialTime(n, loc))196+ }197+ if n == math.Trunc(n) && math.Abs(n) < maxExact {198+ return nuon.IntValue(int64(n))199+ }200+ return nuon.FloatValue(n)201+}202+203+// wholeInt rounds f to an int64, reporting false when it doesn't fit.204+func wholeInt(f float64) (int64, bool) {205+ f = math.Round(f)206+ if math.IsNaN(f) || f >= math.MaxInt64 || f <= math.MinInt64 {207+ return 0, false208+ }209+ return int64(f), true210+}211+212+// serialTime is the time a date serial stands for in loc, to the213+// microsecond: about as fine as a serial's float holds a time today.214+func serialTime(v float64, loc *time.Location) time.Time {215+ days := math.Floor(v)216+ us := math.Round((v - days) * 86400e6)217+ if math.Abs(days) > 1e7 {218+ days = math.Copysign(1e7, days) // past the year 29,000: as far as a sheet shows219+ }220+ return time.Date(1899, 12, 30+int(days), 0, 0, 0, int(us)*1e3, loc)221+}@@ -0,0 +1,108 @@
1+package fileio2+3+import (4+ "bufio"5+ "context"6+ "fmt"7+ "io"8+ "log/slog"9+ "slices"10+11+ "github.com/FelineStateMachine/012/internal/telemetry"12+)13+14+// Streams: 012 - reads a table from standard input, and 012 --pipe15+// writes one to standard output. A stream has no name to tell its16+// format by, so ImportReader looks at the text: NUON (or JSON, which is17+// NUON too) starts with [ or {, and anything else is read as delimited18+// text, CSV or TSV by the delimiter found.19+20+// ImportReader reads a table from r into a new sheet named name (as a21+// file's sheet is named after the file), telling its format from the22+// text; Result.Kind is NUON, JSON, CSV or TSV. It checks ctx between23+// rows and updates opt.Progress.24+func ImportReader(ctx context.Context, name string, r io.Reader, opt Options) (*Result, error) {25+ span := telemetry.ParentFrom(ctx).Start("import", slog.String("format", "stream"))26+ res, err := importReader(telemetry.WithParent(ctx, span.Parent()), r, opt)27+ if err != nil {28+ span.Fail(err)29+ return nil, err30+ }31+ if book := res.Sheet.Book(); book.RenameSheet(res.Sheet, sheetName(name)) == nil {32+ book.ClearHistory()33+ }34+ if p := opt.Progress; p != nil {35+ p.permille.Store(1000)36+ }37+ span.End(slog.String("kind", res.Kind.String()), slog.Int("rows", res.Rows))38+ return res, nil39+}40+41+func importReader(ctx context.Context, r io.Reader, opt Options) (*Result, error) {42+ defer context.AfterFunc(ctx, func() {43+ if c, ok := r.(io.Closer); ok {44+ c.Close() // unblocks a read from a pipe45+ }46+ })()47+ br := bufio.NewReaderSize(r, sniffSize)48+ progress := func(rows int) { opt.Progress.setRows(rows) }49+ switch first, err := firstByte(br); {50+ case err != nil:51+ return nil, err52+ case first == '[' || first == '{':53+ res, json, err := readTable(ctx, br, opt.MaxCells, progress)54+ if err != nil {55+ return nil, err56+ }57+ res.Kind = NUON58+ if json {59+ res.Kind = JSON60+ }61+ return res, nil62+ }63+ s, rows, notes, comma, err := readDelimitedAs(ctx, br, CSV, opt.MaxCells, opt.Locale, progress)64+ if err != nil {65+ return nil, err66+ }67+ res := &Result{Sheet: s, Kind: CSV, Rows: rows, Notes: notes}68+ if comma == '\t' {69+ res.Kind = TSV70+ res.Notes = slices.DeleteFunc(notes, func(n string) bool { return n == "separated by tabs" }) // it's TSV71+ }72+ return res, nil73+}74+75+// firstByte is the first byte of the text past a byte order mark and76+// spaces, without taking it; 0 for text of nothing else.77+func firstByte(br *bufio.Reader) (byte, error) {78+ for n := 1; n <= sniffSize; n++ {79+ b, err := br.Peek(n)80+ if len(b) < n {81+ if err == io.EOF {82+ err = nil83+ }84+ return 0, err85+ }86+ switch c := b[n-1]; {87+ case c == ' ' || c == '\t' || c == '\n' || c == '\r':88+ case n <= 3 && string(b[:n]) == "\xEF\xBB\xBF"[:n]:89+ default:90+ return c, nil91+ }92+ }93+ return 0, nil94+}95+96+// Encode writes a snapshot to w in the text format k (see Kind.IsText):97+// what 012 --pipe writes to standard output.98+func Encode(w io.Writer, k Kind, snap *Snapshot) (*ExportResult, error) {99+ encode := k.format().encode100+ if encode == nil {101+ return nil, fmt.Errorf("can't write %s to a stream", k)102+ }103+ rows, err := encode(w, snap)104+ if err != nil {105+ return nil, err106+ }107+ return &ExportResult{Rows: rows, Notes: formulaNotes(snap)}, nil108+}