Delimited files are read in the importing sheet's locale. The sniffer gives ; the tie where the locale has a decimal comma, and a file whose numbers use the other decimal separator (at least two fields, and twice as many as the other way) is read that way, keeping the locale's date order, with a note. CSV downloads show values in the locale, with ; between fields where it has a decimal comma.
- commit
- aeaf904123e84332564b1ccd88ce720aafea0a25
- parent
- 4ce697c42f461345dec1afc6dcececf72a601c4d
- author
- Dami <dami.etoile@gmail.com>
| M | docs/ | 2 | +- |
| M | internal/ | 23 | ++++-- |
| M | internal/ | 40 | ++++++++-- |
| M | internal/ | 10 | +- |
| A | internal/ | 87 | +++++++++++++++++++++++ |
| A | internal/ | 116 | ++++++++++++++++++++++++++++++ |
| M | internal/ | 4 | + |
| M | internal/ | 10 | ++- |
@@ -20,7 +20,7 @@ it back in its format.
2020 2121 | Format | Import | Download |2222 |---|---|---|23-| CSV, TSV | Delimiter (`,` `;` tab `\|`), UTF-8 BOM, UTF-16 and Windows-1252 detected; entries become numbers, dates, currency and percentages as if typed; formulas stay text | Values as shown, of the sheet shown, as Sheets' Download does; a selection of whole columns or rows downloads their data, not a million blank lines |23+| CSV, TSV | Delimiter (`,` `;` tab `\|`), UTF-8 BOM, UTF-16 and Windows-1252 detected; entries become numbers, dates, currency and percentages as if typed in the sheet's [locale](../sheets/locale.md) (`;` wins a tie there when it has a decimal comma), or with the other decimal separator when the file's numbers are written that way; formulas stay text | Values as shown in the locale, of the sheet shown, as Sheets' Download does, CSV with `;` between fields where the locale has a decimal comma; a selection of whole columns or rows downloads their data, not a million blank lines |2424 | Excel `.xlsx` | Every sheet, with formulas, formats and most of what a sheet holds: see [Excel files](excel.md) | The same |2525 | SQLite | Pick a table or view, or type a query; a header row names the columns | The sheet shown or the selection as a table, first row as column names; a table of that name is replaced |2626 | Parquet | Every column, with dates and timestamps; lists joined with commas | |@@ -7,6 +7,7 @@ import (
77 "strconv"88 "strings"99 10+ "github.com/FelineStateMachine/012/internal/locale"1011 "github.com/FelineStateMachine/012/internal/sheet"1112 "github.com/FelineStateMachine/012/internal/telemetry"1213 )@@ -17,7 +18,8 @@ import (
1718 // back to a formula's cached value when the formula can't be translated,1819 // and counts what didn't fit so the notes can say.1920 type builder struct {20- s *sheet.Sheet21+ s *sheet.Sheet22+ loc *locale.Locale // what entries are typed in (nil for en-US)2123 2224 rowsOut, colsOut int // how far past the sheet's edges the data went2325 values int // formulas kept as their values@@ -129,16 +131,19 @@ func textInput(s string) string {
129131 return s130132 }131133 132-// entryInput is s as if typed into Sheets, for text formats: numbers,133-// dates, currency and percentages become values with formats. Formulas134-// stay text: a data file shouldn't run formulas (or ask JEV questions)135-// just by being opened.136-func entryInput(s string) string {134+// entryInput is s as if typed into Sheets in loc, for text formats:135+// numbers, dates, currency and percentages become values with formats.136+// Formulas stay text: a data file shouldn't run formulas (or ask JEV137+// questions) just by being opened.138+func entryInput(s string, loc *locale.Locale) string {137139 s = flatten(s)138140 if strings.HasPrefix(s, "'") || sheet.IsFormulaEntry(s) {139141 return "'" + s140142 }141- return s143+ if in := sheet.CanonicalEntry(s, loc); !sheet.IsFormulaEntry(in) {144+ return in145+ }146+ return "'" + s // a formula in loc's syntax142147 }143148 144149 // numInput writes v so it parses back to the same number.@@ -164,9 +169,9 @@ func (b *builder) text(a sheet.Addr, s string, f sheet.Format, st sheet.Style) {
164169 b.put(a, textInput(s), f, st)165170 }166171 167-// entry stores s as if typed (see entryInput).172+// entry stores s as if typed in the builder's locale (see entryInput).168173 func (b *builder) entry(a sheet.Addr, s string) {169- b.put(a, entryInput(s), sheet.Format{}, sheet.Style{})174+ b.put(a, entryInput(s, b.loc), sheet.Format{}, sheet.Style{})170175 }171176 172177 // number stores v with format f. Plain text formats keep the digits.@@ -14,6 +14,7 @@ import (
1414 "golang.org/x/text/encoding/charmap"1515 "golang.org/x/text/encoding/unicode"1616 17+ "github.com/FelineStateMachine/012/internal/locale"1718 "github.com/FelineStateMachine/012/internal/sheet"1819 )1920 @@ -32,10 +33,13 @@ type Dialect struct {
3233 3334 // sniff picks the delimiter that splits the sample's records into the3435 // same number of fields most consistently, preferring more fields.35-// Quoted fields may contain delimiters and line breaks.36-func sniff(sample []byte, truncated bool) rune {37- best, bestScore := ',', 0.038- for _, d := range delimiters {36+// Quoted fields may contain delimiters and line breaks. Ties go to37+// first, the locale's own separator (see csvComma), then to delimiters'38+// order.39+func sniff(sample []byte, truncated bool, first rune) rune {40+ best, bestScore := first, 0.041+ order := append([]rune{first}, delimiters...)42+ for _, d := range order {3943 counts := fieldCounts(sample, byte(d), truncated)4044 if len(counts) == 0 {4145 continue@@ -54,6 +58,11 @@ func sniff(sample []byte, truncated bool) rune {
5458 continue5559 }5660 score := float64(freq[mode]) / float64(len(counts)) * (1 + float64(min(mode, 20))/100)61+ if d == first && first != ',' {62+ // 1,5;2,5 splits evenly on both in a locale with decimal63+ // commas; its own separator wins.64+ score *= 1.2565+ }5766 if score > bestScore {5867 best, bestScore = d, score5968 }@@ -166,7 +175,7 @@ func importDelimited(ctx context.Context, name string, k Kind, opt Options) (*Re
166175 size = st.Size()167176 }168177 counter := &countingReader{r: f}169- s, rows, notes, err := readDelimited(ctx, counter, k, opt.MaxCells, func(rows int) {178+ s, rows, notes, err := readDelimited(ctx, counter, k, opt.MaxCells, opt.Locale, func(rows int) {170179 prog.setRows(rows)171180 prog.setFrac(counter.n, size)172181 })@@ -178,8 +187,10 @@ func importDelimited(ctx context.Context, name string, k Kind, opt Options) (*Re
178187 179188 // readDelimited reads CSV or TSV text, sniffing the delimiter of CSV,180189 // keeping up to maxCells cells (see newBuilder). Each field is entered as181-// if typed. progress is called every few hundred rows.182-func readDelimited(ctx context.Context, in io.Reader, k Kind, maxCells int, progress func(rows int)) (*sheet.Sheet, int, []string, error) {190+// if typed in loc, or with the other decimal separator when the file191+// writes its numbers that way (see numberLocale). progress is called192+// every few hundred rows.193+func readDelimited(ctx context.Context, in io.Reader, k Kind, maxCells int, loc *locale.Locale, progress func(rows int)) (*sheet.Sheet, int, []string, error) {183194 br := bufio.NewReaderSize(in, sniffSize)184195 text, enc, err := decode(br)185196 if err != nil {@@ -189,7 +200,7 @@ func readDelimited(ctx context.Context, in io.Reader, k Kind, maxCells int, prog
189200 sample, _ := tr.Peek(sniffSize)190201 comma := '\t'191202 if k == CSV {192- comma = sniff(sample, len(sample) == sniffSize)203+ comma = sniff(sample, len(sample) == sniffSize, csvComma(loc))193204 }194205 cr := csv.NewReader(tr)195206 cr.Comma = comma@@ -198,6 +209,8 @@ func readDelimited(ctx context.Context, in io.Reader, k Kind, maxCells int, prog
198209 cr.ReuseRecord = true199210 200211 b := newBuilder(ctx, maxCells)212+ var otherDecimal bool213+ b.loc, otherDecimal = numberLocale(sample, comma, loc)201214 row := 0202215 for ; ; row++ {203216 // Report before reading on: a pipe may keep the next read waiting.@@ -229,6 +242,9 @@ func readDelimited(ctx context.Context, in io.Reader, k Kind, maxCells int, prog
229242 if k == CSV && comma != ',' {230243 notes = append(notes, "separated by "+delimiterName(comma))231244 }245+ if otherDecimal {246+ notes = append(notes, decimalNote(b.loc))247+ }232248 if enc != "UTF-8" {233249 notes = append(notes, "read as "+enc)234250 }@@ -249,12 +265,15 @@ func delimiterName(r rune) string {
249265 }250266 251267 // exportDelimited writes the snapshot's displayed values, as Sheets'252-// Download as CSV does: formulas become their results, formats show.268+// Download as CSV does: formulas become their results, formats show, in269+// the snapshot's locale, with ; between fields where its decimal270+// separator is a comma.253271 // Rows are written as they are formatted, never all held at once.254272 func exportDelimited(name string, k Kind, snap *Snapshot) (*ExportResult, error) {255273 rows := 0256274 err := writeFile(name, func(out io.Writer) error {257275 w := csv.NewWriter(out)276+ w.Comma = csvComma(snap.Locale)258277 if k == TSV {259278 w.Comma = '\t'260279 }@@ -277,6 +296,9 @@ func exportDelimited(name string, k Kind, snap *Snapshot) (*ExportResult, error)
277296 }278297 }279298 res := &ExportResult{Rows: rows}299+ if k == CSV && csvComma(snap.Locale) != ',' {300+ res.Notes = append(res.Notes, "separated by semicolons")301+ }280302 if formulas > 0 {281303 res.Notes = append(res.Notes, count(formulas, "formula", "formulas")+" saved as values")282304 }@@ -68,7 +68,7 @@ func TestSniff(t *testing.T) {
6868 {"empty", "", ','},6969 } {7070 t.Run(tc.name, func(t *testing.T) {71- if got := sniff([]byte(tc.text), false); got != tc.want {71+ if got := sniff([]byte(tc.text), false, ','); got != tc.want {7272 t.Errorf("sniff = %q, want %q", got, tc.want)7373 }7474 })@@ -77,7 +77,7 @@ func TestSniff(t *testing.T) {
7777 7878 func readText(t *testing.T, k Kind, text string) (*sheet.Sheet, []string) {7979 t.Helper()80- s, _, notes, err := readDelimited(context.Background(), strings.NewReader(text), k, 0, func(int) {})80+ s, _, notes, err := readDelimited(context.Background(), strings.NewReader(text), k, 0, nil, func(int) {})8181 if err != nil {8282 t.Fatal(err)8383 }@@ -188,7 +188,7 @@ func TestReadDelimitedTruncates(t *testing.T) {
188188 for range sheet.MaxRows + 4 {189189 b.WriteString("x\n")190190 }191- s, _, notes, err := readDelimited(context.Background(), strings.NewReader(b.String()), CSV, sheet.MaxRows+sheet.MaxCols, func(int) {})191+ s, _, notes, err := readDelimited(context.Background(), strings.NewReader(b.String()), CSV, sheet.MaxRows+sheet.MaxCols, nil, func(int) {})192192 if err != nil {193193 t.Fatal(err)194194 }@@ -206,7 +206,7 @@ func TestReadDelimitedTruncates(t *testing.T) {
206206 for range 100 {207207 b.WriteString(strings.Repeat("7,", 9) + "7\n")208208 }209- s, _, notes, err = readDelimited(context.Background(), strings.NewReader(b.String()), CSV, 255, func(int) {})209+ s, _, notes, err = readDelimited(context.Background(), strings.NewReader(b.String()), CSV, 255, nil, func(int) {})210210 if err != nil {211211 t.Fatal(err)212212 }@@ -280,6 +280,6 @@ func FuzzReadDelimited(f *testing.F) {
280280 if tsv {281281 k = TSV282282 }283- readDelimited(context.Background(), strings.NewReader(string(data)), k, 0, func(int) {})283+ readDelimited(context.Background(), strings.NewReader(string(data)), k, 0, nil, func(int) {})284284 })285285 }@@ -0,0 +1,87 @@
1+package fileio2+3+import (4+ "bytes"5+ "encoding/csv"6+7+ "github.com/FelineStateMachine/012/internal/locale"8+ "github.com/FelineStateMachine/012/internal/value"9+)10+11+// Delimited files are read and written in the importing workbook's12+// locale, as Sheets does: 1,5 is a number in a CSV file read in de-DE.13+// A file whose numbers are written with the other decimal separator is14+// noticed from its sample and read that way, keeping the locale's date15+// order and currency; and in a locale with a decimal comma, CSV files16+// are written with ; between fields, as its spreadsheets write them.17+18+// numberSample is how many fields of the sample decide how numbers are19+// written.20+const numberSample = 200021+22+// numberLocale is the locale a delimited file's fields are read in: loc,23+// or loc with the other decimal separator (1.5 and 1,5) when at least24+// two of the sample's fields, and more than twice as many as the other25+// way round, read as numbers only that way (a lone 1,5 may be text).26+// other reports the second.27+func numberLocale(sample []byte, comma rune, loc *locale.Locale) (read *locale.Locale, other bool) {28+ loc = loc.Or()29+ alt := swapDecimal(loc)30+ r := csv.NewReader(bytes.NewReader(sample))31+ r.Comma, r.LazyQuotes, r.FieldsPerRecord, r.ReuseRecord = comma, true, -1, true32+ mine, theirs := 0, 033+ for seen := 0; seen < numberSample; {34+ rec, err := r.Read()35+ if err != nil {36+ break37+ }38+ for _, f := range rec {39+ seen++40+ _, _, inMine := value.ParseValueIn(f, loc)41+ _, _, inTheirs := value.ParseValueIn(f, alt)42+ switch {43+ case inMine && !inTheirs:44+ mine++45+ case inTheirs && !inMine:46+ theirs++47+ }48+ }49+ }50+ if theirs >= 2 && theirs > 2*mine {51+ return alt, true52+ }53+ return loc, false54+}55+56+// swapDecimal is loc with the other decimal separator, grouping with the57+// one it gave up: a comma for a point, a point for a comma.58+func swapDecimal(loc *locale.Locale) *locale.Locale {59+ alt := *loc60+ if loc.Decimal == ',' {61+ alt.Decimal, alt.Group = '.', ","62+ } else {63+ alt.Decimal, alt.Group = ',', "."64+ if alt.DateSep == '.' {65+ alt.DateSep = '/'66+ }67+ }68+ return &alt69+}70+71+// csvComma separates the fields of a CSV file written in loc: ; where72+// the decimal separator is a comma.73+func csvComma(loc *locale.Locale) rune {74+ if loc.Or().Decimal == ',' {75+ return ';'76+ }77+ return ','78+}79+80+// decimalNote says how a file's numbers were read when not in the81+// locale's own way.82+func decimalNote(loc *locale.Locale) string {83+ if loc.Decimal == ',' {84+ return "numbers with decimal commas"85+ }86+ return "numbers with decimal points"87+}@@ -0,0 +1,116 @@
1+package fileio2+3+import (4+ "context"5+ "os"6+ "path/filepath"7+ "slices"8+ "strings"9+ "testing"10+11+ "github.com/FelineStateMachine/012/internal/locale"12+ "github.com/FelineStateMachine/012/internal/numfmt"13+ "github.com/FelineStateMachine/012/internal/sheet"14+)15+16+func mustLocale(t *testing.T, tag string) *locale.Locale {17+ t.Helper()18+ l, ok := locale.Lookup(tag)19+ if !ok {20+ t.Fatalf("no locale %s", tag)21+ }22+ return l23+}24+25+// TestReadDelimitedInLocales reads CSV files in a locale: its own26+// separators and date order, or the other decimal separator when the27+// file writes numbers that way.28+func TestReadDelimitedInLocales(t *testing.T) {29+ date := numfmt.DateSerial(2026, 9, 26)30+ cases := []struct {31+ name, tag, text string32+ values map[string]float6433+ texts map[string]string34+ note string35+ }{36+ {"en-US reads decimal commas", "en-US", "name;price\napple;1,5\npear;2,25\n",37+ map[string]float64{"B2": 1.5, "B3": 2.25}, nil, "numbers with decimal commas"},38+ {"de-DE reads its own", "de-DE", "a;b;c\n1.234,5;26.09.2026;12,50 €\n",39+ map[string]float64{"A2": 1234.5, "B2": date, "C2": 12.5}, nil, ""},40+ {"de-DE ties go to semicolons", "de-DE", "1,5;2,5\n3,5;4,5\n",41+ map[string]float64{"A1": 1.5, "B2": 4.5}, nil, ""},42+ {"de-DE reads decimal points", "de-DE", "a,b\n1.25,2.75\n3.5,4.125\n",43+ map[string]float64{"A2": 1.25, "B2": 2.75, "B3": 4.125}, nil, "numbers with decimal points"},44+ {"en-GB reads its date order", "en-GB", "d\n26/09/2026\n",45+ map[string]float64{"A2": date}, nil, ""},46+ {"de-DE keeps its formulas text", "de-DE", "=SUM(1;2)\t-A1*0,5\n",47+ nil, map[string]string{"A1": "=SUM(1;2)", "B1": "-A1*0,5"}, ""},48+ }49+ for _, c := range cases {50+ t.Run(c.name, func(t *testing.T) {51+ k := CSV52+ if strings.Contains(c.text, "\t") {53+ k = TSV54+ }55+ s, _, notes, err := readDelimited(context.Background(), strings.NewReader(c.text), k, 0, mustLocale(t, c.tag), func(int) {})56+ if err != nil {57+ t.Fatal(err)58+ }59+ want := map[string]sheet.Value{}60+ for a, n := range c.values {61+ want[a] = sheet.Value{Kind: sheet.Number, Num: n}62+ }63+ for a, s := range c.texts {64+ want[a] = sheet.Value{Kind: sheet.Text, Str: s}65+ }66+ for a, w := range want {67+ if v := s.Value(addr(t, a)); v != w {68+ t.Errorf("%s = %v, want %v", a, v, w)69+ }70+ }71+ if got := numbersNote(notes); got != c.note {72+ t.Errorf("notes %q, want %q", notes, c.note)73+ }74+ })75+ }76+}77+78+// numbersNote is the note saying how numbers were read, or "".79+func numbersNote(notes []string) string {80+ for _, n := range notes {81+ if strings.HasPrefix(n, "numbers") {82+ return n83+ }84+ }85+ return ""86+}87+88+// TestExportDelimitedInLocale writes CSV as a locale shows it, with ;89+// between fields where the decimal separator is a comma.90+func TestExportDelimitedInLocale(t *testing.T) {91+ src := build(t, map[string]string{"A1": "1234.5", "B1": "9/26/2026", "C1": "Tee, schwarz", "A2": "=A1/4"})92+ src.SetFormat(sheet.NewRect(addr(t, "A1"), addr(t, "A1")), sheet.Preset(sheet.FmtCurrency))93+ src.Book().SetLocale("de-DE")94+ name := filepath.Join(t.TempDir(), "x.csv")95+ res, err := Export(context.Background(), name, CSV, Snap(src, sheet.Rect{}, "x"), ExportOptions{})96+ if err != nil {97+ t.Fatal(err)98+ }99+ data, _ := os.ReadFile(name)100+ if want := "1.234,50 €;26.9.2026;Tee, schwarz\n308,63\u00a0€;;\n"; string(data) != want {101+ t.Errorf("wrote %q, want %q", data, want)102+ }103+ if !slices.Contains(res.Notes, "separated by semicolons") {104+ t.Errorf("notes %q", res.Notes)105+ }106+ // Read back in de-DE, the values are the same.107+ back, _, _, err := readDelimited(context.Background(), strings.NewReader(string(data)), CSV, 0, mustLocale(t, "de-DE"), func(int) {})108+ if err != nil {109+ t.Fatal(err)110+ }111+ for _, a := range []string{"A1", "B1"} {112+ if got, want := back.Value(addr(t, a)), src.Value(addr(t, a)); got != want {113+ t.Errorf("%s read back as %v, want %v", a, got, want)114+ }115+ }116+}@@ -18,6 +18,7 @@ import (
1818 "strings"1919 "sync/atomic"2020 21+ "github.com/FelineStateMachine/012/internal/locale"2122 "github.com/FelineStateMachine/012/internal/sheet"2223 "github.com/FelineStateMachine/012/internal/telemetry"2324 )@@ -79,6 +80,9 @@ type Options struct {
7980 // MaxCells is the most cells to keep, whole rows at a time; 0 is the8081 // max-cells setting (sheet.MaxCells). WK1 files keep their own limits.8182 MaxCells int83+ // Locale is what CSV and TSV fields are read in (see numberLocale);84+ // nil is en-US.85+ Locale *locale.Locale8286 }8387 8488 // Result is an imported sheet and what the import had to leave out or@@ -4,6 +4,7 @@ import (
44 "iter"55 "strconv"66 7+ "github.com/FelineStateMachine/012/internal/locale"78 "github.com/FelineStateMachine/012/internal/sheet"89 )910 @@ -17,6 +18,8 @@ type Snapshot struct {
1718 Cells map[sheet.Addr]SnapCell1819 Widths map[int]int // non-default column widths1920 Name string // what to call the data: a sheet or table name21+ // Locale is the sheet's, which text formats (CSV) are written in.22+ Locale *locale.Locale2023 2124 // ColFormats and RowFormats are the formats of whole columns and2225 // rows, for formats that keep them (XLSX).@@ -87,7 +90,7 @@ func Snap(s *sheet.Sheet, r sheet.Rect, name string) *Snapshot {
8790 } else {8891 r.To = r.From8992 }90- snap := &Snapshot{Range: r, Cells: map[sheet.Addr]SnapCell{}, Widths: s.Widths(), Name: name,93+ snap := &Snapshot{Range: r, Cells: map[sheet.Addr]SnapCell{}, Widths: s.Widths(), Name: name, Locale: s.Locale(),9194 ColFormats: snapLines(s, false), RowFormats: snapLines(s, true), Filter: s.Filter(),9295 CondFormats: s.CondFormats(), Validations: s.Validations()}9396 snap.FrozenRows, snap.FrozenCols = s.Frozen()@@ -166,7 +169,8 @@ func excelRange(ws string, r sheet.Rect) string {
166169 }167170 168171 // textRows yields the snapshot's displayed text row by row across its169-// range, for text formats. The slice is reused from row to row.172+// range as shown in its locale, for text formats. The slice is reused173+// from row to row.170174 func (s *Snapshot) textRows() iter.Seq[[]string] {171175 return func(yield func([]string) bool) {172176 r := s.Range@@ -175,7 +179,7 @@ func (s *Snapshot) textRows() iter.Seq[[]string] {
175179 for col := r.From.Col; col <= r.To.Col; col++ {176180 line[col-r.From.Col] = ""177181 if c, ok := s.Cells[sheet.Addr{Col: col, Row: row}]; ok {178- line[col-r.From.Col] = c.Text()182+ line[col-r.From.Col] = sheet.FormatTextIn(c.Value, c.Format, s.Locale.Or())179183 }180184 }181185 if !yield(line) {