1package fileio23import (4 "bufio"5 "bytes"6 "context"7 "encoding/csv"8 "errors"9 "fmt"10 "io"11 "os"12 "unicode/utf8"1314 "golang.org/x/text/encoding/charmap"15 "golang.org/x/text/encoding/unicode"1617 "src.rest/cat/012/internal/locale"18 "src.rest/cat/012/internal/sheet"19)2021// sniffSize is how much of a file the dialect sniffer looks at.22const sniffSize = 64 << 102324// delimiters are the separators the sniffer considers, in order of25// preference when they score the same.26var delimiters = []rune{',', '\t', ';', '|'}2728// Dialect is how a delimited file is written.29type Dialect struct {30 Comma rune31 Encoding string // "UTF-8", "UTF-16" or "Windows-1252"32}3334// sniff picks the delimiter that splits the sample's records into the35// same number of fields most consistently, preferring more fields.36// Quoted fields may contain delimiters and line breaks. Ties go to37// first, the locale's own separator (see csvComma), then to delimiters'38// order.39func sniff(sample []byte, truncated bool, first rune) rune {40 best, bestScore := first, 0.041 order := append([]rune{first}, delimiters...)42 for _, d := range order {43 counts := fieldCounts(sample, byte(d), truncated)44 if len(counts) == 0 {45 continue46 }47 // The most common count of delimiters per record, and how many48 // records have it.49 freq := map[int]int{}50 mode := 051 for _, n := range counts {52 freq[n]++53 if freq[n] > freq[mode] || freq[n] == freq[mode] && n > mode {54 mode = n55 }56 }57 if mode == 0 {58 continue59 }60 score := float64(freq[mode]) / float64(len(counts)) * (1 + float64(min(mode, 20))/100)61 if d == first && first != ',' {62 // 1,5;2,5 splits evenly on both in a locale with decimal63 // commas; its own separator wins.64 score *= 1.2565 }66 if score > bestScore {67 best, bestScore = d, score68 }69 }70 return best71}7273// fieldCounts counts delimiters outside quotes in each of the first74// records of sample, skipping blank lines and a record cut off at the75// end of a truncated sample.76func fieldCounts(sample []byte, d byte, truncated bool) []int {77 var counts []int78 n, inQuote, blank := 0, false, true79 for _, c := range sample {80 switch {81 case c == '"':82 inQuote = !inQuote83 blank = false84 case inQuote:85 case c == d:86 n++87 blank = false88 case c == '\n':89 if !blank {90 counts = append(counts, n)91 }92 n, blank = 0, true93 if len(counts) == 50 {94 return counts95 }96 case c != '\r':97 blank = false98 }99 }100 if !blank && !truncated {101 counts = append(counts, n)102 }103 return counts104}105106// decode returns a reader of f's text as UTF-8 and the encoding found107// (see encodingOf).108func decode(r *bufio.Reader) (io.Reader, string, error) {109 head, err := r.Peek(sniffSize)110 if err != nil && !errors.Is(err, io.EOF) && !errors.Is(err, bufio.ErrBufferFull) {111 return nil, "", err112 }113 enc, bom := encodingOf(head)114 r.Discard(bom)115 return decodeAs(r, enc), enc, nil116}117118// Encodings of text files, as notes name them.119const (120 encUTF8 = "UTF-8"121 encUTF16LE = "UTF-16"122 encUTF16BE = "UTF-16 big-endian"123 encWin1252 = "Windows-1252"124)125126// encodingOf tells the encoding of text starting with head, and the127// length of its byte order mark: a UTF-8 or UTF-16 mark decides, and128// text that isn't valid UTF-8 is read as Windows-1252, what Excel129// writes on Windows.130func encodingOf(head []byte) (enc string, bom int) {131 switch {132 case bytes.HasPrefix(head, []byte{0xEF, 0xBB, 0xBF}):133 return encUTF8, 3134 case bytes.HasPrefix(head, []byte{0xFF, 0xFE}):135 return encUTF16LE, 2136 case bytes.HasPrefix(head, []byte{0xFE, 0xFF}):137 return encUTF16BE, 2138 }139 // A sample cut mid-character is still UTF-8.140 check := head141 for i := 0; i < utf8.UTFMax && len(check) > 0 && !utf8.Valid(check); i++ {142 check = check[:len(check)-1]143 }144 if len(head) > 0 && !utf8.Valid(check) {145 return encWin1252, 0146 }147 return encUTF8, 0148}149150// decodeAs reads r, text in enc past its byte order mark, as UTF-8.151func decodeAs(r io.Reader, enc string) io.Reader {152 switch enc {153 case encUTF16LE:154 return unicode.UTF16(unicode.LittleEndian, unicode.IgnoreBOM).NewDecoder().Reader(r)155 case encUTF16BE:156 return unicode.UTF16(unicode.BigEndian, unicode.IgnoreBOM).NewDecoder().Reader(r)157 case encWin1252:158 return charmap.Windows1252.NewDecoder().Reader(r)159 }160 return r161}162163// countingReader counts the bytes read through it.164type countingReader struct {165 r io.Reader166 n int64167}168169func (c *countingReader) Read(p []byte) (int, error) {170 n, err := c.r.Read(p)171 c.n += int64(n)172 return n, err173}174175func importCSV(ctx context.Context, name string, opt Options) (*Result, error) {176 return importDelimited(ctx, name, CSV, opt)177}178179func importTSV(ctx context.Context, name string, opt Options) (*Result, error) {180 return importDelimited(ctx, name, TSV, opt)181}182183func exportCSV(_ context.Context, name string, snap *Snapshot, _ ExportOptions) (*ExportResult, error) {184 return exportDelimited(name, CSV, snap)185}186187func exportTSV(_ context.Context, name string, snap *Snapshot, _ ExportOptions) (*ExportResult, error) {188 return exportDelimited(name, TSV, snap)189}190191func encodeCSV(w io.Writer, snap *Snapshot) (int, error) { return encodeDelimited(w, CSV, snap) }192193func encodeTSV(w io.Writer, snap *Snapshot) (int, error) { return encodeDelimited(w, TSV, snap) }194195func importDelimited(ctx context.Context, name string, k Kind, opt Options) (*Result, error) {196 prog := opt.Progress197 f, err := os.Open(name)198 if err != nil {199 return nil, err200 }201 defer f.Close()202 defer context.AfterFunc(ctx, func() { f.Close() })() // unblocks a read from a pipe203 var size int64204 if st, err := f.Stat(); err == nil && st.Mode().IsRegular() {205 size = st.Size()206 }207 counter := &countingReader{r: f}208 s, rows, notes, err := readDelimited(ctx, counter, k, opt.MaxCells, opt.Locale, func(rows int) {209 prog.setRows(rows)210 prog.setFrac(counter.n, size)211 })212 if err != nil {213 return nil, err214 }215 return &Result{Sheet: s, Rows: rows, Notes: notes}, nil216}217218// readDelimited reads CSV or TSV text, sniffing the delimiter of CSV,219// keeping up to maxCells cells (see newBuilder). Each field is entered as220// if typed in loc, or with the other decimal separator when the file221// writes its numbers that way (see numberLocale). progress is called222// every few hundred rows.223func readDelimited(ctx context.Context, in io.Reader, k Kind, maxCells int, loc *locale.Locale, progress func(rows int)) (*sheet.Sheet, int, []string, error) {224 s, rows, notes, _, err := readDelimitedAs(ctx, in, k, maxCells, loc, progress)225 return s, rows, notes, err226}227228// readDelimitedAs is readDelimited, also returning the delimiter.229func readDelimitedAs(ctx context.Context, in io.Reader, k Kind, maxCells int, loc *locale.Locale, progress func(rows int)) (*sheet.Sheet, int, []string, rune, error) {230 br := bufio.NewReaderSize(in, sniffSize)231 text, enc, err := decode(br)232 if err != nil {233 return nil, 0, nil, 0, err234 }235 tr := bufio.NewReaderSize(text, sniffSize)236 sample, _ := tr.Peek(sniffSize)237 comma := '\t'238 if k == CSV {239 comma = sniff(sample, len(sample) == sniffSize, csvComma(loc))240 }241 cr := csv.NewReader(tr)242 cr.Comma = comma243 cr.LazyQuotes = true244 cr.FieldsPerRecord = -1245 cr.ReuseRecord = true246247 b := newBuilder(ctx, maxCells)248 var otherDecimal bool249 b.loc, otherDecimal = numberLocale(sample, comma, loc)250 row := 0251 for ; ; row++ {252 // Report before reading on: a pipe may keep the next read waiting.253 if row%256 == 0 {254 if err := ctx.Err(); err != nil {255 return nil, 0, nil, 0, err256 }257 progress(row)258 }259 rec, err := cr.Read()260 if errors.Is(err, io.EOF) {261 break262 }263 if err != nil {264 var pe *csv.ParseError265 if errors.As(err, &pe) {266 return nil, 0, nil, 0, fmt.Errorf("line %d: %v", pe.Line, pe.Err)267 }268 return nil, 0, nil, 0, err269 }270 for col, field := range rec {271 if field != "" {272 b.entry(sheet.Addr{Col: col, Row: row}, field)273 }274 }275 }276 progress(row)277 var notes []string278 if k == CSV && comma != ',' {279 notes = append(notes, "separated by "+delimiterName(comma))280 }281 if otherDecimal {282 notes = append(notes, decimalNote(b.loc))283 }284 if enc != "UTF-8" {285 notes = append(notes, "read as "+enc)286 }287 s, notes := b.finish(notes)288 return s, row, notes, comma, nil289}290291func delimiterName(r rune) string {292 switch r {293 case '\t':294 return "tabs"295 case ';':296 return "semicolons"297 case '|':298 return "bars"299 }300 return "commas"301}302303// exportDelimited writes the snapshot's displayed values, as Sheets'304// Download as CSV does: formulas become their results, formats show, in305// the snapshot's locale, with ; between fields where its decimal306// separator is a comma.307// Rows are written as they are formatted, never all held at once.308func exportDelimited(name string, k Kind, snap *Snapshot) (*ExportResult, error) {309 res, err := exportText(name, snap, func(w io.Writer, snap *Snapshot) (int, error) { return encodeDelimited(w, k, snap) })310 if err != nil {311 return nil, err312 }313 if k == CSV && csvComma(snap.Locale) != ',' {314 res.Notes = append([]string{"separated by semicolons"}, res.Notes...)315 }316 return res, nil317}318319// encodeDelimited writes the rows of exportDelimited to out, returning320// how many it wrote.321func encodeDelimited(out io.Writer, k Kind, snap *Snapshot) (int, error) {322 rows := 0323 w := csv.NewWriter(out)324 w.Comma = csvComma(snap.Locale)325 if k == TSV {326 w.Comma = '\t'327 }328 for line := range snap.textRows() {329 if err := w.Write(line); err != nil {330 return 0, err331 }332 rows++333 }334 w.Flush()335 return rows, w.Error()336}