1// Package fileio moves data between sheets and other file formats: CSV2// and TSV, Excel workbooks, SQLite databases, Parquet files and Lotus3// 1-2-3 worksheets. Everything is pure Go. The sheet package knows none4// of these formats: importers build a sheet through its public API5// (Load, LoadColWidth, RecalcAll) and exporters read a Snapshot of one.6// Each format is a row of the table in formats.go and a file of its own.7package fileio89import (10 "bufio"11 "context"12 "errors"13 "fmt"14 "io"15 "log/slog"16 "os"17 "path/filepath"18 "strings"19 "sync/atomic"2021 "src.rest/cat/012/internal/locale"22 "src.rest/cat/012/internal/sheet"23 "src.rest/cat/012/internal/telemetry"24)2526// Progress reports how far an import has got. The importer updates it27// from its goroutine; the UI reads it on a timer.28type Progress struct {29 rows atomic.Int6430 permille atomic.Int64 // 0-1000, or -1 while the total isn't known31}3233// NewProgress returns a progress with an unknown total.34func NewProgress() *Progress {35 p := &Progress{}36 p.permille.Store(-1)37 return p38}3940// Get returns the rows read so far and the fraction done, which is41// negative while unknown.42func (p *Progress) Get() (rows int, frac float64) {43 if p == nil {44 return 0, -145 }46 pm := p.permille.Load()47 if pm < 0 {48 return int(p.rows.Load()), -149 }50 return int(p.rows.Load()), float64(pm) / 100051}5253// Report sets the rows read and, with a positive total, the fraction54// done; importers call it, and tests of progress displays.55func (p *Progress) Report(rows int, done, total int64) {56 p.setRows(rows)57 p.setFrac(done, total)58}5960func (p *Progress) setRows(n int) {61 if p != nil {62 p.rows.Store(int64(n))63 }64}6566// setFrac records done out of total; a non-positive total is unknown.67func (p *Progress) setFrac(done, total int64) {68 if p == nil || total <= 0 {69 return70 }71 p.permille.Store(min(done*1000/total, 1000))72}7374// Options tune an import.75type Options struct {76 // Table is the SQLite table to import, or Query a SELECT to run.77 // With neither, a database with one table imports it.78 Table, Query string79 Progress *Progress80 // MaxCells is the most cells to keep, whole rows at a time; 0 is the81 // max-cells setting (sheet.MaxCells). WK1 files keep their own limits.82 MaxCells int83 // Locale is what CSV and TSV fields are read in (see numberLocale);84 // nil is en-US.85 Locale *locale.Locale86}8788// Result is an imported sheet and what the import had to leave out or89// change, in sentences for the context line.90type Result struct {91 Sheet *sheet.Sheet92 Kind Kind93 Rows int // rows of the file that were read, including a header94 Notes []string95}9697// ErrNeedTable is returned for a SQLite database with several tables98// when Options names neither a table nor a query; Tables lists them.99type ErrNeedTable struct{ Tables []TableInfo }100101func (e *ErrNeedTable) Error() string {102 return fmt.Sprintf("the database has %d tables; pick one", len(e.Tables))103}104105// ErrUnsupported is returned for a file whose extension isn't known.106var ErrUnsupported = errors.New("not a format 012 can import")107108// Import reads the file name into a new sheet. It checks ctx between109// rows, so a long import can be cancelled.110func Import(ctx context.Context, name string, opt Options) (*Result, error) {111 k, ok := KindOf(name)112 if !ok {113 return nil, ErrUnsupported114 }115 span := telemetry.ParentFrom(ctx).Start("import", slog.String("format", k.String()))116 // The builder's workbook recalculates under the span (see newBuilder).117 r, err := importKind(telemetry.WithParent(ctx, span.Parent()), name, k, opt)118 if err != nil {119 span.Fail(err)120 return nil, err121 }122 if telemetry.Enabled() {123 span.End(slog.Int64("bytes", fileSize(name)), slog.Int("rows", r.Rows), slog.Int("cells", r.Sheet.Len()), slog.Int("notes", len(r.Notes)))124 }125 return r, nil126}127128func importKind(ctx context.Context, name string, k Kind, opt Options) (*Result, error) {129 r, err := k.format().read(ctx, name, opt)130 if err != nil {131 return nil, err132 }133 // A file of one table becomes one sheet named after it, as in Sheets.134 if book := r.Sheet.Book(); !k.HoldsSheets() && book.Len() == 1 {135 base := strings.TrimSuffix(filepath.Base(name), filepath.Ext(name))136 if opt.Table != "" {137 base = opt.Table138 }139 if book.RenameSheet(r.Sheet, sheetName(base)) == nil {140 book.ClearHistory()141 }142 }143 r.Kind = k144 if !k.format().widths {145 r.Sheet.LoadFitWidths()146 }147 if p := opt.Progress; p != nil && p.permille.Load() >= 0 {148 p.permille.Store(1000)149 }150 return r, nil151}152153// ExportOptions tune an export.154type ExportOptions struct {155 Table string // SQLite: the table to write; replaced if it exists156}157158// ExportResult says what an export wrote and what it couldn't keep.159type ExportResult struct {160 Rows int161 Notes []string162}163164// Export writes a snapshot of a sheet to name in format k.165func Export(ctx context.Context, name string, k Kind, snap *Snapshot, opt ExportOptions) (*ExportResult, error) {166 span := telemetry.ParentFrom(ctx).Start("export", slog.String("format", k.String()), slog.Int("cells", len(snap.Cells)))167 res, err := exportKind(ctx, name, k, snap, opt)168 if err != nil {169 span.Fail(err)170 return nil, err171 }172 if telemetry.Enabled() {173 span.End(slog.Int("rows", res.Rows), slog.Int64("bytes", fileSize(name)))174 }175 return res, nil176}177178func exportKind(ctx context.Context, name string, k Kind, snap *Snapshot, opt ExportOptions) (*ExportResult, error) {179 if !k.CanExport() {180 return nil, fmt.Errorf("can't export %s files", k)181 }182 return k.format().write(ctx, name, snap, opt)183}184185// fileSize is the size of the file name, for telemetry; -1 if unknown.186func fileSize(name string) int64 {187 st, err := os.Stat(name)188 if err != nil {189 return -1190 }191 return st.Size()192}193194// writeFile writes a file atomically: write fills a temporary file195// through a buffer, which is then renamed over the target, so a failed196// export never leaves a half-written file behind.197func writeFile(name string, write func(io.Writer) error) error {198 tmp := name + ".tmp"199 f, err := os.OpenFile(tmp, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0o644)200 if err != nil {201 return err202 }203 bw := bufio.NewWriterSize(f, 64<<10)204 err = write(bw)205 if err == nil {206 err = bw.Flush()207 }208 if cerr := f.Close(); err == nil {209 err = cerr210 }211 if err == nil {212 err = os.Rename(tmp, name)213 }214 if err != nil {215 os.Remove(tmp)216 }217 return err218}