1package fileio23import (4 "cmp"5 "context"6 "errors"7 "fmt"8 "os"9 "slices"1011 "src.rest/cat/012/internal/sheet"12)1314// Sources: a Parquet file, or a SQLite table or query, read in place a15// window of rows at a time rather than imported, for tables too big for16// any grid. A Source streams its rows in its own order for what reads17// them all (aggregates, pivot tables) and fetches the rows a screen18// shows; a SourceView is its rows as a sort and a filter order them,19// pushed down to SQL for SQLite and streamed for Parquet. Values come20// out typed as an import types them.2122// SourceSpec names what a source reads.23type SourceSpec struct {24 // Path is the file.25 Path string26 // Format is "Parquet" or "SQLite", or "" to tell by the extension.27 Format string28 // Table is a SQLite table or view to read, or Query a query to run;29 // both empty reads a database's only table.30 Table, Query string31 // TempDir is where views keep what they build (sorted row orders,32 // SQLite's temporary tables); "" is the system's.33 TempDir string34}3536// SourceColumn is one of a source's columns.37type SourceColumn struct {38 Name string39 // Format is what its values show in when the column's type says (a40 // Parquet DATE column's FmtDate); a value's own may differ.41 Format sheet.Format42 // Numeric is set when its type holds numbers, dates and times43 // included.44 Numeric bool45}4647// SourceSort is a column a view is sorted by.48type SourceSort struct {49 Col int50 Desc bool51}5253// SourceFilter is a condition a view's rows meet on a column.54type SourceFilter struct {55 Col int56 Cond sheet.Condition57}5859// SourceOrder is how a view sorts and filters a source's rows.60type SourceOrder struct {61 Sort []SourceSort62 Filter []SourceFilter63}6465// IsZero reports whether the order is the source's own, every row.66func (o SourceOrder) IsZero() bool { return len(o.Sort) == 0 && len(o.Filter) == 0 }6768// Source is a table read in place. It is safe for use by several69// goroutines at once.70type Source interface {71 // Columns are the source's columns, left to right.72 Columns() []SourceColumn73 // Rows counts its rows, the header not among them.74 Rows() int6475 // Scan streams the rows from row from on (counting from 0) in the76 // source's order, calling fn with each row's number and the values77 // of cols (every column when cols is nil, in that order; the slice78 // is reused) until fn returns false, the rows end or ctx is done.79 Scan(ctx context.Context, from int64, cols []int, fn func(row int64, vals []sheet.LiveCell) bool) error80 // Fetch reads the rows numbered rows, in the order given, with the81 // values of cols (every column when nil).82 Fetch(ctx context.Context, rows []int64, cols []int) ([][]sheet.LiveCell, error)83 // View orders the rows as o says. Building it may read the whole84 // source once, which the caller does in the background.85 View(ctx context.Context, o SourceOrder) (SourceView, error)86 // Close lets the file go, and what its views built.87 Close() error88}8990// SourceView is a source's rows as a sort and a filter order them.91type SourceView interface {92 // Rows counts the rows the view shows.93 Rows() int6494 // Page reads n rows from position from of the view: each one's95 // number in the source and its values.96 Page(ctx context.Context, from int64, n int) ([]int64, [][]sheet.LiveCell, error)97 // Close lets go of what the view built.98 Close() error99}100101// ErrNotSource is what OpenSource says of a file that is neither102// Parquet nor SQLite.103var ErrNotSource = errors.New("only Parquet files and SQLite tables or queries can be linked as sources")104105// SourceKind is the format spec reads, Parquet or SQLite, by its Format106// or else the file's extension.107func SourceKind(spec SourceSpec) (Kind, error) {108 k, ok := KindNamed(spec.Format)109 if spec.Format == "" {110 k, ok = KindOf(spec.Path)111 }112 if !ok || k != Parquet && k != SQLite {113 return 0, ErrNotSource114 }115 return k, nil116}117118// OpenSource opens what spec names.119func OpenSource(ctx context.Context, spec SourceSpec) (Source, error) {120 k, err := SourceKind(spec)121 if err != nil {122 return nil, err123 }124 if spec.TempDir == "" {125 spec.TempDir = os.TempDir()126 }127 if k == Parquet {128 return openParquetSource(spec)129 }130 return openSQLiteSource(ctx, spec)131}132133// allCols is cols, or every one of n columns when cols is nil.134func allCols(cols []int, n int) []int {135 if cols != nil {136 return cols137 }138 out := make([]int, n)139 for i := range out {140 out[i] = i141 }142 return out143}144145// checkCols reports a column past the source's n.146func checkCols(cols []int, n int) error {147 for _, c := range cols {148 if c < 0 || c >= n {149 return fmt.Errorf("the source has no column %d", c+1)150 }151 }152 return nil153}154155// fetchOrder is the positions of rows sorted by row number, so a fetch156// reads forward, the answers going back to the order asked.157func fetchOrder(rows []int64) []int {158 idx := make([]int, len(rows))159 for i := range idx {160 idx[i] = i161 }162 slices.SortFunc(idx, func(a, b int) int { return cmp.Compare(rows[a], rows[b]) })163 return idx164}