- commit
- 0173b14dbd70762991374ced6c1eaf73ea2c06c8
- parent
- 82563374e93545c3cd9762f541ebdddd2c582eb4
- author
- Dami <dami.etoile@gmail.com>
- kind
- refactor risk score 1.9 of 3, Jev's estimate
| M | internal/ | 79 | ++--------- |
| A | internal/ | 203 | ++++++++++++++++++++++++++++++ |
| M | internal/ | 37 | +----- |
@@ -134,78 +134,29 @@ func putParquetRow(b *builder, row int, r parquet.Row, cols []parquetColumn) (re
134134 b.fits(sheet.Addr{Row: row})135135 return false136136 }137- // Values arrive grouped by column; a repeated column has several.137+ eachParquetColumn(r, func(col int, vals []parquet.Value) {138+ if col < 0 || col >= len(cols) {139+ return140+ }141+ c, rep := parquetCells(vals, cols[col])142+ c.put(b, sheet.Addr{Col: col, Row: row})143+ repeated = repeated || rep144+ })145+ return repeated146+}147+148+// eachParquetColumn calls fn with each column's values in a row: they149+// arrive grouped by column, and a repeated column has several.150+func eachParquetColumn(r parquet.Row, fn func(col int, vals []parquet.Value)) {138151 for i := 0; i < len(r); {139152 col := r[i].Column()140153 j := i + 1141154 for j < len(r) && r[j].Column() == col {142155 j++143156 }144- if col < 0 || col >= len(cols) {145- i = j146- continue147- }148- a := sheet.Addr{Col: col, Row: row}149- vals := r[i:j]150- if len(vals) > 1 {151- parts := make([]string, 0, len(vals))152- for _, v := range vals {153- if !v.IsNull() {154- parts = append(parts, parquetText(v, cols[col]))155- }156- }157- if len(parts) > 0 {158- b.text(a, strings.Join(parts, ", "), sheet.Format{}, sheet.Style{})159- repeated = true160- }161- } else if !vals[0].IsNull() {162- putParquetValue(b, a, vals[0], cols[col])163- }157+ fn(col, r[i:j])164158 i = j165159 }166- return repeated167-}168-169-func putParquetValue(b *builder, a sheet.Addr, v parquet.Value, c parquetColumn) {170- switch lt := c.logical.(type) {171- case *format.DateType:172- b.number(a, float64(v.Int32())+serialOf(time.Unix(0, 0).UTC()), sheet.Format{Kind: sheet.FmtDate}, sheet.Style{})173- return174- case *format.TimestampType:175- t := timestamp(v.Int64(), lt.Unit)176- b.number(a, serialOf(t), sheet.Format{Kind: sheet.FmtDateTime}, sheet.Style{})177- return178- case *format.TimeType:179- d := timeUnit(lt.Unit) * time.Duration(timeInt(v))180- b.number(a, d.Hours()/24, sheet.Format{Kind: sheet.FmtTime}, sheet.Style{})181- return182- case *format.DecimalType:183- if x, ok := decimal(v, lt.Scale); ok {184- b.number(a, x, sheet.Format{}, sheet.Style{})185- return186- }187- }188- switch c.kind {189- case parquet.Boolean:190- b.boolean(a, v.Boolean(), sheet.Style{})191- case parquet.Int32:192- b.number(a, float64(v.Int32()), sheet.Format{}, sheet.Style{})193- case parquet.Int64:194- b.number(a, float64(v.Int64()), sheet.Format{}, sheet.Style{})195- case parquet.Float:196- b.number(a, float64(v.Float()), sheet.Format{}, sheet.Style{})197- case parquet.Double:198- b.number(a, v.Double(), sheet.Format{}, sheet.Style{})199- case parquet.Int96:200- // Legacy timestamps: nanoseconds of the day, then the Julian day.201- i := v.Int96()202- nanos := int64(i[1])<<32 | int64(i[0])203- days := int64(i[2]) - 2440588 // the Julian day of 1970-01-01204- t := time.Unix(days*86400, nanos).UTC()205- b.number(a, serialOf(t), sheet.Format{Kind: sheet.FmtDateTime}, sheet.Style{})206- default:207- textOrDate(b, a, parquetText(v, c))208- }209160 }210161 211162 // parquetText is a value as text, for byte arrays and joined lists.@@ -0,0 +1,203 @@
1+package fileio2+3+import (4+ "fmt"5+ "math"6+ "strings"7+ "time"8+ "unicode/utf8"9+10+ "github.com/parquet-go/parquet-go"11+ "github.com/parquet-go/parquet-go/format"12+13+ "github.com/FelineStateMachine/012/internal/sheet"14+)15+16+// How Parquet and SQLite values become cells, shared by imports (which17+// store them through the builder) and sources (which hand them out as18+// they are). Text a file holds as a value comes out raw: an import19+// stores it as typed in its locale (textOrDate) and a source reads the20+// dates in it (typed), so both type a value the same way. Text 01221+// makes (joined lists, a blob's size) stays text.22+23+// fileCell is a value from a file, raw when it is text to be read as a24+// date where it is one.25+type fileCell struct {26+ sheet.LiveCell27+ raw bool28+}29+30+// liveNum is a number in format f; NaN and infinities, which no cell31+// holds, are #NUM!.32+func liveNum(v float64, f sheet.Format) sheet.LiveCell {33+ if math.IsNaN(v) || math.IsInf(v, 0) {34+ return sheet.LiveCell{V: sheet.ErrNum}35+ }36+ return sheet.LiveCell{V: sheet.Value{Kind: sheet.Number, Num: v}, F: f}37+}38+39+func strCell(s string) sheet.LiveCell {40+ return sheet.LiveCell{V: sheet.Value{Kind: sheet.Text, Str: s}}41+}42+43+func liveBool(b bool) sheet.LiveCell {44+ v := sheet.Value{Kind: sheet.Bool}45+ if b {46+ v.Num = 147+ }48+ return sheet.LiveCell{V: v}49+}50+51+// put stores a cell as an import does: raw text through textOrDate,52+// which reads dates in the import's locale.53+func (c fileCell) put(b *builder, a sheet.Addr) {54+ switch c.V.Kind {55+ case sheet.Empty:56+ case sheet.Number:57+ b.number(a, c.V.Num, c.F, sheet.Style{})58+ case sheet.Bool:59+ b.boolean(a, c.V.Num != 0, sheet.Style{})60+ case sheet.Text:61+ if c.raw {62+ textOrDate(b, a, c.V.Str)63+ } else {64+ b.text(a, c.V.Str, c.F, sheet.Style{})65+ }66+ default:67+ b.number(a, math.NaN(), c.F, sheet.Style{})68+ }69+}70+71+// typed is the cell as a source hands it out: raw text that is a date72+// or time (2026-09-26) as that date, as an import would store it, and73+// other text, numbers stored as text included, as text.74+func (c fileCell) typed() sheet.LiveCell {75+ if !c.raw {76+ return c.LiveCell77+ }78+ s := flatten(c.V.Str)79+ if n, f, ok := sheet.ParseValue(s); ok {80+ switch f.Kind {81+ case sheet.FmtDate, sheet.FmtTime, sheet.FmtDateTime:82+ return sheet.LiveCell{V: sheet.Value{Kind: sheet.Number, Num: n}, F: f}83+ }84+ }85+ return strCell(s)86+}87+88+// parquetCells is a row's values of one column as a cell: empty for89+// null, several joined with commas (reporting that they were).90+func parquetCells(vals []parquet.Value, c parquetColumn) (cell fileCell, repeated bool) {91+ if len(vals) == 1 {92+ if vals[0].IsNull() {93+ return fileCell{}, false94+ }95+ return parquetCell(vals[0], c), false96+ }97+ parts := make([]string, 0, len(vals))98+ for _, v := range vals {99+ if !v.IsNull() {100+ parts = append(parts, parquetText(v, c))101+ }102+ }103+ if len(parts) == 0 {104+ return fileCell{}, false105+ }106+ return fileCell{LiveCell: strCell(strings.Join(parts, ", "))}, true107+}108+109+// parquetCell is one value that isn't null.110+func parquetCell(v parquet.Value, c parquetColumn) fileCell {111+ lc, ok := parquetValue(v, c)112+ if !ok {113+ return fileCell{LiveCell: strCell(parquetText(v, c)), raw: true}114+ }115+ return fileCell{LiveCell: lc}116+}117+118+// parquetValue is a value that isn't text, and false for text.119+func parquetValue(v parquet.Value, c parquetColumn) (sheet.LiveCell, bool) {120+ switch lt := c.logical.(type) {121+ case *format.DateType:122+ return liveNum(float64(v.Int32())+serialOf(time.Unix(0, 0).UTC()), sheet.Format{Kind: sheet.FmtDate}), true123+ case *format.TimestampType:124+ return liveNum(serialOf(timestamp(v.Int64(), lt.Unit)), sheet.Format{Kind: sheet.FmtDateTime}), true125+ case *format.TimeType:126+ d := timeUnit(lt.Unit) * time.Duration(timeInt(v))127+ return liveNum(d.Hours()/24, sheet.Format{Kind: sheet.FmtTime}), true128+ case *format.DecimalType:129+ if x, ok := decimal(v, lt.Scale); ok {130+ return liveNum(x, sheet.Format{}), true131+ }132+ }133+ switch c.kind {134+ case parquet.Boolean:135+ return liveBool(v.Boolean()), true136+ case parquet.Int32:137+ return liveNum(float64(v.Int32()), sheet.Format{}), true138+ case parquet.Int64:139+ return liveNum(float64(v.Int64()), sheet.Format{}), true140+ case parquet.Float:141+ return liveNum(float64(v.Float()), sheet.Format{}), true142+ case parquet.Double:143+ return liveNum(v.Double(), sheet.Format{}), true144+ case parquet.Int96:145+ // Legacy timestamps: nanoseconds of the day, then the Julian day.146+ i := v.Int96()147+ nanos := int64(i[1])<<32 | int64(i[0])148+ days := int64(i[2]) - 2440588 // the Julian day of 1970-01-01149+ return liveNum(serialOf(time.Unix(days*86400, nanos).UTC()), sheet.Format{Kind: sheet.FmtDateTime}), true150+ }151+ return sheet.LiveCell{}, false152+}153+154+// parquetFormat is the format a column's type shows its values in, and155+// whether it holds numbers.156+func parquetFormat(c parquetColumn) (sheet.Format, bool) {157+ switch c.logical.(type) {158+ case *format.DateType:159+ return sheet.Format{Kind: sheet.FmtDate}, true160+ case *format.TimestampType:161+ return sheet.Format{Kind: sheet.FmtDateTime}, true162+ case *format.TimeType:163+ return sheet.Format{Kind: sheet.FmtTime}, true164+ case *format.DecimalType:165+ return sheet.Format{}, true166+ }167+ switch c.kind {168+ case parquet.Int32, parquet.Int64, parquet.Float, parquet.Double:169+ return sheet.Format{}, true170+ case parquet.Int96:171+ return sheet.Format{Kind: sheet.FmtDateTime}, true172+ }173+ return sheet.Format{}, false174+}175+176+// sqliteCell is a database value as a cell, reporting whether it was177+// binary data that could only be described.178+func sqliteCell(v any) (cell fileCell, blob bool) {179+ switch v := v.(type) {180+ case nil:181+ return fileCell{}, false182+ case int64:183+ return fileCell{LiveCell: liveNum(float64(v), sheet.Format{})}, false184+ case float64:185+ return fileCell{LiveCell: liveNum(v, sheet.Format{})}, false186+ case bool:187+ return fileCell{LiveCell: liveBool(v)}, false188+ case time.Time:189+ f := sheet.Format{Kind: sheet.FmtDateTime}190+ if h, m, s := v.Clock(); h == 0 && m == 0 && s == 0 && v.Nanosecond() == 0 {191+ f = sheet.Format{Kind: sheet.FmtDate}192+ }193+ return fileCell{LiveCell: liveNum(serialOf(v), f)}, false194+ case string:195+ return fileCell{LiveCell: strCell(v), raw: true}, false196+ case []byte:197+ if utf8.Valid(v) {198+ return fileCell{LiveCell: strCell(string(v)), raw: true}, false199+ }200+ return fileCell{LiveCell: strCell(fmt.Sprintf("(%s)", count(len(v), "byte", "bytes")))}, true201+ }202+ return fileCell{LiveCell: strCell(fmt.Sprint(v))}, false203+}@@ -9,7 +9,6 @@ import (
99 "os"1010 "strings"1111 "time"12- "unicode/utf8"1312 1413 _ "modernc.org/sqlite" // registers the "sqlite" driver1514 @@ -194,7 +193,9 @@ func readSQLiteRows(ctx context.Context, rows *sql.Rows, b *builder, ncols, tota
194193 return 0, 0, sqliteErr(err)195194 }196195 for c, v := range vals {197- if putValue(b, sheet.Addr{Col: c, Row: row}, v) {196+ cell, blob := sqliteCell(v)197+ cell.put(b, sheet.Addr{Col: c, Row: row})198+ if blob {198199 blobs++199200 }200201 }@@ -207,38 +208,6 @@ func readSQLiteRows(ctx context.Context, rows *sql.Rows, b *builder, ncols, tota
207208 return row, blobs, nil208209 }209210 210-// putValue stores a database value, reporting whether it was binary data211-// that could only be described.212-func putValue(b *builder, a sheet.Addr, v any) (blob bool) {213- switch v := v.(type) {214- case nil:215- case int64:216- b.number(a, float64(v), sheet.Format{}, sheet.Style{})217- case float64:218- b.number(a, v, sheet.Format{}, sheet.Style{})219- case bool:220- b.boolean(a, v, sheet.Style{})221- case time.Time:222- f := sheet.Format{Kind: sheet.FmtDateTime}223- if h, m, s := v.Clock(); h == 0 && m == 0 && s == 0 && v.Nanosecond() == 0 {224- f = sheet.Format{Kind: sheet.FmtDate}225- }226- b.number(a, serialOf(v), f, sheet.Style{})227- case string:228- textOrDate(b, a, v)229- case []byte:230- if utf8.Valid(v) {231- textOrDate(b, a, string(v))232- return false233- }234- b.text(a, fmt.Sprintf("(%s)", count(len(v), "byte", "bytes")), sheet.Format{}, sheet.Style{})235- return true236- default:237- b.text(a, fmt.Sprint(v), sheet.Format{}, sheet.Style{})238- }239- return false240-}241-242211 // textOrDate stores database text as text, except dates and times such243212 // as 2026-09-26, which become dates as they would when typed. Numbers244213 // stored as text (zip codes, IDs) stay text.