Exports built every row as strings, then the whole file in memory, before writing it: 104 MB allocated for a full 8192 x 256 sheet, now 30 MB. UTF-16 input was read whole and decoded twice; it now goes through x/text's decoder like Windows-1252. All exports share one atomic write-then-rename helper.
- commit
- 0040bb1c716ae3f7daff9e90e3e670f86f098fd8
- parent
- b5ee00dd062dc2d9079704ab1e0648f79302acc8
- author
- Dami <dami.etoile@gmail.com>
| M | internal/ | 63 | ++++++++++++------------------ |
| M | internal/ | 7 | +++ |
| M | internal/ | 28 | +++++++++++++ |
| M | internal/ | 26 | +++++++----- |
| M | internal/ | 7 | +-- |
@@ -9,10 +9,10 @@ import (
99 "fmt"1010 "io"1111 "os"12- "unicode/utf16"1312 "unicode/utf8"1413 1514 "golang.org/x/text/encoding/charmap"15+ "golang.org/x/text/encoding/unicode"1616 1717 "github.com/FelineStateMachine/012/internal/sheet"1818 )@@ -107,21 +107,12 @@ func decode(r *bufio.Reader) (io.Reader, string, error) {
107107 r.Discard(3)108108 return r, "UTF-8", nil109109 case bytes.HasPrefix(head, []byte{0xFF, 0xFE}), bytes.HasPrefix(head, []byte{0xFE, 0xFF}):110- big := head[0] == 0xFE111- r.Discard(2)112- all, err := io.ReadAll(r)113- if err != nil {114- return nil, "", err115- }116- u := make([]uint16, len(all)/2)117- for i := range u {118- if big {119- u[i] = uint16(all[2*i])<<8 | uint16(all[2*i+1])120- } else {121- u[i] = uint16(all[2*i+1])<<8 | uint16(all[2*i])122- }110+ order := unicode.LittleEndian111+ if head[0] == 0xFE {112+ order = unicode.BigEndian123113 }124- return bytes.NewReader([]byte(string(utf16.Decode(u)))), "UTF-16", nil114+ r.Discard(2)115+ return unicode.UTF16(order, unicode.IgnoreBOM).NewDecoder().Reader(r), "UTF-16", nil125116 }126117 // A sample cut mid-character is still UTF-8.127118 check := head@@ -258,14 +249,24 @@ func delimiterName(r rune) string {
258249 259250 // exportDelimited writes the snapshot's displayed values, as Sheets'260251 // Download as CSV does: formulas become their results, formats show.252+// Rows are written as they are formatted, never all held at once.261253 func exportDelimited(name string, k Kind, snap *Snapshot) (*ExportResult, error) {262- var buf bytes.Buffer263- w := csv.NewWriter(&buf)264- if k == TSV {265- w.Comma = '\t'266- }267- rows := snap.rows()268- if err := w.WriteAll(rows); err != nil {254+ rows := 0255+ err := writeFile(name, func(out io.Writer) error {256+ w := csv.NewWriter(out)257+ if k == TSV {258+ w.Comma = '\t'259+ }260+ for line := range snap.textRows() {261+ if err := w.Write(line); err != nil {262+ return err263+ }264+ rows++265+ }266+ w.Flush()267+ return w.Error()268+ })269+ if err != nil {269270 return nil, err270271 }271272 formulas := 0@@ -274,23 +275,9 @@ func exportDelimited(name string, k Kind, snap *Snapshot) (*ExportResult, error)
274275 formulas++275276 }276277 }277- res := &ExportResult{Rows: len(rows)}278+ res := &ExportResult{Rows: rows}278279 if formulas > 0 {279280 res.Notes = append(res.Notes, count(formulas, "formula", "formulas")+" saved as values")280281 }281- return res, writeFile(name, buf.Bytes())282-}283-284-// writeFile writes data atomically: to a temporary file first, then285-// renamed over the target.286-func writeFile(name string, data []byte) error {287- tmp := name + ".tmp"288- if err := os.WriteFile(tmp, data, 0o644); err != nil {289- return err290- }291- if err := os.Rename(tmp, name); err != nil {292- os.Remove(tmp)293- return err294- }295- return nil282+ return res, nil296283 }@@ -120,6 +120,13 @@ func TestReadDelimited(t *testing.T) {
120120 cells: map[string]string{"A1": "a", "B1": "é", "A2": "1"},121121 note: "read as UTF-16",122122 },123+ {124+ name: "UTF-16 big-endian, surrogate pair",125+ kind: CSV,126+ text: "\xFE\xFF\x00a\x00,\xD8\x3D\xDE\x00\x00\n",127+ cells: map[string]string{"A1": "a", "B1": "😀"},128+ note: "read as UTF-16",129+ },123130 {124131 name: "Windows-1252",125132 kind: CSV,@@ -6,9 +6,11 @@
66 package fileio77 88 import (9+ "bufio"910 "context"1011 "errors"1112 "fmt"13+ "io"1214 "log/slog"1315 "os"1416 "path/filepath"@@ -282,3 +284,29 @@ func fileSize(name string) int64 {
282284 }283285 return st.Size()284286 }287+288+// writeFile writes a file atomically: write fills a temporary file289+// through a buffer, which is then renamed over the target, so a failed290+// export never leaves a half-written file behind.291+func writeFile(name string, write func(io.Writer) error) error {292+ tmp := name + ".tmp"293+ f, err := os.OpenFile(tmp, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0o644)294+ if err != nil {295+ return err296+ }297+ bw := bufio.NewWriterSize(f, 64<<10)298+ err = write(bw)299+ if err == nil {300+ err = bw.Flush()301+ }302+ if cerr := f.Close(); err == nil {303+ err = cerr304+ }305+ if err == nil {306+ err = os.Rename(tmp, name)307+ }308+ if err != nil {309+ os.Remove(tmp)310+ }311+ return err312+}@@ -1,6 +1,7 @@
11 package fileio22 33 import (4+ "iter"45 "strconv"56 67 "github.com/FelineStateMachine/012/internal/sheet"@@ -99,19 +100,22 @@ func excelRange(ws string, r sheet.Rect) string {
99100 return sheet.QuoteSheet(sheetName(ws)) + "!" + ref100101 }101102 102-// rows returns the snapshot's displayed text row by row across its103-// range, for text formats.104-func (s *Snapshot) rows() [][]string {105- r := s.Range106- out := make([][]string, 0, r.To.Row-r.From.Row+1)107- for row := r.From.Row; row <= r.To.Row; row++ {103+// textRows yields the snapshot's displayed text row by row across its104+// range, for text formats. The slice is reused from row to row.105+func (s *Snapshot) textRows() iter.Seq[[]string] {106+ return func(yield func([]string) bool) {107+ r := s.Range108108 line := make([]string, r.To.Col-r.From.Col+1)109- for col := r.From.Col; col <= r.To.Col; col++ {110- if c, ok := s.Cells[sheet.Addr{Col: col, Row: row}]; ok {111- line[col-r.From.Col] = c.Text()109+ for row := r.From.Row; row <= r.To.Row; row++ {110+ for col := r.From.Col; col <= r.To.Col; col++ {111+ line[col-r.From.Col] = ""112+ if c, ok := s.Cells[sheet.Addr{Col: col, Row: row}]; ok {113+ line[col-r.From.Col] = c.Text()114+ }115+ }116+ if !yield(line) {117+ return112118 }113119 }114- out = append(out, line)115120 }116- return out117121 }@@ -3,6 +3,7 @@ package fileio
33 import (44 "context"55 "fmt"6+ "io"67 "math"78 "strconv"89 "strings"@@ -260,15 +261,11 @@ func exportXLSX(_ context.Context, name string, snap *Snapshot, _ ExportOptions)
260261 if err := x.SetCalcProps(&excelize.CalcPropsOptions{FullCalcOnLoad: &yes}); err != nil {261262 return nil, err262263 }263- buf, err := x.WriteToBuffer()264- if err != nil {265- return nil, err266- }267264 if w.values > 0 {268265 res.Notes = append(res.Notes, fmt.Sprintf("%s with no Excel equivalent saved as values, e.g. %s",269266 count(w.values, "formula", "formulas"), w.example))270267 }271- return res, writeFile(name, buf.Bytes())268+ return res, writeFile(name, func(w io.Writer) error { return x.Write(w) })272269 }273270 274271 // xlsxWriter writes sheets into a workbook, sharing cell styles.