- commit
- 617ab46de68e6dd2d8acf206e5aefcc24c95f5e8
- parent
- ef7b6e956733fc0f645c7c46415a36739c30d9cd
- author
- Dami <dami.etoile@gmail.com>
- kind
- fix risk score 1.4 of 3, Jev's estimate
| M | ROADMAP.md | 6 | +- |
| M | docs/ | 64 | +++++++++++- |
| M | docs/ | 49 | ++++++++ |
| M | docs/ | 3 | +- |
| M | docs/ | 4 | +- |
| M | docs/ | 4 | + |
| A | docs/ | 175 | ++++++++++++++++++++++++++++++ |
| M | internal/ | 15 | ++- |
| A | internal/ | 96 | ++++++++++++++++ |
| M | internal/ | 79 | ++++++++++++++ |
| M | internal/ | 12 | +- |
@@ -20,10 +20,7 @@ Sizes: S (a day or two), M (about a week), L (weeks).
2020 The grid is Excel's, 1,048,576 x 16,384, and memory is bounded by the2121 `max-cells` budget (ten million cells) rather than the grid. The budget2222 rises in measured steps; see [Bounds of support](docs/contributing/limits.md#what-would-raise-the-bounds).23-24-| Item | Result | Size |25-|---|---|---|26-| Linked, paged read-only ranges over Parquet and SQLite that feed pivots and formulas by streaming | Sources too big for any grid | L |23+Tables past it are read in place as [linked sources](docs/files/sources.md).2724 2825 ### 2. A TUI Jupyter for nushell2926 @@ -131,6 +128,7 @@ their own.
131128 - Commands for scripts without the screen: `012 get` (text, CSV, TSV, JSON or NUON), `012 set`, `012 recalc` and `012 export`, running notebooks and JEV only behind flags: [Scripts](docs/files/scripts.md)132129 - `012 diff` cell by cell, as git's diff command or textconv, and `012 merge-driver` merging cell by cell with conflicts noted on the cells: [Diff and merge in git](docs/files/git.md)133130 - Following files: a linked region follows a CSV, TSV, JSON lines or NUON file as it grows and any importable file as it's rewritten, keeping every row or the last ones, its rows arriving as the change stream: [Following files](docs/files/following.md)131+- Linked sources: a Parquet file or a SQLite table or query read in place on a tab of its own, scrolled, sorted and filtered however many rows it has, formulas and pivot tables streaming over it: [Linked sources](docs/files/sources.md)134132 - The `.012` format written down with a version, and each release's workbooks kept as fixtures that every later build opens and saves unchanged: [The .012 format](docs/files/format.md#versions), [Releasing](docs/contributing/releasing.md#file-fixtures)135133 - Web pages in 012's look: a sheet, a range or a chart as one self-contained `.html` file, charts as SVG, from File > Download and `012 export`: [Files](docs/files/README.md#web-pages)136134 - A crash keeps unsaved work for recovery, restores the terminal and writes a report, locally and in `012 serve`: [Saving](docs/files/saving.md#if-012-crashes)@@ -20,6 +20,7 @@ internal/locale the locales: separators, date order, currency, formula separato
2020 internal/fileio import and export: CSV, TSV, XLSX, SQLite, Parquet, Lotus .wk1, JSON, NUON; tables on streams2121 internal/nuon nushell's object notation: typed values, tables read a row at a time, written back2222 internal/live the sources of linked regions: files followed as they grow or are rewritten23+internal/paged linked sources read in place: the host answering formulas and pivots by streaming them, the pages a tab shows2324 internal/headless workbook files without the screen: references, get, set, recalc, notebooks and JEV when asked, atomic saves2425 internal/diff workbooks compared and merged cell by cell, from their files' fields2526 internal/chart chart layout, text rendering and kitty image encoding@@ -43,6 +44,7 @@ internal/ui the Bubble Tea model: modes, menus, overlays, rendering
4344 picker the searchable list behind the palette and every picker4445 cmdline the : command line and its completions4546 nbview a notebook tab as Jupyter's: its toolbar, cells, outputs, command and edit modes, mouse, the code editor and its language providers47+ srcview a linked source's tab: its rows a page at a time, the row numbers and scrollbar over every row4648 findbar find and replace, a bar on the context line4749 evalview Data > Evaluate formula: a formula stepped through part by part4850 themepicker File > Settings > Theme, previewing as it moves@@ -72,7 +74,7 @@ flowchart TD
7274 direction LR7375 ui[internal/ui] ~~~ fileio[internal/fileio] ~~~ live[internal/live] ~~~ nushell[internal/nushell]7476 chart[internal/chart] ~~~ jev[internal/jev] ~~~ stress[internal/stress] ~~~ oracle[oracle]75- headless[internal/headless] ~~~ diff[internal/diff] ~~~ room[internal/room]77+ headless[internal/headless] ~~~ diff[internal/diff] ~~~ room[internal/room] ~~~ paged[internal/paged]7678 end7779 callers --> sheet[internal/sheet]7880 sheet --> functions[internal/functions]@@ -260,6 +262,21 @@ style.
260262 table as `nu.name`, resolved when evaluated as named ranges are. The261263 file keeps the definitions (`regionfile.go`), and converts the command262264 regions of earlier files into notebook cells (`notebookfile.go`).265+- **Linked sources.** A [linked source](../files/sources.md) is a tab266+ whose one region is `Paged` (`source.go`): a Parquet file or a SQLite267+ table named as a table is (`sales[amount]`, `nu.sales`), holding no268+ cells, whose table may run past the grid's last row. The engine never269+ reads it. What its host found on opening it (the columns, their270+ formats, the rows) is kept outside the undo history271+ (`SetSourceShape`), and everything that reads it asks the workbook's272+ `SourceHost` (`sourceask.go`): a function given one of its ranges is273+ asked whole as a `functions.StreamCall`, a cell or a range read some274+ other way as its values, and a pivot table over it as its groups275+ (`pivotdata.go`, where a pivot reads a sheet's cells or a source's276+ rows streaming past through one interface). An answer not known yet277+ is `Pending`, the formula or pivot waiting for it, and278+ `RecalcSourced` recomputes what waited once it's known;279+ `SourceChanged` recomputes what read a source whose file changed.263280 - **Notebooks.** A notebook tab is a sheet whose `regionState` holds264281 cells (`notebook.go`), so adding, editing, moving and deleting cells265282 are undo steps as any change is, and the tab is named, moved, hidden@@ -324,6 +341,36 @@ values and names nothing but its region, so applying the same ops in the
324341 same order to the same workbook makes the same cells, whoever applies325342 it. `OnLive` times each op for telemetry.326343 344+### Questions to linked sources345+346+A formula reading a linked source asks, and the answer comes back in347+the background, as a JEV question's does:348+349+```mermaid350+sequenceDiagram351+ participant E as the engine (a recalculation)352+ participant H as paged.Host353+ participant J as a job, on a goroutine354+ participant F as fileio.Source355+ E->>H: SUM(sales[amount]) as a StreamCall356+ H-->>E: not known: queued, the cell Loading…357+ H->>J: the call, and the source as it was opened358+ J->>F: the column, streamed in batches359+ J->>J: functions.EvalStream over a Book that reads the source360+ J-->>H: the answer, kept until the file changes361+ H->>E: RecalcSourced: the cells that waited362+```363+364+The job runs the function library's own function (`EvalStream`) over365+a `Book` that reads the source (`paged/book.go`): an aggregate folds366+the column as it streams, a lookup walks it, and the functions over367+aligned ranges read them a window of rows at a time when the `Reader`368+says it is streaming (`functions/streamed.go`), so every function369+means what it means on a sheet. A function that holds what it reads is370+given a source's range only up to `max-cells` cells. Pivot tables371+gather their groups the same way (`sheet.GatherPivot`), and the engine372+lays them out when the answer arrives.373+327374 ### Functions328375 329376 `internal/functions` holds the function library: the `FuncDef` table@@ -351,6 +398,8 @@ the engine implements once per sheet (`reader` in `recalc.go`):
351398 | `RangeAgg(sheet, r)` | the running aggregate SUM-like functions share within a recalculation |352399 | `Fold(sheet, r, agg)` | a range added to a SUM-like function's aggregate, as the engine reads it |353400 | `Ask(call)` | a JEV question's answer, `Pending` or `ErrNoRemote` |401+| `Paged(sheet)` | whether a sheet is a linked source's tab, whose ranges a function is given whole |402+| `Stream(call)` | the answer to a call over linked sources (`StreamCall`), `Pending` while it's worked out |354403 355404 Sheets are named as references write them, so resolving names, missing356405 sheets (`#REF!`), cycles and the depth limit stay the engine's. The@@ -495,6 +544,7 @@ draw. The components:
495544 | mouse | `mouseState` | drags, hover, double clicks, the fill handle (`mouse.go`, `fill.go`) |496545 | import | `transfer.Transfer`, `pipeState` | the import in progress, its progress display and cancelling (package `transfer`); choosing and placing imports (`transfer.go`, `importplace.go`); standard input read as a sheet, and what a pipeline gets on quitting (`pipe.go`) |497546 | linked files | `followState` | a source (`live.Source`) for each linked region, polled one poll at a time on a command every `live.Interval` and reconciled with the workbook after every update, so undo, opening a file and unlinking need nothing of their own; trust in files outside a workbook's folder (`follow.go`); Data > Linked file and Import's Follow the file (`linked.go`) |547+| linked sources | `sourceState` | the workbook's `paged.Host` (a room's in `012 serve`, run by whoever keeps the room) and its jobs, run a few at a time on commands, their answers applied as they come; each file looked at every `live.Interval` (`sources.go`); the tab shown, package `srcview`, its pages read as it scrolls, and the commands that act on a source (`srcscreen.go`, `srccmds.go`) |498548 | notebooks | `nbview.View`, `nbState` | a view of each notebook tab, its selection, mode, scroll and editor (package `nbview`); the tab on the screen (`nbscreen.go`); the notebook's commands and keys (`notebook.go`); cells running one at a time in the background, their queue, stale outputs, trust (`nbrun.go`); what a run reads (`nbjob.go`); outputs sent to sheets (`nbsend.go`) |499549 | macros | `recorder`, `macroState` | a recording in progress (`macrorec.go`); a macro running, trust in the file's macros (`macrorun.go`); what scripts act on (`macrohost.go`, `macrohostnav.go`); Data > Macros and the manager (`macro.go`, `macromanage.go`) |500550 | others | `clipboard`, `trace`, `traceView`, `chartState`, `jevRunner`, `terminal`, `session` | what Ctrl+V pastes, a trace being stepped through and tracing that stays on (`traceview.go`), chart commands' target, JEV questions in flight, what the terminal supports and the chart images sent to it, what outlasts the file open (the `:` history, whether keys can be held: `keyboard.go`) |@@ -619,6 +669,18 @@ cell run as a stream (`live.Stream`), whose pipeline nu runs printing
619669 each value as NUON on a line of its own (`nushell.Stream`), read by the620670 same `Tail` as it arrives.621671 672+A `Source` (`source.go`) is a Parquet file or a SQLite table or query673+read in place, for [linked sources](../files/sources.md): it streams674+the columns asked for (Parquet reads only their column chunks, from the675+row group holding the first row), fetches rows by number, and orders676+them as a `SourceView`, pushed down to SQL for SQLite (the rowids in677+order in a work database of its own, the source opened read-only) and678+streamed for Parquet (a filter's matches found in one pass, a sort679+merged from sorted runs in temporary files, either kept as a file of680+row numbers a page reads). Values come out typed as an import types681+them (`sourcecell.go`), and every method is safe from several682+goroutines at once.683+622684 XLSX is read by 012's own SpreadsheetML reader on `archive/zip` and623685 `encoding/xml`: the workbook, shared strings (kept end to end in one624686 buffer) and styles first, then each worksheet a token at a time, a row@@ -39,6 +39,7 @@ it lags, and past a second it stalls.
3939 | Find, filter, sort, fill | Filter or sort 8191 rows: 1 to 32 ms; find over 213 k cells: 38 ms; fill 8192 rows: 4 ms | Replace all over 213 k cells: 280 ms | | Per-cell string conversion and regexp |4040 | SSH sessions (`012 serve`) | 50 sessions typing at once: frames within one frame interval (p95 8.7 ms), 1.3 MiB per session plus its sheets (upper bound) | | More than `--max-sessions` (8 by default): turned away | Bubble Tea's 120 fps pacing; per session, the terminal's cell buffers |4141 | Pivot tables | A pivot over 8191 rows recomputes after an edit to its source in 0.8 to 2.5 ms | | Results past the grid (the pivot shows #REF!) | Grouping the source rows |42+| Linked sources | Ten million rows of Parquet or SQLite: opened in 0.4 ms or 0.65 s, a frame scrolling them 0.9 ms at 200 x 60, a page anywhere 0.2 to 12 ms; SUM of a column 0.42 s (Parquet) or 1.4 s (SQLite), COUNTIFS 1.2 or 2.3 s, a pivot table 2.5 or 3.9 s, each in a few MB | Sorting ten million rows: 3.5 s (Parquet) or 19 s (SQLite), in the background | Functions that hold what they read, over more than `max-cells` cells: `#VALUE!`, saying why | Reading the column from the file; sorting |4243 | Following files | A log growing by 10,000 rows a second, formulas over it: each update (about 2,500 rows, four times a second) applied in 7 ms keeping every row, 1.7 to 2 ms keeping the last 1000, 14 ms the last 10,000; frames 2.5 to 3.6 ms at 200 x 60 | | Rows past `max-cells`: left out, with a note | Writing the rows and recalculating what reads them; a window rewrites its rows on every update |4344 | Arrays and spills | FILTER, SORT or UNIQUE over 8192 rows: about 0.1 ms each per edit; 1000 of them spilling 516 k cells, an edit recomputing 500: 52 ms | Full recalculation of those 1000: 106 ms | An array past `maxArray` values (2,097,152) stored: `#VALUE!`; a spill past the sheet's edge or `max-cells` cells: `#REF!` | Computing each array's values; writing only the spilled cells that changed |4445 @@ -373,6 +374,54 @@ change; a growing file is read a megabyte a poll at most, so a large
373374 file loads over several frames. Rows are held once, in the region's374375 cells; a window reads the rows it keeps back from them.375376 377+## Linked sources378+379+A [linked source](../files/sources.md) is read in place: its tab keeps380+at most 48 pages of 128 rows, and what formulas and pivot tables ask381+of it is worked out by streaming the file in the background, a few382+questions at a time, each answer kept until the file changes. None of383+it costs what the source's size does in memory. Measured on ten million384+rows of sales (an id, one of eight categories, an amount and a date;385+`stress.SalesParquet` and `stress.SalesSQLite`, 204 MB and 385 MB), one386+run each, the peak heap above what was live before:387+388+| Reading the file (`BenchmarkSource`, `internal/fileio`) | Parquet | SQLite |389+|---|---|---|390+| Open | 0.39 ms, 0.2 MB | 0.65 s (counting the rows) |391+| Scan one column | 0.35 s (28.5 M rows/s), 4.5 MB | 1.36 s (7.4 M rows/s), 3.4 MB |392+| A page of 60 rows deep in, in the source's order | 0.28 ms | 0.19 ms |393+| Sorting by a column, Z to A | 3.5 s, 113 MB | 19.1 s, in SQLite's own memory and files |394+| A page deep in, sorted | 12 ms (the rows are scattered) | 0.41 ms |395+| Filtering to one category (1,248,497 rows) | 1.26 s, 6.7 MB | 1.19 s |396+| A page deep in, filtered | 0.23 ms | 0.23 ms |397+398+| Answering a question (`BenchmarkSourceFormulas`, `internal/paged`) | Parquet | SQLite |399+|---|---|---|400+| `SUM`, `AVERAGE` of a column | 0.42 s, 3.8 MB | 1.42 s, 2.6 MB |401+| `COUNTIFS`, `SUMIFS` over two columns | 1.2 s, 8.3 MB | 2.3 s, 7.5 MB |402+| `SUMPRODUCT` of two columns | 0.44 s, 9 MB | 1.56 s, 7.7 MB |403+| `XLOOKUP` of the last id | 0.62 s, 5.9 MB | 1.28 s, 4.3 MB |404+| `MATCH` of the middle id | 0.31 s, 5.9 MB | 0.63 s, 3.8 MB |405+| `MEDIAN`, which holds every value: ten million cells are within `max-cells` | 1.15 s, 248 MB | 2.15 s, 247 MB |406+| A pivot table: SUM and COUNT of the amounts by category | 2.45 s, 5.9 MB | 3.9 s, 2.7 MB |407+408+SQLite's own memory, through modernc's libc, isn't in the Go heap.409+Functions over aligned ranges (the criteria functions, `SUMPRODUCT`)410+read their ranges a window of 2048 rows at a time, each column411+streamed by a goroutine of its own in batches of 4096 rows. A function412+that holds what it reads is refused past `max-cells` cells before it413+reads any, so `MEDIAN` of a hundred million values says so at once414+rather than holding them.415+416+Scrolling (`BenchmarkSourceScroll`, `internal/ui`) draws a frame in417+0.26 ms at 80 x 24 and 0.9 ms at 200 x 60 within the pages read, as a418+sheet's frame costs. A jump anywhere, as dragging the scrollbar makes,419+reads the pages it shows and a window either side, in the background420+while the frames go on showing `…`: 0.64 ms at 80 x 24 and 1.5 ms at421+200 x 60, the pages read and the frame drawn.422+The speed gate holds a frame on a 200,000-row source and a SUM over it423+to their baselines (`frame/source-200000x4`, `source/sum-200000`).424+376425 ## Undo377426 378427 Each step keeps the cells it changed as they were, in the form the@@ -107,6 +107,7 @@ there.
107107 | Entries failing validation | A dotted underline, and `Invalid:` with the rule on the context line |108108 | Values an array spilled | Italic, and `Spilled from B2` on the context line |109109 | Linked files' rows, following, paused or failing | Italic; `●`, `‖` or `!` beside the tab's name and on the context line, with the state in words (`Following`, `Paused`, the error) |110+| A linked source's tab, its scrollbar, rows on their way | `▦` before the tab's name and on the context line, `!` after it when the file can't be read; the scrollbar's thumb `┃` on a track of `│`; `…` in place of a row being read |110111 | A notebook's cells and outputs | Marks and words, listed in [Notebooks](../nushell/notebooks.md#reading-a-cell): the bar left of the active cell, a heavy box while it's edited, the prompt's and the box's words for its state, `×` before an error, `OUTPUT` in an output's grid, an output sent to a sheet in italic |111112 | A table's header row | Bold and underlined, when the table styles its header; `Table Sales, column Amount` on the context line |112113 | Pivot table results | Their headings (`SUM of Units`, `Grand Total`), and a note in words when an edit is refused |@@ -114,7 +115,7 @@ there.
114115 | Notes | A `▝` in the cell's top-right corner |115116 | Others in a shared workbook (012 serve) | Their pointer's cell double-underlined, their initial on its row's header, their name on the status line, `▘` in a cell they changed in the last 30 seconds, and `Changed by`, `is here` or their warnings in words on the context line |116117 | Macro recording | `REC` beside the mode indicator |117-| Mode | The mode indicator's word (`READY`, `ENTER`, `POINT`, `MENU`, and on a notebook `NOTEBOOK`, `EDIT` or `OUTPUT`) |118+| Mode | The mode indicator's word (`READY`, `ENTER`, `POINT`, `MENU`, on a notebook `NOTEBOOK`, `EDIT` or `OUTPUT`, and on a linked source's tab `SOURCE`) |118119 | Search matches, traced cells | Reverse video, with a count or the list on the context line; dependents also bold |119120 | Evaluate formula | The part computed next underlined and bold, values in its place italic, and `Next` and `Value` in words |120121 | The copied range | A dashed underline |@@ -38,7 +38,8 @@ SQLite table). Formats that keep no column widths (all but XLSX and
3838 1-2-3) have their columns fitted to the header and the first 200 rows,3939 at most 30 characters wide, as are tables read from standard input. A file can also be followed instead, its table in a4040 linked region that takes in new rows as the file grows or is rewritten:41-see [Following files](following.md). A table can also come in on standard input, and go back41+see [Following files](following.md). A Parquet file or SQLite table too big to import links as a42+[source](sources.md), read in place. A table can also come in on standard input, and go back4243 out on standard output: see [Pipelines](../nushell/pipelines.md).4344 4445 File > Import asks where the data goes, as Sheets' Import location does@@ -98,6 +99,7 @@ line how many rows it left out.
9899 | [Excel files](excel.md) | What comes in from and goes out to `.xlsx` |99100 | [Saving](saving.md) | Atomic saves, overwrite checks, unsaved work under `012 serve` |100101 | [Following files](following.md) | Linked regions that follow a file as it grows or is rewritten, like `tail -f` |102+| [Linked sources](sources.md) | Parquet files and SQLite tables too big for any grid, read in place on a tab of their own, with formulas and pivot tables streaming over them |101103 | [The .012 format](format.md) | The JSON format, its versions and fields |102104 | [Scripts](scripts.md) | `012 get`, `set`, `recalc` and `export`: workbooks read and changed without the screen |103105 | [Diff and merge in git](git.md) | `012 diff` cell by cell, and git's diff and merge drivers for `.012` files |@@ -22,6 +22,10 @@ region recalculate as rows arrive.
2222 file. On a new, empty spreadsheet File > Import opens the file instead,2323 as it always does; link it with Data > Linked file there.2424 25+A linked file's rows are cells, up to `max-cells`. A Parquet file or a26+SQLite table too big for that links as a [source](sources.md) instead,27+read in place a page at a time.28+2529 For a SQLite database with several tables, a picker asks which one. Then2630 a question on the context line asks how many rows to keep:2731 @@ -0,0 +1,175 @@
1+---2+title: "Linked sources"3+sidebar_position: 84+---5+6+# Linked sources7+8+A Parquet file, or a SQLite table or query, too big for any grid links9+as a source: a read-only table on a tab of its own, read in place10+rather than imported. The tab shows a window of the rows as it11+scrolls, with the row numbers and the scrollbar over every row, and12+formulas and pivot tables read the whole source by streaming it. A13+source of ten million rows opens at once and takes a few MB.14+15+16+17+18+## Linking a source19+20+**Data > Linked file > Link a source** lists the Parquet files and21+SQLite databases in the folder, or takes a path. For a database a22+second list asks which table or view to read, or **A query…** to read23+what a `SELECT` returns.24+25+The source gets a tab after the one shown, named after the file26+(`sales` for `sales.parquet`), with `▦` before its name. The source is27+a [table](../sheets/tables.md) of that name, which formulas on any sheet28+read as a table's:29+30+| Formula | Reads |31+|---|---|32+| `=SUM(sales[amount])` | a column's values, every row |33+| `=ROWS(sales)` | the rows under the header |34+| `=COUNTA(nu.sales)` | the whole table, header row included |35+| `=sales!C5` | one cell: row 1 is the header, so row 5 is the source's fourth row |36+37+Renaming the tab keeps the table's name. Undo takes the link back. A38+source's rows may run past the grid's last row, 1,048,576: its table's39+name reaches them all, where a reference typed as `C5` stops at the40+grid's edge.41+42+## The tab43+44+Row 1 is the header, the columns' names, and stays put as the rows45+scroll under it. The rows are numbered as the tab's rows, so the46+source's first row is row 2, and `sales!C2` is its first amount. The47+scrollbar at the right edge is as long as the share of the rows shown,48+and as far down as they are.49+50+| Key or mouse | Does |51+|---|---|52+| Arrows, Tab | Move the active cell |53+| Page Up, Page Down, the wheel | Scroll |54+| Home, End | The first or last column |55+| Ctrl+Up, Ctrl+Down | The first or last row |56+| Ctrl+Home, Ctrl+End | The first cell, the last |57+| A click on the scrollbar, or dragging its thumb | Jumps there |58+| Ctrl+C | Copies the active cell |59+60+The rows come from the file a page (128 rows) at a time, in the61+background, the pages either side of the window read ahead, so62+scrolling finds them there. A row on its way shows `…`. The name box63+shows the active cell as the tab names it, and the formula bar its64+value; the context line says what the source is:65+`▦ sales.parquet 10,000,000 rows, 4 columns`. The mode is `SOURCE`,66+and typing, pasting or formatting there says the tab is read-only.67+68+## Sorting and filtering69+70+The tab's rows sort and filter as a sheet's do, without changing the71+file:72+73+- **Data > Sort sheet A to Z** (or **Z to A**) sorts by the active74+ column; blanks go last either way.75+- **Data > Create a filter** or **Filter by column** asks for a76+ condition on the active column: the conditions of a sheet's77+ [filter](../sheets/sort-filter.md#filter). There is no list of values78+ to tick: a source's are too many to list. Each column's condition79+ applies at once.80+- **Data > Remove filter** drops the conditions, and **Data > Linked81+ file > Original order** shows the source's own rows in its own order.82+83+Each is an undo step and is saved with the workbook. Formulas read the84+source in its own order, every row, as formulas read the rows a sheet's85+filter hides.86+87+Ordering reads the whole source once, in the background, while the88+context line says `sorting and filtering…`; the tab then shows the89+first of the rows. SQLite does the work: the sort and the conditions90+become SQL, the rows' order is kept in a temporary database beside the91+system's temporary files, and the database itself is never written. A92+Parquet file is streamed: a filter is one pass over the columns it93+tests, and a sort writes sorted runs to temporary files and merges94+them, so memory stays a few MB and up to about 100 MB while sorting,95+whatever the size. Either way a page anywhere in the ordered rows96+costs what one at the top does:97+98+```mermaid99+flowchart TD100+ order[a sort or a filter] -->|once, in the background| view[the rows' order: SQL into a temporary table, or streamed into temporary files]101+ scroll[the tab scrolls] --> want[the pages around the window]102+ view --> want103+ want -->|a page not read| read[read 128 rows at their positions]104+ read --> kept[kept, the least used let go past 48 pages]105+ kept --> frame[the next frame draws them]106+```107+108+In SQLite a text condition compares the text as SQLite writes it, so a109+number reads as `1e+20` where the sheet shows `1E+20`, a date held as110+text (`2026-09-26`) sorts and compares as that text, and letters beyond111+ASCII are compared by case.112+113+## Formulas over a source114+115+A function given a source's range is worked out in the background, by116+reading the source; its cell shows `Loading…` until then. The answer117+is kept until the file changes, so another formula asking the same, or118+the same formula recalculating, costs nothing.119+120+These read a source of any size in one pass, holding only what they121+compute:122+123+| Kind | Functions |124+|---|---|125+| Aggregates | SUM, AVERAGE, COUNT, COUNTA, MIN, MAX, PRODUCT |126+| Criteria | COUNTIF, COUNTIFS, SUMIF, SUMIFS, AVERAGEIF, AVERAGEIFS, COUNTBLANK, SUMPRODUCT |127+| Lookups | MATCH, XLOOKUP, VLOOKUP, HLOOKUP, INDEX, ROWS, COLUMNS |128+129+Any other function (MEDIAN, SORT, FILTER, UNIQUE, TEXTJOIN) holds what130+it reads, and is given a source's range as long as that holds no more131+than `max-cells` cells ([Configuration](../reference/config.md#max-cells));132+past it the formula shows `#VALUE!`, and the context line says what it133+would have read. So does an operator over a source's range134+(`=SUM(sales[amount]*2)`) and a function whose other arguments are135+ranges of a sheet or arrays: `SUMPRODUCT((sales[cat]="north")*sales[amount])`136+computes an array as long as the source, where137+`SUMIFS(sales[amount],sales[cat],"north")` streams.138+139+## Pivot tables140+141+**Data > Pivot table** on a source's tab makes a142+[pivot table](../sheets/pivots.md) over the whole source, and **Data >143+Frequency table (column stats)** one of the active column. The source's rows are144+grouped in the background, `Loading…` showing in the pivot table's145+first cell until they are, and again whenever the file changes. A146+filter of a pivot table over a source takes a condition, not values147+ticked.148+149+## When the file changes150+151+012 looks at each source's file four times a second, with the152+operating system's file information alone. Once the file has changed153+and held still for 0.3 s, the source is read again: the tab, and every154+formula and pivot table reading it. **Data > Linked file > Read the155+source again** does the same at once. A file that isn't there shows156+`!` after the tab's name and the reason on the context line, and the157+formulas reading it show `#REF!` with it.158+159+## Saving and sharing160+161+The workbook keeps the link, never the rows: the file's path relative162+to the workbook's folder, the table or query, and the tab's order163+([The .012 format](format.md#linked-sources)). Opening the workbook164+reads the sources again. A workbook from another computer asks before165+reading files outside its own folder, as for166+[linked files](following.md#files-from-elsewhere).167+168+`012 get`, `recalc` and `export` ([Scripts](scripts.md)) read the169+sources in the workbook's folder or below, to the end, before they170+answer. Under `012 serve` sources are read in the served folder, and in171+a [shared workbook](../terminal/ssh.md#sharing-a-workbook) the room reads them, as it follows linked172+files, while each participant scrolls a tab of their own.173+174+What sources cost, measured on ten million rows, is in175+[Bounds of support](../contributing/limits.md#linked-sources).@@ -144,11 +144,20 @@ func (p *Pages) Store(j *PageJob) bool {
144144 }145145 return true146146 }147- delete(p.want, j.n)148- if j.err != nil || j.view != p.view {147+ if j.view != p.view {148+ delete(p.want, j.n)149149 return false150150 }151- p.pages[j.n] = &page{nums: j.nums, rows: j.rows}151+ if j.err != nil {152+ // Still wanted, so not asked for again until the view is153+ // made again: a page that can't be read would be read over154+ // and over.155+ p.err = describe(j.err)156+ return true157+ }158+ delete(p.want, j.n)159+ p.used++ // as the page most lately used, so it isn't the one let go160+ p.pages[j.n] = &page{nums: j.nums, rows: j.rows, used: p.used}152161 p.drop()153162 return true154163 }@@ -0,0 +1,96 @@
1+package paged2+3+import (4+ "context"5+ "path/filepath"6+ "testing"7+8+ "github.com/FelineStateMachine/012/internal/fileio"9+ "github.com/FelineStateMachine/012/internal/sheet"10+)11+12+// runPages runs the pages' jobs until none is left, reporting how many13+// ran.14+func runPages(p *Pages) int {15+ n := 016+ for jobs := p.Jobs(); len(jobs) > 0; jobs = p.Jobs() {17+ for _, j := range jobs {18+ j.Run(context.Background())19+ p.Store(j)20+ n++21+ }22+ }23+ return n24+}25+26+// openSales writes n rows to a Parquet file and opens it as a source.27+func openSales(t *testing.T, n int) fileio.Source {28+ t.Helper()29+ name := filepath.Join(t.TempDir(), "sales.parquet")30+ writeSales(t, name, n, 0)31+ h, err := fileio.OpenSource(context.Background(), fileio.SourceSpec{Path: name})32+ if err != nil {33+ t.Fatal(err)34+ }35+ t.Cleanup(func() { h.Close() })36+ return h37+}38+39+func TestPagesKeepTheLatest(t *testing.T) {40+ h := openSales(t, 100*PageRows)41+ p := NewPages(h, 0, nil)42+ defer p.Close()43+ if runPages(p) != 1 {44+ t.Fatal("the view wasn't built once")45+ }46+ if n, ok := p.Rows(); !ok || n != 100*PageRows {47+ t.Fatalf("%d rows, %v", n, ok)48+ }49+ for i := range int64(90) {50+ row := i * PageRows51+ p.Want(row, row)52+ if got := runPages(p); got != 1 {53+ t.Fatalf("page %d took %d reads", i, got)54+ }55+ if num, vals, ok := p.Row(row); !ok || num != row || vals[0].V.Num != float64(row) {56+ t.Fatalf("row %d: %d %v %v", row, num, vals, ok)57+ }58+ }59+ if len(p.pages) > keepPages {60+ t.Errorf("%d pages kept", len(p.pages))61+ }62+ if _, _, ok := p.Row(0); ok {63+ t.Error("the least used page is kept past the budget")64+ }65+}66+67+func TestPagesSortedAndFiltered(t *testing.T) {68+ h := openSales(t, 1000)69+ o := &sheet.SourceOrder{70+ Sort: []sheet.SourceSort{{Col: 0, Desc: true}},71+ Filter: []sheet.SourceFilter{{Col: 1, Cond: sheet.Condition{Op: sheet.CondExactly, Arg: "north"}}},72+ }73+ p := NewPages(h, 0, o)74+ defer p.Close()75+ if !p.Building() {76+ t.Fatal("an ordered view isn't built")77+ }78+ runPages(p)79+ n, ok := p.Rows()80+ if !ok || n == 0 || n >= 1000 {81+ t.Fatalf("%d rows pass, %v", n, ok)82+ }83+ p.Want(0, n-1)84+ runPages(p)85+ prev := int64(1 << 62)86+ for i := range n {87+ num, vals, ok := p.Row(i)88+ if !ok || num >= prev || !(vals[1].V.Str == "north" || vals[1].V.Str == "North") {89+ t.Fatalf("row %d of the view: %d %v %v", i, num, vals, ok)90+ }91+ prev = num92+ }93+ if !p.Same(h, 0, o) || p.Same(h, 0, nil) || !p.Of(h, 0) {94+ t.Error("Same and Of")95+ }96+}@@ -2,11 +2,13 @@ package ui
22 33 import (44 "os"5+ "path/filepath"56 "strings"67 "testing"78 89 "github.com/parquet-go/parquet-go"910 11+ "github.com/FelineStateMachine/012/internal/room"1012 "github.com/FelineStateMachine/012/internal/sheet"1113 )1214 @@ -177,3 +179,80 @@ func TestSourceCommandsOnTheTab(t *testing.T) {
177179 t.Errorf("each id is there once: %v", v)178180 }179181 }182+183+// The files' ticks come at once in tests, whose ticks are dropped.184+func init() { sourceInterval = 0 }185+186+// A source linked in a room is the room's: the one who keeps the room187+// reads it and answers what anyone's formulas ask of it, each188+// participant scrolls a tab of their own, and whoever keeps the room189+// next goes on reading it.190+func TestSharedSources(t *testing.T) {191+ r := newRooms(t, room.Edit)192+ sum := writeSource(t, filepath.Join(r.dir, "sales.parquet"), 1000)193+ ann := r.open("ann", "@src")194+ bob := r.open("bob", "@src")195+ bob.run(bob.m.runCommand("data.link_source"))196+ bob.press("sales.parquet", "<enter>")197+ for range 5 {198+ r.sync()199+ }200+ if !strings.Contains(bob.screen(), "North") {201+ t.Fatalf("bob's tab:\n%s", bob.screen())202+ }203+ bob.press("<ctrl+down>")204+ r.sync()205+ if !strings.Contains(bob.screen(), "1001 ") || strings.Contains(ann.screen(), "1001 ") {206+ t.Errorf("bob's scroll isn't his own:\n%s\n%s", bob.screen(), ann.screen())207+ }208+ bob.press("<ctrl+pgup>", "=SUM(sales[amount])", "<enter>")209+ for range 5 {210+ r.sync()211+ }212+ var got float64213+ ann.sh.turn(func() { got = ann.m.book().Sheet(0).Value(addr("A1")).Num })214+ if got != sum {215+ t.Fatalf("SUM %v, want %v", got, sum)216+ }217+ if ann.m.LeaveRoom() {218+ t.Fatal("ann was the last")219+ }220+ r.all = r.all[1:]221+ r.sync()222+ bob.press(`=COUNTIF(sales[region],"east")`, "<enter>")223+ for range 5 {224+ r.sync()225+ }226+ bob.sh.turn(func() { got = bob.m.book().Sheet(0).Value(addr("A2")).Num })227+ if got != 250 {228+ t.Errorf("bob, keeping the room now, answered COUNTIF with %v", got)229+ }230+}231+232+func TestMonochromeSource(t *testing.T) {233+ t.Chdir(t.TempDir())234+ m := newModel()235+ linkSales(t, m, 3000)236+ m.note = ""237+ row := monoLine(m, headerLine+3) // the first row, its first cell active238+ if !every(row[6:12], false, isReverse) {239+ t.Errorf("the active cell isn't in reverse video: %+v", row[6:12])240+ }241+ if last := row[len(row)-1]; last.text != "┃" {242+ t.Errorf("the scrollbar's thumb at the top is %q", last.text)243+ }244+ if last := monoLine(m, headerLine+6); last[len(last)-1].text != "│" {245+ t.Errorf("the scrollbar's track is %q", last[len(last)-1].text)246+ }247+ if l := cellText(monoLine(m, contextLine)); !strings.Contains(l, "▦ sales.parquet") {248+ t.Errorf("context line: %q", l)249+ }250+ if err := os.Remove("sales.parquet"); err != nil {251+ t.Fatal(err)252+ }253+ m.sources().host.Reload("sales")254+ pumpSources(t, m)255+ if l := cellText(monoLine(m, m.height-1)); !strings.Contains(l, "▦ sales ! ") {256+ t.Errorf("status line of a source whose file is gone: %q", l)257+ }258+}@@ -154,16 +154,14 @@ func (m *Model) sourceCounts(info sheet.SourceInfo) string {
154154 total := int64(info.Shape.Rows)155155 out := transfer.Thousands(int(total)) + plural(int(min(total, 2)), " row", " rows") + ", " +156156 transfer.Thousands(len(info.Shape.Cols)) + plural(len(info.Shape.Cols), " column", " columns")157- o := info.Source.Order158- if o.IsZero() {159- return out160- }161- p := m.src.pages[info.Name]157+ o, p := info.Source.Order, m.src.pages[info.Name]162158 switch {159+ case p != nil && p.Err() != "":160+ return out + "; " + p.Err()161+ case o.IsZero():162+ return out163163 case p == nil || p.Building():164164 return out + "; sorting and filtering…"165- case p.Err() != "":166- return out + "; " + p.Err()167165 }168166 var parts []string169167 for _, s := range o.Sort {