cat / atlas
clone

an offline map explorer where each world, from a game map to a city's open data, ships as one .atlas file

feat: demand-read native Atlas worlds

Dami committed 871c596
Keep current vNext page zero on outline/index metadata, page visible features through bounded continuation, read raster tiles by verified shard range, preserve exact typed values across the browser seam, and retain honest legacy framing fallback.

Adds demand telemetry and coverage across Go, browser, OS-host, and island boundaries.
commit
871c5962fa6a8eece5a7819087c53cf00980ba9d
parent
a7ab0254919119ce3cbdde68bccc92c9c929bf0f
author
Dami <dami.etoile@gmail.com>
26 files changed, +1462 -107
Mformat/vnext/container.go5+-
Mformat/vnext/container_test.go69++++++++
Mformat/vnext/partition.go101++++++++++++
Mformat/vnext/storage_generation_test.go4+
Minternal/app/app_test.go187++++++++++++++++++++++-
Minternal/app/data.go193+++++++++++++++++++++++-
Minternal/app/hostenv/hostenv.go15++-
Minternal/app/hostenv/oshost/oshost_test.go50++++++
Minternal/app/hostenv/oshost/volumes.go78+++++++---
Ainternal/app/outline_wire.go236+++++++++++++++++++++++++++++
Minternal/app/pages.go14+-
Minternal/app/session.go14++-
Minternal/app/view.go21++-
Minternal/app/vnext_projection_test.go29+++-
Minternal/app/vnext_runtime.go44+++++-
Minternal/app/world.go61++++++-
Mrender/data/payload.ts4+
Mrender/data/plane.ts248++++++++++++++++++++++++++----
Mrender/data/semantic.ts8+-
Arender/data/window.ts15++
Arender/test/demand.test.ts89+++++++++++
Mrender/test/legacy.ts3+-
Mrender/test/vnext.test.ts19++-
Mrender/viewport.ts12+-
Mrender/world/model.ts9+-
Mtests/island/harness_test.go41+++++-
M format/vnext/container.go +3 -2
@@ -18,7 +18,8 @@ import (
1818 const (1919 manifestName = "atlas.json"2020 containerFormat = "atlas-schema-bundle"21- containerFraming = uint16(1)21+ minimumFraming = uint16(1)22+ containerFraming = uint16(2)2223 maxManifestSize = 32 << 202324 )2425
@@ -347,7 +348,7 @@ func readBootstrap(entries map[string]*zip.File) (bootstrap, error) {
347348 if err := decoder.Decode(&struct{}{}); err != io.EOF {348349 return bootstrap{}, fmt.Errorf("bootstrap carries trailing data")349350 }350- if manifest.Format != containerFormat || manifest.Framing != containerFraming {351+ if manifest.Format != containerFormat || manifest.Framing < minimumFraming || manifest.Framing > containerFraming {351352 return bootstrap{}, fmt.Errorf("unsupported Atlas container identity or framing")352353 }353354 if err := ValidSlug(manifest.Volume); err != nil {
M format/vnext/container_test.go +69 -0
@@ -3,6 +3,9 @@ package vnext
33 import (44 "archive/zip"55 "bytes"6+ "crypto/sha256"7+ "encoding/hex"8+ "encoding/json"69 "io"710 "strings"811 "testing"
@@ -52,6 +55,22 @@ func TestBundleIsDeterministicAndKeepsPackedDataStored(t *testing.T) {
5255 }5356 }5457 58+func TestReaderAcceptsKnownContainerFramings(t *testing.T) {59+ t.Parallel()60+61+ current := nativeBytes(t, minimalVolume())62+ for _, framing := range []uint16{minimumFraming, containerFraming} {63+ data := rewriteFraming(t, current, framing)64+ if _, err := Open(bytes.NewReader(data), int64(len(data)), StandardSchema()); err != nil {65+ t.Fatalf("open framing %d: %v", framing, err)66+ }67+ }68+ unknown := rewriteFraming(t, current, containerFraming+1)69+ if _, err := Open(bytes.NewReader(unknown), int64(len(unknown)), StandardSchema()); err == nil || !strings.Contains(err.Error(), "unsupported") {70+ t.Fatalf("unknown framing accepted: %v", err)71+ }72+}73+5574 func TestNewerApplicationReadsOlderFileSchema(t *testing.T) {5675 t.Parallel()5776
@@ -198,6 +217,56 @@ func rewriteAtlas(t *testing.T, source []byte, mutate func(*zip.FileHeader), orp
198217 return output.Bytes()199218 }200219 220+func rewriteFraming(t *testing.T, source []byte, framing uint16) []byte {221+ t.Helper()222+ archive, err := zip.NewReader(bytes.NewReader(source), int64(len(source)))223+ if err != nil {224+ t.Fatal(err)225+ }226+ var output bytes.Buffer227+ writer := zip.NewWriter(&output)228+ for _, entry := range archive.File {229+ reader, err := entry.Open()230+ if err != nil {231+ t.Fatal(err)232+ }233+ data, err := io.ReadAll(reader)234+ reader.Close()235+ if err != nil {236+ t.Fatal(err)237+ }238+ if entry.Name == manifestName {239+ var manifest bootstrap240+ if err := json.Unmarshal(data, &manifest); err != nil {241+ t.Fatal(err)242+ }243+ manifest.Framing = framing244+ manifest.Release.Stamp = ""245+ stampInput, err := json.Marshal(manifest)246+ if err != nil {247+ t.Fatal(err)248+ }249+ stamp := sha256.Sum256(stampInput)250+ manifest.Release.Stamp = hex.EncodeToString(stamp[:])251+ data, err = json.Marshal(manifest)252+ if err != nil {253+ t.Fatal(err)254+ }255+ }256+ destination, err := writer.CreateHeader(&entry.FileHeader)257+ if err != nil {258+ t.Fatal(err)259+ }260+ if _, err := destination.Write(data); err != nil {261+ t.Fatal(err)262+ }263+ }264+ if err := writer.Close(); err != nil {265+ t.Fatal(err)266+ }267+ return output.Bytes()268+}269+201270 func minimalVolume() Volume {202271 return Volume{203272 ID: "minimal", Title: "Minimal",
M format/vnext/partition.go +101 -0
@@ -68,6 +68,59 @@ type FeaturePageResult struct {
6868 BytesRead int646969 }7070 71+// FeatureSetSummary is page-zero cardinality derived from the physical index;72+// reading it never opens a feature partition.73+type FeatureSetSummary struct {74+ FeatureSet string75+ Rows int76+}77+78+// DemandAddressedFeatures reports whether feature rows are physically79+// partitioned instead of carried in the legacy monolithic tables.80+func (reader *Reader) DemandAddressedFeatures() bool {81+ _, held := reader.blobs[featureIndexName]82+ return held83+}84+85+// FeatureSetSummaries returns canonical per-set row counts from the physical86+// partition index without materializing feature rows.87+func (reader *Reader) FeatureSetSummaries() ([]FeatureSetSummary, error) {88+ index, err := reader.featureIndex()89+ if err != nil {90+ return nil, err91+ }92+ if _, indexed := reader.blobs[featureIndexName]; !indexed {93+ volume, err := reader.Volume()94+ if err != nil {95+ return nil, err96+ }97+ var out []FeatureSetSummary98+ for _, world := range volume.Worlds {99+ for _, set := range world.FeatureSets {100+ out = append(out, FeatureSetSummary{FeatureSet: set.ID, Rows: len(set.Features)})101+ }102+ }103+ slices.SortFunc(out, func(left, right FeatureSetSummary) int {104+ return strings.Compare(left.FeatureSet, right.FeatureSet)105+ })106+ return out, nil107+ }108+ bySet := make(map[string]int)109+ for _, partition := range index.Partitions {110+ bySet[partition.FeatureSet] += partition.Rows111+ }112+ sets := make([]string, 0, len(bySet))113+ for set := range bySet {114+ sets = append(sets, set)115+ }116+ slices.Sort(sets)117+ out := make([]FeatureSetSummary, 0, len(sets))118+ for _, set := range sets {119+ out = append(out, FeatureSetSummary{FeatureSet: set, Rows: bySet[set]})120+ }121+ return out, nil122+}123+71124 func partitionFeatureTables(volume Volume, tables map[string]Table) ([]NamedTable, Blob, error) {72125 features := tables[featureTableName]73126 relationships := tables[relationshipTableName]
@@ -393,6 +446,9 @@ func (reader *Reader) FeaturePage(request FeaturePageRequest) (FeaturePageResult
393446 if err != nil {394447 return FeaturePageResult{}, err395448 }449+ if _, indexed := reader.blobs[featureIndexName]; !indexed {450+ return reader.legacyFeaturePage(request, after)451+ }396452 result := FeaturePageResult{}397453 passedAfter := after == ""398454 for _, partition := range index.Partitions {
@@ -429,6 +485,51 @@ func (reader *Reader) FeaturePage(request FeaturePageRequest) (FeaturePageResult
429485 return result, nil430486 }431487 488+func (reader *Reader) legacyFeaturePage(request FeaturePageRequest, after string) (FeaturePageResult, error) {489+ volume, err := reader.Volume()490+ if err != nil {491+ return FeaturePageResult{}, err492+ }493+ result := FeaturePageResult{PartitionsRead: 1}494+ for _, name := range []string{featureTableName, relationshipTableName, provenanceTableName} {495+ result.BytesRead += reader.tables[name].Length496+ }497+ passedAfter := after == ""498+ foundSet := false499+ for _, world := range volume.Worlds {500+ for _, set := range world.FeatureSets {501+ if set.ID != request.FeatureSet {502+ continue503+ }504+ foundSet = true505+ for _, feature := range set.Features {506+ if !passedAfter {507+ passedAfter = feature.ID == after508+ continue509+ }510+ if request.Bounds != nil {511+ bounds, err := geometryBounds(feature.Geometry)512+ if err != nil || !intersects(bounds, *request.Bounds) {513+ continue514+ }515+ }516+ result.Features = append(result.Features, feature)517+ if len(result.Features) == request.Limit {518+ result.Next = encodeContinuation(request, feature.ID)519+ return result, nil520+ }521+ }522+ }523+ }524+ if !foundSet {525+ return FeaturePageResult{}, fmt.Errorf("feature set is not present")526+ }527+ if !passedAfter {528+ return FeaturePageResult{}, fmt.Errorf("feature continuation is not present")529+ }530+ return result, nil531+}532+432533 func (reader *Reader) readFeaturePartition(partition featurePartitionDescriptor) ([]Feature, int64, error) {433534 features, err := reader.Table(partition.Features)434535 if err != nil {
M format/vnext/storage_generation_test.go +4 -0
@@ -110,6 +110,10 @@ func TestOutlineDoesNotReadFeaturePartitions(t *testing.T) {
110110 if err != nil {111111 t.Fatalf("open with unread corrupt partition: %v", err)112112 }113+ summaries, err := reader.FeatureSetSummaries()114+ if err != nil || len(summaries) != 1 || summaries[0].FeatureSet != "roads" || summaries[0].Rows != 300 {115+ t.Fatalf("page-zero feature summary = %#v, %v", summaries, err)116+ }113117 outline, err := reader.Outline()114118 if err != nil {115119 t.Fatalf("outline read a feature partition: %v", err)
M internal/app/app_test.go +182 -5
@@ -1,6 +1,7 @@
11 package app_test22 33 import (4+ "bytes"45 "context"56 "crypto/sha256"67 "encoding/json"
@@ -26,8 +27,10 @@ import (
2627 // window, so what these tests exercise is the handler and not the machine.2728 2829 type fakeVolume struct {29- manifest bundle.Manifest30- entries map[string][]byte30+ manifest bundle.Manifest31+ entries map[string][]byte32+ featurePageCalls int33+ demandAddressed bool3134 }3235 3336 func (v *fakeVolume) Info() hostenv.VolumeInfo {
@@ -43,7 +46,7 @@ func (v *fakeVolume) Info() hostenv.VolumeInfo {
4346 return info4447 }4548 46-func (v *fakeVolume) Semantic() vnext.Volume {49+func (v *fakeVolume) semantic() vnext.Volume {4750 semantic, _ := vnext.ImportV3Volume(v.manifest, func(name string) ([]byte, error) { return v.Blob(name) })4851 for name, data := range v.entries {4952 if !strings.HasPrefix(name, bundle.IconsPrefix) {
@@ -60,12 +63,67 @@ func (v *fakeVolume) Semantic() vnext.Volume {
6063 return semantic6164 }6265 66+func (v *fakeVolume) Outline() vnext.Volume {67+ semantic := v.semantic()68+ for worldIndex := range semantic.Worlds {69+ for setIndex := range semantic.Worlds[worldIndex].FeatureSets {70+ semantic.Worlds[worldIndex].FeatureSets[setIndex].Features = nil71+ }72+ }73+ return semantic74+}75+76+func (v *fakeVolume) nativeReader() (*vnext.Reader, error) {77+ compiled, err := vnext.Compile(v.semantic())78+ if err != nil {79+ return nil, err80+ }81+ compiled.Release.CreatedAt = v.manifest.Version.CreatedAt82+ if compiled.Release.CreatedAt == "" {83+ compiled.Release.CreatedAt = "2026-01-01T00:00:00Z"84+ }85+ var data bytes.Buffer86+ if err := vnext.Write(&data, compiled); err != nil {87+ return nil, err88+ }89+ return vnext.Open(bytes.NewReader(data.Bytes()), int64(data.Len()), vnext.StandardSchema())90+}91+92+func (v *fakeVolume) FeaturePage(request vnext.FeaturePageRequest) (vnext.FeaturePageResult, error) {93+ v.featurePageCalls++94+ reader, err := v.nativeReader()95+ if err != nil {96+ return vnext.FeaturePageResult{}, err97+ }98+ return reader.FeaturePage(request)99+}100+101+func (v *fakeVolume) FeatureSetSummaries() ([]vnext.FeatureSetSummary, error) {102+ var out []vnext.FeatureSetSummary103+ for _, world := range v.semantic().Worlds {104+ for _, set := range world.FeatureSets {105+ out = append(out, vnext.FeatureSetSummary{FeatureSet: set.ID, Rows: len(set.Features)})106+ }107+ }108+ return out, nil109+}110+111+func (v *fakeVolume) DemandAddressedFeatures() bool { return v.demandAddressed }112+113+func (v *fakeVolume) RasterTile(name string) (vnext.RasterTile, error) {114+ data, held := v.entries[name]115+ if !held {116+ return vnext.RasterTile{}, errors.New("no such tile")117+ }118+ return vnext.RasterTile{Name: name, MediaType: mime.TypeByExtension(path.Ext(name)), Data: data}, nil119+}120+63121 func (v *fakeVolume) Blob(name string) ([]byte, error) {64122 held, ok := v.entries[name]65123 if ok {66124 return held, nil67125 }68- compiled, err := vnext.Compile(v.Semantic())126+ compiled, err := vnext.Compile(v.semantic())69127 if err == nil {70128 for _, blob := range compiled.Blobs {71129 if blob.Name == name {
@@ -78,7 +136,7 @@ func (v *fakeVolume) Blob(name string) ([]byte, error) {
78136 79137 func (v *fakeVolume) Schema() ([]byte, error) { return vnext.StandardSchema().Canonical() }80138 func (v *fakeVolume) TableBlock(name string) ([]byte, error) {81- compiled, err := vnext.Compile(v.Semantic())139+ compiled, err := vnext.Compile(v.semantic())82140 if err != nil {83141 return nil, err84142 }
@@ -275,6 +333,14 @@ func TestContentPlane(t *testing.T) {
275333 {"a content-addressed asset", marker, "image/svg+xml", "<svg/>"},276334 {"a tile", base + "/tiles/overworld/0/0/0.jpg", "image/jpeg", "raster"},277335 }336+337+ outline := get(t, handler, base+"/outline.json", nil)338+ if outline.Code != http.StatusOK || outline.Header().Get("X-Atlas-Feature-Partitions") != "0" {339+ t.Fatalf("outline = %d with %q partitions", outline.Code, outline.Header().Get("X-Atlas-Feature-Partitions"))340+ }341+ if tile := get(t, handler, base+"/tiles/overworld/0/0/0.jpg", nil); tile.Header().Get("X-Atlas-Raster-Ranges") != "1" {342+ t.Fatalf("tile read %q raster ranges, want one shard slice", tile.Header().Get("X-Atlas-Raster-Ranges"))343+ }278344 for _, tt := range served {279345 t.Run(tt.name, func(t *testing.T) {280346 got := get(t, handler, tt.path, nil)
@@ -319,6 +385,76 @@ func TestContentPlane(t *testing.T) {
319385 }320386 }321387 388+type extremeVolume struct{ *fakeVolume }389+390+func (v *extremeVolume) Outline() vnext.Volume {391+ outline := v.fakeVolume.Outline()392+ field := func(name string) vnext.Field {393+ return vnext.Field{ID: vnext.IDFromName("example.test", name), Name: name, Kind: vnext.KindInt64}394+ }395+ maximum, minimum := field("sample.maximum"), field("sample.minimum")396+ outline.Worlds[0].Claims = []vnext.Property{397+ {FieldID: maximum.ID, Field: maximum, Value: vnext.Int64Value(int64(^uint64(0) >> 1))},398+ {FieldID: minimum.ID, Field: minimum, Value: vnext.Int64Value(-int64(^uint64(0)>>1) - 1)},399+ }400+ return outline401+}402+403+func (v *extremeVolume) FeaturePage(vnext.FeaturePageRequest) (vnext.FeaturePageResult, error) {404+ field := vnext.Field{ID: vnext.IDFromName("example.test", "extreme"), Name: "sample.extreme", Kind: vnext.KindInt64}405+ feature := func(id string, value int64) vnext.Feature {406+ return vnext.Feature{407+ ID: id, Title: id,408+ Geometry: vnext.Geometry{Kind: vnext.GeometryPoint, Parts: []vnext.GeometryPart{{Rings: [][]vnext.Position{{{0, 0}}}}}},409+ Properties: []vnext.Property{{FieldID: field.ID, Field: field, Value: vnext.Int64Value(value)}},410+ }411+ }412+ return vnext.FeaturePageResult{Features: []vnext.Feature{413+ feature("maximum", int64(^uint64(0)>>1)),414+ feature("minimum", -int64(^uint64(0)>>1)-1),415+ }}, nil416+}417+418+func TestFeaturePageWirePreservesInt64Extremes(t *testing.T) {419+ handler, _ := newApp(t, &extremeVolume{fakeVolume: volume("sample", "Sample", tunicStamp)})420+ base := "/data/v/sample/" + bundle.ShortStamp(tunicStamp)421+ got := get(t, handler, base+"/features/extreme.json", nil)422+ if got.Code != http.StatusOK {423+ t.Fatalf("feature page answered %d: %s", got.Code, got.Body.String())424+ }425+ var page struct {426+ Features []struct {427+ Properties []struct {428+ Value string `json:"value"`429+ } `json:"properties"`430+ } `json:"features"`431+ }432+ if err := json.Unmarshal(got.Body.Bytes(), &page); err != nil {433+ t.Fatal(err)434+ }435+ values := []string{page.Features[0].Properties[0].Value, page.Features[1].Properties[0].Value}436+ if values[0] != "9223372036854775807" || values[1] != "-9223372036854775808" {437+ t.Fatalf("int64 wire values = %v", values)438+ }439+440+ outline := get(t, handler, base+"/outline.json", nil)441+ var pageZero struct {442+ Version int `json:"version"`443+ Worlds []struct {444+ Claims []struct {445+ Value string `json:"value"`446+ } `json:"claims"`447+ } `json:"worlds"`448+ }449+ if err := json.Unmarshal(outline.Body.Bytes(), &pageZero); err != nil {450+ t.Fatal(err)451+ }452+ claims := pageZero.Worlds[0].Claims453+ if pageZero.Version != 1 || claims[0].Value != values[0] || claims[1].Value != values[1] {454+ t.Fatalf("outline wire = version %d, claims %#v", pageZero.Version, claims)455+ }456+}457+322458 // A Range request is answered whole: 200, the full body, no Accept-Ranges.323459 // Tiles are stored uncompressed so that ranges *could* be served, and data.go324460 // says why they deliberately are not; this test is what goes red if somebody
@@ -1170,6 +1306,47 @@ func zonedVolume() *fakeVolume {
11701306 return held11711307 }11721308 1309+func TestFeaturePagesComeFromPhysicalPartitions(t *testing.T) {1310+ handler, _ := newApp(t, zonedVolume())1311+ base := "/data/v/tunic/" + bundle.ShortStamp(tunicStamp)1312+ got := get(t, handler, base+"/features/overworld%2Fset%2F900.json?limit=2&minX=-1000000000000&minY=-1000000000000&maxX=1000000000000&maxY=1000000000000", nil)1313+ if got.Code != http.StatusOK {1314+ t.Fatalf("feature page answered %d: %s", got.Code, got.Body.String())1315+ }1316+ if got.Header().Get("X-Atlas-Feature-Partitions") != "1" {1317+ t.Fatalf("feature page read %q partitions, want one", got.Header().Get("X-Atlas-Feature-Partitions"))1318+ }1319+ var page struct {1320+ Features []struct {1321+ ID string `json:"id"`1322+ } `json:"features"`1323+ Next string `json:"next"`1324+ }1325+ if err := json.Unmarshal(got.Body.Bytes(), &page); err != nil {1326+ t.Fatal(err)1327+ }1328+ if len(page.Features) != 2 || page.Next == "" {1329+ t.Fatalf("feature page = %d features, next %q", len(page.Features), page.Next)1330+ }1331+ invalid := get(t, handler, base+"/features/overworld%2Fset%2F900.json?minX=1", nil)1332+ if invalid.Code != http.StatusBadRequest {1333+ t.Fatalf("incomplete feature bounds answered %d", invalid.Code)1334+ }1335+}1336+1337+func TestExplorerPageZeroDoesNotReadFeaturePartitions(t *testing.T) {1338+ volume := zonedVolume()1339+ volume.demandAddressed = true1340+ handler, _ := newApp(t, volume)1341+ page := get(t, handler, "/v/tunic/overworld", nil)1342+ if page.Code != http.StatusOK {1343+ t.Fatalf("explorer answered %d: %s", page.Code, page.Body.String())1344+ }1345+ if volume.featurePageCalls != 0 {1346+ t.Fatalf("page zero read %d feature pages", volume.featurePageCalls)1347+ }1348+}1349+11731350 // zoneOnlyButton is one zone row's exclusive control, from the attribute that11741351 // names the zone to the end of the button that carries it.11751352 func zoneOnlyButton(page, zone string) string {
M internal/app/data.go +187 -6
@@ -1,8 +1,11 @@
11 package app22 33 import (4+ "encoding/base64"45 "encoding/json"6+ "fmt"57 "log/slog"8+ "math"69 "net/http"710 "sort"811 "strconv"
@@ -122,6 +125,15 @@ func (a *App) handleContent(w http.ResponseWriter, r *http.Request) {
122125 }123126 124127 rest := r.PathValue("rest")128+ if rest == "outline.json" {129+ w.Header().Set("X-Atlas-Feature-Partitions", "0")130+ serveJSON(w, outlineResponse(held.Outline()))131+ return132+ }133+ if strings.HasPrefix(rest, "features/") && strings.HasSuffix(rest, ".json") {134+ a.handleFeaturePage(w, r, held, strings.TrimSuffix(strings.TrimPrefix(rest, "features/"), ".json"))135+ return136+ }125137 if rest == "schema.json" {126138 data, err := held.Schema()127139 if err != nil {
@@ -144,6 +156,12 @@ func (a *App) handleContent(w http.ResponseWriter, r *http.Request) {
144156 http.NotFound(w, r)145157 return146158 }159+ if tile, err := held.RasterTile(rest); err == nil {160+ w.Header().Set("X-Atlas-Raster-Ranges", "1")161+ w.Header().Set("X-Atlas-Raster-Bytes", strconv.Itoa(len(tile.Data)))162+ serveContent(w, tile.Data, tile.MediaType)163+ return164+ }147165 148166 data, err := held.Blob(rest)149167 if err != nil {
@@ -155,12 +173,10 @@ func (a *App) handleContent(w http.ResponseWriter, r *http.Request) {
155173 kind = contentTypes[rest[dot:]]156174 }157175 if kind == "" {158- if semantic, err := a.semanticVolume(held); err == nil {159- for _, asset := range semantic.Assets {160- if asset.Path == rest {161- kind = asset.MediaType162- break163- }176+ for _, asset := range held.Outline().Assets {177+ if asset.Path == rest {178+ kind = asset.MediaType179+ break164180 }165181 }166182 }
@@ -178,6 +194,171 @@ func (a *App) handleContent(w http.ResponseWriter, r *http.Request) {
178194 }179195 }180196 197+func (a *App) handleFeaturePage(w http.ResponseWriter, r *http.Request, volume hostenv.Volume, set string) {198+ limit := 1_000199+ if raw := r.URL.Query().Get("limit"); raw != "" {200+ parsed, err := strconv.Atoi(raw)201+ if err != nil {202+ http.Error(w, "feature page limit is invalid", http.StatusBadRequest)203+ return204+ }205+ limit = parsed206+ }207+ bounds, err := featureBounds(r)208+ if err != nil {209+ http.Error(w, err.Error(), http.StatusBadRequest)210+ return211+ }212+ page, err := volume.FeaturePage(vnext.FeaturePageRequest{213+ FeatureSet: set, Bounds: bounds, After: r.URL.Query().Get("after"), Limit: limit,214+ })215+ if err != nil {216+ http.Error(w, "feature page could not be read", http.StatusBadRequest)217+ return218+ }219+ w.Header().Set("X-Atlas-Feature-Partitions", strconv.Itoa(page.PartitionsRead))220+ w.Header().Set("X-Atlas-Feature-Bytes", strconv.FormatInt(page.BytesRead, 10))221+ serveJSON(w, featurePageResponse(page))222+}223+224+type featurePageWire struct {225+ Features []featureWire `json:"features"`226+ Next string `json:"next,omitempty"`227+}228+229+type featureWire struct {230+ ID string `json:"id"`231+ Title string `json:"title"`232+ Subtitle string `json:"subtitle,omitempty"`233+ Description string `json:"description,omitempty"`234+ Center *vnext.Position `json:"center,omitempty"`235+ Shard string `json:"shard"`236+ Geometry geometryWire `json:"geometry"`237+ Properties []propertyWire `json:"properties,omitempty"`238+ Relationships []relationshipWire `json:"relationships,omitempty"`239+ Provenance []provenanceWire `json:"provenance,omitempty"`240+}241+242+type geometryWire struct {243+ Kind vnext.GeometryKind `json:"kind"`244+ Parts []geometryPartWire `json:"parts"`245+}246+247+type geometryPartWire struct {248+ Rings [][]vnext.Position `json:"rings"`249+}250+251+type propertyWire struct {252+ ID string `json:"id"`253+ Name string `json:"name"`254+ Kind vnext.Kind `json:"kind"`255+ Value interface{} `json:"value"`256+}257+258+type relationshipWire struct {259+ Predicate string `json:"predicate"`260+ Target string `json:"target"`261+}262+263+type provenanceWire struct {264+ Source string `json:"source"`265+ NativeID string `json:"nativeId"`266+ CapturedAt string `json:"capturedAt"`267+}268+269+func featurePageResponse(page vnext.FeaturePageResult) featurePageWire {270+ out := featurePageWire{Next: page.Next, Features: make([]featureWire, 0, len(page.Features))}271+ for _, feature := range page.Features {272+ out.Features = append(out.Features, featureResponse(feature))273+ }274+ return out275+}276+277+func featureResponse(feature vnext.Feature) featureWire {278+ out := featureWire{279+ ID: feature.ID, Title: feature.Title, Subtitle: feature.Subtitle, Description: feature.Description,280+ Center: feature.Center, Shard: strconv.FormatInt(feature.Shard, 10),281+ Geometry: geometryWire{Kind: feature.Geometry.Kind, Parts: make([]geometryPartWire, 0, len(feature.Geometry.Parts))},282+ Properties: make([]propertyWire, 0, len(feature.Properties)),283+ Relationships: make([]relationshipWire, 0, len(feature.Relationships)),284+ Provenance: make([]provenanceWire, 0, len(feature.Provenance)),285+ }286+ for _, part := range feature.Geometry.Parts {287+ out.Geometry.Parts = append(out.Geometry.Parts, geometryPartWire{Rings: part.Rings})288+ }289+ for _, property := range feature.Properties {290+ out.Properties = append(out.Properties, propertyResponse(property))291+ }292+ for _, edge := range feature.Relationships {293+ out.Relationships = append(out.Relationships, relationshipWire{Predicate: edge.Predicate, Target: edge.Target})294+ }295+ for _, evidence := range feature.Provenance {296+ out.Provenance = append(out.Provenance, provenanceWire{297+ Source: evidence.Source, NativeID: evidence.NativeID, CapturedAt: evidence.CapturedAt,298+ })299+ }300+ return out301+}302+303+func propertyResponse(property vnext.Property) propertyWire {304+ id := property.FieldID305+ if id == (vnext.ID{}) {306+ id = property.Field.ID307+ }308+ out := propertyWire{ID: id.String(), Name: property.Field.Name, Kind: property.Value.Kind}309+ switch property.Value.Kind {310+ case vnext.KindBool:311+ out.Value = property.Value.Bool312+ case vnext.KindInt64:313+ out.Value = strconv.FormatInt(property.Value.Int64, 10)314+ case vnext.KindFloat64:315+ out.Value = property.Value.Float64316+ case vnext.KindString:317+ out.Value = property.Value.String318+ case vnext.KindBytes:319+ out.Value = base64.StdEncoding.EncodeToString(property.Value.Bytes)320+ case vnext.KindID:321+ out.Value = property.Value.ID.String()322+ default:323+ out.Value = nil324+ }325+ return out326+}327+328+func featureBounds(r *http.Request) (*vnext.SpatialBounds, error) {329+ names := []string{"minX", "minY", "maxX", "maxY"}330+ values := make([]float64, len(names))331+ present := 0332+ for index, name := range names {333+ raw := r.URL.Query().Get(name)334+ if raw == "" {335+ continue336+ }337+ value, err := strconv.ParseFloat(raw, 64)338+ if err != nil || math.IsNaN(value) || math.IsInf(value, 0) {339+ return nil, fmt.Errorf("feature bounds are invalid")340+ }341+ values[index] = value342+ present++343+ }344+ if present == 0 {345+ return nil, nil346+ }347+ if present != len(names) || values[0] > values[2] || values[1] > values[3] {348+ return nil, fmt.Errorf("feature bounds are incomplete or inverted")349+ }350+ return &vnext.SpatialBounds{MinX: values[0], MinY: values[1], MaxX: values[2], MaxY: values[3]}, nil351+}352+353+func serveJSON(w http.ResponseWriter, value any) {354+ data, err := json.Marshal(value)355+ if err != nil {356+ http.Error(w, "native data could not be encoded", http.StatusInternalServerError)357+ return358+ }359+ serveContent(w, data, "application/json")360+}361+181362 func serveContent(w http.ResponseWriter, data []byte, kind string) {182363 w.Header().Set("Content-Type", kind)183364 w.Header().Set("Content-Length", strconv.Itoa(len(data)))
M internal/app/hostenv/hostenv.go +13 -2
@@ -98,8 +98,17 @@ type Volume interface {
9898 // catalog views without decoding physical blocks.9999 Info() VolumeInfo100100 101- // Semantic is the canonical graph restored from typed blocks.102- Semantic() vnext.Volume101+ // Outline is the canonical graph restored from typed blocks without feature102+ // rows. It is safe to use while composing page zero.103+ Outline() vnext.Volume104+105+ // FeaturePage reads a bounded page from the physical feature partitions.106+ FeaturePage(request vnext.FeaturePageRequest) (vnext.FeaturePageResult, error)107+ FeatureSetSummaries() ([]vnext.FeatureSetSummary, error)108+ DemandAddressedFeatures() bool109+110+ // RasterTile reads one verified logical tile slice from its physical shard.111+ RasterTile(entry string) (vnext.RasterTile, error)103112 104113 // Blob reads one opaque payload such as a raster tile or image asset.105114 Blob(entry string) ([]byte, error)
@@ -131,6 +140,8 @@ type WorldInfo struct {
131140 132141 type TileGrid struct {133142 SourceZoom int `json:"sourceZoom"`143+ OriginX int `json:"originX"`144+ OriginY int `json:"originY"`134145 FirstTile int `json:"firstTile"`135146 TileSize int `json:"tileSize"`136147 Size int `json:"size"`
M internal/app/hostenv/oshost/oshost_test.go +50 -0
@@ -1,11 +1,13 @@
11 package oshost_test22 33 import (4+ "archive/zip"45 "bytes"56 "context"67 "errors"78 "os"89 "path/filepath"10+ "strings"911 "testing"1012 1113 "github.com/FelineStateMachine/atlas/format/vnext"
@@ -88,6 +90,9 @@ func TestVolumesServesTheFoldsWinner(t *testing.T) {
8890 if tunic.Release.CreatedAt != "2026-02-01T00:00:00Z" {8991 t.Errorf("serving the build of %s, want the newest capture", tunic.Release.CreatedAt)9092 }93+ if len(tunic.Worlds) != 1 || tunic.Worlds[0].Points != 1 {94+ t.Fatalf("page-zero feature counts = %#v, want one point", tunic.Worlds)95+ }9196 9297 payload, err := volumes[1].Blob("tiles/overworld/0/0/0.jpg")9398 if err != nil {
@@ -101,6 +106,51 @@ func TestVolumesServesTheFoldsWinner(t *testing.T) {
101106 }102107 }103108 109+func TestVolumesPageZeroDoesNotReadFeaturePartitions(t *testing.T) {110+ dir := t.TempDir()111+ path := build{slug: "sample", title: "Sample", createdAt: "2026-01-01T00:00:00Z"}.write(t, dir)112+ data, err := os.ReadFile(path)113+ if err != nil {114+ t.Fatal(err)115+ }116+ archive, err := zip.NewReader(bytes.NewReader(data), int64(len(data)))117+ if err != nil {118+ t.Fatal(err)119+ }120+ corrupted := false121+ for _, entry := range archive.File {122+ if !strings.HasSuffix(entry.Name, ".features.pack") {123+ continue124+ }125+ offset, err := entry.DataOffset()126+ if err != nil {127+ t.Fatal(err)128+ }129+ data[offset+int64(entry.UncompressedSize64)-1] ^= 0xff130+ corrupted = true131+ break132+ }133+ if !corrupted {134+ t.Fatal("fixture has no physical feature partition")135+ }136+ if err := os.WriteFile(path, data, 0o644); err != nil {137+ t.Fatal(err)138+ }139+140+ store, err := oshost.NewVolumes(dir)141+ if err != nil {142+ t.Fatalf("page zero read a feature partition: %v", err)143+ }144+ t.Cleanup(func() { store.Close() })145+ volumes := store.Volumes()146+ if len(volumes) != 1 || len(volumes[0].Outline().Worlds) != 1 {147+ t.Fatalf("outline did not open: %#v", volumes)148+ }149+ if _, err := volumes[0].FeaturePage(vnext.FeaturePageRequest{FeatureSet: "markers", Limit: 1}); err == nil {150+ t.Fatal("demanded corrupt feature partition was accepted")151+ }152+}153+104154 func TestVolumesRescanReportsWhatMoved(t *testing.T) {105155 dir := t.TempDir()106156 build{slug: "tunic", title: "TUNIC", createdAt: "2026-01-01T00:00:00Z"}.write(t, dir)
M internal/app/hostenv/oshost/volumes.go +56 -22
@@ -6,10 +6,12 @@ import (
66 "log/slog"77 "os"88 "sort"9+ "strings"910 "sync"1011 "sync/atomic"1112 "time"1213 14+ "github.com/FelineStateMachine/atlas/format/semconv"1315 "github.com/FelineStateMachine/atlas/format/vnext"1416 "github.com/FelineStateMachine/atlas/internal/app/hostenv"1517 "github.com/FelineStateMachine/atlas/internal/logging"
@@ -51,14 +53,24 @@ type snapshot struct {
5153 // entry is a seek and a read rather than a fresh open of a file that may be5254 // hundreds of megabytes.5355 type volume struct {54- reader *vnext.File55- locator string56- info hostenv.VolumeInfo57- semantic vnext.Volume56+ reader *vnext.File57+ locator string58+ info hostenv.VolumeInfo59+ outline vnext.Volume5860 }5961 6062 func (v *volume) Info() hostenv.VolumeInfo { return v.info }61-func (v *volume) Semantic() vnext.Volume { return v.semantic }63+func (v *volume) Outline() vnext.Volume { return v.outline }64+func (v *volume) FeaturePage(request vnext.FeaturePageRequest) (vnext.FeaturePageResult, error) {65+ return v.reader.FeaturePage(request)66+}67+func (v *volume) FeatureSetSummaries() ([]vnext.FeatureSetSummary, error) {68+ return v.reader.FeatureSetSummaries()69+}70+func (v *volume) DemandAddressedFeatures() bool { return v.reader.DemandAddressedFeatures() }71+func (v *volume) RasterTile(entry string) (vnext.RasterTile, error) {72+ return v.reader.ReadRasterTile(entry)73+}6274 func (v *volume) Blob(entry string) ([]byte, error) {6375 return v.reader.Blob(entry)6476 }
@@ -219,42 +231,64 @@ func carryOrOpen(previous *snapshot, winner vnext.Descriptor) (*volume, error) {
219231 if err != nil {220232 return nil, err221233 }222- semantic, err := reader.Volume()234+ outline, err := reader.Outline()235+ if err != nil {236+ reader.Close()237+ return nil, err238+ }239+ info, err := volumeInfo(winner, outline, reader.Reader)223240 if err != nil {224241 reader.Close()225242 return nil, err226243 }227- return &volume{reader: reader, locator: winner.Locator, semantic: semantic, info: volumeInfo(winner, semantic)}, nil244+ return &volume{reader: reader, locator: winner.Locator, outline: outline, info: info}, nil228245 }229246 230-func volumeInfo(descriptor vnext.Descriptor, semantic vnext.Volume) hostenv.VolumeInfo {247+func volumeInfo(descriptor vnext.Descriptor, outline vnext.Volume, reader *vnext.Reader) (hostenv.VolumeInfo, error) {231248 info := hostenv.VolumeInfo{232249 Slug: descriptor.Slug, Title: descriptor.Title, Stamp: descriptor.Stamp,233- Release: descriptorRelease(descriptor), Worlds: make([]hostenv.WorldInfo, 0, len(semantic.Worlds)),250+ Release: descriptorRelease(descriptor), Worlds: make([]hostenv.WorldInfo, 0, len(outline.Worlds)),234251 }235- for index, world := range semantic.Worlds {252+ summaries, err := reader.FeatureSetSummaries()253+ if err != nil {254+ return hostenv.VolumeInfo{}, fmt.Errorf("read feature summary: %w", err)255+ }256+ counts := make(map[string]int, len(summaries))257+ for _, summary := range summaries {258+ counts[summary.FeatureSet] = summary.Rows259+ }260+ for index, world := range outline.Worlds {236261 listed := hostenv.WorldInfo{Slug: world.ID, Title: world.Title, UpdatedAt: descriptor.CreatedAt}237262 for _, set := range world.FeatureSets {238- for _, feature := range set.Features {239- switch feature.Geometry.Kind {240- case vnext.GeometryPoint:241- listed.Points++242- case vnext.GeometryLineString:243- listed.Paths++244- case vnext.GeometryPolygon:245- listed.Areas++246- }263+ switch featureSetKind(set) {264+ case semconv.GeometryPath:265+ listed.Paths += counts[set.ID]266+ case semconv.GeometryArea:267+ listed.Areas += counts[set.ID]268+ default:269+ listed.Points += counts[set.ID]247270 }248271 }249272 info.Worlds = append(info.Worlds, listed)250273 if index == 0 {251274 info.TileGrid = hostenv.TileGrid{252- SourceZoom: int(world.CoordinateSpace.SourceZoom), FirstTile: int(world.CoordinateSpace.FirstTile),253- TileSize: int(world.CoordinateSpace.TileSize), Size: int(world.CoordinateSpace.Size),275+ SourceZoom: int(world.CoordinateSpace.SourceZoom),276+ OriginX: int(world.CoordinateSpace.OriginX), OriginY: int(world.CoordinateSpace.OriginY),277+ FirstTile: int(world.CoordinateSpace.FirstTile),278+ TileSize: int(world.CoordinateSpace.TileSize), Size: int(world.CoordinateSpace.Size),254279 }255280 }256281 }257- return info282+ return info, nil283+}284+285+func featureSetKind(set vnext.FeatureSet) string {286+ for _, claim := range set.Claims {287+ if claim.Field.Name == semconv.KeyGeometryKind {288+ return claim.Value.String289+ }290+ }291+ return strings.TrimPrefix(set.SemanticType, "geometry.")258292 }259293 260294 func descriptorRelease(descriptor vnext.Descriptor) vnext.Release {
A internal/app/outline_wire.go +236 -0
@@ -0,0 +1,236 @@
1+package app2+3+import (4+ "encoding/base64"5+ "strconv"6+7+ "github.com/FelineStateMachine/atlas/format/vnext"8+)9+10+const outlineWireVersion = 111+12+type outlineWire struct {13+ Version int `json:"version"`14+ ID string `json:"id"`15+ Title string `json:"title"`16+ Worlds []worldWire `json:"worlds"`17+ Assets []assetWire `json:"assets"`18+}19+20+type worldWire struct {21+ ID string `json:"id"`22+ Title string `json:"title"`23+ CoordinateSpace coordinateSpaceWire `json:"coordinateSpace"`24+ Claims []propertyWire `json:"claims"`25+ FeatureSets []featureSetWire `json:"featureSets"`26+ RasterPyramids []rasterWire `json:"rasterPyramids"`27+ Presentation presentationWire `json:"presentation"`28+}29+30+type coordinateSpaceWire struct {31+ ID string `json:"id"`32+ Kind string `json:"kind"`33+ Unit string `json:"unit"`34+ Definition string `json:"definition"`35+ Extent [4]float64 `json:"extent"`36+ SourceZoom string `json:"sourceZoom"`37+ OriginX string `json:"originX"`38+ OriginY string `json:"originY"`39+ TileSize string `json:"tileSize"`40+ Size string `json:"size"`41+}42+43+type featureSetWire struct {44+ ID string `json:"id"`45+ Title string `json:"title"`46+ SemanticType string `json:"semanticType"`47+ Properties []fieldWire `json:"properties"`48+ Claims []propertyWire `json:"claims"`49+}50+51+type fieldWire struct {52+ ID string `json:"id"`53+ Name string `json:"name"`54+ Kind vnext.Kind `json:"kind"`55+ Optional bool `json:"optional"`56+}57+58+type rasterWire struct {59+ ID string `json:"id"`60+ Name string `json:"name"`61+ Codec string `json:"codec"`62+ TileSize string `json:"tileSize"`63+ MinZoom string `json:"minZoom"`64+ MaxZoom string `json:"maxZoom"`65+ FullZoom string `json:"fullZoom"`66+ SourceZoom string `json:"sourceZoom"`67+ Template string `json:"template"`68+ Formats []string `json:"formats"`69+ Bounds *rasterRectWire `json:"bounds,omitempty"`70+ Surface *rasterRectWire `json:"surface,omitempty"`71+ Interpolate bool `json:"interpolate"`72+ Background string `json:"background,omitempty"`73+ Shard string `json:"shard"`74+ Coverage []coverageWire `json:"coverage"`75+}76+77+type rasterRectWire struct {78+ X float64 `json:"x"`79+ Y float64 `json:"y"`80+ Width float64 `json:"width"`81+ Height float64 `json:"height"`82+}83+84+type coverageWire struct {85+ Zoom string `json:"zoom"`86+ X string `json:"x"`87+ Y string `json:"y"`88+ W string `json:"w"`89+ H string `json:"h"`90+ Bits string `json:"bits"`91+}92+93+type presentationWire struct {94+ ID string `json:"id"`95+ Title string `json:"title"`96+ Styles []styleWire `json:"styles"`97+ Layers []layerWire `json:"layers"`98+ Legend []legendWire `json:"legend"`99+}100+101+type styleWire struct {102+ ID string `json:"id"`103+ Symbol string `json:"symbol,omitempty"`104+ Icon string `json:"icon,omitempty"`105+ IconAsset string `json:"iconAsset,omitempty"`106+ IconPicture bool `json:"iconPicture"`107+ RenderAs string `json:"renderAs,omitempty"`108+ Stroke string `json:"stroke,omitempty"`109+ Fill string `json:"fill,omitempty"`110+}111+112+type layerWire struct {113+ ID string `json:"id"`114+ FeatureSet string `json:"featureSet"`115+ Style string `json:"style"`116+ Group string `json:"group,omitempty"`117+ LabelPolicy string `json:"labelPolicy,omitempty"`118+ Visible bool `json:"visible"`119+ Order string `json:"order"`120+ MinZoom string `json:"minZoom"`121+ MaxZoom string `json:"maxZoom"`122+}123+124+type legendWire struct {125+ Layer string `json:"layer"`126+ Label string `json:"label"`127+ Order string `json:"order"`128+}129+130+type assetWire struct {131+ ID string `json:"id"`132+ MediaType string `json:"mediaType"`133+ Path string `json:"path"`134+ Provenance string `json:"provenance,omitempty"`135+}136+137+func outlineResponse(volume vnext.Volume) outlineWire {138+ out := outlineWire{Version: outlineWireVersion, ID: volume.ID, Title: volume.Title}139+ for _, world := range volume.Worlds {140+ out.Worlds = append(out.Worlds, worldResponse(world))141+ }142+ for _, asset := range volume.Assets {143+ out.Assets = append(out.Assets, assetWire{144+ ID: asset.ID, MediaType: asset.MediaType, Path: asset.Path, Provenance: asset.Provenance,145+ })146+ }147+ return out148+}149+150+func worldResponse(world vnext.World) worldWire {151+ space := world.CoordinateSpace152+ out := worldWire{153+ ID: world.ID, Title: world.Title, Claims: propertyResponses(world.Claims),154+ CoordinateSpace: coordinateSpaceWire{155+ ID: space.ID, Kind: space.Kind, Unit: space.Unit, Definition: space.Definition, Extent: space.Extent,156+ SourceZoom: intText(space.SourceZoom), OriginX: intText(space.OriginX), OriginY: intText(space.OriginY),157+ TileSize: intText(space.TileSize), Size: intText(space.Size),158+ },159+ Presentation: presentationResponse(world.Presentation),160+ }161+ for _, set := range world.FeatureSets {162+ out.FeatureSets = append(out.FeatureSets, featureSetResponse(set))163+ }164+ for _, raster := range world.RasterPyramids {165+ out.RasterPyramids = append(out.RasterPyramids, rasterResponse(raster))166+ }167+ return out168+}169+170+func featureSetResponse(set vnext.FeatureSet) featureSetWire {171+ out := featureSetWire{172+ ID: set.ID, Title: set.Title, SemanticType: set.SemanticType, Claims: propertyResponses(set.Claims),173+ }174+ for _, field := range set.Properties {175+ out.Properties = append(out.Properties, fieldWire{176+ ID: field.ID.String(), Name: field.Name, Kind: field.Kind, Optional: field.Optional,177+ })178+ }179+ return out180+}181+182+func rasterResponse(raster vnext.RasterPyramid) rasterWire {183+ out := rasterWire{184+ ID: raster.ID, Name: raster.Name, Codec: raster.Codec, TileSize: intText(raster.TileSize),185+ MinZoom: intText(raster.MinZoom), MaxZoom: intText(raster.MaxZoom), FullZoom: intText(raster.FullZoom),186+ SourceZoom: intText(raster.SourceZoom), Template: raster.Template, Formats: append([]string(nil), raster.Formats...),187+ Interpolate: raster.Interpolate, Background: raster.Background, Shard: intText(raster.Shard),188+ }189+ if raster.Bounds != nil {190+ out.Bounds = rectResponse(*raster.Bounds)191+ }192+ if raster.Surface != nil {193+ out.Surface = rectResponse(*raster.Surface)194+ }195+ for _, coverage := range raster.Coverage {196+ out.Coverage = append(out.Coverage, coverageWire{197+ Zoom: intText(coverage.Zoom), X: intText(coverage.X), Y: intText(coverage.Y),198+ W: intText(coverage.W), H: intText(coverage.H), Bits: base64.StdEncoding.EncodeToString(coverage.Bits),199+ })200+ }201+ return out202+}203+204+func rectResponse(rect vnext.RasterRect) *rasterRectWire {205+ return &rasterRectWire{X: rect.X, Y: rect.Y, Width: rect.Width, Height: rect.Height}206+}207+208+func presentationResponse(presentation vnext.Presentation) presentationWire {209+ out := presentationWire{ID: presentation.ID, Title: presentation.Title}210+ for _, style := range presentation.Styles {211+ out.Styles = append(out.Styles, styleWire{212+ ID: style.ID, Symbol: style.Symbol, Icon: style.Icon, IconAsset: style.IconAsset, IconPicture: style.IconPicture,213+ RenderAs: style.RenderAs, Stroke: style.Stroke, Fill: style.Fill,214+ })215+ }216+ for _, layer := range presentation.Layers {217+ out.Layers = append(out.Layers, layerWire{218+ ID: layer.ID, FeatureSet: layer.FeatureSet, Style: layer.Style, Group: layer.Group, LabelPolicy: layer.LabelPolicy,219+ Visible: layer.Visible, Order: intText(layer.Order), MinZoom: intText(layer.MinZoom), MaxZoom: intText(layer.MaxZoom),220+ })221+ }222+ for _, legend := range presentation.Legend {223+ out.Legend = append(out.Legend, legendWire{Layer: legend.Layer, Label: legend.Label, Order: intText(legend.Order)})224+ }225+ return out226+}227+228+func propertyResponses(properties []vnext.Property) []propertyWire {229+ out := make([]propertyWire, 0, len(properties))230+ for _, property := range properties {231+ out = append(out, propertyResponse(property))232+ }233+ return out234+}235+236+func intText(value int64) string { return strconv.FormatInt(value, 10) }
M internal/app/pages.go +12 -2
@@ -118,7 +118,11 @@ func (a *App) handleExplorer(w http.ResponseWriter, r *http.Request) {
118118 }119119 session.World = world120120 session.Stamp = vnext.ShortStamp(info.Stamp)121- a.arrange(volume, &session)121+ if volume.DemandAddressedFeatures() {122+ a.arrangePageZero(volume, &session)123+ } else {124+ a.arrange(volume, &session)125+ }122126 if err := a.saveSession(&session); err != nil {123127 slog.Warn("the session could not be written", logging.Op("session"),124128 logging.Volume(slug), slog.Any("error", err))
@@ -128,7 +132,13 @@ func (a *App) handleExplorer(w http.ResponseWriter, r *http.Request) {
128132 logging.Volume(slug), slog.Any("error", err))129133 }130134 131- a.writePage(w, "shell", a.view(held, volume, session))135+ var view View136+ if volume.DemandAddressedFeatures() {137+ view = a.viewPageZero(held, volume, session)138+ } else {139+ view = a.view(held, volume, session)140+ }141+ a.writePage(w, "shell", view)132142 }133143 134144 // handleDetail answers with one feature's card. The card is a fragment rather
M internal/app/session.go +13 -1
@@ -473,6 +473,18 @@ func (a *App) resetSession(w http.ResponseWriter, volume hostenv.Volume, slug st
473473 // hide set afterwards is a reader who asked to see everything, and must not be474474 // mistaken for a record nobody has touched.475475 func (a *App) arrange(volume hostenv.Volume, s *Session) {476+ a.arrangeWith(volume, s, a.world)477+}478+479+func (a *App) arrangePageZero(volume hostenv.Volume, s *Session) {480+ a.arrangeWith(volume, s, a.outlineWorld)481+}482+483+func (a *App) arrangeWith(484+ volume hostenv.Volume,485+ s *Session,486+ load func(hostenv.Volume, string) *worldModel,487+) {476488 if s.Arranged {477489 return478490 }
@@ -483,7 +495,7 @@ func (a *App) arrange(volume hostenv.Volume, s *Session) {
483495 }484496 s.World = info.Worlds[0].Slug485497 }486- model := a.world(volume, s.World)498+ model := load(volume, s.World)487499 if model == nil {488500 return489501 }
M internal/app/view.go +20 -1
@@ -234,6 +234,20 @@ type LabelOverride struct {
234234 // the session's, falling back to the volume's first world, which is the one a235235 // volume opens on when nobody has opened it before.236236 func (a *App) view(held library, volume hostenv.Volume, session Session) View {237+ return a.viewMode(held, volume, session, false)238+}239+240+func (a *App) viewPageZero(held library, volume hostenv.Volume, session Session) View {241+ return a.viewMode(held, volume, session, !sessionNeedsFeatures(session))242+}243+244+func sessionNeedsFeatures(session Session) bool {245+ return session.Search != "" || session.Selected != "" || session.Focused != "" ||246+ len(session.Highlighted) > 0 || len(session.Expanded) > 0 || session.Detail.Open ||247+ session.Dock.Open || session.Grid.Cell != ""248+}249+250+func (a *App) viewMode(held library, volume hostenv.Volume, session Session, pageZero bool) View {237251 out := View{238252 Title: "Atlas",239253 LibraryDir: a.env.Volumes().Location(),
@@ -279,7 +293,12 @@ func (a *App) view(held library, volume hostenv.Volume, session Session) View {
279293 shown.Worlds = append(shown.Worlds, listed)280294 }281295 282- model := a.world(volume, world)296+ var model *worldModel297+ if pageZero {298+ model = a.outlineWorld(volume, world)299+ } else {300+ model = a.world(volume, world)301+ }283302 var lens *payloadLens284303 if model != nil {285304 for at := range model.Lenses {
M internal/app/vnext_projection_test.go +27 -2
@@ -19,6 +19,20 @@ func TestVNextEarthBuildsTheRendererDirectly(t *testing.T) {
1919 if err != nil {2020 t.Fatalf("restore native Earth: %v", err)2121 }22+ for _, set := range semantic.Worlds[0].FeatureSets {23+ if len(set.Features) == 0 {24+ continue25+ }26+ page, err := native.FeaturePage(vnext.FeaturePageRequest{FeatureSet: set.ID, Limit: 1})27+ if err != nil || len(page.Features) != 1 || page.PartitionsRead != 1 || page.BytesRead == 0 {28+ t.Fatalf("framing-1 feature demand = %#v, %v", page, err)29+ }30+ break31+ }32+ summaries, err := native.FeatureSetSummaries()33+ if err != nil || len(summaries) == 0 || summaries[0].Rows == 0 {34+ t.Fatalf("framing-1 feature summaries = %#v, %v", summaries, err)35+ }2236 2337 model, err := buildVNextWorld(semantic.Worlds[0], semantic.Assets)2438 if err != nil {
@@ -55,13 +69,24 @@ func TestNativeCoordinateLabelsRespectTheDeclaredSpace(t *testing.T) {
5569 }5670 }5771 72+func TestTileGridProjectsIndependentColumnAndRowOrigins(t *testing.T) {73+ t.Parallel()74+75+ grid := tileGrid{SourceZoom: 4, OriginX: 3, OriginY: 6, FirstTile: 99, TileSize: 256, Size: 1024}76+ latitude, longitude := grid.unproject(0, 0)77+ x, y := grid.project(latitude, longitude)78+ if x != 0 || y != 0 {79+ t.Fatalf("origin round trip = %v,%v, want 0,0", x, y)80+ }81+}82+5883 func TestNativeGridUsesTheAuthoritativeCoordinateExtent(t *testing.T) {5984 space := vnext.CoordinateSpace{6085 ID: "sample-grid", Kind: "projected", Unit: "metre", Definition: "local sample grid",61- Extent: [4]float64{10, 20, 110, 70},86+ Extent: [4]float64{10, 20, 110, 70}, OriginX: 7, OriginY: 11,6287 }6388 grid := nativeGrid(space)64- if grid.TileSize != 256 || grid.Size != 100 {89+ if grid.TileSize != 256 || grid.Size != 100 || grid.OriginX != 7 || grid.OriginY != 11 {6590 t.Fatalf("native grid = %+v, want conventional tile step over the coordinate extent", grid)6691 }6792 space.SourceZoom, space.TileSize, space.Size = 3, 32, 256
M internal/app/vnext_runtime.go +41 -3
@@ -48,10 +48,45 @@ func (a *App) semanticVolume(source hostenv.Volume) (vnext.Volume, error) {
4848 info := source.Info()4949 key := info.Slug + "@" + vnext.ShortStamp(info.Stamp)5050 return a.semantic.load(key, func() (vnext.Volume, error) {51- return source.Semantic(), nil51+ return materializeVolume(source)5252 })5353 }5454 55+func materializeVolume(source hostenv.Volume) (vnext.Volume, error) {56+ volume := cloneOutline(source.Outline())57+ for worldIndex := range volume.Worlds {58+ for setIndex := range volume.Worlds[worldIndex].FeatureSets {59+ set := &volume.Worlds[worldIndex].FeatureSets[setIndex]60+ for after := ""; ; {61+ page, err := source.FeaturePage(vnext.FeaturePageRequest{62+ FeatureSet: set.ID, After: after, Limit: 1_000,63+ })64+ if err != nil {65+ return vnext.Volume{}, fmt.Errorf("read feature set %s: %w", set.ID, err)66+ }67+ set.Features = append(set.Features, page.Features...)68+ if page.Next == "" {69+ break70+ }71+ after = page.Next72+ }73+ }74+ }75+ return volume, nil76+}77+78+func cloneOutline(source vnext.Volume) vnext.Volume {79+ out := source80+ out.Worlds = append([]vnext.World(nil), source.Worlds...)81+ for worldIndex := range out.Worlds {82+ out.Worlds[worldIndex].FeatureSets = append([]vnext.FeatureSet(nil), source.Worlds[worldIndex].FeatureSets...)83+ for setIndex := range out.Worlds[worldIndex].FeatureSets {84+ out.Worlds[worldIndex].FeatureSets[setIndex].Features = nil85+ }86+ }87+ return out88+}89+5590 func semanticWorld(volume vnext.Volume, slug string) (vnext.World, bool) {5691 for _, world := range volume.Worlds {5792 if world.ID == slug {
@@ -99,7 +134,7 @@ func buildVNextWorld(world vnext.World, assets []vnext.Asset) (*worldModel, erro
99134 kind := featureSetGeometryKind(set)100135 attrs := propertiesToAttrs(set.Claims)101136 collection := &collectionModel{102- ID: layer.ID, Title: legendLabel(world.Presentation.Legend, layer.ID, set.Title), Kind: kind,137+ ID: layer.ID, FeatureSet: set.ID, Title: legendLabel(world.Presentation.Legend, layer.ID, set.Title), Kind: kind,103138 Group: layer.Group, Icon: style.Icon, IconAsset: assetPaths[style.IconAsset],104139 Color: nativeColor(kind, style), Attrs: attrs, Index: index,105140 Curated: layer.LabelPolicy, RenderAs: style.RenderAs, Hidden: !layer.Visible,
@@ -176,7 +211,10 @@ func coordinateLabel(space vnext.CoordinateSpace, grid tileGrid, position vnext.
176211 }177212 178213 func nativeGrid(space vnext.CoordinateSpace) tileGrid {179- grid := tileGrid{SourceZoom: int(space.SourceZoom), FirstTile: int(space.FirstTile), TileSize: int(space.TileSize), Size: int(space.Size)}214+ grid := tileGrid{215+ SourceZoom: int(space.SourceZoom), OriginX: int(space.OriginX), OriginY: int(space.OriginY),216+ FirstTile: int(space.FirstTile), TileSize: int(space.TileSize), Size: int(space.Size),217+ }180218 if grid.TileSize == 0 {181219 grid.TileSize = 256182220 }
M internal/app/world.go +49 -12
@@ -66,11 +66,12 @@ type worldModel struct {
6666 // collectionModel is one collection with everything a row needs already6767 // decided: its kind, its curated label policy, whether it starts hidden.6868 type collectionModel struct {69- ID string70- Title string71- Kind string72- Group string73- Icon string69+ ID string70+ FeatureSet string71+ Title string72+ Kind string73+ Group string74+ Icon string7475 7576 // IconAsset is the artwork the collection wears, as a path under the7677 // build's `icons/`. The map composes a marker from it and the legend
@@ -144,28 +145,41 @@ type point struct{ X, Y float64 }
144145 // grid, overridden by whatever the world declares for itself.145146 type tileGrid struct {146147 SourceZoom int147- FirstTile int148- TileSize int149- Size int148+ OriginX int149+ OriginY int150+ // FirstTile remains only for legacy v3 projections. Native worlds carry151+ // distinct column and row origins.152+ FirstTile int153+ TileSize int154+ Size int150155 }151156 152157 func (g tileGrid) project(lat, lng float64) (x, y float64) {153158 worldTiles := math.Pow(2, float64(g.SourceZoom))154159 xTile := (lng + 180) / 360 * worldTiles155160 yTile := (1 - math.Asinh(math.Tan(lat*math.Pi/180))/math.Pi) / 2 * worldTiles156- return (xTile - float64(g.FirstTile)) * float64(g.TileSize),157- -(yTile - float64(g.FirstTile)) * float64(g.TileSize)161+ originX, originY := g.origins()162+ return (xTile - float64(originX)) * float64(g.TileSize),163+ -(yTile - float64(originY)) * float64(g.TileSize)158164 }159165 160166 func (g tileGrid) unproject(x, y float64) (lat, lng float64) {161167 worldTiles := math.Pow(2, float64(g.SourceZoom))162- xTile := x/float64(g.TileSize) + float64(g.FirstTile)163- yTile := y/float64(g.TileSize) + float64(g.FirstTile)168+ originX, originY := g.origins()169+ xTile := x/float64(g.TileSize) + float64(originX)170+ yTile := y/float64(g.TileSize) + float64(originY)164171 lng = xTile/worldTiles*360 - 180165172 lat = math.Atan(math.Sinh(math.Pi*(1-2*yTile/worldTiles))) * 180 / math.Pi166173 return lat, lng167174 }168175 176+func (g tileGrid) origins() (int, int) {177+ if g.OriginX != 0 || g.OriginY != 0 {178+ return g.OriginX, g.OriginY179+ }180+ return g.FirstTile, g.FirstTile181+}182+169183 // ---------------------------------------------------------------------------170184 // Building171185 // ---------------------------------------------------------------------------
@@ -231,6 +245,29 @@ func (a *App) world(volume hostenv.Volume, slug string) *worldModel {
231245 return model232246 }233247 248+func (a *App) outlineWorld(volume hostenv.Volume, slug string) *worldModel {249+ semantic, held := semanticWorld(volume.Outline(), slug)250+ if !held {251+ return nil252+ }253+ model, err := buildVNextWorld(semantic, volume.Outline().Assets)254+ if err != nil {255+ return nil256+ }257+ summaries, err := volume.FeatureSetSummaries()258+ if err != nil {259+ return nil260+ }261+ counts := make(map[string]int, len(summaries))262+ for _, summary := range summaries {263+ counts[summary.FeatureSet] = summary.Rows264+ }265+ for _, collection := range model.Members {266+ collection.Count = counts[collection.FeatureSet]267+ }268+ return model269+}270+234271 func shapeDepth(model *worldModel, shape *shapeModel, guard int) int {235272 if shape.Parent == "" || guard > 32 {236273 return 0
M render/data/payload.ts +4 -0
@@ -122,6 +122,10 @@ export type TextPayload = Readonly<Record<string, TextEntry>>;
122122 /** The volume's world square and the window its worlds were cut from. */123123 export interface TileGrid {124124 readonly sourceZoom: number;125+ /** Native windows may begin at different source tile columns and rows. */126+ readonly originX?: number;127+ readonly originY?: number;128+ /** Legacy square-window origin; native coordinate spaces do not use it. */125129 readonly firstTile: number;126130 readonly tileSize: number;127131 readonly size: number;
M render/data/plane.ts +213 -35
@@ -24,12 +24,12 @@ import { logger } from "../log.ts";
2424 import type { Catalog, Lens } from "./payload.ts";2525 import { tileFormat, tileTemplate } from "./payload.ts";2626 import {27- decodeVolume, TABLE_NAMES,28- type NativeTables, type Volume, type World,27+ type Asset, type Feature, type Geometry, type Property, type PropertyValue,28+ type Volume, type World,2929 } from "./semantic.ts";30-import { PackedTable, type Schema } from "./vnext.ts";3130 3231 const log = logger("data");32+const MAX_VISIBLE_FEATURE_PAGES = 32;3333 3434 /** A build's URL prefix: `/data/v/<slug>/<stamp12>`. */3535 export type Base = string;
@@ -39,6 +39,12 @@ export interface OpenWorld {
3939 readonly world: World;4040 }4141 42+export interface FeatureWindow {43+ readonly x: number;44+ readonly y: number;45+ readonly zoom: number;46+}47+4248 /** Raised when a base answered 404 — the build moved out from under us. */4349 export class BuildMovedError extends Error {4450 readonly url: string;
@@ -68,11 +74,24 @@ export class DataPlane {
6874 }6975 7076 /** One semantic world, reconstructed from the schema-described tables. */71- async world(base: Base, world: string): Promise<OpenWorld> {72- const volume = await this.volume(base);73- const found = volume.worlds.find((candidate) => candidate.id === world);77+ async world(base: Base, world: string, window: FeatureWindow | null = null): Promise<OpenWorld> {78+ const outline = await this.outline(base);79+ const found = outline.worlds.find((candidate) => candidate.id === world);7480 if (!found) throw new Error(`${base} does not contain world ${world}`);75- return { volume, world: found };81+ const bounds = visibleBounds(found.coordinateSpace.extent, window);82+ const pages = await Promise.all(found.featureSets.map((set) => this.features(base, set.id, bounds)));83+ const opened = {84+ ...found,85+ featureSets: found.featureSets.map((set, index) => ({ ...set, features: pages[index] ?? [] })),86+ };87+ const volume = {88+ ...outline,89+ worlds: outline.worlds.map((candidate) => candidate.id === world ? opened : candidate),90+ };91+ log.info("the semantic world is open", {92+ op: "render", path: base, world, features: pages.reduce((sum, page) => sum + page.length, 0),93+ });94+ return { volume, world: opened };7695 }7796 7897 /** Where an icon asset lives. Icons are fetched by the browser, as images. */
@@ -87,35 +106,33 @@ export class DataPlane {
87106 return `${base}/${fillTemplate(tileTemplate(lens), z, x, y, extension)}`;88107 }89108 90- private volume(base: Base): Promise<Volume> {91- return this.keep(`${base}/@semantic-volume`, async () => {92- const schemaURL = `${base}/schema.json`;93- const schemaBytes = new Uint8Array(await this.bytes(schemaURL));94- const schema = JSON.parse(new TextDecoder().decode(schemaBytes)) as Schema;95- const schemaHash = await sha256(schemaBytes);96- const entries = await Promise.all(TABLE_NAMES.map(async (name) => {97- const url = `${base}/data/${name}.pack`;98- const packed = PackedTable.over(await this.bytes(url));99- if (packed.schemaHash !== schemaHash) throw new Error(`${url} was encoded against another schema`);100- return [name, packed] as const;101- }));102- const tables = new Map(entries) as NativeTables;103- const volume = await decodeVolume(schema, tables);104- log.info("the semantic volume is open", {105- op: "render", path: base, worlds: volume.worlds.length,109+ private outline(base: Base): Promise<Volume> {110+ return this.keep(`${base}/@outline`, async () => decodeOutline(await this.json<unknown>(`${base}/outline.json`)));111+ }112+113+ private async features(base: Base, set: string, bounds: readonly [number, number, number, number]): Promise<readonly Feature[]> {114+ const features: Feature[] = [];115+ let after = "";116+ for (let pageIndex = 0; pageIndex < MAX_VISIBLE_FEATURE_PAGES; pageIndex++) {117+ const query = new URLSearchParams({118+ limit: "1000", minX: String(bounds[0]), minY: String(bounds[1]),119+ maxX: String(bounds[2]), maxY: String(bounds[3]),106120 });107- return volume;108- });121+ if (after) query.set("after", after);122+ const page = await this.json<Record<string, unknown>>(123+ `${base}/features/${encodeURIComponent(set)}.json?${query.toString()}`,124+ );125+ for (const feature of array(page.features)) features.push(decodeFeature(object(feature)));126+ after = text(page.next);127+ if (!after) return features;128+ }129+ throw new Error(`visible feature window for ${set} exceeds ${MAX_VISIBLE_FEATURE_PAGES * 1000} features`);109130 }110131 111132 private json<T>(url: string): Promise<T> {112133 return this.keep(url, async (at) => (await this.response(at)).json() as Promise<T>);113134 }114135 115- private async bytes(url: string): Promise<ArrayBuffer> {116- return (await this.response(url)).arrayBuffer();117- }118-119136 private async response(url: string): Promise<Response> {120137 const response = await fetch(url);121138 if (response.status === 404) {
@@ -142,6 +159,173 @@ export class DataPlane {
142159 }143160 }144161 162+function visibleBounds(163+ extent: readonly [number, number, number, number],164+ window: FeatureWindow | null,165+): readonly [number, number, number, number] {166+ if (!window || !Number.isFinite(window.x) || !Number.isFinite(window.y) || !Number.isFinite(window.zoom)) return extent;167+ const width = extent[2] - extent[0];168+ const height = extent[3] - extent[1];169+ const span = Math.max(width, height) / 2 ** Math.max(0, window.zoom);170+ const x = Math.min(Math.max(window.x, extent[0]), extent[2]);171+ const y = Math.min(Math.max(-window.y, extent[1]), extent[3]);172+ return [173+ Math.max(extent[0], x - span / 2), Math.max(extent[1], y - span / 2),174+ Math.min(extent[2], x + span / 2), Math.min(extent[3], y + span / 2),175+ ];176+}177+178+function decodeOutline(source: unknown): Volume {179+ const root = object(source);180+ if (number(root.version) !== 1) throw new Error("native outline version is unsupported");181+ const assets = array(root.assets).map((entry) => decodeAsset(object(entry)));182+ return {183+ id: text(root.id), title: text(root.title), assets: new Map(assets.map((asset) => [asset.id, asset])),184+ worlds: array(root.worlds).map((entry) => decodeWorld(object(entry))),185+ };186+}187+188+function decodeWorld(raw: Record<string, unknown>): World {189+ const coordinate = object(raw.coordinateSpace);190+ const presentation = object(raw.presentation);191+ return {192+ id: text(raw.id), title: text(raw.title), claims: wireProperties(raw.claims),193+ coordinateSpace: {194+ id: text(coordinate.id), kind: text(coordinate.kind), unit: text(coordinate.unit),195+ definition: text(coordinate.definition), extent: tuple4(coordinate.extent),196+ sourceZoom: number(coordinate.sourceZoom), originX: number(coordinate.originX),197+ originY: number(coordinate.originY), tileSize: number(coordinate.tileSize), size: number(coordinate.size),198+ },199+ featureSets: array(raw.featureSets).map((entry) => {200+ const set = object(entry);201+ return {202+ id: text(set.id), title: text(set.title), semanticType: text(set.semanticType),203+ properties: array(set.properties).map((field) => {204+ const value = object(field);205+ return { id: text(value.id), name: text(value.name), kind: number(value.kind), optional: Boolean(value.optional) };206+ }),207+ claims: wireProperties(set.claims), features: [],208+ };209+ }),210+ rasterPyramids: array(raw.rasterPyramids).map((entry) => {211+ const raster = object(entry);212+ return {213+ id: text(raster.id), name: text(raster.name), codec: text(raster.codec), tileSize: number(raster.tileSize),214+ minZoom: number(raster.minZoom), maxZoom: number(raster.maxZoom), fullZoom: number(raster.fullZoom),215+ sourceZoom: number(raster.sourceZoom), template: text(raster.template), formats: array(raster.formats).map(text),216+ bounds: raster.bounds ? rect(object(raster.bounds)) : null, surface: raster.surface ? rect(object(raster.surface)) : null,217+ interpolate: Boolean(raster.interpolate), background: text(raster.background), shard: number(raster.shard),218+ coverage: array(raster.coverage).map((entry) => {219+ const coverage = object(entry);220+ return {221+ zoom: number(coverage.zoom), x: number(coverage.x), y: number(coverage.y), w: number(coverage.w),222+ h: number(coverage.h), bits: bytes(coverage.bits),223+ };224+ }),225+ };226+ }),227+ presentation: {228+ id: text(presentation.id), title: text(presentation.title),229+ styles: array(presentation.styles).map((entry) => {230+ const style = object(entry);231+ return {232+ id: text(style.id), symbol: text(style.symbol), icon: text(style.icon), iconAsset: text(style.iconAsset),233+ iconPicture: Boolean(style.iconPicture), renderAs: text(style.renderAs), stroke: text(style.stroke), fill: text(style.fill),234+ };235+ }),236+ layers: array(presentation.layers).map((entry) => {237+ const layer = object(entry);238+ return {239+ id: text(layer.id), featureSet: text(layer.featureSet), style: text(layer.style), group: text(layer.group),240+ labelPolicy: text(layer.labelPolicy), visible: Boolean(layer.visible), order: number(layer.order),241+ minZoom: number(layer.minZoom), maxZoom: number(layer.maxZoom),242+ };243+ }),244+ legend: array(presentation.legend).map((entry) => {245+ const legend = object(entry);246+ return { layer: text(legend.layer), label: text(legend.label), order: number(legend.order) };247+ }),248+ },249+ };250+}251+252+function decodeFeature(raw: Record<string, unknown>): Feature {253+ const center = raw.center;254+ return {255+ id: text(raw.id), title: text(raw.title), subtitle: text(raw.subtitle), description: text(raw.description),256+ center: center ? tuple2(center) : null, shard: number(raw.shard), geometry: decodeWireGeometry(object(raw.geometry)),257+ properties: wireProperties(raw.properties),258+ relationships: array(raw.relationships).map((entry) => {259+ const edge = object(entry);260+ return { predicate: text(edge.predicate), target: text(edge.target) };261+ }),262+ provenance: array(raw.provenance).map((entry) => {263+ const evidence = object(entry);264+ return { source: text(evidence.source), nativeID: text(evidence.nativeId), capturedAt: text(evidence.capturedAt) };265+ }),266+ };267+}268+269+function decodeWireGeometry(raw: Record<string, unknown>): Geometry {270+ const kind = number(raw.kind);271+ if (kind !== 1 && kind !== 2 && kind !== 3) throw new Error(`geometry kind ${kind} is unsupported`);272+ return {273+ kind,274+ parts: array(raw.parts).map((entry) => ({275+ rings: array(object(entry).rings).map((ring) => array(ring).map(tuple2)),276+ })),277+ };278+}279+280+function wireProperties(source: unknown): readonly Property[] {281+ return array(source).map((entry) => {282+ const property = object(entry);283+ const kind = number(property.kind);284+ return { id: text(property.id), name: text(property.name), kind, value: wirePropertyValue(kind, property.value) };285+ });286+}287+288+function wirePropertyValue(kind: number, value: unknown): PropertyValue {289+ switch (kind) {290+ case 1: return Boolean(value);291+ case 2: return BigInt(text(value));292+ case 3: return number(value);293+ case 4: return text(value);294+ case 5: return bytes(value);295+ case 6: return text(value);296+ default: throw new Error("property has no typed value");297+ }298+}299+300+function decodeAsset(raw: Record<string, unknown>): Asset {301+ return { id: text(raw.id), mediaType: text(raw.mediaType), path: text(raw.path), provenance: text(raw.provenance) };302+}303+304+function rect(raw: Record<string, unknown>): { x: number; y: number; width: number; height: number } {305+ return { x: number(raw.x), y: number(raw.y), width: number(raw.width), height: number(raw.height) };306+}307+308+function object(value: unknown): Record<string, unknown> {309+ if (!value || typeof value !== "object" || Array.isArray(value)) throw new Error("native response is not an object");310+ return value as Record<string, unknown>;311+}312+313+function array(value: unknown): unknown[] { return Array.isArray(value) ? value : []; }314+function text(value: unknown): string { return typeof value === "string" ? value : ""; }315+function number(value: unknown): number { return typeof value === "number" ? value : Number(value ?? 0); }316+function tuple2(value: unknown): readonly [number, number] {317+ const held = array(value);318+ return [number(held[0]), number(held[1])];319+}320+function tuple4(value: unknown): readonly [number, number, number, number] {321+ const held = array(value);322+ return [number(held[0]), number(held[1]), number(held[2]), number(held[3])];323+}324+function bytes(value: unknown): Uint8Array {325+ if (typeof value !== "string" || !value) return new Uint8Array();326+ return Uint8Array.from(atob(value), (character) => character.charCodeAt(0));327+}328+145329 /**146330 * Where an icon asset lives.147331 *
@@ -163,9 +347,3 @@ export function fillTemplate(template: string, z: number, x: number, y: number,
163347 return template.replaceAll("{z}", String(z)).replaceAll("{x}", String(x))164348 .replaceAll("{y}", String(y)).replaceAll("{format}", format);165349 }166-167-async function sha256(bytes: Uint8Array): Promise<string> {168- const copy = Uint8Array.from(bytes);169- const digest = new Uint8Array(await crypto.subtle.digest("SHA-256", copy.buffer));170- return [...digest].map((byte) => byte.toString(16).padStart(2, "0")).join("");171-}
M render/data/semantic.ts +5 -3
@@ -73,7 +73,8 @@ export interface CoordinateSpace {
7373 readonly definition: string;7474 readonly extent: readonly [number, number, number, number];7575 readonly sourceZoom: number;76- readonly firstTile: number;76+ readonly originX: number;77+ readonly originY: number;7778 readonly tileSize: number;7879 readonly size: number;7980 }
@@ -235,7 +236,7 @@ const CORE_FIELDS = [
235236 "world.id", "world.title", "world.coordinateSpace",236237 "coordinate.id", "coordinate.world", "coordinate.kind", "coordinate.unit",237238 "coordinate.definition", "coordinate.extent", "coordinate.sourceZoom",238- "coordinate.firstTile", "coordinate.tileSize", "coordinate.size",239+ "coordinate.originX", "coordinate.originY", "coordinate.tileSize", "coordinate.size",239240 "featureSet.id", "featureSet.world", "featureSet.title", "featureSet.semanticType",240241 "propertyDefinition.featureSet", "propertyDefinition.field", "propertyDefinition.name",241242 "propertyDefinition.kind", "propertyDefinition.optional",
@@ -410,7 +411,8 @@ function decodeWorlds(table: (name: TableName) => NativeTable): Map<string, Worl
410411 id: coordinates.string("coordinate.id", row), kind: coordinates.string("coordinate.kind", row),411412 unit: coordinates.string("coordinate.unit", row), definition: coordinates.string("coordinate.definition", row),412413 extent: decodeExtent(coordinates.bytes("coordinate.extent", row)),413- sourceZoom: coordinates.int("coordinate.sourceZoom", row), firstTile: coordinates.int("coordinate.firstTile", row),414+ sourceZoom: coordinates.int("coordinate.sourceZoom", row),415+ originX: coordinates.int("coordinate.originX", row), originY: coordinates.int("coordinate.originY", row),414416 tileSize: coordinates.int("coordinate.tileSize", row), size: coordinates.int("coordinate.size", row),415417 };416418 }
A render/data/window.ts +15 -0
@@ -0,0 +1,15 @@
1+import type { Camera } from "../scene/read.ts";2+3+/** A settled camera key precise enough that materially different bboxes differ. */4+export function demandWindowKey(camera: Camera | null): string {5+ if (!camera) return "fit";6+ return `${camera.zoom.toFixed(2)}/${camera.x.toFixed(2)}/${camera.y.toFixed(2)}`;7+}8+9+/** Keep one demanded window per world; a pan replaces rather than accumulates. */10+export function replaceDemandWindow<T>(cache: Map<string, T>, prefix: string, key: string, value: T): void {11+ for (const held of cache.keys()) {12+ if (held.startsWith(prefix) && held !== key) cache.delete(held);13+ }14+ cache.set(key, value);15+}
A render/test/demand.test.ts +89 -0
@@ -0,0 +1,89 @@
1+import { strict as assert } from "node:assert";2+import test from "node:test";3+4+import { DataPlane } from "../data/plane.ts";5+import { demandWindowKey, replaceDemandWindow } from "../data/window.ts";6+7+test("page zero is outline-only and features arrive through bounded pages", async () => {8+ const requested: string[] = [];9+ const original = globalThis.fetch;10+ globalThis.fetch = (async (input: string | URL | Request) => {11+ const url = String(input);12+ requested.push(url);13+ if (url.endsWith("/outline.json")) return Response.json(outline());14+ if (url.includes("/features/roads.json")) {15+ const continued = url.includes("after=more-is-demanded-later");16+ return Response.json({17+ features: continued ? [] : [feature()], next: continued ? "" : "more-is-demanded-later",18+ });19+ }20+ return new Response(null, { status: 404 });21+ }) as typeof fetch;22+ try {23+ const opened = await new DataPlane().world("/data/v/sample/abc123abc123", "sample-region");24+ assert.equal(opened.world.featureSets[0]?.features.length, 1);25+ assert.equal(opened.world.claims[0]?.value, 9223372036854775807n);26+ assert.equal(opened.world.claims[1]?.value, -9223372036854775808n);27+ assert.equal(opened.world.featureSets[0]?.features[0]?.properties[0]?.value, 9223372036854775807n);28+ assert.equal(opened.world.featureSets[0]?.features[0]?.properties[1]?.value, -9223372036854775808n);29+ assert.deepEqual(requested, [30+ "/data/v/sample/abc123abc123/outline.json",31+ "/data/v/sample/abc123abc123/features/roads.json?limit=1000&minX=0&minY=0&maxX=100&maxY=100",32+ "/data/v/sample/abc123abc123/features/roads.json?limit=1000&minX=0&minY=0&maxX=100&maxY=100&after=more-is-demanded-later",33+ ]);34+ assert.ok(requested.every((url) => !url.endsWith("/data/features.pack")));35+ } finally {36+ globalThis.fetch = original;37+ }38+});39+40+test("settled pans replace one world window and zoom changes keep distinct bounds", () => {41+ const cache = new Map<string, number>();42+ const prefix = "/data/v/sample/build/sample-region/";43+ for (let pan = 0; pan < 100; pan++) {44+ const key = `${prefix}${demandWindowKey({ x: pan, y: -pan, zoom: 3.1 + pan / 100, rotation: 0 })}`;45+ replaceDemandWindow(cache, prefix, key, pan);46+ }47+ assert.equal(cache.size, 1);48+ assert.notEqual(49+ demandWindowKey({ x: 10, y: -10, zoom: 3.1, rotation: 0 }),50+ demandWindowKey({ x: 10, y: -10, zoom: 3.9, rotation: 0 }),51+ );52+});53+54+function outline(): Record<string, unknown> {55+ return {56+ version: 1, id: "sample", title: "Sample", assets: [],57+ worlds: [{58+ id: "sample-region", title: "Sample Region", claims: [{59+ id: "33333333-3333-5333-8333-333333333333", name: "sample.maximum", kind: 2,60+ value: "9223372036854775807",61+ }, {62+ id: "44444444-4444-5444-8444-444444444444", name: "sample.minimum", kind: 2,63+ value: "-9223372036854775808",64+ }],65+ coordinateSpace: {66+ id: "sample-space", kind: "projected", unit: "metre", definition: "sample",67+ extent: [0, 0, 100, 100], sourceZoom: "0", originX: "0", originY: "0", tileSize: "256", size: "100",68+ },69+ featureSets: [{ id: "roads", title: "Roads", semanticType: "transport.road", properties: [], claims: [] }],70+ rasterPyramids: [],71+ presentation: { id: "default", title: "Default", styles: [], layers: [], legend: [] },72+ }],73+ };74+}75+76+function feature(): Record<string, unknown> {77+ return {78+ id: "road-1", title: "Sample Road", subtitle: "", description: "", center: null, shard: "0",79+ geometry: { kind: 2, parts: [{ rings: [[[0, 0], [10, 10]]] }] },80+ properties: [{81+ id: "11111111-1111-5111-8111-111111111111", name: "sample.extreme", kind: 2,82+ value: "9223372036854775807",83+ }, {84+ id: "22222222-2222-5222-8222-222222222222", name: "sample.minimum", kind: 2,85+ value: "-9223372036854775808",86+ }],87+ relationships: [], provenance: [],88+ };89+}
M render/test/legacy.ts +2 -1
@@ -54,7 +54,8 @@ export function legacyOpenWorld(
5454 coordinateSpace: {5555 id: `${slug}/space`, kind: "projected", unit: "pixel", definition: "atlas:tile-plane",5656 extent: [0, 0, grid.size, grid.size], sourceZoom: grid.sourceZoom,57- firstTile: grid.firstTile, tileSize: grid.tileSize, size: grid.size,57+ originX: grid.originX ?? grid.firstTile, originY: grid.originY ?? grid.firstTile,58+ tileSize: grid.tileSize, size: grid.size,5859 },5960 claims: properties(payload.attrs), featureSets: sets, rasterPyramids: rasters,6061 presentation: {
M render/test/vnext.test.ts +16 -3
@@ -5,7 +5,7 @@ import {
55 type FieldSchema, type Schema,66 } from "../data/vnext.ts";77 import { coreID, decodeGeometry } from "../data/semantic.ts";8-import { presentedGrid, presentedKind } from "../world/model.ts";8+import { presentedGrid, presentedKind, project } from "../world/model.ts";99 1010 const ROOT = "11111111-1111-5111-8111-111111111111";1111 const TITLE = "22222222-2222-5222-8222-222222222222";
@@ -98,13 +98,26 @@ test("semantic meaning does not have to masquerade as geometry", () => {
9898 test("feature-only coordinate spaces derive a finite presentation grid", () => {9999 assert.deepEqual(presentedGrid({100100 id: "sample-grid", kind: "projected", unit: "metre", definition: "local sample grid",101- extent: [10, 20, 110, 70], sourceZoom: 0, firstTile: 0, tileSize: 0, size: 0,101+ extent: [10, 20, 110, 70], sourceZoom: 0, originX: 7, originY: 11, tileSize: 0, size: 0,102102 }), {103- sourceZoom: 0, firstTile: 0, tileSize: 256, size: 100,103+ sourceZoom: 0, originX: 7, originY: 11, firstTile: 0, tileSize: 256, size: 100,104104 extent: [10, -70, 110, -20],105105 });106106 });107107 108+test("native tile windows use independent column and row origins", () => {109+ const grid = {110+ sourceZoom: 4, originX: 3, originY: 6, firstTile: 99, tileSize: 256, size: 1024,111+ };112+ const atOrigin = project(grid, mercatorLatitude(6 / 16), -112.5);113+ assert.ok(Math.abs(atOrigin[0]) < 1e-9, `${atOrigin[0]} is the local x origin`);114+ assert.ok(Math.abs(atOrigin[1]) < 1e-9, `${atOrigin[1]} is the local y origin`);115+});116+117+function mercatorLatitude(y: number): number {118+ return Math.atan(Math.sinh(Math.PI * (1 - 2 * y))) * 180 / Math.PI;119+}120+108121 function fixtureBlock(): ArrayBuffer {109122 const title = variableColumn([new TextEncoder().encode("Clinic"), null]);110123 const emergency = fixedInt64Column([4n, 5n]);
M render/viewport.ts +8 -4
@@ -23,10 +23,11 @@ import { logger } from "./log.ts";
2323 import { wireKeyboard } from "./keys.ts";2424 import { RowHover } from "./hover.ts";2525 import { DataPlane } from "./data/plane.ts";26+import { demandWindowKey, replaceDemandWindow } from "./data/window.ts";2627 import { reportCamera } from "./data/report.ts";2728 import type { Lens } from "./data/payload.ts";2829 import { SceneWatcher } from "./scene/observe.ts";29-import type { Scene, SceneChange } from "./scene/read.ts";30+import { sceneChange, type Scene, type SceneChange } from "./scene/read.ts";3031 import { WorldModel } from "./world/model.ts";3132 import { Visibility } from "./world/visibility.ts";3233 import type { WorldContext } from "./context.ts";
@@ -61,6 +62,8 @@ export class AtlasViewport extends HTMLElement {
6162 const scene = this.watcher?.scene;6263 if (!scene?.volume) return;6364 reportCamera({ volume: scene.volume, world: scene.world, ...camera });65+ const windowed: Scene = { ...scene, camera };66+ void this.apply(windowed, sceneChange(scene, windowed));6467 });6568 this.unkey?.();6669 this.unkey = wireKeyboard(this);
@@ -423,13 +426,14 @@ export class AtlasViewport extends HTMLElement {
423426 }424427 425428 private model(scene: Scene): Promise<WorldModel> {426- const key = `${scene.base}/${scene.world}`;429+ const prefix = `${scene.base}/${scene.world}/`;430+ const key = `${prefix}${demandWindowKey(scene.camera)}`;427431 const held = this.worlds.get(key);428432 if (held) return held;429433 const building = (async () => {430- return new WorldModel(await this.plane.world(scene.base, scene.world));434+ return new WorldModel(await this.plane.world(scene.base, scene.world, scene.camera));431435 })();432- this.worlds.set(key, building);436+ replaceDemandWindow(this.worlds, prefix, key, building);433437 return building;434438 }435439
M render/world/model.ts +7 -2
@@ -28,7 +28,10 @@ export function project(grid: TileGrid, lat: number, lng: number): Coordinate {
2828 const worldTiles = 2 ** grid.sourceZoom;2929 const xTile = ((lng + 180) / 360) * worldTiles;3030 const yTile = (1 - Math.asinh(Math.tan((lat * Math.PI) / 180)) / Math.PI) / 2 * worldTiles;31- return [(xTile - grid.firstTile) * grid.tileSize, -(yTile - grid.firstTile) * grid.tileSize];31+ return [32+ (xTile - (grid.originX ?? grid.firstTile)) * grid.tileSize,33+ -(yTile - (grid.originY ?? grid.firstTile)) * grid.tileSize,34+ ];3235 }3336 3437 /** Kept for test fixtures extracted before coordinate spaces became native. */
@@ -51,7 +54,9 @@ export function presentedGrid(space: CoordinateSpace): TileGrid {
5154 const height = maxY - minY;5255 return {5356 sourceZoom: space.sourceZoom,54- firstTile: space.firstTile,57+ originX: space.originX,58+ originY: space.originY,59+ firstTile: 0,5560 tileSize: space.tileSize > 0 ? space.tileSize : 256,5661 size: space.size > 0 ? space.size : Math.max(width, height),5762 extent: [minX, -maxY, maxX, -minY],
M tests/island/harness_test.go +40 -1
@@ -189,11 +189,50 @@ func (v *corpusBundle) Info() hostenv.VolumeInfo {
189189 return info190190 }191191 192-func (v *corpusBundle) Semantic() vnext.Volume {192+func (v *corpusBundle) semantic() vnext.Volume {193193 semantic, _ := vnext.ImportV3Volume(v.manifest, func(name string) ([]byte, error) { return v.Blob(name) })194194 return semantic195195 }196196 197+func (v *corpusBundle) Outline() vnext.Volume {198+ semantic := v.semantic()199+ for worldIndex := range semantic.Worlds {200+ for setIndex := range semantic.Worlds[worldIndex].FeatureSets {201+ semantic.Worlds[worldIndex].FeatureSets[setIndex].Features = nil202+ }203+ }204+ return semantic205+}206+207+func (v *corpusBundle) FeaturePage(request vnext.FeaturePageRequest) (vnext.FeaturePageResult, error) {208+ for _, world := range v.semantic().Worlds {209+ for _, set := range world.FeatureSets {210+ if set.ID != request.FeatureSet {211+ continue212+ }213+ return vnext.FeaturePageResult{Features: set.Features}, nil214+ }215+ }216+ return vnext.FeaturePageResult{}, os.ErrNotExist217+}218+219+func (v *corpusBundle) FeatureSetSummaries() ([]vnext.FeatureSetSummary, error) {220+ var out []vnext.FeatureSetSummary221+ for _, world := range v.semantic().Worlds {222+ for _, set := range world.FeatureSets {223+ out = append(out, vnext.FeatureSetSummary{FeatureSet: set.ID, Rows: len(set.Features)})224+ }225+ }226+ return out, nil227+}228+229+func (v *corpusBundle) DemandAddressedFeatures() bool { return false }230+231+func (v *corpusBundle) RasterTile(name string) (vnext.RasterTile, error) {232+ data, err := v.Blob(name)233+ return vnext.RasterTile{Name: name, Data: data}, err234+}235+197236 func (v *corpusBundle) Blob(name string) ([]byte, error) {198237 held, ok := v.entries[name]199238 if !ok {