- commit
- 9a887365dab82948d04c2e6c935944b051c139fa
- parent
- a32d5f24e7a752ef2faecf5887f095a1e89edb07
- author
- Dami <dami.etoile@gmail.com>
| M | format/ | 29 | +- |
| M | format/ | 71 | ++++- |
| M | format/ | 3 | +- |
| M | format/ | 13 | +- |
| A | format/ | 50 | +++ |
| A | format/ | 521 | ++++++++++++++++++++++++++++++ |
| A | format/ | 332 | +++++++++++++++++++ |
| A | format/ | 355 | ++++++++++++++++++++ |
| M | format/ | 3 | +- |
@@ -6,6 +6,7 @@ import (
66 "fmt"77 "math"88 "slices"9+ "strings"910 )1011 1112 const (@@ -35,9 +36,10 @@ type NamedTable struct {
3536 3637 // Blob is an opaque, independently range-readable payload.3738 type Blob struct {38- Name string39- Data []byte40- Path string39+ Name string40+ Data []byte41+ Path string42+ temporary bool4143 }4244 4345 // Bundle is the complete compiled form accepted by Write.@@ -111,14 +113,27 @@ func Compile(volume Volume) (Bundle, error) {
111113 return Bundle{}, err112114 }113115 names := tableNames()114- tables := make([]NamedTable, len(names))115- for index, name := range names {116+ tables := make([]NamedTable, 0, len(names))117+ physical := make(map[string]Table, len(names))118+ for _, name := range names {116119 table := builders[name].table()117120 if err := table.validate(); err != nil {118121 return Bundle{}, fmt.Errorf("compile %s: %w", name, err)119122 }120- tables[index] = NamedTable{Name: name, Table: table}123+ physical[name] = table124+ partitioned := physical[featureTableName].Rows > 0 &&125+ (name == featureTableName || name == relationshipTableName || name == provenanceTableName)126+ if !partitioned {127+ tables = append(tables, NamedTable{Name: name, Table: table})128+ }129+ }130+ partitions, index, err := partitionFeatureTables(volume, physical)131+ if err != nil {132+ return Bundle{}, err121133 }134+ tables = append(tables, partitions...)135+ blobs = append(blobs, index)136+ slices.SortFunc(tables, func(left, right NamedTable) int { return strings.Compare(left.Name, right.Name) })122137 return Bundle{123138 VolumeID: volume.ID,124139 Release: Release{Title: volume.Title, Worlds: len(volume.Worlds)},@@ -261,7 +276,7 @@ func compileCoordinate(world World, builder *tableBuilder) error {
261276 "id": StringValue(space.ID), "world": StringValue(world.ID),262277 "kind": StringValue(space.Kind), "unit": StringValue(space.Unit),263278 "definition": StringValue(space.Definition), "extent": BytesValue(encodeExtent(space.Extent)),264- "sourceZoom": Int64Value(space.SourceZoom), "firstTile": Int64Value(space.FirstTile),279+ "sourceZoom": Int64Value(space.SourceZoom), "originX": Int64Value(space.OriginX), "originY": Int64Value(space.OriginY),265280 "tileSize": Int64Value(space.TileSize), "size": Int64Value(space.Size),266281 }))267282 }@@ -19,7 +19,7 @@ const (
1919 manifestName = "atlas.json"2020 containerFormat = "atlas-schema-bundle"2121 containerFraming = uint16(1)22- maxManifestSize = 1 << 2022+ maxManifestSize = 32 << 202323 )2424 2525 type schemaReference struct {@@ -71,7 +71,7 @@ type Limits struct {
7171 7272 func DefaultLimits() Limits {7373 return Limits{74- MaxEntries: 100_000, MaxTables: 4_096, MaxBlobs: 90_000,74+ MaxEntries: 100_000, MaxTables: 65_536, MaxBlobs: 90_000,7575 MaxTableBytes: 512 << 20, MaxBlobBytes: 512 << 20, MaxTotalBytes: 1 << 40,7676 }7777 }@@ -113,6 +113,12 @@ func Write(out io.Writer, bundle Bundle) error {
113113 // The stamp is always derived here; callers cannot smuggle a stale identity114114 // across a changed schema or blob set.115115 bundle.Release.Stamp = ""116+ packedBlobs, err := packRasterBlobs(bundle.Blobs)117+ if err != nil {118+ return err119+ }120+ defer cleanupTemporaryBlobs(packedBlobs)121+ bundle.Blobs = packedBlobs116122 schemaData, err := bundle.Schema.Canonical()117123 if err != nil {118124 return fmt.Errorf("canonicalize schema: %w", err)@@ -255,6 +261,7 @@ type Reader struct {
255261 tables map[string]tableReference256262 blobs map[string]blobReference257263 limits Limits264+ source io.ReaderAt258265 }259266 260267 // Open reads the bootstrap and embedded schema. Only container framing is a@@ -293,7 +300,7 @@ func OpenWithLimits(source io.ReaderAt, size int64, applicationSchema Schema, li
293300 reader := &Reader{294301 VolumeID: manifest.Volume, Release: manifest.Release, FileSchema: fileSchema, RuntimeSchema: runtimeSchema,295302 entries: entries, tables: make(map[string]tableReference, len(manifest.Tables)),296- blobs: make(map[string]blobReference, len(manifest.Blobs)), limits: limits,303+ blobs: make(map[string]blobReference, len(manifest.Blobs)), limits: limits, source: source,297304 }298305 if err := reader.indexReferences(manifest); err != nil {299306 return nil, err@@ -439,6 +446,9 @@ func validateReference(name, hash string, length, maximum int64, entry *zip.File
439446 func (reader *Reader) Table(name string) (Table, error) {440447 reference, held := reader.tables[name]441448 if !held {449+ if table, partitioned, err := reader.partitionedTable(name); partitioned || err != nil {450+ return table, err451+ }442452 return Table{}, fmt.Errorf("bundle has no table %s", name)443453 }444454 data, err := readReferencedEntry(reader.entries[name], reference.Length, reference.Hash, reader.limits.MaxTableBytes)@@ -483,6 +493,16 @@ func (reader *Reader) Table(name string) (Table, error) {
483493 func (reader *Reader) TableBlock(name string) ([]byte, error) {484494 reference, held := reader.tables[name]485495 if !held {496+ if table, partitioned, err := reader.partitionedTable(name); partitioned || err != nil {497+ if err != nil {498+ return nil, err499+ }500+ schemaHash, err := reader.FileSchema.Hash()501+ if err != nil {502+ return nil, err503+ }504+ return EncodeBlock(schemaHash, table)505+ }486506 return nil, fmt.Errorf("bundle has no table %s", name)487507 }488508 return readReferencedEntry(reader.entries[name], reference.Length, reference.Hash, reader.limits.MaxTableBytes)@@ -500,6 +520,9 @@ func tableHasColumn(table Table, fieldID ID) bool {
500520 func (reader *Reader) Blob(name string) ([]byte, error) {501521 reference, held := reader.blobs[name]502522 if !held {523+ if tile, err := reader.ReadRasterTile(name); err == nil {524+ return tile.Data, nil525+ }503526 return nil, fmt.Errorf("bundle has no blob %s", name)504527 }505528 return readReferencedEntry(reader.entries[name], reference.Length, reference.Hash, reader.limits.MaxBlobBytes)@@ -515,23 +538,57 @@ func (reader *Reader) TableNames() []string {
515538 }516539 517540 func (reader *Reader) BlobNames() []string {541+ names, _ := reader.LogicalBlobNames()542+ return names543+}544+545+// LogicalBlobNames returns public asset and tile names. Unlike BlobNames it546+// reports a malformed logical index instead of degrading to a partial list.547+func (reader *Reader) LogicalBlobNames() ([]string, error) {518548 names := make([]string, 0, len(reader.blobs))519549 for name := range reader.blobs {520- names = append(names, name)550+ if name != rasterIndexName && name != featureIndexName && !strings.HasPrefix(name, rasterShardPrefix) {551+ names = append(names, name)552+ }553+ }554+ index, err := reader.rasterIndex()555+ if err != nil {556+ return nil, err557+ }558+ for _, tile := range index.Tiles {559+ names = append(names, tile.Name)521560 }522561 slices.Sort(names)523- return names562+ return names, nil524563 }525564 526565 // Validate reads and verifies every typed table and opaque blob, then restores527566 // the semantic root. Installers call it before and after committing a file.528567 func (reader *Reader) Validate() error {568+ featureIndex, err := reader.featureIndex()569+ if err != nil {570+ return fmt.Errorf("validate feature index: %w", err)571+ }572+ if _, err := reader.rasterIndex(); err != nil {573+ return fmt.Errorf("validate raster index: %w", err)574+ }529575 for _, name := range reader.TableNames() {530- if _, err := reader.Table(name); err != nil {576+ table, err := reader.Table(name)577+ if err != nil {531578 return fmt.Errorf("validate %s: %w", name, err)532579 }580+ for _, partition := range featureIndex.Partitions {581+ if partition.Features == name && table.Rows != partition.Rows {582+ return fmt.Errorf("validate %s: index row count does not match", name)583+ }584+ }585+ }586+ blobNames := make([]string, 0, len(reader.blobs))587+ for name := range reader.blobs {588+ blobNames = append(blobNames, name)533589 }534- for _, name := range reader.BlobNames() {590+ slices.Sort(blobNames)591+ for _, name := range blobNames {535592 reference := reader.blobs[name]536593 if err := validateReferencedEntry(reader.entries[name], reference.Length, reference.Hash, reader.limits.MaxBlobBytes); err != nil {537594 return fmt.Errorf("validate %s: %w", name, err)@@ -95,7 +95,8 @@ func decodeWorlds(tables map[string]Table) ([]World, map[string]int, error) {
9595 Definition: requiredString(coordinates, row, "coordinate.definition"),9696 Extent: extent,9797 SourceZoom: requiredInt64(coordinates, row, "coordinate.sourceZoom"),98- FirstTile: requiredInt64(coordinates, row, "coordinate.firstTile"),98+ OriginX: requiredInt64(coordinates, row, "coordinate.originX"),99+ OriginY: requiredInt64(coordinates, row, "coordinate.originY"),99100 TileSize: requiredInt64(coordinates, row, "coordinate.tileSize"),100101 Size: requiredInt64(coordinates, row, "coordinate.size"),101102 }@@ -27,9 +27,13 @@ type CoordinateSpace struct {
2727 Definition string2828 Extent [4]float642929 SourceZoom int6430- FirstTile int6431- TileSize int6432- Size int6430+ OriginX int6431+ OriginY int6432+ // FirstTile is deprecated. OriginX and OriginY are authoritative in the33+ // native format; this source-only field exists until all callers migrate.34+ FirstTile int6435+ TileSize int6436+ Size int643337 }3438 3539 // FeatureSet carries semantic membership and its typed property contract.@@ -221,7 +225,8 @@ func StandardSchema() Schema {
221225 coreField("coordinate.definition", KindString, false),222226 coreField("coordinate.extent", KindBytes, false),223227 coreField("coordinate.sourceZoom", KindInt64, false),224- coreField("coordinate.firstTile", KindInt64, false),228+ coreField("coordinate.originX", KindInt64, false),229+ coreField("coordinate.originY", KindInt64, false),225230 coreField("coordinate.tileSize", KindInt64, false),226231 coreField("coordinate.size", KindInt64, false)),227232 coreType("featureSet",@@ -0,0 +1,50 @@
1+package vnext2+3+import "fmt"4+5+// Outline restores the complete semantic catalog except feature rows,6+// relationships, provenance payloads, and asset bytes. It is the page-zero7+// entry point for clients that subsequently demand-page visible features and8+// tiles from the compiled Atlas.9+func (reader *Reader) Outline() (Volume, error) {10+ tables := make(map[string]Table, len(tableNames())-3)11+ for _, name := range tableNames() {12+ if name == featureTableName || name == relationshipTableName || name == provenanceTableName {13+ continue14+ }15+ table, err := reader.Table(name)16+ if err != nil {17+ return Volume{}, err18+ }19+ tables[name] = table20+ }21+ root := tables[volumeTableName]22+ if root.Rows != 1 {23+ return Volume{}, fmt.Errorf("volume table has %d rows", root.Rows)24+ }25+ volume := Volume{ID: requiredString(root, 0, "volume.id"), Title: requiredString(root, 0, "volume.title")}26+ worlds, worldIndex, err := decodeWorlds(tables)27+ if err != nil {28+ return Volume{}, err29+ }30+ volume.Worlds = worlds31+ if _, err := decodeFeatureSets(&volume, tables, worldIndex); err != nil {32+ return Volume{}, err33+ }34+ if err := decodeWorldDetails(&volume, tables, worldIndex); err != nil {35+ return Volume{}, err36+ }37+ volume.Assets = decodeAssetOutline(tables[assetTableName])38+ return volume, nil39+}40+41+func decodeAssetOutline(table Table) []Asset {42+ assets := make([]Asset, table.Rows)43+ for row := 0; row < table.Rows; row++ {44+ assets[row] = Asset{45+ ID: requiredString(table, row, "asset.id"), MediaType: requiredString(table, row, "asset.mediaType"),46+ Path: requiredString(table, row, "asset.path"), Provenance: optionalStringAt(table, row, "asset.provenance"),47+ }48+ }49+ return assets50+}@@ -0,0 +1,521 @@
1+package vnext2+3+import (4+ "bytes"5+ "crypto/sha256"6+ "encoding/base64"7+ "encoding/binary"8+ "encoding/hex"9+ "encoding/json"10+ "fmt"11+ "io"12+ "math"13+ "slices"14+ "strings"15+)16+17+const (18+ featureIndexName = "indexes/features.v1.json"19+ featurePartitionPrefix = "data/feature-partitions/"20+ featurePartitionRows = 4_09621+ maxFeaturePageRows = 1_00022+)23+24+// SpatialBounds is a closed, axis-aligned query rectangle in a world's25+// declared coordinate space.26+type SpatialBounds struct {27+ MinX float64 `json:"minX"`28+ MinY float64 `json:"minY"`29+ MaxX float64 `json:"maxX"`30+ MaxY float64 `json:"maxY"`31+}32+33+type featurePartitionIndex struct {34+ Version uint16 `json:"version"`35+ Partitions []featurePartitionDescriptor `json:"partitions"`36+}37+38+type featurePartitionDescriptor struct {39+ FeatureSet string `json:"featureSet"`40+ Key string `json:"key"`41+ Bounds SpatialBounds `json:"bounds"`42+ Rows int `json:"rows"`43+ Features string `json:"features"`44+ Relationships string `json:"relationships"`45+ Provenance string `json:"provenance"`46+}47+48+type orderedFeature struct {49+ id string50+ key uint6451+ bounds SpatialBounds52+}53+54+// FeaturePageRequest selects a bounded page from one semantic FeatureSet.55+// After is an opaque continuation returned by the preceding page.56+type FeaturePageRequest struct {57+ FeatureSet string58+ Bounds *SpatialBounds59+ After string60+ Limit int61+}62+63+// FeaturePageResult is a demand-read page and its physical read evidence.64+type FeaturePageResult struct {65+ Features []Feature66+ Next string67+ PartitionsRead int68+ BytesRead int6469+}70+71+func partitionFeatureTables(volume Volume, tables map[string]Table) ([]NamedTable, Blob, error) {72+ features := tables[featureTableName]73+ relationships := tables[relationshipTableName]74+ provenance := tables[provenanceTableName]75+ featureRows := rowsByString(features, "feature.id")76+ index := featurePartitionIndex{Version: 1}77+ var partitions []NamedTable78+ for _, world := range volume.Worlds {79+ for _, set := range world.FeatureSets {80+ ordered, err := orderFeatures(set.Features, world.CoordinateSpace.Extent)81+ if err != nil {82+ return nil, Blob{}, fmt.Errorf("partition feature set %s: %w", set.ID, err)83+ }84+ for start := 0; start < len(ordered); start += featurePartitionRows {85+ end := min(start+featurePartitionRows, len(ordered))86+ chunk := ordered[start:end]87+ descriptor, named, err := makeFeaturePartition(set.ID, chunk, featureRows, features, relationships, provenance)88+ if err != nil {89+ return nil, Blob{}, err90+ }91+ index.Partitions = append(index.Partitions, descriptor)92+ partitions = append(partitions, named...)93+ }94+ }95+ }96+ slices.SortFunc(index.Partitions, func(left, right featurePartitionDescriptor) int {97+ if bySet := strings.Compare(left.FeatureSet, right.FeatureSet); bySet != 0 {98+ return bySet99+ }100+ return strings.Compare(left.Key, right.Key)101+ })102+ data, err := json.Marshal(index)103+ if err != nil {104+ return nil, Blob{}, fmt.Errorf("encode feature partition index: %w", err)105+ }106+ return partitions, Blob{Name: featureIndexName, Data: data}, nil107+}108+109+func orderFeatures(features []Feature, extent [4]float64) ([]orderedFeature, error) {110+ ordered := make([]orderedFeature, len(features))111+ for index, feature := range features {112+ bounds, err := geometryBounds(feature.Geometry)113+ if err != nil {114+ return nil, fmt.Errorf("feature %s: %w", feature.ID, err)115+ }116+ ordered[index] = orderedFeature{id: feature.ID, key: spatialKey(bounds, extent), bounds: bounds}117+ }118+ slices.SortFunc(ordered, func(left, right orderedFeature) int {119+ if left.key < right.key {120+ return -1121+ }122+ if left.key > right.key {123+ return 1124+ }125+ return strings.Compare(left.id, right.id)126+ })127+ return ordered, nil128+}129+130+func makeFeaturePartition(131+ setID string,132+ ordered []orderedFeature,133+ featureRows map[string]int,134+ features Table,135+ relationships Table,136+ provenance Table,137+) (featurePartitionDescriptor, []NamedTable, error) {138+ ids := make(map[string]bool, len(ordered))139+ rows := make([]int, len(ordered))140+ bounds := ordered[0].bounds141+ for index, feature := range ordered {142+ row, held := featureRows[feature.id]143+ if !held {144+ return featurePartitionDescriptor{}, nil, fmt.Errorf("feature table omits %s", feature.id)145+ }146+ ids[feature.id] = true147+ rows[index] = row148+ bounds = unionBounds(bounds, feature.bounds)149+ }150+ identity := fmt.Sprintf("%s\x00%016x\x00%s\x00%016x\x00%s", setID, ordered[0].key, ordered[0].id, ordered[len(ordered)-1].key, ordered[len(ordered)-1].id)151+ digest := sha256.Sum256([]byte(identity))152+ token := hex.EncodeToString(digest[:12])153+ base := featurePartitionPrefix + token154+ descriptor := featurePartitionDescriptor{155+ FeatureSet: setID, Key: fmt.Sprintf("%016x/%s", ordered[0].key, ordered[0].id), Bounds: bounds, Rows: len(rows),156+ Features: base + ".features.pack", Relationships: base + ".relationships.pack", Provenance: base + ".provenance.pack",157+ }158+ named := []NamedTable{159+ {Name: descriptor.Features, Table: selectRows(features, rows)},160+ {Name: descriptor.Relationships, Table: selectRowsForIDs(relationships, "relationship.feature", ids)},161+ {Name: descriptor.Provenance, Table: selectRowsForIDs(provenance, "provenance.feature", ids)},162+ }163+ return descriptor, named, nil164+}165+166+func rowsByString(table Table, field string) map[string]int {167+ rows := make(map[string]int, table.Rows)168+ for row := 0; row < table.Rows; row++ {169+ rows[requiredString(table, row, field)] = row170+ }171+ return rows172+}173+174+func selectRowsForIDs(table Table, field string, ids map[string]bool) Table {175+ rows := make([]int, 0)176+ for row := 0; row < table.Rows; row++ {177+ if ids[requiredString(table, row, field)] {178+ rows = append(rows, row)179+ }180+ }181+ slices.SortFunc(rows, func(left, right int) int {182+ return bytes.Compare(stableRowKey(table, left), stableRowKey(table, right))183+ })184+ return selectRows(table, rows)185+}186+187+func stableRowKey(table Table, row int) []byte {188+ values := make([]Value, len(table.Columns))189+ for index, column := range table.Columns {190+ values[index] = column.Values[row]191+ }192+ encoded, _ := json.Marshal(values)193+ return encoded194+}195+196+func selectRows(table Table, rows []int) Table {197+ out := Table{TypeID: table.TypeID, Rows: len(rows), Columns: make([]Column, len(table.Columns))}198+ for columnIndex, column := range table.Columns {199+ values := make([]Value, len(rows))200+ for index, row := range rows {201+ values[index] = column.Values[row]202+ }203+ out.Columns[columnIndex] = Column{Field: column.Field, Values: values}204+ }205+ return canonicalTable(out)206+}207+208+func geometryBounds(geometry Geometry) (SpatialBounds, error) {209+ bounds := SpatialBounds{MinX: math.Inf(1), MinY: math.Inf(1), MaxX: math.Inf(-1), MaxY: math.Inf(-1)}210+ for _, part := range geometry.Parts {211+ for _, ring := range part.Rings {212+ for _, position := range ring {213+ bounds.MinX = math.Min(bounds.MinX, position[0])214+ bounds.MinY = math.Min(bounds.MinY, position[1])215+ bounds.MaxX = math.Max(bounds.MaxX, position[0])216+ bounds.MaxY = math.Max(bounds.MaxY, position[1])217+ }218+ }219+ }220+ if math.IsInf(bounds.MinX, 0) {221+ return SpatialBounds{}, fmt.Errorf("geometry has no positions")222+ }223+ return bounds, nil224+}225+226+func spatialKey(bounds SpatialBounds, extent [4]float64) uint64 {227+ x := quantize((bounds.MinX+bounds.MaxX)/2, extent[0], extent[2])228+ y := quantize((bounds.MinY+bounds.MaxY)/2, extent[1], extent[3])229+ return morton32(x, y)230+}231+232+func quantize(value, minimum, maximum float64) uint32 {233+ if maximum <= minimum || value <= minimum {234+ return 0235+ }236+ if value >= maximum {237+ return math.MaxUint32238+ }239+ return uint32((value - minimum) / (maximum - minimum) * float64(math.MaxUint32))240+}241+242+func morton32(x, y uint32) uint64 {243+ var out uint64244+ for bit := range 32 {245+ out |= uint64((x>>bit)&1) << (bit * 2)246+ out |= uint64((y>>bit)&1) << (bit*2 + 1)247+ }248+ return out249+}250+251+func unionBounds(left, right SpatialBounds) SpatialBounds {252+ return SpatialBounds{253+ MinX: math.Min(left.MinX, right.MinX), MinY: math.Min(left.MinY, right.MinY),254+ MaxX: math.Max(left.MaxX, right.MaxX), MaxY: math.Max(left.MaxY, right.MaxY),255+ }256+}257+258+func intersects(left, right SpatialBounds) bool {259+ return left.MinX <= right.MaxX && left.MaxX >= right.MinX && left.MinY <= right.MaxY && left.MaxY >= right.MinY260+}261+262+func (reader *Reader) featureIndex() (featurePartitionIndex, error) {263+ reference, held := reader.blobs[featureIndexName]264+ if !held {265+ return featurePartitionIndex{}, nil266+ }267+ data, err := readReferencedEntry(reader.entries[featureIndexName], reference.Length, reference.Hash, reader.limits.MaxBlobBytes)268+ if err != nil {269+ return featurePartitionIndex{}, err270+ }271+ var index featurePartitionIndex272+ decoder := json.NewDecoder(bytes.NewReader(data))273+ decoder.DisallowUnknownFields()274+ if err := decoder.Decode(&index); err != nil || index.Version != 1 {275+ return featurePartitionIndex{}, fmt.Errorf("decode feature partition index")276+ }277+ if err := decoder.Decode(&struct{}{}); err != io.EOF {278+ return featurePartitionIndex{}, fmt.Errorf("feature partition index carries trailing data")279+ }280+ canonical, err := json.Marshal(index)281+ if err != nil || !bytes.Equal(canonical, data) {282+ return featurePartitionIndex{}, fmt.Errorf("feature partition index is not canonical")283+ }284+ if err := reader.validateFeatureIndex(index); err != nil {285+ return featurePartitionIndex{}, err286+ }287+ return index, nil288+}289+290+func (reader *Reader) validateFeatureIndex(index featurePartitionIndex) error {291+ seen := make(map[string]bool, len(index.Partitions)*3)292+ previous := ""293+ for _, partition := range index.Partitions {294+ key := partition.FeatureSet + "\x00" + partition.Key295+ finite := !math.IsNaN(partition.Bounds.MinX) && !math.IsNaN(partition.Bounds.MinY) &&296+ !math.IsNaN(partition.Bounds.MaxX) && !math.IsNaN(partition.Bounds.MaxY) &&297+ !math.IsInf(partition.Bounds.MinX, 0) && !math.IsInf(partition.Bounds.MinY, 0) &&298+ !math.IsInf(partition.Bounds.MaxX, 0) && !math.IsInf(partition.Bounds.MaxY, 0)299+ if partition.FeatureSet == "" || partition.Key == "" || key <= previous || partition.Rows <= 0 ||300+ partition.Rows > featurePartitionRows || !finite || partition.Bounds.MinX > partition.Bounds.MaxX ||301+ partition.Bounds.MinY > partition.Bounds.MaxY {302+ return fmt.Errorf("feature partition index is not canonical")303+ }304+ for _, expected := range []struct {305+ name string306+ typeID ID307+ }{308+ {partition.Features, CoreID("type.feature")},309+ {partition.Relationships, CoreID("type.relationship")},310+ {partition.Provenance, CoreID("type.provenance")},311+ } {312+ reference, held := reader.tables[expected.name]313+ if !held || seen[expected.name] || !strings.HasPrefix(expected.name, featurePartitionPrefix) || reference.RootType != expected.typeID {314+ return fmt.Errorf("feature partition %s has an invalid table role", partition.Key)315+ }316+ seen[expected.name] = true317+ }318+ previous = key319+ }320+ for name := range reader.tables {321+ if strings.HasPrefix(name, featurePartitionPrefix) && !seen[name] {322+ return fmt.Errorf("feature partition table %s is not indexed", name)323+ }324+ }325+ return nil326+}327+328+func (reader *Reader) partitionedTable(name string) (Table, bool, error) {329+ index, err := reader.featureIndex()330+ if err != nil || len(index.Partitions) == 0 {331+ return Table{}, false, err332+ }333+ var names []string334+ for _, partition := range index.Partitions {335+ switch name {336+ case featureTableName:337+ names = append(names, partition.Features)338+ case relationshipTableName:339+ names = append(names, partition.Relationships)340+ case provenanceTableName:341+ names = append(names, partition.Provenance)342+ default:343+ return Table{}, false, nil344+ }345+ }346+ var combined Table347+ for _, physical := range names {348+ table, err := reader.Table(physical)349+ if err != nil {350+ return Table{}, true, err351+ }352+ combined, err = appendTable(combined, table)353+ if err != nil {354+ return Table{}, true, err355+ }356+ }357+ return combined, true, nil358+}359+360+func appendTable(left, right Table) (Table, error) {361+ if left.TypeID.isZero() {362+ return right, nil363+ }364+ if left.TypeID != right.TypeID || len(left.Columns) != len(right.Columns) {365+ return Table{}, fmt.Errorf("partition table shapes disagree")366+ }367+ out := Table{TypeID: left.TypeID, Rows: left.Rows + right.Rows, Columns: make([]Column, len(left.Columns))}368+ for index := range left.Columns {369+ if left.Columns[index].Field.ID != right.Columns[index].Field.ID {370+ return Table{}, fmt.Errorf("partition columns disagree")371+ }372+ values := append([]Value(nil), left.Columns[index].Values...)373+ values = append(values, right.Columns[index].Values...)374+ out.Columns[index] = Column{Field: left.Columns[index].Field, Values: values}375+ }376+ return out, nil377+}378+379+// FeaturePage reads only intersecting partitions until the requested page is380+// full. It never calls Volume and its physical counters make that observable.381+func (reader *Reader) FeaturePage(request FeaturePageRequest) (FeaturePageResult, error) {382+ if request.FeatureSet == "" || request.Limit < 0 || request.Limit > maxFeaturePageRows {383+ return FeaturePageResult{}, fmt.Errorf("invalid feature page request")384+ }385+ if request.Limit == 0 {386+ request.Limit = 100387+ }388+ after, err := decodeContinuation(request)389+ if err != nil {390+ return FeaturePageResult{}, err391+ }392+ index, err := reader.featureIndex()393+ if err != nil {394+ return FeaturePageResult{}, err395+ }396+ result := FeaturePageResult{}397+ passedAfter := after == ""398+ for _, partition := range index.Partitions {399+ if partition.FeatureSet != request.FeatureSet || request.Bounds != nil && !intersects(partition.Bounds, *request.Bounds) {400+ continue401+ }402+ features, bytesRead, err := reader.readFeaturePartition(partition)403+ if err != nil {404+ return FeaturePageResult{}, err405+ }406+ result.PartitionsRead++407+ result.BytesRead += bytesRead408+ for _, feature := range features {409+ if !passedAfter {410+ passedAfter = feature.ID == after411+ continue412+ }413+ if request.Bounds != nil {414+ bounds, err := geometryBounds(feature.Geometry)415+ if err != nil || !intersects(bounds, *request.Bounds) {416+ continue417+ }418+ }419+ result.Features = append(result.Features, feature)420+ if len(result.Features) == request.Limit {421+ result.Next = encodeContinuation(request, feature.ID)422+ return result, nil423+ }424+ }425+ }426+ if !passedAfter {427+ return FeaturePageResult{}, fmt.Errorf("feature continuation is not present")428+ }429+ return result, nil430+}431+432+func (reader *Reader) readFeaturePartition(partition featurePartitionDescriptor) ([]Feature, int64, error) {433+ features, err := reader.Table(partition.Features)434+ if err != nil {435+ return nil, 0, err436+ }437+ relationships, err := reader.Table(partition.Relationships)438+ if err != nil {439+ return nil, 0, err440+ }441+ provenance, err := reader.Table(partition.Provenance)442+ if err != nil {443+ return nil, 0, err444+ }445+ out := make([]Feature, features.Rows)446+ byID := make(map[string]int, features.Rows)447+ core, _ := StandardSchema().Type(CoreID("type.feature"))448+ for row := 0; row < features.Rows; row++ {449+ geometry, err := decodeGeometry(requiredBytes(features, row, "feature.geometry"))450+ if err != nil {451+ return nil, 0, err452+ }453+ feature := Feature{454+ ID: requiredString(features, row, "feature.id"), Title: requiredString(features, row, "feature.title"),455+ Subtitle: optionalStringAt(features, row, "feature.subtitle"), Description: optionalStringAt(features, row, "feature.description"),456+ Shard: requiredInt64(features, row, "feature.shard"), Geometry: geometry,457+ }458+ if center := optionalBytesAt(features, row, "feature.center"); center != nil {459+ position, err := decodePosition(center)460+ if err != nil {461+ return nil, 0, err462+ }463+ feature.Center = &position464+ }465+ for _, column := range features.Columns {466+ if _, standard := core.Field(column.Field.ID); !standard && column.Values[row].Kind != KindInvalid {467+ feature.Properties = append(feature.Properties, Property{FieldID: column.Field.ID, Field: column.Field, Value: column.Values[row]})468+ }469+ }470+ out[row] = feature471+ byID[feature.ID] = row472+ }473+ for row := 0; row < relationships.Rows; row++ {474+ if at, held := byID[requiredString(relationships, row, "relationship.feature")]; held {475+ decodeRelationship(&out[at], relationships, row)476+ }477+ }478+ for row := 0; row < provenance.Rows; row++ {479+ if at, held := byID[requiredString(provenance, row, "provenance.feature")]; held {480+ decodeProvenance(&out[at], provenance, row)481+ }482+ }483+ bytesRead := reader.tables[partition.Features].Length + reader.tables[partition.Relationships].Length + reader.tables[partition.Provenance].Length484+ return out, bytesRead, nil485+}486+487+func continuationFingerprint(request FeaturePageRequest) [8]byte {488+ hash := sha256.New()489+ _, _ = hash.Write([]byte(request.FeatureSet))490+ if request.Bounds != nil {491+ for _, value := range []float64{request.Bounds.MinX, request.Bounds.MinY, request.Bounds.MaxX, request.Bounds.MaxY} {492+ var encoded [8]byte493+ binary.LittleEndian.PutUint64(encoded[:], math.Float64bits(value))494+ _, _ = hash.Write(encoded[:])495+ }496+ }497+ var out [8]byte498+ copy(out[:], hash.Sum(nil))499+ return out500+}501+502+func encodeContinuation(request FeaturePageRequest, featureID string) string {503+ fingerprint := continuationFingerprint(request)504+ data := append(fingerprint[:], featureID...)505+ return base64.RawURLEncoding.EncodeToString(data)506+}507+508+func decodeContinuation(request FeaturePageRequest) (string, error) {509+ if request.After == "" {510+ return "", nil511+ }512+ data, err := base64.RawURLEncoding.DecodeString(request.After)513+ if err != nil || len(data) <= 8 {514+ return "", fmt.Errorf("feature continuation is invalid")515+ }516+ fingerprint := continuationFingerprint(request)517+ if !slices.Equal(data[:8], fingerprint[:]) {518+ return "", fmt.Errorf("feature continuation belongs to another query")519+ }520+ return string(data[8:]), nil521+}@@ -0,0 +1,332 @@
1+package vnext2+3+import (4+ "bytes"5+ "crypto/sha256"6+ "encoding/hex"7+ "encoding/json"8+ "fmt"9+ "io"10+ "os"11+ "path/filepath"12+ "slices"13+ "strconv"14+ "strings"15+)16+17+const (18+ rasterIndexName = "indexes/rasters.v1.json"19+ rasterShardPrefix = "data/raster-shards/"20+ maxRasterShardBytes = int64(64 << 20)21+ maxRasterShardTiles = 4_09622+)23+24+type rasterTileIndex struct {25+ Version uint16 `json:"version"`26+ Tiles []rasterTileDescriptor `json:"tiles"`27+}28+29+type rasterTileDescriptor struct {30+ Name string `json:"name"`31+ Shard string `json:"shard"`32+ Offset int64 `json:"offset"`33+ Length int64 `json:"length"`34+ Hash string `json:"hash"`35+ Format string `json:"format"`36+ MediaType string `json:"mediaType"`37+}38+39+type pendingRasterTile struct {40+ name string41+ blob Blob42+ length int6443+ format string44+ mediaType string45+}46+47+type shardRange struct {48+ offset int6449+ length int6450+}51+52+// RasterTile is one verified logical tile read from a shard range. Format and53+// MediaType are per tile, so a pyramid may legally mix PNG and JPEG.54+type RasterTile struct {55+ Name string56+ Format string57+ MediaType string58+ Data []byte59+}60+61+func packRasterBlobs(blobs []Blob) ([]Blob, error) {62+ tiles := make([]pendingRasterTile, 0)63+ kept := make([]Blob, 0, len(blobs))64+ for _, blob := range blobs {65+ format, mediaType, isTile := rasterTileType(blob.Name)66+ if !isTile {67+ kept = append(kept, blob)68+ continue69+ }70+ length, err := blobLength(blob)71+ if err != nil {72+ return nil, fmt.Errorf("pack raster tile %s: %w", blob.Name, err)73+ }74+ if length > maxRasterShardBytes {75+ return nil, fmt.Errorf("pack raster tile %s: tile exceeds shard limit", blob.Name)76+ }77+ tiles = append(tiles, pendingRasterTile{name: blob.Name, blob: blob, length: length, format: format, mediaType: mediaType})78+ }79+ if len(tiles) == 0 {80+ return blobs, nil81+ }82+ slices.SortFunc(tiles, func(left, right pendingRasterTile) int { return strings.Compare(left.name, right.name) })83+ index := rasterTileIndex{Version: 1}84+ for start := 0; start < len(tiles); {85+ end := start86+ var length int6487+ for end < len(tiles) && end-start < maxRasterShardTiles {88+ next := tiles[end].length89+ if end > start && length+next > maxRasterShardBytes {90+ break91+ }92+ length += next93+ end++94+ }95+ shard, descriptors, err := spoolRasterShard(tiles[start:end])96+ if err != nil {97+ cleanupTemporaryBlobs(kept)98+ return nil, err99+ }100+ kept = append(kept, shard)101+ index.Tiles = append(index.Tiles, descriptors...)102+ start = end103+ }104+ data, err := json.Marshal(index)105+ if err != nil {106+ cleanupTemporaryBlobs(kept)107+ return nil, fmt.Errorf("encode raster tile index: %w", err)108+ }109+ kept = append(kept, Blob{Name: rasterIndexName, Data: data})110+ slices.SortFunc(kept, func(left, right Blob) int { return strings.Compare(left.Name, right.Name) })111+ return kept, nil112+}113+114+func blobLength(blob Blob) (int64, error) {115+ if blob.Path == "" {116+ return int64(len(blob.Data)), nil117+ }118+ if len(blob.Data) != 0 {119+ return 0, fmt.Errorf("blob has both data and a source path")120+ }121+ info, err := os.Stat(blob.Path)122+ if err != nil {123+ return 0, err124+ }125+ return info.Size(), nil126+}127+128+func spoolRasterShard(tiles []pendingRasterTile) (Blob, []rasterTileDescriptor, error) {129+ file, err := os.CreateTemp("", "atlas-raster-shard-*.rpack")130+ if err != nil {131+ return Blob{}, nil, fmt.Errorf("stage raster shard: %w", err)132+ }133+ path := file.Name()134+ failed := func(err error) (Blob, []rasterTileDescriptor, error) {135+ file.Close()136+ os.Remove(path)137+ return Blob{}, nil, err138+ }139+ shardHash := sha256.New()140+ descriptors := make([]rasterTileDescriptor, 0, len(tiles))141+ var offset int64142+ for _, tile := range tiles {143+ tileHash := sha256.New()144+ written, err := copyBlob(io.MultiWriter(file, shardHash, tileHash), tile.blob)145+ if err != nil {146+ return failed(fmt.Errorf("write raster tile %s: %w", tile.name, err))147+ }148+ if written != tile.length {149+ return failed(fmt.Errorf("write raster tile %s: length changed", tile.name))150+ }151+ descriptors = append(descriptors, rasterTileDescriptor{152+ Name: tile.name, Offset: offset, Length: written, Hash: hex.EncodeToString(tileHash.Sum(nil)),153+ Format: tile.format, MediaType: tile.mediaType,154+ })155+ offset += written156+ }157+ if err := file.Sync(); err != nil {158+ return failed(fmt.Errorf("sync raster shard: %w", err))159+ }160+ if err := file.Close(); err != nil {161+ os.Remove(path)162+ return Blob{}, nil, fmt.Errorf("close raster shard: %w", err)163+ }164+ name := rasterShardPrefix + hex.EncodeToString(shardHash.Sum(nil)) + ".rpack"165+ for index := range descriptors {166+ descriptors[index].Shard = name167+ }168+ return Blob{Name: name, Path: path, temporary: true}, descriptors, nil169+}170+171+func copyBlob(destination io.Writer, blob Blob) (int64, error) {172+ if blob.Path == "" {173+ written, err := io.Copy(destination, bytes.NewReader(blob.Data))174+ return written, err175+ }176+ source, err := os.Open(blob.Path)177+ if err != nil {178+ return 0, err179+ }180+ written, copyErr := io.Copy(destination, source)181+ closeErr := source.Close()182+ if copyErr != nil {183+ return written, copyErr184+ }185+ return written, closeErr186+}187+188+func cleanupTemporaryBlobs(blobs []Blob) {189+ for _, blob := range blobs {190+ if blob.temporary {191+ _ = os.Remove(blob.Path)192+ }193+ }194+}195+196+func rasterTileType(name string) (string, string, bool) {197+ parts := strings.Split(name, "/")198+ if len(parts) != 5 || parts[0] != "tiles" || parts[1] == "" {199+ return "", "", false200+ }201+ zoom, zoomErr := strconv.ParseUint(parts[2], 10, 6)202+ x, xErr := strconv.ParseUint(parts[3], 10, 63)203+ yText := strings.TrimSuffix(parts[4], filepath.Ext(parts[4]))204+ y, yErr := strconv.ParseUint(yText, 10, 63)205+ if zoomErr != nil || xErr != nil || yErr != nil || x >= uint64(1)<<zoom || y >= uint64(1)<<zoom {206+ return "", "", false207+ }208+ format := strings.TrimPrefix(strings.ToLower(filepath.Ext(parts[4])), ".")209+ switch format {210+ case "png":211+ return format, "image/png", true212+ case "jpg", "jpeg":213+ return format, "image/jpeg", true214+ case "webp":215+ return format, "image/webp", true216+ case "avif":217+ return format, "image/avif", true218+ default:219+ return "", "", false220+ }221+}222+223+func (reader *Reader) rasterIndex() (rasterTileIndex, error) {224+ reference, held := reader.blobs[rasterIndexName]225+ if !held {226+ return rasterTileIndex{}, nil227+ }228+ data, err := readReferencedEntry(reader.entries[rasterIndexName], reference.Length, reference.Hash, reader.limits.MaxBlobBytes)229+ if err != nil {230+ return rasterTileIndex{}, err231+ }232+ var index rasterTileIndex233+ decoder := json.NewDecoder(bytes.NewReader(data))234+ decoder.DisallowUnknownFields()235+ if err := decoder.Decode(&index); err != nil || index.Version != 1 {236+ return rasterTileIndex{}, fmt.Errorf("decode raster tile index")237+ }238+ if err := decoder.Decode(&struct{}{}); err != io.EOF {239+ return rasterTileIndex{}, fmt.Errorf("raster tile index carries trailing data")240+ }241+ canonical, err := json.Marshal(index)242+ if err != nil || !bytes.Equal(canonical, data) {243+ return rasterTileIndex{}, fmt.Errorf("raster tile index is not canonical")244+ }245+ previous := ""246+ ranges := make(map[string][]shardRange)247+ for _, tile := range index.Tiles {248+ format, mediaType, validName := rasterTileType(tile.Name)249+ shard, held := reader.blobs[tile.Shard]250+ decodedHash, hashErr := hex.DecodeString(tile.Hash)251+ if tile.Name <= previous || tile.Offset < 0 || tile.Length < 0 || tile.Length > reader.limits.MaxBlobBytes ||252+ !validName || format != tile.Format || mediaType != tile.MediaType || !held ||253+ tile.Offset > shard.Length || tile.Length > shard.Length-tile.Offset || hashErr != nil ||254+ len(decodedHash) != sha256.Size || tile.Hash != strings.ToLower(tile.Hash) {255+ return rasterTileIndex{}, fmt.Errorf("raster tile index is not canonical")256+ }257+ ranges[tile.Shard] = append(ranges[tile.Shard], shardRange{offset: tile.Offset, length: tile.Length})258+ previous = tile.Name259+ }260+ for shardName, held := range ranges {261+ slices.SortFunc(held, func(left, right shardRange) int {262+ if left.offset < right.offset {263+ return -1264+ }265+ if left.offset > right.offset {266+ return 1267+ }268+ return 0269+ })270+ var end int64271+ for _, part := range held {272+ if part.offset != end {273+ return rasterTileIndex{}, fmt.Errorf("raster shard %s has overlapping or uncovered ranges", shardName)274+ }275+ end += part.length276+ }277+ if end != reader.blobs[shardName].Length {278+ return rasterTileIndex{}, fmt.Errorf("raster shard %s has unindexed bytes", shardName)279+ }280+ }281+ for name := range reader.blobs {282+ if strings.HasPrefix(name, rasterShardPrefix) && ranges[name] == nil {283+ return rasterTileIndex{}, fmt.Errorf("raster shard %s is not indexed", name)284+ }285+ }286+ return index, nil287+}288+289+// ReadRasterTile verifies one tile slice without reading the rest of its290+// shard. The containing shard remains independently hash-verified by Validate.291+func (reader *Reader) ReadRasterTile(name string) (RasterTile, error) {292+ index, err := reader.rasterIndex()293+ if err != nil {294+ return RasterTile{}, err295+ }296+ at, held := slices.BinarySearchFunc(index.Tiles, name, func(tile rasterTileDescriptor, target string) int {297+ return strings.Compare(tile.Name, target)298+ })299+ if !held {300+ return RasterTile{}, fmt.Errorf("bundle has no raster tile %s", name)301+ }302+ tile := index.Tiles[at]303+ data, err := reader.readStoredBlobRange(tile.Shard, tile.Offset, tile.Length)304+ if err != nil {305+ return RasterTile{}, err306+ }307+ digest := sha256.Sum256(data)308+ if hex.EncodeToString(digest[:]) != tile.Hash {309+ return RasterTile{}, fmt.Errorf("raster tile %s hash does not match", name)310+ }311+ return RasterTile{Name: name, Format: tile.Format, MediaType: tile.MediaType, Data: data}, nil312+}313+314+func (reader *Reader) readStoredBlobRange(name string, offset, length int64) ([]byte, error) {315+ reference, held := reader.blobs[name]316+ if !held || offset < 0 || length < 0 || offset > reference.Length || length > reference.Length-offset {317+ return nil, fmt.Errorf("blob range is invalid")318+ }319+ entry := reader.entries[name]320+ if entry == nil || entry.Method != 0 {321+ return nil, fmt.Errorf("blob range is not stored")322+ }323+ dataOffset, err := entry.DataOffset()324+ if err != nil {325+ return nil, fmt.Errorf("locate %s: %w", name, err)326+ }327+ data := make([]byte, length)328+ if _, err := io.ReadFull(io.NewSectionReader(reader.source, dataOffset+offset, length), data); err != nil {329+ return nil, fmt.Errorf("read %s range: %w", name, err)330+ }331+ return data, nil332+}@@ -0,0 +1,355 @@
1+package vnext2+3+import (4+ "archive/zip"5+ "bytes"6+ "fmt"7+ "os"8+ "strings"9+ "testing"10+)11+12+func TestCompilePartitionsFeaturesAndPagesThemWithoutMaterializingVolume(t *testing.T) {13+ t.Parallel()14+15+ volume := sampleRegionVolume(9_000)16+ bundle, err := Compile(volume)17+ if err != nil {18+ t.Fatalf("compile Sample Region: %v", err)19+ }20+ partitionCount := 021+ for _, named := range bundle.Tables {22+ if named.Name == featureTableName {23+ t.Fatal("compiled bundle retained the monolithic feature table")24+ }25+ if strings.HasPrefix(named.Name, featurePartitionPrefix) {26+ partitionCount++27+ }28+ }29+ if partitionCount < 3 {30+ t.Fatalf("feature partitions = %d, want at least 3", partitionCount)31+ }32+33+ bundle.Release.CreatedAt = "2026-08-08T12:00:00Z"34+ var encoded bytes.Buffer35+ if err := Write(&encoded, bundle); err != nil {36+ t.Fatalf("write Sample Region: %v", err)37+ }38+ reader, err := Open(bytes.NewReader(encoded.Bytes()), int64(encoded.Len()), StandardSchema())39+ if err != nil {40+ t.Fatalf("open Sample Region: %v", err)41+ }42+ page, err := reader.FeaturePage(FeaturePageRequest{43+ FeatureSet: "roads",44+ Bounds: &SpatialBounds{MinX: 100, MinY: -1, MaxX: 199, MaxY: 1},45+ Limit: 10,46+ })47+ if err != nil {48+ t.Fatalf("first feature page: %v", err)49+ }50+ if len(page.Features) != 10 || page.Next == "" {51+ t.Fatalf("first page = %d features, next %q", len(page.Features), page.Next)52+ }53+ if page.PartitionsRead > 2 || page.BytesRead <= 0 {54+ t.Fatalf("first page read %d partitions and %d bytes", page.PartitionsRead, page.BytesRead)55+ }56+ second, err := reader.FeaturePage(FeaturePageRequest{57+ FeatureSet: "roads",58+ Bounds: &SpatialBounds{MinX: 100, MinY: -1, MaxX: 199, MaxY: 1},59+ After: page.Next,60+ Limit: 10,61+ })62+ if err != nil {63+ t.Fatalf("second feature page: %v", err)64+ }65+ if len(second.Features) != 10 || second.Features[0].ID == page.Features[0].ID {66+ t.Fatalf("second page did not advance: %#v", second.Features)67+ }68+69+ restored, err := reader.Volume()70+ if err != nil {71+ t.Fatalf("compatibility materialization: %v", err)72+ }73+ if got := len(restored.Worlds[0].FeatureSets[0].Features); got != 9_000 {74+ t.Fatalf("restored features = %d, want 9000", got)75+ }76+ if space := restored.Worlds[0].CoordinateSpace; space.OriginX != 10 || space.OriginY != 20 {77+ t.Fatalf("restored tile origin = %d,%d, want 10,20", space.OriginX, space.OriginY)78+ }79+}80+81+func TestOutlineDoesNotReadFeaturePartitions(t *testing.T) {82+ t.Parallel()83+84+ bundle, err := Compile(sampleRegionVolume(300))85+ if err != nil {86+ t.Fatal(err)87+ }88+ bundle.Release.CreatedAt = "2026-08-08T12:00:00Z"89+ var encoded bytes.Buffer90+ if err := Write(&encoded, bundle); err != nil {91+ t.Fatal(err)92+ }93+ corrupt := append([]byte(nil), encoded.Bytes()...)94+ archive, err := zip.NewReader(bytes.NewReader(corrupt), int64(len(corrupt)))95+ if err != nil {96+ t.Fatal(err)97+ }98+ for _, entry := range archive.File {99+ if !strings.HasSuffix(entry.Name, ".features.pack") {100+ continue101+ }102+ offset, err := entry.DataOffset()103+ if err != nil {104+ t.Fatal(err)105+ }106+ corrupt[offset+int64(entry.UncompressedSize64)-1] ^= 0xff107+ break108+ }109+ reader, err := Open(bytes.NewReader(corrupt), int64(len(corrupt)), StandardSchema())110+ if err != nil {111+ t.Fatalf("open with unread corrupt partition: %v", err)112+ }113+ outline, err := reader.Outline()114+ if err != nil {115+ t.Fatalf("outline read a feature partition: %v", err)116+ }117+ if got := outline.Worlds[0].FeatureSets[0].Features; len(got) != 0 {118+ t.Fatalf("outline materialized %d features", len(got))119+ }120+ if _, err := reader.FeaturePage(FeaturePageRequest{FeatureSet: "roads", Limit: 1}); err == nil {121+ t.Fatal("feature page accepted the corrupt demanded partition")122+ }123+}124+125+func TestPartitionAndShardGenerationIsCanonicalAcrossInputOrder(t *testing.T) {126+ t.Parallel()127+128+ left := sampleRegionVolume(520)129+ right := sampleRegionVolume(520)130+ features := right.Worlds[0].FeatureSets[0].Features131+ for low, high := 0, len(features)-1; low < high; low, high = low+1, high-1 {132+ features[low], features[high] = features[high], features[low]133+ }134+ encode := func(volume Volume, reverseTiles bool) []byte {135+ bundle, err := Compile(volume)136+ if err != nil {137+ t.Fatal(err)138+ }139+ bundle.Release.CreatedAt = "2026-08-08T12:00:00Z"140+ tiles := []Blob{141+ {Name: "tiles/background/0/0/0.png", Data: []byte("zero")},142+ {Name: "tiles/background/1/1/0.jpg", Data: []byte("one")},143+ }144+ if reverseTiles {145+ tiles[0], tiles[1] = tiles[1], tiles[0]146+ }147+ bundle.Blobs = append(bundle.Blobs, tiles...)148+ var output bytes.Buffer149+ if err := Write(&output, bundle); err != nil {150+ t.Fatal(err)151+ }152+ return output.Bytes()153+ }154+ if first, second := encode(left, false), encode(right, true); !bytes.Equal(first, second) {155+ t.Fatal("stable feature IDs and tile names did not canonicalize input order")156+ }157+}158+159+func TestValidateRejectsMalformedLogicalIndexes(t *testing.T) {160+ t.Parallel()161+162+ featureBundle, err := Compile(sampleRegionVolume(1))163+ if err != nil {164+ t.Fatal(err)165+ }166+ for index := range featureBundle.Blobs {167+ if featureBundle.Blobs[index].Name == featureIndexName {168+ featureBundle.Blobs[index].Data = append(featureBundle.Blobs[index].Data, '\n')169+ }170+ }171+ featureBundle.Release.CreatedAt = "2026-08-08T12:00:00Z"172+ var featureBytes bytes.Buffer173+ if err := Write(&featureBytes, featureBundle); err != nil {174+ t.Fatal(err)175+ }176+ featureReader, err := Open(bytes.NewReader(featureBytes.Bytes()), int64(featureBytes.Len()), StandardSchema())177+ if err != nil {178+ t.Fatal(err)179+ }180+ if err := featureReader.Validate(); err == nil || !strings.Contains(err.Error(), "feature index") {181+ t.Fatalf("malformed feature index validation = %v", err)182+ }183+184+ rasterBundle, err := Compile(minimalVolume())185+ if err != nil {186+ t.Fatal(err)187+ }188+ rasterBundle.Release.CreatedAt = "2026-08-08T12:00:00Z"189+ rasterBundle.Blobs = append(rasterBundle.Blobs,190+ Blob{Name: rasterShardPrefix + "missing.rpack", Data: []byte("data")},191+ Blob{Name: rasterIndexName, Data: []byte(`{"version":1,"tiles":[]}`)},192+ )193+ var rasterBytes bytes.Buffer194+ if err := Write(&rasterBytes, rasterBundle); err != nil {195+ t.Fatal(err)196+ }197+ rasterReader, err := Open(bytes.NewReader(rasterBytes.Bytes()), int64(rasterBytes.Len()), StandardSchema())198+ if err != nil {199+ t.Fatal(err)200+ }201+ if _, err := rasterReader.LogicalBlobNames(); err == nil || !strings.Contains(err.Error(), "not indexed") {202+ t.Fatalf("orphan raster shard logical names = %v", err)203+ }204+}205+206+func TestRasterTileRangeRejectsCorruptShardSlice(t *testing.T) {207+ t.Parallel()208+209+ bundle, err := Compile(minimalVolume())210+ if err != nil {211+ t.Fatal(err)212+ }213+ bundle.Release.CreatedAt = "2026-08-08T12:00:00Z"214+ bundle.Blobs = append(bundle.Blobs, Blob{Name: "tiles/background/0/0/0.png", Data: []byte("zero tile")})215+ var encoded bytes.Buffer216+ if err := Write(&encoded, bundle); err != nil {217+ t.Fatal(err)218+ }219+ corrupt := append([]byte(nil), encoded.Bytes()...)220+ archive, err := zip.NewReader(bytes.NewReader(corrupt), int64(len(corrupt)))221+ if err != nil {222+ t.Fatal(err)223+ }224+ for _, entry := range archive.File {225+ if !strings.HasPrefix(entry.Name, rasterShardPrefix) {226+ continue227+ }228+ offset, err := entry.DataOffset()229+ if err != nil {230+ t.Fatal(err)231+ }232+ corrupt[offset] ^= 0xff233+ }234+ reader, err := Open(bytes.NewReader(corrupt), int64(len(corrupt)), StandardSchema())235+ if err != nil {236+ t.Fatal(err)237+ }238+ if _, err := reader.ReadRasterTile("tiles/background/0/0/0.png"); err == nil || !strings.Contains(err.Error(), "hash") {239+ t.Fatalf("corrupt tile read = %v", err)240+ }241+}242+243+func TestRasterPackingSpoolsBoundedShardsWithoutRetainingTileCopies(t *testing.T) {244+ t.Parallel()245+246+ packed, err := packRasterBlobs([]Blob{247+ {Name: "tiles/background/1/0/0.png", Data: bytes.Repeat([]byte{1}, 1<<20)},248+ {Name: "tiles/background/1/1/0.png", Data: bytes.Repeat([]byte{2}, 1<<20)},249+ })250+ if err != nil {251+ t.Fatal(err)252+ }253+ var temporary []string254+ for _, blob := range packed {255+ if !strings.HasPrefix(blob.Name, rasterShardPrefix) {256+ continue257+ }258+ if len(blob.Data) != 0 || blob.Path == "" || !blob.temporary {259+ t.Fatalf("raster shard retained in memory: %#v", blob)260+ }261+ info, err := os.Stat(blob.Path)262+ if err != nil {263+ t.Fatal(err)264+ }265+ if info.Size() > maxRasterShardBytes {266+ t.Fatalf("staged shard size = %d", info.Size())267+ }268+ temporary = append(temporary, blob.Path)269+ }270+ cleanupTemporaryBlobs(packed)271+ for _, path := range temporary {272+ if _, err := os.Stat(path); !os.IsNotExist(err) {273+ t.Fatalf("temporary shard remains at %s: %v", path, err)274+ }275+ }276+}277+278+func TestWritePacksRasterTilesAndReaderVerifiesLogicalTileSlices(t *testing.T) {279+ t.Parallel()280+281+ bundle, err := Compile(minimalVolume())282+ if err != nil {283+ t.Fatal(err)284+ }285+ bundle.Release.CreatedAt = "2026-08-08T12:00:00Z"286+ want := map[string][]byte{287+ "tiles/background/0/0/0.png": []byte("zero tile"),288+ "tiles/background/1/0/0.png": []byte("northwest"),289+ "tiles/background/1/1/0.jpg": []byte("northeast"),290+ }291+ for name, data := range want {292+ bundle.Blobs = append(bundle.Blobs, Blob{Name: name, Data: data})293+ }294+ var encoded bytes.Buffer295+ if err := Write(&encoded, bundle); err != nil {296+ t.Fatal(err)297+ }298+ archive, err := zip.NewReader(bytes.NewReader(encoded.Bytes()), int64(encoded.Len()))299+ if err != nil {300+ t.Fatal(err)301+ }302+ for _, entry := range archive.File {303+ if strings.HasPrefix(entry.Name, "tiles/") {304+ t.Fatalf("tile remained a ZIP entry: %s", entry.Name)305+ }306+ }307+308+ reader, err := Open(bytes.NewReader(encoded.Bytes()), int64(encoded.Len()), StandardSchema())309+ if err != nil {310+ t.Fatal(err)311+ }312+ for name, expected := range want {313+ tile, err := reader.ReadRasterTile(name)314+ if err != nil {315+ t.Fatalf("read %s: %v", name, err)316+ }317+ if !bytes.Equal(tile.Data, expected) {318+ t.Fatalf("read %s = %q, want %q", name, tile.Data, expected)319+ }320+ if strings.HasSuffix(name, ".jpg") && (tile.Format != "jpg" || tile.MediaType != "image/jpeg") {321+ t.Fatalf("mixed tile metadata = %#v", tile)322+ }323+ }324+}325+326+func sampleRegionVolume(features int) Volume {327+ items := make([]Feature, features)328+ for index := range items {329+ items[index] = Feature{330+ ID: fmt.Sprintf("road-%04d", index),331+ Title: fmt.Sprintf("Sample Road %04d", index),332+ Geometry: Geometry{Kind: GeometryPoint, Parts: []GeometryPart{{Rings: [][]Position{{333+ {float64(index), 0},334+ }}}}},335+ Relationships: []Relationship{{Predicate: "next", Target: fmt.Sprintf("road-%04d", (index+1)%max(features, 1))}},336+ Provenance: []Provenance{{Source: "sample", NativeID: fmt.Sprintf("%04d", index), CapturedAt: "2026-08-08T12:00:00Z"}},337+ }338+ }339+ return Volume{340+ ID: "sample-region", Title: "Sample Region",341+ Worlds: []World{{342+ ID: "sample-world", Title: "Sample Region",343+ CoordinateSpace: CoordinateSpace{344+ ID: "sample-space", Kind: "projected", Unit: "metre", Definition: "EPSG:3857",345+ Extent: [4]float64{0, -1, float64(features), 1}, OriginX: 10, OriginY: 20,346+ },347+ FeatureSets: []FeatureSet{{ID: "roads", Title: "Roads", SemanticType: "transport.road", Features: items}},348+ Presentation: Presentation{349+ ID: "default", Title: "Default",350+ Styles: []Style{{ID: "road-style"}},351+ Layers: []Layer{{ID: "roads-layer", FeatureSet: "roads", Style: "road-style", Visible: true, MaxZoom: 22}},352+ },353+ }},354+ }355+}@@ -57,7 +57,8 @@ func validateWorld(world World, ids identities) error {
5757 if world.CoordinateSpace.Kind == "" || world.CoordinateSpace.Unit == "" || world.CoordinateSpace.Definition == "" {5858 return fmt.Errorf("world %s has an incomplete coordinate space", world.ID)5959 }60- if world.CoordinateSpace.SourceZoom < 0 || world.CoordinateSpace.TileSize < 0 || world.CoordinateSpace.Size < 0 {60+ if world.CoordinateSpace.SourceZoom < 0 || world.CoordinateSpace.TileSize < 0 || world.CoordinateSpace.Size < 0 ||61+ world.CoordinateSpace.OriginX < 0 || world.CoordinateSpace.OriginY < 0 {6162 return fmt.Errorf("world %s has invalid coordinate grid metadata", world.ID)6263 }6364 if err := validateClaims(world.Claims, "world"); err != nil {