Introduce the schema-driven vNext container, packed typed tables, v3 importer, browser reader, and semantic presentation model. Route the application through a cached vNext Volume while preserving the current browser payload seam and validating parity against the Earth corpus.
- commit
- 56d23aa3e4a1018aab2a4fc6b3739fb517adf222
- parent
- 107f0677205b46a7ec3cbc26c2c8c86acc3cb95e
- author
- Dami <dami.etoile@gmail.com>
| A | format/ | 388 | +++++++++++++++++++++++ |
| A | format/ | 125 | +++++++ |
| A | format/ | 502 | +++++++++++++++++++++++++++++ |
| A | format/ | 366 | +++++++++++++++++++++ |
| A | format/ | 100 | ++++++ |
| A | format/ | 513 | ++++++++++++++++++++++++++++++ |
| A | format/ | 66 | ++++ |
| A | format/ | 499 | +++++++++++++++++++++++++++++ |
| A | format/ | 125 | +++++++ |
| A | format/ | 319 | +++++++++++++++++++ |
| A | format/ | 65 | ++++ |
| A | format/ | 101 | ++++++ |
| A | format/ | 195 | +++++++++++ |
| A | format/ | 75 | ++++ |
| A | format/ | 143 | ++++++++ |
| A | format/ | 242 | ++++++++++++++ |
| M | internal/ | 7 | +- |
| M | internal/ | 6 | +- |
| M | internal/ | 19 | +- |
| A | internal/ | 127 | +++++++ |
| A | internal/ | 516 | ++++++++++++++++++++++++++++++ |
| M | internal/ | 172 | ++-------- |
| A | render/ | 246 | ++++++++++++++ |
| A | render/ | 149 | +++++++++ |
@@ -0,0 +1,388 @@
1+package vnext2+3+import (4+ "bytes"5+ "encoding/binary"6+ "fmt"7+ "hash/crc32"8+ "math"9+)10+11+const (12+ blockMagic = "ATLASPK\x00"13+ blockFraming = uint16(1)14+ blockHeaderSize = 8015+ directorySize = 6416+ columnFlagOptional = uint16(1)17+ maxBlockRows = 100_000_00018+)19+20+// Block is a decoded typed block. SchemaHash identifies the embedded schema21+// used to write it but does not decide whether a reader may open it.22+type Block struct {23+ SchemaHash [32]byte24+ Table Table25+}26+27+type encodedColumn struct {28+ field Field29+ nulls []byte30+ data []byte31+ crc uint3232+ nullAt uint6433+ dataAt uint6434+}35+36+// EncodeBlock writes a deterministic, self-framed hybrid column block.37+func EncodeBlock(schemaHash [32]byte, table Table) ([]byte, error) {38+ table = canonicalTable(table)39+ if table.Rows > maxBlockRows {40+ return nil, fmt.Errorf("table has %d rows; maximum is %d", table.Rows, maxBlockRows)41+ }42+ if err := table.validate(); err != nil {43+ return nil, err44+ }45+ columns, err := encodeColumns(table)46+ if err != nil {47+ return nil, err48+ }49+ payloadAt := uint64(blockHeaderSize + len(columns)*directorySize)50+ assignOffsets(columns, payloadAt)51+ out := make([]byte, payloadAt)52+ writeBlockHeader(out, schemaHash, table, payloadAt)53+ for index, column := range columns {54+ writeDirectory(out[blockHeaderSize+index*directorySize:], column)55+ out = append(out, column.nulls...)56+ out = append(out, column.data...)57+ }58+ return out, nil59+}60+61+func encodeColumns(table Table) ([]encodedColumn, error) {62+ columns := make([]encodedColumn, len(table.Columns))63+ for index, column := range table.Columns {64+ nulls := presenceBitmap(column.Values)65+ data, err := encodeValues(column.Field.Kind, column.Values)66+ if err != nil {67+ return nil, fmt.Errorf("encode column %s: %w", column.Field.ID, err)68+ }69+ checksum := crc32.NewIEEE()70+ _, _ = checksum.Write(nulls)71+ _, _ = checksum.Write(data)72+ columns[index] = encodedColumn{field: column.Field, nulls: nulls, data: data, crc: checksum.Sum32()}73+ }74+ return columns, nil75+}76+77+func assignOffsets(columns []encodedColumn, at uint64) {78+ for index := range columns {79+ columns[index].nullAt = at80+ at += uint64(len(columns[index].nulls))81+ columns[index].dataAt = at82+ at += uint64(len(columns[index].data))83+ }84+}85+86+func writeBlockHeader(out []byte, schemaHash [32]byte, table Table, payloadAt uint64) {87+ copy(out[:8], blockMagic)88+ binary.LittleEndian.PutUint16(out[8:10], blockFraming)89+ binary.LittleEndian.PutUint32(out[12:16], uint32(table.Rows))90+ binary.LittleEndian.PutUint32(out[16:20], uint32(len(table.Columns)))91+ copy(out[20:52], schemaHash[:])92+ copy(out[52:68], table.TypeID[:])93+ binary.LittleEndian.PutUint32(out[68:72], directorySize)94+ binary.LittleEndian.PutUint64(out[72:80], payloadAt)95+}96+97+func writeDirectory(out []byte, column encodedColumn) {98+ copy(out[:16], column.field.ID[:])99+ out[16] = byte(column.field.Kind)100+ out[17] = byte(column.field.Kind)101+ if column.field.Optional {102+ binary.LittleEndian.PutUint16(out[18:20], columnFlagOptional)103+ }104+ binary.LittleEndian.PutUint64(out[20:28], column.nullAt)105+ binary.LittleEndian.PutUint64(out[28:36], uint64(len(column.nulls)))106+ binary.LittleEndian.PutUint64(out[36:44], column.dataAt)107+ binary.LittleEndian.PutUint64(out[44:52], uint64(len(column.data)))108+ binary.LittleEndian.PutUint32(out[52:56], column.crc)109+}110+111+func presenceBitmap(values []Value) []byte {112+ bitmap := make([]byte, (len(values)+7)/8)113+ for row, value := range values {114+ if value.Kind != KindInvalid {115+ bitmap[row/8] |= 1 << (row % 8)116+ }117+ }118+ return bitmap119+}120+121+func encodeValues(kind Kind, values []Value) ([]byte, error) {122+ switch kind {123+ case KindBool:124+ return encodeBools(values), nil125+ case KindInt64:126+ return encodeInt64s(values), nil127+ case KindFloat64:128+ return encodeFloat64s(values), nil129+ case KindString, KindBytes:130+ return encodeVariable(kind, values)131+ case KindID:132+ return encodeIDs(values), nil133+ default:134+ return nil, fmt.Errorf("unsupported wire kind %d", kind)135+ }136+}137+138+func encodeBools(values []Value) []byte {139+ out := make([]byte, len(values))140+ for index, value := range values {141+ if value.Kind != KindInvalid && value.Bool {142+ out[index] = 1143+ }144+ }145+ return out146+}147+148+func encodeInt64s(values []Value) []byte {149+ out := make([]byte, len(values)*8)150+ for index, value := range values {151+ binary.LittleEndian.PutUint64(out[index*8:], uint64(value.Int64))152+ }153+ return out154+}155+156+func encodeFloat64s(values []Value) []byte {157+ out := make([]byte, len(values)*8)158+ for index, value := range values {159+ binary.LittleEndian.PutUint64(out[index*8:], math.Float64bits(value.Float64))160+ }161+ return out162+}163+164+func encodeIDs(values []Value) []byte {165+ out := make([]byte, len(values)*16)166+ for index, value := range values {167+ copy(out[index*16:], value.ID[:])168+ }169+ return out170+}171+172+func encodeVariable(kind Kind, values []Value) ([]byte, error) {173+ var payload bytes.Buffer174+ offsets := make([]uint32, len(values)+1)175+ for index, value := range values {176+ offsets[index] = uint32(payload.Len())177+ if value.Kind == KindInvalid {178+ continue179+ }180+ if kind == KindString {181+ _, _ = payload.WriteString(value.String)182+ } else {183+ _, _ = payload.Write(value.Bytes)184+ }185+ if uint64(payload.Len()) > math.MaxUint32 {186+ return nil, fmt.Errorf("variable column exceeds 4 GiB")187+ }188+ }189+ offsets[len(values)] = uint32(payload.Len())190+ out := make([]byte, len(offsets)*4+payload.Len())191+ for index, offset := range offsets {192+ binary.LittleEndian.PutUint32(out[index*4:], offset)193+ }194+ copy(out[len(offsets)*4:], payload.Bytes())195+ return out, nil196+}197+198+// DecodeBlock validates and opens one block without consulting an application199+// schema. Unknown fields therefore remain ordinary columns.200+func DecodeBlock(data []byte) (Block, error) {201+ header, err := readBlockHeader(data)202+ if err != nil {203+ return Block{}, err204+ }205+ table := Table{TypeID: header.typeID, Rows: header.rows, Columns: make([]Column, header.columns)}206+ for index := 0; index < header.columns; index++ {207+ at := blockHeaderSize + index*directorySize208+ column, err := decodeColumn(data, data[at:at+directorySize], header.rows)209+ if err != nil {210+ return Block{}, fmt.Errorf("decode column %d: %w", index, err)211+ }212+ table.Columns[index] = column213+ }214+ if err := table.validate(); err != nil {215+ return Block{}, err216+ }217+ return Block{SchemaHash: header.schemaHash, Table: canonicalTable(table)}, nil218+}219+220+type decodedHeader struct {221+ rows int222+ columns int223+ schemaHash [32]byte224+ typeID ID225+}226+227+func readBlockHeader(data []byte) (decodedHeader, error) {228+ if len(data) < blockHeaderSize || string(data[:8]) != blockMagic {229+ return decodedHeader{}, fmt.Errorf("typed block has no Atlas magic")230+ }231+ if framing := binary.LittleEndian.Uint16(data[8:10]); framing != blockFraming {232+ return decodedHeader{}, fmt.Errorf("typed block framing %d is not supported", framing)233+ }234+ columns := int(binary.LittleEndian.Uint32(data[16:20]))235+ directoryEnd := blockHeaderSize + columns*directorySize236+ if columns < 0 || directoryEnd < blockHeaderSize || directoryEnd > len(data) {237+ return decodedHeader{}, fmt.Errorf("typed block directory is truncated")238+ }239+ if binary.LittleEndian.Uint32(data[68:72]) != directorySize || binary.LittleEndian.Uint64(data[72:80]) != uint64(directoryEnd) {240+ return decodedHeader{}, fmt.Errorf("typed block directory framing is invalid")241+ }242+ header := decodedHeader{rows: int(binary.LittleEndian.Uint32(data[12:16])), columns: columns}243+ if header.rows > maxBlockRows || header.columns == 0 && header.rows != 0 {244+ return decodedHeader{}, fmt.Errorf("typed block row count is invalid")245+ }246+ copy(header.schemaHash[:], data[20:52])247+ copy(header.typeID[:], data[52:68])248+ return header, nil249+}250+251+func decodeColumn(data, directory []byte, rows int) (Column, error) {252+ var fieldID ID253+ copy(fieldID[:], directory[:16])254+ kind := Kind(directory[16])255+ if !kind.valid() || directory[17] != byte(kind) {256+ return Column{}, fmt.Errorf("column has unsupported encoding")257+ }258+ optional := binary.LittleEndian.Uint16(directory[18:20])&columnFlagOptional != 0259+ nulls, err := blockSlice(data, directory[20:28], directory[28:36])260+ if err != nil || len(nulls) != (rows+7)/8 {261+ return Column{}, fmt.Errorf("column presence bitmap is invalid")262+ }263+ payload, err := blockSlice(data, directory[36:44], directory[44:52])264+ if err != nil {265+ return Column{}, fmt.Errorf("column payload is invalid")266+ }267+ if checksum(nulls, payload) != binary.LittleEndian.Uint32(directory[52:56]) {268+ return Column{}, fmt.Errorf("column checksum does not match")269+ }270+ values, err := decodeValues(kind, rows, nulls, payload)271+ if err != nil {272+ return Column{}, err273+ }274+ return Column{Field: Field{ID: fieldID, Kind: kind, Optional: optional}, Values: values}, nil275+}276+277+func blockSlice(data, offsetBytes, lengthBytes []byte) ([]byte, error) {278+ offset := binary.LittleEndian.Uint64(offsetBytes)279+ length := binary.LittleEndian.Uint64(lengthBytes)280+ if offset > uint64(len(data)) || length > uint64(len(data))-offset {281+ return nil, fmt.Errorf("range exceeds block")282+ }283+ return data[offset : offset+length], nil284+}285+286+func checksum(parts ...[]byte) uint32 {287+ hash := crc32.NewIEEE()288+ for _, part := range parts {289+ _, _ = hash.Write(part)290+ }291+ return hash.Sum32()292+}293+294+func decodeValues(kind Kind, rows int, present, payload []byte) ([]Value, error) {295+ values := make([]Value, rows)296+ switch kind {297+ case KindBool:298+ return decodeBools(values, present, payload)299+ case KindInt64:300+ return decodeInt64s(values, present, payload)301+ case KindFloat64:302+ return decodeFloat64s(values, present, payload)303+ case KindString, KindBytes:304+ return decodeVariable(kind, values, present, payload)305+ case KindID:306+ return decodeIDs(values, present, payload)307+ default:308+ return nil, fmt.Errorf("unsupported wire kind %d", kind)309+ }310+}311+312+func isPresent(bitmap []byte, row int) bool { return bitmap[row/8]&(1<<(row%8)) != 0 }313+314+func decodeBools(values []Value, present, payload []byte) ([]Value, error) {315+ if len(payload) != len(values) {316+ return nil, fmt.Errorf("boolean column has wrong length")317+ }318+ for row := range values {319+ if isPresent(present, row) {320+ values[row] = BoolValue(payload[row] != 0)321+ }322+ }323+ return values, nil324+}325+326+func decodeInt64s(values []Value, present, payload []byte) ([]Value, error) {327+ if len(payload) != len(values)*8 {328+ return nil, fmt.Errorf("int64 column has wrong length")329+ }330+ for row := range values {331+ if isPresent(present, row) {332+ values[row] = Int64Value(int64(binary.LittleEndian.Uint64(payload[row*8:])))333+ }334+ }335+ return values, nil336+}337+338+func decodeFloat64s(values []Value, present, payload []byte) ([]Value, error) {339+ if len(payload) != len(values)*8 {340+ return nil, fmt.Errorf("float64 column has wrong length")341+ }342+ for row := range values {343+ if isPresent(present, row) {344+ values[row] = Float64Value(math.Float64frombits(binary.LittleEndian.Uint64(payload[row*8:])))345+ }346+ }347+ return values, nil348+}349+350+func decodeIDs(values []Value, present, payload []byte) ([]Value, error) {351+ if len(payload) != len(values)*16 {352+ return nil, fmt.Errorf("ID column has wrong length")353+ }354+ for row := range values {355+ if isPresent(present, row) {356+ var id ID357+ copy(id[:], payload[row*16:(row+1)*16])358+ values[row] = IDValue(id)359+ }360+ }361+ return values, nil362+}363+364+func decodeVariable(kind Kind, values []Value, present, payload []byte) ([]Value, error) {365+ offsetBytes := (len(values) + 1) * 4366+ if len(payload) < offsetBytes {367+ return nil, fmt.Errorf("variable column offsets are truncated")368+ }369+ content := payload[offsetBytes:]370+ previous := uint32(0)371+ for row := range values {372+ start := binary.LittleEndian.Uint32(payload[row*4:])373+ end := binary.LittleEndian.Uint32(payload[(row+1)*4:])374+ if start < previous || end < start || uint64(end) > uint64(len(content)) {375+ return nil, fmt.Errorf("variable column offsets are invalid at row %d", row)376+ }377+ previous = end378+ if !isPresent(present, row) {379+ continue380+ }381+ if kind == KindString {382+ values[row] = StringValue(string(content[start:end]))383+ } else {384+ values[row] = BytesValue(content[start:end])385+ }386+ }387+ return values, nil388+}@@ -0,0 +1,125 @@
1+package vnext2+3+import (4+ "bytes"5+ "testing"6+)7+8+func TestPackedBlockRoundTripsEveryWireKind(t *testing.T) {9+ t.Parallel()10+11+ typeID := IDFromName("com.example", "type.observation")12+ fields := []Field{13+ {ID: IDFromName("com.example", "bool"), Name: "bool", Kind: KindBool, Optional: true},14+ {ID: IDFromName("com.example", "int"), Name: "int", Kind: KindInt64, Optional: true},15+ {ID: IDFromName("com.example", "float"), Name: "float", Kind: KindFloat64, Optional: true},16+ {ID: IDFromName("com.example", "string"), Name: "string", Kind: KindString, Optional: true},17+ {ID: IDFromName("com.example", "bytes"), Name: "bytes", Kind: KindBytes, Optional: true},18+ {ID: IDFromName("com.example", "ref"), Name: "ref", Kind: KindID, Optional: true},19+ }20+ schema := Schema{Types: []Type{{ID: typeID, Name: "Observation", Fields: fields}}}21+ hash, err := schema.Hash()22+ if err != nil {23+ t.Fatalf("hash schema: %v", err)24+ }25+ want := Table{26+ TypeID: typeID,27+ Rows: 2,28+ Columns: []Column{29+ {Field: fields[0], Values: []Value{BoolValue(true), NullValue()}},30+ {Field: fields[1], Values: []Value{Int64Value(-42), Int64Value(9001)}},31+ {Field: fields[2], Values: []Value{Float64Value(3.5), Float64Value(-0.25)}},32+ {Field: fields[3], Values: []Value{StringValue("north"), StringValue("雪")}},33+ {Field: fields[4], Values: []Value{BytesValue([]byte{0, 1, 2}), BytesValue(nil)}},34+ {Field: fields[5], Values: []Value{IDValue(IDFromName("com.example", "one")), IDValue(IDFromName("com.example", "two"))}},35+ },36+ }37+38+ data, err := EncodeBlock(hash, want)39+ if err != nil {40+ t.Fatalf("encode block: %v", err)41+ }42+ got, err := DecodeBlock(data)43+ if err != nil {44+ t.Fatalf("decode block: %v", err)45+ }46+ if diff := diffTable(want, got.Table); diff != "" {47+ t.Fatal(diff)48+ }49+ reencoded, err := EncodeBlock(got.SchemaHash, got.Table)50+ if err != nil {51+ t.Fatalf("re-encode block: %v", err)52+ }53+ if !bytes.Equal(data, reencoded) {54+ t.Fatal("canonical block changed after decode and re-encode")55+ }56+}57+58+func TestOlderRuntimePreservesUnknownColumns(t *testing.T) {59+ t.Parallel()60+61+ typeID := CoreID("type.feature")62+ known := Field{ID: CoreID("feature.title"), Name: "title", Kind: KindString}63+ foreign := Field{ID: IDFromName("com.example.rescue", "feature.triage"), Name: "triage", Kind: KindInt64}64+ readerSchema := Schema{Types: []Type{{ID: typeID, Name: "Feature", Fields: []Field{known}}}}65+ fileSchema, err := UnionSchemas(readerSchema, Schema{Types: []Type{{ID: typeID, Name: "Feature", Fields: []Field{foreign}}}})66+ if err != nil {67+ t.Fatalf("make file schema: %v", err)68+ }69+ table := Table{TypeID: typeID, Rows: 1, Columns: []Column{70+ {Field: known, Values: []Value{StringValue("Clinic")}},71+ {Field: foreign, Values: []Value{Int64Value(4)}},72+ }}73+ hash, err := fileSchema.Hash()74+ if err != nil {75+ t.Fatalf("hash file schema: %v", err)76+ }77+ data, err := EncodeBlock(hash, table)78+ if err != nil {79+ t.Fatalf("encode file block: %v", err)80+ }81+ block, err := DecodeBlock(data)82+ if err != nil {83+ t.Fatalf("decode file block: %v", err)84+ }85+ view, err := ProjectTable(readerSchema, fileSchema, block.Table)86+ if err != nil {87+ t.Fatalf("project through old reader: %v", err)88+ }89+ if len(view.Known.Columns) != 1 || view.Known.Columns[0].Field.ID != known.ID {90+ t.Fatalf("known projection = %#v", view.Known.Columns)91+ }92+ if len(view.Foreign.Columns) != 1 || view.Foreign.Columns[0].Field.ID != foreign.ID {93+ t.Fatalf("foreign projection = %#v", view.Foreign.Columns)94+ }95+96+ reencoded, err := EncodeBlock(block.SchemaHash, view.Complete())97+ if err != nil {98+ t.Fatalf("re-encode projected block: %v", err)99+ }100+ if !bytes.Equal(data, reencoded) {101+ t.Fatal("old reader did not preserve the foreign column byte-for-byte")102+ }103+}104+105+func diffTable(want, got Table) string {106+ if want.TypeID != got.TypeID || want.Rows != got.Rows || len(want.Columns) != len(got.Columns) {107+ return "decoded table shape differs"108+ }109+ byID := make(map[ID]Column, len(got.Columns))110+ for _, column := range got.Columns {111+ byID[column.Field.ID] = column112+ }113+ for _, left := range want.Columns {114+ right, held := byID[left.Field.ID]115+ if !held || left.Field.Kind != right.Field.Kind || left.Field.Optional != right.Field.Optional || len(left.Values) != len(right.Values) {116+ return "decoded column shape differs"117+ }118+ for row := range left.Values {119+ if !left.Values[row].Equal(right.Values[row]) {120+ return "decoded value differs"121+ }122+ }123+ }124+ return ""125+}@@ -0,0 +1,502 @@
1+package vnext2+3+import (4+ "crypto/sha256"5+ "encoding/binary"6+ "fmt"7+ "math"8+ "slices"9+)10+11+const (12+ volumeTableName = "data/volumes.pack"13+ worldTableName = "data/worlds.pack"14+ coordinateTableName = "data/coordinate-spaces.pack"15+ featureSetTableName = "data/feature-sets.pack"16+ propertyDefinitionTableName = "data/property-definitions.pack"17+ featureTableName = "data/features.pack"18+ relationshipTableName = "data/relationships.pack"19+ provenanceTableName = "data/provenance.pack"20+ rasterTableName = "data/raster-pyramids.pack"21+ rasterFormatTableName = "data/raster-formats.pack"22+ rasterCoverageTableName = "data/raster-coverage.pack"23+ presentationTableName = "data/presentations.pack"24+ styleTableName = "data/styles.pack"25+ layerTableName = "data/layers.pack"26+ legendTableName = "data/legend.pack"27+ assetTableName = "data/assets.pack"28+)29+30+// NamedTable assigns a stable container entry to a typed table.31+type NamedTable struct {32+ Name string33+ Table Table34+}35+36+// Blob is an opaque, independently range-readable payload.37+type Blob struct {38+ Name string39+ Data []byte40+}41+42+// Bundle is the complete compiled form accepted by Write.43+type Bundle struct {44+ VolumeID string45+ Schema Schema46+ Tables []NamedTable47+ Blobs []Blob48+}49+50+type tableBuilder struct {51+ typ Type52+ columns map[ID][]Value53+ rows int54+}55+56+func newTableBuilder(schema Schema, typeName string) (*tableBuilder, error) {57+ typ, ok := schema.Type(CoreID("type." + typeName))58+ if !ok {59+ return nil, fmt.Errorf("schema has no %s type", typeName)60+ }61+ columns := make(map[ID][]Value, len(typ.Fields))62+ for _, field := range typ.Fields {63+ columns[field.ID] = nil64+ }65+ return &tableBuilder{typ: typ, columns: columns}, nil66+}67+68+func (builder *tableBuilder) add(values map[ID]Value) error {69+ for id := range values {70+ if _, held := builder.columns[id]; !held {71+ return fmt.Errorf("type %s has no field %s", builder.typ.Name, id)72+ }73+ }74+ for _, field := range builder.typ.Fields {75+ value, held := values[field.ID]76+ if !held {77+ value = NullValue()78+ }79+ builder.columns[field.ID] = append(builder.columns[field.ID], value)80+ }81+ builder.rows++82+ return nil83+}84+85+func (builder *tableBuilder) table() Table {86+ table := Table{TypeID: builder.typ.ID, Rows: builder.rows, Columns: make([]Column, 0, len(builder.typ.Fields))}87+ for _, field := range builder.typ.Fields {88+ table.Columns = append(table.Columns, Column{Field: field, Values: builder.columns[field.ID]})89+ }90+ return canonicalTable(table)91+}92+93+// Compile validates a complete semantic Volume and plans it into typed tables94+// plus content-addressed blobs.95+func Compile(volume Volume) (Bundle, error) {96+ if err := validateVolume(volume); err != nil {97+ return Bundle{}, err98+ }99+ schema, err := schemaFor(volume)100+ if err != nil {101+ return Bundle{}, err102+ }103+ builders, err := makeBuilders(schema)104+ if err != nil {105+ return Bundle{}, err106+ }107+ blobs, err := compileRows(volume, builders)108+ if err != nil {109+ return Bundle{}, err110+ }111+ names := tableNames()112+ tables := make([]NamedTable, len(names))113+ for index, name := range names {114+ table := builders[name].table()115+ if err := table.validate(); err != nil {116+ return Bundle{}, fmt.Errorf("compile %s: %w", name, err)117+ }118+ tables[index] = NamedTable{Name: name, Table: table}119+ }120+ return Bundle{VolumeID: volume.ID, Schema: schema, Tables: tables, Blobs: blobs}, nil121+}122+123+func schemaFor(volume Volume) (Schema, error) {124+ standard := StandardSchema()125+ extensions := make(map[ID]map[ID]Field)126+ for _, world := range volume.Worlds {127+ if err := collectClaims(extensions, standard, "world", world.Claims); err != nil {128+ return Schema{}, err129+ }130+ for _, set := range world.FeatureSets {131+ if err := collectClaims(extensions, standard, "featureSet", set.Claims); err != nil {132+ return Schema{}, err133+ }134+ for _, field := range set.Properties {135+ if err := collectField(extensions, standard, "feature", field); err != nil {136+ return Schema{}, fmt.Errorf("feature set %s: %w", set.ID, err)137+ }138+ }139+ }140+ }141+ fragment := Schema{}142+ for typeID, fields := range extensions {143+ typ, _ := standard.Type(typeID)144+ extension := Type{ID: typeID, Name: typ.Name, Fields: make([]Field, 0, len(fields))}145+ for _, field := range fields {146+ extension.Fields = append(extension.Fields, field)147+ }148+ fragment.Types = append(fragment.Types, extension)149+ }150+ return UnionSchemas(standard, fragment)151+}152+153+func collectClaims(extensions map[ID]map[ID]Field, standard Schema, typeName string, claims []Property) error {154+ for _, claim := range claims {155+ field := claim.Field156+ if field.ID.isZero() {157+ return fmt.Errorf("%s claim %s has no field definition", typeName, claim.fieldID())158+ }159+ if err := collectField(extensions, standard, typeName, field); err != nil {160+ return err161+ }162+ }163+ return nil164+}165+166+func collectField(extensions map[ID]map[ID]Field, standard Schema, typeName string, field Field) error {167+ typeID := CoreID("type." + typeName)168+ core, _ := standard.Type(typeID)169+ if _, held := core.Field(field.ID); held {170+ return fmt.Errorf("extension reuses core field %s", field.ID)171+ }172+ field.Optional = true173+ if extensions[typeID] == nil {174+ extensions[typeID] = make(map[ID]Field)175+ }176+ current, held := extensions[typeID][field.ID]177+ if held && current.Kind != field.Kind {178+ return fmt.Errorf("field %s has conflicting kinds", field.ID)179+ }180+ if held && current.Name < field.Name {181+ field.Name = current.Name182+ }183+ extensions[typeID][field.ID] = field184+ return nil185+}186+187+func makeBuilders(schema Schema) (map[string]*tableBuilder, error) {188+ typeNames := []string{"volume", "world", "coordinate", "featureSet", "propertyDefinition", "feature", "relationship", "provenance", "rasterPyramid", "rasterFormat", "rasterCoverage", "presentation", "style", "layer", "legend", "asset"}189+ names := tableNames()190+ builders := make(map[string]*tableBuilder, len(names))191+ for index, name := range names {192+ builder, err := newTableBuilder(schema, typeNames[index])193+ if err != nil {194+ return nil, err195+ }196+ builders[name] = builder197+ }198+ return builders, nil199+}200+201+func tableNames() []string {202+ return []string{203+ volumeTableName, worldTableName, coordinateTableName, featureSetTableName,204+ propertyDefinitionTableName, featureTableName, relationshipTableName,205+ provenanceTableName, rasterTableName, rasterFormatTableName, rasterCoverageTableName,206+ presentationTableName, styleTableName,207+ layerTableName, legendTableName, assetTableName,208+ }209+}210+211+func compileRows(volume Volume, builders map[string]*tableBuilder) ([]Blob, error) {212+ if err := builders[volumeTableName].add(valuesOf("volume", map[string]Value{213+ "id": StringValue(volume.ID), "title": StringValue(volume.Title),214+ })); err != nil {215+ return nil, err216+ }217+ for _, world := range volume.Worlds {218+ if err := compileWorld(world, builders); err != nil {219+ return nil, fmt.Errorf("world %s: %w", world.ID, err)220+ }221+ }222+ return compileAssets(volume.Assets, builders[assetTableName])223+}224+225+func compileWorld(world World, builders map[string]*tableBuilder) error {226+ worldValues := valuesOf("world", map[string]Value{227+ "id": StringValue(world.ID), "title": StringValue(world.Title),228+ "coordinateSpace": StringValue(world.CoordinateSpace.ID),229+ })230+ addClaims(worldValues, world.Claims)231+ if err := builders[worldTableName].add(worldValues); err != nil {232+ return err233+ }234+ if err := compileCoordinate(world, builders[coordinateTableName]); err != nil {235+ return err236+ }237+ for _, set := range world.FeatureSets {238+ if err := compileFeatureSet(world.ID, set, builders); err != nil {239+ return err240+ }241+ }242+ for _, raster := range world.RasterPyramids {243+ if err := compileRaster(world.ID, raster, builders); err != nil {244+ return err245+ }246+ }247+ return compilePresentation(world.ID, world.Presentation, builders)248+}249+250+func compileCoordinate(world World, builder *tableBuilder) error {251+ space := world.CoordinateSpace252+ return builder.add(valuesOf("coordinate", map[string]Value{253+ "id": StringValue(space.ID), "world": StringValue(world.ID),254+ "kind": StringValue(space.Kind), "unit": StringValue(space.Unit),255+ "definition": StringValue(space.Definition), "extent": BytesValue(encodeExtent(space.Extent)),256+ "sourceZoom": Int64Value(space.SourceZoom), "firstTile": Int64Value(space.FirstTile),257+ "tileSize": Int64Value(space.TileSize), "size": Int64Value(space.Size),258+ }))259+}260+261+func compileFeatureSet(worldID string, set FeatureSet, builders map[string]*tableBuilder) error {262+ setValues := valuesOf("featureSet", map[string]Value{263+ "id": StringValue(set.ID), "world": StringValue(worldID), "title": StringValue(set.Title),264+ "semanticType": StringValue(set.SemanticType),265+ })266+ addClaims(setValues, set.Claims)267+ if err := builders[featureSetTableName].add(setValues); err != nil {268+ return err269+ }270+ contract := make(map[ID]Field, len(set.Properties))271+ for _, field := range set.Properties {272+ contract[field.ID] = field273+ if err := builders[propertyDefinitionTableName].add(valuesOf("propertyDefinition", map[string]Value{274+ "featureSet": StringValue(set.ID), "field": IDValue(field.ID), "name": StringValue(field.Name),275+ "kind": Int64Value(int64(field.Kind)), "optional": BoolValue(field.Optional),276+ })); err != nil {277+ return err278+ }279+ }280+ for _, feature := range set.Features {281+ if err := compileFeature(set.ID, feature, contract, builders); err != nil {282+ return err283+ }284+ }285+ return nil286+}287+288+func compileFeature(setID string, feature Feature, contract map[ID]Field, builders map[string]*tableBuilder) error {289+ geometry, err := encodeGeometry(feature.Geometry)290+ if err != nil {291+ return fmt.Errorf("feature %s geometry: %w", feature.ID, err)292+ }293+ values := valuesOf("feature", map[string]Value{294+ "id": StringValue(feature.ID), "featureSet": StringValue(setID),295+ "title": StringValue(feature.Title), "subtitle": optionalString(feature.Subtitle),296+ "description": optionalString(feature.Description), "center": optionalPosition(feature.Center),297+ "shard": Int64Value(feature.Shard), "geometry": BytesValue(geometry),298+ })299+ seen := make(map[ID]bool, len(feature.Properties))300+ for _, property := range feature.Properties {301+ propertyID := property.fieldID()302+ field, held := contract[propertyID]303+ if !held || seen[propertyID] || property.Value.Kind != field.Kind {304+ return fmt.Errorf("feature %s has an undeclared, duplicate, or mistyped property %s", feature.ID, propertyID)305+ }306+ seen[propertyID] = true307+ values[propertyID] = property.Value308+ }309+ for id, field := range contract {310+ if !field.Optional && !seen[id] {311+ return fmt.Errorf("feature %s omits required property %s", feature.ID, id)312+ }313+ }314+ if err := builders[featureTableName].add(values); err != nil {315+ return err316+ }317+ return compileEvidence(feature, builders)318+}319+320+func compileEvidence(feature Feature, builders map[string]*tableBuilder) error {321+ for _, relationship := range feature.Relationships {322+ if err := builders[relationshipTableName].add(valuesOf("relationship", map[string]Value{323+ "feature": StringValue(feature.ID), "predicate": StringValue(relationship.Predicate), "target": StringValue(relationship.Target),324+ })); err != nil {325+ return err326+ }327+ }328+ for _, provenance := range feature.Provenance {329+ if err := builders[provenanceTableName].add(valuesOf("provenance", map[string]Value{330+ "feature": StringValue(feature.ID), "source": StringValue(provenance.Source),331+ "nativeID": StringValue(provenance.NativeID), "capturedAt": StringValue(provenance.CapturedAt),332+ })); err != nil {333+ return err334+ }335+ }336+ return nil337+}338+339+func compileRaster(worldID string, raster RasterPyramid, builders map[string]*tableBuilder) error {340+ if err := builders[rasterTableName].add(valuesOf("rasterPyramid", map[string]Value{341+ "id": StringValue(raster.ID), "world": StringValue(worldID), "name": StringValue(raster.Name), "codec": StringValue(raster.Codec),342+ "tileSize": Int64Value(raster.TileSize), "minZoom": Int64Value(raster.MinZoom),343+ "maxZoom": Int64Value(raster.MaxZoom), "fullZoom": Int64Value(raster.FullZoom),344+ "sourceZoom": Int64Value(raster.SourceZoom), "template": StringValue(raster.Template),345+ "bounds": optionalRect(raster.Bounds), "surface": optionalRect(raster.Surface),346+ "interpolate": BoolValue(raster.Interpolate), "background": optionalString(raster.Background), "shard": Int64Value(raster.Shard),347+ })); err != nil {348+ return err349+ }350+ for index, format := range raster.Formats {351+ if err := builders[rasterFormatTableName].add(valuesOf("rasterFormat", map[string]Value{352+ "raster": StringValue(raster.ID), "zoom": Int64Value(raster.MinZoom + int64(index)), "format": StringValue(format),353+ })); err != nil {354+ return err355+ }356+ }357+ for _, coverage := range raster.Coverage {358+ if err := builders[rasterCoverageTableName].add(valuesOf("rasterCoverage", map[string]Value{359+ "raster": StringValue(raster.ID), "zoom": Int64Value(coverage.Zoom),360+ "x": Int64Value(coverage.X), "y": Int64Value(coverage.Y), "w": Int64Value(coverage.W), "h": Int64Value(coverage.H),361+ "bits": BytesValue(coverage.Bits),362+ })); err != nil {363+ return err364+ }365+ }366+ return nil367+}368+369+func compilePresentation(worldID string, presentation Presentation, builders map[string]*tableBuilder) error {370+ if err := builders[presentationTableName].add(valuesOf("presentation", map[string]Value{371+ "id": StringValue(presentation.ID), "world": StringValue(worldID), "title": StringValue(presentation.Title),372+ })); err != nil {373+ return err374+ }375+ for _, style := range presentation.Styles {376+ if err := builders[styleTableName].add(valuesOf("style", map[string]Value{377+ "id": StringValue(style.ID), "presentation": StringValue(presentation.ID),378+ "symbol": optionalString(style.Symbol), "icon": optionalString(style.Icon), "iconAsset": optionalString(style.IconAsset),379+ "iconPicture": BoolValue(style.IconPicture), "renderAs": optionalString(style.RenderAs),380+ "stroke": optionalString(style.Stroke), "fill": optionalString(style.Fill),381+ })); err != nil {382+ return err383+ }384+ }385+ for _, layer := range presentation.Layers {386+ if err := builders[layerTableName].add(valuesOf("layer", map[string]Value{387+ "id": StringValue(layer.ID), "presentation": StringValue(presentation.ID),388+ "featureSet": StringValue(layer.FeatureSet), "style": StringValue(layer.Style),389+ "group": optionalString(layer.Group), "labelPolicy": optionalString(layer.LabelPolicy), "visible": BoolValue(layer.Visible),390+ "order": Int64Value(layer.Order), "minZoom": Int64Value(layer.MinZoom), "maxZoom": Int64Value(layer.MaxZoom),391+ })); err != nil {392+ return err393+ }394+ }395+ for _, legend := range presentation.Legend {396+ if err := builders[legendTableName].add(valuesOf("legend", map[string]Value{397+ "presentation": StringValue(presentation.ID), "layer": StringValue(legend.Layer),398+ "label": StringValue(legend.Label), "order": Int64Value(legend.Order),399+ })); err != nil {400+ return err401+ }402+ }403+ return nil404+}405+406+func compileAssets(assets []Asset, builder *tableBuilder) ([]Blob, error) {407+ blobs := make([]Blob, 0, len(assets))408+ held := make(map[string]bool, len(assets))409+ for _, asset := range assets {410+ digest := sha256.Sum256(asset.Data)411+ path := fmt.Sprintf("assets/%x", digest)412+ if err := builder.add(valuesOf("asset", map[string]Value{413+ "id": StringValue(asset.ID), "mediaType": StringValue(asset.MediaType),414+ "path": StringValue(path), "provenance": optionalString(asset.Provenance),415+ })); err != nil {416+ return nil, err417+ }418+ if !held[path] {419+ held[path] = true420+ blobs = append(blobs, Blob{Name: path, Data: append([]byte(nil), asset.Data...)})421+ }422+ }423+ slices.SortFunc(blobs, func(left, right Blob) int {424+ if left.Name < right.Name {425+ return -1426+ }427+ if left.Name > right.Name {428+ return 1429+ }430+ return 0431+ })432+ return blobs, nil433+}434+435+func valuesOf(typeName string, values map[string]Value) map[ID]Value {436+ out := make(map[ID]Value, len(values))437+ for name, value := range values {438+ out[CoreID(typeName+"."+name)] = value439+ }440+ return out441+}442+443+func optionalString(value string) Value {444+ if value == "" {445+ return NullValue()446+ }447+ return StringValue(value)448+}449+450+func addClaims(values map[ID]Value, claims []Property) {451+ for _, claim := range claims {452+ values[claim.fieldID()] = claim.Value453+ }454+}455+456+func optionalRect(rect *RasterRect) Value {457+ if rect == nil {458+ return NullValue()459+ }460+ return BytesValue(encodeRect(*rect))461+}462+463+func optionalPosition(position *Position) Value {464+ if position == nil {465+ return NullValue()466+ }467+ out := make([]byte, 16)468+ binary.LittleEndian.PutUint64(out, math.Float64bits(position[0]))469+ binary.LittleEndian.PutUint64(out[8:], math.Float64bits(position[1]))470+ return BytesValue(out)471+}472+473+func encodeExtent(extent [4]float64) []byte {474+ out := make([]byte, 32)475+ for index, coordinate := range extent {476+ binary.LittleEndian.PutUint64(out[index*8:], math.Float64bits(coordinate))477+ }478+ return out479+}480+481+func encodeGeometry(geometry Geometry) ([]byte, error) {482+ if err := validateGeometry(geometry); err != nil {483+ return nil, err484+ }485+ out := []byte{byte(geometry.Kind)}486+ out = binary.LittleEndian.AppendUint32(out, uint32(len(geometry.Parts)))487+ for _, part := range geometry.Parts {488+ out = binary.LittleEndian.AppendUint32(out, uint32(len(part.Rings)))489+ for _, ring := range part.Rings {490+ out = binary.LittleEndian.AppendUint32(out, uint32(len(ring)))491+ for _, position := range ring {492+ out = binary.LittleEndian.AppendUint64(out, math.Float64bits(position[0]))493+ out = binary.LittleEndian.AppendUint64(out, math.Float64bits(position[1]))494+ }495+ }496+ }497+ return out, nil498+}499+500+func encodeRect(rect RasterRect) []byte {501+ return encodeExtent([4]float64{rect.X, rect.Y, rect.Width, rect.Height})502+}@@ -0,0 +1,366 @@
1+package vnext2+3+import (4+ "archive/zip"5+ "bytes"6+ "crypto/sha256"7+ "encoding/hex"8+ "encoding/json"9+ "fmt"10+ "io"11+ "slices"12+ "strings"13+ "time"14+)15+16+const (17+ manifestName = "atlas.json"18+ containerFormat = "atlas-schema-bundle"19+ containerFraming = uint16(1)20+ maxManifestSize = 1 << 2021+)22+23+type schemaReference struct {24+ Name string `json:"name"`25+ Hash string `json:"hash"`26+}27+28+type tableReference struct {29+ Name string `json:"name"`30+ RootType ID `json:"rootType"`31+ Hash string `json:"hash"`32+ Length int64 `json:"length"`33+}34+35+type blobReference struct {36+ Name string `json:"name"`37+ Hash string `json:"hash"`38+ Length int64 `json:"length"`39+}40+41+type bootstrap struct {42+ Format string `json:"format"`43+ Framing uint16 `json:"framing"`44+ Volume string `json:"volume"`45+ Schema schemaReference `json:"schema"`46+ Tables []tableReference `json:"tables"`47+ Blobs []blobReference `json:"blobs,omitempty"`48+}49+50+type preparedEntry struct {51+ name string52+ data []byte53+ method uint1654+}55+56+// Write produces one deterministic ZIP container with a small JSON bootstrap,57+// an embedded canonical schema, stored typed blocks, and stored opaque blobs.58+func Write(out io.Writer, bundle Bundle) error {59+ if bundle.VolumeID == "" {60+ return fmt.Errorf("bundle has no volume identity")61+ }62+ schemaData, err := bundle.Schema.Canonical()63+ if err != nil {64+ return fmt.Errorf("canonicalize schema: %w", err)65+ }66+ schemaHash := sha256.Sum256(schemaData)67+ schemaName := "schema/" + hex.EncodeToString(schemaHash[:]) + ".json"68+ manifest, entries, err := prepareBundle(bundle, schemaHash, schemaName, schemaData)69+ if err != nil {70+ return err71+ }72+ manifestData, err := json.Marshal(manifest)73+ if err != nil {74+ return fmt.Errorf("marshal bootstrap: %w", err)75+ }76+ entries = append(entries, preparedEntry{name: manifestName, data: manifestData, method: zip.Deflate})77+ slices.SortFunc(entries, func(left, right preparedEntry) int {78+ if left.name == manifestName {79+ return -180+ }81+ if right.name == manifestName {82+ return 183+ }84+ return strings.Compare(left.name, right.name)85+ })86+ return writeArchive(out, entries)87+}88+89+func prepareBundle(bundle Bundle, schemaHash [32]byte, schemaName string, schemaData []byte) (bootstrap, []preparedEntry, error) {90+ manifest := bootstrap{91+ Format: containerFormat, Framing: containerFraming, Volume: bundle.VolumeID,92+ Schema: schemaReference{Name: schemaName, Hash: hex.EncodeToString(schemaHash[:])},93+ }94+ entries := []preparedEntry{{name: schemaName, data: schemaData, method: zip.Deflate}}95+ for _, named := range bundle.Tables {96+ data, err := EncodeBlock(schemaHash, named.Table)97+ if err != nil {98+ return bootstrap{}, nil, fmt.Errorf("encode %s: %w", named.Name, err)99+ }100+ digest := sha256.Sum256(data)101+ manifest.Tables = append(manifest.Tables, tableReference{102+ Name: named.Name, RootType: named.Table.TypeID,103+ Hash: hex.EncodeToString(digest[:]), Length: int64(len(data)),104+ })105+ entries = append(entries, preparedEntry{name: named.Name, data: data, method: zip.Store})106+ }107+ for _, blob := range bundle.Blobs {108+ digest := sha256.Sum256(blob.Data)109+ manifest.Blobs = append(manifest.Blobs, blobReference{110+ Name: blob.Name, Hash: hex.EncodeToString(digest[:]), Length: int64(len(blob.Data)),111+ })112+ entries = append(entries, preparedEntry{name: blob.Name, data: blob.Data, method: zip.Store})113+ }114+ slices.SortFunc(manifest.Tables, func(left, right tableReference) int { return strings.Compare(left.Name, right.Name) })115+ slices.SortFunc(manifest.Blobs, func(left, right blobReference) int { return strings.Compare(left.Name, right.Name) })116+ return manifest, entries, nil117+}118+119+func writeArchive(out io.Writer, entries []preparedEntry) error {120+ archive := zip.NewWriter(out)121+ seen := make(map[string]bool, len(entries))122+ for _, entry := range entries {123+ if err := validEntryName(entry.name); err != nil {124+ return err125+ }126+ if seen[entry.name] {127+ return fmt.Errorf("entry %s is written twice", entry.name)128+ }129+ seen[entry.name] = true130+ header := &zip.FileHeader{Name: entry.name, Method: entry.method}131+ header.SetModTime(time.Date(1980, 1, 1, 0, 0, 0, 0, time.UTC))132+ writer, err := archive.CreateHeader(header)133+ if err != nil {134+ return fmt.Errorf("create %s: %w", entry.name, err)135+ }136+ if _, err := writer.Write(entry.data); err != nil {137+ return fmt.Errorf("write %s: %w", entry.name, err)138+ }139+ }140+ if err := archive.Close(); err != nil {141+ return fmt.Errorf("close Atlas archive: %w", err)142+ }143+ return nil144+}145+146+// Reader is an opened vNext bundle. It retains both the file schema and the147+// application's merged runtime schema.148+type Reader struct {149+ VolumeID string150+ FileSchema Schema151+ RuntimeSchema Schema152+153+ entries map[string]*zip.File154+ tables map[string]tableReference155+ blobs map[string]blobReference156+}157+158+// Open reads the bootstrap and embedded schema. Only container framing is a159+// hard gate; schema differences are merged by identity.160+func Open(source io.ReaderAt, size int64, applicationSchema Schema) (*Reader, error) {161+ archive, err := zip.NewReader(source, size)162+ if err != nil {163+ return nil, fmt.Errorf("open Atlas archive: %w", err)164+ }165+ entries, err := indexEntries(archive.File)166+ if err != nil {167+ return nil, err168+ }169+ manifest, err := readBootstrap(entries)170+ if err != nil {171+ return nil, err172+ }173+ fileSchema, err := readSchema(entries, manifest.Schema)174+ if err != nil {175+ return nil, err176+ }177+ runtimeSchema, err := UnionSchemas(applicationSchema, fileSchema)178+ if err != nil {179+ return nil, fmt.Errorf("merge runtime schema: %w", err)180+ }181+ reader := &Reader{182+ VolumeID: manifest.Volume, FileSchema: fileSchema, RuntimeSchema: runtimeSchema,183+ entries: entries, tables: make(map[string]tableReference, len(manifest.Tables)),184+ blobs: make(map[string]blobReference, len(manifest.Blobs)),185+ }186+ if err := reader.indexReferences(manifest); err != nil {187+ return nil, err188+ }189+ return reader, nil190+}191+192+func indexEntries(files []*zip.File) (map[string]*zip.File, error) {193+ entries := make(map[string]*zip.File, len(files))194+ for _, file := range files {195+ if err := validEntryName(file.Name); err != nil {196+ return nil, err197+ }198+ if _, held := entries[file.Name]; held {199+ return nil, fmt.Errorf("archive carries %s twice", file.Name)200+ }201+ entries[file.Name] = file202+ }203+ return entries, nil204+}205+206+func readBootstrap(entries map[string]*zip.File) (bootstrap, error) {207+ data, err := readLimitedEntry(entries, manifestName, maxManifestSize)208+ if err != nil {209+ return bootstrap{}, err210+ }211+ var manifest bootstrap212+ if err := json.Unmarshal(data, &manifest); err != nil {213+ return bootstrap{}, fmt.Errorf("decode bootstrap: %w", err)214+ }215+ if manifest.Format != containerFormat || manifest.Framing != containerFraming || manifest.Volume == "" {216+ return bootstrap{}, fmt.Errorf("unsupported Atlas container identity or framing")217+ }218+ return manifest, nil219+}220+221+func readSchema(entries map[string]*zip.File, reference schemaReference) (Schema, error) {222+ data, err := readLimitedEntry(entries, reference.Name, 16<<20)223+ if err != nil {224+ return Schema{}, err225+ }226+ digest := sha256.Sum256(data)227+ if hex.EncodeToString(digest[:]) != reference.Hash {228+ return Schema{}, fmt.Errorf("embedded schema hash does not match")229+ }230+ var schema Schema231+ if err := json.Unmarshal(data, &schema); err != nil {232+ return Schema{}, fmt.Errorf("decode embedded schema: %w", err)233+ }234+ canonical, err := schema.Canonical()235+ if err != nil {236+ return Schema{}, err237+ }238+ if !bytes.Equal(canonical, data) {239+ return Schema{}, fmt.Errorf("embedded schema is not canonical")240+ }241+ return schema, nil242+}243+244+func (reader *Reader) indexReferences(manifest bootstrap) error {245+ for _, reference := range manifest.Tables {246+ if _, held := reader.tables[reference.Name]; held || reader.entries[reference.Name] == nil {247+ return fmt.Errorf("table reference %s is duplicate or missing", reference.Name)248+ }249+ reader.tables[reference.Name] = reference250+ }251+ for _, reference := range manifest.Blobs {252+ if _, held := reader.blobs[reference.Name]; held || reader.entries[reference.Name] == nil {253+ return fmt.Errorf("blob reference %s is duplicate or missing", reference.Name)254+ }255+ reader.blobs[reference.Name] = reference256+ }257+ return nil258+}259+260+// Table decodes and integrity-checks one named table.261+func (reader *Reader) Table(name string) (Table, error) {262+ reference, held := reader.tables[name]263+ if !held {264+ return Table{}, fmt.Errorf("bundle has no table %s", name)265+ }266+ data, err := readReferencedEntry(reader.entries[name], reference.Length, reference.Hash)267+ if err != nil {268+ return Table{}, err269+ }270+ block, err := DecodeBlock(data)271+ if err != nil {272+ return Table{}, fmt.Errorf("decode %s: %w", name, err)273+ }274+ schemaHash, err := reader.FileSchema.Hash()275+ if err != nil {276+ return Table{}, err277+ }278+ if block.SchemaHash != schemaHash || block.Table.TypeID != reference.RootType {279+ return Table{}, fmt.Errorf("table %s disagrees with its schema or root type", name)280+ }281+ fileType, held := reader.FileSchema.Type(block.Table.TypeID)282+ if !held {283+ return Table{}, fmt.Errorf("schema has no root type for table %s", name)284+ }285+ for index := range block.Table.Columns {286+ field, held := fileType.Field(block.Table.Columns[index].Field.ID)287+ if !held || field.Kind != block.Table.Columns[index].Field.Kind {288+ return Table{}, fmt.Errorf("schema has no matching field for table %s", name)289+ }290+ block.Table.Columns[index].Field = field291+ }292+ for _, field := range fileType.Fields {293+ if field.Optional {294+ continue295+ }296+ if !tableHasColumn(block.Table, field.ID) {297+ return Table{}, fmt.Errorf("table %s omits required field %s", name, field.ID)298+ }299+ }300+ return block.Table, nil301+}302+303+func tableHasColumn(table Table, fieldID ID) bool {304+ for _, column := range table.Columns {305+ if column.Field.ID == fieldID {306+ return true307+ }308+ }309+ return false310+}311+312+func (reader *Reader) blob(name string) ([]byte, error) {313+ reference, held := reader.blobs[name]314+ if !held {315+ return nil, fmt.Errorf("bundle has no blob %s", name)316+ }317+ return readReferencedEntry(reader.entries[name], reference.Length, reference.Hash)318+}319+320+func readLimitedEntry(entries map[string]*zip.File, name string, maximum int64) ([]byte, error) {321+ entry := entries[name]322+ if entry == nil || int64(entry.UncompressedSize64) > maximum {323+ return nil, fmt.Errorf("entry %s is missing or implausibly large", name)324+ }325+ return readEntry(entry)326+}327+328+func readReferencedEntry(entry *zip.File, length int64, hash string) ([]byte, error) {329+ if entry == nil || int64(entry.UncompressedSize64) != length {330+ return nil, fmt.Errorf("referenced entry length does not match")331+ }332+ data, err := readEntry(entry)333+ if err != nil {334+ return nil, err335+ }336+ digest := sha256.Sum256(data)337+ if hex.EncodeToString(digest[:]) != hash {338+ return nil, fmt.Errorf("referenced entry hash does not match")339+ }340+ return data, nil341+}342+343+func readEntry(entry *zip.File) ([]byte, error) {344+ source, err := entry.Open()345+ if err != nil {346+ return nil, fmt.Errorf("open %s: %w", entry.Name, err)347+ }348+ defer source.Close()349+ data, err := io.ReadAll(source)350+ if err != nil {351+ return nil, fmt.Errorf("read %s: %w", entry.Name, err)352+ }353+ return data, nil354+}355+356+func validEntryName(name string) error {357+ if name == "" || strings.HasPrefix(name, "/") || strings.HasSuffix(name, "/") {358+ return fmt.Errorf("entry name %q is not relative", name)359+ }360+ for segment := range strings.SplitSeq(name, "/") {361+ if segment == "" || segment == "." || segment == ".." {362+ return fmt.Errorf("entry name %q climbs or stutters", name)363+ }364+ }365+ return nil366+}@@ -0,0 +1,100 @@
1+package vnext2+3+import (4+ "archive/zip"5+ "bytes"6+ "testing"7+)8+9+func TestBundleIsDeterministicAndKeepsPackedDataStored(t *testing.T) {10+ t.Parallel()11+12+ bundle, err := Compile(minimalVolume())13+ if err != nil {14+ t.Fatalf("compile: %v", err)15+ }16+ var first, second bytes.Buffer17+ if err := Write(&first, bundle); err != nil {18+ t.Fatalf("write first: %v", err)19+ }20+ if err := Write(&second, bundle); err != nil {21+ t.Fatalf("write second: %v", err)22+ }23+ if !bytes.Equal(first.Bytes(), second.Bytes()) {24+ t.Fatal("same semantic volume produced different archive bytes")25+ }26+27+ archive, err := zip.NewReader(bytes.NewReader(first.Bytes()), int64(first.Len()))28+ if err != nil {29+ t.Fatalf("open zip: %v", err)30+ }31+ if archive.File[0].Name != manifestName {32+ t.Fatalf("first entry = %s, want %s", archive.File[0].Name, manifestName)33+ }34+ for _, entry := range archive.File {35+ if (bytes.HasSuffix([]byte(entry.Name), []byte(".pack")) || bytes.HasPrefix([]byte(entry.Name), []byte("assets/"))) && entry.Method != zip.Store {36+ t.Fatalf("range-readable entry %s uses compression method %d", entry.Name, entry.Method)37+ }38+ }39+}40+41+func TestNewerApplicationReadsOlderFileSchema(t *testing.T) {42+ t.Parallel()43+44+ bundle, err := Compile(minimalVolume())45+ if err != nil {46+ t.Fatalf("compile: %v", err)47+ }48+ var archive bytes.Buffer49+ if err := Write(&archive, bundle); err != nil {50+ t.Fatalf("write: %v", err)51+ }52+ newField := Field{ID: IDFromName("com.example.future", "feature.score"), Name: "score", Kind: KindFloat64, Optional: true}53+ newer, err := UnionSchemas(StandardSchema(), Schema{Types: []Type{{54+ ID: CoreID("type.feature"), Name: "feature", Fields: []Field{newField},55+ }}})56+ if err != nil {57+ t.Fatalf("make newer application schema: %v", err)58+ }59+ reader, err := Open(bytes.NewReader(archive.Bytes()), int64(archive.Len()), newer)60+ if err != nil {61+ t.Fatalf("newer application open older file: %v", err)62+ }63+ featureType, ok := reader.RuntimeSchema.Type(CoreID("type.feature"))64+ if !ok {65+ t.Fatal("runtime schema lost Feature")66+ }67+ if _, ok := featureType.Field(newField.ID); !ok {68+ t.Fatal("runtime schema lost application-only field")69+ }70+ if _, err := reader.Volume(); err != nil {71+ t.Fatalf("restore older volume: %v", err)72+ }73+}74+75+func TestBlockRejectsCorruptColumn(t *testing.T) {76+ t.Parallel()77+78+ field := Field{ID: IDFromName("com.example", "name"), Name: "name", Kind: KindString}79+ table := Table{TypeID: IDFromName("com.example", "type"), Rows: 1, Columns: []Column{{Field: field, Values: []Value{StringValue("atlas")}}}}80+ data, err := EncodeBlock([32]byte{1}, table)81+ if err != nil {82+ t.Fatalf("encode: %v", err)83+ }84+ data[len(data)-1] ^= 0xff85+ if _, err := DecodeBlock(data); err == nil {86+ t.Fatal("decoder accepted a corrupt column")87+ }88+}89+90+func minimalVolume() Volume {91+ return Volume{92+ ID: "minimal", Title: "Minimal",93+ Worlds: []World{{94+ ID: "world", Title: "World",95+ CoordinateSpace: CoordinateSpace{ID: "space", Kind: "synthetic", Unit: "unit", Definition: "atlas:plane", Extent: [4]float64{0, 0, 1, 1}},96+ Presentation: Presentation{ID: "default", Title: "Default"},97+ }},98+ Assets: []Asset{{ID: "empty", MediaType: "application/octet-stream"}},99+ }100+}@@ -0,0 +1,513 @@
1+package vnext2+3+import (4+ "encoding/binary"5+ "fmt"6+ "math"7+)8+9+type setLocation struct {10+ world int11+ set int12+}13+14+// Volume reconstructs the semantic graph from typed tables. No presentation15+// policy is applied here.16+func (reader *Reader) Volume() (Volume, error) {17+ tables := make(map[string]Table, len(reader.tables))18+ for _, name := range tableNames() {19+ table, err := reader.Table(name)20+ if err != nil {21+ return Volume{}, err22+ }23+ tables[name] = table24+ }25+ return decodeVolume(reader, tables)26+}27+28+func decodeVolume(reader *Reader, tables map[string]Table) (Volume, error) {29+ root := tables[volumeTableName]30+ if root.Rows != 1 {31+ return Volume{}, fmt.Errorf("volume table has %d rows", root.Rows)32+ }33+ volume := Volume{34+ ID: requiredString(root, 0, "volume.id"),35+ Title: requiredString(root, 0, "volume.title"),36+ }37+ worlds, worldIndex, err := decodeWorlds(tables)38+ if err != nil {39+ return Volume{}, err40+ }41+ volume.Worlds = worlds42+ sets, err := decodeFeatureSets(&volume, tables, worldIndex)43+ if err != nil {44+ return Volume{}, err45+ }46+ features, err := decodeFeatures(&volume, tables, sets)47+ if err != nil {48+ return Volume{}, err49+ }50+ if err := decodeEvidence(&volume, tables, features); err != nil {51+ return Volume{}, err52+ }53+ if err := decodeWorldDetails(&volume, tables, worldIndex); err != nil {54+ return Volume{}, err55+ }56+ assets, err := decodeAssets(reader, tables[assetTableName])57+ if err != nil {58+ return Volume{}, err59+ }60+ volume.Assets = assets61+ if err := validateVolume(volume); err != nil {62+ return Volume{}, fmt.Errorf("decoded volume: %w", err)63+ }64+ return volume, nil65+}66+67+func decodeWorlds(tables map[string]Table) ([]World, map[string]int, error) {68+ table := tables[worldTableName]69+ worlds := make([]World, table.Rows)70+ index := make(map[string]int, table.Rows)71+ for row := 0; row < table.Rows; row++ {72+ worlds[row] = World{73+ ID: requiredString(table, row, "world.id"),74+ Title: requiredString(table, row, "world.title"),75+ CoordinateSpace: CoordinateSpace{ID: requiredString(table, row, "world.coordinateSpace")},76+ Claims: extensionProperties(table, row, "world"),77+ }78+ index[worlds[row].ID] = row79+ }80+ coordinates := tables[coordinateTableName]81+ for row := 0; row < coordinates.Rows; row++ {82+ worldID := requiredString(coordinates, row, "coordinate.world")83+ at, held := index[worldID]84+ if !held {85+ return nil, nil, fmt.Errorf("coordinate space refers to unknown world %s", worldID)86+ }87+ extent, err := decodeExtent(requiredBytes(coordinates, row, "coordinate.extent"))88+ if err != nil {89+ return nil, nil, err90+ }91+ worlds[at].CoordinateSpace = CoordinateSpace{92+ ID: requiredString(coordinates, row, "coordinate.id"),93+ Kind: requiredString(coordinates, row, "coordinate.kind"),94+ Unit: requiredString(coordinates, row, "coordinate.unit"),95+ Definition: requiredString(coordinates, row, "coordinate.definition"),96+ Extent: extent,97+ SourceZoom: requiredInt64(coordinates, row, "coordinate.sourceZoom"),98+ FirstTile: requiredInt64(coordinates, row, "coordinate.firstTile"),99+ TileSize: requiredInt64(coordinates, row, "coordinate.tileSize"),100+ Size: requiredInt64(coordinates, row, "coordinate.size"),101+ }102+ }103+ return worlds, index, nil104+}105+106+func decodeFeatureSets(volume *Volume, tables map[string]Table, worlds map[string]int) (map[string]setLocation, error) {107+ table := tables[featureSetTableName]108+ sets := make(map[string]setLocation, table.Rows)109+ for row := 0; row < table.Rows; row++ {110+ worldID := requiredString(table, row, "featureSet.world")111+ world, held := worlds[worldID]112+ if !held {113+ return nil, fmt.Errorf("feature set refers to unknown world %s", worldID)114+ }115+ set := FeatureSet{116+ ID: requiredString(table, row, "featureSet.id"),117+ Title: requiredString(table, row, "featureSet.title"),118+ SemanticType: requiredString(table, row, "featureSet.semanticType"),119+ Claims: extensionProperties(table, row, "featureSet"),120+ }121+ volume.Worlds[world].FeatureSets = append(volume.Worlds[world].FeatureSets, set)122+ sets[set.ID] = setLocation{world: world, set: len(volume.Worlds[world].FeatureSets) - 1}123+ }124+ definitions := tables[propertyDefinitionTableName]125+ for row := 0; row < definitions.Rows; row++ {126+ setID := requiredString(definitions, row, "propertyDefinition.featureSet")127+ location, held := sets[setID]128+ if !held {129+ return nil, fmt.Errorf("property definition refers to unknown feature set %s", setID)130+ }131+ field := Field{132+ ID: requiredID(definitions, row, "propertyDefinition.field"),133+ Name: requiredString(definitions, row, "propertyDefinition.name"),134+ Kind: Kind(requiredInt64(definitions, row, "propertyDefinition.kind")),135+ Optional: requiredBool(definitions, row, "propertyDefinition.optional"),136+ }137+ set := &volume.Worlds[location.world].FeatureSets[location.set]138+ set.Properties = append(set.Properties, field)139+ }140+ return sets, nil141+}142+143+func decodeFeatures(volume *Volume, tables map[string]Table, sets map[string]setLocation) (map[string]setLocation, error) {144+ table := tables[featureTableName]145+ features := make(map[string]setLocation, table.Rows)146+ core, _ := StandardSchema().Type(CoreID("type.feature"))147+ for row := 0; row < table.Rows; row++ {148+ setID := requiredString(table, row, "feature.featureSet")149+ location, held := sets[setID]150+ if !held {151+ return nil, fmt.Errorf("feature refers to unknown feature set %s", setID)152+ }153+ geometry, err := decodeGeometry(requiredBytes(table, row, "feature.geometry"))154+ if err != nil {155+ return nil, err156+ }157+ feature := Feature{158+ ID: requiredString(table, row, "feature.id"), Title: requiredString(table, row, "feature.title"),159+ Subtitle: optionalStringAt(table, row, "feature.subtitle"), Description: optionalStringAt(table, row, "feature.description"),160+ Shard: requiredInt64(table, row, "feature.shard"), Geometry: geometry,161+ }162+ if center := optionalBytesAt(table, row, "feature.center"); center != nil {163+ position, err := decodePosition(center)164+ if err != nil {165+ return nil, err166+ }167+ feature.Center = &position168+ }169+ for _, column := range table.Columns {170+ if _, standard := core.Field(column.Field.ID); standard || column.Values[row].Kind == KindInvalid {171+ continue172+ }173+ feature.Properties = append(feature.Properties, Property{FieldID: column.Field.ID, Field: column.Field, Value: column.Values[row]})174+ }175+ set := &volume.Worlds[location.world].FeatureSets[location.set]176+ set.Features = append(set.Features, feature)177+ features[feature.ID] = setLocation{world: location.world, set: location.set}178+ }179+ return features, nil180+}181+182+func decodeEvidence(volume *Volume, tables map[string]Table, features map[string]setLocation) error {183+ for _, spec := range []struct {184+ name string185+ decode func(*Feature, Table, int)186+ }{{relationshipTableName, decodeRelationship}, {provenanceTableName, decodeProvenance}} {187+ table := tables[spec.name]188+ for row := 0; row < table.Rows; row++ {189+ featureID := requiredString(table, row, tablePrefix(spec.name)+".feature")190+ location, held := features[featureID]191+ if !held {192+ return fmt.Errorf("evidence refers to unknown feature %s", featureID)193+ }194+ feature := findFeature(volume, location, featureID)195+ spec.decode(feature, table, row)196+ }197+ }198+ return nil199+}200+201+func findFeature(volume *Volume, location setLocation, id string) *Feature {202+ features := volume.Worlds[location.world].FeatureSets[location.set].Features203+ for index := range features {204+ if features[index].ID == id {205+ return &features[index]206+ }207+ }208+ return nil209+}210+211+func decodeRelationship(feature *Feature, table Table, row int) {212+ feature.Relationships = append(feature.Relationships, Relationship{213+ Predicate: requiredString(table, row, "relationship.predicate"),214+ Target: requiredString(table, row, "relationship.target"),215+ })216+}217+218+func decodeProvenance(feature *Feature, table Table, row int) {219+ feature.Provenance = append(feature.Provenance, Provenance{220+ Source: requiredString(table, row, "provenance.source"),221+ NativeID: requiredString(table, row, "provenance.nativeID"),222+ CapturedAt: requiredString(table, row, "provenance.capturedAt"),223+ })224+}225+226+func tablePrefix(name string) string {227+ if name == relationshipTableName {228+ return "relationship"229+ }230+ return "provenance"231+}232+233+func decodeWorldDetails(volume *Volume, tables map[string]Table, worlds map[string]int) error {234+ if err := decodeRasters(volume, tables, worlds); err != nil {235+ return err236+ }237+ presentations, err := decodePresentations(volume, tables[presentationTableName], worlds)238+ if err != nil {239+ return err240+ }241+ if err := decodeStyles(volume, tables[styleTableName], presentations); err != nil {242+ return err243+ }244+ if err := decodeLayers(volume, tables[layerTableName], presentations); err != nil {245+ return err246+ }247+ return decodeLegend(volume, tables[legendTableName], presentations)248+}249+250+func decodeRasters(volume *Volume, tables map[string]Table, worlds map[string]int) error {251+ table := tables[rasterTableName]252+ rasterLocations := make(map[string][2]int, table.Rows)253+ for row := 0; row < table.Rows; row++ {254+ worldID := requiredString(table, row, "rasterPyramid.world")255+ world, held := worlds[worldID]256+ if !held {257+ return fmt.Errorf("raster pyramid refers to unknown world %s", worldID)258+ }259+ raster := RasterPyramid{260+ ID: requiredString(table, row, "rasterPyramid.id"), Name: requiredString(table, row, "rasterPyramid.name"), Codec: requiredString(table, row, "rasterPyramid.codec"),261+ TileSize: requiredInt64(table, row, "rasterPyramid.tileSize"), MinZoom: requiredInt64(table, row, "rasterPyramid.minZoom"),262+ MaxZoom: requiredInt64(table, row, "rasterPyramid.maxZoom"), FullZoom: requiredInt64(table, row, "rasterPyramid.fullZoom"),263+ SourceZoom: requiredInt64(table, row, "rasterPyramid.sourceZoom"), Template: requiredString(table, row, "rasterPyramid.template"),264+ Interpolate: requiredBool(table, row, "rasterPyramid.interpolate"), Background: optionalStringAt(table, row, "rasterPyramid.background"),265+ Shard: requiredInt64(table, row, "rasterPyramid.shard"),266+ }267+ var err error268+ if data := optionalBytesAt(table, row, "rasterPyramid.bounds"); data != nil {269+ raster.Bounds, err = decodeRect(data)270+ if err != nil {271+ return err272+ }273+ }274+ if data := optionalBytesAt(table, row, "rasterPyramid.surface"); data != nil {275+ raster.Surface, err = decodeRect(data)276+ if err != nil {277+ return err278+ }279+ }280+ volume.Worlds[world].RasterPyramids = append(volume.Worlds[world].RasterPyramids, raster)281+ rasterLocations[raster.ID] = [2]int{world, len(volume.Worlds[world].RasterPyramids) - 1}282+ }283+ formats := tables[rasterFormatTableName]284+ for row := 0; row < formats.Rows; row++ {285+ location, held := rasterLocations[requiredString(formats, row, "rasterFormat.raster")]286+ if !held {287+ return fmt.Errorf("raster format refers to unknown raster")288+ }289+ raster := &volume.Worlds[location[0]].RasterPyramids[location[1]]290+ raster.Formats = append(raster.Formats, requiredString(formats, row, "rasterFormat.format"))291+ }292+ coverage := tables[rasterCoverageTableName]293+ for row := 0; row < coverage.Rows; row++ {294+ location, held := rasterLocations[requiredString(coverage, row, "rasterCoverage.raster")]295+ if !held {296+ return fmt.Errorf("raster coverage refers to unknown raster")297+ }298+ raster := &volume.Worlds[location[0]].RasterPyramids[location[1]]299+ raster.Coverage = append(raster.Coverage, RasterCoverage{300+ Zoom: requiredInt64(coverage, row, "rasterCoverage.zoom"), X: requiredInt64(coverage, row, "rasterCoverage.x"),301+ Y: requiredInt64(coverage, row, "rasterCoverage.y"), W: requiredInt64(coverage, row, "rasterCoverage.w"),302+ H: requiredInt64(coverage, row, "rasterCoverage.h"), Bits: requiredBytes(coverage, row, "rasterCoverage.bits"),303+ })304+ }305+ return nil306+}307+308+func decodePresentations(volume *Volume, table Table, worlds map[string]int) (map[string]int, error) {309+ presentations := make(map[string]int, table.Rows)310+ for row := 0; row < table.Rows; row++ {311+ worldID := requiredString(table, row, "presentation.world")312+ world, held := worlds[worldID]313+ if !held {314+ return nil, fmt.Errorf("presentation refers to unknown world %s", worldID)315+ }316+ presentation := Presentation{ID: requiredString(table, row, "presentation.id"), Title: requiredString(table, row, "presentation.title")}317+ volume.Worlds[world].Presentation = presentation318+ presentations[presentation.ID] = world319+ }320+ return presentations, nil321+}322+323+func decodeStyles(volume *Volume, table Table, presentations map[string]int) error {324+ for row := 0; row < table.Rows; row++ {325+ presentationID := requiredString(table, row, "style.presentation")326+ world, held := presentations[presentationID]327+ if !held {328+ return fmt.Errorf("style refers to unknown presentation %s", presentationID)329+ }330+ volume.Worlds[world].Presentation.Styles = append(volume.Worlds[world].Presentation.Styles, Style{331+ ID: requiredString(table, row, "style.id"), Symbol: optionalStringAt(table, row, "style.symbol"),332+ Icon: optionalStringAt(table, row, "style.icon"), IconAsset: optionalStringAt(table, row, "style.iconAsset"),333+ IconPicture: requiredBool(table, row, "style.iconPicture"), RenderAs: optionalStringAt(table, row, "style.renderAs"),334+ Stroke: optionalStringAt(table, row, "style.stroke"), Fill: optionalStringAt(table, row, "style.fill"),335+ })336+ }337+ return nil338+}339+340+func decodeLayers(volume *Volume, table Table, presentations map[string]int) error {341+ for row := 0; row < table.Rows; row++ {342+ presentationID := requiredString(table, row, "layer.presentation")343+ world, held := presentations[presentationID]344+ if !held {345+ return fmt.Errorf("layer refers to unknown presentation %s", presentationID)346+ }347+ volume.Worlds[world].Presentation.Layers = append(volume.Worlds[world].Presentation.Layers, Layer{348+ ID: requiredString(table, row, "layer.id"), FeatureSet: requiredString(table, row, "layer.featureSet"),349+ Style: requiredString(table, row, "layer.style"), Order: requiredInt64(table, row, "layer.order"),350+ Group: optionalStringAt(table, row, "layer.group"), LabelPolicy: optionalStringAt(table, row, "layer.labelPolicy"),351+ Visible: requiredBool(table, row, "layer.visible"),352+ MinZoom: requiredInt64(table, row, "layer.minZoom"), MaxZoom: requiredInt64(table, row, "layer.maxZoom"),353+ })354+ }355+ return nil356+}357+358+func decodeLegend(volume *Volume, table Table, presentations map[string]int) error {359+ for row := 0; row < table.Rows; row++ {360+ presentationID := requiredString(table, row, "legend.presentation")361+ world, held := presentations[presentationID]362+ if !held {363+ return fmt.Errorf("legend refers to unknown presentation %s", presentationID)364+ }365+ volume.Worlds[world].Presentation.Legend = append(volume.Worlds[world].Presentation.Legend, LegendEntry{366+ Layer: requiredString(table, row, "legend.layer"), Label: requiredString(table, row, "legend.label"),367+ Order: requiredInt64(table, row, "legend.order"),368+ })369+ }370+ return nil371+}372+373+func decodeAssets(reader *Reader, table Table) ([]Asset, error) {374+ assets := make([]Asset, table.Rows)375+ for row := 0; row < table.Rows; row++ {376+ path := requiredString(table, row, "asset.path")377+ data, err := reader.blob(path)378+ if err != nil {379+ return nil, err380+ }381+ assets[row] = Asset{382+ ID: requiredString(table, row, "asset.id"), MediaType: requiredString(table, row, "asset.mediaType"),383+ Data: data, Provenance: optionalStringAt(table, row, "asset.provenance"),384+ }385+ }386+ return assets, nil387+}388+389+func requiredValue(table Table, row int, name string) Value {390+ value, held := table.Value(CoreID(name), row)391+ if !held || value.Kind == KindInvalid {392+ return Value{}393+ }394+ return value395+}396+397+func requiredString(table Table, row int, name string) string {398+ return requiredValue(table, row, name).String399+}400+func requiredInt64(table Table, row int, name string) int64 {401+ return requiredValue(table, row, name).Int64402+}403+func requiredBool(table Table, row int, name string) bool {404+ return requiredValue(table, row, name).Bool405+}406+func requiredID(table Table, row int, name string) ID { return requiredValue(table, row, name).ID }407+func requiredBytes(table Table, row int, name string) []byte {408+ return requiredValue(table, row, name).Bytes409+}410+411+func optionalStringAt(table Table, row int, name string) string {412+ value, _ := table.Value(CoreID(name), row)413+ return value.String414+}415+416+func optionalBytesAt(table Table, row int, name string) []byte {417+ value, held := table.Value(CoreID(name), row)418+ if !held || value.Kind == KindInvalid {419+ return nil420+ }421+ return value.Bytes422+}423+424+func extensionProperties(table Table, row int, typeName string) []Property {425+ core, _ := StandardSchema().Type(CoreID("type." + typeName))426+ var properties []Property427+ for _, column := range table.Columns {428+ if _, standard := core.Field(column.Field.ID); standard || column.Values[row].Kind == KindInvalid {429+ continue430+ }431+ properties = append(properties, Property{FieldID: column.Field.ID, Field: column.Field, Value: column.Values[row]})432+ }433+ return properties434+}435+436+func decodeExtent(data []byte) ([4]float64, error) {437+ var extent [4]float64438+ if len(data) != len(extent)*8 {439+ return extent, fmt.Errorf("coordinate extent has %d bytes", len(data))440+ }441+ for index := range extent {442+ extent[index] = math.Float64frombits(binary.LittleEndian.Uint64(data[index*8:]))443+ }444+ return extent, nil445+}446+447+func decodeGeometry(data []byte) (Geometry, error) {448+ if len(data) < 5 {449+ return Geometry{}, fmt.Errorf("geometry is truncated")450+ }451+ cursor := 1452+ readCount := func() (int, error) {453+ if cursor+4 > len(data) {454+ return 0, fmt.Errorf("geometry is truncated")455+ }456+ count := int(binary.LittleEndian.Uint32(data[cursor : cursor+4]))457+ cursor += 4458+ return count, nil459+ }460+ parts, err := readCount()461+ if err != nil {462+ return Geometry{}, err463+ }464+ geometry := Geometry{Kind: GeometryKind(data[0]), Parts: make([]GeometryPart, parts)}465+ for part := range geometry.Parts {466+ rings, err := readCount()467+ if err != nil {468+ return Geometry{}, err469+ }470+ geometry.Parts[part].Rings = make([][]Position, rings)471+ for ring := range geometry.Parts[part].Rings {472+ positions, err := readCount()473+ if err != nil {474+ return Geometry{}, err475+ }476+ if positions > (len(data)-cursor)/16 {477+ return Geometry{}, fmt.Errorf("geometry position payload is invalid")478+ }479+ geometry.Parts[part].Rings[ring] = make([]Position, positions)480+ for position := range geometry.Parts[part].Rings[ring] {481+ geometry.Parts[part].Rings[ring][position][0] = math.Float64frombits(binary.LittleEndian.Uint64(data[cursor:]))482+ cursor += 8483+ geometry.Parts[part].Rings[ring][position][1] = math.Float64frombits(binary.LittleEndian.Uint64(data[cursor:]))484+ cursor += 8485+ }486+ }487+ }488+ if cursor != len(data) {489+ return Geometry{}, fmt.Errorf("geometry has trailing bytes")490+ }491+ if _, err := encodeGeometry(geometry); err != nil {492+ return Geometry{}, err493+ }494+ return geometry, nil495+}496+497+func decodePosition(data []byte) (Position, error) {498+ if len(data) != 16 {499+ return Position{}, fmt.Errorf("position has %d bytes", len(data))500+ }501+ return Position{502+ math.Float64frombits(binary.LittleEndian.Uint64(data)),503+ math.Float64frombits(binary.LittleEndian.Uint64(data[8:])),504+ }, nil505+}506+507+func decodeRect(data []byte) (*RasterRect, error) {508+ extent, err := decodeExtent(data)509+ if err != nil {510+ return nil, err511+ }512+ return &RasterRect{X: extent[0], Y: extent[1], Width: extent[2], Height: extent[3]}, nil513+}@@ -0,0 +1,66 @@
1+// Package vnext implements the schema-evolved Atlas container beside the2+// frozen version 3 format. Its compatibility boundary is field identity and3+// wire kind, not a document-wide schema version.4+package vnext5+6+import (7+ "crypto/sha256"8+ "encoding/hex"9+ "encoding/json"10+ "fmt"11+ "strings"12+)13+14+// ID is the globally unique identity of a type or field. Blocks use compact15+// local directory positions after naming each field once with this full ID.16+type ID [16]byte17+18+// IDFromName deterministically allocates an ID inside a namespace. Publishers19+// can use a domain they control as the namespace without a central registry.20+func IDFromName(namespace, name string) ID {21+ digest := sha256.Sum256([]byte(namespace + "\x00" + name))22+ var id ID23+ copy(id[:], digest[:len(id)])24+ // Mark the value as a name-derived UUID and set the RFC 4122 variant.25+ id[6] = id[6]&0x0f | 0x5026+ id[8] = id[8]&0x3f | 0x8027+ return id28+}29+30+// CoreID returns an identity in Atlas's own schema namespace.31+func CoreID(name string) ID { return IDFromName("dev.atlas.core", name) }32+33+// ParseID accepts the customary dashed form or 32 hexadecimal digits.34+func ParseID(value string) (ID, error) {35+ var id ID36+ plain := strings.ReplaceAll(value, "-", "")37+ if len(plain) != hex.EncodedLen(len(id)) {38+ return id, fmt.Errorf("ID %q is not 128 bits", value)39+ }40+ if _, err := hex.Decode(id[:], []byte(plain)); err != nil {41+ return ID{}, fmt.Errorf("decode ID %q: %w", value, err)42+ }43+ return id, nil44+}45+46+func (id ID) String() string {47+ plain := hex.EncodeToString(id[:])48+ return plain[:8] + "-" + plain[8:12] + "-" + plain[12:16] + "-" + plain[16:20] + "-" + plain[20:]49+}50+51+func (id ID) isZero() bool { return id == ID{} }52+53+func (id ID) MarshalJSON() ([]byte, error) { return json.Marshal(id.String()) }54+55+func (id *ID) UnmarshalJSON(data []byte) error {56+ var value string57+ if err := json.Unmarshal(data, &value); err != nil {58+ return fmt.Errorf("decode ID string: %w", err)59+ }60+ parsed, err := ParseID(value)61+ if err != nil {62+ return err63+ }64+ *id = parsed65+ return nil66+}@@ -0,0 +1,499 @@
1+package vnext2+3+import (4+ "encoding/base64"5+ "encoding/json"6+ "fmt"7+ "path"8+ "sort"9+ "strconv"10+ "strings"11+12+ "github.com/FelineStateMachine/atlas/format/bundle"13+)14+15+type v3World struct {16+ Attrs map[string]string `json:"attrs"`17+ Grid *v3Grid `json:"grid"`18+ Lenses []v3Lens `json:"lenses"`19+ Collections []v3Collection `json:"collections"`20+ Merged []v3Merge `json:"merged"`21+}22+23+type v3Grid struct {24+ SourceZoom *int `json:"sourceZoom"`25+ FirstTile *int `json:"firstTile"`26+ TileSize *int `json:"tileSize"`27+ Size *int `json:"size"`28+}29+30+type v3Lens struct {31+ Name string `json:"name"`32+ Tiles string `json:"tiles"`33+ MinZoom int64 `json:"minZoom"`34+ MaxZoom int64 `json:"maxZoom"`35+ FullZoom int64 `json:"fullZoom"`36+ SourceZoom int64 `json:"sourceZoom"`37+ Formats []string `json:"formats"`38+ Bounds *RasterRect `json:"bounds"`39+ Surface *RasterRect `json:"surface"`40+ Interpolate bool `json:"interpolate"`41+ Background string `json:"background"`42+ Shard int64 `json:"shard"`43+ Coverage map[string]v3Coverage `json:"coverage"`44+}45+46+type v3Coverage struct {47+ X int64 `json:"x"`48+ Y int64 `json:"y"`49+ W int64 `json:"w"`50+ H int64 `json:"h"`51+ Bits string `json:"bits"`52+}53+54+type v3Collection struct {55+ ID int64 `json:"id"`56+ Title string `json:"title"`57+ Kind string `json:"kind"`58+ Group string `json:"group"`59+ Icon string `json:"icon"`60+ IconAsset string `json:"iconAsset"`61+ IconPicture bool `json:"iconPicture"`62+ Color string `json:"color"`63+ IconColor string `json:"iconColor"`64+ Visible *bool `json:"visible"`65+ Attrs map[string]string `json:"attrs"`66+ Features []v3Feature `json:"features"`67+}68+69+type v3Feature struct {70+ ID int64 `json:"id"`71+ Title string `json:"title"`72+ Subtitle string `json:"subtitle"`73+ Parent *int64 `json:"parent"`74+ Center *bundle.Coordinate `json:"center"`75+ Shard int64 `json:"shard"`76+ Attrs map[string]string `json:"attrs"`77+ Geometry []v3Geometry `json:"geometry"`78+}79+80+type v3Geometry struct {81+ Type string `json:"type"`82+ Coordinates json.RawMessage `json:"coordinates"`83+}84+85+type v3Merge struct {86+ Slug string `json:"slug"`87+ Source string `json:"source"`88+ Origin bool `json:"origin"`89+}90+91+type v3Text struct {92+ Description string `json:"d"`93+ Links []v3Link `json:"l"`94+ Attrs map[string]string `json:"a"`95+}96+97+type v3Link struct {98+ Title string `json:"title"`99+ LocationID int64 `json:"locationId"`100+}101+102+// ImportV3 turns the frozen document-shaped format into the vNext semantic103+// graph. The old format remains a read-only ingress boundary after cutover.104+func ImportV3(reader *bundle.Reader) (Bundle, error) {105+ if err := reader.Validate(); err != nil {106+ return Bundle{}, fmt.Errorf("validate v3 bundle: %w", err)107+ }108+ volume, err := ImportV3Volume(reader.Manifest, reader.ReadEntry)109+ if err != nil {110+ return Bundle{}, err111+ }112+ assets, err := importV3Assets(reader)113+ if err != nil {114+ return Bundle{}, err115+ }116+ volume.Assets = assets117+ compiled, err := Compile(volume)118+ if err != nil {119+ return Bundle{}, fmt.Errorf("compile imported v3 bundle: %w", err)120+ }121+ for _, name := range reader.Names() {122+ if !strings.HasPrefix(name, bundle.TilesPrefix) {123+ continue124+ }125+ data, err := reader.ReadEntry(name)126+ if err != nil {127+ return Bundle{}, err128+ }129+ compiled.Blobs = append(compiled.Blobs, Blob{Name: name, Data: data})130+ }131+ sort.Slice(compiled.Blobs, func(i, j int) bool { return compiled.Blobs[i].Name < compiled.Blobs[j].Name })132+ return compiled, nil133+}134+135+// ImportV3Volume crosses the migration boundary without choosing a physical136+// vNext layout. Runtime adapters use this form to make the semantic graph137+// authoritative while opaque assets continue through their existing door.138+func ImportV3Volume(manifest bundle.Manifest, read func(string) ([]byte, error)) (Volume, error) {139+ volume := Volume{ID: manifest.Volume.Slug, Title: manifest.Volume.Title}140+ for _, entry := range manifest.Worlds {141+ world, err := importV3World(read, manifest, entry)142+ if err != nil {143+ return Volume{}, err144+ }145+ volume.Worlds = append(volume.Worlds, world)146+ }147+ return volume, nil148+}149+150+func importV3World(read func(string) ([]byte, error), manifest bundle.Manifest, entry bundle.WorldEntry) (World, error) {151+ detail, err := read(bundle.WorldEntryName(entry.Slug, bundle.WorldSuffix))152+ if err != nil {153+ return World{}, err154+ }155+ var source v3World156+ if err := json.Unmarshal(detail, &source); err != nil {157+ return World{}, fmt.Errorf("decode world %s: %w", entry.Slug, err)158+ }159+ textData, err := read(bundle.WorldEntryName(entry.Slug, bundle.TextSuffix))160+ if err != nil {161+ return World{}, err162+ }163+ var texts map[string]v3Text164+ if err := json.Unmarshal(textData, &texts); err != nil {165+ return World{}, fmt.Errorf("decode world %s text: %w", entry.Slug, err)166+ }167+ packed, err := read(bundle.WorldEntryName(entry.Slug, bundle.PackedSuffix))168+ if err != nil {169+ return World{}, err170+ }171+ locations, err := bundle.UnpackLocations(packed)172+ if err != nil {173+ return World{}, err174+ }175+176+ grid := manifest.TileGrid177+ if source.Grid != nil {178+ assignInt(&grid.SourceZoom, source.Grid.SourceZoom)179+ assignInt(&grid.FirstTile, source.Grid.FirstTile)180+ assignInt(&grid.TileSize, source.Grid.TileSize)181+ assignInt(&grid.Size, source.Grid.Size)182+ }183+ world := World{184+ ID: entry.Slug, Title: entry.Title,185+ CoordinateSpace: CoordinateSpace{186+ ID: entry.Slug + "/coordinates", Kind: "projected", Unit: "world-pixel", Definition: "atlas:v3/tile-grid",187+ Extent: [4]float64{0, 0, float64(grid.Size), float64(grid.Size)}, SourceZoom: int64(grid.SourceZoom),188+ FirstTile: int64(grid.FirstTile), TileSize: int64(grid.TileSize), Size: int64(grid.Size),189+ },190+ Claims: claimsFromV3("world", source.Attrs),191+ Presentation: Presentation{ID: entry.Slug + "/presentation/default", Title: "Default"},192+ }193+ origin := v3Origin(source.Merged)194+ for index, collection := range source.Collections {195+ set, err := importV3Collection(entry, collection, texts, origin)196+ if err != nil {197+ return World{}, fmt.Errorf("world %s collection %s: %w", entry.Slug, collection.Title, err)198+ }199+ world.FeatureSets = append(world.FeatureSets, set)200+ styleID := entry.Slug + "/style/" + strconv.FormatInt(collection.ID, 10)201+ layerID := entry.Slug + "/layer/" + strconv.FormatInt(collection.ID, 10)202+ color := collection.Color203+ if color == "" {204+ color = collection.IconColor205+ }206+ world.Presentation.Styles = append(world.Presentation.Styles, Style{207+ ID: styleID, Icon: collection.Icon, IconAsset: collection.IconAsset, IconPicture: collection.IconPicture,208+ RenderAs: collection.Attrs["atlas.render.as"], Fill: color, Stroke: color,209+ })210+ visible := collection.Visible == nil || *collection.Visible211+ world.Presentation.Layers = append(world.Presentation.Layers, Layer{212+ ID: layerID, FeatureSet: set.ID, Style: styleID, Group: collection.Group,213+ LabelPolicy: collection.Attrs["atlas.label.policy"], Visible: visible,214+ Order: int64(index), MinZoom: 0, MaxZoom: maxV3Zoom(source.Lenses),215+ })216+ world.Presentation.Legend = append(world.Presentation.Legend, LegendEntry{Layer: layerID, Label: collection.Title, Order: int64(index)})217+ }218+ for _, location := range locations {219+ owner := int(location.Owner)220+ if owner >= len(world.FeatureSets) {221+ return World{}, fmt.Errorf("world %s point owner %d is out of range", entry.Slug, owner)222+ }223+ text := texts[strconv.FormatInt(location.ID, 10)]224+ feature := Feature{225+ ID: v3FeatureID(entry.Slug, location.ID), Title: location.Title, Shard: location.Shard,226+ Geometry: Geometry{Kind: GeometryPoint, Parts: []GeometryPart{{Rings: [][]Position{{{location.Lng, location.Lat}}}}}},227+ Properties: propertiesFromV3(text.Attrs), Provenance: v3Provenance(origin, location.ID, entry.UpdatedAt),228+ Description: text.Description,229+ }230+ ensureV3Contract(&world.FeatureSets[owner], feature.Properties)231+ if location.Member != 0 {232+ feature.Relationships = append(feature.Relationships, Relationship{Predicate: "within", Target: v3FeatureID(entry.Slug, location.Member)})233+ }234+ feature.Relationships = append(feature.Relationships, relationshipsFromV3(entry.Slug, text.Links)...)235+ world.FeatureSets[owner].Features = append(world.FeatureSets[owner].Features, feature)236+ }237+ for index := range world.FeatureSets {238+ sort.Slice(world.FeatureSets[index].Properties, func(i, j int) bool {239+ return world.FeatureSets[index].Properties[i].ID.String() < world.FeatureSets[index].Properties[j].ID.String()240+ })241+ }242+ for index, lens := range source.Lenses {243+ raster, err := importV3Raster(entry.Slug, grid.TileSize, index, lens)244+ if err != nil {245+ return World{}, err246+ }247+ world.RasterPyramids = append(world.RasterPyramids, raster)248+ }249+ return world, nil250+}251+252+func importV3Collection(entry bundle.WorldEntry, source v3Collection, texts map[string]v3Text, origin string) (FeatureSet, error) {253+ set := FeatureSet{254+ ID: entry.Slug + "/set/" + strconv.FormatInt(source.ID, 10), Title: source.Title,255+ SemanticType: "geometry." + source.Kind, Claims: claimsFromV3("collection", source.Attrs),256+ }257+ contract := make(map[string]Field)258+ for _, feature := range source.Features {259+ for key := range feature.Attrs {260+ contract[key] = v3AttributeField("feature", key)261+ }262+ for key := range texts[strconv.FormatInt(feature.ID, 10)].Attrs {263+ contract[key] = v3AttributeField("feature", key)264+ }265+ }266+ for _, field := range contract {267+ set.Properties = append(set.Properties, field)268+ }269+ sort.Slice(set.Properties, func(i, j int) bool { return set.Properties[i].ID.String() < set.Properties[j].ID.String() })270+ for _, sourceFeature := range source.Features {271+ text := texts[strconv.FormatInt(sourceFeature.ID, 10)]272+ geometry, err := geometryFromV3(sourceFeature.Geometry)273+ if err != nil {274+ return FeatureSet{}, fmt.Errorf("feature %d geometry: %w", sourceFeature.ID, err)275+ }276+ feature := Feature{277+ ID: v3FeatureID(entry.Slug, sourceFeature.ID), Title: sourceFeature.Title, Subtitle: sourceFeature.Subtitle,278+ Description: text.Description, Shard: sourceFeature.Shard, Geometry: geometry,279+ Properties: propertiesFromV3(mergeV3Attrs(sourceFeature.Attrs, text.Attrs)),280+ Relationships: relationshipsFromV3(entry.Slug, text.Links), Provenance: v3Provenance(origin, sourceFeature.ID, entry.UpdatedAt),281+ }282+ if sourceFeature.Center != nil {283+ center := Position{sourceFeature.Center.Lng, sourceFeature.Center.Lat}284+ feature.Center = ¢er285+ }286+ if sourceFeature.Parent != nil {287+ feature.Relationships = append(feature.Relationships, Relationship{Predicate: "within", Target: v3FeatureID(entry.Slug, *sourceFeature.Parent)})288+ }289+ set.Features = append(set.Features, feature)290+ }291+ return set, nil292+}293+294+func importV3Raster(worldID string, tileSize, index int, lens v3Lens) (RasterPyramid, error) {295+ raster := RasterPyramid{296+ ID: worldID + "/raster/" + strconv.Itoa(index), Name: lens.Name, Codec: "raster-tile", TileSize: int64(tileSize),297+ MinZoom: lens.MinZoom, MaxZoom: lens.MaxZoom, FullZoom: lens.FullZoom, SourceZoom: lens.SourceZoom,298+ Template: bundle.TilesPrefix + lens.Tiles + "/{z}/{x}/{y}.{format}", Formats: lens.Formats,299+ Bounds: lens.Bounds, Surface: lens.Surface, Interpolate: lens.Interpolate, Background: lens.Background, Shard: lens.Shard,300+ }301+ levels := make([]int, 0, len(lens.Coverage))302+ for level := range lens.Coverage {303+ zoom, err := strconv.Atoi(level)304+ if err != nil {305+ return RasterPyramid{}, fmt.Errorf("raster %s has invalid coverage zoom %q", lens.Name, level)306+ }307+ levels = append(levels, zoom)308+ }309+ sort.Ints(levels)310+ for _, zoom := range levels {311+ coverage := lens.Coverage[strconv.Itoa(zoom)]312+ bits, err := base64.StdEncoding.DecodeString(coverage.Bits)313+ if err != nil {314+ return RasterPyramid{}, fmt.Errorf("decode raster %s coverage: %w", lens.Name, err)315+ }316+ raster.Coverage = append(raster.Coverage, RasterCoverage{Zoom: int64(zoom), X: coverage.X, Y: coverage.Y, W: coverage.W, H: coverage.H, Bits: bits})317+ }318+ return raster, nil319+}320+321+func geometryFromV3(source []v3Geometry) (Geometry, error) {322+ var out Geometry323+ for _, item := range source {324+ switch item.Type {325+ case "Point":326+ var point Position327+ if err := json.Unmarshal(item.Coordinates, &point); err != nil {328+ return Geometry{}, err329+ }330+ out.Kind, out.Parts = GeometryPoint, append(out.Parts, GeometryPart{Rings: [][]Position{{point}}})331+ case "LineString":332+ var line []Position333+ if err := json.Unmarshal(item.Coordinates, &line); err != nil {334+ return Geometry{}, err335+ }336+ out.Kind, out.Parts = GeometryLineString, append(out.Parts, GeometryPart{Rings: [][]Position{line}})337+ case "MultiLineString":338+ var lines [][]Position339+ if err := json.Unmarshal(item.Coordinates, &lines); err != nil {340+ return Geometry{}, err341+ }342+ out.Kind = GeometryLineString343+ for _, line := range lines {344+ out.Parts = append(out.Parts, GeometryPart{Rings: [][]Position{line}})345+ }346+ case "Polygon":347+ var rings [][]Position348+ if err := json.Unmarshal(item.Coordinates, &rings); err != nil {349+ return Geometry{}, err350+ }351+ out.Kind, out.Parts = GeometryPolygon, append(out.Parts, GeometryPart{Rings: rings})352+ case "MultiPolygon":353+ var polygons [][][]Position354+ if err := json.Unmarshal(item.Coordinates, &polygons); err != nil {355+ return Geometry{}, err356+ }357+ out.Kind = GeometryPolygon358+ for _, rings := range polygons {359+ out.Parts = append(out.Parts, GeometryPart{Rings: rings})360+ }361+ default:362+ return Geometry{}, fmt.Errorf("unsupported v3 geometry %q", item.Type)363+ }364+ }365+ return out, validateGeometry(out)366+}367+368+func importV3Assets(reader *bundle.Reader) ([]Asset, error) {369+ var assets []Asset370+ for _, name := range reader.Names() {371+ if !strings.HasPrefix(name, bundle.IconsPrefix) {372+ continue373+ }374+ data, err := reader.ReadEntry(name)375+ if err != nil {376+ return nil, err377+ }378+ assets = append(assets, Asset{ID: strings.TrimPrefix(name, bundle.IconsPrefix), MediaType: mediaTypeForV3(name), Data: data, Provenance: "atlas-bundle/v3"})379+ }380+ return assets, nil381+}382+383+func claimsFromV3(owner string, attrs map[string]string) []Property {384+ properties := propertiesFromV3Namespace(owner, attrs)385+ for index := range properties {386+ properties[index].Field.Optional = true387+ }388+ return properties389+}390+391+func propertiesFromV3(attrs map[string]string) []Property {392+ return propertiesFromV3Namespace("feature", attrs)393+}394+395+func propertiesFromV3Namespace(owner string, attrs map[string]string) []Property {396+ keys := make([]string, 0, len(attrs))397+ for key := range attrs {398+ keys = append(keys, key)399+ }400+ sort.Strings(keys)401+ out := make([]Property, 0, len(keys))402+ for _, key := range keys {403+ field := v3AttributeField(owner, key)404+ out = append(out, Property{FieldID: field.ID, Field: field, Value: StringValue(attrs[key])})405+ }406+ return out407+}408+409+func ensureV3Contract(set *FeatureSet, properties []Property) {410+ for _, property := range properties {411+ id := property.fieldID()412+ found := false413+ for _, field := range set.Properties {414+ if field.ID == id {415+ found = true416+ break417+ }418+ }419+ if !found {420+ set.Properties = append(set.Properties, property.Field)421+ }422+ }423+}424+425+func v3AttributeField(owner, key string) Field {426+ return Field{ID: IDFromName("dev.atlas.v3.attribute."+owner, key), Name: key, Kind: KindString, Optional: true}427+}428+429+func v3FeatureID(world string, id int64) string {430+ return world + "/feature/" + strconv.FormatInt(id, 10)431+}432+433+func v3Origin(merged []v3Merge) string {434+ for _, account := range merged {435+ if account.Origin && account.Source != "" {436+ return account.Source437+ }438+ }439+ for _, account := range merged {440+ if account.Source != "" {441+ return account.Source442+ }443+ }444+ return "atlas-bundle/v3"445+}446+447+func v3Provenance(source string, id int64, capturedAt string) []Provenance {448+ return []Provenance{{Source: source, NativeID: strconv.FormatInt(id, 10), CapturedAt: capturedAt}}449+}450+451+func relationshipsFromV3(world string, links []v3Link) []Relationship {452+ out := make([]Relationship, 0, len(links))453+ for _, link := range links {454+ out = append(out, Relationship{Predicate: "references", Target: v3FeatureID(world, link.LocationID)})455+ }456+ return out457+}458+459+func mergeV3Attrs(left, right map[string]string) map[string]string {460+ out := make(map[string]string, len(left)+len(right))461+ for key, value := range left {462+ out[key] = value463+ }464+ for key, value := range right {465+ out[key] = value466+ }467+ return out468+}469+470+func maxV3Zoom(lenses []v3Lens) int64 {471+ var maximum int64472+ for _, lens := range lenses {473+ if lens.MaxZoom > maximum {474+ maximum = lens.MaxZoom475+ }476+ }477+ return maximum478+}479+480+func assignInt(target *int, source *int) {481+ if source != nil {482+ *target = *source483+ }484+}485+486+func mediaTypeForV3(name string) string {487+ switch strings.ToLower(path.Ext(name)) {488+ case ".svg":489+ return "image/svg+xml"490+ case ".png":491+ return "image/png"492+ case ".jpg", ".jpeg":493+ return "image/jpeg"494+ case ".webp":495+ return "image/webp"496+ default:497+ return "application/octet-stream"498+ }499+}@@ -0,0 +1,125 @@
1+package vnext2+3+import (4+ "bytes"5+ "path/filepath"6+ "runtime"7+ "testing"8+9+ "github.com/FelineStateMachine/atlas/format/bundle"10+)11+12+func TestImportV3TurnsTheIncludedEarthIntoTheVNextGraph(t *testing.T) {13+ t.Parallel()14+15+ legacy, err := bundle.Open(includedEarth(t))16+ if err != nil {17+ t.Fatalf("open v3 Earth: %v", err)18+ }19+ defer legacy.Close()20+21+ imported, err := ImportV3(legacy)22+ if err != nil {23+ t.Fatalf("import v3 Earth: %v", err)24+ }25+ var archive bytes.Buffer26+ if err := Write(&archive, imported); err != nil {27+ t.Fatalf("write imported Earth: %v", err)28+ }29+ reader, err := Open(bytes.NewReader(archive.Bytes()), int64(archive.Len()), StandardSchema())30+ if err != nil {31+ t.Fatalf("open imported Earth: %v", err)32+ }33+ volume, err := reader.Volume()34+ if err != nil {35+ t.Fatalf("restore imported Earth: %v", err)36+ }37+38+ if volume.ID != "earth" || len(volume.Worlds) != 1 {39+ t.Fatalf("imported volume identity = %s, worlds = %d", volume.ID, len(volume.Worlds))40+ }41+ world := volume.Worlds[0]42+ if world.CoordinateSpace.SourceZoom != 5 || world.CoordinateSpace.Size != 8192 {43+ t.Fatalf("coordinate space = %#v", world.CoordinateSpace)44+ }45+ if len(world.RasterPyramids) != 1 || len(world.RasterPyramids[0].Formats) != 7 {46+ t.Fatalf("raster pyramids = %#v", world.RasterPyramids)47+ }48+ if got, want := featureCount(world), 379; got != want {49+ t.Fatalf("features = %d, want %d", got, want)50+ }51+ if len(world.Presentation.Layers) != len(world.FeatureSets) || len(world.Presentation.Legend) != len(world.FeatureSets) {52+ t.Fatalf("presentation did not split every v3 collection: %#v", world.Presentation)53+ }54+55+ countries := featureSetByTitle(world, "Countries")56+ if countries == nil || len(countries.Features) == 0 {57+ t.Fatal("Countries feature set was not imported")58+ }59+ fiji := featureByTitle(*countries, "Fiji")60+ if fiji == nil || fiji.Geometry.Kind != GeometryPolygon || len(fiji.Geometry.Parts) != 3 {61+ t.Fatalf("Fiji geometry = %#v", fiji)62+ }63+ if len(imported.Blobs) < 1000 {64+ t.Fatalf("only %d raster/asset blobs crossed the import", len(imported.Blobs))65+ }66+}67+68+func TestImportV3VolumeStopsAtTheSemanticBoundary(t *testing.T) {69+ t.Parallel()70+71+ legacy, err := bundle.Open(includedEarth(t))72+ if err != nil {73+ t.Fatalf("open v3 Earth: %v", err)74+ }75+ defer legacy.Close()76+77+ volume, err := ImportV3Volume(legacy.Manifest, legacy.ReadEntry)78+ if err != nil {79+ t.Fatalf("import semantic Earth: %v", err)80+ }81+ if volume.ID != "earth" || len(volume.Worlds) != 1 {82+ t.Fatalf("semantic volume = %#v", volume)83+ }84+ if got, want := featureCount(volume.Worlds[0]), 379; got != want {85+ t.Fatalf("semantic features = %d, want %d", got, want)86+ }87+ if len(volume.Assets) != 0 {88+ t.Fatalf("semantic-only import loaded %d assets", len(volume.Assets))89+ }90+}91+92+func includedEarth(t *testing.T) string {93+ t.Helper()94+ _, file, _, ok := runtime.Caller(0)95+ if !ok {96+ t.Fatal("locate importer test")97+ }98+ return filepath.Join(filepath.Dir(file), "..", "..", "included", "earth-20260803-76633cb323ec.atlas")99+}100+101+func featureCount(world World) int {102+ total := 0103+ for _, set := range world.FeatureSets {104+ total += len(set.Features)105+ }106+ return total107+}108+109+func featureSetByTitle(world World, title string) *FeatureSet {110+ for index := range world.FeatureSets {111+ if world.FeatureSets[index].Title == title {112+ return &world.FeatureSets[index]113+ }114+ }115+ return nil116+}117+118+func featureByTitle(set FeatureSet, title string) *Feature {119+ for index := range set.Features {120+ if set.Features[index].Title == title {121+ return &set.Features[index]122+ }123+ }124+ return nil125+}@@ -0,0 +1,319 @@
1+package vnext2+3+// Volume is the durable subject of one Atlas file.4+type Volume struct {5+ ID string6+ Title string7+ Worlds []World8+ Assets []Asset9+}10+11+// World is one coherent spatial environment or snapshot.12+type World struct {13+ ID string14+ Title string15+ CoordinateSpace CoordinateSpace16+ Claims []Property17+ FeatureSets []FeatureSet18+ RasterPyramids []RasterPyramid19+ Presentation Presentation20+}21+22+// CoordinateSpace is the authoritative offline frame for a world.23+type CoordinateSpace struct {24+ ID string25+ Kind string26+ Unit string27+ Definition string28+ Extent [4]float6429+ SourceZoom int6430+ FirstTile int6431+ TileSize int6432+ Size int6433+}34+35+// FeatureSet carries semantic membership and its typed property contract.36+type FeatureSet struct {37+ ID string38+ Title string39+ SemanticType string40+ Properties []Field41+ Claims []Property42+ Features []Feature43+}44+45+// Feature separates spatial facts, typed claims, relationships, and evidence.46+type Feature struct {47+ ID string48+ Title string49+ Subtitle string50+ Description string51+ Center *Position52+ Shard int6453+ Geometry Geometry54+ Properties []Property55+ Relationships []Relationship56+ Provenance []Provenance57+}58+59+// GeometryKind identifies the interpretation of a coordinate sequence.60+type GeometryKind uint861+62+const (63+ GeometryPoint GeometryKind = iota + 164+ GeometryLineString65+ GeometryPolygon66+)67+68+// Position is one x,y pair in a world's CoordinateSpace.69+type Position [2]float6470+71+// GeometryPart is one point, line, or polygon. Lines carry one ring; polygons72+// carry an exterior ring followed by any holes.73+type GeometryPart struct {74+ Rings [][]Position75+}76+77+// Geometry holds every part of a point, multi-line, or multi-polygon without78+// flattening away the boundaries between pieces and holes.79+type Geometry struct {80+ Kind GeometryKind81+ Parts []GeometryPart82+}83+84+// Property is one typed feature claim identified independently of its label.85+type Property struct {86+ FieldID ID87+ Field Field88+ Value Value89+}90+91+func (property Property) fieldID() ID {92+ if !property.FieldID.isZero() {93+ return property.FieldID94+ }95+ return property.Field.ID96+}97+98+// Relationship is a first-class edge to another entity.99+type Relationship struct {100+ Predicate string101+ Target string102+}103+104+// Provenance records where a claim or feature came from.105+type Provenance struct {106+ Source string107+ NativeID string108+ CapturedAt string109+}110+111+// RasterPyramid describes an offline tile matrix whose bytes remain opaque.112+type RasterPyramid struct {113+ ID string114+ Name string115+ Codec string116+ TileSize int64117+ MinZoom int64118+ MaxZoom int64119+ FullZoom int64120+ SourceZoom int64121+ Template string122+ Formats []string123+ Bounds *RasterRect124+ Surface *RasterRect125+ Interpolate bool126+ Background string127+ Shard int64128+ Coverage []RasterCoverage129+}130+131+type RasterRect struct {132+ X float64133+ Y float64134+ Width float64135+ Height float64136+}137+138+type RasterCoverage struct {139+ Zoom int64140+ X int64141+ Y int64142+ W int64143+ H int64144+ Bits []byte145+}146+147+// Presentation is replaceable render intent over semantic feature sets.148+type Presentation struct {149+ ID string150+ Title string151+ Layers []Layer152+ Styles []Style153+ Legend []LegendEntry154+}155+156+// Layer selects a FeatureSet and applies one Style over a visibility range.157+type Layer struct {158+ ID string159+ FeatureSet string160+ Style string161+ Group string162+ LabelPolicy string163+ Visible bool164+ Order int64165+ MinZoom int64166+ MaxZoom int64167+}168+169+// Style names the appearance of a layer without changing feature meaning.170+type Style struct {171+ ID string172+ Symbol string173+ Icon string174+ IconAsset string175+ IconPicture bool176+ RenderAs string177+ Stroke string178+ Fill string179+}180+181+// LegendEntry gives a layer its presented label and order.182+type LegendEntry struct {183+ Layer string184+ Label string185+ Order int64186+}187+188+// Asset is a content-bearing supporting blob.189+type Asset struct {190+ ID string191+ MediaType string192+ Data []byte193+ Provenance string194+}195+196+func coreField(name string, kind Kind, optional bool) Field {197+ return Field{ID: CoreID(name), Name: name, Kind: kind, Optional: optional}198+}199+200+func coreType(name string, fields ...Field) Type {201+ return Type{ID: CoreID("type." + name), Name: name, Fields: fields}202+}203+204+// StandardSchema is the Atlas-owned vocabulary. Extensions union into these205+// types by ID and require no format release or central allocation.206+func StandardSchema() Schema {207+ return canonicalSchema(Schema{Types: []Type{208+ coreType("volume",209+ coreField("volume.id", KindString, false),210+ coreField("volume.title", KindString, false)),211+ coreType("world",212+ coreField("world.id", KindString, false),213+ coreField("world.title", KindString, false),214+ coreField("world.coordinateSpace", KindString, false)),215+ coreType("coordinate",216+ coreField("coordinate.id", KindString, false),217+ coreField("coordinate.world", KindString, false),218+ coreField("coordinate.kind", KindString, false),219+ coreField("coordinate.unit", KindString, false),220+ coreField("coordinate.definition", KindString, false),221+ coreField("coordinate.extent", KindBytes, false),222+ coreField("coordinate.sourceZoom", KindInt64, false),223+ coreField("coordinate.firstTile", KindInt64, false),224+ coreField("coordinate.tileSize", KindInt64, false),225+ coreField("coordinate.size", KindInt64, false)),226+ coreType("featureSet",227+ coreField("featureSet.id", KindString, false),228+ coreField("featureSet.world", KindString, false),229+ coreField("featureSet.title", KindString, false),230+ coreField("featureSet.semanticType", KindString, false)),231+ coreType("propertyDefinition",232+ coreField("propertyDefinition.featureSet", KindString, false),233+ coreField("propertyDefinition.field", KindID, false),234+ coreField("propertyDefinition.name", KindString, false),235+ coreField("propertyDefinition.kind", KindInt64, false),236+ coreField("propertyDefinition.optional", KindBool, false)),237+ coreType("feature",238+ coreField("feature.id", KindString, false),239+ coreField("feature.featureSet", KindString, false),240+ coreField("feature.title", KindString, false),241+ coreField("feature.subtitle", KindString, true),242+ coreField("feature.description", KindString, true),243+ coreField("feature.center", KindBytes, true),244+ coreField("feature.shard", KindInt64, false),245+ coreField("feature.geometry", KindBytes, false)),246+ coreType("relationship",247+ coreField("relationship.feature", KindString, false),248+ coreField("relationship.predicate", KindString, false),249+ coreField("relationship.target", KindString, false)),250+ coreType("provenance",251+ coreField("provenance.feature", KindString, false),252+ coreField("provenance.source", KindString, false),253+ coreField("provenance.nativeID", KindString, false),254+ coreField("provenance.capturedAt", KindString, false)),255+ coreType("rasterPyramid",256+ coreField("rasterPyramid.id", KindString, false),257+ coreField("rasterPyramid.world", KindString, false),258+ coreField("rasterPyramid.name", KindString, false),259+ coreField("rasterPyramid.codec", KindString, false),260+ coreField("rasterPyramid.tileSize", KindInt64, false),261+ coreField("rasterPyramid.minZoom", KindInt64, false),262+ coreField("rasterPyramid.maxZoom", KindInt64, false),263+ coreField("rasterPyramid.fullZoom", KindInt64, false),264+ coreField("rasterPyramid.sourceZoom", KindInt64, false),265+ coreField("rasterPyramid.template", KindString, false),266+ coreField("rasterPyramid.bounds", KindBytes, true),267+ coreField("rasterPyramid.surface", KindBytes, true),268+ coreField("rasterPyramid.interpolate", KindBool, false),269+ coreField("rasterPyramid.background", KindString, true),270+ coreField("rasterPyramid.shard", KindInt64, false)),271+ coreType("rasterFormat",272+ coreField("rasterFormat.raster", KindString, false),273+ coreField("rasterFormat.zoom", KindInt64, false),274+ coreField("rasterFormat.format", KindString, false)),275+ coreType("rasterCoverage",276+ coreField("rasterCoverage.raster", KindString, false),277+ coreField("rasterCoverage.zoom", KindInt64, false),278+ coreField("rasterCoverage.x", KindInt64, false),279+ coreField("rasterCoverage.y", KindInt64, false),280+ coreField("rasterCoverage.w", KindInt64, false),281+ coreField("rasterCoverage.h", KindInt64, false),282+ coreField("rasterCoverage.bits", KindBytes, false)),283+ coreType("presentation",284+ coreField("presentation.id", KindString, false),285+ coreField("presentation.world", KindString, false),286+ coreField("presentation.title", KindString, false)),287+ coreType("style",288+ coreField("style.id", KindString, false),289+ coreField("style.presentation", KindString, false),290+ coreField("style.symbol", KindString, true),291+ coreField("style.icon", KindString, true),292+ coreField("style.iconAsset", KindString, true),293+ coreField("style.iconPicture", KindBool, false),294+ coreField("style.renderAs", KindString, true),295+ coreField("style.stroke", KindString, true),296+ coreField("style.fill", KindString, true)),297+ coreType("layer",298+ coreField("layer.id", KindString, false),299+ coreField("layer.presentation", KindString, false),300+ coreField("layer.featureSet", KindString, false),301+ coreField("layer.style", KindString, false),302+ coreField("layer.group", KindString, true),303+ coreField("layer.labelPolicy", KindString, true),304+ coreField("layer.visible", KindBool, false),305+ coreField("layer.order", KindInt64, false),306+ coreField("layer.minZoom", KindInt64, false),307+ coreField("layer.maxZoom", KindInt64, false)),308+ coreType("legend",309+ coreField("legend.presentation", KindString, false),310+ coreField("legend.layer", KindString, false),311+ coreField("legend.label", KindString, false),312+ coreField("legend.order", KindInt64, false)),313+ coreType("asset",314+ coreField("asset.id", KindString, false),315+ coreField("asset.mediaType", KindString, false),316+ coreField("asset.path", KindString, false),317+ coreField("asset.provenance", KindString, true)),318+ }})319+}@@ -0,0 +1,65 @@
1+package vnext2+3+import (4+ "fmt"5+ "slices"6+)7+8+// PresentedFeature is the scene-facing projection of one semantic feature.9+type PresentedFeature struct {10+ FeatureID string11+ LayerID string12+ StyleID string13+ Legend string14+ Symbol string15+ Stroke string16+ Fill string17+ Geometry Geometry18+ Properties []Property19+}20+21+// Present applies replaceable layer, style, and legend intent to a world's22+// semantic feature graph.23+func Present(world World) ([]PresentedFeature, error) {24+ sets := make(map[string]FeatureSet, len(world.FeatureSets))25+ for _, set := range world.FeatureSets {26+ sets[set.ID] = set27+ }28+ styles := make(map[string]Style, len(world.Presentation.Styles))29+ for _, style := range world.Presentation.Styles {30+ styles[style.ID] = style31+ }32+ legend := make(map[string]string, len(world.Presentation.Legend))33+ for _, entry := range world.Presentation.Legend {34+ legend[entry.Layer] = entry.Label35+ }36+ layers := append([]Layer(nil), world.Presentation.Layers...)37+ slices.SortStableFunc(layers, func(left, right Layer) int {38+ if left.Order < right.Order {39+ return -140+ }41+ if left.Order > right.Order {42+ return 143+ }44+ return 045+ })46+ var presented []PresentedFeature47+ for _, layer := range layers {48+ set, held := sets[layer.FeatureSet]49+ if !held {50+ return nil, fmt.Errorf("layer %s refers to unknown feature set %s", layer.ID, layer.FeatureSet)51+ }52+ style, held := styles[layer.Style]53+ if !held {54+ return nil, fmt.Errorf("layer %s refers to unknown style %s", layer.ID, layer.Style)55+ }56+ for _, feature := range set.Features {57+ presented = append(presented, PresentedFeature{58+ FeatureID: feature.ID, LayerID: layer.ID, StyleID: style.ID, Legend: legend[layer.ID],59+ Symbol: style.Symbol, Stroke: style.Stroke, Fill: style.Fill,60+ Geometry: feature.Geometry, Properties: append([]Property(nil), feature.Properties...),61+ })62+ }63+ }64+ return presented, nil65+}@@ -0,0 +1,101 @@
1+package vnext2+3+import (4+ "bytes"5+ "testing"6+)7+8+func TestVolumeTravelsFromDomainThroughBundleToPresentation(t *testing.T) {9+ t.Parallel()10+11+ emergency := Field{12+ ID: IDFromName("com.example.emergency", "feature.level"),13+ Name: "emergencyLevel",14+ Kind: KindInt64,15+ Optional: true,16+ }17+ volume := Volume{18+ ID: "bend-or",19+ Title: "Bend, Oregon",20+ Worlds: []World{{21+ ID: "2026-08-01",22+ Title: "2026-08-01",23+ CoordinateSpace: CoordinateSpace{24+ ID: "bend-web-mercator",25+ Kind: "projected",26+ Unit: "metre",27+ Definition: "EPSG:3857",28+ Extent: [4]float64{-13500000, 5400000, -13400000, 5500000},29+ SourceZoom: 13,30+ TileSize: 256,31+ Size: 8192,32+ },33+ FeatureSets: []FeatureSet{{34+ ID: "hospitals",35+ Title: "Hospitals",36+ SemanticType: "healthcare.hospital",37+ Properties: []Field{emergency},38+ Features: []Feature{{39+ ID: "osm/node/1842",40+ Title: "St. Charles Bend",41+ Geometry: Geometry{42+ Kind: GeometryPoint,43+ Parts: []GeometryPart{{Rings: [][]Position{{{-121.2691, 44.0671}}}}},44+ },45+ Properties: []Property{{FieldID: emergency.ID, Value: Int64Value(4)}},46+ Relationships: []Relationship{{Predicate: "within", Target: "bend-or"}},47+ Provenance: []Provenance{{Source: "openstreetmap", NativeID: "node/1842", CapturedAt: "2026-08-01T20:13:08Z"}},48+ }},49+ }},50+ RasterPyramids: []RasterPyramid{{51+ ID: "streets", Name: "Streets", Codec: "image/webp", TileSize: 256, MinZoom: 0, MaxZoom: 13, FullZoom: 13, SourceZoom: 13,52+ Template: "rasters/streets/{z}/{x}/{y}.webp",53+ Formats: []string{"webp", "webp", "webp", "webp", "webp", "webp", "webp", "webp", "webp", "webp", "webp", "webp", "webp", "webp"},54+ }},55+ Presentation: Presentation{56+ ID: "default", Title: "Default",57+ Styles: []Style{{ID: "hospital", Symbol: "asset:hospital", Fill: "#bc3b4a"}},58+ Layers: []Layer{{ID: "services-hospitals", FeatureSet: "hospitals", Style: "hospital", Visible: true, Order: 30, MinZoom: 8, MaxZoom: 22}},59+ Legend: []LegendEntry{{Layer: "services-hospitals", Label: "Hospitals", Order: 10}},60+ },61+ }},62+ Assets: []Asset{{ID: "hospital", MediaType: "image/svg+xml", Data: []byte("<svg/>"), Provenance: "custom"}},63+ }64+65+ packed, err := Compile(volume)66+ if err != nil {67+ t.Fatalf("compile volume: %v", err)68+ }69+ var archive bytes.Buffer70+ if err := Write(&archive, packed); err != nil {71+ t.Fatalf("write bundle: %v", err)72+ }73+ opened, err := Open(bytes.NewReader(archive.Bytes()), int64(archive.Len()), StandardSchema())74+ if err != nil {75+ t.Fatalf("open bundle: %v", err)76+ }77+ restored, err := opened.Volume()78+ if err != nil {79+ t.Fatalf("restore domain volume: %v", err)80+ }81+ if restored.ID != volume.ID || restored.Worlds[0].CoordinateSpace.Definition != "EPSG:3857" {82+ t.Fatalf("restored wrong volume: %#v", restored)83+ }84+ if got := restored.Worlds[0].FeatureSets[0].Features[0].Properties[0].Value.Int64; got != 4 {85+ t.Fatalf("typed extension property = %d, want 4", got)86+ }87+ if got := string(restored.Assets[0].Data); got != "<svg/>" {88+ t.Fatalf("asset = %q", got)89+ }90+91+ presented, err := Present(restored.Worlds[0])92+ if err != nil {93+ t.Fatalf("present world: %v", err)94+ }95+ if len(presented) != 1 {96+ t.Fatalf("presented %d features, want 1", len(presented))97+ }98+ if got := presented[0]; got.FeatureID != "osm/node/1842" || got.LayerID != "services-hospitals" || got.Symbol != "asset:hospital" || got.Legend != "Hospitals" {99+ t.Fatalf("presented feature = %#v", got)100+ }101+}@@ -0,0 +1,195 @@
1+package vnext2+3+import (4+ "crypto/sha256"5+ "encoding/json"6+ "fmt"7+ "slices"8+)9+10+// Kind is a stable primitive wire kind. New encodings may be added for a kind11+// without changing its semantic identity.12+type Kind uint813+14+const (15+ KindInvalid Kind = iota16+ KindBool17+ KindInt6418+ KindFloat6419+ KindString20+ KindBytes21+ KindID22+)23+24+func (kind Kind) valid() bool { return kind >= KindBool && kind <= KindID }25+26+// Field describes one independently evolvable claim on a type.27+type Field struct {28+ ID ID `json:"id"`29+ Name string `json:"name"`30+ Kind Kind `json:"kind"`31+ Optional bool `json:"optional,omitempty"`32+}33+34+// Type is a globally identified record whose fields are also globally35+// identified. Declaration order carries no meaning.36+type Type struct {37+ ID ID `json:"id"`38+ Name string `json:"name"`39+ Fields []Field `json:"fields"`40+}41+42+// Field finds a field by identity.43+func (typ Type) Field(id ID) (Field, bool) {44+ for _, field := range typ.Fields {45+ if field.ID == id {46+ return field, true47+ }48+ }49+ return Field{}, false50+}51+52+// Schema is the embedded vocabulary required to interpret a bundle.53+type Schema struct {54+ Types []Type `json:"types"`55+}56+57+// Type finds a record type by identity.58+func (schema Schema) Type(id ID) (Type, bool) {59+ for _, typ := range schema.Types {60+ if typ.ID == id {61+ return typ, true62+ }63+ }64+ return Type{}, false65+}66+67+// Canonical returns the one order-independent JSON encoding of a schema.68+func (schema Schema) Canonical() ([]byte, error) {69+ normalized := Schema{Types: append([]Type(nil), schema.Types...)}70+ for index := range normalized.Types {71+ normalized.Types[index].Fields = append([]Field(nil), normalized.Types[index].Fields...)72+ slices.SortFunc(normalized.Types[index].Fields, func(left, right Field) int {73+ return slices.Compare(left.ID[:], right.ID[:])74+ })75+ }76+ slices.SortFunc(normalized.Types, func(left, right Type) int {77+ return slices.Compare(left.ID[:], right.ID[:])78+ })79+ if err := normalized.validate(); err != nil {80+ return nil, err81+ }82+ data, err := json.Marshal(normalized)83+ if err != nil {84+ return nil, fmt.Errorf("marshal canonical schema: %w", err)85+ }86+ return data, nil87+}88+89+// Hash is the cache identity of the canonical schema, never a compatibility90+// gate. Readers merge the embedded schema with the schema they know.91+func (schema Schema) Hash() ([32]byte, error) {92+ data, err := schema.Canonical()93+ if err != nil {94+ return [32]byte{}, err95+ }96+ return sha256.Sum256(data), nil97+}98+99+// UnionSchemas merges schemas produced independently. A shared ID must retain100+// its wire kind; names are descriptive and converge deterministically.101+func UnionSchemas(schemas ...Schema) (Schema, error) {102+ types := make(map[ID]Type)103+ for _, schema := range schemas {104+ if err := mergeSchema(types, schema); err != nil {105+ return Schema{}, err106+ }107+ }108+ out := Schema{Types: make([]Type, 0, len(types))}109+ for _, typ := range types {110+ out.Types = append(out.Types, typ)111+ }112+ if _, err := out.Canonical(); err != nil {113+ return Schema{}, err114+ }115+ return canonicalSchema(out), nil116+}117+118+func mergeSchema(types map[ID]Type, schema Schema) error {119+ for _, incoming := range schema.Types {120+ if incoming.ID.isZero() || incoming.Name == "" {121+ return fmt.Errorf("schema carries an unnamed or zero-ID type")122+ }123+ current, held := types[incoming.ID]124+ if !held {125+ current = Type{ID: incoming.ID, Name: incoming.Name}126+ } else if incoming.Name < current.Name {127+ current.Name = incoming.Name128+ }129+ fields := make(map[ID]Field, len(current.Fields)+len(incoming.Fields))130+ for _, field := range current.Fields {131+ fields[field.ID] = field132+ }133+ for _, field := range incoming.Fields {134+ if err := mergeField(fields, field, incoming.ID); err != nil {135+ return err136+ }137+ }138+ current.Fields = current.Fields[:0]139+ for _, field := range fields {140+ current.Fields = append(current.Fields, field)141+ }142+ types[incoming.ID] = current143+ }144+ return nil145+}146+147+func mergeField(fields map[ID]Field, incoming Field, typeID ID) error {148+ if incoming.ID.isZero() || incoming.Name == "" || !incoming.Kind.valid() {149+ return fmt.Errorf("type %s carries an invalid field", typeID)150+ }151+ current, held := fields[incoming.ID]152+ if !held {153+ fields[incoming.ID] = incoming154+ return nil155+ }156+ if current.Kind != incoming.Kind {157+ return fmt.Errorf("field %s is both wire kind %d and %d", incoming.ID, current.Kind, incoming.Kind)158+ }159+ if incoming.Name < current.Name {160+ current.Name = incoming.Name161+ }162+ current.Optional = current.Optional || incoming.Optional163+ fields[incoming.ID] = current164+ return nil165+}166+167+func (schema Schema) validate() error {168+ types := make(map[ID]bool, len(schema.Types))169+ for _, typ := range schema.Types {170+ if typ.ID.isZero() || typ.Name == "" || types[typ.ID] {171+ return fmt.Errorf("schema carries a duplicate, unnamed, or zero-ID type")172+ }173+ types[typ.ID] = true174+ fields := make(map[ID]bool, len(typ.Fields))175+ for _, field := range typ.Fields {176+ if field.ID.isZero() || field.Name == "" || !field.Kind.valid() || fields[field.ID] {177+ return fmt.Errorf("type %s carries a duplicate or invalid field", typ.ID)178+ }179+ fields[field.ID] = true180+ }181+ }182+ return nil183+}184+185+func canonicalSchema(schema Schema) Schema {186+ for index := range schema.Types {187+ slices.SortFunc(schema.Types[index].Fields, func(left, right Field) int {188+ return slices.Compare(left.ID[:], right.ID[:])189+ })190+ }191+ slices.SortFunc(schema.Types, func(left, right Type) int {192+ return slices.Compare(left.ID[:], right.ID[:])193+ })194+ return schema195+}@@ -0,0 +1,75 @@
1+package vnext2+3+import (4+ "testing"5+)6+7+func TestSchemaUnionAcceptsIndependentExtensions(t *testing.T) {8+ t.Parallel()9+10+ core := StandardSchema()11+ featureID := CoreID("type.feature")12+ emergency := Field{13+ ID: IDFromName("com.example.emergency", "feature.level"),14+ Name: "emergencyLevel",15+ Kind: KindInt64,16+ }17+ capacity := Field{18+ ID: IDFromName("org.example.planning", "feature.capacity"),19+ Name: "capacity",20+ Kind: KindInt64,21+ }22+23+ branchA := Schema{Types: []Type{{ID: featureID, Name: "Feature", Fields: []Field{emergency}}}}24+ branchB := Schema{Types: []Type{{ID: featureID, Name: "Feature", Fields: []Field{capacity}}}}25+ merged, err := UnionSchemas(core, branchA, branchB)26+ if err != nil {27+ t.Fatalf("union independent schemas: %v", err)28+ }29+30+ feature, ok := merged.Type(featureID)31+ if !ok {32+ t.Fatal("merged schema lost Feature")33+ }34+ if _, ok := feature.Field(emergency.ID); !ok {35+ t.Fatal("merged schema lost branch A field")36+ }37+ if _, ok := feature.Field(capacity.ID); !ok {38+ t.Fatal("merged schema lost branch B field")39+ }40+}41+42+func TestSchemaUnionRejectsIdentityReuseWithAnotherWireKind(t *testing.T) {43+ t.Parallel()44+45+ featureID := CoreID("type.feature")46+ fieldID := IDFromName("com.example", "feature.rating")47+ left := Schema{Types: []Type{{ID: featureID, Name: "Feature", Fields: []Field{{ID: fieldID, Name: "rating", Kind: KindInt64}}}}}48+ right := Schema{Types: []Type{{ID: featureID, Name: "Feature", Fields: []Field{{ID: fieldID, Name: "rating", Kind: KindString}}}}}49+50+ if _, err := UnionSchemas(left, right); err == nil {51+ t.Fatal("union accepted one field identity with incompatible wire kinds")52+ }53+}54+55+func TestCanonicalSchemaIgnoresDeclarationOrder(t *testing.T) {56+ t.Parallel()57+58+ a := Field{ID: IDFromName("com.example", "a"), Name: "a", Kind: KindString}59+ b := Field{ID: IDFromName("com.example", "b"), Name: "b", Kind: KindFloat64}60+ typeID := IDFromName("com.example", "type")61+ left := Schema{Types: []Type{{ID: typeID, Name: "Example", Fields: []Field{a, b}}}}62+ right := Schema{Types: []Type{{ID: typeID, Name: "Example", Fields: []Field{b, a}}}}63+64+ leftHash, err := left.Hash()65+ if err != nil {66+ t.Fatalf("hash left schema: %v", err)67+ }68+ rightHash, err := right.Hash()69+ if err != nil {70+ t.Fatalf("hash right schema: %v", err)71+ }72+ if leftHash != rightHash {73+ t.Fatalf("declaration order changed schema identity: %x != %x", leftHash, rightHash)74+ }75+}@@ -0,0 +1,143 @@
1+package vnext2+3+import (4+ "bytes"5+ "fmt"6+ "math"7+ "slices"8+)9+10+// Value is one scalar in a typed column. KindInvalid is the null value.11+type Value struct {12+ Kind Kind13+ Bool bool14+ Int64 int6415+ Float64 float6416+ String string17+ Bytes []byte18+ ID ID19+}20+21+func NullValue() Value { return Value{} }22+func BoolValue(value bool) Value { return Value{Kind: KindBool, Bool: value} }23+func Int64Value(value int64) Value { return Value{Kind: KindInt64, Int64: value} }24+func Float64Value(value float64) Value { return Value{Kind: KindFloat64, Float64: value} }25+func StringValue(value string) Value { return Value{Kind: KindString, String: value} }26+func IDValue(value ID) Value { return Value{Kind: KindID, ID: value} }27+28+func BytesValue(value []byte) Value {29+ return Value{Kind: KindBytes, Bytes: append([]byte(nil), value...)}30+}31+32+// Equal compares values while preserving float bit patterns and byte content.33+func (value Value) Equal(other Value) bool {34+ return value.Kind == other.Kind &&35+ value.Bool == other.Bool &&36+ value.Int64 == other.Int64 &&37+ math.Float64bits(value.Float64) == math.Float64bits(other.Float64) &&38+ value.String == other.String &&39+ bytes.Equal(value.Bytes, other.Bytes) &&40+ value.ID == other.ID41+}42+43+// Column is one field laid out for every row in a table.44+type Column struct {45+ Field Field46+ Values []Value47+}48+49+// Table is one root type encoded as hybrid columns.50+type Table struct {51+ TypeID ID52+ Rows int53+ Columns []Column54+}55+56+// Value returns one field value from one row.57+func (table Table) Value(fieldID ID, row int) (Value, bool) {58+ for _, column := range table.Columns {59+ if column.Field.ID == fieldID {60+ return column.Values[row], true61+ }62+ }63+ return Value{}, false64+}65+66+func (table Table) validate() error {67+ if table.TypeID.isZero() || table.Rows < 0 {68+ return fmt.Errorf("table has invalid type or row count")69+ }70+ seen := make(map[ID]bool, len(table.Columns))71+ for _, column := range table.Columns {72+ if column.Field.ID.isZero() || !column.Field.Kind.valid() || seen[column.Field.ID] {73+ return fmt.Errorf("table %s carries a duplicate or invalid column", table.TypeID)74+ }75+ seen[column.Field.ID] = true76+ if len(column.Values) != table.Rows {77+ return fmt.Errorf("column %s has %d values for %d rows", column.Field.ID, len(column.Values), table.Rows)78+ }79+ for row, value := range column.Values {80+ if value.Kind == KindInvalid && !column.Field.Optional {81+ return fmt.Errorf("required column %s is null at row %d", column.Field.ID, row)82+ }83+ if value.Kind != KindInvalid && value.Kind != column.Field.Kind {84+ return fmt.Errorf("column %s expects kind %d, got %d at row %d", column.Field.ID, column.Field.Kind, value.Kind, row)85+ }86+ }87+ }88+ return nil89+}90+91+func canonicalTable(table Table) Table {92+ out := Table{TypeID: table.TypeID, Rows: table.Rows, Columns: append([]Column(nil), table.Columns...)}93+ slices.SortFunc(out.Columns, func(left, right Column) int {94+ return slices.Compare(left.Field.ID[:], right.Field.ID[:])95+ })96+ return out97+}98+99+// TableProjection separates columns understood by an application from newer100+// file columns while retaining both for lossless pass-through.101+type TableProjection struct {102+ Known Table103+ Foreign Table104+}105+106+// Complete reunites known and foreign columns in canonical order.107+func (projection TableProjection) Complete() Table {108+ columns := append([]Column(nil), projection.Known.Columns...)109+ columns = append(columns, projection.Foreign.Columns...)110+ return canonicalTable(Table{TypeID: projection.Known.TypeID, Rows: projection.Known.Rows, Columns: columns})111+}112+113+// ProjectTable classifies a file table using an application's schema. The114+// embedded file schema remains authoritative for field metadata.115+func ProjectTable(application, file Schema, table Table) (TableProjection, error) {116+ if _, err := UnionSchemas(application, file); err != nil {117+ return TableProjection{}, fmt.Errorf("merge application and file schemas: %w", err)118+ }119+ fileType, ok := file.Type(table.TypeID)120+ if !ok {121+ return TableProjection{}, fmt.Errorf("file schema does not describe table type %s", table.TypeID)122+ }123+ applicationType, applicationKnowsType := application.Type(table.TypeID)124+ projection := TableProjection{125+ Known: Table{TypeID: table.TypeID, Rows: table.Rows},126+ Foreign: Table{TypeID: table.TypeID, Rows: table.Rows},127+ }128+ for _, column := range table.Columns {129+ field, held := fileType.Field(column.Field.ID)130+ if !held || field.Kind != column.Field.Kind {131+ return TableProjection{}, fmt.Errorf("file schema disagrees with column %s", column.Field.ID)132+ }133+ column.Field = field134+ if _, known := applicationType.Field(field.ID); applicationKnowsType && known {135+ projection.Known.Columns = append(projection.Known.Columns, column)136+ } else {137+ projection.Foreign.Columns = append(projection.Foreign.Columns, column)138+ }139+ }140+ projection.Known = canonicalTable(projection.Known)141+ projection.Foreign = canonicalTable(projection.Foreign)142+ return projection, nil143+}@@ -0,0 +1,242 @@
1+package vnext2+3+import (4+ "fmt"5+ "math"6+ "strings"7+)8+9+type identities map[string]string10+11+func (ids identities) add(id, kind string) error {12+ if id == "" {13+ return fmt.Errorf("%s has no ID", kind)14+ }15+ key := kind + "\x00" + id16+ if previous, held := ids[key]; held {17+ return fmt.Errorf("ID %q is shared by %s and %s", id, previous, kind)18+ }19+ ids[key] = kind20+ return nil21+}22+23+func validateVolume(volume Volume) error {24+ if volume.ID == "" || volume.Title == "" || len(volume.Worlds) == 0 {25+ return fmt.Errorf("volume requires an ID, title, and at least one world")26+ }27+ ids := make(identities)28+ if err := ids.add(volume.ID, "volume"); err != nil {29+ return err30+ }31+ for _, world := range volume.Worlds {32+ if err := validateWorld(world, ids); err != nil {33+ return err34+ }35+ }36+ for _, asset := range volume.Assets {37+ if err := ids.add(asset.ID, "asset"); err != nil {38+ return err39+ }40+ if asset.MediaType == "" {41+ return fmt.Errorf("asset %s has no media type", asset.ID)42+ }43+ }44+ return nil45+}46+47+func validateWorld(world World, ids identities) error {48+ if err := ids.add(world.ID, "world"); err != nil {49+ return err50+ }51+ if world.Title == "" {52+ return fmt.Errorf("world %s has no title", world.ID)53+ }54+ if err := ids.add(world.CoordinateSpace.ID, "coordinate space"); err != nil {55+ return err56+ }57+ if world.CoordinateSpace.Kind == "" || world.CoordinateSpace.Unit == "" || world.CoordinateSpace.Definition == "" {58+ return fmt.Errorf("world %s has an incomplete coordinate space", world.ID)59+ }60+ if world.CoordinateSpace.SourceZoom < 0 || world.CoordinateSpace.TileSize < 0 || world.CoordinateSpace.Size < 0 {61+ return fmt.Errorf("world %s has invalid coordinate grid metadata", world.ID)62+ }63+ if err := validateClaims(world.Claims, "world"); err != nil {64+ return err65+ }66+ sets := make(map[string]bool, len(world.FeatureSets))67+ for _, set := range world.FeatureSets {68+ if err := validateFeatureSet(set, ids); err != nil {69+ return err70+ }71+ sets[set.ID] = true72+ }73+ for _, raster := range world.RasterPyramids {74+ if err := ids.add(raster.ID, "raster pyramid"); err != nil {75+ return err76+ }77+ if raster.Name == "" || raster.Codec == "" || raster.Template == "" || raster.TileSize <= 0 || raster.MinZoom < 0 || raster.MaxZoom < raster.MinZoom || raster.FullZoom < raster.MinZoom || raster.SourceZoom < 0 {78+ return fmt.Errorf("raster pyramid %s is incomplete", raster.ID)79+ }80+ if len(raster.Formats) != int(raster.MaxZoom-raster.MinZoom+1) {81+ return fmt.Errorf("raster pyramid %s needs one format per zoom", raster.ID)82+ }83+ if err := offlineString(raster.Template, "raster template"); err != nil {84+ return err85+ }86+ }87+ return validatePresentation(world.Presentation, sets, ids)88+}89+90+func validateFeatureSet(set FeatureSet, ids identities) error {91+ if err := ids.add(set.ID, "feature set"); err != nil {92+ return err93+ }94+ if set.SemanticType == "" {95+ return fmt.Errorf("feature set %s has no semantic type", set.ID)96+ }97+ if err := validateClaims(set.Claims, "feature set"); err != nil {98+ return err99+ }100+ fields := make(map[ID]bool, len(set.Properties))101+ for _, field := range set.Properties {102+ if field.ID.isZero() || field.Name == "" || !field.Kind.valid() || fields[field.ID] {103+ return fmt.Errorf("feature set %s has an invalid property contract", set.ID)104+ }105+ fields[field.ID] = true106+ }107+ for _, feature := range set.Features {108+ if err := ids.add(feature.ID, "feature"); err != nil {109+ return err110+ }111+ if feature.Title == "" {112+ return fmt.Errorf("feature %s has no title", feature.ID)113+ }114+ if _, err := encodeGeometry(feature.Geometry); err != nil {115+ return fmt.Errorf("feature %s: %w", feature.ID, err)116+ }117+ seen := make(map[ID]bool, len(feature.Properties))118+ for _, property := range feature.Properties {119+ propertyID := property.fieldID()120+ field, held := fieldByID(set.Properties, propertyID)121+ if !held || seen[propertyID] || property.Value.Kind != field.Kind {122+ return fmt.Errorf("feature %s violates its property contract", feature.ID)123+ }124+ seen[propertyID] = true125+ if property.Value.Kind == KindString {126+ if err := offlineString(property.Value.String, "feature property"); err != nil {127+ return err128+ }129+ }130+ }131+ for _, field := range set.Properties {132+ if !field.Optional && !seen[field.ID] {133+ return fmt.Errorf("feature %s omits required property %s", feature.ID, field.ID)134+ }135+ }136+ for _, relationship := range feature.Relationships {137+ if err := offlineString(relationship.Target, "relationship target"); err != nil {138+ return err139+ }140+ }141+ }142+ return nil143+}144+145+func validateClaims(claims []Property, owner string) error {146+ seen := make(map[ID]bool, len(claims))147+ for _, claim := range claims {148+ id := claim.fieldID()149+ if id.isZero() || claim.Field.ID != id || claim.Field.Name == "" || !claim.Field.Kind.valid() || claim.Value.Kind != claim.Field.Kind || seen[id] {150+ return fmt.Errorf("%s has an invalid typed claim", owner)151+ }152+ seen[id] = true153+ if claim.Value.Kind == KindString {154+ if err := offlineString(claim.Value.String, owner+" claim"); err != nil {155+ return err156+ }157+ }158+ }159+ return nil160+}161+162+func validateGeometry(geometry Geometry) error {163+ if geometry.Kind < GeometryPoint || geometry.Kind > GeometryPolygon || len(geometry.Parts) == 0 {164+ return fmt.Errorf("geometry has an invalid kind or no parts")165+ }166+ if geometry.Kind == GeometryPoint && len(geometry.Parts) != 1 {167+ return fmt.Errorf("point geometry must contain exactly one part")168+ }169+ for _, part := range geometry.Parts {170+ if len(part.Rings) == 0 || (geometry.Kind != GeometryPolygon && len(part.Rings) != 1) {171+ return fmt.Errorf("geometry part has an invalid ring count")172+ }173+ for _, ring := range part.Rings {174+ minimum := 1175+ if geometry.Kind == GeometryLineString {176+ minimum = 2177+ } else if geometry.Kind == GeometryPolygon {178+ minimum = 4179+ }180+ if len(ring) < minimum {181+ return fmt.Errorf("geometry ring has too few positions")182+ }183+ for _, position := range ring {184+ if math.IsNaN(position[0]) || math.IsNaN(position[1]) || math.IsInf(position[0], 0) || math.IsInf(position[1], 0) {185+ return fmt.Errorf("geometry contains a non-finite position")186+ }187+ }188+ }189+ }190+ if geometry.Kind == GeometryPoint && len(geometry.Parts[0].Rings[0]) != 1 {191+ return fmt.Errorf("point geometry must contain exactly one position")192+ }193+ return nil194+}195+196+func validatePresentation(presentation Presentation, sets map[string]bool, ids identities) error {197+ if err := ids.add(presentation.ID, "presentation"); err != nil {198+ return err199+ }200+ if presentation.Title == "" {201+ return fmt.Errorf("presentation %s has no title", presentation.ID)202+ }203+ styles := make(map[string]bool, len(presentation.Styles))204+ for _, style := range presentation.Styles {205+ if err := ids.add(style.ID, "style"); err != nil {206+ return err207+ }208+ styles[style.ID] = true209+ }210+ layers := make(map[string]bool, len(presentation.Layers))211+ for _, layer := range presentation.Layers {212+ if err := ids.add(layer.ID, "layer"); err != nil {213+ return err214+ }215+ if !sets[layer.FeatureSet] || !styles[layer.Style] || layer.MaxZoom < layer.MinZoom {216+ return fmt.Errorf("layer %s has an invalid feature set, style, or zoom range", layer.ID)217+ }218+ layers[layer.ID] = true219+ }220+ for _, entry := range presentation.Legend {221+ if !layers[entry.Layer] || entry.Label == "" {222+ return fmt.Errorf("legend has an invalid layer or label")223+ }224+ }225+ return nil226+}227+228+func fieldByID(fields []Field, id ID) (Field, bool) {229+ for _, field := range fields {230+ if field.ID == id {231+ return field, true232+ }233+ }234+ return Field{}, false235+}236+237+func offlineString(value, context string) error {238+ if strings.Contains(value, "http://") || strings.Contains(value, "https://") {239+ return fmt.Errorf("%s carries a runtime URL", context)240+ }241+ return nil242+}@@ -49,8 +49,9 @@ type App struct {
4949 // search, and the keys carry the build's stamp, so a new build is a new5050 // entry and nothing is ever stale. They are memory, not state: two Apps5151 // over one host still answer the same, only slower.52- worlds *worldCache53- texts *textCache52+ worlds *worldCache53+ texts *textCache54+ semantic *semanticCache5455 5556 // writing serializes the read-modify-write of one volume's record. Two5657 // interactions can be in flight at once -- a reader holds a key down, a@@ -77,7 +78,7 @@ type Options struct {
7778 func New(env hostenv.Hostenv, opts Options) *App {7879 a := &App{7980 env: env, static: opts.Static, events: newHub(), mux: http.NewServeMux(),80- worlds: newWorldCache(), texts: newTextCache(),81+ worlds: newWorldCache(), texts: newTextCache(), semantic: newSemanticCache(),8182 }8283 a.routes()8384 return a@@ -111,7 +111,7 @@ func volume(slug, title, stamp string, worlds ...bundle.WorldEntry) *fakeVolume
111111 },112112 entries: map[string][]byte{113113 "worlds/overworld.json": []byte(`{"lenses":[],"collections":[]}`),114- "worlds/overworld.bin": []byte("ATLASLOC"),114+ "worlds/overworld.bin": bundle.PackLocations(nil),115115 "worlds/overworld.text": []byte(`{}`),116116 "icons/marker.svg": []byte("<svg/>"),117117 "tiles/overworld/0/0/0.jpg": []byte("raster"),@@ -210,8 +210,8 @@ func TestContentPlane(t *testing.T) {
210210 kind string211211 size int212212 }{213- {"a world payload", base + "/worlds/overworld.json", "application/json", 30},214- {"the packed locations", base + "/worlds/overworld.bin", "application/octet-stream", 8},213+ {"a world payload", base + "/worlds/overworld.json", "application/json", 126},214+ {"the packed locations", base + "/worlds/overworld.bin", "application/octet-stream", 20},215215 {"the deferred prose", base + "/worlds/overworld.text", "application/json", 2},216216 {"an icon", base + "/icons/marker.svg", "image/svg+xml", 6},217217 {"a tile", base + "/tiles/overworld/0/0/0.jpg", "image/jpeg", 6},@@ -128,8 +128,16 @@ func (a *App) handleContent(w http.ResponseWriter, r *http.Request) {
128128 }129129 130130 rest := r.PathValue("rest")131- if !strings.HasPrefix(rest, bundle.WorldsPrefix) &&132- !strings.HasPrefix(rest, bundle.TilesPrefix) &&131+ if strings.HasPrefix(rest, bundle.WorldsPrefix) {132+ data, kind, projected, err := a.projectedContent(held, rest)133+ if err != nil || !projected {134+ http.NotFound(w, r)135+ return136+ }137+ serveProjectedContent(w, data, kind)138+ return139+ }140+ if !strings.HasPrefix(rest, bundle.TilesPrefix) &&133141 !strings.HasPrefix(rest, bundle.IconsPrefix) {134142 http.NotFound(w, r)135143 return@@ -161,3 +169,10 @@ func (a *App) handleContent(w http.ResponseWriter, r *http.Request) {
161169 logging.Volume(manifest.Volume.Slug), logging.Path(rest), slog.Any("error", err))162170 }163171 }172+173+func serveProjectedContent(w http.ResponseWriter, data []byte, kind string) {174+ w.Header().Set("Content-Type", kind)175+ w.Header().Set("Content-Length", strconv.Itoa(len(data)))176+ w.Header().Set("Cache-Control", "private, max-age=31536000, immutable")177+ _, _ = w.Write(data)178+}@@ -0,0 +1,127 @@
1+package app2+3+import (4+ "encoding/json"5+ "path/filepath"6+ "runtime"7+ "testing"8+9+ "github.com/FelineStateMachine/atlas/format/bundle"10+ "github.com/FelineStateMachine/atlas/format/vnext"11+)12+13+func TestVNextEarthProjectsToTheEstablishedRendererContract(t *testing.T) {14+ t.Parallel()15+16+ legacy, err := bundle.Open(includedEarthPath(t))17+ if err != nil {18+ t.Fatalf("open v3 Earth: %v", err)19+ }20+ defer legacy.Close()21+ semantic, err := vnext.ImportV3Volume(legacy.Manifest, legacy.ReadEntry)22+ if err != nil {23+ t.Fatalf("import v3 Earth: %v", err)24+ }25+26+ payload, locations, texts, err := presentVNextWorld(semantic.Worlds[0])27+ if err != nil {28+ t.Fatalf("present vNext Earth: %v", err)29+ }30+ if len(payload.Collections) != 7 || len(payload.Lenses) != 1 {31+ t.Fatalf("projection has %d collections and %d lenses", len(payload.Collections), len(payload.Lenses))32+ }33+ if len(locations) != 202 {34+ t.Fatalf("projection has %d point locations", len(locations))35+ }36+ if len(texts) != 379 {37+ t.Fatalf("projection has %d text records", len(texts))38+ }39+ countries := projectedCollection(payload, "Countries")40+ if countries == nil || len(countries.Features) != 177 {41+ t.Fatalf("Countries projection = %#v", countries)42+ }43+ fiji := projectedFeature(*countries, "Fiji")44+ if fiji == nil || len(fiji.Geometry) != 1 || fiji.Geometry[0].Type != "MultiPolygon" {45+ t.Fatalf("Fiji projection = %#v", fiji)46+ }47+48+ model, err := buildVNextWorld(semantic.Worlds[0])49+ if err != nil {50+ t.Fatalf("build vNext world: %v", err)51+ }52+ if len(model.Points) != 202 || len(model.Shapes) != 177 || len(model.Members) != 7 {53+ t.Fatalf("runtime model has %d points, %d shapes, %d collections", len(model.Points), len(model.Shapes), len(model.Members))54+ }55+}56+57+func TestVNextEarthServesTheThreeRendererPayloads(t *testing.T) {58+ t.Parallel()59+60+ legacy, err := bundle.Open(includedEarthPath(t))61+ if err != nil {62+ t.Fatalf("open v3 Earth: %v", err)63+ }64+ defer legacy.Close()65+ semantic, err := vnext.ImportV3Volume(legacy.Manifest, legacy.ReadEntry)66+ if err != nil {67+ t.Fatalf("import v3 Earth: %v", err)68+ }69+ world := semantic.Worlds[0]70+71+ payloadBytes, kind, held, err := projectedVNextEntry(world, bundle.WorldEntryName(world.ID, bundle.WorldSuffix))72+ if err != nil || !held || kind != "application/json" {73+ t.Fatalf("project world entry = %q, %t, %v", kind, held, err)74+ }75+ var payload worldPayload76+ if err := json.Unmarshal(payloadBytes, &payload); err != nil {77+ t.Fatalf("decode projected world: %v", err)78+ }79+ if len(payload.Collections) != 7 || len(payload.Lenses[0].Formats) != 7 {80+ t.Fatalf("projected payload = %#v", payload)81+ }82+83+ locationBytes, kind, held, err := projectedVNextEntry(world, bundle.WorldEntryName(world.ID, bundle.PackedSuffix))84+ if err != nil || !held || kind != "application/octet-stream" {85+ t.Fatalf("project locations entry = %q, %t, %v", kind, held, err)86+ }87+ locations, err := bundle.UnpackLocations(locationBytes)88+ if err != nil || len(locations) != 202 {89+ t.Fatalf("projected locations = %d, %v", len(locations), err)90+ }91+92+ textBytes, kind, held, err := projectedVNextEntry(world, bundle.WorldEntryName(world.ID, bundle.TextSuffix))93+ if err != nil || !held || kind != "application/json" {94+ t.Fatalf("project text entry = %q, %t, %v", kind, held, err)95+ }96+ var texts map[string]featureText97+ if err := json.Unmarshal(textBytes, &texts); err != nil || len(texts) != 379 {98+ t.Fatalf("projected text = %d, %v", len(texts), err)99+ }100+}101+102+func includedEarthPath(t *testing.T) string {103+ t.Helper()104+ _, file, _, ok := runtime.Caller(0)105+ if !ok {106+ t.Fatal("locate projection test")107+ }108+ return filepath.Join(filepath.Dir(file), "..", "..", "included", "earth-20260803-76633cb323ec.atlas")109+}110+111+func projectedCollection(payload worldPayload, title string) *payloadCollection {112+ for index := range payload.Collections {113+ if payload.Collections[index].Title == title {114+ return &payload.Collections[index]115+ }116+ }117+ return nil118+}119+120+func projectedFeature(collection payloadCollection, title string) *payloadFeature {121+ for index := range collection.Features {122+ if collection.Features[index].Title == title {123+ return &collection.Features[index]124+ }125+ }126+ return nil127+}@@ -0,0 +1,516 @@
1+package app2+3+import (4+ "encoding/base64"5+ "encoding/json"6+ "fmt"7+ "hash/fnv"8+ "math"9+ "sort"10+ "strconv"11+ "strings"12+ "sync"13+14+ "github.com/FelineStateMachine/atlas/format/bundle"15+ "github.com/FelineStateMachine/atlas/format/semconv"16+ "github.com/FelineStateMachine/atlas/format/vnext"17+ "github.com/FelineStateMachine/atlas/internal/app/cells"18+ "github.com/FelineStateMachine/atlas/internal/app/hostenv"19+)20+21+type semanticCache struct {22+ mu sync.Mutex23+ held map[string]vnext.Volume24+ order []string25+}26+27+func newSemanticCache() *semanticCache {28+ return &semanticCache{held: make(map[string]vnext.Volume)}29+}30+31+func (cache *semanticCache) load(key string, importVolume func() (vnext.Volume, error)) (vnext.Volume, error) {32+ cache.mu.Lock()33+ defer cache.mu.Unlock()34+ if held, ok := cache.held[key]; ok {35+ return held, nil36+ }37+ volume, err := importVolume()38+ if err != nil {39+ return vnext.Volume{}, err40+ }41+ cache.order = append(cache.order, key)42+ for len(cache.order) > worldsHeld {43+ delete(cache.held, cache.order[0])44+ cache.order = cache.order[1:]45+ }46+ cache.held[key] = volume47+ return volume, nil48+}49+50+func (a *App) semanticVolume(source hostenv.Volume) (vnext.Volume, error) {51+ manifest := source.Manifest()52+ key := manifest.Volume.Slug + "@" + bundle.ShortStamp(manifest.Version.Stamp)53+ return a.semantic.load(key, func() (vnext.Volume, error) {54+ return vnext.ImportV3Volume(manifest, func(name string) ([]byte, error) {55+ return readEntry(source, name)56+ })57+ })58+}59+60+func semanticWorld(volume vnext.Volume, slug string) (vnext.World, bool) {61+ for _, world := range volume.Worlds {62+ if world.ID == slug {63+ return world, true64+ }65+ }66+ return vnext.World{}, false67+}68+69+func (a *App) projectedContent(source hostenv.Volume, name string) ([]byte, string, bool, error) {70+ volume, err := a.semanticVolume(source)71+ if err != nil {72+ return nil, "", true, err73+ }74+ for _, world := range volume.Worlds {75+ data, kind, held, err := projectedVNextEntry(world, name)76+ if held {77+ return data, kind, true, err78+ }79+ }80+ return nil, "", false, nil81+}82+83+type compatibilityIDs struct {84+ byName map[string]int6485+ byID map[int64]string86+}87+88+func newCompatibilityIDs() *compatibilityIDs {89+ return &compatibilityIDs{byName: make(map[string]int64), byID: make(map[int64]string)}90+}91+92+func (ids *compatibilityIDs) resolve(name string) (int64, error) {93+ if held, ok := ids.byName[name]; ok {94+ return held, nil95+ }96+ candidate := numericTail(name)97+ if candidate <= 0 {98+ hash := fnv.New64a()99+ _, _ = hash.Write([]byte(name))100+ candidate = int64(hash.Sum64() & math.MaxInt64)101+ if candidate == 0 {102+ candidate = 1103+ }104+ }105+ if owner, held := ids.byID[candidate]; held && owner != name {106+ return 0, fmt.Errorf("compatibility ID %d is shared by %s and %s", candidate, owner, name)107+ }108+ ids.byName[name], ids.byID[candidate] = candidate, name109+ return candidate, nil110+}111+112+func presentVNextWorld(world vnext.World) (worldPayload, []bundle.Location, map[string]featureText, error) {113+ sets := make(map[string]vnext.FeatureSet, len(world.FeatureSets))114+ for _, set := range world.FeatureSets {115+ sets[set.ID] = set116+ }117+ styles := make(map[string]vnext.Style, len(world.Presentation.Styles))118+ for _, style := range world.Presentation.Styles {119+ styles[style.ID] = style120+ }121+ layers := append([]vnext.Layer(nil), world.Presentation.Layers...)122+ sort.SliceStable(layers, func(i, j int) bool { return layers[i].Order < layers[j].Order })123+ projection := worldProjection{124+ world: world, sets: sets, styles: styles,125+ collectionIDs: newCompatibilityIDs(), featureIDs: newCompatibilityIDs(),126+ text: make(map[string]featureText),127+ }128+ return projection.present(layers)129+}130+131+func projectedVNextEntry(world vnext.World, name string) ([]byte, string, bool, error) {132+ payload, locations, text, err := presentVNextWorld(world)133+ if err != nil {134+ return nil, "", true, err135+ }136+ switch name {137+ case bundle.WorldEntryName(world.ID, bundle.WorldSuffix):138+ data, err := json.Marshal(payload)139+ return data, "application/json", true, err140+ case bundle.WorldEntryName(world.ID, bundle.PackedSuffix):141+ return bundle.PackLocations(locations), "application/octet-stream", true, nil142+ case bundle.WorldEntryName(world.ID, bundle.TextSuffix):143+ data, err := json.Marshal(text)144+ return data, "application/json", true, err145+ default:146+ return nil, "", false, nil147+ }148+}149+150+type worldProjection struct {151+ world vnext.World152+ sets map[string]vnext.FeatureSet153+ styles map[string]vnext.Style154+ collectionIDs *compatibilityIDs155+ featureIDs *compatibilityIDs156+ text map[string]featureText157+}158+159+func (projection worldProjection) present(layers []vnext.Layer) (worldPayload, []bundle.Location, map[string]featureText, error) {160+ payload := worldPayload{161+ Grid: &payloadGrid{162+ SourceZoom: intPointer(int(projection.world.CoordinateSpace.SourceZoom)),163+ FirstTile: intPointer(int(projection.world.CoordinateSpace.FirstTile)),164+ TileSize: intPointer(int(projection.world.CoordinateSpace.TileSize)),165+ Size: intPointer(int(projection.world.CoordinateSpace.Size)),166+ },167+ Attrs: claimsToAttrs(projection.world.Claims),168+ }169+ payload.Lenses = projectRasters(projection.world.RasterPyramids)170+ if source := worldOrigin(projection.world); source != "" {171+ payload.Merged = []payloadMerge{{Source: source, Origin: true}}172+ }173+ var locations []bundle.Location174+ for owner, layer := range layers {175+ collection, points, err := projection.collection(layer, owner)176+ if err != nil {177+ return worldPayload{}, nil, nil, err178+ }179+ payload.Collections = append(payload.Collections, collection)180+ locations = append(locations, points...)181+ }182+ return payload, locations, projection.text, nil183+}184+185+func (projection worldProjection) collection(layer vnext.Layer, owner int) (payloadCollection, []bundle.Location, error) {186+ set, held := projection.sets[layer.FeatureSet]187+ if !held {188+ return payloadCollection{}, nil, fmt.Errorf("layer %s refers to missing feature set %s", layer.ID, layer.FeatureSet)189+ }190+ style, held := projection.styles[layer.Style]191+ if !held {192+ return payloadCollection{}, nil, fmt.Errorf("layer %s refers to missing style %s", layer.ID, layer.Style)193+ }194+ id, err := projection.collectionIDs.resolve(set.ID)195+ if err != nil {196+ return payloadCollection{}, nil, err197+ }198+ kind := strings.TrimPrefix(set.SemanticType, "geometry.")199+ collection := payloadCollection{200+ ID: id, Title: set.Title, Kind: kind, Group: layer.Group, Icon: style.Icon,201+ IconAsset: style.IconAsset, IconPicture: style.IconPicture,202+ Color: projectedColor(kind, style), Visible: boolPointer(layer.Visible),203+ Attrs: claimsToAttrs(set.Claims),204+ }205+ collection.Attrs[semconv.KeyGeometryKind] = kind206+ putAttr(collection.Attrs, semconv.KeyLabelPolicy, layer.LabelPolicy)207+ putAttr(collection.Attrs, semconv.KeyRenderAs, style.RenderAs)208+ return projection.features(set, collection, owner)209+}210+211+func (projection worldProjection) features(set vnext.FeatureSet, collection payloadCollection, owner int) (payloadCollection, []bundle.Location, error) {212+ if owner > math.MaxUint16 {213+ return payloadCollection{}, nil, fmt.Errorf("presentation has too many collections")214+ }215+ var locations []bundle.Location216+ for _, feature := range set.Features {217+ id, err := projection.featureIDs.resolve(feature.ID)218+ if err != nil {219+ return payloadCollection{}, nil, err220+ }221+ text, err := projection.featureText(feature)222+ if err != nil {223+ return payloadCollection{}, nil, err224+ }225+ projection.text[strconv.FormatInt(id, 10)] = text226+ if feature.Geometry.Kind == vnext.GeometryPoint {227+ location, err := projection.point(feature, id, owner)228+ if err != nil {229+ return payloadCollection{}, nil, err230+ }231+ locations = append(locations, location)232+ continue233+ }234+ shape, err := projection.shape(feature, id, text)235+ if err != nil {236+ return payloadCollection{}, nil, err237+ }238+ collection.Features = append(collection.Features, shape)239+ }240+ return collection, locations, nil241+}242+243+func (projection worldProjection) point(feature vnext.Feature, id int64, owner int) (bundle.Location, error) {244+ if len(feature.Geometry.Parts) != 1 || len(feature.Geometry.Parts[0].Rings) != 1 || len(feature.Geometry.Parts[0].Rings[0]) != 1 {245+ return bundle.Location{}, fmt.Errorf("point %s has an invalid geometry", feature.ID)246+ }247+ position := feature.Geometry.Parts[0].Rings[0][0]248+ member, err := projection.relationship(feature.Relationships, "within")249+ if err != nil {250+ return bundle.Location{}, err251+ }252+ return bundle.Location{253+ ID: id, Title: feature.Title, Lat: position[1], Lng: position[0],254+ Member: member, Shard: feature.Shard, Owner: uint16(owner),255+ }, nil256+}257+258+func (projection worldProjection) shape(feature vnext.Feature, id int64, text featureText) (payloadFeature, error) {259+ geometry, err := projectGeometry(feature.Geometry)260+ if err != nil {261+ return payloadFeature{}, err262+ }263+ shape := payloadFeature{264+ ID: id, Title: feature.Title, Subtitle: feature.Subtitle, HasText: hasFeatureText(text),265+ Shard: feature.Shard, Attrs: propertiesToAttrs(feature.Properties), Geometry: geometry,266+ }267+ if feature.Center != nil {268+ shape.Center = &bundle.Coordinate{Lat: feature.Center[1], Lng: feature.Center[0]}269+ }270+ if parent, err := projection.relationship(feature.Relationships, "within"); err != nil {271+ return payloadFeature{}, err272+ } else if parent != 0 {273+ shape.Parent = &parent274+ }275+ return shape, nil276+}277+278+func (projection worldProjection) featureText(feature vnext.Feature) (featureText, error) {279+ text := featureText{Description: feature.Description, Attrs: propertiesToAttrs(feature.Properties)}280+ for _, relationship := range feature.Relationships {281+ if relationship.Predicate != "references" {282+ continue283+ }284+ id, err := projection.featureIDs.resolve(relationship.Target)285+ if err != nil {286+ return featureText{}, err287+ }288+ text.Links = append(text.Links, id)289+ }290+ return text, nil291+}292+293+func (projection worldProjection) relationship(relationships []vnext.Relationship, predicate string) (int64, error) {294+ for _, relationship := range relationships {295+ if relationship.Predicate == predicate {296+ return projection.featureIDs.resolve(relationship.Target)297+ }298+ }299+ return 0, nil300+}301+302+func projectRasters(rasters []vnext.RasterPyramid) []payloadLens {303+ out := make([]payloadLens, 0, len(rasters))304+ for _, raster := range rasters {305+ lens := payloadLens{306+ Name: raster.Name, Tiles: rasterTiles(raster.Template), MinZoom: int(raster.MinZoom), MaxZoom: int(raster.MaxZoom),307+ FullZoom: int(raster.FullZoom), SourceZoom: int(raster.SourceZoom), Formats: append([]string(nil), raster.Formats...),308+ Bounds: cellRect(raster.Bounds), Surface: cellRect(raster.Surface), Interpolate: raster.Interpolate,309+ Background: raster.Background, Shard: int(raster.Shard),310+ }311+ if len(raster.Coverage) > 0 {312+ lens.Coverage = make(map[string]payloadCoverage, len(raster.Coverage))313+ for _, coverage := range raster.Coverage {314+ lens.Coverage[strconv.FormatInt(coverage.Zoom, 10)] = payloadCoverage{315+ X: int(coverage.X), Y: int(coverage.Y), W: int(coverage.W), H: int(coverage.H),316+ Bits: base64.StdEncoding.EncodeToString(coverage.Bits),317+ }318+ }319+ }320+ out = append(out, lens)321+ }322+ return out323+}324+325+func projectGeometry(geometry vnext.Geometry) ([]payloadGeometry, error) {326+ if geometry.Kind == vnext.GeometryPolygon {327+ polygons := make([][][]vnext.Position, 0, len(geometry.Parts))328+ for _, part := range geometry.Parts {329+ polygons = append(polygons, part.Rings)330+ }331+ if len(polygons) == 1 {332+ return marshalGeometry("Polygon", polygons[0])333+ }334+ return marshalGeometry("MultiPolygon", polygons)335+ }336+ if geometry.Kind == vnext.GeometryLineString {337+ lines := make([][]vnext.Position, 0, len(geometry.Parts))338+ for _, part := range geometry.Parts {339+ if len(part.Rings) != 1 {340+ return nil, fmt.Errorf("line geometry has an invalid part")341+ }342+ lines = append(lines, part.Rings[0])343+ }344+ if len(lines) == 1 {345+ return marshalGeometry("LineString", lines[0])346+ }347+ return marshalGeometry("MultiLineString", lines)348+ }349+ return nil, fmt.Errorf("geometry kind %d cannot be an inline shape", geometry.Kind)350+}351+352+func marshalGeometry(kind string, coordinates any) ([]payloadGeometry, error) {353+ data, err := json.Marshal(coordinates)354+ if err != nil {355+ return nil, fmt.Errorf("marshal %s geometry: %w", kind, err)356+ }357+ return []payloadGeometry{{Type: kind, Coordinates: data}}, nil358+}359+360+func buildVNextWorld(world vnext.World) (*worldModel, error) {361+ payload, locations, _, err := presentVNextWorld(world)362+ if err != nil {363+ return nil, err364+ }365+ space := world.CoordinateSpace366+ grid := tileGrid{SourceZoom: int(space.SourceZoom), FirstTile: int(space.FirstTile), TileSize: int(space.TileSize), Size: int(space.Size)}367+ return buildProjectedWorld(world.ID, payload, locations, grid), nil368+}369+370+func buildProjectedWorld(slug string, decoded worldPayload, locations []bundle.Location, grid tileGrid) *worldModel {371+ model := &worldModel{372+ Slug: slug, Lenses: decoded.Lenses, Attrs: decoded.Attrs, Grid: grid,373+ ByID: map[string]*collectionModel{}, PointByID: map[string]*pointModel{}, ShapeByID: map[string]*shapeModel{},374+ }375+ for _, account := range decoded.Merged {376+ if account.Origin {377+ model.Origin = account.Source378+ break379+ }380+ }381+ buildProjectedCollections(model, decoded.Collections, grid)382+ buildProjectedPoints(model, locations, grid)383+ for _, shape := range model.Shapes {384+ shape.Depth = shapeDepth(model, shape, 0)385+ }386+ return model387+}388+389+func buildProjectedCollections(model *worldModel, collections []payloadCollection, grid tileGrid) {390+ for at, held := range collections {391+ kind := held.Kind392+ if kind == "" {393+ kind = semconv.GeometryPoint394+ }395+ collection := &collectionModel{396+ ID: strconv.FormatInt(held.ID, 10), Title: held.Title, Kind: kind, Group: held.Group,397+ Icon: held.Icon, IconAsset: held.IconAsset, Color: held.Color, IconColor: held.IconColor,398+ Attrs: held.Attrs, Curated: semconv.LabelPolicy(kind, held.Attrs), RenderAs: semconv.RenderAs(held.Attrs, ""),399+ Hidden: held.Visible != nil && !*held.Visible, Index: at, Count: len(held.Features),400+ }401+ model.Members = append(model.Members, collection)402+ model.ByID[collection.ID] = collection403+ for _, feature := range held.Features {404+ shape := buildShape(feature, collection, grid)405+ collection.Shapes = append(collection.Shapes, shape)406+ model.Shapes = append(model.Shapes, shape)407+ model.ShapeByID[shape.ID] = shape408+ }409+ }410+}411+412+func buildProjectedPoints(model *worldModel, locations []bundle.Location, grid tileGrid) {413+ for _, location := range locations {414+ owner := int(location.Owner)415+ if owner >= len(model.Members) {416+ continue417+ }418+ collection := model.Members[owner]419+ x, y := grid.project(location.Lat, location.Lng)420+ pin := &pointModel{421+ ID: strconv.FormatInt(location.ID, 10), Title: location.Title, Lat: location.Lat, Lng: location.Lng,422+ X: x, Y: y, Shard: location.Shard, Collection: collection,423+ }424+ model.Points = append(model.Points, pin)425+ model.PointByID[pin.ID] = pin426+ collection.Count++427+ }428+}429+430+func claimsToAttrs(claims []vnext.Property) map[string]string { return propertiesToAttrs(claims) }431+432+func propertiesToAttrs(properties []vnext.Property) map[string]string {433+ out := make(map[string]string, len(properties))434+ for _, property := range properties {435+ if property.Field.Name != "" {436+ out[property.Field.Name] = projectedValue(property.Value)437+ }438+ }439+ return out440+}441+442+func projectedValue(value vnext.Value) string {443+ switch value.Kind {444+ case vnext.KindBool:445+ return strconv.FormatBool(value.Bool)446+ case vnext.KindInt64:447+ return strconv.FormatInt(value.Int64, 10)448+ case vnext.KindFloat64:449+ return strconv.FormatFloat(value.Float64, 'g', -1, 64)450+ case vnext.KindString:451+ return value.String452+ case vnext.KindBytes:453+ return base64.StdEncoding.EncodeToString(value.Bytes)454+ case vnext.KindID:455+ return value.ID.String()456+ default:457+ return ""458+ }459+}460+461+func worldOrigin(world vnext.World) string {462+ for _, set := range world.FeatureSets {463+ for _, feature := range set.Features {464+ if len(feature.Provenance) > 0 {465+ return feature.Provenance[0].Source466+ }467+ }468+ }469+ return ""470+}471+472+func projectedColor(kind string, style vnext.Style) string {473+ if kind == semconv.GeometryPath && style.Stroke != "" {474+ return style.Stroke475+ }476+ return style.Fill477+}478+479+func rasterTiles(template string) string {480+ held := strings.TrimPrefix(template, bundle.TilesPrefix)481+ if at := strings.IndexByte(held, '/'); at >= 0 {482+ return held[:at]483+ }484+ return held485+}486+487+func cellRect(rect *vnext.RasterRect) *cells.Rect {488+ if rect == nil {489+ return nil490+ }491+ return &cells.Rect{X: rect.X, Y: rect.Y, Width: rect.Width, Height: rect.Height}492+}493+494+func numericTail(value string) int64 {495+ if at := strings.LastIndexByte(value, '/'); at >= 0 {496+ value = value[at+1:]497+ }498+ parsed, err := strconv.ParseInt(value, 10, 64)499+ if err != nil {500+ return 0501+ }502+ return parsed503+}504+505+func hasFeatureText(text featureText) bool {506+ return text.Description != "" || len(text.Links) > 0 || len(text.Attrs) > 0507+}508+509+func putAttr(attrs map[string]string, key, value string) {510+ if value != "" {511+ attrs[key] = value512+ }513+}514+515+func intPointer(value int) *int { return &value }516+func boolPointer(value bool) *bool { return &value }@@ -55,11 +55,17 @@ type payloadGrid struct {
5555 5656 // payloadLens is one raster pyramid picturing this world.5757 type payloadLens struct {58- Name string `json:"name"`59- Tiles string `json:"tiles"`60- MinZoom int `json:"minZoom"`61- MaxZoom int `json:"maxZoom"`62- Shard int `json:"shard"`58+ Name string `json:"name"`59+ Tiles string `json:"tiles"`60+ MinZoom int `json:"minZoom"`61+ MaxZoom int `json:"maxZoom"`62+ FullZoom int `json:"fullZoom"`63+ SourceZoom int `json:"sourceZoom"`64+ Formats []string `json:"formats"`65+ Interpolate bool `json:"interpolate"`66+ Background string `json:"background,omitempty"`67+ Shard int `json:"shard,omitempty"`68+ Coverage map[string]payloadCoverage `json:"coverage,omitempty"`6369 // Bounds is the raster window the pyramid fills; Surface is the ground6470 // that window pictures. They differ on a split sheet, where the window6571 // was grown to take in a title drawn beside the map, and anything that@@ -68,14 +74,23 @@ type payloadLens struct {
6874 Surface *cells.Rect `json:"surface"`6975 }7076 77+type payloadCoverage struct {78+ X int `json:"x"`79+ Y int `json:"y"`80+ W int `json:"w"`81+ H int `json:"h"`82+ Bits string `json:"bits"`83+}84+7185 // payloadCollection is one ordered group of features.7286 type payloadCollection struct {73- ID int64 `json:"id"`74- Title string `json:"title"`75- Kind string `json:"kind"`76- Group string `json:"group"`77- Icon string `json:"icon"`78- IconAsset string `json:"iconAsset"`87+ ID int64 `json:"id"`88+ Title string `json:"title"`89+ Kind string `json:"kind"`90+ Group string `json:"group"`91+ Icon string `json:"icon"`92+ IconAsset string `json:"iconAsset"`93+ IconPicture bool `json:"iconPicture,omitempty"`7994 // Color is the collection's own accent and IconColor the older spelling8095 // of the same fact. Both are read because the seam reads both, and a8196 // legend that fell back to the palette where the seam honoured a@@ -277,129 +292,20 @@ func (a *App) world(volume hostenv.Volume, slug string) *worldModel {
277292 if held, ok := a.worlds.get(key); ok {278293 return held279294 }280- model, err := buildWorld(volume, manifest, slug)295+ semantic, err := a.semanticVolume(volume)281296 if err != nil {282297 return nil283298 }284- a.worlds.put(key, model)285- return model286-}287-288-func buildWorld(volume hostenv.Volume, manifest bundle.Manifest, slug string) (*worldModel, error) {289- payload, err := readEntry(volume, bundle.WorldEntryName(slug, bundle.WorldSuffix))290- if err != nil {291- return nil, err292- }293- var decoded worldPayload294- if err := json.Unmarshal(payload, &decoded); err != nil {295- return nil, fmt.Errorf("world %s payload: %w", slug, err)296- }297-298- grid := tileGrid{299- SourceZoom: manifest.TileGrid.SourceZoom,300- FirstTile: manifest.TileGrid.FirstTile,301- TileSize: manifest.TileGrid.TileSize,302- Size: manifest.TileGrid.Size,303- }304- if own := decoded.Grid; own != nil {305- if own.SourceZoom != nil {306- grid.SourceZoom = *own.SourceZoom307- }308- if own.FirstTile != nil {309- grid.FirstTile = *own.FirstTile310- }311- if own.TileSize != nil {312- grid.TileSize = *own.TileSize313- }314- if own.Size != nil {315- grid.Size = *own.Size316- }317- }318-319- model := &worldModel{320- Slug: slug,321- Lenses: decoded.Lenses,322- Attrs: decoded.Attrs,323- Grid: grid,324- ByID: map[string]*collectionModel{},325- PointByID: map[string]*pointModel{},326- ShapeByID: map[string]*shapeModel{},327- }328- for _, account := range decoded.Merged {329- if account.Origin {330- model.Origin = account.Source331- break332- }333- }334-335- for at, held := range decoded.Collections {336- kind := held.Kind337- if kind == "" {338- kind = semconv.GeometryPoint339- }340- collection := &collectionModel{341- ID: strconv.FormatInt(held.ID, 10),342- Title: held.Title,343- Kind: kind,344- Group: held.Group,345- Icon: held.Icon,346- IconAsset: held.IconAsset,347- Color: held.Color,348- IconColor: held.IconColor,349-350- Attrs: held.Attrs,351- Curated: semconv.LabelPolicy(kind, held.Attrs),352- RenderAs: semconv.RenderAs(held.Attrs, ""),353- Hidden: held.Visible != nil && !*held.Visible,354- Index: at,355- }356- model.Members = append(model.Members, collection)357- model.ByID[collection.ID] = collection358-359- for _, feature := range held.Features {360- shape := buildShape(feature, collection, grid)361- collection.Shapes = append(collection.Shapes, shape)362- model.Shapes = append(model.Shapes, shape)363- model.ShapeByID[shape.ID] = shape364- }365- collection.Count = len(held.Features)366- }367-368- // Points arrive packed. The owner column indexes the collections array,369- // which is why the array's order is significant and why nothing here370- // sorts it.371- if packed, err := readEntry(volume, bundle.WorldEntryName(slug, bundle.PackedSuffix)); err == nil {372- if locations, err := bundle.UnpackLocations(packed); err == nil {373- for _, location := range locations {374- owner := int(location.Owner)375- if owner < 0 || owner >= len(model.Members) {376- continue377- }378- collection := model.Members[owner]379- x, y := grid.project(location.Lat, location.Lng)380- pin := &pointModel{381- ID: strconv.FormatInt(location.ID, 10),382- Title: location.Title,383- Lat: location.Lat,384- Lng: location.Lng,385- X: x,386- Y: y,387- Shard: location.Shard,388- Collection: collection,389- }390- model.Points = append(model.Points, pin)391- model.PointByID[pin.ID] = pin392- collection.Count++393- }394- }299+ world, held := semanticWorld(semantic, slug)300+ if !held {301+ return nil395302 }396-397- // Depth is the parent chain, so the feature index can indent a398- // sub-watershed under its watershed without the template counting.399- for _, shape := range model.Shapes {400- shape.Depth = shapeDepth(model, shape, 0)303+ model, err := buildVNextWorld(world)304+ if err != nil {305+ return nil401306 }402- return model, nil307+ a.worlds.put(key, model)308+ return model403309 }404310 405311 func shapeDepth(model *worldModel, shape *shapeModel, guard int) int {@@ -597,8 +503,12 @@ func (a *App) text(volume hostenv.Volume, world string) map[string]featureText {
597503 return held598504 }599505 out := map[string]featureText{}600- if data, err := readEntry(volume, bundle.WorldEntryName(world, bundle.TextSuffix)); err == nil {601- _ = json.Unmarshal(data, &out)506+ if semantic, err := a.semanticVolume(volume); err == nil {507+ if held, ok := semanticWorld(semantic, world); ok {508+ if _, _, projected, err := presentVNextWorld(held); err == nil {509+ out = projected510+ }511+ }602512 }603513 a.texts.put(key, out)604514 return out@@ -0,0 +1,246 @@
1+export interface FieldSchema {2+ readonly id: string;3+ readonly name: string;4+ readonly kind: number;5+ readonly optional?: boolean;6+}7+8+export interface TypeSchema {9+ readonly id: string;10+ readonly name: string;11+ readonly fields: readonly FieldSchema[];12+}13+14+export interface Schema {15+ readonly types: readonly TypeSchema[];16+}17+18+export interface FieldProjection {19+ readonly known: readonly string[];20+ readonly foreign: readonly string[];21+}22+23+const HEADER_SIZE = 80;24+const DIRECTORY_SIZE = 64;25+const MAGIC = "ATLASPK\0";26+27+interface ColumnDirectory {28+ readonly id: string;29+ readonly kind: number;30+ readonly optional: boolean;31+ readonly present: Uint8Array;32+ readonly payload: Uint8Array;33+}34+35+export class PackedColumn {36+ readonly id: string;37+ readonly kind: number;38+ readonly optional: boolean;39+ readonly #rows: number;40+ readonly #present: Uint8Array;41+ readonly #payload: Uint8Array;42+ readonly #view: DataView;43+ readonly #decoder = new TextDecoder();44+45+ constructor(rows: number, directory: ColumnDirectory) {46+ this.id = directory.id;47+ this.kind = directory.kind;48+ this.optional = directory.optional;49+ this.#rows = rows;50+ this.#present = directory.present;51+ this.#payload = directory.payload;52+ this.#view = new DataView(directory.payload.buffer, directory.payload.byteOffset, directory.payload.byteLength);53+ this.#validateShape();54+ }55+56+ isNull(row: number): boolean {57+ this.#checkRow(row);58+ const byte = this.#present[Math.floor(row / 8)];59+ return byte === undefined || (byte & (1 << (row % 8))) === 0;60+ }61+62+ bool(row: number): boolean | null {63+ this.#expect(1);64+ return this.isNull(row) ? null : this.#payload[row] !== 0;65+ }66+67+ int64(row: number): bigint | null {68+ this.#expect(2);69+ return this.isNull(row) ? null : this.#view.getBigInt64(row * 8, true);70+ }71+72+ float64(row: number): number | null {73+ this.#expect(3);74+ return this.isNull(row) ? null : this.#view.getFloat64(row * 8, true);75+ }76+77+ string(row: number): string | null {78+ this.#expect(4);79+ const value = this.#variable(row);80+ return value === null ? null : this.#decoder.decode(value);81+ }82+83+ bytes(row: number): Uint8Array | null {84+ this.#expect(5);85+ return this.#variable(row);86+ }87+88+ identifier(row: number): string | null {89+ this.#expect(6);90+ if (this.isNull(row)) return null;91+ return formatID(this.#payload.subarray(row * 16, (row + 1) * 16));92+ }93+94+ #variable(row: number): Uint8Array | null {95+ if (this.isNull(row)) return null;96+ const offsetsLength = (this.#rows + 1) * 4;97+ const start = this.#view.getUint32(row * 4, true);98+ const end = this.#view.getUint32((row + 1) * 4, true);99+ return this.#payload.subarray(offsetsLength + start, offsetsLength + end);100+ }101+102+ #validateShape(): void {103+ const expected = this.kind === 1 ? this.#rows104+ : this.kind === 2 || this.kind === 3 ? this.#rows * 8105+ : this.kind === 6 ? this.#rows * 16106+ : undefined;107+ if (expected !== undefined && this.#payload.byteLength !== expected) throw new Error(`column ${this.id} has an invalid payload length`);108+ if (this.kind === 4 || this.kind === 5) {109+ const offsetsLength = (this.#rows + 1) * 4;110+ if (this.#payload.byteLength < offsetsLength) throw new Error(`column ${this.id} has truncated offsets`);111+ let previous = 0;112+ const contentLength = this.#payload.byteLength - offsetsLength;113+ for (let row = 0; row <= this.#rows; row++) {114+ const offset = this.#view.getUint32(row * 4, true);115+ if (offset < previous || offset > contentLength) throw new Error(`column ${this.id} has invalid offsets`);116+ previous = offset;117+ }118+ }119+ }120+121+ #expect(kind: number): void {122+ if (this.kind !== kind) throw new Error(`column ${this.id} has wire kind ${this.kind}, not ${kind}`);123+ }124+125+ #checkRow(row: number): void {126+ if (!Number.isInteger(row) || row < 0 || row >= this.#rows) throw new RangeError(`row ${row} is outside the table`);127+ }128+}129+130+export class PackedTable {131+ readonly rows: number;132+ readonly rootType: string;133+ readonly schemaHash: string;134+ readonly columns: readonly PackedColumn[];135+ readonly #byID: ReadonlyMap<string, PackedColumn>;136+137+ private constructor(rows: number, rootType: string, schemaHash: string, columns: readonly PackedColumn[]) {138+ this.rows = rows;139+ this.rootType = rootType;140+ this.schemaHash = schemaHash;141+ this.columns = columns;142+ this.#byID = new Map(columns.map((column) => [column.id, column]));143+ }144+145+ static over(source: ArrayBuffer | Uint8Array): PackedTable {146+ const bytes = source instanceof Uint8Array ? source : new Uint8Array(source);147+ const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength);148+ if (bytes.byteLength < HEADER_SIZE || new TextDecoder().decode(bytes.subarray(0, 8)) !== MAGIC) throw new Error("typed block has no Atlas magic");149+ if (view.getUint16(8, true) !== 1) throw new Error("typed block framing is unsupported");150+ const rows = view.getUint32(12, true);151+ const count = view.getUint32(16, true);152+ const directoryEnd = HEADER_SIZE + count * DIRECTORY_SIZE;153+ if (view.getUint32(68, true) !== DIRECTORY_SIZE || view.getBigUint64(72, true) !== BigInt(directoryEnd) || directoryEnd > bytes.byteLength) {154+ throw new Error("typed block directory framing is invalid");155+ }156+ const columns: PackedColumn[] = [];157+ for (let index = 0; index < count; index++) {158+ const at = HEADER_SIZE + index * DIRECTORY_SIZE;159+ const kind = bytes[at + 16];160+ if (kind === undefined || kind < 1 || kind > 6 || bytes[at + 17] !== kind) throw new Error("column has unsupported encoding");161+ const present = boundedSlice(bytes, view.getBigUint64(at + 20, true), view.getBigUint64(at + 28, true));162+ const payload = boundedSlice(bytes, view.getBigUint64(at + 36, true), view.getBigUint64(at + 44, true));163+ if (present.byteLength !== Math.ceil(rows / 8)) throw new Error("column presence bitmap is invalid");164+ if (crc32(present, payload) !== view.getUint32(at + 52, true)) throw new Error("column checksum does not match");165+ columns.push(new PackedColumn(rows, {166+ id: formatID(bytes.subarray(at, at + 16)), kind,167+ optional: (view.getUint16(at + 18, true) & 1) !== 0, present, payload,168+ }));169+ }170+ return new PackedTable(rows, formatID(bytes.subarray(52, 68)), hex(bytes.subarray(20, 52)), columns);171+ }172+173+ column(id: string): PackedColumn | undefined { return this.#byID.get(normalizeID(id)); }174+}175+176+export function mergeSchemas(...schemas: readonly Schema[]): Schema {177+ const types = new Map<string, { id: string; name: string; fields: Map<string, FieldSchema> }>();178+ for (const schema of schemas) {179+ for (const incoming of schema.types) {180+ const typeID = normalizeID(incoming.id);181+ const current = types.get(typeID) ?? { id: typeID, name: incoming.name, fields: new Map<string, FieldSchema>() };182+ if (incoming.name < current.name) current.name = incoming.name;183+ for (const field of incoming.fields) {184+ const fieldID = normalizeID(field.id);185+ const held = current.fields.get(fieldID);186+ if (held && held.kind !== field.kind) throw new Error(`field ${fieldID} changes wire kind from ${held.kind} to ${field.kind}`);187+ const name = held && held.name < field.name ? held.name : field.name;188+ const optional = Boolean(held?.optional || field.optional);189+ current.fields.set(fieldID, optional ? { id: fieldID, name, kind: field.kind, optional: true } : { id: fieldID, name, kind: field.kind });190+ }191+ types.set(typeID, current);192+ }193+ }194+ return {195+ types: [...types.values()].sort(byID).map((type) => ({196+ id: type.id, name: type.name, fields: [...type.fields.values()].sort(byID),197+ })),198+ };199+}200+201+export function projectFields(table: PackedTable, knownSchema: Schema, fileSchema: Schema): FieldProjection {202+ const fileType = fileSchema.types.find((type) => normalizeID(type.id) === table.rootType);203+ if (!fileType) throw new Error(`file schema does not describe table type ${table.rootType}`);204+ const fileKinds = new Map(fileType.fields.map((field) => [normalizeID(field.id), field.kind]));205+ const knownType = knownSchema.types.find((type) => normalizeID(type.id) === table.rootType);206+ const knownKinds = new Map((knownType?.fields ?? []).map((field) => [normalizeID(field.id), field.kind]));207+ const known: string[] = [];208+ const foreign: string[] = [];209+ for (const column of table.columns) {210+ if (fileKinds.get(column.id) !== column.kind) throw new Error(`file schema disagrees with wire kind for ${column.id}`);211+ const expected = knownKinds.get(column.id);212+ if (expected === undefined) foreign.push(column.id);213+ else if (expected !== column.kind) throw new Error(`known schema disagrees with wire kind for ${column.id}`);214+ else known.push(column.id);215+ }216+ return { known, foreign };217+}218+219+function boundedSlice(bytes: Uint8Array, offset64: bigint, length64: bigint): Uint8Array {220+ if (offset64 > BigInt(Number.MAX_SAFE_INTEGER) || length64 > BigInt(Number.MAX_SAFE_INTEGER)) throw new Error("column range exceeds JavaScript precision");221+ const offset = Number(offset64);222+ const length = Number(length64);223+ if (offset > bytes.byteLength || length > bytes.byteLength - offset) throw new Error("column range exceeds block");224+ return bytes.subarray(offset, offset + length);225+}226+227+function normalizeID(id: string): string {228+ const plain = id.toLowerCase().replaceAll("-", "");229+ if (!/^[0-9a-f]{32}$/.test(plain)) throw new Error(`ID ${id} is not 128 bits`);230+ return `${plain.slice(0, 8)}-${plain.slice(8, 12)}-${plain.slice(12, 16)}-${plain.slice(16, 20)}-${plain.slice(20)}`;231+}232+233+function formatID(bytes: Uint8Array): string { return normalizeID(hex(bytes)); }234+function hex(bytes: Uint8Array): string { return [...bytes].map((byte) => byte.toString(16).padStart(2, "0")).join(""); }235+function byID(left: { readonly id: string }, right: { readonly id: string }): number { return left.id.localeCompare(right.id); }236+237+function crc32(...parts: readonly Uint8Array[]): number {238+ let crc = 0xffffffff;239+ for (const part of parts) {240+ for (const byte of part) {241+ crc ^= byte;242+ for (let bit = 0; bit < 8; bit++) crc = (crc >>> 1) ^ (0xedb88320 & -(crc & 1));243+ }244+ }245+ return (crc ^ 0xffffffff) >>> 0;246+}@@ -0,0 +1,149 @@
1+import { strict as assert } from "node:assert";2+import test from "node:test";3+import {4+ mergeSchemas, PackedTable, projectFields,5+ type FieldSchema, type Schema,6+} from "../data/vnext.ts";7+8+const ROOT = "11111111-1111-5111-8111-111111111111";9+const TITLE = "22222222-2222-5222-8222-222222222222";10+const EMERGENCY = "33333333-3333-5333-8333-333333333333";11+12+test("vNext packed columns open as zero-copy typed views", () => {13+ const buffer = fixtureBlock();14+ const envelope = new Uint8Array(buffer.byteLength + 13);15+ envelope.set(new Uint8Array(buffer), 7);16+ const table = PackedTable.over(envelope.subarray(7, 7 + buffer.byteLength));17+ assert.equal(table.rows, 2);18+ assert.equal(table.rootType, ROOT);19+ assert.equal(table.schemaHash, Array.from({ length: 32 }, (_, i) => i.toString(16).padStart(2, "0")).join(""));20+21+ const title = table.column(TITLE);22+ const emergency = table.column(EMERGENCY);23+ assert.ok(title);24+ assert.ok(emergency);25+ assert.equal(title.string(0), "Clinic");26+ assert.equal(title.isNull(1), true);27+ assert.equal(emergency.int64(0), 4n);28+ assert.equal(emergency.int64(1), 5n);29+});30+31+test("old browser schemas classify rather than discard foreign fields", () => {32+ const title: FieldSchema = { id: TITLE, name: "title", kind: 4 };33+ const emergency: FieldSchema = { id: EMERGENCY, name: "emergencyLevel", kind: 2, optional: true };34+ const base: Schema = { types: [{ id: ROOT, name: "Feature", fields: [title] }] };35+ const branch: Schema = { types: [{ id: ROOT, name: "Feature", fields: [emergency] }] };36+ const file = mergeSchemas(base, branch);37+ const projection = projectFields(PackedTable.over(fixtureBlock()), base, file);38+ assert.deepEqual(projection.known, [TITLE]);39+ assert.deepEqual(projection.foreign, [EMERGENCY]);40+});41+42+test("sideward schema union accepts new IDs and refuses changed meanings", () => {43+ const left: Schema = { types: [{ id: ROOT, name: "Feature", fields: [{ id: TITLE, name: "title", kind: 4 }] }] };44+ const right: Schema = { types: [{ id: ROOT, name: "Feature", fields: [{ id: EMERGENCY, name: "level", kind: 2 }] }] };45+ assert.equal(mergeSchemas(left, right).types[0]?.fields.length, 2);46+47+ const conflict: Schema = { types: [{ id: ROOT, name: "Feature", fields: [{ id: TITLE, name: "title", kind: 2 }] }] };48+ assert.throws(() => mergeSchemas(left, conflict), /wire kind/);49+});50+51+test("a corrupt column is refused at the browser boundary", () => {52+ const buffer = fixtureBlock();53+ const bytes = new Uint8Array(buffer);54+ bytes[buffer.byteLength - 1] = (bytes[buffer.byteLength - 1] ?? 0) ^ 0xff;55+ assert.throws(() => PackedTable.over(buffer), /checksum/);56+});57+58+function fixtureBlock(): ArrayBuffer {59+ const title = variableColumn([new TextEncoder().encode("Clinic"), null]);60+ const emergency = fixedInt64Column([4n, 5n]);61+ return block([62+ { id: TITLE, kind: 4, optional: true, ...title },63+ { id: EMERGENCY, kind: 2, optional: false, ...emergency },64+ ]);65+}66+67+interface EncodedColumn {68+ readonly id: string;69+ readonly kind: number;70+ readonly optional: boolean;71+ readonly present: Uint8Array;72+ readonly data: Uint8Array;73+}74+75+function block(columns: readonly EncodedColumn[]): ArrayBuffer {76+ const header = 80;77+ const directory = 64;78+ const payloadAt = header + directory * columns.length;79+ const length = columns.reduce((sum, column) => sum + column.present.length + column.data.length, payloadAt);80+ const buffer = new ArrayBuffer(length);81+ const bytes = new Uint8Array(buffer);82+ const view = new DataView(buffer);83+ bytes.set(new TextEncoder().encode("ATLASPK\0"), 0);84+ view.setUint16(8, 1, true);85+ view.setUint32(12, 2, true);86+ view.setUint32(16, columns.length, true);87+ for (let i = 0; i < 32; i++) bytes[20 + i] = i;88+ bytes.set(idBytes(ROOT), 52);89+ view.setUint32(68, directory, true);90+ view.setBigUint64(72, BigInt(payloadAt), true);91+92+ let payload = payloadAt;93+ columns.forEach((column, index) => {94+ const at = header + index * directory;95+ bytes.set(idBytes(column.id), at);96+ bytes[at + 16] = column.kind;97+ bytes[at + 17] = column.kind;98+ view.setUint16(at + 18, column.optional ? 1 : 0, true);99+ view.setBigUint64(at + 20, BigInt(payload), true);100+ view.setBigUint64(at + 28, BigInt(column.present.length), true);101+ bytes.set(column.present, payload);102+ payload += column.present.length;103+ view.setBigUint64(at + 36, BigInt(payload), true);104+ view.setBigUint64(at + 44, BigInt(column.data.length), true);105+ bytes.set(column.data, payload);106+ payload += column.data.length;107+ view.setUint32(at + 52, crc32(column.present, column.data), true);108+ });109+ return buffer;110+}111+112+function variableColumn(values: readonly (Uint8Array | null)[]): Pick<EncodedColumn, "present" | "data"> {113+ const present = new Uint8Array([0b01]);114+ const offsets = new Uint32Array(values.length + 1);115+ const length = values.reduce((sum, value) => sum + (value?.length ?? 0), 0);116+ const data = new Uint8Array(offsets.byteLength + length);117+ let at = 0;118+ values.forEach((value, index) => {119+ new DataView(data.buffer).setUint32(index * 4, at, true);120+ if (value) {121+ data.set(value, offsets.byteLength + at);122+ at += value.length;123+ }124+ });125+ new DataView(data.buffer).setUint32(values.length * 4, at, true);126+ return { present, data };127+}128+129+function fixedInt64Column(values: readonly bigint[]): Pick<EncodedColumn, "present" | "data"> {130+ const present = new Uint8Array([0b11]);131+ const data = new Uint8Array(values.length * 8);132+ values.forEach((value, index) => new DataView(data.buffer).setBigInt64(index * 8, value, true));133+ return { present, data };134+}135+136+function idBytes(id: string): Uint8Array {137+ return Uint8Array.from(id.replaceAll("-", "").match(/../g) ?? [], (pair) => Number.parseInt(pair, 16));138+}139+140+function crc32(...parts: readonly Uint8Array[]): number {141+ let crc = 0xffffffff;142+ for (const part of parts) {143+ for (const byte of part) {144+ crc ^= byte;145+ for (let bit = 0; bit < 8; bit++) crc = (crc >>> 1) ^ (0xedb88320 & -(crc & 1));146+ }147+ }148+ return (crc ^ 0xffffffff) >>> 0;149+}