1package vnext23import (4 "crypto/sha256"5 "encoding/binary"6 "fmt"7 "math"8 "slices"9 "strings"10)1112const (13 volumeTableName = "data/volumes.pack"14 worldTableName = "data/worlds.pack"15 coordinateTableName = "data/coordinate-spaces.pack"16 featureSetTableName = "data/feature-sets.pack"17 propertyDefinitionTableName = "data/property-definitions.pack"18 featureTableName = "data/features.pack"19 relationshipTableName = "data/relationships.pack"20 provenanceTableName = "data/provenance.pack"21 rasterTableName = "data/raster-pyramids.pack"22 rasterFormatTableName = "data/raster-formats.pack"23 rasterCoverageTableName = "data/raster-coverage.pack"24 presentationTableName = "data/presentations.pack"25 styleTableName = "data/styles.pack"26 layerTableName = "data/layers.pack"27 legendTableName = "data/legend.pack"28 assetTableName = "data/assets.pack"29)3031// NamedTable assigns a stable container entry to a typed table.32type NamedTable struct {33 Name string34 Table Table35}3637// Blob is an opaque, independently range-readable payload.38type Blob struct {39 Name string40 Data []byte41 Path string42 temporary bool43}4445// Bundle is the complete compiled form accepted by Write.46type Bundle struct {47 VolumeID string48 Release Release49 Schema Schema50 Tables []NamedTable51 Blobs []Blob52}5354type tableBuilder struct {55 typ Type56 columns map[ID][]Value57 rows int58}5960func newTableBuilder(schema Schema, typeName string) (*tableBuilder, error) {61 typ, ok := schema.Type(CoreID("type." + typeName))62 if !ok {63 return nil, fmt.Errorf("schema has no %s type", typeName)64 }65 columns := make(map[ID][]Value, len(typ.Fields))66 for _, field := range typ.Fields {67 columns[field.ID] = nil68 }69 return &tableBuilder{typ: typ, columns: columns}, nil70}7172func (builder *tableBuilder) add(values map[ID]Value) error {73 for id := range values {74 if _, held := builder.columns[id]; !held {75 return fmt.Errorf("type %s has no field %s", builder.typ.Name, id)76 }77 }78 for _, field := range builder.typ.Fields {79 value, held := values[field.ID]80 if !held {81 value = NullValue()82 }83 builder.columns[field.ID] = append(builder.columns[field.ID], value)84 }85 builder.rows++86 return nil87}8889func (builder *tableBuilder) table() Table {90 table := Table{TypeID: builder.typ.ID, Rows: builder.rows, Columns: make([]Column, 0, len(builder.typ.Fields))}91 for _, field := range builder.typ.Fields {92 table.Columns = append(table.Columns, Column{Field: field, Values: builder.columns[field.ID]})93 }94 return canonicalTable(table)95}9697// Compile validates a complete semantic Volume and plans it into typed tables98// plus content-addressed blobs.99func Compile(volume Volume) (Bundle, error) {100 if err := validateVolume(volume); err != nil {101 return Bundle{}, err102 }103 schema, err := schemaFor(volume)104 if err != nil {105 return Bundle{}, err106 }107 builders, err := makeBuilders(schema)108 if err != nil {109 return Bundle{}, err110 }111 blobs, err := compileRows(volume, builders)112 if err != nil {113 return Bundle{}, err114 }115 names := tableNames()116 tables := make([]NamedTable, 0, len(names))117 physical := make(map[string]Table, len(names))118 for _, name := range names {119 table := builders[name].table()120 if err := table.validate(); err != nil {121 return Bundle{}, fmt.Errorf("compile %s: %w", name, err)122 }123 physical[name] = table124 partitioned := physical[featureTableName].Rows > 0 &&125 (name == featureTableName || name == relationshipTableName || name == provenanceTableName)126 if !partitioned {127 tables = append(tables, NamedTable{Name: name, Table: table})128 }129 }130 partitions, index, err := partitionFeatureTables(volume, physical)131 if err != nil {132 return Bundle{}, err133 }134 tables = append(tables, partitions...)135 blobs = append(blobs, index)136 slices.SortFunc(tables, func(left, right NamedTable) int { return strings.Compare(left.Name, right.Name) })137 return Bundle{138 VolumeID: volume.ID,139 Release: Release{Title: volume.Title, Worlds: len(volume.Worlds)},140 Schema: schema,141 Tables: tables,142 Blobs: blobs,143 }, nil144}145146func schemaFor(volume Volume) (Schema, error) {147 standard := StandardSchema()148 extensions := make(map[ID]map[ID]Field)149 for _, world := range volume.Worlds {150 if err := collectClaims(extensions, standard, "world", world.Claims); err != nil {151 return Schema{}, err152 }153 for _, set := range world.FeatureSets {154 if err := collectClaims(extensions, standard, "featureSet", set.Claims); err != nil {155 return Schema{}, err156 }157 for _, field := range set.Properties {158 if err := collectField(extensions, standard, "feature", field); err != nil {159 return Schema{}, fmt.Errorf("feature set %s: %w", set.ID, err)160 }161 }162 }163 }164 fragment := Schema{}165 for typeID, fields := range extensions {166 typ, _ := standard.Type(typeID)167 extension := Type{ID: typeID, Name: typ.Name, Fields: make([]Field, 0, len(fields))}168 for _, field := range fields {169 extension.Fields = append(extension.Fields, field)170 }171 fragment.Types = append(fragment.Types, extension)172 }173 return UnionSchemas(standard, fragment)174}175176func collectClaims(extensions map[ID]map[ID]Field, standard Schema, typeName string, claims []Property) error {177 for _, claim := range claims {178 field := claim.Field179 if field.ID.isZero() {180 return fmt.Errorf("%s claim %s has no field definition", typeName, claim.fieldID())181 }182 if err := collectField(extensions, standard, typeName, field); err != nil {183 return err184 }185 }186 return nil187}188189func collectField(extensions map[ID]map[ID]Field, standard Schema, typeName string, field Field) error {190 typeID := CoreID("type." + typeName)191 core, _ := standard.Type(typeID)192 if _, held := core.Field(field.ID); held {193 return fmt.Errorf("extension reuses core field %s", field.ID)194 }195 field.Optional = true196 if extensions[typeID] == nil {197 extensions[typeID] = make(map[ID]Field)198 }199 current, held := extensions[typeID][field.ID]200 if held && current.Kind != field.Kind {201 return fmt.Errorf("field %s has conflicting kinds", field.ID)202 }203 if held && current.Name < field.Name {204 field.Name = current.Name205 }206 extensions[typeID][field.ID] = field207 return nil208}209210func makeBuilders(schema Schema) (map[string]*tableBuilder, error) {211 typeNames := []string{"volume", "world", "coordinate", "featureSet", "propertyDefinition", "feature", "relationship", "provenance", "rasterPyramid", "rasterFormat", "rasterCoverage", "presentation", "style", "layer", "legend", "asset"}212 names := tableNames()213 builders := make(map[string]*tableBuilder, len(names))214 for index, name := range names {215 builder, err := newTableBuilder(schema, typeNames[index])216 if err != nil {217 return nil, err218 }219 builders[name] = builder220 }221 return builders, nil222}223224func tableNames() []string {225 return []string{226 volumeTableName, worldTableName, coordinateTableName, featureSetTableName,227 propertyDefinitionTableName, featureTableName, relationshipTableName,228 provenanceTableName, rasterTableName, rasterFormatTableName, rasterCoverageTableName,229 presentationTableName, styleTableName,230 layerTableName, legendTableName, assetTableName,231 }232}233234func compileRows(volume Volume, builders map[string]*tableBuilder) ([]Blob, error) {235 if err := builders[volumeTableName].add(valuesOf("volume", map[string]Value{236 "id": StringValue(volume.ID), "title": StringValue(volume.Title),237 })); err != nil {238 return nil, err239 }240 for _, world := range volume.Worlds {241 if err := compileWorld(world, builders); err != nil {242 return nil, fmt.Errorf("world %s: %w", world.ID, err)243 }244 }245 return compileAssets(volume.Assets, builders[assetTableName])246}247248func compileWorld(world World, builders map[string]*tableBuilder) error {249 worldValues := valuesOf("world", map[string]Value{250 "id": StringValue(world.ID), "title": StringValue(world.Title),251 "coordinateSpace": StringValue(world.CoordinateSpace.ID),252 })253 addClaims(worldValues, world.Claims)254 if err := builders[worldTableName].add(worldValues); err != nil {255 return err256 }257 if err := compileCoordinate(world, builders[coordinateTableName]); err != nil {258 return err259 }260 for _, set := range world.FeatureSets {261 if err := compileFeatureSet(world.ID, set, builders); err != nil {262 return err263 }264 }265 for _, raster := range world.RasterPyramids {266 if err := compileRaster(world.ID, raster, builders); err != nil {267 return err268 }269 }270 return compilePresentation(world.ID, world.Presentation, builders)271}272273func compileCoordinate(world World, builder *tableBuilder) error {274 space := world.CoordinateSpace275 return builder.add(valuesOf("coordinate", map[string]Value{276 "id": StringValue(space.ID), "world": StringValue(world.ID),277 "kind": StringValue(space.Kind), "unit": StringValue(space.Unit),278 "definition": StringValue(space.Definition), "extent": BytesValue(encodeExtent(space.Extent)),279 "sourceZoom": Int64Value(space.SourceZoom), "originX": Int64Value(space.OriginX), "originY": Int64Value(space.OriginY),280 "tileSize": Int64Value(space.TileSize), "size": Int64Value(space.Size),281 }))282}283284func compileFeatureSet(worldID string, set FeatureSet, builders map[string]*tableBuilder) error {285 setValues := valuesOf("featureSet", map[string]Value{286 "id": StringValue(set.ID), "world": StringValue(worldID), "title": StringValue(set.Title),287 "semanticType": StringValue(set.SemanticType),288 })289 addClaims(setValues, set.Claims)290 if err := builders[featureSetTableName].add(setValues); err != nil {291 return err292 }293 contract := make(map[ID]Field, len(set.Properties))294 for _, field := range set.Properties {295 contract[field.ID] = field296 if err := builders[propertyDefinitionTableName].add(valuesOf("propertyDefinition", map[string]Value{297 "featureSet": StringValue(set.ID), "field": IDValue(field.ID), "name": StringValue(field.Name),298 "kind": Int64Value(int64(field.Kind)), "optional": BoolValue(field.Optional),299 })); err != nil {300 return err301 }302 }303 for _, feature := range set.Features {304 if err := compileFeature(set.ID, feature, contract, builders); err != nil {305 return err306 }307 }308 return nil309}310311func compileFeature(setID string, feature Feature, contract map[ID]Field, builders map[string]*tableBuilder) error {312 geometry, err := encodeGeometry(feature.Geometry)313 if err != nil {314 return fmt.Errorf("feature %s geometry: %w", feature.ID, err)315 }316 values := valuesOf("feature", map[string]Value{317 "id": StringValue(feature.ID), "featureSet": StringValue(setID),318 "title": StringValue(feature.Title), "subtitle": optionalString(feature.Subtitle),319 "description": optionalString(feature.Description), "center": optionalPosition(feature.Center),320 "shard": Int64Value(feature.Shard), "geometry": BytesValue(geometry),321 })322 seen := make(map[ID]bool, len(feature.Properties))323 for _, property := range feature.Properties {324 propertyID := property.fieldID()325 field, held := contract[propertyID]326 if !held || seen[propertyID] || property.Value.Kind != field.Kind {327 return fmt.Errorf("feature %s has an undeclared, duplicate, or mistyped property %s", feature.ID, propertyID)328 }329 seen[propertyID] = true330 values[propertyID] = property.Value331 }332 for id, field := range contract {333 if !field.Optional && !seen[id] {334 return fmt.Errorf("feature %s omits required property %s", feature.ID, id)335 }336 }337 if err := builders[featureTableName].add(values); err != nil {338 return err339 }340 return compileEvidence(feature, builders)341}342343func compileEvidence(feature Feature, builders map[string]*tableBuilder) error {344 for _, relationship := range feature.Relationships {345 if err := builders[relationshipTableName].add(valuesOf("relationship", map[string]Value{346 "feature": StringValue(feature.ID), "predicate": StringValue(relationship.Predicate), "target": StringValue(relationship.Target),347 })); err != nil {348 return err349 }350 }351 for _, provenance := range feature.Provenance {352 if err := builders[provenanceTableName].add(valuesOf("provenance", map[string]Value{353 "feature": StringValue(feature.ID), "source": StringValue(provenance.Source),354 "nativeID": StringValue(provenance.NativeID), "capturedAt": StringValue(provenance.CapturedAt),355 })); err != nil {356 return err357 }358 }359 return nil360}361362func compileRaster(worldID string, raster RasterPyramid, builders map[string]*tableBuilder) error {363 if err := builders[rasterTableName].add(valuesOf("rasterPyramid", map[string]Value{364 "id": StringValue(raster.ID), "world": StringValue(worldID), "name": StringValue(raster.Name), "codec": StringValue(raster.Codec),365 "tileSize": Int64Value(raster.TileSize), "minZoom": Int64Value(raster.MinZoom),366 "maxZoom": Int64Value(raster.MaxZoom), "fullZoom": Int64Value(raster.FullZoom),367 "sourceZoom": Int64Value(raster.SourceZoom), "template": StringValue(raster.Template),368 "bounds": optionalRect(raster.Bounds), "surface": optionalRect(raster.Surface),369 "interpolate": BoolValue(raster.Interpolate), "background": optionalString(raster.Background), "shard": Int64Value(raster.Shard),370 })); err != nil {371 return err372 }373 for index, format := range raster.Formats {374 if err := builders[rasterFormatTableName].add(valuesOf("rasterFormat", map[string]Value{375 "raster": StringValue(raster.ID), "zoom": Int64Value(raster.MinZoom + int64(index)), "format": StringValue(format),376 })); err != nil {377 return err378 }379 }380 for _, coverage := range raster.Coverage {381 if err := builders[rasterCoverageTableName].add(valuesOf("rasterCoverage", map[string]Value{382 "raster": StringValue(raster.ID), "zoom": Int64Value(coverage.Zoom),383 "x": Int64Value(coverage.X), "y": Int64Value(coverage.Y), "w": Int64Value(coverage.W), "h": Int64Value(coverage.H),384 "bits": BytesValue(coverage.Bits),385 })); err != nil {386 return err387 }388 }389 return nil390}391392func compilePresentation(worldID string, presentation Presentation, builders map[string]*tableBuilder) error {393 if err := builders[presentationTableName].add(valuesOf("presentation", map[string]Value{394 "id": StringValue(presentation.ID), "world": StringValue(worldID), "title": StringValue(presentation.Title),395 })); err != nil {396 return err397 }398 for _, style := range presentation.Styles {399 if err := builders[styleTableName].add(valuesOf("style", map[string]Value{400 "id": StringValue(style.ID), "presentation": StringValue(presentation.ID),401 "symbol": optionalString(style.Symbol), "icon": optionalString(style.Icon), "iconAsset": optionalString(style.IconAsset),402 "iconPicture": BoolValue(style.IconPicture), "renderAs": optionalString(style.RenderAs),403 "stroke": optionalString(style.Stroke), "fill": optionalString(style.Fill),404 })); err != nil {405 return err406 }407 }408 for _, layer := range presentation.Layers {409 if err := builders[layerTableName].add(valuesOf("layer", map[string]Value{410 "id": StringValue(layer.ID), "presentation": StringValue(presentation.ID),411 "featureSet": StringValue(layer.FeatureSet), "style": StringValue(layer.Style),412 "group": optionalString(layer.Group), "labelPolicy": optionalString(layer.LabelPolicy), "visible": BoolValue(layer.Visible),413 "order": Int64Value(layer.Order), "minZoom": Int64Value(layer.MinZoom), "maxZoom": Int64Value(layer.MaxZoom),414 })); err != nil {415 return err416 }417 }418 for _, legend := range presentation.Legend {419 if err := builders[legendTableName].add(valuesOf("legend", map[string]Value{420 "presentation": StringValue(presentation.ID), "layer": StringValue(legend.Layer),421 "label": StringValue(legend.Label), "order": Int64Value(legend.Order),422 })); err != nil {423 return err424 }425 }426 return nil427}428429func compileAssets(assets []Asset, builder *tableBuilder) ([]Blob, error) {430 blobs := make([]Blob, 0, len(assets))431 held := make(map[string]bool, len(assets))432 for _, asset := range assets {433 digest := sha256.Sum256(asset.Data)434 path := fmt.Sprintf("assets/%x", digest)435 if err := builder.add(valuesOf("asset", map[string]Value{436 "id": StringValue(asset.ID), "mediaType": StringValue(asset.MediaType),437 "path": StringValue(path), "provenance": optionalString(asset.Provenance),438 })); err != nil {439 return nil, err440 }441 if !held[path] {442 held[path] = true443 blobs = append(blobs, Blob{Name: path, Data: append([]byte(nil), asset.Data...)})444 }445 }446 slices.SortFunc(blobs, func(left, right Blob) int {447 if left.Name < right.Name {448 return -1449 }450 if left.Name > right.Name {451 return 1452 }453 return 0454 })455 return blobs, nil456}457458func valuesOf(typeName string, values map[string]Value) map[ID]Value {459 out := make(map[ID]Value, len(values))460 for name, value := range values {461 out[CoreID(typeName+"."+name)] = value462 }463 return out464}465466func optionalString(value string) Value {467 if value == "" {468 return NullValue()469 }470 return StringValue(value)471}472473func addClaims(values map[ID]Value, claims []Property) {474 for _, claim := range claims {475 values[claim.fieldID()] = claim.Value476 }477}478479func optionalRect(rect *RasterRect) Value {480 if rect == nil {481 return NullValue()482 }483 return BytesValue(encodeRect(*rect))484}485486func optionalPosition(position *Position) Value {487 if position == nil {488 return NullValue()489 }490 out := make([]byte, 16)491 binary.LittleEndian.PutUint64(out, math.Float64bits(position[0]))492 binary.LittleEndian.PutUint64(out[8:], math.Float64bits(position[1]))493 return BytesValue(out)494}495496func encodeExtent(extent [4]float64) []byte {497 out := make([]byte, 32)498 for index, coordinate := range extent {499 binary.LittleEndian.PutUint64(out[index*8:], math.Float64bits(coordinate))500 }501 return out502}503504func encodeGeometry(geometry Geometry) ([]byte, error) {505 if err := validateGeometry(geometry); err != nil {506 return nil, err507 }508 out := []byte{byte(geometry.Kind)}509 out = binary.LittleEndian.AppendUint32(out, uint32(len(geometry.Parts)))510 for _, part := range geometry.Parts {511 out = binary.LittleEndian.AppendUint32(out, uint32(len(part.Rings)))512 for _, ring := range part.Rings {513 out = binary.LittleEndian.AppendUint32(out, uint32(len(ring)))514 for _, position := range ring {515 out = binary.LittleEndian.AppendUint64(out, math.Float64bits(position[0]))516 out = binary.LittleEndian.AppendUint64(out, math.Float64bits(position[1]))517 }518 }519 }520 return out, nil521}522523func encodeRect(rect RasterRect) []byte {524 return encodeExtent([4]float64{rect.X, rect.Y, rect.Width, rect.Height})525}