1package vnext23import (4 "encoding/binary"5 "fmt"6 "math"7)89type setLocation struct {10 world int11 set int12}1314// Volume reconstructs the semantic graph from typed tables. No presentation15// policy is applied here.16func (reader *Reader) Volume() (Volume, error) {17 tables := make(map[string]Table, len(reader.tables))18 for _, name := range tableNames() {19 table, err := reader.Table(name)20 if err != nil {21 return Volume{}, err22 }23 tables[name] = table24 }25 return decodeVolume(reader, tables)26}2728func decodeVolume(reader *Reader, tables map[string]Table) (Volume, error) {29 root := tables[volumeTableName]30 if root.Rows != 1 {31 return Volume{}, fmt.Errorf("volume table has %d rows", root.Rows)32 }33 volume := Volume{34 ID: requiredString(root, 0, "volume.id"),35 Title: requiredString(root, 0, "volume.title"),36 }37 worlds, worldIndex, err := decodeWorlds(tables)38 if err != nil {39 return Volume{}, err40 }41 volume.Worlds = worlds42 sets, err := decodeFeatureSets(&volume, tables, worldIndex)43 if err != nil {44 return Volume{}, err45 }46 features, err := decodeFeatures(&volume, tables, sets)47 if err != nil {48 return Volume{}, err49 }50 if err := decodeEvidence(&volume, tables, features); err != nil {51 return Volume{}, err52 }53 if err := decodeWorldDetails(&volume, tables, worldIndex); err != nil {54 return Volume{}, err55 }56 assets, err := decodeAssets(reader, tables[assetTableName])57 if err != nil {58 return Volume{}, err59 }60 volume.Assets = assets61 if err := validateVolume(volume); err != nil {62 return Volume{}, fmt.Errorf("decoded volume: %w", err)63 }64 return volume, nil65}6667func decodeWorlds(tables map[string]Table) ([]World, map[string]int, error) {68 table := tables[worldTableName]69 worlds := make([]World, table.Rows)70 index := make(map[string]int, table.Rows)71 for row := 0; row < table.Rows; row++ {72 worlds[row] = World{73 ID: requiredString(table, row, "world.id"),74 Title: requiredString(table, row, "world.title"),75 CoordinateSpace: CoordinateSpace{ID: requiredString(table, row, "world.coordinateSpace")},76 Claims: extensionProperties(table, row, "world"),77 }78 index[worlds[row].ID] = row79 }80 coordinates := tables[coordinateTableName]81 for row := 0; row < coordinates.Rows; row++ {82 worldID := requiredString(coordinates, row, "coordinate.world")83 at, held := index[worldID]84 if !held {85 return nil, nil, fmt.Errorf("coordinate space refers to unknown world %s", worldID)86 }87 extent, err := decodeExtent(requiredBytes(coordinates, row, "coordinate.extent"))88 if err != nil {89 return nil, nil, err90 }91 worlds[at].CoordinateSpace = CoordinateSpace{92 ID: requiredString(coordinates, row, "coordinate.id"),93 Kind: requiredString(coordinates, row, "coordinate.kind"),94 Unit: requiredString(coordinates, row, "coordinate.unit"),95 Definition: requiredString(coordinates, row, "coordinate.definition"),96 Extent: extent,97 SourceZoom: requiredInt64(coordinates, row, "coordinate.sourceZoom"),98 OriginX: requiredInt64(coordinates, row, "coordinate.originX"),99 OriginY: requiredInt64(coordinates, row, "coordinate.originY"),100 TileSize: requiredInt64(coordinates, row, "coordinate.tileSize"),101 Size: requiredInt64(coordinates, row, "coordinate.size"),102 }103 }104 return worlds, index, nil105}106107func decodeFeatureSets(volume *Volume, tables map[string]Table, worlds map[string]int) (map[string]setLocation, error) {108 table := tables[featureSetTableName]109 sets := make(map[string]setLocation, table.Rows)110 for row := 0; row < table.Rows; row++ {111 worldID := requiredString(table, row, "featureSet.world")112 world, held := worlds[worldID]113 if !held {114 return nil, fmt.Errorf("feature set refers to unknown world %s", worldID)115 }116 set := FeatureSet{117 ID: requiredString(table, row, "featureSet.id"),118 Title: requiredString(table, row, "featureSet.title"),119 SemanticType: requiredString(table, row, "featureSet.semanticType"),120 Claims: extensionProperties(table, row, "featureSet"),121 }122 volume.Worlds[world].FeatureSets = append(volume.Worlds[world].FeatureSets, set)123 sets[set.ID] = setLocation{world: world, set: len(volume.Worlds[world].FeatureSets) - 1}124 }125 definitions := tables[propertyDefinitionTableName]126 for row := 0; row < definitions.Rows; row++ {127 setID := requiredString(definitions, row, "propertyDefinition.featureSet")128 location, held := sets[setID]129 if !held {130 return nil, fmt.Errorf("property definition refers to unknown feature set %s", setID)131 }132 field := Field{133 ID: requiredID(definitions, row, "propertyDefinition.field"),134 Name: requiredString(definitions, row, "propertyDefinition.name"),135 Kind: Kind(requiredInt64(definitions, row, "propertyDefinition.kind")),136 Optional: requiredBool(definitions, row, "propertyDefinition.optional"),137 }138 set := &volume.Worlds[location.world].FeatureSets[location.set]139 set.Properties = append(set.Properties, field)140 }141 return sets, nil142}143144func decodeFeatures(volume *Volume, tables map[string]Table, sets map[string]setLocation) (map[string]setLocation, error) {145 table := tables[featureTableName]146 features := make(map[string]setLocation, table.Rows)147 core, _ := StandardSchema().Type(CoreID("type.feature"))148 for row := 0; row < table.Rows; row++ {149 setID := requiredString(table, row, "feature.featureSet")150 location, held := sets[setID]151 if !held {152 return nil, fmt.Errorf("feature refers to unknown feature set %s", setID)153 }154 geometry, err := decodeGeometry(requiredBytes(table, row, "feature.geometry"))155 if err != nil {156 return nil, err157 }158 feature := Feature{159 ID: requiredString(table, row, "feature.id"), Title: requiredString(table, row, "feature.title"),160 Subtitle: optionalStringAt(table, row, "feature.subtitle"), Description: optionalStringAt(table, row, "feature.description"),161 Shard: requiredInt64(table, row, "feature.shard"), Geometry: geometry,162 }163 if center := optionalBytesAt(table, row, "feature.center"); center != nil {164 position, err := decodePosition(center)165 if err != nil {166 return nil, err167 }168 feature.Center = &position169 }170 for _, column := range table.Columns {171 if _, standard := core.Field(column.Field.ID); standard || column.Values[row].Kind == KindInvalid {172 continue173 }174 feature.Properties = append(feature.Properties, Property{FieldID: column.Field.ID, Field: column.Field, Value: column.Values[row]})175 }176 set := &volume.Worlds[location.world].FeatureSets[location.set]177 set.Features = append(set.Features, feature)178 features[feature.ID] = setLocation{world: location.world, set: location.set}179 }180 return features, nil181}182183func decodeEvidence(volume *Volume, tables map[string]Table, features map[string]setLocation) error {184 for _, spec := range []struct {185 name string186 decode func(*Feature, Table, int)187 }{{relationshipTableName, decodeRelationship}, {provenanceTableName, decodeProvenance}} {188 table := tables[spec.name]189 for row := 0; row < table.Rows; row++ {190 featureID := requiredString(table, row, tablePrefix(spec.name)+".feature")191 location, held := features[featureID]192 if !held {193 return fmt.Errorf("evidence refers to unknown feature %s", featureID)194 }195 feature := findFeature(volume, location, featureID)196 spec.decode(feature, table, row)197 }198 }199 return nil200}201202func findFeature(volume *Volume, location setLocation, id string) *Feature {203 features := volume.Worlds[location.world].FeatureSets[location.set].Features204 for index := range features {205 if features[index].ID == id {206 return &features[index]207 }208 }209 return nil210}211212func decodeRelationship(feature *Feature, table Table, row int) {213 feature.Relationships = append(feature.Relationships, Relationship{214 Predicate: requiredString(table, row, "relationship.predicate"),215 Target: requiredString(table, row, "relationship.target"),216 })217}218219func decodeProvenance(feature *Feature, table Table, row int) {220 feature.Provenance = append(feature.Provenance, Provenance{221 Source: requiredString(table, row, "provenance.source"),222 NativeID: requiredString(table, row, "provenance.nativeID"),223 CapturedAt: requiredString(table, row, "provenance.capturedAt"),224 })225}226227func tablePrefix(name string) string {228 if name == relationshipTableName {229 return "relationship"230 }231 return "provenance"232}233234func decodeWorldDetails(volume *Volume, tables map[string]Table, worlds map[string]int) error {235 if err := decodeRasters(volume, tables, worlds); err != nil {236 return err237 }238 presentations, err := decodePresentations(volume, tables[presentationTableName], worlds)239 if err != nil {240 return err241 }242 if err := decodeStyles(volume, tables[styleTableName], presentations); err != nil {243 return err244 }245 if err := decodeLayers(volume, tables[layerTableName], presentations); err != nil {246 return err247 }248 return decodeLegend(volume, tables[legendTableName], presentations)249}250251func decodeRasters(volume *Volume, tables map[string]Table, worlds map[string]int) error {252 table := tables[rasterTableName]253 rasterLocations := make(map[string][2]int, table.Rows)254 for row := 0; row < table.Rows; row++ {255 worldID := requiredString(table, row, "rasterPyramid.world")256 world, held := worlds[worldID]257 if !held {258 return fmt.Errorf("raster pyramid refers to unknown world %s", worldID)259 }260 raster := RasterPyramid{261 ID: requiredString(table, row, "rasterPyramid.id"), Name: requiredString(table, row, "rasterPyramid.name"), Codec: requiredString(table, row, "rasterPyramid.codec"),262 TileSize: requiredInt64(table, row, "rasterPyramid.tileSize"), MinZoom: requiredInt64(table, row, "rasterPyramid.minZoom"),263 MaxZoom: requiredInt64(table, row, "rasterPyramid.maxZoom"), FullZoom: requiredInt64(table, row, "rasterPyramid.fullZoom"),264 SourceZoom: requiredInt64(table, row, "rasterPyramid.sourceZoom"), Template: requiredString(table, row, "rasterPyramid.template"),265 Interpolate: requiredBool(table, row, "rasterPyramid.interpolate"), Background: optionalStringAt(table, row, "rasterPyramid.background"),266 Shard: requiredInt64(table, row, "rasterPyramid.shard"),267 }268 var err error269 if data := optionalBytesAt(table, row, "rasterPyramid.bounds"); data != nil {270 raster.Bounds, err = decodeRect(data)271 if err != nil {272 return err273 }274 }275 if data := optionalBytesAt(table, row, "rasterPyramid.surface"); data != nil {276 raster.Surface, err = decodeRect(data)277 if err != nil {278 return err279 }280 }281 volume.Worlds[world].RasterPyramids = append(volume.Worlds[world].RasterPyramids, raster)282 rasterLocations[raster.ID] = [2]int{world, len(volume.Worlds[world].RasterPyramids) - 1}283 }284 formats := tables[rasterFormatTableName]285 for row := 0; row < formats.Rows; row++ {286 location, held := rasterLocations[requiredString(formats, row, "rasterFormat.raster")]287 if !held {288 return fmt.Errorf("raster format refers to unknown raster")289 }290 raster := &volume.Worlds[location[0]].RasterPyramids[location[1]]291 raster.Formats = append(raster.Formats, requiredString(formats, row, "rasterFormat.format"))292 }293 coverage := tables[rasterCoverageTableName]294 for row := 0; row < coverage.Rows; row++ {295 location, held := rasterLocations[requiredString(coverage, row, "rasterCoverage.raster")]296 if !held {297 return fmt.Errorf("raster coverage refers to unknown raster")298 }299 raster := &volume.Worlds[location[0]].RasterPyramids[location[1]]300 raster.Coverage = append(raster.Coverage, RasterCoverage{301 Zoom: requiredInt64(coverage, row, "rasterCoverage.zoom"), X: requiredInt64(coverage, row, "rasterCoverage.x"),302 Y: requiredInt64(coverage, row, "rasterCoverage.y"), W: requiredInt64(coverage, row, "rasterCoverage.w"),303 H: requiredInt64(coverage, row, "rasterCoverage.h"), Bits: requiredBytes(coverage, row, "rasterCoverage.bits"),304 })305 }306 return nil307}308309func decodePresentations(volume *Volume, table Table, worlds map[string]int) (map[string]int, error) {310 presentations := make(map[string]int, table.Rows)311 for row := 0; row < table.Rows; row++ {312 worldID := requiredString(table, row, "presentation.world")313 world, held := worlds[worldID]314 if !held {315 return nil, fmt.Errorf("presentation refers to unknown world %s", worldID)316 }317 presentation := Presentation{ID: requiredString(table, row, "presentation.id"), Title: requiredString(table, row, "presentation.title")}318 volume.Worlds[world].Presentation = presentation319 presentations[presentation.ID] = world320 }321 return presentations, nil322}323324func decodeStyles(volume *Volume, table Table, presentations map[string]int) error {325 for row := 0; row < table.Rows; row++ {326 presentationID := requiredString(table, row, "style.presentation")327 world, held := presentations[presentationID]328 if !held {329 return fmt.Errorf("style refers to unknown presentation %s", presentationID)330 }331 volume.Worlds[world].Presentation.Styles = append(volume.Worlds[world].Presentation.Styles, Style{332 ID: requiredString(table, row, "style.id"), Symbol: optionalStringAt(table, row, "style.symbol"),333 Icon: optionalStringAt(table, row, "style.icon"), IconAsset: optionalStringAt(table, row, "style.iconAsset"),334 IconPicture: requiredBool(table, row, "style.iconPicture"), RenderAs: optionalStringAt(table, row, "style.renderAs"),335 Stroke: optionalStringAt(table, row, "style.stroke"), Fill: optionalStringAt(table, row, "style.fill"),336 })337 }338 return nil339}340341func decodeLayers(volume *Volume, table Table, presentations map[string]int) error {342 for row := 0; row < table.Rows; row++ {343 presentationID := requiredString(table, row, "layer.presentation")344 world, held := presentations[presentationID]345 if !held {346 return fmt.Errorf("layer refers to unknown presentation %s", presentationID)347 }348 volume.Worlds[world].Presentation.Layers = append(volume.Worlds[world].Presentation.Layers, Layer{349 ID: requiredString(table, row, "layer.id"), FeatureSet: requiredString(table, row, "layer.featureSet"),350 Style: requiredString(table, row, "layer.style"), Order: requiredInt64(table, row, "layer.order"),351 Group: optionalStringAt(table, row, "layer.group"), LabelPolicy: optionalStringAt(table, row, "layer.labelPolicy"),352 Visible: requiredBool(table, row, "layer.visible"),353 MinZoom: requiredInt64(table, row, "layer.minZoom"), MaxZoom: requiredInt64(table, row, "layer.maxZoom"),354 })355 }356 return nil357}358359func decodeLegend(volume *Volume, table Table, presentations map[string]int) error {360 for row := 0; row < table.Rows; row++ {361 presentationID := requiredString(table, row, "legend.presentation")362 world, held := presentations[presentationID]363 if !held {364 return fmt.Errorf("legend refers to unknown presentation %s", presentationID)365 }366 volume.Worlds[world].Presentation.Legend = append(volume.Worlds[world].Presentation.Legend, LegendEntry{367 Layer: requiredString(table, row, "legend.layer"), Label: requiredString(table, row, "legend.label"),368 Order: requiredInt64(table, row, "legend.order"),369 })370 }371 return nil372}373374func decodeAssets(reader *Reader, table Table) ([]Asset, error) {375 assets := make([]Asset, table.Rows)376 for row := 0; row < table.Rows; row++ {377 path := requiredString(table, row, "asset.path")378 data, err := reader.Blob(path)379 if err != nil {380 return nil, err381 }382 assets[row] = Asset{383 ID: requiredString(table, row, "asset.id"), MediaType: requiredString(table, row, "asset.mediaType"),384 Path: path, Data: data, Provenance: optionalStringAt(table, row, "asset.provenance"),385 }386 }387 return assets, nil388}389390func requiredValue(table Table, row int, name string) Value {391 value, held := table.Value(CoreID(name), row)392 if !held || value.Kind == KindInvalid {393 return Value{}394 }395 return value396}397398func requiredString(table Table, row int, name string) string {399 return requiredValue(table, row, name).String400}401func requiredInt64(table Table, row int, name string) int64 {402 return requiredValue(table, row, name).Int64403}404func requiredBool(table Table, row int, name string) bool {405 return requiredValue(table, row, name).Bool406}407func requiredID(table Table, row int, name string) ID { return requiredValue(table, row, name).ID }408func requiredBytes(table Table, row int, name string) []byte {409 return requiredValue(table, row, name).Bytes410}411412func optionalStringAt(table Table, row int, name string) string {413 value, _ := table.Value(CoreID(name), row)414 return value.String415}416417func optionalBytesAt(table Table, row int, name string) []byte {418 value, held := table.Value(CoreID(name), row)419 if !held || value.Kind == KindInvalid {420 return nil421 }422 return value.Bytes423}424425func extensionProperties(table Table, row int, typeName string) []Property {426 core, _ := StandardSchema().Type(CoreID("type." + typeName))427 var properties []Property428 for _, column := range table.Columns {429 if _, standard := core.Field(column.Field.ID); standard || column.Values[row].Kind == KindInvalid {430 continue431 }432 properties = append(properties, Property{FieldID: column.Field.ID, Field: column.Field, Value: column.Values[row]})433 }434 return properties435}436437func decodeExtent(data []byte) ([4]float64, error) {438 var extent [4]float64439 if len(data) != len(extent)*8 {440 return extent, fmt.Errorf("coordinate extent has %d bytes", len(data))441 }442 for index := range extent {443 extent[index] = math.Float64frombits(binary.LittleEndian.Uint64(data[index*8:]))444 }445 return extent, nil446}447448func decodeGeometry(data []byte) (Geometry, error) {449 if len(data) < 5 {450 return Geometry{}, fmt.Errorf("geometry is truncated")451 }452 cursor := 1453 readCount := func() (int, error) {454 if cursor+4 > len(data) {455 return 0, fmt.Errorf("geometry is truncated")456 }457 count := int(binary.LittleEndian.Uint32(data[cursor : cursor+4]))458 cursor += 4459 return count, nil460 }461 parts, err := readCount()462 if err != nil {463 return Geometry{}, err464 }465 kind := GeometryKind(data[0])466 if kind < GeometryPoint || kind > GeometryPolygon || parts > (len(data)-cursor)/4 {467 return Geometry{}, fmt.Errorf("geometry part framing is invalid")468 }469 geometry := Geometry{Kind: kind, Parts: make([]GeometryPart, parts)}470 for part := range geometry.Parts {471 rings, err := readCount()472 if err != nil {473 return Geometry{}, err474 }475 if rings > (len(data)-cursor)/4 {476 return Geometry{}, fmt.Errorf("geometry ring framing is invalid")477 }478 geometry.Parts[part].Rings = make([][]Position, rings)479 for ring := range geometry.Parts[part].Rings {480 positions, err := readCount()481 if err != nil {482 return Geometry{}, err483 }484 if positions > (len(data)-cursor)/16 {485 return Geometry{}, fmt.Errorf("geometry position payload is invalid")486 }487 geometry.Parts[part].Rings[ring] = make([]Position, positions)488 for position := range geometry.Parts[part].Rings[ring] {489 geometry.Parts[part].Rings[ring][position][0] = math.Float64frombits(binary.LittleEndian.Uint64(data[cursor:]))490 cursor += 8491 geometry.Parts[part].Rings[ring][position][1] = math.Float64frombits(binary.LittleEndian.Uint64(data[cursor:]))492 cursor += 8493 }494 }495 }496 if cursor != len(data) {497 return Geometry{}, fmt.Errorf("geometry has trailing bytes")498 }499 if _, err := encodeGeometry(geometry); err != nil {500 return Geometry{}, err501 }502 return geometry, nil503}504505func decodePosition(data []byte) (Position, error) {506 if len(data) != 16 {507 return Position{}, fmt.Errorf("position has %d bytes", len(data))508 }509 return Position{510 math.Float64frombits(binary.LittleEndian.Uint64(data)),511 math.Float64frombits(binary.LittleEndian.Uint64(data[8:])),512 }, nil513}514515func decodeRect(data []byte) (*RasterRect, error) {516 extent, err := decodeExtent(data)517 if err != nil {518 return nil, err519 }520 return &RasterRect{X: extent[0], Y: extent[1], Width: extent[2], Height: extent[3]}, nil521}