1package authoring23import (4 "bytes"5 "encoding/base64"6 "encoding/json"7 "fmt"8 "math"9 "sort"10 "strconv"11 "strings"1213 "github.com/FelineStateMachine/atlas/format/vnext"14)1516type observation struct {17 SourceOrder int18 Source string19 NativeID string20 FeatureSet string21 Title string22 Geometry vnext.Geometry23 Values map[string]vnext.Value24 Relations []observedRelation25 Evidence Capture26}2728type observedRelation struct {29 Contract string30 Predicate string31 FeatureSet string32 NativeID string33}3435type geoJSON struct {36 Type string `json:"type"`37 Features []geoJSONFeature `json:"features"`38 Links []geoJSONLink `json:"links,omitempty"`39 NumberMatched *int64 `json:"numberMatched,omitempty"`40 NumberReturned *int64 `json:"numberReturned,omitempty"`41 ExceededTransferLimit bool `json:"exceededTransferLimit,omitempty"`42}4344type geoJSONLink struct {45 Rel string `json:"rel"`46 Href string `json:"href"`47}4849type geoJSONFeature struct {50 Type string `json:"type"`51 ID any `json:"id"`52 Properties map[string]any `json:"properties"`53 Geometry geoJSONGeometry `json:"geometry"`54}5556type geoJSONGeometry struct {57 Type string `json:"type"`58 Coordinates json.RawMessage `json:"coordinates"`59}6061func observe(project Project, sourceOrder int, source Source, capture Capture, body []byte) ([]observation, error) {62 return observePage(project, sourceOrder, source, capture, body, true)63}6465func observeCapturedPage(project Project, sourceOrder int, source Source, capture Capture, body []byte) ([]observation, error) {66 return observePage(project, sourceOrder, source, capture, body, false)67}6869func observePage(project Project, sourceOrder int, source Source, capture Capture, body []byte, enforceCompleteness bool) ([]observation, error) {70 set, exists := featureSetContract(project.FeatureSets, source.Mapping.FeatureSet)71 if !exists {72 return nil, fmt.Errorf("source %s targets unknown feature set %s", source.ID, source.Mapping.FeatureSet)73 }74 decoder := json.NewDecoder(bytes.NewReader(body))75 decoder.UseNumber()76 var collection geoJSON77 if err := decoder.Decode(&collection); err != nil {78 return nil, fmt.Errorf("source %s decode GeoJSON: %w", source.ID, err)79 }80 if collection.Type != "FeatureCollection" {81 return nil, fmt.Errorf("source %s is %q, want FeatureCollection", source.ID, collection.Type)82 }83 if enforceCompleteness {84 if err := validateAdapterCompleteness(source, collection); err != nil {85 return nil, err86 }87 }88 out := make([]observation, 0, len(collection.Features))89 for index, feature := range collection.Features {90 if feature.Type != "" && feature.Type != "Feature" {91 return nil, fmt.Errorf("source %s feature %d has type %q", source.ID, index, feature.Type)92 }93 nativeValue, held := featureValue(feature, source.Mapping.Identity)94 if !held {95 return nil, fmt.Errorf("source %s feature %d has no identity at %s", source.ID, index, source.Mapping.Identity)96 }97 nativeID := scalarString(nativeValue)98 if nativeID == "" {99 return nil, fmt.Errorf("source %s feature %d has an empty identity", source.ID, index)100 }101 titleValue, held := featureValue(feature, source.Mapping.FeatureTitle)102 if !held || scalarString(titleValue) == "" {103 return nil, fmt.Errorf("source %s feature %s has no title at %s", source.ID, nativeID, source.Mapping.FeatureTitle)104 }105 geometry, err := geometryOf(feature.Geometry)106 if err != nil {107 return nil, fmt.Errorf("source %s feature %s: %w", source.ID, nativeID, err)108 }109 geometryMap := source.Mapping.Geometry110 geometryMap.Family = set.Geometry111 geometry, err = transformGeometry(geometry, geometryMap, project.Target.CoordinateSpace)112 if err != nil {113 return nil, fmt.Errorf("source %s feature %s: %w", source.ID, nativeID, err)114 }115 item := observation{116 SourceOrder: sourceOrder, Source: source.ID, NativeID: nativeID,117 FeatureSet: source.Mapping.FeatureSet, Title: scalarString(titleValue),118 Geometry: geometry, Values: make(map[string]vnext.Value),119 Evidence: capture,120 }121 for _, mapping := range source.Mapping.Properties {122 raw, held := featureValue(feature, mapping.Source)123 if !held || raw == nil {124 continue125 }126 contract, exists := propertyContract(set, mapping.Field)127 if !exists {128 return nil, fmt.Errorf("source %s maps unknown property %s", source.ID, mapping.Field)129 }130 kind, _ := kindOf(contract.Type)131 value, err := valueOf(project.SchemaNamespace, kind, raw)132 if err != nil {133 return nil, fmt.Errorf("source %s feature %s property %s: %w", source.ID, nativeID, mapping.Field, err)134 }135 item.Values[mapping.Field] = value136 }137 for _, mapping := range source.Mapping.Relations {138 raw, held := featureValue(feature, mapping.Target)139 if !held || scalarString(raw) == "" {140 continue141 }142 contract, exists := relationshipContract(set, mapping.Relationship)143 if !exists {144 return nil, fmt.Errorf("source %s maps unknown relationship %s", source.ID, mapping.Relationship)145 }146 item.Relations = append(item.Relations, observedRelation{147 Contract: mapping.Relationship, Predicate: contract.Predicate,148 FeatureSet: contract.FeatureSet, NativeID: scalarString(raw),149 })150 }151 out = append(out, item)152 }153 sort.Slice(out, func(i, j int) bool { return out[i].NativeID < out[j].NativeID })154 return out, nil155}156157func validateAdapterCompleteness(source Source, collection geoJSON) error {158 switch source.Adapter {159 case "arcgis-feature-service":160 if collection.ExceededTransferLimit {161 return fmt.Errorf("source %s ArcGIS response exceeded its transfer limit; narrow or partition the configured query", source.ID)162 }163 case "ogc-api-features":164 if collection.NumberReturned != nil && *collection.NumberReturned != int64(len(collection.Features)) {165 return fmt.Errorf("source %s OGC response count differs from its feature payload", source.ID)166 }167 if collection.NumberMatched != nil && *collection.NumberMatched > int64(len(collection.Features)) {168 return fmt.Errorf("source %s OGC response is incomplete: %d matched but only %d returned", source.ID, *collection.NumberMatched, len(collection.Features))169 }170 for _, link := range collection.Links {171 if strings.EqualFold(link.Rel, "next") && strings.TrimSpace(link.Href) != "" {172 return fmt.Errorf("source %s OGC response has a next page; narrow or partition the configured query", source.ID)173 }174 }175 }176 return nil177}178179func featureValue(feature geoJSONFeature, path string) (any, bool) {180 path = strings.TrimSpace(path)181 if strings.HasPrefix(path, "coalesce(") && strings.HasSuffix(path, ")") {182 arguments := strings.Split(strings.TrimSuffix(strings.TrimPrefix(path, "coalesce("), ")"), ",")183 if len(arguments) < 2 {184 return nil, false185 }186 for _, argument := range arguments {187 value, held := featureValue(feature, strings.TrimSpace(argument))188 if held && scalarString(value) != "" {189 return value, true190 }191 }192 return nil, false193 }194 switch path {195 case "id":196 return feature.ID, feature.ID != nil197 }198 const prefix = "properties."199 if !strings.HasPrefix(path, prefix) {200 return nil, false201 }202 value, held := feature.Properties[strings.TrimPrefix(path, prefix)]203 return value, held204}205206func scalarString(value any) string {207 switch held := value.(type) {208 case string:209 return strings.TrimSpace(held)210 case json.Number:211 return held.String()212 case float64:213 return strconv.FormatFloat(held, 'g', -1, 64)214 case bool:215 return strconv.FormatBool(held)216 default:217 return ""218 }219}220221func valueOf(namespace string, kind vnext.Kind, raw any) (vnext.Value, error) {222 switch kind {223 case vnext.KindString:224 value := scalarString(raw)225 if value == "" {226 return vnext.Value{}, fmt.Errorf("value is not a scalar string")227 }228 return vnext.StringValue(value), nil229 case vnext.KindBool:230 if value, ok := raw.(bool); ok {231 return vnext.BoolValue(value), nil232 }233 value, err := strconv.ParseBool(scalarString(raw))234 if err != nil {235 return vnext.Value{}, err236 }237 return vnext.BoolValue(value), nil238 case vnext.KindInt64:239 value, err := strconv.ParseInt(scalarString(raw), 10, 64)240 if err != nil {241 return vnext.Value{}, err242 }243 return vnext.Int64Value(value), nil244 case vnext.KindFloat64:245 value, err := strconv.ParseFloat(scalarString(raw), 64)246 if err != nil || math.IsNaN(value) || math.IsInf(value, 0) {247 return vnext.Value{}, fmt.Errorf("value is not a finite float64")248 }249 return vnext.Float64Value(value), nil250 case vnext.KindBytes:251 value, err := base64.StdEncoding.DecodeString(scalarString(raw))252 if err != nil {253 return vnext.Value{}, err254 }255 return vnext.BytesValue(value), nil256 case vnext.KindID:257 value := scalarString(raw)258 if value == "" {259 return vnext.Value{}, fmt.Errorf("value is not an identity")260 }261 return vnext.IDValue(vnext.IDFromName(namespace, value)), nil262 default:263 return vnext.Value{}, fmt.Errorf("unsupported value type")264 }265}266267func geometryOf(source geoJSONGeometry) (vnext.Geometry, error) {268 switch source.Type {269 case "Point":270 var point vnext.Position271 if err := json.Unmarshal(source.Coordinates, &point); err != nil {272 return vnext.Geometry{}, err273 }274 return vnext.Geometry{Kind: vnext.GeometryPoint, Parts: []vnext.GeometryPart{{Rings: [][]vnext.Position{{point}}}}}, nil275 case "LineString":276 var line []vnext.Position277 if err := json.Unmarshal(source.Coordinates, &line); err != nil {278 return vnext.Geometry{}, err279 }280 return vnext.Geometry{Kind: vnext.GeometryLineString, Parts: []vnext.GeometryPart{{Rings: [][]vnext.Position{line}}}}, nil281 case "MultiLineString":282 var lines [][]vnext.Position283 if err := json.Unmarshal(source.Coordinates, &lines); err != nil {284 return vnext.Geometry{}, err285 }286 geometry := vnext.Geometry{Kind: vnext.GeometryLineString}287 for _, line := range lines {288 geometry.Parts = append(geometry.Parts, vnext.GeometryPart{Rings: [][]vnext.Position{line}})289 }290 return geometry, nil291 case "Polygon":292 var rings [][]vnext.Position293 if err := json.Unmarshal(source.Coordinates, &rings); err != nil {294 return vnext.Geometry{}, err295 }296 return vnext.Geometry{Kind: vnext.GeometryPolygon, Parts: []vnext.GeometryPart{{Rings: rings}}}, nil297 case "MultiPolygon":298 var polygons [][][]vnext.Position299 if err := json.Unmarshal(source.Coordinates, &polygons); err != nil {300 return vnext.Geometry{}, err301 }302 geometry := vnext.Geometry{Kind: vnext.GeometryPolygon}303 for _, rings := range polygons {304 geometry.Parts = append(geometry.Parts, vnext.GeometryPart{Rings: rings})305 }306 return geometry, nil307 default:308 return vnext.Geometry{}, fmt.Errorf("unsupported GeoJSON geometry %q", source.Type)309 }310}