Add versioned adapter planning, complete multi-page capture sets, exact replay receipts, bounded geographic queries, deterministic clipping, and sparse-source mapping expressions.
- commit
- 78d7f3f5dd08896f999906cdbba263b84051dee1
- parent
- 9a887365dab82948d04c2e6c935944b051c139fa
- author
- Dami <dami.etoile@gmail.com>
| A | internal/ | 88 | ++++++ |
| A | internal/ | 422 | ++++++++++++++++++++++++++++++ |
| A | internal/ | 132 | +++++++++ |
| M | internal/ | 3 | +- |
| M | internal/ | 74 | ++---- |
| A | internal/ | 395 | ++++++++++++++++++++++++++++ |
| A | internal/ | 193 | ++++++++++++++ |
| M | internal/ | 191 | +++++++++++++- |
| A | internal/ | 18 | + |
| M | internal/ | 29 | +- |
| M | internal/ | 5 | +- |
| A | internal/ | 151 | +++++++++++ |
| M | internal/ | 28 | ++- |
| M | internal/ | 114 | ++------ |
| M | internal/ | 167 | +++++++++++- |
| M | internal/ | 34 | ++ |
@@ -0,0 +1,88 @@
1+package authoring2+3+import (4+ "context"5+ "fmt"6+ "sort"7+)8+9+type buildEvidence struct {10+ Captures map[string]Capture11+ Sets map[string]CaptureSet12+ Cached map[string]bool13+}14+15+type captureGroup struct {16+ Requests []PlannedRequest17+}18+19+func acquirePlanEvidence(ctx context.Context, plan Plan, cache *Cache, offline bool) (buildEvidence, error) {20+ evidence := buildEvidence{21+ Captures: make(map[string]Capture, len(plan.Requests)),22+ Sets: make(map[string]CaptureSet, len(plan.Requests)),23+ Cached: make(map[string]bool, len(plan.Requests)),24+ }25+ registry := defaultAdapterRegistry()26+ usedRequests := 027+ var usedBytes int6428+ for _, group := range planCaptureGroups(plan) {29+ remainingRequests := plan.Budgets.Requests - usedRequests30+ remainingBytes := plan.Budgets.TotalBytes - usedBytes31+ if remainingRequests <= 0 || remainingBytes <= 0 {32+ return buildEvidence{}, fmt.Errorf("captured evidence exhausts build budgets before source %s", group.Requests[0].Source)33+ }34+ requests := make([]Request, len(group.Requests))35+ for index := range group.Requests {36+ requests[index] = group.Requests[index].Request37+ }38+ set, cached, err := cache.AcquireSet(ctx, requests, registry, CaptureSetOptions{39+ Offline: offline, MaxRequests: remainingRequests, MaxBytes: remainingBytes,40+ })41+ if err != nil {42+ return buildEvidence{}, err43+ }44+ for _, root := range set.Roots {45+ usedRequests += len(root.Pages)46+ }47+ usedBytes += set.TotalBytes48+ if usedRequests > plan.Budgets.Requests || usedBytes > plan.Budgets.TotalBytes {49+ return buildEvidence{}, fmt.Errorf("captured evidence exceeds build budgets")50+ }51+ for _, planned := range group.Requests {52+ root, ok := captureRootForRequest(set, planned.Request)53+ if !ok {54+ return buildEvidence{}, fmt.Errorf("capture set omits request %s", planned.ID)55+ }56+ evidence.Captures[planned.ID] = root.Pages[0]57+ evidence.Sets[planned.ID] = set58+ evidence.Cached[planned.ID] = cached59+ }60+ }61+ return evidence, nil62+}63+64+func planCaptureGroups(plan Plan) []captureGroup {65+ bySource := make(map[string][]PlannedRequest)66+ for _, request := range plan.Requests {67+ key := string(request.Kind) + "\x00" + request.Source68+ bySource[key] = append(bySource[key], request)69+ }70+ groups := make([]captureGroup, 0, len(bySource))71+ for _, requests := range bySource {72+ sort.Slice(requests, func(i, j int) bool {73+ return requestCacheKey(requests[i].Request) < requestCacheKey(requests[j].Request)74+ })75+ groups = append(groups, captureGroup{Requests: requests})76+ }77+ sort.Slice(groups, func(i, j int) bool { return groups[i].Requests[0].ID < groups[j].Requests[0].ID })78+ return groups79+}80+81+func captureRootForRequest(set CaptureSet, request Request) (CaptureRoot, bool) {82+ want := requestCacheKey(request)83+ index := sort.Search(len(set.Roots), func(index int) bool { return set.Roots[index].Acquisition >= want })84+ if index == len(set.Roots) || set.Roots[index].Acquisition != want {85+ return CaptureRoot{}, false86+ }87+ return set.Roots[index], true88+}@@ -0,0 +1,422 @@
1+package authoring2+3+import (4+ "bytes"5+ "encoding/json"6+ "errors"7+ "fmt"8+ "io"9+ "math/big"10+ "net/url"11+ "path/filepath"12+ "strconv"13+ "strings"14+)15+16+const (17+ TerminationSinglePage = "single-page"18+ TerminationNextExhausted = "next-exhausted"19+ TerminationTransferLimitClear = "transfer-limit-clear"20+)21+22+// AdapterRegistry is an immutable lookup table for acquisition protocol23+// behavior. A fresh registry is cheap and avoids process-wide mutable state.24+type AdapterRegistry struct {25+ adapters map[string]Adapter26+}27+28+// Adapter owns protocol versioning and continuation semantics. Mapping source29+// records into typed Atlas features deliberately remains a separate concern.30+type Adapter interface {31+ Name() string32+ Version() string33+ PlanFeature(Project, Source) (Request, error)34+ Next(Request, []byte, *PageProgress) (PageDecision, error)35+}36+37+type PageProgress struct {38+ RootLocator string39+ Returned int6440+ Matched *int6441+ SeenIDs map[string]bool42+ LastID string43+}44+45+type PageDecision struct {46+ Done bool47+ Next Request48+ Termination string49+ Returned int6450+ Matched *int6451+}52+53+type adapterDefinition struct {54+ name string55+ version string56+ next func(Request, []byte, *PageProgress) (PageDecision, error)57+ plan func(Project, Source) (Request, error)58+}59+60+func (adapter adapterDefinition) Name() string { return adapter.name }61+62+func (adapter adapterDefinition) Version() string { return adapter.version }63+64+func (adapter adapterDefinition) PlanFeature(project Project, source Source) (Request, error) {65+ if adapter.plan == nil {66+ return Request{}, fmt.Errorf("adapter %s does not acquire features", adapter.name)67+ }68+ request, err := adapter.plan(project, source)69+ if err != nil {70+ return Request{}, err71+ }72+ request.Adapter = adapter.name73+ request.AdapterVersion = adapter.version74+ request.AcquisitionID = acquisitionID(request)75+ request.ID = requestID(request)76+ return request, nil77+}78+79+func (adapter adapterDefinition) Next(request Request, body []byte, progress *PageProgress) (PageDecision, error) {80+ return adapter.next(request, body, progress)81+}82+83+func defaultAdapterRegistry() AdapterRegistry {84+ definitions := []adapterDefinition{85+ {name: "geojson", version: "geojson/v1", plan: planGeoJSON, next: geoJSONPage},86+ {name: "arcgis-feature-service", version: "arcgis-feature-service/v2", plan: planArcGIS, next: arcGISPage},87+ {name: "ogc-api-features", version: "ogc-api-features/v2", plan: planOGC, next: ogcPage},88+ {name: "raster-file", version: "raster-file/v1", next: singlePage},89+ {name: "xyz", version: "xyz/v1", next: singlePage},90+ {name: "wmts", version: "wmts-rest/v1", next: singlePage},91+ {name: "asset-file", version: "asset-file/v1", next: singlePage},92+ }93+ registry := AdapterRegistry{adapters: make(map[string]Adapter, len(definitions))}94+ for index := range definitions {95+ definition := definitions[index]96+ registry.adapters[definition.name] = definition97+ }98+ return registry99+}100+101+func planGeoJSON(project Project, source Source) (Request, error) {102+ return newFeatureRequest(project, source, project.ResolveLocator(source.Locator), source.Locator), nil103+}104+105+func planArcGIS(project Project, source Source) (Request, error) {106+ parsed, err := url.Parse(project.ResolveLocator(source.Locator))107+ if err != nil {108+ return Request{}, fmt.Errorf("source %s locator: %w", source.ID, err)109+ }110+ path := strings.TrimSuffix(parsed.Path, "/")111+ if source.Query.Layer > 0 && filepath.Base(path) != strconv.Itoa(source.Query.Layer) {112+ path += "/" + strconv.Itoa(source.Query.Layer)113+ }114+ if !strings.HasSuffix(path, "/query") {115+ path += "/query"116+ }117+ parsed.Path = path118+ query := parsed.Query()119+ query.Set("f", "geojson")120+ query.Set("outFields", "*")121+ query.Set("returnGeometry", "true")122+ extent, err := sourceExtent(source.Mapping.Geometry, project.Target.CoordinateSpace)123+ if err != nil {124+ return Request{}, fmt.Errorf("source %s query extent: %w", source.ID, err)125+ }126+ query.Set("geometry", extentString(extent))127+ query.Set("geometryType", "esriGeometryEnvelope")128+ query.Set("spatialRel", "esriSpatialRelIntersects")129+ spatialReference := source.Query.SpatialReference130+ if spatialReference == 0 {131+ spatialReference = recognizedSpatialReference(source.Mapping.Geometry.SourceSpace)132+ }133+ if spatialReference == 0 {134+ return Request{}, fmt.Errorf("source %s requires query.spatial-reference for source space %q", source.ID, source.Mapping.Geometry.SourceSpace)135+ }136+ query.Set("inSR", strconv.Itoa(spatialReference))137+ query.Set("outSR", strconv.Itoa(spatialReference))138+ objectID := source.Query.ObjectID139+ if objectID == "" {140+ objectID = "OBJECTID"141+ }142+ if !validArcGISField(objectID) {143+ return Request{}, fmt.Errorf("source %s has invalid query.object-id %q", source.ID, objectID)144+ }145+ query.Set("orderByFields", objectID+" ASC")146+ where := source.Query.Where147+ if where == "" {148+ where = "1=1"149+ }150+ query.Set("where", where)151+ if source.Query.Limit > 0 {152+ query.Set("resultRecordCount", strconv.Itoa(source.Query.Limit))153+ }154+ query.Set("resultOffset", "0")155+ parsed.RawQuery = query.Encode()156+ return newFeatureRequest(project, source, parsed.String(), parsed.String()), nil157+}158+159+func recognizedSpatialReference(sourceSpace string) int {160+ switch strings.ToUpper(strings.TrimSpace(sourceSpace)) {161+ case "EPSG:4326", "OGC:CRS84", "CRS84", "WGS84", "URN:OGC:DEF:CRS:OGC::CRS84":162+ return 4326163+ case "EPSG:3857", "EPSG:102100", "ESRI:102100", "WEB-MERCATOR":164+ return 3857165+ default:166+ return 0167+ }168+}169+170+func validArcGISField(field string) bool {171+ for index, character := range field {172+ if (character >= 'a' && character <= 'z') || (character >= 'A' && character <= 'Z') || character == '_' || (index > 0 && ((character >= '0' && character <= '9') || character == '.')) {173+ continue174+ }175+ return false176+ }177+ return field != ""178+}179+180+func planOGC(project Project, source Source) (Request, error) {181+ if source.Query.Collection == "" {182+ return Request{}, fmt.Errorf("source %s requires query.collection", source.ID)183+ }184+ parsed, err := url.Parse(project.ResolveLocator(source.Locator))185+ if err != nil {186+ return Request{}, fmt.Errorf("source %s locator: %w", source.ID, err)187+ }188+ parsed.Path = strings.TrimSuffix(parsed.Path, "/") + "/collections/" + url.PathEscape(source.Query.Collection) + "/items"189+ query := parsed.Query()190+ extent, err := sourceExtent(source.Mapping.Geometry, project.Target.CoordinateSpace)191+ if err != nil {192+ return Request{}, fmt.Errorf("source %s query extent: %w", source.ID, err)193+ }194+ query.Set("bbox", extentString(extent))195+ query.Set("f", "json")196+ if source.Query.Limit > 0 {197+ query.Set("limit", strconv.Itoa(source.Query.Limit))198+ }199+ parsed.RawQuery = query.Encode()200+ return newFeatureRequest(project, source, parsed.String(), parsed.String()), nil201+}202+203+func newFeatureRequest(project Project, source Source, locator, identity string) Request {204+ return Request{205+ Kind: RequestFeatures, Source: source.ID, Adapter: source.Adapter,206+ Locator: locator, IdentityLocator: identity, MediaType: source.MediaType,207+ License: source.License, Attribution: source.Attribution,208+ }209+}210+211+func (registry AdapterRegistry) Lookup(name string) (Adapter, error) {212+ adapter, ok := registry.adapters[name]213+ if !ok {214+ return nil, fmt.Errorf("unknown adapter %q", name)215+ }216+ return adapter, nil217+}218+219+func singlePage(_ Request, _ []byte, progress *PageProgress) (PageDecision, error) {220+ return PageDecision{Done: true, Termination: TerminationSinglePage, Returned: progress.Returned}, nil221+}222+223+func geoJSONPage(_ Request, body []byte, progress *PageProgress) (PageDecision, error) {224+ page, err := decodeFeaturePage(body)225+ if err != nil {226+ return PageDecision{}, err227+ }228+ if page.ExceededTransferLimit || nextLink(page.Links) != "" {229+ return PageDecision{}, fmt.Errorf("GeoJSON response declares pagination but the geojson adapter is single-page")230+ }231+ returned, err := pageReturned(page)232+ if err != nil {233+ return PageDecision{}, err234+ }235+ return PageDecision{Done: true, Termination: TerminationSinglePage, Returned: progress.Returned + returned}, nil236+}237+238+func arcGISPage(request Request, body []byte, progress *PageProgress) (PageDecision, error) {239+ page, err := decodeFeaturePage(body)240+ if err != nil {241+ return PageDecision{}, err242+ }243+ returned, err := pageReturned(page)244+ if err != nil {245+ return PageDecision{}, err246+ }247+ total := progress.Returned + returned248+ if err := recordPageIDs("ArcGIS", page, progress, true); err != nil {249+ return PageDecision{}, err250+ }251+ if !page.ExceededTransferLimit {252+ if progress.Returned > 0 && returned == 0 {253+ return PageDecision{}, fmt.Errorf("ArcGIS continuation is missing the page promised by its transfer limit")254+ }255+ return PageDecision{Done: true, Termination: TerminationTransferLimitClear, Returned: total}, nil256+ }257+ if returned == 0 {258+ return PageDecision{}, fmt.Errorf("ArcGIS transfer limit continuation returned no features")259+ }260+ parsed, err := url.Parse(request.IdentityLocator)261+ if err != nil {262+ return PageDecision{}, fmt.Errorf("parse ArcGIS continuation: %w", err)263+ }264+ query := parsed.Query()265+ offset, err := optionalInt64(query.Get("resultOffset"))266+ if err != nil {267+ return PageDecision{}, fmt.Errorf("ArcGIS resultOffset: %w", err)268+ }269+ query.Set("resultOffset", strconv.FormatInt(offset+returned, 10))270+ parsed.RawQuery = query.Encode()271+ next := continuationRequest(request, parsed.String())272+ return PageDecision{Next: next, Returned: total}, nil273+}274+275+func compareStableID(left, right string) int {276+ leftInteger, leftOK := new(big.Int).SetString(left, 10)277+ rightInteger, rightOK := new(big.Int).SetString(right, 10)278+ if leftOK && rightOK {279+ return leftInteger.Cmp(rightInteger)280+ }281+ return strings.Compare(left, right)282+}283+284+func recordPageIDs(protocol string, page geoJSON, progress *PageProgress, ordered bool) error {285+ if progress.SeenIDs == nil {286+ progress.SeenIDs = make(map[string]bool)287+ }288+ for _, feature := range page.Features {289+ id := scalarString(feature.ID)290+ if id == "" {291+ return fmt.Errorf("%s response has a feature without an object ID", protocol)292+ }293+ if progress.SeenIDs[id] {294+ return fmt.Errorf("%s response repeats object ID %s across pages", protocol, id)295+ }296+ if ordered && progress.LastID != "" && compareStableID(progress.LastID, id) >= 0 {297+ return fmt.Errorf("%s response object IDs are not strictly ordered", protocol)298+ }299+ progress.SeenIDs[id] = true300+ progress.LastID = id301+ }302+ return nil303+}304+305+func ogcPage(request Request, body []byte, progress *PageProgress) (PageDecision, error) {306+ page, err := decodeFeaturePage(body)307+ if err != nil {308+ return PageDecision{}, err309+ }310+ returned, err := pageReturned(page)311+ if err != nil {312+ return PageDecision{}, err313+ }314+ total := progress.Returned + returned315+ if err := recordPageIDs("OGC", page, progress, false); err != nil {316+ return PageDecision{}, err317+ }318+ matched, err := reconcileMatched(progress.Matched, page.NumberMatched)319+ if err != nil {320+ return PageDecision{}, err321+ }322+ next := nextLink(page.Links)323+ if next != "" {324+ if matched != nil && total >= *matched {325+ return PageDecision{}, fmt.Errorf("OGC response continues after all %d matched features were returned", *matched)326+ }327+ locator, err := resolveContinuation(progress.RootLocator, request.IdentityLocator, next)328+ if err != nil {329+ return PageDecision{}, err330+ }331+ return PageDecision{Next: continuationRequest(request, locator), Returned: total, Matched: matched}, nil332+ }333+ if matched != nil && total != *matched {334+ return PageDecision{}, fmt.Errorf("OGC response is incomplete: %d matched but only %d returned", *matched, total)335+ }336+ return PageDecision{Done: true, Termination: TerminationNextExhausted, Returned: total, Matched: matched}, nil337+}338+339+func decodeFeaturePage(body []byte) (geoJSON, error) {340+ decoder := json.NewDecoder(bytes.NewReader(body))341+ decoder.UseNumber()342+ var page geoJSON343+ if err := decoder.Decode(&page); err != nil {344+ return geoJSON{}, fmt.Errorf("decode feature page: %w", err)345+ }346+ if err := decoder.Decode(&struct{}{}); !errors.Is(err, io.EOF) {347+ if err == nil {348+ return geoJSON{}, fmt.Errorf("feature page carries more than one JSON value")349+ }350+ return geoJSON{}, fmt.Errorf("decode feature page trailer: %w", err)351+ }352+ if page.Type != "FeatureCollection" {353+ return geoJSON{}, fmt.Errorf("feature page is %q, want FeatureCollection", page.Type)354+ }355+ return page, nil356+}357+358+func pageReturned(page geoJSON) (int64, error) {359+ returned := int64(len(page.Features))360+ if page.NumberReturned != nil && *page.NumberReturned != returned {361+ return 0, fmt.Errorf("response count differs from its feature payload")362+ }363+ return returned, nil364+}365+366+func reconcileMatched(held, next *int64) (*int64, error) {367+ if next == nil {368+ return held, nil369+ }370+ if *next < 0 {371+ return nil, fmt.Errorf("response has a negative matched count")372+ }373+ if held != nil && *held != *next {374+ return nil, fmt.Errorf("response changed numberMatched from %d to %d", *held, *next)375+ }376+ value := *next377+ return &value, nil378+}379+380+func nextLink(links []geoJSONLink) string {381+ for _, link := range links {382+ if strings.EqualFold(link.Rel, "next") && strings.TrimSpace(link.Href) != "" {383+ return strings.TrimSpace(link.Href)384+ }385+ }386+ return ""387+}388+389+func resolveContinuation(rootLocator, currentLocator, reference string) (string, error) {390+ root, err := url.Parse(rootLocator)391+ if err != nil {392+ return "", fmt.Errorf("parse continuation root: %w", err)393+ }394+ current, err := url.Parse(currentLocator)395+ if err != nil {396+ return "", fmt.Errorf("parse current page: %w", err)397+ }398+ next, err := url.Parse(reference)399+ if err != nil {400+ return "", fmt.Errorf("parse next page: %w", err)401+ }402+ resolved := current.ResolveReference(next)403+ if !strings.EqualFold(root.Scheme, resolved.Scheme) || !strings.EqualFold(root.Host, resolved.Host) {404+ return "", fmt.Errorf("next page leaves the configured source origin")405+ }406+ return resolved.String(), nil407+}408+409+func continuationRequest(root Request, locator string) Request {410+ next := root411+ next.Locator = locator412+ next.IdentityLocator = locator413+ finalizeRequest(&next)414+ return next415+}416+417+func optionalInt64(value string) (int64, error) {418+ if value == "" {419+ return 0, nil420+ }421+ return strconv.ParseInt(value, 10, 64)422+}@@ -0,0 +1,132 @@
1+package authoring2+3+import (4+ "net/url"5+ "path/filepath"6+ "strings"7+ "testing"8+)9+10+func TestAdapterRegistryIsTheSingleVersionAuthority(t *testing.T) {11+ registry := defaultAdapterRegistry()12+ tests := map[string]string{13+ "geojson": "geojson/v1", "arcgis-feature-service": "arcgis-feature-service/v2",14+ "ogc-api-features": "ogc-api-features/v2", "raster-file": "raster-file/v1",15+ "xyz": "xyz/v1", "wmts": "wmts-rest/v1", "asset-file": "asset-file/v1",16+ }17+ for name, version := range tests {18+ adapter, err := registry.Lookup(name)19+ if err != nil || adapter.Version() != version {20+ t.Fatalf("adapter %s = %v, %v", name, adapter, err)21+ }22+ }23+ if _, err := registry.Lookup("unknown"); err == nil || !strings.Contains(err.Error(), "unknown adapter") {24+ t.Fatalf("unknown adapter = %v", err)25+ }26+}27+28+func TestArcGISContinuationUsesStableOffsets(t *testing.T) {29+ request := featureSetRequest("sample-roads", "arcgis-feature-service", "https://example.invalid/FeatureServer/0/query?f=geojson&resultRecordCount=2")30+ adapter, err := defaultAdapterRegistry().Lookup(request.Adapter)31+ if err != nil {32+ t.Fatal(err)33+ }34+ decision, err := adapter.Next(request, []byte(`{"type":"FeatureCollection","exceededTransferLimit":true,"features":[{"id":"a"},{"id":"b"}]}`), &PageProgress{})35+ if err != nil {36+ t.Fatal(err)37+ }38+ if decision.Done || !strings.Contains(decision.Next.IdentityLocator, "resultOffset=2") || decision.Returned != 2 {39+ t.Fatalf("ArcGIS decision = %+v", decision)40+ }41+}42+43+func TestArcGISPaginationRefusesOverlapAndMissingContinuations(t *testing.T) {44+ request := featureSetRequest("sample-roads", "arcgis-feature-service", "https://example.invalid/FeatureServer/0/query?f=geojson&resultRecordCount=2")45+ adapter, err := defaultAdapterRegistry().Lookup(request.Adapter)46+ if err != nil {47+ t.Fatal(err)48+ }49+ progress := PageProgress{SeenIDs: make(map[string]bool)}50+ first, err := adapter.Next(request, []byte(`{"type":"FeatureCollection","exceededTransferLimit":true,"features":[{"id":1},{"id":2}]}`), &progress)51+ if err != nil {52+ t.Fatal(err)53+ }54+ progress.Returned = first.Returned55+ if _, err := adapter.Next(first.Next, []byte(`{"type":"FeatureCollection","features":[{"id":2},{"id":3}]}`), &progress); err == nil || !strings.Contains(err.Error(), "repeats object ID") {56+ t.Fatalf("overlapping page = %v", err)57+ }58+59+ progress = PageProgress{Returned: first.Returned, SeenIDs: map[string]bool{"1": true, "2": true}, LastID: "2"}60+ if _, err := adapter.Next(first.Next, []byte(`{"type":"FeatureCollection","features":[]}`), &progress); err == nil || !strings.Contains(err.Error(), "missing") {61+ t.Fatalf("missing continuation = %v", err)62+ }63+}64+65+func TestOGCPaginationRefusesDuplicateFeatureIDsAcrossPages(t *testing.T) {66+ request := featureSetRequest("sample-roads", "ogc-api-features", "https://example.invalid/collections/sample/items")67+ adapter, err := defaultAdapterRegistry().Lookup(request.Adapter)68+ if err != nil {69+ t.Fatal(err)70+ }71+ progress := PageProgress{RootLocator: request.IdentityLocator, SeenIDs: make(map[string]bool)}72+ first, err := adapter.Next(request, []byte(`{"type":"FeatureCollection","numberReturned":1,"features":[{"id":"same"}],"links":[{"rel":"next","href":"?page=2"}]}`), &progress)73+ if err != nil {74+ t.Fatal(err)75+ }76+ progress.Returned = first.Returned77+ if _, err := adapter.Next(first.Next, []byte(`{"type":"FeatureCollection","numberReturned":1,"features":[{"id":"same"}]}`), &progress); err == nil || !strings.Contains(err.Error(), "repeats object ID") {78+ t.Fatalf("duplicate OGC page = %v", err)79+ }80+}81+82+func TestArcGISPlanAlwaysCarriesTheConfiguredSpatialEnvelope(t *testing.T) {83+ project, err := LoadProject(filepath.Join("..", "..", "examples", "sample-region"+ProjectExtension))84+ if err != nil {85+ t.Fatal(err)86+ }87+ source := project.Sources[0]88+ source.Adapter = "arcgis-feature-service"89+ source.Locator = "https://example.invalid/FeatureServer"90+ source.Query.Layer = 291+ source.Query.Limit = 50092+ source.Query.ObjectID = "sample_id"93+ source.Query.SpatialReference = 432694+ request, err := featureRequest(project, source)95+ if err != nil {96+ t.Fatal(err)97+ }98+ parsed, err := url.Parse(request.IdentityLocator)99+ if err != nil {100+ t.Fatal(err)101+ }102+ query := parsed.Query()103+ extent, err := sourceExtent(source.Mapping.Geometry, project.Target.CoordinateSpace)104+ if err != nil {105+ t.Fatal(err)106+ }107+ if parsed.Path != "/FeatureServer/2/query" || query.Get("geometry") != extentString(extent) || query.Get("geometryType") != "esriGeometryEnvelope" || query.Get("spatialRel") != "esriSpatialRelIntersects" || query.Get("resultRecordCount") != "500" || query.Get("resultOffset") != "0" || query.Get("orderByFields") != "sample_id ASC" || query.Get("inSR") != "4326" || query.Get("outSR") != "4326" {108+ t.Fatalf("ArcGIS request = %s", request.IdentityLocator)109+ }110+}111+112+func TestArcGISPlanRefusesAnImplicitUnknownSpatialReference(t *testing.T) {113+ project, err := LoadProject(filepath.Join("..", "..", "examples", "sample-region"+ProjectExtension))114+ if err != nil {115+ t.Fatal(err)116+ }117+ source := project.Sources[0]118+ source.Adapter = "arcgis-feature-service"119+ source.Locator = "https://example.invalid/FeatureServer/0"120+ if _, err := featureRequest(project, source); err == nil || !strings.Contains(err.Error(), "spatial-reference") {121+ t.Fatalf("implicit unknown ArcGIS spatial reference = %v", err)122+ }123+124+ source.Mapping.Geometry.SourceSpace = "EPSG:4326"125+ request, err := featureRequest(project, source)126+ if err != nil {127+ t.Fatal(err)128+ }129+ if !strings.Contains(request.IdentityLocator, "inSR=4326") || !strings.Contains(request.IdentityLocator, "outSR=4326") {130+ t.Fatalf("recognized spatial reference request = %s", request.IdentityLocator)131+ }132+}@@ -22,7 +22,8 @@ func assemble(project Project, observations []observation) (vnext.Volume, error)
2222 ID: project.Target.World, Title: project.Target.Title,2323 CoordinateSpace: vnext.CoordinateSpace{2424 ID: space.ID, Kind: space.Kind, Unit: space.Unit, Definition: space.Definition,25- Extent: space.Extent, SourceZoom: space.SourceZoom, FirstTile: space.FirstTile,25+ Extent: space.Extent, SourceZoom: space.SourceZoom,26+ OriginX: space.OriginX, OriginY: space.OriginY, FirstTile: space.FirstTile,2627 TileSize: space.TileSize, Size: space.Size,2728 },2829 Presentation: vnext.Presentation{ID: project.Presentation.ID, Title: project.Presentation.Title},@@ -54,74 +54,48 @@ func Build(ctx context.Context, options BuildOptions) (BuildResult, error) {
5454 if options.PlanOnly {5555 return result, nil5656 }57- var captures map[string]Capture58- var capturedBytes int6459- accountCapture := func(planned PlannedRequest, capture Capture) error {60- if capture.Body.Length > planned.MaxBytes {61- return fmt.Errorf("captured evidence for %s is %d bytes, exceeding request budget %d", planned.Source, capture.Body.Length, planned.MaxBytes)62- }63- if capture.Body.Length > plan.Budgets.TotalBytes-capturedBytes {64- return fmt.Errorf("captured evidence exceeds total byte budget %d", plan.Budgets.TotalBytes)65- }66- capturedBytes += capture.Body.Length67- return nil68- }57+ var evidence buildEvidence6958 if options.ReplayPath != "" {7059 selection, err := loadEvidenceSelection(options.ReplayPath)7160 if err != nil {7261 return result, err7362 }74- captures, err = replayCaptures(plan, selection, cache)63+ evidence, err = replayPlanEvidence(plan, selection, cache)7564 if err != nil {7665 return result, err7766 }78- for index, planned := range plan.Requests {79- capture := captures[planned.ID]80- if err := accountCapture(planned, capture); err != nil {81- return result, err82- }83- emit(options, Event{Stage: "capture", Message: planned.Source, Current: index + 1, Total: len(plan.Requests), Bytes: capture.Body.Length, Cached: true})84- }8567 } else {86- captures = make(map[string]Capture, len(plan.Requests))87- acquisitions := make(map[string]Capture, len(plan.Requests))88- for index, planned := range plan.Requests {89- key := requestCacheKey(planned.Request)90- capture, cached := acquisitions[key], true91- if capture.RequestHash == "" {92- var err error93- capture, cached, err = cache.Acquire(ctx, planned.Request, options.Offline)94- if err != nil {95- return result, err96- }97- acquisitions[key] = capture98- }99- captures[planned.ID] = capture100- if err := accountCapture(planned, capture); err != nil {101- return result, err102- }103- emit(options, Event{104- Stage: "capture", Message: planned.Source, Current: index + 1,105- Total: len(plan.Requests), Bytes: capture.Body.Length, Cached: cached,106- })68+ evidence, err = acquirePlanEvidence(ctx, plan, cache, options.Offline)69+ if err != nil {70+ return result, err10771 }10872 }73+ for index, planned := range plan.Requests {74+ set := evidence.Sets[planned.ID]75+ emit(options, Event{Stage: "capture", Message: planned.Source, Current: index + 1, Total: len(plan.Requests), Bytes: set.TotalBytes, Cached: evidence.Cached[planned.ID]})76+ }77+ captures := evidence.Captures10978 var observations []observation11079 for sourceIndex, source := range project.Sources {11180 planned, ok := featurePlan(plan, source.ID)11281 if !ok {11382 return result, fmt.Errorf("source %s has no planned request", source.ID)11483 }115- capture := captures[planned.ID]116- body, err := os.ReadFile(cache.BlobPath(capture.Body))117- if err != nil {118- return result, fmt.Errorf("read captured source %s: %w", source.ID, err)84+ root, ok := captureRootForRequest(evidence.Sets[planned.ID], planned.Request)85+ if !ok {86+ return result, fmt.Errorf("source %s has no captured root", source.ID)11987 }120- items, err := observe(project, sourceIndex, source, capture, body)121- if err != nil {122- return result, err88+ for _, capture := range root.Pages {89+ body, err := os.ReadFile(cache.BlobPath(capture.Body))90+ if err != nil {91+ return result, fmt.Errorf("read captured source %s: %w", source.ID, err)92+ }93+ items, err := observeCapturedPage(project, sourceIndex, source, capture, body)94+ if err != nil {95+ return result, err96+ }97+ observations = append(observations, items...)12398 }124- observations = append(observations, items...)12599 emit(options, Event{Stage: "observe", Message: source.ID, Current: sourceIndex + 1, Total: len(project.Sources)})126100 }127101 volume, err := assemble(project, observations)@@ -135,7 +109,7 @@ func Build(ctx context.Context, options BuildOptions) (BuildResult, error) {
135109 if err != nil {136110 return result, err137111 }138- receipt, createdAt, err := buildReceipt(plan, captures)112+ receipt, createdAt, err := buildReceiptSets(plan, captures, evidence.Sets)139113 if err != nil {140114 return result, err141115 }@@ -0,0 +1,395 @@
1+package authoring2+3+import (4+ "bytes"5+ "context"6+ "crypto/sha256"7+ "encoding/hex"8+ "encoding/json"9+ "errors"10+ "fmt"11+ "io"12+ "os"13+ "path/filepath"14+ "sort"15+)16+17+const CaptureSetFormat = "atlas-capture-set/v1"18+19+type CaptureSetOptions struct {20+ Offline bool21+ MaxRequests int22+ MaxBytes int6423+}24+25+// CaptureSet is the atomic evidence unit selected by a build. Its manifest is26+// published only after every root and continuation has terminated completely.27+type CaptureSet struct {28+ Format string `json:"format"`29+ ID string `json:"id"`30+ Digest string `json:"digest"`31+ Adapter string `json:"adapter"`32+ Version string `json:"adapterVersion"`33+ Kind RequestKind `json:"kind"`34+ CapturedAt string `json:"capturedAt"`35+ TotalBytes int64 `json:"totalBytes"`36+ Roots []CaptureRoot `json:"roots"`37+}38+39+type CaptureRoot struct {40+ Acquisition string `json:"acquisition"`41+ Pages []Capture `json:"pages"`42+ Termination CaptureTermination `json:"termination"`43+}44+45+type CaptureTermination struct {46+ Kind string `json:"kind"`47+ Returned int64 `json:"returned,omitempty"`48+ Matched *int64 `json:"matched,omitempty"`49+}50+51+// AcquireSet returns one complete logical acquisition. Offline selection never52+// composes independently-latest pages, so a build cannot observe a torn crawl.53+func (cache *Cache) AcquireSet(ctx context.Context, requests []Request, registry AdapterRegistry, options CaptureSetOptions) (CaptureSet, bool, error) {54+ roots, err := canonicalSetRequests(requests)55+ if err != nil {56+ return CaptureSet{}, false, err57+ }58+ setID := captureSetID(roots)59+ if options.Offline {60+ set, err := cache.LatestSet(setID)61+ if err != nil {62+ return CaptureSet{}, false, fmt.Errorf("capture set %s is not cached: %w", roots[0].Source, err)63+ }64+ if err := validateSetForRequests(set, roots); err != nil {65+ return CaptureSet{}, false, err66+ }67+ return set, true, nil68+ }69+ if options.MaxRequests <= 0 {70+ options.MaxRequests = defaultBuildBudgets.Requests71+ }72+ if options.MaxBytes <= 0 {73+ options.MaxBytes = defaultBuildBudgets.TotalBytes74+ }75+ adapter, err := registry.Lookup(roots[0].Adapter)76+ if err != nil {77+ return CaptureSet{}, false, err78+ }79+ if adapter.Version() != roots[0].AdapterVersion {80+ return CaptureSet{}, false, fmt.Errorf("adapter %s version is %s, request requires %s", adapter.Name(), adapter.Version(), roots[0].AdapterVersion)81+ }82+ set := CaptureSet{83+ Format: CaptureSetFormat, ID: setID, Adapter: adapter.Name(),84+ Version: adapter.Version(), Kind: roots[0].Kind,85+ Roots: make([]CaptureRoot, 0, len(roots)),86+ }87+ usedRequests := 088+ for _, root := range roots {89+ captured, count, err := cache.acquireRoot(ctx, root, adapter, options, usedRequests, set.TotalBytes)90+ if err != nil {91+ return CaptureSet{}, false, fmt.Errorf("capture set %s: %w", root.Source, err)92+ }93+ usedRequests += count94+ for _, page := range captured.Pages {95+ set.TotalBytes += page.Body.Length96+ if page.CapturedAt > set.CapturedAt {97+ set.CapturedAt = page.CapturedAt98+ }99+ }100+ set.Roots = append(set.Roots, captured)101+ }102+ set.Digest, err = captureSetDigest(set)103+ if err != nil {104+ return CaptureSet{}, false, err105+ }106+ data, err := json.Marshal(set)107+ if err != nil {108+ return CaptureSet{}, false, err109+ }110+ path, err := cache.captureSetPath(set.ID, set.Digest)111+ if err != nil {112+ return CaptureSet{}, false, err113+ }114+ written, err := writeOnce(path, append(data, '\n'))115+ if err != nil && !errors.Is(err, errExistingContent) {116+ return CaptureSet{}, false, fmt.Errorf("publish capture set: %w", err)117+ }118+ selected, err := cache.SelectSet(set.ID, set.Digest)119+ if err != nil {120+ return CaptureSet{}, false, fmt.Errorf("validate capture set: %w", err)121+ }122+ return selected, !written, nil123+}124+125+func (cache *Cache) acquireRoot(ctx context.Context, root Request, adapter Adapter, options CaptureSetOptions, usedRequests int, usedBytes int64) (CaptureRoot, int, error) {126+ captured := CaptureRoot{Acquisition: requestCacheKey(root)}127+ current := root128+ progress := PageProgress{RootLocator: root.IdentityLocator, SeenIDs: make(map[string]bool)}129+ seen := make(map[string]bool)130+ for {131+ if usedRequests+len(captured.Pages) >= options.MaxRequests {132+ return CaptureRoot{}, 0, fmt.Errorf("pagination exceeds request budget %d", options.MaxRequests)133+ }134+ if seen[current.IdentityLocator] {135+ return CaptureRoot{}, 0, fmt.Errorf("pagination continuation cycle at %s", current.IdentityLocator)136+ }137+ seen[current.IdentityLocator] = true138+ capture, _, err := cache.Acquire(ctx, current, false)139+ if err != nil {140+ return CaptureRoot{}, 0, err141+ }142+ if capture.Body.Length > options.MaxBytes-usedBytes-rootBytes(captured) {143+ return CaptureRoot{}, 0, fmt.Errorf("pagination exceeds capture-set byte budget %d", options.MaxBytes)144+ }145+ captured.Pages = append(captured.Pages, capture)146+ body, err := os.ReadFile(cache.BlobPath(capture.Body))147+ if err != nil {148+ return CaptureRoot{}, 0, fmt.Errorf("read continuation page: %w", err)149+ }150+ decision, err := adapter.Next(current, body, &progress)151+ if err != nil {152+ return CaptureRoot{}, 0, err153+ }154+ progress.Returned, progress.Matched = decision.Returned, decision.Matched155+ if decision.Done {156+ captured.Termination = CaptureTermination{Kind: decision.Termination, Returned: decision.Returned, Matched: decision.Matched}157+ return captured, len(captured.Pages), nil158+ }159+ current = decision.Next160+ }161+}162+163+func rootBytes(root CaptureRoot) int64 {164+ var total int64165+ for _, page := range root.Pages {166+ total += page.Body.Length167+ }168+ return total169+}170+171+func canonicalSetRequests(requests []Request) ([]Request, error) {172+ if len(requests) == 0 {173+ return nil, fmt.Errorf("capture set has no requests")174+ }175+ roots := append([]Request(nil), requests...)176+ sort.Slice(roots, func(i, j int) bool { return requestCacheKey(roots[i]) < requestCacheKey(roots[j]) })177+ seen := make(map[string]bool, len(roots))178+ for _, request := range roots {179+ key := requestCacheKey(request)180+ if err := validateSHA256("acquisition", key); err != nil {181+ return nil, err182+ }183+ if request.Adapter == "" || request.AdapterVersion == "" || request.Kind == "" || seen[key] {184+ return nil, fmt.Errorf("capture set has an invalid or duplicate request")185+ }186+ if request.Adapter != roots[0].Adapter || request.AdapterVersion != roots[0].AdapterVersion || request.Kind != roots[0].Kind {187+ return nil, fmt.Errorf("capture set mixes adapters, versions, or kinds")188+ }189+ seen[key] = true190+ }191+ return roots, nil192+}193+194+func captureSetID(requests []Request) string {195+ keys := make([]string, len(requests))196+ for index, request := range requests {197+ keys[index] = requestCacheKey(request)198+ }199+ sort.Strings(keys)200+ data, _ := json.Marshal(keys)201+ digest := sha256.Sum256(data)202+ return hex.EncodeToString(digest[:])203+}204+205+func captureSetDigest(set CaptureSet) (string, error) {206+ set.Digest = ""207+ data, err := json.Marshal(set)208+ if err != nil {209+ return "", err210+ }211+ digest := sha256.Sum256(data)212+ return hex.EncodeToString(digest[:]), nil213+}214+215+func (cache *Cache) LatestSet(setID string) (CaptureSet, error) {216+ dir, err := cache.captureSetDir(setID)217+ if err != nil {218+ return CaptureSet{}, err219+ }220+ entries, err := os.ReadDir(dir)221+ if err != nil {222+ return CaptureSet{}, err223+ }224+ var sets []CaptureSet225+ for _, entry := range entries {226+ if entry.IsDir() || filepath.Ext(entry.Name()) != ".json" {227+ continue228+ }229+ set, err := cache.SelectSet(setID, entry.Name()[:len(entry.Name())-len(".json")])230+ if err != nil {231+ return CaptureSet{}, fmt.Errorf("validate capture set %s: %w", entry.Name(), err)232+ }233+ sets = append(sets, set)234+ }235+ if len(sets) == 0 {236+ return CaptureSet{}, os.ErrNotExist237+ }238+ sort.Slice(sets, func(i, j int) bool {239+ if sets[i].CapturedAt != sets[j].CapturedAt {240+ return sets[i].CapturedAt < sets[j].CapturedAt241+ }242+ return sets[i].Digest < sets[j].Digest243+ })244+ return sets[len(sets)-1], nil245+}246+247+func (cache *Cache) SelectSet(setID, digest string) (CaptureSet, error) {248+ path, err := cache.captureSetPath(setID, digest)249+ if err != nil {250+ return CaptureSet{}, err251+ }252+ data, err := os.ReadFile(path)253+ if err != nil {254+ return CaptureSet{}, err255+ }256+ set, err := decodeCaptureSet(data)257+ if err != nil {258+ return CaptureSet{}, err259+ }260+ if err := cache.validateCaptureSet(set, setID, digest); err != nil {261+ return CaptureSet{}, err262+ }263+ return set, nil264+}265+266+func decodeCaptureSet(data []byte) (CaptureSet, error) {267+ decoder := json.NewDecoder(bytes.NewReader(data))268+ decoder.DisallowUnknownFields()269+ var set CaptureSet270+ if err := decoder.Decode(&set); err != nil {271+ return CaptureSet{}, err272+ }273+ if err := decoder.Decode(&struct{}{}); !errors.Is(err, io.EOF) {274+ return CaptureSet{}, fmt.Errorf("capture set has invalid trailing data")275+ }276+ return set, nil277+}278+279+func (cache *Cache) validateCaptureSet(set CaptureSet, setID, digest string) error {280+ if set.Format != CaptureSetFormat || set.ID != setID || set.Digest != digest || set.Adapter == "" || set.Version == "" || len(set.Roots) == 0 {281+ return fmt.Errorf("capture set envelope is invalid")282+ }283+ switch set.Kind {284+ case RequestFeatures, RequestRaster, RequestAsset:285+ default:286+ return fmt.Errorf("capture set kind %q is invalid", set.Kind)287+ }288+ computed, err := captureSetDigest(set)289+ if err != nil || computed != digest {290+ return fmt.Errorf("capture set digest does not match its contents")291+ }292+ var total int64293+ previous := ""294+ latest := ""295+ adapter, err := defaultAdapterRegistry().Lookup(set.Adapter)296+ if err != nil || adapter.Version() != set.Version {297+ return fmt.Errorf("capture set adapter is unavailable")298+ }299+ for _, root := range set.Roots {300+ if validateSHA256("acquisition", root.Acquisition) != nil || root.Acquisition <= previous || len(root.Pages) == 0 || root.Termination.Kind == "" {301+ return fmt.Errorf("capture set root is invalid")302+ }303+ previous = root.Acquisition304+ if root.Pages[0].RequestHash != root.Acquisition {305+ return fmt.Errorf("capture set root does not begin with its acquisition")306+ }307+ rootSource := root.Pages[0].Source308+ for _, page := range root.Pages {309+ if page.Source != rootSource || page.Adapter != set.Adapter || page.Kind != string(set.Kind) {310+ return fmt.Errorf("capture set page envelope differs from its root")311+ }312+ if err := cache.validateCapture(page, page.RequestHash, page.Body.SHA256); err != nil {313+ return err314+ }315+ if page.Body.Length > int64(^uint64(0)>>1)-total {316+ return fmt.Errorf("capture set byte total overflows")317+ }318+ total += page.Body.Length319+ if page.CapturedAt > latest {320+ latest = page.CapturedAt321+ }322+ }323+ if err := cache.validateCaptureRoot(set, root, adapter); err != nil {324+ return err325+ }326+ }327+ if total != set.TotalBytes {328+ return fmt.Errorf("capture set byte total is %d, want %d", total, set.TotalBytes)329+ }330+ if latest != set.CapturedAt {331+ return fmt.Errorf("capture set time does not match its pages")332+ }333+ return nil334+}335+336+func (cache *Cache) validateCaptureRoot(set CaptureSet, root CaptureRoot, adapter Adapter) error {337+ progress := PageProgress{RootLocator: root.Pages[0].Locator, SeenIDs: make(map[string]bool)}338+ for index, page := range root.Pages {339+ body, err := os.ReadFile(cache.BlobPath(page.Body))340+ if err != nil {341+ return err342+ }343+ request := Request{344+ Kind: set.Kind, Source: page.Source, Adapter: set.Adapter, AdapterVersion: set.Version,345+ Locator: page.Locator, IdentityLocator: page.Locator, MediaType: page.Body.MediaType,346+ }347+ decision, err := adapter.Next(request, body, &progress)348+ if err != nil {349+ return fmt.Errorf("validate captured continuation: %w", err)350+ }351+ progress.Returned, progress.Matched = decision.Returned, decision.Matched352+ last := index == len(root.Pages)-1353+ if last && (!decision.Done || !terminationMatches(root.Termination, decision)) {354+ return fmt.Errorf("capture set termination does not match its final page")355+ }356+ if !last && (decision.Done || decision.Next.IdentityLocator != root.Pages[index+1].Locator) {357+ return fmt.Errorf("capture set page order does not follow recorded continuations")358+ }359+ }360+ return nil361+}362+363+func terminationMatches(termination CaptureTermination, decision PageDecision) bool {364+ return termination.Kind == decision.Termination && termination.Returned == decision.Returned && equalOptionalInt64(termination.Matched, decision.Matched)365+}366+367+func validateSetForRequests(set CaptureSet, requests []Request) error {368+ if set.ID != captureSetID(requests) || len(set.Roots) != len(requests) {369+ return fmt.Errorf("cached capture set does not match the planned requests")370+ }371+ for index, request := range requests {372+ if set.Roots[index].Acquisition != requestCacheKey(request) || set.Adapter != request.Adapter || set.Version != request.AdapterVersion || set.Kind != request.Kind {373+ return fmt.Errorf("cached capture set does not match request %s", request.Source)374+ }375+ }376+ return nil377+}378+379+func (cache *Cache) captureSetDir(setID string) (string, error) {380+ if err := validateSHA256("capture set", setID); err != nil {381+ return "", err382+ }383+ return filepath.Join(cache.root, "capture-sets", setID[:2], setID), nil384+}385+386+func (cache *Cache) captureSetPath(setID, digest string) (string, error) {387+ dir, err := cache.captureSetDir(setID)388+ if err != nil {389+ return "", err390+ }391+ if err := validateSHA256("capture set digest", digest); err != nil {392+ return "", err393+ }394+ return filepath.Join(dir, digest+".json"), nil395+}@@ -0,0 +1,193 @@
1+package authoring2+3+import (4+ "context"5+ "fmt"6+ "net/http"7+ "net/http/httptest"8+ "os"9+ "path/filepath"10+ "strings"11+ "sync/atomic"12+ "testing"13+ "time"14+)15+16+func TestCaptureSetFollowsPagesAndReplaysOfflineAtomically(t *testing.T) {17+ var calls atomic.Int6418+ server := httptest.NewServer(http.HandlerFunc(func(rw http.ResponseWriter, request *http.Request) {19+ calls.Add(1)20+ rw.Header().Set("Content-Type", "application/geo+json")21+ switch request.URL.Query().Get("page") {22+ case "2":23+ _, _ = rw.Write([]byte(`{"type":"FeatureCollection","numberMatched":2,"numberReturned":1,"features":[{"type":"Feature","id":"second","properties":{},"geometry":null}]}`))24+ default:25+ _, _ = fmt.Fprintf(rw, `{"type":"FeatureCollection","numberMatched":2,"numberReturned":1,"features":[{"type":"Feature","id":"first","properties":{},"geometry":null}],"links":[{"rel":"next","href":%q}]}`, serverURL(request)+"?page=2")26+ }27+ }))28+ defer server.Close()29+30+ cache := OpenCache(t.TempDir())31+ cache.now = func() time.Time { return time.Date(2026, 8, 8, 12, 0, 0, 0, time.UTC) }32+ request := featureSetRequest("sample-roads", "ogc-api-features", server.URL)33+ registry := defaultAdapterRegistry()34+ set, cached, err := cache.AcquireSet(context.Background(), []Request{request}, registry, CaptureSetOptions{MaxRequests: 8})35+ if err != nil {36+ t.Fatal(err)37+ }38+ if cached || len(set.Roots) != 1 || len(set.Roots[0].Pages) != 2 {39+ t.Fatalf("online set = cached %v, %+v", cached, set)40+ }41+ if set.Roots[0].Termination.Kind != TerminationNextExhausted || set.Roots[0].Termination.Returned != 2 || set.Roots[0].Termination.Matched == nil || *set.Roots[0].Termination.Matched != 2 {42+ t.Fatalf("termination = %+v", set.Roots[0].Termination)43+ }44+ if calls.Load() != 2 {45+ t.Fatalf("online calls = %d, want 2", calls.Load())46+ }47+48+ offline, cached, err := cache.AcquireSet(context.Background(), []Request{request}, registry, CaptureSetOptions{Offline: true, MaxRequests: 8})49+ if err != nil {50+ t.Fatal(err)51+ }52+ if !cached || offline.Digest != set.Digest || calls.Load() != 2 {53+ t.Fatalf("offline set = cached %v digest %s calls %d", cached, offline.Digest, calls.Load())54+ }55+ selected, err := cache.SelectSet(set.ID, set.Digest)56+ if err != nil || selected.Digest != set.Digest {57+ t.Fatalf("select exact set = %+v, %v", selected, err)58+ }59+}60+61+func TestCaptureSetRefusesIncompletePaginationWithoutPublishingASet(t *testing.T) {62+ server := httptest.NewServer(http.HandlerFunc(func(rw http.ResponseWriter, request *http.Request) {63+ rw.Header().Set("Content-Type", "application/geo+json")64+ if request.URL.Query().Get("page") == "2" {65+ _, _ = rw.Write([]byte(`{"type":"FeatureCollection","numberMatched":3,"numberReturned":1,"features":[{"type":"Feature","id":"second","properties":{},"geometry":null}]}`))66+ return67+ }68+ _, _ = fmt.Fprintf(rw, `{"type":"FeatureCollection","numberMatched":3,"numberReturned":1,"features":[{"type":"Feature","id":"first","properties":{},"geometry":null}],"links":[{"rel":"next","href":%q}]}`, serverURL(request)+"?page=2")69+ }))70+ defer server.Close()71+72+ cache := OpenCache(t.TempDir())73+ request := featureSetRequest("sample-roads", "ogc-api-features", server.URL)74+ setID := captureSetID([]Request{request})75+ _, _, err := cache.AcquireSet(context.Background(), []Request{request}, defaultAdapterRegistry(), CaptureSetOptions{MaxRequests: 8})76+ if err == nil || !strings.Contains(err.Error(), "incomplete") {77+ t.Fatalf("incomplete set = %v", err)78+ }79+ if _, err := cache.LatestSet(setID); !os.IsNotExist(err) {80+ t.Fatalf("partial set became selectable: %v", err)81+ }82+ if _, err := os.Stat(filepath.Join(cache.Root(), "capture-sets")); !os.IsNotExist(err) {83+ t.Fatalf("partial set manifest residue = %v", err)84+ }85+}86+87+func TestCaptureSetRefusesContinuationCycles(t *testing.T) {88+ server := httptest.NewServer(http.HandlerFunc(func(rw http.ResponseWriter, request *http.Request) {89+ rw.Header().Set("Content-Type", "application/geo+json")90+ _, _ = fmt.Fprintf(rw, `{"type":"FeatureCollection","numberReturned":0,"features":[],"links":[{"rel":"next","href":%q}]}`, serverURL(request))91+ }))92+ defer server.Close()93+94+ request := featureSetRequest("sample-roads", "ogc-api-features", server.URL)95+ _, _, err := OpenCache(t.TempDir()).AcquireSet(context.Background(), []Request{request}, defaultAdapterRegistry(), CaptureSetOptions{MaxRequests: 8})96+ if err == nil || !strings.Contains(err.Error(), "cycle") {97+ t.Fatalf("continuation cycle = %v", err)98+ }99+}100+101+func TestCaptureSetDoesNotPublishWhenAnyPlannedRootFails(t *testing.T) {102+ root := t.TempDir()103+ firstPath := filepath.Join(root, "first.bin")104+ if err := os.WriteFile(firstPath, []byte("first"), 0o644); err != nil {105+ t.Fatal(err)106+ }107+ requests := []Request{108+ assetSetRequest("sample-assets", "first", firstPath),109+ assetSetRequest("sample-assets", "missing", filepath.Join(root, "missing.bin")),110+ }111+ cache := OpenCache(filepath.Join(root, "cache"))112+ setID := captureSetID(requests)113+ _, _, err := cache.AcquireSet(context.Background(), requests, defaultAdapterRegistry(), CaptureSetOptions{MaxRequests: 8})114+ if err == nil || !strings.Contains(err.Error(), "missing") {115+ t.Fatalf("failed multi-root set = %v", err)116+ }117+ if _, err := cache.LatestSet(setID); !os.IsNotExist(err) {118+ t.Fatalf("partial multi-root set became selectable: %v", err)119+ }120+}121+122+func TestPlanReportsCachedOnlyAfterTheWholeSetIsPublished(t *testing.T) {123+ root := t.TempDir()124+ source := filepath.Join(root, "sample-region.geojson")125+ if err := os.WriteFile(source, []byte(`{"type":"FeatureCollection","features":[]}`), 0o644); err != nil {126+ t.Fatal(err)127+ }128+ cache := OpenCache(filepath.Join(root, "cache"))129+ request := featureSetRequest("sample-features", "geojson", source)130+ plan := Plan{Budgets: defaultBuildBudgets, Requests: []PlannedRequest{{Request: request}}}131+ if _, _, err := cache.Acquire(context.Background(), request, false); err != nil {132+ t.Fatal(err)133+ }134+ markCachedCaptureSets(&plan, cache)135+ if plan.Cached != 0 || plan.Requests[0].Cached {136+ t.Fatal("an individually cached page was reported as a complete set")137+ }138+ if _, _, err := cache.AcquireSet(context.Background(), []Request{request}, defaultAdapterRegistry(), CaptureSetOptions{MaxRequests: 8}); err != nil {139+ t.Fatal(err)140+ }141+ markCachedCaptureSets(&plan, cache)142+ if plan.Cached != 1 || !plan.Requests[0].Cached {143+ t.Fatal("published capture set was not reported as cached")144+ }145+}146+147+func TestCaptureSetDigestIsIndependentOfPlannedRootOrder(t *testing.T) {148+ root := t.TempDir()149+ paths := []string{filepath.Join(root, "first.bin"), filepath.Join(root, "second.bin")}150+ for index, path := range paths {151+ if err := os.WriteFile(path, []byte{byte(index + 1)}, 0o644); err != nil {152+ t.Fatal(err)153+ }154+ }155+ requests := []Request{156+ assetSetRequest("sample-assets", "first", paths[0]),157+ assetSetRequest("sample-assets", "second", paths[1]),158+ }159+ cache := OpenCache(filepath.Join(root, "cache"))160+ first, cached, err := cache.AcquireSet(context.Background(), []Request{requests[1], requests[0]}, defaultAdapterRegistry(), CaptureSetOptions{MaxRequests: 8})161+ if err != nil || cached {162+ t.Fatalf("first ordered set = cached %v, %v", cached, err)163+ }164+ second, cached, err := cache.AcquireSet(context.Background(), requests, defaultAdapterRegistry(), CaptureSetOptions{MaxRequests: 8})165+ if err != nil || !cached {166+ t.Fatalf("reordered set = cached %v, %v", cached, err)167+ }168+ if first.ID != second.ID || first.Digest != second.Digest || first.Roots[0].Acquisition >= first.Roots[1].Acquisition {169+ t.Fatalf("capture set ordering differs: first %+v second %+v", first, second)170+ }171+}172+173+func featureSetRequest(source, adapter, locator string) Request {174+ request := Request{175+ Kind: RequestFeatures, Source: source, Adapter: adapter, Locator: locator,176+ IdentityLocator: locator, MediaType: "application/geo+json", MaxBytes: 1 << 20,177+ }178+ finalizeRequest(&request)179+ return request180+}181+182+func assetSetRequest(source, asset, locator string) Request {183+ request := Request{184+ Kind: RequestAsset, Source: source, Adapter: "asset-file", Asset: asset,185+ Locator: locator, IdentityLocator: locator, MediaType: "application/octet-stream", MaxBytes: 1 << 20,186+ }187+ finalizeRequest(&request)188+ return request189+}190+191+func serverURL(request *http.Request) string {192+ return "http://" + request.Host193+}@@ -28,20 +28,38 @@ type EvidenceSelection struct {
2828 }2929 3030 type SelectedCapture struct {31- Request string `json:"request"`32- Acquisition string `json:"acquisition"`33- Source string `json:"source"`34- Adapter string `json:"adapter"`35- AdapterVersion string `json:"adapterVersion"`36- SHA256 string `json:"sha256"`37- Length int64 `json:"length"`38- MediaType string `json:"mediaType"`39- CapturedAt string `json:"capturedAt"`40- License string `json:"license"`41- Attribution string `json:"attribution"`31+ Request string `json:"request"`32+ Acquisition string `json:"acquisition"`33+ Source string `json:"source"`34+ Adapter string `json:"adapter"`35+ AdapterVersion string `json:"adapterVersion"`36+ SHA256 string `json:"sha256"`37+ Length int64 `json:"length"`38+ MediaType string `json:"mediaType"`39+ CapturedAt string `json:"capturedAt"`40+ License string `json:"license"`41+ Attribution string `json:"attribution"`42+ Set string `json:"set,omitempty"`43+ SetDigest string `json:"setDigest,omitempty"`44+ Termination *CaptureTermination `json:"termination,omitempty"`45+ Pages []SelectedPage `json:"pages,omitempty"`46+}47+48+type SelectedPage struct {49+ Ordinal int `json:"ordinal"`50+ Request string `json:"request"`51+ Locator string `json:"locator"`52+ SHA256 string `json:"sha256"`53+ Length int64 `json:"length"`54+ MediaType string `json:"mediaType"`55+ CapturedAt string `json:"capturedAt"`4256 }4357 4458 func buildReceipt(plan Plan, captures map[string]Capture) ([]byte, string, error) {59+ return buildReceiptSets(plan, captures, nil)60+}61+62+func buildReceiptSets(plan Plan, captures map[string]Capture, sets map[string]CaptureSet) ([]byte, string, error) {4563 selection := EvidenceSelection{4664 Format: BuildReceiptFormat, Project: plan.Project, ProjectDigest: plan.ProjectDigest,4765 }@@ -51,12 +69,32 @@ func buildReceipt(plan Plan, captures map[string]Capture) ([]byte, string, error
5169 if !ok {5270 return nil, "", fmt.Errorf("build selects no capture for request %s", request.ID)5371 }54- selection.Captures = append(selection.Captures, SelectedCapture{72+ selected := SelectedCapture{5573 Request: request.ID, Acquisition: requestCacheKey(request.Request), Source: request.Source, SHA256: capture.Body.SHA256,5674 Adapter: request.Adapter, AdapterVersion: request.AdapterVersion,5775 Length: capture.Body.Length, MediaType: capture.Body.MediaType, CapturedAt: capture.CapturedAt,5876 License: request.License, Attribution: request.Attribution,59- })77+ }78+ if set, ok := sets[request.ID]; ok && set.Digest != "" {79+ root, found := captureRootForRequest(set, request.Request)80+ if !found {81+ return nil, "", fmt.Errorf("capture set selects no root for request %s", request.ID)82+ }83+ termination := root.Termination84+ selected.Set, selected.SetDigest, selected.Termination = set.ID, set.Digest, &termination85+ selected.Pages = make([]SelectedPage, len(root.Pages))86+ for index, page := range root.Pages {87+ selected.Pages[index] = SelectedPage{88+ Ordinal: index, Request: page.RequestHash, Locator: page.Locator,89+ SHA256: page.Body.SHA256, Length: page.Body.Length, MediaType: page.Body.MediaType,90+ CapturedAt: page.CapturedAt,91+ }92+ if page.CapturedAt > createdAt {93+ createdAt = page.CapturedAt94+ }95+ }96+ }97+ selection.Captures = append(selection.Captures, selected)6098 if capture.CapturedAt > createdAt {6199 createdAt = capture.CapturedAt62100 }@@ -135,6 +173,9 @@ func ParseEvidenceSelection(data []byte) (EvidenceSelection, error) {
135173 if _, err := time.Parse(time.RFC3339Nano, capture.CapturedAt); err != nil {136174 return EvidenceSelection{}, fmt.Errorf("invalid replay capture time for request %s", capture.Request)137175 }176+ if err := validateSelectedSet(capture); err != nil {177+ return EvidenceSelection{}, err178+ }138179 if previousRequest > capture.Request || (previousRequest == capture.Request && previousSource > capture.Source) {139180 return EvidenceSelection{}, fmt.Errorf("replay captures are not canonically ordered")140181 }@@ -144,6 +185,130 @@ func ParseEvidenceSelection(data []byte) (EvidenceSelection, error) {
144185 return selection, nil145186 }146187 188+func validateSelectedSet(capture SelectedCapture) error {189+ if capture.Set == "" && capture.SetDigest == "" && capture.Termination == nil && len(capture.Pages) == 0 {190+ return nil191+ }192+ if validateSHA256("capture set", capture.Set) != nil || validateSHA256("capture set digest", capture.SetDigest) != nil || capture.Termination == nil || capture.Termination.Kind == "" || len(capture.Pages) == 0 {193+ return fmt.Errorf("invalid capture set selection for request %s", capture.Request)194+ }195+ seen := make(map[string]bool, len(capture.Pages))196+ for index, page := range capture.Pages {197+ if page.Ordinal != index || validateSHA256("page request", page.Request) != nil || validateSHA256("page body", page.SHA256) != nil || page.Locator == "" || page.Length < 0 || page.MediaType == "" || seen[page.Request] {198+ return fmt.Errorf("invalid selected page for request %s", capture.Request)199+ }200+ if _, err := time.Parse(time.RFC3339Nano, page.CapturedAt); err != nil {201+ return fmt.Errorf("invalid selected page time for request %s", capture.Request)202+ }203+ seen[page.Request] = true204+ }205+ first := capture.Pages[0]206+ if first.Request != capture.Acquisition || first.SHA256 != capture.SHA256 || first.Length != capture.Length || first.MediaType != capture.MediaType || first.CapturedAt != capture.CapturedAt {207+ return fmt.Errorf("capture set primary page differs for request %s", capture.Request)208+ }209+ return nil210+}211+212+func replayPlanEvidence(plan Plan, selection EvidenceSelection, cache *Cache) (buildEvidence, error) {213+ legacy, hasSets := true, false214+ for _, selected := range selection.Captures {215+ if selected.SetDigest != "" {216+ hasSets = true217+ legacy = false218+ }219+ }220+ if !hasSets && legacy {221+ captures, err := replayCaptures(plan, selection, cache)222+ if err != nil {223+ return buildEvidence{}, err224+ }225+ return legacyBuildEvidence(plan, captures), nil226+ }227+ selected := make(map[string]SelectedCapture, len(selection.Captures))228+ for _, capture := range selection.Captures {229+ selected[capture.Request] = capture230+ }231+ if selection.Project != plan.Project || len(selected) != len(plan.Requests) {232+ return buildEvidence{}, fmt.Errorf("replay receipt does not match project %s requests", plan.Project)233+ }234+ evidence := buildEvidence{Captures: make(map[string]Capture), Sets: make(map[string]CaptureSet), Cached: make(map[string]bool)}235+ usedSets := make(map[string]CaptureSet)236+ usedRequests := 0237+ var usedBytes int64238+ for _, group := range planCaptureGroups(plan) {239+ first := selected[group.Requests[0].ID]240+ set, ok := usedSets[first.SetDigest]241+ if !ok {242+ var err error243+ set, err = cache.SelectSet(first.Set, first.SetDigest)244+ if err != nil {245+ return buildEvidence{}, fmt.Errorf("select replay capture set for %s: %w", group.Requests[0].Source, err)246+ }247+ requests := make([]Request, len(group.Requests))248+ for index := range group.Requests {249+ requests[index] = group.Requests[index].Request250+ }251+ if err := validateSetForRequests(set, requests); err != nil {252+ return buildEvidence{}, err253+ }254+ usedSets[first.SetDigest] = set255+ usedBytes += set.TotalBytes256+ for _, root := range set.Roots {257+ usedRequests += len(root.Pages)258+ }259+ }260+ for _, planned := range group.Requests {261+ want, ok := selected[planned.ID]262+ if !ok || want.Set != set.ID || want.SetDigest != set.Digest || want.Source != planned.Source || want.Acquisition != requestCacheKey(planned.Request) || want.Adapter != planned.Adapter || want.AdapterVersion != planned.AdapterVersion || want.License != planned.License || want.Attribution != planned.Attribution {263+ return buildEvidence{}, fmt.Errorf("replay receipt does not select capture set for request %s", planned.ID)264+ }265+ root, ok := captureRootForRequest(set, planned.Request)266+ if !ok || !selectedRootMatches(want, root) {267+ return buildEvidence{}, fmt.Errorf("replay capture set differs for request %s", planned.ID)268+ }269+ evidence.Captures[planned.ID] = root.Pages[0]270+ evidence.Sets[planned.ID] = set271+ evidence.Cached[planned.ID] = true272+ }273+ }274+ if usedRequests > plan.Budgets.Requests || usedBytes > plan.Budgets.TotalBytes {275+ return buildEvidence{}, fmt.Errorf("replay capture sets exceed build budgets")276+ }277+ return evidence, nil278+}279+280+func selectedRootMatches(selected SelectedCapture, root CaptureRoot) bool {281+ if selected.Termination == nil || selected.Termination.Kind != root.Termination.Kind || selected.Termination.Returned != root.Termination.Returned || !equalOptionalInt64(selected.Termination.Matched, root.Termination.Matched) || len(selected.Pages) != len(root.Pages) {282+ return false283+ }284+ for index, page := range root.Pages {285+ want := selected.Pages[index]286+ if want.Ordinal != index || want.Request != page.RequestHash || want.Locator != page.Locator || want.SHA256 != page.Body.SHA256 || want.Length != page.Body.Length || want.MediaType != page.Body.MediaType || want.CapturedAt != page.CapturedAt {287+ return false288+ }289+ }290+ return true291+}292+293+func equalOptionalInt64(left, right *int64) bool {294+ if left == nil || right == nil {295+ return left == nil && right == nil296+ }297+ return *left == *right298+}299+300+func legacyBuildEvidence(plan Plan, captures map[string]Capture) buildEvidence {301+ evidence := buildEvidence{Captures: captures, Sets: make(map[string]CaptureSet, len(captures)), Cached: make(map[string]bool, len(captures))}302+ for _, planned := range plan.Requests {303+ capture := captures[planned.ID]304+ request := planned.Request305+ set := CaptureSet{Format: CaptureSetFormat, ID: captureSetID([]Request{request}), Adapter: request.Adapter, Version: request.AdapterVersion, Kind: request.Kind, CapturedAt: capture.CapturedAt, TotalBytes: capture.Body.Length, Roots: []CaptureRoot{{Acquisition: requestCacheKey(request), Pages: []Capture{capture}, Termination: CaptureTermination{Kind: TerminationSinglePage}}}}306+ evidence.Sets[planned.ID] = set307+ evidence.Cached[planned.ID] = true308+ }309+ return evidence310+}311+147312 func replayCaptures(plan Plan, selection EvidenceSelection, cache *Cache) (map[string]Capture, error) {148313 if selection.Project != plan.Project {149314 return nil, fmt.Errorf("replay receipt belongs to project %s, not %s", selection.Project, plan.Project)@@ -0,0 +1,18 @@
1+package authoring2+3+import "testing"4+5+func TestFeatureValueCoalescesTheFirstPresentScalar(t *testing.T) {6+ t.Parallel()7+ feature := geoJSONFeature{Properties: map[string]any{8+ "NAME": nil,9+ "OID": "road-42",10+ }}11+ value, held := featureValue(feature, "coalesce(properties.NAME, properties.OID)")12+ if !held || scalarString(value) != "road-42" {13+ t.Fatalf("coalesced value = %v, %v", value, held)14+ }15+ if _, held := featureValue(feature, "coalesce(properties.NAME)"); held {16+ t.Fatal("malformed coalesce expression was accepted")17+ }18+}@@ -56,9 +56,13 @@ type CoordinateSpace struct {
5656 Definition string `yaml:"definition"`5757 Extent [4]float64 `yaml:"extent"`5858 SourceZoom int64 `yaml:"source-zoom,omitempty"`59- FirstTile int64 `yaml:"first-tile,omitempty"`60- TileSize int64 `yaml:"tile-size,omitempty"`61- Size int64 `yaml:"size,omitempty"`59+ OriginX int64 `yaml:"origin-x,omitempty"`60+ OriginY int64 `yaml:"origin-y,omitempty"`61+ // FirstTile is accepted only for legacy local projects while their62+ // producers migrate to independent column and row origins.63+ FirstTile int64 `yaml:"first-tile,omitempty"`64+ TileSize int64 `yaml:"tile-size,omitempty"`65+ Size int64 `yaml:"size,omitempty"`6266 }6367 6468 // Source is one configured instance of a reusable feature adapter.@@ -74,13 +78,15 @@ type Source struct {
7478 Mapping Mapping `yaml:"mapping"`7579 }7680 77-// Query carries protocol-neutral selection knobs interpreted only by the78-// selected adapter.81+// Query carries source-selection knobs interpreted only by the selected82+// adapter. Adapter-specific transport details never enter Atlas semantics.7983 type Query struct {80- Layer int `yaml:"layer,omitempty"`81- Collection string `yaml:"collection,omitempty"`82- Where string `yaml:"where,omitempty"`83- Limit int `yaml:"limit,omitempty"`84+ Layer int `yaml:"layer,omitempty"`85+ Collection string `yaml:"collection,omitempty"`86+ Where string `yaml:"where,omitempty"`87+ Limit int `yaml:"limit,omitempty"`88+ ObjectID string `yaml:"object-id,omitempty"`89+ SpatialReference int `yaml:"spatial-reference,omitempty"`8490 }8591 8692 // FeatureSetContract is the source-independent semantic and geometry contract@@ -383,7 +389,7 @@ func validateTarget(target Target) error {
383389 if space.Extent[2] <= space.Extent[0] || space.Extent[3] <= space.Extent[1] {384390 return fmt.Errorf("target coordinate extent is empty")385391 }386- if space.SourceZoom < 0 || space.FirstTile < 0 || space.TileSize < 0 || space.Size < 0 {392+ if space.SourceZoom < 0 || space.OriginX < 0 || space.OriginY < 0 || space.FirstTile < 0 || space.TileSize < 0 || space.Size < 0 {387393 return fmt.Errorf("target coordinate grid metadata is invalid")388394 }389395 return nil@@ -477,6 +483,9 @@ func validateSource(source Source, target CoordinateSpace, sets map[string]Featu
477483 if source.EstimateBytes < 0 {478484 return fmt.Errorf("source %s has a negative size estimate", source.ID)479485 }486+ if source.Query.Layer < 0 || source.Query.Limit < 0 || source.Query.SpatialReference < 0 {487+ return fmt.Errorf("source %s has invalid query settings", source.ID)488+ }480489 mapping := source.Mapping481490 set, exists := sets[mapping.FeatureSet]482491 if !exists {@@ -20,6 +20,8 @@ target:
2020 unit: pixel2121 definition: atlas:tile-plane2222 extent: [0, 0, 256, 256]23+ origin-x: 224+ origin-y: 32325 tile-size: 2562426 size: 2562527 feature-sets:@@ -80,7 +82,8 @@ func TestLoadProject(t *testing.T) {
8082 if err != nil {8183 t.Fatal(err)8284 }83- if project.ID != "sample-region" || project.Target.World != "sample-region" || len(project.Sources) != 1 {85+ if project.ID != "sample-region" || project.Target.World != "sample-region" || len(project.Sources) != 1 ||86+ project.Target.CoordinateSpace.OriginX != 2 || project.Target.CoordinateSpace.OriginY != 3 {8487 t.Fatalf("unexpected project: %+v", project)8588 }8689 want := filepath.Join(filepath.Dir(path), "sample-region.geojson")@@ -0,0 +1,151 @@
1+package authoring2+3+import (4+ "bytes"5+ "context"6+ "fmt"7+ "net/http"8+ "net/http/httptest"9+ "os"10+ "path/filepath"11+ "strings"12+ "sync/atomic"13+ "testing"14+)15+16+func TestMultiPageBuildIsByteIdenticalOnlineOfflineAndExactReplay(t *testing.T) {17+ var calls atomic.Int6418+ server := httptest.NewServer(http.HandlerFunc(func(rw http.ResponseWriter, request *http.Request) {19+ calls.Add(1)20+ rw.Header().Set("Content-Type", "application/geo+json")21+ if request.URL.Query().Get("page") == "2" {22+ _, _ = rw.Write([]byte(samplePointPage("marker-2", "Sample Lookout", 7.5, 3.5, "", 2)))23+ return24+ }25+ next := "http://" + request.Host + request.URL.Path + "?page=2"26+ _, _ = rw.Write([]byte(samplePointPage("marker-1", "Sample Marker", 4, 6, next, 2)))27+ }))28+29+ projectDir := t.TempDir()30+ copySampleProjectFiles(t, projectDir)31+ projectPath := filepath.Join(projectDir, "sample-region"+ProjectExtension)32+ manifest, err := os.ReadFile(projectPath)33+ if err != nil {34+ t.Fatal(err)35+ }36+ updated := strings.Replace(string(manifest), "adapter: geojson", "adapter: ogc-api-features", 1)37+ updated = strings.Replace(updated, "locator: sample-region.geojson", "locator: "+server.URL, 1)38+ updated = strings.Replace(updated, " mapping:\n feature-set: places", " query:\n collection: sample-points\n limit: 1\n mapping:\n feature-set: places", 1)39+ if err := os.WriteFile(projectPath, []byte(updated), 0o644); err != nil {40+ t.Fatal(err)41+ }42+43+ cacheDir := filepath.Join(t.TempDir(), "cache")44+ var onlineEvents []Event45+ online, err := Build(context.Background(), BuildOptions{ProjectPath: projectPath, CacheDir: cacheDir, LibraryDir: filepath.Join(t.TempDir(), "online"), Event: func(event Event) { onlineEvents = append(onlineEvents, event) }})46+ if err != nil {47+ t.Fatal(err)48+ }49+ selection, err := loadEvidenceSelection(online.Path)50+ if err != nil {51+ t.Fatal(err)52+ }53+ selected, ok := selectedSource(selection, "sample-places")54+ if !ok || len(selected.Pages) != 2 || selected.Termination == nil || selected.Termination.Kind != TerminationNextExhausted {55+ t.Fatalf("selected multi-page evidence = %+v", selected)56+ }57+ if !allCaptureEventsCached(onlineEvents, false) {58+ t.Fatalf("first online build cache events = %+v", onlineEvents)59+ }60+ tampered := selection61+ for index := range tampered.Captures {62+ tampered.Captures[index].Pages = append([]SelectedPage(nil), tampered.Captures[index].Pages...)63+ if tampered.Captures[index].Source == "sample-places" {64+ tampered.Captures[index].Pages[1].Locator += "&changed=true"65+ }66+ }67+ if _, err := replayPlanEvidence(online.Plan, tampered, OpenCache(cacheDir)); err == nil || !strings.Contains(err.Error(), "differs") {68+ t.Fatalf("tampered ordered page selection = %v", err)69+ }70+ var repeatedEvents []Event71+ repeated, err := Build(context.Background(), BuildOptions{ProjectPath: projectPath, CacheDir: cacheDir, LibraryDir: filepath.Join(t.TempDir(), "repeated"), Event: func(event Event) { repeatedEvents = append(repeatedEvents, event) }})72+ if err != nil {73+ t.Fatal(err)74+ }75+ if !allCaptureEventsCached(repeatedEvents, true) {76+ t.Fatalf("repeated online build cache events = %+v", repeatedEvents)77+ }78+ server.Close()79+80+ offline, err := Build(context.Background(), BuildOptions{ProjectPath: projectPath, CacheDir: cacheDir, LibraryDir: filepath.Join(t.TempDir(), "offline"), Offline: true})81+ if err != nil {82+ t.Fatal(err)83+ }84+ replayed, err := Build(context.Background(), BuildOptions{ProjectPath: projectPath, CacheDir: cacheDir, LibraryDir: filepath.Join(t.TempDir(), "replay"), ReplayPath: online.Path})85+ if err != nil {86+ t.Fatal(err)87+ }88+ if calls.Load() != 4 {89+ t.Fatalf("source calls = %d, want only the two pages from each online build", calls.Load())90+ }91+ want, err := os.ReadFile(online.Path)92+ if err != nil {93+ t.Fatal(err)94+ }95+ for _, path := range []string{repeated.Path, offline.Path, replayed.Path} {96+ got, err := os.ReadFile(path)97+ if err != nil {98+ t.Fatal(err)99+ }100+ if !bytes.Equal(got, want) {101+ t.Fatalf("build %s is not byte-identical to online build", path)102+ }103+ }104+}105+106+func allCaptureEventsCached(events []Event, want bool) bool {107+ found := false108+ for _, event := range events {109+ if event.Stage != "capture" {110+ continue111+ }112+ found = true113+ if event.Cached != want {114+ return false115+ }116+ }117+ return found118+}119+120+func samplePointPage(id, title string, x, y float64, next string, matched int) string {121+ links := ""122+ if next != "" {123+ links = fmt.Sprintf(`,"links":[{"rel":"next","href":%q}]`, next)124+ }125+ return fmt.Sprintf(`{"type":"FeatureCollection","numberMatched":%d,"numberReturned":1,"features":[{"type":"Feature","id":%q,"properties":{"name":%q,"kind":"landmark","featured":true,"rank":1,"score":9.5,"badge":"AQID","category":"sample.category.marker","area":"area-1"},"geometry":{"type":"Point","coordinates":[%g,%g]}}]%s}`, matched, id, title, x, y, links)126+}127+128+func copySampleProjectFiles(t *testing.T, destination string) {129+ t.Helper()130+ for _, name := range []string{131+ "sample-region.atlas-project", "sample-region.geojson", "sample-region-routes.geojson",132+ "sample-region-areas.geojson", "sample-region-raster.ppm", "sample-region-symbol.svg",133+ } {134+ data, err := os.ReadFile(filepath.Join("..", "..", "examples", name))135+ if err != nil {136+ t.Fatal(err)137+ }138+ if err := os.WriteFile(filepath.Join(destination, name), data, 0o644); err != nil {139+ t.Fatal(err)140+ }141+ }142+}143+144+func selectedSource(selection EvidenceSelection, source string) (SelectedCapture, bool) {145+ for _, selected := range selection.Captures {146+ if selected.Source == source {147+ return selected, true148+ }149+ }150+ return SelectedCapture{}, false151+}@@ -59,6 +59,14 @@ type geoJSONGeometry struct {
5959 }6060 6161 func observe(project Project, sourceOrder int, source Source, capture Capture, body []byte) ([]observation, error) {62+ return observePage(project, sourceOrder, source, capture, body, true)63+}64+65+func observeCapturedPage(project Project, sourceOrder int, source Source, capture Capture, body []byte) ([]observation, error) {66+ return observePage(project, sourceOrder, source, capture, body, false)67+}68+69+func observePage(project Project, sourceOrder int, source Source, capture Capture, body []byte, enforceCompleteness bool) ([]observation, error) {6270 set, exists := featureSetContract(project.FeatureSets, source.Mapping.FeatureSet)6371 if !exists {6472 return nil, fmt.Errorf("source %s targets unknown feature set %s", source.ID, source.Mapping.FeatureSet)@@ -72,8 +80,10 @@ func observe(project Project, sourceOrder int, source Source, capture Capture, b
7280 if collection.Type != "FeatureCollection" {7381 return nil, fmt.Errorf("source %s is %q, want FeatureCollection", source.ID, collection.Type)7482 }75- if err := validateAdapterCompleteness(source, collection); err != nil {76- return nil, err83+ if enforceCompleteness {84+ if err := validateAdapterCompleteness(source, collection); err != nil {85+ return nil, err86+ }7787 }7888 out := make([]observation, 0, len(collection.Features))7989 for index, feature := range collection.Features {@@ -167,6 +177,20 @@ func validateAdapterCompleteness(source Source, collection geoJSON) error {
167177 }168178 169179 func featureValue(feature geoJSONFeature, path string) (any, bool) {180+ path = strings.TrimSpace(path)181+ if strings.HasPrefix(path, "coalesce(") && strings.HasSuffix(path, ")") {182+ arguments := strings.Split(strings.TrimSuffix(strings.TrimPrefix(path, "coalesce("), ")"), ",")183+ if len(arguments) < 2 {184+ return nil, false185+ }186+ for _, argument := range arguments {187+ value, held := featureValue(feature, strings.TrimSpace(argument))188+ if held && scalarString(value) != "" {189+ return value, true190+ }191+ }192+ return nil, false193+ }170194 switch path {171195 case "id":172196 return feature.ID, feature.ID != nil@@ -6,8 +6,6 @@ import (
66 "encoding/json"77 "fmt"88 "math"9- "net/url"10- "path/filepath"119 "sort"1210 "strconv"1311 "strings"@@ -147,13 +145,36 @@ func PlanProject(project Project, cache *Cache, output string) (Plan, error) {
147145 plan.Licenses = append(plan.Licenses, Obligation{Source: asset.ID, License: asset.License, Attribution: asset.Attribution})148146 }149147 sort.Slice(plan.Requests, func(i, j int) bool { return plan.Requests[i].ID < plan.Requests[j].ID })150- for _, request := range plan.Requests {151- if request.Cached {148+ markCachedCaptureSets(&plan, cache)149+ sort.Slice(plan.Licenses, func(i, j int) bool { return plan.Licenses[i].Source < plan.Licenses[j].Source })150+ return plan, nil151+}152+153+func markCachedCaptureSets(plan *Plan, cache *Cache) {154+ plan.Cached = 0155+ requestIndexes := make(map[string]int, len(plan.Requests))156+ for index := range plan.Requests {157+ plan.Requests[index].Cached = false158+ requestIndexes[plan.Requests[index].ID] = index159+ }160+ if cache == nil {161+ return162+ }163+ for _, group := range planCaptureGroups(*plan) {164+ requests := make([]Request, len(group.Requests))165+ for index := range group.Requests {166+ requests[index] = group.Requests[index].Request167+ }168+ set, err := cache.LatestSet(captureSetID(requests))169+ if err != nil || validateSetForRequests(set, requests) != nil {170+ continue171+ }172+ for _, grouped := range group.Requests {173+ index := requestIndexes[grouped.ID]174+ plan.Requests[index].Cached = true152175 plan.Cached++153176 }154177 }155- sort.Slice(plan.Licenses, func(i, j int) bool { return plan.Licenses[i].Source < plan.Licenses[j].Source })156- return plan, nil157178 }158179 159180 func assetRequest(project Project, asset Asset) Request {@@ -168,65 +189,11 @@ func assetRequest(project Project, asset Asset) Request {
168189 }169190 170191 func featureRequest(project Project, source Source) (Request, error) {171- identity := source.Locator172- locator := project.ResolveLocator(identity)173- switch source.Adapter {174- case "geojson":175- case "arcgis-feature-service":176- parsed, err := url.Parse(locator)177- if err != nil {178- return Request{}, fmt.Errorf("source %s locator: %w", source.ID, err)179- }180- path := strings.TrimSuffix(parsed.Path, "/")181- if source.Query.Layer > 0 && filepath.Base(path) != strconv.Itoa(source.Query.Layer) {182- path += "/" + strconv.Itoa(source.Query.Layer)183- }184- if !strings.HasSuffix(path, "/query") {185- path += "/query"186- }187- parsed.Path = path188- query := parsed.Query()189- query.Set("f", "geojson")190- query.Set("outFields", "*")191- query.Set("returnGeometry", "true")192- where := source.Query.Where193- if where == "" {194- where = "1=1"195- }196- query.Set("where", where)197- parsed.RawQuery = query.Encode()198- locator, identity = parsed.String(), parsed.String()199- case "ogc-api-features":200- if source.Query.Collection == "" {201- return Request{}, fmt.Errorf("source %s requires query.collection", source.ID)202- }203- parsed, err := url.Parse(locator)204- if err != nil {205- return Request{}, fmt.Errorf("source %s locator: %w", source.ID, err)206- }207- parsed.Path = strings.TrimSuffix(parsed.Path, "/") + "/collections/" + url.PathEscape(source.Query.Collection) + "/items"208- query := parsed.Query()209- extent, err := sourceExtent(source.Mapping.Geometry, project.Target.CoordinateSpace)210- if err != nil {211- return Request{}, fmt.Errorf("source %s query extent: %w", source.ID, err)212- }213- query.Set("bbox", extentString(extent))214- query.Set("f", "json")215- if source.Query.Limit > 0 {216- query.Set("limit", strconv.Itoa(source.Query.Limit))217- }218- parsed.RawQuery = query.Encode()219- locator, identity = parsed.String(), parsed.String()220- default:221- return Request{}, fmt.Errorf("source %s uses unknown adapter %q", source.ID, source.Adapter)222- }223- request := Request{224- Kind: RequestFeatures, Source: source.ID, Adapter: source.Adapter,225- Locator: locator, IdentityLocator: identity, MediaType: source.MediaType,226- License: source.License, Attribution: source.Attribution,192+ adapter, err := defaultAdapterRegistry().Lookup(source.Adapter)193+ if err != nil {194+ return Request{}, fmt.Errorf("source %s uses %w", source.ID, err)227195 }228- finalizeRequest(&request)229- return request, nil196+ return adapter.PlanFeature(project, source)230197 }231198 232199 func rasterRequests(project Project, raster Raster, budgets BuildBudgets) ([]Request, int64, int64, error) {@@ -333,24 +300,11 @@ func finalizeRequest(request *Request) {
333300 }334301 335302 func adapterVersion(adapter string) string {336- switch adapter {337- case "geojson":338- return "geojson/v1"339- case "arcgis-feature-service":340- return "arcgis-feature-service/v1"341- case "ogc-api-features":342- return "ogc-api-features/v1"343- case "raster-file":344- return "raster-file/v1"345- case "xyz":346- return "xyz/v1"347- case "wmts":348- return "wmts-rest/v1"349- case "asset-file":350- return "asset-file/v1"351- default:303+ registered, err := defaultAdapterRegistry().Lookup(adapter)304+ if err != nil {352305 return adapter + "/unknown"353306 }307+ return registered.Version()354308 }355309 356310 func requestCacheKey(request Request) string {@@ -11,25 +11,172 @@ func transformGeometry(geometry vnext.Geometry, mapping GeometryMap, target Coor
1111 if familyOf(geometry.Kind) != mapping.Family {1212 return vnext.Geometry{}, fmt.Errorf("geometry is %s, feature set requires %s", familyOf(geometry.Kind), mapping.Family)1313 }14- out := vnext.Geometry{Kind: geometry.Kind, Parts: make([]vnext.GeometryPart, len(geometry.Parts))}14+ transformed := vnext.Geometry{Kind: geometry.Kind, Parts: make([]vnext.GeometryPart, len(geometry.Parts))}1515 for partIndex, part := range geometry.Parts {16- out.Parts[partIndex].Rings = make([][]vnext.Position, len(part.Rings))16+ transformed.Parts[partIndex].Rings = make([][]vnext.Position, len(part.Rings))1717 for ringIndex, ring := range part.Rings {18+ if geometry.Kind == vnext.GeometryPolygon && (len(ring) < 4 || ring[0] != ring[len(ring)-1]) {19+ return vnext.Geometry{}, fmt.Errorf("polygon ring is not closed")20+ }1821 positions := make([]vnext.Position, len(ring))1922 for positionIndex, position := range ring {20- transformed := transformPosition(position, mapping.Transform)21- if !inside(transformed, target.Extent) {22- return vnext.Geometry{}, fmt.Errorf("transformed position %v falls outside target extent", transformed)23+ positions[positionIndex] = transformPosition(position, mapping.Transform)24+ }25+ transformed.Parts[partIndex].Rings[ringIndex] = positions26+ }27+ }28+ clipped := clipGeometry(transformed, target.Extent)29+ if len(clipped.Parts) == 0 {30+ return vnext.Geometry{}, fmt.Errorf("transformed geometry falls outside target extent")31+ }32+ return clipped, nil33+}34+35+func clipGeometry(geometry vnext.Geometry, extent [4]float64) vnext.Geometry {36+ out := vnext.Geometry{Kind: geometry.Kind}37+ switch geometry.Kind {38+ case vnext.GeometryPoint:39+ for _, part := range geometry.Parts {40+ if len(part.Rings) == 1 && len(part.Rings[0]) == 1 && inside(part.Rings[0][0], extent) {41+ out.Parts = append(out.Parts, part)42+ }43+ }44+ case vnext.GeometryLineString:45+ for _, part := range geometry.Parts {46+ if len(part.Rings) != 1 {47+ continue48+ }49+ for _, line := range clipLine(part.Rings[0], extent) {50+ out.Parts = append(out.Parts, vnext.GeometryPart{Rings: [][]vnext.Position{line}})51+ }52+ }53+ case vnext.GeometryPolygon:54+ for _, part := range geometry.Parts {55+ if len(part.Rings) == 0 {56+ continue57+ }58+ exterior := clipRing(part.Rings[0], extent)59+ if len(exterior) == 0 {60+ continue61+ }62+ rings := [][]vnext.Position{exterior}63+ for _, hole := range part.Rings[1:] {64+ if clipped := clipRing(hole, extent); len(clipped) > 0 {65+ rings = append(rings, clipped)2366 }24- positions[positionIndex] = transformed2567 }26- if geometry.Kind == vnext.GeometryPolygon && (len(positions) < 4 || positions[0] != positions[len(positions)-1]) {27- return vnext.Geometry{}, fmt.Errorf("polygon ring is not closed")68+ out.Parts = append(out.Parts, vnext.GeometryPart{Rings: rings})69+ }70+ }71+ return out72+}73+74+func clipLine(line []vnext.Position, extent [4]float64) [][]vnext.Position {75+ var out [][]vnext.Position76+ var current []vnext.Position77+ flush := func() {78+ if len(current) >= 2 {79+ out = append(out, current)80+ }81+ current = nil82+ }83+ for index := 1; index < len(line); index++ {84+ start, end, held := clipSegment(line[index-1], line[index], extent)85+ if !held {86+ flush()87+ continue88+ }89+ if len(current) == 0 || current[len(current)-1] != start {90+ flush()91+ current = []vnext.Position{start}92+ }93+ if current[len(current)-1] != end {94+ current = append(current, end)95+ }96+ }97+ flush()98+ return out99+}100+101+func clipRing(ring []vnext.Position, extent [4]float64) []vnext.Position {102+ if len(ring) < 4 || ring[0] != ring[len(ring)-1] {103+ return nil104+ }105+ current := append([]vnext.Position(nil), ring[:len(ring)-1]...)106+ insideEdge := [4]func(vnext.Position) bool{107+ func(point vnext.Position) bool { return point[0] >= extent[0] },108+ func(point vnext.Position) bool { return point[0] <= extent[2] },109+ func(point vnext.Position) bool { return point[1] >= extent[1] },110+ func(point vnext.Position) bool { return point[1] <= extent[3] },111+ }112+ crossing := [4]func(vnext.Position, vnext.Position) vnext.Position{113+ func(from, to vnext.Position) vnext.Position { return crossX(from, to, extent[0]) },114+ func(from, to vnext.Position) vnext.Position { return crossX(from, to, extent[2]) },115+ func(from, to vnext.Position) vnext.Position { return crossY(from, to, extent[1]) },116+ func(from, to vnext.Position) vnext.Position { return crossY(from, to, extent[3]) },117+ }118+ for edge := range insideEdge {119+ if len(current) == 0 {120+ return nil121+ }122+ next := make([]vnext.Position, 0, len(current)+4)123+ for index, to := range current {124+ from := current[(index+len(current)-1)%len(current)]125+ fromInside, toInside := insideEdge[edge](from), insideEdge[edge](to)126+ switch {127+ case fromInside && toInside:128+ next = append(next, to)129+ case fromInside:130+ next = append(next, crossing[edge](from, to))131+ case toInside:132+ next = append(next, crossing[edge](from, to), to)133+ }134+ }135+ current = next136+ }137+ if len(current) < 3 {138+ return nil139+ }140+ return append(current, current[0])141+}142+143+func clipSegment(from, to vnext.Position, extent [4]float64) (vnext.Position, vnext.Position, bool) {144+ enter, leave := 0.0, 1.0145+ dx, dy := to[0]-from[0], to[1]-from[1]146+ narrow := func(p, q float64) bool {147+ if p == 0 {148+ return q >= 0149+ }150+ at := q / p151+ if p < 0 {152+ if at > leave {153+ return false28154 }29- out.Parts[partIndex].Rings[ringIndex] = positions155+ enter = math.Max(enter, at)156+ return true30157 }158+ if at < enter {159+ return false160+ }161+ leave = math.Min(leave, at)162+ return true163+ }164+ if !narrow(-dx, from[0]-extent[0]) || !narrow(dx, extent[2]-from[0]) ||165+ !narrow(-dy, from[1]-extent[1]) || !narrow(dy, extent[3]-from[1]) {166+ return from, to, false31167 }32- return out, nil168+ return vnext.Position{from[0] + enter*dx, from[1] + enter*dy},169+ vnext.Position{from[0] + leave*dx, from[1] + leave*dy}, true170+}171+172+func crossX(from, to vnext.Position, x float64) vnext.Position {173+ at := (x - from[0]) / (to[0] - from[0])174+ return vnext.Position{x, from[1] + at*(to[1]-from[1])}175+}176+177+func crossY(from, to vnext.Position, y float64) vnext.Position {178+ at := (y - from[1]) / (to[1] - from[1])179+ return vnext.Position{from[0] + at*(to[0]-from[0]), y}33180 }34181 35182 func sourceExtent(mapping GeometryMap, target CoordinateSpace) ([4]float64, error) {@@ -58,3 +58,37 @@ func TestTransformGeometryRefusesFamilyExtentAndRingViolations(t *testing.T) {
5858 })5959 }6060 }61+62+func TestTransformGeometryClipsCrossingPathsAndAreasToTheTarget(t *testing.T) {63+ t.Parallel()64+ target := CoordinateSpace{Extent: [4]float64{0, 0, 10, 10}}65+ identity := CoordinateTransform{Kind: "identity"}66+67+ path := vnext.Geometry{Kind: vnext.GeometryLineString, Parts: []vnext.GeometryPart{{Rings: [][]vnext.Position{{68+ {-5, 5}, {5, 5}, {15, 5},69+ }}}}}70+ clippedPath, err := transformGeometry(path, GeometryMap{Family: "path", Transform: identity}, target)71+ if err != nil {72+ t.Fatal(err)73+ }74+ if got := clippedPath.Parts[0].Rings[0]; len(got) != 3 || got[0] != (vnext.Position{0, 5}) || got[2] != (vnext.Position{10, 5}) {75+ t.Fatalf("clipped path = %v", got)76+ }77+78+ area := vnext.Geometry{Kind: vnext.GeometryPolygon, Parts: []vnext.GeometryPart{{Rings: [][]vnext.Position{{79+ {-5, -5}, {15, -5}, {15, 15}, {-5, 15}, {-5, -5},80+ }}}}}81+ clippedArea, err := transformGeometry(area, GeometryMap{Family: "area", Transform: identity}, target)82+ if err != nil {83+ t.Fatal(err)84+ }85+ ring := clippedArea.Parts[0].Rings[0]86+ if len(ring) != 5 || ring[0] != ring[len(ring)-1] {87+ t.Fatalf("clipped area ring = %v", ring)88+ }89+ for _, position := range ring {90+ if !inside(position, target.Extent) {91+ t.Fatalf("clipped position %v leaves target", position)92+ }93+ }94+}