1// The national fetch path. Hub datasets arrive through a download API that2// stages a whole file server-side; a national layer on a plain ArcGIS3// MapServer offers no such staging, so it is paged through the query4// endpoint instead -- the city's padded window as the envelope, true degrees5// asked for by name, pages ordered by the layer's own row identity until6// one comes back short. Fetched geometry is cut to the window at capture7// time: a watershed intersecting the window may extend a hundred windows8// past it, and the trimmed shape is the honest city-sized answer.9package main1011import (12 "context"13 "encoding/json"14 "fmt"15 "net/url"16 "strconv"17 "strings"1819 "github.com/FelineStateMachine/atlas/internal/arcgismap"20)2122// mapServerPage is the most rows one query asks for, matching the services'23// own maxRecordCount.24const mapServerPage = 20002526// fetchMapServerDataset takes one national layer through the query endpoint,27// page by page, and answers features already cut to the window.28func fetchMapServerDataset(29 ctx context.Context,30 fetcher *fetcher,31 window arcgismap.Window,32 dataset arcgismap.Dataset,33) ([]arcgismap.Feature, error) {34 var features []arcgismap.Feature35 claimed := map[int64]bool{}36 for offset := 0; ; offset += mapServerPage {37 address := mapServerQueryURL(dataset, window, offset)38 body, _, err := fetcher.get(ctx, address)39 if err != nil {40 return nil, err41 }42 page, more, err := decodeMapServerPage(dataset, window, body, address)43 if err != nil {44 return nil, err45 }46 for _, feature := range page {47 // A row captured twice means the pages shifted under the48 // crawl; a capture must not hold it two ways, and Normalize's49 // sort would bury the evidence.50 if claimed[feature.ID] {51 return nil, fmt.Errorf("feature %d arrived twice; pages shifted mid-crawl: %s",52 feature.ID, address)53 }54 claimed[feature.ID] = true55 }56 features = append(features, page...)57 if !more {58 return features, nil59 }60 }61}6263// mapServerQueryURL spells one page's request: the window as the envelope,64// degrees in and degrees out, fields limited to the curation, rows ordered65// by the layer's row identity so the pages tile the layer exactly once.66func mapServerQueryURL(dataset arcgismap.Dataset, window arcgismap.Window, offset int) string {67 where := dataset.Where68 if where == "" {69 where = "1=1"70 }71 envelope := strings.Join([]string{72 strconv.FormatFloat(window.West, 'f', -1, 64),73 strconv.FormatFloat(window.South, 'f', -1, 64),74 strconv.FormatFloat(window.East, 'f', -1, 64),75 strconv.FormatFloat(window.North, 'f', -1, 64),76 }, ",")77 values := url.Values{78 "f": {"geojson"},79 "where": {where},80 "geometry": {envelope},81 "geometryType": {"esriGeometryEnvelope"},82 "inSR": {"4326"},83 "spatialRel": {"esriSpatialRelIntersects"},84 "outSR": {"4326"},85 "outFields": {strings.Join(dataset.Keep, ",")},86 "geometryPrecision": {"7"},87 "orderByFields": {dataset.Keep[0]},88 "resultOffset": {strconv.Itoa(offset)},89 "resultRecordCount": {strconv.Itoa(mapServerPage)},90 }91 return fmt.Sprintf("%s/%d/query?%s", dataset.Server, dataset.Layer, values.Encode())92}9394// decodeMapServerPage reads one page into capture features, cut to the95// window, and says whether another page waits. ArcGIS answers errors as96// HTTP 200 with an error member, and spells the truncation flag at the97// collection's top level or under its properties depending on the vintage.98func decodeMapServerPage(99 dataset arcgismap.Dataset,100 window arcgismap.Window,101 body []byte,102 address string,103) ([]arcgismap.Feature, bool, error) {104 var page struct {105 Error *struct {106 Code int `json:"code"`107 Message string `json:"message"`108 } `json:"error"`109 Type string `json:"type"`110 ExceededTransferLimit bool `json:"exceededTransferLimit"`111 Properties struct {112 ExceededTransferLimit bool `json:"exceededTransferLimit"`113 } `json:"properties"`114 Features []struct {115 Properties map[string]any `json:"properties"`116 Geometry struct {117 Type string `json:"type"`118 Coordinates json.RawMessage `json:"coordinates"`119 } `json:"geometry"`120 } `json:"features"`121 }122 if err := json.Unmarshal(body, &page); err != nil {123 return nil, false, fmt.Errorf("decode %s: %w", address, err)124 }125 if page.Error != nil {126 return nil, false, fmt.Errorf("%s answered error %d: %s",127 address, page.Error.Code, page.Error.Message)128 }129 if page.Type != "FeatureCollection" {130 return nil, false, fmt.Errorf("%s is a %q, not a FeatureCollection", address, page.Type)131 }132133 features := make([]arcgismap.Feature, 0, len(page.Features))134 for at, raw := range page.Features {135 fields := arcgismap.Fields{}136 for _, keep := range dataset.Keep {137 if value := arcgismap.FieldString(raw.Properties[keep]); value != "" {138 fields[keep] = value139 }140 }141 id, err := mapServerFeatureID(dataset, fields, raw.Properties)142 if err != nil {143 return nil, false, fmt.Errorf("feature %d of %s: %w", at, address, err)144 }145 geometry, err := arcgisGeometry(dataset.Geometry, raw.Geometry.Type, raw.Geometry.Coordinates)146 if err != nil {147 return nil, false, fmt.Errorf("feature %d: %w", id, err)148 }149 if geometry = clipToWindow(window, geometry); geometry == nil {150 continue151 }152 features = append(features, arcgismap.Feature{ID: id, Fields: fields, Geometry: *geometry})153 }154 more := page.ExceededTransferLimit || page.Properties.ExceededTransferLimit ||155 len(page.Features) == mapServerPage156 return features, more, nil157}158159// mapServerFeatureID is the capture's identity for one national row: the160// curated derivation where the table declares one, the layer's own object161// id otherwise.162func mapServerFeatureID(dataset arcgismap.Dataset, fields arcgismap.Fields, properties map[string]any) (int64, error) {163 if dataset.IDOf == nil {164 return arcgisObjectID(properties)165 }166 id, ok := dataset.IDOf(fields)167 if !ok {168 return 0, fmt.Errorf("no identity in %v", fields)169 }170 return id, nil171}172173// clipToWindow cuts a normalized geometry to the window, answering nil when174// nothing of it remains. Points pass untouched: no national layer captures175// them yet, and a stray point is the stray tolerance's business, not this.176func clipToWindow(window arcgismap.Window, geometry *arcgismap.Geometry) *arcgismap.Geometry {177 if geometry == nil {178 return nil179 }180 switch geometry.Type {181 case arcgismap.GeometryRings:182 geometry.Rings = arcgismap.ClipRings(window, geometry.Rings)183 if len(geometry.Rings) == 0 {184 return nil185 }186 case arcgismap.GeometryLines:187 geometry.Lines = arcgismap.ClipLines(window, geometry.Lines)188 if len(geometry.Lines) == 0 {189 return nil190 }191 }192 return geometry193}