A hub dataset arrives through a download API that stages a whole file; a national layer on a plain MapServer offers no staging, so the crawl pages it through the query endpoint instead -- the city's padded window as the envelope, true degrees asked for by name, pages ordered by the layer's row identity until one comes back short. ArcGIS answers errors as HTTP 200 with an error member and spells its truncation flag two ways by vintage; both are read. A row arriving twice means the pages shifted under the crawl, and the capture refuses it rather than letting Normalize's sort bury the evidence. Fetched geometry is cut to the window before it enters the capture, and the basemap gathers its features from the full curation, so the nation's streams and waterbodies draw under the city's own streets.
- commit
- 80c32362534153e4e5092a3163922dc39bf2cfc7
- parent
- e3e2eaff782f4873be4f466b9fe4528b54643c5b
- author
- Dami <dami.etoile@gmail.com>
- kind
- feat risk score 1.8 of 3, Jev's estimate
| M | tools/ | 20 | ++- |
| A | tools/ | 193 | ++++++++++++++++++++++++++++++ |
| A | tools/ | 120 | +++++++++++++++++++ |
@@ -73,10 +73,17 @@ func runArcgis(ctx context.Context, fetcher *fetcher, o options) error {
7373 Basemap: arcgismap.MapConfig{MaxZoom: maxZoom, Extension: "png"},7474 }7575 76- fmt.Printf("\n== ArcGIS Hub %s / %s (%d datasets)\n", city.Title, date, len(city.Datasets))76+ datasets := city.AllDatasets()77+ fmt.Printf("\n== ArcGIS Hub %s / %s (%d datasets)\n", city.Title, date, len(datasets))7778 pinCount := 078- for _, dataset := range city.Datasets {79- features, err := fetchArcgisDataset(ctx, fetcher, city, dataset)79+ for _, dataset := range datasets {80+ var features []arcgismap.Feature81+ var err error82+ if dataset.Server != "" {83+ features, err = fetchMapServerDataset(ctx, fetcher, window, dataset)84+ } else {85+ features, err = fetchArcgisDataset(ctx, fetcher, city, dataset)86+ }8087 if err != nil {8188 return fmt.Errorf("%s: %w", dataset.Slug, err)8289 }@@ -514,9 +521,10 @@ func renderArcgisBasemap(
514521 // capture's own window, so the raster and the pins can never disagree about515522 // where the ground is.516523 func basemapFeatures(city arcgismap.City, capture *arcgismap.Capture) []basemap.Feature {517- curated := make(map[string]*arcgismap.Dataset, len(city.Datasets))518- for at := range city.Datasets {519- curated[city.Datasets[at].Slug] = &city.Datasets[at]524+ datasets := city.AllDatasets()525+ curated := make(map[string]*arcgismap.Dataset, len(datasets))526+ for at := range datasets {527+ curated[datasets[at].Slug] = &datasets[at]520528 }521529 var features []basemap.Feature522530 for _, dataset := range capture.Datasets {@@ -0,0 +1,193 @@
1+// The national fetch path. Hub datasets arrive through a download API that2+// stages a whole file server-side; a national layer on a plain ArcGIS3+// MapServer offers no such staging, so it is paged through the query4+// endpoint instead -- the city's padded window as the envelope, true degrees5+// asked for by name, pages ordered by the layer's own row identity until6+// one comes back short. Fetched geometry is cut to the window at capture7+// time: a watershed intersecting the window may extend a hundred windows8+// past it, and the trimmed shape is the honest city-sized answer.9+package main10+11+import (12+ "context"13+ "encoding/json"14+ "fmt"15+ "net/url"16+ "strconv"17+ "strings"18+19+ "github.com/FelineStateMachine/atlas/internal/arcgismap"20+)21+22+// mapServerPage is the most rows one query asks for, matching the services'23+// own maxRecordCount.24+const mapServerPage = 200025+26+// fetchMapServerDataset takes one national layer through the query endpoint,27+// page by page, and answers features already cut to the window.28+func fetchMapServerDataset(29+ ctx context.Context,30+ fetcher *fetcher,31+ window arcgismap.Window,32+ dataset arcgismap.Dataset,33+) ([]arcgismap.Feature, error) {34+ var features []arcgismap.Feature35+ claimed := map[int64]bool{}36+ for offset := 0; ; offset += mapServerPage {37+ address := mapServerQueryURL(dataset, window, offset)38+ body, _, err := fetcher.get(ctx, address)39+ if err != nil {40+ return nil, err41+ }42+ page, more, err := decodeMapServerPage(dataset, window, body, address)43+ if err != nil {44+ return nil, err45+ }46+ for _, feature := range page {47+ // A row captured twice means the pages shifted under the48+ // crawl; a capture must not hold it two ways, and Normalize's49+ // sort would bury the evidence.50+ if claimed[feature.ID] {51+ return nil, fmt.Errorf("feature %d arrived twice; pages shifted mid-crawl: %s",52+ feature.ID, address)53+ }54+ claimed[feature.ID] = true55+ }56+ features = append(features, page...)57+ if !more {58+ return features, nil59+ }60+ }61+}62+63+// mapServerQueryURL spells one page's request: the window as the envelope,64+// degrees in and degrees out, fields limited to the curation, rows ordered65+// by the layer's row identity so the pages tile the layer exactly once.66+func mapServerQueryURL(dataset arcgismap.Dataset, window arcgismap.Window, offset int) string {67+ where := dataset.Where68+ if where == "" {69+ where = "1=1"70+ }71+ envelope := strings.Join([]string{72+ strconv.FormatFloat(window.West, 'f', -1, 64),73+ strconv.FormatFloat(window.South, 'f', -1, 64),74+ strconv.FormatFloat(window.East, 'f', -1, 64),75+ strconv.FormatFloat(window.North, 'f', -1, 64),76+ }, ",")77+ values := url.Values{78+ "f": {"geojson"},79+ "where": {where},80+ "geometry": {envelope},81+ "geometryType": {"esriGeometryEnvelope"},82+ "inSR": {"4326"},83+ "spatialRel": {"esriSpatialRelIntersects"},84+ "outSR": {"4326"},85+ "outFields": {strings.Join(dataset.Keep, ",")},86+ "geometryPrecision": {"7"},87+ "orderByFields": {dataset.Keep[0]},88+ "resultOffset": {strconv.Itoa(offset)},89+ "resultRecordCount": {strconv.Itoa(mapServerPage)},90+ }91+ return fmt.Sprintf("%s/%d/query?%s", dataset.Server, dataset.Layer, values.Encode())92+}93+94+// decodeMapServerPage reads one page into capture features, cut to the95+// window, and says whether another page waits. ArcGIS answers errors as96+// HTTP 200 with an error member, and spells the truncation flag at the97+// collection's top level or under its properties depending on the vintage.98+func decodeMapServerPage(99+ dataset arcgismap.Dataset,100+ window arcgismap.Window,101+ body []byte,102+ address string,103+) ([]arcgismap.Feature, bool, error) {104+ var page struct {105+ Error *struct {106+ Code int `json:"code"`107+ Message string `json:"message"`108+ } `json:"error"`109+ Type string `json:"type"`110+ ExceededTransferLimit bool `json:"exceededTransferLimit"`111+ Properties struct {112+ ExceededTransferLimit bool `json:"exceededTransferLimit"`113+ } `json:"properties"`114+ Features []struct {115+ Properties map[string]any `json:"properties"`116+ Geometry struct {117+ Type string `json:"type"`118+ Coordinates json.RawMessage `json:"coordinates"`119+ } `json:"geometry"`120+ } `json:"features"`121+ }122+ if err := json.Unmarshal(body, &page); err != nil {123+ return nil, false, fmt.Errorf("decode %s: %w", address, err)124+ }125+ if page.Error != nil {126+ return nil, false, fmt.Errorf("%s answered error %d: %s",127+ address, page.Error.Code, page.Error.Message)128+ }129+ if page.Type != "FeatureCollection" {130+ return nil, false, fmt.Errorf("%s is a %q, not a FeatureCollection", address, page.Type)131+ }132+133+ features := make([]arcgismap.Feature, 0, len(page.Features))134+ for at, raw := range page.Features {135+ fields := arcgismap.Fields{}136+ for _, keep := range dataset.Keep {137+ if value := arcgismap.FieldString(raw.Properties[keep]); value != "" {138+ fields[keep] = value139+ }140+ }141+ id, err := mapServerFeatureID(dataset, fields, raw.Properties)142+ if err != nil {143+ return nil, false, fmt.Errorf("feature %d of %s: %w", at, address, err)144+ }145+ geometry, err := arcgisGeometry(dataset.Geometry, raw.Geometry.Type, raw.Geometry.Coordinates)146+ if err != nil {147+ return nil, false, fmt.Errorf("feature %d: %w", id, err)148+ }149+ if geometry = clipToWindow(window, geometry); geometry == nil {150+ continue151+ }152+ features = append(features, arcgismap.Feature{ID: id, Fields: fields, Geometry: *geometry})153+ }154+ more := page.ExceededTransferLimit || page.Properties.ExceededTransferLimit ||155+ len(page.Features) == mapServerPage156+ return features, more, nil157+}158+159+// mapServerFeatureID is the capture's identity for one national row: the160+// curated derivation where the table declares one, the layer's own object161+// id otherwise.162+func mapServerFeatureID(dataset arcgismap.Dataset, fields arcgismap.Fields, properties map[string]any) (int64, error) {163+ if dataset.IDOf == nil {164+ return arcgisObjectID(properties)165+ }166+ id, ok := dataset.IDOf(fields)167+ if !ok {168+ return 0, fmt.Errorf("no identity in %v", fields)169+ }170+ return id, nil171+}172+173+// clipToWindow cuts a normalized geometry to the window, answering nil when174+// nothing of it remains. Points pass untouched: no national layer captures175+// them yet, and a stray point is the stray tolerance's business, not this.176+func clipToWindow(window arcgismap.Window, geometry *arcgismap.Geometry) *arcgismap.Geometry {177+ if geometry == nil {178+ return nil179+ }180+ switch geometry.Type {181+ case arcgismap.GeometryRings:182+ geometry.Rings = arcgismap.ClipRings(window, geometry.Rings)183+ if len(geometry.Rings) == 0 {184+ return nil185+ }186+ case arcgismap.GeometryLines:187+ geometry.Lines = arcgismap.ClipLines(window, geometry.Lines)188+ if len(geometry.Lines) == 0 {189+ return nil190+ }191+ }192+ return geometry193+}@@ -0,0 +1,120 @@
1+package main2+3+import (4+ "net/url"5+ "strings"6+ "testing"7+8+ "github.com/FelineStateMachine/atlas/internal/arcgismap"9+)10+11+var testWindow = arcgismap.Window{West: -105.2, South: 39.8, East: -104.9, North: 40.0}12+13+func testDataset() arcgismap.Dataset {14+ for _, dataset := range arcgismap.National {15+ if dataset.Slug == "streams" {16+ return dataset17+ }18+ }19+ panic("streams is not curated")20+}21+22+func TestMapServerQueryURLSpellsThePage(t *testing.T) {23+ address := mapServerQueryURL(testDataset(), testWindow, 4000)24+ parsed, err := url.Parse(address)25+ if err != nil {26+ t.Fatal(err)27+ }28+ if !strings.HasSuffix(parsed.Path, "/6/query") {29+ t.Fatalf("path is %q", parsed.Path)30+ }31+ values := parsed.Query()32+ wants := map[string]string{33+ "f": "geojson",34+ "where": "gnis_name IS NOT NULL",35+ "geometry": "-105.2,39.8,-104.9,40",36+ "geometryType": "esriGeometryEnvelope",37+ "inSR": "4326",38+ "outSR": "4326",39+ "spatialRel": "esriSpatialRelIntersects",40+ "outFields": "OBJECTID,gnis_name",41+ "geometryPrecision": "7",42+ "orderByFields": "OBJECTID",43+ "resultOffset": "4000",44+ "resultRecordCount": "2000",45+ }46+ for key, want := range wants {47+ if got := values.Get(key); got != want {48+ t.Fatalf("%s is %q, want %q", key, got, want)49+ }50+ }51+ // The where-clause must actually be escaped on the wire.52+ if strings.Contains(address, "gnis_name IS") {53+ t.Fatalf("where rides unescaped: %s", address)54+ }55+}56+57+func TestDecodeMapServerPageReadsFeaturesAndClips(t *testing.T) {58+ body := []byte(`{"type":"FeatureCollection","features":[59+ {"properties":{"OBJECTID":7,"gnis_name":"Big Dry Creek"},60+ "geometry":{"type":"LineString","coordinates":[[-105.1,39.9],[-105.0,39.9]]}},61+ {"properties":{"OBJECTID":8,"gnis_name":"Elsewhere Creek"},62+ "geometry":{"type":"LineString","coordinates":[[-99.0,39.9],[-98.9,39.9]]}}63+ ]}`)64+ features, more, err := decodeMapServerPage(testDataset(), testWindow, body, "test")65+ if err != nil {66+ t.Fatal(err)67+ }68+ if more {69+ t.Fatal("a short page claims another waits")70+ }71+ if len(features) != 1 || features[0].ID != 7 {72+ t.Fatalf("features are %+v; the out-of-window line should be cut away", features)73+ }74+ if features[0].Fields["gnis_name"] != "Big Dry Creek" {75+ t.Fatalf("fields are %v", features[0].Fields)76+ }77+}78+79+func TestDecodeMapServerPageReadsTheTruncationFlag(t *testing.T) {80+ spellings := []string{81+ `{"type":"FeatureCollection","exceededTransferLimit":true,"features":[]}`,82+ `{"type":"FeatureCollection","properties":{"exceededTransferLimit":true},"features":[]}`,83+ }84+ for _, body := range spellings {85+ _, more, err := decodeMapServerPage(testDataset(), testWindow, []byte(body), "test")86+ if err != nil {87+ t.Fatal(err)88+ }89+ if !more {90+ t.Fatalf("truncation went unread in %s", body)91+ }92+ }93+}94+95+func TestDecodeMapServerPageRefusesAnErrorAnswer(t *testing.T) {96+ body := []byte(`{"error":{"code":400,"message":"Invalid parameters"}}`)97+ if _, _, err := decodeMapServerPage(testDataset(), testWindow, body, "test"); err == nil {98+ t.Fatal("an error answer decoded as a page")99+ }100+}101+102+func TestMapServerFeatureIDPrefersTheCuratedDerivation(t *testing.T) {103+ var subwatersheds arcgismap.Dataset104+ for _, dataset := range arcgismap.National {105+ if dataset.Slug == arcgismap.SlugSubwatersheds {106+ subwatersheds = dataset107+ }108+ }109+ id, err := mapServerFeatureID(subwatersheds,110+ arcgismap.Fields{"huc12": "101900030304"}, map[string]any{"objectid": float64(9)})111+ if err != nil {112+ t.Fatal(err)113+ }114+ if id != 101900030304 {115+ t.Fatalf("id is %d, want the hydrologic unit code", id)116+ }117+ if _, err := mapServerFeatureID(subwatersheds, arcgismap.Fields{}, nil); err == nil {118+ t.Fatal("a row without its code passed")119+ }120+}