1// The ArcGIS mode fills the archive from a city's open-data hub: the curated2// municipal datasets a city publishes about itself, fetched as GeoJSON3// through the hub's download API, and a basemap rendered from that same4// vector data -- no outside tile service, because the bundle ships offline5// and the city's own streets are the honest picture of the city.6//7// This is the curated-cities importer, and says so. Each city in the8// arcgismap table names verified dataset identities, field names, and a9// bounding box; an unknown city is refused at the door rather than captured10// through unverified curation. Beside the city's hub, every crawl asks the11// USGS national services for the city's window -- the national enrichment12// in arcgismap's National table, fetched through the MapServer query path13// in mapserver.go -- so the capture holds the city and its watershed14// context as one document.15//16// Versioning is the mode's whole reason. Each crawl day registers its own17// map directory, so the archive keeps every day the data moved and the18// bundle's map picker reads as the city's version history. A re-crawl on a19// day already captured is a no-op by content addressing; a new day whose20// data has not moved registers nothing.21package main2223import (24 "bytes"25 "context"26 "crypto/sha256"27 "encoding/hex"28 "encoding/json"29 "errors"30 "fmt"31 "hash/fnv"32 "os"33 "path/filepath"34 "sort"35 "strconv"36 "strings"37 "sync"38 "time"3940 "github.com/FelineStateMachine/atlas/internal/arcgismap"41 "github.com/FelineStateMachine/atlas/internal/basemap"42)4344func runArcgis(ctx context.Context, fetcher *fetcher, o options) error {45 city, known := arcgismap.Cities[o.arcgis]46 if !known {47 cities := make([]string, 0, len(arcgismap.Cities))48 for name := range arcgismap.Cities {49 cities = append(cities, name)50 }51 sort.Strings(cities)52 return fmt.Errorf("the ArcGIS importer knows only %s: each city names verified "+53 "datasets and a verified window, and an unverified one could hang every pin "+54 "on the wrong pixel", strings.Join(cities, ", "))55 }5657 date := o.captureDate58 if date == "" {59 date = time.Now().UTC().Format("2006-01-02")60 }61 if _, err := time.Parse("2006-01-02", date); err != nil {62 return fmt.Errorf("-capture-date %q is not a YYYY-MM-DD day", o.captureDate)63 }6465 maxZoom := city.MaxZoom66 if o.maxZoom > 0 {67 maxZoom = o.maxZoom68 }6970 window := arcgismap.CityWindow(city.BBox)71 capture := &arcgismap.Capture{72 Source: arcgismap.Source,73 City: city.Slug,74 Title: city.Title,75 MapSlug: date,76 Window: window,77 Basemap: arcgismap.MapConfig{MaxZoom: maxZoom, Extension: "png"},78 }7980 datasets := city.AllDatasets()81 fmt.Printf("\n== ArcGIS Hub %s / %s (%d datasets)\n", city.Title, date, len(datasets))82 pinCount := 083 for _, dataset := range datasets {84 var features []arcgismap.Feature85 var err error86 if dataset.Server != "" {87 features, err = fetchMapServerDataset(ctx, fetcher, window, dataset)88 } else {89 features, err = fetchArcgisDataset(ctx, fetcher, city, dataset)90 }91 if err != nil {92 return fmt.Errorf("%s: %w", dataset.Slug, err)93 }94 if dataset.Geometry == "point" {95 pinCount += len(features)96 }97 fmt.Printf(" %-22s %5d features\n", dataset.Slug, len(features))98 capture.Datasets = append(capture.Datasets, arcgismap.CapturedDataset{99 Slug: dataset.Slug,100 Features: features,101 })102 }103 capture.Normalize()104 raw, err := json.MarshalIndent(capture, "", " ")105 if err != nil {106 return err107 }108 // A capture that does not translate would poison the archive for both109 // tools that read it, so it is refused here, while the problem is one110 // crawl old rather than buried in history.111 if _, err := arcgismap.Translate(raw); err != nil {112 return fmt.Errorf("capture does not translate: %w", err)113 }114115 gameID := arcgisArchiveID("arcgis:" + city.Slug)116 mapID := arcgisArchiveID("arcgis:" + city.Slug + "/" + date)117 game := &apiGameFull{ID: gameID, Title: city.Title,118 Config: apiGameConfig{URL: city.HubBase}}119 full := &apiMapFull{ID: mapID, Title: date, Slug: date}120 // The directory says where the capture came from even though the game121 // keeps its plain title, the same custom every non-MapGenie source keeps.122 gameDirectory := resolveGameDirectory(o.archive,123 &apiGameFull{ID: gameID, Title: "ArcGIS Hub " + city.Title})124125 // A new day whose data has not moved registers nothing: the version126 // history means "the city changed", and an unchanged capture would spend127 // a map saying it had not.128 if previous, found, err := latestArcgisCapture(o.archive, gameDirectory, date); err != nil {129 return err130 } else if found && sameArcgisCapture(previous.capture, capture) {131 fmt.Printf(" unchanged since %s; no new version registered\n", previous.day)132 return nil133 }134135 mapDirectory := resolveMapDirectory(o.archive, gameDirectory, full)136 mapDir := filepath.Join(o.archive, mapDirectory)137138 if o.dryRun {139 fmt.Printf(" would write capture to %s and render z%d (%d tiles)\n",140 mapDir, maxZoom, 1<<(2*maxZoom))141 return nil142 }143 if err := writeSnapshotRecord(mapDir, raw, arcgismap.Kind, mapID,144 city.HubBase+"/api/feed/dcat-us/1.1.json"); err != nil {145 return err146 }147148 index, err := readTileIndex(mapDir)149 if err != nil {150 return err151 }152 index.mapID = mapID153 stats, err := renderArcgisBasemap(ctx, o, city, capture, mapDir, index)154 if err != nil {155 return err156 }157 if err := writeTileIndex(mapDir, index); err != nil {158 return err159 }160 if err := writeMapMetadata(mapDir, game, full, index, pinCount); err != nil {161 return err162 }163 if err := registerMap(o.archive, game, full, gameDirectory, mapDirectory, arcgismap.Source); err != nil {164 return err165 }166 fmt.Printf(" %d rendered · %d held · %.1f MB new\n",167 stats.fetched, stats.skipped, float64(stats.bytes)/1e6)168 return nil169}170171// fetchArcgisDataset takes one curated layer through the hub's download API,172// which stages a complete file server-side -- no record-count paging, no173// partial captures -- and answers 202 until the file is ready.174func fetchArcgisDataset(175 ctx context.Context,176 fetcher *fetcher,177 city arcgismap.City,178 dataset arcgismap.Dataset,179) ([]arcgismap.Feature, error) {180 // spatialRefId asks for true degrees by name: without it a hub answers181 // in the layer's own projection -- state-plane feet, for some cities --182 // and every coordinate would be an address in the wrong language.183 url := fmt.Sprintf("%s/api/download/v1/items/%s/geojson?layers=%d&spatialRefId=4326",184 city.HubBase, dataset.ItemID, dataset.Layer)185 var body []byte186 deadline := time.Now().Add(2 * time.Minute)187 for {188 var err error189 body, _, err = fetcher.get(ctx, url)190 if err == nil {191 break192 }193 if !errors.Is(err, errAccepted) {194 return nil, err195 }196 if time.Now().After(deadline) {197 return nil, fmt.Errorf("still staging after two minutes: %s", url)198 }199 select {200 case <-ctx.Done():201 return nil, ctx.Err()202 case <-time.After(2 * time.Second):203 }204 }205206 var collection struct {207 Type string `json:"type"`208 CRS struct {209 Properties struct {210 Name string `json:"name"`211 } `json:"properties"`212 } `json:"crs"`213 Features []struct {214 Properties map[string]any `json:"properties"`215 Geometry struct {216 Type string `json:"type"`217 Coordinates json.RawMessage `json:"coordinates"`218 } `json:"geometry"`219 } `json:"features"`220 }221 if err := json.Unmarshal(body, &collection); err != nil {222 return nil, fmt.Errorf("decode %s: %w", url, err)223 }224 if collection.Type != "FeatureCollection" {225 return nil, fmt.Errorf("%s is a %q, not a FeatureCollection", url, collection.Type)226 }227 // GeoJSON without a crs member is WGS84 by definition; one with a crs228 // must actually say so, or every coordinate is in another language.229 switch collection.CRS.Properties.Name {230 case "", "EPSG:4326", "urn:ogc:def:crs:OGC:1.3:CRS84":231 default:232 return nil, fmt.Errorf("%s answered in %s, not WGS84", url, collection.CRS.Properties.Name)233 }234235 features := make([]arcgismap.Feature, 0, len(collection.Features))236 for at, raw := range collection.Features {237 id, err := arcgisObjectID(raw.Properties)238 if err != nil {239 return nil, fmt.Errorf("feature %d: %w", at, err)240 }241 fields := arcgismap.Fields{}242 for _, keep := range dataset.Keep {243 if value := arcgismap.FieldString(raw.Properties[keep]); value != "" {244 fields[keep] = value245 }246 }247 geometry, err := arcgisGeometry(dataset.Geometry, raw.Geometry.Type, raw.Geometry.Coordinates)248 if err != nil {249 return nil, fmt.Errorf("feature %d: %w", id, err)250 }251 if geometry == nil {252 continue253 }254 features = append(features, arcgismap.Feature{ID: id, Fields: fields, Geometry: *geometry})255 }256 return features, nil257}258259// arcgisObjectID reads a feature's object identifier under either of the260// spellings the hub's layers use: OBJECTID nearly everywhere, OID on the261// occasional legacy layer.262func arcgisObjectID(properties map[string]any) (int64, error) {263 value, held := properties["OBJECTID"]264 if !held {265 value, held = properties["OID"]266 }267 if !held {268 return 0, errors.New("no OBJECTID")269 }270 number, ok := value.(float64)271 if !ok || number != float64(int64(number)) {272 return 0, fmt.Errorf("OBJECTID %v is not an integer", value)273 }274 return int64(number), nil275}276277// arcgisGeometry normalizes a GeoJSON geometry into the capture's three278// shapes, held to the kind the curated table promised, coordinates rounded279// so float spelling cannot masquerade as change. A feature without geometry280// is dropped rather than refused: hubs hold the occasional geometryless row.281func arcgisGeometry(expected, kind string, coordinates json.RawMessage) (*arcgismap.Geometry, error) {282 if kind == "" || len(coordinates) == 0 {283 return nil, nil284 }285 switch expected {286 case "point":287 if kind != "Point" {288 return nil, fmt.Errorf("geometry is %s, not a Point", kind)289 }290 var position []float64291 if err := json.Unmarshal(coordinates, &position); err != nil {292 return nil, err293 }294 if len(position) < 2 {295 return nil, errors.New("point has no position")296 }297 return &arcgismap.Geometry{Type: arcgismap.GeometryPoint,298 Point: []float64{arcgismap.Round7(position[0]), arcgismap.Round7(position[1])}}, nil299 case "line":300 var lines [][][]float64301 switch kind {302 case "LineString":303 var one [][]float64304 if err := json.Unmarshal(coordinates, &one); err != nil {305 return nil, err306 }307 lines = [][][]float64{one}308 case "MultiLineString":309 if err := json.Unmarshal(coordinates, &lines); err != nil {310 return nil, err311 }312 default:313 return nil, fmt.Errorf("geometry is %s, not a line", kind)314 }315 return &arcgismap.Geometry{Type: arcgismap.GeometryLines, Lines: roundLines(lines)}, nil316 case "polygon":317 var polygons [][][][]float64318 switch kind {319 case "Polygon":320 var one [][][]float64321 if err := json.Unmarshal(coordinates, &one); err != nil {322 return nil, err323 }324 polygons = [][][][]float64{one}325 case "MultiPolygon":326 if err := json.Unmarshal(coordinates, &polygons); err != nil {327 return nil, err328 }329 default:330 return nil, fmt.Errorf("geometry is %s, not a polygon", kind)331 }332 for _, polygon := range polygons {333 for at, ring := range polygon {334 polygon[at] = roundPositions(ring)335 }336 }337 return &arcgismap.Geometry{Type: arcgismap.GeometryRings, Rings: polygons}, nil338 }339 return nil, fmt.Errorf("curated geometry %q is unknown", expected)340}341342func roundLines(lines [][][]float64) [][][]float64 {343 for at, line := range lines {344 lines[at] = roundPositions(line)345 }346 return lines347}348349func roundPositions(positions [][]float64) [][]float64 {350 for _, position := range positions {351 for at, value := range position {352 position[at] = arcgismap.Round7(value)353 }354 }355 return positions356}357358// arcgisSnapshot is the slice of game.json this mode reads back: which dated359// maps the game already carries, and where they live.360type previousCapture struct {361 day string362 capture *arcgismap.Capture363}364365// latestArcgisCapture loads the newest capture from a day before this one,366// so an unchanged city can decline to spend a version.367func latestArcgisCapture(archiveRoot, gameDirectory, before string) (previousCapture, bool, error) {368 var gameFile struct {369 Maps []struct {370 Directory string `json:"directory"`371 Slug string `json:"slug"`372 } `json:"maps"`373 }374 err := readJSON(filepath.Join(archiveRoot, gameDirectory, "game.json"), &gameFile)375 if errors.Is(err, os.ErrNotExist) {376 return previousCapture{}, false, nil377 }378 if err != nil {379 return previousCapture{}, false, err380 }381 newest := ""382 directory := ""383 for _, entry := range gameFile.Maps {384 if len(entry.Slug) == len("2006-01-02") && entry.Slug < before && entry.Slug > newest {385 newest, directory = entry.Slug, entry.Directory386 }387 }388 if newest == "" {389 return previousCapture{}, false, nil390 }391392 mapDir := filepath.Join(archiveRoot, directory)393 var snapshots []snapshotRecord394 if err := readJSON(filepath.Join(mapDir, "snapshots", "index.json"), &snapshots); err != nil {395 return previousCapture{}, false, err396 }397 if len(snapshots) == 0 {398 return previousCapture{}, false, nil399 }400 sort.Slice(snapshots, func(a, b int) bool {401 return snapshots[a].CapturedAt < snapshots[b].CapturedAt402 })403 latest := snapshots[len(snapshots)-1]404 raw, err := os.ReadFile(filepath.Join(mapDir, "snapshots", "map", latest.ContentHash+".json"))405 if err != nil {406 return previousCapture{}, false, err407 }408 var capture arcgismap.Capture409 if err := json.Unmarshal(raw, &capture); err != nil {410 return previousCapture{}, false, err411 }412 return previousCapture{day: newest, capture: &capture}, true, nil413}414415// sameArcgisCapture asks whether two captures say the same thing about the416// city, the day itself aside.417func sameArcgisCapture(previous, current *arcgismap.Capture) bool {418 a, b := *previous, *current419 a.MapSlug, b.MapSlug = "", ""420 rawA, errA := json.Marshal(&a)421 rawB, errB := json.Marshal(&b)422 return errA == nil && errB == nil && bytes.Equal(rawA, rawB)423}424425// renderArcgisBasemap draws the deepest level of the city's own ground into426// the archive, every tile of it, exactly as a fetching source would have427// recorded downloaded ones: tools/tiles folds the rest of the pyramid down428// from this level and never knows the wire was absent. Tiles whose bytes429// already sit in the index are held, not rewritten, so a re-render without430// change leaves the archive's working tree untouched.431func renderArcgisBasemap(432 ctx context.Context,433 o options,434 city arcgismap.City,435 capture *arcgismap.Capture,436 mapDir string,437 index *tileIndex,438) (captureStats, error) {439 var stats captureStats440 features := basemapFeatures(city, capture)441 renderer := basemap.NewRenderer(features, capture.Basemap.MaxZoom)442 scope := arcgismap.TileSetPath(city.Slug, capture.MapSlug)443 setID := index.tileSetID(scope)444 template := arcgismap.TileTemplate(city.Slug, capture.MapSlug)445 zoom := capture.Basemap.MaxZoom446 edge := 1 << zoom447 fmt.Printf(" basemap %q z%d, %d tiles from %d features\n",448 scope, zoom, edge*edge, len(features))449450 var mu sync.Mutex451 var group sync.WaitGroup452 var failure error453 gate := make(chan struct{}, o.concurrency)454 for y := range edge {455 for x := range edge {456 select {457 case <-ctx.Done():458 group.Wait()459 return stats, ctx.Err()460 default:461 }462 group.Add(1)463 gate <- struct{}{}464 go func(x, y int) {465 defer group.Done()466 defer func() { <-gate }()467 url := strings.NewReplacer(468 "{z}", strconv.Itoa(zoom), "{x}", strconv.Itoa(x), "{y}", strconv.Itoa(y),469 ).Replace(template)470 path := filepath.Join(mapDir, "tiles",471 "set-"+strconv.FormatInt(setID, 10),472 strconv.Itoa(zoom), strconv.Itoa(x), strconv.Itoa(y)+".png")473474 body, err := basemap.EncodePNG(renderer.Tile(zoom, x, y))475 if err != nil {476 mu.Lock()477 if failure == nil {478 failure = fmt.Errorf("render %d/%d/%d: %w", zoom, x, y, err)479 }480 mu.Unlock()481 return482 }483 sum := sha256.Sum256(body)484 hash := hex.EncodeToString(sum[:])485486 mu.Lock()487 record, held := index.byURL[url]488 mu.Unlock()489 if held && record.Status == "cached" && record.ContentHash == hash {490 if _, err := os.Stat(path); err == nil {491 mu.Lock()492 stats.skipped++493 mu.Unlock()494 return495 }496 }497 if err := writeFile(path, body); err != nil {498 mu.Lock()499 if failure == nil {500 failure = err501 }502 mu.Unlock()503 return504 }505 mu.Lock()506 index.put(tileRecord{507 URL: url, Status: "cached", TileSetID: setID,508 Zoom: zoom, X: x, Y: y,509 ContentHash: hash, ContentType: "image/png",510 ByteLength: len(body),511 CoverageKey: fmt.Sprintf("%d:%d:%d:%d:%d",512 index.mapID, setID, zoom, x, y),513 })514 stats.fetched++515 stats.bytes += int64(len(body))516 mu.Unlock()517 }(x, y)518 }519 }520 group.Wait()521 return stats, failure522}523524// basemapFeatures projects every drawing dataset into world pixels under the525// capture's own window, so the raster and the pins can never disagree about526// where the ground is.527func basemapFeatures(city arcgismap.City, capture *arcgismap.Capture) []basemap.Feature {528 datasets := city.AllDatasets()529 curated := make(map[string]*arcgismap.Dataset, len(datasets))530 for at := range datasets {531 curated[datasets[at].Slug] = &datasets[at]532 }533 var features []basemap.Feature534 for _, dataset := range capture.Datasets {535 table := curated[dataset.Slug]536 if table == nil || table.Role == "" {537 continue538 }539 role := basemap.Role(table.Role)540 for _, feature := range dataset.Features {541 emphasis := 0.0542 if table.Emphasis != nil {543 emphasis = table.Emphasis(feature.Fields)544 }545 switch feature.Geometry.Type {546 case arcgismap.GeometryRings:547 // Each polygon draws on its own, so winding normalization548 // reads every first ring as ground and the rest as holes.549 for _, polygon := range feature.Geometry.Rings {550 drawn := basemap.Feature{Role: role, Emphasis: emphasis}551 for _, ring := range polygon {552 drawn.Rings = append(drawn.Rings, projectPositions(capture.Window, ring))553 }554 if len(drawn.Rings) > 0 {555 features = append(features, drawn)556 }557 }558 case arcgismap.GeometryLines:559 drawn := basemap.Feature{Role: role, Emphasis: emphasis}560 for _, line := range feature.Geometry.Lines {561 drawn.Lines = append(drawn.Lines, projectPositions(capture.Window, line))562 }563 if len(drawn.Lines) > 0 {564 features = append(features, drawn)565 }566 }567 }568 }569 return features570}571572func projectPositions(window arcgismap.Window, positions [][]float64) [][2]float64 {573 out := make([][2]float64, 0, len(positions))574 for _, position := range positions {575 if len(position) < 2 {576 continue577 }578 x, y := window.WorldPixel(position[0], position[1])579 out = append(out, [2]float64{x, y})580 }581 return out582}583584// arcgisArchiveID numbers ArcGIS cities and maps in the archive's registers,585// one bit above Trek's so every source's ids stay apart; see ignArchiveID586// for why the range stays far below a float64's integers.587func arcgisArchiveID(seed string) int64 {588 hash := fnv.New32a()589 _, _ = hash.Write([]byte(seed))590 return int64(hash.Sum32()&0x7fffffff) | 1<<35591}