A new source proves the format's words were never about games: an ArcGIS Hub open-data site, where the city is the "game", each crawl day registers a dated map, and the picker reads as the city's version history -- four captured days are four maps, and the differences between them are differences in the city. Zoning, annexations and wetlands ride the zones, named places ride the pins with their true coordinates in atlas.geo.*, and the one raster layer is the basemap rendered from the city's own streets and trails. The curated table is the whole authority, in Trek's posture: Bend, Oregon is the proof city, verified field by field against the live services, and an uncurated city is refused at the door. A private city can be curated from a git-ignored cities_local.go beside the public table. The hub is asked for true degrees by name -- some hubs answer in state-plane feet otherwise -- and a capture that does not translate is refused while the mistake is one crawl old. Versioning costs nothing new: dated map directories ride the existing newest-snapshot rule, an unchanged day declines to register, a same-day retake is a byte-level no-op, and the capture day rides in the tile-set path so every dated basemap carries its own pyramid through the carry-over stamps.
- commit
- b07f03b5ec06554be1a832f4ecbfd5d1cbf7fdc4
- parent
- 2c28ecbd6ab7e9f2934059eeb76cb0e2bd6e009c
- author
- Dami <dami.etoile@gmail.com>
| M | .gitignore | 3 | + |
| M | cmd/ | 16 | + |
| M | cmd/ | 14 | +- |
| A | internal/ | 867 | ++++++++++++++++++++++++++++++ |
| A | internal/ | 433 | +++++++++++++++ |
| A | internal/ | 4 | + |
| A | internal/ | 4 | + |
| A | internal/ | 4 | + |
| A | internal/ | 4 | + |
| A | internal/ | 4 | + |
| M | tools/ | 7 | + |
| A | tools/ | 579 | ++++++++++++++++++++ |
| M | tools/ | 7 | + |
| M | tools/ | 23 | + |
| M | tools/ | 22 | +- |
| M | tools/ | 2 | + |
| M | tools/ | 4 | + |
@@ -180,6 +180,22 @@ var sources = []Source{
180180 ComponentRaster, ComponentLocations, ComponentMetadata,181181 },182182 },183+ crawlSource{184+ name: "ArcGIS Open Data",185+ slug: "arcgis-hub",186+ description: "A city's ArcGIS Hub open-data site, bending the bundle's words to civic " +187+ "ground: the city is the \"game\", each crawl day registers a dated map, and the " +188+ "picker reads as the city's version history. Zoning, annexations and wetlands ride " +189+ "the zones, named places ride the pins, and the one raster layer is a basemap " +190+ "rendered offline from the city's own streets and trails. A curated importer: each " +191+ "city names verified datasets and a verified window, and only Bend, Oregon is " +192+ "curated so far. Standard-library icons; no source artwork.",193+ flag: "-arcgis",194+ hint: "city, e.g. bend-or (the one curated city)",195+ components: ComponentSet{196+ ComponentRaster, ComponentLocations, ComponentMetadata,197+ },198+ },183199 }184200 185201 // sourceBySlug finds one registered source.@@ -6,21 +6,21 @@ import (
66 )77 88 func TestRegistry(t *testing.T) {9- if len(sources) != 4 {10- t.Fatalf("registry holds %d sources, want 4", len(sources))9+ if len(sources) != 5 {10+ t.Fatalf("registry holds %d sources, want 5", len(sources))1111 }12- for _, slug := range []string{"mapgenie", "ign-wiki", "piggyback", "nasa-trek"} {12+ for _, slug := range []string{"mapgenie", "ign-wiki", "piggyback", "nasa-trek", "arcgis-hub"} {1313 src, ok := sourceBySlug(slug)1414 if !ok {1515 t.Fatalf("registry misses %s", slug)1616 }1717 // Every source publishes a complete map save for icons, which Trek18- // alone leaves to the viewer's fallback glyphs; the game sources19- // differ in quality per component, never in kind.18+ // and the civic importer leave to the standard library's glyphs; the19+ // game sources differ in quality per component, never in kind.2020 wanted := []Component{2121 ComponentRaster, ComponentIcons, ComponentLocations, ComponentMetadata,2222 }23- if slug == "nasa-trek" {23+ if slug == "nasa-trek" || slug == "arcgis-hub" {2424 wanted = []Component{ComponentRaster, ComponentLocations, ComponentMetadata}2525 if src.Components().Has(ComponentIcons) {2626 t.Errorf("%s claims icons it does not fetch", slug)@@ -50,6 +50,7 @@ func TestFetchArgs(t *testing.T) {
5050 {"ign-wiki", "cyberpunk-2077/night-city", []string{"-ign", "cyberpunk-2077/night-city"}},5151 {"piggyback", "cyberpunk-2077/night-city", []string{"-piggyback", "cyberpunk-2077/night-city"}},5252 {"nasa-trek", "mars", []string{"-trek", "mars"}},53+ {"arcgis-hub", "bend-or", []string{"-arcgis", "bend-or"}},5354 }5455 for _, at := range good {5556 src, _ := sourceBySlug(at.source)@@ -77,6 +78,7 @@ func TestFetchArgs(t *testing.T) {
7778 {"piggyback", "night-city/"},7879 {"piggyback", "Night-City/x"}, // slugs are lowercase7980 {"nasa-trek", "mars/global"}, // a body is one slug81+ {"arcgis-hub", "bend/or"}, // a city is one slug8082 }8183 for _, at := range bad {8284 src, _ := sourceBySlug(at.source)@@ -0,0 +1,867 @@
1+// Package arcgismap reads a captured city and speaks MapGenie for it.2+//3+// The capture comes from a city's ArcGIS Hub open-data site: the municipal4+// datasets a city publishes about itself -- its limits, its zoning, its5+// trails, its fire stations -- fetched as GeoJSON and kept verbatim under a6+// curated allowlist of fields. Like ignmap, pbmap and trekmap, translation7+// happens when a capture is read, not when it is taken: the archive keeps8+// what the city published, and every keep-or-leave decision lives here,9+// where changing it re-applies to captures already on disk.10+//11+// The semantics are the point being proven. A city is a "game" to the12+// pipeline; each crawl day registers its own map, so a bundle's map picker13+// becomes the city's version history -- four captured days are four maps,14+// and the differences between them are differences in the city. Polygon15+// datasets ride the bundle's zones, point datasets its pins, and the one16+// raster layer is a basemap rendered from the city's own vector data,17+// because a bundle owes its reader ground to stand on and ships offline.18+//19+// The coordinate design follows trekmap, on a plane. A city sits off the20+// Web-Mercator diagonal, and the viewer's window is a square upon it, so the21+// city's bounding box -- padded, and squared in Mercator's own terms --22+// becomes the 8192-pixel world. A feature's true coordinates land on that23+// square through the Mercator forward, and the pixel becomes synthetic24+// latitude and longitude through the same inverse every translated source25+// uses. The window is declared on the map as the mercator.px/deg mapping,26+// and every pin keeps its true coordinates in atlas.geo.*, so nothing is27+// flattened away.28+package arcgismap29+30+import (31+ "encoding/json"32+ "fmt"33+ "math"34+ "regexp"35+ "sort"36+ "strconv"37+ "strings"38+ "time"39+40+ "github.com/FelineStateMachine/atlas/internal/mgdoc"41+ "github.com/FelineStateMachine/atlas/internal/semconv"42+)43+44+// Kind names ArcGIS Hub captures in a map's snapshot index.45+const Kind = "arcgis-map"46+47+// Source is the value a well-formed capture carries in its source field.48+const Source = "arcgis-hub"49+50+// Fields is one feature's kept attributes, every value spelled as text.51+type Fields map[string]string52+53+// ZoneKey says which zone a feature folds into: the bucket's stable key and54+// the title and subtitle the zone wears. A zero key is a feature that makes55+// no zone.56+type ZoneKey struct {57+ Key string58+ Title string59+ Subtitle string60+}61+62+// Dataset is one curated layer of a city's hub: where to fetch it, which63+// fields to keep, and what it becomes in the bundle -- pins when Group is64+// set, zones when ZoneOf is, basemap linework when Role is. The functions65+// read a feature's kept fields, so re-curating presentation is a table edit66+// and a re-translation of captures already on disk.67+type Dataset struct {68+ Slug string69+ Title string70+ ItemID string71+ Layer int72+ Geometry string // "point" | "line" | "polygon"73+ Keep []string74+75+ // Pins.76+ Group string77+ Icon string78+ TitleOf func(Fields) string79+ Describe func(Fields) string80+81+ // Zones. RibbonWidth turns a line dataset's features into clickable82+ // ground, in world pixels of the 8192 square.83+ ZoneOf func(Fields) ZoneKey84+ RibbonWidth float6485+86+ // Basemap. Emphasis scales a stroke per feature -- an arterial wider87+ // than a lane -- and absent means every feature draws alike.88+ Role string89+ Emphasis func(Fields) float6490+}91+92+// City is one curated hub. The table is the whole authority: an uncurated93+// city is refused at the door, because an unverified bounding box would hang94+// every pin on the wrong pixel, and unverified field names would title every95+// zone with nothing.96+type City struct {97+ Slug string98+ Title string99+ HubBase string100+ MaxZoom int101+ BBox [4]float64 // west, south, east, north, degrees102+ Datasets []Dataset103+}104+105+// Cities is every city the source knows how to capture. A city that should106+// stay out of the repository -- your own, say -- can be registered from a107+// cities_local.go beside this file: that name is ignored by git, and an108+// init function there adding to this map curates privately what this table109+// curates publicly.110+var Cities = map[string]City{111+ "bend-or": bend,112+}113+114+// bend is the City of Bend, Oregon -- the curated proof city, chosen for a115+// hub surface with every shape the format wants to prove: polygon ground,116+// named line work, and titled points, all behind the standard hub download117+// API. Field names were verified against the live services. The water118+// utility layers stay out deliberately: their attribute tables carry staff119+// account names, and an open capture keeps people out of it.120+var bend = City{121+ Slug: "bend-or",122+ Title: "Bend, Oregon",123+ HubBase: "https://data.bendoregon.gov",124+ MaxZoom: 6,125+ BBox: [4]float64{-121.4180, 43.9500, -121.2170, 44.1650},126+ Datasets: []Dataset{127+ {128+ Slug: "mpo-boundary", Title: "MPO Boundary",129+ ItemID: "ec95698d5b7f494eb12dba2c572ed6d1", Layer: 0,130+ Geometry: "polygon",131+ Keep: []string{"OID", "LABEL", "Acres"},132+ ZoneOf: func(f Fields) ZoneKey {133+ return ZoneKey{Key: "mpo-boundary", Title: "MPO Boundary",134+ Subtitle: "Metropolitan planning area"}135+ },136+ Role: "boundary",137+ },138+ {139+ Slug: "zoning", Title: "Zoning",140+ ItemID: "90f0e9717185404581b2c1c865c13f03", Layer: 0,141+ Geometry: "polygon",142+ Keep: []string{"OBJECTID", "ZONE", "ORDINANCE"},143+ ZoneOf: func(f Fields) ZoneKey {144+ code := f["ZONE"]145+ if code == "" {146+ return ZoneKey{}147+ }148+ return ZoneKey{Key: slugify(code), Title: code}149+ },150+ },151+ {152+ Slug: "annexations", Title: "Annexations",153+ ItemID: "7aec052cc38645f69d2a74e8062abd95", Layer: 0,154+ Geometry: "polygon",155+ Keep: []string{"OBJECTID", "DESCRIPT", "EFF_DATE", "ORDIN_NO", "ANNX_ACRES"},156+ // Annexations fold into the decades they took effect in: the157+ // story a version history wants to tell is how the city grew,158+ // not which ordinance grew it.159+ ZoneOf: func(f Fields) ZoneKey {160+ ms, err := strconv.ParseInt(f["EFF_DATE"], 10, 64)161+ if err != nil || ms == 0 {162+ return ZoneKey{Key: "undated", Title: "Annexed, date unknown"}163+ }164+ decade := time.UnixMilli(ms).UTC().Year() / 10 * 10165+ return ZoneKey{166+ Key: strconv.Itoa(decade) + "s",167+ Title: fmt.Sprintf("Annexed %d–%d", decade, decade+9),168+ }169+ },170+ },171+ {172+ Slug: "wetlands", Title: "Wetlands",173+ ItemID: "8c9ded5d2a32423f8b380de1bec50cf4", Layer: 0,174+ Geometry: "polygon",175+ Keep: []string{"OBJECTID", "TYPE", "MAP_CODE", "ACRES"},176+ ZoneOf: func(f Fields) ZoneKey {177+ kind := f["TYPE"]178+ if kind == "" {179+ return ZoneKey{}180+ }181+ return ZoneKey{Key: slugify(kind), Title: kind, Subtitle: "Local Wetlands Inventory"}182+ },183+ Role: "water",184+ },185+ {186+ Slug: "trails", Title: "Paths & Trails",187+ ItemID: "8d006fb5453c4dbfad476033e847eca8", Layer: 0,188+ Geometry: "line",189+ Keep: []string{"OBJECTID", "Trail_Name", "Park", "Status", "Surface_Ma", "Classifica", "Maintained"},190+ // Every existing segment draws into the basemap; the named191+ // trails earn zones, each one clickable end to end.192+ ZoneOf: func(f Fields) ZoneKey {193+ if f["Status"] != "Existing" || f["Trail_Name"] == "" {194+ return ZoneKey{}195+ }196+ return ZoneKey{Key: slugify(f["Trail_Name"]), Title: f["Trail_Name"],197+ Subtitle: title(f["Park"], "Trail")}198+ },199+ RibbonWidth: 12,200+ Role: "trail",201+ },202+ {203+ Slug: "road-centerlines", Title: "Road Centerlines",204+ ItemID: "c303eb99c9a242149b95d5055589e251", Layer: 0,205+ Geometry: "line",206+ Keep: []string{"OBJECTID", "fullname", "roadclass", "onewaydir", "speedlimit"},207+ Role: "street",208+ Emphasis: func(f Fields) float64 {209+ switch f["roadclass"] {210+ case "Local":211+ return 1212+ case "Collector":213+ return 1.4214+ default: // arterials and everything grander215+ return 1.8216+ }217+ },218+ },219+ {220+ Slug: "historic-sites", Title: "Historic Resources",221+ ItemID: "70b88002805f42dfa94d0c470d49eec1", Layer: 0,222+ Geometry: "point",223+ Keep: []string{"OBJECTID", "NAME", "TAB_NAME", "SHORT_DESC", "DESC1", "DESC2", "DESC3", "DESC4", "DESC5"},224+ Group: "Heritage",225+ Icon: "maki/monument",226+ TitleOf: func(f Fields) string { return f["NAME"] },227+ Describe: func(f Fields) string {228+ return join(" · ", f["TAB_NAME"], f["SHORT_DESC"],229+ f["DESC1"], f["DESC2"], f["DESC3"], f["DESC4"], f["DESC5"])230+ },231+ },232+ },233+}234+235+// join glues the non-empty pieces with the separator, which is most of what236+// composing a card from optional fields is.237+func join(sep string, pieces ...string) string {238+ kept := pieces[:0:0]239+ for _, piece := range pieces {240+ if trimmed := strings.TrimSpace(piece); trimmed != "" {241+ kept = append(kept, trimmed)242+ }243+ }244+ return strings.Join(kept, sep)245+}246+247+// Window is the ground the city's world square pictures: the degrees at the248+// padded square's edges. The square is square in Mercator's own terms -- u249+// linear in longitude, v in projected latitude -- which is what lets one250+// number serve as both axes' scale.251+type Window struct {252+ West float64 `json:"west"`253+ North float64 `json:"north"`254+ East float64 `json:"east"`255+ South float64 `json:"south"`256+}257+258+// mercator lands degrees on the global Web-Mercator unit square.259+func mercator(lon, lat float64) (u, v float64) {260+ u = (lon + 180) / 360261+ v = (1 - math.Asinh(math.Tan(lat*math.Pi/180))/math.Pi) / 2262+ return u, v263+}264+265+// degrees is the way back.266+func degrees(u, v float64) (lon, lat float64) {267+ lon = u*360 - 180268+ lat = math.Atan(math.Sinh(math.Pi*(1-2*v))) * 180 / math.Pi269+ return lon, lat270+}271+272+// CityWindow pads a city's bounding box five percent per side and squares it273+// in Mercator terms, growing the shorter axis around its own middle. The274+// same box always makes the same window, which is what keeps every capture275+// of a city in one world.276+func CityWindow(bbox [4]float64) Window {277+ u0, v0 := mercator(bbox[0], bbox[3]) // west, north: the top-left278+ u1, v1 := mercator(bbox[2], bbox[1]) // east, south: the bottom-right279+ padU, padV := (u1-u0)*0.05, (v1-v0)*0.05280+ u0, u1 = u0-padU, u1+padU281+ v0, v1 = v0-padV, v1+padV282+ side := math.Max(u1-u0, v1-v0)283+ uMid, vMid := (u0+u1)/2, (v0+v1)/2284+ u0, u1 = uMid-side/2, uMid+side/2285+ v0, v1 = vMid-side/2, vMid+side/2286+ west, north := degrees(u0, v0)287+ east, south := degrees(u1, v1)288+ return Window{West: west, North: north, East: east, South: south}289+}290+291+// WorldPixel lands true coordinates on the world square.292+func (w Window) WorldPixel(lon, lat float64) (x, y float64) {293+ u0, v0 := mercator(w.West, w.North)294+ u1, _ := mercator(w.East, w.South)295+ side := u1 - u0296+ u, v := mercator(lon, lat)297+ return (u - u0) / side * mgdoc.WorldSize, (v - v0) / side * mgdoc.WorldSize298+}299+300+// PxDeg spells the window as the two map attributes that declare it.301+func (w Window) PxDeg() (px, deg string) {302+ px = fmt.Sprintf("0,0,%d,%d", mgdoc.WorldSize, mgdoc.WorldSize)303+ parts := []string{304+ strconv.FormatFloat(w.West, 'f', -1, 64),305+ strconv.FormatFloat(w.North, 'f', -1, 64),306+ strconv.FormatFloat(w.East, 'f', -1, 64),307+ strconv.FormatFloat(w.South, 'f', -1, 64),308+ }309+ return px, strings.Join(parts, ",")310+}311+312+// TileSetPath is the layer path the basemap's tile records group under,313+// matching what tools/tiles recovers from a record URL's /tiles/ marker.314+// The capture day is part of the path: every dated map renders its own315+// basemap, and the pyramid pipeline tells layers apart by this path alone.316+func TileSetPath(city, day string) string {317+ return city + "/" + day + "/basemap"318+}319+320+// TileTemplate is the record URL the rendered basemap answers to. The scheme321+// is deliberately not a fetchable one: these tiles were never on a wire, and322+// a spelling without http keeps them visibly clear of the offline rule.323+func TileTemplate(city, day string) string {324+ return "arcgis-basemap://tiles/" + TileSetPath(city, day) + "/{z}/{x}/{y}.png"325+}326+327+// Capture is the canonical document the crawler stores: the city, the day,328+// the window, the basemap pyramid rendered beside it, and every curated329+// dataset's features under their kept fields. Fields marshal in declaration330+// order and Normalize sorts the lists, so unchanged data always hashes to331+// the capture already archived.332+type Capture struct {333+ Source string `json:"source"`334+ City string `json:"city"`335+ Title string `json:"title"`336+ MapSlug string `json:"mapSlug"`337+ Window Window `json:"window"`338+ Basemap MapConfig `json:"basemap"`339+ Datasets []CapturedDataset `json:"datasets"`340+}341+342+// MapConfig is the basemap pyramid as rendered: the deepest level drawn, and343+// the extension every tile wears.344+type MapConfig struct {345+ MaxZoom int `json:"maxZoom"`346+ Extension string `json:"extension"`347+}348+349+// CapturedDataset is one curated layer's features, joined back to the table350+// by slug when the capture is read.351+type CapturedDataset struct {352+ Slug string `json:"slug"`353+ Features []Feature `json:"features"`354+}355+356+// Feature is one thing the city published: its object identifier, its kept357+// fields spelled as text, and its ground in true degrees.358+type Feature struct {359+ ID int64 `json:"id"`360+ Fields Fields `json:"fields,omitempty"`361+ Geometry Geometry `json:"geometry"`362+}363+364+// Geometry is a feature's ground, normalized at capture to one of three365+// shapes: a point, lines in MultiLineString nesting, or rings in366+// MultiPolygon nesting. Positions are [longitude, latitude] in WGS84,367+// rounded to seven decimals -- a centimeter -- so float spelling cannot368+// masquerade as change.369+type Geometry struct {370+ Type string `json:"type"`371+ Point []float64 `json:"point,omitempty"`372+ Lines [][][]float64 `json:"lines,omitempty"`373+ Rings [][][][]float64 `json:"rings,omitempty"`374+}375+376+const (377+ GeometryPoint = "point"378+ GeometryLines = "lines"379+ GeometryRings = "rings"380+)381+382+// Normalize puts the capture into its canonical order. The crawler calls383+// this before marshaling, so the hub listing its features differently does384+// not masquerade as a new version.385+func (c *Capture) Normalize() {386+ sort.Slice(c.Datasets, func(a, b int) bool { return c.Datasets[a].Slug < c.Datasets[b].Slug })387+ for _, dataset := range c.Datasets {388+ features := dataset.Features389+ sort.Slice(features, func(a, b int) bool { return features[a].ID < features[b].ID })390+ }391+}392+393+// MaybeTranslate hands other sources' snapshots through untouched and394+// translates ArcGIS Hub captures.395+func MaybeTranslate(kind string, doc []byte) ([]byte, error) {396+ if kind != Kind {397+ return doc, nil398+ }399+ return Translate(doc)400+}401+402+// mapSlugShape holds map slugs to capture days, which is what makes a map403+// picker read as a version history.404+var mapSlugShape = regexp.MustCompile(`^\d{4}-\d{2}-\d{2}$`)405+406+// Translate turns a capture into the MapGenie-shaped document the pipeline407+// reads, deterministically for a given capture.408+func Translate(doc []byte) ([]byte, error) {409+ var capture Capture410+ if err := json.Unmarshal(doc, &capture); err != nil {411+ return nil, fmt.Errorf("decode arcgis capture: %w", err)412+ }413+ if capture.Source != Source {414+ return nil, fmt.Errorf("capture source is %q, not %q", capture.Source, Source)415+ }416+ city, curated := Cities[capture.City]417+ if !curated {418+ return nil, fmt.Errorf("city %q is not curated", capture.City)419+ }420+ if !mapSlugShape.MatchString(capture.MapSlug) {421+ return nil, fmt.Errorf("map slug %q is not a capture day", capture.MapSlug)422+ }423+ if capture.Basemap.MaxZoom < 1 {424+ return nil, fmt.Errorf("capture declares no basemap pyramid")425+ }426+ if capture.Window.East <= capture.Window.West || capture.Window.North <= capture.Window.South {427+ return nil, fmt.Errorf("capture window has no ground")428+ }429+ if len(capture.Datasets) == 0 {430+ return nil, fmt.Errorf("capture carries no datasets")431+ }432+ capture.Normalize()433+434+ byOrder, err := curatedOrder(city, &capture)435+ if err != nil {436+ return nil, err437+ }438+439+ ids := mgdoc.NewIDSpace()440+ scope := capture.City + "/" + capture.MapSlug441+ mapID, err := ids.Claim("arcgis:map:" + scope)442+ if err != nil {443+ return nil, err444+ }445+ gameID, err := ids.Claim("arcgis:game:" + capture.City)446+ if err != nil {447+ return nil, err448+ }449+450+ groups, err := buildGroups(&capture, byOrder, ids)451+ if err != nil {452+ return nil, err453+ }454+ regions, err := buildRegions(&capture, byOrder, ids)455+ if err != nil {456+ return nil, err457+ }458+459+ px, deg := capture.Window.PxDeg()460+ out := mgdoc.Map{461+ ID: mapID,462+ // The day is the map: its slug, and its title, because the picker463+ // listing dates is the version history reading as itself.464+ Title: capture.MapSlug,465+ Slug: capture.MapSlug,466+ InitialLatitude: mgdoc.SyntheticLatitude(mgdoc.WorldSize / 2),467+ InitialLongitude: mgdoc.SyntheticLongitude(mgdoc.WorldSize / 2),468+ Config: mgdoc.Config{TileSets: []mgdoc.TileSet{469+ tileSet(capture.City, capture.MapSlug, capture.Basemap),470+ }},471+ // The map says what it pictures: a plane -- no globe pretends a city472+ // is a world -- cut from Earth by the declared Mercator window, which473+ // is exactly the transform WorldPixel projects features through, so474+ // the two can never disagree without a test noticing.475+ Attrs: map[string]string{476+ semconv.KeyGeometrySurface: semconv.SurfacePlane,477+ semconv.KeyGeometryBody: "earth",478+ semconv.KeyGeometryMercatorPx: px,479+ semconv.KeyGeometryMercatorDeg: deg,480+ },481+ Game: mgdoc.Game{482+ ID: gameID,483+ // A city is a "game" to the pipeline, which is the point being484+ // proven.485+ Title: title(capture.Title, city.Title),486+ Slug: capture.City,487+ },488+ Groups: groups,489+ Regions: regions,490+ }491+ return json.Marshal(out)492+}493+494+func title(given, fallback string) string {495+ if given != "" {496+ return given497+ }498+ return fallback499+}500+501+// curatedOrder joins the capture's datasets back to the table, in the502+// table's own order, refusing anything the table does not name -- the same503+// posture the crawler takes at the door, held again at read time.504+func curatedOrder(city City, capture *Capture) ([]pairing, error) {505+ captured := make(map[string]*CapturedDataset, len(capture.Datasets))506+ for at := range capture.Datasets {507+ dataset := &capture.Datasets[at]508+ if _, doubled := captured[dataset.Slug]; doubled {509+ return nil, fmt.Errorf("dataset %q is captured twice", dataset.Slug)510+ }511+ captured[dataset.Slug] = dataset512+ }513+ var out []pairing514+ for at := range city.Datasets {515+ curated := &city.Datasets[at]516+ if data, taken := captured[curated.Slug]; taken {517+ out = append(out, pairing{curated, data})518+ delete(captured, curated.Slug)519+ }520+ }521+ for slug := range captured {522+ return nil, fmt.Errorf("dataset %q is not curated for %s", slug, city.Slug)523+ }524+ if len(out) == 0 {525+ return nil, fmt.Errorf("capture carries no curated datasets")526+ }527+ return out, nil528+}529+530+type pairing struct {531+ curated *Dataset532+ data *CapturedDataset533+}534+535+// tileSet declares the basemap pyramid in the shape tools/tiles expects.536+// Bounds are declared full-square for every level down to zero, which is537+// what lands the frame derivation on the translated world's own grid --538+// sourceZoom five, first tile zero -- instead of the MapGenie window this539+// map does not sit in.540+func tileSet(citySlug, day string, config MapConfig) mgdoc.TileSet {541+ set := mgdoc.TileSet{542+ Name: "Basemap",543+ Path: TileSetPath(citySlug, day),544+ MinZoom: 0,545+ MaxZoom: config.MaxZoom,546+ Extension: title(config.Extension, "png"),547+ Bounds: make(map[string]mgdoc.Bound, config.MaxZoom+1),548+ }549+ for zoom := 0; zoom <= config.MaxZoom; zoom++ {550+ last := 1<<zoom - 1551+ set.Bounds[strconv.Itoa(zoom)] = mgdoc.Bound{552+ X: mgdoc.Range{Min: 0, Max: last},553+ Y: mgdoc.Range{Min: 0, Max: last},554+ }555+ }556+ return set557+}558+559+// strayTolerance is how much of a point dataset may fall outside the window560+// before the capture is refused rather than trimmed: a hub layer561+// occasionally holds one stray feature, and one stray is not a wrong window.562+const strayTolerance = 0.01563+564+// buildGroups arranges the point datasets the way the pipeline expects: one565+// category per dataset, gathered under the curated group titles in table566+// order.567+func buildGroups(capture *Capture, pairs []pairing, ids *mgdoc.IDSpace) ([]mgdoc.Group, error) {568+ byGroup := make(map[string][]mgdoc.Category)569+ var order []string570+ for _, pair := range pairs {571+ curated, data := pair.curated, pair.data572+ if curated.Group == "" {573+ continue574+ }575+ categoryID, err := ids.Claim("arcgis:cat:" + capture.City + "/" + curated.Slug)576+ if err != nil {577+ return nil, err578+ }579+ attrs := map[string]string{semconv.KeyRenderAs: semconv.RenderAsPin}580+ if curated.Icon != "" {581+ attrs[semconv.KeyIconStd] = curated.Icon582+ }583+ category := mgdoc.Category{584+ ID: categoryID,585+ Title: curated.Title,586+ Icon: slugify(curated.Title),587+ DisplayType: "markers",588+ Visible: true,589+ Locations: make([]mgdoc.Location, 0, len(data.Features)),590+ Attrs: attrs,591+ }592+ strays, untitled := 0, 0593+ for _, feature := range data.Features {594+ if feature.Geometry.Type != GeometryPoint || len(feature.Geometry.Point) < 2 {595+ return nil, fmt.Errorf("%s feature %d has no point", curated.Slug, feature.ID)596+ }597+ lon, lat := feature.Geometry.Point[0], feature.Geometry.Point[1]598+ x, y := capture.Window.WorldPixel(lon, lat)599+ if x < 0 || x > mgdoc.WorldSize || y < 0 || y > mgdoc.WorldSize {600+ strays++601+ continue602+ }603+ name := scrub(curated.TitleOf(feature.Fields))604+ // A nameless row is the source's hygiene, not the curation's: a605+ // pin nobody could search for is left out. A dataset nameless606+ // throughout is different -- that is a wrong field name, and it607+ // is refused below rather than shipped blank.608+ if name == "" {609+ untitled++610+ continue611+ }612+ locationID, err := ids.Claim("arcgis:loc:" + capture.City + "/" + curated.Slug + "/" + strconv.FormatInt(feature.ID, 10))613+ if err != nil {614+ return nil, err615+ }616+ category.Locations = append(category.Locations, mgdoc.Location{617+ ID: locationID,618+ Title: name,619+ Description: scrub(curated.Describe(feature.Fields)),620+ Latitude: mgdoc.SyntheticLatitude(y),621+ Longitude: mgdoc.SyntheticLongitude(x),622+ // The coordinates as the city published them, verbatim:623+ // provenance beside the synthetic position, so no reader624+ // ever has to invert the window to learn where a pin625+ // truly stands.626+ Attrs: map[string]string{627+ semconv.KeyGeoLat: strconv.FormatFloat(lat, 'f', -1, 64),628+ semconv.KeyGeoLon: strconv.FormatFloat(lon, 'f', -1, 64),629+ },630+ })631+ }632+ if total := len(data.Features); total > 0 && float64(strays) > strayTolerance*float64(total) {633+ return nil, fmt.Errorf("%s: %d of %d features fall outside the window", curated.Slug, strays, total)634+ }635+ if total := len(data.Features); total > 0 && untitled == total {636+ return nil, fmt.Errorf("%s: no feature carries a title; is the title field curated right?", curated.Slug)637+ }638+ if _, seen := byGroup[curated.Group]; !seen {639+ order = append(order, curated.Group)640+ }641+ byGroup[curated.Group] = append(byGroup[curated.Group], category)642+ }643+644+ groups := make([]mgdoc.Group, 0, len(order))645+ for _, name := range order {646+ groupID, err := ids.Claim("arcgis:group:" + capture.City + "/" + name)647+ if err != nil {648+ return nil, err649+ }650+ groups = append(groups, mgdoc.Group{ID: groupID, Title: name, Categories: byGroup[name]})651+ }652+ return groups, nil653+}654+655+// zoneLimit is the most zones one dataset may make. The viewer's zone index656+// renders every zone a button and a title; an uncurated explosion -- every657+// parcel a zone -- is refused while the mistake is one table edit old.658+const zoneLimit = 256659+660+// buildRegions folds the polygon and line datasets into zones: every661+// feature whose ZoneOf names a bucket lands its ground in that bucket's662+// region, and line features widen into ribbon polygons first, because a663+// zone is ground and a line has none.664+func buildRegions(capture *Capture, pairs []pairing, ids *mgdoc.IDSpace) ([]mgdoc.Region, error) {665+ regions := []mgdoc.Region{}666+ for _, pair := range pairs {667+ curated, data := pair.curated, pair.data668+ if curated.ZoneOf == nil {669+ continue670+ }671+ buckets := make(map[string]*mgdoc.Region)672+ var order []string673+ for _, feature := range data.Features {674+ key := curated.ZoneOf(feature.Fields)675+ if key.Key == "" {676+ continue677+ }678+ zone, made := buckets[key.Key]679+ if !made {680+ if len(buckets) >= zoneLimit {681+ return nil, fmt.Errorf("%s makes more than %d zones", curated.Slug, zoneLimit)682+ }683+ zoneID, err := ids.Claim("arcgis:zone:" + capture.City + "/" + curated.Slug + "/" + key.Key)684+ if err != nil {685+ return nil, err686+ }687+ zone = &mgdoc.Region{688+ ID: zoneID,689+ Title: scrub(key.Title),690+ Subtitle: scrub(title(key.Subtitle, curated.Title)),691+ }692+ buckets[key.Key] = zone693+ order = append(order, key.Key)694+ }695+ geometry, err := zoneGeometry(capture.Window, curated, feature)696+ if err != nil {697+ return nil, err698+ }699+ if geometry != nil {700+ zone.Features = append(zone.Features, mgdoc.RegionFeature{Geometry: *geometry})701+ }702+ }703+ sort.Strings(order)704+ for _, key := range order {705+ if len(buckets[key].Features) > 0 {706+ regions = append(regions, *buckets[key])707+ }708+ }709+ }710+ return regions, nil711+}712+713+// zoneGeometry lands one feature's ground in the world as a MultiPolygon of714+// synthetic coordinates: rings project position by position, lines widen715+// into one quad per segment.716+func zoneGeometry(window Window, curated *Dataset, feature Feature) (*mgdoc.Geometry, error) {717+ var polygons [][][][]float64718+ switch feature.Geometry.Type {719+ case GeometryRings:720+ for _, polygon := range feature.Geometry.Rings {721+ var rings [][][]float64722+ for _, ring := range polygon {723+ projected := make([][]float64, 0, len(ring))724+ for _, position := range ring {725+ if len(position) < 2 {726+ return nil, fmt.Errorf("%s feature %d has a malformed ring", curated.Slug, feature.ID)727+ }728+ projected = append(projected, synthetic(window, position[0], position[1]))729+ }730+ if len(projected) >= 4 {731+ rings = append(rings, projected)732+ }733+ }734+ if len(rings) > 0 {735+ polygons = append(polygons, rings)736+ }737+ }738+ case GeometryLines:739+ if curated.RibbonWidth <= 0 {740+ return nil, fmt.Errorf("%s zones lines without a ribbon width", curated.Slug)741+ }742+ for _, line := range feature.Geometry.Lines {743+ polygons = append(polygons, ribbon(window, line, curated.RibbonWidth)...)744+ }745+ default:746+ return nil, fmt.Errorf("%s feature %d has no ground to zone", curated.Slug, feature.ID)747+ }748+ if len(polygons) == 0 {749+ return nil, nil750+ }751+ coordinates, err := json.Marshal(polygons)752+ if err != nil {753+ return nil, err754+ }755+ return &mgdoc.Geometry{Type: "MultiPolygon", Coordinates: coordinates}, nil756+}757+758+// synthetic is one position's whole journey: true degrees to world pixel to759+// the synthetic coordinates a zone ring speaks.760+func synthetic(window Window, lon, lat float64) []float64 {761+ x, y := window.WorldPixel(lon, lat)762+ return []float64{mgdoc.SyntheticLongitude(x), mgdoc.SyntheticLatitude(y)}763+}764+765+// ribbon widens a polyline into one closed quad per segment, in world766+// pixels, each spelled as a single-ring polygon of synthetic coordinates.767+// Quads at a joint overlap rather than join; a zone's translucent fill768+// wears that as a slightly darker elbow, which is the price of not writing769+// a polygon clipper for ground that is really a line.770+func ribbon(window Window, line [][]float64, width float64) [][][][]float64 {771+ var quads [][][][]float64772+ half := width / 2773+ for at := 0; at+1 < len(line); at++ {774+ from, to := line[at], line[at+1]775+ if len(from) < 2 || len(to) < 2 {776+ continue777+ }778+ x0, y0 := window.WorldPixel(from[0], from[1])779+ x1, y1 := window.WorldPixel(to[0], to[1])780+ dx, dy := x1-x0, y1-y0781+ length := math.Hypot(dx, dy)782+ if length == 0 {783+ continue784+ }785+ // The unit normal, scaled to half the ribbon.786+ nx, ny := -dy/length*half, dx/length*half787+ ring := [][]float64{788+ syntheticXY(x0+nx, y0+ny),789+ syntheticXY(x1+nx, y1+ny),790+ syntheticXY(x1-nx, y1-ny),791+ syntheticXY(x0-nx, y0-ny),792+ syntheticXY(x0+nx, y0+ny),793+ }794+ quads = append(quads, [][][]float64{ring})795+ }796+ return quads797+}798+799+func syntheticXY(x, y float64) []float64 {800+ return []float64{mgdoc.SyntheticLongitude(x), mgdoc.SyntheticLatitude(y)}801+}802+803+// scrub keeps a live URL out of everything this package emits. Bundles ship804+// offline and the validator refuses any http spelling anywhere in a payload;805+// curated fields should never carry one, and any that slips through loses806+// the token rather than failing the build a capture too late.807+func scrub(text string) string {808+ if !strings.Contains(strings.ToLower(text), "http") {809+ return text810+ }811+ kept := []string{}812+ for token := range strings.FieldsSeq(text) {813+ if strings.Contains(strings.ToLower(token), "http") {814+ continue815+ }816+ kept = append(kept, token)817+ }818+ return strings.Join(kept, " ")819+}820+821+// slugify spells a value the way slugs and icon keys are named.822+func slugify(value string) string {823+ out := make([]rune, 0, len(value))824+ for _, r := range value {825+ switch {826+ case r >= 'a' && r <= 'z' || r >= '0' && r <= '9':827+ out = append(out, r)828+ case r >= 'A' && r <= 'Z':829+ out = append(out, r+'a'-'A')830+ default:831+ if len(out) > 0 && out[len(out)-1] != '-' {832+ out = append(out, '-')833+ }834+ }835+ }836+ for len(out) > 0 && out[len(out)-1] == '-' {837+ out = out[:len(out)-1]838+ }839+ return string(out)840+}841+842+// FieldString spells one GeoJSON property value as capture text: numbers in843+// their shortest exact form, text as itself, absence as emptiness. The844+// crawler runs every kept field through this so a value's JSON spelling845+// cannot wobble the capture's hash.846+func FieldString(value any) string {847+ switch v := value.(type) {848+ case nil:849+ return ""850+ case string:851+ return strings.TrimSpace(v)852+ case float64:853+ return strconv.FormatFloat(v, 'f', -1, 64)854+ case bool:855+ return strconv.FormatBool(v)856+ case json.Number:857+ return v.String()858+ default:859+ return fmt.Sprint(v)860+ }861+}862+863+// Round7 rounds a coordinate to seven decimals -- a centimeter on the864+// ground -- which is where capture determinism and honest precision meet.865+func Round7(value float64) float64 {866+ return math.Round(value*1e7) / 1e7867+}@@ -0,0 +1,433 @@
1+package arcgismap2+3+import (4+ "bytes"5+ "encoding/json"6+ "math"7+ "strconv"8+ "testing"9+10+ "github.com/FelineStateMachine/atlas/internal/mgdoc"11+ "github.com/FelineStateMachine/atlas/internal/semconv"12+)13+14+// The window must be square in Mercator's own terms, or one scale cannot15+// serve both axes.16+func TestCityWindowIsSquare(t *testing.T) {17+ window := CityWindow(bend.BBox)18+ u0, v0 := mercator(window.West, window.North)19+ u1, v1 := mercator(window.East, window.South)20+ du, dv := u1-u0, v1-v021+ if math.Abs(du-dv) > 1e-12 {22+ t.Fatalf("window is not square: du=%v dv=%v", du, dv)23+ }24+ if du <= 0 {25+ t.Fatalf("window has no ground: du=%v", du)26+ }27+ // The curated box sits inside its padded window.28+ if window.West >= bend.BBox[0] || window.East <= bend.BBox[2] {29+ t.Fatalf("window %+v does not contain the bbox longitudes", window)30+ }31+ if window.South >= bend.BBox[1] || window.North <= bend.BBox[3] {32+ t.Fatalf("window %+v does not contain the bbox latitudes", window)33+ }34+}35+36+// A feature's true coordinates, through the window and the synthetic37+// spelling, must land back on the same world pixel the viewer will project38+// them to -- the whole trick, held to a hundredth of a pixel.39+func TestWorldPixelRoundTripsThroughSyntheticCoordinates(t *testing.T) {40+ window := CityWindow(bend.BBox)41+ points := [][2]float64{42+ {-121.30810509, 44.05813637}, // a real historic site43+ {-121.4180, 43.9500}, // the bbox corners44+ {-121.2170, 44.1650},45+ }46+ for _, point := range points {47+ x, y := window.WorldPixel(point[0], point[1])48+ lng := mgdoc.SyntheticLongitude(x)49+ lat := mgdoc.SyntheticLatitude(y)50+ backX := mgdoc.ProjectX(lng, mgdoc.SourceZoom, 0)51+ backY := mgdoc.ProjectY(lat, mgdoc.SourceZoom, 0)52+ if math.Abs(backX-x) > 0.01 || math.Abs(backY-y) > 0.01 {53+ t.Fatalf("pixel (%v,%v) round-trips to (%v,%v)", x, y, backX, backY)54+ }55+ }56+}57+58+// The declared mercator.deg mapping and WorldPixel must be the same59+// transform: a pin's world pixel, inverted through the declared window60+// edges, recovers the published coordinates to well under a meter.61+func TestDeclaredWindowInvertsToTrueCoordinates(t *testing.T) {62+ window := CityWindow(bend.BBox)63+ lon, lat := -121.3153, 44.058264+ x, y := window.WorldPixel(lon, lat)65+ u0, v0 := mercator(window.West, window.North)66+ u1, _ := mercator(window.East, window.South)67+ side := u1 - u068+ backLon, backLat := degrees(u0+x/mgdoc.WorldSize*side, v0+y/mgdoc.WorldSize*side)69+ if math.Abs(backLon-lon) > 1e-9 || math.Abs(backLat-lat) > 1e-9 {70+ t.Fatalf("(%v,%v) inverts to (%v,%v)", lon, lat, backLon, backLat)71+ }72+}73+74+func fixtureCapture() Capture {75+ window := CityWindow(bend.BBox)76+ point := func(lon, lat float64) Geometry {77+ return Geometry{Type: GeometryPoint, Point: []float64{lon, lat}}78+ }79+ square := func(lon, lat, size float64) Geometry {80+ return Geometry{Type: GeometryRings, Rings: [][][][]float64{{{81+ {lon, lat}, {lon + size, lat}, {lon + size, lat + size}, {lon, lat + size}, {lon, lat},82+ }}}}83+ }84+ line := func(positions ...[2]float64) Geometry {85+ one := make([][]float64, 0, len(positions))86+ for _, position := range positions {87+ one = append(one, []float64{position[0], position[1]})88+ }89+ return Geometry{Type: GeometryLines, Lines: [][][]float64{one}}90+ }91+ return Capture{92+ Source: Source,93+ City: "bend-or",94+ Title: "Bend, Oregon",95+ MapSlug: "2026-08-01",96+ Window: window,97+ Basemap: MapConfig{MaxZoom: 3, Extension: "png"},98+ Datasets: []CapturedDataset{99+ {Slug: "historic-sites", Features: []Feature{100+ {ID: 1, Fields: Fields{"NAME": "A.C. Lucas House", "TAB_NAME": "Residential", "SHORT_DESC": "42 NW Hawthorne Ave", "DESC1": "Constructed: 1910"}, Geometry: point(-121.3081, 44.0581)},101+ {ID: 9, Fields: Fields{"NAME": "Drake Park Gazebo", "TAB_NAME": "Civic", "SHORT_DESC": "Drake Park"}, Geometry: point(-121.315, 44.06)},102+ }},103+ {Slug: "zoning", Features: []Feature{104+ {ID: 1, Fields: Fields{"ZONE": "PF"}, Geometry: square(-121.31, 44.05, 0.004)},105+ {ID: 2, Fields: Fields{"ZONE": "RS"}, Geometry: square(-121.30, 44.06, 0.002)},106+ {ID: 3, Fields: Fields{"ZONE": "PF"}, Geometry: square(-121.28, 44.08, 0.003)},107+ }},108+ {Slug: "annexations", Features: []Feature{109+ {ID: 3, Fields: Fields{"DESCRIPT": "Rockridge Park", "EFF_DATE": "1556866800000", "ORDIN_NO": "NS-2327"}, Geometry: square(-121.29, 44.09, 0.002)},110+ }},111+ {Slug: "wetlands", Features: []Feature{112+ {ID: 1, Fields: Fields{"TYPE": "Significant Identified Wetlands", "MAP_CODE": "R9"}, Geometry: square(-121.32, 44.04, 0.003)},113+ }},114+ {Slug: "trails", Features: []Feature{115+ {ID: 6, Fields: Fields{"Trail_Name": "Riley Ranch", "Park": "Riley Ranch Nature Reserve", "Status": "Existing"}, Geometry: line([2]float64{-121.35, 44.11}, [2]float64{-121.345, 44.115})},116+ {ID: 7, Fields: Fields{"Trail_Name": "Planned Spur", "Status": "Proposed"}, Geometry: line([2]float64{-121.33, 44.02}, [2]float64{-121.329, 44.021})},117+ }},118+ {Slug: "mpo-boundary", Features: []Feature{119+ {ID: 21, Fields: Fields{"LABEL": "MPO BOUNDARY"}, Geometry: square(-121.40, 43.97, 0.17)},120+ }},121+ },122+ }123+}124+125+func translated(t *testing.T, capture Capture) mgdoc.Map {126+ t.Helper()127+ raw, err := json.Marshal(&capture)128+ if err != nil {129+ t.Fatal(err)130+ }131+ doc, err := Translate(raw)132+ if err != nil {133+ t.Fatal(err)134+ }135+ var out mgdoc.Map136+ if err := json.Unmarshal(doc, &out); err != nil {137+ t.Fatal(err)138+ }139+ return out140+}141+142+func TestTranslateShapesTheDocument(t *testing.T) {143+ out := translated(t, fixtureCapture())144+145+ if out.Slug != "2026-08-01" || out.Title != "2026-08-01" {146+ t.Fatalf("map is %q titled %q, not the capture day", out.Slug, out.Title)147+ }148+ if out.Game.Slug != "bend-or" || out.Game.Title != "Bend, Oregon" {149+ t.Fatalf("game is %q titled %q", out.Game.Slug, out.Game.Title)150+ }151+152+ if len(out.Config.TileSets) != 1 {153+ t.Fatalf("map has %d tile sets", len(out.Config.TileSets))154+ }155+ set := out.Config.TileSets[0]156+ if set.Path != "bend-or/2026-08-01/basemap" || set.Extension != "png" || set.MaxZoom != 3 {157+ t.Fatalf("tile set is %+v", set)158+ }159+ for zoom := 0; zoom <= set.MaxZoom; zoom++ {160+ bound, declared := set.Bounds[strconv.Itoa(zoom)]161+ if !declared {162+ t.Fatalf("level %d declares no bounds", zoom)163+ }164+ if last := 1<<zoom - 1; bound.X.Max != last || bound.Y.Max != last || bound.X.Min != 0 || bound.Y.Min != 0 {165+ t.Fatalf("level %d bounds are %+v", zoom, bound)166+ }167+ }168+169+ if err := semconv.Validate(semconv.EntityMap, out.Attrs); err != nil {170+ t.Fatalf("map attrs: %v", err)171+ }172+ if out.Attrs[semconv.KeyGeometrySurface] != semconv.SurfacePlane {173+ t.Fatalf("surface is %q", out.Attrs[semconv.KeyGeometrySurface])174+ }175+176+ // One category per point dataset, gathered under curated group titles.177+ if len(out.Groups) != 1 {178+ t.Fatalf("document carries %d groups", len(out.Groups))179+ }180+ group := out.Groups[0]181+ if group.Title != "Heritage" || len(group.Categories) != 1 {182+ t.Fatalf("group is %q with %d categories", group.Title, len(group.Categories))183+ }184+ historic := group.Categories[0]185+ if historic.Title != "Historic Resources" || len(historic.Locations) != 2 {186+ t.Fatalf("category is %q with %d pins", historic.Title, len(historic.Locations))187+ }188+ if err := semconv.Validate(semconv.EntityCategory, historic.Attrs); err != nil {189+ t.Fatalf("category attrs: %v", err)190+ }191+ pin := historic.Locations[0]192+ if pin.Title != "A.C. Lucas House" {193+ t.Fatalf("first pin is %q", pin.Title)194+ }195+ if pin.Description != "Residential · 42 NW Hawthorne Ave · Constructed: 1910" {196+ t.Fatalf("pin card reads %q", pin.Description)197+ }198+ if err := semconv.Validate(semconv.EntityLocation, pin.Attrs); err != nil {199+ t.Fatalf("pin attrs: %v", err)200+ }201+ if pin.Attrs[semconv.KeyGeoLat] != "44.0581" || pin.Attrs[semconv.KeyGeoLon] != "-121.3081" {202+ t.Fatalf("pin keeps %q, %q", pin.Attrs[semconv.KeyGeoLat], pin.Attrs[semconv.KeyGeoLon])203+ }204+205+ // Zones: the boundary as one, zoning bucketed by code, annexations by206+ // decade, wetlands by type, and only the existing named trail -- in207+ // curated dataset order.208+ titles := make([]string, 0, len(out.Regions))209+ for _, region := range out.Regions {210+ titles = append(titles, region.Title)211+ }212+ want := []string{"MPO Boundary", "PF", "RS", "Annexed 2010–2019",213+ "Significant Identified Wetlands", "Riley Ranch"}214+ if len(titles) != len(want) {215+ t.Fatalf("zones are %v", titles)216+ }217+ for at, name := range want {218+ if titles[at] != name {219+ t.Fatalf("zones are %v, want %v", titles, want)220+ }221+ }222+ if out.Regions[1].Subtitle != "Zoning" {223+ t.Fatalf("PF subtitle is %q", out.Regions[1].Subtitle)224+ }225+ if len(out.Regions[1].Features) != 2 {226+ t.Fatalf("PF gathers %d features", len(out.Regions[1].Features))227+ }228+ trail := out.Regions[5]229+ if trail.Subtitle != "Riley Ranch Nature Reserve" || len(trail.Features) != 1 {230+ t.Fatalf("trail zone is %+v", trail)231+ }232+ var ribbonCoords [][][][]float64233+ if err := json.Unmarshal(trail.Features[0].Geometry.Coordinates, &ribbonCoords); err != nil {234+ t.Fatal(err)235+ }236+ if len(ribbonCoords) != 1 || len(ribbonCoords[0][0]) != 5 {237+ t.Fatalf("one segment should ribbon into one closed quad, got %v", ribbonCoords)238+ }239+240+ // Every zone position must project back inside the world square.241+ for _, region := range out.Regions {242+ for _, feature := range region.Features {243+ var polygons [][][][]float64244+ if err := json.Unmarshal(feature.Geometry.Coordinates, &polygons); err != nil {245+ t.Fatal(err)246+ }247+ for _, polygon := range polygons {248+ for _, ring := range polygon {249+ for _, position := range ring {250+ x := mgdoc.ProjectX(position[0], mgdoc.SourceZoom, 0)251+ y := mgdoc.ProjectY(position[1], mgdoc.SourceZoom, 0)252+ if x < -1 || x > mgdoc.WorldSize+1 || y < -1 || y > mgdoc.WorldSize+1 {253+ t.Fatalf("zone %q position projects to (%v,%v)", region.Title, x, y)254+ }255+ }256+ }257+ }258+ }259+ }260+}261+262+// The same data in any order makes byte-identical documents, which is what263+// content-addressed snapshots require.264+func TestTranslateIsDeterministic(t *testing.T) {265+ capture := fixtureCapture()266+ first, err := json.Marshal(&capture)267+ if err != nil {268+ t.Fatal(err)269+ }270+ shuffled := fixtureCapture()271+ for at := range shuffled.Datasets {272+ features := shuffled.Datasets[at].Features273+ for i, j := 0, len(features)-1; i < j; i, j = i+1, j-1 {274+ features[i], features[j] = features[j], features[i]275+ }276+ }277+ shuffled.Datasets[0], shuffled.Datasets[len(shuffled.Datasets)-1] =278+ shuffled.Datasets[len(shuffled.Datasets)-1], shuffled.Datasets[0]279+ shuffled.Normalize()280+ capture.Normalize()281+ second, err := json.Marshal(&capture)282+ if err != nil {283+ t.Fatal(err)284+ }285+ normalizedShuffled, err := json.Marshal(&shuffled)286+ if err != nil {287+ t.Fatal(err)288+ }289+ if !bytes.Equal(second, normalizedShuffled) {290+ t.Fatal("normalized captures differ by input order")291+ }292+ docA, err := Translate(first)293+ if err != nil {294+ t.Fatal(err)295+ }296+ docB, err := Translate(normalizedShuffled)297+ if err != nil {298+ t.Fatal(err)299+ }300+ if !bytes.Equal(docA, docB) {301+ t.Fatal("translation differs by input order")302+ }303+}304+305+func TestTranslateRefusals(t *testing.T) {306+ refuse := func(name string, mutate func(*Capture)) {307+ t.Helper()308+ capture := fixtureCapture()309+ mutate(&capture)310+ raw, err := json.Marshal(&capture)311+ if err != nil {312+ t.Fatal(err)313+ }314+ if _, err := Translate(raw); err == nil {315+ t.Fatalf("%s: translated anyway", name)316+ }317+ }318+ refuse("wrong source", func(c *Capture) { c.Source = "mapgenie" })319+ refuse("uncurated city", func(c *Capture) { c.City = "gotham" })320+ refuse("slug not a day", func(c *Capture) { c.MapSlug = "latest" })321+ refuse("no pyramid", func(c *Capture) { c.Basemap.MaxZoom = 0 })322+ refuse("uncurated dataset", func(c *Capture) { c.Datasets[0].Slug = "parcels" })323+ refuse("dataset doubled", func(c *Capture) { c.Datasets = append(c.Datasets, c.Datasets[0]) })324+ refuse("every pin titleless", func(c *Capture) {325+ for at := range c.Datasets {326+ if c.Datasets[at].Slug == "historic-sites" {327+ for f := range c.Datasets[at].Features {328+ c.Datasets[at].Features[f].Fields["NAME"] = ""329+ }330+ }331+ }332+ })333+ refuse("every pin astray", func(c *Capture) {334+ for at := range c.Datasets {335+ if c.Datasets[at].Slug == "historic-sites" {336+ for f := range c.Datasets[at].Features {337+ c.Datasets[at].Features[f].Geometry.Point = []float64{0, 0}338+ }339+ }340+ }341+ })342+}343+344+// One nameless row is the source's hygiene: the pin is left out and the345+// rest of the dataset stands.346+func TestTranslateSkipsLoneUntitledPin(t *testing.T) {347+ capture := fixtureCapture()348+ for at := range capture.Datasets {349+ if capture.Datasets[at].Slug == "historic-sites" {350+ capture.Datasets[at].Features[0].Fields["NAME"] = ""351+ }352+ }353+ out := translated(t, capture)354+ pins := out.Groups[0].Categories[0].Locations355+ if len(pins) != 1 || pins[0].Title != "Drake Park Gazebo" {356+ t.Fatalf("pins are %+v, want only the titled one", pins)357+ }358+}359+360+func TestScrubDropsLiveURLTokens(t *testing.T) {361+ cases := map[string]string{362+ "plain prose stays": "plain prose stays",363+ "see https://example.com for it": "see for it",364+ "HTTP://LOUD.example": "",365+ "": "",366+ }367+ for in, want := range cases {368+ if got := scrub(in); got != want {369+ t.Fatalf("scrub(%q) = %q, want %q", in, got, want)370+ }371+ }372+}373+374+func TestFieldString(t *testing.T) {375+ cases := []struct {376+ in any377+ want string378+ }{379+ {nil, ""},380+ {" A.C. Lucas House ", "A.C. Lucas House"},381+ {float64(97702), "97702"},382+ {float64(-2208988800000), "-2208988800000"},383+ {0.0202956, "0.0202956"},384+ {true, "true"},385+ }386+ for _, tc := range cases {387+ if got := FieldString(tc.in); got != tc.want {388+ t.Fatalf("FieldString(%v) = %q, want %q", tc.in, got, tc.want)389+ }390+ }391+}392+393+func TestRound7(t *testing.T) {394+ if got := Round7(-121.308105091063071234); got != -121.3081051 {395+ t.Fatalf("Round7 = %v", got)396+ }397+}398+399+// The curated table itself must be internally sound: slugs unique and400+// well-formed, point datasets fully spelled, zone functions only on ground.401+func TestCuratedTableIsSound(t *testing.T) {402+ for slug, city := range Cities {403+ if slug != city.Slug {404+ t.Fatalf("city %q keyed as %q", city.Slug, slug)405+ }406+ if city.MaxZoom < 1 || city.BBox[0] >= city.BBox[2] || city.BBox[1] >= city.BBox[3] {407+ t.Fatalf("city %q declares no ground or no pyramid", slug)408+ }409+ seen := map[string]bool{}410+ for _, dataset := range city.Datasets {411+ if dataset.Slug == "" || seen[dataset.Slug] {412+ t.Fatalf("dataset slug %q is empty or doubled", dataset.Slug)413+ }414+ seen[dataset.Slug] = true415+ if dataset.ItemID == "" || dataset.Title == "" {416+ t.Fatalf("dataset %q is missing its identity", dataset.Slug)417+ }418+ isPoint := dataset.Geometry == "point"419+ if isPoint != (dataset.Group != "") {420+ t.Fatalf("dataset %q: points and only points make pins", dataset.Slug)421+ }422+ if isPoint && (dataset.TitleOf == nil || dataset.Describe == nil) {423+ t.Fatalf("dataset %q makes pins without spelling them", dataset.Slug)424+ }425+ if isPoint && dataset.ZoneOf != nil {426+ t.Fatalf("dataset %q zones points", dataset.Slug)427+ }428+ if dataset.Geometry == "line" && dataset.ZoneOf != nil && dataset.RibbonWidth <= 0 {429+ t.Fatalf("dataset %q zones lines without a ribbon", dataset.Slug)430+ }431+ }432+ }433+}@@ -0,0 +1,4 @@
1+2+<svg width="15" height="15" viewBox="0 0 15 15" xmlns="http://www.w3.org/2000/svg" id="fire-station">3+ <path d="M7.5 14C11.0899 14 14 11 14 7.50003C14 4.5 11.5 2 11.5 2L10.5 5.5L7.5 1L4.5 5.5L3.5 2C3.5 2 1 4.5 1 7.50003C1 11 3.91015 14 7.5 14ZM7.5 12.5C6.11929 12.5 5 11.3807 5 10C5 8.61929 7.5 5.5 7.5 5.5C7.5 5.5 10 8.61929 10 10C10 11.3807 8.88071 12.5 7.5 12.5Z"/>4+</svg>\ No newline at end of file@@ -0,0 +1,4 @@
1+2+<svg version="1.1" id="hospital" xmlns="http://www.w3.org/2000/svg" width="15" height="15" viewBox="0 0 15 15">3+ <path id="rect4194" d="M7,1C6.4,1,6,1.4,6,2v4H2C1.4,6,1,6.4,1,7v1
	c0,0.6,0.4,1,1,1h4v4c0,0.6,0.4,1,1,1h1c0.6,0,1-0.4,1-1V9h4c0.6,0,1-0.4,1-1V7c0-0.6-0.4-1-1-1H9V2c0-0.6-0.4-1-1-1H7z"/>4+</svg>\ No newline at end of file@@ -0,0 +1,4 @@
1+2+<svg width="15" height="15" viewBox="0 0 15 15" xmlns="http://www.w3.org/2000/svg" id="parking">3+ <path d="M4 2V13H6V9H8.5C10.433 9 12 7.433 12 5.5C12 3.567 10.433 2 8.5 2H4ZM6 7V4H8.5C9.32843 4 10 4.67157 10 5.5C10 6.32843 9.32843 7 8.5 7H6Z"/>4+</svg>\ No newline at end of file@@ -0,0 +1,4 @@
1+2+<svg width="15" height="15" viewBox="0 0 15 15" xmlns="http://www.w3.org/2000/svg" id="post">3+ <path d="M13.5 3.65139C13.5 3.86918 13.3912 4.07257 13.2099 4.19338L7.5 8L1.79006 4.19338C1.60885 4.07257 1.5 3.86918 1.5 3.65139C1.5 3.29164 1.79164 3 2.15139 3L12.8486 3C13.2084 3 13.5 3.29164 13.5 3.65139Z M13.5 5.96713V11C13.5 11.5523 13.0523 12 12.5 12H2.5C1.94772 12 1.5 11.5523 1.5 11L1.5 5.96713C1.5 5.76746 1.72254 5.64836 1.88868 5.75912L7.5 9.5L13.1113 5.75912C13.2775 5.64836 13.5 5.76746 13.5 5.96713Z"/>4+</svg>\ No newline at end of file@@ -21,6 +21,11 @@ import (
2121 // thing here. An absent tile is a normal result, not a failure.2222 var errAbsent = errors.New("not published")2323 24+// errAccepted marks a download the server is still staging: ArcGIS Hub's25+// download API answers 202 while it composes a file, and the polite response26+// is to come back shortly, not to fail.27+var errAccepted = errors.New("accepted, still staging")28+2429 // fetcher spaces every request out by a fixed interval and caps how many are2530 // in flight, so a crawl of tens of thousands of tiles stays a light load2631 // rather than a burst. It backs off when the server asks it to.@@ -149,6 +154,8 @@ func (f *fetcher) do(
149154 case response.StatusCode == http.StatusNotFound,150155 response.StatusCode == http.StatusForbidden:151156 return nil, "", fmt.Errorf("%w: %s", errAbsent, url)157+ case response.StatusCode == http.StatusAccepted:158+ return nil, "", fmt.Errorf("%w: %s", errAccepted, url)152159 case response.StatusCode == http.StatusTooManyRequests ||153160 response.StatusCode >= 500:154161 f.penalise(retryAfter(response, time.Duration(attempt+1)*2*time.Second))@@ -0,0 +1,579 @@
1+// The ArcGIS mode fills the archive from a city's open-data hub: the curated2+// municipal datasets a city publishes about itself, fetched as GeoJSON3+// through the hub's download API, and a basemap rendered from that same4+// vector data -- no outside tile service, because the bundle ships offline5+// and the city's own streets are the honest picture of the city.6+//7+// This is the curated-cities importer, and says so. Each city in the8+// arcgismap table names verified dataset identities, field names, and a9+// bounding box; an unknown city is refused at the door rather than captured10+// through unverified curation.11+//12+// Versioning is the mode's whole reason. Each crawl day registers its own13+// map directory, so the archive keeps every day the data moved and the14+// bundle's map picker reads as the city's version history. A re-crawl on a15+// day already captured is a no-op by content addressing; a new day whose16+// data has not moved registers nothing.17+package main18+19+import (20+ "bytes"21+ "context"22+ "crypto/sha256"23+ "encoding/hex"24+ "encoding/json"25+ "errors"26+ "fmt"27+ "hash/fnv"28+ "os"29+ "path/filepath"30+ "sort"31+ "strconv"32+ "strings"33+ "sync"34+ "time"35+36+ "github.com/FelineStateMachine/atlas/internal/arcgismap"37+ "github.com/FelineStateMachine/atlas/internal/basemap"38+)39+40+func runArcgis(ctx context.Context, fetcher *fetcher, o options) error {41+ city, known := arcgismap.Cities[o.arcgis]42+ if !known {43+ cities := make([]string, 0, len(arcgismap.Cities))44+ for name := range arcgismap.Cities {45+ cities = append(cities, name)46+ }47+ sort.Strings(cities)48+ return fmt.Errorf("the ArcGIS importer knows only %s: each city names verified "+49+ "datasets and a verified window, and an unverified one could hang every pin "+50+ "on the wrong pixel", strings.Join(cities, ", "))51+ }52+53+ date := o.captureDate54+ if date == "" {55+ date = time.Now().UTC().Format("2006-01-02")56+ }57+ if _, err := time.Parse("2006-01-02", date); err != nil {58+ return fmt.Errorf("-capture-date %q is not a YYYY-MM-DD day", o.captureDate)59+ }60+61+ maxZoom := city.MaxZoom62+ if o.maxZoom > 0 {63+ maxZoom = o.maxZoom64+ }65+66+ window := arcgismap.CityWindow(city.BBox)67+ capture := &arcgismap.Capture{68+ Source: arcgismap.Source,69+ City: city.Slug,70+ Title: city.Title,71+ MapSlug: date,72+ Window: window,73+ Basemap: arcgismap.MapConfig{MaxZoom: maxZoom, Extension: "png"},74+ }75+76+ fmt.Printf("\n== ArcGIS Hub %s / %s (%d datasets)\n", city.Title, date, len(city.Datasets))77+ pinCount := 078+ for _, dataset := range city.Datasets {79+ features, err := fetchArcgisDataset(ctx, fetcher, city, dataset)80+ if err != nil {81+ return fmt.Errorf("%s: %w", dataset.Slug, err)82+ }83+ if dataset.Geometry == "point" {84+ pinCount += len(features)85+ }86+ fmt.Printf(" %-22s %5d features\n", dataset.Slug, len(features))87+ capture.Datasets = append(capture.Datasets, arcgismap.CapturedDataset{88+ Slug: dataset.Slug,89+ Features: features,90+ })91+ }92+ capture.Normalize()93+ raw, err := json.MarshalIndent(capture, "", " ")94+ if err != nil {95+ return err96+ }97+ // A capture that does not translate would poison the archive for both98+ // tools that read it, so it is refused here, while the problem is one99+ // crawl old rather than buried in history.100+ if _, err := arcgismap.Translate(raw); err != nil {101+ return fmt.Errorf("capture does not translate: %w", err)102+ }103+104+ gameID := arcgisArchiveID("arcgis:" + city.Slug)105+ mapID := arcgisArchiveID("arcgis:" + city.Slug + "/" + date)106+ game := &apiGameFull{ID: gameID, Title: city.Title,107+ Config: apiGameConfig{URL: city.HubBase}}108+ full := &apiMapFull{ID: mapID, Title: date, Slug: date}109+ // The directory says where the capture came from even though the game110+ // keeps its plain title, the same custom every non-MapGenie source keeps.111+ gameDirectory := resolveGameDirectory(o.archive,112+ &apiGameFull{ID: gameID, Title: "ArcGIS Hub " + city.Title})113+114+ // A new day whose data has not moved registers nothing: the version115+ // history means "the city changed", and an unchanged capture would spend116+ // a map saying it had not.117+ if previous, found, err := latestArcgisCapture(o.archive, gameDirectory, date); err != nil {118+ return err119+ } else if found && sameArcgisCapture(previous.capture, capture) {120+ fmt.Printf(" unchanged since %s; no new version registered\n", previous.day)121+ return nil122+ }123+124+ mapDirectory := resolveMapDirectory(o.archive, gameDirectory, full)125+ mapDir := filepath.Join(o.archive, mapDirectory)126+127+ if o.dryRun {128+ fmt.Printf(" would write capture to %s and render z%d (%d tiles)\n",129+ mapDir, maxZoom, 1<<(2*maxZoom))130+ return nil131+ }132+ if err := writeSnapshotRecord(mapDir, raw, arcgismap.Kind, mapID,133+ city.HubBase+"/api/feed/dcat-us/1.1.json"); err != nil {134+ return err135+ }136+137+ index, err := readTileIndex(mapDir)138+ if err != nil {139+ return err140+ }141+ index.mapID = mapID142+ stats, err := renderArcgisBasemap(ctx, o, city, capture, mapDir, index)143+ if err != nil {144+ return err145+ }146+ if err := writeTileIndex(mapDir, index); err != nil {147+ return err148+ }149+ if err := writeMapMetadata(mapDir, game, full, index, pinCount); err != nil {150+ return err151+ }152+ if err := registerMap(o.archive, game, full, gameDirectory, mapDirectory, arcgismap.Source); err != nil {153+ return err154+ }155+ fmt.Printf(" %d rendered · %d held · %.1f MB new\n",156+ stats.fetched, stats.skipped, float64(stats.bytes)/1e6)157+ return nil158+}159+160+// fetchArcgisDataset takes one curated layer through the hub's download API,161+// which stages a complete file server-side -- no record-count paging, no162+// partial captures -- and answers 202 until the file is ready.163+func fetchArcgisDataset(164+ ctx context.Context,165+ fetcher *fetcher,166+ city arcgismap.City,167+ dataset arcgismap.Dataset,168+) ([]arcgismap.Feature, error) {169+ // spatialRefId asks for true degrees by name: without it a hub answers170+ // in the layer's own projection -- state-plane feet, for some cities --171+ // and every coordinate would be an address in the wrong language.172+ url := fmt.Sprintf("%s/api/download/v1/items/%s/geojson?layers=%d&spatialRefId=4326",173+ city.HubBase, dataset.ItemID, dataset.Layer)174+ var body []byte175+ deadline := time.Now().Add(2 * time.Minute)176+ for {177+ var err error178+ body, _, err = fetcher.get(ctx, url)179+ if err == nil {180+ break181+ }182+ if !errors.Is(err, errAccepted) {183+ return nil, err184+ }185+ if time.Now().After(deadline) {186+ return nil, fmt.Errorf("still staging after two minutes: %s", url)187+ }188+ select {189+ case <-ctx.Done():190+ return nil, ctx.Err()191+ case <-time.After(2 * time.Second):192+ }193+ }194+195+ var collection struct {196+ Type string `json:"type"`197+ CRS struct {198+ Properties struct {199+ Name string `json:"name"`200+ } `json:"properties"`201+ } `json:"crs"`202+ Features []struct {203+ Properties map[string]any `json:"properties"`204+ Geometry struct {205+ Type string `json:"type"`206+ Coordinates json.RawMessage `json:"coordinates"`207+ } `json:"geometry"`208+ } `json:"features"`209+ }210+ if err := json.Unmarshal(body, &collection); err != nil {211+ return nil, fmt.Errorf("decode %s: %w", url, err)212+ }213+ if collection.Type != "FeatureCollection" {214+ return nil, fmt.Errorf("%s is a %q, not a FeatureCollection", url, collection.Type)215+ }216+ // GeoJSON without a crs member is WGS84 by definition; one with a crs217+ // must actually say so, or every coordinate is in another language.218+ switch collection.CRS.Properties.Name {219+ case "", "EPSG:4326", "urn:ogc:def:crs:OGC:1.3:CRS84":220+ default:221+ return nil, fmt.Errorf("%s answered in %s, not WGS84", url, collection.CRS.Properties.Name)222+ }223+224+ features := make([]arcgismap.Feature, 0, len(collection.Features))225+ for at, raw := range collection.Features {226+ id, err := arcgisObjectID(raw.Properties)227+ if err != nil {228+ return nil, fmt.Errorf("feature %d: %w", at, err)229+ }230+ fields := arcgismap.Fields{}231+ for _, keep := range dataset.Keep {232+ if value := arcgismap.FieldString(raw.Properties[keep]); value != "" {233+ fields[keep] = value234+ }235+ }236+ geometry, err := arcgisGeometry(dataset.Geometry, raw.Geometry.Type, raw.Geometry.Coordinates)237+ if err != nil {238+ return nil, fmt.Errorf("feature %d: %w", id, err)239+ }240+ if geometry == nil {241+ continue242+ }243+ features = append(features, arcgismap.Feature{ID: id, Fields: fields, Geometry: *geometry})244+ }245+ return features, nil246+}247+248+// arcgisObjectID reads a feature's object identifier under either of the249+// spellings the hub's layers use: OBJECTID nearly everywhere, OID on the250+// occasional legacy layer.251+func arcgisObjectID(properties map[string]any) (int64, error) {252+ value, held := properties["OBJECTID"]253+ if !held {254+ value, held = properties["OID"]255+ }256+ if !held {257+ return 0, errors.New("no OBJECTID")258+ }259+ number, ok := value.(float64)260+ if !ok || number != float64(int64(number)) {261+ return 0, fmt.Errorf("OBJECTID %v is not an integer", value)262+ }263+ return int64(number), nil264+}265+266+// arcgisGeometry normalizes a GeoJSON geometry into the capture's three267+// shapes, held to the kind the curated table promised, coordinates rounded268+// so float spelling cannot masquerade as change. A feature without geometry269+// is dropped rather than refused: hubs hold the occasional geometryless row.270+func arcgisGeometry(expected, kind string, coordinates json.RawMessage) (*arcgismap.Geometry, error) {271+ if kind == "" || len(coordinates) == 0 {272+ return nil, nil273+ }274+ switch expected {275+ case "point":276+ if kind != "Point" {277+ return nil, fmt.Errorf("geometry is %s, not a Point", kind)278+ }279+ var position []float64280+ if err := json.Unmarshal(coordinates, &position); err != nil {281+ return nil, err282+ }283+ if len(position) < 2 {284+ return nil, errors.New("point has no position")285+ }286+ return &arcgismap.Geometry{Type: arcgismap.GeometryPoint,287+ Point: []float64{arcgismap.Round7(position[0]), arcgismap.Round7(position[1])}}, nil288+ case "line":289+ var lines [][][]float64290+ switch kind {291+ case "LineString":292+ var one [][]float64293+ if err := json.Unmarshal(coordinates, &one); err != nil {294+ return nil, err295+ }296+ lines = [][][]float64{one}297+ case "MultiLineString":298+ if err := json.Unmarshal(coordinates, &lines); err != nil {299+ return nil, err300+ }301+ default:302+ return nil, fmt.Errorf("geometry is %s, not a line", kind)303+ }304+ return &arcgismap.Geometry{Type: arcgismap.GeometryLines, Lines: roundLines(lines)}, nil305+ case "polygon":306+ var polygons [][][][]float64307+ switch kind {308+ case "Polygon":309+ var one [][][]float64310+ if err := json.Unmarshal(coordinates, &one); err != nil {311+ return nil, err312+ }313+ polygons = [][][][]float64{one}314+ case "MultiPolygon":315+ if err := json.Unmarshal(coordinates, &polygons); err != nil {316+ return nil, err317+ }318+ default:319+ return nil, fmt.Errorf("geometry is %s, not a polygon", kind)320+ }321+ for _, polygon := range polygons {322+ for at, ring := range polygon {323+ polygon[at] = roundPositions(ring)324+ }325+ }326+ return &arcgismap.Geometry{Type: arcgismap.GeometryRings, Rings: polygons}, nil327+ }328+ return nil, fmt.Errorf("curated geometry %q is unknown", expected)329+}330+331+func roundLines(lines [][][]float64) [][][]float64 {332+ for at, line := range lines {333+ lines[at] = roundPositions(line)334+ }335+ return lines336+}337+338+func roundPositions(positions [][]float64) [][]float64 {339+ for _, position := range positions {340+ for at, value := range position {341+ position[at] = arcgismap.Round7(value)342+ }343+ }344+ return positions345+}346+347+// arcgisSnapshot is the slice of game.json this mode reads back: which dated348+// maps the game already carries, and where they live.349+type previousCapture struct {350+ day string351+ capture *arcgismap.Capture352+}353+354+// latestArcgisCapture loads the newest capture from a day before this one,355+// so an unchanged city can decline to spend a version.356+func latestArcgisCapture(archiveRoot, gameDirectory, before string) (previousCapture, bool, error) {357+ var gameFile struct {358+ Maps []struct {359+ Directory string `json:"directory"`360+ Slug string `json:"slug"`361+ } `json:"maps"`362+ }363+ err := readJSON(filepath.Join(archiveRoot, gameDirectory, "game.json"), &gameFile)364+ if errors.Is(err, os.ErrNotExist) {365+ return previousCapture{}, false, nil366+ }367+ if err != nil {368+ return previousCapture{}, false, err369+ }370+ newest := ""371+ directory := ""372+ for _, entry := range gameFile.Maps {373+ if len(entry.Slug) == len("2006-01-02") && entry.Slug < before && entry.Slug > newest {374+ newest, directory = entry.Slug, entry.Directory375+ }376+ }377+ if newest == "" {378+ return previousCapture{}, false, nil379+ }380+381+ mapDir := filepath.Join(archiveRoot, directory)382+ var snapshots []snapshotRecord383+ if err := readJSON(filepath.Join(mapDir, "snapshots", "index.json"), &snapshots); err != nil {384+ return previousCapture{}, false, err385+ }386+ if len(snapshots) == 0 {387+ return previousCapture{}, false, nil388+ }389+ sort.Slice(snapshots, func(a, b int) bool {390+ return snapshots[a].CapturedAt < snapshots[b].CapturedAt391+ })392+ latest := snapshots[len(snapshots)-1]393+ raw, err := os.ReadFile(filepath.Join(mapDir, "snapshots", "map", latest.ContentHash+".json"))394+ if err != nil {395+ return previousCapture{}, false, err396+ }397+ var capture arcgismap.Capture398+ if err := json.Unmarshal(raw, &capture); err != nil {399+ return previousCapture{}, false, err400+ }401+ return previousCapture{day: newest, capture: &capture}, true, nil402+}403+404+// sameArcgisCapture asks whether two captures say the same thing about the405+// city, the day itself aside.406+func sameArcgisCapture(previous, current *arcgismap.Capture) bool {407+ a, b := *previous, *current408+ a.MapSlug, b.MapSlug = "", ""409+ rawA, errA := json.Marshal(&a)410+ rawB, errB := json.Marshal(&b)411+ return errA == nil && errB == nil && bytes.Equal(rawA, rawB)412+}413+414+// renderArcgisBasemap draws the deepest level of the city's own ground into415+// the archive, every tile of it, exactly as a fetching source would have416+// recorded downloaded ones: tools/tiles folds the rest of the pyramid down417+// from this level and never knows the wire was absent. Tiles whose bytes418+// already sit in the index are held, not rewritten, so a re-render without419+// change leaves the archive's working tree untouched.420+func renderArcgisBasemap(421+ ctx context.Context,422+ o options,423+ city arcgismap.City,424+ capture *arcgismap.Capture,425+ mapDir string,426+ index *tileIndex,427+) (captureStats, error) {428+ var stats captureStats429+ features := basemapFeatures(city, capture)430+ renderer := basemap.NewRenderer(features, capture.Basemap.MaxZoom)431+ scope := arcgismap.TileSetPath(city.Slug, capture.MapSlug)432+ setID := index.tileSetID(scope)433+ template := arcgismap.TileTemplate(city.Slug, capture.MapSlug)434+ zoom := capture.Basemap.MaxZoom435+ edge := 1 << zoom436+ fmt.Printf(" basemap %q z%d, %d tiles from %d features\n",437+ scope, zoom, edge*edge, len(features))438+439+ var mu sync.Mutex440+ var group sync.WaitGroup441+ var failure error442+ gate := make(chan struct{}, o.concurrency)443+ for y := range edge {444+ for x := range edge {445+ select {446+ case <-ctx.Done():447+ group.Wait()448+ return stats, ctx.Err()449+ default:450+ }451+ group.Add(1)452+ gate <- struct{}{}453+ go func(x, y int) {454+ defer group.Done()455+ defer func() { <-gate }()456+ url := strings.NewReplacer(457+ "{z}", strconv.Itoa(zoom), "{x}", strconv.Itoa(x), "{y}", strconv.Itoa(y),458+ ).Replace(template)459+ path := filepath.Join(mapDir, "tiles",460+ "set-"+strconv.FormatInt(setID, 10),461+ strconv.Itoa(zoom), strconv.Itoa(x), strconv.Itoa(y)+".png")462+463+ body, err := basemap.EncodePNG(renderer.Tile(zoom, x, y))464+ if err != nil {465+ mu.Lock()466+ if failure == nil {467+ failure = fmt.Errorf("render %d/%d/%d: %w", zoom, x, y, err)468+ }469+ mu.Unlock()470+ return471+ }472+ sum := sha256.Sum256(body)473+ hash := hex.EncodeToString(sum[:])474+475+ mu.Lock()476+ record, held := index.byURL[url]477+ mu.Unlock()478+ if held && record.Status == "cached" && record.ContentHash == hash {479+ if _, err := os.Stat(path); err == nil {480+ mu.Lock()481+ stats.skipped++482+ mu.Unlock()483+ return484+ }485+ }486+ if err := writeFile(path, body); err != nil {487+ mu.Lock()488+ if failure == nil {489+ failure = err490+ }491+ mu.Unlock()492+ return493+ }494+ mu.Lock()495+ index.put(tileRecord{496+ URL: url, Status: "cached", TileSetID: setID,497+ Zoom: zoom, X: x, Y: y,498+ ContentHash: hash, ContentType: "image/png",499+ ByteLength: len(body),500+ CoverageKey: fmt.Sprintf("%d:%d:%d:%d:%d",501+ index.mapID, setID, zoom, x, y),502+ })503+ stats.fetched++504+ stats.bytes += int64(len(body))505+ mu.Unlock()506+ }(x, y)507+ }508+ }509+ group.Wait()510+ return stats, failure511+}512+513+// basemapFeatures projects every drawing dataset into world pixels under the514+// capture's own window, so the raster and the pins can never disagree about515+// where the ground is.516+func basemapFeatures(city arcgismap.City, capture *arcgismap.Capture) []basemap.Feature {517+ curated := make(map[string]*arcgismap.Dataset, len(city.Datasets))518+ for at := range city.Datasets {519+ curated[city.Datasets[at].Slug] = &city.Datasets[at]520+ }521+ var features []basemap.Feature522+ for _, dataset := range capture.Datasets {523+ table := curated[dataset.Slug]524+ if table == nil || table.Role == "" {525+ continue526+ }527+ role := basemap.Role(table.Role)528+ for _, feature := range dataset.Features {529+ emphasis := 0.0530+ if table.Emphasis != nil {531+ emphasis = table.Emphasis(feature.Fields)532+ }533+ switch feature.Geometry.Type {534+ case arcgismap.GeometryRings:535+ // Each polygon draws on its own, so winding normalization536+ // reads every first ring as ground and the rest as holes.537+ for _, polygon := range feature.Geometry.Rings {538+ drawn := basemap.Feature{Role: role, Emphasis: emphasis}539+ for _, ring := range polygon {540+ drawn.Rings = append(drawn.Rings, projectPositions(capture.Window, ring))541+ }542+ if len(drawn.Rings) > 0 {543+ features = append(features, drawn)544+ }545+ }546+ case arcgismap.GeometryLines:547+ drawn := basemap.Feature{Role: role, Emphasis: emphasis}548+ for _, line := range feature.Geometry.Lines {549+ drawn.Lines = append(drawn.Lines, projectPositions(capture.Window, line))550+ }551+ if len(drawn.Lines) > 0 {552+ features = append(features, drawn)553+ }554+ }555+ }556+ }557+ return features558+}559+560+func projectPositions(window arcgismap.Window, positions [][]float64) [][2]float64 {561+ out := make([][2]float64, 0, len(positions))562+ for _, position := range positions {563+ if len(position) < 2 {564+ continue565+ }566+ x, y := window.WorldPixel(position[0], position[1])567+ out = append(out, [2]float64{x, y})568+ }569+ return out570+}571+572+// arcgisArchiveID numbers ArcGIS cities and maps in the archive's registers,573+// one bit above Trek's so every source's ids stay apart; see ignArchiveID574+// for why the range stays far below a float64's integers.575+func arcgisArchiveID(seed string) int64 {576+ hash := fnv.New32a()577+ _, _ = hash.Write([]byte(seed))578+ return int64(hash.Sum32()&0x7fffffff) | 1<<35579+}@@ -65,6 +65,8 @@ type options struct {
6565 ign string6666 piggyback string6767 trek string68+ arcgis string69+ captureDate string6870 maxZoom int6971 list bool7072 dryRun bool@@ -80,6 +82,8 @@ func parseOptions() options {
8082 flag.StringVar(&o.ign, "ign", "", "IGN wikimap as objectSlug/mapSlug, e.g. cyberpunk-2077/night-city")8183 flag.StringVar(&o.piggyback, "piggyback", "", "Piggyback map as gameSlug/mapSlug, e.g. cyberpunk-2077/night-city")8284 flag.StringVar(&o.trek, "trek", "", "planetary body from NASA Trek and the IAU Gazetteer, e.g. mars")85+ flag.StringVar(&o.arcgis, "arcgis", "", "city from an ArcGIS Hub open-data site, e.g. bend-or")86+ flag.StringVar(&o.captureDate, "capture-date", "", "the day an -arcgis capture answers to, YYYY-MM-DD (default: today)")8387 flag.IntVar(&o.maxZoom, "max-zoom", 0, "deepest zoom to capture (default: the layer's own maximum)")8488 flag.BoolVar(&o.list, "list", false, "list games, or the maps of -game, and exit")8589 flag.BoolVar(&o.dryRun, "dry-run", false, "report what would be fetched without writing anything")@@ -105,6 +109,9 @@ func run(o options) error {
105109 if o.trek != "" {106110 return runTrek(ctx, fetcher, o)107111 }112+ if o.arcgis != "" {113+ return runArcgis(ctx, fetcher, o)114+ }108115 109116 games, err := listGames(ctx, fetcher)110117 if err != nil {@@ -165,6 +165,29 @@ func TestIGNArchiveIDStaysApartFromMapGenie(t *testing.T) {
165165 }166166 }167167 168+func TestArcgisArchiveIDStaysApartFromEveryOtherSource(t *testing.T) {169+ id := arcgisArchiveID("arcgis:bend-or")170+ if id <= 1<<35 {171+ t.Fatalf("id %d could collide with another source's ids", id)172+ }173+ if id >= 1<<53 {174+ t.Fatalf("id %d would lose integers read back through a float64", id)175+ }176+ if again := arcgisArchiveID("arcgis:bend-or"); again != id {177+ t.Fatalf("id is unstable: %d then %d", id, again)178+ }179+}180+181+func TestArcgisTileTemplateRecoversItsTileSetPath(t *testing.T) {182+ // The rendered basemap's record URLs must group under the same layer183+ // path the translator declares, through the same marker every archive184+ // reader cuts on.185+ url := "arcgis-basemap://tiles/bend-or/2026-08-01/basemap/6/12/40.png"186+ if got := tileSetPathOf(url, 6); got != "bend-or/2026-08-01/basemap" {187+ t.Fatalf("tileSetPathOf = %q", got)188+ }189+}190+168191 func TestTileIndexReusesExistingTileSetDirectory(t *testing.T) {169192 // Re-crawling an archive the extension captured must not orphan its files.170193 index := newTileIndex(203)@@ -16,6 +16,7 @@ import (
1616 "strconv"1717 "strings"1818 19+ "github.com/FelineStateMachine/atlas/internal/arcgismap"1920 "github.com/FelineStateMachine/atlas/internal/bundle"2021 "github.com/FelineStateMachine/atlas/internal/icons"2122 "github.com/FelineStateMachine/atlas/internal/ignmap"@@ -318,6 +319,14 @@ var preferredMapOrder = map[string][]string{
318319 "fallout-new-vegas": {"mojave-wasteland"},319320 }320321 322+// newestFirstMaps marks the games whose maps are dated captures of one323+// ground: a version history. The picker's first entry is the map the viewer324+// opens, and a version history should open on the present, so these games325+// sort their date-titled maps newest first.326+var newestFirstMaps = map[string]bool{327+ "bend-or": true,328+}329+321330 // The raster beneath an icon decides which outline is legible, so this is322331 // declared rather than derived. A game whose maps are all drawn the same way323332 // says so once; a single map that differs from its game overrides it.@@ -583,6 +592,12 @@ func sortGameMaps(gameSlug string, maps []catalogMap) {
583592 for index, slug := range preferredMapOrder[gameSlug] {584593 order[slug] = index585594 }595+ // A version-history game reads its date titles backward: the newest596+ // capture opens first, and the past waits one click below it.597+ after := func(a, b string) bool { return a < b }598+ if newestFirstMaps[gameSlug] {599+ after = func(a, b string) bool { return a > b }600+ }586601 titles := make(map[string]string, len(maps))587602 for _, m := range maps {588603 titles[m.Slug] = m.Title@@ -607,12 +622,12 @@ func sortGameMaps(gameSlug string, maps []catalogMap) {
607622 return left < right608623 }609624 if leftTitle != rightTitle {610- return leftTitle < rightTitle625+ return after(leftTitle, rightTitle)611626 }612627 if (maps[i].Parent == "") != (maps[j].Parent == "") {613628 return maps[i].Parent == ""614629 }615- return maps[i].Title < maps[j].Title630+ return after(maps[i].Title, maps[j].Title)616631 })617632 }618633 @@ -653,6 +668,9 @@ func buildMap(
653668 if doc, err = trekmap.MaybeTranslate(latest.Kind, doc); err != nil {654669 return nil, "", err655670 }671+ if doc, err = arcgismap.MaybeTranslate(latest.Kind, doc); err != nil {672+ return nil, "", err673+ }656674 var raw rawMap657675 if err := json.Unmarshal(doc, &raw); err != nil {658676 return nil, "", err@@ -139,6 +139,8 @@ func sourceDisplayLabel(source string) string {
139139 return "Piggyback"140140 case "nasa-trek":141141 return "NASA Trek"142+ case "arcgis-hub":143+ return "ArcGIS Open Data"142144 }143145 return source144146 }@@ -26,6 +26,7 @@ import (
2626 // none, so levels derived from them are written as JPEG or PNG.2727 _ "golang.org/x/image/webp"2828 29+ "github.com/FelineStateMachine/atlas/internal/arcgismap"2930 "github.com/FelineStateMachine/atlas/internal/blend"3031 "github.com/FelineStateMachine/atlas/internal/ignmap"3132 "github.com/FelineStateMachine/atlas/internal/mgdoc"@@ -547,6 +548,9 @@ func inspectMap(mapDir string) ([]tilePlan, string, string, error) {
547548 if doc, err = trekmap.MaybeTranslate(latest.Kind, doc); err != nil {548549 return nil, "", "", err549550 }551+ if doc, err = arcgismap.MaybeTranslate(latest.Kind, doc); err != nil {552+ return nil, "", "", err553+ }550554 var raw rawMap551555 if err := json.Unmarshal(doc, &raw); err != nil {552556 return nil, "", "", err