1// Package arcgismap reads a captured city and speaks MapGenie for it.2//3// The capture comes from a city's ArcGIS Hub open-data site: the municipal4// datasets a city publishes about itself -- its limits, its zoning, its5// trails, its fire stations -- fetched as GeoJSON and kept verbatim under a6// curated allowlist of fields. Like ignmap, pbmap and trekmap, translation7// happens when a capture is read, not when it is taken: the archive keeps8// what the city published, and every keep-or-leave decision lives here,9// where changing it re-applies to captures already on disk.10//11// The semantics are the point being proven. A city is a volume to the12// pipeline; each crawl day registers its own world, so a bundle's picker13// becomes the city's version history -- four captured days are four worlds,14// and the differences between them are differences in the city. Polygon15// datasets ride the bundle's zones, point datasets its pins, and the one16// raster layer is a basemap rendered from the city's own vector data,17// because a bundle owes its reader ground to stand on and ships offline.18//19// The coordinate design follows trekmap, on a plane. A city sits off the20// Web-Mercator diagonal, and the viewer's window is a square upon it, so the21// city's bounding box -- padded, and squared in Mercator's own terms --22// becomes the 8192-pixel world. A feature's true coordinates land on that23// square through the Mercator forward, and the pixel becomes synthetic24// latitude and longitude through the same inverse every translated source25// uses. The window is declared on the map as the mercator.px/deg mapping,26// and every pin keeps its true coordinates in atlas.geo.*, so nothing is27// flattened away.28package arcgismap2930import (31 "encoding/json"32 "fmt"33 "math"34 "regexp"35 "sort"36 "strconv"37 "strings"38 "time"3940 "github.com/FelineStateMachine/atlas/internal/mgdoc"41 "github.com/FelineStateMachine/atlas/internal/semconv"42)4344// Kind names ArcGIS Hub captures in a map's snapshot index.45const Kind = "arcgis-map"4647// Source is the value a well-formed capture carries in its source field.48const Source = "arcgis-hub"4950// Fields is one feature's kept attributes, every value spelled as text.51type Fields map[string]string5253// ZoneKey says which zone a feature folds into: the bucket's stable key and54// the title and subtitle the zone wears. A zero key is a feature that makes55// no zone.56type ZoneKey struct {57 Key string58 Title string59 Subtitle string60}6162// Dataset is one curated layer of a city's hub: where to fetch it, which63// fields to keep, and what it becomes in the bundle -- pins when Group is64// set, zones when ZoneOf is, basemap linework when Role is. The functions65// read a feature's kept fields, so re-curating presentation is a table edit66// and a re-translation of captures already on disk.67type Dataset struct {68 Slug string69 Title string70 ItemID string71 Layer int72 Geometry string // "point" | "line" | "polygon"73 Keep []string7475 // Pins.76 Group string77 Icon string78 TitleOf func(Fields) string79 Describe func(Fields) string8081 // Zones. StrokeWidth is the ground width a line dataset's zones are82 // drawn and clicked at, in world pixels of the 8192 square: the path83 // stays the line it is, and the viewer strokes it. Label is the label84 // policy the dataset's collection declares: "quiet" keeps the zones'85 // names off the map until something highlights them, and empty means86 // the format's default -- always, for areas.87 ZoneOf func(Fields) ZoneKey88 StrokeWidth float6489 Label string9091 // Basemap. Emphasis scales a stroke per feature -- an arterial wider92 // than a lane -- and absent means every feature draws alike.93 Role string94 Emphasis func(Fields) float649596 // National enrichment. A dataset with Server set is fetched from a97 // plain ArcGIS MapServer query endpoint by the city's padded window98 // rather than from the city's hub, Where narrowing the rows, and its99 // geometry is cut to the window at capture time. The first kept field100 // names the layer's row identity and orders the pages; IDOf derives101 // the capture's stable feature identifier from the kept fields when102 // the layer's own object ids churn between upstream refreshes.103 Server string104 Where string105 IDOf func(Fields) (int64, bool)106}107108// City is one curated hub. The table is the whole authority: an uncurated109// city is refused at the door, because an unverified bounding box would hang110// every pin on the wrong pixel, and unverified field names would title every111// zone with nothing.112type City struct {113 Slug string114 Title string115 HubBase string116 MaxZoom int117 BBox [4]float64 // west, south, east, north, degrees118 Datasets []Dataset119}120121// AllDatasets is the city's own curation followed by the national122// enrichment every US city receives by bounding box, in that order, so a123// city's zones keep their places and the nation's sort after them.124func (c City) AllDatasets() []Dataset {125 return append(append(make([]Dataset, 0, len(c.Datasets)+len(National)), c.Datasets...), National...)126}127128// Cities is every city the source knows how to capture. A city that should129// stay out of the repository -- your own, say -- can be registered from a130// cities_local.go beside this file: that name is ignored by git, and an131// init function there adding to this map curates privately what this table132// curates publicly.133var Cities = map[string]City{134 "bend-or": bend,135 "redondo-beach-ca": redondoBeach,136}137138// redondoBeach is the City of Redondo Beach, California -- the second139// proof city. Its land-use layer is parcel-grained and carries the zoning140// code on every polygon, so the zoning zones bucket by code and double as141// the basemap's parcel texture, the first use of that role. Field names142// were verified against the live services.143var redondoBeach = City{144 Slug: "redondo-beach-ca",145 Title: "Redondo Beach, California",146 HubBase: "https://city-of-redondo-beach-gis-redondobeachgis.hub.arcgis.com",147 MaxZoom: 6,148 BBox: [4]float64{-118.4060, 33.8110, -118.3490, 33.8985},149 Datasets: []Dataset{150 {151 Slug: "land-use", Title: "Zoning",152 ItemID: "1d185d3b4bfd43eb9d78ff6c36d4062b", Layer: 3,153 Geometry: "polygon",154 Keep: []string{"OBJECTID", "ZONINGCODE", "LANDUSECODE", "GENERALPLANCODE", "ZONEOVERLAY"},155 ZoneOf: func(f Fields) ZoneKey {156 code := strings.TrimSpace(f["ZONINGCODE"])157 if code == "" {158 return ZoneKey{}159 }160 return ZoneKey{Key: slugify(code), Title: code}161 },162 Role: "parcel",163 },164 {165 Slug: "general-plan", Title: "General Plan",166 ItemID: "1d185d3b4bfd43eb9d78ff6c36d4062b", Layer: 6,167 Geometry: "polygon",168 Keep: []string{"OBJECTID", "GENERALPLAN", "ZONECLASS", "ZONEOVERLAY"},169 ZoneOf: func(f Fields) ZoneKey {170 code := strings.TrimSpace(f["GENERALPLAN"])171 if code == "" {172 return ZoneKey{}173 }174 return ZoneKey{Key: slugify(code), Title: code, Subtitle: "General Plan"}175 },176 },177 {178 Slug: "council-districts", Title: "Council Districts",179 ItemID: "51e571c695ca451999bd7c53097f7bd6", Layer: 0,180 Geometry: "polygon",181 Keep: []string{"OBJECTID", "District", "CouncilMember"},182 ZoneOf: func(f Fields) ZoneKey {183 number := strings.TrimSpace(f["District"])184 if number == "" {185 return ZoneKey{}186 }187 return ZoneKey{Key: "district-" + slugify(number),188 Title: "Council District " + number,189 Subtitle: strings.TrimSpace(f["CouncilMember"])}190 },191 },192 {193 Slug: "district-lines", Title: "Council District Boundaries",194 ItemID: "12d292cd5f4c49838c0c812427ec7f2b", Layer: 0,195 Geometry: "line",196 Keep: []string{"OBJECTID"},197 Role: "boundary",198 },199 {200 Slug: "parks", Title: "Parks & Recreation",201 ItemID: "6bd956cbe6134485a85a20b225a0e5f4", Layer: 1,202 Geometry: "point",203 Keep: []string{"OBJECTID", "name", "address", "facility_size", "web_desc"},204 Group: "Recreation",205 Icon: "maki/park",206 TitleOf: func(f Fields) string { return f["name"] },207 Describe: func(f Fields) string {208 return join(" · ", f["address"], f["facility_size"], f["web_desc"])209 },210 },211 },212}213214// bend is the City of Bend, Oregon -- the curated proof city, chosen for a215// hub surface with every shape the format wants to prove: polygon ground,216// named line work, and titled points, all behind the standard hub download217// API. Field names were verified against the live services. The water218// utility layers stay out deliberately: their attribute tables carry staff219// account names, and an open capture keeps people out of it.220var bend = City{221 Slug: "bend-or",222 Title: "Bend, Oregon",223 HubBase: "https://data.bendoregon.gov",224 MaxZoom: 6,225 BBox: [4]float64{-121.4180, 43.9500, -121.2170, 44.1650},226 Datasets: []Dataset{227 {228 Slug: "mpo-boundary", Title: "MPO Boundary",229 ItemID: "ec95698d5b7f494eb12dba2c572ed6d1", Layer: 0,230 Geometry: "polygon",231 Keep: []string{"OID", "LABEL", "Acres"},232 ZoneOf: func(f Fields) ZoneKey {233 return ZoneKey{Key: "mpo-boundary", Title: "MPO Boundary",234 Subtitle: "Metropolitan planning area"}235 },236 Role: "boundary",237 },238 {239 Slug: "zoning", Title: "Zoning",240 ItemID: "90f0e9717185404581b2c1c865c13f03", Layer: 0,241 Geometry: "polygon",242 Keep: []string{"OBJECTID", "ZONE", "ORDINANCE"},243 ZoneOf: func(f Fields) ZoneKey {244 code := f["ZONE"]245 if code == "" {246 return ZoneKey{}247 }248 return ZoneKey{Key: slugify(code), Title: code}249 },250 },251 {252 Slug: "annexations", Title: "Annexations",253 ItemID: "7aec052cc38645f69d2a74e8062abd95", Layer: 0,254 Geometry: "polygon",255 Keep: []string{"OBJECTID", "DESCRIPT", "EFF_DATE", "ORDIN_NO", "ANNX_ACRES"},256 // Annexations fold into the decades they took effect in: the257 // story a version history wants to tell is how the city grew,258 // not which ordinance grew it.259 ZoneOf: func(f Fields) ZoneKey {260 ms, err := strconv.ParseInt(f["EFF_DATE"], 10, 64)261 if err != nil || ms == 0 {262 return ZoneKey{Key: "undated", Title: "Annexed, date unknown"}263 }264 decade := time.UnixMilli(ms).UTC().Year() / 10 * 10265 return ZoneKey{266 Key: strconv.Itoa(decade) + "s",267 Title: fmt.Sprintf("Annexed %d–%d", decade, decade+9),268 }269 },270 },271 {272 Slug: "wetlands", Title: "Wetlands",273 ItemID: "8c9ded5d2a32423f8b380de1bec50cf4", Layer: 0,274 Geometry: "polygon",275 Keep: []string{"OBJECTID", "TYPE", "MAP_CODE", "ACRES"},276 ZoneOf: func(f Fields) ZoneKey {277 kind := f["TYPE"]278 if kind == "" {279 return ZoneKey{}280 }281 return ZoneKey{Key: slugify(kind), Title: kind, Subtitle: "Local Wetlands Inventory"}282 },283 Role: "water",284 },285 {286 Slug: "trails", Title: "Paths & Trails",287 ItemID: "8d006fb5453c4dbfad476033e847eca8", Layer: 0,288 Geometry: "line",289 Keep: []string{"OBJECTID", "Trail_Name", "Park", "Status", "Surface_Ma", "Classifica", "Maintained"},290 // Every existing segment draws into the basemap; the named291 // trails earn zones, each one clickable end to end.292 ZoneOf: func(f Fields) ZoneKey {293 if f["Status"] != "Existing" || f["Trail_Name"] == "" {294 return ZoneKey{}295 }296 return ZoneKey{Key: slugify(f["Trail_Name"]), Title: f["Trail_Name"],297 Subtitle: title(f["Park"], "Trail")}298 },299 StrokeWidth: 12,300 Role: "trail",301 },302 {303 Slug: "road-centerlines", Title: "Road Centerlines",304 ItemID: "c303eb99c9a242149b95d5055589e251", Layer: 0,305 Geometry: "line",306 Keep: []string{"OBJECTID", "fullname", "roadclass", "onewaydir", "speedlimit"},307 Role: "street",308 Emphasis: func(f Fields) float64 {309 switch f["roadclass"] {310 case "Local":311 return 1312 case "Collector":313 return 1.4314 default: // arterials and everything grander315 return 1.8316 }317 },318 },319 {320 Slug: "historic-sites", Title: "Historic Resources",321 ItemID: "70b88002805f42dfa94d0c470d49eec1", Layer: 0,322 Geometry: "point",323 Keep: []string{"OBJECTID", "NAME", "TAB_NAME", "SHORT_DESC", "DESC1", "DESC2", "DESC3", "DESC4", "DESC5"},324 Group: "Heritage",325 Icon: "maki/monument",326 TitleOf: func(f Fields) string { return f["NAME"] },327 Describe: func(f Fields) string {328 return join(" · ", f["TAB_NAME"], f["SHORT_DESC"],329 f["DESC1"], f["DESC2"], f["DESC3"], f["DESC4"], f["DESC5"])330 },331 },332 },333}334335// join glues the non-empty pieces with the separator, which is most of what336// composing a card from optional fields is.337func join(sep string, pieces ...string) string {338 kept := pieces[:0:0]339 for _, piece := range pieces {340 if trimmed := strings.TrimSpace(piece); trimmed != "" {341 kept = append(kept, trimmed)342 }343 }344 return strings.Join(kept, sep)345}346347// Window is the ground the city's world square pictures: the degrees at the348// padded square's edges. The square is square in Mercator's own terms -- u349// linear in longitude, v in projected latitude -- which is what lets one350// number serve as both axes' scale.351type Window struct {352 West float64 `json:"west"`353 North float64 `json:"north"`354 East float64 `json:"east"`355 South float64 `json:"south"`356}357358// mercator lands degrees on the global Web-Mercator unit square.359func mercator(lon, lat float64) (u, v float64) {360 u = (lon + 180) / 360361 v = (1 - math.Asinh(math.Tan(lat*math.Pi/180))/math.Pi) / 2362 return u, v363}364365// degrees is the way back.366func degrees(u, v float64) (lon, lat float64) {367 lon = u*360 - 180368 lat = math.Atan(math.Sinh(math.Pi*(1-2*v))) * 180 / math.Pi369 return lon, lat370}371372// CityWindow pads a city's bounding box five percent per side and squares it373// in Mercator terms, growing the shorter axis around its own middle. The374// same box always makes the same window, which is what keeps every capture375// of a city in one world.376func CityWindow(bbox [4]float64) Window {377 u0, v0 := mercator(bbox[0], bbox[3]) // west, north: the top-left378 u1, v1 := mercator(bbox[2], bbox[1]) // east, south: the bottom-right379 padU, padV := (u1-u0)*0.05, (v1-v0)*0.05380 u0, u1 = u0-padU, u1+padU381 v0, v1 = v0-padV, v1+padV382 side := math.Max(u1-u0, v1-v0)383 uMid, vMid := (u0+u1)/2, (v0+v1)/2384 u0, u1 = uMid-side/2, uMid+side/2385 v0, v1 = vMid-side/2, vMid+side/2386 west, north := degrees(u0, v0)387 east, south := degrees(u1, v1)388 return Window{West: west, North: north, East: east, South: south}389}390391// WorldPixel lands true coordinates on the world square.392func (w Window) WorldPixel(lon, lat float64) (x, y float64) {393 u0, v0 := mercator(w.West, w.North)394 u1, _ := mercator(w.East, w.South)395 side := u1 - u0396 u, v := mercator(lon, lat)397 return (u - u0) / side * mgdoc.WorldSize, (v - v0) / side * mgdoc.WorldSize398}399400// PxDeg spells the window as the two map attributes that declare it.401func (w Window) PxDeg() (px, deg string) {402 px = fmt.Sprintf("0,0,%d,%d", mgdoc.WorldSize, mgdoc.WorldSize)403 parts := []string{404 strconv.FormatFloat(w.West, 'f', -1, 64),405 strconv.FormatFloat(w.North, 'f', -1, 64),406 strconv.FormatFloat(w.East, 'f', -1, 64),407 strconv.FormatFloat(w.South, 'f', -1, 64),408 }409 return px, strings.Join(parts, ",")410}411412// TileSetPath is the layer path the basemap's tile records group under,413// matching what tools/tiles recovers from a record URL's /tiles/ marker.414// The capture day is part of the path: every dated map renders its own415// basemap, and the pyramid pipeline tells layers apart by this path alone.416func TileSetPath(city, day string) string {417 return city + "/" + day + "/basemap"418}419420// TileTemplate is the record URL the rendered basemap answers to. The scheme421// is deliberately not a fetchable one: these tiles were never on a wire, and422// a spelling without http keeps them visibly clear of the offline rule.423func TileTemplate(city, day string) string {424 return "arcgis-basemap://tiles/" + TileSetPath(city, day) + "/{z}/{x}/{y}.png"425}426427// Capture is the canonical document the crawler stores: the city, the day,428// the window, the basemap pyramid rendered beside it, and every curated429// dataset's features under their kept fields. Fields marshal in declaration430// order and Normalize sorts the lists, so unchanged data always hashes to431// the capture already archived.432type Capture struct {433 Source string `json:"source"`434 City string `json:"city"`435 Title string `json:"title"`436 MapSlug string `json:"mapSlug"`437 Window Window `json:"window"`438 Basemap MapConfig `json:"basemap"`439 Datasets []CapturedDataset `json:"datasets"`440}441442// MapConfig is the basemap pyramid as rendered: the deepest level drawn, and443// the extension every tile wears.444type MapConfig struct {445 MaxZoom int `json:"maxZoom"`446 Extension string `json:"extension"`447}448449// CapturedDataset is one curated layer's features, joined back to the table450// by slug when the capture is read.451type CapturedDataset struct {452 Slug string `json:"slug"`453 Features []Feature `json:"features"`454}455456// Feature is one thing the city published: its object identifier, its kept457// fields spelled as text, and its ground in true degrees.458type Feature struct {459 ID int64 `json:"id"`460 Fields Fields `json:"fields,omitempty"`461 Geometry Geometry `json:"geometry"`462}463464// Geometry is a feature's ground, normalized at capture to one of three465// shapes: a point, lines in MultiLineString nesting, or rings in466// MultiPolygon nesting. Positions are [longitude, latitude] in WGS84,467// rounded to seven decimals -- a centimeter -- so float spelling cannot468// masquerade as change.469type Geometry struct {470 Type string `json:"type"`471 Point []float64 `json:"point,omitempty"`472 Lines [][][]float64 `json:"lines,omitempty"`473 Rings [][][][]float64 `json:"rings,omitempty"`474}475476const (477 GeometryPoint = "point"478 GeometryLines = "lines"479 GeometryRings = "rings"480)481482// Normalize puts the capture into its canonical order. The crawler calls483// this before marshaling, so the hub listing its features differently does484// not masquerade as a new version.485func (c *Capture) Normalize() {486 sort.Slice(c.Datasets, func(a, b int) bool { return c.Datasets[a].Slug < c.Datasets[b].Slug })487 for _, dataset := range c.Datasets {488 features := dataset.Features489 sort.Slice(features, func(a, b int) bool { return features[a].ID < features[b].ID })490 }491}492493// MaybeTranslate hands other sources' snapshots through untouched and494// translates ArcGIS Hub captures.495func MaybeTranslate(kind string, doc []byte) ([]byte, error) {496 if kind != Kind {497 return doc, nil498 }499 return Translate(doc)500}501502// mapSlugShape holds map slugs to capture days, which is what makes a map503// picker read as a version history.504var mapSlugShape = regexp.MustCompile(`^\d{4}-\d{2}-\d{2}$`)505506// Translate turns a capture into the MapGenie-shaped document the pipeline507// reads, deterministically for a given capture.508func Translate(doc []byte) ([]byte, error) {509 var capture Capture510 if err := json.Unmarshal(doc, &capture); err != nil {511 return nil, fmt.Errorf("decode arcgis capture: %w", err)512 }513 if capture.Source != Source {514 return nil, fmt.Errorf("capture source is %q, not %q", capture.Source, Source)515 }516 city, curated := Cities[capture.City]517 if !curated {518 return nil, fmt.Errorf("city %q is not curated", capture.City)519 }520 if !mapSlugShape.MatchString(capture.MapSlug) {521 return nil, fmt.Errorf("map slug %q is not a capture day", capture.MapSlug)522 }523 if capture.Basemap.MaxZoom < 1 {524 return nil, fmt.Errorf("capture declares no basemap pyramid")525 }526 if capture.Window.East <= capture.Window.West || capture.Window.North <= capture.Window.South {527 return nil, fmt.Errorf("capture window has no ground")528 }529 if len(capture.Datasets) == 0 {530 return nil, fmt.Errorf("capture carries no datasets")531 }532 capture.Normalize()533534 byOrder, err := curatedOrder(city, &capture)535 if err != nil {536 return nil, err537 }538539 ids := mgdoc.NewIDSpace()540 scope := capture.City + "/" + capture.MapSlug541 mapID, err := ids.Claim("arcgis:map:" + scope)542 if err != nil {543 return nil, err544 }545 gameID, err := ids.Claim("arcgis:game:" + capture.City)546 if err != nil {547 return nil, err548 }549550 groups, err := buildGroups(&capture, byOrder, ids)551 if err != nil {552 return nil, err553 }554 regions, err := buildRegions(&capture, byOrder, ids, buildHydroIndex(&capture))555 if err != nil {556 return nil, err557 }558559 px, deg := capture.Window.PxDeg()560 out := mgdoc.Map{561 ID: mapID,562 // The day is the map: its slug, and its title, because the picker563 // listing dates is the version history reading as itself.564 Title: capture.MapSlug,565 Slug: capture.MapSlug,566 InitialLatitude: mgdoc.SyntheticLatitude(mgdoc.WorldSize / 2),567 InitialLongitude: mgdoc.SyntheticLongitude(mgdoc.WorldSize / 2),568 Config: mgdoc.Config{TileSets: []mgdoc.TileSet{569 tileSet(capture.City, capture.MapSlug, capture.Basemap),570 }},571 // The map says what it pictures: a plane -- no globe pretends a city572 // is a world -- cut from Earth by the declared Mercator window, which573 // is exactly the transform WorldPixel projects features through, so574 // the two can never disagree without a test noticing.575 Attrs: map[string]string{576 semconv.KeyGeometrySurface: semconv.SurfacePlane,577 semconv.KeyGeometryBody: "earth",578 semconv.KeyGeometryMercatorPx: px,579 semconv.KeyGeometryMercatorDeg: deg,580 },581 Game: mgdoc.Game{582 ID: gameID,583 // A city is a volume to the bundle, spelled in the archive's584 // upstream game shape here, which is the point being585 // proven.586 Title: title(capture.Title, city.Title),587 Slug: capture.City,588 },589 Groups: groups,590 Regions: regions,591 Collections: declaredCollections(byOrder),592 }593 return json.Marshal(out)594}595596func title(given, fallback string) string {597 if given != "" {598 return given599 }600 return fallback601}602603// curatedOrder joins the capture's datasets back to the table, in the604// table's own order, refusing anything the table does not name -- the same605// posture the crawler takes at the door, held again at read time.606func curatedOrder(city City, capture *Capture) ([]pairing, error) {607 captured := make(map[string]*CapturedDataset, len(capture.Datasets))608 for at := range capture.Datasets {609 dataset := &capture.Datasets[at]610 if _, doubled := captured[dataset.Slug]; doubled {611 return nil, fmt.Errorf("dataset %q is captured twice", dataset.Slug)612 }613 captured[dataset.Slug] = dataset614 }615 var out []pairing616 curatedAll := city.AllDatasets()617 for at := range curatedAll {618 curated := &curatedAll[at]619 if data, taken := captured[curated.Slug]; taken {620 out = append(out, pairing{curated, data})621 delete(captured, curated.Slug)622 }623 }624 for slug := range captured {625 return nil, fmt.Errorf("dataset %q is not curated for %s", slug, city.Slug)626 }627 if len(out) == 0 {628 return nil, fmt.Errorf("capture carries no curated datasets")629 }630 return out, nil631}632633type pairing struct {634 curated *Dataset635 data *CapturedDataset636}637638// tileSet declares the basemap pyramid in the shape tools/tiles expects.639// Bounds are declared full-square for every level down to zero, which is640// what lands the frame derivation on the translated world's own grid --641// sourceZoom five, first tile zero -- instead of the MapGenie window this642// map does not sit in.643func tileSet(citySlug, day string, config MapConfig) mgdoc.TileSet {644 set := mgdoc.TileSet{645 Name: "Basemap",646 Path: TileSetPath(citySlug, day),647 MinZoom: 0,648 MaxZoom: config.MaxZoom,649 Extension: title(config.Extension, "png"),650 Bounds: make(map[string]mgdoc.Bound, config.MaxZoom+1),651 }652 for zoom := 0; zoom <= config.MaxZoom; zoom++ {653 last := 1<<zoom - 1654 set.Bounds[strconv.Itoa(zoom)] = mgdoc.Bound{655 X: mgdoc.Range{Min: 0, Max: last},656 Y: mgdoc.Range{Min: 0, Max: last},657 }658 }659 return set660}661662// strayTolerance is how much of a point dataset may fall outside the window663// before the capture is refused rather than trimmed: a hub layer664// occasionally holds one stray feature, and one stray is not a wrong window.665const strayTolerance = 0.01666667// buildGroups arranges the point datasets the way the pipeline expects: one668// category per dataset, gathered under the curated group titles in table669// order.670func buildGroups(capture *Capture, pairs []pairing, ids *mgdoc.IDSpace) ([]mgdoc.Group, error) {671 byGroup := make(map[string][]mgdoc.Category)672 var order []string673 for _, pair := range pairs {674 curated, data := pair.curated, pair.data675 if curated.Group == "" {676 continue677 }678 categoryID, err := ids.Claim("arcgis:cat:" + capture.City + "/" + curated.Slug)679 if err != nil {680 return nil, err681 }682 attrs := map[string]string{semconv.KeyRenderAs: semconv.RenderAsPin}683 if curated.Icon != "" {684 attrs[semconv.KeyIconStd] = curated.Icon685 }686 category := mgdoc.Category{687 ID: categoryID,688 Title: curated.Title,689 Icon: slugify(curated.Title),690 DisplayType: "markers",691 Visible: true,692 Locations: make([]mgdoc.Location, 0, len(data.Features)),693 Attrs: attrs,694 }695 strays, untitled := 0, 0696 for _, feature := range data.Features {697 if feature.Geometry.Type != GeometryPoint || len(feature.Geometry.Point) < 2 {698 return nil, fmt.Errorf("%s feature %d has no point", curated.Slug, feature.ID)699 }700 lon, lat := feature.Geometry.Point[0], feature.Geometry.Point[1]701 x, y := capture.Window.WorldPixel(lon, lat)702 if x < 0 || x > mgdoc.WorldSize || y < 0 || y > mgdoc.WorldSize {703 strays++704 continue705 }706 name := scrub(curated.TitleOf(feature.Fields))707 // A nameless row is the source's hygiene, not the curation's: a708 // pin nobody could search for is left out. A dataset nameless709 // throughout is different -- that is a wrong field name, and it710 // is refused below rather than shipped blank.711 if name == "" {712 untitled++713 continue714 }715 locationID, err := ids.Claim("arcgis:loc:" + capture.City + "/" + curated.Slug + "/" + strconv.FormatInt(feature.ID, 10))716 if err != nil {717 return nil, err718 }719 category.Locations = append(category.Locations, mgdoc.Location{720 ID: locationID,721 Title: name,722 Description: scrub(curated.Describe(feature.Fields)),723 Latitude: mgdoc.SyntheticLatitude(y),724 Longitude: mgdoc.SyntheticLongitude(x),725 // The coordinates as the city published them, verbatim:726 // provenance beside the synthetic position, so no reader727 // ever has to invert the window to learn where a pin728 // truly stands.729 Attrs: map[string]string{730 semconv.KeyGeoLat: strconv.FormatFloat(lat, 'f', -1, 64),731 semconv.KeyGeoLon: strconv.FormatFloat(lon, 'f', -1, 64),732 },733 })734 }735 if total := len(data.Features); total > 0 && float64(strays) > strayTolerance*float64(total) {736 return nil, fmt.Errorf("%s: %d of %d features fall outside the window", curated.Slug, strays, total)737 }738 if total := len(data.Features); total > 0 && untitled == total {739 return nil, fmt.Errorf("%s: no feature carries a title; is the title field curated right?", curated.Slug)740 }741 if _, seen := byGroup[curated.Group]; !seen {742 order = append(order, curated.Group)743 }744 byGroup[curated.Group] = append(byGroup[curated.Group], category)745 }746747 groups := make([]mgdoc.Group, 0, len(order))748 for _, name := range order {749 groupID, err := ids.Claim("arcgis:group:" + capture.City + "/" + name)750 if err != nil {751 return nil, err752 }753 groups = append(groups, mgdoc.Group{ID: groupID, Title: name, Categories: byGroup[name]})754 }755 return groups, nil756}757758// declaredCollections spells the zone-making datasets as the collections759// their zones belong to, in curated order: the table already knows each760// dataset's kind, stroke and label policy, and the declaration says it in761// the registered vocabulary so the generator never has to sniff geometry to762// learn what a region is.763func declaredCollections(pairs []pairing) []mgdoc.CollectionDecl {764 var out []mgdoc.CollectionDecl765 for _, pair := range pairs {766 curated := pair.curated767 if curated.ZoneOf == nil {768 continue769 }770 kind := semconv.GeometryArea771 if curated.Geometry == "line" {772 kind = semconv.GeometryPath773 }774 attrs := map[string]string{semconv.KeyGeometryKind: kind}775 if curated.Geometry == "line" {776 attrs[semconv.KeyStrokeWidthPx] = strconv.FormatFloat(curated.StrokeWidth, 'f', -1, 64)777 }778 // Label policy is an area's word alone: a path is quiet by the779 // registry's own default, so a line dataset's curation says nothing.780 if curated.Label != "" && kind == semconv.GeometryArea {781 attrs[semconv.KeyLabelPolicy] = curated.Label782 }783 out = append(out, mgdoc.CollectionDecl{784 Key: curated.Slug,785 Title: curated.Title,786 Attrs: attrs,787 })788 }789 return out790}791792// zoneLimit is the most zones one dataset may make. The viewer's zone index793// renders every zone a button and a title; an uncurated explosion -- every794// parcel a zone -- is refused while the mistake is one table edit old.795const zoneLimit = 256796797// buildRegions folds the polygon and line datasets into zones: every798// feature whose ZoneOf names a bucket lands its ground in that bucket's799// region, and line features widen into ribbon polygons first, because a800// zone is ground and a line has none. Zones of the city's own datasets --801// never the national ones -- learn their subwatershed from the hydro802// index when their features agree on one.803func buildRegions(capture *Capture, pairs []pairing, ids *mgdoc.IDSpace, hydro *hydroIndex) ([]mgdoc.Region, error) {804 regions := []mgdoc.Region{}805 for _, pair := range pairs {806 curated, data := pair.curated, pair.data807 if curated.ZoneOf == nil {808 continue809 }810 buckets := make(map[string]*mgdoc.Region)811 claims := make(map[string]*hydroClaims)812 var order []string813 for _, feature := range data.Features {814 key := curated.ZoneOf(feature.Fields)815 if key.Key == "" {816 continue817 }818 if hydro != nil && curated.Server == "" {819 claim, tracked := claims[key.Key]820 if !tracked {821 claim = &hydroClaims{}822 claims[key.Key] = claim823 }824 claim.observe(hydro, feature.Geometry)825 }826 zone, made := buckets[key.Key]827 if !made {828 if len(buckets) >= zoneLimit {829 return nil, fmt.Errorf("%s makes more than %d zones", curated.Slug, zoneLimit)830 }831 zoneID, err := ids.Claim("arcgis:zone:" + capture.City + "/" + curated.Slug + "/" + key.Key)832 if err != nil {833 return nil, err834 }835 zone = &mgdoc.Region{836 ID: zoneID,837 Title: scrub(key.Title),838 Subtitle: scrub(title(key.Subtitle, curated.Title)),839 // The zone says which declared collection it belongs840 // to, which is the whole reason declarations exist:841 // the collection speaks for its kind and its stroke,842 // so the zone carries neither.843 Collection: curated.Slug,844 }845 buckets[key.Key] = zone846 order = append(order, key.Key)847 }848 geometry, err := zoneGeometry(capture.Window, curated, feature)849 if err != nil {850 return nil, err851 }852 if geometry != nil {853 zone.Features = append(zone.Features, mgdoc.RegionFeature{Geometry: *geometry})854 }855 }856 sort.Strings(order)857 for _, key := range order {858 if len(buckets[key].Features) == 0 {859 continue860 }861 if claim := claims[key]; claim != nil {862 claim.apply(hydro, buckets[key])863 }864 regions = append(regions, *buckets[key])865 }866 }867 return regions, nil868}869870// zoneGeometry lands one feature's ground in the world in synthetic871// coordinates: rings project position by position into a MultiPolygon, and872// lines stay the MultiLineString they are -- the zone's declared stroke873// width says how wide the viewer draws them.874func zoneGeometry(window Window, curated *Dataset, feature Feature) (*mgdoc.Geometry, error) {875 var polygons [][][][]float64876 switch feature.Geometry.Type {877 case GeometryRings:878 for _, polygon := range feature.Geometry.Rings {879 var rings [][][]float64880 for _, ring := range polygon {881 projected := make([][]float64, 0, len(ring))882 for _, position := range ring {883 if len(position) < 2 {884 return nil, fmt.Errorf("%s feature %d has a malformed ring", curated.Slug, feature.ID)885 }886 projected = append(projected, synthetic(window, position[0], position[1]))887 }888 if len(projected) >= 4 {889 rings = append(rings, projected)890 }891 }892 if len(rings) > 0 {893 polygons = append(polygons, rings)894 }895 }896 case GeometryLines:897 if curated.StrokeWidth <= 0 {898 return nil, fmt.Errorf("%s zones lines without a stroke width", curated.Slug)899 }900 var lines [][][]float64901 for _, line := range feature.Geometry.Lines {902 projected := make([][]float64, 0, len(line))903 for _, position := range line {904 if len(position) < 2 {905 continue906 }907 projected = append(projected, synthetic(window, position[0], position[1]))908 }909 if len(projected) >= 2 {910 lines = append(lines, projected)911 }912 }913 if len(lines) == 0 {914 return nil, nil915 }916 coordinates, err := json.Marshal(lines)917 if err != nil {918 return nil, err919 }920 return &mgdoc.Geometry{Type: "MultiLineString", Coordinates: coordinates}, nil921 default:922 return nil, fmt.Errorf("%s feature %d has no ground to zone", curated.Slug, feature.ID)923 }924 if len(polygons) == 0 {925 return nil, nil926 }927 coordinates, err := json.Marshal(polygons)928 if err != nil {929 return nil, err930 }931 return &mgdoc.Geometry{Type: "MultiPolygon", Coordinates: coordinates}, nil932}933934// synthetic is one position's whole journey: true degrees to world pixel to935// the synthetic coordinates a zone ring speaks.936func synthetic(window Window, lon, lat float64) []float64 {937 x, y := window.WorldPixel(lon, lat)938 return []float64{mgdoc.SyntheticLongitude(x), mgdoc.SyntheticLatitude(y)}939}940941// scrub keeps a live URL out of everything this package emits. Bundles ship942// offline and the validator refuses any http spelling anywhere in a payload;943// curated fields should never carry one, and any that slips through loses944// the token rather than failing the build a capture too late.945func scrub(text string) string {946 if !strings.Contains(strings.ToLower(text), "http") {947 return text948 }949 kept := []string{}950 for token := range strings.FieldsSeq(text) {951 if strings.Contains(strings.ToLower(token), "http") {952 continue953 }954 kept = append(kept, token)955 }956 return strings.Join(kept, " ")957}958959// slugify spells a value the way slugs and icon keys are named.960func slugify(value string) string {961 out := make([]rune, 0, len(value))962 for _, r := range value {963 switch {964 case r >= 'a' && r <= 'z' || r >= '0' && r <= '9':965 out = append(out, r)966 case r >= 'A' && r <= 'Z':967 out = append(out, r+'a'-'A')968 default:969 if len(out) > 0 && out[len(out)-1] != '-' {970 out = append(out, '-')971 }972 }973 }974 for len(out) > 0 && out[len(out)-1] == '-' {975 out = out[:len(out)-1]976 }977 return string(out)978}979980// FieldString spells one GeoJSON property value as capture text: numbers in981// their shortest exact form, text as itself, absence as emptiness. The982// crawler runs every kept field through this so a value's JSON spelling983// cannot wobble the capture's hash.984func FieldString(value any) string {985 switch v := value.(type) {986 case nil:987 return ""988 case string:989 return strings.TrimSpace(v)990 case float64:991 return strconv.FormatFloat(v, 'f', -1, 64)992 case bool:993 return strconv.FormatBool(v)994 case json.Number:995 return v.String()996 default:997 return fmt.Sprint(v)998 }999}10001001// Round7 rounds a coordinate to seven decimals -- a centimeter on the1002// ground -- which is where capture determinism and honest precision meet.1003func Round7(value float64) float64 {1004 return math.Round(value*1e7) / 1e71005}