1// Package arcgishub reads a captured city and translates it into the Atlas2// interchange document.3//4// A capture comes from a city's ArcGIS Hub open-data site: the municipal5// datasets a city publishes about itself -- its limits, its zoning, its trails,6// its historic buildings -- kept verbatim under a reviewed field allowlist, plus7// the national hydrography every US city is enriched with by bounding box.8//9// # What a city is to a bundle10//11// A city is a volume, and **each crawl day is a world**. That is the whole12// design: a volume's world picker becomes the city's version history, four13// captured days are four worlds, and the differences between them are14// differences in the city rather than differences in the pipeline. It is why a15// world's slug is a date and why the reader is told to read them newest first.16//17// Polygon and line datasets become shape collections -- one collection per18// dataset, one feature per curated bucket -- and point datasets become pins. The19// one picture is a basemap rendered offline from the city's own vector data,20// because a bundle owes its reader ground to stand on and ships without a21// network.22//23// # The membership join24//25// A capture carrying the national subwatersheds can say which subwatershed each26// of the city's own zones lies in, computed from the captured polygons at27// translate time. See hydro.go: the claim is made only when every sampled part28// of a zone agrees, so a zone that straddles two says nothing.29//30// # What is refused, and what is passed over31//32// Refused, because the capture is wrong: another source's kind; a world whose33// slug is not a capture day; no basemap pyramid, no window or no datasets; a34// dataset captured twice or not curated at all; a point dataset whose rows are35// all untitled, or too many of whose rows fall outside the declared window; a36// dataset whose curation makes more features than a legend can hold. The reader37// states its preconditions and refuses what does not meet them rather than38// guessing.39//40// Passed over, because the capture is not this reader's: **a city the table does41// not curate**. That is not a malformed capture, it is a capture nothing here42// can answer for, so it wraps ErrNotReady and the caller skips it. The case is43// real rather than theoretical -- an operator may have crawled their own city44// into the same archive, and the privacy rule keeps that city's name out of this45// table -- and a hard refusal would take every other volume in the archive down46// with it.47//48// # Determinism49//50// Datasets are read in the curation table's order, buckets are emitted in sorted51// key order, every identifier is derived from a stable name, and no clock or map52// iteration reaches the output. The same archived bytes give the same document53// on any machine.54package arcgishub5556import (57 "encoding/json"58 "errors"59 "fmt"60 "log/slog"61 "regexp"62 "sort"63 "strconv"64 "strings"6566 "github.com/FelineStateMachine/atlas/format/semconv"67 "github.com/FelineStateMachine/atlas/internal/generate/archive"68 "github.com/FelineStateMachine/atlas/internal/generate/doc"69 "github.com/FelineStateMachine/atlas/internal/logging"70)7172// captureKind is what the archive calls a city capture, and source is what a73// well-formed capture says it is.74const (75 captureKind = "arcgis-map"76 source = "arcgis-hub"77)7879// Source reads ArcGIS Hub city captures.80type Source struct{}8182// New builds the source. It holds no state.83func New() Source { return Source{} }8485// Describe is the source's account of itself.86func (Source) Describe() doc.Provenance {87 return doc.Provenance{88 Name: source,89 Label: "ArcGIS Open Data",90 License: "Open data published by each city under its own terms, and public-domain " +91 "federal hydrography from the USGS Watershed Boundary and National " +92 "Hydrography Datasets.",93 Attribution: "Municipal layers from each city's ArcGIS Hub open-data site. " +94 "Watersheds, subwatersheds, streams and waterbodies from the USGS " +95 "National Hydrography services.",96 // A hub numbers its rows with object ids that are load artifacts and97 // churn between refreshes, and it numbers nothing at all above a row --98 // no map, no layer, no bucket. Every identity here is minted from a99 // stable name instead.100 IDSpace: doc.IDSpaceDerived,101 }102}103104// Translate reads one archived city: every crawl day it holds, in the archive's105// own order, each one a world.106func (s Source) Translate(a *archive.Archive, v archive.VolumeRef, log *slog.Logger) (doc.Document, error) {107 log = log.With(logging.Source(source))108 refs, err := a.Worlds(v)109 if err != nil {110 return doc.Document{}, err111 }112 out := doc.Document{113 Doc: doc.Doc,114 Version: doc.Version,115 Source: s.Describe(),116 }117 for _, ref := range refs {118 world, subject, err := s.translateWorld(a, ref, log)119 if err != nil {120 if errors.Is(err, archive.ErrNotReady) {121 log.Debug("world skipped", logging.Path(archive.TrimRoot(a.Root(), ref.Dir())),122 "reason", err.Error())123 continue124 }125 return doc.Document{}, err126 }127 if out.Volume.Slug == "" {128 out.Volume = subject129 }130 if out.Volume.Slug != subject.Slug {131 return doc.Document{}, fmt.Errorf(132 "the archive directory holds captures of two cities, %s and %s",133 out.Volume.Slug, subject.Slug)134 }135 out.Worlds = append(out.Worlds, world)136 }137 if len(out.Worlds) == 0 {138 return doc.Document{}, fmt.Errorf("%w: volume %s has no readable world", archive.ErrNotReady, v.Title)139 }140 log.Info("volume translated", logging.Volume(out.Volume.Slug), "worlds", len(out.Worlds))141 return out, nil142}143144// daySlug holds a world's slug to a capture day, which is what makes a world145// picker read as a version history.146var daySlug = regexp.MustCompile(`^\d{4}-\d{2}-\d{2}$`)147148func (s Source) translateWorld(149 a *archive.Archive,150 ref archive.WorldRef,151 log *slog.Logger,152) (doc.World, doc.Volume, error) {153 archived, err := a.Newest(ref)154 if err != nil {155 return doc.World{}, doc.Volume{}, err156 }157 if archived.Kind != captureKind {158 return doc.World{}, doc.Volume{}, fmt.Errorf(159 "capture %s is of kind %q; the ArcGIS Hub reader answers only for %q",160 archived.ContentHash, archived.Kind, captureKind)161 }162 body, err := a.Body(ref, archived)163 if err != nil {164 return doc.World{}, doc.Volume{}, err165 }166 var raw capture167 if err := json.Unmarshal(body, &raw); err != nil {168 return doc.World{}, doc.Volume{}, fmt.Errorf("decode capture %s: %w", archived.ContentHash, err)169 }170 curated, err := check(&raw)171 if err != nil {172 return doc.World{}, doc.Volume{}, fmt.Errorf("capture %s: %w", archived.ContentHash, err)173 }174 raw.normalize()175176 pairs, err := curatedOrder(curated, &raw)177 if err != nil {178 return doc.World{}, doc.Volume{}, err179 }180181 ids := doc.NewIDSpace()182 scope := raw.City + "/" + raw.MapSlug183 worldID, err := ids.Claim("arcgis:map:" + scope)184 if err != nil {185 return doc.World{}, doc.Volume{}, err186 }187188 px, deg := raw.Window.pxDeg()189 world := doc.World{190 ID: worldID,191 // The day is the world: its slug and its title, because a picker192 // listing dates is the version history reading as itself.193 Slug: raw.MapSlug,194 Title: raw.MapSlug,195 // A reader opens on the middle of the city's picture.196 Center: doc.SyntheticPosition(doc.SyntheticWorldSize/2, doc.SyntheticWorldSize/2),197 Capture: doc.Capture{198 Kind: archived.Kind,199 ID: archived.SourceID,200 Locator: archived.SourceURL,201 ContentHash: archived.ContentHash,202 CapturedAt: archived.CapturedAt,203 },204 // The world says what it pictures: a plane -- no globe pretends a city205 // is a world -- cut from Earth by the declared Mercator window, which is206 // exactly the transform worldPixel projects features through, so the two207 // cannot disagree without a test noticing.208 Attrs: map[string]string{209 semconv.KeyGeometrySurface: semconv.SurfacePlane,210 semconv.KeyGeometryBody: "earth",211 semconv.KeyGeometryMercatorPx: px,212 semconv.KeyGeometryMercatorDeg: deg,213 },214 // The one picture is drawn, not fetched: a hub serves geometry and no215 // tiles, so the lens carries the drawing its deepest level is216 // rasterized from beside the frame that says how deep that is.217 Lenses: []doc.Lens{{218 Name: "Basemap",219 TileSet: TileSetPath(raw.City, raw.MapSlug),220 Frame: frameOf(raw.Basemap),221 Drawing: drawShapes(&raw, curated),222 }},223 }224225 pins, err := pinCollections(&raw, pairs, ids)226 if err != nil {227 return doc.World{}, doc.Volume{}, err228 }229 shapes, err := shapeCollections(&raw, pairs, ids, buildHydroIndex(&raw))230 if err != nil {231 return doc.World{}, doc.Volume{}, err232 }233 // Pins first, then ground, which is the order a legend reads and the order234 // the packed payload's owner column counts point collections in.235 world.Collections = append(pins, shapes...)236237 log.Debug("world translated", logging.World(raw.MapSlug),238 "collections", len(world.Collections), "capture", archived.ContentHash)239 return world, doc.Volume{Slug: raw.City, Title: named(raw.Title, curated.Title)}, nil240}241242// check states what a capture has to be before anything is read out of it, and243// answers with the curation it is read through.244func check(raw *capture) (city, error) {245 if raw.Source != source {246 return city{}, fmt.Errorf("capture says its source is %q, not %q", raw.Source, source)247 }248 curated, known := cities[raw.City]249 if !known {250 // Not an error: a capture of an uncurated city is a volume this source251 // cannot answer for, and the archive is allowed to hold one. A machine252 // whose operator crawled their own city has exactly that, and the253 // public table may not name it (issue #5, the privacy rule), so the254 // volume is passed over the way any unreadable one is rather than255 // failing every other volume in the archive with it.256 return city{}, fmt.Errorf("%w: city %q is not curated, and an unverified window would "+257 "hang every pin on the wrong pixel", archive.ErrNotReady, raw.City)258 }259 if !daySlug.MatchString(raw.MapSlug) {260 return city{}, fmt.Errorf("world slug %q is not a capture day", raw.MapSlug)261 }262 if raw.Basemap.MaxZoom < 1 {263 return city{}, fmt.Errorf("capture declares no basemap pyramid")264 }265 if raw.Window.East <= raw.Window.West || raw.Window.North <= raw.Window.South {266 return city{}, fmt.Errorf("capture window has no ground")267 }268 if len(raw.Datasets) == 0 {269 return city{}, fmt.Errorf("capture carries no datasets")270 }271 return curated, nil272}273274// pairing is one curated dataset joined to the rows captured for it.275type pairing struct {276 curated *dataset277 rows *capturedDataset278}279280// curatedOrder joins a capture's datasets back to the table, in the table's own281// order, refusing anything the table does not name -- the same posture the282// crawler takes at the door, held again at read time so a capture taken under an283// older table cannot smuggle a layer past a newer one.284func curatedOrder(curated city, raw *capture) ([]pairing, error) {285 captured := make(map[string]*capturedDataset, len(raw.Datasets))286 for at := range raw.Datasets {287 held := &raw.Datasets[at]288 if _, doubled := captured[held.Slug]; doubled {289 return nil, fmt.Errorf("dataset %q is captured twice", held.Slug)290 }291 captured[held.Slug] = held292 }293 var out []pairing294 table := curated.datasets()295 for at := range table {296 entry := &table[at]297 if rows, taken := captured[entry.Slug]; taken {298 out = append(out, pairing{entry, rows})299 delete(captured, entry.Slug)300 }301 }302 for slug := range captured {303 return nil, fmt.Errorf("dataset %q is not curated for %s", slug, curated.Slug)304 }305 if len(out) == 0 {306 return nil, fmt.Errorf("capture carries no curated datasets")307 }308 return out, nil309}310311// frameOf declares the rendered pyramid the way the deriver reads one: every312// level complete, from a single tile at zero down to the deepest level drawn.313// A rendered basemap has no partial level by construction -- the renderer draws314// the whole square -- and saying so is what lands the derivation on the315// translated world's own grid rather than on the shared window a city does not316// sit in.317func frameOf(config mapConfig) *doc.Frame {318 frame := &doc.Frame{319 MinZoom: 0,320 MaxZoom: config.MaxZoom,321 Format: archive.NormalizeFormat(named(config.Extension, "png")),322 Windows: make(map[string]doc.TileWindow, config.MaxZoom+1),323 }324 for zoom := 0; zoom <= config.MaxZoom; zoom++ {325 last := 1<<zoom - 1326 frame.Windows[strconv.Itoa(zoom)] = doc.TileWindow{MaxX: last, MaxY: last}327 }328 return frame329}330331// strayTolerance is how much of a point dataset may fall outside the window332// before the capture is refused rather than trimmed: a hub layer occasionally333// holds one stray row, and one stray is not a wrong window.334const strayTolerance = 0.01335336// pinCollections turns the point datasets into collections, one per dataset,337// each filed under its curated heading and in the table's own order.338func pinCollections(raw *capture, pairs []pairing, ids *doc.IDSpace) ([]doc.Collection, error) {339 var out []doc.Collection340 for _, pair := range pairs {341 curated, rows := pair.curated, pair.rows342 if curated.Group == "" {343 continue344 }345 id, err := ids.Claim("arcgis:cat:" + raw.City + "/" + curated.Slug)346 if err != nil {347 return nil, err348 }349 attrs := map[string]string{semconv.KeyRenderAs: semconv.RenderAsPin}350 // The standard glyph is named, not resolved: resolving it against the351 // icon library is composition's work, and a source that resolved it352 // would be deciding what artwork a bundle carries.353 if curated.Icon != "" {354 attrs[semconv.KeyIconStd] = curated.Icon355 }356 collection := doc.Collection{357 ID: id,358 Title: curated.Title,359 Group: curated.Group,360 Kind: doc.KindPoint,361 Icon: doc.Slugify(curated.Title),362 Visible: true,363 Attrs: attrs,364 Features: make([]doc.Feature, 0, len(rows.Features)),365 }366 strays, untitled := 0, 0367 for _, row := range rows.Features {368 if row.Geometry.Type != geometryPoint || len(row.Geometry.Point) < 2 {369 return nil, fmt.Errorf("%s row %d has no point", curated.Slug, row.ID)370 }371 lon, lat := row.Geometry.Point[0], row.Geometry.Point[1]372 x, y := raw.Window.worldPixel(lon, lat)373 if x < 0 || x > doc.SyntheticWorldSize || y < 0 || y > doc.SyntheticWorldSize {374 strays++375 continue376 }377 // A nameless row is the publisher's hygiene, not the curation's: a378 // pin nobody could search for is left out. A dataset nameless379 // throughout is different -- that is a wrong field name, and it is380 // refused below rather than shipped blank.381 name := scrub(curated.TitleOf(row.Fields))382 if name == "" {383 untitled++384 continue385 }386 featureID, err := ids.Claim("arcgis:loc:" + raw.City + "/" + curated.Slug +387 "/" + strconv.FormatInt(row.ID, 10))388 if err != nil {389 return nil, err390 }391 at := doc.SyntheticPosition(x, y)392 collection.Features = append(collection.Features, doc.Feature{393 ID: featureID,394 Title: name,395 Description: scrub(curated.Describe(row.Fields)),396 At: &at,397 // The coordinates as the city published them, verbatim:398 // provenance beside the synthetic position, so no reader has to399 // invert the window to learn where a pin truly stands.400 Attrs: map[string]string{401 semconv.KeyGeoLat: strconv.FormatFloat(lat, 'f', -1, 64),402 semconv.KeyGeoLon: strconv.FormatFloat(lon, 'f', -1, 64),403 },404 })405 }406 if total := len(rows.Features); total > 0 && float64(strays) > strayTolerance*float64(total) {407 return nil, fmt.Errorf("%s: %d of %d rows fall outside the window", curated.Slug, strays, total)408 }409 if total := len(rows.Features); total > 0 && untitled == total {410 return nil, fmt.Errorf("%s: no row carries a title; is the title field curated right?", curated.Slug)411 }412 out = append(out, collection)413 }414 return out, nil415}416417// zoneLimit is the most features one shape collection may hold. The reader's418// legend renders every one of them a button and a title; an uncurated explosion419// -- every parcel a zone -- is refused while the mistake is one table edit old.420const zoneLimit = 256421422// shapeCollections folds the polygon and line datasets into ground: one423// collection per dataset, and within it one feature per curated bucket, holding424// every row's geometry that fell into that bucket. Buckets of the city's own425// datasets -- never the national ones -- learn their subwatershed from the hydro426// index when all their ground agrees on one.427func shapeCollections(428 raw *capture,429 pairs []pairing,430 ids *doc.IDSpace,431 hydro *hydroIndex,432) ([]doc.Collection, error) {433 var out []doc.Collection434 for _, pair := range pairs {435 curated, rows := pair.curated, pair.rows436 if curated.ZoneOf == nil {437 continue438 }439 kind := doc.KindArea440 if curated.Geometry == "line" {441 kind = doc.KindPath442 }443 attrs := map[string]string{}444 if kind == doc.KindPath {445 attrs[semconv.KeyStrokeWidthPx] = strconv.FormatFloat(curated.StrokeWidth, 'f', -1, 64)446 }447 // Label policy is an area's word alone: a path is quiet by the448 // registry's own default, so a line dataset's curation says nothing.449 if curated.Label != "" && kind == doc.KindArea {450 attrs[semconv.KeyLabelPolicy] = curated.Label451 }452 if len(attrs) == 0 {453 attrs = nil454 }455 collection := doc.Collection{456 Key: curated.Slug,457 Title: curated.Title,458 Kind: kind,459 Visible: true,460 Attrs: attrs,461 }462463 buckets := make(map[string]*doc.Feature)464 claims := make(map[string]*hydroClaims)465 var order []string466 for _, row := range rows.Features {467 key := curated.ZoneOf(row.Fields)468 if key.Key == "" {469 continue470 }471 if hydro != nil && !curated.National {472 claim, tracked := claims[key.Key]473 if !tracked {474 claim = &hydroClaims{}475 claims[key.Key] = claim476 }477 claim.observe(hydro, row.Geometry)478 }479 zone, made := buckets[key.Key]480 if !made {481 if len(buckets) >= zoneLimit {482 return nil, fmt.Errorf("%s makes more than %d features", curated.Slug, zoneLimit)483 }484 zoneID, err := ids.Claim("arcgis:zone:" + raw.City + "/" + curated.Slug + "/" + key.Key)485 if err != nil {486 return nil, err487 }488 zone = &doc.Feature{489 ID: zoneID,490 Title: scrub(key.Title),491 Subtitle: scrub(named(key.Subtitle, curated.Title)),492 }493 buckets[key.Key] = zone494 order = append(order, key.Key)495 }496 parts, err := shapeGeometry(raw.Window, curated, row)497 if err != nil {498 return nil, err499 }500 if parts != nil {501 zone.Geometry = append(zone.Geometry, *parts)502 }503 }504 sort.Strings(order)505 for _, key := range order {506 if len(buckets[key].Geometry) == 0 {507 continue508 }509 if claim := claims[key]; claim != nil {510 claim.apply(hydro, buckets[key])511 }512 collection.Features = append(collection.Features, *buckets[key])513 }514 out = append(out, collection)515 }516 return out, nil517}518519// shapeGeometry lands one row's ground in the world in synthetic coordinates:520// rings project position by position into a MultiPolygon, and lines stay the521// MultiLineString they are -- the collection's declared stroke width says how522// wide the reader draws them.523func shapeGeometry(w window, curated *dataset, row feature) (*doc.Geometry, error) {524 switch row.Geometry.Type {525 case geometryRings:526 var polygons [][][][]float64527 for _, polygon := range row.Geometry.Rings {528 var rings [][][]float64529 for _, ring := range polygon {530 projected := make([][]float64, 0, len(ring))531 for _, position := range ring {532 if len(position) < 2 {533 return nil, fmt.Errorf("%s row %d has a malformed ring", curated.Slug, row.ID)534 }535 projected = append(projected, w.syntheticRing(position[0], position[1]))536 }537 // A ring needs three corners and a closing repeat to enclose538 // anything; anything shorter is a capture artifact.539 if len(projected) >= 4 {540 rings = append(rings, projected)541 }542 }543 if len(rings) > 0 {544 polygons = append(polygons, rings)545 }546 }547 if len(polygons) == 0 {548 return nil, nil549 }550 return marshalGeometry("MultiPolygon", polygons)551 case geometryLines:552 if curated.StrokeWidth <= 0 {553 return nil, fmt.Errorf("%s draws lines without a stroke width", curated.Slug)554 }555 var lines [][][]float64556 for _, line := range row.Geometry.Lines {557 projected := make([][]float64, 0, len(line))558 for _, position := range line {559 if len(position) < 2 {560 continue561 }562 projected = append(projected, w.syntheticRing(position[0], position[1]))563 }564 if len(projected) >= 2 {565 lines = append(lines, projected)566 }567 }568 if len(lines) == 0 {569 return nil, nil570 }571 return marshalGeometry("MultiLineString", lines)572 default:573 return nil, fmt.Errorf("%s row %d has no ground to draw", curated.Slug, row.ID)574 }575}576577func marshalGeometry(kind string, coordinates any) (*doc.Geometry, error) {578 encoded, err := json.Marshal(coordinates)579 if err != nil {580 return nil, err581 }582 return &doc.Geometry{Type: kind, Coordinates: encoded}, nil583}584585// scrub keeps a live URL out of everything this source emits. Bundles ship586// offline and the format refuses an http spelling anywhere in a payload; a587// curated field should never carry one, and any that slips through loses the588// token rather than failing a build a capture too late.589func scrub(text string) string {590 if !strings.Contains(strings.ToLower(text), "http") {591 return text592 }593 kept := []string{}594 for _, token := range strings.Fields(text) {595 if strings.Contains(strings.ToLower(token), "http") {596 continue597 }598 kept = append(kept, token)599 }600 return strings.Join(kept, " ")601}