1// Package doc defines the Atlas interchange document: what one source has to2// say about one volume, in Atlas's own vocabulary.3//4// A document sits between the two halves of the generate lane. A source reads5// archived captures and emits one; composition reads one and writes a .atlas6// bundle. It is Atlas's schema, designed backwards from what composition needs7// -- worlds, lenses, collections, features, geometry, attributes, provenance --8// and no source's shape is privileged in it. A source that cannot say something9// in this vocabulary cannot say it at all, which is the point: nothing10// downstream of a translator knows which source it read.11//12// Three properties are load-bearing:13//14// - Deterministic. The same captures translate to the same document, byte for15// byte, on any machine. Maps are the only unordered thing in it, and JSON16// sorts their keys; everything else is a slice whose order the source17// decides and composition preserves.18// - Replayable. A document is a pure function of archived bytes, so a policy19// change re-runs over the archive instead of over the network. Every world20// names the capture it was read from, by content hash.21// - Self-contained. Icon artwork travels in the document rather than as a22// path back into an archive, so composition never reaches for a file the23// source did not hand it.24//25// The schema is documented for humans in docs/generate.md, which is this26// package's prose twin.27//28// # Coordinates29//30// Positions are latitude and longitude in the volume's own projection -- the31// same world space format/bundle.Coordinate carries and the same one a lens's32// tile pyramid is cut in. For a game map that space is synthetic: an image was33// encoded as a slippy map long ago and the numbers are pixels wearing degrees.34// For a planet or a city it is the real thing. Either way a source publishes35// what its ground means and composition does no reprojection, so a translator36// owns its own projection and nothing else has to know about it.37//38// # Identifiers39//40// Feature and collection identifiers are int64 here and signed 32-bit on the41// wire, where zero reads as absence. A source whose captures carry native42// numeric identities passes them through; a source without them derives stable43// ones from stable names and says so in Provenance.IDSpace. Zero means "no44// such thing" everywhere in this package: Feature.Member and Feature.Parent45// use it for "contained by nothing".46package doc4748import (49 "encoding/json"50 "fmt"51 "sort"52)5354// Doc and Version identify the schema. Version moves when an existing field's55// meaning breaks; a new optional field rides on the old version, because a56// document is never stored -- it is produced and consumed in one run, and the57// version exists to catch a stale debugging dump, not to support archaeology.58const (59 Doc = "atlas-generate-doc"60 Version = 161)6263// The geometry kinds a collection may declare. Every feature in a collection is64// the kind its collection declares; nothing sniffs geometry to find out.65const (66 KindPoint = "point"67 KindPath = "path"68 KindArea = "area"69)7071// Document is one source's whole reading of one volume.72type Document struct {73 Doc string `json:"doc"`74 Version int `json:"version"`75 Volume Volume `json:"volume"`76 Source Provenance `json:"source"`77 Worlds []World `json:"worlds"`78 // Icons is the artwork the volume's collections name, by key. A key no79 // collection names is allowed: composition packs what is used and leaves80 // the rest, so a source may hand over a whole archived icon set without81 // deciding what is worth carrying.82 Icons []Icon `json:"icons,omitempty"`83}8485// Volume is the subject: a game, a planet, a city.86type Volume struct {87 Slug string `json:"slug"`88 Title string `json:"title"`89}9091// Provenance is where a document came from, in the terms a ledger and a source92// card both need. It is the only place a source names itself; nothing else in93// the document, and nothing downstream of composition, mentions a source at94// all.95type Provenance struct {96 // Name is the source's registry name -- the slug a ledger line and a97 // workbench card agree on.98 Name string `json:"name"`99 // ArchivedAs is the name the legacy capture archive files this source's100 // captures under, where that is not Name. It is layout rather than101 // vocabulary: the archive on disk predates Atlas's own naming and is kept102 // verbatim as input data (issue #5 Section 5.1), so a source that Atlas103 // calls one thing may find its captures filed under another. It never104 // rides a document -- nothing downstream of a translator may learn an105 // archive's spelling of anything.106 ArchivedAs string `json:"-"`107 // Label is the same source spelled for a person.108 Label string `json:"label"`109 // License and Attribution are what a volume owes the people whose work it110 // carries. They ride the source registry entry and surface on its card.111 License string `json:"license,omitempty"`112 Attribution string `json:"attribution,omitempty"`113 // IDSpace says where the identifiers in this document came from, so a114 // reader of two documents knows whether their number spaces can be115 // compared. See IDSpaceNative and IDSpaceDerived.116 IDSpace string `json:"idSpace"`117}118119// The id spaces a source may declare.120const (121 // IDSpaceNative means the captures carried their own numeric identities and122 // the document passes them through. Two documents from one native source123 // name the same thing by the same number.124 IDSpaceNative = "native"125 // IDSpaceDerived means the source minted identifiers from stable names.126 // They are stable across runs and meaningless across sources.127 IDSpaceDerived = "derived"128)129130// Capture names the archived bytes a world was read from. It is provenance and131// a determinism receipt: two runs that name the same content hash read the same132// input, whatever the archive looked like around it.133type Capture struct {134 // Kind is the source's own name for what these bytes are. It is the one135 // place a source's vocabulary is allowed to show through, because it names136 // bytes the source captured and nothing else can describe them.137 Kind string `json:"kind"`138 // ID is the capture's identity in the source's own id space.139 ID int64 `json:"id,omitempty"`140 // Locator is where the source fetched the bytes. It is provenance only --141 // no payload ever carries it, and the offline invariant is checked at the142 // bundle, not here.143 Locator string `json:"locator,omitempty"`144 // ContentHash is the archive's content address of the bytes: SHA-256,145 // lowercase hex.146 ContentHash string `json:"contentHash"`147 // CapturedAt is when the archive first saw these bytes, RFC 3339. It is148 // what a build's creation time is derived from, never the build clock.149 CapturedAt string `json:"capturedAt"`150}151152// World is one ground within the volume.153type World struct {154 ID int64 `json:"id"`155 Slug string `json:"slug"`156 Title string `json:"title"`157 // Center is where a reader opening this world is put.158 Center Position `json:"center"`159 Capture Capture `json:"capture"`160 // Lenses are the pictures of this ground, in the order a reader is offered161 // them. A lens here says only what a source knows: what it is called and162 // which tile set pictures it. Everything else about a pyramid -- its zoom163 // range, formats, coverage -- belongs to whatever derived it.164 Lenses []Lens `json:"lenses"`165 // Collections is the world's whole structure in one ordered array. The166 // order is the legend order and it is significant: the packed payload's167 // owner column indexes the point collections by position.168 Collections []Collection `json:"collections"`169 // Attrs is the world speaking the semantic conventions. Producer-strict:170 // every atlas.* key here is checked against the registry before a bundle171 // is written.172 Attrs map[string]string `json:"attrs,omitempty"`173}174175// Position is a point in the volume's own projection.176type Position struct {177 Lat float64 `json:"lat"`178 Lng float64 `json:"lng"`179}180181// Lens is one picture of a world, as its source names it.182type Lens struct {183 // Name is what the reader's lens picker calls it.184 Name string `json:"name"`185 // TileSet is the path the source captured this picture's tiles under. It186 // is how composition finds the derived pyramid; it is a key into a tile187 // set, never a URL.188 TileSet string `json:"tileSet"`189 // Frame is what the source knows about the tiles it captured. Composition190 // never reads it -- what a bundle promises about a raster is what the191 // deriver actually derived, not what a publisher offered -- but the deriver192 // does, because only the source can say which tiles a complete level was193 // supposed to hold. A lens with no frame is a picture nothing can derive.194 Frame *Frame `json:"frame,omitempty"`195 // Drawing, when set, says this picture is drawn rather than fetched: the196 // deriver rasterizes the lens's deepest level from these shapes instead of197 // taking its pixels from a capture. A city is the case it exists for -- no198 // municipal open-data hub serves tiles, so the ground a city's bundle199 // stands on is drawn from the geometry the city publishes about itself.200 Drawing *Drawing `json:"drawing,omitempty"`201}202203// Drawing is a lens's picture stated as shapes rather than as pixels.204//205// It is deliberately not a second copy of the world's collections. A collection206// is what a reader can click, search and filter; a drawing is what the ground207// looks like, and the two do not line up: every road centreline a city publishes208// draws, while only its named trails are worth a legend row, and a shape's209// emphasis varies row by row where a collection's features are buckets of many210// rows. So a source states the drawing separately, in the vocabulary of the211// drawing.212type Drawing struct {213 // Zoom is the local level the shapes are drawn at -- the deepest level of214 // the pyramid, and the only one ever drawn, because every shallower level215 // folds down from it.216 Zoom int `json:"zoom"`217 // Shapes are what to draw. Their order is not significant: shapes sharing a218 // role are unioned rather than painted over one another.219 Shapes []Shape `json:"shapes"`220}221222// Shape is one thing to draw, in world pixels of the square.223type Shape struct {224 // Role is what this shape is to the ground -- water, street, boundary --225 // which is the whole vocabulary the renderer knows. A publisher's own layer226 // names never reach it (issue #5 ยง5.1).227 Role string `json:"role"`228 // Rings is ground: the first ring the outline, every ring after it a hole.229 Rings [][][2]float64 `json:"rings,omitempty"`230 // Lines is linework, each entry one polyline.231 Lines [][][2]float64 `json:"lines,omitempty"`232 // Emphasis scales the role's stroke width for this shape alone, so an233 // arterial can be wider than a lane without being a different role. Zero234 // asks for the role's own width.235 Emphasis float64 `json:"emphasis,omitempty"`236}237238// Frame is the captured pyramid as its source declares it: how deep it goes, in239// what encoding, and which tiles each level was asked for.240//241// The zooms and the tile numbers are the publisher's own, not local ones. A242// source publishing into the corpus's shared window speaks zoom 13 and tiles243// 4064..4095; one publishing its own picture speaks whatever its tile server244// does. The deriver halves a frame's deepest window until it comes to rest in a245// single tile, and that level is where local zoom 0 sits -- so no height is246// assumed, and two publishers cutting the same ground from different heights247// both derive correctly.248type Frame struct {249 MinZoom int `json:"minZoom"`250 MaxZoom int `json:"maxZoom"`251 Format string `json:"format"`252 // Windows is the tile window of each captured zoom, keyed by the zoom as a253 // decimal string, because that is how a publisher spells it and how JSON254 // keys a map. A zoom with no window is measured against the corpus's shared255 // square instead.256 Windows map[string]TileWindow `json:"windows"`257}258259// TileWindow is an inclusive rectangle of tiles at one zoom.260type TileWindow struct {261 MinX int `json:"minX"`262 MinY int `json:"minY"`263 MaxX int `json:"maxX"`264 MaxY int `json:"maxY"`265}266267// Collection is an ordered group of features of one kind.268type Collection struct {269 // ID is the collection's numeric identity on the wire. Zero asks270 // composition to derive one from Key.271 ID int64 `json:"id,omitempty"`272 // Key is the collection's curation and merge identity: the slug273 // collections from different sources meet under. A source whose captures274 // have no such concept leaves it empty and is met by its icon key instead.275 Key string `json:"key,omitempty"`276 Title string `json:"title"`277 // Group is the legend section this collection files under, as free text.278 // It is a heading, not an entity: nothing hangs off it.279 Group string `json:"group,omitempty"`280 // Kind is KindPoint, KindPath or KindArea.281 Kind string `json:"kind"`282 // Icon names artwork in the document's icon set.283 Icon string `json:"icon,omitempty"`284 // Color and IconColor are the collection's resolved colours, "#RRGGBB"285 // upper case. A source resolves whatever inheritance its captures have;286 // composition inherits nothing.287 Color string `json:"color,omitempty"`288 IconColor string `json:"iconColor,omitempty"`289 // Visible is whether the collection is drawn before a reader touches290 // anything.291 Visible bool `json:"visible"`292 Attrs map[string]string `json:"attrs,omitempty"`293294 Features []Feature `json:"features,omitempty"`295}296297// Feature is one thing on a world, whatever its dimensionality.298type Feature struct {299 ID int64 `json:"id"`300 Title string `json:"title"`301 Subtitle string `json:"subtitle,omitempty"`302 Description string `json:"description,omitempty"`303 // At is where a point feature stands. Nil for paths and areas.304 At *Position `json:"at,omitempty"`305 // Geometry is what a path or area draws, as GeoJSON parts. Coordinates are306 // carried opaquely: composition never rewrites them, so a source's numbers307 // reach the payload unrounded.308 Geometry []Geometry `json:"geometry,omitempty"`309 // Center is where a shape's label sits, where the source knows better than310 // the centroid.311 Center *Position `json:"center,omitempty"`312 // Member is the area feature whose ground this feature stands on; Parent is313 // the area feature this area sits inside. Zero is neither.314 Member int64 `json:"member,omitempty"`315 Parent int64 `json:"parent,omitempty"`316 // Links are cross-references to other features of the same world. A source317 // resolves whatever link syntax its captures use into these; nothing318 // outward-facing survives, because a bundle serves offline.319 Links []Link `json:"links,omitempty"`320 Attrs map[string]string `json:"attrs,omitempty"`321}322323// Geometry is one GeoJSON part of a shape feature.324type Geometry struct {325 Type string `json:"type"`326 Coordinates json.RawMessage `json:"coordinates"`327}328329// Link is a cross-reference to another feature of the same world.330type Link struct {331 Title string `json:"title"`332 Feature int64 `json:"feature"`333}334335// Icon is one piece of collection artwork, carried whole.336type Icon struct {337 // Key is what a collection names.338 Key string `json:"key"`339 // File is the asset's name inside a bundle's icons directory, including its340 // extension. The extension is load-bearing: it decides whether the artwork341 // is a glyph a reader tints or a picture it draws as-is.342 File string `json:"file"`343 Data []byte `json:"data"`344}345346// Validate reports the first thing structurally wrong with a document. It is347// the shape check a source owes its consumer: composition may assume everything348// here held, and says so by refusing a document that fails.349//350// It deliberately says nothing about semantic conventions. Those are checked351// against the registry at composition, producer-strict, where a failure can352// name the bundle it would have been written into.353func (d Document) Validate() error {354 if d.Doc != Doc {355 return fmt.Errorf("document says %q, not %q", d.Doc, Doc)356 }357 if d.Version != Version {358 return fmt.Errorf("document version %d, want %d", d.Version, Version)359 }360 if d.Volume.Slug == "" {361 return fmt.Errorf("document names no volume")362 }363 if d.Source.Name == "" {364 return fmt.Errorf("document names no source")365 }366 switch d.Source.IDSpace {367 case IDSpaceNative, IDSpaceDerived:368 default:369 return fmt.Errorf("source %s declares id space %q, which is neither %q nor %q",370 d.Source.Name, d.Source.IDSpace, IDSpaceNative, IDSpaceDerived)371 }372 if len(d.Worlds) == 0 {373 return fmt.Errorf("volume %s carries no world", d.Volume.Slug)374 }375 icons := make(map[string]bool, len(d.Icons))376 for _, icon := range d.Icons {377 if icon.Key == "" || icon.File == "" {378 return fmt.Errorf("icon %q carries no key or no file name", icon.Key)379 }380 if icons[icon.Key] {381 return fmt.Errorf("icon %q is carried twice", icon.Key)382 }383 icons[icon.Key] = true384 }385 slugs := make(map[string]bool, len(d.Worlds))386 for _, world := range d.Worlds {387 if world.Slug == "" {388 return fmt.Errorf("world %q carries no slug", world.Title)389 }390 if slugs[world.Slug] {391 return fmt.Errorf("world %s appears twice", world.Slug)392 }393 slugs[world.Slug] = true394 if err := world.validate(); err != nil {395 return fmt.Errorf("world %s: %w", world.Slug, err)396 }397 }398 return nil399}400401func (w World) validate() error {402 if w.Capture.ContentHash == "" {403 return fmt.Errorf("names no capture")404 }405 if w.Capture.CapturedAt == "" {406 return fmt.Errorf("capture %s carries no time", w.Capture.ContentHash)407 }408 if len(w.Lenses) == 0 {409 return fmt.Errorf("has no lens")410 }411 for _, lens := range w.Lenses {412 if lens.TileSet == "" {413 return fmt.Errorf("lens %q names no tile set", lens.Name)414 }415 }416 // One id space per world: the text payload is keyed by feature id across417 // every collection and kind, so two features sharing a number would lose418 // one of their descriptions.419 seen := make(map[int64]string)420 for _, collection := range w.Collections {421 switch collection.Kind {422 case KindPoint, KindPath, KindArea:423 default:424 return fmt.Errorf("collection %q declares kind %q", collection.Title, collection.Kind)425 }426 for _, feature := range collection.Features {427 if feature.ID == 0 {428 return fmt.Errorf("collection %q holds a feature with no id", collection.Title)429 }430 if held, taken := seen[feature.ID]; taken {431 return fmt.Errorf("features %q and %q share id %d", held, feature.Title, feature.ID)432 }433 seen[feature.ID] = feature.Title434 if collection.Kind == KindPoint && feature.At == nil {435 return fmt.Errorf("point feature %q stands nowhere", feature.Title)436 }437 if collection.Kind != KindPoint && len(feature.Geometry) == 0 {438 return fmt.Errorf("%s feature %q draws nothing", collection.Kind, feature.Title)439 }440 }441 }442 return nil443}444445// NewestCapture is the newest capture time across a document's worlds: the time446// a build of it is versioned by. Times compare as strings, which is the order447// that means something for RFC 3339 and needs no parsing.448func (d Document) NewestCapture() string {449 newest := ""450 for _, world := range d.Worlds {451 if world.Capture.CapturedAt > newest {452 newest = world.Capture.CapturedAt453 }454 }455 return newest456}457458// IconByKey indexes the document's artwork.459func (d Document) IconByKey() map[string]Icon {460 out := make(map[string]Icon, len(d.Icons))461 for _, icon := range d.Icons {462 out[icon.Key] = icon463 }464 return out465}466467// Marshal encodes a document the one way it is ever written: indented, with a468// trailing newline, so a debugging dump is readable and two dumps diff line by469// line. Nothing stamps over these bytes -- a bundle's stamp is taken over the470// bundle -- so the encoding is chosen for people.471func (d Document) Marshal() ([]byte, error) {472 data, err := json.MarshalIndent(d, "", " ")473 if err != nil {474 return nil, fmt.Errorf("marshal document: %w", err)475 }476 return append(data, '\n'), nil477}478479// Unmarshal reads a document back and holds it to Validate.480func Unmarshal(data []byte) (Document, error) {481 var d Document482 if err := json.Unmarshal(data, &d); err != nil {483 return Document{}, fmt.Errorf("decode document: %w", err)484 }485 if err := d.Validate(); err != nil {486 return Document{}, err487 }488 return d, nil489}490491// AttrKeys lists an attribute set's keys in sorted order. Producers walk492// attributes this way so a report about two documents reads the same twice.493func AttrKeys(attrs map[string]string) []string {494 keys := make([]string, 0, len(attrs))495 for key := range attrs {496 keys = append(keys, key)497 }498 sort.Strings(keys)499 return keys500}