1// Package compose turns an interchange document into a valid volume.2//3// It is the second half of the generate lane: a source has said what it has to4// say, and composition makes a .atlas file of it. That means normalizing to the5// format's unified model, speaking the semantic conventions on the volume's6// behalf and holding every attribute to the registry producer-strict, splitting7// what needs splitting, packing the three payload parts and the tile pyramids,8// stamping the result, and writing it into a registry so carefully that a reader9// scanning mid-write sees either no file or a whole one.10//11// Composition is single-source. What one source said travels here whole and12// unmixed; folding two sources' readings of one volume together is the enrich13// lane's work (docs/enrich.md).14//15// # Determinism16//17// The same document and the same tile set give the same bundle, byte for byte,18// on any machine -- the same stamp, and therefore the same file name. Nothing19// here reads a clock: a build's creation time is the newest capture time in its20// document, so a rebuild years later of an archive nobody touched writes the21// file that is already there. That is the whole reason a directory of these22// files can stand as a registry.23package compose2425import (26 "encoding/json"27 "fmt"28 "log/slog"29 "sort"3031 "github.com/FelineStateMachine/atlas/format/semconv"32 "github.com/FelineStateMachine/atlas/internal/generate/curation"33 "github.com/FelineStateMachine/atlas/internal/generate/doc"34 "github.com/FelineStateMachine/atlas/internal/generate/icons"35 "github.com/FelineStateMachine/atlas/internal/generate/tiles"36 "github.com/FelineStateMachine/atlas/internal/logging"37)3839// PolicyRevision orders builds of the same capture. The data has not moved40// between them, but what this lane makes of a capture has -- a merge rule, a41// kept field, a dropped collection -- and among equal captures a registry42// serves the highest revision. It is bumped when a policy change should43// supersede the builds already in every library.44//45// 1 first revisioned builds46// 2 merge resolution: subset names, adoption, one-to-one matches47// 3 origin provenance on every world; overlap merges across world slugs48// 4 semantic conventions: attributes ride every payload49// 5 standard icons resolved for collections that declare one50// 6 geometry declared: spheres say so, pins carry true coordinates51// 7 attribute-level merge resolution; ledgers name canonical source slugs52// 8 shape prose defers to the text payload; shapes mark hasText53// 9 format v3: one collections array of point, path, and area features54// 10 native vNext semantic tables, embedded schema, and opaque raster blobs55const PolicyRevision = 105657// Options is one composition.58type Options struct {59 // Document is the source's reading of the volume.60 Document doc.Document61 // Tiles is the derived tile set the volume's lenses are drawn from.62 Tiles *tiles.Set63 // Curation is the editorial data the composition consults.64 Curation curation.Tables65 // Ledger is a world's whole provenance, by world slug, where the caller has66 // one to hand. It is how the enrich lane's accounts reach a payload without67 // either lane importing the other: the accounts arrive already serialized68 // and composition writes them as they are. A world the caller says nothing69 // about gets the single origin account composition opens itself.70 Ledger map[string][]json.RawMessage71 // Revision is the build revision the manifest carries. Zero is this lane's72 // own PolicyRevision, which is what a plain single-source build writes; the73 // enrich lane hands its own, which is how an enriched build of one capture74 // deterministically outranks the plain build beside it.75 Revision int76 // BundleDir is the registry the finished bundle is installed into. Empty77 // composes and validates without writing anything, which is what a check78 // run wants.79 BundleDir string80 // Log is the run's event stream; nil is a logger that discards.81 Log *slog.Logger82}8384// Result is what a composition produced.85type Result struct {86 Volume string87 Stamp string88 // File is the name the build carries in a registry.89 File string90 // Path is where it was written, empty when nothing was written.91 Path string92 // Present reports that the build was already in the registry, so nothing93 // was written. It is the normal outcome of recomposing an unchanged94 // archive, and it is what makes a rebuild cheap.95 Present bool96 // Worlds, Tiles and Icons are what the bundle holds.97 Worlds, Tiles, Icons int98}99100// composedWorld is one world under composition: the document's world, resolved101// against a tile set and curation, on its way to a payload.102type composedWorld struct {103 ID int64104 Slug string105 Title string106 Parent string107 IconOutset string108 Center doc.Position109 CapturedAt string110 Grid *worldGrid111 Lenses []lens112 // Pyramids is the tile set entry behind each lens, in the same order.113 // A lens carries only what a payload says; a pyramid carries what a stamp114 // and a copy need.115 Pyramids []tiles.Pyramid116 Collections []composedCollection117 Attrs map[string]string118 Merged []json.RawMessage119}120121// composedCollection is a document collection with what composition knows added122// to it: the artwork it resolved, and the numbers a wire needs.123type composedCollection struct {124 ID int64125 Key string126 Title string127 Group string128 Kind string129 Icon string130 Color string131 IconColor string132 Visible bool133 Attrs map[string]string134 // IconAsset is the artwork's name inside the bundle, empty where the135 // document carried none.136 IconAsset string137 // IconPicture says the artwork is a picture drawn as-is rather than a glyph138 // the reader tints.139 IconPicture bool140 Features []composedFeature141}142143// composedFeature is a document feature with its place in a split world.144type composedFeature struct {145 doc.Feature146 // Shard is the layer of a split world this feature belongs to, zero when147 // the world is whole. It is composition's to decide: a source describes one148 // sheet, and whether that sheet is really several places is curation.149 Shard int64150}151152// Compose builds a volume and, where a registry is named, installs it.153func Compose(o Options) (Result, error) {154 log := o.Log155 if log == nil {156 log = slog.New(slog.DiscardHandler)157 }158 log = log.With(logging.Op("compose"), logging.Volume(o.Document.Volume.Slug))159160 if err := o.Document.Validate(); err != nil {161 return Result{}, fmt.Errorf("document: %w", err)162 }163 if o.Tiles == nil {164 return Result{}, fmt.Errorf("volume %s: no tile set to draw from", o.Document.Volume.Slug)165 }166 grid := worldGrid{167 SourceZoom: o.Curation.Window.SourceZoom,168 FirstTile: o.Curation.Window.FirstTile,169 }170171 var worlds []composedWorld172 for _, source := range o.Document.Worlds {173 pieces, err := resolveWorld(source, o, grid, log)174 if err != nil {175 return Result{}, fmt.Errorf("world %s: %w", source.Slug, err)176 }177 worlds = append(worlds, pieces...)178 }179 if len(worlds) == 0 {180 return Result{}, fmt.Errorf("volume %s: no world is ready to compose", o.Document.Volume.Slug)181 }182 orderWorlds(o.Curation, o.Document.Volume.Slug, worlds)183 artwork := attachArtwork(o.Document, worlds)184 if err := resolveStandardIcons(worlds, artwork); err != nil {185 return Result{}, fmt.Errorf("volume %s: %w", o.Document.Volume.Slug, err)186 }187 if err := speakConventions(o.Curation, o.Document.Volume.Slug, worlds); err != nil {188 return Result{}, fmt.Errorf("volume %s: %w", o.Document.Volume.Slug, err)189 }190 return write(o, worlds, artwork, log)191}192193// resolveStandardIcons makes good on every atlas.icon.std declaration: the named194// library glyph lands in the volume's artwork under its provenance-spelling195// name, and the collection wears it exactly the way it would wear artwork the196// source shipped itself.197//198// A source that has no artwork at all -- a gazetteer, a city's open data --199// names a glyph rather than inventing one, and naming is all it may do: which200// picture answers to the name is the library's business, and packing it is201// composition's. It runs after the document's own artwork has been attached and202// only where a source's own icon has not already won the slot, because a203// publisher's own drawing of a thing always beats a generic one.204//205// A declaration the library cannot answer fails the build. The promise was made206// in a translator; it is kept here or heard about, rather than reaching a207// reader as a collection with a hole where its marker should be.208func resolveStandardIcons(worlds []composedWorld, artwork map[string][]byte) error {209 for worldIndex := range worlds {210 world := &worlds[worldIndex]211 for index := range world.Collections {212 collection := &world.Collections[index]213 if collection.Kind != doc.KindPoint {214 continue215 }216 ref := collection.Attrs[semconv.KeyIconStd]217 if ref == "" || collection.IconAsset != "" {218 continue219 }220 data, asset, err := icons.Standard(ref)221 if err != nil {222 return fmt.Errorf("world %s collection %q: %w", world.Slug, collection.Title, err)223 }224 artwork[asset] = data225 collection.IconAsset = asset226 collection.IconPicture = false227 }228 }229 return nil230}231232// resolveWorld turns one document world into the worlds under composition it233// becomes: its lenses found in the tile set, its collections numbered, a curated234// sheet taken apart, the ground each piece covers measured, and every piece's235// own account of where it came from opened.236//237// It answers with a slice because one sheet may hold several places. All but one238// volume in the corpus answer with exactly one world, and the one that does not239// is declared in curation rather than detected.240func resolveWorld(source doc.World, o Options, shared worldGrid, log *slog.Logger) ([]composedWorld, error) {241 out := composedWorld{242 ID: source.ID,243 Slug: source.Slug,244 Title: source.Title,245 IconOutset: o.Curation.IconOutset(o.Document.Volume.Slug, source.Slug),246 Center: source.Center,247 CapturedAt: source.Capture.CapturedAt,248 Attrs: source.Attrs,249 }250 window := shared251 for index, declared := range source.Lenses {252 pyramid, ok := o.Tiles.Native(declared.TileSet)253 if !ok {254 return nil, fmt.Errorf("no pyramid is derived from tile set %s", declared.TileSet)255 }256 found := worldGrid{SourceZoom: pyramid.Window.SourceZoom, FirstTile: pyramid.Window.FirstTile}257 if found == (worldGrid{}) {258 // A tile set derived before pyramids carried their window: the259 // shared one is what everything in it was cut from.260 found = shared261 }262 // Every lens of a world is a picture of the same ground, so they agree263 // on the window it is cut from. One that did not would be a world drawn264 // in two places at once, and no feature could be placed on it.265 if index == 0 {266 window = found267 if found != shared {268 grid := found269 out.Grid = &grid270 }271 } else if found != window {272 return nil, fmt.Errorf(273 "tile set %s sits in a different window from %s",274 declared.TileSet, source.Lenses[0].TileSet)275 }276 out.Lenses = append(out.Lenses, lensOf(declared.Name, pyramid, o.Document.Volume.Slug))277 out.Pyramids = append(out.Pyramids, pyramid)278 // Another source's raster, resampled into this world, arrives as one279 // more way to see the same ground. It was rendered in this world's280 // window, so it passes the same agreement every native lens passes.281 for _, aligned := range o.Tiles.Aligned(declared.TileSet) {282 alignedWindow := worldGrid{283 SourceZoom: aligned.Window.SourceZoom,284 FirstTile: aligned.Window.FirstTile,285 }286 if alignedWindow != window {287 return nil, fmt.Errorf(288 "aligned pyramid %s was rendered in a different window", aligned.Name)289 }290 out.Lenses = append(out.Lenses, lensOf(aligned.LensName, aligned, o.Document.Volume.Slug))291 out.Pyramids = append(out.Pyramids, aligned)292 }293 }294295 collections, err := numberCollections(source)296 if err != nil {297 return nil, err298 }299 out.Collections = collections300301 grid := surfaceGrid{302 SourceZoom: window.SourceZoom,303 FirstTile: window.FirstTile,304 TileSize: o.Tiles.TileSize,305 Size: o.Tiles.Size,306 }307 // A sheet holding several separate places is taken apart here, where the308 // window it was cut from is finally known, and before the ground each piece309 // covers is measured: a piece's surface is its own, not the sheet's.310 mode := o.Curation.Shard(o.Document.Volume.Slug, source.Slug)311 pieces, err := splitWorld(out, mode, grid)312 if err != nil {313 return nil, fmt.Errorf("splitting into %s: %w", mode, err)314 }315 for index := range pieces {316 markSurfaces(&pieces[index], grid)317 // Every world opens its account with where it came from, split or318 // not: provenance is part of a world, not a side effect of319 // composition. A caller with a whole ledger to hand — the enrich320 // lane's ⊕ — supplies it by world slug, already serialized, and it321 // is written as it is.322 if ledger, held := o.Ledger[pieces[index].Slug]; held && len(ledger) > 0 {323 pieces[index].Merged = ledger324 } else {325 account, err := json.Marshal(origin{326 Source: o.Document.Source.Label,327 Slug: o.Document.Source.Name,328 Origin: true,329 DonorFeatures: tally(pieces[index].Collections),330 })331 if err != nil {332 return nil, fmt.Errorf("marshal origin account: %w", err)333 }334 pieces[index].Merged = []json.RawMessage{account}335 }336 log.Debug("world composed", logging.World(pieces[index].Slug),337 "lenses", len(pieces[index].Lenses),338 "collections", len(pieces[index].Collections))339 }340 return pieces, nil341}342343func lensOf(name string, p tiles.Pyramid, volume string) lens {344 return lens{345 Name: name,346 Tiles: tiles.LocalName(volume, p.Name),347 MinZoom: p.MinZoom,348 MaxZoom: p.MaxZoom,349 FullZoom: p.FullZoom,350 SourceZoom: p.SourceZoom,351 Formats: p.Formats,352 Bounds: p.Bounds,353 Interpolate: p.Interpolate,354 Background: p.Background,355 Coverage: p.Coverage,356 }357}358359// numberCollections gives every collection a numeric identity on the wire.360// A source that has native numbers passes them through; one that declares only361// a key gets a number derived from that key and the world it sits in, because362// a reader's hide and unfold sets are keyed by number and a key is not one.363//364// The derivation is FNV-1a over "<world id>:collection:<key>", masked into the365// positive range of a signed 32-bit integer, with zero -- which the wire reads366// as absence -- moved to one. A collision with a number the source already367// claimed fails the build rather than quietly renaming somebody's collection.368func numberCollections(source doc.World) ([]composedCollection, error) {369 used := make(map[int64]string, len(source.Collections))370 for _, collection := range source.Collections {371 if collection.ID != 0 {372 used[collection.ID] = collection.Title373 }374 }375 out := make([]composedCollection, 0, len(source.Collections))376 for _, collection := range source.Collections {377 id := collection.ID378 if id == 0 {379 if collection.Key == "" {380 return nil, fmt.Errorf("collection %q carries neither an id nor a key", collection.Title)381 }382 claimed, err := claimID(source.ID, collection.Key, used)383 if err != nil {384 return nil, err385 }386 id = claimed387 }388 features := make([]composedFeature, 0, len(collection.Features))389 for _, feature := range collection.Features {390 features = append(features, composedFeature{Feature: feature})391 }392 out = append(out, composedCollection{393 ID: id,394 Key: collection.Key,395 Title: collection.Title,396 Group: collection.Group,397 Kind: collection.Kind,398 Icon: collection.Icon,399 Color: collection.Color,400 IconColor: collection.IconColor,401 Visible: collection.Visible,402 Attrs: collection.Attrs,403 Features: features,404 })405 }406 return out, nil407}408409// orderWorlds puts a volume's primary world first and keeps a split sheet's410// pieces with the sheet they came from.411func orderWorlds(tables curation.Tables, volume string, worlds []composedWorld) {412 order := make(map[string]int)413 for index, slug := range tables.PreferredWorlds(volume) {414 order[slug] = index415 }416 // A version-history volume reads its date titles backward: the newest417 // capture opens, and the past waits one click below it.418 before := func(a, b string) bool { return a < b }419 if tables.NewestFirst(volume) {420 before = func(a, b string) bool { return a > b }421 }422 titles := make(map[string]string, len(worlds))423 for _, world := range worlds {424 titles[world.Slug] = world.Title425 }426 // Worlds sort as families: a piece carries its sheet's position and follows427 // it, so a split sheet stays together in the picker.428 family := func(w composedWorld) (string, string) {429 if w.Parent == "" {430 return w.Slug, w.Title431 }432 return w.Parent, titles[w.Parent]433 }434 sort.SliceStable(worlds, func(i, j int) bool {435 leftSlug, leftTitle := family(worlds[i])436 rightSlug, rightTitle := family(worlds[j])437 left, leftPreferred := order[leftSlug]438 right, rightPreferred := order[rightSlug]439 if leftPreferred != rightPreferred {440 return leftPreferred441 }442 if leftPreferred && left != right {443 return left < right444 }445 if leftTitle != rightTitle {446 return before(leftTitle, rightTitle)447 }448 if (worlds[i].Parent == "") != (worlds[j].Parent == "") {449 return worlds[i].Parent == ""450 }451 return before(worlds[i].Title, worlds[j].Title)452 })453}454455// attachArtwork resolves every point collection's artwork against the456// document's icon set, and reports the assets the bundle will carry. Artwork457// nothing names is left behind: a source may hand over a whole archived icon458// set, and a bundle carries what is used.459func attachArtwork(d doc.Document, worlds []composedWorld) map[string][]byte {460 byKey := d.IconByKey()461 out := make(map[string][]byte)462 for worldIndex := range worlds {463 collections := worlds[worldIndex].Collections464 for index := range collections {465 collection := &collections[index]466 if collection.Kind != doc.KindPoint || collection.Icon == "" {467 continue468 }469 icon, held := byKey[collection.Icon]470 if !held {471 continue472 }473 out[icon.File] = icon.Data474 collection.IconAsset = icon.File475 collection.IconPicture = isPicture(icon.File)476 }477 }478 return out479}480481// isPicture reads the one thing an asset's name decides: whether the artwork is482// a glyph a reader may tint, or a picture that must be drawn as it is.483func isPicture(file string) bool {484 return len(file) >= 4 && file[len(file)-4:] == ".png"485}