1// Package merge folds other readings of one volume into the reading that2// serves.3//4// When two sources capture one game, the newest capture is the one a registry5// serves, and the other reading would otherwise ride along unseen in a shadowed6// bundle. Here the serving volume absorbs it: each other reading's ground is7// aligned into the serving world by the transformation their shared named8// places determine -- the same alignment a resampled raster stands on -- and9// every feature is resolved against what the serving world already holds.10//11// A place both readings pin is recorded as one place and never drawn twice. A12// place only the other reading knows is added, into the serving world's own13// collection where the two spell the same concept, and under a collection named14// for its source where they do not. A place the resolution cannot be sure about15// is held back, with the reason written down. The ledger carries the whole16// account, so a merge is a documented derivation over immutable captures rather17// than an edit.18package merge1920import (21 "fmt"22 "log/slog"23 "math"2425 "github.com/FelineStateMachine/atlas/format/semconv"26 "github.com/FelineStateMachine/atlas/internal/enrich"27 "github.com/FelineStateMachine/atlas/internal/enrich/align"28 "github.com/FelineStateMachine/atlas/internal/logging"29)3031// Name is what curation queues this enricher as.32const Name = "merge"3334// Enricher folds donor readings into the serving volume.35type Enricher struct{}3637// New builds the enricher. It carries no state: everything it consults arrives38// in the context, which is what lets one instance serve every volume in a run.39func New() *Enricher { return &Enricher{} }4041func (*Enricher) Name() string { return Name }4243// Declares says what a merge may write onto the serving volume. Prose is not44// here: a description travels through the contribution's own prose operation,45// under a name the conventions registry deliberately does not know, because no46// payload carries it as an attribute.47func (*Enricher) Declares() []string {48 return []string{semconv.KeyGeoLat, semconv.KeyGeoLon, semconv.KeyIconStd}49}5051// Enrich folds every donor reading into the serving volume, in the order the52// donors were given.53//54// Each donor is resolved against the volume as the donors before it left it,55// which is the only order that can be right: a place two donors both know is56// one place, and the second donor has to be able to see that the first already57// contributed it.58func (e *Enricher) Enrich(v *enrich.Volume, ctx enrich.Context) (enrich.Contribution, error) {59 out := enrich.Contribution{Enricher: Name, Volume: v.Slug}60 if len(ctx.Donors) == 0 {61 return out, nil62 }63 log := ctx.Log64 if log == nil {65 log = slog.New(slog.DiscardHandler)66 }67 log = log.With(logging.Op("merge"), logging.Volume(v.Slug))6869 // The donors are folded into a copy, so that each donor sees what the ones70 // before it contributed while the caller's volume stays untouched until the71 // whole contribution is applied.72 working := v.Clone()73 for _, donor := range ctx.Donors {74 if donor.NewestCapture() > working.NewestCapture() {75 return enrich.Contribution{}, fmt.Errorf(76 "%s captured %s is newer than the serving reading captured %s; "+77 "the newest capture serves and the others are folded into it",78 donor.Source.Name, donor.NewestCapture(), working.NewestCapture())79 }80 ops, err := foldVolume(working, donor, ctx, log)81 if err != nil {82 return enrich.Contribution{}, err83 }84 if len(ops) == 0 {85 continue86 }87 step := enrich.Contribution{Enricher: Name, Volume: v.Slug, Ops: ops}88 if err := enrich.Apply(working, step); err != nil {89 return enrich.Contribution{}, err90 }91 out.Ops = append(out.Ops, ops...)92 }93 return out, nil94}9596// foldVolume folds one donor reading into the working volume.97func foldVolume(working, donor *enrich.Volume, ctx enrich.Context, log *slog.Logger) ([]enrich.Op, error) {98 var ops []enrich.Op99 for index := range donor.Worlds {100 donorWorld := &donor.Worlds[index]101 target := working.World(donorWorld.Slug)102 if target == nil {103 // Sources do not divide the world the same way: one reading's104 // separate ground may be ground another draws inside a larger105 // sheet, in its own projection. A shared slug is only the cheapest106 // evidence of shared ground -- the real test is whether the places107 // both name determine a transformation -- so before a world is108 // taken as new, it is tried against every world the volume already109 // draws.110 target = overlapping(working, donorWorld, log)111 }112 if target == nil {113 contributed, err := wholeWorld(working, donor, donorWorld)114 if err != nil {115 return nil, err116 }117 log.Info("world joins whole", logging.World(donorWorld.Slug),118 logging.Source(donor.Source.Name))119 ops = append(ops, contributed...)120 continue121 }122 folded, account, merged, err := foldWorld(working, target, donor, donorWorld, ctx, log)123 if err != nil {124 return nil, err125 }126 if !merged {127 continue128 }129 if err := enrich.GateAccount(account); err != nil {130 return nil, err131 }132 log.Info("world merged", logging.World(target.Slug), logging.Source(donor.Source.Name),133 "offered", account.DonorFeatures.Total(), "matched", account.MatchedN(),134 "enriched", account.EnrichedN(), "added", account.Added,135 "adopted", account.AdoptedN(), "held", account.HeldN(),136 "rejected", account.RejectedN(), "alignment", account.Alignment)137 ops = append(ops, folded...)138 ops = append(ops, enrich.Op{Kind: enrich.OpLedger, World: target.Slug, Account: &account})139 }140 return ops, nil141}142143// overlapping finds the serving world that pictures the same ground as a donor144// world under another slug, if any does: the one whose named places fit the145// donor's through an affine that closes. The fit is the compatibility test the146// differing projections make necessary -- coordinates cannot be compared, but147// the places can.148func overlapping(working *enrich.Volume, donorWorld *enrich.World, log *slog.Logger) *enrich.World {149 donorAnchors := anchorsOf(donorWorld)150 var best *enrich.World151 bestAnchors := 0152 for index := range working.Worlds {153 candidate := &working.Worlds[index]154 _, report, err := align.Fit(donorAnchors, anchorsOf(candidate))155 if err != nil {156 continue157 }158 if report.Anchors > bestAnchors {159 best, bestAnchors = candidate, report.Anchors160 }161 }162 if best != nil {163 log.Info("world pictures ground already drawn", logging.World(donorWorld.Slug),164 "drawn_as", best.Slug, "anchors", bestAnchors)165 }166 return best167}168169// wholeWorld carries one reading's ground into the volume untouched: its170// collections, features and prose as captured, and its artwork brought across171// -- under its own key where that key is free or already holds the same bytes,172// and under a source-prefixed key where the volume spells something else with173// it.174func wholeWorld(working, donor *enrich.Volume, donorWorld *enrich.World) ([]enrich.Op, error) {175 contributed := donorWorld.Clone()176 if len(contributed.Ledger) == 0 {177 // A ground that joins whole opens its own account, exactly as it would178 // have if it had been composed on its own: it came from this reading,179 // and it arrived holding this.180 contributed.Ledger = []enrich.Account{{181 Source: donor.Source.Label,182 Slug: donor.Source.Name,183 Origin: true,184 DonorFeatures: enrich.Tally(&contributed),185 }}186 }187 tag := slugify(donor.Source.Label)188 carried := newArtwork(working, donor, tag, false)189 for index := range contributed.Collections {190 collection := &contributed.Collections[index]191 key, err := carried.carry(collection.Icon)192 if err != nil {193 return nil, err194 }195 collection.Icon = key196 }197 ops := carried.ops198 return append(ops, enrich.Op{Kind: enrich.OpAddWorld, NewWorld: &contributed}), nil199}200201// foldWorld resolves one donor world into one serving world.202func foldWorld(203 working *enrich.Volume,204 serving *enrich.World,205 donor *enrich.Volume,206 donorWorld *enrich.World,207 ctx enrich.Context,208 log *slog.Logger,209) ([]enrich.Op, enrich.Account, bool, error) {210 if !serving.Grid.Ready() || !donorWorld.Grid.Ready() {211 return nil, enrich.Account{}, false, fmt.Errorf(212 "world %s: a merge measures distance, and one of the two readings is cut from no window",213 serving.Slug)214 }215 affine, report, err := align.Fit(anchorsOf(donorWorld), anchorsOf(serving))216 if err != nil {217 // A fit that will not close is a merge that does not happen. The two218 // readings stay apart, whole, in separate builds, which is exactly what219 // they were before anybody tried -- and a person should eventually hear220 // that two readings of one ground could not be brought together.221 log.Warn("readings stay apart", logging.World(serving.Slug),222 logging.Source(donor.Source.Name), "reason", err.Error())223 return nil, enrich.Account{}, false, nil224 }225226 account := enrich.Account{227 Source: donor.Source.Label,228 Slug: donor.Source.Name,229 Alignment: report.String(),230 }231 // A nameless neighbour matters out to where the alignment's own noise232 // reaches: two readings place the same shop apart by their residuals, so233 // the radius listens to the fit rather than assuming precision.234 nearbyRadius := math.Max(ctx.Curation.NearbyFloorPx(), 2*report.P90Px)235 index := indexServing(serving)236 // Artwork carried into a world that already draws its own always lands237 // under a source-tagged file, so a contributed picture can never be238 // written over one the serving reading ships under the same name.239 carried := newArtwork(working, donor, slugify(donor.Source.Label), true)240241 var ops []enrich.Op242 var kept []enrich.Collection243 for _, donorCollection := range donorWorld.Collections {244 if donorCollection.Kind != enrich.KindPoint {245 // Matching is point-only, but the ledger is not: every donor path246 // and area feature is held on the record, one line each with its247 // title, rather than dropped without a word.248 for _, shape := range donorCollection.Features {249 if donorCollection.Kind == enrich.KindPath {250 account.DonorFeatures.Path++251 } else {252 account.DonorFeatures.Area++253 }254 account.Held = append(account.Held, enrich.HeldItem{255 Donor: shape.ID, Title: shape.Title, Reason: enrich.HeldShapeReason,256 })257 }258 continue259 }260 identity := enrich.MergeIdentity(donorCollection)261 adoptive := index.collections[identity]262263 // Attribute-level resolution at the collection: a serving collection264 // with no artwork of any kind takes the donor's standard-icon265 // declaration, and the ledger says so.266 if adoptive != nil && ctx.Curation.DonorFillsEmpty(semconv.KeyIconStd) &&267 adoptive.IconAsset == "" && adoptive.Icon == "" &&268 adoptive.Attrs[semconv.KeyIconStd] == "" &&269 donorCollection.Attrs[semconv.KeyIconStd] != "" {270 ops = append(ops, enrich.Op{271 Kind: enrich.OpSetAttr,272 World: serving.Slug,273 Collection: adoptive.ID,274 Entity: semconv.EntityCollection,275 Key: semconv.KeyIconStd,276 Value: donorCollection.Attrs[semconv.KeyIconStd],277 })278 account.Enriched = append(account.Enriched, enrich.CollectionTake{279 Collection: identity, Key: semconv.KeyIconStd,280 })281 }282283 keeping := donorCollection284 keeping.Features = nil285 for _, feature := range donorCollection.Features {286 account.DonorFeatures.Point++287 if feature.At == nil {288 account.Rejected = append(account.Rejected, enrich.HeldItem{289 Donor: feature.ID, Title: feature.Title, Reason: placelessReason,290 })291 continue292 }293 x, y := affine.Apply(294 donorWorld.Grid.ProjectX(feature.At.Lng),295 donorWorld.Grid.ProjectY(feature.At.Lat),296 )297 outcome := resolve(feature, x, y, index, identity, nearbyRadius, ctx.Curation)298 switch outcome.kind {299 case matched:300 index.claimed[outcome.match.ID] = feature.ID301 pair := enrich.MatchedPair{302 Donor: feature.ID,303 Winner: outcome.match.ID,304 DistancePx: int(outcome.distance + 0.5),305 }306 // Attribute-level: the policy table says, key by key, what a307 // serving feature takes from its counterpart, and every take is308 // ledgered. The description travels under its policy name, and309 // true coordinates fill in the same way words do: only where310 // the serving side has none of its own.311 if ctx.Curation.DonorFillsEmpty(semconv.KeyNoteText) &&312 outcome.match.Description == "" && feature.Description != "" {313 ops = append(ops, enrich.Op{314 Kind: enrich.OpSetProse,315 World: serving.Slug,316 Feature: outcome.match.ID,317 Value: feature.Description,318 })319 pair.Enriched = true320 pair.Took = append(pair.Took, semconv.KeyNoteText)321 }322 for _, key := range []string{semconv.KeyGeoLat, semconv.KeyGeoLon} {323 if !ctx.Curation.DonorFillsEmpty(key) {324 continue325 }326 if feature.Attrs[key] == "" || outcome.match.Attrs[key] != "" {327 continue328 }329 ops = append(ops, enrich.Op{330 Kind: enrich.OpSetAttr,331 World: serving.Slug,332 Feature: outcome.match.ID,333 Entity: semconv.EntityFeature,334 Key: key,335 Value: feature.Attrs[key],336 })337 pair.Took = append(pair.Took, key)338 }339 account.Matched = append(account.Matched, pair)340 case held:341 account.Held = append(account.Held, enrich.HeldItem{342 Donor: feature.ID, Title: feature.Title, Reason: outcome.reason,343 })344 case distinct:345 if x < 0 || y < 0 || x > float64(serving.Grid.Size) || y > float64(serving.Grid.Size) {346 account.Rejected = append(account.Rejected, enrich.HeldItem{347 Donor: feature.ID, Title: feature.Title,348 Reason: outsideReason,349 })350 continue351 }352 if holder, taken := index.ids[feature.ID]; taken {353 return nil, enrich.Account{}, false, fmt.Errorf(354 "feature id %d (%s) collides with serving feature %q",355 feature.ID, feature.Title, holder)356 }357 index.ids[feature.ID] = feature.Title358 moved := feature.Clone()359 moved.At = &enrich.Position{360 Lat: serving.Grid.UnprojectLat(y),361 Lng: serving.Grid.UnprojectLng(x),362 }363 // A contributed feature stands on the serving ground, not on364 // whatever ground it stood on in the reading it came from.365 moved.Member = 0366 account.Added++367 if adoptive != nil {368 ops = append(ops, enrich.Op{369 Kind: enrich.OpAddFeature,370 World: serving.Slug,371 Collection: adoptive.ID,372 NewFeature: &moved,373 })374 account.Adopted = append(account.Adopted, enrich.AdoptedItem{375 Donor: feature.ID, Into: identity,376 })377 continue378 }379 keeping.Features = append(keeping.Features, moved)380 }381 }382 if len(keeping.Features) == 0 {383 continue384 }385 kept = append(kept, keeping)386 }387388 for index := range kept {389 collection := &kept[index]390 // The kept collections file under a group named for their source, so391 // the legend says where they came from without machinery of its own.392 collection.Group = donor.Source.Label393 key, err := carried.carry(collection.Icon)394 if err != nil {395 return nil, enrich.Account{}, false, err396 }397 collection.Icon = key398 collection.IconAsset = ""399 collection.IconPicture = false400 ops = append(ops, enrich.Op{401 Kind: enrich.OpAddCollection, World: serving.Slug, NewCollection: collection,402 })403 }404 return append(carried.ops, ops...), account, true, nil405}406407// servingIndex is everything the serving world holds, arranged for resolution:408// each point feature with its place, name and name tokens; the collections by409// merge identity; and the identifiers already spoken for.410type servingIndex struct {411 features []placed412 byName map[string][]int413 collections map[string]*enrich.Collection414 ids map[int64]string415 // claimed maps each serving feature already matched to the donor that416 // matched it. A place is one place: the next donor bearing the same name417 // must find its own, or say why it cannot.418 claimed map[int64]int64419}420421type placed struct {422 feature *enrich.Feature423 identity string424 x, y float64425 tokens map[string]bool426}427428func indexServing(serving *enrich.World) *servingIndex {429 index := &servingIndex{430 byName: make(map[string][]int),431 collections: make(map[string]*enrich.Collection),432 ids: make(map[int64]string),433 claimed: make(map[int64]int64),434 }435 for collectionIndex := range serving.Collections {436 collection := &serving.Collections[collectionIndex]437 if collection.Kind != enrich.KindPoint {438 for _, feature := range collection.Features {439 index.ids[feature.ID] = feature.Title440 }441 continue442 }443 identity := enrich.MergeIdentity(*collection)444 if _, held := index.collections[identity]; !held && identity != "" {445 index.collections[identity] = collection446 }447 for featureIndex := range collection.Features {448 feature := &collection.Features[featureIndex]449 name := align.NormalizeTitle(feature.Title)450 index.byName[name] = append(index.byName[name], len(index.features))451 var x, y float64452 if feature.At != nil {453 x = serving.Grid.ProjectX(feature.At.Lng)454 y = serving.Grid.ProjectY(feature.At.Lat)455 }456 index.features = append(index.features, placed{457 feature: feature,458 identity: identity,459 x: x,460 y: y,461 tokens: align.Tokens(name),462 })463 index.ids[feature.ID] = feature.Title464 }465 }466 return index467}468469// The reasons a feature is held back or refused.470//471// These strings are frozen, letter for letter. They are not messages about the472// code: they are data, written into the ledger of every merged bundle already473// published, and a reader comparing two builds' accounts compares them474// directly. Their vocabulary is a vocabulary behind the rest of this lane -- a475// feature is a "pin" here, a collection a "category" -- and that is the price476// of not rewriting what every bundle in every library already says about477// itself.478const (479 namedLikeReason = "named like %q %.0fpx away; too far to merge, too near to double"480 besideReason = "beside %q in the same category; names disagree"481 alreadyMatchedReason = "every nearby pin of this name is already matched"482 outsideReason = "outside the shared world"483 placelessReason = "stands nowhere"484)485486// The outcomes one donor feature may have.487const (488 distinct = iota489 matched490 held491)492493type outcome struct {494 kind int495 match *enrich.Feature496 distance float64497 reason string498}499500// resolve decides what one donor feature is, against everything the serving501// world holds.502//503// The same name near where the alignment predicts is the same place, and so is504// a name one reading spells inside the other's -- "Northside Apartment" inside505// "Northside, Watson Apartment" -- when the two share a collection. The same506// name far beyond the radius is a different place bearing it; the stretch507// between is left undecided. A nameless neighbour inside the same collection is508// likewise held rather than guessed -- proximity alone never merges -- and only509// a feature resembling nothing at all is added.510func resolve(511 donor enrich.Feature,512 x, y float64,513 index *servingIndex,514 identity string,515 nearbyRadius float64,516 tables enrich.Curation,517) outcome {518 name := align.NormalizeTitle(donor.Title)519 donorTokens := align.Tokens(name)520521 var nearest *placed522 nearestDistance := math.Inf(1)523 alreadyClaimed := false524 consider := func(candidate *placed, distance float64) {525 // A serving feature another donor already resolved to is one place, not526 // two: it cannot be matched again, but its nearness is remembered so527 // the refusal can say what stood in the way.528 if _, taken := index.claimed[candidate.feature.ID]; taken {529 if distance <= tables.MatchRadiusPx() {530 alreadyClaimed = true531 }532 return533 }534 if distance < nearestDistance {535 nearest, nearestDistance = candidate, distance536 }537 }538 if name != "" {539 for _, at := range index.byName[name] {540 candidate := &index.features[at]541 consider(candidate, math.Hypot(candidate.x-x, candidate.y-y))542 }543 // One reading's name written inside the other's counts only with the544 // collection agreeing: a bare "Apartment" must not roam the world for a545 // long-named cousin.546 if nearest == nil && len(donorTokens) >= 2 {547 for at := range index.features {548 candidate := &index.features[at]549 if candidate.identity != identity || !tokenSubset(donorTokens, candidate.tokens) {550 continue551 }552 consider(candidate, math.Hypot(candidate.x-x, candidate.y-y))553 }554 }555 }556 if nearest != nil {557 switch {558 case nearestDistance <= tables.MatchRadiusPx():559 return outcome{kind: matched, match: nearest.feature, distance: nearestDistance}560 case nearestDistance <= tables.SeparateRadiusPx():561 return outcome{kind: held, reason: fmt.Sprintf(namedLikeReason,562 nearest.feature.Title, nearestDistance)}563 }564 }565 if alreadyClaimed {566 return outcome{kind: held, reason: alreadyMatchedReason}567 }568 if _, shared := index.collections[identity]; shared {569 for at := range index.features {570 candidate := &index.features[at]571 if candidate.identity != identity {572 continue573 }574 if math.Hypot(candidate.x-x, candidate.y-y) <= nearbyRadius {575 return outcome{kind: held, reason: fmt.Sprintf(besideReason, candidate.feature.Title)}576 }577 }578 }579 return outcome{kind: distinct}580}581582// tokenSubset reports whether one name is spelled entirely inside the other,583// either way round. The shorter side must carry at least two words: a single584// word inside a longer name -- every "Apartment" inside every apartment -- says585// nothing.586func tokenSubset(a, b map[string]bool) bool {587 small, big := a, b588 if len(small) > len(big) {589 small, big = big, small590 }591 if len(small) < 2 {592 return false593 }594 for token := range small {595 if !big[token] {596 return false597 }598 }599 return true600}601602func anchorsOf(w *enrich.World) []align.Anchor {603 var anchors []align.Anchor604 for _, collection := range w.Collections {605 if collection.Kind != enrich.KindPoint {606 continue607 }608 for _, feature := range collection.Features {609 if feature.At == nil {610 continue611 }612 anchors = append(anchors, align.Anchor{613 Title: feature.Title,614 X: w.Grid.ProjectX(feature.At.Lng),615 Y: w.Grid.ProjectY(feature.At.Lat),616 })617 }618 }619 return anchors620}621622// artwork carries a donor's icons across, so that nothing a donor ships can623// displace what the serving reading already drew.624//625// Two names travel with one picture, and they are tagged on different rules626// because they answer different questions. The **file** is where the bytes land627// in the volume's icons directory, and a contribution's bytes may never land on628// the serving volume's: when a donor's collections are folded into a ground the629// serving reading already draws, the file always takes the source's tag. The630// **key** is what a collection names its artwork by, and it stays the donor's631// own -- there is nothing to displace unless the serving volume already names632// different artwork with it, and only then is the key tagged too.633type artwork struct {634 into *enrich.Volume635 donor *enrich.Volume636 tag string637 // tagFiles marks a contribution into a volume that is already drawing:638 // every file it carries is tagged, whether or not the name was free.639 tagFiles bool640 byKey map[string]enrich.Icon641 renamed map[string]string642 ops []enrich.Op643}644645func newArtwork(into, donor *enrich.Volume, tag string, tagFiles bool) *artwork {646 byKey := make(map[string]enrich.Icon, len(donor.Icons))647 for _, icon := range donor.Icons {648 byKey[icon.Key] = icon649 }650 return &artwork{651 into: into, donor: donor, tag: tag, tagFiles: tagFiles,652 byKey: byKey, renamed: map[string]string{},653 }654}655656// carry brings one icon across and answers with the key the contributed657// collection should name. An icon the donor does not actually hold answers with658// no key: a collection with artwork nobody shipped draws without it, which is659// what it did in the reading it came from.660func (a *artwork) carry(key string) (string, error) {661 if key == "" {662 return "", nil663 }664 if replacement, seen := a.renamed[key]; seen {665 return replacement, nil666 }667 icon, held := a.byKey[key]668 if !held {669 a.renamed[key] = ""670 return "", nil671 }672 name, file := key, icon.File673 if a.tagFiles {674 file = a.tag + "--" + icon.File675 }676 for _, standing := range a.into.Icons {677 if standing.Key != key {678 continue679 }680 if string(standing.Data) == string(icon.Data) {681 // The same artwork under the same name is the same artwork.682 a.renamed[key] = key683 return key, nil684 }685 // The name is spoken for by something else, so the contribution686 // answers to a tagged one instead.687 name = a.tag + "--" + key688 file = a.tag + "--" + icon.File689 break690 }691 carried := enrich.Icon{Key: name, File: file, Data: icon.Data}692 a.ops = append(a.ops, enrich.Op{Kind: enrich.OpAddAsset, Asset: &carried})693 a.renamed[key] = name694 return name, nil695}696697// slugify folds a source's label into the shape a file name may carry.698func slugify(label string) string {699 out := make([]rune, 0, len(label))700 for _, r := range label {701 switch {702 case r >= 'a' && r <= 'z' || r >= '0' && r <= '9':703 out = append(out, r)704 case r >= 'A' && r <= 'Z':705 out = append(out, r+'a'-'A')706 default:707 if len(out) > 0 && out[len(out)-1] != '-' {708 out = append(out, '-')709 }710 }711 }712 for len(out) > 0 && out[len(out)-1] == '-' {713 out = out[:len(out)-1]714 }715 return string(out)716}