1// Package maturity scores a build of a volume.2//3// # What a score is4//5// It is unbounded, additive, and monotone. Every feature earns points for each6// quality it verifiably has -- a name, prose, prose with something in it,7// resolved coordinates, a membership its ground was joined to, a collection8// that resolves artwork, geometry actually drawn, another reading that9// corroborates it -- and those sums roll up through collections and worlds to10// the volume. Collections earn for the conventions they declare; worlds earn11// for what they say about their surface and for how much raster a reader can12// actually get to.13//14// There are no denominators and no ceilings, and that is the whole of why this15// scores points rather than percentages: a share moves when its denominator16// moves, so a build that added five hundred features and described half of them17// reads as a regression. The five absolute axes survive as diagnostics -- good18// for reading, useless as a gate -- and a described *share* above 100% is not19// a defect that can be reached from here, because nothing is divided by20// anything.21//22// # The gate23//24// An enrichment build whose score declines fails (see [Gate]). The comparison25// is only ever between builds scored under the same table version: a26// re-weighting is a new version of the table, not a mass failure of every build27// in the library. A decrease that the ledger accounts for -- data removed28// because it was wrong -- is permitted, because the gate exists to reward good29// data and never to punish a correction.30package maturity3132import (33 "archive/zip"34 _ "embed"35 "encoding/json"36 "fmt"37 "io"38 "math"39 "path/filepath"40 "sort"41 "strconv"42 "strings"4344 "github.com/FelineStateMachine/atlas/format/bundle"45 "github.com/FelineStateMachine/atlas/format/semconv"46 "github.com/FelineStateMachine/atlas/internal/enrich"47)4849//go:embed points.json50var embeddedTable []byte5152// Table is the versioned point table: what each quality is worth.53type Table struct {54 Schema int `json:"schema"`55 Version int `json:"version"`56 Feature struct {57 Name int `json:"name"`58 Prose int `json:"prose"`59 ProseSubstance int `json:"proseSubstance"`60 SubstanceChars int `json:"substanceChars"`61 Geo int `json:"geo"`62 Membership int `json:"membership"`63 Icon int `json:"icon"`64 GeometryLog int `json:"geometryLog"`65 Corroboration int `json:"corroboration"`66 } `json:"feature"`67 Collection struct {68 Convention int `json:"convention"`69 } `json:"collection"`70 World struct {71 Convention int `json:"convention"`72 Lens int `json:"lens"`73 LensZoom int `json:"lensZoom"`74 } `json:"world"`75}7677// TableSchema is the file layout this package reads.78const TableSchema = 17980// Points reads the embedded point table.81func Points() (Table, error) { return ReadTable(embeddedTable) }8283// ReadTable reads a point table from bytes, for a test that wants to state its84// own weights.85func ReadTable(data []byte) (Table, error) {86 var table Table87 if err := json.Unmarshal(data, &table); err != nil {88 return Table{}, fmt.Errorf("decode point table: %w", err)89 }90 if table.Schema != TableSchema {91 return Table{}, fmt.Errorf("point table schema %d, want %d", table.Schema, TableSchema)92 }93 if table.Version <= 0 {94 return Table{}, fmt.Errorf("point table declares no version")95 }96 if table.Feature.SubstanceChars <= 0 {97 return Table{}, fmt.Errorf("point table declares no substance threshold")98 }99 return table, nil100}101102// MembershipKeys are the attributes that say a feature's ground was joined to103// somewhere. One point each; the list grows with the joins.104var MembershipKeys = []string{semconv.KeyHydroHUC12}105106// Score is one build, scored.107type Score struct {108 // TableVersion is which point table produced these numbers. Two scores are109 // comparable only when it agrees.110 TableVersion int111112 File string113 Path string114 Volume string115 Title string116 Stamp string117 CreatedAt string118 Revision int119 // Enriched reports that this build's revision was written by the enrich120 // lane, and under which enrich policy.121 Enriched bool122 EnrichPolicy int123124 Total int125 Worlds []WorldScore126127 // Axes are the absolute measurements, kept as diagnostics for a reader128 // comparing two builds. Nothing gates on them.129 Axes Axes130131 // Ledger is every provenance account the build's payloads carry.132 Ledger []LedgerLine133}134135// WorldScore is one world's contribution to the total.136type WorldScore struct {137 Slug string138 Total int139 Features int140 Collections int141 World int142}143144// LedgerLine is one account, with the world it belongs to.145type LedgerLine struct {146 World string147 Account enrich.Account148}149150// Axes are the five absolute measurement axes, kept as a diagnostic.151type Axes struct {152 // Annotation.153 Points int154 // Features is every feature of every kind, and it is the denominator a155 // described share is actually a share of: prose defers into one text156 // payload whatever kind of feature wrote it, so dividing that whole by the157 // point features alone is how a city whose zones are all described comes158 // to report that 235% of it is described. Scoring in points retired the159 // consequence; naming the right denominator here retires the arithmetic.160 Features int161 Described int162 MedianLength int163 // Cartography.164 TileCount int165 RasterBytes int64166 Depth int167 DepthTiles int168 // Structure.169 Collections int170 Groups int171 TextSets int172 Shapes int173 Vertices int174 Lenses int175 // Icons.176 IconsCarried int177 IconsWanted int178 // Conventions.179 Conventions int180 RenderDeclared int181 StdIcons int182 GeoFeatures int183 Memberships int184 Geometry string185 UnknownAttrs int186}187188// DescribedShare, IconShare and RenderShare are diagnostics, printed and never189// gated on. They are what percentages are for: reading, not deciding.190func (a Axes) DescribedShare() string { return share(a.Described, a.Features) }191func (a Axes) IconShare() string { return share(a.IconsCarried, a.IconsWanted) }192func (a Axes) RenderShare() string { return share(a.RenderDeclared, a.Collections) }193194// RasterMB is the unique raster weight in megabytes, one decimal.195func (a Axes) RasterMB() string { return fmt.Sprintf("%.1f", float64(a.RasterBytes)/1e6) }196197func share(part, whole int) string {198 if whole == 0 {199 return "—"200 }201 return strconv.Itoa(part*100/whole) + "%"202}203204// the sliver of a world payload a score reads.205type worldPayload struct {206 Attrs map[string]string `json:"attrs"`207 Collections []struct {208 ID int64 `json:"id"`209 Title string `json:"title"`210 Group string `json:"group"`211 Kind string `json:"kind"`212 IconAsset string `json:"iconAsset"`213 Attrs map[string]string `json:"attrs"`214 Features []struct {215 ID int64 `json:"id"`216 Title string `json:"title"`217 HasText bool `json:"hasText"`218 Attrs map[string]string `json:"attrs"`219 Geometry []struct {220 Coordinates json.RawMessage `json:"coordinates"`221 } `json:"geometry"`222 } `json:"features"`223 } `json:"collections"`224 Lenses []struct {225 Name string `json:"name"`226 MinZoom int `json:"minZoom"`227 MaxZoom int `json:"maxZoom"`228 } `json:"lenses"`229 Merged []enrich.Account `json:"merged"`230}231232type featureText struct {233 Description string `json:"d"`234 Attrs map[string]string `json:"a"`235}236237// WorldParts is one world's three payloads, as a score reads them. It exists238// so that scoring is a pure function of bytes: [Measure] gets them out of an239// archive, and a test gets them out of a fixture directory, and both are240// scoring the same thing.241type WorldParts struct {242 Slug string243 // Payload and Text are worlds/<slug>.json and worlds/<slug>.text, verbatim.244 Payload []byte245 Text []byte246 // Locations are the world's packed point features, unpacked.247 Locations []bundle.Location248}249250// Measure scores one .atlas file.251//252// It reads the archive directly rather than only through the reader, because253// the cartography diagnostic needs what only the table of contents knows: every254// tile entry's checksum and size, for de-duplicating filler.255func Measure(path string, table Table) (*Score, error) {256 reader, err := zip.OpenReader(path)257 if err != nil {258 return nil, err259 }260 defer reader.Close()261262 entries := make(map[string]*zip.File, len(reader.File))263 for _, file := range reader.File {264 entries[file.Name] = file265 }266 var manifest bundle.Manifest267 if err := readEntry(entries, bundle.ManifestName, &manifest); err != nil {268 return nil, err269 }270271 parts := make([]WorldParts, 0, len(manifest.Worlds))272 for _, entry := range manifest.Worlds {273 payload, err := readBytes(entries, bundle.WorldEntryName(entry.Slug, bundle.WorldSuffix))274 if err != nil {275 return nil, err276 }277 text, err := readBytes(entries, bundle.WorldEntryName(entry.Slug, bundle.TextSuffix))278 if err != nil {279 return nil, err280 }281 packed, err := readBytes(entries, bundle.WorldEntryName(entry.Slug, bundle.PackedSuffix))282 if err != nil {283 return nil, err284 }285 locations, err := bundle.UnpackLocations(packed)286 if err != nil {287 return nil, fmt.Errorf("world %s: %w", entry.Slug, err)288 }289 parts = append(parts, WorldParts{Slug: entry.Slug, Payload: payload, Text: text, Locations: locations})290 }291292 score, err := ScoreParts(manifest, parts, table)293 if err != nil {294 return nil, err295 }296 score.Path = path297 score.File = filepath.Base(path)298299 seen := make(map[[2]uint64]bool)300 for name, file := range entries {301 if !strings.HasPrefix(name, "tiles/") {302 continue303 }304 score.Axes.TileCount++305 key := [2]uint64{uint64(file.CRC32), file.UncompressedSize64}306 if !seen[key] {307 seen[key] = true308 score.Axes.RasterBytes += int64(file.UncompressedSize64)309 }310 if zoom := tileZoom(name); zoom == score.Axes.Depth {311 score.Axes.DepthTiles++312 }313 }314 return score, nil315}316317// ScoreParts scores a build from its manifest and its worlds' payloads. It318// touches no filesystem and reads no rasters: the cartography axis is the only319// thing an archive knows that these bytes do not, and it is a diagnostic.320func ScoreParts(manifest bundle.Manifest, parts []WorldParts, table Table) (*Score, error) {321 if err := manifest.Validate(); err != nil {322 return nil, err323 }324 policy, enriched := enrich.Enriched(manifest.Version.Revision)325 score := &Score{326 TableVersion: table.Version,327 Volume: manifest.Volume.Slug,328 Title: manifest.Volume.Title,329 Stamp: manifest.Version.Stamp,330 CreatedAt: manifest.Version.CreatedAt,331 Revision: manifest.Version.Revision,332 Enriched: enriched,333 EnrichPolicy: policy,334 }335 score.Axes.Conventions = manifest.Conventions336 score.Axes.Geometry = semconv.SurfacePlane + "-default"337338 byWorld := make(map[string]WorldParts, len(parts))339 for _, part := range parts {340 byWorld[part.Slug] = part341 }342 var lengths []int343 // One group set for the whole volume, built once outside the loop: a group344 // title shared by thirteen worlds of a split sheet is one group, and a set345 // rebuilt per world would count it thirteen times.346 groups := map[string]bool{}347 for _, entry := range manifest.Worlds {348 part, held := byWorld[entry.Slug]349 if !held {350 return nil, fmt.Errorf("world %s carries no payload to score", entry.Slug)351 }352 var payload worldPayload353 if err := json.Unmarshal(part.Payload, &payload); err != nil {354 return nil, fmt.Errorf("world %s payload: %w", entry.Slug, err)355 }356 var text map[string]featureText357 if err := json.Unmarshal(part.Text, &text); err != nil {358 return nil, fmt.Errorf("world %s prose: %w", entry.Slug, err)359 }360 world, err := scoreWorld(score, table, entry.Slug, payload, text, part.Locations, &lengths, groups)361 if err != nil {362 return nil, err363 }364 score.Worlds = append(score.Worlds, world)365 score.Total += world.Total366 }367 sort.Ints(lengths)368 if len(lengths) > 0 {369 score.Axes.MedianLength = lengths[len(lengths)/2]370 }371 return score, nil372}373374func scoreWorld(375 score *Score,376 table Table,377 slug string,378 payload worldPayload,379 text map[string]featureText,380 locations []bundle.Location,381 lengths *[]int,382 groups map[string]bool,383) (WorldScore, error) {384 out := WorldScore{Slug: slug}385386 // A world earns for what it declares about its own ground, and for how much387 // picture a reader can actually get to.388 out.World += table.World.Convention * registered(payload.Attrs, semconv.EntityWorld)389 score.Axes.UnknownAttrs += unknown(payload.Attrs)390 switch payload.Attrs[semconv.KeyGeometrySurface] {391 case semconv.SurfaceSphere:392 score.Axes.Geometry = semconv.SurfaceSphere393 case semconv.SurfacePlane:394 if score.Axes.Geometry != semconv.SurfaceSphere {395 score.Axes.Geometry = semconv.SurfacePlane396 }397 }398 score.Axes.Lenses += len(payload.Lenses)399 for _, lens := range payload.Lenses {400 out.World += table.World.Lens401 if depth := lens.MaxZoom - lens.MinZoom + 1; depth > 0 {402 out.World += table.World.LensZoom * depth403 }404 score.Axes.Depth = max(score.Axes.Depth, lens.MaxZoom)405 }406407 // Corroboration is read off the ledger: a serving feature another reading408 // also pinned is a feature two sources agree about.409 corroborated := make(map[int64]int)410 for _, account := range payload.Merged {411 score.Ledger = append(score.Ledger, LedgerLine{World: slug, Account: account})412 for _, pair := range account.Matched {413 corroborated[pair.Winner]++414 }415 }416417 // Point collections come first in the payload and own the packed locations418 // by position.419 byOwner := make(map[int][]bundle.Location)420 for _, location := range locations {421 byOwner[int(location.Owner)] = append(byOwner[int(location.Owner)], location)422 }423424 pointIndex := 0425 for _, collection := range payload.Collections {426 score.Axes.UnknownAttrs += unknown(collection.Attrs)427 out.Collections += table.Collection.Convention * registered(collection.Attrs, semconv.EntityCollection)428429 resolvesIcon := collection.IconAsset != ""430 if collection.Kind == semconv.GeometryPoint {431 score.Axes.Collections++432 if !groups[collection.Group] {433 groups[collection.Group] = true434 score.Axes.Groups++435 }436 if _, declared := collection.Attrs[semconv.KeyRenderAs]; declared {437 score.Axes.RenderDeclared++438 }439 if collection.Attrs[semconv.KeyIconStd] != "" {440 score.Axes.StdIcons++441 }442 labels := semconv.RenderAs(collection.Attrs, "") == semconv.RenderAsText443 if labels {444 score.Axes.TextSets++445 } else {446 score.Axes.IconsWanted++447 if resolvesIcon {448 score.Axes.IconsCarried++449 }450 }451 for _, location := range byOwner[pointIndex] {452 score.Axes.Features++453 held := text[strconv.FormatInt(location.ID, 10)]454 out.Features += scoreFeature(score, table, featureFacts{455 title: location.Title,456 description: held.Description,457 attrs: held.Attrs,458 resolvesIcon: resolvesIcon,459 corroboration: corroborated[location.ID],460 }, lengths)461 score.Axes.Points++462 }463 pointIndex++464 continue465 }466467 for _, feature := range collection.Features {468 vertices := 0469 for _, geometry := range feature.Geometry {470 vertices += countVertices(geometry.Coordinates)471 }472 score.Axes.Shapes++473 score.Axes.Features++474 score.Axes.Vertices += vertices475 description := ""476 if feature.HasText {477 description = text[strconv.FormatInt(feature.ID, 10)].Description478 }479 out.Features += scoreFeature(score, table, featureFacts{480 title: feature.Title,481 description: description,482 attrs: feature.Attrs,483 resolvesIcon: resolvesIcon,484 vertices: vertices,485 corroboration: corroborated[feature.ID],486 }, lengths)487 }488 }489 out.Total = out.Features + out.Collections + out.World490 return out, nil491}492493type featureFacts struct {494 title string495 description string496 attrs map[string]string497 resolvesIcon bool498 vertices int499 corroboration int500}501502// scoreFeature is the whole per-feature rule, in one place: every point a503// feature can earn, and nothing it can lose.504func scoreFeature(score *Score, table Table, f featureFacts, lengths *[]int) int {505 points := 0506 if strings.TrimSpace(f.title) != "" {507 points += table.Feature.Name508 }509 cleaned := strings.Join(strings.Fields(f.description), " ")510 if cleaned != "" {511 points += table.Feature.Prose512 if len(cleaned) > table.Feature.SubstanceChars {513 points += table.Feature.ProseSubstance514 }515 score.Axes.Described++516 *lengths = append(*lengths, len(cleaned))517 }518 if f.attrs[semconv.KeyGeoLat] != "" && f.attrs[semconv.KeyGeoLon] != "" {519 points += table.Feature.Geo520 score.Axes.GeoFeatures++521 }522 for _, key := range MembershipKeys {523 if f.attrs[key] != "" {524 points += table.Feature.Membership525 score.Axes.Memberships++526 }527 }528 if f.resolvesIcon {529 points += table.Feature.Icon530 }531 if f.vertices > 0 {532 points += table.Feature.GeometryLog * int(math.Log2(float64(1+f.vertices)))533 }534 points += table.Feature.Corroboration * f.corroboration535 score.Axes.UnknownAttrs += unknown(f.attrs)536 return points537}538539// registered counts the attributes that are this entity's registered540// vocabulary. An unregistered key earns nothing: the score measures adoption of541// the shared vocabulary, and a private key is not that.542func registered(attrs map[string]string, entity semconv.Entity) int {543 count := 0544 for key := range attrs {545 if held, known := semconv.EntityOf(key); known && held == entity {546 count++547 }548 }549 return count550}551552// unknown counts the atlas-namespace keys the registry does not know.553// Validation refuses them at the producer, so a count above zero means a bundle554// written by a newer vocabulary than this build of the tools.555func unknown(attrs map[string]string) int {556 count := 0557 for key := range attrs {558 if !strings.HasPrefix(key, semconv.Namespace) {559 continue560 }561 if _, known := semconv.EntityOf(key); !known {562 count++563 }564 }565 return count566}567568func readEntry(entries map[string]*zip.File, name string, into any) error {569 data, err := readBytes(entries, name)570 if err != nil {571 return err572 }573 return json.Unmarshal(data, into)574}575576func readBytes(entries map[string]*zip.File, name string) ([]byte, error) {577 file, ok := entries[name]578 if !ok {579 return nil, fmt.Errorf("%s is missing", name)580 }581 opened, err := file.Open()582 if err != nil {583 return nil, err584 }585 defer opened.Close()586 return io.ReadAll(opened)587}588589// tileZoom reads the level out of tiles/<pyramid>/<zoom>/<x>/<y>.<ext>.590func tileZoom(name string) int {591 parts := strings.Split(name, "/")592 if len(parts) < 5 {593 return -1594 }595 zoom, err := strconv.Atoi(parts[len(parts)-3])596 if err != nil {597 return -1598 }599 return zoom600}601602// countVertices counts coordinate pairs through however many rings and603// nestings a geometry holds.604func countVertices(raw json.RawMessage) int {605 var value any606 if err := json.Unmarshal(raw, &value); err != nil {607 return 0608 }609 numbers := 0610 var walk func(any)611 walk = func(node any) {612 switch typed := node.(type) {613 case []any:614 for _, child := range typed {615 walk(child)616 }617 case float64:618 numbers++619 }620 }621 walk(value)622 return numbers / 2623}