1// Package maturity scores a build of a volume.2//3// # What a score is4//5// It is unbounded, additive, and monotone. Every feature earns points for each6// quality it verifiably has -- a name, prose, prose with something in it,7// resolved coordinates, a membership its ground was joined to, a collection8// that resolves artwork, geometry actually drawn, another reading that9// corroborates it -- and those sums roll up through collections and worlds to10// the volume. Collections earn for the conventions they declare; worlds earn11// for what they say about their surface and for how much raster a reader can12// actually get to.13//14// There are no denominators and no ceilings, and that is the whole of why this15// scores points rather than percentages: a share moves when its denominator16// moves, so a build that added five hundred features and described half of them17// reads as a regression. The five absolute axes survive as diagnostics -- good18// for reading, useless as a gate -- and a described *share* above 100% is not19// a defect that can be reached from here, because nothing is divided by20// anything.21//22// # The gate23//24// An enrichment build whose score declines fails (see [Gate]). The comparison25// is only ever between builds scored under the same table version: a26// re-weighting is a new version of the table, not a mass failure of every build27// in the library. A decrease that the ledger accounts for -- data removed28// because it was wrong -- is permitted, because the gate exists to reward good29// data and never to punish a correction.30package maturity3132import (33 "archive/zip"34 _ "embed"35 "encoding/json"36 "fmt"37 "io"38 "math"39 "path/filepath"40 "sort"41 "strconv"42 "strings"4344 "github.com/FelineStateMachine/atlas/format/bundle"45 "github.com/FelineStateMachine/atlas/format/semconv"46 "github.com/FelineStateMachine/atlas/format/vnext"47 "github.com/FelineStateMachine/atlas/internal/enrich"48)4950//go:embed points.json51var embeddedTable []byte5253// Table is the versioned point table: what each quality is worth.54type Table struct {55 Schema int `json:"schema"`56 Version int `json:"version"`57 Feature struct {58 Name int `json:"name"`59 Prose int `json:"prose"`60 ProseSubstance int `json:"proseSubstance"`61 SubstanceChars int `json:"substanceChars"`62 Geo int `json:"geo"`63 Membership int `json:"membership"`64 Icon int `json:"icon"`65 GeometryLog int `json:"geometryLog"`66 Corroboration int `json:"corroboration"`67 } `json:"feature"`68 Collection struct {69 Convention int `json:"convention"`70 } `json:"collection"`71 World struct {72 Convention int `json:"convention"`73 Lens int `json:"lens"`74 LensZoom int `json:"lensZoom"`75 } `json:"world"`76}7778// TableSchema is the file layout this package reads.79const TableSchema = 18081// Points reads the embedded point table.82func Points() (Table, error) { return ReadTable(embeddedTable) }8384// ReadTable reads a point table from bytes, for a test that wants to state its85// own weights.86func ReadTable(data []byte) (Table, error) {87 var table Table88 if err := json.Unmarshal(data, &table); err != nil {89 return Table{}, fmt.Errorf("decode point table: %w", err)90 }91 if table.Schema != TableSchema {92 return Table{}, fmt.Errorf("point table schema %d, want %d", table.Schema, TableSchema)93 }94 if table.Version <= 0 {95 return Table{}, fmt.Errorf("point table declares no version")96 }97 if table.Feature.SubstanceChars <= 0 {98 return Table{}, fmt.Errorf("point table declares no substance threshold")99 }100 return table, nil101}102103// MembershipKeys are the attributes that say a feature's ground was joined to104// somewhere. One point each; the list grows with the joins.105var MembershipKeys = []string{semconv.KeyHydroHUC12}106107// Score is one build, scored.108type Score struct {109 // TableVersion is which point table produced these numbers. Two scores are110 // comparable only when it agrees.111 TableVersion int112113 File string114 Path string115 Volume string116 Title string117 Stamp string118 CreatedAt string119 Revision int120 // Enriched reports that this build's revision was written by the enrich121 // lane, and under which enrich policy.122 Enriched bool123 EnrichPolicy int124125 Total int126 Worlds []WorldScore127128 // Axes are the absolute measurements, kept as diagnostics for a reader129 // comparing two builds. Nothing gates on them.130 Axes Axes131132 // Ledger is every provenance account the build's payloads carry.133 Ledger []LedgerLine134}135136// WorldScore is one world's contribution to the total.137type WorldScore struct {138 Slug string139 Total int140 Features int141 Collections int142 World int143}144145// LedgerLine is one account, with the world it belongs to.146type LedgerLine struct {147 World string148 Account enrich.Account149}150151// Axes are the five absolute measurement axes, kept as a diagnostic.152type Axes struct {153 // Annotation.154 Points int155 // Features is every feature of every kind, and it is the denominator a156 // described share is actually a share of: prose defers into one text157 // payload whatever kind of feature wrote it, so dividing that whole by the158 // point features alone is how a city whose zones are all described comes159 // to report that 235% of it is described. Scoring in points retired the160 // consequence; naming the right denominator here retires the arithmetic.161 Features int162 Described int163 MedianLength int164 // Cartography.165 TileCount int166 RasterBytes int64167 Depth int168 DepthTiles int169 // Structure.170 Collections int171 Groups int172 TextSets int173 Shapes int174 Vertices int175 Lenses int176 // Icons.177 IconsCarried int178 IconsWanted int179 // Conventions.180 Conventions int181 RenderDeclared int182 StdIcons int183 GeoFeatures int184 Memberships int185 Geometry string186 UnknownAttrs int187}188189// DescribedShare, IconShare and RenderShare are diagnostics, printed and never190// gated on. They are what percentages are for: reading, not deciding.191func (a Axes) DescribedShare() string { return share(a.Described, a.Features) }192func (a Axes) IconShare() string { return share(a.IconsCarried, a.IconsWanted) }193func (a Axes) RenderShare() string { return share(a.RenderDeclared, a.Collections) }194195// RasterMB is the unique raster weight in megabytes, one decimal.196func (a Axes) RasterMB() string { return fmt.Sprintf("%.1f", float64(a.RasterBytes)/1e6) }197198func share(part, whole int) string {199 if whole == 0 {200 return "—"201 }202 return strconv.Itoa(part*100/whole) + "%"203}204205// the sliver of a world payload a score reads.206type worldPayload struct {207 Attrs map[string]string `json:"attrs"`208 Collections []struct {209 ID int64 `json:"id"`210 Title string `json:"title"`211 Group string `json:"group"`212 Kind string `json:"kind"`213 IconAsset string `json:"iconAsset"`214 Attrs map[string]string `json:"attrs"`215 Features []struct {216 ID int64 `json:"id"`217 Title string `json:"title"`218 HasText bool `json:"hasText"`219 Attrs map[string]string `json:"attrs"`220 Geometry []struct {221 Coordinates json.RawMessage `json:"coordinates"`222 } `json:"geometry"`223 } `json:"features"`224 } `json:"collections"`225 Lenses []struct {226 Name string `json:"name"`227 MinZoom int `json:"minZoom"`228 MaxZoom int `json:"maxZoom"`229 } `json:"lenses"`230 Merged []enrich.Account `json:"merged"`231}232233type featureText struct {234 Description string `json:"d"`235 Attrs map[string]string `json:"a"`236}237238// WorldParts is one world's three payloads, as a score reads them. It exists239// so that scoring is a pure function of bytes: [Measure] gets them out of an240// archive, and a test gets them out of a fixture directory, and both are241// scoring the same thing.242type WorldParts struct {243 Slug string244 // Payload and Text are worlds/<slug>.json and worlds/<slug>.text, verbatim.245 Payload []byte246 Text []byte247 // Locations are the world's packed point features, unpacked.248 Locations []bundle.Location249}250251// Measure scores one .atlas file.252//253// It reads the archive directly rather than only through the reader, because254// the cartography diagnostic needs what only the table of contents knows: every255// tile entry's checksum and size, for de-duplicating filler.256func Measure(path string, table Table) (*Score, error) {257 native, err := vnext.OpenFile(path, vnext.StandardSchema())258 if err != nil {259 return nil, err260 }261 defer native.Close()262 volume, err := native.Volume()263 if err != nil {264 return nil, err265 }266 score, err := scoreNativeVolume(volume, native.Release, table)267 if err != nil {268 return nil, err269 }270 score.Path = path271 score.File = filepath.Base(path)272273 archive, err := zip.OpenReader(path)274 if err != nil {275 return nil, err276 }277 defer archive.Close()278279 seen := make(map[[2]uint64]bool)280 for _, file := range archive.File {281 name := file.Name282 if !strings.HasPrefix(name, "tiles/") {283 continue284 }285 score.Axes.TileCount++286 key := [2]uint64{uint64(file.CRC32), file.UncompressedSize64}287 if !seen[key] {288 seen[key] = true289 score.Axes.RasterBytes += int64(file.UncompressedSize64)290 }291 if zoom := tileZoom(name); zoom == score.Axes.Depth {292 score.Axes.DepthTiles++293 }294 }295 return score, nil296}297298func scoreNativeVolume(volume vnext.Volume, release vnext.Release, table Table) (*Score, error) {299 policy, enriched := enrich.Enriched(release.Revision)300 score := &Score{301 TableVersion: table.Version, Volume: volume.ID, Title: release.Title, Stamp: release.Stamp,302 CreatedAt: release.CreatedAt, Revision: release.Revision, Enriched: enriched, EnrichPolicy: policy,303 }304 score.Axes.Conventions = 2305 score.Axes.Geometry = semconv.SurfacePlane + "-default"306 assets := make(map[string]bool, len(volume.Assets))307 for _, asset := range volume.Assets {308 assets[asset.ID] = true309 }310 groups := make(map[string]bool)311 var lengths []int312 for _, world := range volume.Worlds {313 worldScore := scoreNativeWorld(score, table, world, assets, &lengths, groups)314 score.Worlds = append(score.Worlds, worldScore)315 score.Total += worldScore.Total316 }317 sort.Ints(lengths)318 if len(lengths) > 0 {319 score.Axes.MedianLength = lengths[len(lengths)/2]320 }321 return score, nil322}323324func scoreNativeWorld(score *Score, table Table, world vnext.World, assets map[string]bool, lengths *[]int, groups map[string]bool) WorldScore {325 out := WorldScore{Slug: world.ID}326 worldAttrs := nativeAttrs(world.Claims)327 out.World += table.World.Convention * registered(worldAttrs, semconv.EntityWorld)328 score.Axes.UnknownAttrs += unknown(worldAttrs)329 switch worldAttrs[semconv.KeyGeometrySurface] {330 case semconv.SurfaceSphere:331 score.Axes.Geometry = semconv.SurfaceSphere332 case semconv.SurfacePlane:333 if score.Axes.Geometry != semconv.SurfaceSphere {334 score.Axes.Geometry = semconv.SurfacePlane335 }336 }337 for _, raster := range world.RasterPyramids {338 score.Axes.Lenses++339 out.World += table.World.Lens340 if depth := int(raster.MaxZoom-raster.MinZoom) + 1; depth > 0 {341 out.World += table.World.LensZoom * depth342 }343 score.Axes.Depth = max(score.Axes.Depth, int(raster.MaxZoom))344 }345 sets := make(map[string]vnext.FeatureSet, len(world.FeatureSets))346 for _, set := range world.FeatureSets {347 sets[set.ID] = set348 }349 styles := make(map[string]vnext.Style, len(world.Presentation.Styles))350 for _, style := range world.Presentation.Styles {351 styles[style.ID] = style352 }353 accounts := make(map[string]*enrich.Account)354 var accountOrder []string355 for _, layer := range world.Presentation.Layers {356 set, ok := sets[layer.FeatureSet]357 if !ok {358 continue359 }360 style := styles[layer.Style]361 attrs := nativeAttrs(set.Claims)362 kind := strings.TrimPrefix(set.SemanticType, "geometry.")363 score.Axes.UnknownAttrs += unknown(attrs)364 out.Collections += table.Collection.Convention * registered(attrs, semconv.EntityCollection)365 score.Axes.Collections++366 if !groups[layer.Group] {367 groups[layer.Group] = true368 score.Axes.Groups++369 }370 if style.RenderAs != "" {371 score.Axes.RenderDeclared++372 }373 labels := style.RenderAs == semconv.RenderAsText374 resolvesIcon := style.IconAsset != "" && assets[style.IconAsset]375 if labels {376 score.Axes.TextSets++377 } else if kind == semconv.GeometryPoint {378 score.Axes.IconsWanted++379 if resolvesIcon {380 score.Axes.IconsCarried++381 }382 }383 for _, feature := range set.Features {384 featureAttrs := nativeAttrs(feature.Properties)385 vertices := nativeVertices(feature.Geometry)386 score.Axes.Features++387 if feature.Geometry.Kind == vnext.GeometryPoint {388 score.Axes.Points++389 } else {390 score.Axes.Shapes++391 score.Axes.Vertices += vertices392 }393 corroboration := max(0, len(feature.Provenance)-1)394 for index, provenance := range feature.Provenance {395 account := accounts[provenance.Source]396 if account == nil {397 account = &enrich.Account{Source: provenance.Source, Origin: index == 0}398 accounts[provenance.Source] = account399 accountOrder = append(accountOrder, provenance.Source)400 }401 switch feature.Geometry.Kind {402 case vnext.GeometryPoint:403 account.DonorFeatures.Point++404 case vnext.GeometryLineString:405 account.DonorFeatures.Path++406 case vnext.GeometryPolygon:407 account.DonorFeatures.Area++408 }409 }410 out.Features += scoreFeature(score, table, featureFacts{411 title: feature.Title, description: feature.Description, attrs: featureAttrs,412 resolvesIcon: resolvesIcon, vertices: vertices, corroboration: corroboration,413 }, lengths)414 }415 }416 for _, source := range accountOrder {417 score.Ledger = append(score.Ledger, LedgerLine{World: world.ID, Account: *accounts[source]})418 }419 out.Total = out.Features + out.Collections + out.World420 return out421}422423func nativeAttrs(properties []vnext.Property) map[string]string {424 out := make(map[string]string, len(properties))425 for _, property := range properties {426 if property.Field.Name != "" {427 out[property.Field.Name] = nativeValue(property.Value)428 }429 }430 return out431}432433func nativeValue(value vnext.Value) string {434 switch value.Kind {435 case vnext.KindBool:436 return strconv.FormatBool(value.Bool)437 case vnext.KindInt64:438 return strconv.FormatInt(value.Int64, 10)439 case vnext.KindFloat64:440 return strconv.FormatFloat(value.Float64, 'g', -1, 64)441 case vnext.KindString:442 return value.String443 case vnext.KindBytes:444 return fmt.Sprintf("%x", value.Bytes)445 case vnext.KindID:446 return value.ID.String()447 default:448 return ""449 }450}451452func nativeVertices(geometry vnext.Geometry) int {453 total := 0454 for _, part := range geometry.Parts {455 for _, ring := range part.Rings {456 total += len(ring)457 }458 }459 return total460}461462// ScoreParts scores a build from its manifest and its worlds' payloads. It463// touches no filesystem and reads no rasters: the cartography axis is the only464// thing an archive knows that these bytes do not, and it is a diagnostic.465func ScoreParts(manifest bundle.Manifest, parts []WorldParts, table Table) (*Score, error) {466 if err := manifest.Validate(); err != nil {467 return nil, err468 }469 policy, enriched := enrich.Enriched(manifest.Version.Revision)470 score := &Score{471 TableVersion: table.Version,472 Volume: manifest.Volume.Slug,473 Title: manifest.Volume.Title,474 Stamp: manifest.Version.Stamp,475 CreatedAt: manifest.Version.CreatedAt,476 Revision: manifest.Version.Revision,477 Enriched: enriched,478 EnrichPolicy: policy,479 }480 score.Axes.Conventions = manifest.Conventions481 score.Axes.Geometry = semconv.SurfacePlane + "-default"482483 byWorld := make(map[string]WorldParts, len(parts))484 for _, part := range parts {485 byWorld[part.Slug] = part486 }487 var lengths []int488 // One group set for the whole volume, built once outside the loop: a group489 // title shared by thirteen worlds of a split sheet is one group, and a set490 // rebuilt per world would count it thirteen times.491 groups := map[string]bool{}492 for _, entry := range manifest.Worlds {493 part, held := byWorld[entry.Slug]494 if !held {495 return nil, fmt.Errorf("world %s carries no payload to score", entry.Slug)496 }497 var payload worldPayload498 if err := json.Unmarshal(part.Payload, &payload); err != nil {499 return nil, fmt.Errorf("world %s payload: %w", entry.Slug, err)500 }501 var text map[string]featureText502 if err := json.Unmarshal(part.Text, &text); err != nil {503 return nil, fmt.Errorf("world %s prose: %w", entry.Slug, err)504 }505 world, err := scoreWorld(score, table, entry.Slug, payload, text, part.Locations, &lengths, groups)506 if err != nil {507 return nil, err508 }509 score.Worlds = append(score.Worlds, world)510 score.Total += world.Total511 }512 sort.Ints(lengths)513 if len(lengths) > 0 {514 score.Axes.MedianLength = lengths[len(lengths)/2]515 }516 return score, nil517}518519func scoreWorld(520 score *Score,521 table Table,522 slug string,523 payload worldPayload,524 text map[string]featureText,525 locations []bundle.Location,526 lengths *[]int,527 groups map[string]bool,528) (WorldScore, error) {529 out := WorldScore{Slug: slug}530531 // A world earns for what it declares about its own ground, and for how much532 // picture a reader can actually get to.533 out.World += table.World.Convention * registered(payload.Attrs, semconv.EntityWorld)534 score.Axes.UnknownAttrs += unknown(payload.Attrs)535 switch payload.Attrs[semconv.KeyGeometrySurface] {536 case semconv.SurfaceSphere:537 score.Axes.Geometry = semconv.SurfaceSphere538 case semconv.SurfacePlane:539 if score.Axes.Geometry != semconv.SurfaceSphere {540 score.Axes.Geometry = semconv.SurfacePlane541 }542 }543 score.Axes.Lenses += len(payload.Lenses)544 for _, lens := range payload.Lenses {545 out.World += table.World.Lens546 if depth := lens.MaxZoom - lens.MinZoom + 1; depth > 0 {547 out.World += table.World.LensZoom * depth548 }549 score.Axes.Depth = max(score.Axes.Depth, lens.MaxZoom)550 }551552 // Corroboration is read off the ledger: a serving feature another reading553 // also pinned is a feature two sources agree about.554 corroborated := make(map[int64]int)555 for _, account := range payload.Merged {556 score.Ledger = append(score.Ledger, LedgerLine{World: slug, Account: account})557 for _, pair := range account.Matched {558 corroborated[pair.Winner]++559 }560 }561562 // Point collections come first in the payload and own the packed locations563 // by position.564 byOwner := make(map[int][]bundle.Location)565 for _, location := range locations {566 byOwner[int(location.Owner)] = append(byOwner[int(location.Owner)], location)567 }568569 pointIndex := 0570 for _, collection := range payload.Collections {571 score.Axes.UnknownAttrs += unknown(collection.Attrs)572 out.Collections += table.Collection.Convention * registered(collection.Attrs, semconv.EntityCollection)573574 resolvesIcon := collection.IconAsset != ""575 if collection.Kind == semconv.GeometryPoint {576 score.Axes.Collections++577 if !groups[collection.Group] {578 groups[collection.Group] = true579 score.Axes.Groups++580 }581 if _, declared := collection.Attrs[semconv.KeyRenderAs]; declared {582 score.Axes.RenderDeclared++583 }584 if collection.Attrs[semconv.KeyIconStd] != "" {585 score.Axes.StdIcons++586 }587 labels := semconv.RenderAs(collection.Attrs, "") == semconv.RenderAsText588 if labels {589 score.Axes.TextSets++590 } else {591 score.Axes.IconsWanted++592 if resolvesIcon {593 score.Axes.IconsCarried++594 }595 }596 for _, location := range byOwner[pointIndex] {597 score.Axes.Features++598 held := text[strconv.FormatInt(location.ID, 10)]599 out.Features += scoreFeature(score, table, featureFacts{600 title: location.Title,601 description: held.Description,602 attrs: held.Attrs,603 resolvesIcon: resolvesIcon,604 corroboration: corroborated[location.ID],605 }, lengths)606 score.Axes.Points++607 }608 pointIndex++609 continue610 }611612 for _, feature := range collection.Features {613 vertices := 0614 for _, geometry := range feature.Geometry {615 vertices += countVertices(geometry.Coordinates)616 }617 score.Axes.Shapes++618 score.Axes.Features++619 score.Axes.Vertices += vertices620 description := ""621 if feature.HasText {622 description = text[strconv.FormatInt(feature.ID, 10)].Description623 }624 out.Features += scoreFeature(score, table, featureFacts{625 title: feature.Title,626 description: description,627 attrs: feature.Attrs,628 resolvesIcon: resolvesIcon,629 vertices: vertices,630 corroboration: corroborated[feature.ID],631 }, lengths)632 }633 }634 out.Total = out.Features + out.Collections + out.World635 return out, nil636}637638type featureFacts struct {639 title string640 description string641 attrs map[string]string642 resolvesIcon bool643 vertices int644 corroboration int645}646647// scoreFeature is the whole per-feature rule, in one place: every point a648// feature can earn, and nothing it can lose.649func scoreFeature(score *Score, table Table, f featureFacts, lengths *[]int) int {650 points := 0651 if strings.TrimSpace(f.title) != "" {652 points += table.Feature.Name653 }654 cleaned := strings.Join(strings.Fields(f.description), " ")655 if cleaned != "" {656 points += table.Feature.Prose657 if len(cleaned) > table.Feature.SubstanceChars {658 points += table.Feature.ProseSubstance659 }660 score.Axes.Described++661 *lengths = append(*lengths, len(cleaned))662 }663 if f.attrs[semconv.KeyGeoLat] != "" && f.attrs[semconv.KeyGeoLon] != "" {664 points += table.Feature.Geo665 score.Axes.GeoFeatures++666 }667 for _, key := range MembershipKeys {668 if f.attrs[key] != "" {669 points += table.Feature.Membership670 score.Axes.Memberships++671 }672 }673 if f.resolvesIcon {674 points += table.Feature.Icon675 }676 if f.vertices > 0 {677 points += table.Feature.GeometryLog * int(math.Log2(float64(1+f.vertices)))678 }679 points += table.Feature.Corroboration * f.corroboration680 score.Axes.UnknownAttrs += unknown(f.attrs)681 return points682}683684// registered counts the attributes that are this entity's registered685// vocabulary. An unregistered key earns nothing: the score measures adoption of686// the shared vocabulary, and a private key is not that.687func registered(attrs map[string]string, entity semconv.Entity) int {688 count := 0689 for key := range attrs {690 if held, known := semconv.EntityOf(key); known && held == entity {691 count++692 }693 }694 return count695}696697// unknown counts the atlas-namespace keys the registry does not know.698// Validation refuses them at the producer, so a count above zero means a bundle699// written by a newer vocabulary than this build of the tools.700func unknown(attrs map[string]string) int {701 count := 0702 for key := range attrs {703 if !strings.HasPrefix(key, semconv.Namespace) {704 continue705 }706 if _, known := semconv.EntityOf(key); !known {707 count++708 }709 }710 return count711}712713func readEntry(entries map[string]*zip.File, name string, into any) error {714 data, err := readBytes(entries, name)715 if err != nil {716 return err717 }718 return json.Unmarshal(data, into)719}720721func readBytes(entries map[string]*zip.File, name string) ([]byte, error) {722 file, ok := entries[name]723 if !ok {724 return nil, fmt.Errorf("%s is missing", name)725 }726 opened, err := file.Open()727 if err != nil {728 return nil, err729 }730 defer opened.Close()731 return io.ReadAll(opened)732}733734// tileZoom reads the level out of tiles/<pyramid>/<zoom>/<x>/<y>.<ext>.735func tileZoom(name string) int {736 parts := strings.Split(name, "/")737 if len(parts) < 5 {738 return -1739 }740 zoom, err := strconv.Atoi(parts[len(parts)-3])741 if err != nil {742 return -1743 }744 return zoom745}746747// countVertices counts coordinate pairs through however many rings and748// nestings a geometry holds.749func countVertices(raw json.RawMessage) int {750 var value any751 if err := json.Unmarshal(raw, &value); err != nil {752 return 0753 }754 numbers := 0755 var walk func(any)756 walk = func(node any) {757 switch typed := node.(type) {758 case []any:759 for _, child := range typed {760 walk(child)761 }762 case float64:763 numbers++764 }765 }766 walk(value)767 return numbers / 2768}