clone

an offline map explorer where each world, from a game map to a city's open data, ships as one .atlas file

generate: normalize worlds into collections of features

Dami committed 37327fa
Zones, paths and points are about to hold equal rank on the wire, and
that cannot happen while the generator itself thinks in the MapGenie
shape -- pins under groups and categories, regions standing apart as
zones, every pass written twice because of it. The catalog now holds
one model instead: collections of features, where each category is a
point collection and a world's regions fold into one implicit area
collection. Every pass over a map -- conventions, icons, links,
sharding, surfaces, merging, the merge gate -- reads that model.

The wire does not move an inch yet. The v2 groups and zones are
reconstructed from the collections at the moment a payload is written,
and the emission-only structs stay behind to spell them; regrouping by
first-seen group id reproduces the original tree because categories
within a group were contiguous in every capture and every pass keeps
them so. A rebuilt Westminster bundle lands under the same stamp, and
its manifest and world payloads compare byte for byte against the
pre-refactor build -- which is the proof that the model holds
everything the wire says, and the licence for the wire to move later.

The pin count is no longer a field that could drift: the manifest
counts the point features it actually has, and the merge gate holds
that count to the sum of the map's own ledger.
commit
37327fa0fa66c5c50b789baac693355d8715dd0f
parent
6467a1c789b7e72ffc8bda662d024f980234b165
author
Dami <dami.etoile@gmail.com>
kind
refactor risk score 2.0 of 3, Jev's estimate
9 files changed, +778 -469
Mtools/generate/bundles.go2+-
Mtools/generate/main.go290+++++++++-------------
Mtools/generate/main_test.go66++---
Mtools/generate/merge.go389++++++++++++++++--------------
Atools/generate/normalize.go244+++++++++++++++++++
Atools/generate/normalize_test.go103++++++++
Mtools/generate/shard.go132+++++-----
Mtools/generate/split.go8+-
Mtools/generate/split_test.go13+-
M tools/generate/bundles.go +1 -1
M tools/generate/main.go +118 -172
@@ -170,14 +170,16 @@ type catalogWorld struct {
170170 Attrs map[string]string `json:"attrs,omitempty"`171171 // Grid is carried only by a map whose window is not the shared one, so the172172 // catalog reads the same as it did for every map that is.173- Grid *worldGrid `json:"grid,omitempty"`174- Lenses []lens `json:"lenses"`175- Groups []catalogGroup `json:"groups"`176- Zones []zone `json:"zones,omitempty"`173+ Grid *worldGrid `json:"grid,omitempty"`174+ Lenses []lens `json:"lenses"`175+ // Collections is everything the map holds, in one model: each category of176+ // pins as a point collection, and the regions folded into the implicit177+ // collection (normalize.go). The v2 wire's groups and zones are178+ // reconstructed from it only when a payload is written.179+ Collections []worldCollection `json:"-"`177180 // Parent names the map this one was split out of, so an inset sorts with178181 // the sheet it came from rather than alphabetically among unrelated maps.179182 Parent string `json:"parent,omitempty"`180- PinCount int `json:"pinCount"`181183 UpdatedAt string `json:"updatedAt"`182184 // Merged is the account of every other source folded into this map:183185 // what matched, what was added, what was held back and why. It rides in
@@ -252,6 +254,10 @@ type tileLensManifest struct {
252254 AlignedWith string `json:"alignedWith"`253255 }254256 257+// The v2 wire shapes. The catalog itself holds collections of features258+// (normalize.go); these structs exist only to be reconstructed at emission259+// time, so every byte a bundle says stays exactly what today's readers260+// expect until the wire itself is ready to move.255261 type catalogGroup struct {256262 ID int64 `json:"id"`257263 Title string `json:"title"`
@@ -302,6 +308,8 @@ type catalogLink struct {
302308 LocationID int64 `json:"locationId"`303309 }304310 311+// zone is emission-only, like the group tree above it: rebuilt from the312+// implicit region collection's features when a payload is written.305313 type zone struct {306314 ID int64 `json:"id"`307315 Title string `json:"title"`
@@ -481,7 +489,7 @@ func buildVolume(
481489 Source: sourceDisplayLabel(ref.Source),482490 Slug: canonicalSourceSlug(ref.Source),483491 Origin: true,484- DonorPins: pieces[index].PinCount,492+ DonorPins: pieces[index].pinCount(),485493 }}486494 }487495 game.Worlds = append(game.Worlds, pieces...)
@@ -513,42 +521,42 @@ func speakConventions(game *catalogVolume) error {
513521 if err := semconv.Validate(semconv.EntityWorld, m.Attrs); err != nil {514522 return fmt.Errorf("world %s: %w", m.Slug, err)515523 }516- for zoneIndex := range m.Zones {517- if err := semconv.Validate(semconv.EntityZone, m.Zones[zoneIndex].Attrs); err != nil {518- return fmt.Errorf("world %s zone %q: %w", m.Slug, m.Zones[zoneIndex].Title, err)519- }520- }521- for groupIndex := range m.Groups {522- categories := m.Groups[groupIndex].Categories523- for categoryIndex := range categories {524- category := &categories[categoryIndex]525- if _, declared := category.Attrs[semconv.KeyRenderAs]; !declared {526- category.Attrs = withAttr(category.Attrs, semconv.KeyRenderAs,527- semconv.RenderAs(nil, category.DisplayType))528- }529- // Where two sources are known to spell one concept two ways,530- // the shared name rides the category itself, so the merge531- // reads identity off the payload rather than a table of its532- // own.533- if shared := categoryEquivalents[game.Slug][category.Icon]; shared != "" {534- if _, declared := category.Attrs[semconv.KeyCategoryKey]; !declared {535- category.Attrs = withAttr(category.Attrs, semconv.KeyCategoryKey, shared)524+ for collectionIndex := range m.Collections {525+ collection := &m.Collections[collectionIndex]526+ if collection.Kind != kindPoint {527+ for _, shape := range collection.Features {528+ if err := semconv.Validate(semconv.EntityZone, shape.Attrs); err != nil {529+ return fmt.Errorf("world %s zone %q: %w", m.Slug, shape.Title, err)536530 }537531 }538- if category.IconAsset != "" {539- kind := semconv.IconKindGlyph540- if category.IconPicture {541- kind = semconv.IconKindPicture542- }543- category.Attrs = withAttr(category.Attrs, semconv.KeyIconKind, kind)532+ continue533+ }534+ if _, declared := collection.Attrs[semconv.KeyRenderAs]; !declared {535+ collection.Attrs = withAttr(collection.Attrs, semconv.KeyRenderAs,536+ semconv.RenderAs(nil, collection.DisplayType))537+ }538+ // Where two sources are known to spell one concept two ways,539+ // the shared name rides the collection itself, so the merge540+ // reads identity off the payload rather than a table of its541+ // own.542+ if shared := categoryEquivalents[game.Slug][collection.Icon]; shared != "" {543+ if _, declared := collection.Attrs[semconv.KeyCategoryKey]; !declared {544+ collection.Attrs = withAttr(collection.Attrs, semconv.KeyCategoryKey, shared)544545 }545- if err := semconv.Validate(semconv.EntityCategory, category.Attrs); err != nil {546- return fmt.Errorf("world %s category %q: %w", m.Slug, category.Title, err)546+ }547+ if collection.IconAsset != "" {548+ kind := semconv.IconKindGlyph549+ if collection.IconPicture {550+ kind = semconv.IconKindPicture547551 }548- for _, location := range category.Locations {549- if err := semconv.Validate(semconv.EntityLocation, location.Attrs); err != nil {550- return fmt.Errorf("world %s pin %q: %w", m.Slug, location.Title, err)551- }552+ collection.Attrs = withAttr(collection.Attrs, semconv.KeyIconKind, kind)553+ }554+ if err := semconv.Validate(semconv.EntityCategory, collection.Attrs); err != nil {555+ return fmt.Errorf("world %s category %q: %w", m.Slug, collection.Title, err)556+ }557+ for _, pin := range collection.Features {558+ if err := semconv.Validate(semconv.EntityLocation, pin.Attrs); err != nil {559+ return fmt.Errorf("world %s pin %q: %w", m.Slug, pin.Title, err)552560 }553561 }554562 }
@@ -576,26 +584,26 @@ func withAttr(attrs map[string]string, key, value string) map[string]string {
576584 func resolveStandardIcons(game *catalogVolume) error {577585 for mapIndex := range game.Worlds {578586 m := &game.Worlds[mapIndex]579- for groupIndex := range m.Groups {580- categories := m.Groups[groupIndex].Categories581- for categoryIndex := range categories {582- category := &categories[categoryIndex]583- ref := category.Attrs[semconv.KeyIconStd]584- if ref == "" || category.IconAsset != "" {585- continue586- }587- data, asset, err := icons.Standard(ref)588- if err != nil {589- return fmt.Errorf("world %s category %q: %w", m.Slug, category.Title, err)590- }591- if game.Icons == nil {592- game.Icons = make(map[string][]byte)593- }594- game.Icons[asset] = data595- category.IconAsset = asset596- category.IconPicture = false597- category.Attrs = withAttr(category.Attrs, semconv.KeyIconKind, semconv.IconKindGlyph)587+ for collectionIndex := range m.Collections {588+ collection := &m.Collections[collectionIndex]589+ if collection.Kind != kindPoint {590+ continue591+ }592+ ref := collection.Attrs[semconv.KeyIconStd]593+ if ref == "" || collection.IconAsset != "" {594+ continue598595 }596+ data, asset, err := icons.Standard(ref)597+ if err != nil {598+ return fmt.Errorf("world %s category %q: %w", m.Slug, collection.Title, err)599+ }600+ if game.Icons == nil {601+ game.Icons = make(map[string][]byte)602+ }603+ game.Icons[asset] = data604+ collection.IconAsset = asset605+ collection.IconPicture = false606+ collection.Attrs = withAttr(collection.Attrs, semconv.KeyIconKind, semconv.IconKindGlyph)599607 }600608 }601609 return nil
@@ -766,72 +774,11 @@ func buildWorld(
766774 })767775 }768776 }769- for _, rawGroup := range raw.Groups {770- group := catalogGroup{ID: rawGroup.ID, Title: rawGroup.Title}771- for _, rawCategory := range rawGroup.Categories {772- category := catalogCategory{773- ID: rawCategory.ID,774- Title: rawCategory.Title,775- Icon: rawCategory.Icon,776- Color: resolvedCategoryColor(rawGroup, rawCategory),777- IconColor: resolvedIconColor(rawGroup, rawCategory),778- DisplayType: rawCategory.DisplayType,779- Visible: rawCategory.Visible,780- Attrs: rawCategory.Attrs,781- }782- for _, rawLocation := range rawCategory.Locations {783- lat, err := number(rawLocation.Latitude)784- if err != nil {785- return nil, "", fmt.Errorf("location %d latitude: %w", rawLocation.ID, err)786- }787- lng, err := number(rawLocation.Longitude)788- if err != nil {789- return nil, "", fmt.Errorf("location %d longitude: %w", rawLocation.ID, err)790- }791- category.Locations = append(category.Locations, catalogLocation{792- ID: rawLocation.ID,793- Title: rawLocation.Title,794- Description: rawLocation.Description,795- Latitude: lat,796- Longitude: lng,797- RegionID: rawLocation.RegionID,798- Attrs: rawLocation.Attrs,799- })800- m.PinCount++801- }802- group.Categories = append(group.Categories, category)803- }804- m.Groups = append(m.Groups, group)805- }806- for _, rawRegion := range raw.Regions {807- z := zone{808- ID: rawRegion.ID,809- Title: rawRegion.Title,810- Subtitle: rawRegion.Subtitle,811- Description: rawRegion.Description,812- ParentRegionID: rawRegion.ParentRegionID,813- Attrs: rawRegion.Attrs,814- }815- centerX, hasX, err := optionalNumber(rawRegion.CenterX)816- if err != nil {817- return nil, "", fmt.Errorf("region %d center_x: %w", rawRegion.ID, err)818- }819- centerY, hasY, err := optionalNumber(rawRegion.CenterY)820- if err != nil {821- return nil, "", fmt.Errorf("region %d center_y: %w", rawRegion.ID, err)822- }823- if hasX && hasY {824- z.Center = &coordinate{Latitude: centerY, Longitude: centerX}825- }826- for _, feature := range rawRegion.Features {827- if feature.Geometry.Type != "" && len(feature.Geometry.Coordinates) > 0 {828- z.Features = append(z.Features, feature.Geometry)829- }830- }831- if len(z.Features) > 0 {832- m.Zones = append(m.Zones, z)833- }777+ collections, err := normalizeWorld(raw)778+ if err != nil {779+ return nil, "", err834780 }781+ m.Collections = collections835782 resolveDescriptionLinks(&m)836783 pieces, err := splitWorld(m, raw, grid)837784 if err != nil {
@@ -860,42 +807,43 @@ var bareURL = regexp.MustCompile(`\s*\(?\s*https?://[^\s)]+\)?`)
860807 // navigate to instead.861808 func resolveDescriptionLinks(m *catalogWorld) {862809 known := make(map[int64]bool)863- for _, group := range m.Groups {864- for _, category := range group.Categories {865- for _, location := range category.Locations {866- known[location.ID] = true867- }810+ for _, collection := range m.Collections {811+ if collection.Kind != kindPoint {812+ continue813+ }814+ for _, pin := range collection.Features {815+ known[pin.ID] = true868816 }869817 }870- for groupIndex := range m.Groups {871- categories := m.Groups[groupIndex].Categories872- for categoryIndex := range categories {873- locations := categories[categoryIndex].Locations874- for locationIndex := range locations {875- location := &locations[locationIndex]876- if !strings.Contains(location.Description, "http") {877- continue878- }879- location.Description = markdownLink.ReplaceAllStringFunc(880- location.Description,881- func(match string) string {882- parts := markdownLink.FindStringSubmatch(match)883- label, target := parts[1], parts[2]884- id := mapgenieLocation.FindStringSubmatch(target)885- if id != nil && !strings.HasPrefix(label, "!") {886- if value, err := strconv.ParseInt(id[1], 10, 64); err == nil &&887- known[value] && value != location.ID {888- location.Links = append(location.Links, catalogLink{889- Title: label,890- LocationID: value,891- })892- }893- }894- return label895- },896- )897- location.Description = strings.TrimSpace(bareURL.ReplaceAllString(location.Description, ""))818+ for collectionIndex := range m.Collections {819+ collection := &m.Collections[collectionIndex]820+ if collection.Kind != kindPoint {821+ continue822+ }823+ for pinIndex := range collection.Features {824+ pin := &collection.Features[pinIndex]825+ if !strings.Contains(pin.Description, "http") {826+ continue898827 }828+ pin.Description = markdownLink.ReplaceAllStringFunc(829+ pin.Description,830+ func(match string) string {831+ parts := markdownLink.FindStringSubmatch(match)832+ label, target := parts[1], parts[2]833+ id := mapgenieLocation.FindStringSubmatch(target)834+ if id != nil && !strings.HasPrefix(label, "!") {835+ if value, err := strconv.ParseInt(id[1], 10, 64); err == nil &&836+ known[value] && value != pin.ID {837+ pin.Links = append(pin.Links, catalogLink{838+ Title: label,839+ LocationID: value,840+ })841+ }842+ }843+ return label844+ },845+ )846+ pin.Description = strings.TrimSpace(bareURL.ReplaceAllString(pin.Description, ""))899847 }900848 }901849 }
@@ -907,25 +855,23 @@ func attachVolumeIcons(gamePath string, game *catalogVolume) error {
907855 game.Icons = make(map[string][]byte)908856 copied := make(map[string]string)909857 for mapIndex := range game.Worlds {910- for groupIndex := range game.Worlds[mapIndex].Groups {911- categories := game.Worlds[mapIndex].Groups[groupIndex].Categories912- for categoryIndex := range categories {913- category := &categories[categoryIndex]914- if !validIconKey(category.Icon) {915- continue916- }917- asset, found := copied[category.Icon]918- if !found {919- var err error920- asset, err = readVolumeIcon(gamePath, game, category.Icon)921- if err != nil {922- return err923- }924- copied[category.Icon] = asset858+ collections := game.Worlds[mapIndex].Collections859+ for collectionIndex := range collections {860+ collection := &collections[collectionIndex]861+ if collection.Kind != kindPoint || !validIconKey(collection.Icon) {862+ continue863+ }864+ asset, found := copied[collection.Icon]865+ if !found {866+ var err error867+ asset, err = readVolumeIcon(gamePath, game, collection.Icon)868+ if err != nil {869+ return err925870 }926- category.IconAsset = asset927- category.IconPicture = strings.HasSuffix(asset, ".png")871+ copied[collection.Icon] = asset928872 }873+ collection.IconAsset = asset874+ collection.IconPicture = strings.HasSuffix(asset, ".png")929875 }930876 }931877 return nil
M tools/generate/main_test.go +31 -35
@@ -56,24 +56,22 @@ func TestAttachGameIcons(t *testing.T) {
5656 game := catalogVolume{5757 Slug: "pokemon-red-blue-yellow",5858 Worlds: []catalogWorld{{59- Groups: []catalogGroup{{60- Categories: []catalogCategory{61- {Icon: "pokemon_center"},62- {Icon: "missing"},63- },64- }},59+ Collections: []worldCollection{60+ {Kind: kindPoint, Icon: "pokemon_center"},61+ {Kind: kindPoint, Icon: "missing"},62+ },6563 }},6664 }6765 if err := attachVolumeIcons(source, &game); err != nil {6866 t.Fatal(err)6967 }7068 71- categories := game.Worlds[0].Groups[0].Categories72- if got, want := categories[0].IconAsset, "pokemon_center.svg"; got != want {69+ collections := game.Worlds[0].Collections70+ if got, want := collections[0].IconAsset, "pokemon_center.svg"; got != want {7371 t.Fatalf("icon asset = %q, want %q", got, want)7472 }75- if categories[1].IconAsset != "" {76- t.Fatalf("missing icon asset = %q, want empty", categories[1].IconAsset)73+ if collections[1].IconAsset != "" {74+ t.Fatalf("missing icon asset = %q, want empty", collections[1].IconAsset)7775 }7876 if got := string(game.Icons["pokemon_center.svg"]); got != svg {7977 t.Fatalf("carried SVG = %q, want %q", got, svg)
@@ -129,14 +127,14 @@ func TestSortGameMapsPrefersPrimaryMap(t *testing.T) {
129127 // this way once, and every layer's pins drew over every other.130128 func TestBuildPayloadPacksEveryColumn(t *testing.T) {131129 region := int64(77)132- m := catalogWorld{Groups: []catalogGroup{{Categories: []catalogCategory{133- {Locations: []catalogLocation{134- {ID: 11, Title: "Sky Mine", Latitude: 1.5, Longitude: -2.5, Shard: 1783},130+ m := catalogWorld{Collections: []worldCollection{131+ {Kind: kindPoint, Features: []feature{132+ {ID: 11, Title: "Sky Mine", Lat: 1.5, Lng: -2.5, Shard: 1783},135133 }},136- {Locations: []catalogLocation{137- {ID: 12, Title: "Deep Well", Latitude: -3, Longitude: 4, RegionID: &region, Shard: 1785},134+ {Kind: kindPoint, Features: []feature{135+ {ID: 12, Title: "Deep Well", Lat: -3, Lng: 4, Member: &region, Shard: 1785},138136 }},139- }}}}137+ }}140138 141139 _, packed, _ := buildPayload(m)142140
@@ -206,24 +204,23 @@ func TestGrowIntoGapsKeepsEveryPieceAroundItsGround(t *testing.T) {
206204 // stretch that measurement back over everything it left out.207205 func TestMarkSurfacesMeasuresContentsNotTheWindow(t *testing.T) {208206 grid := tileGrid{SourceZoom: 13, FirstTile: 4064, TileSize: 256, Size: 8192}209- at := func(x, y float64) catalogLocation {210- return catalogLocation{211- Latitude: unprojectLatitude(y, grid),212- Longitude: unprojectLongitude(x, grid),207+ at := func(x, y float64) feature {208+ return feature{209+ Lat: unprojectLatitude(y, grid),210+ Lng: unprojectLongitude(x, grid),213211 }214212 }215213 window := contentBounds{X: 2000, Y: 2000, Width: 3000, Height: 3000}216214 m := catalogWorld{217215 Lenses: []lens{{Bounds: &window}},218- Groups: []catalogGroup{{219- Categories: []catalogCategory{{220- Locations: []catalogLocation{221- at(3000, 3000),222- at(4000, 4000),223- // Off the sheet, and so no part of what the map covers.224- at(6000, 6000),225- },226- }},216+ Collections: []worldCollection{{217+ Kind: kindPoint,218+ Features: []feature{219+ at(3000, 3000),220+ at(4000, 4000),221+ // Off the sheet, and so no part of what the map covers.222+ at(6000, 6000),223+ },227224 }},228225 }229226
@@ -255,12 +252,11 @@ func TestMarkSurfacesKeepsGroundAlreadyMeasured(t *testing.T) {
255252 ground := contentBounds{X: 2500, Y: 2500, Width: 2000, Height: 2000}256253 m := catalogWorld{257254 Lenses: []lens{{Bounds: &window, Surface: &ground}},258- Groups: []catalogGroup{{259- Categories: []catalogCategory{{260- Locations: []catalogLocation{{261- Latitude: unprojectLatitude(3000, grid),262- Longitude: unprojectLongitude(3000, grid),263- }},255+ Collections: []worldCollection{{256+ Kind: kindPoint,257+ Features: []feature{{258+ Lat: unprojectLatitude(3000, grid),259+ Lng: unprojectLongitude(3000, grid),264260 }},265261 }},266262 }
M tools/generate/merge.go +203 -186
@@ -313,34 +313,31 @@ func contributeWorld(winner, donor *catalogVolume, donorMap *catalogWorld) error
313313 contributed := *donorMap314314 sourceTag := slugifyLabel(sourceLabelOf(donorMap, donor))315315 renamed := make(map[string]string)316- for groupIndex := range contributed.Groups {317- categories := contributed.Groups[groupIndex].Categories318- for categoryIndex := range categories {319- category := &categories[categoryIndex]320- if category.IconAsset == "" {321- continue322- }323- data, held := donor.Icons[category.IconAsset]324- if !held {325- category.IconAsset = ""326- category.IconPicture = false327- continue328- }329- name := category.IconAsset330- if replacement, seen := renamed[name]; seen {331- category.IconAsset = replacement332- continue333- }334- if winner.Icons == nil {335- winner.Icons = make(map[string][]byte)336- }337- if existing, taken := winner.Icons[name]; taken && !bytes.Equal(existing, data) {338- name = sourceTag + "--" + name339- }340- winner.Icons[name] = data341- renamed[category.IconAsset] = name342- category.IconAsset = name316+ for collectionIndex := range contributed.Collections {317+ collection := &contributed.Collections[collectionIndex]318+ if collection.IconAsset == "" {319+ continue320+ }321+ data, held := donor.Icons[collection.IconAsset]322+ if !held {323+ collection.IconAsset = ""324+ collection.IconPicture = false325+ continue326+ }327+ name := collection.IconAsset328+ if replacement, seen := renamed[name]; seen {329+ collection.IconAsset = replacement330+ continue331+ }332+ if winner.Icons == nil {333+ winner.Icons = make(map[string][]byte)343334 }335+ if existing, taken := winner.Icons[name]; taken && !bytes.Equal(existing, data) {336+ name = sourceTag + "--" + name337+ }338+ winner.Icons[name] = data339+ renamed[collection.IconAsset] = name340+ collection.IconAsset = name344341 }345342 winner.Worlds = append(winner.Worlds, contributed)346343 return nil
@@ -384,115 +381,119 @@ func mergeWorld(
384381 // Categories meet under their declared merge identity -- the385382 // atlas.category.key their payloads carry, their icon key otherwise --386383 // so the equivalence curation lives in the payloads, not here.387- var keptCategories []catalogCategory388- for _, group := range donor.Groups {389- for _, category := range group.Categories {390- kept := category391- kept.Locations = nil392- mappedKey := categoryKey(category.Icon, category.Attrs)393- adoptive := index.categories[mappedKey]394- // Attribute-level resolution at the category: a serving category395- // with no artwork of any kind takes the donor's standard-icon396- // declaration, and the ledger says so.397- if adoptive != nil && attributeMergePolicy[semconv.KeyIconStd] == donorFillsEmpty &&398- adoptive.IconAsset == "" && adoptive.Attrs[semconv.KeyIconStd] == "" &&399- category.Attrs[semconv.KeyIconStd] != "" {400- adoptive.Attrs = withAttr(adoptive.Attrs, semconv.KeyIconStd,401- category.Attrs[semconv.KeyIconStd])402- merge.Enriched = append(merge.Enriched, categoryTake{403- Category: mappedKey, Key: semconv.KeyIconStd,404- })405- }406- for _, location := range category.Locations {407- merge.DonorPins++408- x, y := affine.Apply(409- projectX(location.Longitude, donorGrid),410- projectY(location.Latitude, donorGrid),411- )412- outcome := resolvePin(location, x, y, index, mappedKey, nearbyRadius)413- switch outcome.kind {414- case pinMatched:415- index.claimed[outcome.match] = location.ID416- pair := mergedPair{417- Donor: location.ID,418- Winner: outcome.match.ID,419- DistancePx: int(outcome.distance + 0.5),420- }421- // Attribute-level: the policy table says, key by key,422- // what a serving pin takes from its donor, and every423- // take is ledgered. The description travels under its424- // policy name, and a pin's true coordinates fill in the425- // same way words do: only where the serving side has426- // none of its own.427- if attributeMergePolicy[semconv.KeyNoteText] == donorFillsEmpty &&428- outcome.match.Description == "" && location.Description != "" {429- outcome.match.Description = location.Description430- pair.Enriched = true431- pair.Took = append(pair.Took, semconv.KeyNoteText)432- }433- for _, key := range []string{semconv.KeyGeoLat, semconv.KeyGeoLon} {434- if attributeMergePolicy[key] != donorFillsEmpty {435- continue436- }437- if location.Attrs[key] == "" || outcome.match.Attrs[key] != "" {438- continue439- }440- outcome.match.Attrs = withAttr(outcome.match.Attrs, key, location.Attrs[key])441- pair.Took = append(pair.Took, key)442- }443- merge.Matched = append(merge.Matched, pair)444- case pinHeld:445- merge.Held = append(merge.Held, heldPin{446- Donor: location.ID, Title: location.Title, Reason: outcome.reason,447- })448- case pinDistinct:449- if x < 0 || y < 0 || x > float64(shared.Size) || y > float64(shared.Size) {450- merge.Rejected = append(merge.Rejected, heldPin{451- Donor: location.ID, Title: location.Title,452- Reason: "outside the shared world",453- })384+ var keptCollections []worldCollection385+ for _, donorCollection := range donor.Collections {386+ if donorCollection.Kind != kindPoint {387+ continue388+ }389+ kept := donorCollection390+ kept.Features = nil391+ mappedKey := categoryKey(donorCollection.Icon, donorCollection.Attrs)392+ adoptive := index.categories[mappedKey]393+ // Attribute-level resolution at the category: a serving category394+ // with no artwork of any kind takes the donor's standard-icon395+ // declaration, and the ledger says so.396+ if adoptive != nil && attributeMergePolicy[semconv.KeyIconStd] == donorFillsEmpty &&397+ adoptive.IconAsset == "" && adoptive.Attrs[semconv.KeyIconStd] == "" &&398+ donorCollection.Attrs[semconv.KeyIconStd] != "" {399+ adoptive.Attrs = withAttr(adoptive.Attrs, semconv.KeyIconStd,400+ donorCollection.Attrs[semconv.KeyIconStd])401+ merge.Enriched = append(merge.Enriched, categoryTake{402+ Category: mappedKey, Key: semconv.KeyIconStd,403+ })404+ }405+ for _, pin := range donorCollection.Features {406+ merge.DonorPins++407+ x, y := affine.Apply(408+ projectX(pin.Lng, donorGrid),409+ projectY(pin.Lat, donorGrid),410+ )411+ outcome := resolvePin(pin, x, y, index, mappedKey, nearbyRadius)412+ switch outcome.kind {413+ case pinMatched:414+ index.claimed[outcome.match] = pin.ID415+ pair := mergedPair{416+ Donor: pin.ID,417+ Winner: outcome.match.ID,418+ DistancePx: int(outcome.distance + 0.5),419+ }420+ // Attribute-level: the policy table says, key by key,421+ // what a serving pin takes from its donor, and every422+ // take is ledgered. The description travels under its423+ // policy name, and a pin's true coordinates fill in the424+ // same way words do: only where the serving side has425+ // none of its own.426+ if attributeMergePolicy[semconv.KeyNoteText] == donorFillsEmpty &&427+ outcome.match.Description == "" && pin.Description != "" {428+ outcome.match.Description = pin.Description429+ pair.Enriched = true430+ pair.Took = append(pair.Took, semconv.KeyNoteText)431+ }432+ for _, key := range []string{semconv.KeyGeoLat, semconv.KeyGeoLon} {433+ if attributeMergePolicy[key] != donorFillsEmpty {454434 continue455435 }456- if holder, taken := index.ids[location.ID]; taken {457- return fmt.Errorf("pin id %d (%s) collides with serving pin %q",458- location.ID, location.Title, holder)459- }460- index.ids[location.ID] = location.Title461- moved := location462- moved.Longitude = unprojectLongitude(x, winnerGrid)463- moved.Latitude = unprojectLatitude(y, winnerGrid)464- moved.RegionID = nil465- moved.Shard = 0466- merge.Added++467- if adoptive != nil {468- adoptive.Locations = append(adoptive.Locations, moved)469- merge.Adopted = append(merge.Adopted, adoptedPin{470- Donor: location.ID, Into: mappedKey,471- })472- } else {473- kept.Locations = append(kept.Locations, moved)436+ if pin.Attrs[key] == "" || outcome.match.Attrs[key] != "" {437+ continue474438 }439+ outcome.match.Attrs = withAttr(outcome.match.Attrs, key, pin.Attrs[key])440+ pair.Took = append(pair.Took, key)441+ }442+ merge.Matched = append(merge.Matched, pair)443+ case pinHeld:444+ merge.Held = append(merge.Held, heldPin{445+ Donor: pin.ID, Title: pin.Title, Reason: outcome.reason,446+ })447+ case pinDistinct:448+ if x < 0 || y < 0 || x > float64(shared.Size) || y > float64(shared.Size) {449+ merge.Rejected = append(merge.Rejected, heldPin{450+ Donor: pin.ID, Title: pin.Title,451+ Reason: "outside the shared world",452+ })453+ continue454+ }455+ if holder, taken := index.ids[pin.ID]; taken {456+ return fmt.Errorf("pin id %d (%s) collides with serving pin %q",457+ pin.ID, pin.Title, holder)458+ }459+ index.ids[pin.ID] = pin.Title460+ moved := pin461+ moved.Lng = unprojectLongitude(x, winnerGrid)462+ moved.Lat = unprojectLatitude(y, winnerGrid)463+ moved.Member = nil464+ moved.Shard = 0465+ merge.Added++466+ if adoptive != nil {467+ adoptive.Features = append(adoptive.Features, moved)468+ merge.Adopted = append(merge.Adopted, adoptedPin{469+ Donor: pin.ID, Into: mappedKey,470+ })471+ } else {472+ kept.Features = append(kept.Features, moved)475473 }476474 }477- if len(kept.Locations) == 0 {478- continue479- }480- if err := carryIcon(winnerGame, donorGame, &kept, sourceLabel); err != nil {481- return err482- }483- keptCategories = append(keptCategories, kept)484475 }476+ if len(kept.Features) == 0 {477+ continue478+ }479+ if err := carryIcon(winnerGame, donorGame, &kept, sourceLabel); err != nil {480+ return err481+ }482+ keptCollections = append(keptCollections, kept)485483 }486484 487- if len(keptCategories) > 0 {488- winner.Groups = append(winner.Groups, catalogGroup{489- ID: mergedGroupID(winner),490- Title: sourceLabel,491- Categories: keptCategories,492- })485+ if len(keptCollections) > 0 {486+ // The source-named group exists only on the wire; here it is a group487+ // title and number the kept collections share, and the emission488+ // regroups them under it exactly as the old group append did.489+ groupID := mergedGroupID(winner)490+ for index := range keptCollections {491+ keptCollections[index].Group = sourceLabel492+ keptCollections[index].GroupID = groupID493+ }494+ winner.Collections = append(winner.Collections, keptCollections...)493495 }494496 winner.Merged = append(winner.Merged, *merge)495- winner.PinCount += merge.Added496497 // The merged account is as fresh as its freshest ingredient, so a newer497498 // donor capture re-versions the merged bundle even when the serving498499 // capture stood still.
@@ -515,16 +516,16 @@ func mergeWorld(
515516 type winnerIndex struct {516517 pins []placedPin517518 byName map[string][]int518- categories map[string]*catalogCategory519+ categories map[string]*worldCollection519520 ids map[int64]string520521 // claimed maps each serving pin already matched to the donor that521522 // matched it. A place is one place: the next donor bearing the same name522523 // must find its own, or say why it cannot.523- claimed map[*catalogLocation]int64524+ claimed map[*feature]int64524525 }525526 526527 type placedPin struct {527- location *catalogLocation528+ feature *feature528529 category string529530 x, y float64530531 tokens map[string]bool
@@ -533,30 +534,31 @@ type placedPin struct {
533534 func indexWinner(winner *catalogWorld, grid tileGrid) *winnerIndex {534535 index := &winnerIndex{535536 byName: make(map[string][]int),536- categories: make(map[string]*catalogCategory),537+ categories: make(map[string]*worldCollection),537538 ids: make(map[int64]string),538- claimed: make(map[*catalogLocation]int64),539- }540- for groupIndex := range winner.Groups {541- for categoryIndex := range winner.Groups[groupIndex].Categories {542- category := &winner.Groups[groupIndex].Categories[categoryIndex]543- key := categoryKey(category.Icon, category.Attrs)544- if _, held := index.categories[key]; !held {545- index.categories[key] = category546- }547- for locationIndex := range category.Locations {548- location := &category.Locations[locationIndex]549- name := blend.NormalizeTitle(location.Title)550- index.byName[name] = append(index.byName[name], len(index.pins))551- index.pins = append(index.pins, placedPin{552- location: location,553- category: key,554- x: projectX(location.Longitude, grid),555- y: projectY(location.Latitude, grid),556- tokens: tokensOf(name),557- })558- index.ids[location.ID] = location.Title559- }539+ claimed: make(map[*feature]int64),540+ }541+ for collectionIndex := range winner.Collections {542+ collection := &winner.Collections[collectionIndex]543+ if collection.Kind != kindPoint {544+ continue545+ }546+ key := categoryKey(collection.Icon, collection.Attrs)547+ if _, held := index.categories[key]; !held {548+ index.categories[key] = collection549+ }550+ for pinIndex := range collection.Features {551+ pin := &collection.Features[pinIndex]552+ name := blend.NormalizeTitle(pin.Title)553+ index.byName[name] = append(index.byName[name], len(index.pins))554+ index.pins = append(index.pins, placedPin{555+ feature: pin,556+ category: key,557+ x: projectX(pin.Lng, grid),558+ y: projectY(pin.Lat, grid),559+ tokens: tokensOf(name),560+ })561+ index.ids[pin.ID] = pin.Title560562 }561563 }562564 return index
@@ -564,7 +566,7 @@ func indexWinner(winner *catalogWorld, grid tileGrid) *winnerIndex {
564566 565567 type pinOutcome struct {566568 kind int567- match *catalogLocation569+ match *feature568570 distance float64569571 reason string570572 }
@@ -584,7 +586,7 @@ const (
584586 // same mapped category is likewise held rather than guessed -- proximity585587 // alone never merges -- and only a pin resembling nothing at all is added.586588 func resolvePin(587- donor catalogLocation,589+ donor feature,588590 x, y float64,589591 index *winnerIndex,590592 mappedKey string,
@@ -600,7 +602,7 @@ func resolvePin(
600602 // A serving pin another donor already resolved to is one place, not601603 // two: it cannot be matched again, but its nearness is remembered so602604 // the refusal can say what stood in the way.603- if _, taken := index.claimed[pin.location]; taken {605+ if _, taken := index.claimed[pin.feature]; taken {604606 if distance <= matchRadiusPx {605607 alreadyClaimed = true606608 }
@@ -631,11 +633,11 @@ func resolvePin(
631633 if nearest != nil {632634 switch {633635 case nearestDistance <= matchRadiusPx:634- return pinOutcome{kind: pinMatched, match: nearest.location, distance: nearestDistance}636+ return pinOutcome{kind: pinMatched, match: nearest.feature, distance: nearestDistance}635637 case nearestDistance <= separateRadiusPx:636638 return pinOutcome{kind: pinHeld, reason: fmt.Sprintf(637639 "named like %q %.0fpx away; too far to merge, too near to double",638- nearest.location.Title, nearestDistance)}640+ nearest.feature.Title, nearestDistance)}639641 }640642 }641643 if alreadyClaimed {
@@ -649,7 +651,7 @@ func resolvePin(
649651 }650652 if math.Hypot(pin.x-x, pin.y-y) <= nearbyRadius {651653 return pinOutcome{kind: pinHeld, reason: fmt.Sprintf(652- "beside %q in the same category; names disagree", pin.location.Title)}654+ "beside %q in the same category; names disagree", pin.feature.Title)}653655 }654656 }655657 }
@@ -700,22 +702,22 @@ func tokensOf(normalized string) map[string]bool {
700702 // carryIcon brings a merged category's icon across from the donor's archive701703 // under a source-prefixed name, so it cannot displace a serving icon that702704 // shares its key.703-func carryIcon(winnerGame, donorGame *catalogVolume, category *catalogCategory, sourceLabel string) error {704- if category.IconAsset == "" {705+func carryIcon(winnerGame, donorGame *catalogVolume, collection *worldCollection, sourceLabel string) error {706+ if collection.IconAsset == "" {705707 return nil706708 }707- data, held := donorGame.Icons[category.IconAsset]709+ data, held := donorGame.Icons[collection.IconAsset]708710 if !held {709- category.IconAsset = ""710- category.IconPicture = false711+ collection.IconAsset = ""712+ collection.IconPicture = false711713 return nil712714 }713- carried := slugifyLabel(sourceLabel) + "--" + category.IconAsset715+ carried := slugifyLabel(sourceLabel) + "--" + collection.IconAsset714716 if winnerGame.Icons == nil {715717 winnerGame.Icons = make(map[string][]byte)716718 }717719 winnerGame.Icons[carried] = data718- category.IconAsset = carried720+ collection.IconAsset = carried719721 return nil720722 }721723
@@ -723,8 +725,10 @@ func carryIcon(winnerGame, donorGame *catalogVolume, category *catalogCategory,
723725 // already uses.724726 func mergedGroupID(winner *catalogWorld) int64 {725727 used := make(map[int64]bool)726- for _, group := range winner.Groups {727- used[group.ID] = true728+ for _, collection := range winner.Collections {729+ if collection.Kind == kindPoint {730+ used[collection.GroupID] = true731+ }728732 }729733 id := int64(1)730734 for used[id] {
@@ -793,15 +797,16 @@ func gridOf(m *catalogWorld, shared tileGrid) tileGrid {
793797 794798 func anchorsOfWorld(m *catalogWorld, grid tileGrid) []blend.Anchor {795799 var anchors []blend.Anchor796- for _, group := range m.Groups {797- for _, category := range group.Categories {798- for _, location := range category.Locations {799- anchors = append(anchors, blend.Anchor{800- Title: location.Title,801- X: projectX(location.Longitude, grid),802- Y: projectY(location.Latitude, grid),803- })804- }800+ for _, collection := range m.Collections {801+ if collection.Kind != kindPoint {802+ continue803+ }804+ for _, pin := range collection.Features {805+ anchors = append(anchors, blend.Anchor{806+ Title: pin.Title,807+ X: projectX(pin.Lng, grid),808+ Y: projectY(pin.Lat, grid),809+ })805810 }806811 }807812 return anchors
@@ -858,19 +863,31 @@ func mergeGate(merge *mergedSource, winner *catalogWorld) error {
858863 }859864 seen := make(map[int64]string)860865 counted := 0861- for _, group := range winner.Groups {862- for _, category := range group.Categories {863- for _, location := range category.Locations {864- if holder, taken := seen[location.ID]; taken {865- return fmt.Errorf("pin id %d held by both %q and %q", location.ID, holder, location.Title)866- }867- seen[location.ID] = location.Title868- counted++866+ for _, collection := range winner.Collections {867+ if collection.Kind != kindPoint {868+ continue869+ }870+ for _, pin := range collection.Features {871+ if holder, taken := seen[pin.ID]; taken {872+ return fmt.Errorf("pin id %d held by both %q and %q", pin.ID, holder, pin.Title)869873 }874+ seen[pin.ID] = pin.Title875+ counted++876+ }877+ }878+ // The count the map claims is no longer a field to drift; it is the sum879+ // of its own ledger -- what it opened with, plus what every merge says it880+ // added -- and the map must actually hold that many.881+ ledgered := 0882+ for _, account := range winner.Merged {883+ if account.Origin {884+ ledgered += account.DonorPins885+ } else {886+ ledgered += account.Added870887 }871888 }872- if counted != winner.PinCount {873- return fmt.Errorf("world holds %d pins but claims %d", counted, winner.PinCount)889+ if counted != ledgered {890+ return fmt.Errorf("world holds %d pins but its ledger claims %d", counted, ledgered)874891 }875892 return nil876893 }
A tools/generate/normalize.go +244 -0
@@ -0,0 +1,244 @@
1+package main2+3+import "fmt"4+5+// A map's contents used to mirror the MapGenie shape all the way through the6+// generator: pins under groups and categories, regions standing apart as7+// zones, and every pass over a map written twice because of it. The catalog8+// now holds one model instead -- collections of features, each feature a9+// geometry with attributes -- and the v2 wire keeps its exact bytes by being10+// reconstructed from that model at emission time. The wire can only move once11+// everything it says provably fits the model, and the byte-identical rebuild12+// is that proof.13+14+// A collection's kind is the dimensionality of its features. Nothing15+// classifies geometry yet -- paths still travel as pretend areas -- so the16+// vocabulary is declared in full but only points and areas are spoken.17+const (18+ kindPoint = "point"19+ kindPath = "path"20+ kindArea = "area"21+)22+23+// regionsCollectionKey names the implicit collection every world's regions24+// fold into. MapGenie never declared its regions as a collection, so the25+// generator declares one for them.26+const regionsCollectionKey = "regions"27+28+// worldCollection is one legend entry's worth of features: a category of29+// pins, or the world's regions gathered under the implicit key.30+type worldCollection struct {31+ ID int6432+ // Key is the collection's curation and merge identity; legacy categories33+ // carry none and are still met by their icon key instead.34+ Key string35+ Title string36+ // Group is the legend section this collection files under -- today's37+ // group title -- and GroupID is the number that lets the v2 groups be38+ // re-emitted byte-identically. The implicit region collection has neither.39+ Group string40+ GroupID int6441+ // Kind is "point", "path" or "area". The implicit region collection says42+ // "area" nominally; its features keep whatever geometry the source drew.43+ Kind string44+ Icon, IconAsset string45+ IconPicture bool46+ Color, IconColor string47+ // DisplayType is the legacy render field, kept only because the v2 wire48+ // still spells it.49+ DisplayType string50+ Visible bool51+ Attrs map[string]string52+ Features []feature53+}54+55+// feature is one thing on the map, whatever its dimensionality: a point56+// carries Lat and Lng, a shape carries Geometry, and everything else --57+// prose, attributes, shard, nesting -- reads the same for both.58+type feature struct {59+ ID int6460+ Title string61+ Subtitle string62+ Description string63+ Lat, Lng float6464+ Geometry []geometry65+ // Member is the area feature containing this point -- the region a66+ // MapGenie pin names -- where Parent nests one shape inside another.67+ Member *int6468+ Parent *int6469+ Center *coordinate70+ Shard int6471+ // HasText stays unset until the wire carries features directly: the v272+ // emission derives its marker from Description at packing time.73+ HasText bool74+ Links []catalogLink75+ Attrs map[string]string76+}77+78+// normalizeWorld folds a decoded capture into the uniform model: each79+// category becomes a point collection carrying its resolved colours, and80+// every region becomes a shape feature of one implicit collection. A region81+// whose geometry all came through empty is dropped here, exactly as it always82+// was, because a shape with nothing to draw is not a shape.83+func normalizeWorld(raw rawMap) ([]worldCollection, error) {84+ var collections []worldCollection85+ for _, rawGroup := range raw.Groups {86+ for _, rawCategory := range rawGroup.Categories {87+ collection := worldCollection{88+ ID: rawCategory.ID,89+ Title: rawCategory.Title,90+ Group: rawGroup.Title,91+ GroupID: rawGroup.ID,92+ Kind: kindPoint,93+ Icon: rawCategory.Icon,94+ Color: resolvedCategoryColor(rawGroup, rawCategory),95+ IconColor: resolvedIconColor(rawGroup, rawCategory),96+ DisplayType: rawCategory.DisplayType,97+ Visible: rawCategory.Visible,98+ Attrs: rawCategory.Attrs,99+ }100+ for _, rawLocation := range rawCategory.Locations {101+ lat, err := number(rawLocation.Latitude)102+ if err != nil {103+ return nil, fmt.Errorf("location %d latitude: %w", rawLocation.ID, err)104+ }105+ lng, err := number(rawLocation.Longitude)106+ if err != nil {107+ return nil, fmt.Errorf("location %d longitude: %w", rawLocation.ID, err)108+ }109+ collection.Features = append(collection.Features, feature{110+ ID: rawLocation.ID,111+ Title: rawLocation.Title,112+ Description: rawLocation.Description,113+ Lat: lat,114+ Lng: lng,115+ Member: rawLocation.RegionID,116+ Attrs: rawLocation.Attrs,117+ })118+ }119+ collections = append(collections, collection)120+ }121+ }122+ regions := worldCollection{123+ Key: regionsCollectionKey,124+ Title: "Regions",125+ Kind: kindArea,126+ }127+ for _, rawRegion := range raw.Regions {128+ shape := feature{129+ ID: rawRegion.ID,130+ Title: rawRegion.Title,131+ Subtitle: rawRegion.Subtitle,132+ Description: rawRegion.Description,133+ Parent: rawRegion.ParentRegionID,134+ Attrs: rawRegion.Attrs,135+ }136+ centerX, hasX, err := optionalNumber(rawRegion.CenterX)137+ if err != nil {138+ return nil, fmt.Errorf("region %d center_x: %w", rawRegion.ID, err)139+ }140+ centerY, hasY, err := optionalNumber(rawRegion.CenterY)141+ if err != nil {142+ return nil, fmt.Errorf("region %d center_y: %w", rawRegion.ID, err)143+ }144+ if hasX && hasY {145+ shape.Center = &coordinate{Latitude: centerY, Longitude: centerX}146+ }147+ for _, part := range rawRegion.Features {148+ if part.Geometry.Type != "" && len(part.Geometry.Coordinates) > 0 {149+ shape.Geometry = append(shape.Geometry, part.Geometry)150+ }151+ }152+ if len(shape.Geometry) > 0 {153+ regions.Features = append(regions.Features, shape)154+ }155+ }156+ if len(regions.Features) > 0 {157+ collections = append(collections, regions)158+ }159+ return collections, nil160+}161+162+// v2Groups rebuilds the wire's group tree from the collections. Categories163+// within a group were contiguous in every capture and every pass keeps them164+// so, which is why regrouping by first-seen GroupID reproduces the original165+// structure byte for byte.166+func (m catalogWorld) v2Groups() []catalogGroup {167+ var groups []catalogGroup168+ at := make(map[int64]int)169+ for _, collection := range m.Collections {170+ if collection.Kind != kindPoint {171+ continue172+ }173+ index, seen := at[collection.GroupID]174+ if !seen {175+ index = len(groups)176+ at[collection.GroupID] = index177+ groups = append(groups, catalogGroup{ID: collection.GroupID, Title: collection.Group})178+ }179+ category := catalogCategory{180+ ID: collection.ID,181+ Title: collection.Title,182+ Icon: collection.Icon,183+ IconAsset: collection.IconAsset,184+ IconPicture: collection.IconPicture,185+ Color: collection.Color,186+ IconColor: collection.IconColor,187+ DisplayType: collection.DisplayType,188+ Visible: collection.Visible,189+ Attrs: collection.Attrs,190+ }191+ for _, f := range collection.Features {192+ category.Locations = append(category.Locations, catalogLocation{193+ ID: f.ID,194+ Title: f.Title,195+ Description: f.Description,196+ Latitude: f.Lat,197+ Longitude: f.Lng,198+ RegionID: f.Member,199+ Shard: f.Shard,200+ Links: f.Links,201+ Attrs: f.Attrs,202+ })203+ }204+ groups[index].Categories = append(groups[index].Categories, category)205+ }206+ return groups207+}208+209+// v2Zones rebuilds the wire's zone list from the implicit region collection,210+// in the order the capture told them.211+func (m catalogWorld) v2Zones() []zone {212+ var zones []zone213+ for _, collection := range m.Collections {214+ if collection.Key != regionsCollectionKey {215+ continue216+ }217+ for _, f := range collection.Features {218+ zones = append(zones, zone{219+ ID: f.ID,220+ Title: f.Title,221+ Subtitle: f.Subtitle,222+ Description: f.Description,223+ ParentRegionID: f.Parent,224+ Center: f.Center,225+ Shard: f.Shard,226+ Features: f.Geometry,227+ Attrs: f.Attrs,228+ })229+ }230+ }231+ return zones232+}233+234+// pinCount is how many point features the map holds, which is what the235+// manifest has always reported for it.236+func (m catalogWorld) pinCount() int {237+ total := 0238+ for _, collection := range m.Collections {239+ if collection.Kind == kindPoint {240+ total += len(collection.Features)241+ }242+ }243+ return total244+}
A tools/generate/normalize_test.go +103 -0
@@ -0,0 +1,103 @@
1+package main2+3+import (4+ "encoding/json"5+ "reflect"6+ "testing"7+)8+9+// The collections model only earns its place if the v2 wire can be rebuilt10+// from it without a byte moving: a capture normalized into collections and11+// re-emitted must say exactly what the old direct build said. This walks one12+// fixture through normalizeWorld and the emission-time reconstruction and13+// holds the result to the shapes the old code produced.14+func TestNormalizeWorldRoundTripsTheV2Shapes(t *testing.T) {15+ oldTown := int64(9001)16+ ring := json.RawMessage(`[[[[0,0],[1,0],[1,1]]]]`)17+ raw := rawMap{18+ Groups: []rawGroup{19+ {ID: 1, Title: "Districts", Color: "38344C", IconColor: "ffffff", Categories: []rawCategory{20+ {21+ ID: 10, Title: "Shops", Icon: "shop", Color: "aabbcc",22+ DisplayType: "pin", Visible: true,23+ Locations: []rawLocation{{24+ ID: 100,25+ Title: "Bakery",26+ Description: "Fresh bread.",27+ Latitude: json.RawMessage(`"1.5"`),28+ Longitude: json.RawMessage(`-2.25`),29+ RegionID: &oldTown,30+ Attrs: map[string]string{"atlas.geo.lat": "39.8"},31+ }},32+ },33+ {ID: 11, Title: "Signs", Icon: "sign", DisplayType: "text", Visible: false},34+ }},35+ {ID: 2, Title: "Nature", Categories: []rawCategory{36+ {ID: 12, Title: "Trees", Icon: "tree", Visible: true},37+ }},38+ },39+ Regions: []rawRegion{40+ {41+ ID: 9001, Title: "Old Town", Subtitle: "OT", Description: "The old quarter.",42+ CenterX: json.RawMessage(`"12.5"`),43+ CenterY: json.RawMessage(`3.25`),44+ Features: []rawFeature{45+ {Geometry: geometry{Type: "MultiPolygon", Coordinates: ring}},46+ // An empty part never survived the old build either.47+ {Geometry: geometry{}},48+ },49+ },50+ // A region whose geometry all came through empty has nothing to51+ // draw, and the old build dropped it whole.52+ {ID: 9002, Title: "Ghost", Features: []rawFeature{{Geometry: geometry{Type: "Polygon"}}}},53+ },54+ }55+56+ collections, err := normalizeWorld(raw)57+ if err != nil {58+ t.Fatal(err)59+ }60+ m := catalogWorld{Collections: collections}61+62+ wantGroups := []catalogGroup{63+ {ID: 1, Title: "Districts", Categories: []catalogCategory{64+ {65+ ID: 10, Title: "Shops", Icon: "shop", Color: "#AABBCC", IconColor: "#FFFFFF",66+ DisplayType: "pin", Visible: true,67+ Locations: []catalogLocation{{68+ ID: 100, Title: "Bakery", Description: "Fresh bread.",69+ Latitude: 1.5, Longitude: -2.25, RegionID: &oldTown,70+ Attrs: map[string]string{"atlas.geo.lat": "39.8"},71+ }},72+ },73+ {74+ ID: 11, Title: "Signs", Icon: "sign", Color: "#38344C", IconColor: "#FFFFFF",75+ DisplayType: "text",76+ },77+ }},78+ {ID: 2, Title: "Nature", Categories: []catalogCategory{79+ {ID: 12, Title: "Trees", Icon: "tree", Visible: true},80+ }},81+ }82+ if got := m.v2Groups(); !reflect.DeepEqual(got, wantGroups) {83+ t.Fatalf("v2 groups = %+v\nwant %+v", got, wantGroups)84+ }85+86+ wantZones := []zone{{87+ ID: 9001, Title: "Old Town", Subtitle: "OT", Description: "The old quarter.",88+ Center: &coordinate{Latitude: 3.25, Longitude: 12.5},89+ Features: []geometry{{Type: "MultiPolygon", Coordinates: ring}},90+ }}91+ if got := m.v2Zones(); !reflect.DeepEqual(got, wantZones) {92+ t.Fatalf("v2 zones = %+v\nwant %+v", got, wantZones)93+ }94+95+ if got := m.pinCount(); got != 1 {96+ t.Fatalf("pin count = %d, want 1", got)97+ }98+ last := collections[len(collections)-1]99+ if last.Key != regionsCollectionKey || last.Kind != kindArea {100+ t.Fatalf("implicit collection = {key %q, kind %q}, want {%q, %q}",101+ last.Key, last.Kind, regionsCollectionKey, kindArea)102+ }103+}
M tools/generate/shard.go +63 -69
@@ -451,9 +451,10 @@ func asMaps(m catalogWorld, pieces []shard) []catalogWorld {
451451 out := make([]catalogWorld, 0, len(pieces))452452 for index, piece := range pieces {453453 copied := m454- copied.Groups = keepLocations(m.Groups, piece.Locations)455- copied.Zones = keepZones(m.Zones, piece.Regions, piece.Region.ID)456- copied.PinCount = countLocations(copied.Groups)454+ copied.Collections = keepShapeFeatures(455+ keepPointFeatures(m.Collections, piece.Locations),456+ piece.Regions, piece.Region.ID,457+ )457458 copied.Center = piece.Center458459 copied.Lenses = boundVariants(m.Lenses, piece.Bounds, piece.Surface, 0)459460 if index > 0 {
@@ -479,26 +480,22 @@ func asVariants(m catalogWorld, pieces []shard) catalogWorld {
479480 bound[index].Name = piece.Region.Title480481 }481482 variants = append(variants, bound...)482- for groupIndex := range m.Groups {483- categories := m.Groups[groupIndex].Categories484- for categoryIndex := range categories {485- locations := categories[categoryIndex].Locations486- for locationIndex := range locations {487- if piece.Locations[locations[locationIndex].ID] {488- locations[locationIndex].Shard = piece.Region.ID489- }490- }483+ for collectionIndex := range m.Collections {484+ collection := &m.Collections[collectionIndex]485+ claims := piece.Regions486+ if collection.Kind == kindPoint {487+ claims = piece.Locations491488 }492- }493- for zoneIndex := range m.Zones {494- if piece.Regions[m.Zones[zoneIndex].ID] {495- m.Zones[zoneIndex].Shard = piece.Region.ID489+ for featureIndex := range collection.Features {490+ if claims[collection.Features[featureIndex].ID] {491+ collection.Features[featureIndex].Shard = piece.Region.ID492+ }496493 }497494 }498495 dropped = append(dropped, piece.Region.ID)499496 }500497 m.Lenses = variants501- m.Zones = keepZones(m.Zones, nil, dropped...)498+ m.Collections = keepShapeFeatures(m.Collections, nil, dropped...)502499 if len(pieces) > 0 {503500 m.Center = pieces[0].Center504501 }
@@ -576,25 +573,21 @@ func contentExtent(m catalogWorld, shard int64, window contentBounds, grid tileG
576573 found = true577574 }578575 579- for _, group := range m.Groups {580- for _, category := range group.Categories {581- for _, location := range category.Locations {582- if shard != 0 && location.Shard != shard {583- continue584- }585- x, y := projectPoint(location.Latitude, location.Longitude, grid)586- grow(x, y)576+ for _, collection := range m.Collections {577+ for _, f := range collection.Features {578+ if shard != 0 && f.Shard != shard {579+ continue587580 }588- }589- }590- for _, z := range m.Zones {591- if shard != 0 && z.Shard != shard {592- continue593- }594- for _, feature := range z.Features {595- for _, point := range flattenCoordinates(feature.Coordinates) {596- x, y := projectPoint(point[1], point[0], grid)581+ if collection.Kind == kindPoint {582+ x, y := projectPoint(f.Lat, f.Lng, grid)597583 grow(x, y)584+ continue585+ }586+ for _, part := range f.Geometry {587+ for _, point := range flattenCoordinates(part.Coordinates) {588+ x, y := projectPoint(point[1], point[0], grid)589+ grow(x, y)590+ }598591 }599592 }600593 }
@@ -635,58 +628,59 @@ func intersectBounds(a, b contentBounds) contentBounds {
635628 return contentBounds{X: left, Y: top, Width: right - left, Height: bottom - top}636629 }637630 638-func keepLocations(groups []catalogGroup, wanted map[int64]bool) []catalogGroup {639- out := make([]catalogGroup, 0, len(groups))640- for _, group := range groups {641- kept := make([]catalogCategory, 0, len(group.Categories))642- for _, category := range group.Categories {643- locations := make([]catalogLocation, 0, len(category.Locations))644- for _, location := range category.Locations {645- if wanted[location.ID] {646- locations = append(locations, location)647- }648- }649- if len(locations) == 0 {650- continue631+// keepPointFeatures pares each point collection down to the piece's own pins.632+// A collection left with none is dropped, which is also what empties a group633+// out of the piece's payload: no collection carries its number any more.634+func keepPointFeatures(collections []worldCollection, wanted map[int64]bool) []worldCollection {635+ out := make([]worldCollection, 0, len(collections))636+ for _, collection := range collections {637+ if collection.Kind != kindPoint {638+ out = append(out, collection)639+ continue640+ }641+ kept := make([]feature, 0, len(collection.Features))642+ for _, pin := range collection.Features {643+ if wanted[pin.ID] {644+ kept = append(kept, pin)651645 }652- category.Locations = locations653- kept = append(kept, category)654646 }655647 if len(kept) == 0 {656648 continue657649 }658- group.Categories = kept659- out = append(out, group)650+ collection.Features = kept651+ out = append(out, collection)660652 }661653 return out662654 }663655 664-// keepZones drops the piece's own outline. Once a region has been used to cut665-// a map or a layer out of a sheet, drawing it again just traces the edge of666-// what the reader is already looking at.667-func keepZones(zones []zone, wanted map[int64]bool, drop ...int64) []zone {656+// keepShapeFeatures drops the piece's own outline. Once a region has been657+// used to cut a map or a layer out of a sheet, drawing it again just traces658+// the edge of what the reader is already looking at.659+func keepShapeFeatures(collections []worldCollection, wanted map[int64]bool, drop ...int64) []worldCollection {668660 discard := make(map[int64]bool, len(drop))669661 for _, id := range drop {670662 discard[id] = true671663 }672- out := make([]zone, 0, len(zones))673- for _, z := range zones {674- if discard[z.ID] || (wanted != nil && !wanted[z.ID]) {664+ out := make([]worldCollection, 0, len(collections))665+ for _, collection := range collections {666+ if collection.Kind == kindPoint {667+ out = append(out, collection)675668 continue676669 }677- out = append(out, z)678- }679- return out680-}681-682-func countLocations(groups []catalogGroup) int {683- total := 0684- for _, group := range groups {685- for _, category := range group.Categories {686- total += len(category.Locations)670+ kept := make([]feature, 0, len(collection.Features))671+ for _, shape := range collection.Features {672+ if discard[shape.ID] || (wanted != nil && !wanted[shape.ID]) {673+ continue674+ }675+ kept = append(kept, shape)687676 }677+ if len(kept) == 0 {678+ continue679+ }680+ collection.Features = kept681+ out = append(out, collection)688682 }689- return total683+ return out690684 }691685 692686 func slugify(value string) string {
M tools/generate/split.go +6 -2
M tools/generate/split_test.go +9 -4