Piggyback is the official guide house, and its maps carry what the wikimaps do not: prose. Pins arrive named and described from two open API requests under the map's own address, and both survive into the capture -- internal/pbmap keeps them through translation, so the bundle is the first whose pin details read as a guide rather than a label. District names, filed by the API under the reader's own favorites category, become the floating text labels they are on Piggyback's map. The tiles need more care than IGN's. The CDN answers only requests that arrive as the map's own, publishes no bounds, and prices its deepest two levels as premium: the crawler names the page it stands in for, surveys the pyramid level by level following only the children of tiles that exist, and stops at the free ceiling unless -max-zoom asks past it. What the survey finds becomes the capture's account of the pyramid, and pins project through the map's own Leaflet transformation -- recorded in the capture -- into the shared world.
- commit
- 22bce7439944b1ebd64205a7c33eadf9170b6408
- parent
- abe71f06e0892ddbe6a049182b2766cdf550292e
- author
- Dami <dami.etoile@gmail.com>
| A | internal/ | 451 | ++++++++++++++++++++++++++++++ |
| A | internal/ | 231 | +++++++++++++++ |
| M | tools/ | 16 | +- |
| M | tools/ | 5 | +- |
| M | tools/ | 13 | +- |
| M | tools/ | 5 | + |
| A | tools/ | 456 | ++++++++++++++++++++++++++++++ |
@@ -0,0 +1,451 @@
1+// Package pbmap reads a captured Piggyback map and speaks MapGenie for it.2+//3+// Piggyback is the official guide house, and its maps carry what IGN's do not:4+// prose. Pins arrive with names and descriptions, and both survive into the5+// bundle. Like ignmap, this package translates at read time -- the archive6+// keeps the capture as Piggyback served it, and every keep-or-leave decision7+// lives here where changing it re-applies to captures already on disk.8+//9+// Piggyback draws its world in game coordinates on a Leaflet CRS.Simple map:10+// a linear transformation squeezes them onto the unit tile at zoom zero. The11+// capture records that transformation, and pins pass through it, then through12+// the shared inverse Mercator, to land on the same pixels Piggyback draws13+// them at.14+package pbmap15+16+import (17+ "encoding/json"18+ "fmt"19+ "sort"20+ "strconv"21+ "strings"22+23+ "github.com/FelineStateMachine/atlas/internal/mgdoc"24+)25+26+// Kind names Piggyback captures in a map's snapshot index.27+const Kind = "piggyback-map"28+29+// Source is the value a well-formed capture carries in its source field.30+const Source = "piggyback"31+32+// Capture is the canonical document the crawler stores: the map's settings,33+// the category tree, every pin, and the tile windows the crawler observed --34+// Piggyback publishes no bounds, so the crawler's survey is the only account35+// of where the pyramid is drawn. Fields marshal in declaration order and every36+// list is sorted by Normalize, so unchanged data always hashes to the capture37+// already archived.38+type Capture struct {39+ Source string `json:"source"`40+ GameSlug string `json:"gameSlug"`41+ MapSlug string `json:"mapSlug"`42+ GameTitle string `json:"gameTitle"`43+ MapTitle string `json:"mapTitle"`44+ Map MapConfig `json:"map"`45+ Labels Labels `json:"labels"`46+ Categories []Category `json:"categories"`47+ Pins []Pin `json:"pins"`48+ Levels []Level `json:"levels"`49+}50+51+// MapConfig keeps the settings the page ships and the transformation the52+// map's own scripts project through. MaxZoom is the deepest level actually53+// captured; the premium ceiling rides along as a matter of record.54+type MapConfig struct {55+ TileServer string `json:"tileServer"`56+ MinZoom int `json:"minZoom"`57+ MaxZoom int `json:"maxZoom"`58+ PremiumMaxZoom int `json:"premiumMaxZoom,omitempty"`59+ Transform Transform `json:"transform"`60+}61+62+// Transform is Leaflet's affine pair: a point (x, y) in game coordinates63+// draws at (a*x + b, c*y + d) on the unit tile at zoom zero.64+type Transform struct {65+ A float64 `json:"a"`66+ B float64 `json:"b"`67+ C float64 `json:"c"`68+ D float64 `json:"d"`69+}70+71+// Labels carries the display names the page's language bundle gives the72+// category and type keys, which the data API never repeats.73+type Labels struct {74+ Categories []Label `json:"categories"`75+ Types []Label `json:"types"`76+}77+78+type Label struct {79+ Key string `json:"key"`80+ Label string `json:"label"`81+}82+83+type Category struct {84+ ID string `json:"id"`85+ Key string `json:"key"`86+ Position float64 `json:"position"`87+ Types []Type `json:"types"`88+}89+90+type Type struct {91+ ID string `json:"id"`92+ Key string `json:"key"`93+ Position float64 `json:"position"`94+}95+96+// Pin keeps a pin the way Piggyback serves it, coordinates included as the97+// strings they arrive in.98+type Pin struct {99+ ID string `json:"id"`100+ X string `json:"x"`101+ Y string `json:"y"`102+ CategoryKey string `json:"categoryKey"`103+ TypeKey string `json:"typeKey"`104+ Name string `json:"name"`105+ Description string `json:"description,omitempty"`106+}107+108+// Level is one observed tile window: at Zoom, tiles exist from MinX,MinY109+// through MaxX,MaxY inclusive.110+type Level struct {111+ Zoom int `json:"zoom"`112+ MinX int `json:"minX"`113+ MinY int `json:"minY"`114+ MaxX int `json:"maxX"`115+ MaxY int `json:"maxY"`116+}117+118+// Normalize puts the capture into its canonical order.119+func (c *Capture) Normalize() {120+ sort.Slice(c.Labels.Categories, func(a, b int) bool {121+ return c.Labels.Categories[a].Key < c.Labels.Categories[b].Key122+ })123+ sort.Slice(c.Labels.Types, func(a, b int) bool {124+ return c.Labels.Types[a].Key < c.Labels.Types[b].Key125+ })126+ sort.SliceStable(c.Categories, func(a, b int) bool {127+ if c.Categories[a].Position != c.Categories[b].Position {128+ return c.Categories[a].Position < c.Categories[b].Position129+ }130+ return c.Categories[a].Key < c.Categories[b].Key131+ })132+ for index := range c.Categories {133+ types := c.Categories[index].Types134+ sort.SliceStable(types, func(a, b int) bool {135+ if types[a].Position != types[b].Position {136+ return types[a].Position < types[b].Position137+ }138+ return types[a].Key < types[b].Key139+ })140+ }141+ sort.Slice(c.Pins, func(a, b int) bool { return c.Pins[a].ID < c.Pins[b].ID })142+ sort.Slice(c.Levels, func(a, b int) bool { return c.Levels[a].Zoom < c.Levels[b].Zoom })143+}144+145+// districtTypes are Piggyback's region name markers. They arrive filed under146+// the user-state "favorites" category, which is nothing to build a legend147+// from; they become floating text labels, which is what they are on148+// Piggyback's own map.149+var districtTypes = map[string]bool{150+ "province": true,151+ "sub-region": true,152+}153+154+// MaybeTranslate hands other sources' snapshots through untouched and155+// translates Piggyback captures.156+func MaybeTranslate(kind string, doc []byte) ([]byte, error) {157+ if kind != Kind {158+ return doc, nil159+ }160+ return Translate(doc)161+}162+163+// Translate turns a capture into the MapGenie-shaped document the pipeline164+// reads, deterministically for a given capture.165+func Translate(doc []byte) ([]byte, error) {166+ var capture Capture167+ if err := json.Unmarshal(doc, &capture); err != nil {168+ return nil, fmt.Errorf("decode piggyback capture: %w", err)169+ }170+ if capture.Source != Source {171+ return nil, fmt.Errorf("capture source is %q, not %q", capture.Source, Source)172+ }173+ if capture.GameSlug == "" || capture.MapSlug == "" {174+ return nil, fmt.Errorf("capture names no map")175+ }176+ if len(capture.Levels) == 0 {177+ return nil, fmt.Errorf("capture observed no tile levels")178+ }179+ if capture.Map.Transform.A == 0 || capture.Map.Transform.C == 0 {180+ return nil, fmt.Errorf("capture carries no usable transformation")181+ }182+ capture.Normalize()183+184+ ids := mgdoc.NewIDSpace()185+ scope := capture.GameSlug + "/" + capture.MapSlug186+ mapID, err := ids.Claim("pb:map:" + scope)187+ if err != nil {188+ return nil, err189+ }190+ gameID, err := ids.Claim("pb:game:" + capture.GameSlug)191+ if err != nil {192+ return nil, err193+ }194+195+ set, err := tileSet(&capture)196+ if err != nil {197+ return nil, err198+ }199+ groups, err := buildGroups(&capture, ids, scope)200+ if err != nil {201+ return nil, err202+ }203+204+ centerX, centerY := deepestWindowCenter(&capture)205+ out := mgdoc.Map{206+ ID: mapID,207+ Title: title(capture.MapTitle, capture.MapSlug),208+ Slug: capture.MapSlug,209+ InitialLatitude: mgdoc.SyntheticLatitude(centerY),210+ InitialLongitude: mgdoc.SyntheticLongitude(centerX),211+ Config: mgdoc.Config{TileSets: []mgdoc.TileSet{set}},212+ Game: mgdoc.Game{213+ ID: gameID,214+ // The plain slug, deliberately shared with every other source's215+ // capture of the same game: their bundles answer for one library216+ // entry, and the newest capture is the one the reader sees.217+ Title: title(capture.GameTitle, capture.GameSlug),218+ Slug: capture.GameSlug,219+ },220+ Groups: groups,221+ Regions: []struct{}{},222+ }223+ return json.Marshal(out)224+}225+226+func title(given, slug string) string {227+ if given != "" {228+ return given229+ }230+ return mgdoc.SpellOut(slug)231+}232+233+// TileSetPath cuts the layer path out of the tile URL template: everything234+// between "/tiles/" and the zoom placeholder, which is how tile records group235+// into layers downstream.236+func TileSetPath(template string) (string, error) {237+ _, rest, found := strings.Cut(template, "/tiles/")238+ if !found {239+ return "", fmt.Errorf("tile template %q has no /tiles/ segment", template)240+ }241+ path, _, found := strings.Cut(rest, "/{z}")242+ if !found || path == "" {243+ return "", fmt.Errorf("tile template %q does not name a layer before its zoom", template)244+ }245+ return path, nil246+}247+248+// TileExtension reads the image format off the tile URL template.249+func TileExtension(template string) string {250+ if dot := strings.LastIndex(template, "."); dot >= 0 && dot < len(template)-1 {251+ return strings.ToLower(template[dot+1:])252+ }253+ return "webp"254+}255+256+// tileSet declares the pyramid from the crawler's survey. Bounds are declared257+// for every observed level: a level without them would be measured against258+// the MapGenie window this map does not sit in.259+func tileSet(capture *Capture) (mgdoc.TileSet, error) {260+ path, err := TileSetPath(capture.Map.TileServer)261+ if err != nil {262+ return mgdoc.TileSet{}, err263+ }264+ deepest := capture.Levels[len(capture.Levels)-1]265+ set := mgdoc.TileSet{266+ Name: "Default",267+ Path: path,268+ MinZoom: capture.Levels[0].Zoom,269+ MaxZoom: deepest.Zoom,270+ Extension: TileExtension(capture.Map.TileServer),271+ Bounds: make(map[string]mgdoc.Bound, len(capture.Levels)),272+ }273+ for _, level := range capture.Levels {274+ set.Bounds[strconv.Itoa(level.Zoom)] = mgdoc.Bound{275+ X: mgdoc.Range{Min: level.MinX, Max: level.MaxX},276+ Y: mgdoc.Range{Min: level.MinY, Max: level.MaxY},277+ }278+ }279+ return set, nil280+}281+282+// worldPixel runs a pin's game coordinates through the map's transformation283+// onto the zoom-zero tile, then scales to the shared 8192-pixel world.284+func worldPixel(transform Transform, x, y float64) (float64, float64) {285+ const zoomZeroSpan = 256286+ scale := float64(mgdoc.WorldSize) / zoomZeroSpan287+ return (transform.A*x + transform.B) * scale, (transform.C*y + transform.D) * scale288+}289+290+func deepestWindowCenter(capture *Capture) (float64, float64) {291+ deepest := capture.Levels[len(capture.Levels)-1]292+ span := float64(mgdoc.WorldSize) / float64(int(1)<<deepest.Zoom)293+ x := (float64(deepest.MinX) + float64(deepest.MaxX+1)) / 2 * span294+ y := (float64(deepest.MinY) + float64(deepest.MaxY+1)) / 2 * span295+ return x, y296+}297+298+// buildGroups turns Piggyback's category tree into the pipeline's groups. A299+// declared category becomes a group, its types become categories, and a type300+// no pin uses is left out. District name pins -- filed under the user-state301+// favorites category -- gather instead into their own group as floating text302+// labels. Anything else a pin references without a declaration is an error:303+// better a loud crawl than a pin silently dropped.304+func buildGroups(capture *Capture, ids *mgdoc.IDSpace, scope string) ([]mgdoc.Group, error) {305+ categoryLabels := labelIndex(capture.Labels.Categories)306+ typeLabels := labelIndex(capture.Labels.Types)307+308+ pinsByType := make(map[string][]Pin)309+ for _, pin := range capture.Pins {310+ pinsByType[pin.TypeKey] = append(pinsByType[pin.TypeKey], pin)311+ }312+313+ declared := make(map[string]bool)314+ var groups []mgdoc.Group315+ for _, category := range capture.Categories {316+ if category.Key == "favorites" {317+ // The reader's own favorites, not the map's data. Its stowaway318+ // district pins are picked up below.319+ continue320+ }321+ var categories []mgdoc.Category322+ for _, t := range category.Types {323+ declared[t.Key] = true324+ built, ok, err := buildCategory(capture, ids, scope, category.Key, t.Key,325+ label(typeLabels, t.Key), "markers", pinsByType)326+ if err != nil {327+ return nil, err328+ }329+ if ok {330+ categories = append(categories, built)331+ }332+ }333+ if len(categories) == 0 {334+ continue335+ }336+ groupID, err := ids.Claim("pb:category:" + scope + ":" + category.Key)337+ if err != nil {338+ return nil, err339+ }340+ groups = append(groups, mgdoc.Group{341+ ID: groupID,342+ Title: label(categoryLabels, category.Key),343+ Categories: categories,344+ })345+ }346+347+ // District names, and a loud failure for anything else undeclared.348+ var districts []mgdoc.Category349+ for _, key := range sortedKeys(pinsByType) {350+ if declared[key] {351+ continue352+ }353+ if !districtTypes[key] {354+ return nil, fmt.Errorf("pins use type %q, which no category declares", key)355+ }356+ built, ok, err := buildCategory(capture, ids, scope, "districts", key,357+ label(typeLabels, key), "text", pinsByType)358+ if err != nil {359+ return nil, err360+ }361+ if ok {362+ districts = append(districts, built)363+ }364+ }365+ if len(districts) > 0 {366+ groupID, err := ids.Claim("pb:category:" + scope + ":districts")367+ if err != nil {368+ return nil, err369+ }370+ groups = append(groups, mgdoc.Group{ID: groupID, Title: "Districts", Categories: districts})371+ }372+ return groups, nil373+}374+375+func buildCategory(376+ capture *Capture,377+ ids *mgdoc.IDSpace,378+ scope, categoryKey, typeKey, typeLabel, displayType string,379+ pinsByType map[string][]Pin,380+) (mgdoc.Category, bool, error) {381+ pins := pinsByType[typeKey]382+ if len(pins) == 0 {383+ return mgdoc.Category{}, false, nil384+ }385+ categoryID, err := ids.Claim("pb:type:" + scope + ":" + categoryKey + ":" + typeKey)386+ if err != nil {387+ return mgdoc.Category{}, false, err388+ }389+ category := mgdoc.Category{390+ ID: categoryID,391+ Title: typeLabel,392+ // The type key, so icons dropped into the archive later attach on the393+ // next generation without a policy change; until then the viewer's394+ // fallback glyph stands in.395+ Icon: typeKey,396+ DisplayType: displayType,397+ Visible: true,398+ Locations: make([]mgdoc.Location, 0, len(pins)),399+ }400+ for _, pin := range pins {401+ locationID, err := ids.Claim("pb:pin:" + pin.ID)402+ if err != nil {403+ return mgdoc.Category{}, false, err404+ }405+ x, err := strconv.ParseFloat(pin.X, 64)406+ if err != nil {407+ return mgdoc.Category{}, false, fmt.Errorf("pin %s x %q: %w", pin.ID, pin.X, err)408+ }409+ y, err := strconv.ParseFloat(pin.Y, 64)410+ if err != nil {411+ return mgdoc.Category{}, false, fmt.Errorf("pin %s y %q: %w", pin.ID, pin.Y, err)412+ }413+ pixelX, pixelY := worldPixel(capture.Map.Transform, x, y)414+ name := pin.Name415+ if name == "" {416+ name = typeLabel417+ }418+ category.Locations = append(category.Locations, mgdoc.Location{419+ ID: locationID,420+ Title: name,421+ Description: pin.Description,422+ Latitude: mgdoc.SyntheticLatitude(pixelY),423+ Longitude: mgdoc.SyntheticLongitude(pixelX),424+ })425+ }426+ return category, true, nil427+}428+429+func labelIndex(labels []Label) map[string]string {430+ out := make(map[string]string, len(labels))431+ for _, l := range labels {432+ out[l.Key] = l.Label433+ }434+ return out435+}436+437+func label(index map[string]string, key string) string {438+ if l, ok := index[key]; ok && l != "" {439+ return l440+ }441+ return mgdoc.SpellOut(key)442+}443+444+func sortedKeys(pins map[string][]Pin) []string {445+ keys := make([]string, 0, len(pins))446+ for key := range pins {447+ keys = append(keys, key)448+ }449+ sort.Strings(keys)450+ return keys451+}@@ -0,0 +1,231 @@
1+package pbmap2+3+import (4+ "bytes"5+ "encoding/json"6+ "math"7+ "strings"8+ "testing"9+10+ "github.com/FelineStateMachine/atlas/internal/mgdoc"11+)12+13+// fixture exercises everything Translate decides: a declared category with a14+// used and an unused type, district pins under the favorites category, prose15+// descriptions, and the night-city transformation.16+func fixture() Capture {17+ return Capture{18+ Source: Source,19+ GameSlug: "cyberpunk-2077",20+ MapSlug: "night-city",21+ GameTitle: "Cyberpunk 2077",22+ MapTitle: "Night City",23+ Map: MapConfig{24+ TileServer: "https://storage-cdn.piggyback.com/images/tiles/cbp/{z}/{x}/{y}.webp",25+ MinZoom: 3,26+ MaxZoom: 7,27+ PremiumMaxZoom: 9,28+ Transform: Transform{A: 0.015625, B: 128, C: -0.015625, D: 128},29+ },30+ Labels: Labels{31+ Categories: []Label{{Key: "gigs", Label: "Gigs"}},32+ Types: []Label{33+ {Key: "agent-saboteur", Label: "Agent Saboteur"},34+ {Key: "province", Label: "Province"},35+ },36+ },37+ Categories: []Category{38+ {ID: "cat-fav", Key: "favorites", Position: 0},39+ {ID: "cat-gigs", Key: "gigs", Position: 30, Types: []Type{40+ {ID: "t-as", Key: "agent-saboteur", Position: 10},41+ {ID: "t-empty", Key: "thievery", Position: 20},42+ }},43+ },44+ Pins: []Pin{45+ {ID: "pin-b", X: "-1992.51", Y: "4097.82", CategoryKey: "gigs", TypeKey: "agent-saboteur",46+ Name: "WELCOME TO AMERICA, COMRADE", Description: "Unlock Condition – Street Cred Tier 1"},47+ {ID: "pin-a", X: "0", Y: "0", CategoryKey: "gigs", TypeKey: "agent-saboteur", Name: "Centre Gig"},48+ {ID: "pin-d", X: "-4000", Y: "5000", CategoryKey: "favorites", TypeKey: "province", Name: "Watson"},49+ },50+ Levels: []Level{51+ {Zoom: 3, MinX: 0, MinY: 0, MaxX: 6, MaxY: 6},52+ {Zoom: 7, MinX: 0, MinY: 0, MaxX: 111, MaxY: 111},53+ },54+ }55+}56+57+func translate(t *testing.T, capture Capture) mgdoc.Map {58+ t.Helper()59+ doc, err := json.Marshal(capture)60+ if err != nil {61+ t.Fatal(err)62+ }63+ out, err := Translate(doc)64+ if err != nil {65+ t.Fatal(err)66+ }67+ var m mgdoc.Map68+ if err := json.Unmarshal(out, &m); err != nil {69+ t.Fatal(err)70+ }71+ return m72+}73+74+func TestTranslateShape(t *testing.T) {75+ m := translate(t, fixture())76+77+ if m.Slug != "night-city" || m.Game.Slug != "cyberpunk-2077" {78+ t.Fatalf("identity: %q / %q", m.Game.Slug, m.Slug)79+ }80+ set := m.Config.TileSets[0]81+ if set.Path != "cbp" || set.Extension != "webp" {82+ t.Fatalf("tile set: %+v", set)83+ }84+ if set.MinZoom != 3 || set.MaxZoom != 7 {85+ t.Fatalf("zoom range: %d..%d", set.MinZoom, set.MaxZoom)86+ }87+ if bound := set.Bounds["7"]; bound.X.Max != 111 || bound.Y.Max != 111 {88+ t.Fatalf("deepest bounds: %+v", bound)89+ }90+91+ if len(m.Groups) != 2 {92+ t.Fatalf("groups: %+v", m.Groups)93+ }94+ gigs := m.Groups[0]95+ if gigs.Title != "Gigs" || len(gigs.Categories) != 1 {96+ t.Fatalf("gigs group: %+v", gigs)97+ }98+ category := gigs.Categories[0]99+ if category.Title != "Agent Saboteur" || category.Icon != "agent-saboteur" ||100+ category.DisplayType != "markers" {101+ t.Fatalf("gig category: %+v", category)102+ }103+ if len(category.Locations) != 2 {104+ t.Fatalf("gig pins: %+v", category.Locations)105+ }106+ // Pins sort by id, so pin-a comes first.107+ if category.Locations[0].Title != "Centre Gig" {108+ t.Fatalf("pin order: %+v", category.Locations)109+ }110+ if category.Locations[1].Description != "Unlock Condition – Street Cred Tier 1" {111+ t.Fatalf("description lost: %+v", category.Locations[1])112+ }113+114+ districts := m.Groups[1]115+ if districts.Title != "Districts" || len(districts.Categories) != 1 {116+ t.Fatalf("districts group: %+v", districts)117+ }118+ province := districts.Categories[0]119+ if province.Title != "Province" || province.DisplayType != "text" {120+ t.Fatalf("province category: %+v", province)121+ }122+ if province.Locations[0].Title != "Watson" {123+ t.Fatalf("district pin: %+v", province.Locations[0])124+ }125+126+ // The empty thievery type keeps no category.127+ for _, group := range m.Groups {128+ for _, category := range group.Categories {129+ if category.Icon == "thievery" {130+ t.Fatal("a pinless type kept its category")131+ }132+ }133+ }134+}135+136+// TestCoordinatesRoundTrip runs a pin's synthetic coordinates through the137+// viewer's projection and expects the pixel Piggyback draws it at: the game138+// coordinate through the map's own transformation, scaled to the world.139+func TestCoordinatesRoundTrip(t *testing.T) {140+ m := translate(t, fixture())141+142+ project := func(latitude, longitude float64) (float64, float64) {143+ worldTiles := math.Pow(2, mgdoc.SourceZoom)144+ x := ((longitude + 180) / 360) * worldTiles * mgdoc.TileSize145+ y := (1 - math.Asinh(math.Tan(latitude*math.Pi/180))/math.Pi) / 2 * worldTiles * mgdoc.TileSize146+ return x, y147+ }148+ expected := map[string][2]float64{149+ "Centre Gig": {4096, 4096},150+ "WELCOME TO AMERICA, COMRADE": {-1992.51/2 + 4096, -4097.82/2 + 4096},151+ "Watson": {-4000.0/2 + 4096, -5000.0/2 + 4096},152+ }153+ seen := 0154+ for _, group := range m.Groups {155+ for _, category := range group.Categories {156+ for _, location := range category.Locations {157+ want, ok := expected[location.Title]158+ if !ok {159+ continue160+ }161+ seen++162+ x, y := project(location.Latitude, location.Longitude)163+ if math.Abs(x-want[0]) > 1e-6 || math.Abs(y-want[1]) > 1e-6 {164+ t.Fatalf("%s projects to %.6f,%.6f, want %.3f,%.3f",165+ location.Title, x, y, want[0], want[1])166+ }167+ }168+ }169+ }170+ if seen != len(expected) {171+ t.Fatalf("saw %d of %d pins", seen, len(expected))172+ }173+}174+175+func TestTranslateIsDeterministic(t *testing.T) {176+ doc, _ := json.Marshal(fixture())177+ first, err := Translate(doc)178+ if err != nil {179+ t.Fatal(err)180+ }181+ shuffled := fixture()182+ shuffled.Pins[0], shuffled.Pins[2] = shuffled.Pins[2], shuffled.Pins[0]183+ shuffled.Categories[0], shuffled.Categories[1] = shuffled.Categories[1], shuffled.Categories[0]184+ reordered, _ := json.Marshal(shuffled)185+ second, err := Translate(reordered)186+ if err != nil {187+ t.Fatal(err)188+ }189+ if !bytes.Equal(first, second) {190+ t.Fatal("the same capture translated to different bytes")191+ }192+}193+194+func TestTranslateRejectsBadCaptures(t *testing.T) {195+ undeclared := fixture()196+ undeclared.Pins[0].TypeKey = "never-declared"197+ if _, err := translateErr(undeclared); err == nil || !strings.Contains(err.Error(), "no category declares") {198+ t.Fatalf("undeclared type: %v", err)199+ }200+201+ unsurveyed := fixture()202+ unsurveyed.Levels = nil203+ if _, err := translateErr(unsurveyed); err == nil {204+ t.Fatal("a capture with no observed levels was accepted")205+ }206+207+ flat := fixture()208+ flat.Map.Transform = Transform{}209+ if _, err := translateErr(flat); err == nil {210+ t.Fatal("a capture with no transformation was accepted")211+ }212+}213+214+func translateErr(capture Capture) ([]byte, error) {215+ doc, err := json.Marshal(capture)216+ if err != nil {217+ return nil, err218+ }219+ return Translate(doc)220+}221+222+func TestMaybeTranslatePassesOthersThrough(t *testing.T) {223+ doc := []byte(`{"id": 5}`)224+ out, err := MaybeTranslate("ign-map", doc)225+ if err != nil {226+ t.Fatal(err)227+ }228+ if !bytes.Equal(out, doc) {229+ t.Fatal("another source's snapshot was rewritten")230+ }231+}@@ -82,8 +82,22 @@ func (f *fetcher) penalise(d time.Duration) {
8282 const maxAttempts = 48383 8484 func (f *fetcher) get(ctx context.Context, url string) ([]byte, string, error) {85+ return f.getReferred(ctx, url, "")86+}87+88+// getReferred names the page the request stands in for. Piggyback's tile CDN89+// serves only requests that arrive as its own map's, and answers everything90+// else 403 -- which get would read as "not published".91+func (f *fetcher) getReferred(ctx context.Context, url, referer string) ([]byte, string, error) {8592 return f.do(ctx, url, func() (*http.Request, error) {86- return http.NewRequestWithContext(ctx, http.MethodGet, url, nil)93+ request, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)94+ if err != nil {95+ return nil, err96+ }97+ if referer != "" {98+ request.Header.Set("Referer", referer)99+ }100+ return request, nil87101 })88102 }89103 @@ -116,9 +116,10 @@ func readTileIndex(mapDir string) (*tileIndex, error) {
116116 117117 // tileSetPathOf recovers the layer path from a tile URL, matching how118118 // tools/tiles groups levels: everything between the source's marker segment119-// and "/<zoom>/". MapGenie serves tiles under /games/, IGN under /wikimaps/.119+// and "/<zoom>/". MapGenie serves tiles under /games/, IGN under /wikimaps/,120+// Piggyback under /tiles/.120121 func tileSetPathOf(url string, zoom int) string {121- for _, marker := range []string{"/games/", "/wikimaps/"} {122+ for _, marker := range []string{"/games/", "/wikimaps/", "/tiles/"} {122123 _, rest, found := strings.Cut(url, marker)123124 if !found {124125 continue@@ -351,7 +351,7 @@ func captureIGNTiles(
351351 for zoom := 0; zoom <= deepest; zoom++ {352352 maxX, maxY := ignmap.LevelExtent(capture.Map.Width, capture.Map.Height, zoom)353353 window := tileWindow{minX: 0, minY: 0, maxX: maxX, maxY: maxY}354- results, err := fetchIGNLevel(ctx, fetcher, o, template, setID, extension,354+ results, err := fetchTemplateLevel(ctx, fetcher, o, template, "", setID, extension,355355 mapDir, zoom, window.tiles(), index)356356 if err != nil {357357 return stats, err@@ -370,11 +370,15 @@ func captureIGNTiles(
370370 return stats, nil371371 }372372 373-func fetchIGNLevel(373+// fetchTemplateLevel takes one level of a pyramid whose URLs come from a374+// {z}/{x}/{y} template, in whatever punctuation the source spells the tokens.375+// The hashes of held tiles are recorded alongside fetched ones, so a resumed376+// crawl sees the same picture of the level a fresh one would.377+func fetchTemplateLevel(374378 ctx context.Context,375379 fetcher *fetcher,376380 o options,377- template string,381+ template, referer string,378382 setID int64,379383 extension, mapDir string,380384 zoom int,@@ -411,6 +415,7 @@ func fetchIGNLevel(
411415 if record.Status == "cached" {412416 if _, err := os.Stat(path); err == nil {413417 result.skipped++418+ result.hashes[coordinate] = record.ContentHash414419 continue415420 }416421 }@@ -426,7 +431,7 @@ func fetchIGNLevel(
426431 defer group.Done()427432 defer func() { <-gate }()428433 429- body, contentType, err := fetcher.get(ctx, url)434+ body, contentType, err := fetcher.getReferred(ctx, url, referer)430435 mu.Lock()431436 defer mu.Unlock()432437 if err != nil {@@ -62,6 +62,7 @@ type options struct {
6262 game string6363 mapTarget string6464 ign string65+ piggyback string6566 maxZoom int6667 list bool6768 dryRun bool@@ -75,6 +76,7 @@ func parseOptions() options {
7576 flag.StringVar(&o.game, "game", "", "game slug or id")7677 flag.StringVar(&o.mapTarget, "map", "", "map slug or id (default: every map of the game)")7778 flag.StringVar(&o.ign, "ign", "", "IGN wikimap as objectSlug/mapSlug, e.g. cyberpunk-2077/night-city")79+ flag.StringVar(&o.piggyback, "piggyback", "", "Piggyback map as gameSlug/mapSlug, e.g. cyberpunk-2077/night-city")7880 flag.IntVar(&o.maxZoom, "max-zoom", 0, "deepest zoom to capture (default: the layer's own maximum)")7981 flag.BoolVar(&o.list, "list", false, "list games, or the maps of -game, and exit")8082 flag.BoolVar(&o.dryRun, "dry-run", false, "report what would be fetched without writing anything")@@ -94,6 +96,9 @@ func run(o options) error {
9496 if o.ign != "" {9597 return runIGN(ctx, fetcher, o)9698 }99+ if o.piggyback != "" {100+ return runPB(ctx, fetcher, o)101+ }97102 98103 games, err := listGames(ctx, fetcher)99104 if err != nil {@@ -0,0 +1,456 @@
1+// The Piggyback mode fills the archive from maps.piggyback.com. The page2+// carries its settings and its display names; the categories and pins come3+// from two open API requests under the map's own address; the tiles are4+// surveyed level by level, because Piggyback publishes no bounds -- the5+// crawler's own account of where tiles exist is what the capture carries.6+package main7+8+import (9+ "context"10+ "encoding/json"11+ "errors"12+ "fmt"13+ "hash/fnv"14+ "path/filepath"15+ "strconv"16+ "strings"17+18+ "github.com/FelineStateMachine/atlas/internal/mgdoc"19+ "github.com/FelineStateMachine/atlas/internal/pbmap"20+)21+22+// pbPage is what the crawl needs from the page's embedded state: the tile23+// server and zoom range, and the language bundle's names for category and24+// type keys, which the data API never repeats.25+type pbPage struct {26+ tileServer string27+ minZoom int28+ maxZoom int29+ premiumMaxZoom int30+ categoryLabels map[string]string31+ typeLabels map[string]string32+}33+34+// pbTransform is the transformation Piggyback's Cyberpunk map projects35+// through: game coordinates spanning ±8192 onto the unit tile at zoom zero.36+// It lives in the map's scripts rather than its data, so the crawler asserts37+// it instead of reading it; the capture records what was assumed, and a game38+// drawn to a different rule would need this looked up in its own scripts.39+var pbTransform = pbmap.Transform{A: 0.015625, B: 128, C: -0.015625, D: 128}40+41+func runPB(ctx context.Context, fetcher *fetcher, o options) error {42+ gameSlug, mapSlug, ok := strings.Cut(o.piggyback, "/")43+ if !ok || gameSlug == "" || mapSlug == "" {44+ return errors.New("-piggyback wants gameSlug/mapSlug, as the address bar spells it")45+ }46+ pageURL := fmt.Sprintf("https://maps.piggyback.com/%s/maps/%s", gameSlug, mapSlug)47+48+ page, err := fetchPBPage(ctx, fetcher, pageURL)49+ if err != nil {50+ return err51+ }52+ categories, err := fetchPBCategories(ctx, fetcher, pageURL)53+ if err != nil {54+ return err55+ }56+ pins, err := fetchPBPins(ctx, fetcher, pageURL)57+ if err != nil {58+ return err59+ }60+61+ capture := composePBCapture(gameSlug, mapSlug, page, categories, pins)62+ scope, err := pbmap.TileSetPath(page.tileServer)63+ if err != nil {64+ return err65+ }66+67+ gameID := pbArchiveID("pb:" + gameSlug)68+ mapID := pbArchiveID("pb:" + gameSlug + "/" + mapSlug)69+ game := &apiGameFull{ID: gameID, Title: capture.GameTitle,70+ Config: apiGameConfig{URL: pageURL}}71+ full := &apiMapFull{ID: mapID, Title: capture.MapTitle, Slug: mapSlug}72+ // The directory says where the capture came from even though the game73+ // keeps its plain title: other sources' captures of the same game live in74+ // their own directories, and they must never resolve to one.75+ gameDirectory := resolveGameDirectory(o.archive,76+ &apiGameFull{ID: gameID, Title: "Piggyback " + capture.GameTitle})77+ mapDirectory := resolveMapDirectory(o.archive, gameDirectory, full)78+ mapDir := filepath.Join(o.archive, mapDirectory)79+80+ fmt.Printf("\n== Piggyback %s / %s (%d pins, %d categories)\n",81+ capture.GameTitle, capture.MapTitle, len(capture.Pins), len(capture.Categories))82+83+ if o.dryRun {84+ fmt.Printf(" would survey tiles z%d-%d and write the capture to %s\n",85+ page.minZoom, pbDeepest(page, o), mapDir)86+ return nil87+ }88+89+ index, err := readTileIndex(mapDir)90+ if err != nil {91+ return err92+ }93+ index.mapID = mapID94+ stats, levels, err := capturePBTiles(ctx, fetcher, o, page, pageURL, scope, mapDir, index)95+ if err != nil {96+ return err97+ }98+ capture.Levels = levels99+100+ raw, err := json.MarshalIndent(capture, "", " ")101+ if err != nil {102+ return err103+ }104+ // A capture that does not translate would poison the archive for both105+ // tools that read it, so it is refused here, while the problem is one106+ // crawl old rather than buried in history.107+ if _, err := pbmap.Translate(raw); err != nil {108+ return fmt.Errorf("capture does not translate: %w", err)109+ }110+ if err := writeSnapshotRecord(mapDir, raw, pbmap.Kind, mapID, pageURL); err != nil {111+ return err112+ }113+ if err := writeTileIndex(mapDir, index); err != nil {114+ return err115+ }116+ if err := writeMapMetadata(mapDir, game, full, index, len(capture.Pins)); err != nil {117+ return err118+ }119+ if err := registerMap(o.archive, game, full, gameDirectory, mapDirectory, "piggyback"); err != nil {120+ return err121+ }122+ fmt.Printf(" %d fetched · %d held · %d not published · %d failed · %.1f MB new\n",123+ stats.fetched, stats.skipped, stats.absent, stats.failed, float64(stats.bytes)/1e6)124+ return nil125+}126+127+func fetchPBPage(ctx context.Context, f *fetcher, pageURL string) (*pbPage, error) {128+ body, _, err := f.get(ctx, pageURL)129+ if err != nil {130+ return nil, err131+ }132+ const openTag = `<script id="__NEXT_DATA__" type="application/json">`133+ _, rest, found := strings.Cut(string(body), openTag)134+ if !found {135+ return nil, fmt.Errorf("%s embeds no page state", pageURL)136+ }137+ doc, _, found := strings.Cut(rest, "</script>")138+ if !found {139+ return nil, fmt.Errorf("%s embeds unterminated page state", pageURL)140+ }141+ var state struct {142+ Props struct {143+ PageProps struct {144+ Settings struct {145+ TileServerURL string `json:"tileServerUrl"`146+ MinZoom string `json:"minZoom"`147+ MaxZoom string `json:"maxZoom"`148+ MaxZoomPremium string `json:"maxZoomPremium"`149+ } `json:"settings"`150+ I18N struct {151+ Store map[string]struct {152+ Common struct {153+ Category map[string]string `json:"category"`154+ Type map[string]string `json:"type"`155+ } `json:"common"`156+ } `json:"initialI18nStore"`157+ } `json:"_nextI18Next"`158+ } `json:"pageProps"`159+ } `json:"props"`160+ }161+ if err := json.Unmarshal([]byte(doc), &state); err != nil {162+ return nil, fmt.Errorf("decode %s page state: %w", pageURL, err)163+ }164+ settings := state.Props.PageProps.Settings165+ if settings.TileServerURL == "" {166+ return nil, fmt.Errorf("%s describes no map", pageURL)167+ }168+ page := &pbPage{169+ tileServer: settings.TileServerURL,170+ minZoom: pbAtoi(settings.MinZoom, 0),171+ maxZoom: pbAtoi(settings.MaxZoom, 0),172+ premiumMaxZoom: pbAtoi(settings.MaxZoomPremium, 0),173+ categoryLabels: map[string]string{},174+ typeLabels: map[string]string{},175+ }176+ if page.maxZoom <= 0 {177+ return nil, fmt.Errorf("%s declares no zoom range", pageURL)178+ }179+ // English labels when the page carries them, any language over none.180+ store := state.Props.PageProps.I18N.Store181+ languages := []string{"en"}182+ for language := range store {183+ languages = append(languages, language)184+ }185+ for _, language := range languages {186+ bundle, ok := store[language]187+ if !ok {188+ continue189+ }190+ for key, label := range bundle.Common.Category {191+ if _, taken := page.categoryLabels[key]; !taken {192+ page.categoryLabels[key] = label193+ }194+ }195+ for key, label := range bundle.Common.Type {196+ if _, taken := page.typeLabels[key]; !taken {197+ page.typeLabels[key] = label198+ }199+ }200+ }201+ return page, nil202+}203+204+// The two data requests every visitor's map makes, under the map's own path.205+type pbAPICategory struct {206+ ID string `json:"id"`207+ Key string `json:"key"`208+ Position float64 `json:"position"`209+ Types []struct {210+ ID string `json:"id"`211+ Key string `json:"key"`212+ Position float64 `json:"position"`213+ } `json:"types"`214+}215+216+type pbAPIPin struct {217+ ID string `json:"id"`218+ X string `json:"x"`219+ Y string `json:"y"`220+ Category struct {221+ Key string `json:"key"`222+ } `json:"category"`223+ Type struct {224+ Key string `json:"key"`225+ } `json:"type"`226+ Data []struct {227+ Name string `json:"name"`228+ Description string `json:"description"`229+ } `json:"data"`230+}231+232+func fetchPBCategories(ctx context.Context, f *fetcher, pageURL string) ([]pbAPICategory, error) {233+ var out []pbAPICategory234+ if err := fetchPBResult(ctx, f, pageURL+"/api/trpc/category.getCategory", &out); err != nil {235+ return nil, err236+ }237+ if len(out) == 0 {238+ return nil, errors.New("the map declares no categories")239+ }240+ return out, nil241+}242+243+func fetchPBPins(ctx context.Context, f *fetcher, pageURL string) ([]pbAPIPin, error) {244+ var out []pbAPIPin245+ if err := fetchPBResult(ctx, f, pageURL+"/api/trpc/pin.getPins", &out); err != nil {246+ return nil, err247+ }248+ if len(out) == 0 {249+ return nil, errors.New("the map answered with no pins at all")250+ }251+ return out, nil252+}253+254+func fetchPBResult(ctx context.Context, f *fetcher, url string, into any) error {255+ body, _, err := f.get(ctx, url)256+ if err != nil {257+ return err258+ }259+ var envelope struct {260+ Result struct {261+ Data struct {262+ JSON json.RawMessage `json:"json"`263+ } `json:"data"`264+ } `json:"result"`265+ Error json.RawMessage `json:"error"`266+ }267+ if err := json.Unmarshal(body, &envelope); err != nil {268+ return fmt.Errorf("decode %s: %w", url, err)269+ }270+ if len(envelope.Error) > 0 {271+ return fmt.Errorf("%s answered: %s", url, envelope.Error)272+ }273+ if err := json.Unmarshal(envelope.Result.Data.JSON, into); err != nil {274+ return fmt.Errorf("decode %s payload: %w", url, err)275+ }276+ return nil277+}278+279+func composePBCapture(280+ gameSlug, mapSlug string,281+ page *pbPage,282+ categories []pbAPICategory,283+ pins []pbAPIPin,284+) *pbmap.Capture {285+ capture := &pbmap.Capture{286+ Source: pbmap.Source,287+ GameSlug: gameSlug,288+ MapSlug: mapSlug,289+ // The page's language bundle names categories and types, never the290+ // game or the map themselves; their slugs, spelled out, are all the291+ // naming there is.292+ GameTitle: mgdoc.SpellOut(gameSlug),293+ MapTitle: mgdoc.SpellOut(mapSlug),294+ }295+ capture.Map = pbmap.MapConfig{296+ TileServer: page.tileServer,297+ MinZoom: page.minZoom,298+ MaxZoom: page.maxZoom,299+ PremiumMaxZoom: page.premiumMaxZoom,300+ Transform: pbTransform,301+ }302+ for _, category := range categories {303+ entry := pbmap.Category{ID: category.ID, Key: category.Key, Position: category.Position}304+ for _, t := range category.Types {305+ entry.Types = append(entry.Types, pbmap.Type{ID: t.ID, Key: t.Key, Position: t.Position})306+ }307+ capture.Categories = append(capture.Categories, entry)308+ }309+ for _, pin := range pins {310+ entry := pbmap.Pin{311+ ID: pin.ID,312+ X: pin.X,313+ Y: pin.Y,314+ CategoryKey: pin.Category.Key,315+ TypeKey: pin.Type.Key,316+ }317+ // One entry is the rule; a pin stacking several keeps them all, the318+ // extras folded into the description rather than dropped.319+ if len(pin.Data) > 0 {320+ entry.Name = pin.Data[0].Name321+ entry.Description = pin.Data[0].Description322+ for _, extra := range pin.Data[1:] {323+ addition := strings.TrimSpace(extra.Name + " — " + extra.Description)324+ entry.Description = strings.TrimSpace(entry.Description + "\n\n" + addition)325+ }326+ }327+ capture.Pins = append(capture.Pins, entry)328+ }329+ // Labels for every key the data actually uses, from the page's bundle.330+ seenCategories := map[string]bool{}331+ seenTypes := map[string]bool{}332+ for _, category := range capture.Categories {333+ seenCategories[category.Key] = true334+ for _, t := range category.Types {335+ seenTypes[t.Key] = true336+ }337+ }338+ for _, pin := range capture.Pins {339+ seenCategories[pin.CategoryKey] = true340+ seenTypes[pin.TypeKey] = true341+ }342+ for key := range seenCategories {343+ if label := page.categoryLabels[key]; label != "" {344+ capture.Labels.Categories = append(capture.Labels.Categories, pbmap.Label{Key: key, Label: label})345+ }346+ }347+ for key := range seenTypes {348+ if label := page.typeLabels[key]; label != "" {349+ capture.Labels.Types = append(capture.Labels.Types, pbmap.Label{Key: key, Label: label})350+ }351+ }352+ capture.Normalize()353+ return capture354+}355+356+func pbDeepest(page *pbPage, o options) int {357+ deepest := page.maxZoom358+ if o.maxZoom > 0 {359+ // An explicit ask may pass the free ceiling, up to what exists at all.360+ limit := page.premiumMaxZoom361+ if limit <= 0 {362+ limit = page.maxZoom363+ }364+ deepest = min(o.maxZoom, limit)365+ }366+ return deepest367+}368+369+// capturePBTiles surveys and captures the pyramid in one walk. The first370+// published level is asked for whole; every deeper level asks only after the371+// children of tiles that exist, which is what bounds the walk over a mostly372+// empty world square. What each level turns out to hold becomes the capture's373+// account of the pyramid.374+func capturePBTiles(375+ ctx context.Context,376+ fetcher *fetcher,377+ o options,378+ page *pbPage,379+ pageURL, scope, mapDir string,380+ index *tileIndex,381+) (captureStats, []pbmap.Level, error) {382+ var stats captureStats383+ var levels []pbmap.Level384+ setID := index.tileSetID(scope)385+ extension := pbmap.TileExtension(page.tileServer)386+ deepest := pbDeepest(page, o)387+ referer := pageURL + "/"388+ fmt.Printf(" layer %q z%d-%d\n", scope, page.minZoom, deepest)389+390+ var present map[[2]int]bool391+ for zoom := page.minZoom; zoom <= deepest; zoom++ {392+ across := 1 << zoom393+ window := tileWindow{minX: 0, minY: 0, maxX: across - 1, maxY: across - 1}394+ var wanted [][2]int395+ if present == nil {396+ wanted = window.tiles()397+ } else {398+ wanted = childrenOf(present, window)399+ }400+ if len(wanted) == 0 {401+ break402+ }403+ results, err := fetchTemplateLevel(ctx, fetcher, o, page.tileServer, referer,404+ setID, extension, mapDir, zoom, wanted, index)405+ if err != nil {406+ return stats, nil, err407+ }408+ stats.merge(results)409+410+ present = make(map[[2]int]bool, len(results.hashes))411+ level := pbmap.Level{Zoom: zoom, MinX: across, MinY: across, MaxX: -1, MaxY: -1}412+ for coordinate := range results.hashes {413+ present[coordinate] = true414+ level.MinX = min(level.MinX, coordinate[0])415+ level.MinY = min(level.MinY, coordinate[1])416+ level.MaxX = max(level.MaxX, coordinate[0])417+ level.MaxY = max(level.MaxY, coordinate[1])418+ }419+ fmt.Printf(" z%-2d %4d fetched · %4d held · %4d not published\n",420+ zoom, results.fetched, results.skipped, results.absent)421+ if level.MaxX < 0 {422+ fmt.Printf(" z%-2d nothing published; the survey stops here\n", zoom)423+ break424+ }425+ // The pipeline trusts a level to fill its declared window. Holes426+ // inside the window are survivable -- the level rides along partially427+ // with its coverage recorded -- but they deserve a loud word.428+ expected := (level.MaxX - level.MinX + 1) * (level.MaxY - level.MinY + 1)429+ if len(present) < expected {430+ fmt.Printf(" WARNING: z%d holds %d of the %d tiles in its window\n",431+ zoom, len(present), expected)432+ }433+ levels = append(levels, level)434+ }435+ if len(levels) == 0 {436+ return stats, nil, errors.New("no tiles at all; is the CDN refusing the survey?")437+ }438+ return stats, levels, nil439+}440+441+// pbArchiveID numbers Piggyback games and maps in the archive's registers,442+// one bit above IGN's so every source's ids stay apart; see ignArchiveID for443+// why the range stays far below a float64's integers.444+func pbArchiveID(seed string) int64 {445+ hash := fnv.New32a()446+ _, _ = hash.Write([]byte(seed))447+ return int64(hash.Sum32()&0x7fffffff) | 1<<33448+}449+450+func pbAtoi(value string, fallback int) int {451+ parsed, err := strconv.Atoi(strings.TrimSpace(value))452+ if err != nil {453+ return fallback454+ }455+ return parsed456+}