1// The Piggyback mode fills the archive from maps.piggyback.com. The page2// carries its settings and its display names; the categories and pins come3// from two open API requests under the map's own address; the tiles are4// surveyed level by level, because Piggyback publishes no bounds -- the5// crawler's own account of where tiles exist is what the capture carries.6package main78import (9 "bytes"10 "context"11 "encoding/json"12 "errors"13 "fmt"14 "hash/fnv"15 "image"16 "image/png"17 "os"18 "path/filepath"19 "sort"20 "strconv"21 "strings"2223 // Piggyback draws its icons only as WebP; they decode here and land in24 // the archive as the PNG the pipeline reads.25 _ "golang.org/x/image/webp"2627 "github.com/FelineStateMachine/atlas/internal/mgdoc"28 "github.com/FelineStateMachine/atlas/internal/pbmap"29)3031// pbPage is what the crawl needs from the page's embedded state: the tile32// server and zoom range, and the language bundle's names for category and33// type keys, which the data API never repeats.34type pbPage struct {35 tileServer string36 minZoom int37 maxZoom int38 premiumMaxZoom int39 categoryLabels map[string]string40 typeLabels map[string]string41}4243// This is the Cyberpunk importer, and says so. Piggyback publishes guides44// for many games, but each map's coordinate transformation lives in that45// map's own scripts rather than its data: the crawler asserts Cyberpunk's --46// game coordinates spanning ±8192 onto the unit tile at zoom zero -- and the47// capture records what was assumed. Another Piggyback game would need its48// transformation read out of its own scripts and verified; until someone49// does that work, an unknown game is refused at the door rather than50// captured through the wrong lens, where every pin would land somewhere51// plausible and nothing would say it was wrong.52const pbSupportedGame = "cyberpunk-2077"5354var pbTransform = pbmap.Transform{A: 0.015625, B: 128, C: -0.015625, D: 128}5556func runPB(ctx context.Context, fetcher *fetcher, o options) error {57 gameSlug, mapSlug, ok := strings.Cut(o.piggyback, "/")58 if !ok || gameSlug == "" || mapSlug == "" {59 return errors.New("-piggyback wants gameSlug/mapSlug, as the address bar spells it")60 }61 if gameSlug != pbSupportedGame {62 return fmt.Errorf("the Piggyback importer knows only %s: %s draws through a "+63 "transformation this crawler has not verified, and a wrong one moves every pin",64 pbSupportedGame, gameSlug)65 }66 pageURL := fmt.Sprintf("https://maps.piggyback.com/%s/maps/%s", gameSlug, mapSlug)6768 page, err := fetchPBPage(ctx, fetcher, pageURL)69 if err != nil {70 return err71 }72 categories, err := fetchPBCategories(ctx, fetcher, pageURL)73 if err != nil {74 return err75 }76 pins, err := fetchPBPins(ctx, fetcher, pageURL)77 if err != nil {78 return err79 }8081 capture := composePBCapture(gameSlug, mapSlug, page, categories, pins)82 scope, err := pbmap.TileSetPath(page.tileServer)83 if err != nil {84 return err85 }8687 gameID := pbArchiveID("pb:" + gameSlug)88 mapID := pbArchiveID("pb:" + gameSlug + "/" + mapSlug)89 game := &apiGameFull{ID: gameID, Title: capture.GameTitle,90 Config: apiGameConfig{URL: pageURL}}91 full := &apiMapFull{ID: mapID, Title: capture.MapTitle, Slug: mapSlug}92 // The directory says where the capture came from even though the game93 // keeps its plain title: other sources' captures of the same game live in94 // their own directories, and they must never resolve to one.95 gameDirectory := resolveGameDirectory(o.archive,96 &apiGameFull{ID: gameID, Title: "Piggyback " + capture.GameTitle})97 mapDirectory := resolveMapDirectory(o.archive, gameDirectory, full)98 mapDir := filepath.Join(o.archive, mapDirectory)99100 fmt.Printf("\n== Piggyback %s / %s (%d pins, %d categories)\n",101 capture.GameTitle, capture.MapTitle, len(capture.Pins), len(capture.Categories))102103 if o.dryRun {104 fmt.Printf(" would survey tiles z%d-%d and write the capture to %s\n",105 page.minZoom, pbDeepest(page, o), mapDir)106 return nil107 }108109 if err := fetchPBIcons(ctx, fetcher, gameSlug, pageURL,110 filepath.Join(o.archive, gameDirectory), capture); err != nil {111 return err112 }113114 index, err := readTileIndex(mapDir)115 if err != nil {116 return err117 }118 index.mapID = mapID119 stats, levels, err := capturePBTiles(ctx, fetcher, o, page, pageURL, scope, mapDir, index)120 if err != nil {121 return err122 }123 capture.Levels = levels124125 raw, err := json.MarshalIndent(capture, "", " ")126 if err != nil {127 return err128 }129 // A capture that does not translate would poison the archive for both130 // tools that read it, so it is refused here, while the problem is one131 // crawl old rather than buried in history.132 if _, err := pbmap.Translate(raw); err != nil {133 return fmt.Errorf("capture does not translate: %w", err)134 }135 if err := writeSnapshotRecord(mapDir, raw, pbmap.Kind, mapID, pageURL); err != nil {136 return err137 }138 if err := writeTileIndex(mapDir, index); err != nil {139 return err140 }141 if err := writeMapMetadata(mapDir, game, full, index, len(capture.Pins)); err != nil {142 return err143 }144 if err := registerMap(o.archive, game, full, gameDirectory, mapDirectory, "piggyback"); err != nil {145 return err146 }147 fmt.Printf(" %d fetched · %d held · %d not published · %d failed · %.1f MB new\n",148 stats.fetched, stats.skipped, stats.absent, stats.failed, float64(stats.bytes)/1e6)149 return nil150}151152func fetchPBPage(ctx context.Context, f *fetcher, pageURL string) (*pbPage, error) {153 body, _, err := f.get(ctx, pageURL)154 if err != nil {155 return nil, err156 }157 const openTag = `<script id="__NEXT_DATA__" type="application/json">`158 _, rest, found := strings.Cut(string(body), openTag)159 if !found {160 return nil, fmt.Errorf("%s embeds no page state", pageURL)161 }162 doc, _, found := strings.Cut(rest, "</script>")163 if !found {164 return nil, fmt.Errorf("%s embeds unterminated page state", pageURL)165 }166 var state struct {167 Props struct {168 PageProps struct {169 Settings struct {170 TileServerURL string `json:"tileServerUrl"`171 MinZoom string `json:"minZoom"`172 MaxZoom string `json:"maxZoom"`173 MaxZoomPremium string `json:"maxZoomPremium"`174 } `json:"settings"`175 I18N struct {176 Store map[string]struct {177 Common struct {178 Category map[string]string `json:"category"`179 Type map[string]string `json:"type"`180 } `json:"common"`181 } `json:"initialI18nStore"`182 } `json:"_nextI18Next"`183 } `json:"pageProps"`184 } `json:"props"`185 }186 if err := json.Unmarshal([]byte(doc), &state); err != nil {187 return nil, fmt.Errorf("decode %s page state: %w", pageURL, err)188 }189 settings := state.Props.PageProps.Settings190 if settings.TileServerURL == "" {191 return nil, fmt.Errorf("%s describes no map", pageURL)192 }193 page := &pbPage{194 tileServer: settings.TileServerURL,195 minZoom: pbAtoi(settings.MinZoom, 0),196 maxZoom: pbAtoi(settings.MaxZoom, 0),197 premiumMaxZoom: pbAtoi(settings.MaxZoomPremium, 0),198 categoryLabels: map[string]string{},199 typeLabels: map[string]string{},200 }201 if page.maxZoom <= 0 {202 return nil, fmt.Errorf("%s declares no zoom range", pageURL)203 }204 // English labels when the page carries them, any language over none.205 store := state.Props.PageProps.I18N.Store206 languages := []string{"en"}207 for language := range store {208 languages = append(languages, language)209 }210 for _, language := range languages {211 bundle, ok := store[language]212 if !ok {213 continue214 }215 for key, label := range bundle.Common.Category {216 if _, taken := page.categoryLabels[key]; !taken {217 page.categoryLabels[key] = label218 }219 }220 for key, label := range bundle.Common.Type {221 if _, taken := page.typeLabels[key]; !taken {222 page.typeLabels[key] = label223 }224 }225 }226 return page, nil227}228229// The two data requests every visitor's map makes, under the map's own path.230type pbAPICategory struct {231 ID string `json:"id"`232 Key string `json:"key"`233 Position float64 `json:"position"`234 Types []struct {235 ID string `json:"id"`236 Key string `json:"key"`237 Position float64 `json:"position"`238 } `json:"types"`239}240241type pbAPIPin struct {242 ID string `json:"id"`243 X string `json:"x"`244 Y string `json:"y"`245 Category struct {246 Key string `json:"key"`247 } `json:"category"`248 Type struct {249 Key string `json:"key"`250 } `json:"type"`251 Data []struct {252 Name string `json:"name"`253 Description string `json:"description"`254 } `json:"data"`255}256257func fetchPBCategories(ctx context.Context, f *fetcher, pageURL string) ([]pbAPICategory, error) {258 var out []pbAPICategory259 if err := fetchPBResult(ctx, f, pageURL+"/api/trpc/category.getCategory", &out); err != nil {260 return nil, err261 }262 if len(out) == 0 {263 return nil, errors.New("the map declares no categories")264 }265 return out, nil266}267268func fetchPBPins(ctx context.Context, f *fetcher, pageURL string) ([]pbAPIPin, error) {269 var out []pbAPIPin270 if err := fetchPBResult(ctx, f, pageURL+"/api/trpc/pin.getPins", &out); err != nil {271 return nil, err272 }273 if len(out) == 0 {274 return nil, errors.New("the map answered with no pins at all")275 }276 return out, nil277}278279func fetchPBResult(ctx context.Context, f *fetcher, url string, into any) error {280 body, _, err := f.get(ctx, url)281 if err != nil {282 return err283 }284 var envelope struct {285 Result struct {286 Data struct {287 JSON json.RawMessage `json:"json"`288 } `json:"data"`289 } `json:"result"`290 Error json.RawMessage `json:"error"`291 }292 if err := json.Unmarshal(body, &envelope); err != nil {293 return fmt.Errorf("decode %s: %w", url, err)294 }295 if len(envelope.Error) > 0 {296 return fmt.Errorf("%s answered: %s", url, envelope.Error)297 }298 if err := json.Unmarshal(envelope.Result.Data.JSON, into); err != nil {299 return fmt.Errorf("decode %s payload: %w", url, err)300 }301 return nil302}303304func composePBCapture(305 gameSlug, mapSlug string,306 page *pbPage,307 categories []pbAPICategory,308 pins []pbAPIPin,309) *pbmap.Capture {310 capture := &pbmap.Capture{311 Source: pbmap.Source,312 GameSlug: gameSlug,313 MapSlug: mapSlug,314 // The page's language bundle names categories and types, never the315 // game or the map themselves; their slugs, spelled out, are all the316 // naming there is.317 GameTitle: mgdoc.SpellOut(gameSlug),318 MapTitle: mgdoc.SpellOut(mapSlug),319 }320 capture.Map = pbmap.MapConfig{321 TileServer: page.tileServer,322 MinZoom: page.minZoom,323 MaxZoom: page.maxZoom,324 PremiumMaxZoom: page.premiumMaxZoom,325 Transform: pbTransform,326 }327 for _, category := range categories {328 entry := pbmap.Category{ID: category.ID, Key: category.Key, Position: category.Position}329 for _, t := range category.Types {330 entry.Types = append(entry.Types, pbmap.Type{ID: t.ID, Key: t.Key, Position: t.Position})331 }332 capture.Categories = append(capture.Categories, entry)333 }334 for _, pin := range pins {335 entry := pbmap.Pin{336 ID: pin.ID,337 X: pin.X,338 Y: pin.Y,339 CategoryKey: pin.Category.Key,340 TypeKey: pin.Type.Key,341 }342 // One entry is the rule; a pin stacking several keeps them all, the343 // extras folded into the description rather than dropped.344 if len(pin.Data) > 0 {345 entry.Name = pin.Data[0].Name346 entry.Description = pin.Data[0].Description347 for _, extra := range pin.Data[1:] {348 addition := strings.TrimSpace(extra.Name + " — " + extra.Description)349 entry.Description = strings.TrimSpace(entry.Description + "\n\n" + addition)350 }351 }352 capture.Pins = append(capture.Pins, entry)353 }354 // Labels for every key the data actually uses, from the page's bundle.355 seenCategories := map[string]bool{}356 seenTypes := map[string]bool{}357 for _, category := range capture.Categories {358 seenCategories[category.Key] = true359 for _, t := range category.Types {360 seenTypes[t.Key] = true361 }362 }363 for _, pin := range capture.Pins {364 seenCategories[pin.CategoryKey] = true365 seenTypes[pin.TypeKey] = true366 }367 for key := range seenCategories {368 if label := page.categoryLabels[key]; label != "" {369 capture.Labels.Categories = append(capture.Labels.Categories, pbmap.Label{Key: key, Label: label})370 }371 }372 for key := range seenTypes {373 if label := page.typeLabels[key]; label != "" {374 capture.Labels.Types = append(capture.Labels.Types, pbmap.Label{Key: key, Label: label})375 }376 }377 capture.Normalize()378 return capture379}380381// fetchPBIcons stores each type's marker icon under the key the translator382// gives its category, so the icons attach at the next generation. Piggyback383// serves them only as WebP under an asset prefix of the app's own -- for384// cyberpunk-2077 the assets live under "cyberpunk" -- so the prefix is found385// by asking, and each icon is decoded and archived as the PNG the pipeline386// reads. A type without an icon, like the district name labels, is simply a387// type the viewer keeps its fallback glyph for.388func fetchPBIcons(389 ctx context.Context,390 f *fetcher,391 gameSlug, pageURL, gameDir string,392 capture *pbmap.Capture,393) error {394 keys := make(map[string]bool)395 for _, category := range capture.Categories {396 for _, t := range category.Types {397 keys[t.Key] = true398 }399 }400 for _, pin := range capture.Pins {401 keys[pin.TypeKey] = true402 }403 ordered := make([]string, 0, len(keys))404 for key := range keys {405 ordered = append(ordered, key)406 }407 sort.Strings(ordered)408409 prefix, found := "", false410 fetched, absent := 0, 0411 for _, key := range ordered {412 path := filepath.Join(gameDir, "icons", key+".png")413 if _, err := os.Stat(path); err == nil {414 continue415 }416 if !found {417 prefix, found = discoverPBIconPrefix(ctx, f, gameSlug, pageURL, ordered)418 if !found {419 fmt.Println(" no icon prefix answers; the viewer will use its fallback glyphs")420 return nil421 }422 }423 body, _, err := f.get(ctx, pageURL+"/"+prefix+"/icons/"+key+"@2x.webp")424 if errors.Is(err, errAbsent) {425 absent++426 continue427 }428 if err != nil {429 return fmt.Errorf("icon %s: %w", key, err)430 }431 decoded, _, err := image.Decode(bytes.NewReader(body))432 if err != nil {433 return fmt.Errorf("decode icon %s: %w", key, err)434 }435 var out bytes.Buffer436 if err := png.Encode(&out, decoded); err != nil {437 return fmt.Errorf("encode icon %s: %w", key, err)438 }439 if err := writeFile(path, out.Bytes()); err != nil {440 return err441 }442 fetched++443 }444 if fetched > 0 || absent > 0 {445 fmt.Printf(" %d icons · %d types draw no icon\n", fetched, absent)446 }447 return nil448}449450// discoverPBIconPrefix finds the asset prefix by asking: the game's slug, and451// then the slug with its trailing words dropped, against the first few types.452func discoverPBIconPrefix(453 ctx context.Context,454 f *fetcher,455 gameSlug, pageURL string,456 keys []string,457) (string, bool) {458 candidates := []string{gameSlug}459 for at := strings.LastIndex(gameSlug, "-"); at > 0; at = strings.LastIndex(gameSlug[:at], "-") {460 candidates = append(candidates, gameSlug[:at])461 }462 probes := keys463 if len(probes) > 3 {464 probes = probes[:3]465 }466 for _, candidate := range candidates {467 for _, key := range probes {468 _, _, err := f.get(ctx, pageURL+"/"+candidate+"/icons/"+key+"@2x.webp")469 if err == nil {470 return candidate, true471 }472 if !errors.Is(err, errAbsent) {473 return "", false474 }475 }476 }477 return "", false478}479480func pbDeepest(page *pbPage, o options) int {481 deepest := page.maxZoom482 if o.maxZoom > 0 {483 // An explicit ask may pass the free ceiling, up to what exists at all.484 limit := page.premiumMaxZoom485 if limit <= 0 {486 limit = page.maxZoom487 }488 deepest = min(o.maxZoom, limit)489 }490 return deepest491}492493// capturePBTiles surveys and captures the pyramid in one walk. The first494// published level is asked for whole; every deeper level asks only after the495// children of tiles that exist, which is what bounds the walk over a mostly496// empty world square. What each level turns out to hold becomes the capture's497// account of the pyramid.498func capturePBTiles(499 ctx context.Context,500 fetcher *fetcher,501 o options,502 page *pbPage,503 pageURL, scope, mapDir string,504 index *tileIndex,505) (captureStats, []pbmap.Level, error) {506 var stats captureStats507 var levels []pbmap.Level508 setID := index.tileSetID(scope)509 extension := pbmap.TileExtension(page.tileServer)510 deepest := pbDeepest(page, o)511 referer := pageURL + "/"512 fmt.Printf(" layer %q z%d-%d\n", scope, page.minZoom, deepest)513514 var present map[[2]int]bool515 for zoom := page.minZoom; zoom <= deepest; zoom++ {516 across := 1 << zoom517 window := tileWindow{minX: 0, minY: 0, maxX: across - 1, maxY: across - 1}518 var wanted [][2]int519 if present == nil {520 wanted = window.tiles()521 } else {522 wanted = childrenOf(present, window)523 }524 if len(wanted) == 0 {525 break526 }527 results, err := fetchTemplateLevel(ctx, fetcher, o, page.tileServer, "", referer,528 setID, extension, mapDir, zoom, wanted, index)529 if err != nil {530 return stats, nil, err531 }532 stats.merge(results)533534 present = make(map[[2]int]bool, len(results.hashes))535 level := pbmap.Level{Zoom: zoom, MinX: across, MinY: across, MaxX: -1, MaxY: -1}536 for coordinate := range results.hashes {537 present[coordinate] = true538 level.MinX = min(level.MinX, coordinate[0])539 level.MinY = min(level.MinY, coordinate[1])540 level.MaxX = max(level.MaxX, coordinate[0])541 level.MaxY = max(level.MaxY, coordinate[1])542 }543 fmt.Printf(" z%-2d %4d fetched · %4d held · %4d not published\n",544 zoom, results.fetched, results.skipped, results.absent)545 if level.MaxX < 0 {546 fmt.Printf(" z%-2d nothing published; the survey stops here\n", zoom)547 break548 }549 // The pipeline trusts a level to fill its declared window. Holes550 // inside the window are survivable -- the level rides along partially551 // with its coverage recorded -- but they deserve a loud word.552 expected := (level.MaxX - level.MinX + 1) * (level.MaxY - level.MinY + 1)553 if len(present) < expected {554 fmt.Printf(" WARNING: z%d holds %d of the %d tiles in its window\n",555 zoom, len(present), expected)556 }557 levels = append(levels, level)558 }559 if len(levels) == 0 {560 return stats, nil, errors.New("no tiles at all; is the CDN refusing the survey?")561 }562 return stats, levels, nil563}564565// pbArchiveID numbers Piggyback games and maps in the archive's registers,566// one bit above IGN's so every source's ids stay apart; see ignArchiveID for567// why the range stays far below a float64's integers.568func pbArchiveID(seed string) int64 {569 hash := fnv.New32a()570 _, _ = hash.Write([]byte(seed))571 return int64(hash.Sum32()&0x7fffffff) | 1<<33572}573574func pbAtoi(value string, fallback int) int {575 parsed, err := strconv.Atoi(strings.TrimSpace(value))576 if err != nil {577 return fallback578 }579 return parsed580}