1// The IGN mode fills the same archive from IGN's wikimaps. A wikimap page2// carries its whole configuration in the page itself; the markers come from3// one GraphQL request; the tiles are a small pyramid asked for level by level.4// What lands in the archive is a canonical capture -- the ignmap document --5// stored and versioned exactly the way MapGenie snapshots are.6package main78import (9 "context"10 "crypto/sha256"11 "encoding/hex"12 "encoding/json"13 "errors"14 "fmt"15 "hash/fnv"16 "os"17 "path/filepath"18 "strconv"19 "strings"20 "sync"2122 "github.com/FelineStateMachine/atlas/internal/ignmap"23)2425const ignGraphQL = "https://mollusk.apis.ign.com/graphql"2627// ignPageMap is the slice of the page's embedded state a capture is built28// from. Everything else in __NEXT_DATA__ -- build ids, trace baggage, session29// scaffolding -- is volatile and never stored.30type ignPageMap struct {31 Width float64 `json:"width"`32 Height float64 `json:"height"`33 MinZoom int `json:"minZoom"`34 MaxZoom int `json:"maxZoom"`35 InitialLat float64 `json:"initialLat"`36 InitialLng float64 `json:"initialLng"`37 BackgroundColor string `json:"backgroundColor"`38 MapName string `json:"mapName"`39 MapType string `json:"mapType"`40 ObjectName string `json:"objectName"`41 MapGenieGameID json.RawMessage `json:"mapGenieGameId"`42 Tilesets []string `json:"tilesets"`43 Types []ignPageType `json:"types"`44}4546type ignPageType struct {47 TypeSlug string `json:"typeSlug"`48 TypeName string `json:"typeName"`49 ParentTypeSlug string `json:"parentTypeSlug"`50 MarkerIcon struct {51 URL string `json:"url"`52 Width int `json:"width"`53 Height int `json:"height"`54 } `json:"markerIcon"`55}5657func runIGN(ctx context.Context, fetcher *fetcher, o options) error {58 objectSlug, mapSlug, ok := strings.Cut(o.ign, "/")59 if !ok || objectSlug == "" || mapSlug == "" {60 return errors.New("-ign wants objectSlug/mapSlug, as the address bar spells it")61 }62 pageURL := fmt.Sprintf("https://www.ign.com/maps/%s/%s", objectSlug, mapSlug)6364 page, err := fetchIGNPage(ctx, fetcher, pageURL)65 if err != nil {66 return err67 }68 // A wikimap page can also front a MapGenie map. That data has its own69 // crawler, and capturing it here would archive the same source twice under70 // two names.71 if page.MapType != "ign" {72 return fmt.Errorf("%s is not a native IGN wikimap (map type %q)", pageURL, page.MapType)73 }74 if id := strings.TrimSpace(string(page.MapGenieGameID)); id != "" && id != "null" {75 return fmt.Errorf("%s proxies MapGenie game %s; crawl it with -game instead", pageURL, id)76 }77 if len(page.Tilesets) == 0 {78 return fmt.Errorf("%s publishes no tileset", pageURL)79 }80 if len(page.Tilesets) > 1 {81 fmt.Printf(" note: %d tilesets published; capturing the first\n", len(page.Tilesets))82 }83 template := page.Tilesets[0]84 scope := objectSlug + "/" + mapSlug85 if !strings.Contains(template, "/wikimaps/"+scope+"/") {86 return fmt.Errorf("tileset %q does not sit under /wikimaps/%s/, and the pipeline groups tiles by that path", template, scope)87 }8889 markers, err := fetchIGNMarkers(ctx, fetcher, objectSlug, mapSlug, page.Types)90 if err != nil {91 return err92 }9394 capture := composeIGNCapture(objectSlug, mapSlug, page, markers)95 raw, err := json.MarshalIndent(capture, "", " ")96 if err != nil {97 return err98 }99 // A capture that does not translate would poison the archive for both100 // tools that read it, so it is refused here, while the problem is one101 // crawl old rather than buried in history.102 if _, err := ignmap.Translate(raw); err != nil {103 return fmt.Errorf("capture does not translate: %w", err)104 }105106 gameID := ignArchiveID("ign:" + objectSlug)107 mapID := ignArchiveID("ign:" + scope)108 game := &apiGameFull{ID: gameID, Title: capture.GameTitle,109 Config: apiGameConfig{URL: pageURL}}110 full := &apiMapFull{ID: mapID, Title: capture.MapTitle, Slug: mapSlug}111 // The directory says where the capture came from even though the game112 // keeps its plain title: a MapGenie capture of the same game lives in its113 // own directory, and the two must never resolve to one.114 gameDirectory := resolveGameDirectory(o.archive,115 &apiGameFull{ID: gameID, Title: "IGN " + capture.GameTitle})116 mapDirectory := resolveMapDirectory(o.archive, gameDirectory, full)117 mapDir := filepath.Join(o.archive, mapDirectory)118119 fmt.Printf("\n== IGN %s / %s (%d markers, %d types)\n",120 capture.GameTitle, capture.MapTitle, len(capture.Markers), len(capture.Types))121122 if o.dryRun {123 fmt.Printf(" would write capture to %s\n", mapDir)124 } else if err := writeSnapshotRecord(mapDir, raw, ignmap.Kind, mapID, pageURL); err != nil {125 return err126 }127128 if err := fetchIGNIcons(ctx, fetcher, o, filepath.Join(o.archive, gameDirectory), page.Types); err != nil {129 return err130 }131132 index, err := readTileIndex(mapDir)133 if err != nil {134 return err135 }136 index.mapID = mapID137 stats, err := captureIGNTiles(ctx, fetcher, o, capture, template, scope, mapDir, index)138 if err != nil {139 return err140 }141 if o.dryRun {142 return nil143 }144 if err := writeTileIndex(mapDir, index); err != nil {145 return err146 }147 if err := writeMapMetadata(mapDir, game, full, index, len(capture.Markers)); err != nil {148 return err149 }150 if err := registerMap(o.archive, game, full, gameDirectory, mapDirectory, "ign"); err != nil {151 return err152 }153 fmt.Printf(" %d fetched · %d held · %d not published · %d failed · %.1f MB new\n",154 stats.fetched, stats.skipped, stats.absent, stats.failed, float64(stats.bytes)/1e6)155 return nil156}157158// fetchIGNPage pulls the map's configuration out of the page. The page embeds159// its state as one JSON document in a script tag, which is the same data the160// page's own scripts boot from.161func fetchIGNPage(ctx context.Context, f *fetcher, pageURL string) (*ignPageMap, error) {162 body, _, err := f.get(ctx, pageURL)163 if err != nil {164 return nil, err165 }166 const openTag = `<script id="__NEXT_DATA__" type="application/json">`167 _, rest, found := strings.Cut(string(body), openTag)168 if !found {169 return nil, fmt.Errorf("%s embeds no page state", pageURL)170 }171 doc, _, found := strings.Cut(rest, "</script>")172 if !found {173 return nil, fmt.Errorf("%s embeds unterminated page state", pageURL)174 }175 var page struct {176 Props struct {177 PageProps struct {178 Page struct {179 Map ignPageMap `json:"map"`180 } `json:"page"`181 } `json:"pageProps"`182 } `json:"props"`183 }184 if err := json.Unmarshal([]byte(doc), &page); err != nil {185 return nil, fmt.Errorf("decode %s page state: %w", pageURL, err)186 }187 m := page.Props.PageProps.Page.Map188 if m.Width <= 0 || m.Height <= 0 || len(m.Types) == 0 {189 return nil, fmt.Errorf("%s describes no wikimap", pageURL)190 }191 return &m, nil192}193194// fetchIGNMarkers asks for every marker of every type in one request, which is195// how IGN's own page loads a map with everything switched on.196func fetchIGNMarkers(197 ctx context.Context,198 f *fetcher,199 objectSlug, mapSlug string,200 types []ignPageType,201) ([]ignmap.Marker, error) {202 typeSlugs := make([]string, 0, len(types))203 for _, t := range types {204 typeSlugs = append(typeSlugs, t.TypeSlug)205 }206 request, err := json.Marshal(map[string]any{207 "query": `query MapMarkers($objectSlug: String!, $mapSlug: String!, $typeSlugs: [String!]!) {208 markersMultiple(objectSlug: $objectSlug, mapSlug: $mapSlug, typeSlugs: $typeSlugs) {209 id lat lng markerName markerSlug typeSlug wikiPage iconSlug regionId checklistTaskId210 }211}`,212 "variables": map[string]any{213 "objectSlug": objectSlug,214 "mapSlug": mapSlug,215 "typeSlugs": typeSlugs,216 },217 })218 if err != nil {219 return nil, err220 }221 body, _, err := f.post(ctx, ignGraphQL, "application/json", request)222 if err != nil {223 return nil, err224 }225 var response struct {226 Data struct {227 MarkersMultiple []ignmap.Marker `json:"markersMultiple"`228 } `json:"data"`229 Errors []struct {230 Message string `json:"message"`231 } `json:"errors"`232 }233 if err := json.Unmarshal(body, &response); err != nil {234 return nil, fmt.Errorf("decode markers: %w", err)235 }236 if len(response.Errors) > 0 {237 return nil, fmt.Errorf("markers query: %s", response.Errors[0].Message)238 }239 if len(response.Data.MarkersMultiple) == 0 {240 return nil, errors.New("the map answered with no markers at all")241 }242 return response.Data.MarkersMultiple, nil243}244245func composeIGNCapture(246 objectSlug, mapSlug string,247 page *ignPageMap,248 markers []ignmap.Marker,249) *ignmap.Capture {250 capture := &ignmap.Capture{251 Source: ignmap.Source,252 ObjectSlug: objectSlug,253 MapSlug: mapSlug,254 GameTitle: page.ObjectName,255 MapTitle: page.MapName,256 Map: ignmap.MapConfig{257 Width: page.Width,258 Height: page.Height,259 MinZoom: page.MinZoom,260 MaxZoom: page.MaxZoom,261 InitialLat: page.InitialLat,262 InitialLng: page.InitialLng,263 BackgroundColor: page.BackgroundColor,264 Tileset: page.Tilesets[0],265 },266 Markers: markers,267 }268 if capture.GameTitle == "" {269 capture.GameTitle = objectSlug270 }271 for _, t := range page.Types {272 capture.Types = append(capture.Types, ignmap.Type{273 TypeSlug: t.TypeSlug,274 TypeName: t.TypeName,275 ParentTypeSlug: t.ParentTypeSlug,276 IconURL: t.MarkerIcon.URL,277 IconWidth: t.MarkerIcon.Width,278 IconHeight: t.MarkerIcon.Height,279 })280 }281 capture.Normalize()282 return capture283}284285// fetchIGNIcons stores each type's marker icon under the key tools/generate286// looks icons up by: the category's icon name, which for IGN is the type slug.287func fetchIGNIcons(288 ctx context.Context,289 f *fetcher,290 o options,291 gameDir string,292 types []ignPageType,293) error {294 fetched := 0295 for _, t := range types {296 if t.MarkerIcon.URL == "" {297 continue298 }299 extension := "png"300 if dot := strings.LastIndex(t.MarkerIcon.URL, "."); dot >= 0 && dot < len(t.MarkerIcon.URL)-1 {301 extension = strings.ToLower(t.MarkerIcon.URL[dot+1:])302 }303 path := filepath.Join(gameDir, "icons", t.TypeSlug+"."+extension)304 if _, err := os.Stat(path); err == nil {305 continue306 }307 if o.dryRun {308 fetched++309 continue310 }311 body, _, err := f.get(ctx, t.MarkerIcon.URL)312 if err != nil {313 if errors.Is(err, errAbsent) {314 fmt.Printf(" icon %s is not published\n", t.TypeSlug)315 continue316 }317 return fmt.Errorf("icon %s: %w", t.TypeSlug, err)318 }319 if err := writeFile(path, body); err != nil {320 return err321 }322 fetched++323 }324 if fetched > 0 {325 fmt.Printf(" %d icons\n", fetched)326 }327 return nil328}329330// captureIGNTiles takes the whole pyramid level by level. There is no pruning331// to plan: the deepest level of a wikimap is at most a few thousand tiles, and332// the deepest level must be complete for the pyramid the bundle is built from,333// so every tile of every level is asked for once.334func captureIGNTiles(335 ctx context.Context,336 fetcher *fetcher,337 o options,338 capture *ignmap.Capture,339 template, scope, mapDir string,340 index *tileIndex,341) (captureStats, error) {342 var stats captureStats343 setID := index.tileSetID(scope)344 extension := ignmap.TileExtension(template)345 deepest := capture.Map.MaxZoom346 if o.maxZoom > 0 && o.maxZoom < deepest {347 deepest = o.maxZoom348 }349 fmt.Printf(" layer %q z0-%d\n", scope, deepest)350351 for zoom := 0; zoom <= deepest; zoom++ {352 maxX, maxY := ignmap.LevelExtent(capture.Map.Width, capture.Map.Height, zoom)353 window := tileWindow{minX: 0, minY: 0, maxX: maxX, maxY: maxY}354 results, err := fetchTemplateLevel(ctx, fetcher, o, template, "", "", setID, extension,355 mapDir, zoom, window.tiles(), index)356 if err != nil {357 return stats, err358 }359 stats.merge(results)360 fmt.Printf(" z%-2d %4d fetched · %4d held · %4d not published\n",361 zoom, results.fetched, results.skipped, results.absent)362 // The bundle is built from the deepest complete level. Holes there mean363 // the pyramid tops out shallower than the map is drawn, which deserves364 // more than a counter in the summary line.365 if zoom == deepest && results.absent > 0 {366 fmt.Printf(" WARNING: %d tiles of the deepest level are not published; "+367 "the bundle will be built from a shallower level\n", results.absent)368 }369 }370 return stats, nil371}372373// fetchTemplateLevel takes one level of a pyramid whose URLs come from a374// {z}/{x}/{y} template, in whatever punctuation the source spells the tokens.375// The hashes of held tiles are recorded alongside fetched ones, so a resumed376// crawl sees the same picture of the level a fresh one would.377//378// A source that spells its addresses differently from the pipeline -- Trek379// names the row before the column, one zoom down -- passes a second template380// naming where the bytes actually live; the first template stays the address381// of record, the one resume checks and every reader of the archive derives382// the layer from. An empty fetchTemplate means the record address is the383// fetch address, which is every other source.384func fetchTemplateLevel(385 ctx context.Context,386 fetcher *fetcher,387 o options,388 template, fetchTemplate, referer string,389 setID int64,390 extension, mapDir string,391 zoom int,392 wanted [][2]int,393 index *tileIndex,394) (levelResult, error) {395 result := levelResult{hashes: make(map[[2]int]string, len(wanted))}396397 var mu sync.Mutex398 var group sync.WaitGroup399 var failure error400 gate := make(chan struct{}, o.concurrency)401402 replacer := func(template string, x, y int) string {403 return strings.NewReplacer(404 "{z}", strconv.Itoa(zoom),405 "{x}", strconv.Itoa(x),406 "{y}", strconv.Itoa(y),407 ).Replace(template)408 }409 if fetchTemplate == "" {410 fetchTemplate = template411 }412413 for _, coordinate := range wanted {414 url := replacer(template, coordinate[0], coordinate[1])415 path := filepath.Join(mapDir, "tiles",416 "set-"+strconv.FormatInt(setID, 10),417 strconv.Itoa(zoom), strconv.Itoa(coordinate[0]),418 strconv.Itoa(coordinate[1])+"."+extension)419420 if record, held := index.byURL[url]; held {421 if record.Status == "absent" {422 result.absent++423 continue424 }425 if record.Status == "cached" {426 if _, err := os.Stat(path); err == nil {427 result.skipped++428 result.hashes[coordinate] = record.ContentHash429 continue430 }431 }432 }433 if o.dryRun {434 result.fetched++435 continue436 }437438 group.Add(1)439 gate <- struct{}{}440 go func(coordinate [2]int, url, fetchURL, path string) {441 defer group.Done()442 defer func() { <-gate }()443444 body, contentType, err := fetcher.getReferred(ctx, fetchURL, referer)445 mu.Lock()446 defer mu.Unlock()447 if err != nil {448 status, message := "failed", err.Error()449 if errors.Is(err, errAbsent) {450 status, message = "absent", ""451 result.absent++452 } else {453 result.failed++454 if failure == nil {455 failure = err456 }457 }458 index.put(tileRecord{459 URL: url, Status: status, TileSetID: setID,460 Zoom: zoom, X: coordinate[0], Y: coordinate[1],461 ContentType: "application/octet-stream",462 Error: message,463 })464 return465 }466 if err := writeFile(path, body); err != nil {467 failure = err468 return469 }470 hash := hashBytes(body)471 index.put(tileRecord{472 URL: url, Status: "cached", TileSetID: setID,473 Zoom: zoom, X: coordinate[0], Y: coordinate[1],474 ContentHash: hash, ContentType: contentType,475 ByteLength: len(body),476 CoverageKey: fmt.Sprintf("%d:%d:%d:%d:%d",477 index.mapID, setID, zoom, coordinate[0], coordinate[1]),478 })479 result.fetched++480 result.bytes += int64(len(body))481 result.hashes[coordinate] = hash482 }(coordinate, url, replacer(fetchTemplate, coordinate[0], coordinate[1]), path)483 }484 group.Wait()485 return result, failure486}487488func hashBytes(body []byte) string {489 sum := sha256.Sum256(body)490 return hex.EncodeToString(sum[:])491}492493// ignArchiveID numbers IGN games and maps in the archive's registers. The high494// bit set above the int32 range keeps them apart from every MapGenie id while495// staying far below the point where a float64 -- which is how the register496// reads them back -- loses integers.497func ignArchiveID(seed string) int64 {498 hash := fnv.New32a()499 _, _ = hash.Write([]byte(seed))500 return int64(hash.Sum32()&0x7fffffff) | 1<<32501}