A run names the map the way the address bar does -- crawl -ign cyberpunk-2077/night-city -- and captures everything it needs in one pass: the configuration embedded in the page, every marker from one GraphQL request, each type's icon, and the whole tile pyramid, which for a wikimap is small enough to take level by level without pruning. The capture lands in the same archive under the same disciplines the MapGenie crawler established: content-addressed snapshots that record a new version only when the data changed, a tile index that lets an interrupted run resume, and registration that makes the map visible to the tools downstream. A capture is refused at the door if it does not translate, if the page fronts a MapGenie map that has its own crawler, or if the tiles are served from a path the pipeline cannot group.
- commit
- 90ea57d21af9699876a45bf037a00f030963709f
- parent
- 50a20a7dcb51941af0f783d9bd674f75761679e8
- author
- Dami <dami.etoile@gmail.com>
| M | tools/ | 27 | ++- |
| M | tools/ | 56 | ++- |
| A | tools/ | 484 | ++++++++++++++++++++++++++++++ |
| M | tools/ | 10 | +- |
| M | tools/ | 19 | + |
@@ -1,6 +1,7 @@
11 package main22 33 import (4+ "bytes"45 "context"56 "encoding/json"67 "errors"@@ -81,6 +82,30 @@ func (f *fetcher) penalise(d time.Duration) {
8182 const maxAttempts = 48283 8384 func (f *fetcher) get(ctx context.Context, url string) ([]byte, string, error) {85+ return f.do(ctx, url, func() (*http.Request, error) {86+ return http.NewRequestWithContext(ctx, http.MethodGet, url, nil)87+ })88+}89+90+// post carries the same manners as get -- spacing, backoff, penalties shared91+// with every other request -- because an API asked with POST is no less a92+// guest's request than a tile asked with GET.93+func (f *fetcher) post(ctx context.Context, url, contentType string, body []byte) ([]byte, string, error) {94+ return f.do(ctx, url, func() (*http.Request, error) {95+ request, err := http.NewRequestWithContext(ctx, http.MethodPost, url, bytes.NewReader(body))96+ if err != nil {97+ return nil, err98+ }99+ request.Header.Set("Content-Type", contentType)100+ return request, nil101+ })102+}103+104+func (f *fetcher) do(105+ ctx context.Context,106+ url string,107+ build func() (*http.Request, error),108+) ([]byte, string, error) {84109 var lastErr error85110 for attempt := range maxAttempts {86111 if attempt > 0 {@@ -91,7 +116,7 @@ func (f *fetcher) get(ctx context.Context, url string) ([]byte, string, error) {
91116 return nil, "", err92117 }93118 94- request, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)119+ request, err := build()95120 if err != nil {96121 return nil, "", err97122 }@@ -115,19 +115,21 @@ func readTileIndex(mapDir string) (*tileIndex, error) {
115115 }116116 117117 // tileSetPathOf recovers the layer path from a tile URL, matching how118-// tools/tiles groups levels: everything between "/games/" and "/<zoom>/".118+// tools/tiles groups levels: everything between the source's marker segment119+// and "/<zoom>/". MapGenie serves tiles under /games/, IGN under /wikimaps/.119120 func tileSetPathOf(url string, zoom int) string {120- const marker = "/games/"121- start := strings.Index(url, marker)122- if start < 0 {123- return ""124- }125- rest := url[start+len(marker):]126- end := strings.Index(rest, "/"+strconv.Itoa(zoom)+"/")127- if end < 0 {128- return ""121+ for _, marker := range []string{"/games/", "/wikimaps/"} {122+ _, rest, found := strings.Cut(url, marker)123+ if !found {124+ continue125+ }126+ path, _, found := strings.Cut(rest, "/"+strconv.Itoa(zoom)+"/")127+ if !found {128+ continue129+ }130+ return path129131 }130- return rest[:end]132+ return ""131133 }132134 133135 func writeTileIndex(mapDir string, index *tileIndex) error {@@ -139,6 +141,14 @@ func writeTileIndex(mapDir string, index *tileIndex) error {
139141 // writeSnapshot stores the response exactly as received, addressed by its hash,140142 // and records it only if that content is new.141143 func writeSnapshot(mapDir string, full *apiMapFull, raw []byte) error {144+ return writeSnapshotRecord(mapDir, raw, "map", full.ID,145+ fmt.Sprintf("/api/v1/maps/%d/full", full.ID))146+}147+148+// writeSnapshotRecord is the shape every source's capture takes: the document149+// addressed by its hash, and one index record per distinct content. A capture150+// whose bytes are already archived leaves no trace of having been retaken.151+func writeSnapshotRecord(mapDir string, raw []byte, kind string, sourceID int64, sourceURL string) error {142152 sum := sha256.Sum256(raw)143153 hash := hex.EncodeToString(sum[:])144154 @@ -160,9 +170,9 @@ func writeSnapshot(mapDir string, full *apiMapFull, raw []byte) error {
160170 snapshots = append(snapshots, snapshotRecord{161171 CapturedAt: time.Now().UTC().Format(time.RFC3339Nano),162172 ContentHash: hash,163- Kind: "map",164- SourceID: full.ID,165- SourceURL: fmt.Sprintf("/api/v1/maps/%d/full", full.ID),173+ Kind: kind,174+ SourceID: sourceID,175+ SourceURL: sourceURL,166176 })167177 sort.Slice(snapshots, func(a, b int) bool {168178 return snapshots[a].CapturedAt < snapshots[b].CapturedAt@@ -175,6 +185,7 @@ func writeMapMetadata(
175185 game *apiGameFull,176186 full *apiMapFull,177187 index *tileIndex,188+ pinCount int,178189 ) error {179190 cachedByZoom := map[string]int{}180191 totalsByZoom := map[string]int{}@@ -202,7 +213,7 @@ func writeMapMetadata(
202213 "mapSlug": full.Slug,203214 "mapTitle": full.Title,204215 "phase": "complete",205- "pinCount": full.pinCount(),216+ "pinCount": pinCount,206217 "snapshotCount": 1,207218 "tileBytes": bytes,208219 "tileCount": count,@@ -211,11 +222,13 @@ func writeMapMetadata(
211222 212223 // registerMap makes the map discoverable to tools/tiles and tools/generate by213224 // listing it in game.json and archive.json, leaving any existing entries alone.225+// A source name, when there is one, rides along so a reader of the archive can226+// tell which crawler a game came from; the tools never look at it.214227 func registerMap(215228 archiveRoot string,216229 game *apiGameFull,217230 full *apiMapFull,218- gameDirectory, mapDirectory string,231+ gameDirectory, mapDirectory, source string,219232 ) error {220233 gamePath := filepath.Join(archiveRoot, gameDirectory, "game.json")221234 entry := map[string]any{@@ -235,6 +248,9 @@ func registerMap(
235248 gameFile["title"] = game.Title236249 gameFile["url"] = game.Config.URL237250 gameFile["icons"] = filepath.ToSlash(filepath.Join(gameDirectory, "icons", "index.json"))251+ if source != "" {252+ gameFile["source"] = source253+ }238254 gameFile["maps"] = upsertByID(gameFile["maps"], entry, full.ID)239255 if err := writeJSON(gamePath, gameFile); err != nil {240256 return err@@ -248,12 +264,16 @@ func registerMap(
248264 }249265 archiveFile = map[string]any{"format": "fmg-git-working-tree"}250266 }251- archiveFile["games"] = upsertByID(archiveFile["games"], map[string]any{267+ gameEntry := map[string]any{252268 "directory": gameDirectory,253269 "id": game.ID,254270 "title": game.Title,255271 "url": game.Config.URL,256- }, game.ID)272+ }273+ if source != "" {274+ gameEntry["source"] = source275+ }276+ archiveFile["games"] = upsertByID(archiveFile["games"], gameEntry, game.ID)257277 return writeJSON(archivePath, archiveFile)258278 }259279 @@ -0,0 +1,484 @@
1+// The IGN mode fills the same archive from IGN's wikimaps. A wikimap page2+// carries its whole configuration in the page itself; the markers come from3+// one GraphQL request; the tiles are a small pyramid asked for level by level.4+// What lands in the archive is a canonical capture -- the ignmap document --5+// stored and versioned exactly the way MapGenie snapshots are.6+package main7+8+import (9+ "context"10+ "crypto/sha256"11+ "encoding/hex"12+ "encoding/json"13+ "errors"14+ "fmt"15+ "hash/fnv"16+ "os"17+ "path/filepath"18+ "strconv"19+ "strings"20+ "sync"21+22+ "github.com/FelineStateMachine/atlas/internal/ignmap"23+)24+25+const ignGraphQL = "https://mollusk.apis.ign.com/graphql"26+27+// ignPageMap is the slice of the page's embedded state a capture is built28+// from. Everything else in __NEXT_DATA__ -- build ids, trace baggage, session29+// scaffolding -- is volatile and never stored.30+type ignPageMap struct {31+ Width float64 `json:"width"`32+ Height float64 `json:"height"`33+ MinZoom int `json:"minZoom"`34+ MaxZoom int `json:"maxZoom"`35+ InitialLat float64 `json:"initialLat"`36+ InitialLng float64 `json:"initialLng"`37+ BackgroundColor string `json:"backgroundColor"`38+ MapName string `json:"mapName"`39+ MapType string `json:"mapType"`40+ ObjectName string `json:"objectName"`41+ MapGenieGameID json.RawMessage `json:"mapGenieGameId"`42+ Tilesets []string `json:"tilesets"`43+ Types []ignPageType `json:"types"`44+}45+46+type ignPageType struct {47+ TypeSlug string `json:"typeSlug"`48+ TypeName string `json:"typeName"`49+ ParentTypeSlug string `json:"parentTypeSlug"`50+ MarkerIcon struct {51+ URL string `json:"url"`52+ Width int `json:"width"`53+ Height int `json:"height"`54+ } `json:"markerIcon"`55+}56+57+func runIGN(ctx context.Context, fetcher *fetcher, o options) error {58+ objectSlug, mapSlug, ok := strings.Cut(o.ign, "/")59+ if !ok || objectSlug == "" || mapSlug == "" {60+ return errors.New("-ign wants objectSlug/mapSlug, as the address bar spells it")61+ }62+ pageURL := fmt.Sprintf("https://www.ign.com/maps/%s/%s", objectSlug, mapSlug)63+64+ page, err := fetchIGNPage(ctx, fetcher, pageURL)65+ if err != nil {66+ return err67+ }68+ // A wikimap page can also front a MapGenie map. That data has its own69+ // crawler, and capturing it here would archive the same source twice under70+ // two names.71+ if page.MapType != "ign" {72+ return fmt.Errorf("%s is not a native IGN wikimap (map type %q)", pageURL, page.MapType)73+ }74+ if id := strings.TrimSpace(string(page.MapGenieGameID)); id != "" && id != "null" {75+ return fmt.Errorf("%s proxies MapGenie game %s; crawl it with -game instead", pageURL, id)76+ }77+ if len(page.Tilesets) == 0 {78+ return fmt.Errorf("%s publishes no tileset", pageURL)79+ }80+ if len(page.Tilesets) > 1 {81+ fmt.Printf(" note: %d tilesets published; capturing the first\n", len(page.Tilesets))82+ }83+ template := page.Tilesets[0]84+ scope := objectSlug + "/" + mapSlug85+ if !strings.Contains(template, "/wikimaps/"+scope+"/") {86+ return fmt.Errorf("tileset %q does not sit under /wikimaps/%s/, and the pipeline groups tiles by that path", template, scope)87+ }88+89+ markers, err := fetchIGNMarkers(ctx, fetcher, objectSlug, mapSlug, page.Types)90+ if err != nil {91+ return err92+ }93+94+ capture := composeIGNCapture(objectSlug, mapSlug, page, markers)95+ raw, err := json.MarshalIndent(capture, "", " ")96+ if err != nil {97+ return err98+ }99+ // A capture that does not translate would poison the archive for both100+ // tools that read it, so it is refused here, while the problem is one101+ // crawl old rather than buried in history.102+ if _, err := ignmap.Translate(raw); err != nil {103+ return fmt.Errorf("capture does not translate: %w", err)104+ }105+106+ gameID := ignArchiveID("ign:" + objectSlug)107+ mapID := ignArchiveID("ign:" + scope)108+ game := &apiGameFull{ID: gameID, Title: capture.GameTitle,109+ Config: apiGameConfig{URL: pageURL}}110+ full := &apiMapFull{ID: mapID, Title: capture.MapTitle, Slug: mapSlug}111+ // The directory says where the capture came from even though the game112+ // keeps its plain title: a MapGenie capture of the same game lives in its113+ // own directory, and the two must never resolve to one.114+ gameDirectory := resolveGameDirectory(o.archive,115+ &apiGameFull{ID: gameID, Title: "IGN " + capture.GameTitle})116+ mapDirectory := resolveMapDirectory(o.archive, gameDirectory, full)117+ mapDir := filepath.Join(o.archive, mapDirectory)118+119+ fmt.Printf("\n== IGN %s / %s (%d markers, %d types)\n",120+ capture.GameTitle, capture.MapTitle, len(capture.Markers), len(capture.Types))121+122+ if o.dryRun {123+ fmt.Printf(" would write capture to %s\n", mapDir)124+ } else if err := writeSnapshotRecord(mapDir, raw, ignmap.Kind, mapID, pageURL); err != nil {125+ return err126+ }127+128+ if err := fetchIGNIcons(ctx, fetcher, o, filepath.Join(o.archive, gameDirectory), page.Types); err != nil {129+ return err130+ }131+132+ index, err := readTileIndex(mapDir)133+ if err != nil {134+ return err135+ }136+ index.mapID = mapID137+ stats, err := captureIGNTiles(ctx, fetcher, o, capture, template, scope, mapDir, index)138+ if err != nil {139+ return err140+ }141+ if o.dryRun {142+ return nil143+ }144+ if err := writeTileIndex(mapDir, index); err != nil {145+ return err146+ }147+ if err := writeMapMetadata(mapDir, game, full, index, len(capture.Markers)); err != nil {148+ return err149+ }150+ if err := registerMap(o.archive, game, full, gameDirectory, mapDirectory, "ign"); err != nil {151+ return err152+ }153+ fmt.Printf(" %d fetched · %d held · %d not published · %d failed · %.1f MB new\n",154+ stats.fetched, stats.skipped, stats.absent, stats.failed, float64(stats.bytes)/1e6)155+ return nil156+}157+158+// fetchIGNPage pulls the map's configuration out of the page. The page embeds159+// its state as one JSON document in a script tag, which is the same data the160+// page's own scripts boot from.161+func fetchIGNPage(ctx context.Context, f *fetcher, pageURL string) (*ignPageMap, error) {162+ body, _, err := f.get(ctx, pageURL)163+ if err != nil {164+ return nil, err165+ }166+ const openTag = `<script id="__NEXT_DATA__" type="application/json">`167+ _, rest, found := strings.Cut(string(body), openTag)168+ if !found {169+ return nil, fmt.Errorf("%s embeds no page state", pageURL)170+ }171+ doc, _, found := strings.Cut(rest, "</script>")172+ if !found {173+ return nil, fmt.Errorf("%s embeds unterminated page state", pageURL)174+ }175+ var page struct {176+ Props struct {177+ PageProps struct {178+ Page struct {179+ Map ignPageMap `json:"map"`180+ } `json:"page"`181+ } `json:"pageProps"`182+ } `json:"props"`183+ }184+ if err := json.Unmarshal([]byte(doc), &page); err != nil {185+ return nil, fmt.Errorf("decode %s page state: %w", pageURL, err)186+ }187+ m := page.Props.PageProps.Page.Map188+ if m.Width <= 0 || m.Height <= 0 || len(m.Types) == 0 {189+ return nil, fmt.Errorf("%s describes no wikimap", pageURL)190+ }191+ return &m, nil192+}193+194+// fetchIGNMarkers asks for every marker of every type in one request, which is195+// how IGN's own page loads a map with everything switched on.196+func fetchIGNMarkers(197+ ctx context.Context,198+ f *fetcher,199+ objectSlug, mapSlug string,200+ types []ignPageType,201+) ([]ignmap.Marker, error) {202+ typeSlugs := make([]string, 0, len(types))203+ for _, t := range types {204+ typeSlugs = append(typeSlugs, t.TypeSlug)205+ }206+ request, err := json.Marshal(map[string]any{207+ "query": `query MapMarkers($objectSlug: String!, $mapSlug: String!, $typeSlugs: [String!]!) {208+ markersMultiple(objectSlug: $objectSlug, mapSlug: $mapSlug, typeSlugs: $typeSlugs) {209+ id lat lng markerName markerSlug typeSlug wikiPage iconSlug regionId checklistTaskId210+ }211+}`,212+ "variables": map[string]any{213+ "objectSlug": objectSlug,214+ "mapSlug": mapSlug,215+ "typeSlugs": typeSlugs,216+ },217+ })218+ if err != nil {219+ return nil, err220+ }221+ body, _, err := f.post(ctx, ignGraphQL, "application/json", request)222+ if err != nil {223+ return nil, err224+ }225+ var response struct {226+ Data struct {227+ MarkersMultiple []ignmap.Marker `json:"markersMultiple"`228+ } `json:"data"`229+ Errors []struct {230+ Message string `json:"message"`231+ } `json:"errors"`232+ }233+ if err := json.Unmarshal(body, &response); err != nil {234+ return nil, fmt.Errorf("decode markers: %w", err)235+ }236+ if len(response.Errors) > 0 {237+ return nil, fmt.Errorf("markers query: %s", response.Errors[0].Message)238+ }239+ if len(response.Data.MarkersMultiple) == 0 {240+ return nil, errors.New("the map answered with no markers at all")241+ }242+ return response.Data.MarkersMultiple, nil243+}244+245+func composeIGNCapture(246+ objectSlug, mapSlug string,247+ page *ignPageMap,248+ markers []ignmap.Marker,249+) *ignmap.Capture {250+ capture := &ignmap.Capture{251+ Source: ignmap.Source,252+ ObjectSlug: objectSlug,253+ MapSlug: mapSlug,254+ GameTitle: page.ObjectName,255+ MapTitle: page.MapName,256+ Map: ignmap.MapConfig{257+ Width: page.Width,258+ Height: page.Height,259+ MinZoom: page.MinZoom,260+ MaxZoom: page.MaxZoom,261+ InitialLat: page.InitialLat,262+ InitialLng: page.InitialLng,263+ BackgroundColor: page.BackgroundColor,264+ Tileset: page.Tilesets[0],265+ },266+ Markers: markers,267+ }268+ if capture.GameTitle == "" {269+ capture.GameTitle = objectSlug270+ }271+ for _, t := range page.Types {272+ capture.Types = append(capture.Types, ignmap.Type{273+ TypeSlug: t.TypeSlug,274+ TypeName: t.TypeName,275+ ParentTypeSlug: t.ParentTypeSlug,276+ IconURL: t.MarkerIcon.URL,277+ IconWidth: t.MarkerIcon.Width,278+ IconHeight: t.MarkerIcon.Height,279+ })280+ }281+ capture.Normalize()282+ return capture283+}284+285+// fetchIGNIcons stores each type's marker icon under the key tools/generate286+// looks icons up by: the category's icon name, which for IGN is the type slug.287+func fetchIGNIcons(288+ ctx context.Context,289+ f *fetcher,290+ o options,291+ gameDir string,292+ types []ignPageType,293+) error {294+ fetched := 0295+ for _, t := range types {296+ if t.MarkerIcon.URL == "" {297+ continue298+ }299+ extension := "png"300+ if dot := strings.LastIndex(t.MarkerIcon.URL, "."); dot >= 0 && dot < len(t.MarkerIcon.URL)-1 {301+ extension = strings.ToLower(t.MarkerIcon.URL[dot+1:])302+ }303+ path := filepath.Join(gameDir, "icons", t.TypeSlug+"."+extension)304+ if _, err := os.Stat(path); err == nil {305+ continue306+ }307+ if o.dryRun {308+ fetched++309+ continue310+ }311+ body, _, err := f.get(ctx, t.MarkerIcon.URL)312+ if err != nil {313+ if errors.Is(err, errAbsent) {314+ fmt.Printf(" icon %s is not published\n", t.TypeSlug)315+ continue316+ }317+ return fmt.Errorf("icon %s: %w", t.TypeSlug, err)318+ }319+ if err := writeFile(path, body); err != nil {320+ return err321+ }322+ fetched++323+ }324+ if fetched > 0 {325+ fmt.Printf(" %d icons\n", fetched)326+ }327+ return nil328+}329+330+// captureIGNTiles takes the whole pyramid level by level. There is no pruning331+// to plan: the deepest level of a wikimap is at most a few thousand tiles, and332+// the deepest level must be complete for the pyramid the bundle is built from,333+// so every tile of every level is asked for once.334+func captureIGNTiles(335+ ctx context.Context,336+ fetcher *fetcher,337+ o options,338+ capture *ignmap.Capture,339+ template, scope, mapDir string,340+ index *tileIndex,341+) (captureStats, error) {342+ var stats captureStats343+ setID := index.tileSetID(scope)344+ extension := ignmap.TileExtension(template)345+ deepest := capture.Map.MaxZoom346+ if o.maxZoom > 0 && o.maxZoom < deepest {347+ deepest = o.maxZoom348+ }349+ fmt.Printf(" layer %q z0-%d\n", scope, deepest)350+351+ for zoom := 0; zoom <= deepest; zoom++ {352+ maxX, maxY := ignmap.LevelExtent(capture.Map.Width, capture.Map.Height, zoom)353+ window := tileWindow{minX: 0, minY: 0, maxX: maxX, maxY: maxY}354+ results, err := fetchIGNLevel(ctx, fetcher, o, template, setID, extension,355+ mapDir, zoom, window.tiles(), index)356+ if err != nil {357+ return stats, err358+ }359+ stats.merge(results)360+ fmt.Printf(" z%-2d %4d fetched · %4d held · %4d not published\n",361+ zoom, results.fetched, results.skipped, results.absent)362+ // The bundle is built from the deepest complete level. Holes there mean363+ // the pyramid tops out shallower than the map is drawn, which deserves364+ // more than a counter in the summary line.365+ if zoom == deepest && results.absent > 0 {366+ fmt.Printf(" WARNING: %d tiles of the deepest level are not published; "+367+ "the bundle will be built from a shallower level\n", results.absent)368+ }369+ }370+ return stats, nil371+}372+373+func fetchIGNLevel(374+ ctx context.Context,375+ fetcher *fetcher,376+ o options,377+ template string,378+ setID int64,379+ extension, mapDir string,380+ zoom int,381+ wanted [][2]int,382+ index *tileIndex,383+) (levelResult, error) {384+ result := levelResult{hashes: make(map[[2]int]string, len(wanted))}385+386+ var mu sync.Mutex387+ var group sync.WaitGroup388+ var failure error389+ gate := make(chan struct{}, o.concurrency)390+391+ replacer := func(x, y int) string {392+ return strings.NewReplacer(393+ "{z}", strconv.Itoa(zoom),394+ "{x}", strconv.Itoa(x),395+ "{y}", strconv.Itoa(y),396+ ).Replace(template)397+ }398+399+ for _, coordinate := range wanted {400+ url := replacer(coordinate[0], coordinate[1])401+ path := filepath.Join(mapDir, "tiles",402+ "set-"+strconv.FormatInt(setID, 10),403+ strconv.Itoa(zoom), strconv.Itoa(coordinate[0]),404+ strconv.Itoa(coordinate[1])+"."+extension)405+406+ if record, held := index.byURL[url]; held {407+ if record.Status == "absent" {408+ result.absent++409+ continue410+ }411+ if record.Status == "cached" {412+ if _, err := os.Stat(path); err == nil {413+ result.skipped++414+ continue415+ }416+ }417+ }418+ if o.dryRun {419+ result.fetched++420+ continue421+ }422+423+ group.Add(1)424+ gate <- struct{}{}425+ go func(coordinate [2]int, url, path string) {426+ defer group.Done()427+ defer func() { <-gate }()428+429+ body, contentType, err := fetcher.get(ctx, url)430+ mu.Lock()431+ defer mu.Unlock()432+ if err != nil {433+ status, message := "failed", err.Error()434+ if errors.Is(err, errAbsent) {435+ status, message = "absent", ""436+ result.absent++437+ } else {438+ result.failed++439+ if failure == nil {440+ failure = err441+ }442+ }443+ index.put(tileRecord{444+ URL: url, Status: status, TileSetID: setID,445+ Zoom: zoom, X: coordinate[0], Y: coordinate[1],446+ ContentType: "application/octet-stream",447+ Error: message,448+ })449+ return450+ }451+ if err := writeFile(path, body); err != nil {452+ failure = err453+ return454+ }455+ index.put(tileRecord{456+ URL: url, Status: "cached", TileSetID: setID,457+ Zoom: zoom, X: coordinate[0], Y: coordinate[1],458+ ContentHash: hashBytes(body), ContentType: contentType,459+ ByteLength: len(body),460+ CoverageKey: fmt.Sprintf("%d:%d:%d:%d:%d",461+ index.mapID, setID, zoom, coordinate[0], coordinate[1]),462+ })463+ result.fetched++464+ result.bytes += int64(len(body))465+ }(coordinate, url, path)466+ }467+ group.Wait()468+ return result, failure469+}470+471+func hashBytes(body []byte) string {472+ sum := sha256.Sum256(body)473+ return hex.EncodeToString(sum[:])474+}475+476+// ignArchiveID numbers IGN games and maps in the archive's registers. The high477+// bit set above the int32 range keeps them apart from every MapGenie id while478+// staying far below the point where a float64 -- which is how the register479+// reads them back -- loses integers.480+func ignArchiveID(seed string) int64 {481+ hash := fnv.New32a()482+ _, _ = hash.Write([]byte(seed))483+ return int64(hash.Sum32()&0x7fffffff) | 1<<32484+}@@ -61,6 +61,7 @@ type options struct {
6161 archive string6262 game string6363 mapTarget string64+ ign string6465 maxZoom int6566 list bool6667 dryRun bool@@ -73,6 +74,7 @@ func parseOptions() options {
7374 flag.StringVar(&o.archive, "archive", "../gamemap/fmg-archive", "FMG archive root")7475 flag.StringVar(&o.game, "game", "", "game slug or id")7576 flag.StringVar(&o.mapTarget, "map", "", "map slug or id (default: every map of the game)")77+ flag.StringVar(&o.ign, "ign", "", "IGN wikimap as objectSlug/mapSlug, e.g. cyberpunk-2077/night-city")7678 flag.IntVar(&o.maxZoom, "max-zoom", 0, "deepest zoom to capture (default: the layer's own maximum)")7779 flag.BoolVar(&o.list, "list", false, "list games, or the maps of -game, and exit")7880 flag.BoolVar(&o.dryRun, "dry-run", false, "report what would be fetched without writing anything")@@ -89,6 +91,10 @@ func run(o options) error {
8991 fetcher := newFetcher(o.concurrency, o.interval)9092 ctx := context.Background()9193 94+ if o.ign != "" {95+ return runIGN(ctx, fetcher, o)96+ }97+9298 games, err := listGames(ctx, fetcher)9399 if err != nil {94100 return err@@ -166,10 +172,10 @@ func crawlMap(
166172 if err := writeTileIndex(mapDir, index); err != nil {167173 return err168174 }169- if err := writeMapMetadata(mapDir, game, full, index); err != nil {175+ if err := writeMapMetadata(mapDir, game, full, index, full.pinCount()); err != nil {170176 return err171177 }172- if err := registerMap(o.archive, game, full, gameDirectory, mapDirectory); err != nil {178+ if err := registerMap(o.archive, game, full, gameDirectory, mapDirectory, ""); err != nil {173179 return err174180 }175181 fmt.Printf(" %d fetched · %d held · %d not published · %d failed · %.1f MB new\n",@@ -144,6 +144,25 @@ func TestTileSetPathOf(t *testing.T) {
144144 if got := tileSetPathOf(url, 13); got != "" {145145 t.Fatalf("tileSetPathOf with the wrong zoom = %q, want empty", got)146146 }147+ // IGN serves its pyramids under /wikimaps/, and a re-crawl has to recover148+ // the same layer path from those URLs or it orphans every captured tile.149+ ign := "https://oyster.ignimgs.com/ignmedia/wikimaps/cyberpunk-2077/night-city/5/12-3.jpg"150+ if got := tileSetPathOf(ign, 5); got != "cyberpunk-2077/night-city" {151+ t.Fatalf("tileSetPathOf(ign) = %q", got)152+ }153+}154+155+func TestIGNArchiveIDStaysApartFromMapGenie(t *testing.T) {156+ id := ignArchiveID("ign:cyberpunk-2077")157+ if id <= 1<<32 {158+ t.Fatalf("id %d could collide with a MapGenie id", id)159+ }160+ if id >= 1<<53 {161+ t.Fatalf("id %d would lose integers read back through a float64", id)162+ }163+ if again := ignArchiveID("ign:cyberpunk-2077"); again != id {164+ t.Fatalf("id is unstable: %d then %d", id, again)165+ }147166 }148167 149168 func TestTileIndexReusesExistingTileSetDirectory(t *testing.T) {