1// Command crawl fills the FMG archive directly from the MapGenie API.2//3// A run names a game, optionally a single map, and captures everything that4// target needs in one pass: the map metadata snapshot and every tile of its5// pyramid. There are no separate phases to run in a particular order, and6// re-running is cheap because anything already archived is skipped.7//8// MapGenie serves this data to every visitor of a map page, but a crawl asks9// for far more of it at once than a person browsing would. Requests are10// therefore spaced out, limited in number, backed off when the server pushes11// back, and never repeated for something already on disk.12package main1314import (15 "context"16 "crypto/sha256"17 "encoding/hex"18 "errors"19 "flag"20 "fmt"21 "os"22 "os/exec"23 "path/filepath"24 "sort"25 "strconv"26 "strings"27 "sync"28 "time"29)3031const (32 apiBase = "https://mapgenie.io/api/v1"33 // MapGenie's CDN answers 403 to requests without a browser user agent.34 userAgent = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) " +35 "AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36"36 // Levels holding at most this many tiles are captured whole. Deeper levels37 // follow only the parents that held content, which is what keeps a z1538 // capture affordable: empty space stays empty at every zoom below it.39 wholeLevelBudget = 102440)4142// lineEnd redraws the progress line in place on a terminal, clearing whatever43// the previous longer line left behind, and leaves ordinary lines when the44// output is being piped into a log.45var lineEnd = func() string {46 info, err := os.Stdout.Stat()47 if err == nil && info.Mode()&os.ModeCharDevice != 0 {48 return "\033[K\r"49 }50 return "\n"51}()5253func main() {54 options := parseOptions()55 if err := run(options); err != nil {56 fmt.Fprintln(os.Stderr, "crawl:", err)57 os.Exit(1)58 }59}6061type options struct {62 archive string63 game string64 mapTarget string65 ign string66 piggyback string67 trek string68 arcgis string69 captureDate string70 maxZoom int71 list bool72 dryRun bool73 concurrency int74 interval time.Duration75}7677func parseOptions() options {78 var o options79 flag.StringVar(&o.archive, "archive", "../gamemap/fmg-archive", "FMG archive root")80 flag.StringVar(&o.game, "game", "", "game slug or id")81 flag.StringVar(&o.mapTarget, "map", "", "map slug or id (default: every map of the game)")82 flag.StringVar(&o.ign, "ign", "", "IGN wikimap as objectSlug/mapSlug, e.g. cyberpunk-2077/night-city")83 flag.StringVar(&o.piggyback, "piggyback", "", "Piggyback map as gameSlug/mapSlug, e.g. cyberpunk-2077/night-city")84 flag.StringVar(&o.trek, "trek", "", "planetary body from NASA Trek and the IAU Gazetteer, e.g. mars")85 flag.StringVar(&o.arcgis, "arcgis", "", "city from an ArcGIS Hub open-data site, e.g. bend-or")86 flag.StringVar(&o.captureDate, "capture-date", "", "the day an -arcgis capture answers to, YYYY-MM-DD (default: today)")87 flag.IntVar(&o.maxZoom, "max-zoom", 0, "deepest zoom to capture (default: the layer's own maximum)")88 flag.BoolVar(&o.list, "list", false, "list games, or the maps of -game, and exit")89 flag.BoolVar(&o.dryRun, "dry-run", false, "report what would be fetched without writing anything")90 flag.IntVar(&o.concurrency, "concurrency", 4, "concurrent requests")91 flag.DurationVar(&o.interval, "interval", 150*time.Millisecond, "minimum spacing between requests")92 flag.Parse()93 return o94}9596func run(o options) error {97 if o.concurrency < 1 {98 return errors.New("-concurrency must be at least 1")99 }100 fetcher := newFetcher(o.concurrency, o.interval)101 ctx := context.Background()102103 if o.ign != "" {104 return runIGN(ctx, fetcher, o)105 }106 if o.piggyback != "" {107 return runPB(ctx, fetcher, o)108 }109 if o.trek != "" {110 return runTrek(ctx, fetcher, o)111 }112 if o.arcgis != "" {113 return runArcgis(ctx, fetcher, o)114 }115116 games, err := listGames(ctx, fetcher)117 if err != nil {118 return err119 }120 if o.game == "" {121 if !o.list {122 return errors.New("-game is required; use -list to see what is available")123 }124 for _, game := range games {125 fmt.Printf("%-38s id=%d\n", game.Slug, game.ID)126 }127 return nil128 }129130 game, ok := findGame(games, o.game)131 if !ok {132 return fmt.Errorf("no game matches %q; use -list to see what is available", o.game)133 }134 targets, err := selectMaps(game, o.mapTarget)135 if err != nil {136 return err137 }138 if o.list {139 for _, target := range targets {140 fmt.Printf("%-38s id=%d %s\n", target.Slug, target.ID, target.Title)141 }142 return nil143 }144145 for _, target := range targets {146 if err := crawlMap(ctx, fetcher, o, game, target); err != nil {147 return fmt.Errorf("%s / %s: %w", game.Title, target.Title, err)148 }149 }150 if !o.dryRun {151 renderIcons(ctx, o, game)152 }153 return nil154}155156// renderIcons resolves the game's icon font into the archive once its maps157// are captured. MapGenie ships icons as a font behind a stylesheet, which the158// Node script under tools/icons untangles; it is quick, idempotent, and was159// forgotten by hand often enough that the crawl now runs it itself. A machine160// without node is told what it is missing rather than failed: the tiles and161// pins are already safely on disk, and icons can be rendered later.162func renderIcons(ctx context.Context, o options, game *apiGameFull) {163 script := filepath.Join("tools", "icons", "render-icons.mjs")164 if _, err := os.Stat(script); err != nil {165 fmt.Printf(" icons not rendered (no %s here); run it from the atlas checkout:\n"+166 " node %s --archive %s --game %s\n", script, script, o.archive, game.Slug)167 return168 }169 command := exec.CommandContext(ctx, "node", script,170 "--archive", o.archive, "--game", game.Slug)171 command.Stdout = os.Stdout172 command.Stderr = os.Stderr173 if err := command.Run(); err != nil {174 fmt.Printf(" icons not rendered: %v; run when ready:\n"+175 " node %s --archive %s --game %s\n", err, script, o.archive, game.Slug)176 }177}178179// crawlMap captures one map completely: its metadata snapshot, then its tiles.180// Both are idempotent, so an interrupted run resumes by simply being repeated.181func crawlMap(182 ctx context.Context,183 fetcher *fetcher,184 o options,185 game *apiGameFull,186 target apiMapRef,187) error {188 fmt.Printf("\n== %s / %s (map %d)\n", game.Title, target.Title, target.ID)189190 full, raw, err := fetchMap(ctx, fetcher, target.ID)191 if err != nil {192 return err193 }194 gameDirectory := resolveGameDirectory(o.archive, game)195 mapDirectory := resolveMapDirectory(o.archive, gameDirectory, full)196 mapDir := filepath.Join(o.archive, mapDirectory)197198 if o.dryRun {199 fmt.Printf(" would write snapshot to %s\n", mapDir)200 } else if err := writeSnapshot(mapDir, full, raw); err != nil {201 return err202 }203204 index, err := readTileIndex(mapDir)205 if err != nil {206 return err207 }208 index.mapID = full.ID209 stats, err := captureTiles(ctx, fetcher, o, game, full, mapDir, index)210 if err != nil {211 return err212 }213 if o.dryRun {214 return nil215 }216 if err := writeTileIndex(mapDir, index); err != nil {217 return err218 }219 if err := writeMapMetadata(mapDir, game, full, index, full.pinCount()); err != nil {220 return err221 }222 if err := registerMap(o.archive, game, full, gameDirectory, mapDirectory, ""); err != nil {223 return err224 }225 fmt.Printf(" %d fetched · %d held · %d not published · %d failed · %.1f MB new\n",226 stats.fetched, stats.skipped, stats.absent, stats.failed, float64(stats.bytes)/1e6)227 return nil228}229230// captureTiles walks each layer from its shallowest zoom down. Levels small231// enough to take whole are taken whole; past that only the children of tiles232// that held content are requested, so blank regions cost nothing at depth.233func captureTiles(234 ctx context.Context,235 fetcher *fetcher,236 o options,237 game *apiGameFull,238 full *apiMapFull,239 mapDir string,240 index *tileIndex,241) (captureStats, error) {242 var stats captureStats243 base := game.Config.TilesBaseURL244 if base == "" {245 base = "https://tiles.mapgenie.io"246 }247248 for _, set := range full.Config.TileSets {249 setID := index.tileSetID(set.Path)250 deepest := set.MaxZoom251 if o.maxZoom > 0 && o.maxZoom < deepest {252 deepest = o.maxZoom253 }254 fmt.Printf(" layer %q (%s) z%d-%d\n", set.Name, set.Path, set.MinZoom, deepest)255256 var live map[[2]int]bool257 var background string258 transposed, probed := false, false259 for zoom := set.MinZoom; zoom <= deepest; zoom++ {260 window, ok := set.window(zoom)261 if !ok {262 var err error263 window, ok, err = unpublishedWindow(264 ctx, fetcher, strings.TrimSuffix(base, "/"), set, full, zoom, live,265 &transposed, &probed,266 )267 if err != nil {268 return stats, err269 }270 if !ok {271 fmt.Printf(" z%-2d skipped: no window published, and none found\n", zoom)272 live = nil273 continue274 }275 }276 // A window one tile across settles nothing about which way round the277 // axes are read, so the question waits for a level wide enough to278 // answer it rather than being marked answered by default.279 if !probed && window.maxX > window.minX && window.maxY > window.minY {280 extension := tileExtension(set)281 order, err := probeAxisOrder(ctx, fetcher, strings.TrimSuffix(base, "/"), set, zoom, window, extension)282 if err != nil {283 return stats, err284 }285 transposed, probed = order, true286 if transposed {287 fmt.Printf(" axes are published transposed; reading /{zoom}/{y}/{x}\n")288 }289 }290291 wanted := window.tiles()292 whole := len(wanted) <= wholeLevelBudget || live == nil293 if !whole {294 wanted = childrenOf(live, window)295 }296 if len(wanted) == 0 {297 fmt.Printf(" z%-2d nothing left to follow\n", zoom)298 break299 }300301 results, err := fetchLevel(ctx, fetcher, o, set, setID, base, mapDir, zoom, wanted, index, transposed)302 if err != nil {303 return stats, err304 }305 stats.merge(results)306307 // Each level re-encodes its own background tile, so the hash is308 // re-read from every level captured whole. It is never taken from a309 // pruned level, whose handful of real tiles could otherwise vote310 // themselves away, and a level that yields no answer keeps the last311 // one rather than losing it.312 if whole {313 if found := dominantHash(results.hashes); found != "" {314 background = found315 }316 }317 live = liveTiles(results.hashes, background)318 fmt.Printf(" z%-2d %4d fetched · %4d held · %4d not published · %4d carry content\n",319 zoom, results.fetched, results.skipped, results.absent, len(live))320 if background != "" && len(live) == 0 {321 break322 }323 }324 }325 return stats, nil326}327328// unpublishedWindow stands in for the window a layer never published. Below the329// first level captured there is nothing to search for: the tiles that held330// content name where their children can be. The first level is searched for331// outward from the point the map opens on, and finding it there also settles332// which way round the axes are read, which no single tile could have said.333func unpublishedWindow(334 ctx context.Context,335 fetcher *fetcher,336 base string,337 set apiTileSet,338 full *apiMapFull,339 zoom int,340 live map[[2]int]bool,341 transposed, probed *bool,342) (tileWindow, bool, error) {343 if len(live) > 0 {344 window, ok := childWindow(live)345 return window, ok, nil346 }347 centre, ok := startTile(full, zoom)348 if !ok {349 return tileWindow{}, false, nil350 }351 found, err := searchWindow(ctx, fetcher, base, set, zoom, centre, tileExtension(set))352 if err != nil || !found.found {353 return tileWindow{}, false, err354 }355 fmt.Printf(" z%-2d no window published; found x %d-%d y %d-%d from %d/%d\n",356 zoom, found.window.minX, found.window.maxX, found.window.minY, found.window.maxY,357 centre[0], centre[1])358 if found.transposed {359 fmt.Printf(" axes are published transposed; reading /{zoom}/{y}/{x}\n")360 *transposed, *probed = true, true361 }362 return found.window, true, nil363}364365func fetchLevel(366 ctx context.Context,367 fetcher *fetcher,368 o options,369 set apiTileSet,370 setID int64,371 base, mapDir string,372 zoom int,373 wanted [][2]int,374 index *tileIndex,375 transposed bool,376) (levelResult, error) {377 result := levelResult{hashes: make(map[[2]int]string, len(wanted))}378 extension := tileExtension(set)379380 var mu sync.Mutex381 var group sync.WaitGroup382 var failure error383 gate := make(chan struct{}, o.concurrency)384385 // A deep level is tens of thousands of tiles and takes hours. Report as it386 // goes, so the run can be watched rather than waited on. Held under mu with387 // the counters it reads.388 started := time.Now()389 lastReport := started390 report := func(done bool) {391 if !done && (time.Since(lastReport) < 15*time.Second) {392 return393 }394 lastReport = time.Now()395 seen := result.fetched + result.skipped + result.absent + result.failed396 if seen == 0 {397 return398 }399 elapsed := time.Since(started)400 remaining := time.Duration(0)401 if seen < len(wanted) {402 remaining = time.Duration(float64(elapsed) / float64(seen) * float64(len(wanted)-seen))403 }404 fmt.Printf(" z%-2d %6d/%d %5.1f MB %s elapsed %s left%s",405 zoom, seen, len(wanted), float64(result.bytes)/1e6,406 elapsed.Truncate(time.Second), remaining.Truncate(time.Second), lineEnd)407 }408409 for _, coordinate := range wanted {410 // A transposed layer names the row first. The tile is still stored at411 // its true position, so everything downstream stays in one layout.412 first, second := coordinate[0], coordinate[1]413 if transposed {414 first, second = second, first415 }416 url := fmt.Sprintf("%s/games/%s/%d/%d/%d.%s",417 strings.TrimSuffix(base, "/"), set.Path, zoom, first, second, extension)418 path := filepath.Join(mapDir, "tiles",419 "set-"+strconv.FormatInt(setID, 10),420 strconv.Itoa(zoom), strconv.Itoa(coordinate[0]),421 strconv.Itoa(coordinate[1])+"."+extension)422423 // Resume: a tile already recorded and present on disk is never asked424 // for a second time.425 if record, held := index.byURL[url]; held {426 if record.Status == "absent" {427 continue428 }429 if record.Status == "cached" {430 if _, err := os.Stat(path); err == nil {431 mu.Lock()432 result.skipped++433 result.hashes[coordinate] = record.ContentHash434 mu.Unlock()435 continue436 }437 }438 }439 if o.dryRun {440 mu.Lock()441 result.fetched++442 mu.Unlock()443 continue444 }445446 group.Add(1)447 gate <- struct{}{}448 go func(coordinate [2]int, url, path string) {449 defer group.Done()450 defer func() { <-gate }()451452 body, contentType, err := fetcher.get(ctx, url)453 mu.Lock()454 defer mu.Unlock()455 if err != nil {456 status, message := "failed", err.Error()457 if errors.Is(err, errAbsent) {458 // Recorded so a later run does not ask again.459 status, message = "absent", ""460 result.absent++461 } else {462 result.failed++463 if failure == nil {464 failure = err465 }466 }467 report(false)468 index.put(tileRecord{469 URL: url, Status: status, TileSetID: setID,470 Zoom: zoom, X: coordinate[0], Y: coordinate[1],471 ContentType: "application/octet-stream",472 Error: message,473 })474 return475 }476 if err := writeFile(path, body); err != nil {477 failure = err478 return479 }480 sum := sha256.Sum256(body)481 hash := hex.EncodeToString(sum[:])482 index.put(tileRecord{483 URL: url, Status: "cached", TileSetID: setID,484 Zoom: zoom, X: coordinate[0], Y: coordinate[1],485 ContentHash: hash, ContentType: contentType,486 ByteLength: len(body),487 CoverageKey: fmt.Sprintf("%d:%d:%d:%d:%d",488 index.mapID, setID, zoom, coordinate[0], coordinate[1]),489 })490 result.fetched++491 result.bytes += int64(len(body))492 result.hashes[coordinate] = hash493 report(false)494 }(coordinate, url, path)495 }496 group.Wait()497 mu.Lock()498 if result.fetched > 0 && lineEnd != "\n" {499 report(true)500 fmt.Println()501 }502 mu.Unlock()503 return result, failure504}505506type levelResult struct {507 fetched int508 skipped int509 absent int510 failed int511 bytes int64512 hashes map[[2]int]string513}514515type captureStats struct {516 fetched int517 skipped int518 absent int519 failed int520 bytes int64521}522523func (s *captureStats) merge(r levelResult) {524 s.fetched += r.fetched525 s.skipped += r.skipped526 s.absent += r.absent527 s.failed += r.failed528 s.bytes += r.bytes529}530531// backgroundSampleFloor is the smallest level worth judging. The top of a532// pyramid is a handful of tiles showing the whole map, where "repeated over533// half the level" means nothing.534const backgroundSampleFloor = 16535536// dominantHash reports the tile a level uses for empty space. Two tiles of real537// map are never byte-identical, so any tile repeated across a level is the538// background, however small a share of the level it occupies -- a map whose539// content fills most of its bounds still has a background in the corners.540//541// A level whose tiles are *all* identical says nothing: there is no content to542// contrast against, so the question is left to a deeper level.543func dominantHash(hashes map[[2]int]string) string {544 if len(hashes) < backgroundSampleFloor {545 return ""546 }547 counts := make(map[string]int, len(hashes))548 var best string549 for _, hash := range hashes {550 if hash == "" {551 continue552 }553 counts[hash]++554 if counts[hash] > counts[best] {555 best = hash556 }557 }558 repeats := max(8, len(hashes)/20)559 if best == "" || counts[best] < repeats || counts[best] == len(hashes) {560 return ""561 }562 return best563}564565func liveTiles(hashes map[[2]int]string, background string) map[[2]int]bool {566 live := make(map[[2]int]bool, len(hashes))567 for coordinate, hash := range hashes {568 if hash == "" || (background != "" && hash == background) {569 continue570 }571 live[coordinate] = true572 }573 return live574}575576func childrenOf(live map[[2]int]bool, window tileWindow) [][2]int {577 children := make([][2]int, 0, len(live)*4)578 for parent := range live {579 for dy := 0; dy < 2; dy++ {580 for dx := 0; dx < 2; dx++ {581 child := [2]int{parent[0]*2 + dx, parent[1]*2 + dy}582 if window.holds(child) {583 children = append(children, child)584 }585 }586 }587 }588 sortCoordinates(children)589 return children590}591592func tileExtension(set apiTileSet) string {593 extension := strings.TrimPrefix(strings.ToLower(set.Extension), ".")594 if extension == "" {595 return "jpg"596 }597 return extension598}599600func sortCoordinates(coordinates [][2]int) {601 sort.Slice(coordinates, func(i, j int) bool {602 if coordinates[i][1] != coordinates[j][1] {603 return coordinates[i][1] < coordinates[j][1]604 }605 return coordinates[i][0] < coordinates[j][0]606 })607}608609type tileWindow struct{ minX, minY, maxX, maxY int }610611func (w tileWindow) holds(c [2]int) bool {612 return c[0] >= w.minX && c[0] <= w.maxX && c[1] >= w.minY && c[1] <= w.maxY613}614615func (w tileWindow) tiles() [][2]int {616 out := make([][2]int, 0, (w.maxX-w.minX+1)*(w.maxY-w.minY+1))617 for y := w.minY; y <= w.maxY; y++ {618 for x := w.minX; x <= w.maxX; x++ {619 out = append(out, [2]int{x, y})620 }621 }622 return out623}624625func (s apiTileSet) window(zoom int) (tileWindow, bool) {626 bounds, ok := s.Bounds[strconv.Itoa(zoom)]627 if !ok {628 return tileWindow{}, false629 }630 return tileWindow{631 minX: bounds.X.Min, minY: bounds.Y.Min,632 maxX: bounds.X.Max, maxY: bounds.Y.Max,633 }, true634}