The archive was filled by a Chrome extension that drove a MapGenie map
around in a sweep tab and captured whatever tiles the page happened to
request. That is lossy against an exact tile grid -- L.A. Noire holds 523
of 4096 tiles at z14 -- and it caps how deep a capture can go, since sweep
positions grow fourfold per level.
Neither half of the data needs a browser. Tiles are public: a request with
no headers at all returns them. Map metadata comes from /api/v1/maps/{id}/full,
which is gated only by a user-agent filter; across ten maps the anonymous
response carries the same locations the extension archived while signed in.
tools/crawl names a game, optionally one map, and captures both in a single
pass. There are no phases to run in order and no state to keep between runs:
anything already on disk is skipped, so an interrupted crawl resumes by being
repeated. Tiles are enumerated from the layer's published bounds rather than
hoped for, and past the point where a level is small enough to take whole,
only the children of tiles that held content are requested. Empty space stays
empty at every zoom below it, so L.A. Noire reaches z14 in 652 requests
instead of 4096.
A crawl asks for far more at once than a person browsing would, so requests
are spaced out, capped in flight, and backed off when the server pushes back.
The origin answers 403 rather than 404 for a tile it does not hold, and the
published bounds are a box that content rarely fills exactly, so both are
recorded as absent rather than treated as failures.
Category glyphs stay in Node. MapGenie ships them as an icon font, and
resolving key -> codepoint -> outline runs once per game rather than per tile.
The stylesheet also describes layered icons, whose glyphs stack with their own
colours; Atlas draws these as background images, where currentColor has
nothing to resolve against, so the legend colour is carried on the root.
Also fixes contentBoundsFor, which scaled bounds up for levels above the
reference zoom but never down for levels below it. Nothing reached those
levels until deep capture worked, and the error is invisible -- the raster
layer clips to a larger extent than it should rather than a smaller one.- commit
- c3f7b509e62ef5071e6b4f677872c5c68e0ba9b2
- parent
- 4047acee4997c97a4d57875497db4014db772356
- author
- Dami <dami.etoile@gmail.com>
| M | .gitignore | 1 | + |
| M | main.go | 7 | + |
| A | tools/ | 285 | ++++++++++++++++++ |
| A | tools/ | 371 | ++++++++++++++++++++++++ |
| A | tools/ | 466 | ++++++++++++++++++++++++++++++ |
| A | tools/ | 191 | ++++++++++++ |
| A | tools/ | 294 | +++++++++++++++++++ |
| M | tools/ | 17 | +- |
@@ -15,6 +15,13 @@ import (
1515 "github.com/wailsapp/wails/v2/pkg/options"1616 )1717 18+// Archive capture is deliberately not a generate step: it reaches out to19+// MapGenie and takes as long as the chosen depth demands. Run it by hand for20+// the game or map you want, then regenerate.21+//22+// go run ./tools/crawl -game skyrim -map solstheim -max-zoom 1523+// node tools/icons/render-icons.mjs --game skyrim24+1825 //go:generate go run ./tools/tiles -source ../gamemap/fmg-archive -output assets/tiles1926 //go:generate go run ./tools/generate -source ../gamemap -tiles assets/tiles/index.json -output assets/catalog.json2027 //go:generate npm --prefix frontend ci@@ -0,0 +1,285 @@
1+package main2+3+import (4+ "context"5+ "encoding/json"6+ "errors"7+ "fmt"8+ "io"9+ "math"10+ "net/http"11+ "strconv"12+ "strings"13+ "sync"14+ "time"15+)16+17+// errAbsent marks a tile the CDN does not hold. The published bounds of a layer18+// are a bounding box, and content rarely fills it exactly; the origin answers19+// 403 rather than 404 for an object that is not there, so both mean the same20+// thing here. An absent tile is a normal result, not a failure.21+var errAbsent = errors.New("not published")22+23+// fetcher spaces every request out by a fixed interval and caps how many are24+// in flight, so a crawl of tens of thousands of tiles stays a light load25+// rather than a burst. It backs off when the server asks it to.26+type fetcher struct {27+ http *http.Client28+ interval time.Duration29+30+ mu sync.Mutex31+ next time.Time32+}33+34+func newFetcher(concurrency int, interval time.Duration) *fetcher {35+ return &fetcher{36+ http: &http.Client{37+ Timeout: 60 * time.Second,38+ Transport: &http.Transport{39+ MaxIdleConnsPerHost: concurrency,40+ },41+ },42+ interval: interval,43+ }44+}45+46+// wait returns once this caller owns the next slot in the request schedule.47+func (f *fetcher) wait(ctx context.Context) error {48+ f.mu.Lock()49+ now := time.Now()50+ slot := f.next51+ if slot.Before(now) {52+ slot = now53+ }54+ f.next = slot.Add(f.interval)55+ f.mu.Unlock()56+57+ delay := time.Until(slot)58+ if delay <= 0 {59+ return nil60+ }61+ timer := time.NewTimer(delay)62+ defer timer.Stop()63+ select {64+ case <-timer.C:65+ return nil66+ case <-ctx.Done():67+ return ctx.Err()68+ }69+}70+71+// penalise pushes the whole schedule back, so a 429 slows every worker rather72+// than just the one that saw it.73+func (f *fetcher) penalise(d time.Duration) {74+ f.mu.Lock()75+ defer f.mu.Unlock()76+ if until := time.Now().Add(d); until.After(f.next) {77+ f.next = until78+ }79+}80+81+const maxAttempts = 482+83+func (f *fetcher) get(ctx context.Context, url string) ([]byte, string, error) {84+ var lastErr error85+ for attempt := range maxAttempts {86+ if attempt > 0 {87+ backoff := time.Duration(math.Pow(2, float64(attempt))) * time.Second88+ f.penalise(backoff)89+ }90+ if err := f.wait(ctx); err != nil {91+ return nil, "", err92+ }93+94+ request, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)95+ if err != nil {96+ return nil, "", err97+ }98+ request.Header.Set("User-Agent", userAgent)99+ request.Header.Set("Accept", "*/*")100+101+ response, err := f.http.Do(request)102+ if err != nil {103+ lastErr = err104+ continue105+ }106+ body, readErr := io.ReadAll(response.Body)107+ response.Body.Close()108+109+ switch {110+ case response.StatusCode == http.StatusNotFound,111+ response.StatusCode == http.StatusForbidden:112+ return nil, "", fmt.Errorf("%w: %s", errAbsent, url)113+ case response.StatusCode == http.StatusTooManyRequests ||114+ response.StatusCode >= 500:115+ f.penalise(retryAfter(response, time.Duration(attempt+1)*2*time.Second))116+ lastErr = fmt.Errorf("HTTP %d from %s", response.StatusCode, url)117+ continue118+ case response.StatusCode != http.StatusOK:119+ return nil, "", fmt.Errorf("HTTP %d from %s", response.StatusCode, url)120+ case readErr != nil:121+ lastErr = readErr122+ continue123+ }124+ return body, response.Header.Get("Content-Type"), nil125+ }126+ return nil, "", lastErr127+}128+129+func retryAfter(response *http.Response, fallback time.Duration) time.Duration {130+ if value := response.Header.Get("Retry-After"); value != "" {131+ if seconds, err := strconv.Atoi(strings.TrimSpace(value)); err == nil && seconds >= 0 {132+ return time.Duration(seconds) * time.Second133+ }134+ }135+ return fallback136+}137+138+func (f *fetcher) getJSON(ctx context.Context, url string, into any) ([]byte, error) {139+ body, _, err := f.get(ctx, url)140+ if err != nil {141+ return nil, err142+ }143+ if err := json.Unmarshal(body, into); err != nil {144+ return nil, fmt.Errorf("decode %s: %w", url, err)145+ }146+ return body, nil147+}148+149+type apiGameConfig struct {150+ TilesBaseURL string `json:"tiles_base_url"`151+ CDNURL string `json:"cdn_url"`152+ URL string `json:"url"`153+}154+155+// apiGameFull is what /games already returns for every game: identity, config,156+// and the full map list. The /games/{id}/full endpoint requires an account, and157+// carries nothing extra that a crawl needs.158+type apiGameFull struct {159+ ID int64 `json:"id"`160+ Title string `json:"title"`161+ Slug string `json:"slug"`162+ Config apiGameConfig `json:"config"`163+ Maps []apiMapRef `json:"maps"`164+}165+166+type apiMapRef struct {167+ ID int64 `json:"id"`168+ Title string `json:"title"`169+ Slug string `json:"slug"`170+}171+172+type apiRange struct {173+ Min int `json:"min"`174+ Max int `json:"max"`175+}176+177+type apiBound struct {178+ X apiRange `json:"x"`179+ Y apiRange `json:"y"`180+}181+182+type apiTileSet struct {183+ Name string `json:"name"`184+ Path string `json:"path"`185+ MinZoom int `json:"min_zoom"`186+ MaxZoom int `json:"max_zoom"`187+ Extension string `json:"extension"`188+ Bounds map[string]apiBound `json:"bounds"`189+}190+191+type apiMapFull struct {192+ ID int64 `json:"id"`193+ GameID int64 `json:"game_id"`194+ Title string `json:"title"`195+ Slug string `json:"slug"`196+ URL string `json:"url"`197+ Config struct {198+ TileSets []apiTileSet `json:"tile_sets"`199+ } `json:"config"`200+ Groups []struct {201+ Categories []struct {202+ Locations []json.RawMessage `json:"locations"`203+ } `json:"categories"`204+ } `json:"groups"`205+}206+207+func (m *apiMapFull) pinCount() int {208+ total := 0209+ for _, group := range m.Groups {210+ for _, category := range group.Categories {211+ total += len(category.Locations)212+ }213+ }214+ return total215+}216+217+func listGames(ctx context.Context, f *fetcher) ([]apiGameFull, error) {218+ var games []apiGameFull219+ if _, err := f.getJSON(ctx, apiBase+"/games", &games); err != nil {220+ return nil, err221+ }222+ return games, nil223+}224+225+func findGame(games []apiGameFull, target string) (*apiGameFull, bool) {226+ if id, err := strconv.ParseInt(target, 10, 64); err == nil {227+ for index := range games {228+ if games[index].ID == id {229+ return &games[index], true230+ }231+ }232+ }233+ target = strings.ToLower(target)234+ for index := range games {235+ if strings.ToLower(games[index].Slug) == target {236+ return &games[index], true237+ }238+ }239+ for index := range games {240+ if strings.Contains(strings.ToLower(games[index].Title), target) {241+ return &games[index], true242+ }243+ }244+ return nil, false245+}246+247+// fetchMap returns both the decoded map and its exact response bytes: the248+// archive stores what the server actually sent, hashed as received.249+func fetchMap(ctx context.Context, f *fetcher, id int64) (*apiMapFull, []byte, error) {250+ var full apiMapFull251+ url := fmt.Sprintf("%s/maps/%d/full", apiBase, id)252+ raw, err := f.getJSON(ctx, url, &full)253+ if err != nil {254+ return nil, nil, err255+ }256+ return &full, raw, nil257+}258+259+func selectMaps(game *apiGameFull, target string) ([]apiMapRef, error) {260+ if target == "" {261+ if len(game.Maps) == 0 {262+ return nil, errors.New("game publishes no maps")263+ }264+ return game.Maps, nil265+ }266+ if id, err := strconv.ParseInt(target, 10, 64); err == nil {267+ for _, candidate := range game.Maps {268+ if candidate.ID == id {269+ return []apiMapRef{candidate}, nil270+ }271+ }272+ }273+ lowered := strings.ToLower(target)274+ for _, candidate := range game.Maps {275+ if strings.ToLower(candidate.Slug) == lowered {276+ return []apiMapRef{candidate}, nil277+ }278+ }279+ available := make([]string, 0, len(game.Maps))280+ for _, candidate := range game.Maps {281+ available = append(available, candidate.Slug)282+ }283+ return nil, fmt.Errorf("no map matches %q; available: %s",284+ target, strings.Join(available, ", "))285+}@@ -0,0 +1,371 @@
1+package main2+3+import (4+ "crypto/sha256"5+ "encoding/hex"6+ "encoding/json"7+ "errors"8+ "fmt"9+ "hash/fnv"10+ "os"11+ "path/filepath"12+ "sort"13+ "strconv"14+ "strings"15+ "time"16+)17+18+// The archive layout is the one the FMG crawl dashboard produced, so tools/tiles19+// and tools/generate read a crawled archive without knowing the difference.20+21+type tileRecord struct {22+ ByteLength int `json:"byteLength"`23+ ContentHash string `json:"contentHash"`24+ ContentType string `json:"contentType"`25+ CoverageKey string `json:"coverageKey,omitempty"`26+ Error string `json:"error,omitempty"`27+ Status string `json:"status"`28+ TileSetID int64 `json:"tileSetId,omitempty"`29+ URL string `json:"url"`30+ X int `json:"x,omitempty"`31+ Y int `json:"y,omitempty"`32+ Zoom int `json:"zoom,omitempty"`33+}34+35+type snapshotRecord struct {36+ CapturedAt string `json:"capturedAt"`37+ ContentHash string `json:"contentHash"`38+ Kind string `json:"kind"`39+ SourceID int64 `json:"sourceId"`40+ SourceURL string `json:"sourceUrl"`41+}42+43+// tileIndex is the working set of tiles/index.json: keyed by URL for resume44+// checks, and remembering which directory each layer's tiles live under.45+type tileIndex struct {46+ mapID int6447+ byURL map[string]tileRecord48+ order []string49+ setIDs map[string]int6450+ usedIDs map[int64]bool51+}52+53+func newTileIndex(mapID int64) *tileIndex {54+ return &tileIndex{55+ mapID: mapID,56+ byURL: make(map[string]tileRecord),57+ setIDs: make(map[string]int64),58+ usedIDs: make(map[int64]bool),59+ }60+}61+62+func (i *tileIndex) put(record tileRecord) {63+ if _, seen := i.byURL[record.URL]; !seen {64+ i.order = append(i.order, record.URL)65+ }66+ i.byURL[record.URL] = record67+}68+69+func (i *tileIndex) records() []tileRecord {70+ out := make([]tileRecord, 0, len(i.order))71+ for _, url := range i.order {72+ out = append(out, i.byURL[url])73+ }74+ return out75+}76+77+// tileSetID keeps the directory a layer already uses, so re-crawling an archive78+// captured by the extension does not orphan its files. New layers get a stable79+// id derived from their path.80+func (i *tileIndex) tileSetID(path string) int64 {81+ if id, ok := i.setIDs[path]; ok {82+ return id83+ }84+ hash := fnv.New32a()85+ _, _ = hash.Write([]byte(path))86+ id := int64(hash.Sum32()%9000) + 100087+ for i.usedIDs[id] {88+ id++89+ }90+ i.setIDs[path] = id91+ i.usedIDs[id] = true92+ return id93+}94+95+func readTileIndex(mapDir string) (*tileIndex, error) {96+ index := newTileIndex(0)97+ var records []tileRecord98+ if err := readJSON(filepath.Join(mapDir, "tiles", "index.json"), &records); err != nil {99+ if errors.Is(err, os.ErrNotExist) {100+ return index, nil101+ }102+ return nil, err103+ }104+ for _, record := range records {105+ index.put(record)106+ if record.TileSetID == 0 || record.Zoom == 0 {107+ continue108+ }109+ if path := tileSetPathOf(record.URL, record.Zoom); path != "" {110+ index.setIDs[path] = record.TileSetID111+ index.usedIDs[record.TileSetID] = true112+ }113+ }114+ return index, nil115+}116+117+// tileSetPathOf recovers the layer path from a tile URL, matching how118+// tools/tiles groups levels: everything between "/games/" and "/<zoom>/".119+func tileSetPathOf(url string, zoom int) string {120+ const marker = "/games/"121+ start := strings.Index(url, marker)122+ if start < 0 {123+ return ""124+ }125+ rest := url[start+len(marker):]126+ end := strings.Index(rest, "/"+strconv.Itoa(zoom)+"/")127+ if end < 0 {128+ return ""129+ }130+ return rest[:end]131+}132+133+func writeTileIndex(mapDir string, index *tileIndex) error {134+ records := index.records()135+ sort.SliceStable(records, func(a, b int) bool { return records[a].URL < records[b].URL })136+ return writeJSON(filepath.Join(mapDir, "tiles", "index.json"), records)137+}138+139+// writeSnapshot stores the response exactly as received, addressed by its hash,140+// and records it only if that content is new.141+func writeSnapshot(mapDir string, full *apiMapFull, raw []byte) error {142+ sum := sha256.Sum256(raw)143+ hash := hex.EncodeToString(sum[:])144+145+ path := filepath.Join(mapDir, "snapshots", "map", hash+".json")146+ if err := writeFile(path, raw); err != nil {147+ return err148+ }149+150+ indexPath := filepath.Join(mapDir, "snapshots", "index.json")151+ var snapshots []snapshotRecord152+ if err := readJSON(indexPath, &snapshots); err != nil && !errors.Is(err, os.ErrNotExist) {153+ return err154+ }155+ for _, snapshot := range snapshots {156+ if snapshot.ContentHash == hash {157+ return nil158+ }159+ }160+ snapshots = append(snapshots, snapshotRecord{161+ CapturedAt: time.Now().UTC().Format(time.RFC3339Nano),162+ ContentHash: hash,163+ Kind: "map",164+ SourceID: full.ID,165+ SourceURL: fmt.Sprintf("/api/v1/maps/%d/full", full.ID),166+ })167+ sort.Slice(snapshots, func(a, b int) bool {168+ return snapshots[a].CapturedAt < snapshots[b].CapturedAt169+ })170+ return writeJSON(indexPath, snapshots)171+}172+173+func writeMapMetadata(174+ mapDir string,175+ game *apiGameFull,176+ full *apiMapFull,177+ index *tileIndex,178+) error {179+ cachedByZoom := map[string]int{}180+ totalsByZoom := map[string]int{}181+ var bytes int64182+ var count int183+ for _, record := range index.records() {184+ if record.Zoom == 0 {185+ continue186+ }187+ zoom := strconv.Itoa(record.Zoom)188+ totalsByZoom[zoom]++189+ if record.Status == "cached" {190+ cachedByZoom[zoom]++191+ bytes += int64(record.ByteLength)192+ count++193+ }194+ }195+ return writeJSON(filepath.Join(mapDir, "map.json"), map[string]any{196+ "cachedTilesByZoom": cachedByZoom,197+ "tileTotalsByZoom": totalsByZoom,198+ "gameId": game.ID,199+ "gameTitle": game.Title,200+ "gameUrl": game.Config.URL,201+ "mapId": full.ID,202+ "mapSlug": full.Slug,203+ "mapTitle": full.Title,204+ "phase": "complete",205+ "pinCount": full.pinCount(),206+ "snapshotCount": 1,207+ "tileBytes": bytes,208+ "tileCount": count,209+ })210+}211+212+// registerMap makes the map discoverable to tools/tiles and tools/generate by213+// listing it in game.json and archive.json, leaving any existing entries alone.214+func registerMap(215+ archiveRoot string,216+ game *apiGameFull,217+ full *apiMapFull,218+ gameDirectory, mapDirectory string,219+) error {220+ gamePath := filepath.Join(archiveRoot, gameDirectory, "game.json")221+ entry := map[string]any{222+ "directory": mapDirectory,223+ "id": full.ID,224+ "slug": full.Slug,225+ "title": full.Title,226+ }227+ var gameFile map[string]any228+ if err := readJSON(gamePath, &gameFile); err != nil {229+ if !errors.Is(err, os.ErrNotExist) {230+ return err231+ }232+ gameFile = map[string]any{}233+ }234+ gameFile["id"] = game.ID235+ gameFile["title"] = game.Title236+ gameFile["url"] = game.Config.URL237+ gameFile["icons"] = filepath.ToSlash(filepath.Join(gameDirectory, "icons", "index.json"))238+ gameFile["maps"] = upsertByID(gameFile["maps"], entry, full.ID)239+ if err := writeJSON(gamePath, gameFile); err != nil {240+ return err241+ }242+243+ archivePath := filepath.Join(archiveRoot, "archive.json")244+ var archiveFile map[string]any245+ if err := readJSON(archivePath, &archiveFile); err != nil {246+ if !errors.Is(err, os.ErrNotExist) {247+ return err248+ }249+ archiveFile = map[string]any{"format": "fmg-git-working-tree"}250+ }251+ archiveFile["games"] = upsertByID(archiveFile["games"], map[string]any{252+ "directory": gameDirectory,253+ "id": game.ID,254+ "title": game.Title,255+ "url": game.Config.URL,256+ }, game.ID)257+ return writeJSON(archivePath, archiveFile)258+}259+260+// upsertByID replaces the entry carrying id, or appends it, preserving whatever261+// order and extra fields the archive already had.262+func upsertByID(existing any, entry map[string]any, id int64) []any {263+ list, _ := existing.([]any)264+ for index, candidate := range list {265+ record, ok := candidate.(map[string]any)266+ if !ok {267+ continue268+ }269+ if value, ok := record["id"].(float64); ok && int64(value) == id {270+ for key, replacement := range entry {271+ record[key] = replacement272+ }273+ list[index] = record274+ return list275+ }276+ }277+ return append(list, entry)278+}279+280+// resolveGameDirectory prefers the directory the archive already uses for this281+// game. The existing names came from slugified titles with accents folded282+// ("Pokémon Red/Blue/Yellow" became pokemon-red-blue-yellow), so reusing the283+// recorded name is safer than trying to reproduce that transformation.284+func resolveGameDirectory(archiveRoot string, game *apiGameFull) string {285+ var archiveFile struct {286+ Games []struct {287+ Directory string `json:"directory"`288+ ID int64 `json:"id"`289+ } `json:"games"`290+ }291+ if err := readJSON(filepath.Join(archiveRoot, "archive.json"), &archiveFile); err == nil {292+ for _, candidate := range archiveFile.Games {293+ if candidate.ID == game.ID && candidate.Directory != "" {294+ return candidate.Directory295+ }296+ }297+ }298+ return filepath.ToSlash(filepath.Join("games", directoryName(game.Title, game.ID)))299+}300+301+func resolveMapDirectory(archiveRoot, gameDirectory string, full *apiMapFull) string {302+ var gameFile struct {303+ Maps []struct {304+ Directory string `json:"directory"`305+ ID int64 `json:"id"`306+ } `json:"maps"`307+ }308+ path := filepath.Join(archiveRoot, gameDirectory, "game.json")309+ if err := readJSON(path, &gameFile); err == nil {310+ for _, candidate := range gameFile.Maps {311+ if candidate.ID == full.ID && candidate.Directory != "" {312+ return candidate.Directory313+ }314+ }315+ }316+ return filepath.ToSlash(filepath.Join(gameDirectory, "maps", directoryName(full.Title, full.ID)))317+}318+319+// directoryName names a game or map the archive has never seen. Latin letters320+// with diacritics fold to their base form so titles such as "Pokémon" keep a321+// readable directory rather than losing the character to a separator.322+func directoryName(title string, id int64) string {323+ var builder strings.Builder324+ previousDash := false325+ for _, r := range strings.ToLower(title) {326+ if folded, ok := foldedLatin[r]; ok {327+ r = folded328+ }329+ switch {330+ case r >= 'a' && r <= 'z', r >= '0' && r <= '9':331+ builder.WriteRune(r)332+ previousDash = false333+ case !previousDash:334+ builder.WriteRune('-')335+ previousDash = true336+ }337+ }338+ return strings.Trim(builder.String(), "-") + "-" + strconv.FormatInt(id, 10)339+}340+341+var foldedLatin = map[rune]rune{342+ 'á': 'a', 'à': 'a', 'â': 'a', 'ä': 'a', 'ã': 'a', 'å': 'a',343+ 'é': 'e', 'è': 'e', 'ê': 'e', 'ë': 'e',344+ 'í': 'i', 'ì': 'i', 'î': 'i', 'ï': 'i',345+ 'ó': 'o', 'ò': 'o', 'ô': 'o', 'ö': 'o', 'õ': 'o', 'ø': 'o',346+ 'ú': 'u', 'ù': 'u', 'û': 'u', 'ü': 'u',347+ 'ñ': 'n', 'ç': 'c', 'ß': 's', 'ý': 'y',348+}349+350+func writeFile(path string, data []byte) error {351+ if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {352+ return err353+ }354+ return os.WriteFile(path, data, 0o644)355+}356+357+func writeJSON(path string, value any) error {358+ data, err := json.MarshalIndent(value, "", " ")359+ if err != nil {360+ return err361+ }362+ return writeFile(path, append(data, '\n'))363+}364+365+func readJSON(path string, into any) error {366+ data, err := os.ReadFile(path)367+ if err != nil {368+ return err369+ }370+ return json.Unmarshal(data, into)371+}@@ -0,0 +1,466 @@
1+// Command crawl fills the FMG archive directly from the MapGenie API.2+//3+// A run names a game, optionally a single map, and captures everything that4+// target needs in one pass: the map metadata snapshot and every tile of its5+// pyramid. There are no separate phases to run in a particular order, and6+// re-running is cheap because anything already archived is skipped.7+//8+// MapGenie serves this data to every visitor of a map page, but a crawl asks9+// for far more of it at once than a person browsing would. Requests are10+// therefore spaced out, limited in number, backed off when the server pushes11+// back, and never repeated for something already on disk.12+package main13+14+import (15+ "context"16+ "crypto/sha256"17+ "encoding/hex"18+ "errors"19+ "flag"20+ "fmt"21+ "os"22+ "path/filepath"23+ "sort"24+ "strconv"25+ "strings"26+ "sync"27+ "time"28+)29+30+const (31+ apiBase = "https://mapgenie.io/api/v1"32+ // MapGenie's CDN answers 403 to requests without a browser user agent.33+ userAgent = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) " +34+ "AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36"35+ // Levels holding at most this many tiles are captured whole. Deeper levels36+ // follow only the parents that held content, which is what keeps a z1537+ // capture affordable: empty space stays empty at every zoom below it.38+ wholeLevelBudget = 102439+)40+41+func main() {42+ options := parseOptions()43+ if err := run(options); err != nil {44+ fmt.Fprintln(os.Stderr, "crawl:", err)45+ os.Exit(1)46+ }47+}48+49+type options struct {50+ archive string51+ game string52+ mapTarget string53+ maxZoom int54+ list bool55+ dryRun bool56+ concurrency int57+ interval time.Duration58+}59+60+func parseOptions() options {61+ var o options62+ flag.StringVar(&o.archive, "archive", "../gamemap/fmg-archive", "FMG archive root")63+ flag.StringVar(&o.game, "game", "", "game slug or id")64+ flag.StringVar(&o.mapTarget, "map", "", "map slug or id (default: every map of the game)")65+ flag.IntVar(&o.maxZoom, "max-zoom", 0, "deepest zoom to capture (default: the layer's own maximum)")66+ flag.BoolVar(&o.list, "list", false, "list games, or the maps of -game, and exit")67+ flag.BoolVar(&o.dryRun, "dry-run", false, "report what would be fetched without writing anything")68+ flag.IntVar(&o.concurrency, "concurrency", 4, "concurrent requests")69+ flag.DurationVar(&o.interval, "interval", 150*time.Millisecond, "minimum spacing between requests")70+ flag.Parse()71+ return o72+}73+74+func run(o options) error {75+ if o.concurrency < 1 {76+ return errors.New("-concurrency must be at least 1")77+ }78+ fetcher := newFetcher(o.concurrency, o.interval)79+ ctx := context.Background()80+81+ games, err := listGames(ctx, fetcher)82+ if err != nil {83+ return err84+ }85+ if o.game == "" {86+ if !o.list {87+ return errors.New("-game is required; use -list to see what is available")88+ }89+ for _, game := range games {90+ fmt.Printf("%-38s id=%d\n", game.Slug, game.ID)91+ }92+ return nil93+ }94+95+ game, ok := findGame(games, o.game)96+ if !ok {97+ return fmt.Errorf("no game matches %q; use -list to see what is available", o.game)98+ }99+ targets, err := selectMaps(game, o.mapTarget)100+ if err != nil {101+ return err102+ }103+ if o.list {104+ for _, target := range targets {105+ fmt.Printf("%-38s id=%d %s\n", target.Slug, target.ID, target.Title)106+ }107+ return nil108+ }109+110+ for _, target := range targets {111+ if err := crawlMap(ctx, fetcher, o, game, target); err != nil {112+ return fmt.Errorf("%s / %s: %w", game.Title, target.Title, err)113+ }114+ }115+ return nil116+}117+118+// crawlMap captures one map completely: its metadata snapshot, then its tiles.119+// Both are idempotent, so an interrupted run resumes by simply being repeated.120+func crawlMap(121+ ctx context.Context,122+ fetcher *fetcher,123+ o options,124+ game *apiGameFull,125+ target apiMapRef,126+) error {127+ fmt.Printf("\n== %s / %s (map %d)\n", game.Title, target.Title, target.ID)128+129+ full, raw, err := fetchMap(ctx, fetcher, target.ID)130+ if err != nil {131+ return err132+ }133+ gameDirectory := resolveGameDirectory(o.archive, game)134+ mapDirectory := resolveMapDirectory(o.archive, gameDirectory, full)135+ mapDir := filepath.Join(o.archive, mapDirectory)136+137+ if o.dryRun {138+ fmt.Printf(" would write snapshot to %s\n", mapDir)139+ } else if err := writeSnapshot(mapDir, full, raw); err != nil {140+ return err141+ }142+143+ index, err := readTileIndex(mapDir)144+ if err != nil {145+ return err146+ }147+ index.mapID = full.ID148+ stats, err := captureTiles(ctx, fetcher, o, game, full, mapDir, index)149+ if err != nil {150+ return err151+ }152+ if o.dryRun {153+ return nil154+ }155+ if err := writeTileIndex(mapDir, index); err != nil {156+ return err157+ }158+ if err := writeMapMetadata(mapDir, game, full, index); err != nil {159+ return err160+ }161+ if err := registerMap(o.archive, game, full, gameDirectory, mapDirectory); err != nil {162+ return err163+ }164+ fmt.Printf(" %d fetched · %d held · %d not published · %d failed · %.1f MB new\n",165+ stats.fetched, stats.skipped, stats.absent, stats.failed, float64(stats.bytes)/1e6)166+ return nil167+}168+169+// captureTiles walks each layer from its shallowest zoom down. Levels small170+// enough to take whole are taken whole; past that only the children of tiles171+// that held content are requested, so blank regions cost nothing at depth.172+func captureTiles(173+ ctx context.Context,174+ fetcher *fetcher,175+ o options,176+ game *apiGameFull,177+ full *apiMapFull,178+ mapDir string,179+ index *tileIndex,180+) (captureStats, error) {181+ var stats captureStats182+ base := game.Config.TilesBaseURL183+ if base == "" {184+ base = "https://tiles.mapgenie.io"185+ }186+187+ for _, set := range full.Config.TileSets {188+ setID := index.tileSetID(set.Path)189+ deepest := set.MaxZoom190+ if o.maxZoom > 0 && o.maxZoom < deepest {191+ deepest = o.maxZoom192+ }193+ fmt.Printf(" layer %q (%s) z%d-%d\n", set.Name, set.Path, set.MinZoom, deepest)194+195+ var live map[[2]int]bool196+ var background string197+ for zoom := set.MinZoom; zoom <= deepest; zoom++ {198+ window, ok := set.window(zoom)199+ if !ok {200+ fmt.Printf(" z%-2d skipped: no bounds published\n", zoom)201+ live = nil202+ continue203+ }204+ wanted := window.tiles()205+ whole := len(wanted) <= wholeLevelBudget || live == nil206+ if !whole {207+ wanted = childrenOf(live, window)208+ }209+ if len(wanted) == 0 {210+ fmt.Printf(" z%-2d nothing left to follow\n", zoom)211+ break212+ }213+214+ results, err := fetchLevel(ctx, fetcher, o, set, setID, base, mapDir, zoom, wanted, index)215+ if err != nil {216+ return stats, err217+ }218+ stats.merge(results)219+220+ // Each level re-encodes its own background tile, so the hash is221+ // re-read from every level captured whole. It is never taken from a222+ // pruned level, whose handful of real tiles could otherwise vote223+ // themselves away, and a level that yields no answer keeps the last224+ // one rather than losing it.225+ if whole {226+ if found := dominantHash(results.hashes); found != "" {227+ background = found228+ }229+ }230+ live = liveTiles(results.hashes, background)231+ fmt.Printf(" z%-2d %4d fetched · %4d held · %4d not published · %4d carry content\n",232+ zoom, results.fetched, results.skipped, results.absent, len(live))233+ if background != "" && len(live) == 0 {234+ break235+ }236+ }237+ }238+ return stats, nil239+}240+241+func fetchLevel(242+ ctx context.Context,243+ fetcher *fetcher,244+ o options,245+ set apiTileSet,246+ setID int64,247+ base, mapDir string,248+ zoom int,249+ wanted [][2]int,250+ index *tileIndex,251+) (levelResult, error) {252+ result := levelResult{hashes: make(map[[2]int]string, len(wanted))}253+ extension := strings.TrimPrefix(strings.ToLower(set.Extension), ".")254+ if extension == "" {255+ extension = "jpg"256+ }257+258+ var mu sync.Mutex259+ var group sync.WaitGroup260+ var failure error261+ gate := make(chan struct{}, o.concurrency)262+263+ for _, coordinate := range wanted {264+ url := fmt.Sprintf("%s/games/%s/%d/%d/%d.%s",265+ strings.TrimSuffix(base, "/"), set.Path, zoom, coordinate[0], coordinate[1], extension)266+ path := filepath.Join(mapDir, "tiles",267+ "set-"+strconv.FormatInt(setID, 10),268+ strconv.Itoa(zoom), strconv.Itoa(coordinate[0]),269+ strconv.Itoa(coordinate[1])+"."+extension)270+271+ // Resume: a tile already recorded and present on disk is never asked272+ // for a second time.273+ if record, held := index.byURL[url]; held {274+ if record.Status == "absent" {275+ continue276+ }277+ if record.Status == "cached" {278+ if _, err := os.Stat(path); err == nil {279+ mu.Lock()280+ result.skipped++281+ result.hashes[coordinate] = record.ContentHash282+ mu.Unlock()283+ continue284+ }285+ }286+ }287+ if o.dryRun {288+ mu.Lock()289+ result.fetched++290+ mu.Unlock()291+ continue292+ }293+294+ group.Add(1)295+ gate <- struct{}{}296+ go func(coordinate [2]int, url, path string) {297+ defer group.Done()298+ defer func() { <-gate }()299+300+ body, contentType, err := fetcher.get(ctx, url)301+ mu.Lock()302+ defer mu.Unlock()303+ if err != nil {304+ status, message := "failed", err.Error()305+ if errors.Is(err, errAbsent) {306+ // Recorded so a later run does not ask again.307+ status, message = "absent", ""308+ result.absent++309+ } else {310+ result.failed++311+ if failure == nil {312+ failure = err313+ }314+ }315+ index.put(tileRecord{316+ URL: url, Status: status, TileSetID: setID,317+ Zoom: zoom, X: coordinate[0], Y: coordinate[1],318+ ContentType: "application/octet-stream",319+ Error: message,320+ })321+ return322+ }323+ if err := writeFile(path, body); err != nil {324+ failure = err325+ return326+ }327+ sum := sha256.Sum256(body)328+ hash := hex.EncodeToString(sum[:])329+ index.put(tileRecord{330+ URL: url, Status: "cached", TileSetID: setID,331+ Zoom: zoom, X: coordinate[0], Y: coordinate[1],332+ ContentHash: hash, ContentType: contentType,333+ ByteLength: len(body),334+ CoverageKey: fmt.Sprintf("%d:%d:%d:%d:%d",335+ index.mapID, setID, zoom, coordinate[0], coordinate[1]),336+ })337+ result.fetched++338+ result.bytes += int64(len(body))339+ result.hashes[coordinate] = hash340+ }(coordinate, url, path)341+ }342+ group.Wait()343+ return result, failure344+}345+346+type levelResult struct {347+ fetched int348+ skipped int349+ absent int350+ failed int351+ bytes int64352+ hashes map[[2]int]string353+}354+355+type captureStats struct {356+ fetched int357+ skipped int358+ absent int359+ failed int360+ bytes int64361+}362+363+func (s *captureStats) merge(r levelResult) {364+ s.fetched += r.fetched365+ s.skipped += r.skipped366+ s.absent += r.absent367+ s.failed += r.failed368+ s.bytes += r.bytes369+}370+371+// backgroundSampleFloor is the smallest level worth judging. The top of a372+// pyramid is a handful of tiles showing the whole map, where "repeated over373+// half the level" means nothing.374+const backgroundSampleFloor = 16375+376+// dominantHash reports the tile a level uses for empty space. Two tiles of real377+// map are never byte-identical, so any tile repeated across a level is the378+// background, however small a share of the level it occupies -- a map whose379+// content fills most of its bounds still has a background in the corners.380+//381+// A level whose tiles are *all* identical says nothing: there is no content to382+// contrast against, so the question is left to a deeper level.383+func dominantHash(hashes map[[2]int]string) string {384+ if len(hashes) < backgroundSampleFloor {385+ return ""386+ }387+ counts := make(map[string]int, len(hashes))388+ var best string389+ for _, hash := range hashes {390+ if hash == "" {391+ continue392+ }393+ counts[hash]++394+ if counts[hash] > counts[best] {395+ best = hash396+ }397+ }398+ repeats := max(8, len(hashes)/20)399+ if best == "" || counts[best] < repeats || counts[best] == len(hashes) {400+ return ""401+ }402+ return best403+}404+405+func liveTiles(hashes map[[2]int]string, background string) map[[2]int]bool {406+ live := make(map[[2]int]bool, len(hashes))407+ for coordinate, hash := range hashes {408+ if hash == "" || (background != "" && hash == background) {409+ continue410+ }411+ live[coordinate] = true412+ }413+ return live414+}415+416+func childrenOf(live map[[2]int]bool, window tileWindow) [][2]int {417+ children := make([][2]int, 0, len(live)*4)418+ for parent := range live {419+ for dy := 0; dy < 2; dy++ {420+ for dx := 0; dx < 2; dx++ {421+ child := [2]int{parent[0]*2 + dx, parent[1]*2 + dy}422+ if window.holds(child) {423+ children = append(children, child)424+ }425+ }426+ }427+ }428+ sortCoordinates(children)429+ return children430+}431+432+func sortCoordinates(coordinates [][2]int) {433+ sort.Slice(coordinates, func(i, j int) bool {434+ if coordinates[i][1] != coordinates[j][1] {435+ return coordinates[i][1] < coordinates[j][1]436+ }437+ return coordinates[i][0] < coordinates[j][0]438+ })439+}440+441+type tileWindow struct{ minX, minY, maxX, maxY int }442+443+func (w tileWindow) holds(c [2]int) bool {444+ return c[0] >= w.minX && c[0] <= w.maxX && c[1] >= w.minY && c[1] <= w.maxY445+}446+447+func (w tileWindow) tiles() [][2]int {448+ out := make([][2]int, 0, (w.maxX-w.minX+1)*(w.maxY-w.minY+1))449+ for y := w.minY; y <= w.maxY; y++ {450+ for x := w.minX; x <= w.maxX; x++ {451+ out = append(out, [2]int{x, y})452+ }453+ }454+ return out455+}456+457+func (s apiTileSet) window(zoom int) (tileWindow, bool) {458+ bounds, ok := s.Bounds[strconv.Itoa(zoom)]459+ if !ok {460+ return tileWindow{}, false461+ }462+ return tileWindow{463+ minX: bounds.X.Min, minY: bounds.Y.Min,464+ maxX: bounds.X.Max, maxY: bounds.Y.Max,465+ }, true466+}@@ -0,0 +1,191 @@
1+package main2+3+import "testing"4+5+func TestDominantHashIgnoresTinySamples(t *testing.T) {6+ // The top of a pyramid is a handful of tiles showing the whole map. Calling7+ // the only tile there "background" would stop a crawl before it started.8+ single := map[[2]int]string{{0, 0}: "a"}9+ if got := dominantHash(single); got != "" {10+ t.Fatalf("dominantHash(single tile) = %q, want %q", got, "")11+ }12+13+ four := map[[2]int]string{{0, 0}: "a", {1, 0}: "a", {0, 1}: "a", {1, 1}: "b"}14+ if got := dominantHash(four); got != "" {15+ t.Fatalf("dominantHash(4 tiles) = %q, want %q", got, "")16+ }17+}18+19+func TestDominantHashFindsBackgroundBelowMajority(t *testing.T) {20+ // A map whose content fills most of its bounds still has empty corners, so21+ // the background need not be a majority to be the background.22+ hashes := make(map[[2]int]string, 32)23+ for index := range 32 {24+ key := [2]int{index, 0}25+ if index < 10 {26+ hashes[key] = "empty"27+ continue28+ }29+ hashes[key] = string(rune('a' + index))30+ }31+ if got := dominantHash(hashes); got != "empty" {32+ t.Fatalf("dominantHash = %q, want %q", got, "empty")33+ }34+}35+36+func TestDominantHashRejectsUniformLevel(t *testing.T) {37+ // Every tile identical gives nothing to contrast against; the question is38+ // left to a deeper level rather than answered wrongly.39+ hashes := make(map[[2]int]string, 20)40+ for index := range 20 {41+ hashes[[2]int{index, 0}] = "same"42+ }43+ if got := dominantHash(hashes); got != "" {44+ t.Fatalf("dominantHash(uniform) = %q, want %q", got, "")45+ }46+}47+48+func TestDominantHashRejectsDistinctTiles(t *testing.T) {49+ hashes := make(map[[2]int]string, 40)50+ for index := range 40 {51+ hashes[[2]int{index, 0}] = string(rune('a' + index))52+ }53+ if got := dominantHash(hashes); got != "" {54+ t.Fatalf("dominantHash(all distinct) = %q, want %q", got, "")55+ }56+}57+58+func TestLiveTilesDropsBackground(t *testing.T) {59+ hashes := map[[2]int]string{60+ {0, 0}: "empty",61+ {1, 0}: "content",62+ {2, 0}: "empty",63+ }64+ live := liveTiles(hashes, "empty")65+ if len(live) != 1 || !live[[2]int{1, 0}] {66+ t.Fatalf("liveTiles = %v, want only {1,0}", live)67+ }68+ // With no background known, nothing is pruned.69+ if all := liveTiles(hashes, ""); len(all) != 3 {70+ t.Fatalf("liveTiles(no background) kept %d, want 3", len(all))71+ }72+}73+74+func TestChildrenOfStaysInsideWindow(t *testing.T) {75+ live := map[[2]int]bool{{2, 2}: true}76+ window := tileWindow{minX: 4, minY: 4, maxX: 5, maxY: 4}77+ children := childrenOf(live, window)78+ if len(children) != 2 {79+ t.Fatalf("childrenOf = %v, want the two children inside the window", children)80+ }81+ for _, child := range children {82+ if !window.holds(child) {83+ t.Errorf("child %v falls outside the window", child)84+ }85+ }86+}87+88+func TestChildrenOfPrunesEmptySpace(t *testing.T) {89+ // One live parent out of a 4x4 level yields 4 requests at the next zoom90+ // rather than the 64 a whole level would need.91+ live := map[[2]int]bool{{0, 0}: true}92+ window := tileWindow{minX: 0, minY: 0, maxX: 7, maxY: 7}93+ if children := childrenOf(live, window); len(children) != 4 {94+ t.Fatalf("childrenOf = %d tiles, want 4", len(children))95+ }96+}97+98+func TestWindowTiles(t *testing.T) {99+ window := tileWindow{minX: 3, minY: 7, maxX: 4, maxY: 8}100+ tiles := window.tiles()101+ if len(tiles) != 4 {102+ t.Fatalf("tiles = %v, want 4", tiles)103+ }104+ if tiles[0] != [2]int{3, 7} || tiles[3] != [2]int{4, 8} {105+ t.Fatalf("tiles = %v, want row-major from the minimum corner", tiles)106+ }107+}108+109+func TestTileSetWindowMissingZoom(t *testing.T) {110+ set := apiTileSet{Bounds: map[string]apiBound{111+ "13": {X: apiRange{Min: 4064, Max: 4095}, Y: apiRange{Min: 4064, Max: 4095}},112+ }}113+ if _, ok := set.window(14); ok {114+ t.Fatal("window(14) reported bounds the layer does not publish")115+ }116+ window, ok := set.window(13)117+ if !ok || window.minX != 4064 || window.maxY != 4095 {118+ t.Fatalf("window(13) = %+v, %v", window, ok)119+ }120+}121+122+func TestDirectoryName(t *testing.T) {123+ cases := map[string]string{124+ "L.A. Noire": "l-a-noire-59",125+ "Pokémon Red/Blue/Yellow": "pokemon-red-blue-yellow-59",126+ "Assassin's Creed (Resynced)": "assassin-s-creed-resynced-59",127+ "Skyrim": "skyrim-59",128+ }129+ for title, want := range cases {130+ if got := directoryName(title, 59); got != want {131+ t.Errorf("directoryName(%q) = %q, want %q", title, got, want)132+ }133+ }134+}135+136+func TestTileSetPathOf(t *testing.T) {137+ url := "https://tiles.mapgenie.io/games/la-noire/los-angeles/default-v2/10/508/508.png"138+ if got := tileSetPathOf(url, 10); got != "la-noire/los-angeles/default-v2" {139+ t.Fatalf("tileSetPathOf = %q", got)140+ }141+ if got := tileSetPathOf(url, 13); got != "" {142+ t.Fatalf("tileSetPathOf with the wrong zoom = %q, want empty", got)143+ }144+}145+146+func TestTileIndexReusesExistingTileSetDirectory(t *testing.T) {147+ // Re-crawling an archive the extension captured must not orphan its files.148+ index := newTileIndex(203)149+ index.put(tileRecord{150+ URL: "https://tiles.mapgenie.io/games/a/b/c-v1/10/1/2.png",151+ Zoom: 10,152+ TileSetID: 206,153+ Status: "cached",154+ })155+ index.setIDs["a/b/c-v1"] = 206156+ index.usedIDs[206] = true157+158+ if got := index.tileSetID("a/b/c-v1"); got != 206 {159+ t.Fatalf("tileSetID(known layer) = %d, want 206", got)160+ }161+ fresh := index.tileSetID("x/y/z-v1")162+ if fresh == 206 {163+ t.Fatal("a new layer reused the existing directory")164+ }165+ if again := index.tileSetID("x/y/z-v1"); again != fresh {166+ t.Fatalf("tileSetID is unstable: %d then %d", fresh, again)167+ }168+}169+170+func TestUpsertByIDReplacesInPlace(t *testing.T) {171+ existing := []any{172+ map[string]any{"id": float64(1), "directory": "games/one", "extra": "kept"},173+ map[string]any{"id": float64(2), "directory": "games/two"},174+ }175+ result := upsertByID(existing, map[string]any{"directory": "games/one-moved"}, 1)176+ if len(result) != 2 {177+ t.Fatalf("upsertByID changed the entry count to %d", len(result))178+ }179+ first := result[0].(map[string]any)180+ if first["directory"] != "games/one-moved" {181+ t.Errorf("directory = %v, want the replacement", first["directory"])182+ }183+ if first["extra"] != "kept" {184+ t.Errorf("upsertByID dropped a field the archive already had")185+ }186+187+ appended := upsertByID(existing, map[string]any{"id": float64(3)}, 3)188+ if len(appended) != 3 {189+ t.Fatalf("upsertByID did not append an unknown id")190+ }191+}@@ -0,0 +1,294 @@
1+2+// Renders a game's category glyphs into the FMG archive as standalone SVGs.3+//4+// MapGenie ships its category icons as an icon font referenced from a5+// stylesheet, so the glyph for "police_station" is a codepoint rather than a6+// file. This resolves that indirection: the stylesheet gives key -> codepoint,7+// the SVG font gives codepoint -> outline, and each pairing is written to8+// <archive>/games/<game-dir>/icons/<key>.svg, which is where tools/generate9+// looks for them.10+//11+// It runs once per game rather than per tile, which is why it stays a small12+// script instead of moving into the Go crawler.13+//14+// node tools/icons/render-icons.mjs --archive ../gamemap/fmg-archive --game skyrim15+//16+// Without --game, every game already present in the archive is refreshed.17+18+import { createHash } from "node:crypto";19+import { mkdir, readFile, writeFile } from "node:fs/promises";20+import path from "node:path";21+import process from "node:process";22+23+const API = "https://mapgenie.io/api/v1";24+const CDN = "https://cdn.mapgenie.io";25+// The CDN answers 403 to requests that do not look like a browser.26+// Used when a category carries no legend colour of its own; light enough to27+// stay legible on the dark map.28+const FALLBACK_COLOR = "#c8ccbd";29+const USER_AGENT =30+ "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 " +31+ "(KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36";32+33+const parseArguments = (argv) => {34+ const options = { archive: "../gamemap/fmg-archive", game: null };35+ for (let index = 0; index < argv.length; index += 1) {36+ const flag = argv[index];37+ if (flag === "--archive") options.archive = argv[++index];38+ else if (flag === "--game") options.game = argv[++index];39+ else if (flag === "--help" || flag === "-h") options.help = true;40+ else throw new Error(`unknown argument ${flag}`);41+ }42+ return options;43+};44+45+let lastRequest = 0;46+// A single game needs only a handful of requests, but they are still spaced47+// out so a loop over every game stays a light load.48+const fetchPolitely = async (url, { optional = false } = {}) => {49+ const wait = Math.max(0, lastRequest + 150 - Date.now());50+ if (wait > 0) await new Promise((resolve) => setTimeout(resolve, wait));51+ lastRequest = Date.now();52+53+ const response = await fetch(url, {54+ headers: { "User-Agent": USER_AGENT, Accept: "*/*" },55+ });56+ if (!response.ok) {57+ if (optional && (response.status === 403 || response.status === 404)) return null;58+ throw new Error(`HTTP ${response.status} from ${url}`);59+ }60+ return response;61+};62+63+const fetchText = async (url, options) => {64+ const response = await fetchPolitely(url, options);65+ return response && response.text();66+};67+68+const fetchJSON = async (url, options) => {69+ const response = await fetchPolitely(url, options);70+ return response && response.json();71+};72+73+const cssColor = (value) => {74+ if (!value) return null;75+ const rgb = /rgba?\(\s*(\d+)\s*,\s*(\d+)\s*,\s*(\d+)/i.exec(value);76+ if (rgb) {77+ return `#${[1, 2, 3]78+ .map((index) => Number(rgb[index]).toString(16).padStart(2, "0"))79+ .join("")}`;80+ }81+ return /#[0-9a-f]{3,8}/i.exec(value)?.[0] ?? null;82+};83+84+// An icon is either a single glyph, `.icon-<key>:before`, or several stacked85+// layers, `.icon-<key> .path1:before` .. `.pathN:before`, each with its own86+// colour. Both forms are collected per key, in the order the layers paint.87+const readIconLayers = (css) => {88+ const icons = new Map();89+ const layerAt = (key, order) => {90+ if (!icons.has(key)) icons.set(key, []);91+ const layers = icons.get(key);92+ const existing = layers.find((layer) => layer.order === order);93+ if (existing) return existing;94+ const layer = { order };95+ layers.push(layer);96+ return layer;97+ };98+99+ const rule =100+ /\.icon-([a-z0-9_-]+)(?:\s+\.path(\d+))?:before\s*\{([^}]*)\}/gi;101+ for (const match of css.matchAll(rule)) {102+ const [, key, pathNumber, body] = match;103+ const codepoint = /content:\s*"\\([0-9a-fA-F]+)"/.exec(body)?.[1];104+ if (!codepoint) continue;105+ const layer = layerAt(key, pathNumber ? Number(pathNumber) : 0);106+ layer.codepoint = codepoint.toLowerCase();107+ layer.color = cssColor(/color:\s*([^;]+)/i.exec(body)?.[1]);108+ const opacity = /opacity:\s*([0-9.]+)/i.exec(body)?.[1];109+ if (opacity) layer.opacity = Number(opacity);110+ }111+112+ for (const layers of icons.values()) layers.sort((a, b) => a.order - b.order);113+ return icons;114+};115+116+const readFontURL = (css) => {117+ const match = css.match(/url\(['"]?([^'")]*\.svg[^'")]*)['"]?\)/i);118+ if (!match) return null;119+ return new URL(match[1].replace(/#.*$/, ""), `${CDN}/css/themes/icons/`).href;120+};121+122+// An SVG font is XML, so the glyphs can be read directly rather than through a123+// font parser: each carries its codepoint and its outline.124+const readGlyphs = (svg) => {125+ const glyphs = new Map();126+ const element = /<glyph\b([^>]*)\/?>/gi;127+ for (const match of svg.matchAll(element)) {128+ const attributes = match[1];129+ const unicode = /unicode="&#x([0-9a-fA-F]+);"/.exec(attributes);130+ const outline = /\sd="([^"]*)"/.exec(attributes);131+ if (!unicode || !outline || !outline[1].trim()) continue;132+ glyphs.set(unicode[1].toLowerCase(), outline[1]);133+ }134+ const face = /<font-face\b([^>]*)/i.exec(svg)?.[1] ?? "";135+ const number = (name, fallback) =>136+ Number(new RegExp(`${name}="(-?[0-9.]+)"`).exec(face)?.[1] ?? fallback);137+ return {138+ glyphs,139+ unitsPerEm: number("units-per-em", 1024),140+ ascent: number("ascent", 960),141+ };142+};143+144+// Font outlines are y-up from the baseline; SVG is y-down from the top corner,145+// so the glyph is flipped about the ascent line to sit in a square viewBox.146+//147+// Layers paint in order. Atlas draws these as background images, where nothing148+// outside the file can tint them and `currentColor` has no inherited value to149+// resolve against, so a layer without its own colour needs the legend colour150+// carried on the root element rather than left to the page.151+const renderSVG = (key, layers, unitsPerEm, ascent, defaultColor) => {152+ const paths = layers153+ .map(({ outline, color, opacity }) => {154+ const fill = color ? `fill="${color}"` : `fill="currentColor"`;155+ const alpha = opacity === undefined ? "" : ` fill-opacity="${opacity}"`;156+ return ` <path ${fill}${alpha} d="${outline}"/>`;157+ })158+ .join("\n");159+ return (160+ `<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 ${unitsPerEm} ${unitsPerEm}" ` +161+ `color="${defaultColor}" role="img">\n` +162+ ` <title>${key}</title>\n` +163+ ` <g transform="translate(0, ${ascent}) scale(1, -1)">\n` +164+ `${paths}\n` +165+ ` </g>\n` +166+ `</svg>\n`167+ );168+};169+170+const renderGame = async (archiveRoot, game, directory) => {171+ const css = await fetchText(`${CDN}/css/themes/icons/${game.slug}-icons.css`, {172+ optional: true,173+ });174+ if (!css) {175+ console.log(` ${game.slug}: no icon stylesheet published`);176+ return 0;177+ }178+ const icons = readIconLayers(css);179+ const fontURL = readFontURL(css);180+ if (!fontURL || icons.size === 0) {181+ console.log(` ${game.slug}: stylesheet carries no usable glyph mapping`);182+ return 0;183+ }184+ const { glyphs, unitsPerEm, ascent } = readGlyphs(await fetchText(fontURL));185+186+ // Which glyphs matter, and what colour the legend gives them, comes from the187+ // maps themselves.188+ const usages = new Map();189+ for (const reference of game.maps ?? []) {190+ const full = await fetchJSON(`${API}/maps/${reference.id}/full`, { optional: true });191+ if (!full) continue;192+ for (const group of full.groups ?? []) {193+ for (const category of group.categories ?? []) {194+ if (!category.icon) continue;195+ if (!usages.has(category.icon)) usages.set(category.icon, []);196+ usages.get(category.icon).push({197+ categoryId: category.id,198+ categoryTitle: category.title,199+ groupId: group.id,200+ legendColor: (category.color ?? "").replace(/^#/, "") || undefined,201+ mapId: full.id,202+ });203+ }204+ }205+ }206+207+ const iconDirectory = path.join(archiveRoot, directory, "icons");208+ await mkdir(iconDirectory, { recursive: true });209+210+ const index = [];211+ let missing = 0;212+ for (const [key, uses] of [...usages].sort(([a], [b]) => a.localeCompare(b))) {213+ const drawn = (icons.get(key) ?? [])214+ .map((layer) => ({ ...layer, outline: glyphs.get(layer.codepoint) }))215+ .filter((layer) => layer.outline);216+ if (drawn.length === 0) {217+ missing += 1;218+ continue;219+ }220+ const legend = uses.find((use) => use.legendColor)?.legendColor;221+ const defaultColor = legend ? `#${legend.toLowerCase()}` : FALLBACK_COLOR;222+ const svg = renderSVG(key, drawn, unitsPerEm, ascent, defaultColor);223+ await writeFile(path.join(iconDirectory, `${key}.svg`), svg);224+ index.push({225+ codepoint: drawn.map((layer) => layer.codepoint).join("+"),226+ contentHash: createHash("sha256").update(svg).digest("hex"),227+ defaultColor,228+ file: `${key}.svg`,229+ key,230+ sourceCssUrl: `${CDN}/css/themes/icons/${game.slug}-icons.css`,231+ sourceFontUrl: fontURL,232+ usages: uses,233+ });234+ }235+ await writeFile(236+ path.join(iconDirectory, "index.json"),237+ `${JSON.stringify(index, null, 2)}\n`238+ );239+ console.log(240+ ` ${game.slug}: ${index.length} icons written` +241+ (missing ? ` · ${missing} referenced but not in the font` : "")242+ );243+ return index.length;244+};245+246+const main = async () => {247+ const options = parseArguments(process.argv.slice(2));248+ if (options.help) {249+ console.log(250+ "usage: render-icons.mjs [--archive PATH] [--game SLUG]\n" +251+ "Renders category glyphs into <archive>/games/<game>/icons/."252+ );253+ return;254+ }255+256+ const archiveRoot = path.resolve(options.archive);257+ const archiveFile = JSON.parse(258+ await readFile(path.join(archiveRoot, "archive.json"), "utf8")259+ );260+ const directories = new Map(261+ (archiveFile.games ?? []).map((entry) => [entry.id, entry.directory])262+ );263+264+ const games = await fetchJSON(`${API}/games`);265+ const wanted = games.filter((game) =>266+ options.game267+ ? game.slug === options.game || String(game.id) === options.game268+ : directories.has(game.id)269+ );270+ if (wanted.length === 0) {271+ throw new Error(272+ options.game273+ ? `no game matches ${options.game}`274+ : "archive.json lists no games"275+ );276+ }277+278+ console.log(`Rendering icons for ${wanted.length} game(s)`);279+ let total = 0;280+ for (const game of wanted) {281+ const directory = directories.get(game.id);282+ if (!directory) {283+ console.log(` ${game.slug}: not in this archive, skipping`);284+ continue;285+ }286+ total += await renderGame(archiveRoot, game, directory);287+ }288+ console.log(`${total} icons rendered`);289+};290+291+main().catch((error) => {292+ console.error("render-icons:", error.message);293+ process.exit(1);294+});@@ -765,16 +765,21 @@ func contentBoundsFor(files []tileFile, zoom int) *contentBounds {
765765 if maxX < minX || maxY < minY {766766 return nil767767 }768+ // Bounds are reported in the shared world space, which is sized at the769+ // reference zoom. A shallower level covers more world per tile and a deeper770+ // one covers less, so the tile pitch is scaled both ways.768771 origin, _ := worldTileRange(zoom)769- scale := 1772+ pitch, divisor := tileSize, 1770773 if zoom < referenceZoom {771- scale = 1 << (referenceZoom - zoom)774+ pitch = tileSize << (referenceZoom - zoom)775+ } else if zoom > referenceZoom {776+ divisor = 1 << (zoom - referenceZoom)772777 }773778 bounds := &contentBounds{774- X: (minX - origin) * tileSize * scale,775- Y: (minY - origin) * tileSize * scale,776- Width: (maxX - minX + 1) * tileSize * scale,777- Height: (maxY - minY + 1) * tileSize * scale,779+ X: (minX - origin) * pitch / divisor,780+ Y: (minY - origin) * pitch / divisor,781+ Width: (maxX - minX + 1) * pitch / divisor,782+ Height: (maxY - minY + 1) * pitch / divisor,778783 }779784 if bounds.X == 0 && bounds.Y == 0 && bounds.Width == worldSize && bounds.Height == worldSize {780785 return nil