1package main23import (4 "crypto/sha256"5 "encoding/hex"6 "encoding/json"7 "errors"8 "fmt"9 "hash/fnv"10 "os"11 "path/filepath"12 "sort"13 "strconv"14 "strings"15 "time"16)1718// The archive layout is the one the FMG crawl dashboard produced, so tools/tiles19// and tools/generate read a crawled archive without knowing the difference.2021type tileRecord struct {22 ByteLength int `json:"byteLength"`23 ContentHash string `json:"contentHash"`24 ContentType string `json:"contentType"`25 CoverageKey string `json:"coverageKey,omitempty"`26 Error string `json:"error,omitempty"`27 Status string `json:"status"`28 TileSetID int64 `json:"tileSetId,omitempty"`29 URL string `json:"url"`30 X int `json:"x,omitempty"`31 Y int `json:"y,omitempty"`32 Zoom int `json:"zoom,omitempty"`33}3435type snapshotRecord struct {36 CapturedAt string `json:"capturedAt"`37 ContentHash string `json:"contentHash"`38 Kind string `json:"kind"`39 SourceID int64 `json:"sourceId"`40 SourceURL string `json:"sourceUrl"`41}4243// tileIndex is the working set of tiles/index.json: keyed by URL for resume44// checks, and remembering which directory each layer's tiles live under.45type tileIndex struct {46 mapID int6447 byURL map[string]tileRecord48 order []string49 setIDs map[string]int6450 usedIDs map[int64]bool51}5253func newTileIndex(mapID int64) *tileIndex {54 return &tileIndex{55 mapID: mapID,56 byURL: make(map[string]tileRecord),57 setIDs: make(map[string]int64),58 usedIDs: make(map[int64]bool),59 }60}6162func (i *tileIndex) put(record tileRecord) {63 if _, seen := i.byURL[record.URL]; !seen {64 i.order = append(i.order, record.URL)65 }66 i.byURL[record.URL] = record67}6869func (i *tileIndex) records() []tileRecord {70 out := make([]tileRecord, 0, len(i.order))71 for _, url := range i.order {72 out = append(out, i.byURL[url])73 }74 return out75}7677// tileSetID keeps the directory a layer already uses, so re-crawling an archive78// captured by the extension does not orphan its files. New layers get a stable79// id derived from their path.80func (i *tileIndex) tileSetID(path string) int64 {81 if id, ok := i.setIDs[path]; ok {82 return id83 }84 hash := fnv.New32a()85 _, _ = hash.Write([]byte(path))86 id := int64(hash.Sum32()%9000) + 100087 for i.usedIDs[id] {88 id++89 }90 i.setIDs[path] = id91 i.usedIDs[id] = true92 return id93}9495func readTileIndex(mapDir string) (*tileIndex, error) {96 index := newTileIndex(0)97 var records []tileRecord98 if err := readJSON(filepath.Join(mapDir, "tiles", "index.json"), &records); err != nil {99 if errors.Is(err, os.ErrNotExist) {100 return index, nil101 }102 return nil, err103 }104 for _, record := range records {105 index.put(record)106 if record.TileSetID == 0 || record.Zoom == 0 {107 continue108 }109 if path := tileSetPathOf(record.URL, record.Zoom); path != "" {110 index.setIDs[path] = record.TileSetID111 index.usedIDs[record.TileSetID] = true112 }113 }114 return index, nil115}116117// tileSetPathOf recovers the layer path from a tile URL, matching how118// tools/tiles groups levels: everything between the source's marker segment119// and "/<zoom>/". MapGenie serves tiles under /games/, IGN under /wikimaps/,120// Piggyback under /tiles/.121func tileSetPathOf(url string, zoom int) string {122 for _, marker := range []string{"/games/", "/wikimaps/", "/tiles/"} {123 _, rest, found := strings.Cut(url, marker)124 if !found {125 continue126 }127 path, _, found := strings.Cut(rest, "/"+strconv.Itoa(zoom)+"/")128 if !found {129 continue130 }131 return path132 }133 return ""134}135136func writeTileIndex(mapDir string, index *tileIndex) error {137 records := index.records()138 sort.SliceStable(records, func(a, b int) bool { return records[a].URL < records[b].URL })139 return writeJSON(filepath.Join(mapDir, "tiles", "index.json"), records)140}141142// writeSnapshot stores the response exactly as received, addressed by its hash,143// and records it only if that content is new.144func writeSnapshot(mapDir string, full *apiMapFull, raw []byte) error {145 return writeSnapshotRecord(mapDir, raw, "map", full.ID,146 fmt.Sprintf("/api/v1/maps/%d/full", full.ID))147}148149// writeSnapshotRecord is the shape every source's capture takes: the document150// addressed by its hash, and one index record per distinct content. A capture151// whose bytes are already archived leaves no trace of having been retaken.152func writeSnapshotRecord(mapDir string, raw []byte, kind string, sourceID int64, sourceURL string) error {153 sum := sha256.Sum256(raw)154 hash := hex.EncodeToString(sum[:])155156 path := filepath.Join(mapDir, "snapshots", "map", hash+".json")157 if err := writeFile(path, raw); err != nil {158 return err159 }160161 indexPath := filepath.Join(mapDir, "snapshots", "index.json")162 var snapshots []snapshotRecord163 if err := readJSON(indexPath, &snapshots); err != nil && !errors.Is(err, os.ErrNotExist) {164 return err165 }166 for _, snapshot := range snapshots {167 if snapshot.ContentHash == hash {168 return nil169 }170 }171 snapshots = append(snapshots, snapshotRecord{172 CapturedAt: time.Now().UTC().Format(time.RFC3339Nano),173 ContentHash: hash,174 Kind: kind,175 SourceID: sourceID,176 SourceURL: sourceURL,177 })178 sort.Slice(snapshots, func(a, b int) bool {179 return snapshots[a].CapturedAt < snapshots[b].CapturedAt180 })181 return writeJSON(indexPath, snapshots)182}183184func writeMapMetadata(185 mapDir string,186 game *apiGameFull,187 full *apiMapFull,188 index *tileIndex,189 pinCount int,190) error {191 cachedByZoom := map[string]int{}192 totalsByZoom := map[string]int{}193 var bytes int64194 var count int195 for _, record := range index.records() {196 if record.Zoom == 0 {197 continue198 }199 zoom := strconv.Itoa(record.Zoom)200 totalsByZoom[zoom]++201 if record.Status == "cached" {202 cachedByZoom[zoom]++203 bytes += int64(record.ByteLength)204 count++205 }206 }207 return writeJSON(filepath.Join(mapDir, "map.json"), map[string]any{208 "cachedTilesByZoom": cachedByZoom,209 "tileTotalsByZoom": totalsByZoom,210 "gameId": game.ID,211 "gameTitle": game.Title,212 "gameUrl": game.Config.URL,213 "mapId": full.ID,214 "mapSlug": full.Slug,215 "mapTitle": full.Title,216 "phase": "complete",217 "pinCount": pinCount,218 "snapshotCount": 1,219 "tileBytes": bytes,220 "tileCount": count,221 })222}223224// registerMap makes the map discoverable to tools/tiles and tools/generate by225// listing it in game.json and archive.json, leaving any existing entries alone.226// A source name, when there is one, rides along so a reader of the archive can227// tell which crawler a game came from; the tools never look at it.228func registerMap(229 archiveRoot string,230 game *apiGameFull,231 full *apiMapFull,232 gameDirectory, mapDirectory, source string,233) error {234 gamePath := filepath.Join(archiveRoot, gameDirectory, "game.json")235 entry := map[string]any{236 "directory": mapDirectory,237 "id": full.ID,238 "slug": full.Slug,239 "title": full.Title,240 }241 var gameFile map[string]any242 if err := readJSON(gamePath, &gameFile); err != nil {243 if !errors.Is(err, os.ErrNotExist) {244 return err245 }246 gameFile = map[string]any{}247 }248 gameFile["id"] = game.ID249 gameFile["title"] = game.Title250 gameFile["url"] = game.Config.URL251 gameFile["icons"] = filepath.ToSlash(filepath.Join(gameDirectory, "icons", "index.json"))252 if source != "" {253 gameFile["source"] = source254 }255 gameFile["maps"] = upsertByID(gameFile["maps"], entry, full.ID)256 if err := writeJSON(gamePath, gameFile); err != nil {257 return err258 }259260 archivePath := filepath.Join(archiveRoot, "archive.json")261 var archiveFile map[string]any262 if err := readJSON(archivePath, &archiveFile); err != nil {263 if !errors.Is(err, os.ErrNotExist) {264 return err265 }266 archiveFile = map[string]any{"format": "fmg-git-working-tree"}267 }268 gameEntry := map[string]any{269 "directory": gameDirectory,270 "id": game.ID,271 "title": game.Title,272 "url": game.Config.URL,273 }274 if source != "" {275 gameEntry["source"] = source276 }277 archiveFile["games"] = upsertByID(archiveFile["games"], gameEntry, game.ID)278 return writeJSON(archivePath, archiveFile)279}280281// upsertByID replaces the entry carrying id, or appends it, preserving whatever282// order and extra fields the archive already had.283func upsertByID(existing any, entry map[string]any, id int64) []any {284 list, _ := existing.([]any)285 for index, candidate := range list {286 record, ok := candidate.(map[string]any)287 if !ok {288 continue289 }290 if value, ok := record["id"].(float64); ok && int64(value) == id {291 for key, replacement := range entry {292 record[key] = replacement293 }294 list[index] = record295 return list296 }297 }298 return append(list, entry)299}300301// resolveGameDirectory prefers the directory the archive already uses for this302// game. The existing names came from slugified titles with accents folded303// ("Pokémon Red/Blue/Yellow" became pokemon-red-blue-yellow), so reusing the304// recorded name is safer than trying to reproduce that transformation.305func resolveGameDirectory(archiveRoot string, game *apiGameFull) string {306 var archiveFile struct {307 Games []struct {308 Directory string `json:"directory"`309 ID int64 `json:"id"`310 } `json:"games"`311 }312 if err := readJSON(filepath.Join(archiveRoot, "archive.json"), &archiveFile); err == nil {313 for _, candidate := range archiveFile.Games {314 if candidate.ID == game.ID && candidate.Directory != "" {315 return candidate.Directory316 }317 }318 }319 return filepath.ToSlash(filepath.Join("games", directoryName(game.Title, game.ID)))320}321322func resolveMapDirectory(archiveRoot, gameDirectory string, full *apiMapFull) string {323 var gameFile struct {324 Maps []struct {325 Directory string `json:"directory"`326 ID int64 `json:"id"`327 } `json:"maps"`328 }329 path := filepath.Join(archiveRoot, gameDirectory, "game.json")330 if err := readJSON(path, &gameFile); err == nil {331 for _, candidate := range gameFile.Maps {332 if candidate.ID == full.ID && candidate.Directory != "" {333 return candidate.Directory334 }335 }336 }337 return filepath.ToSlash(filepath.Join(gameDirectory, "maps", directoryName(full.Title, full.ID)))338}339340// directoryName names a game or map the archive has never seen. Latin letters341// with diacritics fold to their base form so titles such as "Pokémon" keep a342// readable directory rather than losing the character to a separator.343func directoryName(title string, id int64) string {344 var builder strings.Builder345 previousDash := false346 for _, r := range strings.ToLower(title) {347 if folded, ok := foldedLatin[r]; ok {348 r = folded349 }350 switch {351 case r >= 'a' && r <= 'z', r >= '0' && r <= '9':352 builder.WriteRune(r)353 previousDash = false354 case !previousDash:355 builder.WriteRune('-')356 previousDash = true357 }358 }359 return strings.Trim(builder.String(), "-") + "-" + strconv.FormatInt(id, 10)360}361362var foldedLatin = map[rune]rune{363 'á': 'a', 'à': 'a', 'â': 'a', 'ä': 'a', 'ã': 'a', 'å': 'a',364 'é': 'e', 'è': 'e', 'ê': 'e', 'ë': 'e',365 'í': 'i', 'ì': 'i', 'î': 'i', 'ï': 'i',366 'ó': 'o', 'ò': 'o', 'ô': 'o', 'ö': 'o', 'õ': 'o', 'ø': 'o',367 'ú': 'u', 'ù': 'u', 'û': 'u', 'ü': 'u',368 'ñ': 'n', 'ç': 'c', 'ß': 's', 'ý': 'y',369}370371func writeFile(path string, data []byte) error {372 if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {373 return err374 }375 return os.WriteFile(path, data, 0o644)376}377378func writeJSON(path string, value any) error {379 data, err := json.MarshalIndent(value, "", " ")380 if err != nil {381 return err382 }383 return writeFile(path, append(data, '\n'))384}385386func readJSON(path string, into any) error {387 data, err := os.ReadFile(path)388 if err != nil {389 return err390 }391 return json.Unmarshal(data, into)392}