1// Command tiles prepares bounded, multi-resolution raster tile pyramids from2// the FMG archive. Complete photographic source levels retain their encoded3// tiles; pixel-art levels are normalized to lossless PNG. Missing lower levels4// are derived from the next complete or generated level.5package main67import (8 "encoding/base64"9 "encoding/json"10 "errors"11 "flag"12 "fmt"13 "image"14 "image/color"15 "image/draw"16 "image/jpeg"17 "image/png"18 "io"19 "os"20 "path/filepath"21 "sort"22 "strconv"23 "strings"2425 // WebP sources decode through image.Decode like any other; Go encodes26 // none, so levels derived from them are written as JPEG or PNG.27 _ "golang.org/x/image/webp"2829 "github.com/FelineStateMachine/atlas/internal/arcgismap"30 "github.com/FelineStateMachine/atlas/internal/blend"31 "github.com/FelineStateMachine/atlas/internal/ignmap"32 "github.com/FelineStateMachine/atlas/internal/mgdoc"33 "github.com/FelineStateMachine/atlas/internal/pbmap"34 "github.com/FelineStateMachine/atlas/internal/trekmap"35)3637const (38 referenceZoom = 1339 firstTile = 406440 tileSize = 25641 worldSize = 819242 // worldZoomLevels is how far the world square sits above the level that43 // holds a map in one tile: 8192 units of world across, 256 to a tile, so44 // five halvings. It is what makes a unit of world one pixel of the45 // reference level, whichever level that turns out to be.46 worldZoomLevels = 547)4849// A tile set's window is where its map sits in the source grid, and the games50// do not agree on it. The maps captured most recently are cut from a 32-tile51// square at zoom 13; Grand Theft Auto 5 is cut from a square at the origin52// whose deepest level is 43 tiles across and whose shallowest is 3. Both are53// the same kind of pyramid seen from a different height, so no one height is54// assumed: each layer is measured, and the zoom whose window collapses into a55// single tile is the floor its local zooms are counted from.56type tileFrame struct {57 // BaseZoom is the source zoom that holds the whole window in one tile, and58 // BaseTile is that tile. Local zoom 0 is this level, so a layer's local59 // zooms are its source zooms less BaseZoom.60 BaseZoom int61 BaseTile int62}6364// frameFor halves a tile set's declared window until it is a single tile. A set65// that declares nothing is measured against the shared window instead, which is66// where every map cut from a 32-tile square at zoom 13 lands.67func frameFor(set rawTileSet) (tileFrame, bool) {68 bounds := expectedBounds(set, set.MaxZoom)69 for shift := 0; shift <= set.MaxZoom; shift++ {70 x, y := bounds.X.Min>>shift, bounds.Y.Min>>shift71 if x != bounds.X.Max>>shift || y != bounds.Y.Max>>shift {72 continue73 }74 // The viewer places a pin from one origin for both axes, so a window75 // whose axes come to rest on different tiles would draw the map in one76 // place and its pins in another. Nothing in the archive is shaped that77 // way; a layer that is says so rather than being silently a tile out.78 if x != y {79 return tileFrame{}, false80 }81 return tileFrame{BaseZoom: set.MaxZoom - shift, BaseTile: x}, true82 }83 return tileFrame{}, false84}8586// origin is the first tile of a source level: the base tile, halved back down87// the pyramid the same way the window was.88func (f tileFrame) origin(zoom int) int {89 return f.BaseTile << (zoom - f.BaseZoom)90}9192// grid is where this layer's world space sits in the source grid, in the terms93// the viewer needs to place a pin: the zoom whose pixels are the world's units,94// and the first tile of the window at that zoom. A map in the shared window95// lands back on zoom 13 and tile 4064, which is where the numbers the viewer96// used to assume came from.97func (f tileFrame) grid() gridSpec {98 reference := f.BaseZoom + worldZoomLevels99 return gridSpec{SourceZoom: reference, FirstTile: f.origin(reference)}100}101102type archive struct {103 Games []archiveGame `json:"games"`104}105106type archiveGame struct {107 Directory string `json:"directory"`108 Title string `json:"title"`109}110111type snapshotIndex struct {112 CapturedAt string `json:"capturedAt"`113 ContentHash string `json:"contentHash"`114 Kind string `json:"kind"`115}116117type rawMap struct {118 Title string `json:"title"`119 Slug string `json:"slug"`120 Config rawConfig `json:"config"`121 Game rawGame `json:"game"`122 Groups []rawGroup `json:"groups"`123}124125type rawGroup struct {126 Categories []struct {127 Locations []rawLocation `json:"locations"`128 } `json:"categories"`129}130131// rawLocation reads only what alignment needs: a name and a place. The132// coordinates arrive as numbers from translated captures and as either133// numbers or quoted strings from MapGenie's own snapshots.134type rawLocation struct {135 Title string `json:"title"`136 Latitude json.RawMessage `json:"latitude"`137 Longitude json.RawMessage `json:"longitude"`138}139140type rawGame struct {141 Slug string `json:"slug"`142}143144type rawConfig struct {145 TileSets []rawTileSet `json:"tile_sets"`146}147148type rawTileSet struct {149 Extension string `json:"extension"`150 MaxZoom int `json:"max_zoom"`151 MinZoom int `json:"min_zoom"`152 Name string `json:"name"`153 Path string `json:"path"`154 Bounds map[string]rawTileSetBound `json:"bounds"`155}156157type rawTileSetBound struct {158 X rawRange `json:"x"`159 Y rawRange `json:"y"`160}161162type rawRange struct {163 Min int `json:"min"`164 Max int `json:"max"`165}166167type tileRecord struct {168 ContentHash string `json:"contentHash"`169 Status string `json:"status"`170 TileSetID int64 `json:"tileSetId"`171 URL string `json:"url"`172 X int `json:"x"`173 Y int `json:"y"`174 Zoom int `json:"zoom"`175}176177type tileFile struct {178 Record tileRecord179 Path string180 Format string181}182183type tilePlan struct {184 AssetPath string185 Bounds *contentBounds186 // Levels holds every usable source level. Levels at or below MaxFullZoom187 // cover the map completely; deeper ones may be partially captured and are188 // carried as extra detail on top of the complete pyramid.189 Levels map[int][]tileFile190 Frame tileFrame191 Interpolate bool192 MapDir string193 MaxFullZoom int194 MaxSourceZoom int195 SourcePath string196 PreferredFormat string197 // The map this layer pictures, and the places its capture pins on it.198 // Two sources capturing the same game and map are pictures of the same199 // ground, and the shared names among their anchors are what align them.200 GameSlug string201 MapSlug string202 SetName string203 Anchors []blend.Anchor204 // Warp, when set, makes this a derived plan: the donor layer resampled205 // into another layer's world so both rasters answer to one grid.206 Warp *warpSpec207}208209// warpSpec says how a donor layer lands in a base layer's world.210type warpSpec struct {211 Donor *tilePlan212 Base *tilePlan213 Affine blend.Affine214 TargetZoom int // local zoom in the base frame the warp renders at215}216217// pendingPlan is one layer waiting to be derived, with the names its218// progress is reported under.219type pendingPlan struct {220 gameTitle string221 title string222 plan tilePlan223}224225type manifest struct {226 TileSize int `json:"tileSize"`227 Size int `json:"size"`228 Lenses []lensManifest `json:"lenses"`229}230231type lensManifest struct {232 SourcePath string `json:"sourcePath"`233 AssetPath string `json:"assetPath"`234 MinZoom int `json:"minZoom"`235 MaxZoom int `json:"maxZoom"`236 // FullZoom is the deepest level with complete coverage. Levels beyond it237 // exist only where Coverage says so; the viewer falls back to the parent238 // tile elsewhere.239 FullZoom int `json:"fullZoom"`240 SourceZoom int `json:"sourceZoom"`241 // Grid places this layer's world space in the source tile grid. Pins arrive242 // as latitudes and longitudes in that grid, so it is what lets them be laid243 // over the raster.244 Grid gridSpec `json:"grid"`245 Formats []string `json:"formats"`246 Bounds *contentBounds `json:"bounds,omitempty"`247 Interpolate bool `json:"interpolate"`248 Background string `json:"background,omitempty"`249 Coverage map[string]*levelCoverage `json:"coverage,omitempty"`250 // Stamp names everything this pyramid was derived from -- the captured251 // tiles and the tool that reduced them -- so a later run can tell that252 // nothing has moved and keep what it already built.253 Stamp string `json:"stamp,omitempty"`254 // Name and AlignedWith mark a warped variant: a donor layer resampled255 // into the world of the layer AlignedWith names, offered to the catalog256 // as an additional way to see that map. Ordinary layers leave both empty257 // and take their name from their map's own configuration.258 Name string `json:"name,omitempty"`259 AlignedWith string `json:"alignedWith,omitempty"`260}261262type gridSpec struct {263 SourceZoom int `json:"sourceZoom"`264 FirstTile int `json:"firstTile"`265}266267// levelCoverage is a row-major bitset over the [X,Y,W,H] tile window of one268// level. A set bit means the tile was written.269type levelCoverage struct {270 X int `json:"x"`271 Y int `json:"y"`272 W int `json:"w"`273 H int `json:"h"`274 Bits string `json:"bits"`275}276277// coverageBuilder accumulates the tiles actually written for one level and278// encodes them as a bitset once the level is finished.279type coverageBuilder struct {280 present map[[2]int]bool281 total int282}283284func (b *coverageBuilder) mark(x, y int) {285 if b.present == nil {286 b.present = make(map[[2]int]bool)287 }288 b.present[[2]int{x, y}] = true289}290291// build returns nil when the level is fully covered, so complete levels cost292// nothing in the manifest.293func (b *coverageBuilder) build() *levelCoverage {294 if len(b.present) == 0 || len(b.present) == b.total {295 return nil296 }297 minX, minY := int(^uint(0)>>1), int(^uint(0)>>1)298 maxX, maxY := -1, -1299 for key := range b.present {300 minX, minY = min(minX, key[0]), min(minY, key[1])301 maxX, maxY = max(maxX, key[0]), max(maxY, key[1])302 }303 width, height := maxX-minX+1, maxY-minY+1304 bits := make([]byte, (width*height+7)/8)305 for key := range b.present {306 index := (key[1]-minY)*width + (key[0] - minX)307 bits[index/8] |= 1 << (index % 8)308 }309 return &levelCoverage{310 X: minX,311 Y: minY,312 W: width,313 H: height,314 Bits: base64.StdEncoding.EncodeToString(bits),315 }316}317318type contentBounds struct {319 X int `json:"x"`320 Y int `json:"y"`321 Width int `json:"width"`322 Height int `json:"height"`323}324325func main() {326 source := flag.String("source", "", "FMG archive root")327 output := flag.String("output", "", "embedded tile destination")328 force := flag.Bool("force", false, "derive every pyramid again, even one nothing has changed under")329 flag.Parse()330331 if *source == "" || *output == "" {332 fmt.Fprintln(os.Stderr, "tiles: -source and -output are required")333 os.Exit(2)334 }335 if err := run(*source, *output, *force); err != nil {336 fmt.Fprintln(os.Stderr, "tiles:", err)337 os.Exit(1)338 }339}340341func run(source, output string, force bool) error {342 var index archive343 if err := readJSON(filepath.Join(source, "archive.json"), &index); err != nil {344 return err345 }346347 parent := filepath.Dir(output)348 if err := os.MkdirAll(parent, 0o755); err != nil {349 return fmt.Errorf("create output parent: %w", err)350 }351 temp, err := os.MkdirTemp(parent, "."+filepath.Base(output)+"-")352 if err != nil {353 return fmt.Errorf("create temporary output: %w", err)354 }355 defer os.RemoveAll(temp)356357 // What the last run left, so a layer nothing has changed under can be kept358 // where it is rather than derived again.359 built := manifestBySource(readManifest(output))360 var carried int361 derived := make(map[string]bool)362363 var pending []pendingPlan364 for _, game := range index.Games {365 mapDirs, err := filepath.Glob(filepath.Join(source, game.Directory, "maps", "*"))366 if err != nil {367 return err368 }369 sort.Strings(mapDirs)370 for _, mapDir := range mapDirs {371 info, err := os.Stat(mapDir)372 if err != nil {373 return err374 }375 if !info.IsDir() {376 continue377 }378 plans, title, reason, err := inspectMap(mapDir)379 if err != nil {380 return err381 }382 if len(plans) == 0 {383 fmt.Printf("skip %s / %s: %s\n", game.Title, filepath.Base(mapDir), reason)384 continue385 }386 for _, plan := range plans {387 pending = append(pending, pendingPlan{gameTitle: game.Title, title: title, plan: plan})388 }389 }390 }391392 // Two sources capturing the same game name the same directory for the393 // same map. Every collider takes its layer path into its name -- all of394 // them, so the outcome does not depend on the order the archive lists395 // its games in.396 byAssetPath := make(map[string][]int, len(pending))397 for index := range pending {398 byAssetPath[pending[index].plan.AssetPath] = append(399 byAssetPath[pending[index].plan.AssetPath], index)400 }401 for _, colliders := range byAssetPath {402 if len(colliders) < 2 {403 continue404 }405 for _, at := range colliders {406 pending[at].plan.AssetPath += "__" + slugify(pending[at].plan.SourcePath)407 }408 }409410 pending = append(pending, planWarps(pending)...)411412 out := manifest{TileSize: tileSize, Size: worldSize}413 for _, entry := range pending {414 plan := entry.plan415 stamp := planStamp(plan)416 if kept, ok := carry(built, plan, stamp, output, force); ok {417 // The window is read from the archive rather than derived418 // from the pyramid, so it is answered from the plan even419 // for a layer nothing has changed under.420 if plan.Warp == nil {421 kept.Grid = plan.Frame.grid()422 } else {423 kept.Grid = plan.Warp.Base.Frame.grid()424 }425 out.Lenses = append(out.Lenses, kept)426 carried++427 continue428 }429 fmt.Printf("tile %s / %s / %s\n", entry.gameTitle, entry.title, plan.SourcePath)430 var variant lensManifest431 var err error432 if plan.Warp == nil {433 variant, err = buildPyramid(temp, plan)434 } else {435 variant, err = buildWarpedPyramid(temp, plan)436 }437 if err != nil {438 return fmt.Errorf("%s / %s: %w", entry.title, plan.SourcePath, err)439 }440 variant.Stamp = stamp441 out.Lenses = append(out.Lenses, variant)442 derived[plan.AssetPath] = true443 }444 sort.Slice(out.Lenses, func(i, j int) bool {445 return out.Lenses[i].SourcePath < out.Lenses[j].SourcePath446 })447 if err := installPyramids(temp, output, out, derived); err != nil {448 return err449 }450 fmt.Printf("%d layers derived · %d carried over\n", len(derived), carried)451 return nil452}453454// carry keeps a pyramid the last run left, when nothing it was derived from has455// moved. Keeping one is doing nothing at all: it stays where it is, and only456// the layers that were derived again are installed over the top.457func carry(458 built map[string]lensManifest,459 plan tilePlan,460 stamp, output string,461 force bool,462) (lensManifest, bool) {463 if force {464 return lensManifest{}, false465 }466 entry, ok := built[plan.SourcePath]467 if !ok || entry.Stamp == "" || entry.Stamp != stamp || entry.AssetPath != plan.AssetPath {468 return lensManifest{}, false469 }470 if _, err := os.Stat(filepath.Join(output, plan.AssetPath)); err != nil {471 // Whatever the last run left is not there to be kept, so derive it.472 return lensManifest{}, false473 }474 return entry, true475}476477// installPyramids moves the layers that were derived into the output, takes out478// any layer the archive no longer offers, and writes the index last.479//480// A layer arrives by one rename, so a reader of the output never sees a481// half-written pyramid. The index naming the old stamps until the end is what482// makes a run that dies partway safe: the next run finds a stamp that does not483// match what is on disk and derives that layer again.484func installPyramids(temp, output string, out manifest, derived map[string]bool) error {485 if err := os.MkdirAll(output, 0o755); err != nil {486 return fmt.Errorf("create tile output: %w", err)487 }488 for asset := range derived {489 destination := filepath.Join(output, asset)490 if err := os.RemoveAll(destination); err != nil {491 return fmt.Errorf("replace %s: %w", asset, err)492 }493 if err := os.Rename(filepath.Join(temp, asset), destination); err != nil {494 return fmt.Errorf("install %s: %w", asset, err)495 }496 }497498 wanted := make(map[string]bool, len(out.Lenses))499 for _, variant := range out.Lenses {500 wanted[variant.AssetPath] = true501 }502 entries, err := os.ReadDir(output)503 if err != nil {504 return err505 }506 for _, entry := range entries {507 if !entry.IsDir() || wanted[entry.Name()] {508 continue509 }510 if err := os.RemoveAll(filepath.Join(output, entry.Name())); err != nil {511 return fmt.Errorf("remove %s: %w", entry.Name(), err)512 }513 }514515 staged := filepath.Join(temp, "index.json")516 if err := writeJSON(staged, out); err != nil {517 return err518 }519 return os.Rename(staged, filepath.Join(output, "index.json"))520}521522func inspectMap(mapDir string) ([]tilePlan, string, string, error) {523 var snapshots []snapshotIndex524 if err := readJSON(filepath.Join(mapDir, "snapshots", "index.json"), &snapshots); err != nil {525 if errors.Is(err, os.ErrNotExist) {526 return nil, "", "no snapshot index", nil527 }528 return nil, "", "", err529 }530 if len(snapshots) == 0 {531 return nil, "", "no captured snapshot", nil532 }533 sort.Slice(snapshots, func(i, j int) bool { return snapshots[i].CapturedAt < snapshots[j].CapturedAt })534535 // A capture from another source is translated into the MapGenie shape on536 // the way in; a MapGenie snapshot passes through untouched.537 latest := snapshots[len(snapshots)-1]538 doc, err := os.ReadFile(filepath.Join(mapDir, "snapshots", "map", latest.ContentHash+".json"))539 if err != nil {540 return nil, "", "", err541 }542 if doc, err = ignmap.MaybeTranslate(latest.Kind, doc); err != nil {543 return nil, "", "", err544 }545 if doc, err = pbmap.MaybeTranslate(latest.Kind, doc); err != nil {546 return nil, "", "", err547 }548 if doc, err = trekmap.MaybeTranslate(latest.Kind, doc); err != nil {549 return nil, "", "", err550 }551 if doc, err = arcgismap.MaybeTranslate(latest.Kind, doc); err != nil {552 return nil, "", "", err553 }554 var raw rawMap555 if err := json.Unmarshal(doc, &raw); err != nil {556 return nil, "", "", err557 }558 if len(raw.Config.TileSets) == 0 {559 return nil, raw.Title, "no configured tile sets", nil560 }561562 var records []tileRecord563 if err := readJSON(filepath.Join(mapDir, "tiles", "index.json"), &records); err != nil {564 if errors.Is(err, os.ErrNotExist) {565 return nil, raw.Title, "no tile index", nil566 }567 return nil, "", "", err568 }569 byPath := make(map[string]map[int][]tileRecord)570 for _, record := range records {571 if record.Status != "cached" || record.ContentHash == "" {572 continue573 }574 path := tileSetPath(record.URL, record.Zoom)575 if path == "" {576 continue577 }578 if byPath[path] == nil {579 byPath[path] = make(map[int][]tileRecord)580 }581 byPath[path][record.Zoom] = append(byPath[path][record.Zoom], record)582 }583584 plans := make([]tilePlan, 0, len(raw.Config.TileSets))585 for _, set := range raw.Config.TileSets {586 frame, ok := frameFor(set)587 if !ok {588 return nil, raw.Title, fmt.Sprintf("layer %q has no square window", set.Name), nil589 }590 levels := make(map[int][]tileFile)591 maxFullZoom, maxSourceZoom := -1, -1592 for zoom, level := range byPath[set.Path] {593 if zoom < frame.BaseZoom || zoom > set.MaxZoom {594 continue595 }596 files, full, err := readLevel(mapDir, set, zoom, level)597 if err != nil {598 return nil, "", "", err599 }600 if len(files) == 0 {601 continue602 }603 levels[zoom] = files604 maxSourceZoom = max(maxSourceZoom, zoom)605 if full {606 maxFullZoom = max(maxFullZoom, zoom)607 }608 }609 if maxFullZoom < frame.BaseZoom {610 return nil, raw.Title, fmt.Sprintf("layer %q has no complete source level", set.Name), nil611 }612 // A partial level is only usable if every level beneath it exists;613 // otherwise the viewer would jump across a missing resolution.614 for zoom := maxFullZoom + 1; zoom <= maxSourceZoom; zoom++ {615 if len(levels[zoom]) == 0 {616 maxSourceZoom = zoom - 1617 break618 }619 }620 assetPath := raw.Game.Slug + "__" + raw.Slug621 if len(raw.Config.TileSets) > 1 {622 assetPath += "__" + slugify(set.Name)623 }624 plans = append(plans, tilePlan{625 AssetPath: assetPath,626 Bounds: contentBoundsFor(levels[maxFullZoom], maxFullZoom, frame),627 Frame: frame,628 Levels: levels,629 Interpolate: !isPixelArt(raw.Game.Slug),630 MapDir: mapDir,631 MaxFullZoom: maxFullZoom,632 MaxSourceZoom: maxSourceZoom,633 PreferredFormat: normalizeFormat(set.Extension),634 SourcePath: set.Path,635 GameSlug: raw.Game.Slug,636 MapSlug: raw.Slug,637 SetName: set.Name,638 Anchors: anchorsOf(raw, frame.grid()),639 })640 }641 return plans, raw.Title, "", nil642}643644// anchorsOf lands every named location in the layer's own world pixels.645// These are the places alignment stands on when another source pictures the646// same map.647func anchorsOf(raw rawMap, grid gridSpec) []blend.Anchor {648 var anchors []blend.Anchor649 for _, group := range raw.Groups {650 for _, category := range group.Categories {651 for _, location := range category.Locations {652 latitude, ok := numberValue(location.Latitude)653 if !ok {654 continue655 }656 longitude, ok := numberValue(location.Longitude)657 if !ok {658 continue659 }660 anchors = append(anchors, blend.Anchor{661 Title: location.Title,662 X: mgdoc.ProjectX(longitude, grid.SourceZoom, grid.FirstTile),663 Y: mgdoc.ProjectY(latitude, grid.SourceZoom, grid.FirstTile),664 })665 }666 }667 }668 return anchors669}670671// numberValue reads the coordinate spellings the snapshots hold: a number,672// or the same number in quotes.673func numberValue(raw json.RawMessage) (float64, bool) {674 trimmed := strings.TrimSpace(string(raw))675 trimmed = strings.Trim(trimmed, `"`)676 if trimmed == "" || trimmed == "null" {677 return 0, false678 }679 value, err := strconv.ParseFloat(trimmed, 64)680 return value, err == nil681}682683// readLevel collects every cached tile of one source level. It reports whether684// the level covers its expected window completely; partial levels are still685// returned so they can be carried as extra detail above the complete pyramid.686func readLevel(mapDir string, set rawTileSet, zoom int, records []tileRecord) ([]tileFile, bool, error) {687 bounds := expectedBounds(set, zoom)688 expected := (bounds.X.Max - bounds.X.Min + 1) * (bounds.Y.Max - bounds.Y.Min + 1)689690 seen := make(map[[2]int]bool, len(records))691 files := make([]tileFile, 0, len(records))692 for _, record := range records {693 key := [2]int{record.X, record.Y}694 if seen[key] || record.X < bounds.X.Min || record.X > bounds.X.Max ||695 record.Y < bounds.Y.Min || record.Y > bounds.Y.Max {696 continue697 }698 path, err := sourceTilePath(mapDir, record)699 if errors.Is(err, os.ErrNotExist) {700 continue701 }702 if err != nil {703 return nil, false, err704 }705 seen[key] = true706 files = append(files, tileFile{707 Record: record,708 Path: path,709 Format: normalizeFormat(strings.TrimPrefix(filepath.Ext(path), ".")),710 })711 }712 sort.Slice(files, func(i, j int) bool {713 if files[i].Record.Y != files[j].Record.Y {714 return files[i].Record.Y < files[j].Record.Y715 }716 return files[i].Record.X < files[j].Record.X717 })718 return files, expected > 0 && len(files) == expected, nil719}720721func expectedBounds(set rawTileSet, zoom int) rawTileSetBound {722 if bounds, ok := set.Bounds[strconv.Itoa(zoom)]; ok {723 return bounds724 }725 minimum, maximum := worldTileRange(zoom)726 return rawTileSetBound{727 X: rawRange{Min: minimum, Max: maximum},728 Y: rawRange{Min: minimum, Max: maximum},729 }730}731732func worldTileRange(zoom int) (int, int) {733 const worldTilesAtReference = worldSize / tileSize734 if zoom <= referenceZoom {735 shift := referenceZoom - zoom736 return firstTile >> shift, ((firstTile + worldTilesAtReference) >> shift) - 1737 }738 shift := zoom - referenceZoom739 return firstTile << shift, ((firstTile + worldTilesAtReference) << shift) - 1740}741742func buildPyramid(root string, plan tilePlan) (lensManifest, error) {743 maxZoom := plan.MaxSourceZoom - plan.Frame.BaseZoom744 fullZoom := plan.MaxFullZoom - plan.Frame.BaseZoom745 formats := make([]string, maxZoom+1)746 coverage := make(map[string]*levelCoverage)747748 // Resolve the background tile once, from the level we trust. Reusing one749 // hash across every level keeps a sparse deep level from having its handful750 // of real tiles voted "background" by its own majority, and skipping the751 // omission entirely when no flat colour could be sampled means we never752 // leave a hole we cannot paint over.753 full := plan.Levels[plan.MaxFullZoom]754 placeholder := placeholderHash(full)755 backgroundHex, err := placeholderColor(full, placeholder)756 if err != nil {757 return lensManifest{}, err758 }759 if backgroundHex == "" {760 placeholder = ""761 }762 background := parseHexColor(backgroundHex)763764 // The deepest complete level is the one we trust. Everything below it is765 // folded down from it rather than taken from whatever intermediate levels766 // the capture happens to hold: those are separately-encoded JPEGs of767 // varying completeness, and a box filter over the level we already have is768 // both more consistent and never leaves a level empty.769 format, mask, err := copyLevel(root, plan, plan.MaxFullZoom, fullZoom, full, placeholder)770 if err != nil {771 return lensManifest{}, err772 }773 formats[fullZoom] = format774 if mask != nil {775 coverage[strconv.Itoa(fullZoom)] = mask776 }777 for localZoom := fullZoom - 1; localZoom >= 0; localZoom-- {778 derivedFormat := plan.PreferredFormat779 if !plan.Interpolate || derivedFormat == "png" {780 derivedFormat = "png"781 }782 // A WebP source level copies through as it came, but nothing here783 // encodes WebP, so the levels folded down from it take JPEG.784 if derivedFormat == "webp" {785 derivedFormat = "jpg"786 }787 mask, err := deriveLevel(root, plan.AssetPath, localZoom, formats[localZoom+1], derivedFormat, !plan.Interpolate, background)788 if err != nil {789 return lensManifest{}, err790 }791 formats[localZoom] = derivedFormat792 if mask != nil {793 coverage[strconv.Itoa(localZoom)] = mask794 }795 }796797 // Partially captured levels above it. These have gaps by definition, so the798 // viewer falls back to the parent tile wherever coverage says nothing. A799 // level that survives with nothing in it ends the pyramid: advertising a800 // zoom with no tiles would only produce misses.801 for localZoom := fullZoom + 1; localZoom <= maxZoom; localZoom++ {802 sourceZoom := localZoom + plan.Frame.BaseZoom803 format, mask, err := copyLevel(root, plan, sourceZoom, localZoom, plan.Levels[sourceZoom], placeholder)804 if err != nil {805 return lensManifest{}, err806 }807 if mask == nil {808 maxZoom = localZoom - 1809 formats = formats[:localZoom]810 break811 }812 formats[localZoom] = format813 coverage[strconv.Itoa(localZoom)] = mask814 }815816 if len(coverage) == 0 {817 coverage = nil818 }819 return lensManifest{820 SourcePath: plan.SourcePath,821 AssetPath: plan.AssetPath,822 MinZoom: 0,823 MaxZoom: maxZoom,824 FullZoom: fullZoom,825 SourceZoom: plan.MaxSourceZoom,826 Grid: plan.Frame.grid(),827 Formats: formats,828 Bounds: plan.Bounds,829 Interpolate: plan.Interpolate,830 Background: backgroundHex,831 Coverage: coverage,832 }, nil833}834835// parseHexColor turns "#rrggbb" into a colour, defaulting to opaque black when836// the level had no flat background to sample.837func parseHexColor(value string) color.Color {838 if len(value) != 7 || value[0] != '#' {839 return color.NRGBA{A: 0xff}840 }841 component, err := strconv.ParseUint(value[1:], 16, 32)842 if err != nil {843 return color.NRGBA{A: 0xff}844 }845 return color.NRGBA{846 R: uint8(component >> 16),847 G: uint8(component >> 8),848 B: uint8(component),849 A: 0xff,850 }851}852853// copyLevel writes one source level, omitting tiles that are byte-identical854// copies of the level's background placeholder. Source bytes are passed through855// untouched: re-encoding cannot add detail the capture never had, and for the856// JPEG-sourced pixel-art maps it only stores compression noise at a larger size.857func copyLevel(858 root string,859 plan tilePlan,860 sourceZoom int,861 localZoom int,862 files []tileFile,863 placeholder string,864) (string, *levelCoverage, error) {865 if len(files) == 0 {866 return "", nil, fmt.Errorf("source level %d has no tiles", sourceZoom)867 }868 outputFormat := files[0].Format869 origin := plan.Frame.origin(sourceZoom)870 span := 1 << localZoom871 builder := &coverageBuilder{total: span * span}872873 for _, file := range files {874 if file.Format != outputFormat {875 return "", nil, fmt.Errorf("source level %d mixes %s and %s", sourceZoom, outputFormat, file.Format)876 }877 if placeholder != "" && file.Record.ContentHash == placeholder {878 continue879 }880 x := file.Record.X - origin881 y := file.Record.Y - origin882 if err := copyFile(file.Path, tilePath(root, plan.AssetPath, localZoom, x, y, outputFormat)); err != nil {883 return "", nil, err884 }885 builder.mark(x, y)886 }887 return outputFormat, builder.build(), nil888}889890// placeholderHash returns the content hash of the level's background tile, or891// "" when no single tile holds a majority. The majority rule matches892// contentBoundsFor: only an unambiguous filler tile is treated as empty space,893// so a map whose real content happens to repeat is never punched full of holes.894func placeholderHash(files []tileFile) string {895 counts := make(map[string]int, len(files))896 var dominant string897 for _, file := range files {898 hash := file.Record.ContentHash899 counts[hash]++900 if counts[hash] > counts[dominant] {901 dominant = hash902 }903 }904 if dominant == "" || counts[dominant] <= len(files)/2 {905 return ""906 }907 return dominant908}909910// placeholderColor reports the flat colour of the level's background tile so911// the viewer can paint it behind the raster layer. Tiles we omit then look912// exactly as they did when they were shipped as files. It returns "" when the913// tile is not flat enough for a single colour to stand in for it.914func placeholderColor(files []tileFile, hash string) (string, error) {915 if hash == "" {916 return "", nil917 }918 var sample string919 for _, file := range files {920 if file.Record.ContentHash == hash {921 sample = file.Path922 break923 }924 }925 if sample == "" {926 return "", nil927 }928 value, err := decodeImage(sample)929 if err != nil {930 return "", err931 }932 bounds := value.Bounds()933 var sums [3]int64934 var count int64935 minimum := [3]uint32{^uint32(0), ^uint32(0), ^uint32(0)}936 maximum := [3]uint32{}937 for y := bounds.Min.Y; y < bounds.Max.Y; y += 4 {938 for x := bounds.Min.X; x < bounds.Max.X; x += 4 {939 r, g, b, _ := value.At(x, y).RGBA()940 for channel, component := range [3]uint32{r >> 8, g >> 8, b >> 8} {941 sums[channel] += int64(component)942 minimum[channel] = min(minimum[channel], component)943 maximum[channel] = max(maximum[channel], component)944 }945 count++946 }947 }948 if count == 0 {949 return "", nil950 }951 for channel := range minimum {952 if maximum[channel]-minimum[channel] > 12 {953 return "", nil954 }955 }956 return fmt.Sprintf("#%02x%02x%02x",957 sums[0]/count, sums[1]/count, sums[2]/count), nil958}959960// deriveLevel folds the level above down by two. Parents with no surviving961// children are skipped, so omitted background propagates down the pyramid.962func deriveLevel(963 root, assetPath string,964 zoom int,965 childFormat, outputFormat string,966 nearest bool,967 background color.Color,968) (*levelCoverage, error) {969 childRoot := filepath.Join(root, assetPath, strconv.Itoa(zoom+1))970 parents := make(map[[2]int]bool)971 err := filepath.WalkDir(childRoot, func(path string, entry os.DirEntry, walkErr error) error {972 if errors.Is(walkErr, os.ErrNotExist) {973 return nil974 }975 if walkErr != nil {976 return walkErr977 }978 if entry.IsDir() || normalizeFormat(strings.TrimPrefix(filepath.Ext(path), ".")) != childFormat {979 return nil980 }981 y, err := strconv.Atoi(strings.TrimSuffix(entry.Name(), filepath.Ext(entry.Name())))982 if err != nil {983 return fmt.Errorf("decode child y from %s: %w", path, err)984 }985 x, err := strconv.Atoi(filepath.Base(filepath.Dir(path)))986 if err != nil {987 return fmt.Errorf("decode child x from %s: %w", path, err)988 }989 parents[[2]int{x / 2, y / 2}] = true990 return nil991 })992 if err != nil {993 return nil, err994 }995996 keys := make([][2]int, 0, len(parents))997 for key := range parents {998 keys = append(keys, key)999 }1000 sort.Slice(keys, func(i, j int) bool {1001 if keys[i][1] != keys[j][1] {1002 return keys[i][1] < keys[j][1]1003 }1004 return keys[i][0] < keys[j][0]1005 })1006 span := 1 << zoom1007 builder := &coverageBuilder{total: span * span}1008 for _, parent := range keys {1009 composite := image.NewNRGBA(image.Rect(0, 0, tileSize*2, tileSize*2))1010 // Quadrants whose child was omitted as background must still carry the1011 // background colour: a derived JPEG has no alpha to fall back on.1012 draw.Draw(composite, composite.Bounds(), &image.Uniform{background}, image.Point{}, draw.Src)1013 for offsetY := 0; offsetY < 2; offsetY++ {1014 for offsetX := 0; offsetX < 2; offsetX++ {1015 childX := parent[0]*2 + offsetX1016 childY := parent[1]*2 + offsetY1017 path := tilePath(root, assetPath, zoom+1, childX, childY, childFormat)1018 child, err := decodeImage(path)1019 if errors.Is(err, os.ErrNotExist) {1020 continue1021 }1022 if err != nil {1023 return nil, err1024 }1025 target := image.Rect(offsetX*tileSize, offsetY*tileSize, (offsetX+1)*tileSize, (offsetY+1)*tileSize)1026 draw.Draw(composite, target, child, child.Bounds().Min, draw.Src)1027 }1028 }1029 parentImage := downsample(composite, nearest)1030 destination := tilePath(root, assetPath, zoom, parent[0], parent[1], outputFormat)1031 if err := encodeImage(destination, parentImage, outputFormat); err != nil {1032 return nil, err1033 }1034 builder.mark(parent[0], parent[1])1035 }1036 return builder.build(), nil1037}10381039func downsample(source *image.NRGBA, nearest bool) *image.NRGBA {1040 target := image.NewNRGBA(image.Rect(0, 0, tileSize, tileSize))1041 for y := 0; y < tileSize; y++ {1042 for x := 0; x < tileSize; x++ {1043 destination := target.PixOffset(x, y)1044 sourceX, sourceY := x*2, y*21045 if nearest {1046 copy(target.Pix[destination:destination+4], source.Pix[source.PixOffset(sourceX, sourceY):][:4])1047 continue1048 }1049 offsets := [4]int{1050 source.PixOffset(sourceX, sourceY),1051 source.PixOffset(sourceX+1, sourceY),1052 source.PixOffset(sourceX, sourceY+1),1053 source.PixOffset(sourceX+1, sourceY+1),1054 }1055 for channel := 0; channel < 4; channel++ {1056 sum := 01057 for _, offset := range offsets {1058 sum += int(source.Pix[offset+channel])1059 }1060 target.Pix[destination+channel] = uint8((sum + 2) / 4)1061 }1062 }1063 }1064 return target1065}10661067func contentBoundsFor(files []tileFile, zoom int, frame tileFrame) *contentBounds {1068 counts := make(map[string]int)1069 var placeholder string1070 for _, file := range files {1071 hash := file.Record.ContentHash1072 counts[hash]++1073 if counts[hash] > counts[placeholder] {1074 placeholder = hash1075 }1076 }1077 excludePlaceholder := counts[placeholder] > len(files)/21078 minX, minY := int(^uint(0)>>1), int(^uint(0)>>1)1079 maxX, maxY := -1, -11080 for _, file := range files {1081 if excludePlaceholder && file.Record.ContentHash == placeholder {1082 continue1083 }1084 minX = min(minX, file.Record.X)1085 minY = min(minY, file.Record.Y)1086 maxX = max(maxX, file.Record.X)1087 maxY = max(maxY, file.Record.Y)1088 }1089 if maxX < minX || maxY < minY {1090 return nil1091 }1092 // Bounds are reported in world space, which is sized at this layer's own1093 // reference zoom -- the level whose pixels are its units. A shallower level1094 // covers more world per tile and a deeper one covers less, so the tile pitch1095 // is scaled both ways.1096 reference := frame.BaseZoom + worldZoomLevels1097 origin := frame.origin(zoom)1098 pitch, divisor := tileSize, 11099 if zoom < reference {1100 pitch = tileSize << (reference - zoom)1101 } else if zoom > reference {1102 divisor = 1 << (zoom - reference)1103 }1104 bounds := &contentBounds{1105 X: (minX - origin) * pitch / divisor,1106 Y: (minY - origin) * pitch / divisor,1107 Width: (maxX - minX + 1) * pitch / divisor,1108 Height: (maxY - minY + 1) * pitch / divisor,1109 }1110 if bounds.X == 0 && bounds.Y == 0 && bounds.Width == worldSize && bounds.Height == worldSize {1111 return nil1112 }1113 return bounds1114}11151116func sourceTilePath(mapDir string, record tileRecord) (string, error) {1117 pattern := filepath.Join(1118 mapDir,1119 "tiles",1120 "set-"+strconv.FormatInt(record.TileSetID, 10),1121 strconv.Itoa(record.Zoom),1122 strconv.Itoa(record.X),1123 strconv.Itoa(record.Y)+".*",1124 )1125 matches, err := filepath.Glob(pattern)1126 if err != nil {1127 return "", err1128 }1129 if len(matches) == 0 {1130 return "", os.ErrNotExist1131 }1132 sort.Strings(matches)1133 return matches[0], nil1134}11351136// tileSetPath recovers the layer path a record belongs to: everything between1137// the source's marker segment and "/<zoom>/". MapGenie serves tiles under1138// /games/, IGN under /wikimaps/, Piggyback under /tiles/.1139func tileSetPath(rawURL string, zoom int) string {1140 for _, marker := range []string{"/games/", "/wikimaps/", "/tiles/"} {1141 _, rest, found := strings.Cut(rawURL, marker)1142 if !found {1143 continue1144 }1145 path, _, found := strings.Cut(rest, "/"+strconv.Itoa(zoom)+"/")1146 if !found {1147 continue1148 }1149 return path1150 }1151 return ""1152}11531154func tilePath(root, assetPath string, zoom, x, y int, format string) string {1155 return filepath.Join(root, assetPath, strconv.Itoa(zoom), strconv.Itoa(x), strconv.Itoa(y)+"."+format)1156}11571158func copyFile(source, destination string) error {1159 if err := os.MkdirAll(filepath.Dir(destination), 0o755); err != nil {1160 return err1161 }1162 input, err := os.Open(source)1163 if err != nil {1164 return err1165 }1166 defer input.Close()1167 output, err := os.Create(destination)1168 if err != nil {1169 return err1170 }1171 if _, err := io.Copy(output, input); err != nil {1172 output.Close()1173 return err1174 }1175 return output.Close()1176}11771178func decodeImage(path string) (image.Image, error) {1179 file, err := os.Open(path)1180 if err != nil {1181 return nil, err1182 }1183 defer file.Close()1184 value, _, err := image.Decode(file)1185 if err != nil {1186 return nil, fmt.Errorf("decode %s: %w", path, err)1187 }1188 return value, nil1189}11901191func encodeImage(path string, value image.Image, format string) error {1192 if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {1193 return err1194 }1195 file, err := os.Create(path)1196 if err != nil {1197 return err1198 }1199 var encodeErr error1200 switch format {1201 case "png":1202 encoder := png.Encoder{CompressionLevel: png.BestSpeed}1203 encodeErr = encoder.Encode(file, value)1204 default:1205 encodeErr = jpeg.Encode(file, value, &jpeg.Options{Quality: 90})1206 }1207 if encodeErr != nil {1208 file.Close()1209 return fmt.Errorf("encode %s: %w", path, encodeErr)1210 }1211 return file.Close()1212}12131214func writeJSON(path string, value any) error {1215 data, err := json.Marshal(value)1216 if err != nil {1217 return err1218 }1219 if err := os.WriteFile(path, append(data, '\n'), 0o644); err != nil {1220 return fmt.Errorf("write %s: %w", path, err)1221 }1222 return nil1223}12241225func readJSON(path string, destination any) error {1226 data, err := os.ReadFile(path)1227 if err != nil {1228 return fmt.Errorf("read %s: %w", path, err)1229 }1230 if err := json.Unmarshal(data, destination); err != nil {1231 return fmt.Errorf("decode %s: %w", path, err)1232 }1233 return nil1234}12351236func normalizeFormat(value string) string {1237 value = strings.ToLower(strings.TrimPrefix(value, "."))1238 switch value {1239 case "jpeg":1240 return "jpg"1241 case "png", "webp":1242 return value1243 default:1244 return "jpg"1245 }1246}12471248// isPixelArt marks maps drawn on a pixel grid. They are folded down with1249// nearest-neighbour sampling and rendered without interpolation, so the grid1250// survives instead of being blurred into it.1251func isPixelArt(gameSlug string) bool {1252 if strings.HasPrefix(gameSlug, "pokemon-") {1253 return true1254 }1255 switch gameSlug {1256 case "old-school-runescape-DISABLED":1257 return true1258 }1259 return false1260}12611262func slugify(value string) string {1263 return strings.Trim(strings.Map(func(r rune) rune {1264 switch {1265 case r >= 'a' && r <= 'z', r >= '0' && r <= '9':1266 return r1267 case r >= 'A' && r <= 'Z':1268 return r + ('a' - 'A')1269 default:1270 return '-'1271 }1272 }, value), "-")1273}