1package archive23import (4 "errors"5 "fmt"6 "os"7 "path/filepath"8 "sort"9 "strconv"10 "strings"11)1213// The captured rasters, as against the derived ones.14//15// A crawler writes what a tile server answered, tile for tile, under the path16// the server used. Nothing composes those bytes: they are frames, and the17// deriver folds them into a pyramid first. So the vocabulary here is the18// crawler's -- a level, a column, a row, the hash of what came back -- and it19// stops at this package's edge, exactly as the capture vocabulary does.20//21// A record's status says whether the bytes are here. Only "cached" ones are,22// and only those are answered for: a level that answered 404 for its corners23// records the refusal so a re-crawl does not ask again, and a deriver reading24// those records would otherwise count a tile that does not exist.2526// TileRef is one captured raster: where it sits in the publisher's own grid, and27// the hash of what came back.28type TileRef struct {29 // TileSet is the publisher's path for the pyramid this belongs to,30 // recovered from the URL it was fetched at.31 TileSet string32 // Zoom, X and Y are the publisher's own numbering, not a local one.33 Zoom, X, Y int34 // ContentHash is the archive's content address of the bytes: SHA-256,35 // lowercase hex. It is what a derivation stamp is taken over, so a deriver36 // never has to read a raster to know whether anything moved.37 ContentHash string38 // SetID is the numbered directory the bytes sit in. It is the layout's, not39 // the vocabulary's, and only Raster needs it.40 SetID int6441}4243// Tiles lists a world's captured rasters that are actually on disk, grouped by44// the publisher's pyramid path and then by level, each level sorted by row and45// then column -- the order a level is walked in, and the order its stamp is46// taken in.47//48// A world with no captured raster is not an error. Some worlds are pictures49// somebody else derived, and some are simply not crawled yet.50func (a *Archive) Tiles(w WorldRef) (map[string]map[int][]TileRef, error) {51 var records []struct {52 ContentHash string `json:"contentHash"`53 Status string `json:"status"`54 TileSetID int64 `json:"tileSetId"`55 URL string `json:"url"`56 X int `json:"x"`57 Y int `json:"y"`58 Zoom int `json:"zoom"`59 }60 path := filepath.Join(w.dir, "tiles", "index.json")61 if err := readJSON(path, &records); err != nil {62 if errors.Is(err, os.ErrNotExist) {63 return nil, fmt.Errorf("%w: %s has no tile index", ErrNotReady, w.dir)64 }65 return nil, err66 }67 out := make(map[string]map[int][]TileRef)68 for _, record := range records {69 if record.Status != "cached" || record.ContentHash == "" {70 continue71 }72 set := tileSetOf(record.URL, record.Zoom)73 if set == "" {74 continue75 }76 if out[set] == nil {77 out[set] = make(map[int][]TileRef)78 }79 out[set][record.Zoom] = append(out[set][record.Zoom], TileRef{80 TileSet: set,81 Zoom: record.Zoom,82 X: record.X,83 Y: record.Y,84 ContentHash: record.ContentHash,85 SetID: record.TileSetID,86 })87 }88 for _, levels := range out {89 for _, level := range levels {90 sort.Slice(level, func(i, j int) bool {91 if level[i].Y != level[j].Y {92 return level[i].Y < level[j].Y93 }94 return level[i].X < level[j].X95 })96 }97 }98 return out, nil99}100101// Raster is where a captured tile's bytes sit, and the encoding the file name102// gives them. A record whose file is gone reports os.ErrNotExist, which a103// deriver treats as a tile that is not there rather than as a failure: an104// archive is filled by hand and an interrupted crawl leaves records ahead of105// bytes.106func (a *Archive) Raster(w WorldRef, t TileRef) (path, format string, err error) {107 pattern := filepath.Join(w.dir, "tiles",108 "set-"+strconv.FormatInt(t.SetID, 10),109 strconv.Itoa(t.Zoom), strconv.Itoa(t.X), strconv.Itoa(t.Y)+".*")110 matches, err := filepath.Glob(pattern)111 if err != nil {112 return "", "", err113 }114 if len(matches) == 0 {115 return "", "", os.ErrNotExist116 }117 sort.Strings(matches)118 return matches[0], NormalizeFormat(strings.TrimPrefix(filepath.Ext(matches[0]), ".")), nil119}120121// NormalizeFormat spells an encoding the one way the pipeline spells it. JPEG122// answers to two names on disk and to one everywhere else; anything unrecognized123// is read as JPEG, which is what a tile server that names nothing serves.124func NormalizeFormat(value string) string {125 value = strings.ToLower(strings.TrimPrefix(value, "."))126 switch value {127 case "jpeg":128 return "jpg"129 case "png", "webp":130 return value131 default:132 return "jpg"133 }134}135136// tileSetOf recovers the publisher's pyramid path from the URL a tile was137// fetched at: everything between the publisher's own marker segment and the138// zoom. The markers are the layout's, kept verbatim like the rest of it --139// MapGenie serves under /games/, IGN under /wikimaps/, and everyone else under140// /tiles/, including the rendered basemaps whose URLs were never fetchable at141// all.142func tileSetOf(rawURL string, zoom int) string {143 for _, marker := range []string{"/games/", "/wikimaps/", "/tiles/"} {144 _, rest, found := strings.Cut(rawURL, marker)145 if !found {146 continue147 }148 path, _, found := strings.Cut(rest, "/"+strconv.Itoa(zoom)+"/")149 if !found {150 continue151 }152 return path153 }154 return ""155}