A crawl reads the window a layer's tiles occupy at each zoom and asks for what is inside it. Some games publish no window at all -- Fallout 76 and Marathon among them -- and the crawl then had nothing to ask for: every level was skipped, and the map came away with its metadata and not one tile. What every map does say is where it opens, and that point is on the map by definition. So the shallowest level is now found by fetching the tile under it and spreading outward from whatever answers, eight ways at a time, until the frontier runs out. Appalachia's shallowest level is a single tile, and its neighbours answer 403 exactly as an unpublished tile always has. The levels below need no window of their own: they follow the tiles that held content, which is what a published layer does after its first level anyway. A layer with no window also leaves the axis probe nothing to measure, so the search settles that instead -- whichever way round the opening tile answers is the way the rest is read. The probe itself now waits for a level at least two tiles across before calling the question answered; Skyrim's shallowest window is one tile square, so its axes were never actually measured. The search is capped at 512 tiles. The shallowest level of a map is a handful of them, so a search that has read more than that has misread something and should stop rather than walk the world.
- commit
- a8a6da93c9f424d6bd6d6201310984f714af17fe
- parent
- f528bdcd49e657457b1403a16fa65fdaf3ae7f15
- author
- Dami <dami.etoile@gmail.com>
| M | tools/ | 6 | +- |
| M | tools/ | 58 | ++++++++- |
| M | tools/ | 72 | +++++++++++- |
| A | tools/ | 192 | ++++++++++++++++++++++++++++++ |
@@ -194,7 +194,11 @@ type apiMapFull struct {
194194 Title string `json:"title"`195195 Slug string `json:"slug"`196196 URL string `json:"url"`197- Config struct {197+ // Where the map opens, which is the one point known to be on it. A layer198+ // published without a window is searched outward from here.199+ InitialLatitude json.RawMessage `json:"initial_latitude"`200+ InitialLongitude json.RawMessage `json:"initial_longitude"`201+ Config struct {198202 TileSets []apiTileSet `json:"tile_sets"`199203 } `json:"config"`200204 Groups []struct {@@ -209,11 +209,24 @@ func captureTiles(
209209 for zoom := set.MinZoom; zoom <= deepest; zoom++ {210210 window, ok := set.window(zoom)211211 if !ok {212- fmt.Printf(" z%-2d skipped: no bounds published\n", zoom)213- live = nil214- continue212+ var err error213+ window, ok, err = unpublishedWindow(214+ ctx, fetcher, strings.TrimSuffix(base, "/"), set, full, zoom, live,215+ &transposed, &probed,216+ )217+ if err != nil {218+ return stats, err219+ }220+ if !ok {221+ fmt.Printf(" z%-2d skipped: no window published, and none found\n", zoom)222+ live = nil223+ continue224+ }215225 }216- if !probed {226+ // A window one tile across settles nothing about which way round the227+ // axes are read, so the question waits for a level wide enough to228+ // answer it rather than being marked answered by default.229+ if !probed && window.maxX > window.minX && window.maxY > window.minY {217230 extension := tileExtension(set)218231 order, err := probeAxisOrder(ctx, fetcher, strings.TrimSuffix(base, "/"), set, zoom, window, extension)219232 if err != nil {@@ -262,6 +275,43 @@ func captureTiles(
262275 return stats, nil263276 }264277 278+// unpublishedWindow stands in for the window a layer never published. Below the279+// first level captured there is nothing to search for: the tiles that held280+// content name where their children can be. The first level is searched for281+// outward from the point the map opens on, and finding it there also settles282+// which way round the axes are read, which no single tile could have said.283+func unpublishedWindow(284+ ctx context.Context,285+ fetcher *fetcher,286+ base string,287+ set apiTileSet,288+ full *apiMapFull,289+ zoom int,290+ live map[[2]int]bool,291+ transposed, probed *bool,292+) (tileWindow, bool, error) {293+ if len(live) > 0 {294+ window, ok := childWindow(live)295+ return window, ok, nil296+ }297+ centre, ok := startTile(full, zoom)298+ if !ok {299+ return tileWindow{}, false, nil300+ }301+ found, err := searchWindow(ctx, fetcher, base, set, zoom, centre, tileExtension(set))302+ if err != nil || !found.found {303+ return tileWindow{}, false, err304+ }305+ fmt.Printf(" z%-2d no window published; found x %d-%d y %d-%d from %d/%d\n",306+ zoom, found.window.minX, found.window.maxX, found.window.minY, found.window.maxY,307+ centre[0], centre[1])308+ if found.transposed {309+ fmt.Printf(" axes are published transposed; reading /{zoom}/{y}/{x}\n")310+ *transposed, *probed = true, true311+ }312+ return found.window, true, nil313+}314+265315 func fetchLevel(266316 ctx context.Context,267317 fetcher *fetcher,@@ -1,6 +1,9 @@
11 package main22 3-import "testing"3+import (4+ "encoding/json"5+ "testing"6+)47 58 func TestDominantHashIgnoresTinySamples(t *testing.T) {69 // The top of a pyramid is a handful of tiles showing the whole map. Calling@@ -189,3 +192,70 @@ func TestUpsertByIDReplacesInPlace(t *testing.T) {
189192 t.Fatalf("upsertByID did not append an unknown id")190193 }191194 }195+196+func TestChildWindowCoversEveryChild(t *testing.T) {197+ live := map[[2]int]bool{{2, 3}: true, {5, 4}: true}198+ window, ok := childWindow(live)199+ if !ok {200+ t.Fatal("two live tiles yielded no window")201+ }202+ want := tileWindow{minX: 4, minY: 6, maxX: 11, maxY: 9}203+ if window != want {204+ t.Fatalf("childWindow() = %+v, want %+v", window, want)205+ }206+ for parent := range live {207+ for _, child := range [][2]int{208+ {parent[0] * 2, parent[1] * 2},209+ {parent[0]*2 + 1, parent[1]*2 + 1},210+ } {211+ if !window.holds(child) {212+ t.Errorf("window drops child %v of %v", child, parent)213+ }214+ }215+ }216+ if _, ok := childWindow(nil); ok {217+ t.Error("an empty level yielded a window")218+ }219+}220+221+// The search for an unpublished window starts at the tile the map opens on, so222+// that tile has to be the one the map is actually drawn in. Appalachia opens on223+// the single tile its shallowest level holds.224+func TestStartTileFindsTheOpeningTile(t *testing.T) {225+ full := &apiMapFull{226+ InitialLatitude: json.RawMessage("0.50662148513874"),227+ InitialLongitude: json.RawMessage("-0.62060295831731"),228+ }229+ at, ok := startTile(full, 8)230+ if !ok {231+ t.Fatal("a map with a start coordinate gave no tile")232+ }233+ if at != [2]int{127, 127} {234+ t.Fatalf("startTile(8) = %v, want [127 127]", at)235+ }236+ if at, ok := startTile(full, 9); !ok || at != [2]int{255, 255} {237+ t.Fatalf("startTile(9) = %v %v, want [255 255] true", at, ok)238+ }239+ if _, ok := startTile(&apiMapFull{}, 8); ok {240+ t.Error("a map with no start coordinate gave a tile")241+ }242+}243+244+func TestNumberValueReadsQuotedAndPlain(t *testing.T) {245+ for _, tt := range []struct {246+ raw string247+ want float64248+ ok bool249+ }{250+ {raw: "1.5", want: 1.5, ok: true},251+ {raw: `"-0.25"`, want: -0.25, ok: true},252+ {raw: "null"},253+ {raw: `""`},254+ {raw: `"east"`},255+ } {256+ got, ok := numberValue(json.RawMessage(tt.raw))257+ if ok != tt.ok || (ok && got != tt.want) {258+ t.Errorf("numberValue(%s) = %v %v, want %v %v", tt.raw, got, ok, tt.want, tt.ok)259+ }260+ }261+}@@ -0,0 +1,192 @@
1+package main2+3+import (4+ "context"5+ "encoding/json"6+ "errors"7+ "fmt"8+ "math"9+ "strconv"10+ "strings"11+)12+13+// A layer usually publishes the window its tiles occupy at each zoom, and the14+// crawl asks for what is inside it. Some games publish none at all -- Fallout15+// 76 and Marathon among them -- and a crawl with no window to read had nothing16+// to ask for and captured nothing.17+//18+// What every map does say is where it opens, and that point is on the map by19+// definition. So the shallowest level is found by fetching the tile under it20+// and spreading outward from whatever answers, until the frontier runs out.21+// The levels below need no window of their own: they follow the tiles that22+// held content, which is what a published layer does after its first level too.23+24+// searchBudget caps the spread. The shallowest level of a map is a handful of25+// tiles -- Appalachia is one -- so a search that has read this many has26+// misread something, and should stop rather than walk the whole world.27+const searchBudget = 51228+29+// foundWindow is where a layer's tiles turned out to be, and which way round30+// they had to be asked for to find them.31+type foundWindow struct {32+ window tileWindow33+ transposed bool34+ found bool35+}36+37+// searchWindow spreads outward from the tile the map opens on. Neighbours are38+// taken in all eight directions, so a coastline meeting a corner does not end39+// the search early.40+func searchWindow(41+ ctx context.Context,42+ fetcher *fetcher,43+ base string,44+ set apiTileSet,45+ zoom int,46+ centre [2]int,47+ extension string,48+) (foundWindow, error) {49+ // The two axes are published the other way round on some layers, and with50+ // no window to compare neighbours in there is nothing yet to measure that51+ // against. The tile the map opens on settles it instead: whichever way52+ // round answers is the way the search reads the rest.53+ for _, transposed := range []bool{false, true} {54+ exists, err := tileExists(ctx, fetcher, base, set, zoom, centre, extension, transposed)55+ if err != nil {56+ return foundWindow{}, err57+ }58+ if !exists {59+ continue60+ }61+ window, err := spread(ctx, fetcher, base, set, zoom, centre, extension, transposed)62+ if err != nil {63+ return foundWindow{}, err64+ }65+ return foundWindow{window: window, transposed: transposed, found: true}, nil66+ }67+ return foundWindow{}, nil68+}69+70+func spread(71+ ctx context.Context,72+ fetcher *fetcher,73+ base string,74+ set apiTileSet,75+ zoom int,76+ centre [2]int,77+ extension string,78+ transposed bool,79+) (tileWindow, error) {80+ span := 1 << zoom81+ window := tileWindow{minX: centre[0], minY: centre[1], maxX: centre[0], maxY: centre[1]}82+ tested := map[[2]int]bool{centre: true}83+ queue := [][2]int{centre}84+85+ for len(queue) > 0 && len(tested) <= searchBudget {86+ at := queue[0]87+ queue = queue[1:]88+ if at != centre {89+ exists, err := tileExists(ctx, fetcher, base, set, zoom, at, extension, transposed)90+ if err != nil {91+ return tileWindow{}, err92+ }93+ if !exists {94+ continue95+ }96+ }97+ window.minX, window.minY = min(window.minX, at[0]), min(window.minY, at[1])98+ window.maxX, window.maxY = max(window.maxX, at[0]), max(window.maxY, at[1])99+ for dy := -1; dy <= 1; dy++ {100+ for dx := -1; dx <= 1; dx++ {101+ next := [2]int{at[0] + dx, at[1] + dy}102+ if (dx == 0 && dy == 0) || tested[next] ||103+ next[0] < 0 || next[1] < 0 || next[0] >= span || next[1] >= span {104+ continue105+ }106+ tested[next] = true107+ queue = append(queue, next)108+ }109+ }110+ }111+ return window, nil112+}113+114+func tileExists(115+ ctx context.Context,116+ fetcher *fetcher,117+ base string,118+ set apiTileSet,119+ zoom int,120+ at [2]int,121+ extension string,122+ transposed bool,123+) (bool, error) {124+ first, second := at[0], at[1]125+ if transposed {126+ first, second = second, first127+ }128+ url := fmt.Sprintf("%s/games/%s/%d/%d/%d.%s", base, set.Path, zoom, first, second, extension)129+ if _, _, err := fetcher.get(ctx, url); err != nil {130+ if errors.Is(err, errAbsent) {131+ return false, nil132+ }133+ return false, err134+ }135+ return true, nil136+}137+138+// childWindow is where the next level down can hold anything, given what the139+// level above turned out to hold. It stands in for a published window on a140+// layer that has none.141+func childWindow(live map[[2]int]bool) (tileWindow, bool) {142+ first := true143+ var window tileWindow144+ for parent := range live {145+ left, top := parent[0]*2, parent[1]*2146+ if first {147+ window = tileWindow{minX: left, minY: top, maxX: left + 1, maxY: top + 1}148+ first = false149+ continue150+ }151+ window.minX, window.minY = min(window.minX, left), min(window.minY, top)152+ window.maxX, window.maxY = max(window.maxX, left+1), max(window.maxY, top+1)153+ }154+ return window, !first155+}156+157+// startTile is the tile holding the point the map opens on, in the same158+// projection the tile pyramid is cut in.159+func startTile(full *apiMapFull, zoom int) ([2]int, bool) {160+ latitude, ok := numberValue(full.InitialLatitude)161+ if !ok {162+ return [2]int{}, false163+ }164+ longitude, ok := numberValue(full.InitialLongitude)165+ if !ok {166+ return [2]int{}, false167+ }168+ span := float64(int(1) << zoom)169+ x := (longitude + 180) / 360 * span170+ y := (1 - math.Asinh(math.Tan(latitude*math.Pi/180))/math.Pi) / 2 * span171+ if math.IsNaN(x) || math.IsNaN(y) {172+ return [2]int{}, false173+ }174+ limit := (1 << zoom) - 1175+ return [2]int{176+ min(max(int(x), 0), limit),177+ min(max(int(y), 0), limit),178+ }, true179+}180+181+// numberValue reads a JSON number that the archive has also been seen to quote.182+func numberValue(raw json.RawMessage) (float64, bool) {183+ text := strings.Trim(strings.TrimSpace(string(raw)), `"`)184+ if text == "" || text == "null" {185+ return 0, false186+ }187+ value, err := strconv.ParseFloat(text, 64)188+ if err != nil {189+ return 0, false190+ }191+ return value, true192+}