clone

an offline map explorer where each world, from a game map to a city's open data, ships as one .atlas file

fix: detect layers whose tile axes are published transposed

Dami committed 12650e5
RuneScape addresses a tile as /{zoom}/{y}/{x} rather than /{zoom}/{x}/{y}.
Nothing in the API says so, and a transposed layer still returns a real tile
for every request, so the mistake is invisible until the tiles are laid side
by side. Measured as the colour distance across a tile seam, RuneScape scored
44.8 against 3.8 for a map known to render correctly.

The layout is now measured rather than assumed: one small square block is
read once per layer -- the same set of URLs either way round -- and whichever
arrangement lines its seams up wins. RuneScape then measures 2.0 against a
1.3 control, and maps that were already correct are untouched.

Category icons gain a third form. Some games publish a marker strip that CSS
windows into with background-position rather than an icon font, so the strip
is sliced into one image per key. That is the only raster work the icon step
does, and Node's zlib covers it, so it needs no image library. tools/generate
takes whichever form the archive holds.
commit
12650e5c55028df1f0e6916e73e88929dd7096f5
parent
d44423871cdfff7b27cb3dfc7536fe608744e451
author
Dami <dami.etoile@gmail.com>
6 files changed, +471 -23
Mfrontend/app.js3+
Atools/crawl/axis.go135++++++++++++++++++++
Mtools/crawl/main.go46+++++--
Mtools/generate/main.go26+++-
Atools/icons/png.mjs200++++++++++++++++++++++++++++++
Mtools/icons/render-icons.mjs84++++++++++++-
M frontend/app.js +3 -0
A tools/crawl/axis.go +135 -0
@@ -0,0 +1,135 @@
1+package main2+3+import (4+ "bytes"5+ "context"6+ "fmt"7+ "image"8+ _ "image/jpeg"9+ _ "image/png"10+)11+12+// Most maps address a tile as /{zoom}/{x}/{y}, but some publish the two axes13+// the other way round. Nothing in the API says which, and a transposed map14+// still returns a real tile for every request, so the mistake is invisible15+// until the tiles are laid side by side: neighbours that belong together share16+// an edge, and neighbours that do not are unrelated pictures.17+//18+// probeAxisOrder measures that. It reads one small square block -- the same set19+// of URLs either way round -- and reports whether reading the first component20+// as the row lines the block up better.21+const (22+ axisProbeSpan = 323+ axisProbeSamples = 3224+)25+26+func probeAxisOrder(27+ ctx context.Context,28+ fetcher *fetcher,29+ base string,30+ set apiTileSet,31+ zoom int,32+ window tileWindow,33+ extension string,34+) (bool, error) {35+ span := axisProbeSpan36+ if width := window.maxX - window.minX + 1; span > width {37+ span = width38+ }39+ if height := window.maxY - window.minY + 1; span > height {40+ span = height41+ }42+ if span < 2 {43+ return false, nil44+ }45+ // Sample from the middle, where a map is likeliest to hold content: two46+ // blank tiles share an edge perfectly whichever way round they are read.47+ originA := window.minX + (window.maxX-window.minX+1-span)/248+ originB := window.minY + (window.maxY-window.minY+1-span)/249+50+ block := make(map[[2]int]image.Image, span*span)51+ for a := range span {52+ for b := range span {53+ url := fmt.Sprintf("%s/games/%s/%d/%d/%d.%s",54+ base, set.Path, zoom, originA+a, originB+b, extension)55+ body, _, err := fetcher.get(ctx, url)56+ if err != nil {57+ continue58+ }59+ decoded, _, err := image.Decode(bytes.NewReader(body))60+ if err != nil {61+ continue62+ }63+ block[[2]int{a, b}] = decoded64+ }65+ }66+ if len(block) < span*span {67+ // An incomplete probe cannot be trusted either way; assume the usual68+ // order rather than transpose on partial evidence.69+ return false, nil70+ }71+72+ direct := seamCost(block, false, span)73+ transposed := seamCost(block, true, span)74+ if direct < 0 || transposed < 0 {75+ return false, nil76+ }77+ return transposed < direct, nil78+}79+80+// seamCost is the average colour distance across the vertical seams of a block,81+// laid out either directly or transposed. Lower means the tiles belong together.82+func seamCost(block map[[2]int]image.Image, transpose bool, span int) float64 {83+ at := func(column, row int) image.Image {84+ if transpose {85+ return block[[2]int{row, column}]86+ }87+ return block[[2]int{column, row}]88+ }89+ var total float6490+ var seams int91+ for row := range span {92+ for column := 0; column < span-1; column++ {93+ left, right := at(column, row), at(column+1, row)94+ if left == nil || right == nil {95+ continue96+ }97+ total += edgeDistance(left, right)98+ seams++99+ }100+ }101+ if seams == 0 {102+ return -1103+ }104+ return total / float64(seams)105+}106+107+// edgeDistance compares the right-hand column of one tile with the left-hand108+// column of the tile that should sit beside it.109+func edgeDistance(left, right image.Image) float64 {110+ lb, rb := left.Bounds(), right.Bounds()111+ height := min(lb.Dy(), rb.Dy())112+ if height == 0 {113+ return 0114+ }115+ step := max(1, height/axisProbeSamples)116+ var total float64117+ var count int118+ for offset := 0; offset < height; offset += step {119+ lr, lg, lbl, _ := left.At(lb.Max.X-1, lb.Min.Y+offset).RGBA()120+ rr, rg, rbl, _ := right.At(rb.Min.X, rb.Min.Y+offset).RGBA()121+ total += absDiff(lr, rr) + absDiff(lg, rg) + absDiff(lbl, rbl)122+ count += 3123+ }124+ if count == 0 {125+ return 0126+ }127+ return total / float64(count) / 257 // back to 0-255128+}129+130+func absDiff(a, b uint32) float64 {131+ if a > b {132+ return float64(a - b)133+ }134+ return float64(b - a)135+}
M tools/crawl/main.go +36 -10
@@ -38,12 +38,13 @@ const (
3838 wholeLevelBudget = 10243939 )4040 41-// lineEnd redraws the progress line in place on a terminal, and leaves ordinary42-// lines behind when the output is being piped into a log.41+// lineEnd redraws the progress line in place on a terminal, clearing whatever42+// the previous longer line left behind, and leaves ordinary lines when the43+// output is being piped into a log.4344 var lineEnd = func() string {4445 info, err := os.Stdout.Stat()4546 if err == nil && info.Mode()&os.ModeCharDevice != 0 {46- return "\r"47+ return "\033[K\r"4748 }4849 return "\n"4950 }()
@@ -204,6 +205,7 @@ func captureTiles(
204205 205206 var live map[[2]int]bool206207 var background string208+ transposed, probed := false, false207209 for zoom := set.MinZoom; zoom <= deepest; zoom++ {208210 window, ok := set.window(zoom)209211 if !ok {
@@ -211,6 +213,18 @@ func captureTiles(
211213 live = nil212214 continue213215 }216+ if !probed {217+ extension := tileExtension(set)218+ order, err := probeAxisOrder(ctx, fetcher, strings.TrimSuffix(base, "/"), set, zoom, window, extension)219+ if err != nil {220+ return stats, err221+ }222+ transposed, probed = order, true223+ if transposed {224+ fmt.Printf(" axes are published transposed; reading /{zoom}/{y}/{x}\n")225+ }226+ }227+214228 wanted := window.tiles()215229 whole := len(wanted) <= wholeLevelBudget || live == nil216230 if !whole {
@@ -221,7 +235,7 @@ func captureTiles(
221235 break222236 }223237 224- results, err := fetchLevel(ctx, fetcher, o, set, setID, base, mapDir, zoom, wanted, index)238+ results, err := fetchLevel(ctx, fetcher, o, set, setID, base, mapDir, zoom, wanted, index, transposed)225239 if err != nil {226240 return stats, err227241 }
@@ -258,12 +272,10 @@ func fetchLevel(
258272 zoom int,259273 wanted [][2]int,260274 index *tileIndex,275+ transposed bool,261276 ) (levelResult, error) {262277 result := levelResult{hashes: make(map[[2]int]string, len(wanted))}263- extension := strings.TrimPrefix(strings.ToLower(set.Extension), ".")264- if extension == "" {265- extension = "jpg"266- }278+ extension := tileExtension(set)267279 268280 var mu sync.Mutex269281 var group sync.WaitGroup
@@ -295,8 +307,14 @@ func fetchLevel(
295307 }296308 297309 for _, coordinate := range wanted {310+ // A transposed layer names the row first. The tile is still stored at311+ // its true position, so everything downstream stays in one layout.312+ first, second := coordinate[0], coordinate[1]313+ if transposed {314+ first, second = second, first315+ }298316 url := fmt.Sprintf("%s/games/%s/%d/%d/%d.%s",299- strings.TrimSuffix(base, "/"), set.Path, zoom, coordinate[0], coordinate[1], extension)317+ strings.TrimSuffix(base, "/"), set.Path, zoom, first, second, extension)300318 path := filepath.Join(mapDir, "tiles",301319 "set-"+strconv.FormatInt(setID, 10),302320 strconv.Itoa(zoom), strconv.Itoa(coordinate[0]),
@@ -377,7 +395,7 @@ func fetchLevel(
377395 }378396 group.Wait()379397 mu.Lock()380- if result.fetched > 0 && lineEnd == "\r" {398+ if result.fetched > 0 && lineEnd != "\n" {381399 report(true)382400 fmt.Println()383401 }
@@ -471,6 +489,14 @@ func childrenOf(live map[[2]int]bool, window tileWindow) [][2]int {
471489 return children472490 }473491 492+func tileExtension(set apiTileSet) string {493+ extension := strings.TrimPrefix(strings.ToLower(set.Extension), ".")494+ if extension == "" {495+ return "jpg"496+ }497+ return extension498+}499+474500 func sortCoordinates(coordinates [][2]int) {475501 sort.Slice(coordinates, func(i, j int) bool {476502 if coordinates[i][1] != coordinates[j][1] {
M tools/generate/main.go +19 -7
A tools/icons/png.mjs +200 -0
@@ -0,0 +1,200 @@
1+// Enough PNG to cut one icon out of a marker sprite sheet.2+//3+// Some games publish category icons as a single strip that CSS windows into4+// with background-position, rather than as a font. Slicing that strip is the5+// only raster work the icon step needs, and Node's zlib covers it, so it stays6+// dependency-free rather than pulling in an image library for one operation.7+//8+// Only what MapGenie actually serves is handled: 8-bit non-interlaced images.9+10+import zlib from "node:zlib";11+12+const SIGNATURE = Buffer.from([137, 80, 78, 71, 13, 10, 26, 10]);13+14+const crcTable = (() => {15+ const table = new Int32Array(256);16+ for (let index = 0; index < 256; index += 1) {17+ let value = index;18+ for (let bit = 0; bit < 8; bit += 1) {19+ value = value & 1 ? 0xedb88320 ^ (value >>> 1) : value >>> 1;20+ }21+ table[index] = value;22+ }23+ return table;24+})();25+26+const crc32 = (buffer) => {27+ let value = -1;28+ for (const byte of buffer) value = crcTable[(value ^ byte) & 0xff] ^ (value >>> 8);29+ return (value ^ -1) >>> 0;30+};31+32+const chunk = (type, body) => {33+ const header = Buffer.alloc(8);34+ header.writeUInt32BE(body.length, 0);35+ header.write(type, 4, "ascii");36+ const trailer = Buffer.alloc(4);37+ trailer.writeUInt32BE(crc32(Buffer.concat([header.subarray(4), body])), 0);38+ return Buffer.concat([header, body, trailer]);39+};40+41+const CHANNELS = { 0: 1, 2: 3, 4: 2, 6: 4 };42+43+// Each row is filtered against the one above it, so decoding is sequential.44+const unfilter = (raw, width, height, bytesPerPixel) => {45+ const stride = width * bytesPerPixel;46+ const out = Buffer.alloc(height * stride);47+ let source = 0;48+ for (let y = 0; y < height; y += 1) {49+ const filter = raw[source];50+ source += 1;51+ const row = y * stride;52+ const above = row - stride;53+ for (let index = 0; index < stride; index += 1) {54+ const value = raw[source + index];55+ const left = index >= bytesPerPixel ? out[row + index - bytesPerPixel] : 0;56+ const up = y > 0 ? out[above + index] : 0;57+ const upLeft =58+ y > 0 && index >= bytesPerPixel ? out[above + index - bytesPerPixel] : 0;59+ let restored;60+ switch (filter) {61+ case 0:62+ restored = value;63+ break;64+ case 1:65+ restored = value + left;66+ break;67+ case 2:68+ restored = value + up;69+ break;70+ case 3:71+ restored = value + ((left + up) >> 1);72+ break;73+ case 4: {74+ const estimate = left + up - upLeft;75+ const dLeft = Math.abs(estimate - left);76+ const dUp = Math.abs(estimate - up);77+ const dUpLeft = Math.abs(estimate - upLeft);78+ restored =79+ value +80+ (dLeft <= dUp && dLeft <= dUpLeft ? left : dUp <= dUpLeft ? up : upLeft);81+ break;82+ }83+ default:84+ throw new Error(`unsupported PNG row filter ${filter}`);85+ }86+ out[row + index] = restored & 0xff;87+ }88+ source += stride;89+ }90+ return out;91+};92+93+export const decodePNG = (buffer) => {94+ if (!buffer.subarray(0, 8).equals(SIGNATURE)) throw new Error("not a PNG");95+ let offset = 8;96+ let header = null;97+ let palette = null;98+ let alpha = null;99+ const data = [];100+101+ while (offset < buffer.length) {102+ const length = buffer.readUInt32BE(offset);103+ const type = buffer.toString("ascii", offset + 4, offset + 8);104+ const body = buffer.subarray(offset + 8, offset + 8 + length);105+ offset += 12 + length;106+107+ if (type === "IHDR") {108+ header = {109+ width: body.readUInt32BE(0),110+ height: body.readUInt32BE(4),111+ depth: body[8],112+ colorType: body[9],113+ interlace: body[12],114+ };115+ } else if (type === "PLTE") palette = Buffer.from(body);116+ else if (type === "tRNS") alpha = Buffer.from(body);117+ else if (type === "IDAT") data.push(Buffer.from(body));118+ else if (type === "IEND") break;119+ }120+121+ if (!header) throw new Error("PNG has no header");122+ if (header.depth !== 8) throw new Error(`unsupported PNG bit depth ${header.depth}`);123+ if (header.interlace !== 0) throw new Error("interlaced PNG is not supported");124+125+ const { width, height, colorType } = header;126+ const raw = zlib.inflateSync(Buffer.concat(data));127+128+ // Everything is normalised to RGBA so the caller only deals with one layout.129+ const rgba = Buffer.alloc(width * height * 4);130+ if (colorType === 3) {131+ if (!palette) throw new Error("indexed PNG has no palette");132+ const indices = unfilter(raw, width, height, 1);133+ for (let pixel = 0; pixel < width * height; pixel += 1) {134+ const entry = indices[pixel];135+ rgba[pixel * 4] = palette[entry * 3];136+ rgba[pixel * 4 + 1] = palette[entry * 3 + 1];137+ rgba[pixel * 4 + 2] = palette[entry * 3 + 2];138+ rgba[pixel * 4 + 3] = alpha && entry < alpha.length ? alpha[entry] : 255;139+ }140+ return { width, height, rgba };141+ }142+143+ const channels = CHANNELS[colorType];144+ if (!channels) throw new Error(`unsupported PNG colour type ${colorType}`);145+ const pixels = unfilter(raw, width, height, channels);146+ for (let pixel = 0; pixel < width * height; pixel += 1) {147+ const source = pixel * channels;148+ const target = pixel * 4;149+ if (colorType === 0 || colorType === 4) {150+ const grey = pixels[source];151+ rgba[target] = grey;152+ rgba[target + 1] = grey;153+ rgba[target + 2] = grey;154+ rgba[target + 3] = colorType === 4 ? pixels[source + 1] : 255;155+ } else {156+ rgba[target] = pixels[source];157+ rgba[target + 1] = pixels[source + 1];158+ rgba[target + 2] = pixels[source + 2];159+ rgba[target + 3] = colorType === 6 ? pixels[source + 3] : 255;160+ }161+ }162+ return { width, height, rgba };163+};164+165+export const encodePNG = ({ width, height, rgba }) => {166+ const stride = width * 4;167+ const raw = Buffer.alloc(height * (stride + 1));168+ for (let y = 0; y < height; y += 1) {169+ raw[y * (stride + 1)] = 0; // no filtering; these images are tiny170+ rgba.copy(raw, y * (stride + 1) + 1, y * stride, (y + 1) * stride);171+ }172+ const header = Buffer.alloc(13);173+ header.writeUInt32BE(width, 0);174+ header.writeUInt32BE(height, 4);175+ header[8] = 8; // bit depth176+ header[9] = 6; // RGBA177+ return Buffer.concat([178+ SIGNATURE,179+ chunk("IHDR", header),180+ chunk("IDAT", zlib.deflateSync(raw, { level: 9 })),181+ chunk("IEND", Buffer.alloc(0)),182+ ]);183+};184+185+export const cropPNG = (image, x, y, width, height) => {186+ const rgba = Buffer.alloc(width * height * 4);187+ for (let row = 0; row < height; row += 1) {188+ const source = ((y + row) * image.width + x) * 4;189+ image.rgba.copy(rgba, row * width * 4, source, source + width * 4);190+ }191+ return { width, height, rgba };192+};193+194+// A slice that is entirely transparent is a gap in the sprite, not an icon.195+export const isBlank = ({ rgba }) => {196+ for (let index = 3; index < rgba.length; index += 4) {197+ if (rgba[index] !== 0) return false;198+ }199+ return true;200+};
M tools/icons/render-icons.mjs +78 -6
@@ -20,6 +20,8 @@ import { mkdir, readFile, writeFile } from "node:fs/promises";
2020 import path from "node:path";2121 import process from "node:process";2222 23+import { cropPNG, decodePNG, encodePNG, isBlank } from "./png.mjs";24+2325 const API = "https://mapgenie.io/api/v1";2426 const CDN = "https://cdn.mapgenie.io";2527 // The CDN answers 403 to requests that do not look like a browser.
@@ -70,6 +72,11 @@ const fetchJSON = async (url, options) => {
7072 return response && response.json();7173 };7274 75+const fetchBuffer = async (url, options) => {76+ const response = await fetchPolitely(url, options);77+ return response && Buffer.from(await response.arrayBuffer());78+};79+7380 const cssColor = (value) => {7481 if (!value) return null;7582 const rgb = /rgba?\(\s*(\d+)\s*,\s*(\d+)\s*,\s*(\d+)/i.exec(value);
@@ -119,6 +126,33 @@ const readFontURL = (css) => {
119126 return new URL(match[1].replace(/#.*$/, ""), `${CDN}/css/themes/icons/`).href;120127 };121128 129+// Some games publish one marker strip that CSS windows into, rather than a130+// font: `[class^="icon-"] { background: url(sprite.png); background-size: 15px }`131+// with `.icon-<key> { background-position: 0 -Npx }` naming each row.132+const readSprite = (css) => {133+ const source = /\[class\^?[*]?="icon-"\][^{]*\{([^}]*)\}/i.exec(css)?.[1];134+ if (!source) return null;135+ const url = /url\(['"]?([^'")]+\.png[^'")]*)['"]?\)/i.exec(source)?.[1];136+ if (!url) return null;137+ const size = Number(/background-size:\s*([0-9.]+)px/i.exec(source)?.[1] ?? 0);138+ const cell = Number(/width:\s*([0-9.]+)px/i.exec(source)?.[1] ?? size);139+ if (!cell) return null;140+141+ const offsets = new Map();142+ const rule = /\.icon-([a-z0-9_-]+)\s*\{[^}]*background-position:\s*[^;]*?(-?[0-9.]+)px\s*;?[^}]*\}/gi;143+ for (const match of css.matchAll(rule)) {144+ // A key repeated in the strip keeps its first row; later ones are spares.145+ if (!offsets.has(match[1])) offsets.set(match[1], Math.abs(Number(match[2])));146+ }147+ if (offsets.size === 0) return null;148+ return {149+ url: new URL(url, `${CDN}/css/themes/icons/`).href,150+ cell,151+ displayWidth: size || cell,152+ offsets,153+ };154+};155+122156 // An SVG font is XML, so the glyphs can be read directly rather than through a123157 // font parser: each carries its codepoint and its outline.124158 const readGlyphs = (svg) => {
@@ -177,11 +211,13 @@ const renderGame = async (archiveRoot, game, directory) => {
177211 }178212 const icons = readIconLayers(css);179213 const fontURL = readFontURL(css);180- if (!fontURL || icons.size === 0) {214+ const sprite = fontURL ? null : readSprite(css);215+ if ((!fontURL || icons.size === 0) && !sprite) {181216 console.log(` ${game.slug}: stylesheet carries no usable glyph mapping`);182217 return 0;183218 }184- const { glyphs, unitsPerEm, ascent } = readGlyphs(await fetchText(fontURL));219+ const font = fontURL ? readGlyphs(await fetchText(fontURL)) : null;220+ const strip = sprite ? decodePNG(await fetchBuffer(sprite.url)) : null;185221 186222 // Which glyphs matter, and what colour the legend gives them, comes from the187223 // maps themselves.
@@ -210,16 +246,52 @@ const renderGame = async (archiveRoot, game, directory) => {
210246 const index = [];211247 let missing = 0;212248 for (const [key, uses] of [...usages].sort(([a], [b]) => a.localeCompare(b))) {249+ const legend = uses.find((use) => use.legendColor)?.legendColor;250+ const defaultColor = legend ? `#${legend.toLowerCase()}` : FALLBACK_COLOR;251+252+ if (strip) {253+ const offset = sprite.offsets.get(key);254+ if (offset === undefined) {255+ missing += 1;256+ continue;257+ }258+ // Offsets are in displayed pixels; the strip may be published at a259+ // multiple of that, so they are scaled to source pixels before cutting.260+ const scale = strip.width / sprite.displayWidth;261+ const cell = Math.round(sprite.cell * scale);262+ const top = Math.round(offset * scale);263+ if (top + cell > strip.height) {264+ missing += 1;265+ continue;266+ }267+ const slice = cropPNG(strip, 0, top, Math.min(cell, strip.width), cell);268+ if (isBlank(slice)) {269+ missing += 1;270+ continue;271+ }272+ const png = encodePNG(slice);273+ await writeFile(path.join(iconDirectory, `${key}.png`), png);274+ index.push({275+ contentHash: createHash("sha256").update(png).digest("hex"),276+ defaultColor,277+ file: `${key}.png`,278+ key,279+ spriteOffset: offset,280+ sourceCssUrl: `${CDN}/css/themes/icons/${game.slug}-icons.css`,281+ sourceSpriteUrl: sprite.url,282+ usages: uses,283+ });284+ continue;285+ }286+213287 const drawn = (icons.get(key) ?? [])214- .map((layer) => ({ ...layer, outline: glyphs.get(layer.codepoint) }))288+ .map((layer) => ({ ...layer, outline: font.glyphs.get(layer.codepoint) }))215289 .filter((layer) => layer.outline);216290 if (drawn.length === 0) {217291 missing += 1;218292 continue;219293 }220- const legend = uses.find((use) => use.legendColor)?.legendColor;221- const defaultColor = legend ? `#${legend.toLowerCase()}` : FALLBACK_COLOR;222- const svg = renderSVG(key, drawn, unitsPerEm, ascent, defaultColor);294+ const svg = renderSVG(key, drawn, font.unitsPerEm, font.ascent, defaultColor);223295 await writeFile(path.join(iconDirectory, `${key}.svg`), svg);224296 index.push({225297 codepoint: drawn.map((layer) => layer.codepoint).join("+"),