perf: derive a tile pyramid only when something under it has moved

Dami committed Jul 31c428b04

Every pyramid was derived afresh on every run. Capturing one new game meant
re-decoding and re-reducing the other seventeen -- half a minute of work to
arrive at the bytes that were already on disk -- because the tool built the
whole tree into a temporary directory and swapped it in wholesale.

A layer is derived from tiles the archive already stores under their content
hashes, so what went into a pyramid can be written down. Each one now carries
a stamp over the hash of every source tile, the shape the plan settled on, and
the source of the tool itself: changing how a level is reduced or where the
content bounds land has to invalidate everything, and a stamp that only
watched the archive would quietly keep serving the old derivation. A layer
whose stamp still matches is left exactly where it is.

That means installing per layer rather than swapping the tree: what was
derived is moved into place by one rename each, layers the archive no longer
offers are taken out, and the index is written last. A run that dies partway
leaves an index naming the old stamps for anything it had replaced, so the
next run finds the mismatch and derives those layers again.

Deriving all thirty-one layers takes 32s; a run with nothing to do now takes
2.2s, all of it reading the archive's own indexes to work out that there is
nothing to do. Rebuilding this repository's tiles from scratch under the new
code changed no tile byte -- only the index, which now carries the stamps.
`-force` derives everything again for when that is what is wanted.

commit: c428b04d5077690ad13725f5fc3c9425a1eef586
parent: 88b67d9c2380ebe5b6fc0f280057018a65da4eea
author: Dami <dami.etoile@gmail.com>

6 files changed, +389 -28wrap lines

M	.gitignore	1	+
M	README.md	5	+
M	assets/tiles/index.json	2	+-
M	tools/tiles/main.go	117	+++++++++++++++++-----
A	tools/tiles/stamp.go	135	++++++++++++++++++++++++++
A	tools/tiles/stamp_test.go	157	++++++++++++++++++++++++++++++

M .gitignore +1 -0

@@ -5,3 +5,4 @@

55 /output/66 /crawl 77 /generate 8+/tiles

M README.md +5 -0

@@ -63,6 +63,11 @@ go generate .
6363    normalized to lossless PNG and use nearest-neighbor reduction. Photo6464    pyramids use smooth box reduction. Repeated placeholder borders are6565    excluded from fit bounds.66+67+   Each pyramid records a stamp over the tiles it was derived from and the68+   tool that derived them, so a layer nothing has changed under is left where69+   it is: a run that captures one new game costs seconds rather than the half70+   minute of re-deriving the rest. `-force` derives everything again.6671 2. `tools/generate` builds `assets/catalog.json` and copies referenced archive6772    SVGs into `assets/icons`. Maps with missing snapshots or incomplete6873    configured layers are omitted.

M assets/tiles/index.json +1 -1

@@ -1 +1 @@

M tools/tiles/main.go +90 -27

@@ -131,6 +131,10 @@ type variantManifest struct {
131131 	Interpolate bool                      `json:"interpolate"`132132 	Background  string                    `json:"background,omitempty"`133133 	Coverage    map[string]*levelCoverage `json:"coverage,omitempty"`134+	// Stamp names everything this pyramid was derived from -- the captured135+	// tiles and the tool that reduced them -- so a later run can tell that136+	// nothing has moved and keep what it already built.137+	Stamp string `json:"stamp,omitempty"`134138 }135139 136140 // levelCoverage is a row-major bitset over the [X,Y,W,H] tile window of one@@ -194,19 +198,20 @@ type contentBounds struct {
194198 func main() {195199 	source := flag.String("source", "", "FMG archive root")196200 	output := flag.String("output", "", "embedded tile destination")201+	force := flag.Bool("force", false, "derive every pyramid again, even one nothing has changed under")197202 	flag.Parse()198203 199204 	if *source == "" || *output == "" {200205 		fmt.Fprintln(os.Stderr, "tiles: -source and -output are required")201206 		os.Exit(2)202207 	}203-	if err := run(*source, *output); err != nil {208+	if err := run(*source, *output, *force); err != nil {204209 		fmt.Fprintln(os.Stderr, "tiles:", err)205210 		os.Exit(1)206211 	}207212 }208213 209-func run(source, output string) error {214+func run(source, output string, force bool) error {210215 	var index archive211216 	if err := readJSON(filepath.Join(source, "archive.json"), &index); err != nil {212217 		return err@@ -222,6 +227,12 @@ func run(source, output string) error {
222227 	}223228 	defer os.RemoveAll(temp)224229 230+	// What the last run left, so a layer nothing has changed under can be kept231+	// where it is rather than derived again.232+	built := manifestBySource(readManifest(output))233+	var carried int234+	derived := make(map[string]bool)235+225236 	out := manifest{TileSize: tileSize, Size: worldSize}226237 	for _, game := range index.Games {227238 		mapDirs, err := filepath.Glob(filepath.Join(source, game.Directory, "maps", "*"))@@ -246,25 +257,99 @@ func run(source, output string) error {
246257 				continue247258 			}248259 			for _, plan := range plans {260+				stamp := planStamp(plan)261+				if kept, ok := carry(built, plan, stamp, output, force); ok {262+					out.Variants = append(out.Variants, kept)263+					carried++264+					continue265+				}249266 				fmt.Printf("tile %s / %s / %s\n", game.Title, title, plan.SourcePath)250267 				entry, err := buildPyramid(temp, plan)251268 				if err != nil {252269 					return fmt.Errorf("%s / %s: %w", title, plan.SourcePath, err)253270 				}271+				entry.Stamp = stamp254272 				out.Variants = append(out.Variants, entry)273+				derived[plan.AssetPath] = true255274 			}256275 		}257276 	}258277 	sort.Slice(out.Variants, func(i, j int) bool {259278 		return out.Variants[i].SourcePath < out.Variants[j].SourcePath260279 	})261-	if err := writeJSON(filepath.Join(temp, "index.json"), out); err != nil {280+	if err := installPyramids(temp, output, out, derived); err != nil {281+		return err282+	}283+	fmt.Printf("%d layers derived · %d carried over\n", len(derived), carried)284+	return nil285+}286+287+// carry keeps a pyramid the last run left, when nothing it was derived from has288+// moved. Keeping one is doing nothing at all: it stays where it is, and only289+// the layers that were derived again are installed over the top.290+func carry(291+	built map[string]variantManifest,292+	plan tilePlan,293+	stamp, output string,294+	force bool,295+) (variantManifest, bool) {296+	if force {297+		return variantManifest{}, false298+	}299+	entry, ok := built[plan.SourcePath]300+	if !ok || entry.Stamp == "" || entry.Stamp != stamp || entry.AssetPath != plan.AssetPath {301+		return variantManifest{}, false302+	}303+	if _, err := os.Stat(filepath.Join(output, plan.AssetPath)); err != nil {304+		// Whatever the last run left is not there to be kept, so derive it.305+		return variantManifest{}, false306+	}307+	return entry, true308+}309+310+// installPyramids moves the layers that were derived into the output, takes out311+// any layer the archive no longer offers, and writes the index last.312+//313+// A layer arrives by one rename, so a reader of the output never sees a314+// half-written pyramid. The index naming the old stamps until the end is what315+// makes a run that dies partway safe: the next run finds a stamp that does not316+// match what is on disk and derives that layer again.317+func installPyramids(temp, output string, out manifest, derived map[string]bool) error {318+	if err := os.MkdirAll(output, 0o755); err != nil {319+		return fmt.Errorf("create tile output: %w", err)320+	}321+	for asset := range derived {322+		destination := filepath.Join(output, asset)323+		if err := os.RemoveAll(destination); err != nil {324+			return fmt.Errorf("replace %s: %w", asset, err)325+		}326+		if err := os.Rename(filepath.Join(temp, asset), destination); err != nil {327+			return fmt.Errorf("install %s: %w", asset, err)328+		}329+	}330+331+	wanted := make(map[string]bool, len(out.Variants))332+	for _, variant := range out.Variants {333+		wanted[variant.AssetPath] = true334+	}335+	entries, err := os.ReadDir(output)336+	if err != nil {262337 		return err263338 	}264-	if err := replaceDirectory(temp, output); err != nil {339+	for _, entry := range entries {340+		if !entry.IsDir() || wanted[entry.Name()] {341+			continue342+		}343+		if err := os.RemoveAll(filepath.Join(output, entry.Name())); err != nil {344+			return fmt.Errorf("remove %s: %w", entry.Name(), err)345+		}346+	}347+348+	staged := filepath.Join(temp, "index.json")349+	if err := writeJSON(staged, out); err != nil {265350 		return err266351 	}267-	return nil352+	return os.Rename(staged, filepath.Join(output, "index.json"))268353 }269354 270355 func inspectMap(mapDir string) ([]tilePlan, string, string, error) {@@ -881,28 +966,6 @@ func encodeImage(path string, value image.Image, format string) error {
881966 	return file.Close()882967 }883968 884-func replaceDirectory(temp, output string) error {885-	backup := output + ".previous"886-	if err := os.RemoveAll(backup); err != nil {887-		return err888-	}889-	if _, err := os.Stat(output); err == nil {890-		if err := os.Rename(output, backup); err != nil {891-			return fmt.Errorf("backup old tile output: %w", err)892-		}893-	} else if !errors.Is(err, os.ErrNotExist) {894-		return err895-	}896-	if err := os.Rename(temp, output); err != nil {897-		_ = os.Rename(backup, output)898-		return fmt.Errorf("install tile output: %w", err)899-	}900-	if err := os.RemoveAll(backup); err != nil {901-		return fmt.Errorf("remove old tile output: %w", err)902-	}903-	return nil904-}905-906969 func writeJSON(path string, value any) error {907970 	data, err := json.Marshal(value)908971 	if err != nil {

A tools/tiles/stamp.go +135 -0

@@ -0,0 +1,135 @@
1+package main2+3+import (4+	"crypto/sha256"5+	"embed"6+	"encoding/hex"7+	"fmt"8+	"io/fs"9+	"os"10+	"path/filepath"11+	"sort"12+	"sync"13+)14+15+// Every pyramid was derived afresh on every run: adding one game re-decoded and16+// re-reduced the other seventeen, half a minute of work to arrive at the bytes17+// that were already there. A layer is derived from tiles that are stored under18+// their own content hash, so what went into a pyramid can be written down and19+// compared, and a layer whose capture has not moved can be carried over from20+// the last run instead.21+//22+// The tool's own source counts as an input. Changing how a level is reduced or23+// where the content bounds land has to invalidate every pyramid, and a stamp24+// that only watched the archive would quietly keep serving the old derivation.25+//26+//go:embed main.go stamp.go27+var toolSource embed.FS28+29+var toolStamp = sync.OnceValue(func() string {30+	sum := sha256.New()31+	entries, err := fs.ReadDir(toolSource, ".")32+	if err != nil {33+		// The files are embedded at build time, so this cannot fail in a built34+		// binary. Falling back to a random-looking constant would defeat the35+		// point of the stamp, and rebuilding everything is the safe answer.36+		return "unstamped"37+	}38+	names := make([]string, 0, len(entries))39+	for _, entry := range entries {40+		names = append(names, entry.Name())41+	}42+	sort.Strings(names)43+	for _, name := range names {44+		data, err := toolSource.ReadFile(name)45+		if err != nil {46+			return "unstamped"47+		}48+		fmt.Fprintf(sum, "%s\x00%d\x00", name, len(data))49+		sum.Write(data)50+	}51+	return hex.EncodeToString(sum.Sum(nil))52+})53+54+// planStamp names everything a pyramid is derived from: the tool that derives55+// it, the shape the plan settled on, and the content hash of every source tile56+// that goes into it. Two runs that would write the same bytes stamp the same.57+func planStamp(plan tilePlan) string {58+	sum := sha256.New()59+	fmt.Fprintf(sum, "tool\x00%s\x00", toolStamp())60+	fmt.Fprintf(sum, "layer\x00%s\x00%s\x00", plan.SourcePath, plan.AssetPath)61+	fmt.Fprintf(sum, "zooms\x00%d\x00%d\x00", plan.MaxFullZoom, plan.MaxSourceZoom)62+	fmt.Fprintf(sum, "format\x00%s\x00%t\x00", plan.PreferredFormat, plan.Interpolate)63+	if plan.Bounds != nil {64+		fmt.Fprintf(sum, "bounds\x00%d\x00%d\x00%d\x00%d\x00",65+			plan.Bounds.X, plan.Bounds.Y, plan.Bounds.Width, plan.Bounds.Height)66+	}67+68+	zooms := make([]int, 0, len(plan.Levels))69+	for zoom := range plan.Levels {70+		zooms = append(zooms, zoom)71+	}72+	sort.Ints(zooms)73+	for _, zoom := range zooms {74+		files := append([]tileFile(nil), plan.Levels[zoom]...)75+		sort.Slice(files, func(i, j int) bool {76+			if files[i].Record.X != files[j].Record.X {77+				return files[i].Record.X < files[j].Record.X78+			}79+			return files[i].Record.Y < files[j].Record.Y80+		})81+		fmt.Fprintf(sum, "level\x00%d\x00%d\x00", zoom, len(files))82+		for _, file := range files {83+			fmt.Fprintf(sum, "%d\x00%d\x00%s\x00%s\x00",84+				file.Record.X, file.Record.Y, file.Record.ContentHash, file.Format)85+		}86+	}87+	return hex.EncodeToString(sum.Sum(nil))88+}89+90+// linkTree reproduces a directory as links to the files it already holds, and91+// reports how many it brought across. Directories are visited before their92+// contents, so each one is made once rather than checked for again per file.93+func linkTree(from, to string) (int, error) {94+	tiles := 095+	err := filepath.WalkDir(from, func(path string, entry fs.DirEntry, err error) error {96+		if err != nil {97+			return err98+		}99+		relative, err := filepath.Rel(from, path)100+		if err != nil {101+			return err102+		}103+		destination := filepath.Join(to, relative)104+		if entry.IsDir() {105+			return os.Mkdir(destination, 0o755)106+		}107+		tiles++108+		if err := os.Link(path, destination); err == nil {109+			return nil110+		}111+		// A link can fail across filesystems, and on one that has none. The112+		// bytes are what matter, so fall back to writing them out.113+		return copyFile(path, destination)114+	})115+	return tiles, err116+}117+118+// readManifest is the index left by the last run, or an empty one where there119+// was no last run. A manifest that cannot be read is not an error: it means120+// everything is derived again, which is what a first run does anyway.121+func readManifest(output string) manifest {122+	var previous manifest123+	if err := readJSON(filepath.Join(output, "index.json"), &previous); err != nil {124+		return manifest{}125+	}126+	return previous127+}128+129+func manifestBySource(previous manifest) map[string]variantManifest {130+	bySource := make(map[string]variantManifest, len(previous.Variants))131+	for _, variant := range previous.Variants {132+		bySource[variant.SourcePath] = variant133+	}134+	return bySource135+}

A tools/tiles/stamp_test.go +157 -0

@@ -0,0 +1,157 @@
1+package main2+3+import (4+	"os"5+	"path/filepath"6+	"testing"7+)8+9+func samplePlan() tilePlan {10+	return tilePlan{11+		AssetPath:       "tunic__world",12+		SourcePath:      "tunic/world/default-v2",13+		MaxFullZoom:     12,14+		MaxSourceZoom:   13,15+		Interpolate:     true,16+		PreferredFormat: "jpg",17+		Bounds:          &contentBounds{X: 16, Y: 32, Width: 512, Height: 512},18+		Levels: map[int][]tileFile{19+			12: {20+				{Record: tileRecord{X: 1, Y: 1, ContentHash: "aaa"}, Format: "jpg"},21+				{Record: tileRecord{X: 1, Y: 2, ContentHash: "bbb"}, Format: "jpg"},22+			},23+			13: {24+				{Record: tileRecord{X: 2, Y: 2, ContentHash: "ccc"}, Format: "jpg"},25+			},26+		},27+	}28+}29+30+// The stamp decides whether half a minute of derivation is skipped, so it has31+// to say the same thing about the same capture and something different about32+// every part of it that changes what would be written.33+func TestPlanStampAnswersForEveryInput(t *testing.T) {34+	base := planStamp(samplePlan())35+	if base == planStamp(tilePlan{}) {36+		t.Fatal("an empty plan stamps the same as a captured one")37+	}38+39+	// The order tiles happen to be listed in is not an input.40+	shuffled := samplePlan()41+	shuffled.Levels[12] = []tileFile{shuffled.Levels[12][1], shuffled.Levels[12][0]}42+	if got := planStamp(shuffled); got != base {43+		t.Error("the same tiles in another order stamp differently")44+	}45+46+	changes := map[string]func(*tilePlan){47+		"a tile recaptured": func(plan *tilePlan) {48+			plan.Levels[12][0].Record.ContentHash = "zzz"49+		},50+		"a tile added": func(plan *tilePlan) {51+			plan.Levels[12] = append(plan.Levels[12],52+				tileFile{Record: tileRecord{X: 9, Y: 9, ContentHash: "ddd"}, Format: "jpg"})53+		},54+		"a level captured deeper":  func(plan *tilePlan) { plan.MaxSourceZoom = 14 },55+		"the complete level moved": func(plan *tilePlan) { plan.MaxFullZoom = 11 },56+		"the content bounds moved": func(plan *tilePlan) { plan.Bounds.Width = 1024 },57+		"the reduction changed":    func(plan *tilePlan) { plan.Interpolate = false },58+		"the format changed":       func(plan *tilePlan) { plan.PreferredFormat = "png" },59+		"the layer renamed":        func(plan *tilePlan) { plan.AssetPath = "tunic__other" },60+	}61+	for name, change := range changes {62+		t.Run(name, func(t *testing.T) {63+			plan := samplePlan()64+			change(&plan)65+			if planStamp(plan) == base {66+				t.Errorf("%s stamps the same as before it", name)67+			}68+		})69+	}70+}71+72+// The tool derives the pyramid, so a change to the tool has to invalidate one.73+func TestToolStampIsRecorded(t *testing.T) {74+	stamp := toolStamp()75+	if stamp == "" || stamp == "unstamped" {76+		t.Fatalf("tool stamp = %q, so nothing would notice the tool changing", stamp)77+	}78+	if planStamp(samplePlan()) == "" {79+		t.Fatal("a plan stamped as nothing")80+	}81+}82+83+func TestLinkTreeBringsEveryFileAcross(t *testing.T) {84+	from, to := filepath.Join(t.TempDir(), "from"), filepath.Join(t.TempDir(), "to")85+	for _, name := range []string{"3/0/0.jpg", "3/0/1.jpg", "4/1/0.png"} {86+		path := filepath.Join(from, name)87+		if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {88+			t.Fatal(err)89+		}90+		if err := os.WriteFile(path, []byte(name), 0o644); err != nil {91+			t.Fatal(err)92+		}93+	}94+95+	tiles, err := linkTree(from, to)96+	if err != nil {97+		t.Fatal(err)98+	}99+	if tiles != 3 {100+		t.Errorf("linked %d tiles, want 3", tiles)101+	}102+	for _, name := range []string{"3/0/0.jpg", "3/0/1.jpg", "4/1/0.png"} {103+		data, err := os.ReadFile(filepath.Join(to, name))104+		if err != nil {105+			t.Fatal(err)106+		}107+		if string(data) != name {108+			t.Errorf("%s = %q, want %q", name, data, name)109+		}110+	}111+}112+113+// A run installs what it derived and takes out what the archive no longer114+// offers, leaving everything it carried where it already was.115+func TestInstallPyramidsKeepsCarriedAndPrunesRetired(t *testing.T) {116+	temp, output := t.TempDir(), t.TempDir()117+	write := func(root, name string, body string) {118+		path := filepath.Join(root, name)119+		if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {120+			t.Fatal(err)121+		}122+		if err := os.WriteFile(path, []byte(body), 0o644); err != nil {123+			t.Fatal(err)124+		}125+	}126+	write(output, "carried/3/0/0.jpg", "kept")127+	write(output, "derived/3/0/0.jpg", "old")128+	write(output, "retired/3/0/0.jpg", "gone")129+	write(temp, "derived/3/0/0.jpg", "new")130+131+	out := manifest{Variants: []variantManifest{132+		{AssetPath: "carried", SourcePath: "a"},133+		{AssetPath: "derived", SourcePath: "b"},134+	}}135+	if err := installPyramids(temp, output, out, map[string]bool{"derived": true}); err != nil {136+		t.Fatal(err)137+	}138+139+	for name, want := range map[string]string{140+		"carried/3/0/0.jpg": "kept",141+		"derived/3/0/0.jpg": "new",142+	} {143+		data, err := os.ReadFile(filepath.Join(output, name))144+		if err != nil {145+			t.Fatal(err)146+		}147+		if string(data) != want {148+			t.Errorf("%s = %q, want %q", name, data, want)149+		}150+	}151+	if _, err := os.Stat(filepath.Join(output, "retired")); !os.IsNotExist(err) {152+		t.Error("a layer the archive no longer offers was left behind")153+	}154+	if _, err := os.Stat(filepath.Join(output, "index.json")); err != nil {155+		t.Error("the index was not written")156+	}157+}