1package crawl23import (4 "encoding/json"5 "errors"6 "fmt"7 "os"8 "path/filepath"9 "sort"10 "strconv"11 "strings"12 "time"13)1415// The write path.16//17// Everything a crawl produces lands through this file, and it lands the way18// docs/generate.md ยง3 describes the archive: a volume register at the root, a19// world register per volume, a capture index and its bodies per world, and a20// tile index beside the rasters. The layout is the historical one and is kept21// verbatim; what is new is that only these functions write it.22//23// Two habits are load-bearing and are worth reading twice.24//25// A register is updated **in place, by identity**. An entry is found by its id,26// its known fields are merged over whatever was there, and its position in the27// list never changes. That is what keeps a directory named once named forever,28// and it is why a field this package has never heard of survives a run.29//30// A capture is deduplicated **by content hash alone**. The body is written --31// idempotently, to a path that is its own hash -- and the index is appended to32// only if that hash has never been seen. So `capturedAt` is first-seen rather33// than last-verified, and a re-crawl of unchanged data leaves the archive byte34// for byte as it was. Everything downstream stands on that: the translators35// replay, the stamps stand still, and a rebuild writes the file that is already36// there.3738// Archive is an archive open for writing.39type Archive struct{ root string }4041// Open readies an archive for writing. The directory need not exist.42func Open(root string) (*Archive, error) {43 if root == "" {44 return nil, errors.New("an archive needs a root")45 }46 return &Archive{root: root}, nil47}4849// Root is where the archive sits.50func (a *Archive) Root() string { return a.root }5152// Volume is a volume as a crawl registers it.53type Volume struct {54 // ID is the volume's identity in the crawling source's id space. It is what55 // the register is keyed by, so it must be stable across runs.56 ID int6457 // Title is the volume's name as the publisher gives it -- the plain one, so58 // two sources' captures of one game answer for one library entry.59 Title string60 // Locator is where the publisher publishes it. Provenance only.61 Locator string62 // Source is the crawler's registry name.63 Source string64 // DirectoryTitle is the title the volume's directory is named from on first65 // sight, and only then. It carries the source's own prefix -- "IGN66 // Cyberpunk 2077" -- so that two publishers describing one game do not67 // fight over one directory while both still register the plain title.68 DirectoryTitle string69}7071// World is a world as a crawl registers it.72type World struct {73 ID int6474 Slug string75 Title string76}7778// RegisterVolume finds or opens a volume's directory and records it in the79// archive's register. It answers the directory, absolute.80func (a *Archive) RegisterVolume(v Volume) (string, error) {81 register, err := readList(filepath.Join(a.root, "archive.json"), "games")82 if err != nil {83 return "", err84 }85 directory := findDirectory(register, v.ID)86 if directory == "" {87 title := v.DirectoryTitle88 if title == "" {89 title = v.Title90 }91 directory = "games/" + DirectoryName(title, v.ID)92 }93 entry := map[string]any{94 "directory": directory,95 "id": v.ID,96 "title": v.Title,97 "url": v.Locator,98 }99 if v.Source != "" {100 entry["source"] = v.Source101 }102 register = upsert(register, v.ID, entry)103 if err := writeJSON(filepath.Join(a.root, "archive.json"), map[string]any{104 "format": "fmg-git-working-tree",105 "games": register,106 }); err != nil {107 return "", err108 }109 return filepath.Join(a.root, filepath.FromSlash(directory)), nil110}111112// RegisterWorld finds or opens a world's directory under a volume and records113// it in the volume's register. It answers the directory, absolute.114func (a *Archive) RegisterWorld(v Volume, volumeDir string, w World) (string, error) {115 path := filepath.Join(volumeDir, "game.json")116 worlds, err := readList(path, "maps")117 if err != nil {118 return "", err119 }120 relative, err := filepath.Rel(a.root, volumeDir)121 if err != nil {122 return "", err123 }124 directory := findDirectory(worlds, w.ID)125 if directory == "" {126 directory = filepath.ToSlash(filepath.Join(relative, "maps", DirectoryName(w.Title, w.ID)))127 }128 worlds = upsert(worlds, w.ID, map[string]any{129 "directory": directory,130 "id": w.ID,131 "slug": w.Slug,132 "title": w.Title,133 })134 volume := map[string]any{135 "id": v.ID,136 "title": v.Title,137 "url": v.Locator,138 "icons": filepath.ToSlash(filepath.Join(relative, "icons", "index.json")),139 "maps": worlds,140 }141 if v.Source != "" {142 volume["source"] = v.Source143 }144 if err := writeJSON(path, volume); err != nil {145 return "", err146 }147 return filepath.Join(a.root, filepath.FromSlash(directory)), nil148}149150// Capture is one archived document: what the source called it, where it came151// from, and the bytes themselves.152type Capture struct {153 Kind string154 SourceID int64155 SourceURL string156 // CapturedAt pins the moment a capture answers to, for a source whose157 // upstream is itself pinned: a fixed publication captured under a recorded158 // time reproduces the same archive on any machine. Empty takes the clock's159 // first sight, which is what a live origin gets.160 CapturedAt string161 Body []byte162}163164// WriteCapture archives one capture and reports whether anything moved.165//166// The body is written unconditionally -- to a path that is its own content hash,167// so writing it twice writes the same bytes -- and the index is appended to only168// when that hash is new. A re-crawl of unchanged data therefore leaves the169// working tree untouched, which is the property every replayable thing170// downstream stands on.171func (a *Archive) WriteCapture(worldDir string, c Capture) (hash string, fresh bool, err error) {172 hash = Hash(c.Body)173 if err := writeFile(filepath.Join(worldDir, "snapshots", "map", hash+".json"), c.Body); err != nil {174 return "", false, err175 }176 path := filepath.Join(worldDir, "snapshots", "index.json")177 var index []snapshot178 if err := readJSON(path, &index); err != nil && !errors.Is(err, os.ErrNotExist) {179 return "", false, err180 }181 for _, held := range index {182 if held.ContentHash == hash {183 return hash, false, nil184 }185 }186 capturedAt := c.CapturedAt187 if capturedAt == "" {188 capturedAt = time.Now().UTC().Format(time.RFC3339Nano)189 }190 index = append(index, snapshot{191 // First-seen, and never refreshed. A volume's creation time is derived192 // from this, so a re-verified capture must not move it.193 CapturedAt: capturedAt,194 ContentHash: hash,195 Kind: c.Kind,196 SourceID: c.SourceID,197 SourceURL: c.SourceURL,198 })199 sort.Slice(index, func(i, j int) bool { return index[i].CapturedAt < index[j].CapturedAt })200 if err := writeJSON(path, index); err != nil {201 return "", false, err202 }203 return hash, true, nil204}205206// snapshot is a capture index record. Its field order is its wire order.207type snapshot struct {208 CapturedAt string `json:"capturedAt"`209 ContentHash string `json:"contentHash"`210 Kind string `json:"kind"`211 SourceID int64 `json:"sourceId"`212 SourceURL string `json:"sourceUrl"`213}214215// TileRecord is one tile as the index records it. Its field order is its wire216// order, and the omissions are the layout's own.217type TileRecord struct {218 ByteLength int `json:"byteLength"`219 ContentHash string `json:"contentHash"`220 ContentType string `json:"contentType"`221 CoverageKey string `json:"coverageKey,omitempty"`222 Error string `json:"error,omitempty"`223 Status string `json:"status"`224 TileSetID int64 `json:"tileSetId,omitempty"`225 URL string `json:"url"`226 X int `json:"x,omitempty"`227 Y int `json:"y,omitempty"`228 Zoom int `json:"zoom,omitempty"`229}230231// TileIndex is a world's tile records, open for writing. It is keyed by URL,232// because the URL is what a resumed run looks a tile up by.233type TileIndex struct {234 byURL map[string]TileRecord235 // worldID and setIDs are what a coverage key is spelled from.236 worldID int64237 setIDs map[string]int64238 usedIDs map[int64]bool239}240241// OpenTileIndex reads what a previous run recorded, so this one can resume: a242// tile already cached is not fetched again, and a tile the origin said it never243// published is not asked for again.244func (a *Archive) OpenTileIndex(worldDir string, worldID int64) (*TileIndex, error) {245 var records []TileRecord246 if err := readJSON(filepath.Join(worldDir, "tiles", "index.json"), &records); err != nil &&247 !errors.Is(err, os.ErrNotExist) {248 return nil, err249 }250 index := &TileIndex{251 byURL: make(map[string]TileRecord, len(records)),252 worldID: worldID,253 setIDs: make(map[string]int64),254 usedIDs: make(map[int64]bool),255 }256 for _, record := range records {257 index.byURL[record.URL] = record258 // The numbered directories a previous run assigned are recovered rather259 // than reassigned: a fresh number would orphan every tile already on260 // disk under the old one.261 if record.TileSetID != 0 && record.Zoom != 0 {262 if set := tileSetOf(record.URL, record.Zoom); set != "" {263 index.setIDs[set] = record.TileSetID264 index.usedIDs[record.TileSetID] = true265 }266 }267 }268 return index, nil269}270271// Held is what a previous run recorded for a URL, if anything.272func (i *TileIndex) Held(url string) (TileRecord, bool) {273 record, ok := i.byURL[url]274 return record, ok275}276277// Put records one tile.278func (i *TileIndex) Put(record TileRecord) { i.byURL[record.URL] = record }279280// SetID is the numbered directory a publisher's pyramid path is stored under,281// assigned once and remembered. The number is derived from the path so a fresh282// archive is deterministic, and probed forward on collision so two paths never283// share a directory.284func (i *TileIndex) SetID(path string) int64 {285 if id, known := i.setIDs[path]; known {286 return id287 }288 id := int64(fnv32a(path)%9000) + 1000289 for i.usedIDs[id] {290 id++291 }292 i.setIDs[path] = id293 i.usedIDs[id] = true294 return id295}296297// CoverageKey is what a tile record carries so a reader of the index can tell298// two worlds' tiles apart without parsing a URL.299func (i *TileIndex) CoverageKey(setID int64, zoom, x, y int) string {300 return strconv.FormatInt(i.worldID, 10) + ":" + strconv.FormatInt(setID, 10) + ":" +301 strconv.Itoa(zoom) + ":" + strconv.Itoa(x) + ":" + strconv.Itoa(y)302}303304// TilePath is where a tile's bytes sit.305func TilePath(worldDir string, setID int64, zoom, x, y int, format string) string {306 return filepath.Join(worldDir, "tiles", "set-"+strconv.FormatInt(setID, 10),307 strconv.Itoa(zoom), strconv.Itoa(x), strconv.Itoa(y)+"."+format)308}309310// WriteTile puts a tile's bytes on disk.311func WriteTile(path string, data []byte) error { return writeFile(path, data) }312313// Close writes the index. Records are sorted by URL, so a diff of two runs reads314// as what changed rather than as what was fetched in what order.315func (i *TileIndex) Close(worldDir string) error {316 records := make([]TileRecord, 0, len(i.byURL))317 for _, record := range i.byURL {318 records = append(records, record)319 }320 sort.SliceStable(records, func(a, b int) bool { return records[a].URL < records[b].URL })321 return writeJSON(filepath.Join(worldDir, "tiles", "index.json"), records)322}323324// WriteArtwork stores one piece of a volume's collection artwork.325func WriteArtwork(volumeDir, key, extension string, data []byte) error {326 return writeFile(filepath.Join(volumeDir, "icons", key+extension), data)327}328329// upsert merges an entry over whatever the register already held for that330// identity, keeping its position. Anything the register carried that this331// package has never heard of survives, which is what lets the layout outlive the332// code that writes it.333func upsert(list []map[string]any, id int64, entry map[string]any) []map[string]any {334 for index, held := range list {335 if identity(held) != id {336 continue337 }338 for key, value := range entry {339 held[key] = value340 }341 list[index] = held342 return list343 }344 return append(list, entry)345}346347func findDirectory(list []map[string]any, id int64) string {348 for _, held := range list {349 if identity(held) != id {350 continue351 }352 if directory, ok := held["directory"].(string); ok {353 return directory354 }355 }356 return ""357}358359// identity reads a register entry's id back. It arrives as a float64 because360// that is what JSON numbers decode to, and every id in the archive is held under361// 2^53 so the round trip is exact.362func identity(entry map[string]any) int64 {363 value, ok := entry["id"].(float64)364 if !ok {365 return 0366 }367 return int64(value)368}369370func readList(path, key string) ([]map[string]any, error) {371 var held map[string]any372 if err := readJSON(path, &held); err != nil {373 if errors.Is(err, os.ErrNotExist) {374 return nil, nil375 }376 return nil, err377 }378 raw, ok := held[key].([]any)379 if !ok {380 return nil, nil381 }382 out := make([]map[string]any, 0, len(raw))383 for _, entry := range raw {384 if record, ok := entry.(map[string]any); ok {385 out = append(out, record)386 }387 }388 return out, nil389}390391func readJSON(path string, into any) error {392 data, err := os.ReadFile(path)393 if err != nil {394 return err395 }396 if err := json.Unmarshal(data, into); err != nil {397 return fmt.Errorf("decode %s: %w", path, err)398 }399 return nil400}401402// writeJSON writes an index the way the layout spells one: two-space indent and403// a trailing newline, so a git diff of an archive reads line by line.404func writeJSON(path string, value any) error {405 data, err := json.MarshalIndent(value, "", " ")406 if err != nil {407 return err408 }409 return writeFile(path, append(data, '\n'))410}411412func writeFile(path string, data []byte) error {413 if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {414 return err415 }416 return os.WriteFile(path, data, 0o644)417}418419// fnv32a is FNV-1a, the same derivation every stable-name-to-number in the lane420// uses.421func fnv32a(value string) uint32 {422 const offset, prime = 2166136261, 16777619423 hash := uint32(offset)424 for i := 0; i < len(value); i++ {425 hash ^= uint32(value[i])426 hash *= prime427 }428 return hash429}430431// tileSetOf recovers a publisher's pyramid path from a tile URL, the way the432// archive's reader does. The two must agree: a record this cannot parse is a433// record the deriver will not see.434func tileSetOf(rawURL string, zoom int) string {435 for _, marker := range []string{"/games/", "/wikimaps/", "/tiles/"} {436 _, rest, found := strings.Cut(rawURL, marker)437 if !found {438 continue439 }440 path, _, found := strings.Cut(rest, "/"+strconv.Itoa(zoom)+"/")441 if !found {442 continue443 }444 return path445 }446 return ""447}