1// Package archive reads the capture archive: content-addressed bytes a crawler2// wrote, arranged so a translator can find them again without a network.3//4// The archive is a source-neutral concept -- captures addressed by the hash of5// their bytes, grouped by volume and by world, each carrying the time it was6// first seen -- but the archive on disk is a specific, historical layout. Years7// of captured history are data, not code, so that layout is kept verbatim as8// input and read here, behind the vocabulary of this package. Nothing new9// inherits its naming: a caller asks for a volume's worlds and their captures,10// and never learns which directory answered.11//12// # The layout this package reads13//14// <root>/archive.json the volume register15// <root>/<volumeDir>/game.json the world register, and where icons live16// <root>/<volumeDir>/icons/<key>.svg collection artwork, svg or png17// <root>/<volumeDir>/<worldDir>/snapshots/index.json18// <root>/<volumeDir>/<worldDir>/snapshots/map/<contentHash>.json19// <root>/<volumeDir>/<worldDir>/tiles/index.json20// <root>/<volumeDir>/<worldDir>/tiles/set-<id>/<z>/<x>/<y>.<ext>21//22// The register files spell their directories as slash-separated paths relative23// to the root, and the JSON keys are the ones the layout was written with. Two24// of its habits are load-bearing and are translated here rather than passed on:25// a volume whose register entry names no source is a MapGenie capture, because26// MapGenie predates the field; and a capture's identity lives in the path it27// sits at rather than in the record, so the volume and world a capture belongs28// to are recovered structurally.29//30// # What a reader may assume31//32// Captures are deduplicated by content hash alone: unchanged bytes record33// nothing, so a re-crawl that fetched the same thing leaves the index byte for34// byte as it was and a rebuild produces the same stamp. Only the newest capture35// of a world is ever read; older ones are history kept on disk. Times compare36// as strings, which is the order that means something for RFC 3339.37package archive3839import (40 "encoding/json"41 "errors"42 "fmt"43 "os"44 "path/filepath"45 "sort"46 "strings"47)4849// DefaultSource is the source a volume register entry with no source names.50// The field was added after the first crawler, so its absence is not a gap.51const DefaultSource = "mapgenie"5253// Archive is an opened capture archive.54type Archive struct {55 root string56 volumes []VolumeRef57}5859// VolumeRef is one volume as the archive registers it. It is a pointer into the60// layout, not a document: opening it is what reads anything.61type VolumeRef struct {62 // Source names the crawler that filled this directory, defaulted.63 Source string64 // ID is the volume's identity in that source's id space.65 ID int6466 // Title is the volume's name as the source published it.67 Title string68 // Locator is where the source published it. Provenance only.69 Locator string7071 dir string72}7374// WorldRef is one world of a volume as the archive registers it.75type WorldRef struct {76 ID int6477 Slug string78 Title string7980 dir string81}8283// CaptureRef is one archived capture: bytes, when they were first seen, and84// what the source called them.85type CaptureRef struct {86 Kind string87 SourceID int6488 SourceURL string89 ContentHash string90 CapturedAt string91}9293// ErrNotReady marks an archive entry that is present but not yet worth reading:94// a world crawled halfway, a register naming a directory that holds no capture.95// A caller skips these rather than failing, because a partial crawl is a normal96// state of an archive that is filled by hand.97var ErrNotReady = errors.New("capture is not ready")9899// Open reads an archive's volume register.100func Open(root string) (*Archive, error) {101 var register struct {102 Volumes []struct {103 Directory string `json:"directory"`104 ID int64 `json:"id"`105 Title string `json:"title"`106 URL string `json:"url"`107 Source string `json:"source"`108 } `json:"games"`109 }110 if err := readJSON(filepath.Join(root, "archive.json"), ®ister); err != nil {111 return nil, err112 }113 out := &Archive{root: root}114 for _, entry := range register.Volumes {115 source := entry.Source116 if source == "" {117 source = DefaultSource118 }119 out.volumes = append(out.volumes, VolumeRef{120 Source: source,121 ID: entry.ID,122 Title: entry.Title,123 Locator: entry.URL,124 dir: filepath.Join(root, filepath.FromSlash(entry.Directory)),125 })126 }127 return out, nil128}129130// Root is where the archive was opened.131func (a *Archive) Root() string { return a.root }132133// Volumes lists every registered volume, in register order.134func (a *Archive) Volumes() []VolumeRef {135 out := make([]VolumeRef, len(a.volumes))136 copy(out, a.volumes)137 return out138}139140// Dir is where a volume's captures sit. It is exported because a source that141// needs a file the archive has no vocabulary for -- a sprite sheet, a stylesheet142// -- reads it from here, and because a caller reporting a failure needs to name143// the directory that failed.144func (v VolumeRef) Dir() string { return v.dir }145146// Dir is where a world's captures sit.147func (w WorldRef) Dir() string { return w.dir }148149// Worlds lists a volume's worlds. The directory listing is the truth -- the150// world register is a convenience that a crawl interrupted mid-world may not151// have updated -- and the register supplies each world's title and slug where152// it knows them.153func (a *Archive) Worlds(v VolumeRef) ([]WorldRef, error) {154 var register struct {155 Worlds []struct {156 Directory string `json:"directory"`157 ID int64 `json:"id"`158 Slug string `json:"slug"`159 Title string `json:"title"`160 } `json:"maps"`161 }162 if err := readJSON(filepath.Join(v.dir, "game.json"), ®ister); err != nil {163 if errors.Is(err, os.ErrNotExist) {164 // The register names a directory nothing has filled yet. An archive165 // is crawled by hand, one volume at a time, so this is an ordinary166 // state and not a fault.167 return nil, fmt.Errorf("%w: %s holds no world register", ErrNotReady, v.dir)168 }169 return nil, err170 }171 known := make(map[string]int, len(register.Worlds))172 for index, entry := range register.Worlds {173 known[filepath.Base(filepath.FromSlash(entry.Directory))] = index174 }175 entries, err := os.ReadDir(filepath.Join(v.dir, "maps"))176 if errors.Is(err, os.ErrNotExist) {177 return nil, fmt.Errorf("%w: %s holds no world", ErrNotReady, v.dir)178 }179 if err != nil {180 return nil, fmt.Errorf("list worlds of %s: %w", v.dir, err)181 }182 var out []WorldRef183 for _, entry := range entries {184 if !entry.IsDir() {185 continue186 }187 world := WorldRef{dir: filepath.Join(v.dir, "maps", entry.Name())}188 if index, ok := known[entry.Name()]; ok {189 world.ID = register.Worlds[index].ID190 world.Slug = register.Worlds[index].Slug191 world.Title = register.Worlds[index].Title192 }193 out = append(out, world)194 }195 sort.Slice(out, func(i, j int) bool { return out[i].dir < out[j].dir })196 return out, nil197}198199// Captures lists a world's captures, oldest first.200func (a *Archive) Captures(w WorldRef) ([]CaptureRef, error) {201 var index []struct {202 CapturedAt string `json:"capturedAt"`203 ContentHash string `json:"contentHash"`204 Kind string `json:"kind"`205 SourceID int64 `json:"sourceId"`206 SourceURL string `json:"sourceUrl"`207 }208 path := filepath.Join(w.dir, "snapshots", "index.json")209 if err := readJSON(path, &index); err != nil {210 if errors.Is(err, os.ErrNotExist) {211 return nil, fmt.Errorf("%w: %s has no capture index", ErrNotReady, w.dir)212 }213 return nil, err214 }215 if len(index) == 0 {216 return nil, fmt.Errorf("%w: %s has an empty capture index", ErrNotReady, w.dir)217 }218 out := make([]CaptureRef, 0, len(index))219 for _, entry := range index {220 out = append(out, CaptureRef{221 Kind: entry.Kind,222 SourceID: entry.SourceID,223 SourceURL: entry.SourceURL,224 ContentHash: entry.ContentHash,225 CapturedAt: entry.CapturedAt,226 })227 }228 sort.SliceStable(out, func(i, j int) bool { return out[i].CapturedAt < out[j].CapturedAt })229 return out, nil230}231232// Newest is a world's most recent capture.233func (a *Archive) Newest(w WorldRef) (CaptureRef, error) {234 captures, err := a.Captures(w)235 if err != nil {236 return CaptureRef{}, err237 }238 return captures[len(captures)-1], nil239}240241// Body reads a capture's archived bytes. The bytes are what the source242// published, untouched: translation happens here, at read time, so an editorial243// change replays over the archive rather than needing a re-crawl.244func (a *Archive) Body(w WorldRef, c CaptureRef) ([]byte, error) {245 path := filepath.Join(w.dir, "snapshots", "map", c.ContentHash+".json")246 data, err := os.ReadFile(path)247 if err != nil {248 return nil, fmt.Errorf("read capture %s: %w", c.ContentHash, err)249 }250 return data, nil251}252253// Artwork reads one piece of a volume's collection artwork, trying each254// extension in turn and reporting the asset name of the one that answered. A255// key nothing answers for is not an error: a collection may name artwork the256// archive never captured, and it simply goes without.257func (a *Archive) Artwork(v VolumeRef, key string, extensions ...string) (name string, data []byte, err error) {258 if !ValidArtworkKey(key) {259 return "", nil, nil260 }261 for _, extension := range extensions {262 path := filepath.Join(v.dir, "icons", key+extension)263 data, err := os.ReadFile(path)264 if errors.Is(err, os.ErrNotExist) {265 continue266 }267 if err != nil {268 return "", nil, fmt.Errorf("read artwork %s: %w", path, err)269 }270 return key + extension, data, nil271 }272 return "", nil, nil273}274275// ValidArtworkKey admits the keys that can name a file without escaping the276// artwork directory. A key that needs more than letters, digits, underscore and277// hyphen is not a name this archive holds.278func ValidArtworkKey(key string) bool {279 if key == "" {280 return false281 }282 for _, r := range key {283 switch {284 case r >= 'a' && r <= 'z', r >= 'A' && r <= 'Z', r >= '0' && r <= '9', r == '_', r == '-':285 default:286 return false287 }288 }289 return true290}291292func readJSON(path string, dst any) error {293 data, err := os.ReadFile(path)294 if err != nil {295 return fmt.Errorf("read %s: %w", path, err)296 }297 if err := json.Unmarshal(data, dst); err != nil {298 return fmt.Errorf("decode %s: %w", path, err)299 }300 return nil301}302303// TrimRoot shortens a path for a log line, so an event names the archive304// entry rather than the reader's home directory.305func TrimRoot(root, path string) string {306 if rel, err := filepath.Rel(root, path); err == nil && !strings.HasPrefix(rel, "..") {307 return filepath.ToSlash(rel)308 }309 return path310}