1// Package crawl fetches what publishers serve and writes it into the capture2// archive. It is the one package in Atlas permitted to reach the network, and3// `depcheck`'s netconfine rule is what makes that true rather than customary.4//5// # Why the boundary is here6//7// A bundle serves offline, forever. That invariant is kept at two ends: a8// payload carries no runtime URL, and only one package can produce one. Every9// other part of the pipeline is a pure function of bytes already on disk, which10// is what lets an editorial change replay over years of captures instead of11// re-crawling them — and what lets a translator be tested without a network,12// a fixture, or a mock.13//14// Fetching is crawling, wherever it happens. The enrich lane's national15// hydrography evidence is captured here too, and travels in the archive, so the16// join re-runs against the archive rather than against a live endpoint.17//18// # What a crawl promises19//20// - **Unchanged bytes record nothing.** A capture is deduplicated by content21// hash alone. A re-crawl that fetched the same thing leaves the archive byte22// for byte as it was: the payload file is rewritten to the same path with23// the same bytes, and the index is not touched at all. `capturedAt` is24// first-seen, never last-verified, so nothing downstream moves and a rebuild25// computes the same stamp.26// - **Politeness is global.** One schedule spaces every request of a run,27// whatever its concurrency and whatever host it is aimed at. A 429 or a 5xx28// pushes the whole schedule back, not just the worker that saw it.29// - **Absence is a result.** A tile origin answers 403 or 404 for an object it30// never published, and that is an ordinary outcome recorded as such, so a31// re-crawl does not ask again.32// - **Nothing is renamed.** A volume or world the archive already holds a33// directory for keeps it forever, found by identity rather than by name.34//35// # The archive this writes36//37// The layout is the historical one, kept verbatim because years of captured38// history are data rather than code. `internal/generate/archive` reads it and39// docs/generate.md §3 is normative for it. What this package adds is the write40// path: where a capture's bytes go, how the registers are updated in place, and41// what is recorded about a tile that was fetched, one that was not published,42// and one that failed.43package crawl4445import (46 "crypto/sha256"47 "encoding/hex"48 "strconv"49 "strings"50)5152// Hash is the archive's content address: SHA-256 of the bytes, lowercase hex.53// It addresses a capture body, and it is what a tile record carries so a54// derivation stamp can be taken without reading a raster.55func Hash(data []byte) string {56 sum := sha256.Sum256(data)57 return hex.EncodeToString(sum[:])58}5960// The statuses a tile record carries. Only Cached means the bytes are on disk.61const (62 // StatusCached: the bytes are here, under the path the record names.63 StatusCached = "cached"64 // StatusAbsent: the origin says it never published this tile. A pyramid is65 // a rectangle and its corners are often empty, so this is the ordinary66 // answer at the edges and it is recorded so a re-crawl does not ask again.67 StatusAbsent = "absent"68 // StatusFailed: something went wrong that is not absence. The error is69 // recorded beside it, and a later run will try again.70 StatusFailed = "failed"71)7273// DirectoryName is how a volume or a world's directory is named on first sight:74// its title, folded to something a filesystem is comfortable with, with its75// identity appended so two publishers describing the same game never collide.76//77// It is only ever consulted on first sight. A directory the archive already78// holds is found by identity and kept forever, whatever it was called, because79// renaming one would orphan every capture and every tile inside it.80func DirectoryName(title string, id int64) string {81 out := make([]rune, 0, len(title)+8)82 for _, r := range strings.ToLower(title) {83 if folded, known := foldedLatin[r]; known {84 r = folded85 }86 switch {87 case r >= 'a' && r <= 'z', r >= '0' && r <= '9':88 out = append(out, r)89 default:90 if len(out) > 0 && out[len(out)-1] != '-' {91 out = append(out, '-')92 }93 }94 }95 for len(out) > 0 && out[len(out)-1] == '-' {96 out = out[:len(out)-1]97 }98 return string(out) + "-" + strconv.FormatInt(id, 10)99}100101// foldedLatin is the diacritics the corpus actually contains. It is a table102// rather than a normalization library because the answer has to be stable103// forever: a directory named once is named for good, and a Unicode table that104// improved would rename somebody's archive.105var foldedLatin = map[rune]rune{106 'á': 'a', 'à': 'a', 'â': 'a', 'ä': 'a', 'ã': 'a', 'å': 'a',107 'é': 'e', 'è': 'e', 'ê': 'e', 'ë': 'e',108 'í': 'i', 'ì': 'i', 'î': 'i', 'ï': 'i',109 'ó': 'o', 'ò': 'o', 'ô': 'o', 'ö': 'o', 'õ': 'o', 'ø': 'o',110 'ú': 'u', 'ù': 'u', 'û': 'u', 'ü': 'u',111 'ñ': 'n', 'ç': 'c', 'ß': 's', 'ý': 'y',112}