1package crawl23import (4 "context"5 "errors"6 "fmt"7 "log/slog"8 "time"9)1011// The crawler seam.12//13// A crawler is the outward half of a source: it knows one publisher's endpoints14// the way that source's reader knows its bytes. The two are deliberately15// separate packages, because they have opposite properties -- one is a pure16// function of an archive and the other is the only thing in Atlas that can fail17// because somebody else's server is having a bad afternoon -- and because the18// import rule that keeps the network out of the rest of the lane is a rule about19// this package, not about a habit.20//21// A crawler is handed a run and hands back nothing. What it produces lands in22// the archive through this package's write path, which is the only way anything23// reaches an archive.2425// Run is one crawl: what it is aimed at, what it may do, and where it writes.26type Run struct {27 // Target is the crawler's own spelling of what to fetch -- a game slug, an28 // "object/map" pair, a planetary body, a city. Each crawler documents its29 // own, because only it can say what one means.30 Target string31 // Archive is where captures land.32 Archive *Archive33 // Fetch is the run's shared schedule. Every request goes through it.34 Fetch *Fetcher35 // Concurrency is how many requests may be in flight. The schedule still36 // spaces them, so this hides latency rather than raising the rate.37 Concurrency int38 // MaxZoom clamps how deep a pyramid is captured; zero takes what the39 // publisher offers.40 MaxZoom int41 // On is the day a dated capture answers to, empty for today. Only a source42 // whose worlds are dated captures of one ground reads it.43 On string44 // DryRun reports what would be fetched and writes nothing. The discovery45 // requests still happen: what a crawl would do cannot be known without46 // asking what is there.47 DryRun bool48 // Log is the run's event stream; nil is a logger that discards.49 Log *slog.Logger50}5152// Logger is the run's stream, never nil.53func (r Run) Logger() *slog.Logger {54 if r.Log == nil {55 return slog.New(slog.DiscardHandler)56 }57 return r.Log58}5960// A Crawler fetches one publisher's captures into the archive.61type Crawler interface {62 // Name is the crawler's registry name, and it is the source name the63 // archive records for what it writes, so a reader knows which translator to64 // read it back with.65 Name() string66 // Usage is one line about what Target means for this crawler, printed by67 // the subcommand's help.68 Usage() string69 // Crawl runs. It is expected to be interrupted: a run that stops halfway70 // leaves an archive a later run resumes from, because every write is71 // idempotent and every fetched thing is recorded before the next is asked72 // for.73 Crawl(ctx context.Context, run Run) error74}7576// ErrNotRunnable marks a crawler whose endpoints this checkout will not reach:77// the game sources' captures are archived and their endpoints are somebody78// else's editorial work, so their crawlers are kept complete and are not run.79// It is a refusal a caller can recognise rather than a message it has to read.80var ErrNotRunnable = errors.New("this crawler is not run against live endpoints from here")8182// Registry is the assembly point, and -- exactly as with the source registry --83// the one file outside a crawler's own scope permitted to name one.84func Registry() []Crawler {85 return []Crawler{86 ignCrawler{},87 blueMarbleCrawler{},88 }89}9091// For finds the crawler registered under a name.92func For(name string) (Crawler, error) {93 for _, crawler := range Registry() {94 if crawler.Name() == name {95 return crawler, nil96 }97 }98 return nil, fmt.Errorf("no crawler is registered as %q", name)99}100101// Names lists the registered crawlers, in registration order.102func Names() []string {103 crawlers := Registry()104 out := make([]string, 0, len(crawlers))105 for _, crawler := range crawlers {106 out = append(out, crawler.Name())107 }108 return out109}110111// Day is the date a dated capture answers to: the one a run named, or today in112// UTC. A crawl of a city is a crawl of a day, and the day is what its world is113// called, so it is a run's decision rather than a clock's.114func Day(on string) (string, error) {115 if on == "" {116 return time.Now().UTC().Format(time.DateOnly), nil117 }118 if _, err := time.Parse(time.DateOnly, on); err != nil {119 return "", fmt.Errorf("capture day %q is not a YYYY-MM-DD date", on)120 }121 return on, nil122}