1package crawl23import (4 "context"5 "errors"6 "fmt"7 "io"8 "math"9 "net/http"10 "strconv"11 "strings"12 "sync"13 "time"14)1516// Politeness.17//18// A crawl of this corpus is thousands of small requests to somebody else's19// origin, run by hand, for a personal archive. The rule the Fetcher encodes is20// that Atlas is a guest: one schedule spaces every request of a run whatever its21// concurrency, and a refusal slows the whole run rather than the one worker that22// heard it.23//24// The schedule is a single monotonically advancing instant taken under a lock,25// not a token bucket. A bucket lets a run that idled spend its savings in a26// burst, which is exactly the shape a rate limiter is watching for; an advancing27// instant simply never lets two requests be closer together than the interval,28// however many goroutines are waiting.2930// Defaults a run uses unless it says otherwise. The interval is the load-bearing31// one: at 150 ms a crawl asks for under seven things a second, which is slower32// than a person clicking around the same site.33const (34 DefaultInterval = 150 * time.Millisecond35 DefaultConcurrency = 436 DefaultTimeout = 60 * time.Second37 // Attempts counts the first try. Beyond four, an origin that is refusing is38 // refusing, and asking again is rudeness rather than resilience.39 Attempts = 440)4142// UserAgent is what a crawl calls itself.43//44// It is a browser's string, and that is a decision rather than an oversight:45// several of these origins answer 403 to anything else, and a 403 is46// indistinguishable from "never published" in a tile pyramid, so an honest47// user-agent would silently punch holes in an archive. The politeness that48// matters -- the interval, the backoff, the refusal to re-ask -- is in the49// behaviour, not the header.50const UserAgent = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) " +51 "AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36"5253// ErrAbsent marks something the origin says it never published: a 404, or the54// 403 several tile origins answer with instead. A pyramid is a rectangle and its55// corners are usually empty, so this is an ordinary result and a caller records56// it rather than failing on it.57var ErrAbsent = errors.New("not published")5859// ErrStaging marks a 202: an origin that has accepted the request and is60// preparing the answer. A caller waits and asks again.61var ErrStaging = errors.New("still being prepared")6263// Fetcher is one run's whole outward face: its schedule, its client, and its64// manners. It is safe for concurrent use, which is the point -- every worker of65// a run shares one.66type Fetcher struct {67 client *http.Client68 interval time.Duration6970 mu sync.Mutex71 next time.Time72}7374// NewFetcher opens a run's schedule.75func NewFetcher(interval time.Duration, concurrency int) *Fetcher {76 if interval <= 0 {77 interval = DefaultInterval78 }79 if concurrency < 1 {80 concurrency = DefaultConcurrency81 }82 return &Fetcher{83 interval: interval,84 client: &http.Client{85 Timeout: DefaultTimeout,86 Transport: &http.Transport{MaxIdleConnsPerHost: concurrency},87 },88 }89}9091// Get fetches one thing, and reports the content type the origin gave it.92func (f *Fetcher) Get(ctx context.Context, url string, headers map[string]string) ([]byte, string, error) {93 return f.do(ctx, func() (*http.Request, error) {94 request, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)95 if err != nil {96 return nil, err97 }98 for name, value := range headers {99 request.Header.Set(name, value)100 }101 return request, nil102 }, url)103}104105// Post sends one request body, which is how one of the corpus's origins is106// asked anything at all.107func (f *Fetcher) Post(ctx context.Context, url, contentType string, body []byte) ([]byte, string, error) {108 return f.do(ctx, func() (*http.Request, error) {109 request, err := http.NewRequestWithContext(ctx, http.MethodPost, url, strings.NewReader(string(body)))110 if err != nil {111 return nil, err112 }113 request.Header.Set("Content-Type", contentType)114 return request, nil115 }, url)116}117118// do is the whole retry and backoff policy, in one place so that every request119// of every crawler obeys it.120func (f *Fetcher) do(ctx context.Context, build func() (*http.Request, error), url string) ([]byte, string, error) {121 var last error122 for attempt := range Attempts {123 if attempt > 0 {124 // Exponential, and applied to the shared schedule rather than to125 // this goroutine: an origin under strain should see the whole run126 // slow down.127 f.penalise(time.Duration(math.Pow(2, float64(attempt))) * time.Second)128 }129 if err := f.wait(ctx); err != nil {130 return nil, "", err131 }132 request, err := build()133 if err != nil {134 return nil, "", err135 }136 request.Header.Set("User-Agent", UserAgent)137 if request.Header.Get("Accept") == "" {138 request.Header.Set("Accept", "*/*")139 }140 response, err := f.client.Do(request)141 if err != nil {142 last = err143 continue144 }145 body, readErr := io.ReadAll(response.Body)146 response.Body.Close()147148 switch status := response.StatusCode; {149 case status == http.StatusNotFound, status == http.StatusForbidden:150 // A tile origin answers 403 for an object it never published, so151 // the two are one answer here. Treating a 403 as a failure would152 // make a run retry the empty corners of every pyramid four times.153 return nil, "", fmt.Errorf("%w: %s", ErrAbsent, url)154 case status == http.StatusAccepted:155 return nil, "", fmt.Errorf("%w: %s", ErrStaging, url)156 case status == http.StatusTooManyRequests, status >= 500:157 f.penalise(retryAfter(response, time.Duration(attempt+1)*2*time.Second))158 last = fmt.Errorf("HTTP %d from %s", status, url)159 continue160 case status != http.StatusOK:161 // Anything else is a disagreement rather than a hiccup, and asking162 // again will not settle it.163 return nil, "", fmt.Errorf("HTTP %d from %s", status, url)164 case readErr != nil:165 last = readErr166 continue167 }168 return body, response.Header.Get("Content-Type"), nil169 }170 return nil, "", last171}172173// wait takes this request's slot in the run's schedule and sleeps until it.174func (f *Fetcher) wait(ctx context.Context) error {175 f.mu.Lock()176 now := time.Now()177 slot := f.next178 if slot.Before(now) {179 slot = now180 }181 f.next = slot.Add(f.interval)182 f.mu.Unlock()183184 delay := time.Until(slot)185 if delay <= 0 {186 return ctx.Err()187 }188 timer := time.NewTimer(delay)189 defer timer.Stop()190 select {191 case <-ctx.Done():192 return ctx.Err()193 case <-timer.C:194 return nil195 }196}197198// penalise pushes the whole run's schedule forward. Slowing one worker would199// only hand its slot to another; slowing the schedule is the only thing an200// origin can actually feel.201func (f *Fetcher) penalise(d time.Duration) {202 f.mu.Lock()203 defer f.mu.Unlock()204 if until := time.Now().Add(d); until.After(f.next) {205 f.next = until206 }207}208209// retryAfter reads the origin's own answer to "how long", falling back to the210// caller's guess. Only the seconds form is read: the HTTP-date form would need a211// clock the two ends agree on, and the fallback is safe.212func retryAfter(response *http.Response, fallback time.Duration) time.Duration {213 seconds, err := strconv.Atoi(strings.TrimSpace(response.Header.Get("Retry-After")))214 if err != nil || seconds < 0 {215 return fallback216 }217 return time.Duration(seconds) * time.Second218}