1package main23import (4 "bytes"5 "context"6 "encoding/json"7 "errors"8 "fmt"9 "io"10 "math"11 "net/http"12 "strconv"13 "strings"14 "sync"15 "time"16)1718// errAbsent marks a tile the CDN does not hold. The published bounds of a layer19// are a bounding box, and content rarely fills it exactly; the origin answers20// 403 rather than 404 for an object that is not there, so both mean the same21// thing here. An absent tile is a normal result, not a failure.22var errAbsent = errors.New("not published")2324// errAccepted marks a download the server is still staging: ArcGIS Hub's25// download API answers 202 while it composes a file, and the polite response26// is to come back shortly, not to fail.27var errAccepted = errors.New("accepted, still staging")2829// fetcher spaces every request out by a fixed interval and caps how many are30// in flight, so a crawl of tens of thousands of tiles stays a light load31// rather than a burst. It backs off when the server asks it to.32type fetcher struct {33 http *http.Client34 interval time.Duration3536 mu sync.Mutex37 next time.Time38}3940func newFetcher(concurrency int, interval time.Duration) *fetcher {41 return &fetcher{42 http: &http.Client{43 Timeout: 60 * time.Second,44 Transport: &http.Transport{45 MaxIdleConnsPerHost: concurrency,46 },47 },48 interval: interval,49 }50}5152// wait returns once this caller owns the next slot in the request schedule.53func (f *fetcher) wait(ctx context.Context) error {54 f.mu.Lock()55 now := time.Now()56 slot := f.next57 if slot.Before(now) {58 slot = now59 }60 f.next = slot.Add(f.interval)61 f.mu.Unlock()6263 delay := time.Until(slot)64 if delay <= 0 {65 return nil66 }67 timer := time.NewTimer(delay)68 defer timer.Stop()69 select {70 case <-timer.C:71 return nil72 case <-ctx.Done():73 return ctx.Err()74 }75}7677// penalise pushes the whole schedule back, so a 429 slows every worker rather78// than just the one that saw it.79func (f *fetcher) penalise(d time.Duration) {80 f.mu.Lock()81 defer f.mu.Unlock()82 if until := time.Now().Add(d); until.After(f.next) {83 f.next = until84 }85}8687const maxAttempts = 48889func (f *fetcher) get(ctx context.Context, url string) ([]byte, string, error) {90 return f.getReferred(ctx, url, "")91}9293// getReferred names the page the request stands in for. Piggyback's tile CDN94// serves only requests that arrive as its own map's, and answers everything95// else 403 -- which get would read as "not published".96func (f *fetcher) getReferred(ctx context.Context, url, referer string) ([]byte, string, error) {97 return f.do(ctx, url, func() (*http.Request, error) {98 request, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)99 if err != nil {100 return nil, err101 }102 if referer != "" {103 request.Header.Set("Referer", referer)104 }105 return request, nil106 })107}108109// post carries the same manners as get -- spacing, backoff, penalties shared110// with every other request -- because an API asked with POST is no less a111// guest's request than a tile asked with GET.112func (f *fetcher) post(ctx context.Context, url, contentType string, body []byte) ([]byte, string, error) {113 return f.do(ctx, url, func() (*http.Request, error) {114 request, err := http.NewRequestWithContext(ctx, http.MethodPost, url, bytes.NewReader(body))115 if err != nil {116 return nil, err117 }118 request.Header.Set("Content-Type", contentType)119 return request, nil120 })121}122123func (f *fetcher) do(124 ctx context.Context,125 url string,126 build func() (*http.Request, error),127) ([]byte, string, error) {128 var lastErr error129 for attempt := range maxAttempts {130 if attempt > 0 {131 backoff := time.Duration(math.Pow(2, float64(attempt))) * time.Second132 f.penalise(backoff)133 }134 if err := f.wait(ctx); err != nil {135 return nil, "", err136 }137138 request, err := build()139 if err != nil {140 return nil, "", err141 }142 request.Header.Set("User-Agent", userAgent)143 request.Header.Set("Accept", "*/*")144145 response, err := f.http.Do(request)146 if err != nil {147 lastErr = err148 continue149 }150 body, readErr := io.ReadAll(response.Body)151 response.Body.Close()152153 switch {154 case response.StatusCode == http.StatusNotFound,155 response.StatusCode == http.StatusForbidden:156 return nil, "", fmt.Errorf("%w: %s", errAbsent, url)157 case response.StatusCode == http.StatusAccepted:158 return nil, "", fmt.Errorf("%w: %s", errAccepted, url)159 case response.StatusCode == http.StatusTooManyRequests ||160 response.StatusCode >= 500:161 f.penalise(retryAfter(response, time.Duration(attempt+1)*2*time.Second))162 lastErr = fmt.Errorf("HTTP %d from %s", response.StatusCode, url)163 continue164 case response.StatusCode != http.StatusOK:165 return nil, "", fmt.Errorf("HTTP %d from %s", response.StatusCode, url)166 case readErr != nil:167 lastErr = readErr168 continue169 }170 return body, response.Header.Get("Content-Type"), nil171 }172 return nil, "", lastErr173}174175func retryAfter(response *http.Response, fallback time.Duration) time.Duration {176 if value := response.Header.Get("Retry-After"); value != "" {177 if seconds, err := strconv.Atoi(strings.TrimSpace(value)); err == nil && seconds >= 0 {178 return time.Duration(seconds) * time.Second179 }180 }181 return fallback182}183184func (f *fetcher) getJSON(ctx context.Context, url string, into any) ([]byte, error) {185 body, _, err := f.get(ctx, url)186 if err != nil {187 return nil, err188 }189 if err := json.Unmarshal(body, into); err != nil {190 return nil, fmt.Errorf("decode %s: %w", url, err)191 }192 return body, nil193}194195type apiGameConfig struct {196 TilesBaseURL string `json:"tiles_base_url"`197 CDNURL string `json:"cdn_url"`198 URL string `json:"url"`199}200201// apiGameFull is what /games already returns for every game: identity, config,202// and the full map list. The /games/{id}/full endpoint requires an account, and203// carries nothing extra that a crawl needs.204type apiGameFull struct {205 ID int64 `json:"id"`206 Title string `json:"title"`207 Slug string `json:"slug"`208 Config apiGameConfig `json:"config"`209 Maps []apiMapRef `json:"maps"`210}211212type apiMapRef struct {213 ID int64 `json:"id"`214 Title string `json:"title"`215 Slug string `json:"slug"`216}217218type apiRange struct {219 Min int `json:"min"`220 Max int `json:"max"`221}222223type apiBound struct {224 X apiRange `json:"x"`225 Y apiRange `json:"y"`226}227228type apiTileSet struct {229 Name string `json:"name"`230 Path string `json:"path"`231 MinZoom int `json:"min_zoom"`232 MaxZoom int `json:"max_zoom"`233 Extension string `json:"extension"`234 Bounds map[string]apiBound `json:"bounds"`235}236237type apiMapFull struct {238 ID int64 `json:"id"`239 GameID int64 `json:"game_id"`240 Title string `json:"title"`241 Slug string `json:"slug"`242 URL string `json:"url"`243 // Where the map opens, which is the one point known to be on it. A layer244 // published without a window is searched outward from here.245 InitialLatitude json.RawMessage `json:"initial_latitude"`246 InitialLongitude json.RawMessage `json:"initial_longitude"`247 Config struct {248 TileSets []apiTileSet `json:"tile_sets"`249 } `json:"config"`250 Groups []struct {251 Categories []struct {252 Locations []json.RawMessage `json:"locations"`253 } `json:"categories"`254 } `json:"groups"`255}256257func (m *apiMapFull) pinCount() int {258 total := 0259 for _, group := range m.Groups {260 for _, category := range group.Categories {261 total += len(category.Locations)262 }263 }264 return total265}266267func listGames(ctx context.Context, f *fetcher) ([]apiGameFull, error) {268 var games []apiGameFull269 if _, err := f.getJSON(ctx, apiBase+"/games", &games); err != nil {270 return nil, err271 }272 return games, nil273}274275func findGame(games []apiGameFull, target string) (*apiGameFull, bool) {276 if id, err := strconv.ParseInt(target, 10, 64); err == nil {277 for index := range games {278 if games[index].ID == id {279 return &games[index], true280 }281 }282 }283 target = strings.ToLower(target)284 for index := range games {285 if strings.ToLower(games[index].Slug) == target {286 return &games[index], true287 }288 }289 for index := range games {290 if strings.Contains(strings.ToLower(games[index].Title), target) {291 return &games[index], true292 }293 }294 return nil, false295}296297// fetchMap returns both the decoded map and its exact response bytes: the298// archive stores what the server actually sent, hashed as received.299func fetchMap(ctx context.Context, f *fetcher, id int64) (*apiMapFull, []byte, error) {300 var full apiMapFull301 url := fmt.Sprintf("%s/maps/%d/full", apiBase, id)302 raw, err := f.getJSON(ctx, url, &full)303 if err != nil {304 return nil, nil, err305 }306 return &full, raw, nil307}308309func selectMaps(game *apiGameFull, target string) ([]apiMapRef, error) {310 if target == "" {311 if len(game.Maps) == 0 {312 return nil, errors.New("game publishes no maps")313 }314 return game.Maps, nil315 }316 if id, err := strconv.ParseInt(target, 10, 64); err == nil {317 for _, candidate := range game.Maps {318 if candidate.ID == id {319 return []apiMapRef{candidate}, nil320 }321 }322 }323 lowered := strings.ToLower(target)324 for _, candidate := range game.Maps {325 if strings.ToLower(candidate.Slug) == lowered {326 return []apiMapRef{candidate}, nil327 }328 }329 available := make([]string, 0, len(game.Maps))330 for _, candidate := range game.Maps {331 available = append(available, candidate.Slug)332 }333 return nil, fmt.Errorf("no map matches %q; available: %s",334 target, strings.Join(available, ", "))335}