1// Package web holds the shared HTTP client used for public data sources.2package web34import (5 "context"6 "fmt"7 "io"8 "net/http"9 "time"10)1112// UserAgent identifies requests. Some listing endpoints reject the Go13// default, so present as a desktop browser.14const UserAgent = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/130.0 Safari/537.36"1516type uaTransport struct{ next http.RoundTripper }1718func (t uaTransport) RoundTrip(r *http.Request) (*http.Response, error) {19 if r.Header.Get("User-Agent") == "" {20 r = r.Clone(r.Context())21 r.Header.Set("User-Agent", UserAgent)22 }23 return t.next.RoundTrip(r)24}2526// NewClient returns a client with a browser User-Agent and a sane timeout.27func NewClient() *http.Client {28 return &http.Client{29 Timeout: 45 * time.Second,30 Transport: uaTransport{next: http.DefaultTransport},31 }32}3334// Get fetches url and returns the body, failing on non-200 responses.35func Get(ctx context.Context, c *http.Client, url string) ([]byte, error) {36 req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)37 if err != nil {38 return nil, err39 }40 resp, err := c.Do(req)41 if err != nil {42 return nil, err43 }44 defer resp.Body.Close()45 body, err := io.ReadAll(resp.Body)46 if err != nil {47 return nil, err48 }49 if resp.StatusCode != http.StatusOK {50 return nil, fmt.Errorf("GET %s: HTTP %d", url, resp.StatusCode)51 }52 return body, nil53}