1package crawl23import (4 "context"5 "encoding/json"6 "errors"7 "fmt"8 "log/slog"9 "strconv"10 "strings"11 "sync"1213 "github.com/FelineStateMachine/atlas/internal/generate/sources/ign"14 "github.com/FelineStateMachine/atlas/internal/logging"15)1617// The IGN wikimap crawler.18//19// It is complete and it is not run. IGN's wikimaps are already archived, their20// endpoints are somebody else's editorial work, and a clean-room rewrite has21// nothing to learn from asking for bytes it already has. What it is for is the22// shape: this is what a game-source crawler is, so that the archive layout, the23// politeness, the resume behaviour and the source gate are written down as code24// rather than as a description of code that used to exist.25//26// # The gate27//28// Some IGN wikimap pages are MapGenie maps in an IGN frame: the page declares a29// MapGenie game id and serves MapGenie's tiles. Archiving one here would put a30// second, worse copy of data Atlas already reads properly into the archive,31// where a later merge would fold a source into itself and report a beautiful32// agreement between a thing and itself. The page's own declaration is the33// evidence, and it exists only while the crawler is looking at the page -- which34// is why the refusal lives here and the reader says it cannot check.3536const (37 ignPage = "https://www.ign.com/maps/"38 ignGraphQL = "https://mollusk.apis.ign.com/graphql"39 // ignIDBit separates IGN's identities from every other source's in the40 // archive register. The low bits are a hash of a stable name and the high41 // bit says whose space it is, so a game two publishers both describe never42 // collides on a number and every id stays under 2^53, where a JSON round43 // trip is exact.44 ignIDBit = int64(1) << 3245)4647type ignCrawler struct{}4849func (ignCrawler) Name() string { return "ign" }5051func (ignCrawler) Usage() string {52 return "an IGN wikimap as <objectSlug>/<mapSlug>, e.g. cyberpunk-2077/night-city"53}5455func (c ignCrawler) Crawl(ctx context.Context, run Run) error {56 object, mapSlug, ok := strings.Cut(run.Target, "/")57 if !ok || object == "" || mapSlug == "" {58 return fmt.Errorf("%s names a wikimap as <objectSlug>/<mapSlug>, not %q", c.Name(), run.Target)59 }60 log := run.Logger().With(logging.Source(c.Name()), logging.Op("crawl"))6162 page, err := c.fetchPage(ctx, run, object, mapSlug)63 if err != nil {64 return err65 }66 capture, err := c.fetchMarkers(ctx, run, object, mapSlug, page)67 if err != nil {68 return err69 }70 body, err := json.MarshalIndent(capture, "", " ")71 if err != nil {72 return err73 }74 volume := Volume{75 ID: ignArchiveID("ign:" + object),76 Title: page.ObjectName,77 Locator: ignPage + object + "/" + mapSlug,78 Source: c.Name(),79 // The same-slug policy: two publishers describe Night City, and each80 // registers the plain title while naming its directory after itself.81 DirectoryTitle: "IGN " + page.ObjectName,82 }83 world := World{84 ID: ignArchiveID("ign:" + object + "/" + mapSlug),85 Slug: mapSlug,86 Title: page.MapName,87 }88 if run.DryRun {89 log.Info("crawl would archive", logging.Volume(object), logging.World(mapSlug),90 "types", len(capture.Types), "markers", len(capture.Markers),91 "bytes", len(body))92 return nil93 }9495 volumeDir, err := run.Archive.RegisterVolume(volume)96 if err != nil {97 return err98 }99 worldDir, err := run.Archive.RegisterWorld(volume, volumeDir, world)100 if err != nil {101 return err102 }103 hash, fresh, err := run.Archive.WriteCapture(worldDir, Capture{104 Kind: "ign-map",105 SourceID: world.ID,106 SourceURL: volume.Locator,107 Body: body,108 })109 if err != nil {110 return err111 }112 log.Info("capture archived", logging.Volume(object), logging.World(mapSlug),113 logging.Stamp(hash[:12]), "fresh", fresh)114115 if err := c.fetchArtwork(ctx, run, volumeDir, capture.Types, log); err != nil {116 return err117 }118 return c.fetchTiles(ctx, run, worldDir, world.ID, object, mapSlug, page, log)119}120121// ignPageState is the slice of the page's embedded state a capture needs.122type ignPageState struct {123 MapType string `json:"mapType"`124 MapGenieGameID string `json:"mapGenieGameId"`125 ObjectName string `json:"objectName"`126 MapName string `json:"mapName"`127 Width float64 `json:"width"`128 Height float64 `json:"height"`129 MinZoom int `json:"minZoom"`130 MaxZoom int `json:"maxZoom"`131 InitialLat float64 `json:"initialLat"`132 InitialLng float64 `json:"initialLng"`133 Background string `json:"backgroundColor"`134 Tilesets []string `json:"tilesets"`135 Types []struct {136 TypeSlug string `json:"typeSlug"`137 TypeName string `json:"typeName"`138 ParentSlug string `json:"parentTypeSlug"`139 MarkerIcon struct {140 URL string `json:"url"`141 Width int `json:"width"`142 Height int `json:"height"`143 } `json:"markerIcon"`144 } `json:"types"`145}146147// fetchPage reads the wikimap's page and holds it to every gate before a single148// tile is asked for.149func (c ignCrawler) fetchPage(ctx context.Context, run Run, object, mapSlug string) (ignPageState, error) {150 url := ignPage + object + "/" + mapSlug151 body, _, err := run.Fetch.Get(ctx, url, nil)152 if err != nil {153 return ignPageState{}, fmt.Errorf("read %s: %w", url, err)154 }155 const open = `<script id="__NEXT_DATA__" type="application/json">`156 _, rest, found := strings.Cut(string(body), open)157 if !found {158 return ignPageState{}, fmt.Errorf("%s carries no embedded state", url)159 }160 state, _, found := strings.Cut(rest, "</script>")161 if !found {162 return ignPageState{}, fmt.Errorf("%s carries no embedded state", url)163 }164 var envelope struct {165 Props struct {166 PageProps struct {167 Page struct {168 Map ignPageState `json:"map"`169 } `json:"page"`170 } `json:"pageProps"`171 } `json:"props"`172 }173 if err := json.Unmarshal([]byte(state), &envelope); err != nil {174 return ignPageState{}, fmt.Errorf("decode the state of %s: %w", url, err)175 }176 page := envelope.Props.PageProps.Page.Map177178 switch {179 case page.MapType != "ign":180 return ignPageState{}, fmt.Errorf(181 "%s is not a native IGN wikimap: the page calls itself %q", url, page.MapType)182 case page.MapGenieGameID != "" && page.MapGenieGameID != "null":183 // The gate. Archiving this would put a second, worse copy of data Atlas184 // already reads properly into the archive.185 return ignPageState{}, fmt.Errorf(186 "%s is a MapGenie map in an IGN frame (it declares MapGenie game %s); "+187 "crawl it through the MapGenie source instead", url, page.MapGenieGameID)188 case len(page.Tilesets) == 0:189 return ignPageState{}, fmt.Errorf("%s offers no tile set", url)190 case page.Width <= 0 || page.Height <= 0:191 return ignPageState{}, fmt.Errorf("%s draws a map with no size", url)192 case len(page.Types) == 0:193 return ignPageState{}, fmt.Errorf("%s declares no marker type", url)194 }195 template := page.Tilesets[0]196 if want := "/wikimaps/" + object + "/" + mapSlug + "/"; !strings.Contains(template, want) {197 // The archive groups tiles by the path in their URL, and the reader198 // looks them up under the same path. A template that does not carry it199 // would archive tiles nothing could find again.200 return ignPageState{}, fmt.Errorf(201 "the tile template %q does not carry %s, so its tiles could not be grouped",202 template, want)203 }204 return page, nil205}206207// ignCapture is the canonical document archived for a wikimap. Its shape is the208// source reader's, because the reader is what has to read it back.209type ignCapture struct {210 Source string `json:"source"`211 ObjectSlug string `json:"objectSlug"`212 MapSlug string `json:"mapSlug"`213 GameTitle string `json:"gameTitle"`214 MapTitle string `json:"mapTitle"`215 Map ignCaptureMap `json:"map"`216 Types []ignType `json:"types"`217 Markers []json.RawMessage `json:"markers"`218}219220type ignCaptureMap struct {221 Width float64 `json:"width"`222 Height float64 `json:"height"`223 MinZoom int `json:"minZoom"`224 MaxZoom int `json:"maxZoom"`225 InitialLat float64 `json:"initialLat"`226 InitialLng float64 `json:"initialLng"`227 BackgroundColor string `json:"backgroundColor"`228 Tileset string `json:"tileset"`229}230231type ignType struct {232 TypeSlug string `json:"typeSlug"`233 TypeName string `json:"typeName"`234 ParentTypeSlug string `json:"parentTypeSlug,omitempty"`235 IconURL string `json:"iconUrl,omitempty"`236 IconWidth int `json:"iconWidth,omitempty"`237 IconHeight int `json:"iconHeight,omitempty"`238}239240// fetchMarkers asks for every marker of every declared type in one query, and241// assembles the capture.242func (c ignCrawler) fetchMarkers(243 ctx context.Context, run Run, object, mapSlug string, page ignPageState,244) (ignCapture, error) {245 slugs := make([]string, 0, len(page.Types))246 for _, t := range page.Types {247 slugs = append(slugs, t.TypeSlug)248 }249 query := map[string]any{250 "query": `query MapMarkers($objectSlug: String!, $mapSlug: String!, $typeSlugs: [String!]!) {251 markersMultiple(objectSlug: $objectSlug, mapSlug: $mapSlug, typeSlugs: $typeSlugs) {252 id lat lng markerName markerSlug typeSlug wikiPage iconSlug regionId checklistTaskId253 }254}`,255 "variables": map[string]any{256 "objectSlug": object,257 "mapSlug": mapSlug,258 "typeSlugs": slugs,259 },260 }261 body, err := json.Marshal(query)262 if err != nil {263 return ignCapture{}, err264 }265 answer, _, err := run.Fetch.Post(ctx, ignGraphQL, "application/json", body)266 if err != nil {267 return ignCapture{}, fmt.Errorf("ask for the markers of %s/%s: %w", object, mapSlug, err)268 }269 var envelope struct {270 Data struct {271 Markers []json.RawMessage `json:"markersMultiple"`272 } `json:"data"`273 Errors []struct {274 Message string `json:"message"`275 } `json:"errors"`276 }277 if err := json.Unmarshal(answer, &envelope); err != nil {278 return ignCapture{}, fmt.Errorf("decode the markers of %s/%s: %w", object, mapSlug, err)279 }280 if len(envelope.Errors) > 0 && envelope.Errors[0].Message != "" {281 return ignCapture{}, fmt.Errorf("the marker endpoint refused: %s", envelope.Errors[0].Message)282 }283 if len(envelope.Data.Markers) == 0 {284 return ignCapture{}, fmt.Errorf("%s/%s answered with no marker at all", object, mapSlug)285 }286287 capture := ignCapture{288 Source: "ign-wikimaps",289 ObjectSlug: object,290 MapSlug: mapSlug,291 GameTitle: page.ObjectName,292 MapTitle: page.MapName,293 Map: ignCaptureMap{294 Width: page.Width,295 Height: page.Height,296 MinZoom: page.MinZoom,297 MaxZoom: page.MaxZoom,298 InitialLat: page.InitialLat,299 InitialLng: page.InitialLng,300 BackgroundColor: page.Background,301 Tileset: page.Tilesets[0],302 },303 Markers: envelope.Data.Markers,304 }305 for _, t := range page.Types {306 capture.Types = append(capture.Types, ignType{307 TypeSlug: t.TypeSlug,308 TypeName: t.TypeName,309 ParentTypeSlug: t.ParentSlug,310 IconURL: t.MarkerIcon.URL,311 IconWidth: t.MarkerIcon.Width,312 IconHeight: t.MarkerIcon.Height,313 })314 }315 return capture, nil316}317318// fetchArtwork stores the sprite each marker type draws with. A sprite the319// origin never published is not a failure: the collection goes without and a320// reader falls back to its own glyph.321func (c ignCrawler) fetchArtwork(322 ctx context.Context, run Run, volumeDir string, types []ignType, log *slog.Logger,323) error {324 for _, t := range types {325 if t.IconURL == "" {326 continue327 }328 extension := ".png"329 if dot := strings.LastIndex(t.IconURL, "."); dot >= 0 && dot < len(t.IconURL)-1 {330 extension = "." + strings.ToLower(t.IconURL[dot+1:])331 }332 if run.DryRun {333 continue334 }335 data, _, err := run.Fetch.Get(ctx, t.IconURL, nil)336 if errors.Is(err, ErrAbsent) {337 log.Warn("artwork not published", "key", t.TypeSlug, logging.Path(t.IconURL))338 continue339 }340 if err != nil {341 return err342 }343 if err := WriteArtwork(volumeDir, t.TypeSlug, extension, data); err != nil {344 return err345 }346 }347 return nil348}349350// fetchTiles walks every level of the pyramid the page declares. A wikimap is a351// flat image and its levels are rectangles, so there is nothing to prune: what352// is asked for is exactly what the source's own frame declaration says a353// complete level holds, which is what lets the deriver judge completeness later.354func (c ignCrawler) fetchTiles(355 ctx context.Context, run Run, worldDir string, worldID int64,356 object, mapSlug string, page ignPageState, log *slog.Logger,357) error {358 if run.DryRun {359 return nil360 }361 index, err := run.Archive.OpenTileIndex(worldDir, worldID)362 if err != nil {363 return err364 }365 template := page.Tilesets[0]366 scope := ign.TileSetPath(object, mapSlug)367 format := ign.TileExtension(template)368 setID := index.SetID(scope)369370 deepest := page.MaxZoom371 if run.MaxZoom > 0 && run.MaxZoom < deepest {372 deepest = run.MaxZoom373 }374 concurrency := run.Concurrency375 if concurrency < 1 {376 concurrency = DefaultConcurrency377 }378379 for zoom := 0; zoom <= deepest; zoom++ {380 maxX, maxY := ign.LevelExtent(page.Width, page.Height, zoom)381 var (382 mu sync.Mutex383 wait sync.WaitGroup384 gate = make(chan struct{}, concurrency)385 failure error386 fetched, absent, skipped int387 )388 for y := 0; y <= maxY; y++ {389 for x := 0; x <= maxX; x++ {390 url := strings.NewReplacer(391 "{z}", strconv.Itoa(zoom), "{x}", strconv.Itoa(x), "{y}", strconv.Itoa(y),392 ).Replace(template)393 // Resume: a tile already cached, or already known not to exist,394 // is not asked for again.395 if held, known := index.Held(url); known {396 switch held.Status {397 case StatusAbsent:398 absent++399 continue400 case StatusCached:401 skipped++402 continue403 }404 }405 wait.Add(1)406 gate <- struct{}{}407 go func(zoom, x, y int, url string) {408 defer wait.Done()409 defer func() { <-gate }()410 data, contentType, err := run.Fetch.Get(ctx, url, nil)411 record := TileRecord{412 URL: url, Zoom: zoom, X: x, Y: y,413 TileSetID: setID,414 CoverageKey: index.CoverageKey(setID, zoom, x, y),415 ContentType: "application/octet-stream",416 }417 mu.Lock()418 defer mu.Unlock()419 switch {420 case errors.Is(err, ErrAbsent):421 record.Status = StatusAbsent422 absent++423 case err != nil:424 record.Status = StatusFailed425 record.Error = err.Error()426 if failure == nil {427 failure = err428 }429 default:430 record.Status = StatusCached431 record.ContentHash = Hash(data)432 record.ContentType = contentType433 record.ByteLength = len(data)434 if writeErr := WriteTile(435 TilePath(worldDir, setID, zoom, x, y, format), data); writeErr != nil {436 failure = writeErr437 return438 }439 fetched++440 }441 index.Put(record)442 }(zoom, x, y, url)443 }444 }445 wait.Wait()446 // The index is written after every level, so a run interrupted between447 // levels resumes rather than starting over.448 if err := index.Close(worldDir); err != nil {449 return err450 }451 log.Info("level captured", logging.World(mapSlug), "zoom", zoom,452 "fetched", fetched, "skipped", skipped, "absent", absent)453 if failure != nil {454 return failure455 }456 }457 return nil458}459460// ignArchiveID mints an archive identity for an IGN thing: FNV-1a over a stable461// name in the low bits, IGN's own bit above them.462func ignArchiveID(name string) int64 {463 return int64(fnv32a(name)&0x7fffffff) | ignIDBit464}