1package crawl23import (4 "bytes"5 "context"6 "encoding/json"7 "fmt"8 "image"9 "image/jpeg"10 "log/slog"11 "os"12 "path/filepath"13 "strconv"1415 "github.com/FelineStateMachine/atlas/internal/generate/doc"16 "github.com/FelineStateMachine/atlas/internal/generate/sources/nasabluemarble"17 "github.com/FelineStateMachine/atlas/internal/logging"18)1920// The Blue Marble crawler: the one networked step behind the included Earth21// volume.22//23// Every other crawler follows a publisher wherever its endpoints lead. This one24// is aimed at a single pinned publication -- NASA Earth Observatory's Blue25// Marble Next Generation base map, July 2004, one image at one URL with one26// digest -- and it refuses anything else. The digest is the whole contract:27// bytes that do not hash to the pin are not the pinned publication, whatever28// the URL served, and accepting them silently would put an unaccounted picture29// into every library that ships the included volume.30//31// What lands in the archive is the ordinary shape every capture takes: a32// capture body naming the product, and the reference-level tiles the deriver33// folds down -- the source image cut to the shared whole-sphere window through34// the deterministic resampler beside this file. The downloaded source itself is35// kept beside the capture, content-addressed, so a re-run derives without36// asking NASA again; it is a working file of the archive, never part of a37// bundle and never read at runtime.3839// blueMarblePin is everything fixed about the capture: where the publication40// is, what its bytes must hash to, when it was first captured for this41// artifact, and the derivation policy the tiles are cut under. It is a struct42// rather than constants so the write path can be exercised against a stated43// pin; the shipped pin is blueMarble below.44type blueMarblePin struct {45 // Page is the product's own page, and Asset the exact image fetched.46 Page string47 Asset string48 // SHA256 is the digest the fetched bytes must carry. A mismatch is a49 // refusal, never a warning.50 SHA256 string51 // CapturedAt is the moment the pinned publication was first captured for52 // this artifact. It is part of the pin so the archive -- and everything53 // stamped downstream of it -- reproduces on any machine.54 CapturedAt string55 // Width and Height are the source image's declared pixels.56 Width, Height int57 // Quality is the JPEG setting the reference tiles are encoded at. It is58 // spelled here, once, because two runs of one pipeline may not disagree.59 Quality int60 // FeaturesEdition names the vector publication the capture marries to the61 // raster, and Borders and Places pin its two files: country polygons and62 // populated places, each held to its own digest exactly as the image is.63 FeaturesEdition string64 Borders, Places blueMarbleAsset65}6667// blueMarbleAsset is one pinned file: where it is, and what its bytes must68// hash to.69type blueMarbleAsset struct {70 Asset string71 SHA256 string72}7374// blueMarble is the shipped pin: Blue Marble Next Generation with topography75// and bathymetry, July 2004, as published by NASA Earth Observatory -- the76// variant whose shaded relief and ocean floor give the planet its texture,77// which is also the rendering NASA's own maps lead with. included/README.md is78// the recipe that carries these same facts for a person.79var blueMarble = blueMarblePin{80 Page: "https://science.nasa.gov/earth/earth-observatory/blue-marble-next-generation/base-topography-bathymetry/",81 Asset: "https://assets.science.nasa.gov/content/dam/science/esd/eo/images/bmng/bmng-topography-bathymetry/july/world.topo.bathy.200407.3x21600x10800.jpg",82 SHA256: "d225f1f35a6448a4d1d8f6de6e48f3433e470085b70a35800e64f384f269a7b0",83 CapturedAt: "2026-08-03T16:21:07Z",84 Width: 21600,85 Height: 10800,86 Quality: 85,87 // The feature data is Natural Earth, public domain, at a tagged release:88 // country polygons at 1:110m -- the light cut, made for whole-planet maps89 // -- and the populated places its capitals are read from. Boundaries and90 // capital designations are carried as Natural Earth draws them, de facto.91 FeaturesEdition: "Natural Earth 1:110m cultural vectors v5.1.2",92 Borders: blueMarbleAsset{93 Asset: "https://raw.githubusercontent.com/nvkelso/natural-earth-vector/v5.1.2/geojson/ne_110m_admin_0_countries.geojson",94 SHA256: "6866c877d39cba9c357620878839b336d569f8c662d3cfab4cb1dbe2d39c977f",95 },96 Places: blueMarbleAsset{97 Asset: "https://raw.githubusercontent.com/nvkelso/natural-earth-vector/v5.1.2/geojson/ne_110m_populated_places_simple.geojson",98 SHA256: "0dbd25c9ad8bd797ddf164b067f563be5c16be2c002254eb594862377963f9dc",99 },100}101102// blueMarbleResampler names the cut for the capture body: the resampler beside103// this file, at its fixed-point precision. A change to how the reference level104// is cut renames this, and the renamed policy reads as a new capture.105const blueMarbleResampler = "catmull-rom-fixed15"106107// blueMarbleZoom is the level the source is cut to: one deeper than the108// reference level whose pixels are the world square's own, so the pyramid109// holds a 16384×8192 picture the 21600-wide source still fills with real110// detail. The deriver folds the reference level and everything shallower down111// from this cut.112const blueMarbleZoom = doc.SyntheticZoom + 1113114// blueMarbleIDBit marks Blue Marble identities in the archive register, beside115// the bits the other sources' identities carry.116const blueMarbleIDBit = int64(1) << 36117118type blueMarbleCrawler struct{}119120func (blueMarbleCrawler) Name() string { return nasabluemarble.SourceName }121122func (blueMarbleCrawler) Usage() string {123 return "the pinned Blue Marble base map; the target is \"earth\""124}125126func (c blueMarbleCrawler) Crawl(ctx context.Context, run Run) error {127 if run.Target != nasabluemarble.Body {128 return fmt.Errorf("%s captures the pinned Earth base map; the target is %q, not %q",129 c.Name(), nasabluemarble.Body, run.Target)130 }131 log := run.Logger().With(logging.Source(c.Name()), logging.Op("crawl"))132133 volume := Volume{134 ID: blueMarbleArchiveID("bluemarble:" + nasabluemarble.Body),135 Title: doc.Title(nasabluemarble.Body),136 Locator: blueMarble.Page,137 Source: c.Name(),138 // The same-slug policy: any source describing the Earth registers the139 // plain title while naming its directory after itself.140 DirectoryTitle: "NASA Blue Marble Earth",141 }142 world := World{143 ID: blueMarbleArchiveID("bluemarble:" + nasabluemarble.Body + "/" + nasabluemarble.Body),144 Slug: nasabluemarble.Body,145 Title: doc.Title(nasabluemarble.Body),146 }147 if run.DryRun {148 log.Info("crawl would archive", logging.Volume(volume.Title), logging.World(world.Slug),149 "asset", blueMarble.Asset, "sha256", blueMarble.SHA256)150 return nil151 }152153 volumeDir, err := run.Archive.RegisterVolume(volume)154 if err != nil {155 return err156 }157 worldDir, err := run.Archive.RegisterWorld(volume, volumeDir, world)158 if err != nil {159 return err160 }161 source, err := c.asset(ctx, run, worldDir, blueMarble.Asset, blueMarble.SHA256, ".jpg", log)162 if err != nil {163 return err164 }165 borders, err := c.asset(ctx, run, worldDir, blueMarble.Borders.Asset, blueMarble.Borders.SHA256, ".geojson", log)166 if err != nil {167 return err168 }169 places, err := c.asset(ctx, run, worldDir, blueMarble.Places.Asset, blueMarble.Places.SHA256, ".geojson", log)170 if err != nil {171 return err172 }173 features, err := distillBlueMarbleFeatures(blueMarble, borders, places)174 if err != nil {175 return err176 }177 if err := writeBlueMarble(run.Archive, worldDir, world.ID, blueMarble, source, features, log); err != nil {178 return err179 }180 return nil181}182183// asset answers one pinned file's bytes: the copy already kept beside the184// capture when there is one, the origin exactly once when there is not. Either185// way the bytes are held to the pinned digest before anything is derived from186// them.187func (c blueMarbleCrawler) asset(188 ctx context.Context, run Run, worldDir, url, sha256, extension string, log *slog.Logger,189) ([]byte, error) {190 cached := filepath.Join(worldDir, "source", sha256+extension)191 if data, err := os.ReadFile(cached); err == nil {192 if err := verifyBlueMarble(data, sha256); err != nil {193 return nil, fmt.Errorf("cached source %s: %w", cached, err)194 }195 log.Info("source already captured", logging.Path(cached), "bytes", len(data))196 return data, nil197 }198 data, _, err := run.Fetch.Get(ctx, url, nil)199 if err != nil {200 return nil, fmt.Errorf("fetch %s: %w", url, err)201 }202 if err := verifyBlueMarble(data, sha256); err != nil {203 return nil, err204 }205 if err := writeFile(cached, data); err != nil {206 return nil, err207 }208 log.Info("source captured", logging.Path(cached), "bytes", len(data))209 return data, nil210}211212// verifyBlueMarble holds bytes to a pinned digest. It runs before any decode:213// bytes that are not the pinned publication are not decoded, not derived from,214// and not archived.215func verifyBlueMarble(data []byte, sha256 string) error {216 if sum := Hash(data); sum != sha256 {217 return fmt.Errorf(218 "source digest mismatch: fetched bytes hash to %s where the pin requires %s; "+219 "refusing changed upstream bytes", sum, sha256)220 }221 return nil222}223224// distillBlueMarbleFeatures reads the two verified Natural Earth files down to225// what the capture keeps: each country's name, code, continent, label point226// and rings, and each primary capital's name, country and place. Coordinates227// travel verbatim, as published; putting them on the picture is the reader's228// work. Order is the publication's own.229func distillBlueMarbleFeatures(pin blueMarblePin, borders, places []byte) (blueMarbleFeatures, error) {230 out := blueMarbleFeatures{231 Edition: pin.FeaturesEdition,232 BordersSHA256: pin.Borders.SHA256,233 PlacesSHA256: pin.Places.SHA256,234 }235236 var countries struct {237 Features []struct {238 Properties struct {239 Name string `json:"NAME"`240 A3 string `json:"ADM0_A3"`241 Continent string `json:"CONTINENT"`242 LabelLon float64 `json:"LABEL_X"`243 LabelLat float64 `json:"LABEL_Y"`244 } `json:"properties"`245 Geometry struct {246 Type string `json:"type"`247 Coordinates json.RawMessage `json:"coordinates"`248 } `json:"geometry"`249 } `json:"features"`250 }251 if err := json.Unmarshal(borders, &countries); err != nil {252 return out, fmt.Errorf("decode country borders: %w", err)253 }254 for _, feature := range countries.Features {255 country := blueMarbleCountry{256 Name: feature.Properties.Name,257 A3: feature.Properties.A3,258 Continent: feature.Properties.Continent,259 LabelLon: feature.Properties.LabelLon,260 LabelLat: feature.Properties.LabelLat,261 }262 switch feature.Geometry.Type {263 case "Polygon":264 var rings [][][2]float64265 if err := json.Unmarshal(feature.Geometry.Coordinates, &rings); err != nil {266 return out, fmt.Errorf("country %q: %w", country.Name, err)267 }268 country.Polygons = [][][][2]float64{rings}269 case "MultiPolygon":270 if err := json.Unmarshal(feature.Geometry.Coordinates, &country.Polygons); err != nil {271 return out, fmt.Errorf("country %q: %w", country.Name, err)272 }273 default:274 return out, fmt.Errorf("country %q draws a %s; borders are ground",275 country.Name, feature.Geometry.Type)276 }277 out.Countries = append(out.Countries, country)278 }279280 var placed struct {281 Features []struct {282 Properties struct {283 Class string `json:"featurecla"`284 Name string `json:"name"`285 Country string `json:"adm0name"`286 A3 string `json:"adm0_a3"`287 Lat float64 `json:"latitude"`288 Lon float64 `json:"longitude"`289 } `json:"properties"`290 } `json:"features"`291 }292 if err := json.Unmarshal(places, &placed); err != nil {293 return out, fmt.Errorf("decode populated places: %w", err)294 }295 for _, feature := range placed.Features {296 // Only the primary designations: the alternates are second seats and297 // former capitals, which is more editorial weight than a demo carries.298 if feature.Properties.Class != "Admin-0 capital" {299 continue300 }301 out.Capitals = append(out.Capitals, blueMarbleCapital{302 Name: feature.Properties.Name,303 Country: feature.Properties.Country,304 A3: feature.Properties.A3,305 Lat: feature.Properties.Lat,306 Lon: feature.Properties.Lon,307 })308 }309 return out, nil310}311312// writeBlueMarble derives the capture from verified source bytes: the image313// decoded, held to its declared size, cut to the capture level through the314// deterministic resampler, tiled, and recorded -- and the capture body written315// last, carrying the product's identity and the policy the cut was made under.316func writeBlueMarble(317 store *Archive, worldDir string, worldID int64,318 pin blueMarblePin, source []byte, features blueMarbleFeatures, log *slog.Logger,319) error {320 decoded, kind, err := image.Decode(bytes.NewReader(source))321 if err != nil {322 return fmt.Errorf("decode source: %w", err)323 }324 bounds := decoded.Bounds()325 if kind != "jpeg" || bounds.Dx() != pin.Width || bounds.Dy() != pin.Height {326 return fmt.Errorf("source is a %d×%d %s where the pin requires a %d×%d jpeg",327 bounds.Dx(), bounds.Dy(), kind, pin.Width, pin.Height)328 }329330 maxX, maxY := doc.EquirectLevelExtent(blueMarbleZoom)331 ground := resample(toNRGBA(decoded), (maxX+1)*doc.SyntheticTileSize, (maxY+1)*doc.SyntheticTileSize)332333 index, err := store.OpenTileIndex(worldDir, worldID)334 if err != nil {335 return err336 }337 setID := index.SetID(nasabluemarble.TileSet)338 for y := 0; y <= maxY; y++ {339 for x := 0; x <= maxX; x++ {340 tile := ground.SubImage(image.Rect(341 x*doc.SyntheticTileSize, y*doc.SyntheticTileSize,342 (x+1)*doc.SyntheticTileSize, (y+1)*doc.SyntheticTileSize))343 var encoded bytes.Buffer344 if err := jpeg.Encode(&encoded, tile, &jpeg.Options{Quality: pin.Quality}); err != nil {345 return fmt.Errorf("encode tile %d/%d: %w", x, y, err)346 }347 data := encoded.Bytes()348 if err := WriteTile(TilePath(worldDir, setID, blueMarbleZoom, x, y, "jpg"), data); err != nil {349 return err350 }351 index.Put(TileRecord{352 ByteLength: len(data),353 ContentHash: Hash(data),354 ContentType: "image/jpeg",355 CoverageKey: index.CoverageKey(setID, blueMarbleZoom, x, y),356 Status: StatusCached,357 TileSetID: setID,358 URL: blueMarbleTileURL(blueMarbleZoom, x, y),359 X: x,360 Y: y,361 Zoom: blueMarbleZoom,362 })363 }364 }365 if err := index.Close(worldDir); err != nil {366 return err367 }368369 body, err := json.MarshalIndent(blueMarbleCapture{370 Source: nasabluemarble.SourceName,371 Body: nasabluemarble.Body,372 Product: "Blue Marble: Next Generation w/ Topography and Bathymetry, July 2004",373 Credit: "NASA Earth Observatory",374 AssetSHA256: pin.SHA256,375 Width: pin.Width,376 Height: pin.Height,377 Map: blueMarbleMosaic{378 MaxZoom: blueMarbleZoom,379 Extension: "jpg",380 LayerTitle: "Blue Marble",381 },382 Derive: blueMarbleDerivation{383 Resampler: blueMarbleResampler,384 JPEGQuality: pin.Quality,385 },386 Features: features,387 }, "", " ")388 if err != nil {389 return err390 }391 hash, fresh, err := store.WriteCapture(worldDir, Capture{392 Kind: nasabluemarble.CaptureKind,393 SourceID: worldID,394 SourceURL: pin.Asset,395 CapturedAt: pin.CapturedAt,396 Body: body,397 })398 if err != nil {399 return err400 }401 log.Info("capture archived", logging.World(nasabluemarble.Body),402 logging.Stamp(hash[:12]), "fresh", fresh, "tiles", (maxX+1)*(maxY+1))403 return nil404}405406// blueMarbleTileURL is the address a reference tile is recorded under. These407// tiles were derived here and never served anywhere, so the address is honest408// about that -- an .invalid host, like the rendered basemaps' -- while carrying409// the marker and pyramid path the archive's reader recovers the tile set from.410func blueMarbleTileURL(zoom, x, y int) string {411 return "https://blue-marble.invalid/tiles/" + nasabluemarble.TileSet +412 "/" + strconv.Itoa(zoom) + "/" + strconv.Itoa(x) + "/" + strconv.Itoa(y) + ".jpg"413}414415// blueMarbleCapture is the capture body as this crawler writes it, mirrored by416// the reader's own declaration in sources/nasabluemarble.417type blueMarbleCapture struct {418 Source string `json:"source"`419 Body string `json:"body"`420 Product string `json:"product"`421 Credit string `json:"credit"`422 AssetSHA256 string `json:"assetSha256"`423 Width int `json:"width"`424 Height int `json:"height"`425 Map blueMarbleMosaic `json:"map"`426 Derive blueMarbleDerivation `json:"derive"`427 Features blueMarbleFeatures `json:"features"`428}429430type blueMarbleMosaic struct {431 MaxZoom int `json:"maxZoom"`432 Extension string `json:"extension"`433 LayerTitle string `json:"layerTitle"`434}435436type blueMarbleDerivation struct {437 Resampler string `json:"resampler"`438 JPEGQuality int `json:"jpegQuality"`439}440441// blueMarbleFeatures is the vector half of the capture: the publication it was442// distilled from, the digests its files carried, and the distillation itself.443type blueMarbleFeatures struct {444 Edition string `json:"edition"`445 BordersSHA256 string `json:"bordersSha256"`446 PlacesSHA256 string `json:"placesSha256"`447 Countries []blueMarbleCountry `json:"countries"`448 Capitals []blueMarbleCapital `json:"capitals"`449}450451// blueMarbleCountry is one country as the capture keeps it: identity, the452// continent the publication files it under, its label point, and its rings --453// polygons, then rings, then positions, longitude first, all verbatim.454type blueMarbleCountry struct {455 Name string `json:"name"`456 A3 string `json:"a3"`457 Continent string `json:"continent"`458 LabelLon float64 `json:"labelLon"`459 LabelLat float64 `json:"labelLat"`460 Polygons [][][][2]float64 `json:"polygons"`461}462463// blueMarbleCapital is one primary capital as the capture keeps it.464type blueMarbleCapital struct {465 Name string `json:"name"`466 Country string `json:"country"`467 A3 string `json:"a3"`468 Lat float64 `json:"lat"`469 Lon float64 `json:"lon"`470}471472func blueMarbleArchiveID(name string) int64 {473 return int64(fnv32a(name)&0x7fffffff) | blueMarbleIDBit474}