1// The Trek mode fills the archive from two public planetary publications at2// once: NASA Trek's global tile mosaics for the raster, and the IAU Gazetteer3// of Planetary Nomenclature's downloadable feature list for the pins. Both4// are plainly published for taking -- the mosaic behind an open WMTS5// endpoint, the features as a shapefile whose attribute table carries name,6// place, size, and the origin story of every name -- and nothing else is7// touched: no page prose, nothing the data files do not carry.8//9// This is the curated-bodies importer, and says so. A Trek mosaic's tiling is10// asserted from its own WMTS capabilities before anything is fetched, but11// which mosaic represents a body is an editorial choice; each body in the12// table names one verified global equirectangular layer, and an unknown body13// is refused at the door rather than captured through an unverified one.14package main1516import (17 "archive/zip"18 "bytes"19 "context"20 "encoding/binary"21 "encoding/json"22 "encoding/xml"23 "errors"24 "fmt"25 "hash/fnv"26 "io"27 "path/filepath"28 "strconv"29 "strings"3031 "github.com/FelineStateMachine/atlas/internal/mgdoc"32 "github.com/FelineStateMachine/atlas/internal/trekmap"33)3435const trekTiles = "https://trek.nasa.gov/tiles"3637// trekDefaultZoom is how deep a crawl goes unasked, in the pipeline's square38// world where level z holds Trek's z-1. Level 6 is Trek's 5: 2048 tiles for39// the deepest level and a third of that above it, a courteous take of a40// pyramid whose deepest published levels run to hundreds of thousands.41const trekDefaultZoom = 64243// trekBody is one body the importer knows: how Trek's paths spell it, which44// global mosaic pictures it, the sibling mosaics captured beside that one,45// and where the Gazetteer's feature list for it lives.46type trekBody struct {47 pathName string48 layer string49 layerTitle string50 variants []trekLayer51 portal string52 gazetteer string53}5455// trekLayer names one verified sibling mosaic and how a person knows it.56type trekLayer struct {57 layer string58 title string59}6061var trekBodies = map[string]trekBody{62 "mars": {63 pathName: "Mars",64 layer: "Mars_Viking_MDIM21_ClrMosaic_global_232m",65 layerTitle: "Viking MDIM 2.1",66 variants: []trekLayer{67 // The laser altimeter's colored shaded relief: the same ground as68 // height instead of photograph, the other classic way Mars is seen.69 {layer: "Mars_MGS_MOLA_ClrShade_merge_global_463m", title: "MOLA Elevation"},70 },71 portal: "https://trek.nasa.gov/mars",72 gazetteer: "https://asc-planetarynames-data.s3.us-west-2.amazonaws.com/MARS_nomenclature_center_pts.zip",73 },74}7576func runTrek(ctx context.Context, fetcher *fetcher, o options) error {77 slug := o.trek78 body, known := trekBodies[slug]79 if !known {80 bodies := make([]string, 0, len(trekBodies))81 for name := range trekBodies {82 bodies = append(bodies, name)83 }84 return fmt.Errorf("the Trek importer knows only %s: each body names one verified "+85 "global mosaic, and an unverified one could hang every pin on the wrong pixel",86 strings.Join(bodies, ", "))87 }8889 // Every mosaic of the body answers for its own tiling before anything is90 // fetched, and takes its own depth: the default courtesy ceiling, under91 // whatever each one actually publishes.92 layers := append([]trekLayer{{layer: body.layer, title: body.layerTitle}}, body.variants...)93 depths := make([]int, len(layers))94 for at, entry := range layers {95 published, err := fetchTrekPyramid(ctx, fetcher, trekLayerBase(body, entry.layer))96 if err != nil {97 return fmt.Errorf("%s: %w", entry.layer, err)98 }99 depths[at] = published100 if o.maxZoom > 0 {101 depths[at] = min(depths[at], o.maxZoom)102 } else {103 depths[at] = min(depths[at], trekDefaultZoom)104 }105 }106107 features, err := fetchGazetteer(ctx, fetcher, body.gazetteer)108 if err != nil {109 return err110 }111112 capture := &trekmap.Capture{113 Source: trekmap.Source,114 Body: slug,115 Layer: body.layer,116 MapSlug: "global",117 MapTitle: "Global",118 Map: trekmap.MapConfig{119 MaxZoom: depths[0],120 Extension: "jpg",121 LayerTitle: body.layerTitle,122 },123 Features: features,124 }125 for at, entry := range layers[1:] {126 capture.Variants = append(capture.Variants, trekmap.Variant{127 Layer: entry.layer,128 Title: entry.title,129 MaxZoom: depths[at+1],130 Extension: "jpg",131 })132 }133 capture.Normalize()134 raw, err := json.MarshalIndent(capture, "", " ")135 if err != nil {136 return err137 }138 // A capture that does not translate would poison the archive for both139 // tools that read it, so it is refused here, while the problem is one140 // crawl old rather than buried in history.141 if _, err := trekmap.Translate(raw); err != nil {142 return fmt.Errorf("capture does not translate: %w", err)143 }144145 gameID := trekArchiveID("trek:" + slug)146 mapID := trekArchiveID("trek:" + slug + "/" + capture.MapSlug)147 game := &apiGameFull{ID: gameID, Title: mgdoc.SpellOut(slug),148 Config: apiGameConfig{URL: body.portal}}149 full := &apiMapFull{ID: mapID, Title: capture.MapTitle, Slug: capture.MapSlug}150 // The directory says where the capture came from even though the game151 // keeps its plain title: another source picturing the same body would152 // live in its own directory, and the two must never resolve to one.153 gameDirectory := resolveGameDirectory(o.archive,154 &apiGameFull{ID: gameID, Title: "NASA Trek " + game.Title})155 mapDirectory := resolveMapDirectory(o.archive, gameDirectory, full)156 mapDir := filepath.Join(o.archive, mapDirectory)157158 fmt.Printf("\n== NASA Trek %s / %s (%d features, %d layers)\n",159 game.Title, capture.MapTitle, len(capture.Features), len(layers))160161 if o.dryRun {162 fmt.Printf(" would write capture to %s\n", mapDir)163 } else if err := writeSnapshotRecord(mapDir, raw, trekmap.Kind, mapID, body.gazetteer); err != nil {164 return err165 }166167 index, err := readTileIndex(mapDir)168 if err != nil {169 return err170 }171 index.mapID = mapID172 var stats captureStats173 for at, entry := range layers {174 taken, err := captureTrekTiles(ctx, fetcher, o, slug, body, entry.layer, depths[at], mapDir, index)175 if err != nil {176 return err177 }178 stats.fetched += taken.fetched179 stats.skipped += taken.skipped180 stats.absent += taken.absent181 stats.failed += taken.failed182 stats.bytes += taken.bytes183 }184 if o.dryRun {185 return nil186 }187 if err := writeTileIndex(mapDir, index); err != nil {188 return err189 }190 if err := writeMapMetadata(mapDir, game, full, index, len(capture.Features)); err != nil {191 return err192 }193 if err := registerMap(o.archive, game, full, gameDirectory, mapDirectory, "nasa-trek"); err != nil {194 return err195 }196 fmt.Printf(" %d fetched · %d held · %d not published · %d failed · %.1f MB new\n",197 stats.fetched, stats.skipped, stats.absent, stats.failed, float64(stats.bytes)/1e6)198 return nil199}200201// wmtsCapabilities is the slice of a layer's WMTS document the crawl stands202// on: the tile matrices, whose sizes and origin say how the mosaic is tiled.203type wmtsCapabilities struct {204 Contents struct {205 TileMatrixSets []struct {206 Matrices []struct {207 TopLeftCorner string `xml:"TopLeftCorner"`208 TileWidth int `xml:"TileWidth"`209 TileHeight int `xml:"TileHeight"`210 MatrixWidth float64 `xml:"MatrixWidth"`211 MatrixHeight float64 `xml:"MatrixHeight"`212 } `xml:"TileMatrix"`213 } `xml:"TileMatrixSet"`214 } `xml:"Contents"`215}216217// fetchTrekPyramid reads the layer's own account of its tiling and holds it218// to the shape the translator promises: an equirectangular pyramid anchored219// at longitude -180, latitude 90, two tiles wide for every one tall, in220// 256-pixel tiles. It returns the deepest pipeline zoom published -- one221// above the deepest Trek matrix, since the pipeline's square world sits one222// level over the 2:1 mosaic.223func fetchTrekPyramid(ctx context.Context, f *fetcher, layerBase string) (int, error) {224 capsURL := layerBase + "/1.0.0/WMTSCapabilities.xml"225 body, _, err := f.get(ctx, capsURL)226 if err != nil {227 return 0, fmt.Errorf("capabilities: %w", err)228 }229 var caps wmtsCapabilities230 if err := xml.Unmarshal(body, &caps); err != nil {231 return 0, fmt.Errorf("decode %s: %w", capsURL, err)232 }233 if len(caps.Contents.TileMatrixSets) != 1 {234 return 0, fmt.Errorf("%s declares %d tile matrix sets, not the one global scheme",235 capsURL, len(caps.Contents.TileMatrixSets))236 }237 matrices := caps.Contents.TileMatrixSets[0].Matrices238 if len(matrices) == 0 {239 return 0, fmt.Errorf("%s declares no tile matrices", capsURL)240 }241 for level, matrix := range matrices {242 wide, tall := float64(int(2)<<level), float64(int(1)<<level)243 corner := strings.Fields(matrix.TopLeftCorner)244 if len(corner) != 2 || corner[0] != "-180.0" || corner[1] != "90.0" {245 return 0, fmt.Errorf("level %d anchors at %q, not the -180/90 corner the "+246 "projection assumes", level, matrix.TopLeftCorner)247 }248 if matrix.TileWidth != 256 || matrix.TileHeight != 256 ||249 matrix.MatrixWidth != wide || matrix.MatrixHeight != tall {250 return 0, fmt.Errorf("level %d is %vx%v tiles of %dx%d px; the projection "+251 "assumes the 2:1 equirectangular scheme", level,252 matrix.MatrixWidth, matrix.MatrixHeight, matrix.TileWidth, matrix.TileHeight)253 }254 }255 return len(matrices), nil256}257258// trekLayerBase is where one of the body's mosaics actually lives, in Trek's259// own spelling of the body's name.260func trekLayerBase(body trekBody, layer string) string {261 return trekTiles + "/" + body.pathName + "/EQ/" + layer262}263264// captureTrekTiles takes one layer's pyramid level by level, whole: a global265// mosaic has no empty space to prune around. Trek spells a tile as matrix/266// row/column, one zoom below and with its axes named the other way round267// from the pipeline's zoom/x/y -- so each level fetches under Trek's268// spelling and records under the canonical one, which is the address every269// reader of the archive derives the layer and its window from.270func captureTrekTiles(271 ctx context.Context,272 fetcher *fetcher,273 o options,274 slug string,275 body trekBody,276 layer string,277 deepest int,278 mapDir string,279 index *tileIndex,280) (captureStats, error) {281 var stats captureStats282 scope := trekmap.TileSetPath(slug, layer)283 setID := index.tileSetID(scope)284 fmt.Printf(" layer %q z1-%d\n", scope, deepest)285286 for zoom := 1; zoom <= deepest; zoom++ {287 maxX, maxY := trekmap.LevelExtent(zoom)288 window := tileWindow{minX: 0, minY: 0, maxX: maxX, maxY: maxY}289 record := trekTiles + "/" + scope + "/{z}/{x}/{y}.jpg"290 fetch := trekLayerBase(body, layer) + "/1.0.0/default/default028mm/" +291 strconv.Itoa(zoom-1) + "/{y}/{x}.jpg"292 results, err := fetchTemplateLevel(ctx, fetcher, o, record, fetch, "", setID, "jpg",293 mapDir, zoom, window.tiles(), index)294 if err != nil {295 return stats, err296 }297 stats.merge(results)298 fmt.Printf(" z%-2d %4d fetched · %4d held · %4d not published\n",299 zoom, results.fetched, results.skipped, results.absent)300 // The bundle is built from the deepest complete level. Holes there mean301 // the mosaic is thinner than its capabilities claim, which deserves302 // more than a counter in the summary line.303 if zoom == deepest && results.absent > 0 {304 fmt.Printf(" WARNING: %d tiles of the deepest level are not published; "+305 "the bundle will be built from a shallower level\n", results.absent)306 }307 }308 return stats, nil309}310311// fetchGazetteer takes the body's feature list as the Gazetteer publishes it312// -- a zipped shapefile -- and reads the attribute table inside, which313// carries everything the capture keeps: names, places, sizes, feature types,314// and the origin text explaining each name.315func fetchGazetteer(ctx context.Context, f *fetcher, url string) ([]trekmap.Feature, error) {316 body, _, err := f.get(ctx, url)317 if err != nil {318 return nil, fmt.Errorf("gazetteer: %w", err)319 }320 archive, err := zip.NewReader(bytes.NewReader(body), int64(len(body)))321 if err != nil {322 return nil, fmt.Errorf("read gazetteer archive: %w", err)323 }324 for _, file := range archive.File {325 if !strings.HasSuffix(strings.ToLower(file.Name), ".dbf") {326 continue327 }328 reader, err := file.Open()329 if err != nil {330 return nil, err331 }332 table, err := io.ReadAll(reader)333 reader.Close()334 if err != nil {335 return nil, err336 }337 return parseNomenclature(table)338 }339 return nil, fmt.Errorf("%s holds no attribute table", url)340}341342// parseNomenclature reads the shapefile's dBASE attribute table. The format343// is fixed-width rows behind a field directory; the Gazetteer writes it in344// UTF-8, per the encoding sidecar in the same archive.345func parseNomenclature(table []byte) ([]trekmap.Feature, error) {346 if len(table) < 32 {347 return nil, errors.New("attribute table is truncated")348 }349 records := int(binary.LittleEndian.Uint32(table[4:8]))350 headerSize := int(binary.LittleEndian.Uint16(table[8:10]))351 recordSize := int(binary.LittleEndian.Uint16(table[10:12]))352353 type column struct {354 name string355 start int356 length int357 }358 var columns []column359 // Each row leads with a deletion flag byte, so fields start at one.360 at := 1361 for pos := 32; ; pos += 32 {362 if pos >= len(table) || pos >= headerSize {363 return nil, errors.New("attribute table's field directory never ends")364 }365 if table[pos] == 0x0d {366 break367 }368 if pos+32 > len(table) {369 return nil, errors.New("attribute table is truncated inside a field")370 }371 name := string(bytes.TrimRight(table[pos:pos+11], "\x00"))372 length := int(table[pos+16])373 columns = append(columns, column{name: name, start: at, length: length})374 at += length375 }376 if at != recordSize {377 return nil, fmt.Errorf("fields span %d bytes of a %d-byte record", at, recordSize)378 }379 if headerSize+records*recordSize > len(table) {380 return nil, fmt.Errorf("attribute table holds fewer than its %d records", records)381 }382383 wanted := map[string]bool{384 "name": true, "type": true, "code": true, "origin": true,385 "diameter": true, "center_lat": true, "center_lon": true, "link": true,386 }387 features := make([]trekmap.Feature, 0, records)388 for number := range records {389 row := table[headerSize+number*recordSize:][:recordSize]390 if row[0] == '*' {391 continue392 }393 cells := make(map[string]string, len(wanted))394 for _, col := range columns {395 if !wanted[col.name] {396 continue397 }398 cells[col.name] = strings.TrimSpace(string(row[col.start : col.start+col.length]))399 }400 feature, err := composeFeature(cells)401 if err != nil {402 return nil, fmt.Errorf("record %d: %w", number, err)403 }404 features = append(features, feature)405 }406 return dedupeFeatures(features)407}408409// dedupeFeatures folds the table's own repeats. The Mars list has been seen410// carrying one crater twice, byte for byte; a doubled row saying the same411// thing is the source's hygiene, not a different feature, but two rows412// claiming one identifier for different facts would be, and are refused.413func dedupeFeatures(features []trekmap.Feature) ([]trekmap.Feature, error) {414 seen := make(map[int64]trekmap.Feature, len(features))415 kept := features[:0]416 for _, feature := range features {417 previous, held := seen[feature.ID]418 if !held {419 seen[feature.ID] = feature420 kept = append(kept, feature)421 continue422 }423 if previous != feature {424 return nil, fmt.Errorf("feature %d is both %q and %q", feature.ID,425 previous.Name, feature.Name)426 }427 }428 return kept, nil429}430431// composeFeature turns one attribute row into the capture's terms. The432// Gazetteer's own feature identifier comes off the record's link, which is433// the only place the table spells it.434func composeFeature(cells map[string]string) (trekmap.Feature, error) {435 _, tail, found := strings.Cut(cells["link"], "/Feature/")436 if !found {437 return trekmap.Feature{}, fmt.Errorf("no feature identifier in link %q", cells["link"])438 }439 id, err := strconv.ParseInt(tail, 10, 64)440 if err != nil {441 return trekmap.Feature{}, fmt.Errorf("feature identifier %q: %w", tail, err)442 }443 latitude, err := strconv.ParseFloat(cells["center_lat"], 64)444 if err != nil {445 return trekmap.Feature{}, fmt.Errorf("latitude %q: %w", cells["center_lat"], err)446 }447 longitude, err := strconv.ParseFloat(cells["center_lon"], 64)448 if err != nil {449 return trekmap.Feature{}, fmt.Errorf("longitude %q: %w", cells["center_lon"], err)450 }451 diameter := 0.0452 if cells["diameter"] != "" {453 if diameter, err = strconv.ParseFloat(cells["diameter"], 64); err != nil {454 return trekmap.Feature{}, fmt.Errorf("diameter %q: %w", cells["diameter"], err)455 }456 }457 return trekmap.Feature{458 ID: id,459 Name: cells["name"],460 Type: cells["type"],461 Code: cells["code"],462 Latitude: latitude,463 Longitude: longitude,464 DiameterKM: diameter,465 Origin: cells["origin"],466 }, nil467}468469// trekArchiveID numbers Trek bodies and maps in the archive's registers, one470// bit above Piggyback's so every source's ids stay apart; see ignArchiveID471// for why the range stays far below a float64's integers.472func trekArchiveID(seed string) int64 {473 hash := fnv.New32a()474 _, _ = hash.Write([]byte(seed))475 return int64(hash.Sum32()&0x7fffffff) | 1<<34476}