Mars gains its second raster: the MOLA laser altimeter's colored shaded relief, riding the same map as a variant beside the Viking photograph, the way a game offers satellite beside atlas. A trek capture now carries sibling mosaics -- each verified against its own WMTS capabilities, each taking its own depth, all sharing the one window the synthetic projection already lands pins in -- and a capture without any is simply a body pictured a single way, so the first captures translate as they always did. The exercise this source exists for continues: the bundle format answers to no one standard -- not MapGenie's grid, not the games' habits, not the sciences' own tiling -- and a planet's elevation joining its photograph through the same door a wiki's markers came through is the proof restated.
- commit
- c19cfc51cf071b595ce1031cea7d55bc62ac946b
- parent
- df7148951d53dca8012e905ff9668fdf3eaca3be
- author
- Dami <dami.etoile@gmail.com>
| M | internal/ | 65 | +++++++++++++++----- |
| M | internal/ | 41 | +++++++++---- |
| M | tools/ | 98 | +++++++++++++++++++++--------- |
@@ -39,9 +39,11 @@ const Kind = "trek-map"
3939 const Source = "nasa-trek"4040 4141 // Capture is the canonical document the crawler stores: which body and mosaic42-// the raster is, and every named feature the Gazetteer publishes for the body.43-// Fields marshal in declaration order and Normalize sorts the features, so44-// unchanged data always hashes to the capture already archived.42+// the raster is, every sibling mosaic captured beside it, and every named43+// feature the Gazetteer publishes for the body. Fields marshal in declaration44+// order and Normalize sorts the lists, so unchanged data always hashes to the45+// capture already archived. Variants is newer than the field's first captures;46+// a capture without one is simply a body pictured a single way.4547 type Capture struct {4648 Source string `json:"source"`4749 Body string `json:"body"`@@ -49,6 +51,7 @@ type Capture struct {
4951 MapSlug string `json:"mapSlug"`5052 MapTitle string `json:"mapTitle"`5153 Map MapConfig `json:"map"`54+ Variants []Variant `json:"variants,omitempty"`5255 Features []Feature `json:"features"`5356 }5457 @@ -62,6 +65,17 @@ type MapConfig struct {
6265 LayerTitle string `json:"layerTitle"`6366 }6467 68+// Variant is a sibling mosaic of the same ground: another way of seeing the69+// body -- elevation beside photograph -- captured into the same window, so70+// the viewer offers it the way it offers a game's satellite layer beside its71+// atlas. Its pyramid is its own; siblings need not agree on depth.72+type Variant struct {73+ Layer string `json:"layer"`74+ Title string `json:"title"`75+ MaxZoom int `json:"maxZoom"`76+ Extension string `json:"extension"`77+}78+6579 // Feature is one Gazetteer entry, as the feature list publishes it: the IAU's6680 // own feature identifier, planetocentric latitude, east-positive longitude in6781 // the Gazetteer's 0..360 spelling, diameter in kilometers, and the origin text@@ -81,6 +95,7 @@ type Feature struct {
8195 // before marshaling, so the Gazetteer listing its features differently does8296 // not masquerade as a new version.8397 func (c *Capture) Normalize() {98+ sort.Slice(c.Variants, func(a, b int) bool { return c.Variants[a].Layer < c.Variants[b].Layer })8499 sort.Slice(c.Features, func(a, b int) bool { return c.Features[a].ID < c.Features[b].ID })85100 }86101 @@ -130,6 +145,16 @@ func Translate(doc []byte) ([]byte, error) {
130145 if len(capture.Features) == 0 {131146 return nil, fmt.Errorf("capture carries no features")132147 }148+ taken := map[string]bool{capture.Layer: true}149+ for _, variant := range capture.Variants {150+ if variant.Layer == "" || variant.MaxZoom < 1 {151+ return nil, fmt.Errorf("variant %q declares no pyramid", variant.Layer)152+ }153+ if taken[variant.Layer] {154+ return nil, fmt.Errorf("layer %q is captured twice", variant.Layer)155+ }156+ taken[variant.Layer] = true157+ }133158 capture.Normalize()134159 135160 ids := mgdoc.NewIDSpace()@@ -148,6 +173,13 @@ func Translate(doc []byte) ([]byte, error) {
148173 return nil, err149174 }150175 176+ sets := []mgdoc.TileSet{tileSet(capture.Body, capture.Layer,177+ title(capture.Map.LayerTitle, "default"), capture.Map.Extension, capture.Map.MaxZoom)}178+ for _, variant := range capture.Variants {179+ sets = append(sets, tileSet(capture.Body, variant.Layer,180+ title(variant.Title, "default"), variant.Extension, variant.MaxZoom))181+ }182+151183 out := mgdoc.Map{152184 ID: mapID,153185 Title: title(capture.MapTitle, capture.MapSlug),@@ -157,7 +189,7 @@ func Translate(doc []byte) ([]byte, error) {
157189 // sits in the square.158190 InitialLatitude: mgdoc.SyntheticLatitude(mgdoc.WorldSize / 4),159191 InitialLongitude: mgdoc.SyntheticLongitude(mgdoc.WorldSize / 2),160- Config: mgdoc.Config{TileSets: []mgdoc.TileSet{tileSet(&capture)}},192+ Config: mgdoc.Config{TileSets: sets},161193 Game: mgdoc.Game{162194 ID: gameID,163195 // A planet is a "game" to the pipeline, which is the point being@@ -179,21 +211,22 @@ func title(given, slug string) string {
179211 return mgdoc.SpellOut(slug)180212 }181213 182-// tileSet declares the pyramid in the shape tools/tiles expects. Bounds are183-// declared for every level down to zero: a level without them would be184-// measured against the MapGenie window this map does not sit in, and the185-// half-height windows are what tell the frame derivation where the planet186-// actually is.187-func tileSet(capture *Capture) mgdoc.TileSet {214+// tileSet declares one layer's pyramid in the shape tools/tiles expects.215+// Bounds are declared for every level down to zero: a level without them216+// would be measured against the MapGenie window this map does not sit in,217+// and the half-height windows are what tell the frame derivation where the218+// planet actually is. Every layer of a body shares that window whatever its219+// depth, which is what lets siblings ride one map as its variants.220+func tileSet(body, layer, name, ext string, maxZoom int) mgdoc.TileSet {188221 set := mgdoc.TileSet{189- Name: title(capture.Map.LayerTitle, "default"),190- Path: TileSetPath(capture.Body, capture.Layer),222+ Name: name,223+ Path: TileSetPath(body, layer),191224 MinZoom: 0,192- MaxZoom: capture.Map.MaxZoom,193- Extension: extension(capture.Map.Extension),194- Bounds: make(map[string]mgdoc.Bound, capture.Map.MaxZoom+1),225+ MaxZoom: maxZoom,226+ Extension: extension(ext),227+ Bounds: make(map[string]mgdoc.Bound, maxZoom+1),195228 }196- for zoom := 0; zoom <= capture.Map.MaxZoom; zoom++ {229+ for zoom := 0; zoom <= maxZoom; zoom++ {197230 maxX, maxY := LevelExtent(zoom)198231 set.Bounds[strconv.Itoa(zoom)] = mgdoc.Bound{199232 X: mgdoc.Range{Min: 0, Max: maxX},@@ -26,6 +26,10 @@ func fixture() Capture {
2626 Extension: "jpg",2727 LayerTitle: "Viking MDIM 2.1",2828 },29+ Variants: []Variant{30+ {Layer: "Mars_MGS_MOLA_ClrShade_merge_global_463m",31+ Title: "MOLA Elevation", MaxZoom: 6, Extension: "jpg"},32+ },2933 Features: []Feature{3034 {ID: 4453, Name: "Olympus Mons", Type: "Mons, montes", Code: "MO",3135 Latitude: 18.6528, Longitude: 226.1975, DiameterKM: 610.13,@@ -86,7 +90,7 @@ func TestTranslateShape(t *testing.T) {
8690 t.Fatalf("game identity: %q %q", m.Game.Slug, m.Game.Title)8791 }8892 89- if len(m.Config.TileSets) != 1 {93+ if len(m.Config.TileSets) != 2 {9094 t.Fatalf("tile sets: %d", len(m.Config.TileSets))9195 }9296 set := m.Config.TileSets[0]@@ -99,19 +103,28 @@ func TestTranslateShape(t *testing.T) {
99103 if set.MinZoom != 0 || set.MaxZoom != 6 {100104 t.Fatalf("zoom range: %d..%d", set.MinZoom, set.MaxZoom)101105 }106+ // The sibling mosaic rides the same map as a second layer in the same107+ // window, under its own name and path.108+ sibling := m.Config.TileSets[1]109+ if sibling.Name != "MOLA Elevation" ||110+ sibling.Path != "mars/EQ/Mars_MGS_MOLA_ClrShade_merge_global_463m" {111+ t.Fatalf("variant tile set: %+v", sibling)112+ }102113 // The planet fills the width of the world square and the top half of its103- // height, at every level of the pyramid.104- for zoom := 0; zoom <= 6; zoom++ {105- bounds, ok := set.Bounds[strconv.Itoa(zoom)]106- if !ok {107- t.Fatalf("no bounds declared for zoom %d", zoom)108- }109- wantX, wantY := 1<<zoom-1, 1<<zoom/2-1110- if zoom == 0 {111- wantX, wantY = 0, 0112- }113- if bounds.X.Min != 0 || bounds.Y.Min != 0 || bounds.X.Max != wantX || bounds.Y.Max != wantY {114- t.Fatalf("zoom %d bounds: %+v", zoom, bounds)114+ // height, at every level of both pyramids.115+ for _, layer := range m.Config.TileSets {116+ for zoom := 0; zoom <= layer.MaxZoom; zoom++ {117+ bounds, ok := layer.Bounds[strconv.Itoa(zoom)]118+ if !ok {119+ t.Fatalf("%s declares no bounds for zoom %d", layer.Name, zoom)120+ }121+ wantX, wantY := 1<<zoom-1, 1<<zoom/2-1122+ if zoom == 0 {123+ wantX, wantY = 0, 0124+ }125+ if bounds.X.Min != 0 || bounds.Y.Min != 0 || bounds.X.Max != wantX || bounds.Y.Max != wantY {126+ t.Fatalf("%s zoom %d bounds: %+v", layer.Name, zoom, bounds)127+ }115128 }116129 }117130 @@ -243,6 +256,8 @@ func TestTranslateRejectsBadCaptures(t *testing.T) {
243256 {"latitude off the planet", func(c *Capture) { c.Features[0].Latitude = 91 }, "latitude"},244257 {"longitude off the planet", func(c *Capture) { c.Features[0].Longitude = -5 }, "longitude"},245258 {"doubled feature", func(c *Capture) { c.Features[1].ID = c.Features[0].ID }, "collision"},259+ {"doubled layer", func(c *Capture) { c.Variants[0].Layer = c.Layer }, "twice"},260+ {"depthless variant", func(c *Capture) { c.Variants[0].MaxZoom = 0 }, "pyramid"},246261 }247262 for _, test := range cases {248263 capture := fixture()@@ -41,23 +41,35 @@ const trekTiles = "https://trek.nasa.gov/tiles"
4141 const trekDefaultZoom = 64242 4343 // trekBody is one body the importer knows: how Trek's paths spell it, which44-// global mosaic pictures it, and where the Gazetteer's feature list for it45-// lives.44+// global mosaic pictures it, the sibling mosaics captured beside that one,45+// and where the Gazetteer's feature list for it lives.4646 type trekBody struct {4747 pathName string4848 layer string4949 layerTitle string50+ variants []trekLayer5051 portal string5152 gazetteer string5253 }5354 55+// trekLayer names one verified sibling mosaic and how a person knows it.56+type trekLayer struct {57+ layer string58+ title string59+}60+5461 var trekBodies = map[string]trekBody{5562 "mars": {5663 pathName: "Mars",5764 layer: "Mars_Viking_MDIM21_ClrMosaic_global_232m",5865 layerTitle: "Viking MDIM 2.1",59- portal: "https://trek.nasa.gov/mars",60- gazetteer: "https://asc-planetarynames-data.s3.us-west-2.amazonaws.com/MARS_nomenclature_center_pts.zip",66+ variants: []trekLayer{67+ // The laser altimeter's colored shaded relief: the same ground as68+ // height instead of photograph, the other classic way Mars is seen.69+ {layer: "Mars_MGS_MOLA_ClrShade_merge_global_463m", title: "MOLA Elevation"},70+ },71+ portal: "https://trek.nasa.gov/mars",72+ gazetteer: "https://asc-planetarynames-data.s3.us-west-2.amazonaws.com/MARS_nomenclature_center_pts.zip",6173 },6274 }6375 @@ -74,16 +86,22 @@ func runTrek(ctx context.Context, fetcher *fetcher, o options) error {
7486 strings.Join(bodies, ", "))7587 }7688 77- layerBase := trekTiles + "/" + body.pathName + "/EQ/" + body.layer78- published, err := fetchTrekPyramid(ctx, fetcher, layerBase)79- if err != nil {80- return err81- }82- deepest := published83- if o.maxZoom > 0 {84- deepest = min(deepest, o.maxZoom)85- } else {86- deepest = min(deepest, trekDefaultZoom)89+ // Every mosaic of the body answers for its own tiling before anything is90+ // fetched, and takes its own depth: the default courtesy ceiling, under91+ // whatever each one actually publishes.92+ layers := append([]trekLayer{{layer: body.layer, title: body.layerTitle}}, body.variants...)93+ depths := make([]int, len(layers))94+ for at, entry := range layers {95+ published, err := fetchTrekPyramid(ctx, fetcher, trekLayerBase(body, entry.layer))96+ if err != nil {97+ return fmt.Errorf("%s: %w", entry.layer, err)98+ }99+ depths[at] = published100+ if o.maxZoom > 0 {101+ depths[at] = min(depths[at], o.maxZoom)102+ } else {103+ depths[at] = min(depths[at], trekDefaultZoom)104+ }87105 }88106 89107 features, err := fetchGazetteer(ctx, fetcher, body.gazetteer)@@ -98,12 +116,20 @@ func runTrek(ctx context.Context, fetcher *fetcher, o options) error {
98116 MapSlug: "global",99117 MapTitle: "Global",100118 Map: trekmap.MapConfig{101- MaxZoom: deepest,119+ MaxZoom: depths[0],102120 Extension: "jpg",103121 LayerTitle: body.layerTitle,104122 },105123 Features: features,106124 }125+ for at, entry := range layers[1:] {126+ capture.Variants = append(capture.Variants, trekmap.Variant{127+ Layer: entry.layer,128+ Title: entry.title,129+ MaxZoom: depths[at+1],130+ Extension: "jpg",131+ })132+ }107133 capture.Normalize()108134 raw, err := json.MarshalIndent(capture, "", " ")109135 if err != nil {@@ -129,8 +155,8 @@ func runTrek(ctx context.Context, fetcher *fetcher, o options) error {
129155 mapDirectory := resolveMapDirectory(o.archive, gameDirectory, full)130156 mapDir := filepath.Join(o.archive, mapDirectory)131157 132- fmt.Printf("\n== NASA Trek %s / %s (%d features, pyramid z1-%d of %d published)\n",133- game.Title, capture.MapTitle, len(capture.Features), deepest, published)158+ fmt.Printf("\n== NASA Trek %s / %s (%d features, %d layers)\n",159+ game.Title, capture.MapTitle, len(capture.Features), len(layers))134160 135161 if o.dryRun {136162 fmt.Printf(" would write capture to %s\n", mapDir)@@ -143,9 +169,17 @@ func runTrek(ctx context.Context, fetcher *fetcher, o options) error {
143169 return err144170 }145171 index.mapID = mapID146- stats, err := captureTrekTiles(ctx, fetcher, o, slug, body, layerBase, deepest, mapDir, index)147- if err != nil {148- return err172+ var stats captureStats173+ for at, entry := range layers {174+ taken, err := captureTrekTiles(ctx, fetcher, o, slug, body, entry.layer, depths[at], mapDir, index)175+ if err != nil {176+ return err177+ }178+ stats.fetched += taken.fetched179+ stats.skipped += taken.skipped180+ stats.absent += taken.absent181+ stats.failed += taken.failed182+ stats.bytes += taken.bytes149183 }150184 if o.dryRun {151185 return nil@@ -221,25 +255,31 @@ func fetchTrekPyramid(ctx context.Context, f *fetcher, layerBase string) (int, e
221255 return len(matrices), nil222256 }223257 224-// captureTrekTiles takes the pyramid level by level, whole: a global mosaic225-// has no empty space to prune around. Trek spells a tile as matrix/row/226-// column, one zoom below and with its axes named the other way round from227-// the pipeline's zoom/x/y -- so each level fetches under Trek's spelling and228-// records under the canonical one, which is the address every reader of the229-// archive derives the layer and its window from.258+// trekLayerBase is where one of the body's mosaics actually lives, in Trek's259+// own spelling of the body's name.260+func trekLayerBase(body trekBody, layer string) string {261+ return trekTiles + "/" + body.pathName + "/EQ/" + layer262+}263+264+// captureTrekTiles takes one layer's pyramid level by level, whole: a global265+// mosaic has no empty space to prune around. Trek spells a tile as matrix/266+// row/column, one zoom below and with its axes named the other way round267+// from the pipeline's zoom/x/y -- so each level fetches under Trek's268+// spelling and records under the canonical one, which is the address every269+// reader of the archive derives the layer and its window from.230270 func captureTrekTiles(231271 ctx context.Context,232272 fetcher *fetcher,233273 o options,234274 slug string,235275 body trekBody,236- layerBase string,276+ layer string,237277 deepest int,238278 mapDir string,239279 index *tileIndex,240280 ) (captureStats, error) {241281 var stats captureStats242- scope := trekmap.TileSetPath(slug, body.layer)282+ scope := trekmap.TileSetPath(slug, layer)243283 setID := index.tileSetID(scope)244284 fmt.Printf(" layer %q z1-%d\n", scope, deepest)245285 @@ -247,7 +287,7 @@ func captureTrekTiles(
247287 maxX, maxY := trekmap.LevelExtent(zoom)248288 window := tileWindow{minX: 0, minY: 0, maxX: maxX, maxY: maxY}249289 record := trekTiles + "/" + scope + "/{z}/{x}/{y}.jpg"250- fetch := layerBase + "/1.0.0/default/default028mm/" +290+ fetch := trekLayerBase(body, layer) + "/1.0.0/default/default028mm/" +251291 strconv.Itoa(zoom-1) + "/{y}/{x}.jpg"252292 results, err := fetchTemplateLevel(ctx, fetcher, o, record, fetch, "", setID, "jpg",253293 mapDir, zoom, window.tiles(), index)