1package vnext23import (4 "bytes"5 "crypto/sha256"6 "encoding/hex"7 "encoding/json"8 "fmt"9 "io"10 "os"11 "path/filepath"12 "slices"13 "strconv"14 "strings"15)1617const (18 rasterIndexName = "indexes/rasters.v1.json"19 rasterShardPrefix = "data/raster-shards/"20 maxRasterShardBytes = int64(64 << 20)21 maxRasterShardTiles = 4_09622)2324type rasterTileIndex struct {25 Version uint16 `json:"version"`26 Tiles []rasterTileDescriptor `json:"tiles"`27}2829type rasterTileDescriptor struct {30 Name string `json:"name"`31 Shard string `json:"shard"`32 Offset int64 `json:"offset"`33 Length int64 `json:"length"`34 Hash string `json:"hash"`35 Format string `json:"format"`36 MediaType string `json:"mediaType"`37}3839type pendingRasterTile struct {40 name string41 blob Blob42 length int6443 format string44 mediaType string45}4647type shardRange struct {48 offset int6449 length int6450}5152// RasterTile is one verified logical tile read from a shard range. Format and53// MediaType are per tile, so a pyramid may legally mix PNG and JPEG.54type RasterTile struct {55 Name string56 Format string57 MediaType string58 Data []byte59}6061func packRasterBlobs(blobs []Blob) ([]Blob, error) {62 tiles := make([]pendingRasterTile, 0)63 kept := make([]Blob, 0, len(blobs))64 for _, blob := range blobs {65 format, mediaType, isTile := rasterTileType(blob.Name)66 if !isTile {67 kept = append(kept, blob)68 continue69 }70 length, err := blobLength(blob)71 if err != nil {72 return nil, fmt.Errorf("pack raster tile %s: %w", blob.Name, err)73 }74 if length > maxRasterShardBytes {75 return nil, fmt.Errorf("pack raster tile %s: tile exceeds shard limit", blob.Name)76 }77 tiles = append(tiles, pendingRasterTile{name: blob.Name, blob: blob, length: length, format: format, mediaType: mediaType})78 }79 if len(tiles) == 0 {80 return blobs, nil81 }82 slices.SortFunc(tiles, func(left, right pendingRasterTile) int { return strings.Compare(left.name, right.name) })83 index := rasterTileIndex{Version: 1}84 for start := 0; start < len(tiles); {85 end := start86 var length int6487 for end < len(tiles) && end-start < maxRasterShardTiles {88 next := tiles[end].length89 if end > start && length+next > maxRasterShardBytes {90 break91 }92 length += next93 end++94 }95 shard, descriptors, err := spoolRasterShard(tiles[start:end])96 if err != nil {97 cleanupTemporaryBlobs(kept)98 return nil, err99 }100 kept = append(kept, shard)101 index.Tiles = append(index.Tiles, descriptors...)102 start = end103 }104 data, err := json.Marshal(index)105 if err != nil {106 cleanupTemporaryBlobs(kept)107 return nil, fmt.Errorf("encode raster tile index: %w", err)108 }109 kept = append(kept, Blob{Name: rasterIndexName, Data: data})110 slices.SortFunc(kept, func(left, right Blob) int { return strings.Compare(left.Name, right.Name) })111 return kept, nil112}113114func blobLength(blob Blob) (int64, error) {115 if blob.Path == "" {116 return int64(len(blob.Data)), nil117 }118 if len(blob.Data) != 0 {119 return 0, fmt.Errorf("blob has both data and a source path")120 }121 info, err := os.Stat(blob.Path)122 if err != nil {123 return 0, err124 }125 return info.Size(), nil126}127128func spoolRasterShard(tiles []pendingRasterTile) (Blob, []rasterTileDescriptor, error) {129 file, err := os.CreateTemp("", "atlas-raster-shard-*.rpack")130 if err != nil {131 return Blob{}, nil, fmt.Errorf("stage raster shard: %w", err)132 }133 path := file.Name()134 failed := func(err error) (Blob, []rasterTileDescriptor, error) {135 file.Close()136 os.Remove(path)137 return Blob{}, nil, err138 }139 shardHash := sha256.New()140 descriptors := make([]rasterTileDescriptor, 0, len(tiles))141 var offset int64142 for _, tile := range tiles {143 tileHash := sha256.New()144 written, err := copyBlob(io.MultiWriter(file, shardHash, tileHash), tile.blob)145 if err != nil {146 return failed(fmt.Errorf("write raster tile %s: %w", tile.name, err))147 }148 if written != tile.length {149 return failed(fmt.Errorf("write raster tile %s: length changed", tile.name))150 }151 descriptors = append(descriptors, rasterTileDescriptor{152 Name: tile.name, Offset: offset, Length: written, Hash: hex.EncodeToString(tileHash.Sum(nil)),153 Format: tile.format, MediaType: tile.mediaType,154 })155 offset += written156 }157 if err := file.Sync(); err != nil {158 return failed(fmt.Errorf("sync raster shard: %w", err))159 }160 if err := file.Close(); err != nil {161 os.Remove(path)162 return Blob{}, nil, fmt.Errorf("close raster shard: %w", err)163 }164 name := rasterShardPrefix + hex.EncodeToString(shardHash.Sum(nil)) + ".rpack"165 for index := range descriptors {166 descriptors[index].Shard = name167 }168 return Blob{Name: name, Path: path, temporary: true}, descriptors, nil169}170171func copyBlob(destination io.Writer, blob Blob) (int64, error) {172 if blob.Path == "" {173 written, err := io.Copy(destination, bytes.NewReader(blob.Data))174 return written, err175 }176 source, err := os.Open(blob.Path)177 if err != nil {178 return 0, err179 }180 written, copyErr := io.Copy(destination, source)181 closeErr := source.Close()182 if copyErr != nil {183 return written, copyErr184 }185 return written, closeErr186}187188func cleanupTemporaryBlobs(blobs []Blob) {189 for _, blob := range blobs {190 if blob.temporary {191 _ = os.Remove(blob.Path)192 }193 }194}195196func rasterTileType(name string) (string, string, bool) {197 parts := strings.Split(name, "/")198 if len(parts) != 5 || parts[0] != "tiles" || parts[1] == "" {199 return "", "", false200 }201 zoom, zoomErr := strconv.ParseUint(parts[2], 10, 6)202 x, xErr := strconv.ParseUint(parts[3], 10, 63)203 yText := strings.TrimSuffix(parts[4], filepath.Ext(parts[4]))204 y, yErr := strconv.ParseUint(yText, 10, 63)205 if zoomErr != nil || xErr != nil || yErr != nil || x >= uint64(1)<<zoom || y >= uint64(1)<<zoom {206 return "", "", false207 }208 format := strings.TrimPrefix(strings.ToLower(filepath.Ext(parts[4])), ".")209 switch format {210 case "png":211 return format, "image/png", true212 case "jpg", "jpeg":213 return format, "image/jpeg", true214 case "webp":215 return format, "image/webp", true216 case "avif":217 return format, "image/avif", true218 default:219 return "", "", false220 }221}222223func (reader *Reader) rasterIndex() (rasterTileIndex, error) {224 reference, held := reader.blobs[rasterIndexName]225 if !held {226 return rasterTileIndex{}, nil227 }228 data, err := readReferencedEntry(reader.entries[rasterIndexName], reference.Length, reference.Hash, reader.limits.MaxBlobBytes)229 if err != nil {230 return rasterTileIndex{}, err231 }232 var index rasterTileIndex233 decoder := json.NewDecoder(bytes.NewReader(data))234 decoder.DisallowUnknownFields()235 if err := decoder.Decode(&index); err != nil || index.Version != 1 {236 return rasterTileIndex{}, fmt.Errorf("decode raster tile index")237 }238 if err := decoder.Decode(&struct{}{}); err != io.EOF {239 return rasterTileIndex{}, fmt.Errorf("raster tile index carries trailing data")240 }241 canonical, err := json.Marshal(index)242 if err != nil || !bytes.Equal(canonical, data) {243 return rasterTileIndex{}, fmt.Errorf("raster tile index is not canonical")244 }245 previous := ""246 ranges := make(map[string][]shardRange)247 for _, tile := range index.Tiles {248 format, mediaType, validName := rasterTileType(tile.Name)249 shard, held := reader.blobs[tile.Shard]250 decodedHash, hashErr := hex.DecodeString(tile.Hash)251 if tile.Name <= previous || tile.Offset < 0 || tile.Length < 0 || tile.Length > reader.limits.MaxBlobBytes ||252 !validName || format != tile.Format || mediaType != tile.MediaType || !held ||253 tile.Offset > shard.Length || tile.Length > shard.Length-tile.Offset || hashErr != nil ||254 len(decodedHash) != sha256.Size || tile.Hash != strings.ToLower(tile.Hash) {255 return rasterTileIndex{}, fmt.Errorf("raster tile index is not canonical")256 }257 ranges[tile.Shard] = append(ranges[tile.Shard], shardRange{offset: tile.Offset, length: tile.Length})258 previous = tile.Name259 }260 for shardName, held := range ranges {261 slices.SortFunc(held, func(left, right shardRange) int {262 if left.offset < right.offset {263 return -1264 }265 if left.offset > right.offset {266 return 1267 }268 return 0269 })270 var end int64271 for _, part := range held {272 if part.offset != end {273 return rasterTileIndex{}, fmt.Errorf("raster shard %s has overlapping or uncovered ranges", shardName)274 }275 end += part.length276 }277 if end != reader.blobs[shardName].Length {278 return rasterTileIndex{}, fmt.Errorf("raster shard %s has unindexed bytes", shardName)279 }280 }281 for name := range reader.blobs {282 if strings.HasPrefix(name, rasterShardPrefix) && ranges[name] == nil {283 return rasterTileIndex{}, fmt.Errorf("raster shard %s is not indexed", name)284 }285 }286 return index, nil287}288289// ReadRasterTile verifies one tile slice without reading the rest of its290// shard. The containing shard remains independently hash-verified by Validate.291func (reader *Reader) ReadRasterTile(name string) (RasterTile, error) {292 index, err := reader.rasterIndex()293 if err != nil {294 return RasterTile{}, err295 }296 at, held := slices.BinarySearchFunc(index.Tiles, name, func(tile rasterTileDescriptor, target string) int {297 return strings.Compare(tile.Name, target)298 })299 if !held {300 return RasterTile{}, fmt.Errorf("bundle has no raster tile %s", name)301 }302 tile := index.Tiles[at]303 data, err := reader.readStoredBlobRange(tile.Shard, tile.Offset, tile.Length)304 if err != nil {305 return RasterTile{}, err306 }307 digest := sha256.Sum256(data)308 if hex.EncodeToString(digest[:]) != tile.Hash {309 return RasterTile{}, fmt.Errorf("raster tile %s hash does not match", name)310 }311 return RasterTile{Name: name, Format: tile.Format, MediaType: tile.MediaType, Data: data}, nil312}313314func (reader *Reader) readStoredBlobRange(name string, offset, length int64) ([]byte, error) {315 reference, held := reader.blobs[name]316 if !held || offset < 0 || length < 0 || offset > reference.Length || length > reference.Length-offset {317 return nil, fmt.Errorf("blob range is invalid")318 }319 entry := reader.entries[name]320 if entry == nil || entry.Method != 0 {321 return nil, fmt.Errorf("blob range is not stored")322 }323 dataOffset, err := entry.DataOffset()324 if err != nil {325 return nil, fmt.Errorf("locate %s: %w", name, err)326 }327 data := make([]byte, length)328 if _, err := io.ReadFull(io.NewSectionReader(reader.source, dataOffset+offset, length), data); err != nil {329 return nil, fmt.Errorf("read %s range: %w", name, err)330 }331 return data, nil332}