1package authoring23import (4 "bytes"5 "context"6 "crypto/sha256"7 "encoding/hex"8 "encoding/json"9 "errors"10 "fmt"11 "io"12 "net/http"13 "os"14 "path/filepath"15 "sort"16 "strings"17 "time"18)1920const CaptureFormat = "atlas-capture/v1"2122const sha256HexLength = sha256.Size * 22324var errExistingContent = errors.New("path already holds different content")2526type BlobRef struct {27 SHA256 string `json:"sha256"`28 Length int64 `json:"length"`29 MediaType string `json:"mediaType"`30}3132// Capture is an immutable source-neutral envelope around exact evidence bytes.33type Capture struct {34 Format string `json:"format"`35 RequestHash string `json:"requestHash"`36 Source string `json:"source"`37 Adapter string `json:"adapter"`38 Kind string `json:"kind"`39 Locator string `json:"locator"`40 CapturedAt string `json:"capturedAt"`41 License string `json:"license,omitempty"`42 Attribution string `json:"attribution,omitempty"`43 Body BlobRef `json:"body"`44}4546// Cache is shared by projects. The clock and client are replaceable for a47// deterministic test; production uses UTC now and a bounded HTTP client.48type Cache struct {49 root string50 client *http.Client51 now func() time.Time52}5354func OpenCache(root string) *Cache {55 return &Cache{56 root: root, client: &http.Client{Timeout: 2 * time.Minute},57 now: func() time.Time { return time.Now().UTC() },58 }59}6061func (cache *Cache) Root() string { return cache.root }6263func (cache *Cache) Has(requestID string) bool {64 _, err := cache.Latest(requestID)65 return err == nil66}6768func (cache *Cache) Latest(requestID string) (Capture, error) {69 dir, err := cache.captureDir(requestID)70 if err != nil {71 return Capture{}, err72 }73 entries, err := os.ReadDir(dir)74 if err != nil {75 return Capture{}, err76 }77 sort.Slice(entries, func(i, j int) bool { return entries[i].Name() < entries[j].Name() })78 var captures []Capture79 for _, entry := range entries {80 if entry.IsDir() || filepath.Ext(entry.Name()) != ".json" {81 continue82 }83 bodySHA := strings.TrimSuffix(entry.Name(), ".json")84 capture, err := cache.Select(requestID, bodySHA)85 if err != nil {86 return Capture{}, fmt.Errorf("validate capture %s: %w", entry.Name(), err)87 }88 captures = append(captures, capture)89 }90 if len(captures) == 0 {91 return Capture{}, os.ErrNotExist92 }93 sort.Slice(captures, func(i, j int) bool {94 if captures[i].CapturedAt != captures[j].CapturedAt {95 return captures[i].CapturedAt < captures[j].CapturedAt96 }97 return captures[i].Body.SHA256 < captures[j].Body.SHA25698 })99 return captures[len(captures)-1], nil100}101102// Select returns one exact, validated capture for a request and body hash.103func (cache *Cache) Select(requestID, bodySHA string) (Capture, error) {104 dir, err := cache.captureDir(requestID)105 if err != nil {106 return Capture{}, err107 }108 if err := validateSHA256("body", bodySHA); err != nil {109 return Capture{}, err110 }111 path := filepath.Join(dir, bodySHA+".json")112 data, err := os.ReadFile(path)113 if err != nil {114 return Capture{}, fmt.Errorf("read capture %s: %w", bodySHA, err)115 }116 capture, err := decodeCapture(data)117 if err != nil {118 return Capture{}, fmt.Errorf("decode capture %s: %w", bodySHA, err)119 }120 if err := cache.validateCapture(capture, requestID, bodySHA); err != nil {121 return Capture{}, err122 }123 return capture, nil124}125126func (cache *Cache) BlobPath(ref BlobRef) string {127 path, err := cache.blobPath(ref.SHA256)128 if err != nil {129 return filepath.Join(cache.root, "blobs", "sha256", "invalid")130 }131 return path132}133134// Acquire fetches or reads one planned request, records only changed bytes and135// returns the immutable capture selected for this build.136func (cache *Cache) Acquire(ctx context.Context, request Request, offline bool) (Capture, bool, error) {137 requestKey := requestCacheKey(request)138 if err := validateSHA256("request", requestKey); err != nil {139 return Capture{}, false, err140 }141 if offline {142 capture, err := cache.Latest(requestKey)143 if err != nil {144 return Capture{}, false, fmt.Errorf("request %s is not cached: %w", request.Source, err)145 }146 return capture, true, nil147 }148 body, mediaType, err := cache.read(ctx, request)149 if err != nil {150 return Capture{}, false, err151 }152 digest := sha256.Sum256(body)153 hash := hex.EncodeToString(digest[:])154 if held, err := cache.Select(requestKey, hash); err == nil {155 return held, true, nil156 } else if !errors.Is(err, os.ErrNotExist) {157 return Capture{}, false, fmt.Errorf("select captured source %s: %w", request.Source, err)158 }159 if mediaType == "" {160 mediaType = request.MediaType161 }162 if mediaType == "" {163 mediaType = "application/octet-stream"164 }165 ref := BlobRef{SHA256: hash, Length: int64(len(body)), MediaType: mediaType}166 blobPath, err := cache.blobPath(hash)167 if err != nil {168 return Capture{}, false, err169 }170 if _, err := writeOnce(blobPath, body); err != nil {171 return Capture{}, false, fmt.Errorf("cache evidence: %w", err)172 }173 capture := Capture{174 Format: CaptureFormat, RequestHash: requestKey, Source: request.Source,175 Adapter: request.Adapter, Kind: string(request.Kind), Locator: request.IdentityLocator,176 CapturedAt: cache.now().UTC().Format(time.RFC3339Nano),177 License: request.License, Attribution: request.Attribution, Body: ref,178 }179 data, err := json.Marshal(capture)180 if err != nil {181 return Capture{}, false, err182 }183 data = append(data, '\n')184 dir, err := cache.captureDir(requestKey)185 if err != nil {186 return Capture{}, false, err187 }188 path := filepath.Join(dir, hash+".json")189 written, writeErr := writeOnce(path, data)190 if writeErr != nil && !errors.Is(writeErr, errExistingContent) {191 return Capture{}, false, fmt.Errorf("cache capture: %w", writeErr)192 }193 selected, err := cache.Select(requestKey, hash)194 if err != nil {195 return Capture{}, false, fmt.Errorf("validate cached capture: %w", err)196 }197 return selected, !written, nil198}199200func (cache *Cache) captureDir(requestID string) (string, error) {201 if err := validateSHA256("request", requestID); err != nil {202 return "", err203 }204 return filepath.Join(cache.root, "captures", requestID[:2], requestID), nil205}206207func (cache *Cache) blobPath(bodySHA string) (string, error) {208 if err := validateSHA256("body", bodySHA); err != nil {209 return "", err210 }211 return filepath.Join(cache.root, "blobs", "sha256", bodySHA[:2], bodySHA), nil212}213214func (cache *Cache) validateCapture(capture Capture, requestID, bodySHA string) error {215 if capture.Format != CaptureFormat {216 return fmt.Errorf("capture format %q, want %q", capture.Format, CaptureFormat)217 }218 if capture.RequestHash != requestID {219 return fmt.Errorf("capture request hash %q does not match %q", capture.RequestHash, requestID)220 }221 if capture.Source == "" || capture.Adapter == "" || capture.Locator == "" {222 return fmt.Errorf("capture envelope is incomplete")223 }224 switch RequestKind(capture.Kind) {225 case RequestFeatures, RequestRaster, RequestAsset:226 default:227 return fmt.Errorf("capture kind %q is invalid", capture.Kind)228 }229 if _, err := time.Parse(time.RFC3339Nano, capture.CapturedAt); err != nil {230 return fmt.Errorf("capture time %q: %w", capture.CapturedAt, err)231 }232 if capture.Body.SHA256 != bodySHA {233 return fmt.Errorf("capture body %q does not match path %q", capture.Body.SHA256, bodySHA)234 }235 if capture.Body.Length < 0 || strings.TrimSpace(capture.Body.MediaType) == "" {236 return fmt.Errorf("capture body metadata is incomplete")237 }238 return cache.validateBlob(capture.Body)239}240241func (cache *Cache) validateBlob(ref BlobRef) error {242 path, err := cache.blobPath(ref.SHA256)243 if err != nil {244 return err245 }246 file, err := os.Open(path)247 if err != nil {248 return fmt.Errorf("read captured body %s: %w", ref.SHA256, err)249 }250 defer file.Close()251 info, err := file.Stat()252 if err != nil {253 return fmt.Errorf("stat captured body %s: %w", ref.SHA256, err)254 }255 if !info.Mode().IsRegular() || info.Size() != ref.Length {256 return fmt.Errorf("captured body %s length is %d, want %d", ref.SHA256, info.Size(), ref.Length)257 }258 hasher := sha256.New()259 written, err := io.Copy(hasher, file)260 if err != nil {261 return fmt.Errorf("hash captured body %s: %w", ref.SHA256, err)262 }263 if written != ref.Length {264 return fmt.Errorf("captured body %s length is %d, want %d", ref.SHA256, written, ref.Length)265 }266 if actual := hex.EncodeToString(hasher.Sum(nil)); actual != ref.SHA256 {267 return fmt.Errorf("captured body hash is %s, want %s", actual, ref.SHA256)268 }269 return nil270}271272func decodeCapture(data []byte) (Capture, error) {273 decoder := json.NewDecoder(bytes.NewReader(data))274 decoder.DisallowUnknownFields()275 var capture Capture276 if err := decoder.Decode(&capture); err != nil {277 return Capture{}, err278 }279 if err := decoder.Decode(&struct{}{}); !errors.Is(err, io.EOF) {280 if err == nil {281 return Capture{}, fmt.Errorf("capture carries more than one JSON value")282 }283 return Capture{}, fmt.Errorf("decode capture trailer: %w", err)284 }285 return capture, nil286}287288func validateSHA256(name, value string) error {289 if len(value) != sha256HexLength || strings.ToLower(value) != value {290 return fmt.Errorf("%s SHA-256 %q is not lowercase hexadecimal", name, value)291 }292 if _, err := hex.DecodeString(value); err != nil {293 return fmt.Errorf("%s SHA-256 %q: %w", name, value, err)294 }295 return nil296}297298func (cache *Cache) read(ctx context.Context, request Request) ([]byte, string, error) {299 if request.MaxBytes <= 0 {300 request.MaxBytes = defaultBuildBudgets.RequestBytes301 }302 if !isRemote(request.Locator) {303 file, err := os.Open(request.Locator)304 if err != nil {305 return nil, "", fmt.Errorf("read %s source %s: %w", request.Adapter, request.Source, err)306 }307 defer file.Close()308 if info, err := file.Stat(); err != nil {309 return nil, "", fmt.Errorf("stat %s source %s: %w", request.Adapter, request.Source, err)310 } else if info.Size() > request.MaxBytes {311 return nil, "", fmt.Errorf("read %s source %s: %d bytes exceeds request budget %d", request.Adapter, request.Source, info.Size(), request.MaxBytes)312 }313 data, err := io.ReadAll(io.LimitReader(file, request.MaxBytes+1))314 if err != nil {315 return nil, "", fmt.Errorf("read %s source %s: %w", request.Adapter, request.Source, err)316 }317 if int64(len(data)) > request.MaxBytes {318 return nil, "", fmt.Errorf("read %s source %s exceeds request budget %d", request.Adapter, request.Source, request.MaxBytes)319 }320 return data, request.MediaType, nil321 }322 req, err := http.NewRequestWithContext(ctx, http.MethodGet, request.Locator, nil)323 if err != nil {324 return nil, "", err325 }326 req.Header.Set("Accept", request.MediaType)327 req.Header.Set("User-Agent", "Atlas authoring/1")328 response, err := cache.client.Do(req)329 if err != nil {330 return nil, "", fmt.Errorf("fetch %s source %s: %w", request.Adapter, request.Source, err)331 }332 defer response.Body.Close()333 if response.StatusCode != http.StatusOK {334 _, _ = io.Copy(io.Discard, response.Body)335 return nil, "", fmt.Errorf("fetch %s source %s: %s", request.Adapter, request.Source, response.Status)336 }337 if response.ContentLength > request.MaxBytes {338 return nil, "", fmt.Errorf("fetch %s source %s: %d bytes exceeds request budget %d", request.Adapter, request.Source, response.ContentLength, request.MaxBytes)339 }340 body, err := io.ReadAll(io.LimitReader(response.Body, request.MaxBytes+1))341 if err != nil {342 return nil, "", fmt.Errorf("read %s source %s: %w", request.Adapter, request.Source, err)343 }344 if int64(len(body)) > request.MaxBytes {345 return nil, "", fmt.Errorf("read %s source %s exceeds request budget %d", request.Adapter, request.Source, request.MaxBytes)346 }347 mediaType := strings.TrimSpace(strings.Split(response.Header.Get("Content-Type"), ";")[0])348 return body, mediaType, nil349}350351func writeOnce(path string, data []byte) (bool, error) {352 if held, err := os.ReadFile(path); err == nil {353 if !bytes.Equal(held, data) {354 return false, fmt.Errorf("%w at %s", errExistingContent, path)355 }356 return false, nil357 } else if !errors.Is(err, os.ErrNotExist) {358 return false, err359 }360 if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {361 return false, err362 }363 stage, err := os.CreateTemp(filepath.Dir(path), ".writing-*")364 if err != nil {365 return false, err366 }367 name := stage.Name()368 defer os.Remove(name)369 if _, err := stage.Write(data); err != nil {370 _ = stage.Close()371 return false, err372 }373 if err := stage.Sync(); err != nil {374 _ = stage.Close()375 return false, err376 }377 if err := stage.Close(); err != nil {378 return false, err379 }380 if err := os.Chmod(name, 0o644); err != nil {381 return false, err382 }383 if err := os.Link(name, path); err != nil {384 if !errors.Is(err, os.ErrExist) {385 return false, err386 }387 held, readErr := os.ReadFile(path)388 if readErr != nil {389 return false, readErr390 }391 if !bytes.Equal(held, data) {392 return false, fmt.Errorf("%w at %s", errExistingContent, path)393 }394 return false, nil395 }396 return true, nil397}