- commit
- 8867da128f1648c5fd19070f2b144ab6ebefc238
- parent
- fe0e11cd305bc053375e215480da7a08588585f6
- author
- Dami <dami.etoile@gmail.com>
| M | internal/ | 110 | +++++++++++++++++++++++++++--- |
| M | internal/ | 106 | +++++++++++++++++++++++++++++ |
@@ -61,6 +61,10 @@ type Config struct {
6161 // Limits is read for each upload so policy changes apply without reopening6262 // the blob service. Zero values mean unlimited.6363 Limits func() Limits64+ // UploadTerms returns what an uploader's standing imposes on its uploads:65+ // an agent grant with a sites ttl gives its claims an expiry, and one with66+ // the encrypted flag refuses plain uploads. Zero values mean no terms.67+ UploadTerms func(context.Context, string) (UploadTerms, error)6468 }6569 6670 type Limits struct {@@ -68,6 +72,20 @@ type Limits struct {
6872 UserStorageBytes int646973 }7074 75+// UploadTerms is what an uploader's grant imposes on the blobs it stores.76+// ExpiresAt is the unix time its claims lapse, 0 for never; Encrypted77+// requires the stored bytes to be an encrypted blob or manifest.78+type UploadTerms struct {79+ ExpiresAt int6480+ Encrypted bool81+}82+83+// encryptedTypes are the content types the browser uploader gives encrypted84+// objects: AES-GCM ciphertext for files and chunks, and the BUD-16 directory85+// type for encrypted manifests. Anything the relay recognizes as a page,86+// image, video, audio or document is plain.87+var encryptedTypes = map[string]bool{"application/octet-stream": true, "application/vnd.blossom.directory+msgpack": true}88+7189 type Blob struct {7290 SHA256 string `json:"sha256"`7391 Size int64 `json:"size"`@@ -107,6 +125,7 @@ var ErrHashMismatch = errors.New("conflict: mirrored blob hash does not match th
107125 var ErrFileTooLarge = errors.New("invalid: blob exceeds the per-file size limit")108126 var ErrQuotaExceeded = errors.New("restricted: uploader storage quota exceeded")109127 var ErrMultipartConflict = errors.New("conflict: upload metadata differs from existing session")128+var ErrNotEncrypted = errors.New("restricted: agent grant requires encrypted uploads")110129 111130 var shaPattern = regexp.MustCompile(`^[0-9a-f]{64}$`)112131 var blobPathPattern = regexp.MustCompile(`^/([0-9a-f]{64})(?:\.[a-z0-9]{1,8})?$`)@@ -157,6 +176,14 @@ func New(ctx context.Context, config Config) (*Service, error) {
157176 SELECT sha256,uploader,uploaded FROM blobs WHERE uploader != ''`); err != nil {158177 return nil, fmt.Errorf("initialize blob metadata: %w", err)159178 }179+ // Claims made under an agent grant with a ttl carry an expiry; every180+ // earlier claim keeps 0, which never lapses.181+ if _, err := config.Store.DB().ExecContext(ctx, "ALTER TABLE blob_claims ADD COLUMN expires INTEGER NOT NULL DEFAULT 0"); err != nil && !strings.Contains(err.Error(), "duplicate column") {182+ return nil, fmt.Errorf("add blob claim expiry: %w", err)183+ }184+ if _, err := config.Store.DB().ExecContext(ctx, "CREATE INDEX IF NOT EXISTS blob_claims_expires ON blob_claims(expires) WHERE expires>0"); err != nil {185+ return nil, fmt.Errorf("index blob claim expiry: %w", err)186+ }160187 if err := reconcile(ctx, config.Store, filepath.Join(config.Root, "blobs")); err != nil {161188 return nil, err162189 }@@ -649,6 +676,10 @@ func (s *Service) installUpload(ctx context.Context, candidate uploadCandidate)
649676 if s.blocked(ctx, sha) {650677 return Blob{}, false, ErrBlocked651678 }679+ terms, err := s.uploadTerms(ctx, uploader)680+ if err != nil {681+ return Blob{}, false, err682+ }652683 limits := s.currentLimits()653684 if limits.MaxFileBytes > 0 && count > limits.MaxFileBytes {654685 return Blob{}, false, ErrFileTooLarge@@ -657,6 +688,9 @@ func (s *Service) installUpload(ctx context.Context, candidate uploadCandidate)
657688 return Blob{}, false, ErrQuotaExceeded658689 }659690 if existing, err := s.lookup(ctx, sha); err == nil {691+ if terms.Encrypted && !encryptedTypes[existing.Type] {692+ return Blob{}, false, ErrNotEncrypted693+ }660694 if uploader != "" {661695 if limits.UserStorageBytes > 0 && !s.hasClaim(ctx, sha, uploader) {662696 used, usageErr := s.quotaUsageExcept(ctx, uploader, candidate.reservation)@@ -667,7 +701,7 @@ func (s *Service) installUpload(ctx context.Context, candidate uploadCandidate)
667701 return Blob{}, false, ErrQuotaExceeded668702 }669703 }670- if err := s.claim(ctx, sha, uploader); err != nil {704+ if err := s.claim(ctx, sha, uploader, terms.ExpiresAt); err != nil {671705 return Blob{}, false, err672706 }673707 }@@ -689,6 +723,16 @@ func (s *Service) installUpload(ctx context.Context, candidate uploadCandidate)
689723 return Blob{}, false, ErrQuotaExceeded690724 }691725 }726+ // The type is settled before the bytes are installed so a plain upload727+ // under an encrypted-only grant leaves nothing behind. A declared octet728+ // stream is sniffed: encrypted bytes stay one, a page or an image does not.729+ typ = contentType(typ)730+ if typ == "application/octet-stream" {731+ typ = detectFileType(candidate.path)732+ }733+ if terms.Encrypted && !encryptedTypes[typ] {734+ return Blob{}, false, ErrNotEncrypted735+ }692736 final := filepath.Join(s.root, sha)693737 if err := os.Rename(candidate.path, final); err != nil && !errors.Is(err, os.ErrExist) {694738 return Blob{}, false, fmt.Errorf("install blob: %w", err)@@ -697,12 +741,8 @@ func (s *Service) installUpload(ctx context.Context, candidate uploadCandidate)
697741 return Blob{}, false, fmt.Errorf("sync blob directory: %w", err)698742 }699743 now := time.Now().UTC().Unix()700- typ = contentType(typ)701- if typ == "application/octet-stream" {702- typ = detectFileType(final)703- }704744 entry := Blob{SHA256: sha, Size: count, Type: typ, Uploader: uploader, Uploaded: now}705- err := s.store.WithTx(ctx, func(tx *sql.Tx) error {745+ err = s.store.WithTx(ctx, func(tx *sql.Tx) error {706746 if _, err := tx.ExecContext(ctx, "DELETE FROM blob_tombstones WHERE sha256=?", entry.SHA256); err != nil {707747 return fmt.Errorf("clear blob deletion tombstone: %w", err)708748 }@@ -710,7 +750,7 @@ func (s *Service) installUpload(ctx context.Context, candidate uploadCandidate)
710750 return fmt.Errorf("record blob: %w", err)711751 }712752 if entry.Uploader != "" {713- if _, err := tx.ExecContext(ctx, "INSERT OR IGNORE INTO blob_claims(sha256,uploader,claimed_at) VALUES(?,?,?)", entry.SHA256, entry.Uploader, entry.Uploaded); err != nil {753+ if _, err := tx.ExecContext(ctx, claimSQL, entry.SHA256, entry.Uploader, entry.Uploaded, terms.ExpiresAt); err != nil {714754 return fmt.Errorf("record blob claim: %w", err)715755 }716756 }@@ -745,14 +785,64 @@ func (s *Service) quotaUsageExcept(ctx context.Context, uploader, reservation st
745785 return used.Int64, nil746786 }747787 748-func (s *Service) claim(ctx context.Context, sha, uploader string) error {749- _, err := s.store.DB().ExecContext(ctx, "INSERT OR IGNORE INTO blob_claims(sha256,uploader,claimed_at) VALUES(?,?,?)", sha, uploader, time.Now().UTC().Unix())788+// claimSQL records a claim. The insert trigger on blobs has usually made the789+// row already, without an expiry, so the newest upload's terms replace it.790+const claimSQL = "INSERT INTO blob_claims(sha256,uploader,claimed_at,expires) VALUES(?,?,?,?) ON CONFLICT(sha256,uploader) DO UPDATE SET expires=excluded.expires"791+792+func (s *Service) claim(ctx context.Context, sha, uploader string, expires int64) error {793+ _, err := s.store.DB().ExecContext(ctx, claimSQL, sha, uploader, time.Now().UTC().Unix(), expires)750794 if err != nil {751795 return fmt.Errorf("record blob claim: %w", err)752796 }753797 return nil754798 }755799 800+func (s *Service) uploadTerms(ctx context.Context, uploader string) (UploadTerms, error) {801+ if s.config.UploadTerms == nil || uploader == "" {802+ return UploadTerms{}, nil803+ }804+ terms, err := s.config.UploadTerms(ctx, uploader)805+ if err != nil {806+ return UploadTerms{}, fmt.Errorf("check upload terms: %w", err)807+ }808+ return terms, nil809+}810+811+// SweepExpired removes the blobs whose every claim has lapsed and releases812+// lapsed claims on blobs someone else still holds, so an agent's files leave813+// with its site while a person's claim keeps them. It returns how many blobs814+// were removed.815+func (s *Service) SweepExpired(ctx context.Context, now int64) (int, error) {816+ s.quotaMu.Lock()817+ defer s.quotaMu.Unlock()818+ rows, err := s.store.DB().QueryContext(ctx, `SELECT DISTINCT sha256 FROM blob_claims c WHERE c.expires>0 AND c.expires<=?819+ AND NOT EXISTS(SELECT 1 FROM blob_claims live WHERE live.sha256=c.sha256 AND (live.expires=0 OR live.expires>?))`, now, now)820+ if err != nil {821+ return 0, fmt.Errorf("list expired blobs: %w", err)822+ }823+ var expired []string824+ for rows.Next() {825+ var sha string826+ if err := rows.Scan(&sha); err != nil {827+ _ = rows.Close()828+ return 0, err829+ }830+ expired = append(expired, sha)831+ }832+ if err := errors.Join(rows.Err(), rows.Close()); err != nil {833+ return 0, err834+ }835+ for _, sha := range expired {836+ if err := s.Delete(ctx, sha); err != nil {837+ return 0, err838+ }839+ }840+ if _, err := s.store.DB().ExecContext(ctx, "DELETE FROM blob_claims WHERE expires>0 AND expires<=?", now); err != nil {841+ return 0, fmt.Errorf("release expired blob claims: %w", err)842+ }843+ return len(expired), nil844+}845+756846 func (s *Service) currentLimits() Limits {757847 if s.config.Limits == nil {758848 return Limits{}@@ -1309,7 +1399,7 @@ func statusFor(err error) int {
13091399 if errors.Is(err, ErrQuotaExceeded) {13101400 return http.StatusForbidden13111401 }1312- if errors.Is(err, ErrBlocked) {1402+ if errors.Is(err, ErrBlocked) || errors.Is(err, ErrNotEncrypted) {13131403 return http.StatusForbidden13141404 }13151405 if strings.HasPrefix(err.Error(), "auth-required:") {@@ -291,6 +291,112 @@ func TestBlobClaimsPreserveDeduplicatedOwnershipAndCursor(t *testing.T) {
291291 }292292 }293293 294+// TestAgentUploadsExpireUnlessAPersonClaimsThem covers the ttl an agent295+// grant puts on uploads: claims carry the expiry, the sweep removes blobs296+// whose every claim lapsed and keeps those a person holds.297+func TestAgentUploadsExpireUnlessAPersonClaimsThem(t *testing.T) {298+ service := testService(t)299+ agent := strings.Repeat("a", 64)300+ person := strings.Repeat("b", 64)301+ now := time.Now().Unix()302+ service.config.UploadTerms = func(_ context.Context, uploader string) (UploadTerms, error) {303+ if uploader == agent {304+ return UploadTerms{ExpiresAt: now + 10}, nil305+ }306+ return UploadTerms{}, nil307+ }308+ ctx := context.Background()309+ shared, err := service.Put(ctx, PutOptions{Reader: strings.NewReader("shared site file"), Type: "text/html", Uploader: agent})310+ if err != nil {311+ t.Fatal(err)312+ }313+ alone, err := service.Put(ctx, PutOptions{Reader: strings.NewReader("agent only file"), Type: "text/html", Uploader: agent})314+ if err != nil {315+ t.Fatal(err)316+ }317+ var expires int64318+ if err := service.store.DB().QueryRow("SELECT expires FROM blob_claims WHERE sha256=? AND uploader=?", alone.SHA256, agent).Scan(&expires); err != nil || expires != now+10 {319+ t.Fatalf("agent claim expires=%d err=%v, want %d", expires, err, now+10)320+ }321+ if _, err := service.Put(ctx, PutOptions{Reader: strings.NewReader("shared site file"), Type: "text/html", Uploader: person}); err != nil {322+ t.Fatal(err)323+ }324+ if err := service.store.DB().QueryRow("SELECT expires FROM blob_claims WHERE sha256=? AND uploader=?", shared.SHA256, person).Scan(&expires); err != nil || expires != 0 {325+ t.Fatalf("person claim expires=%d err=%v, want 0", expires, err)326+ }327+ if removed, err := service.SweepExpired(ctx, now+9); err != nil || removed != 0 {328+ t.Fatalf("early sweep removed %d err=%v", removed, err)329+ }330+ if removed, err := service.SweepExpired(ctx, now+10); err != nil || removed != 1 {331+ t.Fatalf("sweep removed %d err=%v, want 1", removed, err)332+ }333+ if _, _, err := service.Get(ctx, alone.SHA256); err == nil {334+ t.Fatal("expired unclaimed blob still served")335+ }336+ if _, err := os.Stat(filepath.Join(service.root, alone.SHA256)); !errors.Is(err, os.ErrNotExist) {337+ t.Fatalf("expired blob file remains: %v", err)338+ }339+ if _, _, err := service.Get(ctx, shared.SHA256); err != nil {340+ t.Fatalf("blob a person claimed was removed: %v", err)341+ }342+ mine, err := service.ListMetadata(ctx, agent)343+ if err != nil || len(mine) != 0 {344+ t.Fatalf("agent still lists %d blobs after expiry: %v", len(mine), err)345+ }346+ theirs, err := service.ListMetadata(ctx, person)347+ if err != nil || len(theirs) != 1 {348+ t.Fatalf("person lists %d blobs: %v", len(theirs), err)349+ }350+ // A restart keeps the expiry column and the earlier rows.351+ if _, err := New(ctx, service.config); err != nil {352+ t.Fatal(err)353+ }354+}355+356+func TestEncryptedOnlyTermsRefusePlainUploads(t *testing.T) {357+ service := testService(t)358+ agent := strings.Repeat("a", 64)359+ service.config.UploadTerms = func(_ context.Context, uploader string) (UploadTerms, error) {360+ return UploadTerms{Encrypted: uploader == agent}, nil361+ }362+ ctx := context.Background()363+ png := "\x89PNG\r\n\x1a\n" + strings.Repeat("\x00", 64)364+ for name, options := range map[string]PutOptions{365+ "declared page": {Reader: strings.NewReader("<html>plain</html>"), Type: "text/html", Uploader: agent},366+ "sniffed image": {Reader: strings.NewReader(png), Type: "application/octet-stream", Uploader: agent},367+ "declared manifest": {Reader: strings.NewReader("{}"), Type: "application/json", Uploader: agent},368+ } {369+ if _, err := service.Put(ctx, options); !errors.Is(err, ErrNotEncrypted) {370+ t.Fatalf("%s: err=%v, want ErrNotEncrypted", name, err)371+ }372+ }373+ if entries, _ := os.ReadDir(service.root); len(entries) != 0 {374+ t.Fatalf("refused uploads left %d files", len(entries))375+ }376+ cipher := "\x8f\x1a\x9c\x03" + strings.Repeat("\xe2\x71\x05\x9b\x4c", 20)377+ if _, err := service.Put(ctx, PutOptions{Reader: strings.NewReader(cipher), Type: "application/octet-stream", Uploader: agent}); err != nil {378+ t.Fatalf("ciphertext refused: %v", err)379+ }380+ if _, err := service.Put(ctx, PutOptions{Reader: strings.NewReader("\x82\xa1t\x01"), Type: "application/vnd.blossom.directory+msgpack", Uploader: agent}); err != nil {381+ t.Fatalf("encrypted manifest refused: %v", err)382+ }383+ // Claiming a plain blob someone else stored is refused too.384+ if _, err := service.Put(ctx, PutOptions{Reader: strings.NewReader("<html>theirs</html>"), Type: "text/html", Uploader: strings.Repeat("b", 64)}); err != nil {385+ t.Fatal(err)386+ }387+ if _, err := service.Put(ctx, PutOptions{Reader: strings.NewReader("<html>theirs</html>"), Type: "text/html", Uploader: agent}); !errors.Is(err, ErrNotEncrypted) {388+ t.Fatalf("claim of a plain blob: err=%v", err)389+ }390+ // The door answers 403 with the reason.391+ service.config.Authorize = func(*http.Request, Action) (string, error) { return agent, nil }392+ req := httptest.NewRequest(http.MethodPut, "https://relay.test/upload", strings.NewReader("<html>plain</html>"))393+ req.Header.Set("Content-Type", "text/html")394+ response := record(service.Handler().ServeHTTP, req)395+ if response.Code != http.StatusForbidden || !strings.Contains(response.Body.String(), "requires encrypted uploads") {396+ t.Fatalf("plain upload = %d %s", response.Code, response.Body.String())397+ }398+}399+294400 const testUploader = "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"295401 296402 func testService(t *testing.T) *Service {