Add stable manifest limits for final emitted feature and geometry-position counts, enforce them after fusion and clipping but before native compilation, and reject rather than truncate. Covers exact boundaries, checked overflow, multi-page and multi-source fusion, and identical online, offline, and exact-replay refusal.
- commit
- 2df65e928197171311bc4d8e281db233c10bea80
- parent
- 871c5962fa6a8eece5a7819087c53cf00980ba9d
- author
- Dami <dami.etoile@gmail.com>
| M | docs/ | 13 | +++- |
| M | internal/ | 3 | + |
| M | internal/ | 32 | +++++++---- |
| M | internal/ | 17 | +++++- |
| M | internal/ | 6 | ++ |
| A | internal/ | 62 | +++++++++++++++++++++ |
| A | internal/ | 88 | ++++++++++++++++++++++++++++++ |
@@ -61,10 +61,15 @@ Their defaults are the application-owned cache and library.
6161 reports validated cache presence, estimated bytes, and obligations; identical6262 acquisitions may be shared across projects without merging their semantics.6363 - Every plan resolves stable refusal budgets for request bytes, total evidence,64- request count, raster tile count, and raster pixel count. The manifest may65- state smaller or deliberately larger positive limits. Planning rejects tile66- explosions before enumeration; capture rejects at `limit + 1`; offline and67- exact replay apply the same limits. Atlas never truncates to fit a budget.64+ request count, raster tile count, raster pixel count, emitted feature count,65+ and emitted geometry positions. The `features` default is 2,000,000 fused66+ features and the `geometry-positions` default is 50,000,000 positions; the67+ manifest may state smaller or deliberately larger positive limits. Planning68+ rejects tile explosions before enumeration; capture rejects at `limit + 1`;69+ semantic assembly counts the70+ final fused output with checked arithmetic before native compilation. Online,71+ offline, and exact replay apply the same limits. Atlas never truncates to fit72+ a budget.6873 - Capture envelopes retain request identity, body hash, media type, timestamp,6974 licence, and attribution. Changed bodies append history instead of replacing7075 evidence. Cache reads verify the envelope, body length, and body SHA-256;@@ -102,6 +102,9 @@ func Build(ctx context.Context, options BuildOptions) (BuildResult, error) {
102102 if err != nil {103103 return result, err104104 }105+ if err := enforceSemanticBudgets(volume, plan.Budgets); err != nil {106+ return result, err107+ }105108 if err := attachAssets(project, plan, captures, cache, &volume); err != nil {106109 return result, err107110 }@@ -238,23 +238,27 @@ type Release struct {
238238 // compiler default; a manifest can choose a smaller or deliberately larger239239 // positive value, but evidence is never silently truncated.240240 type BuildBudgets struct {241- RequestBytes int64 `yaml:"request-bytes,omitempty" json:"requestBytes"`242- TotalBytes int64 `yaml:"total-bytes,omitempty" json:"totalBytes"`243- Requests int `yaml:"requests,omitempty" json:"requests"`244- RasterTiles int64 `yaml:"raster-tiles,omitempty" json:"rasterTiles"`245- RasterPixels int64 `yaml:"raster-pixels,omitempty" json:"rasterPixels"`241+ RequestBytes int64 `yaml:"request-bytes,omitempty" json:"requestBytes"`242+ TotalBytes int64 `yaml:"total-bytes,omitempty" json:"totalBytes"`243+ Requests int `yaml:"requests,omitempty" json:"requests"`244+ RasterTiles int64 `yaml:"raster-tiles,omitempty" json:"rasterTiles"`245+ RasterPixels int64 `yaml:"raster-pixels,omitempty" json:"rasterPixels"`246+ Features int64 `yaml:"features,omitempty" json:"features"`247+ GeometryPositions int64 `yaml:"geometry-positions,omitempty" json:"geometryPositions"`246248 }247249 248250 var defaultBuildBudgets = BuildBudgets{249- RequestBytes: 512 << 20,250- TotalBytes: 8 << 30,251- Requests: 100_000,252- RasterTiles: 90_000,253- RasterPixels: 16_000_000_000,251+ RequestBytes: 512 << 20,252+ TotalBytes: 8 << 30,253+ Requests: 100_000,254+ RasterTiles: 90_000,255+ RasterPixels: 16_000_000_000,256+ Features: 2_000_000,257+ GeometryPositions: 50_000_000,254258 }255259 256260 func (budgets BuildBudgets) resolved() (BuildBudgets, error) {257- if budgets.RequestBytes < 0 || budgets.TotalBytes < 0 || budgets.Requests < 0 || budgets.RasterTiles < 0 || budgets.RasterPixels < 0 {261+ if budgets.RequestBytes < 0 || budgets.TotalBytes < 0 || budgets.Requests < 0 || budgets.RasterTiles < 0 || budgets.RasterPixels < 0 || budgets.Features < 0 || budgets.GeometryPositions < 0 {258262 return BuildBudgets{}, fmt.Errorf("build budgets must be positive")259263 }260264 resolved := budgets@@ -273,6 +277,12 @@ func (budgets BuildBudgets) resolved() (BuildBudgets, error) {
273277 if resolved.RasterPixels == 0 {274278 resolved.RasterPixels = defaultBuildBudgets.RasterPixels275279 }280+ if resolved.Features == 0 {281+ resolved.Features = defaultBuildBudgets.Features282+ }283+ if resolved.GeometryPositions == 0 {284+ resolved.GeometryPositions = defaultBuildBudgets.GeometryPositions285+ }276286 if resolved.RequestBytes > resolved.TotalBytes {277287 return BuildBudgets{}, fmt.Errorf("request byte budget %d exceeds total byte budget %d", resolved.RequestBytes, resolved.TotalBytes)278288 }@@ -36,6 +36,7 @@ func TestMultiPageBuildIsByteIdenticalOnlineOfflineAndExactReplay(t *testing.T)
3636 updated := strings.Replace(string(manifest), "adapter: geojson", "adapter: ogc-api-features", 1)3737 updated = strings.Replace(updated, "locator: sample-region.geojson", "locator: "+server.URL, 1)3838 updated = strings.Replace(updated, " mapping:\n feature-set: places", " query:\n collection: sample-points\n limit: 1\n mapping:\n feature-set: places", 1)39+ updated = strings.Replace(updated, "presentation:\n", "budgets:\n features: 4\n geometry-positions: 11\n\npresentation:\n", 1)3940 if err := os.WriteFile(projectPath, []byte(updated), 0o644); err != nil {4041 t.Fatal(err)4142 }@@ -75,8 +76,6 @@ func TestMultiPageBuildIsByteIdenticalOnlineOfflineAndExactReplay(t *testing.T)
7576 if !allCaptureEventsCached(repeatedEvents, true) {7677 t.Fatalf("repeated online build cache events = %+v", repeatedEvents)7778 }78- server.Close()79-8079 offline, err := Build(context.Background(), BuildOptions{ProjectPath: projectPath, CacheDir: cacheDir, LibraryDir: filepath.Join(t.TempDir(), "offline"), Offline: true})8180 if err != nil {8281 t.Fatal(err)@@ -101,6 +100,20 @@ func TestMultiPageBuildIsByteIdenticalOnlineOfflineAndExactReplay(t *testing.T)
101100 t.Fatalf("build %s is not byte-identical to online build", path)102101 }103102 }103+ tooSmall := strings.Replace(updated, "features: 4", "features: 3", 1)104+ if err := os.WriteFile(projectPath, []byte(tooSmall), 0o644); err != nil {105+ t.Fatal(err)106+ }107+ for _, options := range []BuildOptions{108+ {ProjectPath: projectPath, CacheDir: cacheDir, LibraryDir: filepath.Join(t.TempDir(), "online-over")},109+ {ProjectPath: projectPath, CacheDir: cacheDir, LibraryDir: filepath.Join(t.TempDir(), "offline-over"), Offline: true},110+ {ProjectPath: projectPath, CacheDir: cacheDir, LibraryDir: filepath.Join(t.TempDir(), "replay-over"), ReplayPath: online.Path},111+ } {112+ if _, err := Build(context.Background(), options); err == nil || !strings.Contains(err.Error(), "emits 4 features, budget is 3") {113+ t.Fatalf("semantic budget mode error = %v", err)114+ }115+ }116+ server.Close()104117 }105118 106119 func allCaptureEventsCached(events []Event, want bool) bool {@@ -103,4 +103,10 @@ func TestBuildBudgetsResolveStableDefaultsAndRejectInversion(t *testing.T) {
103103 if _, err := (BuildBudgets{RequestBytes: 2, TotalBytes: 1}).resolved(); err == nil {104104 t.Fatal("request budget larger than total was accepted")105105 }106+ if _, err := (BuildBudgets{Features: -1}).resolved(); err == nil {107+ t.Fatal("negative feature budget was accepted")108+ }109+ if _, err := (BuildBudgets{GeometryPositions: -1}).resolved(); err == nil {110+ t.Fatal("negative geometry-position budget was accepted")111+ }106112 }@@ -0,0 +1,62 @@
1+package authoring2+3+import (4+ "fmt"5+ "math"6+7+ "github.com/FelineStateMachine/atlas/format/vnext"8+)9+10+type semanticCounts struct {11+ features int6412+ positions int6413+}14+15+func enforceSemanticBudgets(volume vnext.Volume, budgets BuildBudgets) error {16+ counts, err := countSemantics(volume)17+ if err != nil {18+ return err19+ }20+ if counts.features > budgets.Features {21+ return fmt.Errorf("semantic output emits %d features, budget is %d", counts.features, budgets.Features)22+ }23+ if counts.positions > budgets.GeometryPositions {24+ return fmt.Errorf("semantic output emits %d geometry positions, budget is %d", counts.positions, budgets.GeometryPositions)25+ }26+ return nil27+}28+29+func countSemantics(volume vnext.Volume) (semanticCounts, error) {30+ var counts semanticCounts31+ for _, world := range volume.Worlds {32+ for _, set := range world.FeatureSets {33+ var err error34+ counts.features, err = checkedSemanticAdd(counts.features, len(set.Features))35+ if err != nil {36+ return semanticCounts{}, fmt.Errorf("count emitted features: %w", err)37+ }38+ for _, feature := range set.Features {39+ for _, part := range feature.Geometry.Parts {40+ for _, ring := range part.Rings {41+ counts.positions, err = checkedSemanticAdd(counts.positions, len(ring))42+ if err != nil {43+ return semanticCounts{}, fmt.Errorf("count emitted geometry positions: %w", err)44+ }45+ }46+ }47+ }48+ }49+ }50+ return counts, nil51+}52+53+func checkedSemanticAdd(total int64, count int) (int64, error) {54+ if count < 0 || uint64(count) > math.MaxInt64 {55+ return 0, fmt.Errorf("count overflows int64")56+ }57+ increment := int64(count)58+ if total > math.MaxInt64-increment {59+ return 0, fmt.Errorf("total overflows int64")60+ }61+ return total + increment, nil62+}@@ -0,0 +1,88 @@
1+package authoring2+3+import (4+ "math"5+ "strings"6+ "testing"7+8+ "github.com/FelineStateMachine/atlas/format/vnext"9+)10+11+func TestSemanticBudgetsAcceptExactEmittedBoundary(t *testing.T) {12+ volume := semanticBudgetVolume(13+ vnext.Geometry{Kind: vnext.GeometryPoint, Parts: []vnext.GeometryPart{{Rings: [][]vnext.Position{{{1, 1}}}}}},14+ vnext.Geometry{Kind: vnext.GeometryLineString, Parts: []vnext.GeometryPart{{Rings: [][]vnext.Position{{{0, 0}, {1, 1}, {2, 2}}}}}},15+ )16+ budgets := defaultBuildBudgets17+ budgets.Features = 218+ budgets.GeometryPositions = 419+20+ if err := enforceSemanticBudgets(volume, budgets); err != nil {21+ t.Fatalf("exact semantic budget boundary: %v", err)22+ }23+}24+25+func TestSemanticBudgetsRejectWithoutTruncating(t *testing.T) {26+ volume := semanticBudgetVolume(27+ vnext.Geometry{Kind: vnext.GeometryPoint, Parts: []vnext.GeometryPart{{Rings: [][]vnext.Position{{{1, 1}}}}}},28+ vnext.Geometry{Kind: vnext.GeometryLineString, Parts: []vnext.GeometryPart{{Rings: [][]vnext.Position{{{0, 0}, {1, 1}, {2, 2}}}}}},29+ )30+ tests := []struct {31+ name string32+ budget BuildBudgets33+ message string34+ }{35+ {name: "features", budget: BuildBudgets{Features: 1, GeometryPositions: 4}, message: "emits 2 features, budget is 1"},36+ {name: "positions", budget: BuildBudgets{Features: 2, GeometryPositions: 3}, message: "emits 4 geometry positions, budget is 3"},37+ }38+ for _, test := range tests {39+ t.Run(test.name, func(t *testing.T) {40+ if err := enforceSemanticBudgets(volume, test.budget); err == nil || !strings.Contains(err.Error(), test.message) {41+ t.Fatalf("semantic budget error = %v, want %q", err, test.message)42+ }43+ })44+ }45+ if got := len(volume.Worlds[0].FeatureSets[0].Features); got != 2 {46+ t.Fatalf("refusal truncated volume to %d features", got)47+ }48+}49+50+func TestSemanticBudgetsCountFusedOutputRatherThanObservations(t *testing.T) {51+ project := contractTestProject()52+ geometry := vnext.Geometry{Kind: vnext.GeometryPoint, Parts: []vnext.GeometryPart{{Rings: [][]vnext.Position{{{1, 1}}}}}}53+ observations := []observation{54+ {SourceOrder: 0, Source: "page-one", NativeID: "one", FeatureSet: "places", Title: "One", Geometry: geometry, Values: map[string]vnext.Value{"kind": vnext.StringValue("sample")}},55+ {SourceOrder: 0, Source: "page-two", NativeID: "two", FeatureSet: "places", Title: "Two", Geometry: geometry, Values: map[string]vnext.Value{"kind": vnext.StringValue("sample")}},56+ {SourceOrder: 1, Source: "secondary", NativeID: "one", FeatureSet: "places", Title: "One", Geometry: geometry, Values: map[string]vnext.Value{"kind": vnext.StringValue("alternate")}},57+ }58+ volume, err := assemble(project, observations)59+ if err != nil {60+ t.Fatal(err)61+ }62+ budgets := defaultBuildBudgets63+ budgets.Features = 264+ budgets.GeometryPositions = 265+ if err := enforceSemanticBudgets(volume, budgets); err != nil {66+ t.Fatalf("fused output at exact boundary: %v", err)67+ }68+ if got := len(volume.Worlds[0].FeatureSets[0].Features[0].Provenance); got != 2 {69+ t.Fatalf("fused feature provenance = %d, want both sources", got)70+ }71+}72+73+func TestSemanticBudgetArithmeticRefusesOverflow(t *testing.T) {74+ if got, err := checkedSemanticAdd(math.MaxInt64-1, 1); err != nil || got != math.MaxInt64 {75+ t.Fatalf("checked boundary = %d, %v", got, err)76+ }77+ if _, err := checkedSemanticAdd(math.MaxInt64-1, 2); err == nil || !strings.Contains(err.Error(), "overflows") {78+ t.Fatalf("checked overflow = %v", err)79+ }80+}81+82+func semanticBudgetVolume(geometries ...vnext.Geometry) vnext.Volume {83+ features := make([]vnext.Feature, 0, len(geometries))84+ for index, geometry := range geometries {85+ features = append(features, vnext.Feature{ID: string(rune('a' + index)), Geometry: geometry})86+ }87+ return vnext.Volume{Worlds: []vnext.World{{FeatureSets: []vnext.FeatureSet{{Features: features}}}}}88+}