From ac189cc5507097f410f7bc7e85735b4a6e9c922d Mon Sep 17 00:00:00 2001 From: Cristian Magherusan-Stanciu Date: Tue, 13 Oct 2026 20:52:06 +0200 Subject: [PATCH 1/5] fix(gcp/computeengine): price CUDs per vCPU and GiB from the real SKU taxonomy Match core and RAM SKUs by family label, region and term instead of looking for the machine type in the description (never present in the live catalog), multiply by the recommendation's vCPU and GiB, follow every catalog page, and report commitment, on-demand and savings as consistent term totals. Refs #92 #245 Closes #271 Closes #321 Co-Authored-By: Claude Sonnet 5.5 --- .../gcp/services/computeengine/client.go | 151 ++------ .../gcp/services/computeengine/client_test.go | 232 +------------ .../gcp/services/computeengine/pricing.go | 324 ++++++++++++++++++ .../services/computeengine/pricing_test.go | 193 +++++++++++ .../services/computeengine/usage_type_test.go | 43 --- 5 files changed, 571 insertions(+), 372 deletions(-) create mode 100644 providers/gcp/services/computeengine/pricing.go create mode 100644 providers/gcp/services/computeengine/pricing_test.go delete mode 100644 providers/gcp/services/computeengine/usage_type_test.go diff --git a/providers/gcp/services/computeengine/client.go b/providers/gcp/services/computeengine/client.go index 7c3f6167..ab38e456 100644 --- a/providers/gcp/services/computeengine/client.go +++ b/providers/gcp/services/computeengine/client.go @@ -12,6 +12,7 @@ import ( "regexp" "strconv" "strings" + "sync" "time" compute "cloud.google.com/go/compute/apiv1" @@ -29,7 +30,6 @@ import ( "github.com/LeanerCloud/cloud-commitments-go/pkg/common" "github.com/LeanerCloud/cloud-commitments-go/pkg/retry" - "github.com/LeanerCloud/cloud-commitments-go/providers/gcp/internal/skumatch" ) // maxRecsPages caps GCP Recommender API iteration to avoid burning a Lambda @@ -267,6 +267,9 @@ type Client struct { machineTypesService MachineTypesService billingService BillingService recommenderClient RecommenderClient + + skuMu sync.Mutex + skuCache []*cloudbilling.Sku } // NewClient creates a new GCP Compute Engine client. @@ -292,6 +295,9 @@ func (c *Client) SetMachineTypesService(svc MachineTypesService) { // SetBillingService sets the billing service (for testing). func (c *Client) SetBillingService(svc BillingService) { c.billingService = svc + c.skuMu.Lock() + c.skuCache = nil + c.skuMu.Unlock() } // SetRecommenderClient sets the recommender client (for testing). @@ -335,11 +341,27 @@ func (r *realMachineTypesService) Close() error { // realBillingService wraps the real cloudbilling.APIService. type realBillingService struct { + ctx context.Context service *cloudbilling.APIService } +// ListSKUs returns every SKU of the service. The catalog is paged (5000 SKUs per +// page, 33k for Compute Engine), so all pages are followed. func (r *realBillingService) ListSKUs(serviceID string) (*cloudbilling.ListSkusResponse, error) { - return r.service.Services.Skus.List(serviceID).Do() + all := &cloudbilling.ListSkusResponse{} + pages := 0 + err := r.service.Services.Skus.List(serviceID).PageSize(skuPageSize).Pages(r.ctx, func(page *cloudbilling.ListSkusResponse) error { + pages++ + if pages > maxSKUPages { + return fmt.Errorf("computeengine: billing SKU listing page cap (%d pages) reached", maxSKUPages) + } + all.Skus = append(all.Skus, page.Skus...) + return nil + }) + if err != nil { + return nil, err + } + return all, nil } // realRecommenderIterator wraps the real recommender iterator. @@ -960,7 +982,11 @@ func (c *Client) GetOfferingDetails(ctx context.Context, rec common.Recommendati termYears = 3 } - pricing, err := c.getComputePricing(ctx, rec.ResourceType, c.region, termYears) + res, err := resourcesFromRecommendation(rec) + if err != nil { + return nil, fmt.Errorf("failed to get pricing: %w", err) + } + pricing, err := c.getComputePricing(ctx, rec.ResourceType, c.region, termYears, res) if err != nil { return nil, fmt.Errorf("failed to get pricing: %w", err) } @@ -1042,52 +1068,6 @@ func (c *Client) GetValidResourceTypes(ctx context.Context) ([]string, error) { return machineTypes, nil } -// ComputePricing contains pricing information for Compute Engine. -type ComputePricing struct { - HourlyRate float64 - CommitmentPrice float64 - OnDemandPrice float64 - Currency string - SavingsPercentage float64 -} - -// getComputePricing gets pricing from GCP Cloud Billing Catalog API. -// It returns an error when commitment pricing is absent from the catalog rather -// than fabricating a price from a hardcoded discount factor (issue #1020). -func (c *Client) getComputePricing(ctx context.Context, machineType, region string, termYears int) (*ComputePricing, error) { - svc, err := c.getOrCreateBillingService(ctx) - if err != nil { - return nil, err - } - - skus, err := svc.ListSKUs("services/6F81-5844-456A") - if err != nil { - return nil, fmt.Errorf("failed to list SKUs: %w", err) - } - - onDemandPrice, commitmentPrice, currency, err := extractComputePricingFromSKUs(skus.Skus, machineType, region) - if err != nil { - return nil, err - } - if onDemandPrice == 0 { - return nil, fmt.Errorf("no on-demand pricing found for machine type %s", machineType) - } - if commitmentPrice == 0 { - return nil, fmt.Errorf("no commitment pricing found for machine type %s in region %s: catalog has no CUD SKU; cannot compute savings percentage", machineType, region) - } - - hoursInTerm := 8760.0 * float64(termYears) - savingsPercentage := calculateComputeSavingsPercentage(onDemandPrice, hoursInTerm, commitmentPrice) - - return &ComputePricing{ - HourlyRate: commitmentPrice / hoursInTerm, - CommitmentPrice: commitmentPrice, - OnDemandPrice: onDemandPrice * hoursInTerm, - Currency: currency, - SavingsPercentage: savingsPercentage, - }, nil -} - // getOrCreateBillingService returns the billing service, creating it if needed. func (c *Client) getOrCreateBillingService(ctx context.Context) (BillingService, error) { if c.billingService != nil { @@ -1099,71 +1079,7 @@ func (c *Client) getOrCreateBillingService(ctx context.Context) (BillingService, return nil, fmt.Errorf("failed to create billing service: %w", err) } - return &realBillingService{service: service}, nil -} - -// extractComputePricingFromSKUs extracts on-demand and commitment pricing from SKU list. -func extractComputePricingFromSKUs(skus []*cloudbilling.Sku, machineType, region string) (onDemand, commitment float64, currency string, err error) { - currency = "USD" - - for _, sku := range skus { - if !skuMatchesMachineType(sku, machineType, region) { - continue - } - - price, curr := extractComputePriceFromSKU(sku) - if price == 0 { - continue - } - - if curr != "" { - currency = curr - } - - slot, slotErr := skumatch.Slot(sku) - if slotErr != nil { - return 0, 0, "", slotErr - } - switch slot { - case skumatch.SlotCommitment: - commitment = price - case skumatch.SlotOnDemand: - onDemand = price - } - } - - return onDemand, commitment, currency, nil -} - -// extractComputePriceFromSKU extracts the unit price from a SKU. -func extractComputePriceFromSKU(sku *cloudbilling.Sku) (price float64, currency string) { - if len(sku.PricingInfo) == 0 { - return 0, "" - } - - pricingInfo := sku.PricingInfo[0] - if pricingInfo.PricingExpression == nil || len(pricingInfo.PricingExpression.TieredRates) == 0 { - return 0, "" - } - - rate := pricingInfo.PricingExpression.TieredRates[0] - if rate.UnitPrice == nil { - return 0, "" - } - - price = float64(rate.UnitPrice.Units) + float64(rate.UnitPrice.Nanos)/1e9 - return price, rate.UnitPrice.CurrencyCode -} - -// calculateComputeSavingsPercentage calculates the savings percentage. -func calculateComputeSavingsPercentage(onDemandPrice, hoursInTerm, commitmentPrice float64) float64 { - onDemandTotal := onDemandPrice * hoursInTerm - return ((onDemandTotal - commitmentPrice) / onDemandTotal) * 100 -} - -// skuMatchesMachineType checks if a SKU matches the machine type and region. -func skuMatchesMachineType(sku *cloudbilling.Sku, machineType, region string) bool { - return skumatch.HasToken(sku.Description, machineType) && skumatch.InRegion(sku, region) + return &realBillingService{ctx: ctx, service: service}, nil } // convertGCPRecommendation converts a GCP Recommender recommendation to common format. @@ -1245,7 +1161,12 @@ func (c *Client) enrichRecWithPricing(ctx context.Context, rec *common.Recommend return } termYears := termYearsFromTerm(rec.Term) - pricing, err := c.getComputePricing(ctx, rec.ResourceType, c.region, termYears) + res, err := resourcesFromRecommendation(*rec) + if err != nil { + log.Printf("computeengine: pricing unavailable for %s in %s: %v", rec.ResourceType, c.region, err) + return + } + pricing, err := c.getComputePricing(ctx, rec.ResourceType, c.region, termYears, res) if err != nil { log.Printf("computeengine: pricing unavailable for %s in %s (issue #1020): %v", rec.ResourceType, c.region, err) return diff --git a/providers/gcp/services/computeengine/client_test.go b/providers/gcp/services/computeengine/client_test.go index 304274a2..e91ddc58 100644 --- a/providers/gcp/services/computeengine/client_test.go +++ b/providers/gcp/services/computeengine/client_test.go @@ -345,90 +345,6 @@ func TestComputeEngineClient_GetRegion(t *testing.T) { } } -func TestSkuMatchesMachineType(t *testing.T) { - tests := []struct { - name string - sku *cloudbilling.Sku - machineType string - region string - expected bool - }{ - { - name: "SKU matches machine type and region", - sku: &cloudbilling.Sku{ - Description: "n1-standard-1 VM running in Americas", - Category: &cloudbilling.Category{UsageType: "OnDemand"}, - ServiceRegions: []string{"us-central1"}, - }, - machineType: "n1-standard-1", - region: "us-central1", - expected: true, - }, - { - name: "SKU matches machine type but not region", - sku: &cloudbilling.Sku{ - Description: "n1-standard-1 VM running in Europe", - Category: &cloudbilling.Category{UsageType: "OnDemand"}, - ServiceRegions: []string{"europe-west1"}, - }, - machineType: "n1-standard-1", - region: "us-central1", - expected: false, - }, - { - name: "SKU does not match machine type", - sku: &cloudbilling.Sku{ - Description: "n2-highmem-4 VM running in Americas", - Category: &cloudbilling.Category{UsageType: "OnDemand"}, - ServiceRegions: []string{"us-central1"}, - }, - machineType: "n1-standard-1", - region: "us-central1", - expected: false, - }, - { - name: "SKU with nil service regions matches no region", - sku: &cloudbilling.Sku{ - Description: "n1-standard-1 VM", - Category: &cloudbilling.Category{UsageType: "OnDemand"}, - ServiceRegions: nil, - }, - machineType: "n1-standard-1", - region: "us-central1", - expected: false, - }, - { - name: "SKU for a longer machine type that starts with the same text is a decoy", - sku: &cloudbilling.Sku{ - Description: "n1-standard-16 VM running in Americas", - Category: &cloudbilling.Category{UsageType: "OnDemand"}, - ServiceRegions: []string{"us-central1"}, - }, - machineType: "n1-standard-1", - region: "us-central1", - expected: false, - }, - { - name: "Case insensitive machine type match", - sku: &cloudbilling.Sku{ - Description: "N1-STANDARD-1 VM running in Americas", - Category: &cloudbilling.Category{UsageType: "OnDemand"}, - ServiceRegions: []string{"us-central1"}, - }, - machineType: "n1-standard-1", - region: "us-central1", - expected: true, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - result := skuMatchesMachineType(tt.sku, tt.machineType, tt.region) - assert.Equal(t, tt.expected, result) - }) - } -} - func TestComputePricingStructure(t *testing.T) { pricing := ComputePricing{ HourlyRate: 0.10, @@ -1076,69 +992,24 @@ func TestComputeEngineClient_PurchaseCommitment_WaitError(t *testing.T) { func TestComputeEngineClient_GetOfferingDetails_WithMock(t *testing.T) { ctx := context.Background() client, _ := NewClient(ctx, "test-project", "us-central1") - - // Both on-demand and commitment SKUs are required; without a commitment SKU - // getComputePricing now returns an error (issue #1020 fix). - mockService := &MockBillingService{ - skus: &cloudbilling.ListSkusResponse{ - Skus: []*cloudbilling.Sku{ - { - Description: "n1-standard-1 VM running in Americas", - Category: &cloudbilling.Category{UsageType: "OnDemand"}, - ServiceRegions: []string{"us-central1"}, - PricingInfo: []*cloudbilling.PricingInfo{ - { - PricingExpression: &cloudbilling.PricingExpression{ - TieredRates: []*cloudbilling.TierRate{ - { - UnitPrice: &cloudbilling.Money{ - Units: 0, - Nanos: 50000000, - CurrencyCode: "USD", - }, - }, - }, - }, - }, - }, - }, - { - Description: "n1-standard-1 commitment 1yr in Americas", - Category: &cloudbilling.Category{UsageType: "Commit1Yr"}, - ServiceRegions: []string{"us-central1"}, - PricingInfo: []*cloudbilling.PricingInfo{ - { - PricingExpression: &cloudbilling.PricingExpression{ - TieredRates: []*cloudbilling.TierRate{ - { - UnitPrice: &cloudbilling.Money{ - Units: 0, - Nanos: 32000000, - CurrencyCode: "USD", - }, - }, - }, - }, - }, - }, - }, - }, - }, - } - client.SetBillingService(mockService) + client.SetBillingService(mockBillingWithCommitment()) rec := common.Recommendation{ - ResourceType: "n1-standard-1", + ResourceType: "n1-standard-4", Term: "1yr", - PaymentOption: "upfront", + PaymentOption: "monthly", + Count: 4, + Details: common.ComputeDetails{MemoryGB: 15}, } details, err := client.GetOfferingDetails(ctx, rec) require.NoError(t, err) - assert.Equal(t, "n1-standard-1", details.ResourceType) + assert.Equal(t, "n1-standard-4", details.ResourceType) assert.Equal(t, "1yr", details.Term) assert.Equal(t, "USD", details.Currency) - assert.Greater(t, details.TotalCost, float64(0)) + // (4 vCPU * 0.019915 + 15 GiB * 0.002669) per hour over 8760 h. + assert.InDelta(t, (4*0.019915+15*0.002669)*8760, details.TotalCost, 1e-6) + assert.InDelta(t, details.TotalCost/12, details.RecurringCost, 1e-6) } func TestComputeEngineClient_GetOfferingDetails_NoPricing(t *testing.T) { @@ -1152,11 +1023,11 @@ func TestComputeEngineClient_GetOfferingDetails_NoPricing(t *testing.T) { } client.SetBillingService(mockService) - rec := common.Recommendation{ResourceType: "n1-standard-1"} + rec := common.Recommendation{ResourceType: "n1-standard-1", Count: 1, Details: common.ComputeDetails{MemoryGB: 3.75}} _, err := client.GetOfferingDetails(ctx, rec) assert.Error(t, err) - assert.Contains(t, err.Error(), "no on-demand pricing found") + assert.Contains(t, err.Error(), "no on-demand core and RAM pricing found") } func TestComputeEngineClient_GetRecommendations_WithMock(t *testing.T) { @@ -1392,55 +1263,11 @@ func realisticCUDRecommendation() *recommenderpb.Recommendation { } } -// mockBillingWithCommitment returns a MockBillingService that has both an -// on-demand SKU and a commitment SKU for n1-standard machines in us-central1. +// mockBillingWithCommitment returns the live-catalog-shaped N1 core and RAM SKUs +// (on demand and 1-year commitment, us-central1) that the n1-standard-4 fixtures +// are priced against. func mockBillingWithCommitment() *MockBillingService { - return &MockBillingService{ - skus: &cloudbilling.ListSkusResponse{ - Skus: []*cloudbilling.Sku{ - { - Description: "n1-standard-4 VM running in Americas", - Category: &cloudbilling.Category{UsageType: "OnDemand"}, - ServiceRegions: []string{"us-central1"}, - PricingInfo: []*cloudbilling.PricingInfo{ - { - PricingExpression: &cloudbilling.PricingExpression{ - TieredRates: []*cloudbilling.TierRate{ - { - UnitPrice: &cloudbilling.Money{ - Units: 0, - Nanos: 190000000, - CurrencyCode: "USD", - }, - }, - }, - }, - }, - }, - }, - { - Description: "n1-standard-4 commitment 1yr in Americas", - Category: &cloudbilling.Category{UsageType: "Commit1Yr"}, - ServiceRegions: []string{"us-central1"}, - PricingInfo: []*cloudbilling.PricingInfo{ - { - PricingExpression: &cloudbilling.PricingExpression{ - TieredRates: []*cloudbilling.TierRate{ - { - UnitPrice: &cloudbilling.Money{ - Units: 0, - Nanos: 120000000, - CurrencyCode: "USD", - }, - }, - }, - }, - }, - }, - }, - }, - }, - } + return &MockBillingService{skus: &cloudbilling.ListSkusResponse{Skus: n1Americas1YrSKUs()}} } // TestConverterToInsert_CountNonZero_VCPUAmountSet is the primary regression test @@ -1567,35 +1394,12 @@ func TestGetComputePricing_NoCommitmentSKUReturnsError(t *testing.T) { ctx := context.Background() client, _ := NewClient(ctx, "test-project", "us-central1") - // Only on-demand SKU -- no commitment SKU. + // Only the on-demand SKUs -- no commitment SKU. client.SetBillingService(&MockBillingService{ - skus: &cloudbilling.ListSkusResponse{ - Skus: []*cloudbilling.Sku{ - { - Description: "n1-standard-4 VM running in Americas", - Category: &cloudbilling.Category{UsageType: "OnDemand"}, - ServiceRegions: []string{"us-central1"}, - PricingInfo: []*cloudbilling.PricingInfo{ - { - PricingExpression: &cloudbilling.PricingExpression{ - TieredRates: []*cloudbilling.TierRate{ - { - UnitPrice: &cloudbilling.Money{ - Units: 0, - Nanos: 190000000, - CurrencyCode: "USD", - }, - }, - }, - }, - }, - }, - }, - }, - }, + skus: &cloudbilling.ListSkusResponse{Skus: n1Americas1YrSKUs()[:2]}, }) - pricing, err := client.getComputePricing(ctx, "n1-standard-4", "us-central1", 1) + pricing, err := client.getComputePricing(ctx, "n1-standard-4", "us-central1", 1, computeResources{vcpus: 4, memoryGiB: 15}) require.Error(t, err, "getComputePricing must return an error when no commitment SKU exists (issue #1020)") assert.Contains(t, err.Error(), "no commitment pricing found") assert.Nil(t, pricing, "no pricing struct must be returned when commitment SKU is missing") diff --git a/providers/gcp/services/computeengine/pricing.go b/providers/gcp/services/computeengine/pricing.go new file mode 100644 index 00000000..a6cc14a4 --- /dev/null +++ b/providers/gcp/services/computeengine/pricing.go @@ -0,0 +1,324 @@ +package computeengine + +import ( + "context" + "fmt" + "regexp" + "strings" + + "google.golang.org/api/cloudbilling/v1" + + "github.com/LeanerCloud/cloud-commitments-go/pkg/common" + "github.com/LeanerCloud/cloud-commitments-go/providers/gcp/internal/skumatch" +) + +const ( + // computeServiceID is the Cloud Billing catalog service for Compute Engine. + computeServiceID = "services/6F81-5844-456A" + + // skuPageSize is the largest page the catalog serves. + skuPageSize = 5000 + + // maxSKUPages caps catalog paging; Compute Engine is about 7 pages. + maxSKUPages = 50 + + hoursPerYear = 8760.0 + + usageUnitHour = "h" + usageUnitGiBHour = "GiBy.h" + + usageTypeCommit1Yr = "Commit1Yr" + usageTypeCommit3Yr = "Commit3Yr" +) + +// ComputePricing is the price of a commitment's resources, for one term. +type ComputePricing struct { + // HourlyRate is the committed price per hour for the whole resource set. + HourlyRate float64 + // CommitmentPrice is the committed price over the whole term. + CommitmentPrice float64 + // OnDemandPrice is the on-demand price of the same resources over the term. + OnDemandPrice float64 + Currency string + SavingsPercentage float64 +} + +// computeResources is the amount a commitment covers. +type computeResources struct { + vcpus int + memoryGiB float64 +} + +// resourcesFromRecommendation reads the vCPU count and memory a recommendation +// covers; both are required to price it. +func resourcesFromRecommendation(rec common.Recommendation) (computeResources, error) { + if rec.Count <= 0 { + return computeResources{}, fmt.Errorf("recommendation for %s has no vCPU count; cannot price it", rec.ResourceType) + } + memMB, err := memoryMBFromDetails(rec) + if err != nil { + return computeResources{}, err + } + return computeResources{vcpus: rec.Count, memoryGiB: float64(memMB) / 1024.0}, nil +} + +// skuFamilyLabels holds how the catalog names a machine family. The labels are +// not derivable from the machine type prefix for every family. +type skuFamilyLabels struct { + onDemand string // "