feat(ladder): convert existing coverage at observed commitment rates

internal/server/handler_ladder.go

@@ -285,8 +285,10 @@ func (app *Application) processOneLadderConfig(
 // below): a safe under-buy, and a deliberate deviation from go's InFlight doc.
 //
 // Units: the provider's existing and expiring coverage arrive in commitment
-// dollars and are first converted to on-demand-equivalent dollars with the
-// same rates (ladderLayerStatesOnDemand). Coverage on a layer with no rate
+// dollars and are first converted to on-demand-equivalent dollars at the
+// observed rate of the commitments held when the provider reports one, else at
+// the new-purchase rate (ladderConversionRates, ladderLayerStatesOnDemand).
+// New tranches are always sized at the new-purchase rate. Coverage on a layer with no rate
 // cannot be converted, so the gap cannot be sized: the run is persisted as a
 // Hold-only run with reason_code missing_discount_rate that buys nothing
 // (planMissingRateHold), not an Errored run. Allocate sizes the gap in
@@ -331,7 +333,8 @@ func (app *Application) executeLadderRun(
 
 	// Rates are provider data, fetched outside the transaction. No capability or
 	// a fetch error fails the run: the allocator never gets a guessed rate. An
-	// empty map is valid and yields a Hold-only run.
+	// empty map is valid and yields a Hold-only run. Observed rates are
+	// optional: a capability without them converts at the marginal rates.
 	rater, ok := capability.(pkgladder.CommitmentRateCapability)
 	if !ok {
 		return fmt.Errorf("capability %T does not provide commitment rates", capability)
@@ -343,9 +346,16 @@ func (app *Application) executeLadderRun(
 
 	// Providers report existing coverage in commitment dollars; Allocate and
 	// the plan snapshot compare it with an on-demand-equivalent target (#946).
-	// Layers holding coverage with no rate come back in missing; the run is
-	// then a Hold that buys nothing and says why.
-	layerStates, missing, err := ladderLayerStatesOnDemand(layerStates, rates)
+	// Existing coverage converts at observed rates where reported, else at the
+	// marginal rates; the planner below keeps the marginal rates for new
+	// tranches. Layers holding coverage with no rate at all come back in
+	// missing; the run is then a Hold that buys nothing and says why.
+	conv, observed, err := ladderConversionRates(ctx, capability, scope, rates)
+	if err != nil {
+		return fmt.Errorf("observed commitment rates: %w", err)
+	}
+	logObservedOverrides(dbCfg.ID, observed, rates)
+	layerStates, missing, err := ladderLayerStatesOnDemand(layerStates, conv)
 	if err != nil {
 		return fmt.Errorf("existing coverage: %w", err)
 	}
@@ -414,7 +424,7 @@ func (p *ladderRunPlanner) reasonCode() string {
 // not a failure, and the run counts as planned.
 func (p *ladderRunPlanner) logPersisted(run *config.LadderRunDB) {
 	if len(p.missing) > 0 {
-		logging.Warnf("ladder_run: config %s: persisted Hold-only run %s (reason_code=%s): %d layer(s) hold existing coverage with no discount rate; nothing bought",
+		logging.Warnf("ladder_run: config %s: persisted Hold-only run %s (reason_code=%s): %d layer(s) hold existing coverage with no observed or new-purchase rate; nothing bought",
 			p.dbCfg.ID, run.ID, ladderHoldMissingDiscountRate, len(p.missing))
 		return
 	}

internal/server/ladder_existing_units.go

@@ -1,7 +1,9 @@
 package server
 
 import (
+	"context"
 	"fmt"
+	"log"
 	"maps"
 	"math"
 	"math/big"
@@ -25,11 +27,11 @@ import (
 //
 //	od = commitment / r
 //
-// so 5 commitment $/hr at r = 0.5 covers 10 OD $/hr. The rate is today's
-// new-purchase rate for the config's term and payment option, from
-// GetCommitmentRates (option 2 on #946): DescribeSavingsPlans does not expose
-// each plan's own rate, so plans bought at a different discount are
-// approximated.
+// so 5 commitment $/hr at r = 0.5 covers 10 OD $/hr. The rate is the observed
+// rate of the commitments held, from GetObservedCommitmentRates, when the
+// provider reports one for the layer; otherwise today's new-purchase rate for
+// the config's term and payment option, from GetCommitmentRates, which
+// approximates plans bought at a different discount (ladderConversionRates).
 //
 // Rounding favors under-buying. When the exact quotient is not a float64,
 // existing is rounded up and expiring down, so the net existing coverage
@@ -49,6 +51,47 @@ type ladderMissingRate struct {
 	existing, expiring *float64
 }
 
+// ladderConversionRates returns the rates used to convert existing coverage
+// to on-demand-equivalent dollars: the marginal rates, overridden per layer by
+// the observed rates when the capability reports them. observed is the set of
+// overrides (nil when the capability has none). An observed-rates error fails
+// the run; a missing observed rate for a layer falls back to its marginal rate.
+func ladderConversionRates(
+	ctx context.Context,
+	capability pkgladder.LadderCapability,
+	scope pkgladder.Scope,
+	marginal map[pkgladder.LayerType]float64,
+) (conv, observed map[pkgladder.LayerType]float64, err error) {
+	conv = make(map[pkgladder.LayerType]float64, len(marginal))
+	for k, v := range marginal {
+		conv[k] = v
+	}
+	oc, ok := capability.(pkgladder.ObservedCommitmentRateCapability)
+	if !ok {
+		return conv, nil, nil
+	}
+	observed, err = oc.GetObservedCommitmentRates(ctx, scope)
+	if err != nil {
+		return nil, nil, err
+	}
+	for k, v := range observed {
+		conv[k] = v
+	}
+	return conv, observed, nil
+}
+
+// logObservedOverrides records, per layer, the observed rate that replaced the
+// marginal one. Nothing else persists the conversion rate.
+func logObservedOverrides(cfgID string, observed, marginal map[pkgladder.LayerType]float64) {
+	for _, layer := range slices.Sorted(maps.Keys(observed)) {
+		if m, ok := marginal[layer]; ok {
+			log.Printf("ladder_run: config %s: layer %s converts existing coverage at observed rate %v (marginal %v)", cfgID, layer, observed[layer], m)
+			continue
+		}
+		log.Printf("ladder_run: config %s: layer %s converts existing coverage at observed rate %v (marginal: none)", cfgID, layer, observed[layer])
+	}
+}
+
 // ladderLayerStatesOnDemand returns a copy of states with ExistingUSDPerHour
 // and ExpiringUSDPerHour converted from commitment dollars to
 // on-demand-equivalent dollars by each layer's rate. The input map and its
@@ -205,7 +248,7 @@ func (p *ladderRunPlanner) planMissingRateHold() (*config.LadderRunDB, []config.
 // as "covered", so it is not mistaken for a target-met Hold.
 func (m ladderMissingRate) rationale(term pkgladder.Term, payment pkgladder.PaymentOption) string {
 	return fmt.Sprintf("Can't size this yet because layer %s holds existing coverage (%s commitment USD/hr, %s expiring) "+
-		"and the provider returned no discount rate for it at %s %s, so that coverage cannot be converted to on-demand-equivalent dollars. "+
+		"and has no rate: neither an observed rate from the commitments held nor a new-purchase rate at %s %s, so that coverage cannot be converted to on-demand-equivalent dollars. "+
 		"Nothing is bought this run: the rate is never estimated or assumed 1:1. The ladder sizes the gap once a rate is available.",
 		m.layer, fmtFloatPtr(m.existing), fmtFloatPtr(m.expiring), term, payment)
 }

internal/server/ladder_missing_rate_hold_test.go

@@ -86,7 +86,7 @@ func TestExecuteLadderRun_MissingDiscountRatePersistsHoldWithReason(t *testing.T
 			assert.Equal(t, string(missing), hold.Layer)
 			assert.Regexp(t, `^Can't size this yet because layer `+string(missing)+` `, hold.Rationale)
 			assert.Contains(t, hold.Rationale, "2.5 commitment USD/hr, 0.5 expiring")
-			assert.Contains(t, hold.Rationale, "no discount rate")
+			assert.Contains(t, hold.Rationale, "neither an observed rate from the commitments held nor a new-purchase rate")
 			assert.Contains(t, hold.Rationale, "1yr no-upfront")
 			assert.NotContains(t, hold.Rationale, "already met")
 		})

internal/server/ladder_observed_rates_test.go

@@ -0,0 +1,129 @@
+package server
+
+import (
+	"context"
+	"errors"
+	"testing"
+
+	pkgladder "github.com/LeanerCloud/cloud-commitments-go/pkg/ladder"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	"github.com/LeanerCloud/cloud-commitments-platform/internal/testutil"
+)
+
+// observedFake adds GetObservedCommitmentRates to the ladder fake.
+type observedFake struct {
+	*fakeLadderCapability
+	observedRates map[pkgladder.LayerType]float64
+	observedErr   error
+}
+
+func (o *observedFake) GetObservedCommitmentRates(_ context.Context, _ pkgladder.Scope) (map[pkgladder.LayerType]float64, error) {
+	if o.observedErr != nil {
+		return nil, o.observedErr
+	}
+	return o.observedRates, nil
+}
+
+// singleFlexFake is a one-flex-layer fake (no base layer takes the gap) with
+// low-water 25 (target 20 at 80% coverage) and a state holding existing
+// commitment USD/hr on that layer. All rates here are exact in binary.
+func singleFlexFake(t *testing.T, layer pkgladder.LayerType, existing float64, marginal map[pkgladder.LayerType]float64) *fakeLadderCapability {
+	t.Helper()
+	return &fakeLadderCapability{
+		t: t, baseline: testBaselineLowWater(25), rates: marginal,
+		supportedLayers: []pkgladder.LayerSpec{{Type: layer, Roles: []pkgladder.LayerRole{pkgladder.RoleFlex}}},
+		layerStates: map[pkgladder.LayerType]pkgladder.LayerState{
+			layer: {Layer: layer, ExistingUSDPerHour: f64(existing), ExpiringUSDPerHour: f64(0)},
+		},
+	}
+}
+
+func TestExecuteLadderRun_ObservedRateConvertsExistingPlannerKeepsMarginal(t *testing.T) {
+	store := &ladderTestStore{}
+	base := singleFlexFake(t, pkgladder.LayerEC2InstanceSP, 5, map[pkgladder.LayerType]float64{pkgladder.LayerEC2InstanceSP: 0.75})
+	cap := &observedFake{fakeLadderCapability: base, observedRates: map[pkgladder.LayerType]float64{pkgladder.LayerEC2InstanceSP: 0.5}}
+	require.NoError(t, runWithRates(t, store, cap))
+
+	run := store.savedRun
+	require.NotNil(t, run)
+	require.NotNil(t, run.ExistingUSDHr)
+	assert.InDelta(t, 10.0, *run.ExistingUSDHr, 1e-9, "5 commitment at the observed 1/2 is 10 OD (marginal would give 6.67)")
+	require.Len(t, store.savedTranches, 1)
+	tr := store.savedTranches[0]
+	assert.InDelta(t, 7.5, tr.AmountUSDHr, 1e-9, "the gap of 10 OD is sized at the marginal 3/4")
+	require.NotNil(t, tr.CommitmentRate)
+	assert.Equal(t, "3/4", *tr.CommitmentRate)
+}
+
+func TestExecuteLadderRun_AbsentObservedKeyFallsBackToMarginal(t *testing.T) {
+	store := &ladderTestStore{}
+	base := singleFlexFake(t, pkgladder.LayerEC2InstanceSP, 5, map[pkgladder.LayerType]float64{pkgladder.LayerEC2InstanceSP: 0.5})
+	cap := &observedFake{fakeLadderCapability: base, observedRates: map[pkgladder.LayerType]float64{pkgladder.LayerComputeSP: 0.75}}
+	require.NoError(t, runWithRates(t, store, cap))
+
+	require.NotNil(t, store.savedRun.ExistingUSDHr)
+	assert.InDelta(t, 10.0, *store.savedRun.ExistingUSDHr, 1e-9, "no observed EC2 rate: marginal 1/2 converts")
+}
+
+func TestExecuteLadderRun_NoObservedCapabilityConvertsAtMarginal(t *testing.T) {
+	store := &ladderTestStore{}
+	cap := singleFlexFake(t, pkgladder.LayerEC2InstanceSP, 5, map[pkgladder.LayerType]float64{pkgladder.LayerEC2InstanceSP: 0.5})
+	require.NoError(t, runWithRates(t, store, cap))
+
+	require.NotNil(t, store.savedRun.ExistingUSDHr)
+	assert.InDelta(t, 10.0, *store.savedRun.ExistingUSDHr, 1e-9)
+}
+
+// Compute has only an observed rate: its coverage converts (no
+// missing_discount_rate), and the planner, with no marginal rate, holds the gap.
+func TestExecuteLadderRun_ObservedOnlyLayerConvertsButPlannerHolds(t *testing.T) {
+	store := &ladderTestStore{}
+	base := singleFlexFake(t, pkgladder.LayerComputeSP, 2, map[pkgladder.LayerType]float64{})
+	cap := &observedFake{fakeLadderCapability: base, observedRates: map[pkgladder.LayerType]float64{pkgladder.LayerComputeSP: 0.5}}
+	require.NoError(t, runWithRates(t, store, cap))
+
+	run := store.savedRun
+	require.NotNil(t, run)
+	plan := decodeLadderPlan(t, run.Plan)
+	assert.NotEqual(t, "missing_discount_rate", plan.ReasonCode)
+	require.NotNil(t, run.ExistingUSDHr)
+	assert.InDelta(t, 4.0, *run.ExistingUSDHr, 1e-9, "2 commitment at the observed 1/2 is 4 OD")
+	assert.Empty(t, store.savedTranches, "new tranches need a marginal rate")
+	require.Len(t, plan.Actions, 1)
+	assert.Equal(t, string(pkgladder.ActionHold), plan.Actions[0].Action)
+	assert.Contains(t, plan.Actions[0].Rationale, "no commitment rate for layer compute-sp")
+}
+
+func TestExecuteLadderRun_LayerWithNeitherRateIsMissingHold(t *testing.T) {
+	store := &ladderTestStore{}
+	base := singleFlexFake(t, pkgladder.LayerComputeSP, 2, map[pkgladder.LayerType]float64{})
+	cap := &observedFake{fakeLadderCapability: base, observedRates: map[pkgladder.LayerType]float64{}}
+	require.NoError(t, runWithRates(t, store, cap))
+
+	plan := decodeLadderPlan(t, store.savedRun.Plan)
+	assert.Equal(t, "missing_discount_rate", plan.ReasonCode)
+	require.Len(t, plan.Actions, 1)
+	assert.Contains(t, plan.Actions[0].Rationale, "neither an observed rate from the commitments held nor a new-purchase rate")
+	assert.Empty(t, store.savedTranches)
+}
+
+func TestExecuteLadderRun_ObservedRatesErrorFailsRun(t *testing.T) {
+	store := &ladderTestStore{}
+	base := singleFlexFake(t, pkgladder.LayerEC2InstanceSP, 5, map[pkgladder.LayerType]float64{pkgladder.LayerEC2InstanceSP: 0.5})
+	cap := &observedFake{fakeLadderCapability: base, observedErr: errors.New("ce throttled")}
+	err := runWithRates(t, store, cap)
+	require.ErrorContains(t, err, "observed commitment rates")
+	require.ErrorContains(t, err, "ce throttled")
+	assert.Nil(t, store.savedRun)
+}
+
+func TestLadderConversionRates_NilMarginalWithObserved(t *testing.T) {
+	base := singleFlexFake(t, pkgladder.LayerComputeSP, 2, nil)
+	cap := &observedFake{fakeLadderCapability: base, observedRates: map[pkgladder.LayerType]float64{pkgladder.LayerComputeSP: 0.5}}
+	conv, observed, err := ladderConversionRates(testutil.TestContext(t), cap, pkgladder.Scope{}, nil)
+	require.NoError(t, err)
+	assert.Equal(t, map[pkgladder.LayerType]float64{pkgladder.LayerComputeSP: 0.5}, conv)
+	assert.Equal(t, cap.observedRates, observed)
+}