From 116befa72e3a228124a07500ea0413f1553109c4 Mon Sep 17 00:00:00 2001 From: krisztian-gajdar Date: Thu, 5 Nov 2026 17:16:01 +0100 Subject: [PATCH 1/5] feat(placement): plan with GPU availability as measured evidence Some GPU products take minutes to place a container on, or fail to place one at all. The planner now reads that from boot evidence and bounds plans by the accelerators an environment can obtain. - Boot evidence records placementWaitS, the time an attempt waited for a container, and outcome unplaced for an attempt that never got one. - A cold warmth's come-alive is its placement wait plus its boot, each at the same percentile: the boot over the identity's attempts at the cell, and the placement wait over every cold attempt on the GPU product. From 20 attempts the 95th percentile by nearest rank stands, below that the slowest, and a failed or unplaced attempt ranks above every success. - A product whose cold attempts were more than 5% unplaced is not chosen for an identity whose comeAliveP95S fits the new hold_budget_s parameter, unless a candidate list names a second product where the identity has a receipt meeting its warm bounds. Plan lanes gain fallbackCells for that list, which is priced at its dearest member. - A placement-capacity document states the total, per-product and shared-quota ceilings. plan raises CapacityExceededError, carrying the refused plan and its report, when the envelope maxima exceed one, and the report lists each ceiling's use. - The report's scarcity section names every cheaper option that scarcity alone blocks, so a reviewer sees when it keeps an identity warm. --- packages/sie_placement/README.md | 230 ++++-- .../src/sie_placement/planner.py | 447 ++++++++-- .../sie_placement/src/sie_placement/report.py | 131 ++- .../sie_placement/src/sie_placement/schema.py | 124 ++- .../tests/fixtures/boot-evidence.json | 1 + .../tests/fixtures/capacity.json | 19 + .../sie_placement/tests/fixtures/plan.json | 6 + .../fixtures/planner/expected-report.json | 3 + .../sie_placement/tests/test_availability.py | 781 ++++++++++++++++++ packages/sie_placement/tests/test_planner.py | 22 +- packages/sie_placement/tests/test_schema.py | 132 +++ 11 files changed, 1750 insertions(+), 146 deletions(-) create mode 100644 packages/sie_placement/tests/fixtures/capacity.json create mode 100644 packages/sie_placement/tests/test_availability.py diff --git a/packages/sie_placement/README.md b/packages/sie_placement/README.md index 7a8be6e86..304094fcf 100644 --- a/packages/sie_placement/README.md +++ b/packages/sie_placement/README.md @@ -29,7 +29,8 @@ name the offending field by its JSON path, for example | `placement-boot-evidence` | `BootEvidence` | One measured boot, snapshot restore or resident load of one identity on one cell and container shape | | `placement-price-list` | `PriceList` | Accelerator, core and memory rates per machine profile, and the region multiplier per placement | | `placement-demand` | `Demand` | Hourly request counts per identity and operation over a UTC window | -| `placement-plan` | `Plan` | Lanes with their cell, container shape, concurrency, ceiling and envelope, and the lane and warmth of every identity | +| `placement-capacity` | `Capacity` | The accelerators a plan may hold: in total, per GPU product, and in a quota it shares with other environments | +| `placement-plan` | `Plan` | Lanes with their cell, fallback cells, container shape, concurrency, ceiling and envelope, and the lane and warmth of every identity | Each type parses with `from_dict(data)` and serializes with `to_dict()`. Constructing a type in code applies the same rules as parsing it: @@ -74,11 +75,16 @@ A plan gives every identity one warmth: `pinned`, `cold-with-bound`, `lazy-resident` identity sits on a lane whose envelope keeps `floorMin` at 1 or more; every other warmth sits on a lane whose `floorMin` is 0. Boot evidence records `cold-with-bound` boots, `snapshot-cold` restores and -`lazy-resident` loads, with outcome `succeeded`, `failed` or `timed-out`. -`borrowed: true` marks a boot measured in another environment whose -boot-shaping inputs match: the identity and its model revision, the image, the -realized GPU product and count, the container shape, the warmth and the -placement class. Whoever supplies the evidence decides the match. +`lazy-resident` loads, one attempt per document, with outcome `succeeded`, +`failed`, `timed-out` or `unplaced`. `placementWaitS` is the time the attempt +waited for a container to be placed before its `bootSeconds` began. An +`unplaced` attempt never got a container: it records no `bootSeconds`, and +`placementWaitS`, when present, is how long it waited before giving up. A +`lazy-resident` load runs on a container that is already placed, so it records +neither. `borrowed: true` marks an attempt measured in another environment +whose boot-shaping inputs match: the identity and its model revision, the +image, the realized GPU product and count, the container shape, the warmth and +the placement class. Whoever supplies the evidence decides the match. ## Lanes @@ -93,6 +99,27 @@ refers to its lane by that name. Two plan lanes may carry the same three tokens, and so name one chart lane, which cannot give them separate envelopes or concurrency; the planner's report lists every such chart lane. +`fallbackCells` is an optional ordered candidate list: cells, after `cell`, +that the lane's containers may be placed on when `cell` cannot be placed. No +fallback cell repeats `cell` or another fallback cell. A realized card is +billed at its own price, so the planner prices such a lane at its dearest +member. + +## Capacity + +A `placement-capacity` document states the accelerators an environment can +obtain. Each field is optional, and at least one is declared: + +| Field | Meaning | +| --- | --- | +| `total` | The accelerators every lane of the plan may hold together | +| `types` | Per `gpuProduct`, the `accelerators` the lanes that may be placed on that product may hold together | +| `sharedQuota` | A quota of `accelerators` that this environment shares with others, which hold `heldElsewhere` of it | + +A lane holds `floorMax` containers of its `gpuCount` accelerators each. A lane +with fallback cells counts against the ceiling of every product it names, and +once against `total`. + ## Envelope A plan lane's envelope bounds what an online tuner may change without a new @@ -115,10 +142,17 @@ result = plan( demand, current_plan, PlannerParameters(placement="us-central1", max_containers=4, fallback_floor=2, cooldown_s=900), + capacity=capacity, ) print(render(result.plan.to_dict()), render(result.report.to_dict())) ``` +`capacity` is optional. A plan whose envelope maxima exceed a capacity ceiling +is not returned: `plan` raises `CapacityExceededError`, a `PlannerInputError` +whose message names each exceeded ceiling and the lanes that hold it, whose +`exceeded` lists those ceilings, and whose `result` carries the refused plan +and its report. + `plan` is a pure function of its documents. The same inputs render to the same bytes in whatever order the receipts, boot evidence, identities and demand series arrive. An identity that stays on its current placement keeps its @@ -136,6 +170,7 @@ may rejoin that lane as a lazy resident on the next run. | `fallback_floor` | The least containers an active zero-floor lane holds, the `sie-cluster` chart's `autoscaling.fallbackReplicas` | | `cooldown_s` | How long an active lane keeps its containers after its last request, the chart's `autoscaling.cooldownPeriod` | | `buffer_containers` | The declared warm buffer above a lane's floor, `0` by default | +| `hold_budget_s` | The longest a request for a cold lane is held for it before it is refused, `0` by default. An identity whose `comeAliveP95S` is at most this is served by holding its requests, and avoids unreliable GPU products | | `hours_per_month` | The hours a monthly figure covers, `730` by default | | `queue_pool`, `bundle`, `bundles` | Every lane's queue pool, and each identity's bundle: its `bundles` entry, otherwise `bundle` | @@ -154,11 +189,19 @@ document: class, the cell of its current lane, is blocked by the undeclared bound, and an identity without a current lane has no unblocked cell. 3. A cell offers `pinned`, and `cold-with-bound` or `snapshot-cold` where boot - evidence for the identity, cell and container shape exists. Below 20 - successful boots the slowest stands for the come-alive percentile; from 20, - their 95th percentile by nearest rank does, so one slow boot stops deciding - once enough boots exist. A boot that did not succeed withdraws the warmth, - and the environment's own evidence replaces borrowed evidence. A + evidence for the identity, cell and container shape exists. A cold + warmth's come-alive is its placement wait plus its boot, each taken at the + same percentile. The boot is taken over the identity's attempts at that + cell, container shape and warmth that got a container. The placement wait + is taken over every cold attempt on the cell's GPU product, whatever the + identity, that records `placementWaitS` or was `unplaced`; a product + without such an attempt adds no wait. Below 20 attempts the slowest stands + for the percentile; from 20, their 95th percentile by nearest rank does, so + one slow attempt stops deciding once enough exist. An attempt that did not + succeed, a failed or timed-out boot or an unplaced placement, ranks above + every success, so the warmth is withdrawn when the percentile falls on one. + The environment's own attempts replace borrowed ones, per cell, container + shape and warmth for the boot and per GPU product for the placement wait. A `comeAliveP95S` of `0` admits `pinned` only. 4. A cell also offers `lazy-resident` on each host lane with its cell, container shape and concurrency. A host lane is a lane of the current plan @@ -169,31 +212,43 @@ document: so no other choice moves them. The identity's own current lane is also a host when it is a lazy resident there already. The come-alive is the identity's `lazy-resident` boot evidence, its measured load onto that cell - and container shape, under the same statistic. The option costs what the + and container shape, under the same statistic, with no placement wait, + since the host's container is already placed. The option costs what the identity's demand needs above the host lane's floor, plus the floor itself on a lane where no such identity stays pinned, since no other identity pays - it. -5. The cheapest unblocked option wins. Ties go to the current placement, - then to the faster come-alive. + it. A lane with fallback cells hosts no lazy resident. +5. A GPU product is unreliable when more than 5% of the cold attempts on it + were `unplaced`, counting the environment's own attempts in place of + borrowed ones. For an identity whose `comeAliveP95S` is at most + `hold_budget_s`, every option on an unreliable product is blocked by + `placementFailureRate`. Such an identity may still be pinned on an + unreliable product through a candidate list: a lane whose fallback cell is + on another product no cheaper than the first, where the identity has a + receipt meeting its warm bounds at the same container shape and + concurrency on both. The list is priced at the dearer product and at the + slower of the two receipts. +6. The cheapest unblocked option wins. Ties go to the current placement, + then to the faster come-alive, then to a lane without fallback cells. The current placement is the identity's current lane when that lane has the -option's cell, container shape and concurrency, the identity's warmth is the -option's, and the lane's `queuePool` and `bundle` are the ones the parameters -give the identity; a `lazy-resident` option must also be hosted by that lane. -A current placement that no option matches lacks evidence: a receipt at its -lane's cell, container shape and concurrency, a boot for a cold warmth, or a -load for a lazy resident. Unless it is known to miss a bound, through a receipt -at the lane that misses a warm bound, a measured come-alive over the bound, a -boot or load at the lane that did not succeed, or a cold or `lazy-resident` -warmth under a bound of `0`, the identity keeps that placement and leaves it -only for an unblocked option at least a cent cheaper than the placement. The -placement is priced as an option of its warmth at the lane's fastest receipt, -or without one at its floor alone: one container all month for `pinned`, for -`lazy-resident` nothing beyond the lane's floor, or the floor itself where no -identity whose `comeAliveP95S` is `0` stays pinned on the lane, and the burst -tails for a cold warmth. What the placement lacks becomes a measurement -request. Such a placement on a lane the price list cannot price has no cost to -compare, so the identity keeps it. +option's cell and fallback cells, container shape and concurrency, the +identity's warmth is the option's, and the lane's `queuePool` and `bundle` are +the ones the parameters give the identity; a `lazy-resident` option must also be +hosted by that lane. A current placement that no option matches lacks evidence: +a receipt at its lane's cell, container shape and concurrency, a boot for a cold +warmth, or a load for a lazy resident. Unless it is known to miss a bound, +through a receipt at the lane that misses a warm bound, a measured come-alive +over the bound, a boot or load at the lane that did not succeed, a cold or +`lazy-resident` warmth under a bound of `0`, or an unreliable product without a +fallback cell on another product under a bound `hold_budget_s` fits, the +identity keeps that placement and leaves it only for an unblocked option at +least a cent cheaper than the placement. The placement is priced as an option of +its warmth at the lane's fastest receipt, or without one at its floor alone: one +container all month for `pinned`, for `lazy-resident` nothing beyond the lane's +floor, or the floor itself where no identity whose `comeAliveP95S` is `0` stays +pinned on the lane, and the burst tails for a cold warmth. What the placement +lacks becomes a measurement request. Such a placement on a lane the price list +cannot price has no cost to compare, so the identity keeps it. An identity that stays on its current placement keeps that lane exactly as it is, beside whoever else the lane hosts: its cell, container shape, @@ -225,7 +280,9 @@ resident's demand needs beyond the floor: on a lane with a floor, the containers an hour needs above it, and on a zero-floor lane, the work and the tails a cold warmth pays. A placed identity's demand is priced at its chosen receipt, and a kept identity's at the slowest receipt it has for its lane's -cell, container shape and concurrency, under that lane's own ceiling. +cells, container shape and concurrency, under that lane's own ceiling. A lane +with fallback cells costs the hourly rate of its dearest cell, and a candidate +list's demand is priced at the slower of its two receipts. ### Envelope @@ -235,7 +292,10 @@ the ceiling. A floor raised on a zero-floor lane realizes as at least `fallback_floor`, so a zero-floor lane's nonzero `floorMax` is at least `fallback_floor` capped by the ceiling. The envelope carries the floor range only. A lane carried from the current plan keeps its envelope, capped by the -ceiling. +ceiling. Given a `capacity` document, the envelope maxima of every lane must +fit each of its ceilings, or the plan is refused as described above. A +loosened service level whose plan the ceilings would refuse saves nothing in +the report. ### Report @@ -248,26 +308,36 @@ The report renders as a `placement-report` document: effective service level and demand, its lane and `monthlyCost`, what its demand costs there with the lane's floor, the chosen cell, warmth, monthly cost and come-alive with its basis (`warm-floor`, `boot-evidence` or - `borrowed-boot-evidence`) and, for a lazy resident, its host `lane`, the peak - containers its demand needs, and `nextCheaper`: the nearest option cheaper - than the choice, or than a kept identity's current placement, that a - service level blocks, the bounds that block it, and `saving`, the fall in - the plan total found by planning again from this plan with each of those - bounds loosened to what the option measured, rounded up to a whole second or - millisecond and never below 1. A bound the identity does not declare blocks - with no `bound`, and loosening it declares it. The loosened bounds may admit - a cheaper option still, and the saving counts it. The saving is 0 when the - total would not fall, as for an identity whose lane keeps its floor for its - other residents. An identity without a placement names its - `unplacedReason`: `no-receipt`, `warm-bound-missed`, - `warm-bound-undeclared` when no cell is on its current class, - `current-unmeasured` when it keeps a current placement that lacks evidence - and no unblocked option is cheaper, or `current-unpriced` when it keeps a - current placement on a lane the price list cannot price. A kept identity's - `blockedBy` names the bounds its kept placement is known to miss: its - receipt's warm latencies, a measured come-alive over the bound, and with no - `measured` value a boot or load at its lane that did not succeed, or a cold - or `lazy-resident` warmth under a bound of `0`, which needs no measurement. + `borrowed-boot-evidence`), the `placementWaitS` part of a cold come-alive + where a placement wait was measured, the `fallbackCells` of a candidate list + and, for a lazy resident, its host `lane`, the peak containers its demand + needs, and `nextCheaper`: the nearest option cheaper than the choice, or than + a kept identity's current placement, that a service level blocks, the bounds + that block it, and `saving`, the fall in the plan total found by planning + again from this plan with each of those bounds loosened to what the option + measured, rounded up to a whole second or millisecond and never below 1. A + bound the identity does not declare blocks with no `bound`, and loosening it + declares it. A `placementFailureRate` blocker, whose `bound` is `0.05` and + `measured` the product's failure rate, is loosened by raising `comeAliveP95S` + to one second above `hold_budget_s`. The loosened bounds may admit a cheaper + option still, and the saving counts it. The saving is 0 when the total would + not fall, as for an identity whose lane keeps its floor for its other + residents. An identity without a placement names its `unplacedReason`: + `no-receipt`, `warm-bound-missed`, `warm-bound-undeclared` when no cell is on + its current class, `placement-unreliable` when its options are on unreliable + products only, `current-unmeasured` when it keeps a current placement that + lacks evidence and no unblocked option is cheaper, or `current-unpriced` when + it keeps a current placement on a lane the price list cannot price. A kept + identity's `blockedBy` names the bounds its kept placement is known to miss: + its receipt's warm latencies, a measured come-alive over the bound, an + unreliable product, and with no `measured` value a boot or load at its lane + that did not succeed, or a cold or `lazy-resident` warmth under a bound of + `0`, which needs no measurement. +- `capacity`: for each ceiling of the `capacity` document, its `scope` + (`total`, `type` with its `gpuProduct`, or `shared-quota` with + `heldElsewhere`), the `ceiling`, the `accelerators` the plan's envelope + maxima hold against it, the `lanes` that hold them, and whether it is + `exceeded`. - `lanes`: each lane's hourly cost, its monthly cost, its floor's monthly cost and the monthly cost of its `floorMax`, which a floor budget prices. A lane on a cell the price list cannot price carries `unpricedReason` and no cost, @@ -278,28 +348,42 @@ The report renders as a `placement-report` document: - `measurementRequests`: options that lack evidence, ranked by the saving they could unlock. A profile without a receipt is bounded by price alone: one accelerator at its rate, kept warm or, where the come-alive bound allows, - paying burst tails only. A cell whose receipt has no successful boot asks for - boot evidence and is bounded by its cold cost, and a host lane without the - identity's load asks for `lazy-resident` boot evidence on it, bounded by + paying burst tails only. A profile on an unreliable product is not requested + for an identity whose bound `hold_budget_s` fits. A cell whose receipt has no + successful boot asks for boot evidence and is bounded by its cold cost, with + reason `failed`, or `unplaced` when no attempt got a container or the + product's placement wait falls on an unplaced attempt, and a host lane without + the identity's load asks for `lazy-resident` boot evidence on it, bounded by what that option would cost. Savings are measured against the choice, or against a kept identity's current placement. A request that cannot save is - dropped, and the requests of an identity without a placement or a current - one carry no bound and rank first. The evidence a kept current placement - lacks is requested at its lane, named in `lane` with its `concurrency`, and - also ranks first without a bound; a profile it asks for there is not asked - for again. -- `comeAliveLevers`: cold and `lazy-resident` warmths whose measured - come-alive misses the bound on a cell that meets the warm bounds for less - than the choice, or than a kept identity's current placement, listed only - when loosening the identity's bound to `permittedBoundS`, the bound the - measured come-alive would permit, lowers the plan total. Both savings are - found by planning again from this plan and taking the fall in its total: + dropped, and the requests of an identity without a placement or a current one + carry no bound and rank first. The evidence a kept current placement lacks is + requested at its lane, named in `lane` with its `concurrency`, and also ranks + first without a bound; a profile it asks for there is not asked for again. +- `comeAliveLevers`: cold and `lazy-resident` warmths whose measured come-alive + misses the bound on a cell that meets the warm bounds for less than the + choice, or than a kept identity's current placement, listed only when + loosening the identity's bound to `permittedBoundS`, the bound the measured + come-alive would permit, lowers the plan total. Both savings are found by + planning again from this plan and taking the fall in its total: `savingAtPermittedBound` with the bound loosened, and `savingUnderBound` with - the warmth's boot measured at the identity's bound, which is 0 under a bound - of 0, since no boot comes under it, and when the total would not fall. An - identity that shares its lane saves nothing of the lane's floor by leaving, - since the other residents keep it. Levers rank by the saving under the bound, - then by the saving at the permitted bound. + the warmth's boot measured at the identity's bound less its placement wait, + which is 0 under a bound of 0, since no boot comes under it, when the + placement wait alone exceeds the bound, and when the total would not fall. A + lever names the `placementWaitS` part of its come-alive where one was + measured. An identity that shares its lane saves nothing of the lane's floor + by leaving, since the other residents keep it. Levers rank by the saving under + the bound, then by the saving at the permitted bound. +- `scarcity`: every option cheaper than an identity's choice, or than a kept + identity's current placement, that scarcity alone blocks, so scarcity is + part of why the identity stays warm or pays more: `placement-wait`, a cold + warmth whose boot fits the come-alive bound and whose boot plus placement + wait does not; `unplaced`, a cold warmth whose boot fits the bound and whose + product's placement wait falls on an unplaced attempt; and + `unreliable-type`, an option on an unreliable product under a bound + `hold_budget_s` fits. Each names the option's cell, container shape, warmth + and monthly cost, the bound, the boot and placement wait it measured, and + the product's `placementFailureRate`. - `queueWaitUnattained`: every identity that declares `queueWaitP99Ms`. No plan claims that bound as met. - `warmBoundUndeclared`: every identity that has a cell and declares neither diff --git a/packages/sie_placement/src/sie_placement/planner.py b/packages/sie_placement/src/sie_placement/planner.py index b3e82d23b..bf286ef09 100644 --- a/packages/sie_placement/src/sie_placement/planner.py +++ b/packages/sie_placement/src/sie_placement/planner.py @@ -13,6 +13,8 @@ from sie_placement.report import ( Blocker, + CapacityScope, + CapacityUse, ComeAliveBasis, IdentityChange, IdentityReport, @@ -27,6 +29,8 @@ RefusedRecord, Report, RequestReason, + Scarcity, + ScarcityReason, SharedChartLane, UnplacedReason, ) @@ -35,6 +39,7 @@ WARM_FLOOR_WARMTHS, BootEvidence, BootOutcome, + Capacity, Cell, ContainerShape, Demand, @@ -58,6 +63,8 @@ PINNED_FLOOR = 1 BOOT_PERCENTILE = 95 BOOT_PERCENTILE_MIN_ROWS = 20 +PLACEMENT_FAILURE_RATE_LIMIT = 0.05 +PLACEMENT_FAILURE_FIELD = "placementFailureRate" COLD_WARMTHS = (Warmth.COLD_WITH_BOUND, Warmth.SNAPSHOT_COLD) _SLUG = re.compile(r"[^a-z0-9]+") @@ -90,6 +97,7 @@ class PlannerParameters: fallback_floor: int cooldown_s: int buffer_containers: int = 0 + hold_budget_s: int = 0 hours_per_month: float = HOURS_PER_MONTH queue_pool: str = "default" bundle: str = "default" @@ -101,6 +109,7 @@ def __post_init__(self) -> None: _require_minimum("fallbackFloor", self.fallback_floor, 0) _require_minimum("cooldownS", self.cooldown_s, 0) _require_minimum("bufferContainers", self.buffer_containers, 0) + _require_minimum("holdBudgetS", self.hold_budget_s, 0) if not (math.isfinite(self.hours_per_month) and self.hours_per_month > 0): raise SchemaError("hoursPerMonth", f"must be greater than 0, got {self.hours_per_month!r}") _require_name("queuePool", self.queue_pool) @@ -123,6 +132,7 @@ def to_dict(self) -> dict[str, object]: "fallbackFloor": self.fallback_floor, "cooldownS": self.cooldown_s, "bufferContainers": self.buffer_containers, + "holdBudgetS": self.hold_budget_s, "hoursPerMonth": self.hours_per_month, "queuePool": self.queue_pool, "bundle": self.bundle, @@ -138,6 +148,13 @@ class PlannerResult: report: Report +class CapacityExceededError(PlannerInputError): + def __init__(self, result: PlannerResult) -> None: + self.result = result + self.exceeded = tuple(use for use in result.report.capacity if use.exceeded) + super().__init__("; ".join(use.describe() for use in self.exceeded)) + + def container_hourly_cost(price: MachineProfilePrice, shape: ContainerShape, region_multiplier: float) -> float: return region_multiplier * ( price.accelerator_per_hour * shape.gpu_count @@ -189,14 +206,46 @@ def cold_container_hours(self, requests: Sequence[int], requests_per_second: flo return seconds / SECONDS_PER_HOUR * self.scale -@dataclass(frozen=True, slots=True) +@dataclass(frozen=True, slots=True, kw_only=True) +class _Placement: + wait_s: float | None + measured: bool + failure_rate: float + borrowed: bool + + +_UNMEASURED_PLACEMENT = _Placement(wait_s=0.0, measured=False, failure_rate=0.0, borrowed=False) + + +@dataclass(frozen=True, slots=True, kw_only=True) class _Boot: - seconds: float | None + boot_s: float | None borrowed: bool + placement: _Placement = _UNMEASURED_PLACEMENT + booted: bool = True + + @property + def seconds(self) -> float | None: + if self.boot_s is None or self.placement.wait_s is None: + return None + if not self.placement.measured: + return self.boot_s + return round(self.boot_s + self.placement.wait_s, 3) + + @property + def placement_wait_s(self) -> float | None: + return self.placement.wait_s if self.placement.measured else None @property def basis(self) -> ComeAliveBasis: - return ComeAliveBasis.BORROWED_BOOT_EVIDENCE if self.borrowed else ComeAliveBasis.BOOT_EVIDENCE + borrowed = self.borrowed or (self.placement.measured and self.placement.borrowed) + return ComeAliveBasis.BORROWED_BOOT_EVIDENCE if borrowed else ComeAliveBasis.BOOT_EVIDENCE + + @property + def reason(self) -> RequestReason: + if not self.booted or (self.boot_s is not None and self.placement.wait_s is None): + return RequestReason.UNPLACED + return RequestReason.FAILED type _BootKey = tuple[str, Cell, ContainerShape, Warmth] @@ -213,14 +262,33 @@ class _Option: blockers: tuple[Blocker, ...] current: bool host: str | None = None + fallbacks: tuple[Receipt, ...] = () + boot_s: float | None = None + placement_wait_s: float | None = None @property def feasible(self) -> bool: return not self.blockers @property - def rank(self) -> tuple[float, bool, float, str, str, str]: - return (self.cost, not self.current, self.come_alive_s, self.order, self.warmth.value, self.host or "") + def cells(self) -> tuple[Cell, ...]: + return (self.receipt.cell, *(receipt.cell for receipt in self.fallbacks)) + + @property + def pricing_receipt(self) -> Receipt: + return min((self.receipt, *self.fallbacks), key=lambda receipt: receipt.throughput.requests_per_second) + + @property + def rank(self) -> tuple[float, bool, float, int, str, str, str]: + return ( + self.cost, + not self.current, + self.come_alive_s, + len(self.fallbacks), + self.order, + self.warmth.value, + self.host or "", + ) def summary(self, saving: float | None = None) -> Option: return Option( @@ -235,6 +303,8 @@ def summary(self, saving: float | None = None) -> Option: blocked_by=self.blockers, saving=saving, lane=self.host, + fallback_cells=self.cells[1:], + placement_wait_s=self.placement_wait_s, ) @@ -250,6 +320,7 @@ class _Choice: held: float | None = None gaps: tuple[MeasurementRequest, ...] = () unpriced: bool = False + withdrawn: tuple[_Option, ...] = () @property def baseline(self) -> float | None: @@ -263,6 +334,8 @@ def unplaced_reason(self) -> UnplacedReason: return UnplacedReason.NO_RECEIPT if not _declares_warm_bound(self.level): return UnplacedReason.WARM_BOUND_UNDECLARED + if any(_fields(option.blockers) == [PLACEMENT_FAILURE_FIELD] for option in self.options): + return UnplacedReason.PLACEMENT_UNRELIABLE return UnplacedReason.WARM_BOUND_MISSED @@ -290,6 +363,14 @@ def _refusal_reason(cell: Cell, prices: Mapping[str, MachineProfilePrice]) -> st return None +def _lane_refusal(lane: PlanLane, prices: Mapping[str, MachineProfilePrice]) -> str | None: + return next((reason for cell in lane.cells if (reason := _refusal_reason(cell, prices)) is not None), None) + + +def _fields(blockers: Iterable[Blocker]) -> list[str]: + return [blocker.field for blocker in blockers] + + def _incomparable_reason(level: ServiceLevel, receipt: Receipt) -> str | None: generate = receipt.request_shape.operation is Operation.GENERATE if level.is_generation and not generate: @@ -334,25 +415,65 @@ def _come_alive_blocker(level: ServiceLevel, seconds: float) -> Blocker | None: return Blocker(field="comeAliveP95S", bound=bound, measured=seconds) -def _come_alive(seconds: Iterable[float]) -> float: - ordered = sorted(float(value) for value in seconds) - if len(ordered) < BOOT_PERCENTILE_MIN_ROWS: - return ordered[-1] - return ordered[(len(ordered) * BOOT_PERCENTILE + 99) // 100 - 1] +def _percentile(samples: Sequence[float | None]) -> float | None: + succeeded = sorted(value for value in samples if value is not None) + if len(samples) < BOOT_PERCENTILE_MIN_ROWS: + return succeeded[-1] if len(succeeded) == len(samples) else None + rank = (len(samples) * BOOT_PERCENTILE + 99) // 100 + return succeeded[rank - 1] if rank <= len(succeeded) else None + + +def _boot_sample(record: BootEvidence) -> float | None: + if record.outcome is not BootOutcome.SUCCEEDED or record.boot_seconds is None: + return None + return float(record.boot_seconds) -def _boot_index(records: Iterable[BootEvidence]) -> dict[_BootKey, _Boot]: +def _placement_index(records: Iterable[BootEvidence]) -> dict[str, _Placement]: + grouped: defaultdict[str, list[BootEvidence]] = defaultdict(list) + for record in records: + if record.warmth in COLD_WARMTHS: + grouped[record.cell.gpu_product].append(record) + index: dict[str, _Placement] = {} + for product, group in grouped.items(): + own = [record for record in group if not record.borrowed] + used = own or group + samples = [ + None if record.outcome is BootOutcome.UNPLACED else float(record.placement_wait_s or 0.0) + for record in used + if record.outcome is BootOutcome.UNPLACED or record.placement_wait_s is not None + ] + unplaced = sum(record.outcome is BootOutcome.UNPLACED for record in used) + index[product] = _Placement( + wait_s=_percentile(samples) if samples else 0.0, + measured=bool(samples), + failure_rate=unplaced / len(used), + borrowed=not own, + ) + return index + + +def _boot_index(records: Iterable[BootEvidence], placements: Mapping[str, _Placement]) -> dict[_BootKey, _Boot]: grouped: defaultdict[_BootKey, list[BootEvidence]] = defaultdict(list) for record in records: grouped[(record.identity, record.cell, record.container_shape, record.warmth)].append(record) index: dict[_BootKey, _Boot] = {} for key, group in grouped.items(): - own = [record for record in group if not record.borrowed] - used = own or group - if any(record.outcome is not BootOutcome.SUCCEEDED for record in used): - index[key] = _Boot(seconds=None, borrowed=not own) - else: - index[key] = _Boot(seconds=_come_alive(record.boot_seconds for record in used), borrowed=not own) + _, cell, _, warmth = key + placement = placements.get(cell.gpu_product, _UNMEASURED_PLACEMENT) + if warmth not in COLD_WARMTHS: + placement = _UNMEASURED_PLACEMENT + booted = [record for record in group if record.outcome is not BootOutcome.UNPLACED] + own = [record for record in booted if not record.borrowed] + used = own or booted + if not used: + index[key] = _Boot( + boot_s=None, borrowed=all(record.borrowed for record in group), placement=placement, booted=False + ) + continue + index[key] = _Boot( + boot_s=_percentile([_boot_sample(record) for record in used]), borrowed=not own, placement=placement + ) return index @@ -368,7 +489,7 @@ def _hourly_requests(demand: Demand) -> dict[str, tuple[int, ...]]: def _runs_on(receipt: Receipt, lane: PlanLane) -> bool: return ( - lane.cell == receipt.cell + receipt.cell in lane.cells and lane.container_shape == receipt.container_shape and lane.concurrency == receipt.concurrency ) @@ -384,6 +505,7 @@ def __init__( demand: Demand, current_plan: Plan | None, parameters: PlannerParameters, + capacity: Capacity | None = None, ) -> None: receipts = tuple(receipts) boot_evidence = tuple(boot_evidence) @@ -396,6 +518,7 @@ def __init__( f"the current plan is for placement {current_plan.placement!r}, not {parameters.placement!r}" ) self.parameters = parameters + self.capacity = capacity self.service_levels = service_levels self.receipt_records = receipts self.boot_records = boot_evidence @@ -425,7 +548,10 @@ def __init__( self.refused_profiles: set[tuple[str, str]] = set() self.savings: dict[tuple[str, tuple[tuple[str, int], ...]], float] = {} self.receipts = self._accept_receipts(receipts) - self.boots = _boot_index(self._accept_boot_evidence(boot_evidence)) + self.placements = _placement_index( + record for record in boot_evidence if _refusal_reason(record.cell, self.prices) is None + ) + self.boots = _boot_index(self._accept_boot_evidence(boot_evidence), self.placements) def _accept_receipts(self, receipts: Iterable[Receipt]) -> dict[str, list[tuple[str, Receipt]]]: accepted: defaultdict[str, list[tuple[str, Receipt]]] = defaultdict(list) @@ -456,7 +582,7 @@ def _accept_boot_evidence(self, records: Iterable[BootEvidence]) -> list[BootEvi def _refuse(self, record: RefusedRecord, identity: str, cell: Cell, source: SourceRef, reason: str) -> None: self.refusals.append(Refusal(record=record, identity=identity, cell=cell, source=source, reason=reason)) - def is_current(self, identity: str, receipt: Receipt, warmth: Warmth) -> bool: + def is_current(self, identity: str, receipt: Receipt, warmth: Warmth, fallbacks: tuple[Cell, ...] = ()) -> bool: current = self.current.get(identity) if current is None: return False @@ -465,9 +591,45 @@ def is_current(self, identity: str, receipt: Receipt, warmth: Warmth) -> bool: entry.warmth is warmth and lane.queue_pool == self.parameters.queue_pool and lane.bundle == self.parameters.bundle_of(identity) + and lane.cells == (receipt.cell, *fallbacks) and _runs_on(receipt, lane) ) + def hourly(self, receipt: Receipt) -> float: + return container_hourly_cost( + self.prices[receipt.cell.machine_profile], receipt.container_shape, self.multiplier + ) + + def lane_hourly(self, lane: PlanLane) -> float: + return max( + container_hourly_cost(self.prices[cell.machine_profile], lane.container_shape, self.multiplier) + for cell in lane.cells + ) + + def tight(self, level: ServiceLevel) -> bool: + bound = level.come_alive_p95_s + return bound is not None and bound <= self.parameters.hold_budget_s + + def unreliable(self, gpu_product: str) -> bool: + placement = self.placements.get(gpu_product) + return placement is not None and placement.failure_rate > PLACEMENT_FAILURE_RATE_LIMIT + + def reliability_blockers(self, level: ServiceLevel, cells: Sequence[Cell]) -> tuple[Blocker, ...]: + first = cells[0].gpu_product + if not self.tight(level) or not self.unreliable(first): + return () + if any(cell.gpu_product != first for cell in cells[1:]): + return () + rate = self.placements[first].failure_rate + return (Blocker(field=PLACEMENT_FAILURE_FIELD, bound=PLACEMENT_FAILURE_RATE_LIMIT, measured=rate),) + + @staticmethod + def warm_blockers(level: ServiceLevel, receipt: Receipt, current_class: Cell | None) -> tuple[Blocker, ...]: + warm = _warm_blockers(level, receipt) + if receipt.cell != current_class: + warm = (*warm, *_undeclared_blockers(level, receipt)) + return warm + def current_class(self, identity: str) -> Cell | None: current = self.current.get(identity) return None if current is None else current[1].cell @@ -504,6 +666,7 @@ def lazy_hosts(self, identity: str) -> tuple[PlanLane, ...]: for name, lane in sorted(self.current_lanes.items()) if (lazy if name == home else self.floor_paid(name, identity)) and lane.envelope.floor_min >= PINNED_FLOOR + and not lane.fallback_cells and lane.queue_pool == self.parameters.queue_pool and lane.bundle == bundle ) @@ -519,9 +682,9 @@ def resident_hours(self, lane: PlanLane, identity: str, requests: Sequence[int], return floor + window.above_floor_hours(requests, rate, capped.envelope.floor_min) def current_cost(self, identity: str, entry: PlanIdentity, lane: PlanLane) -> float | None: - if _refusal_reason(lane.cell, self.prices) is not None: + if _lane_refusal(lane, self.prices) is not None: return None - hourly = container_hourly_cost(self.prices[lane.cell.machine_profile], lane.container_shape, self.multiplier) + hourly = self.lane_hourly(lane) requests = self.requests.get(identity, ()) rate = max( ( @@ -574,19 +737,19 @@ def choose(self, identity: str) -> _Choice: level = self.levels[identity] requests = self.requests.get(identity, ()) options: list[_Option] = [] + withdrawn: list[_Option] = [] boot_requests: dict[tuple[Cell, ContainerShape], MeasurementRequest] = {} load_requests: dict[str, MeasurementRequest] = {} receipts = self.receipts.get(identity, []) current_class = self.current_class(identity) hosts = self.lazy_hosts(identity) for order, receipt in receipts: - hourly = container_hourly_cost( - self.prices[receipt.cell.machine_profile], receipt.container_shape, self.multiplier - ) + hourly = self.hourly(receipt) rate = receipt.throughput.requests_per_second - warm = _warm_blockers(level, receipt) - if receipt.cell != current_class: - warm = (*warm, *_undeclared_blockers(level, receipt)) + blocked = ( + *self.warm_blockers(level, receipt, current_class), + *self.reliability_blockers(level, (receipt.cell,)), + ) options.append( _Option( receipt=receipt, @@ -595,7 +758,7 @@ def choose(self, identity: str) -> _Choice: cost=hourly * self.window.warm_container_hours(requests, rate), come_alive_s=0.0, basis=ComeAliveBasis.WARM_FLOOR, - blockers=warm, + blockers=blocked, current=self.is_current(identity, receipt, Warmth.PINNED), ) ) @@ -603,7 +766,7 @@ def choose(self, identity: str) -> _Choice: for warmth in COLD_WARMTHS: boot = self.boots.get((identity, receipt.cell, receipt.container_shape, warmth)) if boot is None or boot.seconds is None: - if warmth is Warmth.COLD_WITH_BOUND and not warm and level.come_alive_p95_s != 0: + if warmth is Warmth.COLD_WITH_BOUND and not blocked and level.come_alive_p95_s != 0: key = (receipt.cell, receipt.container_shape) known = boot_requests.get(key) if known is None or cold_cost < known.cost_bound: @@ -613,10 +776,24 @@ def choose(self, identity: str) -> _Choice: container_shape=receipt.container_shape, measure=(Measurement.BOOT_EVIDENCE,), warmth=warmth, - reason=RequestReason.UNMEASURED if boot is None else RequestReason.FAILED, + reason=RequestReason.UNMEASURED if boot is None else boot.reason, cost_bound=cold_cost, saving_bound=None, ) + if boot is not None and boot.boot_s is not None and not blocked: + withdrawn.append( + _Option( + receipt=receipt, + order=order, + warmth=warmth, + cost=cold_cost, + come_alive_s=math.inf, + basis=boot.basis, + blockers=(Blocker(field="comeAliveP95S", bound=level.come_alive_p95_s, measured=None),), + current=False, + boot_s=boot.boot_s, + ) + ) continue blocker = _come_alive_blocker(level, boot.seconds) options.append( @@ -627,8 +804,10 @@ def choose(self, identity: str) -> _Choice: cost=cold_cost, come_alive_s=boot.seconds, basis=boot.basis, - blockers=warm if blocker is None else (*warm, blocker), + blockers=blocked if blocker is None else (*blocked, blocker), current=self.is_current(identity, receipt, warmth), + boot_s=boot.boot_s, + placement_wait_s=boot.placement_wait_s, ) ) for host in hosts: @@ -638,7 +817,7 @@ def choose(self, identity: str) -> _Choice: load = self.boots.get((identity, host.cell, host.container_shape, Warmth.LAZY_RESIDENT)) if load is None or load.seconds is None: known = load_requests.get(host.name) - if not warm and level.come_alive_p95_s != 0 and (known is None or lazy_cost < known.cost_bound): + if not blocked and level.come_alive_p95_s != 0 and (known is None or lazy_cost < known.cost_bound): load_requests[host.name] = MeasurementRequest( identity=identity, cell=host.cell, @@ -661,12 +840,15 @@ def choose(self, identity: str) -> _Choice: cost=lazy_cost, come_alive_s=load.seconds, basis=load.basis, - blockers=warm if blocker is None else (*warm, blocker), + blockers=blocked if blocker is None else (*blocked, blocker), current=self.is_current(identity, receipt, Warmth.LAZY_RESIDENT) and self.current[identity][1].name == host.name, host=host.name, + boot_s=load.boot_s, ) ) + if self.tight(level): + options.extend(self.candidate_lists(identity, level, receipts, requests, current_class)) feasible = [option for option in options if option.feasible] chosen = min(feasible, key=lambda option: option.rank) if feasible else None held = None @@ -703,17 +885,57 @@ def choose(self, identity: str) -> _Choice: held=held, gaps=gaps, unpriced=unpriced, + withdrawn=tuple(withdrawn), ) + def candidate_lists( + self, + identity: str, + level: ServiceLevel, + receipts: Sequence[tuple[str, Receipt]], + requests: Sequence[int], + current_class: Cell | None, + ) -> list[_Option]: + lists = [] + for order, receipt in receipts: + if not self.unreliable(receipt.cell.gpu_product) or self.warm_blockers(level, receipt, current_class): + continue + for second_order, second in receipts: + if ( + second.cell.gpu_product == receipt.cell.gpu_product + or self.hourly(second) < self.hourly(receipt) + or second.container_shape != receipt.container_shape + or second.concurrency != receipt.concurrency + or self.warm_blockers(level, second, current_class) + ): + continue + hourly = max(self.hourly(receipt), self.hourly(second)) + rate = min(receipt.throughput.requests_per_second, second.throughput.requests_per_second) + lists.append( + _Option( + receipt=receipt, + fallbacks=(second,), + order=f"{order}\n{second_order}", + warmth=Warmth.PINNED, + cost=hourly * self.window.warm_container_hours(requests, rate), + come_alive_s=0.0, + basis=ComeAliveBasis.WARM_FLOOR, + blockers=(), + current=self.is_current(identity, receipt, Warmth.PINNED, (second.cell,)), + ) + ) + return lists + def receipt_requests(self, choice: _Choice, baseline: float | None) -> list[MeasurementRequest]: measured = {receipt.cell.machine_profile for _, receipt in self.receipts.get(choice.identity, [])} measured.update( request.cell.machine_profile for request in choice.gaps if Measurement.RECEIPT in request.measure ) bound = choice.level.come_alive_p95_s + tight = self.tight(choice.level) requests = [] for profile in sorted(self.price_list.machine_profiles, key=lambda price: price.name): - if profile.name in measured: + if profile.name in measured or (tight and self.unreliable(profile.gpu_product)): continue accelerator = profile.accelerator_per_hour * self.multiplier candidates = [(accelerator * self.window.warm_container_hours((), 1.0), Warmth.PINNED)] @@ -770,8 +992,12 @@ def replanned_total( self.demand, current, self.parameters, + self.capacity, ) - return planner.run(price_savings=False).report.monthly_cost + report = planner.run(price_savings=False).report + if any(use.exceeded for use in report.capacity): + return math.inf + return report.monthly_cost def loosened(self, identity: str, bounds: Mapping[str, int]) -> ServiceLevels: fields = {_LEVEL_FIELDS[name]: bound for name, bound in bounds.items()} @@ -790,11 +1016,25 @@ def saving_when_loosened(self, identity: str, bounds: Mapping[str, int], written self.savings[key] = _saving(total, replanned) return self.savings[key] + def loosening(self, blockers: Iterable[Blocker]) -> dict[str, int]: + bounds: dict[str, int] = {} + for blocker in blockers: + if blocker.field == PLACEMENT_FAILURE_FIELD: + name, bound = "comeAliveP95S", self.parameters.hold_budget_s + 1 + elif blocker.measured is not None: + name, bound = blocker.field, _permitted(blocker.measured) + else: + continue + bounds[name] = max(bound, bounds.get(name, 0)) + return bounds + def booted_within(self, identity: str, option: _Option, bound: int) -> tuple[BootEvidence, ...]: key = (identity, option.receipt.cell, option.receipt.container_shape, option.warmth) + budget = max(0.0, bound - (option.placement_wait_s or 0.0)) return tuple( - replace(record, boot_seconds=min(record.boot_seconds, bound)) + replace(record, boot_seconds=min(record.boot_seconds, budget)) if (record.identity, record.cell, record.container_shape, record.warmth) == key + and record.boot_seconds is not None else record for record in self.boot_records ) @@ -839,6 +1079,7 @@ def levers(self, choice: _Choice, written: Plan, total: float) -> list[Lever]: saving_under_bound=under, permitted_bound_s=permitted, saving_at_permitted_bound=saving, + placement_wait_s=option.placement_wait_s, ) ) return levers @@ -851,15 +1092,91 @@ def next_cheaper(self, choice: _Choice, baseline: float | None, written: Plan, t return None cost = max(option.cost for option in cheaper) nearest = min((option for option in cheaper if option.cost == cost), key=lambda option: option.rank) - bounds = { - blocker.field: _permitted(blocker.measured) for blocker in nearest.blockers if blocker.measured is not None - } + bounds = self.loosening(nearest.blockers) return nearest.summary(saving=self.saving_when_loosened(choice.identity, bounds, written, total)) + def scarcity(self, choice: _Choice) -> list[Scarcity]: + baseline = choice.baseline + if baseline is None: + return [] + bound = choice.level.come_alive_p95_s + candidates = [(option, _scarcity_reason(option, bound)) for option in choice.options] + candidates.extend( + (option, ScarcityReason.UNPLACED) + for option in choice.withdrawn + if bound is None or (bound > 0 and (option.boot_s or 0.0) <= bound) + ) + best: dict[tuple[Cell, ContainerShape, Warmth, str], tuple[_Option, ScarcityReason]] = {} + for option, reason in candidates: + if reason is None or round(baseline - option.cost, 2) <= 0: + continue + key = (option.receipt.cell, option.receipt.container_shape, option.warmth, option.host or "") + known = best.get(key) + if known is None or option.rank < known[0].rank: + best[key] = (option, reason) + return [ + Scarcity( + identity=choice.identity, + reason=reason, + cell=option.receipt.cell, + container_shape=option.receipt.container_shape, + warmth=option.warmth, + monthly_cost=option.cost, + placement_failure_rate=self.placements.get( + option.receipt.cell.gpu_product, _UNMEASURED_PLACEMENT + ).failure_rate, + come_alive_bound_s=bound, + boot_s=option.boot_s, + placement_wait_s=option.placement_wait_s, + lane=option.host, + ) + for option, reason in best.values() + ] + + def capacity_use(self, plan: Plan) -> tuple[CapacityUse, ...]: + capacity = self.capacity + if capacity is None: + return () + + def held(lanes: Iterable[PlanLane]) -> tuple[int, tuple[str, ...]]: + counted = [lane for lane in lanes if lane.envelope.floor_max] + accelerators = sum(lane.envelope.floor_max * lane.container_shape.gpu_count for lane in counted) + return accelerators, tuple(lane.name for lane in counted) + + total, names = held(plan.lanes) + uses = [] + if capacity.total is not None: + uses.append(CapacityUse(scope=CapacityScope.TOTAL, ceiling=capacity.total, accelerators=total, lanes=names)) + for ceiling in sorted(capacity.types, key=lambda entry: entry.gpu_product): + accelerators, lanes = held( + lane for lane in plan.lanes if any(cell.gpu_product == ceiling.gpu_product for cell in lane.cells) + ) + uses.append( + CapacityUse( + scope=CapacityScope.TYPE, + gpu_product=ceiling.gpu_product, + ceiling=ceiling.accelerators, + accelerators=accelerators, + lanes=lanes, + ) + ) + if capacity.shared_quota is not None: + quota = capacity.shared_quota + uses.append( + CapacityUse( + scope=CapacityScope.SHARED_QUOTA, + ceiling=quota.accelerators, + accelerators=total + quota.held_elsewhere, + held_elsewhere=quota.held_elsewhere, + lanes=names, + ) + ) + return tuple(uses) + @staticmethod def lane_names(placed: Mapping[str, _Option], reserved: set[str]) -> dict[str, str]: bases = { - identity: _slug(f"{identity}-{option.receipt.cell.machine_profile}") or "lane" + identity: _slug("-".join((identity, *(cell.machine_profile for cell in option.cells)))) or "lane" for identity, option in placed.items() } counts = Counter(bases.values()) @@ -882,6 +1199,7 @@ def placed_lane(self, name: str, identity: str, option: _Option) -> PlanLane: queue_pool=self.parameters.queue_pool, bundle=self.parameters.bundle_of(identity), cell=receipt.cell, + fallback_cells=option.cells[1:], container_shape=receipt.container_shape, concurrency=receipt.concurrency, max_containers=self.parameters.max_containers, @@ -917,6 +1235,7 @@ def kept_blockers( blockers.extend(() if blocker is None else (blocker,)) elif boot is not None or level.come_alive_p95_s == 0: blockers.append(Blocker(field="comeAliveP95S", bound=level.come_alive_p95_s, measured=None)) + blockers.extend(self.reliability_blockers(level, lane.cells)) return tuple(blockers) def demand_hours(self, lane: PlanLane, identity: str, receipt: Receipt | None) -> float | None: @@ -973,15 +1292,17 @@ def run(self, *, price_savings: bool = True) -> PlannerResult: lanes=tuple(lanes[name] for name in sorted(lanes)), identities=tuple(sorted(identities, key=lambda entry: entry.identity)), ) - receipts: dict[str, Receipt | None] = {identity: option.receipt for identity, option in placed.items()} + receipts: dict[str, Receipt | None] = {identity: option.pricing_receipt for identity, option in placed.items()} receipts.update({identity: self.slowest_receipt(identity, lane) for identity, (_, lane) in kept.items()}) lane_reports, costs = self._pricing(new_plan, receipts) total = sum(lane.monthly_cost for lane in lane_reports if lane.monthly_cost is not None) reports: list[IdentityReport] = [] requests: list[MeasurementRequest] = [] levers: list[Lever] = [] + scarcity: list[Scarcity] = [] for choice in choices: baseline = choice.baseline + scarcity.extend(self.scarcity(choice)) requests.extend(choice.gaps) requests.extend(self.receipt_requests(choice, baseline)) requests.extend(self.boot_requests(choice, baseline)) @@ -1015,6 +1336,8 @@ def run(self, *, price_savings: bool = True) -> PlannerResult: for identity in sorted(self.levels) if not _declares_warm_bound(self.levels[identity]) and self.receipts.get(identity) ), + capacity=self.capacity_use(new_plan), + scarcity=tuple(sorted(scarcity, key=_scarcity_rank)), ) return PlannerResult(plan=new_plan, report=report) @@ -1050,7 +1373,7 @@ def _identity_report( next_cheaper=next_cheaper, ) return replace(report, unplaced_reason=reason) - rate = chosen.receipt.throughput.requests_per_second + rate = chosen.pricing_receipt.throughput.requests_per_second return replace( report, status=IdentityStatus.PLACED, @@ -1072,13 +1395,11 @@ def _pricing( costs: dict[str, float] = {} for lane in plan.lanes: hosted = tuple(sorted(residents[lane.name])) - reason = _refusal_reason(lane.cell, self.prices) + reason = _lane_refusal(lane, self.prices) if reason is not None: reports.append(LaneReport(name=lane.name, identities=hosted, unpriced_reason=reason)) continue - hourly = container_hourly_cost( - self.prices[lane.cell.machine_profile], lane.container_shape, self.multiplier - ) + hourly = self.lane_hourly(lane) floor = lane.envelope.floor_min * hours demand = {identity: self.demand_hours(lane, identity, receipts.get(identity)) for identity in hosted} priced = {identity: value for identity, value in demand.items() if value is not None} @@ -1097,6 +1418,17 @@ def _pricing( return tuple(reports), costs +def _scarcity_reason(option: _Option, bound: int | None) -> ScarcityReason | None: + fields = _fields(option.blockers) + if fields == [PLACEMENT_FAILURE_FIELD]: + return ScarcityReason.UNRELIABLE_TYPE + if option.warmth not in COLD_WARMTHS or fields != ["comeAliveP95S"] or not option.placement_wait_s: + return None + if not bound or option.boot_s is None or option.boot_s > bound: + return None + return ScarcityReason.PLACEMENT_WAIT + + def _saving(total: float, replanned: float) -> float: saving = total - replanned return saving if round(saving, 2) > 0 else 0.0 @@ -1165,6 +1497,18 @@ def _lever_rank(lever: Lever) -> tuple[object, ...]: ) +def _scarcity_rank(entry: Scarcity) -> tuple[object, ...]: + return ( + entry.identity, + entry.reason.value, + entry.cell.machine_profile, + entry.cell.gpu_product, + _shape_key(entry.container_shape), + entry.warmth.value, + entry.lane or "", + ) + + def _refusal_rank(refusal: Refusal) -> tuple[str, ...]: return ( refusal.record.value, @@ -1185,5 +1529,12 @@ def plan( demand: Demand, current_plan: Plan | None, parameters: PlannerParameters, + *, + capacity: Capacity | None = None, ) -> PlannerResult: - return _Planner(service_levels, receipts, boot_evidence, price_list, demand, current_plan, parameters).run() + result = _Planner( + service_levels, receipts, boot_evidence, price_list, demand, current_plan, parameters, capacity + ).run() + if any(use.exceeded for use in result.report.capacity): + raise CapacityExceededError(result) + return result diff --git a/packages/sie_placement/src/sie_placement/report.py b/packages/sie_placement/src/sie_placement/report.py index 585f5f8d0..d63ffc25d 100644 --- a/packages/sie_placement/src/sie_placement/report.py +++ b/packages/sie_placement/src/sie_placement/report.py @@ -33,6 +33,7 @@ class UnplacedReason(StrEnum): WARM_BOUND_UNDECLARED = "warm-bound-undeclared" CURRENT_UNMEASURED = "current-unmeasured" CURRENT_UNPRICED = "current-unpriced" + PLACEMENT_UNRELIABLE = "placement-unreliable" class ComeAliveBasis(StrEnum): @@ -50,6 +51,7 @@ class RequestReason(StrEnum): UNMEASURED = "unmeasured" REFUSED = "refused" FAILED = "failed" + UNPLACED = "unplaced" class RefusedRecord(StrEnum): @@ -57,6 +59,18 @@ class RefusedRecord(StrEnum): BOOT_EVIDENCE = "boot-evidence" +class ScarcityReason(StrEnum): + PLACEMENT_WAIT = "placement-wait" + UNPLACED = "unplaced" + UNRELIABLE_TYPE = "unreliable-type" + + +class CapacityScope(StrEnum): + TOTAL = "total" + TYPE = "type" + SHARED_QUOTA = "shared-quota" + + def _money(value: float) -> float: return round(value, 2) + 0.0 @@ -72,7 +86,7 @@ def _compact(fields: dict[str, object | None]) -> dict[str, object]: @dataclass(frozen=True, slots=True, kw_only=True) class Blocker: field: str - bound: int | None + bound: float | None measured: float | None def to_dict(self) -> dict[str, object]: @@ -92,11 +106,14 @@ class Option: blocked_by: tuple[Blocker, ...] = () saving: float | None = None lane: str | None = None + fallback_cells: tuple[Cell, ...] = () + placement_wait_s: float | None = None def to_dict(self) -> dict[str, object]: return _compact( { "cell": self.cell.to_dict(), + "fallbackCells": [cell.to_dict() for cell in self.fallback_cells] or None, "containerShape": self.container_shape.to_dict(), "concurrency": self.concurrency, "lane": self.lane, @@ -104,6 +121,7 @@ def to_dict(self) -> dict[str, object]: "monthlyCost": _money(self.monthly_cost), "saving": None if self.saving is None else _money(self.saving), "comeAliveS": self.come_alive_s, + "placementWaitS": self.placement_wait_s, "comeAliveBasis": self.come_alive_basis.value, "blockedBy": [blocker.to_dict() for blocker in self.blocked_by] or None, "receipt": self.receipt.to_dict(), @@ -236,21 +254,102 @@ class Lever: saving_under_bound: float permitted_bound_s: int saving_at_permitted_bound: float + placement_wait_s: float | None = None def to_dict(self) -> dict[str, object]: - return { - "identity": self.identity, - "cell": self.cell.to_dict(), - "containerShape": self.container_shape.to_dict(), - "warmth": self.warmth.value, - "comeAliveS": self.come_alive_s, - "comeAliveBasis": self.come_alive_basis.value, - "comeAliveBoundS": self.bound_s, - "monthlyCost": _money(self.monthly_cost), - "savingUnderBound": _money(self.saving_under_bound), - "permittedBoundS": self.permitted_bound_s, - "savingAtPermittedBound": _money(self.saving_at_permitted_bound), - } + return _compact( + { + "identity": self.identity, + "cell": self.cell.to_dict(), + "containerShape": self.container_shape.to_dict(), + "warmth": self.warmth.value, + "comeAliveS": self.come_alive_s, + "placementWaitS": self.placement_wait_s, + "comeAliveBasis": self.come_alive_basis.value, + "comeAliveBoundS": self.bound_s, + "monthlyCost": _money(self.monthly_cost), + "savingUnderBound": _money(self.saving_under_bound), + "permittedBoundS": self.permitted_bound_s, + "savingAtPermittedBound": _money(self.saving_at_permitted_bound), + } + ) + + +@dataclass(frozen=True, slots=True, kw_only=True) +class Scarcity: + identity: str + reason: ScarcityReason + cell: Cell + container_shape: ContainerShape + warmth: Warmth + monthly_cost: float + placement_failure_rate: float + come_alive_bound_s: int | None = None + boot_s: float | None = None + placement_wait_s: float | None = None + lane: str | None = None + + def to_dict(self) -> dict[str, object]: + return _compact( + { + "identity": self.identity, + "reason": self.reason.value, + "cell": self.cell.to_dict(), + "containerShape": self.container_shape.to_dict(), + "lane": self.lane, + "warmth": self.warmth.value, + "monthlyCost": _money(self.monthly_cost), + "comeAliveBoundS": self.come_alive_bound_s, + "bootS": self.boot_s, + "placementWaitS": self.placement_wait_s, + "placementFailureRate": _rate(self.placement_failure_rate), + } + ) + + +@dataclass(frozen=True, slots=True, kw_only=True) +class CapacityUse: + scope: CapacityScope + ceiling: int + accelerators: int + lanes: tuple[str, ...] + gpu_product: str | None = None + held_elsewhere: int | None = None + + @property + def exceeded(self) -> bool: + return self.accelerators > self.ceiling + + def describe(self) -> str: + lanes = ", ".join(self.lanes) or "no lane" + if self.scope is CapacityScope.TYPE: + return ( + f"the envelope maxima of {lanes} hold {self.accelerators} {self.gpu_product} accelerators, " + f"above the type ceiling of {self.ceiling}" + ) + if self.scope is CapacityScope.SHARED_QUOTA: + elsewhere = self.held_elsewhere or 0 + return ( + f"the envelope maxima of {lanes} hold {self.accelerators - elsewhere} accelerators, " + f"which with {elsewhere} held elsewhere exceed the shared quota of {self.ceiling}" + ) + return ( + f"the envelope maxima of {lanes} hold {self.accelerators} accelerators, " + f"above the total ceiling of {self.ceiling}" + ) + + def to_dict(self) -> dict[str, object]: + return _compact( + { + "scope": self.scope.value, + "gpuProduct": self.gpu_product, + "ceiling": self.ceiling, + "accelerators": self.accelerators, + "heldElsewhere": self.held_elsewhere, + "lanes": list(self.lanes), + "exceeded": self.exceeded, + } + ) @dataclass(frozen=True, slots=True, kw_only=True) @@ -318,6 +417,8 @@ class Report: identity_changes: tuple[IdentityChange, ...] lane_changes: tuple[LaneChange, ...] warm_bound_undeclared: tuple[str, ...] = () + capacity: tuple[CapacityUse, ...] = () + scarcity: tuple[Scarcity, ...] = () @property def monthly_cost(self) -> float: @@ -355,9 +456,11 @@ def to_dict(self) -> dict[str, object]: }, "identities": [entry.to_dict() for entry in self.identities], "lanes": [lane.to_dict() for lane in self.lanes], + "capacity": [use.to_dict() for use in self.capacity], "sharedChartLanes": [shared.to_dict() for shared in self.shared_chart_lanes], "measurementRequests": [request.to_dict() for request in self.measurement_requests], "comeAliveLevers": [lever.to_dict() for lever in self.come_alive_levers], + "scarcity": [entry.to_dict() for entry in self.scarcity], "queueWaitUnattained": list(self.queue_wait_unattained), "warmBoundUndeclared": list(self.warm_bound_undeclared), "refusals": [refusal.to_dict() for refusal in self.refusals], diff --git a/packages/sie_placement/src/sie_placement/schema.py b/packages/sie_placement/src/sie_placement/schema.py index ba9e691ba..93b9d9cd0 100644 --- a/packages/sie_placement/src/sie_placement/schema.py +++ b/packages/sie_placement/src/sie_placement/schema.py @@ -17,6 +17,7 @@ BOOT_EVIDENCE_KIND = "placement-boot-evidence" PRICE_LIST_KIND = "placement-price-list" DEMAND_KIND = "placement-demand" +CAPACITY_KIND = "placement-capacity" PLAN_KIND = "placement-plan" ROOT_PATH = "$" @@ -57,6 +58,7 @@ class BootOutcome(StrEnum): SUCCEEDED = "succeeded" FAILED = "failed" TIMED_OUT = "timed-out" + UNPLACED = "unplaced" def _join(path: str, key: str) -> str: @@ -667,9 +669,10 @@ class BootEvidence: cell: Cell container_shape: ContainerShape warmth: Warmth - boot_seconds: float outcome: BootOutcome source: SourceRef + boot_seconds: float | None = None + placement_wait_s: float | None = None borrowed: bool = False def __post_init__(self) -> None: @@ -677,6 +680,17 @@ def __post_init__(self) -> None: if self.warmth not in BOOT_WARMTHS: choices = ", ".join(repr(warmth.value) for warmth in Warmth if warmth in BOOT_WARMTHS) raise SchemaError("warmth", f"boot evidence records one of {choices}, got {self.warmth.value!r}") + if self.warmth is Warmth.LAZY_RESIDENT: + if self.outcome is BootOutcome.UNPLACED: + raise SchemaError("outcome", "a lazy-resident load runs on a placed container and is never unplaced") + _require_absent( + "placementWaitS", self.placement_wait_s, "for a lazy-resident load, which runs on a placed container" + ) + if self.outcome is BootOutcome.UNPLACED: + _require_absent("bootSeconds", self.boot_seconds, "for an unplaced attempt, which never booted") + elif self.boot_seconds is None: + raise SchemaError("bootSeconds", "is required for an attempt that was placed") + _require_at_least("placementWaitS", self.placement_wait_s, 0) _require_at_least("bootSeconds", self.boot_seconds, 0) @classmethod @@ -689,7 +703,8 @@ def from_dict(cls, data: object, path: str = ROOT_PATH) -> Self: cell=reader.child("cell", Cell.from_dict), container_shape=reader.child("containerShape", ContainerShape.from_dict), warmth=reader.enum("warmth", Warmth), - boot_seconds=reader.number("bootSeconds"), + placement_wait_s=reader.optional_number("placementWaitS"), + boot_seconds=reader.optional_number("bootSeconds"), outcome=reader.enum("outcome", BootOutcome), borrowed=reader.optional_boolean("borrowed") is True, source=reader.child("source", SourceRef.from_dict), @@ -704,6 +719,7 @@ def to_dict(self) -> dict[str, object]: "cell": self.cell.to_dict(), "containerShape": self.container_shape.to_dict(), "warmth": self.warmth.value, + "placementWaitS": self.placement_wait_s, "bootSeconds": self.boot_seconds, "outcome": self.outcome.value, "borrowed": True if self.borrowed else None, @@ -900,6 +916,81 @@ def to_dict(self) -> dict[str, object]: } +@dataclass(frozen=True, slots=True, kw_only=True) +class TypeCeiling: + gpu_product: str + accelerators: int + + def __post_init__(self) -> None: + _require_identifier("gpuProduct", self.gpu_product) + _require_at_least("accelerators", self.accelerators, 0) + + @classmethod + def from_dict(cls, data: object, path: str = ROOT_PATH) -> Self: + reader = _Reader(data, path) + return reader.build(cls, gpu_product=reader.string("gpuProduct"), accelerators=reader.integer("accelerators")) + + def to_dict(self) -> dict[str, object]: + return {"gpuProduct": self.gpu_product, "accelerators": self.accelerators} + + +@dataclass(frozen=True, slots=True, kw_only=True) +class SharedQuota: + accelerators: int + held_elsewhere: int = 0 + + def __post_init__(self) -> None: + _require_at_least("accelerators", self.accelerators, 0) + _require_at_least("heldElsewhere", self.held_elsewhere, 0) + + @classmethod + def from_dict(cls, data: object, path: str = ROOT_PATH) -> Self: + reader = _Reader(data, path) + return reader.build( + cls, + accelerators=reader.integer("accelerators"), + held_elsewhere=reader.optional_integer("heldElsewhere") or 0, + ) + + def to_dict(self) -> dict[str, object]: + return {"accelerators": self.accelerators, "heldElsewhere": self.held_elsewhere} + + +@dataclass(frozen=True, slots=True, kw_only=True) +class Capacity: + total: int | None = None + types: tuple[TypeCeiling, ...] = () + shared_quota: SharedQuota | None = None + + def __post_init__(self) -> None: + _require_at_least("total", self.total, 0) + _require_unique("types", "gpuProduct", (ceiling.gpu_product for ceiling in self.types)) + if self.total is None and not self.types and self.shared_quota is None: + raise SchemaError("", "must declare total, types or sharedQuota") + + @classmethod + def from_dict(cls, data: object, path: str = ROOT_PATH) -> Self: + reader = _Reader(data, path) + reader.header(CAPACITY_KIND) + return reader.build( + cls, + total=reader.optional_integer("total"), + types=reader.children("types", TypeCeiling.from_dict, required=False), + shared_quota=reader.optional_child("sharedQuota", SharedQuota.from_dict), + ) + + def to_dict(self) -> dict[str, object]: + return _compact( + { + "kind": CAPACITY_KIND, + "schemaVersion": SCHEMA_VERSION, + "total": self.total, + "types": [ceiling.to_dict() for ceiling in self.types] or None, + "sharedQuota": None if self.shared_quota is None else self.shared_quota.to_dict(), + } + ) + + @dataclass(frozen=True, slots=True, kw_only=True) class Envelope: floor_min: int @@ -959,6 +1050,7 @@ class PlanLane: concurrency: int max_containers: int envelope: Envelope + fallback_cells: tuple[Cell, ...] = () def __post_init__(self) -> None: _require_identifier("name", self.name) @@ -966,12 +1058,22 @@ def __post_init__(self) -> None: _require_identifier("bundle", self.bundle) _require_at_least("concurrency", self.concurrency, 1) _require_at_least("maxContainers", self.max_containers, 1) + for index, cell in enumerate(self.fallback_cells): + if cell in self.cells[: index + 1]: + raise SchemaError( + f"fallbackCells[{index}]", + f"repeats {cell.machine_profile!r} with {cell.gpu_product!r}, already a cell of this lane", + ) if self.envelope.floor_max > self.max_containers: raise SchemaError( "envelope.floorMax", f"must not exceed maxContainers ({self.max_containers}), got {self.envelope.floor_max}", ) + @property + def cells(self) -> tuple[Cell, ...]: + return (self.cell, *self.fallback_cells) + @classmethod def from_dict(cls, data: object, path: str = ROOT_PATH) -> Self: reader = _Reader(data, path) @@ -981,6 +1083,7 @@ def from_dict(cls, data: object, path: str = ROOT_PATH) -> Self: queue_pool=reader.string("queuePool"), bundle=reader.string("bundle"), cell=reader.child("cell", Cell.from_dict), + fallback_cells=reader.children("fallbackCells", Cell.from_dict, required=False), container_shape=reader.child("containerShape", ContainerShape.from_dict), concurrency=reader.integer("concurrency"), max_containers=reader.integer("maxContainers"), @@ -988,16 +1091,23 @@ def from_dict(cls, data: object, path: str = ROOT_PATH) -> Self: ) def to_dict(self) -> dict[str, object]: - return { + document: dict[str, object] = { "name": self.name, "queuePool": self.queue_pool, "bundle": self.bundle, "cell": self.cell.to_dict(), - "containerShape": self.container_shape.to_dict(), - "concurrency": self.concurrency, - "maxContainers": self.max_containers, - "envelope": self.envelope.to_dict(), } + if self.fallback_cells: + document["fallbackCells"] = [cell.to_dict() for cell in self.fallback_cells] + document.update( + { + "containerShape": self.container_shape.to_dict(), + "concurrency": self.concurrency, + "maxContainers": self.max_containers, + "envelope": self.envelope.to_dict(), + } + ) + return document @dataclass(frozen=True, slots=True, kw_only=True) diff --git a/packages/sie_placement/tests/fixtures/boot-evidence.json b/packages/sie_placement/tests/fixtures/boot-evidence.json index be9e58814..73648db0a 100644 --- a/packages/sie_placement/tests/fixtures/boot-evidence.json +++ b/packages/sie_placement/tests/fixtures/boot-evidence.json @@ -12,6 +12,7 @@ "memoryMiB": 87040 }, "warmth": "cold-with-bound", + "placementWaitS": 41.2, "bootSeconds": 187.4, "outcome": "succeeded", "source": { diff --git a/packages/sie_placement/tests/fixtures/capacity.json b/packages/sie_placement/tests/fixtures/capacity.json new file mode 100644 index 000000000..da79b0646 --- /dev/null +++ b/packages/sie_placement/tests/fixtures/capacity.json @@ -0,0 +1,19 @@ +{ + "kind": "placement-capacity", + "schemaVersion": 1, + "total": 12, + "types": [ + { + "gpuProduct": "NVIDIA-L4", + "accelerators": 8 + }, + { + "gpuProduct": "NVIDIA-A100-SXM4-80GB", + "accelerators": 4 + } + ], + "sharedQuota": { + "accelerators": 16, + "heldElsewhere": 6 + } +} diff --git a/packages/sie_placement/tests/fixtures/plan.json b/packages/sie_placement/tests/fixtures/plan.json index ed9a2f5fa..940112335 100644 --- a/packages/sie_placement/tests/fixtures/plan.json +++ b/packages/sie_placement/tests/fixtures/plan.json @@ -35,6 +35,12 @@ "machineProfile": "a100-80gb", "gpuProduct": "NVIDIA-A100-SXM4-80GB" }, + "fallbackCells": [ + { + "machineProfile": "h100", + "gpuProduct": "NVIDIA-H100-80GB-HBM3" + } + ], "containerShape": { "gpuCount": 1, "cpuCores": 12, diff --git a/packages/sie_placement/tests/fixtures/planner/expected-report.json b/packages/sie_placement/tests/fixtures/planner/expected-report.json index 076b55a12..de243df5b 100644 --- a/packages/sie_placement/tests/fixtures/planner/expected-report.json +++ b/packages/sie_placement/tests/fixtures/planner/expected-report.json @@ -9,6 +9,7 @@ "fallbackFloor": 2, "cooldownS": 900, "bufferContainers": 1, + "holdBudgetS": 0, "hoursPerMonth": 730.0, "queuePool": "default", "bundle": "default", @@ -323,6 +324,7 @@ "envelopeMaxMonthlyCost": 1518.4 } ], + "capacity": [], "sharedChartLanes": [ { "queuePool": "default", @@ -525,6 +527,7 @@ "savingAtPermittedBound": 379.6 } ], + "scarcity": [], "queueWaitUnattained": [ "BAAI/bge-m3", "BAAI/bge-m3:sparse", diff --git a/packages/sie_placement/tests/test_availability.py b/packages/sie_placement/tests/test_availability.py new file mode 100644 index 000000000..eac0a3852 --- /dev/null +++ b/packages/sie_placement/tests/test_availability.py @@ -0,0 +1,781 @@ +from __future__ import annotations + +import math +import random +from dataclasses import replace +from datetime import UTC, datetime, timedelta + +import pytest +from sie_placement.planner import CapacityExceededError, PlannerInputError, PlannerParameters, plan +from sie_placement.report import ( + Blocker, + CapacityScope, + ComeAliveBasis, + IdentityStatus, + Measurement, + RequestReason, + ScarcityReason, + UnplacedReason, + render, +) +from sie_placement.schema import ( + BootEvidence, + BootOutcome, + Capacity, + Cell, + ContainerShape, + Demand, + DemandHour, + DemandSeries, + Envelope, + IdentityServiceLevel, + MachineProfilePrice, + Operation, + Percentiles, + Placement, + Plan, + PlanIdentity, + PlanLane, + PriceList, + Receipt, + RequestShape, + ServiceLevel, + ServiceLevels, + SharedQuota, + SourceRef, + Throughput, + TypeCeiling, + Warmth, +) + +START = datetime(2026, 11, 2, tzinfo=UTC) +MONTH = 730.0 +SCALE = MONTH / 24 +L4 = Cell(machine_profile="l4", gpu_product="NVIDIA-L4") +A100 = Cell(machine_profile="a100-80gb", gpu_product="NVIDIA-A100-SXM4-80GB") +H100 = Cell(machine_profile="h100", gpu_product="NVIDIA-H100-80GB-HBM3") +RATES = {L4: 1.0, A100: 3.0, H100: 4.0} +SHAPE = ContainerShape(gpu_count=1, cpu_cores=4, memory_mib=16384) +PAIR = ContainerShape(gpu_count=2, cpu_cores=8, memory_mib=32768) +IDENTITY = "org/model" +OTHER = "org/other" +PRICES = PriceList( + currency="USD", + machine_profiles=tuple( + MachineProfilePrice( + name=cell.machine_profile, + gpu_product=cell.gpu_product, + accelerator_per_hour=rate, + core_per_hour=0.0, + memory_gib_per_hour=0.0, + ) + for cell, rate in RATES.items() + ), + placements=(Placement(name="test", region_multiplier=1),), +) +UNRELIABLE = 1 / 3 + + +def receipt(identity=IDENTITY, cell=L4, *, shape=SHAPE, concurrency=8, rps=10.0, p99=100.0) -> Receipt: + return Receipt( + identity=identity, + cell=cell, + container_shape=shape, + concurrency=concurrency, + request_shape=RequestShape(operation=Operation.ENCODE, items_per_request=1), + throughput=Throughput(requests_per_second=rps), + latency=Percentiles(p99_ms=p99), + source=SourceRef(uri=f"receipts/{identity}/{cell.machine_profile}/{shape.gpu_count}/{concurrency}/{rps}/{p99}"), + ) + + +def attempt( + identity=IDENTITY, + cell=L4, + *, + warmth=Warmth.COLD_WITH_BOUND, + boot=60.0, + wait=None, + outcome=BootOutcome.SUCCEEDED, + borrowed=False, + shape=SHAPE, + index=0, +) -> BootEvidence: + return BootEvidence( + identity=identity, + cell=cell, + container_shape=shape, + warmth=warmth, + outcome=outcome, + boot_seconds=None if outcome is BootOutcome.UNPLACED else boot, + placement_wait_s=wait, + borrowed=borrowed, + source=SourceRef(uri=f"boots/{identity}/{cell.machine_profile}/{warmth.value}/{index}"), + ) + + +def unplaced(cell=L4, *, index=0, identity=OTHER) -> BootEvidence: + return attempt(identity, cell, outcome=BootOutcome.UNPLACED, index=index) + + +def scarce(cell=L4, *, placed=2, failed=1) -> list[BootEvidence]: + rows = [attempt(OTHER, cell, boot=30.0, index=index) for index in range(placed)] + rows.extend(unplaced(cell, index=placed + index) for index in range(failed)) + return rows + + +def levels(**bounds) -> ServiceLevels: + return ServiceLevels( + identities=tuple( + IdentityServiceLevel(identity=identity, level=ServiceLevel(come_alive_p95_s=bound, warm_p99_ms=250)) + for identity, bound in sorted(bounds.items()) + ) + ) + + +def one(bound) -> ServiceLevels: + return levels(**{IDENTITY: bound}) + + +def demand(hourly=None) -> Demand: + return Demand( + start=START, + end=START + timedelta(hours=24), + series=tuple( + DemandSeries( + identity=identity, + operation=Operation.ENCODE, + hours=tuple( + DemandHour(start=START + timedelta(hours=hour), requests=count) + for hour, count in sorted(counts.items()) + ), + ) + for identity, counts in sorted((hourly or {}).items()) + ), + ) + + +def parameters(**overrides) -> PlannerParameters: + values = {"placement": "test", "max_containers": 4, "fallback_floor": 2, "cooldown_s": 900} | overrides + return PlannerParameters(**values) + + +def run(service_levels, receipts=(), boots=(), *, hourly=None, current=None, capacity=None, **overrides): + return plan( + service_levels, receipts, boots, PRICES, demand(hourly), current, parameters(**overrides), capacity=capacity + ) + + +def rendered(result) -> tuple[str, str]: + return render(result.plan.to_dict()), render(result.report.to_dict()) + + +def lane(name, cell=L4, *, fallbacks=(), floor=1, buffer=0) -> PlanLane: + return PlanLane( + name=name, + queue_pool="default", + bundle="default", + cell=cell, + fallback_cells=tuple(fallbacks), + container_shape=SHAPE, + concurrency=8, + max_containers=4, + envelope=Envelope(floor_min=floor, floor_max=floor + buffer), + ) + + +def plan_of(lanes, residents) -> Plan: + return Plan( + placement="test", + lanes=tuple(lanes), + identities=tuple( + PlanIdentity(identity=identity, lane=name, warmth=warmth) + for identity, (name, warmth) in sorted(residents.items()) + ), + ) + + +def test_a_cold_come_alive_adds_the_placement_wait_to_the_boot(): + result = run(one(120), [receipt()], [attempt(boot=50.0, wait=30.0)]) + chosen = result.report.identity(IDENTITY).chosen + assert (chosen.warmth, chosen.come_alive_s, chosen.placement_wait_s) == (Warmth.COLD_WITH_BOUND, 80.0, 30.0) + assert chosen.to_dict()["placementWaitS"] == 30.0 + assert result.report.scarcity == () + + +def test_placement_waits_pool_over_every_attempt_on_a_gpu_type(): + boots = [ + attempt(boot=50.0), + attempt(OTHER, boot=500.0, wait=100.0, index=1), + attempt(OTHER, A100, boot=10.0, wait=900.0, index=2), + ] + result = run(one(120), [receipt()], boots) + report = result.report.identity(IDENTITY) + assert report.chosen.warmth is Warmth.PINNED + assert report.next_cheaper.blocked_by == (Blocker(field="comeAliveP95S", bound=120, measured=150.0),) + assert report.next_cheaper.placement_wait_s == 100.0 + (entry,) = result.report.scarcity + assert (entry.identity, entry.reason, entry.cell, entry.warmth) == ( + IDENTITY, + ScarcityReason.PLACEMENT_WAIT, + L4, + Warmth.COLD_WITH_BOUND, + ) + assert (entry.boot_s, entry.placement_wait_s, entry.come_alive_bound_s, entry.monthly_cost) == (50.0, 100.0, 120, 0) + assert entry.to_dict()["reason"] == "placement-wait" + + +@pytest.mark.parametrize(("fast", "come_alive"), [(18, None), (19, 60.0), (37, None), (38, 60.0)]) +def test_the_placement_wait_takes_the_95th_percentile_from_twenty_attempts(fast, come_alive): + boots = [attempt(boot=50.0)] + boots.extend(attempt(OTHER, boot=1.0, wait=10.0, index=index) for index in range(fast)) + boots.extend(attempt(OTHER, boot=1.0, wait=200.0, index=100 + index) for index in range(1 if fast < 30 else 2)) + chosen = run(one(120), [receipt()], boots).report.identity(IDENTITY).chosen + if come_alive is None: + assert chosen.warmth is Warmth.PINNED + else: + assert (chosen.warmth, chosen.come_alive_s) == (Warmth.COLD_WITH_BOUND, come_alive) + + +@pytest.mark.parametrize(("placed", "selected"), [(17, False), (18, True)]) +def test_an_unplaced_attempt_ranks_above_every_placed_one(placed, selected): + boots = [attempt(boot=50.0, wait=10.0)] + boots.extend(attempt(OTHER, boot=1.0, wait=10.0, index=index) for index in range(placed)) + boots.append(unplaced(index=99)) + result = run(one(120), [receipt()], boots) + chosen = result.report.identity(IDENTITY).chosen + if selected: + assert (chosen.warmth, chosen.come_alive_s) == (Warmth.COLD_WITH_BOUND, 60.0) + assert result.report.scarcity == () + return + assert chosen.warmth is Warmth.PINNED + (request,) = [request for request in result.report.measurement_requests if request.cell == L4] + assert (request.measure, request.warmth, request.reason) == ( + (Measurement.BOOT_EVIDENCE,), + Warmth.COLD_WITH_BOUND, + RequestReason.UNPLACED, + ) + assert request.to_dict()["reason"] == "unplaced" + (entry,) = result.report.scarcity + assert (entry.reason, entry.boot_s, entry.placement_wait_s) == (ScarcityReason.UNPLACED, 50.0, None) + assert entry.placement_failure_rate == pytest.approx(1 / (placed + 2)) + + +def test_an_attempt_without_a_placement_wait_leaves_its_boot_to_stand_for_the_come_alive(): + result = run(one(120), [receipt()], [attempt(boot=95.0)]) + chosen = result.report.identity(IDENTITY).chosen + assert (chosen.come_alive_s, chosen.placement_wait_s) == (95.0, None) + assert "placementWaitS" not in chosen.to_dict() + + +def test_own_attempts_on_a_type_replace_borrowed_ones(): + boots = [attempt(OTHER, boot=1.0, wait=200.0, borrowed=True, index=1), attempt(boot=50.0)] + chosen = run(one(120), [receipt()], boots).report.identity(IDENTITY).chosen + assert (chosen.come_alive_s, chosen.come_alive_basis) == (50.0, ComeAliveBasis.BOOT_EVIDENCE) + borrowed = [attempt(boot=50.0, wait=20.0, borrowed=True)] + chosen = run(one(120), [receipt()], borrowed).report.identity(IDENTITY).chosen + assert (chosen.come_alive_s, chosen.come_alive_basis) == (70.0, ComeAliveBasis.BORROWED_BOOT_EVIDENCE) + + +def test_an_unplaced_attempt_of_the_identity_withdraws_its_cold_cell_until_it_boots(): + result = run(one(120), [receipt()], [unplaced(identity=IDENTITY)]) + assert result.report.identity(IDENTITY).chosen.warmth is Warmth.PINNED + (request,) = [request for request in result.report.measurement_requests if request.cell == L4] + assert request.reason is RequestReason.UNPLACED + assert result.report.scarcity == () + + +def test_a_snapshot_restore_waits_for_placement_too(): + boots = [attempt(warmth=Warmth.SNAPSHOT_COLD, boot=5.0, wait=30.0)] + result = run(one(20), [receipt()], boots) + report = result.report.identity(IDENTITY) + assert report.chosen.warmth is Warmth.PINNED + assert (report.next_cheaper.warmth, report.next_cheaper.come_alive_s) == (Warmth.SNAPSHOT_COLD, 35.0) + assert [(entry.warmth, entry.reason) for entry in result.report.scarcity] == [ + (Warmth.SNAPSHOT_COLD, ScarcityReason.PLACEMENT_WAIT) + ] + + +def test_a_lazy_resident_load_waits_for_no_placement(): + current = plan_of( + [lane("own"), lane("shared")], {OTHER: ("shared", Warmth.PINNED), IDENTITY: ("own", Warmth.PINNED)} + ) + service_levels = levels(**{OTHER: 0, IDENTITY: 60}) + boots = [attempt(warmth=Warmth.LAZY_RESIDENT, boot=20.0), attempt(OTHER, boot=1.0, wait=500.0, index=1)] + result = run(service_levels, [receipt(OTHER), receipt()], boots, current=current) + chosen = result.report.identity(IDENTITY).chosen + assert (chosen.warmth, chosen.lane, chosen.come_alive_s, chosen.placement_wait_s) == ( + Warmth.LAZY_RESIDENT, + "shared", + 20.0, + None, + ) + + +@pytest.mark.parametrize(("wait", "under"), [(30.0, MONTH), (70.0, 0.0)]) +def test_a_lever_under_its_bound_leaves_room_for_the_placement_wait(wait, under): + result = run(one(60), [receipt()], [attempt(boot=50.0, wait=wait)]) + (lever,) = result.report.come_alive_levers + assert (lever.come_alive_s, lever.placement_wait_s, lever.permitted_bound_s) == (50.0 + wait, wait, 50 + wait) + assert lever.saving_at_permitted_bound == pytest.approx(MONTH) + assert lever.saving_under_bound == pytest.approx(under) + assert lever.to_dict()["placementWaitS"] == wait + + +def test_an_unreliable_type_is_not_chosen_under_a_bound_the_hold_budget_fits(): + result = run(one(0), [receipt(), receipt(cell=A100)], scarce()) + report = result.report.identity(IDENTITY) + assert (report.chosen.cell, report.chosen.warmth) == (A100, Warmth.PINNED) + assert report.chosen.monthly_cost == pytest.approx(3 * MONTH) + assert report.next_cheaper.cell == L4 + assert report.next_cheaper.blocked_by == ( + Blocker(field="placementFailureRate", bound=0.05, measured=pytest.approx(UNRELIABLE)), + ) + assert report.next_cheaper.saving == pytest.approx(2 * MONTH) + (entry,) = result.report.scarcity + assert (entry.reason, entry.cell, entry.warmth, entry.monthly_cost) == ( + ScarcityReason.UNRELIABLE_TYPE, + L4, + Warmth.PINNED, + pytest.approx(MONTH), + ) + assert entry.to_dict()["placementFailureRate"] == 0.3333 + assert result.report.to_dict()["identities"][0]["nextCheaper"]["blockedBy"][0]["field"] == "placementFailureRate" + + +def test_loosening_a_bound_above_the_hold_budget_admits_an_unreliable_type(): + report = run(one(0), [receipt(), receipt(cell=A100)], scarce(), hold_budget_s=20).report.identity(IDENTITY) + loosened = run(one(21), [receipt(), receipt(cell=A100)], scarce(), hold_budget_s=20).report + assert report.chosen.cell == A100 + assert loosened.identity(IDENTITY).chosen.cell == L4 + assert report.next_cheaper.saving == pytest.approx(2 * MONTH) + + +@pytest.mark.parametrize(("hold", "cell"), [(20, A100), (19, L4), (0, L4)]) +def test_the_hold_budget_decides_which_bounds_avoid_unreliable_types(hold, cell): + result = run(one(20), [receipt(), receipt(cell=A100)], scarce(), hold_budget_s=hold) + assert result.report.identity(IDENTITY).chosen.cell == cell + assert bool(result.report.scarcity) is (cell == A100) + + +def test_an_undeclared_come_alive_bound_may_use_an_unreliable_type(): + result = run(one(None), [receipt(), receipt(cell=A100)], scarce(), hold_budget_s=600) + assert result.report.identity(IDENTITY).chosen.cell == L4 + + +def test_a_placement_failure_rate_of_five_percent_is_reliable(): + boots = scarce(placed=19, failed=1) + assert run(one(0), [receipt(), receipt(cell=A100)], boots).report.identity(IDENTITY).chosen.cell == L4 + boots = scarce(placed=18, failed=1) + assert run(one(0), [receipt(), receipt(cell=A100)], boots).report.identity(IDENTITY).chosen.cell == A100 + + +def test_the_failure_rate_counts_the_environments_own_attempts_over_borrowed_ones(): + borrowed = [replace(row, borrowed=True) for row in scarce(placed=0, failed=3)] + own = [attempt(OTHER, boot=30.0, index=10)] + result = run(one(0), [receipt(), receipt(cell=A100)], [*borrowed, *own]) + assert result.report.identity(IDENTITY).chosen.cell == L4 + result = run(one(0), [receipt(), receipt(cell=A100)], borrowed) + assert result.report.identity(IDENTITY).chosen.cell == A100 + + +def test_an_identity_whose_only_types_are_unreliable_is_unplaced_and_says_why(): + result = run(one(0), [receipt()], scarce()) + report = result.report.identity(IDENTITY) + assert (report.status, report.unplaced_reason) == (IdentityStatus.UNPLACED, UnplacedReason.PLACEMENT_UNRELIABLE) + assert report.to_dict()["unplacedReason"] == "placement-unreliable" + assert result.plan.identities == () + + +def test_unreliable_types_are_not_requested_for_a_bound_the_hold_budget_fits(): + result = run(one(0), [receipt(cell=H100)], scarce()) + assert [request.cell for request in result.report.measurement_requests] == [A100] + result = run(one(600), [receipt(cell=H100)], scarce()) + assert sorted(request.cell.machine_profile for request in result.report.measurement_requests) == [ + "a100-80gb", + "h100", + "l4", + ] + + +def test_a_current_placement_on_an_unreliable_type_is_known_to_miss_its_bound(): + current = plan_of([lane("own")], {IDENTITY: ("own", Warmth.PINNED)}) + moved = run(one(0), [receipt(), receipt(cell=A100)], scarce(), current=current) + assert moved.report.identity(IDENTITY).chosen.cell == A100 + assert [entry.lane for entry in moved.plan.identities] == ["org-model-a100-80gb"] + kept = run(one(0), [receipt()], scarce(), current=current) + report = kept.report.identity(IDENTITY) + assert (report.status, report.lane, report.unplaced_reason) == ( + IdentityStatus.KEPT, + "own", + UnplacedReason.PLACEMENT_UNRELIABLE, + ) + assert report.blocked_by == (Blocker(field="placementFailureRate", bound=0.05, measured=pytest.approx(UNRELIABLE)),) + assert kept.plan == current + + +def test_a_candidate_list_names_a_second_type_when_every_single_type_is_unreliable(): + boots = [*scarce(L4), *scarce(A100)] + receipts = [receipt(), receipt(cell=A100)] + result = run(one(0), receipts, boots) + chosen = result.report.identity(IDENTITY).chosen + assert (chosen.cell, chosen.fallback_cells, chosen.warmth) == (L4, (A100,), Warmth.PINNED) + assert chosen.monthly_cost == pytest.approx(3 * MONTH) + assert chosen.to_dict()["fallbackCells"] == [A100.to_dict()] + (planned,) = result.plan.lanes + assert (planned.name, planned.cell, planned.fallback_cells) == ("org-model-l4-a100-80gb", L4, (A100,)) + assert result.report.monthly_cost == pytest.approx(3 * MONTH) + assert Plan.from_dict(result.plan.to_dict()) == result.plan + again = run(one(0), receipts, boots, current=result.plan) + assert again.plan == result.plan + assert again.report.unchanged + + +def test_a_candidate_list_is_not_offered_where_a_reliable_type_is_cheaper_or_as_cheap(): + result = run(one(0), [receipt(), receipt(cell=A100)], scarce(L4)) + assert result.report.identity(IDENTITY).chosen.fallback_cells == () + assert result.plan.lanes[0].fallback_cells == () + + +@pytest.mark.parametrize( + "second", + [ + receipt(cell=A100, p99=300.0), + receipt(cell=A100, concurrency=16), + receipt(cell=A100, shape=PAIR), + ], +) +def test_a_list_member_meets_the_warm_bound_at_the_lane_shape_and_concurrency(second): + result = run(one(0), [receipt(), second], [*scarce(L4), *scarce(A100)]) + assert result.report.identity(IDENTITY).status is IdentityStatus.UNPLACED + + +def test_a_candidate_list_is_priced_at_its_dearest_member_and_its_slowest_receipt(): + receipts = [receipt(rps=10.0), receipt(cell=A100, rps=5.0)] + result = run(one(0), receipts, [*scarce(L4), *scarce(A100)], hourly={IDENTITY: {0: 100000}}) + report = result.report.identity(IDENTITY) + above = 3 * SCALE + assert report.chosen.monthly_cost == pytest.approx(3 * (MONTH + above)) + assert report.monthly_cost == pytest.approx(3 * (MONTH + above)) + assert report.peak_containers == 6 + (lane_report,) = result.report.lanes + assert lane_report.hourly_cost == 3.0 + + +def test_a_current_candidate_list_stays_when_a_plain_type_costs_the_same(): + current = plan_of([lane("listed", fallbacks=(A100,))], {IDENTITY: ("listed", Warmth.PINNED)}) + result = run(one(0), [receipt(), receipt(cell=A100)], scarce(L4), current=current) + assert result.plan == current + assert result.report.unchanged + assert result.report.identity(IDENTITY).chosen.fallback_cells == (A100,) + + +def test_a_lane_with_fallback_cells_hosts_no_lazy_resident(): + current = plan_of( + [lane("own"), lane("listed", fallbacks=(A100,))], + {OTHER: ("listed", Warmth.PINNED), IDENTITY: ("own", Warmth.PINNED)}, + ) + service_levels = levels(**{OTHER: 0, IDENTITY: 60}) + boots = [attempt(warmth=Warmth.LAZY_RESIDENT, boot=20.0), *scarce(L4)] + result = run(service_levels, [receipt(OTHER), receipt(OTHER, A100), receipt()], boots, current=current) + assert {entry.identity: entry.lane for entry in result.plan.identities}[OTHER] == "listed" + assert result.report.identity(IDENTITY).chosen.warmth is Warmth.PINNED + assert all(request.warmth is not Warmth.LAZY_RESIDENT for request in result.report.measurement_requests) + + +def two_lanes(**overrides): + receipts = [receipt("org/a"), receipt("org/b", A100)] + return run(levels(**{"org/a": 0, "org/b": 0}), receipts, buffer_containers=1, **overrides) + + +def test_capacity_reports_what_the_envelope_maxima_hold_against_each_ceiling(): + capacity = Capacity( + total=4, + types=( + TypeCeiling(gpu_product="NVIDIA-L4", accelerators=2), + TypeCeiling(gpu_product=A100.gpu_product, accelerators=2), + ), + shared_quota=SharedQuota(accelerators=6, held_elsewhere=2), + ) + report = two_lanes(capacity=capacity).report + assert [(use.scope, use.gpu_product, use.ceiling, use.accelerators, use.lanes) for use in report.capacity] == [ + (CapacityScope.TOTAL, None, 4, 4, ("org-a-l4", "org-b-a100-80gb")), + (CapacityScope.TYPE, A100.gpu_product, 2, 2, ("org-b-a100-80gb",)), + (CapacityScope.TYPE, "NVIDIA-L4", 2, 2, ("org-a-l4",)), + (CapacityScope.SHARED_QUOTA, None, 6, 6, ("org-a-l4", "org-b-a100-80gb")), + ] + assert not any(use.exceeded for use in report.capacity) + assert report.to_dict()["capacity"][3] == { + "scope": "shared-quota", + "ceiling": 6, + "accelerators": 6, + "heldElsewhere": 2, + "lanes": ["org-a-l4", "org-b-a100-80gb"], + "exceeded": False, + } + + +@pytest.mark.parametrize( + ("capacity", "message"), + [ + ( + Capacity(types=(TypeCeiling(gpu_product="NVIDIA-L4", accelerators=1),)), + "the envelope maxima of org-a-l4 hold 2 NVIDIA-L4 accelerators, above the type ceiling of 1", + ), + ( + Capacity(total=3), + "the envelope maxima of org-a-l4, org-b-a100-80gb hold 4 accelerators, above the total ceiling of 3", + ), + ( + Capacity(shared_quota=SharedQuota(accelerators=6, held_elsewhere=3)), + "the envelope maxima of org-a-l4, org-b-a100-80gb hold 4 accelerators, " + "which with 3 held elsewhere exceed the shared quota of 6", + ), + ], +) +def test_a_plan_whose_envelope_maxima_exceed_a_ceiling_is_refused_and_says_why(capacity, message): + with pytest.raises(CapacityExceededError) as caught: + two_lanes(capacity=capacity) + assert isinstance(caught.value, PlannerInputError) + assert str(caught.value) == message + (exceeded,) = caught.value.exceeded + assert exceeded.exceeded + refused = caught.value.result + assert [lane.name for lane in refused.plan.lanes] == ["org-a-l4", "org-b-a100-80gb"] + assert [use["exceeded"] for use in refused.report.to_dict()["capacity"]] == [True] + + +def test_every_exceeded_ceiling_is_named(): + capacity = Capacity(total=3, types=(TypeCeiling(gpu_product="NVIDIA-L4", accelerators=1),)) + with pytest.raises(CapacityExceededError) as caught: + two_lanes(capacity=capacity) + assert [use.scope for use in caught.value.exceeded] == [CapacityScope.TOTAL, CapacityScope.TYPE] + assert str(caught.value).count("the envelope maxima") == 2 + + +def test_a_candidate_list_lane_counts_against_every_member_type(): + capacity = Capacity( + types=( + TypeCeiling(gpu_product="NVIDIA-L4", accelerators=1), + TypeCeiling(gpu_product=A100.gpu_product, accelerators=1), + ) + ) + result = run(one(0), [receipt(), receipt(cell=A100)], [*scarce(L4), *scarce(A100)], capacity=capacity) + assert [use.accelerators for use in result.report.capacity] == [1, 1] + capacity = Capacity(types=(TypeCeiling(gpu_product=A100.gpu_product, accelerators=0),)) + with pytest.raises(CapacityExceededError, match="hold 1 NVIDIA-A100-SXM4-80GB accelerators"): + run(one(0), [receipt(), receipt(cell=A100)], [*scarce(L4), *scarce(A100)], capacity=capacity) + + +def test_each_container_holds_its_gpu_count(): + result = run(one(0), [receipt(shape=PAIR)], capacity=Capacity(total=2), buffer_containers=0) + assert [use.accelerators for use in result.report.capacity] == [2] + with pytest.raises(CapacityExceededError, match="hold 2 accelerators"): + run(one(0), [receipt(shape=PAIR)], capacity=Capacity(total=1)) + + +def test_a_zero_floor_lane_holds_nothing_until_its_envelope_may_raise_it(): + result = run(one(600), [receipt()], [attempt(boot=60.0)], capacity=Capacity(total=0)) + assert result.report.identity(IDENTITY).chosen.warmth is Warmth.COLD_WITH_BOUND + assert [(use.accelerators, use.lanes) for use in result.report.capacity] == [(0, ())] + + +def test_a_loosening_whose_plan_the_ceilings_would_refuse_saves_nothing(): + receipts, boots = [receipt()], [attempt(boot=240.0)] + free = run(one(60), receipts, boots, buffer_containers=1, fallback_floor=3) + assert [lever.permitted_bound_s for lever in free.report.come_alive_levers] == [240] + capacity = Capacity(types=(TypeCeiling(gpu_product="NVIDIA-L4", accelerators=2),)) + bounded = run(one(60), receipts, boots, buffer_containers=1, fallback_floor=3, capacity=capacity) + assert bounded.plan == free.plan + assert bounded.report.come_alive_levers == () + assert bounded.report.identity(IDENTITY).next_cheaper.saving == 0 + with pytest.raises(CapacityExceededError): + run(one(240), receipts, boots, buffer_containers=1, fallback_floor=3, capacity=capacity, current=free.plan) + + +def test_without_capacity_the_report_lists_no_ceiling(): + report = two_lanes().report + assert report.capacity == () + assert report.to_dict()["capacity"] == [] + + +CELLS = (L4, A100, H100) +FIELDS = {"comeAliveP95S": "come_alive_p95_s", "warmP99Ms": "warm_p99_ms"} + + +@pytest.mark.parametrize("seed", range(120)) +def test_random_estates_respect_placement_evidence_and_capacity(seed): + rng = random.Random(seed) # noqa: S311 -- reproducible test data + identities = [f"org/model-{index}" for index in range(rng.randint(1, 4))] + service_levels = ServiceLevels( + identities=tuple( + IdentityServiceLevel( + identity=identity, + level=ServiceLevel( + come_alive_p95_s=rng.choice([None, 0, 10, 30, 120, 600]), + warm_p99_ms=rng.choice([150, 250, 400]), + ), + ) + for identity in identities + ) + ) + receipts = [ + receipt( + identity, cell, concurrency=rng.choice([4, 8]), rps=rng.choice([0.5, 5.0]), p99=rng.choice([100.0, 300.0]) + ) + for identity in identities + for cell in CELLS + if rng.random() < 0.6 + ] + boots = [] + for identity in identities: + for cell in CELLS: + for warmth in (Warmth.COLD_WITH_BOUND, Warmth.SNAPSHOT_COLD): + for index in range(rng.randint(0, 3)): + outcome = rng.choice( + [BootOutcome.SUCCEEDED, BootOutcome.SUCCEEDED, BootOutcome.FAILED, BootOutcome.UNPLACED] + ) + boots.append( + attempt( + identity, + cell, + warmth=warmth, + boot=rng.choice([5.0, 20.0, 90.0]), + wait=rng.choice([None, 0.0, 15.0, 120.0]), + outcome=outcome, + borrowed=rng.random() < 0.2, + index=index, + ) + ) + for index in range(rng.randint(0, 6)): + boots.append(unplaced(rng.choice(CELLS), index=index)) + hold = rng.choice([0, 20, 60]) + capacity = rng.choice( + [ + None, + Capacity(total=rng.randint(0, 8)), + Capacity( + types=tuple(TypeCeiling(gpu_product=cell.gpu_product, accelerators=rng.randint(0, 4)) for cell in CELLS) + ), + Capacity(shared_quota=SharedQuota(accelerators=8, held_elsewhere=rng.randint(0, 8))), + ] + ) + overrides = {"hold_budget_s": hold, "buffer_containers": rng.randint(0, 1)} + hourly = { + identity: {hour: rng.choice([1, 500]) for hour in rng.sample(range(24), rng.randint(0, 3))} + for identity in identities + } + + def attempt_plan(current=None, order=False, given_levels=service_levels): + given_receipts, given_boots = list(receipts), list(boots) + if order: + rng.shuffle(given_receipts) + given_boots.reverse() + try: + return run( + given_levels, + given_receipts, + given_boots, + hourly=hourly, + current=current, + capacity=capacity, + **overrides, + ), None + except CapacityExceededError as error: + return error.result, error + + def relaxed(identity, blockers): + bounds = {} + for blocker in blockers: + if blocker.field == "placementFailureRate": + name, value = "come_alive_p95_s", hold + 1 + elif blocker.measured is not None: + name, value = FIELDS[blocker.field], max(1, math.ceil(blocker.measured)) + else: + continue + bounds[name] = max(value, bounds.get(name, 0)) + return ServiceLevels( + identities=tuple( + replace(entry, level=replace(entry.level, **bounds)) if entry.identity == identity else entry + for entry in service_levels.identities + ) + ) + + def fall(identity, blockers): + again, again_error = attempt_plan(current=result.plan, given_levels=relaxed(identity, blockers)) + return 0.0 if again_error is not None else max(0.0, result.report.monthly_cost - again.report.monthly_cost) + + result, error = attempt_plan() + attempts: dict[str, list[BootEvidence]] = {} + for row in boots: + if row.warmth in (Warmth.COLD_WITH_BOUND, Warmth.SNAPSHOT_COLD): + attempts.setdefault(row.cell.gpu_product, []).append(row) + + def failure_rate(product): + rows = attempts.get(product, []) + own = [row for row in rows if not row.borrowed] or rows + return sum(row.outcome is BootOutcome.UNPLACED for row in own) / len(own) if own else 0.0 + + by_uri = {document.source.uri: document for document in receipts} + levels_by_identity = {entry.identity: entry.level for entry in service_levels.identities} + for entry in result.report.identities: + if entry.chosen is None: + continue + chosen = entry.chosen + level = levels_by_identity[entry.identity] + assert chosen.blocked_by == () + tight = level.come_alive_p95_s is not None and level.come_alive_p95_s <= hold + if tight and not chosen.fallback_cells: + assert failure_rate(chosen.cell.gpu_product) <= 0.05 + if chosen.fallback_cells: + assert chosen.warmth is Warmth.PINNED + assert tight + assert failure_rate(chosen.cell.gpu_product) > 0.05 + (second,) = chosen.fallback_cells + assert second.gpu_product != chosen.cell.gpu_product + assert RATES[second] >= RATES[chosen.cell] + assert any( + document.identity == entry.identity + and document.cell == second + and document.concurrency == chosen.concurrency + and document.latency.p99_ms <= level.warm_p99_ms + for document in receipts + ) + assert by_uri[chosen.receipt.uri].latency.p99_ms <= level.warm_p99_ms + if chosen.warmth is not Warmth.PINNED: + assert level.come_alive_p95_s is None or chosen.come_alive_s <= level.come_alive_p95_s + if chosen.placement_wait_s is not None: + assert chosen.come_alive_s >= chosen.placement_wait_s + for scarcity in result.report.scarcity: + chosen = result.report.identity(scarcity.identity).chosen + if chosen is not None: + assert scarcity.monthly_cost < chosen.monthly_cost + if scarcity.reason is ScarcityReason.UNRELIABLE_TYPE: + assert scarcity.placement_failure_rate > 0.05 + for use in result.report.capacity: + lanes = [ + planned + for planned in result.plan.lanes + if use.scope is not CapacityScope.TYPE or use.gpu_product in {cell.gpu_product for cell in planned.cells} + ] + held = sum(planned.envelope.floor_max * planned.container_shape.gpu_count for planned in lanes) + elsewhere = use.held_elsewhere or 0 + assert use.accelerators == held + elsewhere + assert (error is not None) is any(use.exceeded for use in result.report.capacity) + if error is not None: + assert error.exceeded == tuple(use for use in result.report.capacity if use.exceeded) + reordered, _ = attempt_plan(order=True) + assert rendered(reordered) == rendered(result) + if error is None: + for entry in result.report.identities: + if entry.next_cheaper is not None: + assert entry.next_cheaper.saving == pytest.approx( + fall(entry.identity, entry.next_cheaper.blocked_by), abs=0.01 + ) + for lever in result.report.come_alive_levers: + measured = Blocker(field="comeAliveP95S", bound=None, measured=lever.permitted_bound_s) + assert lever.saving_at_permitted_bound == pytest.approx(fall(lever.identity, [measured]), abs=0.01) + again, again_error = attempt_plan(current=result.plan) + assert again_error is None + assert again.plan == result.plan + assert again.report.unchanged diff --git a/packages/sie_placement/tests/test_planner.py b/packages/sie_placement/tests/test_planner.py index 0141707b1..2bcbd0718 100644 --- a/packages/sie_placement/tests/test_planner.py +++ b/packages/sie_placement/tests/test_planner.py @@ -505,11 +505,23 @@ def test_twenty_or_more_boots_stand_for_the_come_alive_at_their_95th_percentile( assert rendered(run(level(come_alive_p95_s=600), [receipt()], boots[::-1], prices=L4_ONLY)) == rendered(result) -def test_a_failed_boot_withdraws_the_warmth_however_many_boots_succeeded(): - boots = [boot(seconds=50.0, uri=f"boots/{index}.json") for index in range(30)] - boots.append(boot(seconds=0.0, outcome=BootOutcome.FAILED, uri="boots/failed.json")) +@pytest.mark.parametrize( + ("succeeded", "failed", "come_alive"), + [(18, 1, None), (19, 1, 50.0), (30, 1, 50.0), (29, 2, None), (38, 2, 50.0), (37, 3, None)], +) +def test_a_failed_boot_ranks_above_every_success_at_the_95th_percentile(succeeded, failed, come_alive): + boots = [boot(seconds=50.0, uri=f"boots/{index}.json") for index in range(succeeded)] + boots.extend( + boot(seconds=0.0, outcome=BootOutcome.FAILED, uri=f"boots/failed-{index}.json") for index in range(failed) + ) result = run(level(come_alive_p95_s=600), [receipt()], boots, prices=L4_ONLY) - assert result.report.identity(IDENTITY).chosen.warmth is Warmth.PINNED + chosen = result.report.identity(IDENTITY).chosen + if come_alive is None: + assert chosen.warmth is Warmth.PINNED + (request,) = result.report.measurement_requests + assert request.reason is RequestReason.FAILED + else: + assert (chosen.warmth, chosen.come_alive_s) == (Warmth.COLD_WITH_BOUND, come_alive) def test_boot_seconds_render_the_same_whatever_their_number_type(): @@ -1456,6 +1468,7 @@ def test_the_region_multiplier_scales_every_price(): ({"fallback_floor": -1}, "fallbackFloor"), ({"cooldown_s": -1}, "cooldownS"), ({"buffer_containers": -1}, "bufferContainers"), + ({"hold_budget_s": -1}, "holdBudgetS"), ({"hours_per_month": 0}, "hoursPerMonth"), ({"hours_per_month": math.inf}, "hoursPerMonth"), ({"queue_pool": "two words"}, "queuePool"), @@ -1478,6 +1491,7 @@ def test_parameters_echo_into_the_report(): "fallbackFloor": 2, "cooldownS": 900, "bufferContainers": 0, + "holdBudgetS": 0, "hoursPerMonth": 730.0, "queuePool": "default", "bundle": "default", diff --git a/packages/sie_placement/tests/test_schema.py b/packages/sie_placement/tests/test_schema.py index c05a873c3..841c8eead 100644 --- a/packages/sie_placement/tests/test_schema.py +++ b/packages/sie_placement/tests/test_schema.py @@ -10,6 +10,7 @@ import pytest from sie_placement.schema import ( BootEvidence, + Capacity, Cell, ContainerShape, Demand, @@ -27,6 +28,8 @@ SchemaError, ServiceLevel, ServiceLevels, + SharedQuota, + TypeCeiling, Warmth, ) @@ -39,6 +42,7 @@ ("boot-evidence.json", BootEvidence.from_dict), ("price-list.json", PriceList.from_dict), ("demand.json", Demand.from_dict), + ("capacity.json", Capacity.from_dict), ("plan.json", Plan.from_dict), ] @@ -354,6 +358,100 @@ def test_boot_evidence_borrowed_is_a_strict_boolean(value, described): assert str(schema_error(BootEvidence.from_dict, data)) == f"$.borrowed: expected a boolean, got {described}" +def test_boot_evidence_records_an_unplaced_attempt_without_a_boot(): + data = load("boot-evidence.json") + data["outcome"] = "unplaced" + del data["bootSeconds"] + evidence = BootEvidence.from_dict(data) + assert (evidence.outcome.value, evidence.boot_seconds, evidence.placement_wait_s) == ("unplaced", None, 41.2) + assert evidence.to_dict() == data + del data["placementWaitS"] + assert BootEvidence.from_dict(data).to_dict() == data + data["bootSeconds"] = 30 + assert str(schema_error(BootEvidence.from_dict, data)) == ( + "$.bootSeconds: must be omitted for an unplaced attempt, which never booted" + ) + + +@pytest.mark.parametrize("outcome", ["succeeded", "failed", "timed-out"]) +def test_boot_evidence_of_a_placed_attempt_records_its_boot(outcome): + data = load("boot-evidence.json") + data["outcome"] = outcome + del data["bootSeconds"] + assert str(schema_error(BootEvidence.from_dict, data)) == ( + "$.bootSeconds: is required for an attempt that was placed" + ) + + +def test_boot_evidence_placement_waits_are_optional_and_non_negative(): + data = load("boot-evidence.json") + del data["placementWaitS"] + evidence = BootEvidence.from_dict(data) + assert evidence.placement_wait_s is None + assert "placementWaitS" not in evidence.to_dict() + data["placementWaitS"] = -1 + assert str(schema_error(BootEvidence.from_dict, data)) == "$.placementWaitS: must be at least 0, got -1" + + +def test_a_lazy_resident_load_runs_on_a_placed_container(): + data = load("boot-evidence.json") + data["warmth"] = "lazy-resident" + assert str(schema_error(BootEvidence.from_dict, data)) == ( + "$.placementWaitS: must be omitted for a lazy-resident load, which runs on a placed container" + ) + del data["placementWaitS"] + assert BootEvidence.from_dict(data).warmth is Warmth.LAZY_RESIDENT + data["outcome"] = "unplaced" + del data["bootSeconds"] + assert str(schema_error(BootEvidence.from_dict, data)) == ( + "$.outcome: a lazy-resident load runs on a placed container and is never unplaced" + ) + + +def test_capacity_declares_a_total_type_ceilings_or_a_shared_quota(): + assert str(schema_error(Capacity.from_dict, {"kind": "placement-capacity", "schemaVersion": 1})) == ( + "$: must declare total, types or sharedQuota" + ) + assert Capacity(total=0).to_dict() == {"kind": "placement-capacity", "schemaVersion": 1, "total": 0} + quota = Capacity.from_dict( + {"kind": "placement-capacity", "schemaVersion": 1, "sharedQuota": {"accelerators": 4}} + ).shared_quota + assert quota == SharedQuota(accelerators=4, held_elsewhere=0) + assert quota.to_dict() == {"accelerators": 4, "heldElsewhere": 0} + + +@pytest.mark.parametrize( + ("mutate", "message"), + [ + (lambda data: data.update(total=-1), "$.total: must be at least 0, got -1"), + (lambda data: data["types"][1].update(accelerators=-2), "$.types[1].accelerators: must be at least 0, got -2"), + ( + lambda data: data["types"][1].update(gpuProduct="NVIDIA-L4"), + "$.types[1].gpuProduct: duplicates 'NVIDIA-L4'", + ), + (lambda data: data["types"][0].update(gpuProduct="NVIDIA L4"), "$.types[0].gpuProduct: must be a non-empty"), + ( + lambda data: data["sharedQuota"].update(heldElsewhere=-1), + "$.sharedQuota.heldElsewhere: must be at least 0, got -1", + ), + (lambda data: data["sharedQuota"].pop("accelerators"), "$.sharedQuota.accelerators: is required"), + (lambda data: data["types"][0].update(max=3), "$.types[0].max: is not a field of this object"), + ], +) +def test_capacity_ceilings_are_validated(mutate, message): + data = load("capacity.json") + mutate(data) + assert str(schema_error(Capacity.from_dict, data)).startswith(message) + + +def test_capacity_built_in_code_names_type_ceilings_by_gpu_product(): + capacity = Capacity(types=(TypeCeiling(gpu_product="NVIDIA-L4", accelerators=2),)) + assert Capacity.from_dict(json.loads(json.dumps(capacity.to_dict()))) == capacity + with pytest.raises(SchemaError) as caught: + TypeCeiling(gpu_product="", accelerators=1) + assert caught.value.path == "gpuProduct" + + def test_price_lists_validate_currency_rates_and_names(): data = load("price-list.json") data["currency"] = "usd" @@ -523,6 +621,40 @@ def test_plan_accepts_every_zero_floor_warmth(warmth): assert Plan.from_dict(data).identities[2].warmth.value == warmth +def test_plan_lanes_name_fallback_cells_after_their_cell(): + data = load("plan.json") + lane = Plan.from_dict(data).lanes[1] + assert lane.cells == ( + Cell(machine_profile="a100-80gb", gpu_product="NVIDIA-A100-SXM4-80GB"), + Cell(machine_profile="h100", gpu_product="NVIDIA-H100-80GB-HBM3"), + ) + assert "fallbackCells" not in Plan.from_dict(data).lanes[0].to_dict() + assert Plan.from_dict(data).lanes[0].cells == (Plan.from_dict(data).lanes[0].cell,) + + +@pytest.mark.parametrize( + ("cells", "message"), + [ + ( + [{"machineProfile": "a100-80gb", "gpuProduct": "NVIDIA-A100-SXM4-80GB"}], + "$.lanes[1].fallbackCells[0]: repeats 'a100-80gb' with 'NVIDIA-A100-SXM4-80GB', already a cell of this lane", + ), + ( + [ + {"machineProfile": "h100", "gpuProduct": "NVIDIA-H100-80GB-HBM3"}, + {"machineProfile": "h100", "gpuProduct": "NVIDIA-H100-80GB-HBM3"}, + ], + "$.lanes[1].fallbackCells[1]: repeats 'h100' with 'NVIDIA-H100-80GB-HBM3', already a cell of this lane", + ), + ({"machineProfile": "h100"}, "$.lanes[1].fallbackCells: expected an array, got an object"), + ], +) +def test_plan_lane_fallback_cells_are_distinct_from_each_other_and_the_cell(cells, message): + data = load("plan.json") + data["lanes"][1]["fallbackCells"] = cells + assert str(schema_error(Plan.from_dict, data)) == message + + def test_plan_lane_names_and_identities_are_unique(): data = load("plan.json") data["lanes"][1]["name"] = "default-l4-default" From f6f890807809cde4745d51aae747b5592f45fe4d Mon Sep 17 00:00:00 2001 From: krisztian-gajdar Date: Thu, 5 Nov 2026 18:28:23 +0100 Subject: [PATCH 2/5] fix(placement): rank every cold attempt and hold lanes to the ceilings at maxContainers - The placement wait ranks every cold attempt on a GPU product. A placed attempt without placementWaitS counts as one that waited none, so the 20-attempt threshold and the 95th percentile no longer fall on a subset made mostly of unplaced attempts, and agree with the failure rate, which already counted every attempt. A product none of whose attempts records a wait or was unplaced still adds no wait. - The total and per-type ceilings now hold every covered lane at its maxContainers as well as at its floorMax, which is what a deployment compiler enforcing those ceilings refuses. The report gives peakAccelerators for those scopes, and the refusal names the container ceilings that exceed. The shared quota still bounds the envelope maxima. --- packages/sie_placement/README.md | 36 +++--- .../src/sie_placement/planner.py | 35 ++++-- .../sie_placement/src/sie_placement/report.py | 19 ++- .../sie_placement/tests/test_availability.py | 112 ++++++++++++++---- 4 files changed, 143 insertions(+), 59 deletions(-) diff --git a/packages/sie_placement/README.md b/packages/sie_placement/README.md index 304094fcf..caf70a449 100644 --- a/packages/sie_placement/README.md +++ b/packages/sie_placement/README.md @@ -116,9 +116,13 @@ obtain. Each field is optional, and at least one is declared: | `types` | Per `gpuProduct`, the `accelerators` the lanes that may be placed on that product may hold together | | `sharedQuota` | A quota of `accelerators` that this environment shares with others, which hold `heldElsewhere` of it | -A lane holds `floorMax` containers of its `gpuCount` accelerators each. A lane -with fallback cells counts against the ceiling of every product it names, and -once against `total`. +Every container holds the `gpuCount` accelerators of its lane's container +shape. `total` and each of `types` must hold every lane they cover at its +container ceiling, `maxContainers` containers, at once, and so at its envelope +maximum, `floorMax` containers, too. `sharedQuota` bounds the envelope maxima, +the most a tuner may hold warm, together with `heldElsewhere`. A lane with +fallback cells counts against the ceiling of every product it names, and once +against `total`. ## Envelope @@ -147,8 +151,8 @@ result = plan( print(render(result.plan.to_dict()), render(result.report.to_dict())) ``` -`capacity` is optional. A plan whose envelope maxima exceed a capacity ceiling -is not returned: `plan` raises `CapacityExceededError`, a `PlannerInputError` +`capacity` is optional. A plan whose lanes exceed a capacity ceiling, as +[Capacity](#capacity) states, is not returned: `plan` raises `CapacityExceededError`, a `PlannerInputError` whose message names each exceeded ceiling and the lanes that hold it, whose `exceeded` lists those ceilings, and whose `result` carries the refused plan and its report. @@ -194,12 +198,14 @@ document: same percentile. The boot is taken over the identity's attempts at that cell, container shape and warmth that got a container. The placement wait is taken over every cold attempt on the cell's GPU product, whatever the - identity, that records `placementWaitS` or was `unplaced`; a product - without such an attempt adds no wait. Below 20 attempts the slowest stands - for the percentile; from 20, their 95th percentile by nearest rank does, so - one slow attempt stops deciding once enough exist. An attempt that did not - succeed, a failed or timed-out boot or an unplaced placement, ranks above - every success, so the warmth is withdrawn when the percentile falls on one. + identity. An attempt that got a container without recording + `placementWaitS` waited none, and a product none of whose cold attempts + records `placementWaitS` or was `unplaced` adds no wait. Below 20 attempts + the slowest stands for the percentile; from 20, their 95th percentile by + nearest rank does, so one slow attempt stops deciding once enough exist. An + attempt that did not succeed, a failed or timed-out boot or an unplaced + placement, ranks above every success, so the warmth is withdrawn when the + percentile falls on one. The environment's own attempts replace borrowed ones, per cell, container shape and warmth for the boot and per GPU product for the placement wait. A `comeAliveP95S` of `0` admits `pinned` only. @@ -292,8 +298,9 @@ the ceiling. A floor raised on a zero-floor lane realizes as at least `fallback_floor`, so a zero-floor lane's nonzero `floorMax` is at least `fallback_floor` capped by the ceiling. The envelope carries the floor range only. A lane carried from the current plan keeps its envelope, capped by the -ceiling. Given a `capacity` document, the envelope maxima of every lane must -fit each of its ceilings, or the plan is refused as described above. A +ceiling. Given a `capacity` document, the lanes must fit each of its ceilings, +at their container ceilings and envelope maxima as [Capacity](#capacity) +states, or the plan is refused as described above. A loosened service level whose plan the ceilings would refuse saves nothing in the report. @@ -336,7 +343,8 @@ The report renders as a `placement-report` document: - `capacity`: for each ceiling of the `capacity` document, its `scope` (`total`, `type` with its `gpuProduct`, or `shared-quota` with `heldElsewhere`), the `ceiling`, the `accelerators` the plan's envelope - maxima hold against it, the `lanes` that hold them, and whether it is + maxima hold against it, for `total` and `type` the `peakAccelerators` its + lanes hold at `maxContainers`, the `lanes` that hold them, and whether it is `exceeded`. - `lanes`: each lane's hourly cost, its monthly cost, its floor's monthly cost and the monthly cost of its `floorMax`, which a floor budget prices. A lane diff --git a/packages/sie_placement/src/sie_placement/planner.py b/packages/sie_placement/src/sie_placement/planner.py index bf286ef09..48e5ff752 100644 --- a/packages/sie_placement/src/sie_placement/planner.py +++ b/packages/sie_placement/src/sie_placement/planner.py @@ -439,14 +439,13 @@ def _placement_index(records: Iterable[BootEvidence]) -> dict[str, _Placement]: own = [record for record in group if not record.borrowed] used = own or group samples = [ - None if record.outcome is BootOutcome.UNPLACED else float(record.placement_wait_s or 0.0) - for record in used - if record.outcome is BootOutcome.UNPLACED or record.placement_wait_s is not None + None if record.outcome is BootOutcome.UNPLACED else float(record.placement_wait_s or 0.0) for record in used ] + measured = any(record.outcome is BootOutcome.UNPLACED or record.placement_wait_s is not None for record in used) unplaced = sum(record.outcome is BootOutcome.UNPLACED for record in used) index[product] = _Placement( - wait_s=_percentile(samples) if samples else 0.0, - measured=bool(samples), + wait_s=_percentile(samples) if measured else 0.0, + measured=measured, failure_rate=unplaced / len(used), borrowed=not own, ) @@ -1138,17 +1137,26 @@ def capacity_use(self, plan: Plan) -> tuple[CapacityUse, ...]: if capacity is None: return () - def held(lanes: Iterable[PlanLane]) -> tuple[int, tuple[str, ...]]: - counted = [lane for lane in lanes if lane.envelope.floor_max] - accelerators = sum(lane.envelope.floor_max * lane.container_shape.gpu_count for lane in counted) - return accelerators, tuple(lane.name for lane in counted) + def held(lanes: Iterable[PlanLane]) -> tuple[int, int, tuple[str, ...]]: + lanes = tuple(lanes) + envelope = sum(lane.envelope.floor_max * lane.container_shape.gpu_count for lane in lanes) + peak = sum(lane.max_containers * lane.container_shape.gpu_count for lane in lanes) + return envelope, peak, tuple(lane.name for lane in lanes) - total, names = held(plan.lanes) + total, peak, names = held(plan.lanes) uses = [] if capacity.total is not None: - uses.append(CapacityUse(scope=CapacityScope.TOTAL, ceiling=capacity.total, accelerators=total, lanes=names)) + uses.append( + CapacityUse( + scope=CapacityScope.TOTAL, + ceiling=capacity.total, + accelerators=total, + peak_accelerators=peak, + lanes=names, + ) + ) for ceiling in sorted(capacity.types, key=lambda entry: entry.gpu_product): - accelerators, lanes = held( + accelerators, peak_accelerators, lanes = held( lane for lane in plan.lanes if any(cell.gpu_product == ceiling.gpu_product for cell in lane.cells) ) uses.append( @@ -1157,6 +1165,7 @@ def held(lanes: Iterable[PlanLane]) -> tuple[int, tuple[str, ...]]: gpu_product=ceiling.gpu_product, ceiling=ceiling.accelerators, accelerators=accelerators, + peak_accelerators=peak_accelerators, lanes=lanes, ) ) @@ -1168,7 +1177,7 @@ def held(lanes: Iterable[PlanLane]) -> tuple[int, tuple[str, ...]]: ceiling=quota.accelerators, accelerators=total + quota.held_elsewhere, held_elsewhere=quota.held_elsewhere, - lanes=names, + lanes=tuple(lane.name for lane in plan.lanes if lane.envelope.floor_max), ) ) return tuple(uses) diff --git a/packages/sie_placement/src/sie_placement/report.py b/packages/sie_placement/src/sie_placement/report.py index d63ffc25d..29bbb4a34 100644 --- a/packages/sie_placement/src/sie_placement/report.py +++ b/packages/sie_placement/src/sie_placement/report.py @@ -315,28 +315,26 @@ class CapacityUse: lanes: tuple[str, ...] gpu_product: str | None = None held_elsewhere: int | None = None + peak_accelerators: int | None = None @property def exceeded(self) -> bool: - return self.accelerators > self.ceiling + return max(self.accelerators, self.peak_accelerators or 0) > self.ceiling def describe(self) -> str: lanes = ", ".join(self.lanes) or "no lane" - if self.scope is CapacityScope.TYPE: - return ( - f"the envelope maxima of {lanes} hold {self.accelerators} {self.gpu_product} accelerators, " - f"above the type ceiling of {self.ceiling}" - ) if self.scope is CapacityScope.SHARED_QUOTA: elsewhere = self.held_elsewhere or 0 return ( f"the envelope maxima of {lanes} hold {self.accelerators - elsewhere} accelerators, " f"which with {elsewhere} held elsewhere exceed the shared quota of {self.ceiling}" ) - return ( - f"the envelope maxima of {lanes} hold {self.accelerators} accelerators, " - f"above the total ceiling of {self.ceiling}" - ) + accelerators = "accelerators" if self.gpu_product is None else f"{self.gpu_product} accelerators" + held = f"the envelope maxima of {lanes} hold {self.accelerators}" + if self.peak_accelerators is not None and self.peak_accelerators > self.ceiling: + held = f"the container ceilings of {lanes} hold {self.peak_accelerators}" + scope = "type" if self.scope is CapacityScope.TYPE else "total" + return f"{held} {accelerators}, above the {scope} ceiling of {self.ceiling}" def to_dict(self) -> dict[str, object]: return _compact( @@ -345,6 +343,7 @@ def to_dict(self) -> dict[str, object]: "gpuProduct": self.gpu_product, "ceiling": self.ceiling, "accelerators": self.accelerators, + "peakAccelerators": self.peak_accelerators, "heldElsewhere": self.held_elsewhere, "lanes": list(self.lanes), "exceeded": self.exceeded, diff --git a/packages/sie_placement/tests/test_availability.py b/packages/sie_placement/tests/test_availability.py index eac0a3852..ae226c927 100644 --- a/packages/sie_placement/tests/test_availability.py +++ b/packages/sie_placement/tests/test_availability.py @@ -225,7 +225,7 @@ def test_placement_waits_pool_over_every_attempt_on_a_gpu_type(): assert entry.to_dict()["reason"] == "placement-wait" -@pytest.mark.parametrize(("fast", "come_alive"), [(18, None), (19, 60.0), (37, None), (38, 60.0)]) +@pytest.mark.parametrize(("fast", "come_alive"), [(17, None), (18, 60.0), (36, None), (37, 60.0)]) def test_the_placement_wait_takes_the_95th_percentile_from_twenty_attempts(fast, come_alive): boots = [attempt(boot=50.0)] boots.extend(attempt(OTHER, boot=1.0, wait=10.0, index=index) for index in range(fast)) @@ -261,6 +261,35 @@ def test_an_unplaced_attempt_ranks_above_every_placed_one(placed, selected): assert entry.placement_failure_rate == pytest.approx(1 / (placed + 2)) +@pytest.mark.parametrize(("placed", "selected"), [(17, False), (18, True), (100, True)]) +def test_a_placed_attempt_without_a_placement_wait_ranks_as_one_that_waited_none(placed, selected): + boots = [attempt(boot=50.0)] + boots.extend(attempt(OTHER, boot=1.0, wait=0.0 if index % 2 else None, index=index) for index in range(placed)) + boots.append(unplaced(index=999)) + result = run(one(120), [receipt()], boots) + chosen = result.report.identity(IDENTITY).chosen + if not selected: + assert chosen.warmth is Warmth.PINNED + (entry,) = result.report.scarcity + assert entry.reason is ScarcityReason.UNPLACED + return + assert (chosen.warmth, chosen.come_alive_s, chosen.placement_wait_s) == (Warmth.COLD_WITH_BOUND, 50.0, 0.0) + assert result.report.scarcity == () + assert not [request for request in result.report.measurement_requests if request.cell == L4] + + +def test_placed_attempts_without_a_placement_wait_count_toward_the_failure_rate_and_the_percentile_alike(): + boots = [attempt(boot=50.0), *(attempt(OTHER, boot=1.0, index=index) for index in range(99))] + boots.append(unplaced(index=999)) + result = run(one(120), [receipt()], boots) + assert result.report.identity(IDENTITY).chosen.come_alive_s == 50.0 + boots = [*boots, *(unplaced(index=1000 + index) for index in range(5))] + result = run(one(0), [receipt(), receipt(cell=A100)], boots) + assert result.report.identity(IDENTITY).chosen.cell == A100 + (entry,) = result.report.scarcity + assert (entry.reason, entry.placement_failure_rate) == (ScarcityReason.UNRELIABLE_TYPE, pytest.approx(6 / 106)) + + def test_an_attempt_without_a_placement_wait_leaves_its_boot_to_stand_for_the_come_alive(): result = run(one(120), [receipt()], [attempt(boot=95.0)]) chosen = result.report.identity(IDENTITY).chosen @@ -488,7 +517,7 @@ def two_lanes(**overrides): return run(levels(**{"org/a": 0, "org/b": 0}), receipts, buffer_containers=1, **overrides) -def test_capacity_reports_what_the_envelope_maxima_hold_against_each_ceiling(): +def test_capacity_reports_what_the_envelope_maxima_and_container_ceilings_hold_against_each_ceiling(): capacity = Capacity( total=4, types=( @@ -497,14 +526,25 @@ def test_capacity_reports_what_the_envelope_maxima_hold_against_each_ceiling(): ), shared_quota=SharedQuota(accelerators=6, held_elsewhere=2), ) - report = two_lanes(capacity=capacity).report - assert [(use.scope, use.gpu_product, use.ceiling, use.accelerators, use.lanes) for use in report.capacity] == [ - (CapacityScope.TOTAL, None, 4, 4, ("org-a-l4", "org-b-a100-80gb")), - (CapacityScope.TYPE, A100.gpu_product, 2, 2, ("org-b-a100-80gb",)), - (CapacityScope.TYPE, "NVIDIA-L4", 2, 2, ("org-a-l4",)), - (CapacityScope.SHARED_QUOTA, None, 6, 6, ("org-a-l4", "org-b-a100-80gb")), + report = two_lanes(capacity=capacity, max_containers=2).report + assert [ + (use.scope, use.gpu_product, use.ceiling, use.accelerators, use.peak_accelerators, use.lanes) + for use in report.capacity + ] == [ + (CapacityScope.TOTAL, None, 4, 4, 4, ("org-a-l4", "org-b-a100-80gb")), + (CapacityScope.TYPE, A100.gpu_product, 2, 2, 2, ("org-b-a100-80gb",)), + (CapacityScope.TYPE, "NVIDIA-L4", 2, 2, 2, ("org-a-l4",)), + (CapacityScope.SHARED_QUOTA, None, 6, 6, None, ("org-a-l4", "org-b-a100-80gb")), ] assert not any(use.exceeded for use in report.capacity) + assert report.to_dict()["capacity"][0] == { + "scope": "total", + "ceiling": 4, + "accelerators": 4, + "peakAccelerators": 4, + "lanes": ["org-a-l4", "org-b-a100-80gb"], + "exceeded": False, + } assert report.to_dict()["capacity"][3] == { "scope": "shared-quota", "ceiling": 6, @@ -520,11 +560,15 @@ def test_capacity_reports_what_the_envelope_maxima_hold_against_each_ceiling(): [ ( Capacity(types=(TypeCeiling(gpu_product="NVIDIA-L4", accelerators=1),)), - "the envelope maxima of org-a-l4 hold 2 NVIDIA-L4 accelerators, above the type ceiling of 1", + "the container ceilings of org-a-l4 hold 4 NVIDIA-L4 accelerators, above the type ceiling of 1", ), ( Capacity(total=3), - "the envelope maxima of org-a-l4, org-b-a100-80gb hold 4 accelerators, above the total ceiling of 3", + "the container ceilings of org-a-l4, org-b-a100-80gb hold 8 accelerators, above the total ceiling of 3", + ), + ( + Capacity(total=4), + "the container ceilings of org-a-l4, org-b-a100-80gb hold 8 accelerators, above the total ceiling of 4", ), ( Capacity(shared_quota=SharedQuota(accelerators=6, held_elsewhere=3)), @@ -533,7 +577,7 @@ def test_capacity_reports_what_the_envelope_maxima_hold_against_each_ceiling(): ), ], ) -def test_a_plan_whose_envelope_maxima_exceed_a_ceiling_is_refused_and_says_why(capacity, message): +def test_a_plan_whose_lanes_exceed_a_ceiling_is_refused_and_says_why(capacity, message): with pytest.raises(CapacityExceededError) as caught: two_lanes(capacity=capacity) assert isinstance(caught.value, PlannerInputError) @@ -550,7 +594,19 @@ def test_every_exceeded_ceiling_is_named(): with pytest.raises(CapacityExceededError) as caught: two_lanes(capacity=capacity) assert [use.scope for use in caught.value.exceeded] == [CapacityScope.TOTAL, CapacityScope.TYPE] - assert str(caught.value).count("the envelope maxima") == 2 + assert str(caught.value).count("the container ceilings") == 2 + + +def test_the_container_ceilings_bind_where_the_envelope_maxima_fit(): + capacity = Capacity(total=4, types=(TypeCeiling(gpu_product="NVIDIA-L4", accelerators=2),)) + with pytest.raises(CapacityExceededError) as caught: + two_lanes(capacity=capacity) + assert [(use.scope, use.accelerators, use.peak_accelerators) for use in caught.value.exceeded] == [ + (CapacityScope.TOTAL, 4, 8), + (CapacityScope.TYPE, 2, 4), + ] + result = two_lanes(capacity=capacity, max_containers=2) + assert [planned.max_containers for planned in result.plan.lanes] == [2, 2] def test_a_candidate_list_lane_counts_against_every_member_type(): @@ -560,31 +616,36 @@ def test_a_candidate_list_lane_counts_against_every_member_type(): TypeCeiling(gpu_product=A100.gpu_product, accelerators=1), ) ) - result = run(one(0), [receipt(), receipt(cell=A100)], [*scarce(L4), *scarce(A100)], capacity=capacity) - assert [use.accelerators for use in result.report.capacity] == [1, 1] + boots = [*scarce(L4), *scarce(A100)] + result = run(one(0), [receipt(), receipt(cell=A100)], boots, capacity=capacity, max_containers=1) + assert [(use.accelerators, use.peak_accelerators) for use in result.report.capacity] == [(1, 1), (1, 1)] capacity = Capacity(types=(TypeCeiling(gpu_product=A100.gpu_product, accelerators=0),)) with pytest.raises(CapacityExceededError, match="hold 1 NVIDIA-A100-SXM4-80GB accelerators"): - run(one(0), [receipt(), receipt(cell=A100)], [*scarce(L4), *scarce(A100)], capacity=capacity) + run(one(0), [receipt(), receipt(cell=A100)], boots, capacity=capacity, max_containers=1) def test_each_container_holds_its_gpu_count(): - result = run(one(0), [receipt(shape=PAIR)], capacity=Capacity(total=2), buffer_containers=0) - assert [use.accelerators for use in result.report.capacity] == [2] + result = run(one(0), [receipt(shape=PAIR)], capacity=Capacity(total=2), buffer_containers=0, max_containers=1) + assert [(use.accelerators, use.peak_accelerators) for use in result.report.capacity] == [(2, 2)] with pytest.raises(CapacityExceededError, match="hold 2 accelerators"): - run(one(0), [receipt(shape=PAIR)], capacity=Capacity(total=1)) + run(one(0), [receipt(shape=PAIR)], capacity=Capacity(total=1), max_containers=1) -def test_a_zero_floor_lane_holds_nothing_until_its_envelope_may_raise_it(): - result = run(one(600), [receipt()], [attempt(boot=60.0)], capacity=Capacity(total=0)) +def test_a_zero_floor_lane_holds_no_envelope_maximum_but_holds_its_container_ceiling(): + result = run(one(600), [receipt()], [attempt(boot=60.0)], capacity=Capacity(total=4)) assert result.report.identity(IDENTITY).chosen.warmth is Warmth.COLD_WITH_BOUND - assert [(use.accelerators, use.lanes) for use in result.report.capacity] == [(0, ())] + assert [(use.accelerators, use.peak_accelerators, use.lanes) for use in result.report.capacity] == [ + (0, 4, ("org-model-l4",)) + ] + with pytest.raises(CapacityExceededError, match="the container ceilings of org-model-l4 hold 4 accelerators"): + run(one(600), [receipt()], [attempt(boot=60.0)], capacity=Capacity(total=3)) def test_a_loosening_whose_plan_the_ceilings_would_refuse_saves_nothing(): receipts, boots = [receipt()], [attempt(boot=240.0)] free = run(one(60), receipts, boots, buffer_containers=1, fallback_floor=3) assert [lever.permitted_bound_s for lever in free.report.come_alive_levers] == [240] - capacity = Capacity(types=(TypeCeiling(gpu_product="NVIDIA-L4", accelerators=2),)) + capacity = Capacity(shared_quota=SharedQuota(accelerators=2)) bounded = run(one(60), receipts, boots, buffer_containers=1, fallback_floor=3, capacity=capacity) assert bounded.plan == free.plan assert bounded.report.come_alive_levers == () @@ -761,6 +822,13 @@ def failure_rate(product): held = sum(planned.envelope.floor_max * planned.container_shape.gpu_count for planned in lanes) elsewhere = use.held_elsewhere or 0 assert use.accelerators == held + elsewhere + if use.scope is CapacityScope.SHARED_QUOTA: + assert use.peak_accelerators is None + assert use.exceeded is (use.accelerators > use.ceiling) + else: + peak = sum(planned.max_containers * planned.container_shape.gpu_count for planned in lanes) + assert use.peak_accelerators == peak + assert use.exceeded is (peak > use.ceiling) assert (error is not None) is any(use.exceeded for use in result.report.capacity) if error is not None: assert error.exceeded == tuple(use for use in result.report.capacity if use.exceeded) From 1d4809bb82aa784f94f416bfbd74b97beb9dc1ac Mon Sep 17 00:00:00 2001 From: krisztian-gajdar Date: Thu, 5 Nov 2026 19:01:26 +0100 Subject: [PATCH 3/5] fix(placement): hold lanes to the shared quota at maxContainers - The shared quota now holds every lane at its container ceiling, with the accelerators held elsewhere, as the total and per-type ceilings already do. The report gives peakAccelerators for every scope, and the refusal names every lane the quota covers and the container ceilings that exceed it. - A plan that does not fit is refused whole: the refused plan keeps every lane's maxContainers, which the tests now pin. - A loosening whose new lane would exceed the quota at its container ceiling saves nothing, while the carried lane it would leave fits. --- packages/sie_placement/README.md | 20 +++---- .../src/sie_placement/planner.py | 3 +- .../sie_placement/src/sie_placement/report.py | 10 ++-- .../sie_placement/tests/test_availability.py | 53 ++++++++++++++----- 4 files changed, 58 insertions(+), 28 deletions(-) diff --git a/packages/sie_placement/README.md b/packages/sie_placement/README.md index caf70a449..4e39543e6 100644 --- a/packages/sie_placement/README.md +++ b/packages/sie_placement/README.md @@ -117,12 +117,14 @@ obtain. Each field is optional, and at least one is declared: | `sharedQuota` | A quota of `accelerators` that this environment shares with others, which hold `heldElsewhere` of it | Every container holds the `gpuCount` accelerators of its lane's container -shape. `total` and each of `types` must hold every lane they cover at its -container ceiling, `maxContainers` containers, at once, and so at its envelope -maximum, `floorMax` containers, too. `sharedQuota` bounds the envelope maxima, -the most a tuner may hold warm, together with `heldElsewhere`. A lane with -fallback cells counts against the ceiling of every product it names, and once -against `total`. +shape. Each ceiling must hold every lane it covers at its container ceiling, +`maxContainers` containers, at once, as a deployment compiler that enforces the +ceiling charges it, and so at its envelope maximum, `floorMax` containers, too. +`total` and `sharedQuota` cover every lane, and `sharedQuota` holds them +together with its `heldElsewhere` accelerators. A lane with fallback cells +counts against the ceiling of every product it names, and once against `total` +and `sharedQuota`. The planner never lowers a lane's `maxContainers` or +`floorMax` to fit a capacity document: a plan that does not fit is refused. ## Envelope @@ -343,9 +345,9 @@ The report renders as a `placement-report` document: - `capacity`: for each ceiling of the `capacity` document, its `scope` (`total`, `type` with its `gpuProduct`, or `shared-quota` with `heldElsewhere`), the `ceiling`, the `accelerators` the plan's envelope - maxima hold against it, for `total` and `type` the `peakAccelerators` its - lanes hold at `maxContainers`, the `lanes` that hold them, and whether it is - `exceeded`. + maxima hold against it, the `peakAccelerators` its lanes hold at + `maxContainers`, both with `heldElsewhere` for `shared-quota`, the `lanes` + that hold them, and whether it is `exceeded`. - `lanes`: each lane's hourly cost, its monthly cost, its floor's monthly cost and the monthly cost of its `floorMax`, which a floor budget prices. A lane on a cell the price list cannot price carries `unpricedReason` and no cost, diff --git a/packages/sie_placement/src/sie_placement/planner.py b/packages/sie_placement/src/sie_placement/planner.py index 48e5ff752..505dff824 100644 --- a/packages/sie_placement/src/sie_placement/planner.py +++ b/packages/sie_placement/src/sie_placement/planner.py @@ -1176,8 +1176,9 @@ def held(lanes: Iterable[PlanLane]) -> tuple[int, int, tuple[str, ...]]: scope=CapacityScope.SHARED_QUOTA, ceiling=quota.accelerators, accelerators=total + quota.held_elsewhere, + peak_accelerators=peak + quota.held_elsewhere, held_elsewhere=quota.held_elsewhere, - lanes=tuple(lane.name for lane in plan.lanes if lane.envelope.floor_max), + lanes=names, ) ) return tuple(uses) diff --git a/packages/sie_placement/src/sie_placement/report.py b/packages/sie_placement/src/sie_placement/report.py index 29bbb4a34..6d4a81bc1 100644 --- a/packages/sie_placement/src/sie_placement/report.py +++ b/packages/sie_placement/src/sie_placement/report.py @@ -323,18 +323,18 @@ def exceeded(self) -> bool: def describe(self) -> str: lanes = ", ".join(self.lanes) or "no lane" + held, count = "envelope maxima", self.accelerators + if self.peak_accelerators is not None and self.peak_accelerators > self.ceiling: + held, count = "container ceilings", self.peak_accelerators if self.scope is CapacityScope.SHARED_QUOTA: elsewhere = self.held_elsewhere or 0 return ( - f"the envelope maxima of {lanes} hold {self.accelerators - elsewhere} accelerators, " + f"the {held} of {lanes} hold {count - elsewhere} accelerators, " f"which with {elsewhere} held elsewhere exceed the shared quota of {self.ceiling}" ) accelerators = "accelerators" if self.gpu_product is None else f"{self.gpu_product} accelerators" - held = f"the envelope maxima of {lanes} hold {self.accelerators}" - if self.peak_accelerators is not None and self.peak_accelerators > self.ceiling: - held = f"the container ceilings of {lanes} hold {self.peak_accelerators}" scope = "type" if self.scope is CapacityScope.TYPE else "total" - return f"{held} {accelerators}, above the {scope} ceiling of {self.ceiling}" + return f"the {held} of {lanes} hold {count} {accelerators}, above the {scope} ceiling of {self.ceiling}" def to_dict(self) -> dict[str, object]: return _compact( diff --git a/packages/sie_placement/tests/test_availability.py b/packages/sie_placement/tests/test_availability.py index ae226c927..0304772b6 100644 --- a/packages/sie_placement/tests/test_availability.py +++ b/packages/sie_placement/tests/test_availability.py @@ -534,7 +534,7 @@ def test_capacity_reports_what_the_envelope_maxima_and_container_ceilings_hold_a (CapacityScope.TOTAL, None, 4, 4, 4, ("org-a-l4", "org-b-a100-80gb")), (CapacityScope.TYPE, A100.gpu_product, 2, 2, 2, ("org-b-a100-80gb",)), (CapacityScope.TYPE, "NVIDIA-L4", 2, 2, 2, ("org-a-l4",)), - (CapacityScope.SHARED_QUOTA, None, 6, 6, None, ("org-a-l4", "org-b-a100-80gb")), + (CapacityScope.SHARED_QUOTA, None, 6, 6, 6, ("org-a-l4", "org-b-a100-80gb")), ] assert not any(use.exceeded for use in report.capacity) assert report.to_dict()["capacity"][0] == { @@ -549,6 +549,7 @@ def test_capacity_reports_what_the_envelope_maxima_and_container_ceilings_hold_a "scope": "shared-quota", "ceiling": 6, "accelerators": 6, + "peakAccelerators": 6, "heldElsewhere": 2, "lanes": ["org-a-l4", "org-b-a100-80gb"], "exceeded": False, @@ -572,9 +573,14 @@ def test_capacity_reports_what_the_envelope_maxima_and_container_ceilings_hold_a ), ( Capacity(shared_quota=SharedQuota(accelerators=6, held_elsewhere=3)), - "the envelope maxima of org-a-l4, org-b-a100-80gb hold 4 accelerators, " + "the container ceilings of org-a-l4, org-b-a100-80gb hold 8 accelerators, " "which with 3 held elsewhere exceed the shared quota of 6", ), + ( + Capacity(shared_quota=SharedQuota(accelerators=10, held_elsewhere=3)), + "the container ceilings of org-a-l4, org-b-a100-80gb hold 8 accelerators, " + "which with 3 held elsewhere exceed the shared quota of 10", + ), ], ) def test_a_plan_whose_lanes_exceed_a_ceiling_is_refused_and_says_why(capacity, message): @@ -598,17 +604,37 @@ def test_every_exceeded_ceiling_is_named(): def test_the_container_ceilings_bind_where_the_envelope_maxima_fit(): - capacity = Capacity(total=4, types=(TypeCeiling(gpu_product="NVIDIA-L4", accelerators=2),)) + capacity = Capacity( + total=4, + types=(TypeCeiling(gpu_product="NVIDIA-L4", accelerators=2),), + shared_quota=SharedQuota(accelerators=7, held_elsewhere=3), + ) with pytest.raises(CapacityExceededError) as caught: two_lanes(capacity=capacity) assert [(use.scope, use.accelerators, use.peak_accelerators) for use in caught.value.exceeded] == [ (CapacityScope.TOTAL, 4, 8), (CapacityScope.TYPE, 2, 4), + (CapacityScope.SHARED_QUOTA, 7, 11), ] result = two_lanes(capacity=capacity, max_containers=2) assert [planned.max_containers for planned in result.plan.lanes] == [2, 2] +def test_a_refused_plan_keeps_every_container_ceiling_it_would_need(): + capacity = Capacity(total=5, shared_quota=SharedQuota(accelerators=5)) + with pytest.raises(CapacityExceededError) as caught: + two_lanes(capacity=capacity) + refused = caught.value.result.plan + assert [(planned.name, planned.max_containers) for planned in refused.lanes] == [ + ("org-a-l4", 4), + ("org-b-a100-80gb", 4), + ] + assert refused == two_lanes().plan + for use in caught.value.exceeded: + assert use.lanes == ("org-a-l4", "org-b-a100-80gb") + assert "org-a-l4, org-b-a100-80gb hold 8 accelerators" in use.describe() + + def test_a_candidate_list_lane_counts_against_every_member_type(): capacity = Capacity( types=( @@ -643,14 +669,18 @@ def test_a_zero_floor_lane_holds_no_envelope_maximum_but_holds_its_container_cei def test_a_loosening_whose_plan_the_ceilings_would_refuse_saves_nothing(): receipts, boots = [receipt()], [attempt(boot=240.0)] - free = run(one(60), receipts, boots, buffer_containers=1, fallback_floor=3) + current = plan_of( + [replace(lane("org-model-l4", buffer=1), max_containers=2)], {IDENTITY: ("org-model-l4", Warmth.PINNED)} + ) + free = run(one(60), receipts, boots, buffer_containers=1, fallback_floor=3, current=current) + assert free.plan == current assert [lever.permitted_bound_s for lever in free.report.come_alive_levers] == [240] capacity = Capacity(shared_quota=SharedQuota(accelerators=2)) - bounded = run(one(60), receipts, boots, buffer_containers=1, fallback_floor=3, capacity=capacity) + bounded = run(one(60), receipts, boots, buffer_containers=1, fallback_floor=3, capacity=capacity, current=current) assert bounded.plan == free.plan assert bounded.report.come_alive_levers == () assert bounded.report.identity(IDENTITY).next_cheaper.saving == 0 - with pytest.raises(CapacityExceededError): + with pytest.raises(CapacityExceededError, match="the container ceilings of org-model-l4 hold 4 accelerators"): run(one(240), receipts, boots, buffer_containers=1, fallback_floor=3, capacity=capacity, current=free.plan) @@ -820,15 +850,12 @@ def failure_rate(product): if use.scope is not CapacityScope.TYPE or use.gpu_product in {cell.gpu_product for cell in planned.cells} ] held = sum(planned.envelope.floor_max * planned.container_shape.gpu_count for planned in lanes) + peak = sum(planned.max_containers * planned.container_shape.gpu_count for planned in lanes) elsewhere = use.held_elsewhere or 0 assert use.accelerators == held + elsewhere - if use.scope is CapacityScope.SHARED_QUOTA: - assert use.peak_accelerators is None - assert use.exceeded is (use.accelerators > use.ceiling) - else: - peak = sum(planned.max_containers * planned.container_shape.gpu_count for planned in lanes) - assert use.peak_accelerators == peak - assert use.exceeded is (peak > use.ceiling) + assert use.peak_accelerators == peak + elsewhere + assert use.exceeded is (peak + elsewhere > use.ceiling) + assert use.lanes == tuple(planned.name for planned in lanes) assert (error is not None) is any(use.exceeded for use in result.report.capacity) if error is not None: assert error.exceeded == tuple(use for use in result.report.capacity if use.exceeded) From 4799e51872a238e7af25813876d41aa9328a84b5 Mon Sep 17 00:00:00 2001 From: krisztian-gajdar Date: Fri, 6 Nov 2026 10:15:14 +0100 Subject: [PATCH 4/5] fix(placement): charge the shared quota at envelope maxima - The shared quota charges every lane at its envelope maximum, the most a tuner may hold warm, together with the accelerators held elsewhere. Warm capacity is held continuously, so it must fit the quota that environments share. A burst above the floors is bounded by each environment's own total and per-type ceilings, environments sharing one accelerator quota rarely burst together, and a burst that meets the quota shows as a measured placement wait, which the planner adds to a cold come-alive. - The total and per-type ceilings still charge every lane at maxContainers, and a plan that does not fit is still refused whole. - The report gives peakAccelerators for total and type ceilings only, and a shared-quota refusal names the lanes whose envelope maxima it charges. --- packages/sie_placement/README.md | 29 +++++--- .../src/sie_placement/planner.py | 3 +- .../sie_placement/tests/test_availability.py | 72 +++++++++++-------- 3 files changed, 62 insertions(+), 42 deletions(-) diff --git a/packages/sie_placement/README.md b/packages/sie_placement/README.md index 4e39543e6..016b2eb56 100644 --- a/packages/sie_placement/README.md +++ b/packages/sie_placement/README.md @@ -117,14 +117,20 @@ obtain. Each field is optional, and at least one is declared: | `sharedQuota` | A quota of `accelerators` that this environment shares with others, which hold `heldElsewhere` of it | Every container holds the `gpuCount` accelerators of its lane's container -shape. Each ceiling must hold every lane it covers at its container ceiling, -`maxContainers` containers, at once, as a deployment compiler that enforces the -ceiling charges it, and so at its envelope maximum, `floorMax` containers, too. -`total` and `sharedQuota` cover every lane, and `sharedQuota` holds them -together with its `heldElsewhere` accelerators. A lane with fallback cells -counts against the ceiling of every product it names, and once against `total` -and `sharedQuota`. The planner never lowers a lane's `maxContainers` or -`floorMax` to fit a capacity document: a plan that does not fit is refused. +shape. `total` and each of `types` must hold every lane they cover at its +container ceiling, `maxContainers` containers, at once, as a deployment +compiler that enforces the ceiling charges it, and so at its envelope maximum, +`floorMax` containers, too. `sharedQuota` must hold every lane at its envelope +maximum, the most a tuner may hold warm, together with its `heldElsewhere` +accelerators. Warm capacity is held continuously, so it must fit the quota the +environments share. A burst above the floors is bounded by each environment's +own `total` and `types`, environments that share a quota rarely burst +together, and a burst that meets the shared quota shows as a measured +placement wait, which the planner adds to a cold come-alive. A lane with +fallback cells counts against the ceiling of every product it names, and once +against `total` and `sharedQuota`. The planner never lowers a lane's +`maxContainers` or `floorMax` to fit a capacity document: a plan that does not +fit is refused whole. ## Envelope @@ -345,9 +351,10 @@ The report renders as a `placement-report` document: - `capacity`: for each ceiling of the `capacity` document, its `scope` (`total`, `type` with its `gpuProduct`, or `shared-quota` with `heldElsewhere`), the `ceiling`, the `accelerators` the plan's envelope - maxima hold against it, the `peakAccelerators` its lanes hold at - `maxContainers`, both with `heldElsewhere` for `shared-quota`, the `lanes` - that hold them, and whether it is `exceeded`. + maxima hold against it, with `heldElsewhere` for `shared-quota`, for `total` + and `type` the `peakAccelerators` its lanes hold at `maxContainers`, the + `lanes` that hold them, and whether it is `exceeded`. `total` and `type` are + charged at `peakAccelerators`, and `shared-quota` at `accelerators`. - `lanes`: each lane's hourly cost, its monthly cost, its floor's monthly cost and the monthly cost of its `floorMax`, which a floor budget prices. A lane on a cell the price list cannot price carries `unpricedReason` and no cost, diff --git a/packages/sie_placement/src/sie_placement/planner.py b/packages/sie_placement/src/sie_placement/planner.py index 505dff824..48e5ff752 100644 --- a/packages/sie_placement/src/sie_placement/planner.py +++ b/packages/sie_placement/src/sie_placement/planner.py @@ -1176,9 +1176,8 @@ def held(lanes: Iterable[PlanLane]) -> tuple[int, int, tuple[str, ...]]: scope=CapacityScope.SHARED_QUOTA, ceiling=quota.accelerators, accelerators=total + quota.held_elsewhere, - peak_accelerators=peak + quota.held_elsewhere, held_elsewhere=quota.held_elsewhere, - lanes=names, + lanes=tuple(lane.name for lane in plan.lanes if lane.envelope.floor_max), ) ) return tuple(uses) diff --git a/packages/sie_placement/tests/test_availability.py b/packages/sie_placement/tests/test_availability.py index 0304772b6..8874bb72c 100644 --- a/packages/sie_placement/tests/test_availability.py +++ b/packages/sie_placement/tests/test_availability.py @@ -534,7 +534,7 @@ def test_capacity_reports_what_the_envelope_maxima_and_container_ceilings_hold_a (CapacityScope.TOTAL, None, 4, 4, 4, ("org-a-l4", "org-b-a100-80gb")), (CapacityScope.TYPE, A100.gpu_product, 2, 2, 2, ("org-b-a100-80gb",)), (CapacityScope.TYPE, "NVIDIA-L4", 2, 2, 2, ("org-a-l4",)), - (CapacityScope.SHARED_QUOTA, None, 6, 6, 6, ("org-a-l4", "org-b-a100-80gb")), + (CapacityScope.SHARED_QUOTA, None, 6, 6, None, ("org-a-l4", "org-b-a100-80gb")), ] assert not any(use.exceeded for use in report.capacity) assert report.to_dict()["capacity"][0] == { @@ -549,7 +549,6 @@ def test_capacity_reports_what_the_envelope_maxima_and_container_ceilings_hold_a "scope": "shared-quota", "ceiling": 6, "accelerators": 6, - "peakAccelerators": 6, "heldElsewhere": 2, "lanes": ["org-a-l4", "org-b-a100-80gb"], "exceeded": False, @@ -573,14 +572,9 @@ def test_capacity_reports_what_the_envelope_maxima_and_container_ceilings_hold_a ), ( Capacity(shared_quota=SharedQuota(accelerators=6, held_elsewhere=3)), - "the container ceilings of org-a-l4, org-b-a100-80gb hold 8 accelerators, " + "the envelope maxima of org-a-l4, org-b-a100-80gb hold 4 accelerators, " "which with 3 held elsewhere exceed the shared quota of 6", ), - ( - Capacity(shared_quota=SharedQuota(accelerators=10, held_elsewhere=3)), - "the container ceilings of org-a-l4, org-b-a100-80gb hold 8 accelerators, " - "which with 3 held elsewhere exceed the shared quota of 10", - ), ], ) def test_a_plan_whose_lanes_exceed_a_ceiling_is_refused_and_says_why(capacity, message): @@ -614,25 +608,42 @@ def test_the_container_ceilings_bind_where_the_envelope_maxima_fit(): assert [(use.scope, use.accelerators, use.peak_accelerators) for use in caught.value.exceeded] == [ (CapacityScope.TOTAL, 4, 8), (CapacityScope.TYPE, 2, 4), - (CapacityScope.SHARED_QUOTA, 7, 11), ] result = two_lanes(capacity=capacity, max_containers=2) assert [planned.max_containers for planned in result.plan.lanes] == [2, 2] -def test_a_refused_plan_keeps_every_container_ceiling_it_would_need(): - capacity = Capacity(total=5, shared_quota=SharedQuota(accelerators=5)) +def test_the_shared_quota_holds_the_envelope_maxima_where_the_container_ceilings_exceed_it(): + capacity = Capacity(total=8, shared_quota=SharedQuota(accelerators=7, held_elsewhere=3)) + result = two_lanes(capacity=capacity) + assert [(planned.max_containers, planned.envelope.floor_max) for planned in result.plan.lanes] == [(4, 2), (4, 2)] + assert [(use.scope, use.accelerators, use.peak_accelerators, use.exceeded) for use in result.report.capacity] == [ + (CapacityScope.TOTAL, 4, 8, False), + (CapacityScope.SHARED_QUOTA, 7, None, False), + ] + with pytest.raises(CapacityExceededError) as caught: + two_lanes(capacity=replace(capacity, total=7)) + assert [use.scope for use in caught.value.exceeded] == [CapacityScope.TOTAL] + with pytest.raises(CapacityExceededError) as caught: + two_lanes(capacity=replace(capacity, shared_quota=SharedQuota(accelerators=6, held_elsewhere=3))) + assert [use.scope for use in caught.value.exceeded] == [CapacityScope.SHARED_QUOTA] + + +def test_a_refused_plan_keeps_every_container_ceiling_and_envelope_maximum_it_would_need(): + capacity = Capacity(total=5, shared_quota=SharedQuota(accelerators=3)) with pytest.raises(CapacityExceededError) as caught: two_lanes(capacity=capacity) refused = caught.value.result.plan - assert [(planned.name, planned.max_containers) for planned in refused.lanes] == [ - ("org-a-l4", 4), - ("org-b-a100-80gb", 4), + assert [(planned.name, planned.max_containers, planned.envelope.floor_max) for planned in refused.lanes] == [ + ("org-a-l4", 4, 2), + ("org-b-a100-80gb", 4, 2), ] assert refused == two_lanes().plan - for use in caught.value.exceeded: - assert use.lanes == ("org-a-l4", "org-b-a100-80gb") - assert "org-a-l4, org-b-a100-80gb hold 8 accelerators" in use.describe() + assert [use.describe() for use in caught.value.exceeded] == [ + "the container ceilings of org-a-l4, org-b-a100-80gb hold 8 accelerators, above the total ceiling of 5", + "the envelope maxima of org-a-l4, org-b-a100-80gb hold 4 accelerators, " + "which with 0 held elsewhere exceed the shared quota of 3", + ] def test_a_candidate_list_lane_counts_against_every_member_type(): @@ -658,10 +669,12 @@ def test_each_container_holds_its_gpu_count(): def test_a_zero_floor_lane_holds_no_envelope_maximum_but_holds_its_container_ceiling(): - result = run(one(600), [receipt()], [attempt(boot=60.0)], capacity=Capacity(total=4)) + capacity = Capacity(total=4, shared_quota=SharedQuota(accelerators=0)) + result = run(one(600), [receipt()], [attempt(boot=60.0)], capacity=capacity) assert result.report.identity(IDENTITY).chosen.warmth is Warmth.COLD_WITH_BOUND assert [(use.accelerators, use.peak_accelerators, use.lanes) for use in result.report.capacity] == [ - (0, 4, ("org-model-l4",)) + (0, 4, ("org-model-l4",)), + (0, None, ()), ] with pytest.raises(CapacityExceededError, match="the container ceilings of org-model-l4 hold 4 accelerators"): run(one(600), [receipt()], [attempt(boot=60.0)], capacity=Capacity(total=3)) @@ -669,18 +682,14 @@ def test_a_zero_floor_lane_holds_no_envelope_maximum_but_holds_its_container_cei def test_a_loosening_whose_plan_the_ceilings_would_refuse_saves_nothing(): receipts, boots = [receipt()], [attempt(boot=240.0)] - current = plan_of( - [replace(lane("org-model-l4", buffer=1), max_containers=2)], {IDENTITY: ("org-model-l4", Warmth.PINNED)} - ) - free = run(one(60), receipts, boots, buffer_containers=1, fallback_floor=3, current=current) - assert free.plan == current + free = run(one(60), receipts, boots, buffer_containers=1, fallback_floor=3) assert [lever.permitted_bound_s for lever in free.report.come_alive_levers] == [240] capacity = Capacity(shared_quota=SharedQuota(accelerators=2)) - bounded = run(one(60), receipts, boots, buffer_containers=1, fallback_floor=3, capacity=capacity, current=current) + bounded = run(one(60), receipts, boots, buffer_containers=1, fallback_floor=3, capacity=capacity) assert bounded.plan == free.plan assert bounded.report.come_alive_levers == () assert bounded.report.identity(IDENTITY).next_cheaper.saving == 0 - with pytest.raises(CapacityExceededError, match="the container ceilings of org-model-l4 hold 4 accelerators"): + with pytest.raises(CapacityExceededError, match="the envelope maxima of org-model-l4 hold 3 accelerators"): run(one(240), receipts, boots, buffer_containers=1, fallback_floor=3, capacity=capacity, current=free.plan) @@ -853,9 +862,14 @@ def failure_rate(product): peak = sum(planned.max_containers * planned.container_shape.gpu_count for planned in lanes) elsewhere = use.held_elsewhere or 0 assert use.accelerators == held + elsewhere - assert use.peak_accelerators == peak + elsewhere - assert use.exceeded is (peak + elsewhere > use.ceiling) - assert use.lanes == tuple(planned.name for planned in lanes) + if use.scope is CapacityScope.SHARED_QUOTA: + assert use.peak_accelerators is None + assert use.exceeded is (held + elsewhere > use.ceiling) + assert use.lanes == tuple(planned.name for planned in lanes if planned.envelope.floor_max) + else: + assert use.peak_accelerators == peak + assert use.exceeded is (peak > use.ceiling) + assert use.lanes == tuple(planned.name for planned in lanes) assert (error is not None) is any(use.exceeded for use in result.report.capacity) if error is not None: assert error.exceeded == tuple(use for use in result.report.capacity if use.exceeded) From 78fe4e31696174816a97ad052f9d2ea2162fdf46 Mon Sep 17 00:00:00 2001 From: krisztian-gajdar Date: Fri, 6 Nov 2026 10:30:05 +0100 Subject: [PATCH 5/5] fix(placement): check capacity counts as integers in documents built in code - total, each type ceiling's accelerators and the shared quota's accelerators and heldElsewhere go through the integer check, so a capacity document built in code refuses booleans and floats as parsing does. --- packages/sie_placement/src/sie_placement/schema.py | 8 ++++---- packages/sie_placement/tests/test_schema.py | 3 +++ 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/packages/sie_placement/src/sie_placement/schema.py b/packages/sie_placement/src/sie_placement/schema.py index 7f0f83acc..b445aba45 100644 --- a/packages/sie_placement/src/sie_placement/schema.py +++ b/packages/sie_placement/src/sie_placement/schema.py @@ -949,7 +949,7 @@ class TypeCeiling: def __post_init__(self) -> None: _require_identifier("gpuProduct", self.gpu_product) - _require_at_least("accelerators", self.accelerators, 0) + _require_count("accelerators", self.accelerators, 0) @classmethod def from_dict(cls, data: object, path: str = ROOT_PATH) -> Self: @@ -966,8 +966,8 @@ class SharedQuota: held_elsewhere: int = 0 def __post_init__(self) -> None: - _require_at_least("accelerators", self.accelerators, 0) - _require_at_least("heldElsewhere", self.held_elsewhere, 0) + _require_count("accelerators", self.accelerators, 0) + _require_count("heldElsewhere", self.held_elsewhere, 0) @classmethod def from_dict(cls, data: object, path: str = ROOT_PATH) -> Self: @@ -989,7 +989,7 @@ class Capacity: shared_quota: SharedQuota | None = None def __post_init__(self) -> None: - _require_at_least("total", self.total, 0) + _require_count("total", self.total, 0) _require_unique("types", "gpuProduct", (ceiling.gpu_product for ceiling in self.types)) if self.total is None and not self.types and self.shared_quota is None: raise SchemaError("", "must declare total, types or sharedQuota") diff --git a/packages/sie_placement/tests/test_schema.py b/packages/sie_placement/tests/test_schema.py index b828daece..454ad6c4b 100644 --- a/packages/sie_placement/tests/test_schema.py +++ b/packages/sie_placement/tests/test_schema.py @@ -187,6 +187,9 @@ def test_numbers_built_in_code_must_be_numbers(value, described): (lambda value: ServiceLevel(come_alive_p95_s=value), "comeAliveP95S"), (lambda value: Envelope(floor_min=0, floor_max=value), "floorMax"), (lambda value: DemandHour(start=datetime(2026, 1, 1, tzinfo=UTC), requests=value), "requests"), + (lambda value: Capacity(total=value), "total"), + (lambda value: TypeCeiling(gpu_product="NVIDIA-L4", accelerators=value), "accelerators"), + (lambda value: SharedQuota(accelerators=8, held_elsewhere=value), "heldElsewhere"), ], ) @pytest.mark.parametrize(("value", "described"), [(1.5, "number 1.5"), (2.0, "number 2.0"), (True, "boolean true")])